\begin{algorithm}[t]
	\caption{PPO-based Optimization Workflow}
	\begin{algorithmic}[1]
		\INPUT the optimized reward model $r_{\phi}(\cdot)$, the reference model $\pi_{\theta_{\mathrm{ref}}}(\cdot)$, the initialized value model $V_{\omega}(\cdot)$, the policy model $\pi_{\theta}(\cdot)$, the input-only dataset $S_{x}$;
		\OUTPUT the aligned $\pi_{\theta}(\cdot)$;
        \State \%\%\% suppose the batch size is set to 1
        \For{$\mathbf{x} \in S_x$}
        \State $\pi_{\theta_\mathrm{old}(\cdot)} = \mathrm{Clone}(\pi_{\theta}(\cdot))$
		\State sample an output from $\pi_{\theta_\mathrm{old}}$: $\mathbf{y} \sim \pi_{\theta_\mathrm{old}}(\cdot|\mathbf{x})$ 
        \State compute the reward for $[\mathbf{x},\mathbf{y}]$: $r_{\phi}(\mathbf{x},\mathbf{y})$
        \State \%\%\% roll out
        \For{ppo\_epoch\_idx=1 to $\mathrm{PPO\_EPOCH}$}
        \State \%\%\% \textit{compute policy model loss and value model loss through advantages and returns}
        \State initialize the loss values: $\mathrm{loss}_{p} = 0$, $\mathrm{loss}_{v} = 0$
        \For{t=1 to T}
        \State compute the penalty through Eq. (\ref{eq:penalty}): $\mathrm{Penalty}_{t}=\log \mathrm{Pr}_{\theta}(y_t|\mathbf{x},\mathbf{y}_{<t}) - \log \mathrm{Pr}_{\theta_{\mathrm{ref}}}(y_t|\mathbf{x},\mathbf{y}_{<t})$
        \State \textbf{if} t==T \textbf{then} 
        \State \hspace{0.5cm} $r_{t} = r_{\phi}(\mathbf{x}, \mathbf{y}) -\beta \mathrm{Penalty}_{t}$ 
        \State \textbf{else}
        \State \hspace{0.5cm} $r_{t} = -\beta \mathrm{Penalty}_{t}$
        \State \textbf{end if}
        \State predict the value $V_{\omega}(\mathbf{x},\mathbf{y}_{<t},y_{t})$
        \State \textbf{if} t==T \textbf{then}
        \State \hspace{0.5cm} $V_{\omega}(\mathbf{x},\mathbf{y}_{<t+1},y_{t+1})=0$
        \State \textbf{else}
        \State \hspace{0.5cm} predict the next time step value $V_{\omega}(\mathbf{x},\mathbf{y}_{<t+1},y_{t+1})$
        \State \textbf{end if}
        \State \%\%\% \textit{compute policy model loss}
        \State compute the advantage $A(\mathbf{x},\mathbf{y}_{<t},y_{t})$ through Eq. (\ref{eq:advantage}): $A(\mathbf{x},\mathbf{y}_{<t},y_{t})=r_{t}+\gamma V_{\omega}(\mathbf{x},\mathbf{y}_{<t+1},y_{t+1})-V_{\omega}(\mathbf{x},\mathbf{y}_{<t},y_{t})$
        \State compute the policy model loss through Eq. (\ref{eq:ppo-loss}) and add it to $\mathrm{loss}_{p}$: $\mathrm{loss}_{p} = \mathrm{loss}_{p} + \mathrm{Clip}\Big( \frac{\pi_{\theta}(y_t|\mathbf{x},\mathbf{y}_{<t})}{\pi_{\theta_{\mathrm{old}}}(y_t|\mathbf{x},\mathbf{y}_{<t})} \Big) A(\mathbf{x},\mathbf{y}_{<t},y_t)$
        \State \%\%\% \textit{compute value model loss}
        \State compute the return $\mathrm{Return}_{t}$: $\mathrm{Return}_{t}=A(\mathbf{x},\mathbf{y}_{<t},y_t)+V_{\omega}(\mathbf{x},\mathbf{y}_{<t},y_{t})$
        \State compute the value model loss through Eq. (\ref{eq:value-loss}) and add it to $\mathrm{loss}_{v}$: $\mathrm{loss}_v = \mathrm{loss}_v + (\mathrm{Return}_{t}-V_{\omega}(\mathbf{x},\mathbf{y}_{<t},y_{t}))^2$
        \EndFor
        \State update the parameters of $\pi_{\theta}(\cdot)$ through $\mathrm{loss}_{p}$
        \State update the parameters of $V_{\omega}(\cdot)$ through $\mathrm{loss}_{v}$
        \EndFor
        \EndFor
		\State return $\pi_{\theta}$
	\end{algorithmic}
	\label{alg:rlhf}
\end{algorithm}