title={Deep unsupervised learning using nonequilibrium thermodynamics},
author={Sohl-Dickstein, Jascha and Weiss, Eric and Maheswaranathan, Niru and Ganguli, Surya},
booktitle={International conference on machine learning},
pages={2256--2265},
year={2015},
organization={pmlr}
}
@article{croitoru-etal:diffusion,
title={Diffusion models in vision: A survey},
author={Croitoru, Florinel-Alin and Hondru, Vlad and Ionescu, Radu Tudor and Shah, Mubarak},
journal={IEEE transactions on pattern analysis and machine intelligence},
volume={45},
number={9},
pages={10850--10869},
year={2023},
publisher={Ieee}
}
@article{yang-etal:diffusion,
title={Diffusion models: A comprehensive survey of methods and applications},
author={Yang, Ling and Zhang, Zhilong and Song, Yang and Hong, Shenda and Xu, Runsheng and Zhao, Yue and Zhang, Wentao and Cui, Bin and Yang, Ming-Hsuan},
journal={ACM computing surveys},
volume={56},
number={4},
pages={1--39},
year={2023},
publisher={ACM New York, NY, USA}
}
@article{chang-etal:efficient,
title={Efficient prompting methods for large language models: A survey},
author={Chang, Kaiyan and Xu, Songcheng and Wang, Chenglong and Luo, Yingfeng and Liu, Xiaoqian and Xiao, Tong and Zhu, Jingbo},
journal={arXiv preprint arXiv:2404.01077},
year={2024}
}
@article{fan-etal:dpok,
title={Dpok: Reinforcement learning for fine-tuning text-to-image diffusion models},
author={Fan, Ying and Watkins, Olivia and Du, Yuqing and Liu, Hao and Ryu, Moonkyung and Boutilier, Craig and Abbeel, Pieter and Ghavamzadeh, Mohammad and Lee, Kangwook and Lee, Kimin},
journal={Advances in neural information processing systems},
volume={36},
pages={79858--79885},
year={2023}
}
@article{lee-etal:aligning,
title={Aligning text-to-image models using human feedback},
author={Lee, Kimin and Liu, Hao and Ryu, Moonkyung and Watkins, Olivia and Du, Yuqing and Boutilier, Craig and Abbeel, Pieter and Ghavamzadeh, Mohammad and Gu, Shixiang Shane},
journal={arXiv preprint arXiv:2302.12192},
year={2023}
}
@inproceedings{wang-etal:probing,
title={Probing preference representations: A multi-dimensional evaluation and analysis method for reward models},
author={Wang, Chenglong and Huo, Yifu and Gan, Yang and Mu, Yongyu and He, Qiaozhi and Yang, Murun and Li, Bei and Zhang, Chunliang and Liu, Tongran and Ma, Anxiang and others},
booktitle={Proceedings of the AAAI Conference on Artificial Intelligence},
\section{Improved Reinforcement Learning for LLMs}
\section{Improved Reinforcement Learning for LLMs}
\vspace{-2mm}
In the previous section, we introduced some improvements to RL, such as importance sampling and reward baseline techniques. However, directly applying them to train LLMs still presents numerous challenges. In this section, we will delve deeper into the improvements for using RL to train LLMs.
In the previous section, we introduced some improvements to RL, such as importance sampling and reward baseline techniques. However, directly applying them to train LLMs still presents numerous challenges. In this section, we will delve deeper into the improvements for using RL to train LLMs.
\subsection{Advanced Reward Models}
\subsection{Advanced Reward Models}
...
@@ -214,12 +215,27 @@ Although incorporating CoT rationales into generative reward models improves pre
...
@@ -214,12 +215,27 @@ Although incorporating CoT rationales into generative reward models improves pre
\subsubsection{Reward Model Evaluation}
\subsubsection{Reward Model Evaluation}
After training a reward model, an important question is \textit{how to evaluate whether the learned reward function can accurately capture human preferences}. Unlike conventional supervised models, reward models do not directly predict explicit labels, but instead learn to assign scores that reflect the relative quality of different outputs. As a result, existing evaluation methods mainly focus on measuring the preference modeling ability of reward models or their effectiveness in RL training. We summarize three commonly used evaluation methods as follows.
After training a reward model, an important question is \textit{how to evaluate whether the learned reward function can accurately capture human preferences}. Unlike conventional supervised models that directly predict explicit labels, reward models learn to assign scores that reflect the relative quality of different outputs. As a result, existing evaluation approaches mainly focus on measuring the preference modeling ability of reward models or their effectiveness in downstream RL optimization. Figure~\ref{fig:rm-evaluation} compares three mainstream evaluation paradigms, including RL-based evaluation, pairwise ranking evaluation, and listwise ranking evaluation. We summarize these commonly used evaluation approaches as follows.
For the RL-based evaluation approach, the reward model is used to provide reward signals for policy optimization, and its quality is measured by the downstream performance of the optimized policy.
For the pairwise evaluation approach, the reward model compares a preferred output with a dispreferred output, and its prediction is checked against human preference annotations.
For the listwise evaluation approach, the reward model selects the best output from multiple candidates, and the selected output is compared with the human-preferred one.
}
\label{fig:rm-evaluation}
\end{figure*}
\begin{itemize}
\begin{itemize}
\item\textbf{RL-based Evaluation.}
\item\textbf{RL-based Evaluation.}
A straightforward approach to evaluate a reward model is to measure its effectiveness in downstream RL training. Specifically, given multiple reward models $\{\mathcal{M}_r^1,\cdots,\mathcal{M}_r^k\}$, we use each reward model to provide reward signals for training a policy model while keeping the training data, RL algorithm, and hyperparameters identical. After RL training, we obtain a set of policies $\{\mathcal{P}_1,\cdots,\mathcal{P}_k\}$, which are evaluated on human preference benchmarks or downstream tasks. The performance of each optimized policy is then used as an indirect measure of the corresponding reward model quality. In this way, we consider that a high-quality reward model should provide more reliable reward signals, enabling the policy to achieve better final performance.
A straightforward approach to evaluate a reward model is to measure its effectiveness in downstream RL training. Specifically, given multiple reward models $\{\mathcal{M}_r^1,\cdots,\mathcal{M}_r^k\}$, we use each reward model to provide reward signals for training a policy model while keeping the training data, RL algorithm, and hyperparameters identical\citep{wang-etal:probing,frick-etal:ppe}. After RL training, we obtain a set of policies $\{\mathcal{P}_1,\cdots,\mathcal{P}_k\}$, which are evaluated on human preference benchmarks or downstream tasks. The performance of each optimized policy is then used as an indirect measure of the corresponding reward model quality. In this way, we consider that a high-quality reward model should provide more reliable reward signals, enabling the policy to achieve better final performance.
\item\textbf{Pairwise Ranking Evaluation.}
\item\textbf{Pairwise Ranking Evaluation.}
Although RL-based evaluation directly measures whether a reward model can improve downstream policy optimization, it suffers from two limitations. First, it introduces significant evaluation costs. Since RL training requires substantial computational resources and time, it is difficult to efficiently compare different reward models. Second, the evaluation results can be sensitive to other factors in the RL pipeline, such as the choice of RL algorithms, which may introduce additional variations beyond the quality of the reward model itself. To address these limitations, a more efficient approach is to directly evaluate the preference ranking ability of reward models. Given a prompt $\mathbf{x}$ and two candidate outputs $\mathbf{y}^{+}$ and $\mathbf{y}^{-}$, where $\mathbf{y}^{+}$ is preferred over $\mathbf{y}^{-}$ according to human preference annotations, the reward model predicts their relative preference. Specifically, for discriminative reward models, we compare the predicted scores of the two outputs. If the reward model assigns a higher score to $\mathbf{y}^{+}$, its prediction is consistent with human preference. For generative reward models, the model directly selects $\mathbf{y}^{+}$ as the preferred output based on its generated preference. Otherwise, the prediction is considered incorrect. By constructing a large number of pairwise ranking samples, we can evaluate the reward model using ranking accuracy:
Although RL-based evaluation directly measures whether a reward model can improve downstream policy optimization, it suffers from two limitations. First, it introduces significant evaluation costs. Since RL training requires substantial computational resources and time, it is difficult to efficiently compare different reward models. Second, the evaluation results can be sensitive to other factors in the RL pipeline, such as the choice of RL algorithms, which may introduce additional variations beyond the quality of the reward model itself. To address these limitations, a more efficient approach is to directly evaluate the preference ranking ability of reward models. Given a prompt $\mathbf{x}$ and two candidate outputs $\mathbf{y}^{+}$ and $\mathbf{y}^{-}$, where $\mathbf{y}^{+}$ is preferred over $\mathbf{y}^{-}$ according to human preference annotations, the reward model predicts their relative preference. Specifically, for discriminative reward models, we compare the predicted scores of the two outputs. If the reward model assigns a higher score to $\mathbf{y}^{+}$, its prediction is consistent with human preference. For generative reward models, the model directly selects $\mathbf{y}^{+}$ as the preferred output based on its generated preference. Otherwise, the prediction is considered incorrect. By constructing a large number of pairwise ranking samples, we can evaluate the reward model using ranking accuracy:
...
@@ -263,10 +279,6 @@ where $N$ denotes the number of evaluation instances.
...
@@ -263,10 +279,6 @@ where $N$ denotes the number of evaluation instances.
\end{itemize}
\end{itemize}
-> probing approach?
\subsection{Better Advantage Estimation}
\subsection{Better Advantage Estimation}
Accurate advantage estimation is critical in RL, particularly when optimizing the policy model to truly reflect the potential benefits of different actions (or tokens in training LLMs). In this subsection, we will delve into methods for improving advantage estimation, providing more accurate advantages in the process of optimizing the policy model.
Accurate advantage estimation is critical in RL, particularly when optimizing the policy model to truly reflect the potential benefits of different actions (or tokens in training LLMs). In this subsection, we will delve into methods for improving advantage estimation, providing more accurate advantages in the process of optimizing the policy model.
@@ -144,7 +144,7 @@ where $r_p$ indicates that the overall plan is reasonable, $r_1$ and $r_2$ indic
...
@@ -144,7 +144,7 @@ where $r_p$ indicates that the overall plan is reasonable, $r_1$ and $r_2$ indic
\subsubsection{Tool Use}
\subsubsection{Tool Use}
Tool use is another fundamental capability of LLM-based agents. Tools extend the agent beyond its internal parameters. They allow the agent to access external knowledge, perform accurate computation, and execute actions in external systems. Early studies show that LLMs can use tools through prompt-based reasoning-action patterns. ReAct is a representative example, where the model alternates between reasoning steps and tool-use actions \citep{yao-etal:react}. However, prompt-based tool use is often unstable. It also depends heavily on the capability of the model. Thus, SFT methods train LLMs on tool-use trajectories, so that the model can learn when and how to invoke tools from demonstrations \citep{komeili-etal:Internet,schick-etal:toolformer}. Still, SFT mainly teaches imitation. It cannot directly optimize whether the agent should use a tool, which tool it should select, or how to balance tool benefit with tool cost. RL provides a natural framework for efficient tool use. We can define rewards based on task success, tool-use correctness, tool-use completeness, and tool cost \citep{qian-etal:toolrl,singh-etal:agentic}.
Tool use is another fundamental capability of LLM-based agents. Tools extend the agent beyond its internal parameters. They allow the agent to access external knowledge, perform accurate computation, and execute actions in external systems. Early studies show that LLMs can use tools through prompt-based reasoning-action patterns\citep{chang-etal:efficient,yao-etal:react}. ReAct is a representative example, where the model alternates between reasoning steps and tool-use actions \citep{yao-etal:react}. However, prompt-based tool use is often unstable. It also depends heavily on the capability of the model. Thus, SFT methods train LLMs on tool-use trajectories, so that the model can learn when and how to invoke tools from demonstrations \citep{komeili-etal:Internet,schick-etal:toolformer}. Still, SFT mainly teaches imitation. It cannot directly optimize whether the agent should use a tool, which tool it should select, or how to balance tool benefit with tool cost. RL provides a natural framework for efficient tool use. We can define rewards based on task success, tool-use correctness, tool-use completeness, and tool cost \citep{qian-etal:toolrl,singh-etal:agentic}.
