






@article{monea-etal:llms,
  title={Llms are in-context reinforcement learners},
  author={Monea, Giovanni and Bosselut, Antoine and Brantley, Kiant{\'e} and Artzi, Yoav},
  year={2024}
}


@inproceedings{song-etal:trial,
  title={Trial and error: Exploration-based trajectory optimization of LLM agents},
  author={Song, Yifan and Yin, Da and Yue, Xiang and Huang, Jie and Li, Sujian and Lin, Bill Yuchen},
  booktitle={Proceedings of the 62nd Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers)},
  pages={7584--7600},
  year={2024}
}

@inproceedings{fu-etal:agentrefine,
  title={Agentrefine: Enhancing agent generalization through refinement tuning},
  author={Fu, Dayuan and He, Keqing and Wang, Yejie and Hong, Wentao and Gongque, Zhuoma and Zeng, Weihao and Wang, Wei and Wang, Jingang and Cai, Xunliang and Xu, Weiran},
  booktitle={International Conference on Learning Representations},
  volume={2025},
  pages={65185--65204},
  year={2025}
}

@article{yuan-etal:agent,
  title={Agent-r: Training language model agents to reflect via iterative self-training},
  author={Yuan, Siyu and Chen, Zehui and Xi, Zhiheng and Ye, Junjie and Du, Zhengyin and Chen, Jiecao},
  journal={arXiv preprint arXiv:2501.11425},
  year={2025}
}

@inproceedings{wang-etal:steca,
  title={Steca: Step-level trajectory calibration for llm agent learning},
  author={Wang, Hanlin and Wang, Jian and Leong, Chak Tou and Li, Wenjie},
  booktitle={Findings of the Association for Computational Linguistics: ACL 2025},
  pages={11597--11614},
  year={2025}
}

@inproceedings{liu-etal:pract,
  title={Pract: Optimizing principled reasoning and acting of llm agent},
  author={Liu, Zhiwei and Yao, Weiran and Zhang, Jianguo and Liu, Zuxin and Yang, Liangwei and RN, Rithesh and Lan, Tian and Zhu, Ming and Tan, Juntao and Kokane, Shirley and others},
  booktitle={Proceedings of the 28th Conference on Computational Natural Language Learning},
  pages={442--446},
  year={2024}
}

@article{shinn-etal:reflexion,
  title={Reflexion: Language agents with verbal reinforcement learning},
  author={Shinn, Noah and Cassano, Federico and Gopinath, Ashwin and Narasimhan, Karthik and Yao, Shunyu},
  journal={Advances in neural information processing systems},
  volume={36},
  pages={8634--8652},
  year={2023}
}


@article{fu-etal:self,
      title={Self-Play Meets Skill Evolution: Self-Evolving Search Agents that Pose, Solve, and Remember}, 
      author={Zenghuang Fu and Zhaoyang Li and Qiuyuan Ai and Haoyu Wu and Minghui Wu and Chenxu Zhao and Ante Wang and Guannan He and Changwei Wang},
      journal={arXiv preprint arXiv:2607.29468},
      year={2026},
}


@inproceedings{wang-etal:reinforcement,
  title={Reinforcement learning for self-improving agent with skill library},
  author={Wang, Jiongxiao and Yan, Qiaojing and Wang, Yawei and Tian, Yijun and Mishra, Soumya Smruti and Xu, Zhichao and Gandhi, Megha and Xu, Panpan and Cheong, Lin Lee},
  booktitle={Proceedings of the 64th Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers)},
  pages={1529--1550},
  year={2026}
}

@article{xia-etal:skillrl,
  title={Skillrl: Evolving agents via recursive skill-augmented reinforcement learning},
  author={Xia, Peng and Chen, Jianwen and Wang, Hanyang and Liu, Jiaqi and Zeng, Kaide and Wang, Yu and Han, Siwei and Zhou, Yiyang and Zhao, Xujiang and Chen, Haifeng and others},
  journal={arXiv preprint arXiv:2602.08234},
  year={2026}
}

@article{yu-etal:skill,
  title={Skill is Not One-Size-Fits-All: Model-Aware Skill Alignment for LLM Agents},
  author={Yu, Jianxiang and Zhu, Jiapeng and Lin, Bochen and Cui, Qier and Ding, Zichen and Li, Xiang},
  journal={arXiv preprint arXiv:2605.30723},
  year={2026}
}

@article{wang-etal:mem,
  title={Mem-$\{$$\backslash$alpha$\}$: Learning memory construction via reinforcement learning},
  author={Wang, Yu and Takanobu, Ryuichi and Liang, Zhiqi and Mao, Yuzhen and Hu, Yuanzhe and McAuley, Julian and Wu, Xiaojian},
  journal={arXiv preprint arXiv:2509.25911},
  year={2025}
}

@article{xu-etal:a-mem,
  title={A-mem: Agentic memory for llm agents},
  author={Xu, Wujiang and Liang, Zujie and Mei, Kai and Gao, Hang and Tan, Juntao and Zhang, Yongfeng},
  journal={Advances in Neural Information Processing Systems},
  volume={38},
  pages={17577--17604},
  year={2026}
}

@inproceedings{yan-etal:memory-r1,
  title={Memory-r1: Enhancing large language model agents to manage and utilize memories via reinforcement learning},
  author={Yan, Sikuan and Yang, Xiufeng and Huang, Zuchao and Nie, Ercong and Ding, Zifeng and Li, Zonggen and Ma, Xiaowen and Bi, Jinhe and Kersting, Kristian and Pan, Jeff Z and others},
  booktitle={Proceedings of the 64th Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers)},
  pages={12805--12825},
  year={2026}
}

@article{chhikara-etal:mem0,
  title={Mem0: Building production-ready ai agents with scalable long-term memory},
  author={Chhikara, Prateek and Khant, Dev and Aryan, Saket and Singh, Taranjeet and Yadav, Deshraj},
  journal={arXiv preprint arXiv:2504.19413},
  year={2025}
}

@inproceedings{li-etal:word2world,
  title={From word to world: Can large language models be implicit text-based world models?},
  author={Li, Yixia and Wang, Hongru and Qiu, Jiahao and Yin, Zhenfei and Zhang, Dongdong and Qian, Cheng and Li, Zeping and Ma, Xiaoteng and Chen, Guanhua and Ji, Heng},
  booktitle={Proceedings of the 64th Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers)},
  pages={8084--8111},
  year={2026}
}

@article{sutton-etal:welcome,
  title={Welcome to the era of experience},
  author={Silver, David and Sutton, Richard S},
  journal={Google AI},
  volume={1},
  pages={11},
  year={2025}
}

@inproceedings{chen-etal:failure-attribution,
    title     = {Where Did It Go Wrong? Capability-Oriented Failure Attribution for Vision-and-Language Navigation Agents},
    author    = {Chen, Jianming and Wang, Yawen and Wang, Junjie and Xie, Xiaofei and Li, Shoubin and Wang, Qing and Xu, Fanjiang},
    booktitle = {Findings of the Association for Computational Linguistics: ACL 2026},
    pages     = {28132--28150},
    address   = {San Diego, California, United States},
    publisher = {Association for Computational Linguistics},
    month     = jul,
    year      = {2026},
    doi       = {10.18653/v1/2026.findings-acl.1402},
    url       = {https://aclanthology.org/2026.findings-acl.1402/}
}

@article{sun-etal:learning-from-failure,
    title   = {Learning from Failure: Inference-Time Self-Improvement for Computer-Use Agents},
    author  = {Sun, Xueqiao and Wang, Xiaohan and Schmidt, Ludwig and Yeung-Levy, Serena and Zhang, Yuhui},
    journal = {arXiv preprint arXiv:2606.31270},
    year    = {2026},
    url     = {https://arxiv.org/abs/2606.31270}
}

@article{zhang-etal:autoenv,
  title         = {AutoEnv: Automated Environments for Measuring
                   Cross-Environment Agent Learning},
  author        = {Zhang, Jiayi and Peng, Yiran and Kong, Fanqi and Cheng, Yang
                   and Wu, Yifan and Yu, Zhaoyang and Xiang, Jinyu
                   and Ruan, Jianhao and Wang, Jinlin and Song, Maojia
                   and Liu, HongZhang and Tang, Xiangru and Liu, Bang
                   and Wu, Chenglin and Luo, Yuyu},
  journal       = {arXiv preprint arXiv:2511.19304},
  year          = {2025},
  eprint        = {2511.19304},
  archivePrefix = {arXiv},
  primaryClass  = {cs.AI}
}

@article{cai-etal:autoforge,
  title         = {AutoForge: Automated Environment Synthesis for
                   Agentic Reinforcement Learning},
  author        = {Cai, Shihao and Fang, Runnan and Wu, Jialong and Li, Baixuan
                   and Wang, Xinyu and Jiang, Yong and Su, Liangcai
                   and Zhang, Liwen and Yin, Wenbiao and Zhang, Zhen
                   and Feng, Fuli and Xie, Pengjun and Wang, Xiaobin},
  journal       = {arXiv preprint arXiv:2512.22857},
  year          = {2025},
  eprint        = {2512.22857},
  archivePrefix = {arXiv},
  primaryClass  = {cs.AI}
}

@article{wang-etal:awm,
  title         = {Agent World Model: Infinity Synthetic Environments for
                   Agentic Reinforcement Learning},
  author        = {Wang, Zhaoyang and Xu, Canwen and Liu, Boyi and Wang, Yite
                   and Han, Siwei and Yao, Zhewei and Yao, Huaxiu
                   and He, Yuxiong},
  journal       = {arXiv preprint arXiv:2602.10090},
  year          = {2026},
  eprint        = {2602.10090},
  archivePrefix = {arXiv},
  primaryClass  = {cs.AI}
}

@article{song-etal:envscaler,
  title         = {EnvScaler: Scaling Tool-Interactive Environments for
                   LLM Agent via Programmatic Synthesis},
  author        = {Song, Xiaoshuai and Chang, Haofei and Dong, Guanting
                   and Zhu, Yutao and Dou, Zhicheng and Wen, Ji-Rong},
  journal       = {arXiv preprint arXiv:2601.05808},
  year          = {2026},
  eprint        = {2601.05808},
  archivePrefix = {arXiv},
  primaryClass  = {cs.CL}
}

@article{deepseek-ai:deepseekv32,
  title         = {DeepSeek-V3.2: Pushing the Frontier of Open Large Language Models},
  author        = {DeepSeek-AI},
  journal       = {arXiv preprint arXiv:2512.02556},
  year          = {2025},
  eprint        = {2512.02556},
  archivePrefix = {arXiv},
  primaryClass  = {cs.CL}
}

@article{fang-etal:agentscaler,
  title         = {Towards General Agentic Intelligence via Environment Scaling},
  author        = {Fang, Runnan and Cai, Shihao and Li, Baixuan and Wu, Jialong
                   and Li, Guangyu and Yin, Wenbiao and Wang, Xinyu
                   and Wang, Xiaobin and Su, Liangcai and Zhang, Zhen
                   and Wu, Shibin and Tao, Zhengwei and Jiang, Yong
                   and Xie, Pengjun and Huang, Fei and Zhou, Jingren},
  journal       = {arXiv preprint arXiv:2509.13311},
  year          = {2025},
  eprint        = {2509.13311},
  archivePrefix = {arXiv},
  primaryClass  = {cs.AI}
}

@article{sullivan-etal:randomworld,
  title         = {Procedural Environment Generation for Tool-Use Agents},
  author        = {Sullivan, Michael and Hartmann, Mareike and Koller, Alexander},
  journal       = {arXiv preprint arXiv:2506.11045},
  year          = {2025},
  eprint        = {2506.11045},
  archivePrefix = {arXiv},
  primaryClass  = {cs.CL}
}

@article{chen-etal:research,
  title = {ReSearch: Learning to Reason with Search for LLMs via Reinforcement Learning},
  author = {Chen, Mingyang and Li, Tianpeng and Sun, Haoze and Zhou, Yijie and Zhu, Chenzheng and Yang, Fan and Zhou, Zenan and Chen, Weipeng and Wang, Haofen and Pan, Jeff Z. and Zhang, Wen and Chen, Huajun},
  journal = {arXiv preprint arXiv:2503.19470},
  year = {2025}
}

@article{jin-etal:searchr1,
  title = {Search-R1: Training LLMs to Reason and Leverage Search Engines with Reinforcement Learning},
  author = {Jin, Bowen and Zeng, Hansi and Yue, Zhenrui and Wang, Dong and Zamani, Hamed and Han, Jiawei},
  journal = {arXiv preprint arXiv:2503.09516},
  year = {2025}
}

@article{singh-etal:artist,
  title = {Agentic Reasoning and Tool Integration for LLMs via Reinforcement Learning},
  author = {Singh, Joykirat and Magazine, Raghav and Pandya, Yash and Nambi, Akshay},
  journal = {arXiv preprint arXiv:2505.01441},
  year = {2025}
}

@article{feng-etal:retool,
  title = {ReTool: Reinforcement Learning for Strategic Tool Use in LLMs},
  author = {Feng, Jiazhan and Huang, Shijue and Qu, Xingwei and Zhang, Ge and Qin, Yujia and Zhong, Baoquan and Jiang, Chengquan and Chi, Jinxin and Zhong, Wanjun},
  journal = {arXiv preprint arXiv:2504.11536},
  year = {2025}
}


@article{qian-etal:toolrl,
  title = {ToolRL: Reward is All Tool Learning Needs},
  author = {Qian, Cheng and Acikgoz, Emre Can and He, Qi and Wang, Hongru and Chen, Xiusi and Hakkani-Tr, Dilek and Tur, Gokhan and Ji, Heng},
  journal = {arXiv preprint arXiv:2504.13958},
  year = {2025}
}

@inproceedings{patil-etal:gorilla,
  title = {Gorilla: Large Language Model Connected with Massive APIs},
  author = {Patil, Shishir G. and Zhang, Tianjun and Wang, Xin and Gonzalez, Joseph E.},
  booktitle = {Advances in Neural Information Processing Systems},
  year = {2024}
}

@inproceedings{qin-etal:toolllm,
  title = {ToolLLM: Facilitating Large Language Models to Master 16000+ Real-world APIs},
  author = {Qin, Yujia and Liang, Shihao and Ye, Yining and Zhu, Kunlun and Yan, Lan and Lu, Yaxi and Lin, Yankai and Cong, Xin and Tang, Xiangru and Qian, Bill and Zhao, Sihan and Hong, Lauren and Tian, Runchu and Xie, Ruobing and Zhou, Jie and Gerstein, Mark and Li, Dahai and Liu, Zhiyuan and Sun, Maosong},
  booktitle = {The Twelfth International Conference on Learning Representations},
  year = {2024}
}

@inproceedings{li-etal:apibank,
  title = {API-Bank: A Comprehensive Benchmark for Tool-Augmented LLMs},
  author = {Li, Minghao and Zhao, Yingxiu and Yu, Bowen and Song, Feifan and Li, Hangyu and Yu, Haiyang and Li, Zhoujun and Huang, Fei and Li, Yongbin},
  booktitle = {Proceedings of the 2023 Conference on Empirical Methods in Natural Language Processing},
  year = {2023}
}

@inproceedings{schick-etal:toolformer,
  title = {Toolformer: Language Models Can Teach Themselves to Use Tools},
  author = {Schick, Timo and Dwivedi-Yu, Jane and Dess'i, Roberto and Raileanu, Roberta and Lomeli, Maria and Hambro, Eric and Zettlemoyer, Luke and Cancedda, Nicola and Scialom, Thomas},
  booktitle = {Advances in Neural Information Processing Systems},
  year = {2023}
}


@inproceedings{komeili-etal:Internet,
  title={Internet-augmented dialogue generation},
  author={Komeili, Mojtaba and Shuster, Kurt and Weston, Jason},
  booktitle={Proceedings of the 60th Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers)},
  pages={8460--8478},
  year={2022}
}

@inproceedings{singh-etal:agentic,
  title={Agentic Reasoning and Tool Integration for LLMs via Reinforcement Learning},
  author={Singh, Joykirat and Pandya, Yash and Vajreshwari, Pranav and Magazine, Raghav and Nambi, Akshay},
  booktitle={First Workshop on Foundations of Reasoning in Language Models},
  year={2025}
}

@inproceedings{melnyk:2024distributional,
  title={Distributional Preference Alignment of LLMs via Optimal Transport},
  author={Melnyk, Igor and Mroueh, Youssef and Belgodere, Brian and Rigotti, Mattia and Nitsure, Apoorva and Yurochkin, Mikhail and Greenewald, Kristjan and Navratil, Jiri and Ross, Jerret},
  booktitle={Advances in Neural Information Processing Systems},
  year={2024}
}

@inproceedings{zeng:2024token,
  title={Token-level Direct Preference Optimization},
  author={Zeng, Yongcheng and Liu, Guoqing and Ma, Weiyu and Yang, Ning and Zhang, Haifeng and Wang, Jun},
  booktitle={Proceedings of the 41st International Conference on Machine Learning},
  year={2024}
}

@inproceedings{xiao:2024cal,
  title={Cal-DPO: Calibrated Direct Preference Optimization for Language Model Alignment},
  author={Xiao, Teng and Yuan, Yige and Zhu, Huaisheng and Li, Mingxiao and Honavar, Vasant G.},
  booktitle={The Thirty-eighth Annual Conference on Neural Information Processing Systems},
  year={2024}
}

@article{amini:2024direct,
  title={Direct Preference Optimization with an Offset},
  author={Amini, Afra and Vieira, Tim and Cotterell, Ryan},
  journal={arXiv preprint arXiv:2402.10571},
  year={2024}
}

@article{yao-etal:react,
  title={React: Synergizing reasoning and acting in language models},
  author={Yao, Shunyu and Zhao, Jeffrey and Yu, Dian and Du, Nan and Shafran, Izhak and Narasimhan, Karthik and Cao, Yuan},
  journal={arXiv preprint arXiv:2210.03629},
  year={2022}
}

@article{wang-etal:steppo,
  title={StepPO: Step-Aligned Policy Optimization for Agentic Reinforcement Learning},
  author={Wang, Daoyu and Li, Qingchuan and Cheng, Mingyue and Ouyang, Jie and Yu, Shuo and Liu, Qi and Chen, Enhong},
  journal={arXiv preprint arXiv:2604.18401},
  year={2026}
}

@article{djuhera-etal:tsr,
  title={TSR: Trajectory-Search Rollouts for Multi-Turn RL of LLM Agents},
  author={Djuhera, Aladin and Kadhe, Swanand Ravindra and Ahmed, Farhan and Zawad, Syed and Ludwig, Heiko and Boche, Holger},
  journal={arXiv preprint arXiv:2602.11767},
  year={2026}
}

@article{wang-etal:ragen,
  title={Ragen: Understanding self-evolution in llm agents via multi-turn reinforcement learning},
  author={Wang, Zihan and Wang, Kangrui and Wang, Qineng and Zhang, Pingyue and Li, Linjie and Yang, Zhengyuan and Jin, Xing and Yu, Kefan and Nguyen, Minh Nhat and Liu, Licheng and others},
  journal={arXiv preprint arXiv:2504.20073},
  year={2025}
}

@inproceedings{xi-etal:agentprm,
  title={Agentprm: Process reward models for llm agents via step-wise promise and progress},
  author={Xi, Zhiheng and Liao, Chenyang and Li, Guanyu and Zhang, Zhihao and Chen, Wenxiang and Wang, Binghai and Jin, Senjie and Zhou, Yuhao and Guan, Jian and Wu, Wei and others},
  booktitle={Proceedings of the ACM Web Conference 2026},
  pages={4184--4195},
  year={2026}
}

@inproceedings{li-etal:encouraging,
  title={Encouraging good processes without the need for good answers: Reinforcement learning for llm agent planning},
  author={Li, Zhiwei and Hu, Yong and Wang, Wenqing},
  booktitle={Proceedings of the 2025 Conference on Empirical Methods in Natural Language Processing: Industry Track},
  pages={1654--1666},
  year={2025}
}

@inproceedings{hu-etal:agentgen,
  title={Agentgen: Enhancing planning abilities for large language model based agent via environment and task generation},
  author={Hu, Mengkang and Zhao, Pu and Xu, Can and Sun, Qingfeng and Lou, Jian-Guang and Lin, Qingwei and Luo, Ping and Rajmohan, Saravan},
  booktitle={Proceedings of the 31st ACM SIGKDD Conference on Knowledge Discovery and Data Mining V. 1},
  pages={496--507},
  year={2025}
}

@inproceedings{yin-etal:agentlumos,
  title={Agent lumos: Unified and modular training for open-source language agents},
  author={Yin, Da and Brahman, Faeze and Ravichander, Abhilasha and Chandu, Khyathi and Chang, Kai-Wei and Choi, Yejin and Lin, Bill Yuchen},
  booktitle={Proceedings of the 62nd Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers)},
  pages={12380--12403},
  year={2024}
}

@article{zhou-etal:lima,
  title={Lima: Less is more for alignment},
  author={Zhou, Chunting and Liu, Pengfei and Xu, Puxin and Iyer, Srinivasan and Sun, Jiao and Mao, Yuning and Ma, Xuezhe and Efrat, Avia and Yu, Ping and Yu, Lili and others},
  journal={Advances in Neural Information Processing Systems},
  volume={36},
  pages={55006--55021},
  year={2023}
}

@inproceedings{longpre-etal:flan,
  title={The flan collection: Designing data and methods for effective instruction tuning},
  author={Longpre, Shayne and Hou, Le and Vu, Tu and Webson, Albert and Chung, Hyung Won and Tay, Yi and Zhou, Denny and Le, Quoc V and Zoph, Barret and Wei, Jason and others},
  booktitle={International conference on machine learning},
  pages={22631--22648},
  year={2023},
  organization={PMLR}
}

@inproceedings{zeng-etal:agenttuning,
  title={Agenttuning: Enabling generalized agent abilities for llms},
  author={Zeng, Aohan and Liu, Mingdao and Lu, Rui and Wang, Bowen and Liu, Xiao and Dong, Yuxiao and Tang, Jie},
  booktitle={Findings of the Association for Computational Linguistics: ACL 2024},
  pages={3053--3077},
  year={2024}
}

@article{zhang-etal:agentohana,
  title={Agentohana: Design unified data and training pipeline for effective agent learning},
  author={Zhang, Jianguo and Lan, Tian and Murthy, Rithesh and Liu, Zhiwei and Yao, Weiran and Zhu, Ming and Tan, Juntao and Hoang, Thai and Liu, Zuxin and Yang, Liangwei and others},
  journal={arXiv preprint arXiv:2402.15506},
  year={2024}
}

@article{chen-etal:fireact,
  title={Fireact: Toward language agent fine-tuning},
  author={Chen, Baian and Shu, Chang and Shareghi, Ehsan and Collier, Nigel and Narasimhan, Karthik and Yao, Shunyu},
  journal={arXiv preprint arXiv:2310.05915},
  year={2023}
}

@inproceedings{singh-etal:singhagentic,
  title={Agentic Reasoning and Tool Integration for LLMs via Reinforcement Learning},
  author={Singh, Joykirat and Pandya, Yash and Vajreshwari, Pranav and Magazine, Raghav and Nambi, Akshay},
  booktitle={First Workshop on Foundations of Reasoning in Language Models},
  year={2025},
}

@article{lee-etal:lee2024learning,
  title={Learning robust autonomous navigation and locomotion for wheeled-legged robots},
  author={Lee, Joonho and Bjelonic, Marko and Reske, Alexander and Wellhausen, Lorenz and Miki, Takahiro and Hutter, Marco},
  journal={Science Robotics},
  volume={9},
  number={89},
  pages={eadi9641},
  year={2024},
  publisher={American Association for the Advancement of Science}
}

@article{silver-etal:silver2017mastering,
  title={Mastering the game of go without human knowledge},
  author={Silver, David and Schrittwieser, Julian and Simonyan, Karen and Antonoglou, Ioannis and Huang, Aja and Guez, Arthur and Hubert, Thomas and Baker, Lucas and Lai, Matthew and Bolton, Adrian and others},
  journal={nature},
  volume={550},
  number={7676},
  pages={354--359},
  year={2017},
  publisher={Nature Publishing Group UK London}
}

@article{mnih-etal:mnih2013playing,
  title={Playing atari with deep reinforcement learning},
  author={Mnih, Volodymyr and Kavukcuoglu, Koray and Silver, David and Graves, Alex and Antonoglou, Ioannis and Wierstra, Daan and Riedmiller, Martin},
  journal={arXiv preprint arXiv:1312.5602},
  year={2013}
}

@article{wang2026msrl,
  title={MSRL: Scaling Generative Multimodal Reward Modeling via Multi-Stage Reinforcement Learning},
  author={Wang, Chenglong and Huo, Yifu and Gan, Yang and He, Qiaozhi and Meng, Qi and Li, Bei and Wang, Yan and Liu, Junfu and Zhou, Tianhua and Zhu, Jingbo and others},
  journal={arXiv preprint arXiv:2603.25108},
  year={2026}
}

@inproceedings{wang-etal:wang2026gramrr,
  title={Gram-r$^2$: Self-training generative foundation reward models for reward reasoning},
  author={Wang, Chenglong and Mu, Yongyu and Zhou, Hang and Huo, Yifu and Zhu, Ziming and Zeng, Jiali and Yang, Murun and Li, Bei and Hao, Xiaoyang and Zhang, Chunliang and others},
  booktitle={Proceedings of the AAAI Conference on Artificial Intelligence},
  volume={40},
  number={39},
  pages={33395--33403},
  year={2026}
}

@article{wang-etal:wang2025gram,
  title={Gram: A generative foundation reward model for reward generalization},
  author={Wang, Chenglong and Gan, Yang and Huo, Yifu and Mu, Yongyu and He, Qiaozhi and Yang, Murun and Li, Bei and Xiao, Tong and Zhang, Chunliang and Liu, Tongran and others},
  journal={arXiv preprint arXiv:2506.14175},
  year={2025}
}

@article{zhang-etal:zhang2024generative,
  title={Generative verifiers: Reward modeling as next-token prediction},
  author={Zhang, Lunjun and Hosseini, Arian and Bansal, Hritik and Kazemi, Mehran and Kumar, Aviral and Agarwal, Rishabh},
  journal={arXiv preprint arXiv:2408.15240},
  year={2024}
}

@article{sun-etal:2023aligning,
  title={Aligning large multimodal models with factually augmented rlhf},
  author={Sun, Zhiqing and Shen, Sheng and Cao, Shengcao and Liu, Haotian and Li, Chunyuan and Shen, Yikang and Gan, Chuang and Gui, Liang-Yan and Wang, Yu-Xiong and Yang, Yiming and others},
  journal={arXiv preprint arXiv:2309.14525},
  year={2023}
}

@article{ji2025safe,
  title={Safe RLHF-V: Safe Reinforcement Learning from Human Feedback in Multimodal Large Language Models},
  author={Ji, Jiaming and Chen, Xinyu and Pan, Rui and Zhu, Han and Zhang, Conghui and Li, Jiahao and Hong, Donghai and Chen, Boyuan and Zhou, Jiayi and Wang, Kaile and others},
  journal={arXiv preprint arXiv:2503.17682},
  year={2025}
}

@article{zang2025internlm,
  title={InternLM-XComposer2. 5-Reward: A Simple Yet Effective Multi-Modal Reward Model},
  author={Zang, Yuhang and Dong, Xiaoyi and Zhang, Pan and Cao, Yuhang and Liu, Ziyu and Ding, Shengyuan and Wu, Shenxi and Ma, Yubo and Duan, Haodong and Zhang, Wenwei and others},
  journal={arXiv preprint arXiv:2501.12368},
  year={2025}
}

@article{cheng-etal:2023everyone,
  title={Everyone deserves a reward: Learning customized human preferences},
  author={Cheng, Pengyu and Xie, Jiawen and Bai, Ke and Dai, Yong and Du, Nan},
  journal={arXiv preprint arXiv:2309.03126},
  year={2023}
}

@article{wu-etal:2024reuse,
  title={Reuse your rewards: Reward model transfer for zero-shot cross-lingual alignment},
  author={Wu, Zhaofeng and Balashankar, Ananth and Kim, Yoon and Eisenstein, Jacob and Beirami, Ahmad},
  journal={arXiv preprint arXiv:2404.12318},
  year={2024}
}

@article{xia-etal:2024less,
  title={Less: Selecting influential data for targeted instruction tuning},
  author={Xia, Mengzhou and Malladi, Sadhika and Gururangan, Suchin and Arora, Sanjeev and Chen, Danqi},
  journal={arXiv preprint arXiv:2402.04333},
  year={2024}
}

@article{wang-etal:2024rovrm,
  title={RoVRM: A Robust Visual Reward Model Optimized via Auxiliary Textual Preference Data},
  author={Wang, Chenglong and Gan, Yang and Huo, Yifu and Mu, Yongyu and Yang, Murun and He, Qiaozhi and Xiao, Tong and Zhang, Chunliang and Liu, Tongran and Du, Quan and others},
  journal={arXiv preprint arXiv:2408.12109},
  year={2024}
}

@article{yu-etal:2024rlaif,
  title={Rlaif-v: Aligning mllms through open-source ai feedback for super gpt-4v trustworthiness},
  author={Yu, Tianyu and Zhang, Haoye and Yao, Yuan and Dang, Yunkai and Chen, Da and Lu, Xiaoman and Cui, Ganqu and He, Taiwen and Liu, Zhiyuan and Chua, Tat-Seng and others},
  journal={arXiv preprint arXiv:2405.17220},
  year={2024}
}

@article{zhang-etal:2024mm,
  title={Mm-llms: Recent advances in multimodal large language models},
  author={Zhang, Duzhen and Yu, Yahan and Dong, Jiahua and Li, Chenxing and Su, Dan and Chu, Chenhui and Yu, Dong},
  journal={arXiv preprint arXiv:2401.13601},
  year={2024}
}

@inproceedings{yu-etal:2024rlhf,
  title={Rlhf-v: Towards trustworthy mllms via behavior alignment from fine-grained correctional human feedback},
  author={Yu, Tianyu and Yao, Yuan and Zhang, Haoye and He, Taiwen and Han, Yifeng and Cui, Ganqu and Hu, Jinyi and Liu, Zhiyuan and Zheng, Hai-Tao and Sun, Maosong and others},
  booktitle={Proceedings of the IEEE/CVF Conference on Computer Vision and Pattern Recognition},
  pages={13807--13816},
  year={2024}
}

@article{grattafiori-etal:2024llama,
  title={The llama 3 herd of models},
  author={Grattafiori, Aaron and Dubey, Abhimanyu and Jauhri, Abhinav and Pandey, Abhinav and Kadian, Abhishek and Al-Dahle, Ahmad and Letman, Aiesha and Mathur, Akhil and Schelten, Alan and Vaughan, Alex and others},
  journal={arXiv preprint arXiv:2407.21783},
  year={2024}
}

@misc{qwenTeam:2025qwen2.5-VL,
    title = {Qwen2.5-VL},
    url = {https://qwenlm.github.io/blog/qwen2.5-vl/},
    author = {Qwen Team},
    month = {January},
    year = {2025}
}

@article{xu-etal:2025qwen2,
  title={Qwen2. 5-Omni Technical Report},
  author={Xu, Jin and Guo, Zhifang and He, Jinzheng and Hu, Hangrui and He, Ting and Bai, Shuai and Chen, Keqin and Wang, Jialin and Fan, Yang and Dang, Kai and others},
  journal={arXiv preprint arXiv:2503.20215},
  year={2025}
}

@misc{radford-etal:2021learning,
      title={Learning Transferable Visual Models From Natural Language Supervision}, 
      author={Alec Radford and Jong Wook Kim and Chris Hallacy and Aditya Ramesh and Gabriel Goh and Sandhini Agarwal and Girish Sastry and Amanda Askell and Pamela Mishkin and Jack Clark and Gretchen Krueger and Ilya Sutskever},
      year={2021},
      eprint={2103.00020},
      archivePrefix={arXiv},
      primaryClass={cs.CV},
      url={https://arxiv.org/abs/2103.00020}, 
}

@article{baltruvsaitis-etal:2018multimodal,
  title={Multimodal machine learning: A survey and taxonomy},
  author={Baltru{\v{s}}aitis, Tadas and Ahuja, Chaitanya and Morency, Louis-Philippe},
  journal={IEEE transactions on pattern analysis and machine intelligence},
  volume={41},
  number={2},
  pages={423--443},
  year={2018},
  publisher={IEEE}
}

@incollection{deb-etal:2016multi,
  title={Multi-objective optimization},
  author={Deb, Kalyanmoy and Sindhya, Karthik and Hakanen, Jussi},
  booktitle={Decision sciences},
  pages={161--200},
  year={2016},
  publisher={CRC Press}
}

@incollection{miettinen-etal:2008introduction,
  title={Introduction to multiobjective optimization: interactive approaches},
  author={Miettinen, Kaisa and Ruiz, Francisco and Wierzbicki, Andrzej P},
  booktitle={Multiobjective optimization: interactive and evolutionary approaches},
  pages={27--57},
  year={2008},
  publisher={Springer}
}

@article{gorbatovski-etal:2024learn,
 author = {Gorbatovski, Alexey and Shaposhnikov, Boris and Malakhov, Alexey and Surnachev, Nikita and Aksenov, Yaroslav and Maksimov, Ian and Balagansky, Nikita and Gavrilov, Daniil},
 journal = {ArXiv preprint},
 title = {Learn your reference model for real good alignment},
 url = {https://arxiv.org/abs/2404.09656},
 volume = {abs/2404.09656},
 year = {2024}
}

@inproceedings{yuan-etal:2024selfrewarding,
 author = {Weizhe Yuan and
Richard Yuanzhe Pang and
Kyunghyun Cho and
Xian Li and
Sainbayar Sukhbaatar and
Jing Xu and
Jason Weston},
 bibsource = {dblp computer science bibliography, https://dblp.org},
 biburl = {https://dblp.org/rec/conf/icml/YuanPCLSXW24.bib},
 booktitle = {Forty-first International Conference on Machine Learning, {ICML} 2024,
Vienna, Austria, July 21-27, 2024},
 publisher = {OpenReview.net},
 timestamp = {Mon, 02 Sep 2024 01:00:00 +0200},
 title = {Self-Rewarding Language Models},
 url = {https://openreview.net/forum?id=0NphYCmgua},
 year = {2024}
}

@article{zeng-etal:2025revisiting,
 author = {Zeng, Zhiyuan and Cheng, Qinyuan and Yin, Zhangyue and Zhou, Yunhua and Qiu, Xipeng},
 journal = {ArXiv preprint},
 title = {Revisiting the Test-Time Scaling of o1-like Models: Do they Truly Possess Test-Time Scaling Capabilities?},
 url = {https://arxiv.org/abs/2502.12215},
 volume = {abs/2502.12215},
 year = {2025}
}

@article{yuan-etal:2023scaling,
 author = {Yuan, Zheng and Yuan, Hongyi and Li, Chengpeng and Dong, Guanting and Lu, Keming and Tan, Chuanqi and Zhou, Chang and Zhou, Jingren},
 journal = {ArXiv preprint},
 title = {Scaling relationship on learning mathematical reasoning with large language models},
 url = {https://arxiv.org/abs/2308.01825},
 volume = {abs/2308.01825},
 year = {2023}
}

@inproceedings{yao-etal:2023tree,
 author = {Shunyu Yao and
Dian Yu and
Jeffrey Zhao and
Izhak Shafran and
Tom Griffiths and
Yuan Cao and
Karthik Narasimhan},
 bibsource = {dblp computer science bibliography, https://dblp.org},
 biburl = {https://dblp.org/rec/conf/nips/YaoYZS00N23.bib},
 booktitle = {Advances in Neural Information Processing Systems 36: Annual Conference
on Neural Information Processing Systems 2023, NeurIPS 2023, New Orleans,
LA, USA, December 10 - 16, 2023},
 editor = {Alice Oh and
Tristan Naumann and
Amir Globerson and
Kate Saenko and
Moritz Hardt and
Sergey Levine},
 timestamp = {Fri, 01 Mar 2024 00:00:00 +0100},
 title = {Tree of Thoughts: Deliberate Problem Solving with Large Language Models},
 url = {http://papers.nips.cc/paper\_files/paper/2023/hash/271db9922b8d1f4dd7aaef84ed5ac703-Abstract-Conference.html},
 year = {2023}
}

@article{xie-etal:2025logic,
 author = {Xie, Tian and Gao, Zitian and Ren, Qingnan and Luo, Haoming and Hong, Yuqian and Dai, Bryan and Zhou, Joey and Qiu, Kai and Wu, Zhirong and Luo, Chong},
 journal = {ArXiv preprint},
 title = {Logic-rl: Unleashing llm reasoning with rule-based reinforcement learning},
 url = {https://arxiv.org/abs/2502.14768},
 volume = {abs/2502.14768},
 year = {2025}
}

@article{wang-etal:2023math,
 author = {Wang, Peiyi and Li, Lei and Shao, Zhihong and Xu, RX and Dai, Damai and Li, Yifei and Chen, Deli and Wu, Yu and Sui, Zhifang},
 journal = {ArXiv preprint},
 title = {Math-shepherd: Verify and reinforce llms step-by-step without human annotations},
 url = {https://arxiv.org/abs/2312.08935},
 volume = {abs/2312.08935},
 year = {2023}
}

@inproceedings{lightman-etal:2024lets,
 author = {Hunter Lightman and
Vineet Kosaraju and
Yuri Burda and
Harrison Edwards and
Bowen Baker and
Teddy Lee and
Jan Leike and
John Schulman and
Ilya Sutskever and
Karl Cobbe},
 bibsource = {dblp computer science bibliography, https://dblp.org},
 biburl = {https://dblp.org/rec/conf/iclr/LightmanKBEBLLS24.bib},
 booktitle = {The Twelfth International Conference on Learning Representations,
{ICLR} 2024, Vienna, Austria, May 7-11, 2024},
 publisher = {OpenReview.net},
 timestamp = {Wed, 07 Aug 2024 01:00:00 +0200},
 title = {Let's Verify Step by Step},
 url = {https://openreview.net/forum?id=v8L0pN6EOi},
 year = {2024}
}

@article{setlur-etal:2024rewarding,
 author = {Setlur, Amrith and Nagpal, Chirag and Fisch, Adam and Geng, Xinyang and Eisenstein, Jacob and Agarwal, Rishabh and Agarwal, Alekh and Berant, Jonathan and Kumar, Aviral},
 journal = {ArXiv preprint},
 title = {Rewarding progress: Scaling automated process verifiers for llm reasoning},
 url = {https://arxiv.org/abs/2410.08146},
 volume = {abs/2410.08146},
 year = {2024}
}

@article{touvron-etal:2023llama2,
 author = {Hugo Touvron and Louis Martin and Kevin Stone and Peter Albert and Amjad Almahairi and Yasmine Babaei and Nikolay Bashlykov and Soumya Batra and Prajjwal Bhargava and Shruti Bhosale and Dan Bikel and Lukas Blecher and Cristian Canton Ferrer and Moya Chen and Guillem Cucurull and David Esiobu and Jude Fernandes and Jeremy Fu and Wenyin Fu and Brian Fuller and Cynthia Gao and Vedanuj Goswami and Naman Goyal and Anthony Hartshorn and Saghar Hosseini and Rui Hou and Hakan Inan and Marcin Kardas and Viktor Kerkez and Madian Khabsa and Isabel Kloumann and Artem Korenev and Punit Singh Koura and Marie-Anne Lachaux and Thibaut Lavril and Jenya Lee and Diana Liskovich and Yinghai Lu and Yuning Mao and Xavier Martinet and Todor Mihaylov and Pushkar Mishra and Igor Molybog and Yixin Nie and Andrew Poulton and Jeremy Reizenstein and Rashi Rungta and Kalyan Saladi and Alan Schelten and Ruan Silva and Eric Michael Smith and Ranjan Subramanian and Xiaoqing Ellen Tan and Binh Tang and Ross Taylor and Adina Williams and Jian Xiang Kuan and Puxin Xu and Zheng Yan and Iliyan Zarov and Yuchen Zhang and Angela Fan and Melanie Kambadur and Sharan Narang and Aurelien Rodriguez and Robert Stojnic and Sergey Edunov and Thomas Scialom},
 journal = {ArXiv preprint},
 title = {Llama 2: Open foundation and fine-tuned chat models},
 url = {https://arxiv.org/abs/2307.09288},
 volume = {abs/2307.09288},
 year = {2023}
}

@article{nakano-etal:2021webgpt,
 author = {Reiichiro Nakano and Jacob Hilton and Suchir Balaji and Jeff Wu and Long Ouyang and Christina Kim and Christopher Hesse and Shantanu Jain and Vineet Kosaraju and William Saunders and Xu Jiang and Karl Cobbe and Tyna Eloundou and Gretchen Krueger and Kevin Button and Matthew Knight and Benjamin Chess and John Schulman},
 journal = {ArXiv preprint},
 title = {Webgpt: Browser-assisted question-answering with human feedback},
 url = {https://arxiv.org/abs/2112.09332},
 volume = {abs/2112.09332},
 year = {2021}
}

@article{xiao-and-zhu:2025foundations,
 author = {Xiao, Tong and Zhu, Jingbo},
 journal = {ArXiv preprint},
 title = {Foundations of Large Language Models},
 url = {https://arxiv.org/abs/2501.09223},
 volume = {abs/2501.09223},
 year = {2025}
}

@article{vinyals-rtal:2019grandmaster,
 author = {Vinyals, Oriol and Babuschkin, Igor and Czarnecki, Wojciech M and Mathieu, Micha{\"e}l and Dudzik, Andrew and Chung, Junyoung and Choi, David H and Powell, Richard and Ewalds, Timo and Georgiev, Petko and others},
 journal = {nature},
 number = {7782},
 pages = {350--354},
 publisher = {Nature Publishing Group},
 title = {Grandmaster level in StarCraft II using multi-agent reinforcement learning},
 volume = {575},
 year = {2019}
}

@article{silver-etal:2016mastering,
 author = {Silver, David and Huang, Aja and Maddison, Chris J and Guez, Arthur and Sifre, Laurent and Van Den Driessche, George and Schrittwieser, Julian and Antonoglou, Ioannis and Panneershelvam, Veda and Lanctot, Marc and others},
 journal = {nature},
 number = {7587},
 pages = {484--489},
 publisher = {Nature Publishing Group},
 title = {Mastering the game of Go with deep neural networks and tree search},
 volume = {529},
 year = {2016}
}

@article{kimi-team:2025kimi,
 author = {Team, Kimi and Du, Angang and Gao, Bofei and Xing, Bowei and Jiang, Changjiu and Chen, Cheng and Li, Cheng and Xiao, Chenjun and Du, Chenzhuang and Liao, Chonghua and others},
 journal = {ArXiv preprint},
 title = {Kimi k1. 5: Scaling reinforcement learning with llms},
 url = {https://arxiv.org/abs/2501.12599},
 volume = {abs/2501.12599},
 year = {2025}
}

@article{xiao-etal:2023introduction,
 author = {Xiao, Tong and Zhu, Jingbo},
 journal = {ArXiv preprint},
 title = {Introduction to transformers: an nlp perspective},
 url = {https://arxiv.org/abs/2311.17633},
 volume = {abs/2311.17633},
 year = {2023}
}

@inproceedings{wang-etal:2023far,
 author = {Yizhong Wang and
Hamish Ivison and
Pradeep Dasigi and
Jack Hessel and
Tushar Khot and
Khyathi Chandu and
David Wadden and
Kelsey MacMillan and
Noah A. Smith and
Iz Beltagy and
Hannaneh Hajishirzi},
 bibsource = {dblp computer science bibliography, https://dblp.org},
 biburl = {https://dblp.org/rec/conf/nips/WangIDHKCWMSBH23.bib},
 booktitle = {Advances in Neural Information Processing Systems 36: Annual Conference
on Neural Information Processing Systems 2023, NeurIPS 2023, New Orleans,
LA, USA, December 10 - 16, 2023},
 editor = {Alice Oh and
Tristan Naumann and
Amir Globerson and
Kate Saenko and
Moritz Hardt and
Sergey Levine},
 timestamp = {Fri, 01 Mar 2024 00:00:00 +0100},
 title = {How Far Can Camels Go? Exploring the State of Instruction Tuning on
Open Resources},
 url = {http://papers.nips.cc/paper\_files/paper/2023/hash/ec6413875e4ab08d7bc4d8e225263398-Abstract-Datasets\_and\_Benchmarks.html},
 year = {2023}
}

@article{morimura-etal:2024filtered,
 author = {Morimura, Tetsuro and Sakamoto, Mitsuki and Jinnai, Yuu and Abe, Kenshi and Ariu, Kaito},
 journal = {ArXiv preprint},
 title = {Filtered direct preference optimization},
 url = {https://arxiv.org/abs/2404.13846},
 volume = {abs/2404.13846},
 year = {2024}
}

@inproceedings{cui-etal:2023ultrafeedback,
 author = {Ganqu Cui and
Lifan Yuan and
Ning Ding and
Guanming Yao and
Bingxiang He and
Wei Zhu and
Yuan Ni and
Guotong Xie and
Ruobing Xie and
Yankai Lin and
Zhiyuan Liu and
Maosong Sun},
 bibsource = {dblp computer science bibliography, https://dblp.org},
 biburl = {https://dblp.org/rec/conf/icml/CuiY0YH0NXXL0024.bib},
 booktitle = {Forty-first International Conference on Machine Learning, {ICML} 2024,
Vienna, Austria, July 21-27, 2024},
 publisher = {OpenReview.net},
 timestamp = {Mon, 02 Sep 2024 01:00:00 +0200},
 title = {{ULTRAFEEDBACK:} Boosting Language Models with Scaled {AI} Feedback},
 url = {https://openreview.net/forum?id=BOorDpKHiJ},
 year = {2024}
}

@article{wang-etal:2023learning,
 author = {Wang, Chenglong and Zhou, Hang and Chang, Kaiyan and Liu, Tongran and Zhang, Chunliang and Du, Quan and Xiao, Tong and Zhu, Jingbo},
 journal = {ArXiv preprint},
 title = {Learning evaluation models from large language models for sequence generation},
 url = {https://arxiv.org/abs/2308.04386},
 volume = {abs/2308.04386},
 year = {2023}
}

@inproceedings{sun-etal:2023simple,
 author = {Mingjie Sun and
Zhuang Liu and
Anna Bair and
J. Zico Kolter},
 bibsource = {dblp computer science bibliography, https://dblp.org},
 biburl = {https://dblp.org/rec/conf/iclr/Sun0BK24.bib},
 booktitle = {The Twelfth International Conference on Learning Representations,
{ICLR} 2024, Vienna, Austria, May 7-11, 2024},
 publisher = {OpenReview.net},
 timestamp = {Wed, 07 Aug 2024 01:00:00 +0200},
 title = {A Simple and Effective Pruning Approach for Large Language Models},
 url = {https://openreview.net/forum?id=PxoFut3dWW},
 year = {2024}
}

@article{masoudnia-etal:2014mixture,
 author = {Masoudnia, Saeed and Ebrahimpour, Reza},
 journal = {Artificial Intelligence Review},
 pages = {275--293},
 publisher = {Springer},
 title = {Mixture of experts: a literature survey},
 volume = {42},
 year = {2014}
}

@inproceedings{li-etal:2023remax,
 author = {Ziniu Li and
Tian Xu and
Yushun Zhang and
Zhihang Lin and
Yang Yu and
Ruoyu Sun and
Zhi{-}Quan Luo},
 bibsource = {dblp computer science bibliography, https://dblp.org},
 biburl = {https://dblp.org/rec/conf/icml/LiXZL00L24.bib},
 booktitle = {Forty-first International Conference on Machine Learning, {ICML} 2024,
Vienna, Austria, July 21-27, 2024},
 publisher = {OpenReview.net},
 timestamp = {Sat, 14 Dec 2024 00:00:00 +0100},
 title = {ReMax: {A} Simple, Effective, and Efficient Reinforcement Learning
Method for Aligning Large Language Models},
 url = {https://openreview.net/forum?id=Stn8hXkpe6},
 year = {2024}
}

@article{hu:2025reinforce++,
 author = {Hu, Jian},
 journal = {ArXiv preprint},
 title = {REINFORCE++: A Simple and Efficient Approach for Aligning Large Language Models},
 url = {https://arxiv.org/abs/2501.03262},
 volume = {abs/2501.03262},
 year = {2025}
}

@article{wang-etal:2024hybrid,
 author = {Wang, Chenglong and Zhou, Hang and Chang, Kaiyan and Li, Bei and Mu, Yongyu and Xiao, Tong and Liu, Tongran and Zhu, Jingbo},
 journal = {ArXiv preprint},
 title = {Hybrid Alignment Training for Large Language Models},
 url = {https://arxiv.org/abs/2406.15178},
 volume = {abs/2406.15178},
 year = {2024}
}

@article{sutton-and-richard:1988learning,
 author = {Sutton, Richard S},
 journal = {Machine learning},
 pages = {9--44},
 publisher = {Springer},
 title = {Learning to predict by the methods of temporal differences},
 volume = {3},
 year = {1988}
}

@inproceedings{bahdanau-etal:2016actor,
 author = {Dzmitry Bahdanau and
Philemon Brakel and
Kelvin Xu and
Anirudh Goyal and
Ryan Lowe and
Joelle Pineau and
Aaron C. Courville and
Yoshua Bengio},
 bibsource = {dblp computer science bibliography, https://dblp.org},
 biburl = {https://dblp.org/rec/conf/iclr/BahdanauBXGLPCB17.bib},
 booktitle = {5th International Conference on Learning Representations, {ICLR} 2017,
Toulon, France, April 24-26, 2017, Conference Track Proceedings},
 publisher = {OpenReview.net},
 timestamp = {Thu, 25 Jul 2019 01:00:00 +0200},
 title = {An Actor-Critic Algorithm for Sequence Prediction},
 url = {https://openreview.net/forum?id=SJDaqqveg},
 year = {2017}
}

@article{kumar-etal:2024training,
 author = {Kumar, Aviral and Zhuang, Vincent and Agarwal, Rishabh and Su, Yi and Co-Reyes, John D and Singh, Avi and Baumli, Kate and Iqbal, Shariq and Bishop, Colton and Roelofs, Rebecca and others},
 journal = {ArXiv preprint},
 title = {Training language models to self-correct via reinforcement learning},
 url = {https://arxiv.org/abs/2409.12917},
 volume = {abs/2409.12917},
 year = {2024}
}

@inproceedings{wu-etal:2023fine,
 author = {Zeqiu Wu and
Yushi Hu and
Weijia Shi and
Nouha Dziri and
Alane Suhr and
Prithviraj Ammanabrolu and
Noah A. Smith and
Mari Ostendorf and
Hannaneh Hajishirzi},
 bibsource = {dblp computer science bibliography, https://dblp.org},
 biburl = {https://dblp.org/rec/conf/nips/WuHSDSASOH23.bib},
 booktitle = {Advances in Neural Information Processing Systems 36: Annual Conference
on Neural Information Processing Systems 2023, NeurIPS 2023, New Orleans,
LA, USA, December 10 - 16, 2023},
 editor = {Alice Oh and
Tristan Naumann and
Amir Globerson and
Kate Saenko and
Moritz Hardt and
Sergey Levine},
 timestamp = {Fri, 01 Mar 2024 00:00:00 +0100},
 title = {Fine-Grained Human Feedback Gives Better Rewards for Language Model
Training},
 url = {http://papers.nips.cc/paper\_files/paper/2023/hash/b8c90b65739ae8417e61eadb521f63d5-Abstract-Conference.html},
 year = {2023}
}

@inproceedings{lightman-etal:2023let,
 author = {Hunter Lightman and
Vineet Kosaraju and
Yuri Burda and
Harrison Edwards and
Bowen Baker and
Teddy Lee and
Jan Leike and
John Schulman and
Ilya Sutskever and
Karl Cobbe},
 bibsource = {dblp computer science bibliography, https://dblp.org},
 biburl = {https://dblp.org/rec/conf/iclr/LightmanKBEBLLS24.bib},
 booktitle = {The Twelfth International Conference on Learning Representations,
{ICLR} 2024, Vienna, Austria, May 7-11, 2024},
 publisher = {OpenReview.net},
 timestamp = {Wed, 07 Aug 2024 01:00:00 +0200},
 title = {Let's Verify Step by Step},
 url = {https://openreview.net/forum?id=v8L0pN6EOi},
 year = {2024}
}

@article{bromley-etal:1993signature,
 author = {Bromley, Jane and Guyon, Isabelle and LeCun, Yann and S{\"a}ckinger, Eduard and Shah, Roopak},
 journal = {Advances in neural information processing systems},
 title = {Signature verification using a" siamese" time delay neural network},
 volume = {6},
 year = {1993}
}

@article{singhal-etal:2023long,
 author = {Singhal, Prasann and Goyal, Tanya and Xu, Jiacheng and Durrett, Greg},
 journal = {ArXiv preprint},
 title = {A long way to go: Investigating length correlations in rlhf},
 url = {https://arxiv.org/abs/2310.03716},
 volume = {abs/2310.03716},
 year = {2023}
}

@article{havrilla-etal:2024teaching,
 author = {Havrilla, Alex and Du, Yuqing and Raparthy, Sharath Chandra and Nalmpantis, Christoforos and Dwivedi-Yu, Jane and Zhuravinskyi, Maksym and Hambro, Eric and Sukhbaatar, Sainbayar and Raileanu, Roberta},
 journal = {ArXiv preprint},
 title = {Teaching large language models to reason with reinforcement learning},
 url = {https://arxiv.org/abs/2403.04642},
 volume = {abs/2403.04642},
 year = {2024}
}

@inproceedings{wang-etal:2021selective,
 address = {Online},
 author = {Wang, Fusheng  and
Yan, Jianhao  and
Meng, Fandong  and
Zhou, Jie},
 booktitle = {Proceedings of the 59th Annual Meeting of the Association for Computational Linguistics and the 11th International Joint Conference on Natural Language Processing (Volume 1: Long Papers)},
 doi = {10.18653/v1/2021.acl-long.504},
 editor = {Zong, Chengqing  and
Xia, Fei  and
Li, Wenjie  and
Navigli, Roberto},
 pages = {6456--6466},
 publisher = {Association for Computational Linguistics},
 title = {Selective Knowledge Distillation for Neural Machine Translation},
 url = {https://aclanthology.org/2021.acl-long.504},
 year = {2021}
}

@inproceedings{chen-etal:2023alpagasus,
 author = {Lichang Chen and
Shiyang Li and
Jun Yan and
Hai Wang and
Kalpa Gunaratna and
Vikas Yadav and
Zheng Tang and
Vijay Srinivasan and
Tianyi Zhou and
Heng Huang and
Hongxia Jin},
 bibsource = {dblp computer science bibliography, https://dblp.org},
 biburl = {https://dblp.org/rec/conf/iclr/ChenLYWGYTS0HJ24.bib},
 booktitle = {The Twelfth International Conference on Learning Representations,
{ICLR} 2024, Vienna, Austria, May 7-11, 2024},
 publisher = {OpenReview.net},
 timestamp = {Wed, 07 Aug 2024 01:00:00 +0200},
 title = {AlpaGasus: Training a Better Alpaca with Fewer Data},
 url = {https://openreview.net/forum?id=FdVXgSJhvz},
 year = {2024}
}

@article{li-etal:2025limr,
 author = {Li, Xuefeng and Zou, Haoyang and Liu, Pengfei},
 journal = {ArXiv preprint},
 title = {LIMR: Less is More for RL Scaling},
 url = {https://arxiv.org/abs/2502.11886},
 volume = {abs/2502.11886},
 year = {2025}
}

@inproceedings{wang-and-zhou:2024chain,
 author = {Xuezhi Wang and
Denny Zhou},
 bibsource = {dblp computer science bibliography, https://dblp.org},
 biburl = {https://dblp.org/rec/conf/nips/0002Z24.bib},
 booktitle = {Advances in Neural Information Processing Systems 38: Annual Conference
on Neural Information Processing Systems 2024, NeurIPS 2024, Vancouver,
BC, Canada, December 10 - 15, 2024},
 editor = {Amir Globersons and
Lester Mackey and
Danielle Belgrave and
Angela Fan and
Ulrich Paquet and
Jakub M. Tomczak and
Cheng Zhang},
 timestamp = {Thu, 13 Feb 2025 00:00:00 +0100},
 title = {Chain-of-Thought Reasoning Without Prompting},
 url = {http://papers.nips.cc/paper\_files/paper/2024/hash/7a8e7fd295aa04eac4b470ae27f8785c-Abstract-Conference.html},
 year = {2024}
}

@article{chen-etal:2023accelerating,
 author = {Chen, Charlie and Borgeaud, Sebastian and Irving, Geoffrey and Lespiau, Jean-Baptiste and Sifre, Laurent and Jumper, John},
 journal = {ArXiv preprint},
 title = {Accelerating large language model decoding with speculative sampling},
 url = {https://arxiv.org/abs/2302.01318},
 volume = {abs/2302.01318},
 year = {2023}
}

@article{zhao-etal:2024atom,
 author = {Zhao, Yilong and Lin, Chien-Yu and Zhu, Kan and Ye, Zihao and Chen, Lequn and Zheng, Size and Ceze, Luis and Krishnamurthy, Arvind and Chen, Tianqi and Kasikci, Baris},
 journal = {Proceedings of Machine Learning and Systems},
 pages = {196--209},
 title = {Atom: Low-bit quantization for efficient and accurate llm serving},
 volume = {6},
 year = {2024}
}

@article{pope-etal:2023efficiently,
 author = {Pope, Reiner and Douglas, Sholto and Chowdhery, Aakanksha and Devlin, Jacob and Bradbury, James and Heek, Jonathan and Xiao, Kefan and Agrawal, Shivani and Dean, Jeff},
 journal = {Proceedings of Machine Learning and Systems},
 pages = {606--624},
 title = {Efficiently scaling transformer inference},
 volume = {5},
 year = {2023}
}

@inproceedings{vaswani-etal:2017attention,
 author = {Ashish Vaswani and
Noam Shazeer and
Niki Parmar and
Jakob Uszkoreit and
Llion Jones and
Aidan N. Gomez and
Lukasz Kaiser and
Illia Polosukhin},
 bibsource = {dblp computer science bibliography, https://dblp.org},
 biburl = {https://dblp.org/rec/conf/nips/VaswaniSPUJGKP17.bib},
 booktitle = {Advances in Neural Information Processing Systems 30: Annual Conference
on Neural Information Processing Systems 2017, December 4-9, 2017,
Long Beach, CA, {USA}},
 editor = {Isabelle Guyon and
Ulrike von Luxburg and
Samy Bengio and
Hanna M. Wallach and
Rob Fergus and
S. V. N. Vishwanathan and
Roman Garnett},
 pages = {5998--6008},
 timestamp = {Thu, 21 Jan 2021 00:00:00 +0100},
 title = {Attention is All you Need},
 url = {https://proceedings.neurips.cc/paper/2017/hash/3f5ee243547dee91fbd053c1c4a845aa-Abstract.html},
 year = {2017}
}

@inproceedings{schulman-etal:2015high,
 author = {John Schulman and
Philipp Moritz and
Sergey Levine and
Michael I. Jordan and
Pieter Abbeel},
 bibsource = {dblp computer science bibliography, https://dblp.org},
 biburl = {https://dblp.org/rec/journals/corr/SchulmanMLJA15.bib},
 booktitle = {4th International Conference on Learning Representations, {ICLR} 2016,
San Juan, Puerto Rico, May 2-4, 2016, Conference Track Proceedings},
 editor = {Yoshua Bengio and
Yann LeCun},
 timestamp = {Thu, 25 Jul 2019 01:00:00 +0200},
 title = {High-Dimensional Continuous Control Using Generalized Advantage Estimation},
 url = {http://arxiv.org/abs/1506.02438},
 year = {2016}
}

@article{wang-etal:2023large,
 author = {Wang, Peiyi and Li, Lei and Chen, Liang and Cai, Zefan and Zhu, Dawei and Lin, Binghuai and Cao, Yunbo and Liu, Qi and Liu, Tianyu and Sui, Zhifang},
 journal = {ArXiv preprint},
 title = {Large language models are not fair evaluators},
 year = {2023}
}

@article{mahan-etal:2024generative,
 author = {Mahan, Dakota and Van Phung, Duy and Rafailov, Rafael and Blagden, Chase and Lile, Nathan and Castricato, Louis and Fr{\"a}nken, Jan-Philipp and Finn, Chelsea and Albalak, Alon},
 journal = {ArXiv preprint},
 title = {Generative reward models},
 url = {https://arxiv.org/abs/2410.12832},
 volume = {abs/2410.12832},
 year = {2024}
}

@article{zhang-etal:2024generative,
 author = {Zhang, Lunjun and Hosseini, Arian and Bansal, Hritik and Kazemi, Mehran and Kumar, Aviral and Agarwal, Rishabh},
 journal = {ArXiv preprint},
 title = {Generative verifiers: Reward modeling as next-token prediction},
 url = {https://arxiv.org/abs/2408.15240},
 volume = {abs/2408.15240},
 year = {2024}
}

@inproceedings{yang-etal:2024regularizing,
 author = {Rui Yang and
Ruomeng Ding and
Yong Lin and
Huan Zhang and
Tong Zhang},
 bibsource = {dblp computer science bibliography, https://dblp.org},
 biburl = {https://dblp.org/rec/conf/nips/YangDLZZ24.bib},
 booktitle = {Advances in Neural Information Processing Systems 38: Annual Conference
on Neural Information Processing Systems 2024, NeurIPS 2024, Vancouver,
BC, Canada, December 10 - 15, 2024},
 editor = {Amir Globersons and
Lester Mackey and
Danielle Belgrave and
Angela Fan and
Ulrich Paquet and
Jakub M. Tomczak and
Cheng Zhang},
 timestamp = {Thu, 13 Feb 2025 00:00:00 +0100},
 title = {Regularizing Hidden States Enables Learning Generalizable Reward Model
for LLMs},
 url = {http://papers.nips.cc/paper\_files/paper/2024/hash/71f7154547c748c8041505521ca433ab-Abstract-Conference.html},
 year = {2024}
}

@book{Bishop:2006,
 author = {Christopher M. Bishop},
 publisher = {Springer},
 title = {Pattern Recognition and Machine Learning},
 year = {2006}
}

@book{miettinen:1999nonlinear,
 author = {Miettinen, Kaisa},
 publisher = {Springer Science \& Business Media},
 title = {Nonlinear multiobjective optimization},
 volume = {12},
 year = {1999}
}

@article{lin-etal:2024dogerm,
 author = {Lin, Tzu-Han and Li, Chen-An and Lee, Hung-yi and Chen, Yun-Nung},
 journal = {ArXiv preprint},
 title = {Dogerm: Equipping reward models with domain knowledge through model merging},
 url = {https://arxiv.org/abs/2407.01470},
 volume = {abs/2407.01470},
 year = {2024}
}

@inproceedings{coste-etal:2024reward,
 author = {Thomas Coste and
Usman Anwar and
Robert Kirk and
David Krueger},
 bibsource = {dblp computer science bibliography, https://dblp.org},
 biburl = {https://dblp.org/rec/conf/iclr/CosteAK024.bib},
 booktitle = {The Twelfth International Conference on Learning Representations,
{ICLR} 2024, Vienna, Austria, May 7-11, 2024},
 publisher = {OpenReview.net},
 timestamp = {Wed, 07 Aug 2024 01:00:00 +0200},
 title = {Reward Model Ensembles Help Mitigate Overoptimization},
 url = {https://openreview.net/forum?id=dcjtMYkpXx},
 year = {2024}
}

@article{eisenstein-etal:2023helping,
 author = {Eisenstein, Jacob and Nagpal, Chirag and Agarwal, Alekh and Beirami, Ahmad and D'Amour, Alex and Dvijotham, DJ and Fisch, Adam and Heller, Katherine and Pfohl, Stephen and Ramachandran, Deepak and others},
 journal = {ArXiv preprint},
 title = {Helping or herding? reward model ensembles mitigate but do not eliminate reward hacking},
 url = {https://arxiv.org/abs/2312.09244},
 volume = {abs/2312.09244},
 year = {2023}
}

@inproceedings{gao-etal:2023scaling,
 author = {Leo Gao and
John Schulman and
Jacob Hilton},
 bibsource = {dblp computer science bibliography, https://dblp.org},
 biburl = {https://dblp.org/rec/conf/icml/GaoSH23.bib},
 booktitle = {International Conference on Machine Learning, {ICML} 2023, 23-29 July
2023, Honolulu, Hawaii, {USA}},
 editor = {Andreas Krause and
Emma Brunskill and
Kyunghyun Cho and
Barbara Engelhardt and
Sivan Sabato and
Jonathan Scarlett},
 pages = {10835--10866},
 publisher = {{PMLR}},
 series = {Proceedings of Machine Learning Research},
 timestamp = {Mon, 28 Aug 2023 01:00:00 +0200},
 title = {Scaling Laws for Reward Model Overoptimization},
 url = {https://proceedings.mlr.press/v202/gao23h.html},
 volume = {202},
 year = {2023}
}

@inproceedings{liu-etal:2023GEval,
 address = {Singapore},
 author = {Liu, Yang  and
Iter, Dan  and
Xu, Yichong  and
Wang, Shuohang  and
Xu, Ruochen  and
Zhu, Chenguang},
 booktitle = {Proceedings of the 2023 Conference on Empirical Methods in Natural Language Processing},
 doi = {10.18653/v1/2023.emnlp-main.153},
 editor = {Bouamor, Houda  and
Pino, Juan  and
Bali, Kalika},
 pages = {2511--2522},
 publisher = {Association for Computational Linguistics},
 title = {{G}-Eval: {NLG} Evaluation using Gpt-4 with Better Human Alignment},
 url = {https://aclanthology.org/2023.emnlp-main.153},
 year = {2023}
}

@inproceedings{zheng-etal:2023judging,
 author = {Lianmin Zheng and
Wei{-}Lin Chiang and
Ying Sheng and
Siyuan Zhuang and
Zhanghao Wu and
Yonghao Zhuang and
Zi Lin and
Zhuohan Li and
Dacheng Li and
Eric P. Xing and
Hao Zhang and
Joseph E. Gonzalez and
Ion Stoica},
 bibsource = {dblp computer science bibliography, https://dblp.org},
 biburl = {https://dblp.org/rec/conf/nips/ZhengC00WZL0LXZ23.bib},
 booktitle = {Advances in Neural Information Processing Systems 36: Annual Conference
on Neural Information Processing Systems 2023, NeurIPS 2023, New Orleans,
LA, USA, December 10 - 16, 2023},
 editor = {Alice Oh and
Tristan Naumann and
Amir Globerson and
Kate Saenko and
Moritz Hardt and
Sergey Levine},
 timestamp = {Thu, 04 Jul 2024 01:00:00 +0200},
 title = {Judging LLM-as-a-Judge with MT-Bench and Chatbot Arena},
 url = {http://papers.nips.cc/paper\_files/paper/2023/hash/91f18a1287b398d378ef22505bf41832-Abstract-Datasets\_and\_Benchmarks.html},
 year = {2023}
}

@inproceedings{ng1999-etal:policy,
 author = {Ng, Andrew Y and Harada, Daishi and Russell, Stuart J},
 booktitle = {Proceedings of the Sixteenth International Conference on Machine Learning},
 pages = {278--287},
 title = {Policy Invariance Under Reward Transformations: Theory and Application to Reward Shaping},
 year = {1999}
}

@inproceedings{dubois-etal:2024alpacafarm,
 author = {Yann Dubois and
Chen Xuechen Li and
Rohan Taori and
Tianyi Zhang and
Ishaan Gulrajani and
Jimmy Ba and
Carlos Guestrin and
Percy Liang and
Tatsunori B. Hashimoto},
 bibsource = {dblp computer science bibliography, https://dblp.org},
 biburl = {https://dblp.org/rec/conf/nips/DuboisLTZGBGLH23.bib},
 booktitle = {Advances in Neural Information Processing Systems 36: Annual Conference
on Neural Information Processing Systems 2023, NeurIPS 2023, New Orleans,
LA, USA, December 10 - 16, 2023},
 editor = {Alice Oh and
Tristan Naumann and
Amir Globerson and
Kate Saenko and
Moritz Hardt and
Sergey Levine},
 timestamp = {Fri, 01 Mar 2024 00:00:00 +0100},
 title = {AlpacaFarm: {A} Simulation Framework for Methods that Learn from Human
Feedback},
 url = {http://papers.nips.cc/paper\_files/paper/2023/hash/5fc47800ee5b30b8777fdd30abcaaf3b-Abstract-Conference.html},
 year = {2023}
}

@inproceedings{cui-etal:2024ultra,
 author = {Ganqu Cui and
Lifan Yuan and
Ning Ding and
Guanming Yao and
Bingxiang He and
Wei Zhu and
Yuan Ni and
Guotong Xie and
Ruobing Xie and
Yankai Lin and
Zhiyuan Liu and
Maosong Sun},
 bibsource = {dblp computer science bibliography, https://dblp.org},
 biburl = {https://dblp.org/rec/conf/icml/CuiY0YH0NXXL0024.bib},
 booktitle = {Forty-first International Conference on Machine Learning, {ICML} 2024,
Vienna, Austria, July 21-27, 2024},
 publisher = {OpenReview.net},
 timestamp = {Mon, 02 Sep 2024 01:00:00 +0200},
 title = {{ULTRAFEEDBACK:} Boosting Language Models with Scaled {AI} Feedback},
 url = {https://openreview.net/forum?id=BOorDpKHiJ},
 year = {2024}
}

@article{lee-etal:2023rlaif,
 author = {Lee, Harrison and Phatale, Samrat and Mansoor, Hassan and Lu, Kellie Ren and Mesnard, Thomas and Ferret, Johan and Bishop, Colton and Hall, Ethan and Carbune, Victor and Rastogi, Abhinav},
 journal = {ArXiv preprint},
 title = {Rlaif: Scaling reinforcement learning from human feedback with AI feedback},
 url = {https://arxiv.org/abs/2309.00267},
 volume = {abs/2309.00267},
 year = {2023}
}

@article{shao-etal:2024deepseekmath,
 author = {Shao, Zhihong and Wang, Peiyi and Zhu, Qihao and Xu, Runxin and Song, Junxiao and Bi, Xiao and Zhang, Haowei and Zhang, Mingchuan and Li, YK and Wu, Y and others},
 journal = {ArXiv preprint},
 title = {Deepseekmath: Pushing the limits of mathematical reasoning in open language models},
 url = {https://arxiv.org/abs/2402.03300},
 volume = {abs/2402.03300},
 year = {2024}
}

@inproceedings{wang-etal:2024esrl,
 author = {Chenglong Wang and
Hang Zhou and
Yimin Hu and
Yifu Huo and
Bei Li and
Tongran Liu and
Tong Xiao and
Jingbo Zhu},
 bibsource = {dblp computer science bibliography, https://dblp.org},
 biburl = {https://dblp.org/rec/conf/aaai/WangZHHLLXZ24.bib},
 booktitle = {Thirty-Eighth {AAAI} Conference on Artificial Intelligence, {AAAI}
2024, Thirty-Sixth Conference on Innovative Applications of Artificial
Intelligence, {IAAI} 2024, Fourteenth Symposium on Educational Advances
in Artificial Intelligence, {EAAI} 2014, February 20-27, 2024, Vancouver,
Canada},
 doi = {10.1609/AAAI.V38I17.29878},
 editor = {Michael J. Wooldridge and
Jennifer G. Dy and
Sriraam Natarajan},
 pages = {19107--19115},
 publisher = {{AAAI} Press},
 timestamp = {Tue, 02 Apr 2024 01:00:00 +0200},
 title = {{ESRL:} Efficient Sampling-Based Reinforcement Learning for Sequence
Generation},
 url = {https://doi.org/10.1609/aaai.v38i17.29878},
 year = {2024}
}

@inproceedings{schulman-etal:2015trust,
 author = {John Schulman and
Sergey Levine and
Pieter Abbeel and
Michael I. Jordan and
Philipp Moritz},
 bibsource = {dblp computer science bibliography, https://dblp.org},
 biburl = {https://dblp.org/rec/conf/icml/SchulmanLAJM15.bib},
 booktitle = {Proceedings of the 32nd International Conference on Machine Learning,
{ICML} 2015, Lille, France, 6-11 July 2015},
 editor = {Francis R. Bach and
David M. Blei},
 pages = {1889--1897},
 publisher = {JMLR.org},
 series = {{JMLR} Workshop and Conference Proceedings},
 timestamp = {Wed, 29 May 2019 01:00:00 +0200},
 title = {Trust Region Policy Optimization},
 url = {http://proceedings.mlr.press/v37/schulman15.html},
 volume = {37},
 year = {2015}
}

@article{schulman-etal:2017proximal,
 author = {Schulman, John and Wolski, Filip and Dhariwal, Prafulla and Radford, Alec and Klimov, Oleg},
 journal = {ArXiv preprint},
 title = {Proximal policy optimization algorithms},
 url = {https://arxiv.org/abs/1707.06347},
 volume = {abs/1707.06347},
 year = {2017}
}

@article{bradley-and-terry:rank,
 author = {Ralph Allan Bradley and Milton E. Terry},
 journal = {Biometrika},
 number = {3/4},
 pages = {324--345},
 title = {Rank Analysis of Incomplete Block Designs: I. The Method of Paired Comparisons},
 volume = {39},
 year = {1952}
}

@inproceedings{stiennon-etal:2020learning,
 author = {Nisan Stiennon and
Long Ouyang and
Jeffrey Wu and
Daniel M. Ziegler and
Ryan Lowe and
Chelsea Voss and
Alec Radford and
Dario Amodei and
Paul F. Christiano},
 bibsource = {dblp computer science bibliography, https://dblp.org},
 biburl = {https://dblp.org/rec/conf/nips/StiennonO0ZLVRA20.bib},
 booktitle = {Advances in Neural Information Processing Systems 33: Annual Conference
on Neural Information Processing Systems 2020, NeurIPS 2020, December
6-12, 2020, virtual},
 editor = {Hugo Larochelle and
Marc'Aurelio Ranzato and
Raia Hadsell and
Maria{-}Florina Balcan and
Hsuan{-}Tien Lin},
 timestamp = {Tue, 19 Jan 2021 00:00:00 +0100},
 title = {Learning to summarize with human feedback},
 url = {https://proceedings.neurips.cc/paper/2020/hash/1f89885d556929e98d3ef9b86448f951-Abstract.html},
 year = {2020}
}

@inproceedings{christiano-etal:2017deep,
 author = {Paul F. Christiano and
Jan Leike and
Tom B. Brown and
Miljan Martic and
Shane Legg and
Dario Amodei},
 bibsource = {dblp computer science bibliography, https://dblp.org},
 biburl = {https://dblp.org/rec/conf/nips/ChristianoLBMLA17.bib},
 booktitle = {Advances in Neural Information Processing Systems 30: Annual Conference
on Neural Information Processing Systems 2017, December 4-9, 2017,
Long Beach, CA, {USA}},
 editor = {Isabelle Guyon and
Ulrike von Luxburg and
Samy Bengio and
Hanna M. Wallach and
Rob Fergus and
S. V. N. Vishwanathan and
Roman Garnett},
 pages = {4299--4307},
 timestamp = {Thu, 21 Jan 2021 00:00:00 +0100},
 title = {Deep Reinforcement Learning from Human Preferences},
 url = {https://proceedings.neurips.cc/paper/2017/hash/d5e2c0adad503c91f91df240d0cd4e49-Abstract.html},
 year = {2017}
}

@inproceedings{wei-etal:2022finetuned,
 author = {Jason Wei and
Maarten Bosma and
Vincent Y. Zhao and
Kelvin Guu and
Adams Wei Yu and
Brian Lester and
Nan Du and
Andrew M. Dai and
Quoc V. Le},
 bibsource = {dblp computer science bibliography, https://dblp.org},
 biburl = {https://dblp.org/rec/conf/iclr/WeiBZGYLDDL22.bib},
 booktitle = {The Tenth International Conference on Learning Representations, {ICLR}
2022, Virtual Event, April 25-29, 2022},
 publisher = {OpenReview.net},
 timestamp = {Sat, 20 Aug 2022 01:00:00 +0200},
 title = {Finetuned Language Models are Zero-Shot Learners},
 url = {https://openreview.net/forum?id=gEZrGCozdqR},
 year = {2022}
}

@article{havrilla:2024teaching,
 author = {Havrilla, Alex and Du, Yuqing and Raparthy, Sharath Chandra and Nalmpantis, Christoforos and Dwivedi-Yu, Jane and Zhuravinskyi, Maksym and Hambro, Eric and Sukhbaatar, Sainbayar and Raileanu, Roberta},
 journal = {ArXiv preprint},
 title = {Teaching large language models to reason with reinforcement learning},
 url = {https://arxiv.org/abs/2403.04642},
 volume = {abs/2403.04642},
 year = {2024}
}

@inproceedings{shen:2015minimum,
 address = {Berlin, Germany},
 author = {Shen, Shiqi  and
Cheng, Yong  and
He, Zhongjun  and
He, Wei  and
Wu, Hua  and
Sun, Maosong  and
Liu, Yang},
 booktitle = {Proceedings of the 54th Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers)},
 doi = {10.18653/v1/P16-1159},
 editor = {Erk, Katrin  and
Smith, Noah A.},
 pages = {1683--1692},
 publisher = {Association for Computational Linguistics},
 title = {Minimum Risk Training for Neural Machine Translation},
 url = {https://aclanthology.org/P16-1159},
 year = {2016}
}

@article{guo:2025deepseek,
 author = {Guo, Daya and Yang, Dejian and Zhang, Haowei and Song, Junxiao and Zhang, Ruoyu and Xu, Runxin and Zhu, Qihao and Ma, Shirong and Wang, Peiyi and Bi, Xiao and others},
 journal = {ArXiv preprint},
 title = {Deepseek-r1: Incentivizing reasoning capability in llms via reinforcement learning},
 url = {https://arxiv.org/abs/2501.12948},
 volume = {abs/2501.12948},
 year = {2025}
}

@article{touvron:2023llama,
 author = {Touvron, Hugo and Martin, Louis and Stone, Kevin and Albert, Peter and Almahairi, Amjad and Babaei, Yasmine and Bashlykov, Nikolay and Batra, Soumya and Bhargava, Prajjwal and Bhosale, Shruti and others},
 journal = {ArXiv preprint},
 title = {Llama 2: Open foundation and fine-tuned chat models},
 url = {https://arxiv.org/abs/2307.09288},
 volume = {abs/2307.09288},
 year = {2023}
}

@inproceedings{ouyang:2022training,
 author = {Long Ouyang and
Jeffrey Wu and
Xu Jiang and
Diogo Almeida and
Carroll L. Wainwright and
Pamela Mishkin and
Chong Zhang and
Sandhini Agarwal and
Katarina Slama and
Alex Ray and
John Schulman and
Jacob Hilton and
Fraser Kelton and
Luke Miller and
Maddie Simens and
Amanda Askell and
Peter Welinder and
Paul F. Christiano and
Jan Leike and
Ryan Lowe},
 bibsource = {dblp computer science bibliography, https://dblp.org},
 biburl = {https://dblp.org/rec/conf/nips/Ouyang0JAWMZASR22.bib},
 booktitle = {Advances in Neural Information Processing Systems 35: Annual Conference
on Neural Information Processing Systems 2022, NeurIPS 2022, New Orleans,
LA, USA, November 28 - December 9, 2022},
 editor = {Sanmi Koyejo and
S. Mohamed and
A. Agarwal and
Danielle Belgrave and
K. Cho and
A. Oh},
 timestamp = {Mon, 08 Jan 2024 00:00:00 +0100},
 title = {Training language models to follow instructions with human feedback},
 url = {http://papers.nips.cc/paper\_files/paper/2022/hash/b1efde53be364a73914f58805a001731-Abstract-Conference.html},
 year = {2022}
}

@inproceedings{bahdanau:2016actor,
 author = {Dzmitry Bahdanau and
Philemon Brakel and
Kelvin Xu and
Anirudh Goyal and
Ryan Lowe and
Joelle Pineau and
Aaron C. Courville and
Yoshua Bengio},
 bibsource = {dblp computer science bibliography, https://dblp.org},
 biburl = {https://dblp.org/rec/conf/iclr/BahdanauBXGLPCB17.bib},
 booktitle = {5th International Conference on Learning Representations, {ICLR} 2017,
Toulon, France, April 24-26, 2017, Conference Track Proceedings},
 publisher = {OpenReview.net},
 timestamp = {Thu, 25 Jul 2019 01:00:00 +0200},
 title = {An Actor-Critic Algorithm for Sequence Prediction},
 url = {https://openreview.net/forum?id=SJDaqqveg},
 year = {2017}
}

@inproceedings{mnih-etal:2016asynchronous,
 author = {Volodymyr Mnih and
Adri{\`{a}} Puigdom{\`{e}}nech Badia and
Mehdi Mirza and
Alex Graves and
Timothy P. Lillicrap and
Tim Harley and
David Silver and
Koray Kavukcuoglu},
 bibsource = {dblp computer science bibliography, https://dblp.org},
 biburl = {https://dblp.org/rec/conf/icml/MnihBMGLHSK16.bib},
 booktitle = {Proceedings of the 33nd International Conference on Machine Learning,
{ICML} 2016, New York City, NY, USA, June 19-24, 2016},
 editor = {Maria{-}Florina Balcan and
Kilian Q. Weinberger},
 pages = {1928--1937},
 publisher = {JMLR.org},
 series = {{JMLR} Workshop and Conference Proceedings},
 timestamp = {Wed, 29 May 2019 01:00:00 +0200},
 title = {Asynchronous Methods for Deep Reinforcement Learning},
 url = {http://proceedings.mlr.press/v48/mniha16.html},
 volume = {48},
 year = {2016}
}

@article{szepesvari:2010algorithms,
 author = {Szepesv{\'a}ri, Csaba},
 journal = {Synthesis Lectures on Artificial Intelligence and Machine Learning},
 number = {1},
 pages = {1--103},
 publisher = {Springer Science and Business Media LLC},
 title = {Algorithms for Reinforcement Learning},
 volume = {4},
 year = {2010}
}

@article{williams:1992simple,
 author = {Williams, Ronald J},
 journal = {Machine learning},
 pages = {229--256},
 publisher = {Springer},
 title = {Simple statistical gradient-following algorithms for connectionist reinforcement learning},
 volume = {8},
 year = {1992}
}

@book{Sutton-and-Barto:2018RL,
 author = {Richard S. Sutton and Andrew G. Barto},
 publisher = {The MIT Press},
 title = {Reinforcement Learning: An Introduction (2nd ed.)},
 year = {2018}
}

@article{bradley:1952rank,
 author = {Bradley, Ralph Allan and Terry, Milton E},
 journal = {Biometrika},
 number = {3/4},
 pages = {324--345},
 publisher = {JSTOR},
 title = {Rank analysis of incomplete block designs: I. The method of paired comparisons},
 volume = {39},
 year = {1952}
}

@inproceedings{rafailov:2023direct,
 author = {Rafael Rafailov and
Archit Sharma and
Eric Mitchell and
Christopher D. Manning and
Stefano Ermon and
Chelsea Finn},
 bibsource = {dblp computer science bibliography, https://dblp.org},
 biburl = {https://dblp.org/rec/conf/nips/RafailovSMMEF23.bib},
 booktitle = {Advances in Neural Information Processing Systems 36: Annual Conference
on Neural Information Processing Systems 2023, NeurIPS 2023, New Orleans,
LA, USA, December 10 - 16, 2023},
 editor = {Alice Oh and
Tristan Naumann and
Amir Globerson and
Kate Saenko and
Moritz Hardt and
Sergey Levine},
 timestamp = {Fri, 01 Mar 2024 00:00:00 +0100},
 title = {Direct Preference Optimization: Your Language Model is Secretly a
Reward Model},
 url = {http://papers.nips.cc/paper\_files/paper/2023/hash/a85b405ed65c6477a4fe8302b5e06ce7-Abstract-Conference.html},
 year = {2023}
}

@inproceedings{azar:2024general,
 author = {Mohammad Gheshlaghi Azar and
Zhaohan Daniel Guo and
Bilal Piot and
R{\'{e}}mi Munos and
Mark Rowland and
Michal Valko and
Daniele Calandriello},
 bibsource = {dblp computer science bibliography, https://dblp.org},
 biburl = {https://dblp.org/rec/conf/aistats/AzarGPMRVC24.bib},
 booktitle = {International Conference on Artificial Intelligence and Statistics,
2-4 May 2024, Palau de Congressos, Valencia, Spain},
 editor = {Sanjoy Dasgupta and
Stephan Mandt and
Yingzhen Li},
 pages = {4447--4455},
 publisher = {{PMLR}},
 series = {Proceedings of Machine Learning Research},
 timestamp = {Mon, 13 May 2024 01:00:00 +0200},
 title = {A General Theoretical Paradigm to Understand Learning from Human Preferences},
 url = {https://proceedings.mlr.press/v238/gheshlaghi-azar24a.html},
 volume = {238},
 year = {2024}
}

@inproceedings{xu:2024contrastive,
 author = {Haoran Xu and
Amr Sharaf and
Yunmo Chen and
Weiting Tan and
Lingfeng Shen and
Benjamin Van Durme and
Kenton Murray and
Young Jin Kim},
 bibsource = {dblp computer science bibliography, https://dblp.org},
 biburl = {https://dblp.org/rec/conf/icml/XuSCTSDM024.bib},
 booktitle = {Forty-first International Conference on Machine Learning, {ICML} 2024,
Vienna, Austria, July 21-27, 2024},
 publisher = {OpenReview.net},
 timestamp = {Mon, 02 Sep 2024 01:00:00 +0200},
 title = {Contrastive Preference Optimization: Pushing the Boundaries of {LLM}
Performance in Machine Translation},
 url = {https://openreview.net/forum?id=51iwkioZpn},
 year = {2024}
}

@article{ethayarajh:2024kto,
 author = {Ethayarajh, Kawin and Xu, Winnie and Muennighoff, Niklas and Jurafsky, Dan and Kiela, Douwe},
 journal = {ArXiv preprint},
 title = {Kto: Model alignment as prospect theoretic optimization},
 url = {https://arxiv.org/abs/2402.01306},
 volume = {abs/2402.01306},
 year = {2024}
}

@article{hong:2024orpo,
 author = {Hong, Jiwoo and Lee, Noah and Thorne, James},
 journal = {ArXiv preprint},
 title = {Orpo: Monolithic preference optimization without reference model},
 url = {https://arxiv.org/abs/2403.07691},
 volume = {abs/2403.07691},
 year = {2024}
}

@article{gallego:2024refined,
 author = {Gallego, V{\'\i}ctor},
 journal = {ArXiv preprint},
 title = {Refined direct preference optimization with synthetic data for behavioral alignment of llms},
 url = {https://arxiv.org/abs/2402.08005},
 volume = {abs/2402.08005},
 year = {2024}
}

@inproceedings{meng:2025simpo,
 author = {Yu Meng and
Mengzhou Xia and
Danqi Chen},
 bibsource = {dblp computer science bibliography, https://dblp.org},
 biburl = {https://dblp.org/rec/conf/nips/0001X024.bib},
 booktitle = {Advances in Neural Information Processing Systems 38: Annual Conference
on Neural Information Processing Systems 2024, NeurIPS 2024, Vancouver,
BC, Canada, December 10 - 15, 2024},
 editor = {Amir Globersons and
Lester Mackey and
Danielle Belgrave and
Angela Fan and
Ulrich Paquet and
Jakub M. Tomczak and
Cheng Zhang},
 timestamp = {Thu, 13 Feb 2025 00:00:00 +0100},
 title = {SimPO: Simple Preference Optimization with a Reference-Free Reward},
 url = {http://papers.nips.cc/paper\_files/paper/2024/hash/e099c1c9699814af0be873a175361713-Abstract-Conference.html},
 year = {2024}
}

@inproceedings{zhou:2024prior,
 author = {Zhou, Hang and Wang, Chenglong and Hu, Yimin and Xiao, Tong and Zhang, Chunliang and Zhu, Jingbo},
 booktitle = {China National Conference on Chinese Computational Linguistics},
 organization = {Springer},
 pages = {555--570},
 title = {Prior constraints-based reward model training for aligning large language models},
 year = {2024}
}

@article{shao:2025earlier,
 author = {Shao, Ruichen and Li, Bei and Liu, Gangao and Chen, Yang and Zhou, Xiang and Wang, Jingang and Cai, Xunliang and Li, Peng},
 journal = {ArXiv preprint},
 title = {Earlier Tokens Contribute More: Learning Direct Preference Optimization From Temporal Decay Perspective},
 url = {https://arxiv.org/abs/2502.14340},
 volume = {abs/2502.14340},
 year = {2025}
}

@misc{openai:2024learning,
    title = {Learning to Reason with LLMs},
    url = {https://openai.com/index/learning-to-reason-with-llms/},
    author = {OpenAI},
    month = {September},
    year = {2024}
}

@article{deepseek:2025r1,
    title={Deepseek-r1: Incentivizing reasoning capability in llms via reinforcement learning},
    author={Deepseek},
    journal={arXiv preprint arXiv:2501.12948},
    year={2025}
}