Skip to content

R
rl-introduction
Commit 7689c0d3 by wangchenglong
Options

update.

parent e0398fef
正在显示 包含 0 行增加 和 132 行删除
rl-introduction.blg deleted 100644 → 0
This is BibTeX, Version 0.99e (TeX Live 2026)
Capacity: max_strings=200000, hash_size=200000, hash_prime=170003
The top-level auxiliary file: rl-introduction.aux
The style file: rl-introduction.bst
Database file #1: rl-introduction.bib
Warning--can't use both volume and number fields in wang-etal:wang2026gram
You've used 98 entries,
2773 wiz_defined-function locations,
1106 strings with 33536 characters,
and the built_in function-call counts, 71100 in all, are:
= -- 5525
> -- 7412
< -- 34
+ -- 2481
- -- 2372
* -- 6800
:= -- 11734
add.period$ -- 380
call.type$ -- 98
change.case$ -- 1032
chr.to.int$ -- 87
cite$ -- 197
duplicate$ -- 2238
empty$ -- 3704
format.name$ -- 2482
if$ -- 14216
int.to.chr$ -- 12
int.to.str$ -- 1
missing$ -- 98
newline$ -- 562
num.names$ -- 430
pop$ -- 2052
preamble$ -- 1
purify$ -- 937
quote$ -- 0
skip$ -- 2138
stack$ -- 0
substring$ -- 926
swap$ -- 304
text.length$ -- 19
text.prefix$ -- 0
top$ -- 0
type$ -- 1063
warning$ -- 1
while$ -- 370
width$ -- 0
write$ -- 1394
(There was 1 warning)
rl-introduction.out deleted 100644 → 0
\BOOKMARK [1][-]{section.1}{\376\377\000I\000n\000t\000r\000o\000d\000u\000c\000t\000i\000o\000n}{}% 1
\BOOKMARK [1][-]{section.2}{\376\377\000P\000r\000e\000l\000i\000m\000i\000n\000a\000r\000y}{}% 2
\BOOKMARK [2][-]{subsection.2.1}{\376\377\000F\000u\000n\000d\000a\000m\000e\000n\000t\000a\000l\000s\000\040\000o\000f\000\040\000L\000a\000r\000g\000e\000\040\000L\000a\000n\000g\000u\000a\000g\000e\000\040\000M\000o\000d\000e\000l\000s}{section.2}% 3
\BOOKMARK [2][-]{subsection.2.2}{\376\377\000F\000u\000n\000d\000a\000m\000e\000n\000t\000a\000l\000s\000\040\000o\000f\000\040\000R\000e\000i\000n\000f\000o\000r\000c\000e\000m\000e\000n\000t\000\040\000L\000e\000a\000r\000n\000i\000n\000g}{section.2}% 4
\BOOKMARK [3][-]{subsubsection.2.2.1}{\376\377\000G\000e\000n\000e\000r\000a\000l\000\040\000F\000r\000a\000m\000e\000w\000o\000r\000k\000\040\000o\000f\000\040\000R\000e\000i\000n\000f\000o\000r\000c\000e\000m\000e\000n\000t\000\040\000L\000e\000a\000r\000n\000i\000n\000g}{subsection.2.2}% 5
\BOOKMARK [3][-]{subsubsection.2.2.2}{\376\377\000K\000e\000y\000\040\000E\000l\000e\000m\000e\000n\000t\000s}{subsection.2.2}% 6
\BOOKMARK [1][-]{section.3}{\376\377\000A\000n\000\040\000E\000x\000a\000m\000p\000l\000e\000\040\000o\000f\000\040\000U\000s\000i\000n\000g\000\040\000R\000e\000i\000n\000f\000o\000r\000c\000e\000m\000e\000n\000t\000\040\000L\000e\000a\000r\000n\000i\000n\000g\000\040\000t\000o\000\040\000T\000r\000a\000i\000n\000\040\000L\000L\000M\000s}{}% 7
\BOOKMARK [2][-]{subsection.3.1}{\376\377\000P\000o\000l\000i\000c\000y\000\040\000G\000r\000a\000d\000i\000e\000n\000t}{section.3}% 8
\BOOKMARK [2][-]{subsection.3.2}{\376\377\000T\000e\000m\000p\000o\000r\000a\000l\000\040\000D\000e\000c\000o\000m\000p\000o\000s\000i\000t\000i\000o\000n}{section.3}% 9
\BOOKMARK [2][-]{subsection.3.3}{\376\377\000R\000e\000d\000u\000c\000i\000n\000g\000\040\000G\000r\000a\000d\000i\000e\000n\000t\000\040\000V\000a\000r\000i\000a\000n\000c\000e}{section.3}% 10
\BOOKMARK [2][-]{subsection.3.4}{\376\377\000I\000m\000p\000o\000r\000t\000a\000n\000c\000e\000\040\000S\000a\000m\000p\000l\000i\000n\000g}{section.3}% 11
\BOOKMARK [2][-]{subsection.3.5}{\376\377\000P\000r\000o\000x\000i\000m\000a\000l\000\040\000P\000o\000l\000i\000c\000y\000\040\000O\000p\000t\000i\000m\000i\000z\000a\000t\000i\000o\000n}{section.3}% 12
\BOOKMARK [2][-]{subsection.3.6}{\376\377\000T\000r\000a\000i\000n\000i\000n\000g\000\040\000R\000e\000w\000a\000r\000d\000\040\000M\000o\000d\000e\000l\000s}{section.3}% 13
\BOOKMARK [2][-]{subsection.3.7}{\376\377\000A\000\040\000C\000o\000m\000p\000l\000e\000t\000e\000\040\000W\000o\000r\000k\000f\000l\000o\000w}{section.3}% 14
\BOOKMARK [1][-]{section.4}{\376\377\000I\000m\000p\000r\000o\000v\000e\000d\000\040\000R\000e\000i\000n\000f\000o\000r\000c\000e\000m\000e\000n\000t\000\040\000L\000e\000a\000r\000n\000i\000n\000g\000\040\000f\000o\000r\000\040\000L\000L\000M\000s}{}% 15
\BOOKMARK [2][-]{subsection.4.1}{\376\377\000A\000d\000v\000a\000n\000c\000e\000d\000\040\000R\000e\000w\000a\000r\000d\000\040\000M\000o\000d\000e\000l\000s}{section.4}% 16
\BOOKMARK [3][-]{subsubsection.4.1.1}{\376\377\000A\000u\000t\000o\000m\000a\000t\000i\000c\000\040\000P\000r\000e\000f\000e\000r\000e\000n\000c\000e\000\040\000D\000a\000t\000a\000\040\000G\000e\000n\000e\000r\000a\000t\000i\000o\000n}{subsection.4.1}% 17
\BOOKMARK [3][-]{subsubsection.4.1.2}{\376\377\000R\000e\000w\000a\000r\000d\000\040\000S\000h\000a\000p\000i\000n\000g}{subsection.4.1}% 18
\BOOKMARK [3][-]{subsubsection.4.1.3}{\376\377\000I\000m\000p\000r\000o\000v\000e\000d\000\040\000R\000e\000w\000a\000r\000d\000\040\000G\000e\000n\000e\000r\000a\000l\000i\000z\000a\000t\000i\000o\000n}{subsection.4.1}% 19
\BOOKMARK [3][-]{subsubsection.4.1.4}{\376\377\000G\000e\000n\000e\000r\000a\000t\000i\000v\000e\000\040\000R\000e\000w\000a\000r\000d\000\040\000M\000o\000d\000e\000l\000s}{subsection.4.1}% 20
\BOOKMARK [2][-]{subsection.4.2}{\376\377\000B\000e\000t\000t\000e\000r\000\040\000A\000d\000v\000a\000n\000t\000a\000g\000e\000\040\000E\000s\000t\000i\000m\000a\000t\000i\000o\000n}{section.4}% 21
\BOOKMARK [3][-]{subsubsection.4.2.1}{\376\377\000T\000e\000m\000p\000o\000r\000a\000l\000\040\000D\000i\000f\000f\000e\000r\000e\000n\000c\000e\000-\000b\000a\000s\000e\000d\000\040\000A\000d\000v\000a\000n\000t\000a\000g\000e\000\040\000E\000s\000t\000i\000m\000a\000t\000i\000o\000n}{subsection.4.2}% 22
\BOOKMARK [3][-]{subsubsection.4.2.2}{\376\377\000G\000e\000n\000e\000r\000a\000l\000i\000z\000e\000d\000\040\000A\000d\000v\000a\000n\000t\000a\000g\000e\000\040\000E\000s\000t\000i\000m\000a\000t\000i\000o\000n}{subsection.4.2}% 23
\BOOKMARK [3][-]{subsubsection.4.2.3}{\376\377\000G\000r\000o\000u\000p\000\040\000R\000e\000l\000a\000t\000i\000v\000e\000\040\000P\000o\000l\000i\000c\000y\000\040\000O\000p\000t\000i\000m\000i\000z\000a\000t\000i\000o\000n}{subsection.4.2}% 24
\BOOKMARK [2][-]{subsection.4.3}{\376\377\000E\000f\000f\000i\000c\000i\000e\000n\000t\000\040\000R\000L\000\040\000M\000e\000t\000h\000o\000d\000s}{section.4}% 25
\BOOKMARK [3][-]{subsubsection.4.3.1}{\376\377\000D\000y\000n\000a\000m\000i\000c\000\040\000S\000a\000m\000p\000l\000i\000n\000g}{subsection.4.3}% 26
\BOOKMARK [3][-]{subsubsection.4.3.2}{\376\377\000L\000i\000g\000h\000t\000w\000e\000i\000g\000h\000t\000\040\000R\000e\000w\000a\000r\000d\000\040\000M\000e\000t\000h\000o\000d\000s}{subsection.4.3}% 27
\BOOKMARK [2][-]{subsection.4.4}{\376\377\000D\000i\000r\000e\000c\000t\000\040\000P\000r\000e\000f\000e\000r\000e\000n\000c\000e\000\040\000O\000p\000t\000i\000m\000i\000z\000a\000t\000i\000o\000n}{section.4}% 28
\BOOKMARK [1][-]{section.5}{\376\377\000R\000e\000i\000n\000f\000o\000r\000c\000e\000m\000e\000n\000t\000\040\000L\000e\000a\000r\000n\000i\000n\000g\000\040\000f\000o\000r\000\040\000L\000L\000M\000\040\000R\000e\000a\000s\000o\000n\000i\000n\000g}{}% 29
\BOOKMARK [2][-]{subsection.5.1}{\376\377\000T\000e\000s\000t\000-\000t\000i\000m\000e\000\040\000S\000c\000a\000l\000i\000n\000g}{section.5}% 30
\BOOKMARK [3][-]{subsubsection.5.1.1}{\376\377\000B\000e\000s\000t\000-\000o\000f\000-\000N\000\040\000S\000a\000m\000p\000l\000i\000n\000g}{subsection.5.1}% 31
\BOOKMARK [3][-]{subsubsection.5.1.2}{\376\377\000S\000t\000e\000p\000-\000b\000y\000-\000s\000t\000e\000p\000\040\000V\000e\000r\000i\000f\000i\000c\000a\000t\000i\000o\000n}{subsection.5.1}% 32
\BOOKMARK [3][-]{subsubsection.5.1.3}{\376\377\000M\000o\000n\000t\000e\000\040\000C\000a\000r\000l\000o\000\040\000T\000r\000e\000e\000\040\000S\000e\000a\000r\000c\000h}{subsection.5.1}% 33
\BOOKMARK [2][-]{subsection.5.2}{\376\377\000L\000a\000r\000g\000e\000-\000s\000c\000a\000l\000e\000\040\000R\000e\000i\000n\000f\000o\000r\000c\000e\000m\000e\000n\000t\000\040\000L\000e\000a\000r\000n\000i\000n\000g}{section.5}% 34
\BOOKMARK [2][-]{subsection.5.3}{\376\377\000I\000t\000e\000r\000a\000t\000i\000v\000e\000\040\000R\000e\000i\000n\000f\000o\000r\000c\000e\000m\000e\000n\000t\000\040\000L\000e\000a\000r\000n\000i\000n\000g}{section.5}% 35
\BOOKMARK [1][-]{section.6}{\376\377\000R\000e\000i\000n\000f\000o\000r\000c\000e\000m\000e\000n\000t\000\040\000L\000e\000a\000r\000n\000i\000n\000g\000\040\000f\000o\000r\000\040\000L\000L\000M\000-\000b\000a\000s\000e\000d\000\040\000A\000g\000e\000n\000t\000s}{}% 36
\BOOKMARK [1][-]{section.7}{\376\377\000R\000e\000i\000n\000f\000o\000r\000c\000e\000m\000e\000n\000t\000\040\000L\000e\000a\000r\000n\000i\000n\000g\000\040\000f\000o\000r\000\040\000M\000u\000l\000t\000i\000m\000o\000d\000a\000l\000\040\000M\000o\000d\000e\000l\000s}{}% 37
\BOOKMARK [2][-]{subsection.7.1}{\376\377\000V\000i\000s\000u\000a\000l\000\040\000L\000a\000n\000g\000u\000a\000g\000e\000\040\000M\000o\000d\000e\000l\000s}{section.7}% 38
\BOOKMARK [2][-]{subsection.7.2}{\376\377\000S\000p\000e\000e\000c\000h\000\040\000G\000e\000n\000e\000r\000a\000t\000i\000o\000n\000\040\000M\000o\000d\000e\000l\000s}{section.7}% 39
\BOOKMARK [2][-]{subsection.7.3}{\376\377\000D\000i\000f\000f\000u\000s\000i\000o\000n\000\040\000M\000o\000d\000e\000l\000s}{section.7}% 40
\BOOKMARK [1][-]{section.8}{\376\377\000S\000u\000m\000m\000a\000r\000y}{}% 41
\BOOKMARK [1][-]{section.9}{\376\377\000S\000y\000s\000t\000e\000m\000s\000\040\000a\000n\000d\000\040\000D\000a\000t\000a\000s\000e\000t\000s}{}% 42
rl-introduction.toc deleted 100644 → 0
\contentsline {section}{\numberline {1}Introduction}{3}{section.1}%
\contentsline {section}{\numberline {2}Preliminary}{3}{section.2}%
\contentsline {subsection}{\numberline {2.1}Fundamentals of Large Language Models}{3}{subsection.2.1}%
\contentsline {subsection}{\numberline {2.2}Fundamentals of Reinforcement Learning}{4}{subsection.2.2}%
\contentsline {subsubsection}{\numberline {2.2.1}General Framework of Reinforcement Learning}{4}{subsubsection.2.2.1}%
\contentsline {subsubsection}{\numberline {2.2.2}Key Elements}{5}{subsubsection.2.2.2}%
\contentsline {section}{\numberline {3}An Example of Using Reinforcement Learning to Train LLMs}{6}{section.3}%
\contentsline {subsection}{\numberline {3.1}Policy Gradient}{7}{subsection.3.1}%
\contentsline {subsection}{\numberline {3.2}Temporal Decomposition}{9}{subsection.3.2}%
\contentsline {subsection}{\numberline {3.3}Reducing Gradient Variance}{10}{subsection.3.3}%
\contentsline {subsection}{\numberline {3.4}Importance Sampling}{12}{subsection.3.4}%
\contentsline {subsection}{\numberline {3.5}Proximal Policy Optimization}{14}{subsection.3.5}%
\contentsline {subsection}{\numberline {3.6}Training Reward Models}{15}{subsection.3.6}%
\contentsline {subsection}{\numberline {3.7}A Complete Workflow}{18}{subsection.3.7}%
\contentsline {section}{\numberline {4}Improved Reinforcement Learning for LLMs}{20}{section.4}%
\contentsline {subsection}{\numberline {4.1}Advanced Reward Models}{20}{subsection.4.1}%
\contentsline {subsubsection}{\numberline {4.1.1}Automatic Preference Data Generation}{20}{subsubsection.4.1.1}%
\contentsline {subsubsection}{\numberline {4.1.2}Reward Shaping}{21}{subsubsection.4.1.2}%
\contentsline {subsubsection}{\numberline {4.1.3}Improved Reward Generalization}{23}{subsubsection.4.1.3}%
\contentsline {subsubsection}{\numberline {4.1.4}Generative Reward Models}{24}{subsubsection.4.1.4}%
\contentsline {subsection}{\numberline {4.2}Better Advantage Estimation}{26}{subsection.4.2}%
\contentsline {subsubsection}{\numberline {4.2.1}Temporal Difference-based Advantage Estimation}{26}{subsubsection.4.2.1}%
\contentsline {subsubsection}{\numberline {4.2.2}Generalized Advantage Estimation}{26}{subsubsection.4.2.2}%
\contentsline {subsubsection}{\numberline {4.2.3}Group Relative Policy Optimization}{27}{subsubsection.4.2.3}%
\contentsline {subsection}{\numberline {4.3}Efficient RL Methods}{29}{subsection.4.3}%
\contentsline {subsubsection}{\numberline {4.3.1}Dynamic Sampling}{29}{subsubsection.4.3.1}%
\contentsline {subsubsection}{\numberline {4.3.2}Lightweight Reward Methods}{30}{subsubsection.4.3.2}%
\contentsline {subsection}{\numberline {4.4}Direct Preference Optimization}{32}{subsection.4.4}%
\contentsline {section}{\numberline {5}Reinforcement Learning for LLM Reasoning}{34}{section.5}%
\contentsline {subsection}{\numberline {5.1}Test-time Scaling}{35}{subsection.5.1}%
\contentsline {subsubsection}{\numberline {5.1.1}Best-of-N Sampling}{35}{subsubsection.5.1.1}%
\contentsline {subsubsection}{\numberline {5.1.2}Step-by-step Verification}{37}{subsubsection.5.1.2}%
\contentsline {subsubsection}{\numberline {5.1.3}Monte Carlo Tree Search}{38}{subsubsection.5.1.3}%
\contentsline {subsection}{\numberline {5.2}Large-scale Reinforcement Learning}{40}{subsection.5.2}%
\contentsline {subsection}{\numberline {5.3}Iterative Reinforcement Learning}{41}{subsection.5.3}%
\contentsline {section}{\numberline {6}Reinforcement Learning for LLM-based Agents}{43}{section.6}%
\contentsline {section}{\numberline {7}Reinforcement Learning for Multimodal Models}{43}{section.7}%
\contentsline {subsection}{\numberline {7.1}Visual Language Models}{43}{subsection.7.1}%
\contentsline {subsection}{\numberline {7.2}Speech Generation Models}{45}{subsection.7.2}%
\contentsline {subsection}{\numberline {7.3}Diffusion Models}{45}{subsection.7.3}%
\contentsline {section}{\numberline {8}Summary}{45}{section.8}%
\contentsline {section}{\numberline {9}Systems and Datasets}{46}{section.9}%
Markdown 格式
0% 或
您添加了 0 人 到此讨论。请谨慎行事。
请先完成此评论的编辑！
请 注册 或者 后发表评论