update.

rl-introduction.bbl

-\begin{thebibliography}{175}
+\begin{thebibliography}{176}
 \providecommand{\natexlab}[1]{#1}
 \providecommand{\natexlab}[1]{#1}
 \providecommand{\url}[1]{\texttt{#1}}
 \providecommand{\url}[1]{\texttt{#1}}
 \expandafter\ifx\csname urlstyle\endcsname\relax
 \expandafter\ifx\csname urlstyle\endcsname\relax
   \providecommand{\doi}[1]{doi: #1}\else
   \providecommand{\doi}[1]{doi: #1}\else
   \providecommand{\doi}{doi: \begingroup \urlstyle{rm}\Url}\fi
   \providecommand{\doi}{doi: \begingroup \urlstyle{rm}\Url}\fi
 
 
+\bibitem[Agarwal et~al.(2024)Agarwal, Vieillard, Zhou, Stanczyk, Ramos~Garea, Geist, and Bachem]{agarwal-etal:onpolicy}
+Rishabh Agarwal, Nino Vieillard, Yongchao Zhou, Piotr Stanczyk, Sabela Ramos~Garea, Matthieu Geist, and Olivier Bachem.
+\newblock On-policy distillation of language models: Learning from self-generated mistakes.
+\newblock In \emph{International Conference on Learning Representations}, volume 2024, pp.\  21246--21263, 2024.
+
 \bibitem[Amini et~al.(2024)Amini, Vieira, and Cotterell]{amini:2024direct}
 \bibitem[Amini et~al.(2024)Amini, Vieira, and Cotterell]{amini:2024direct}
 Afra Amini, Tim Vieira, and Ryan Cotterell.
 Afra Amini, Tim Vieira, and Ryan Cotterell.
 \newblock Direct preference optimization with an offset.
 \newblock Direct preference optimization with an offset.

rl-introduction.bib

 
 
 
 
 
 
+@inproceedings{agarwal-etal:onpolicy,
+  title={On-policy distillation of language models: Learning from self-generated mistakes},
+  author={Agarwal, Rishabh and Vieillard, Nino and Zhou, Yongchao and Stanczyk, Piotr and Ramos Garea, Sabela and Geist, Matthieu and Bachem, Olivier},
+  booktitle={International Conference on Learning Representations},
+  volume={2024},
+  pages={21246--21263},
+  year={2024}
+}
 
 
 @inproceedings{mcallister-etal:flow,
 @inproceedings{mcallister-etal:flow,
   title={Flow matching policy gradients},
   title={Flow matching policy gradients},

rl-introduction.tex

@@ -207,7 +207,6 @@ The aim of this paper is to provide a comprehensive introduction to RL from the
 \input{section8/section8}
 \input{section8/section8}
 
 
 % \clearpage
 % \clearpage
-% systems & datasets  todo: ganyang
 \input{section9/section9}
 \input{section9/section9}
 
 
 
 

section6/section6.tex

@@ -451,10 +451,6 @@ The above way of generating skills often suffer from quality issues. First, a ge
 
 
 \begin{figure*}[!t]
 \begin{figure*}[!t]
     \centering
     \centering
-    % \resizebox{\linewidth}{!}{
-    %     \input{section6/Figures/skillrl.tex}
-    % }
-    %   \includegraphics[width=0.5\textwidth]{section6/Figures/skillrl_draft.png}
     \input{section6/Figures/skillrl_illustration.tex}
     \input{section6/Figures/skillrl_illustration.tex}
     \caption{
     \caption{
         Illustration of SkillRL \citep{xia-etal:skillrl}. SkillRL maintains a pool of agentic trajectories and initially abstracts reusable skills from these trajectories. The extracted skills are then used to guide RL training. As the agent improves through RL training, it generates higher-quality trajectories, which are further used to refine the skill bank.
         Illustration of SkillRL \citep{xia-etal:skillrl}. SkillRL maintains a pool of agentic trajectories and initially abstracts reusable skills from these trajectories. The extracted skills are then used to guide RL training. As the agent improves through RL training, it generates higher-quality trajectories, which are further used to refine the skill bank.

section8/section8.tex

 
 
-\section{Summary}
+\section{Conclusions and Future Directions}
 
 
 % future work
 % future work
 % 强化学习作为一种预训练方式
 % 强化学习作为一种预训练方式