Skip to content

R
rl-introduction
Commit 285edfde by wangchenglong
Options

update.

parent 7e3220a0
正在显示 包含 4194 行增加 和 107 行删除
rl-introduction.aux 0 → 100644
\relax
\providecommand\hyper@newdestlabel[2]{}
\providecommand\HyField@AuxAddToFields[1]{}
\providecommand\HyField@AuxAddToCoFields[2]{}
\def\@LN@column{1}
\def\@LN@column{1}
\citation{christiano-etal:2017deep}
\citation{openai:2024learning,deepseek:2025r1}
\citation{Sutton-and-Barto:2018RL}
\citation{ouyang:2022training,wei-etal:2022finetuned}
\def\@LN@column{1}
\@writefile{toc}{\contentsline {section}{\numberline {1}Introduction}{3}{section.1}\protected@file@percent }
\@writefile{toc}{\contentsline {section}{\numberline {2}Preliminary}{3}{section.2}\protected@file@percent }
\@writefile{toc}{\contentsline {subsection}{\numberline {2.1}Fundamentals of Large Language Models}{3}{subsection.2.1}\protected@file@percent }
\citation{xiao-and-zhu:2025foundations}
\citation{Sutton-and-Barto:2018RL}
\def\@LN@column{1}
\newlabel{eq:instruction-fine-tuning}{{1}{4}{Fundamentals of Large Language Models}{equation.1}{}}
\@writefile{toc}{\contentsline {subsection}{\numberline {2.2}Fundamentals of Reinforcement Learning}{4}{subsection.2.2}\protected@file@percent }
\@writefile{toc}{\contentsline {subsubsection}{\numberline {2.2.1}General Framework of Reinforcement Learning}{4}{subsubsection.2.2.1}\protected@file@percent }
\def\@LN@column{1}
\@writefile{lof}{\contentsline {figure}{\numberline {1}{\ignorespaces A general framework of deep reinforcement learning. The policy is parameterized by neural networks, and the agent learns an optimal policy through interactions with the environment to maximize cumulative rewards. }}{5}{figure.1}\protected@file@percent }
\newlabel{fig:rl_framework}{{1}{5}{A general framework of deep reinforcement learning. The policy is parameterized by neural networks, and the agent learns an optimal policy through interactions with the environment to maximize cumulative rewards}{figure.1}{}}
\@writefile{toc}{\contentsline {subsubsection}{\numberline {2.2.2}Key Elements}{5}{subsubsection.2.2.2}\protected@file@percent }
\def\@LN@column{1}
\@writefile{toc}{\contentsline {section}{\numberline {3}An Example of Using Reinforcement Learning to Train LLMs}{6}{section.3}\protected@file@percent }
\newlabel{sec:example-using-rl-training-llms}{{3}{6}{An Example of Using Reinforcement Learning to Train LLMs}{section.3}{}}
\def\@LN@column{1}
\@writefile{toc}{\contentsline {subsection}{\numberline {3.1}Policy Gradient}{7}{subsection.3.1}\protected@file@percent }
\newlabel{sec:policy-gradient}{{3.1}{7}{Policy Gradient}{subsection.3.1}{}}
\newlabel{eq:langth-based-reward-function}{{7}{7}{Policy Gradient}{equation.7}{}}
\def\@LN@column{1}
\@writefile{lof}{\contentsline {figure}{\numberline {2}{\ignorespaces Illustration of training an LLM using policy gradient. The central component is an LLM. Given an input $\mathbf {x}=x_1...x_m$, we sample an output $\mathbf {y}$ from the LLM. Note that gradients are not computed during the sampling to prevent extensive memory consumption because the sampling is performed via an autoregressive mode. Instead, by leveraging the parallel computation capability of the LLM, similar to SFT, the sampled output $\mathbf {y}$ is used for a subsequent forward pass to compute the gradients produced during the generation of each token. }}{8}{figure.2}\protected@file@percent }
\newlabel{fig:update-with-policy-gradient}{{2}{8}{Illustration of training an LLM using policy gradient. The central component is an LLM. Given an input $\mathbf {x}=x_1...x_m$, we sample an output $\mathbf {y}$ from the LLM. Note that gradients are not computed during the sampling to prevent extensive memory consumption because the sampling is performed via an autoregressive mode. Instead, by leveraging the parallel computation capability of the LLM, similar to SFT, the sampled output $\mathbf {y}$ is used for a subsequent forward pass to compute the gradients produced during the generation of each token}{figure.2}{}}
\newlabel{eq:rl-gradient-j-theta}{{11}{8}{Policy Gradient}{equation.11}{}}
\newlabel{eq:rl-j-theta-gradient-simplified}{{12}{8}{Policy Gradient}{equation.12}{}}
\def\@LN@column{1}
\@writefile{lof}{\contentsline {figure}{\numberline {3}{\ignorespaces The mechanism of RL in training LLMs from the perspective of hypothesis space reranking: given an input $\mathbf {x}$, the model increases the probability of longer outputs, such as $\mathbf {y}_3$, within the hypothesis space by assigning higher rewards to them; conversely, shorter outputs like $\mathbf {y}_1$ receive lower rewards, which reduces their probabilities. }}{9}{figure.3}\protected@file@percent }
\newlabel{fig:understand-policy-gradient}{{3}{9}{The mechanism of RL in training LLMs from the perspective of hypothesis space reranking: given an input $\mathbf {x}$, the model increases the probability of longer outputs, such as $\mathbf {y}_3$, within the hypothesis space by assigning higher rewards to them; conversely, shorter outputs like $\mathbf {y}_1$ receive lower rewards, which reduces their probabilities}{figure.3}{}}
\@writefile{toc}{\contentsline {subsection}{\numberline {3.2}Temporal Decomposition}{9}{subsection.3.2}\protected@file@percent }
\newlabel{sec:temporal-decomposition}{{3.2}{9}{Temporal Decomposition}{subsection.3.2}{}}
\def\@LN@column{1}
\@writefile{lof}{\contentsline {figure}{\numberline {4}{\ignorespaces Example of sampled outputs demonstrating that the same tokens will receive significantly different rewards (i.e., 16 vs. 75). This introduces high gradient variance in the optimization process of the policy gradient. }}{10}{figure.4}\protected@file@percent }
\newlabel{fig:an-overlap-example}{{4}{10}{Example of sampled outputs demonstrating that the same tokens will receive significantly different rewards (i.e., 16 vs. 75). This introduces high gradient variance in the optimization process of the policy gradient}{figure.4}{}}
\@writefile{toc}{\contentsline {subsection}{\numberline {3.3}Reducing Gradient Variance}{10}{subsection.3.3}\protected@file@percent }
\newlabel{sec:reduce-gradient-variance}{{3.3}{10}{Reducing Gradient Variance}{subsection.3.3}{}}
\citation{wang-and-zhou:2024chain}
\def\@LN@column{1}
\newlabel{eq:modified-gradient-simlified}{{15}{11}{Reducing Gradient Variance}{equation.15}{}}
\newlabel{eq:monte-carlo-advantage}{{16}{11}{Reducing Gradient Variance}{equation.16}{}}
\newlabel{eq:modified-gradient-simlified-advantage}{{17}{11}{Reducing Gradient Variance}{equation.17}{}}
\newlabel{eq:loss-function-llm-update}{{18}{11}{Reducing Gradient Variance}{equation.18}{}}
\citation{mnih-etal:2016asynchronous}
\citation{szepesvari:2010algorithms,Sutton-and-Barto:2018RL}
\def\@LN@column{1}
\@writefile{lof}{\contentsline {figure}{\numberline {5}{\ignorespaces Architecture of the value model based on a pre-trained LLM. The main component of this model is still an LLM. For a given sequence $\mathrm {seq}_{\mathbf {x}, \mathbf {y}}=[\mathbf {x}, \mathbf {y}]$, the model extracts representations corresponding to the positions of the tokens in $\mathbf {y}$. These representations are then individually mapped to scalars through a linear transformation. Each scalar represents the value associated with each token in $\mathbf {y}$. Note that the linear mapping matrix used for the transformations is typically shared across all tokens. }}{12}{figure.5}\protected@file@percent }
\newlabel{fig:architecture-value-function}{{5}{12}{Architecture of the value model based on a pre-trained LLM. The main component of this model is still an LLM. For a given sequence $\mathrm {seq}_{\mathbf {x}, \mathbf {y}}=[\mathbf {x}, \mathbf {y}]$, the model extracts representations corresponding to the positions of the tokens in $\mathbf {y}$. These representations are then individually mapped to scalars through a linear transformation. Each scalar represents the value associated with each token in $\mathbf {y}$. Note that the linear mapping matrix used for the transformations is typically shared across all tokens}{figure.5}{}}
\@writefile{toc}{\contentsline {subsection}{\numberline {3.4}Importance Sampling}{12}{subsection.3.4}\protected@file@percent }
\newlabel{sec:importance-sampling}{{3.4}{12}{Importance Sampling}{subsection.3.4}{}}
\newlabel{eq:loss-function-training-llm-importance-sampling}{{22}{12}{Importance Sampling}{equation.22}{}}
\citation{schulman-etal:2015trust}
\def\@LN@column{1}
\@writefile{lof}{\contentsline {figure}{\numberline {6}{\ignorespaces Workflow of integrating important sampling in the training of LLMs with policy gradient. In Step 1, the current policy is synchronized to establish the reference policy. In Step 2, we sample an output $\mathbf {y}$ from this reference policy for a given input $\mathbf {x}$. Finally, in Step 3, the sequence $[\mathbf {x}, \mathbf {y}]$ is used to optimize the current policy multiple times. After optimization, the updated policy is synchronized to become the reference policy. }}{13}{figure.6}\protected@file@percent }
\newlabel{fig:policy-gradient-with-importance-sampling}{{6}{13}{Workflow of integrating important sampling in the training of LLMs with policy gradient. In Step 1, the current policy is synchronized to establish the reference policy. In Step 2, we sample an output $\mathbf {y}$ from this reference policy for a given input $\mathbf {x}$. Finally, in Step 3, the sequence $[\mathbf {x}, \mathbf {y}]$ is used to optimize the current policy multiple times. After optimization, the updated policy is synchronized to become the reference policy}{figure.6}{}}
\citation{schulman-etal:2015trust}
\citation{schulman-etal:2017proximal}
\def\@LN@column{1}
\@writefile{lof}{\contentsline {figure}{\numberline {7}{\ignorespaces Illustration of the impact of a terrible step on sampling within an LLM training cycle using importance sampling. The left shows poor output sampled with slight repetition resulting in a terrible step. The right shows subsequent output sampled with more severe repetition after a terrible step. }}{14}{figure.7}\protected@file@percent }
\newlabel{fig:terrible-step-importance-sampling}{{7}{14}{Illustration of the impact of a terrible step on sampling within an LLM training cycle using importance sampling. The left shows poor output sampled with slight repetition resulting in a terrible step. The right shows subsequent output sampled with more severe repetition after a terrible step}{figure.7}{}}
\newlabel{eq:importance-sampling-with-penalty}{{25}{14}{Importance Sampling}{equation.25}{}}
\@writefile{toc}{\contentsline {subsection}{\numberline {3.5}Proximal Policy Optimization}{14}{subsection.3.5}\protected@file@percent }
\newlabel{eq:ppo-loss}{{26}{14}{Proximal Policy Optimization}{equation.26}{}}
\newlabel{eq:cliping-function}{{27}{14}{Proximal Policy Optimization}{equation.27}{}}
\def\@LN@column{1}
\@writefile{lof}{\contentsline {figure}{\numberline {8}{\ignorespaces PPO clipping at time step $t$. The basic idea can be represented with a hypothesis space as described in Figure \ref {fig:understand-policy-gradient}. The sub-figure (a) shows the case where the advantage $A_{t}$ is positive, suggesting that the output is preferred. Here, the probability of the sampled output is increased within a defined constraint (up to $1 + \epsilon $) in the hypothesis space. The sub-figure (b) shows the case where $A_{t}$ is negative, indicating a dispreferred output. In this case, the probability of the sample output is reduced to a minimum (down to $1 - \epsilon $) in the hypothesis space. }}{15}{figure.8}\protected@file@percent }
\newlabel{fig:ppo-clip}{{8}{15}{PPO clipping at time step $t$. The basic idea can be represented with a hypothesis space as described in Figure \ref {fig:understand-policy-gradient}. The sub-figure (a) shows the case where the advantage $A_{t}$ is positive, suggesting that the output is preferred. Here, the probability of the sampled output is increased within a defined constraint (up to $1 + \epsilon $) in the hypothesis space. The sub-figure (b) shows the case where $A_{t}$ is negative, indicating a dispreferred output. In this case, the probability of the sample output is reduced to a minimum (down to $1 - \epsilon $) in the hypothesis space}{figure.8}{}}
\@writefile{toc}{\contentsline {subsection}{\numberline {3.6}Training Reward Models}{15}{subsection.3.6}\protected@file@percent }
\newlabel{sec:training-reward-models}{{3.6}{15}{Training Reward Models}{subsection.3.6}{}}
\citation{bradley-and-terry:rank}
\citation{bromley-etal:1993signature}
\citation{bromley-etal:1993signature}
\def\@LN@column{1}
\@writefile{lof}{\contentsline {figure}{\numberline {9}{\ignorespaces Architecture of the reward model based on an LLM. Unlike the value model depicted in Figure \ref {fig:architecture-value-function}, this model extracts the representation from the last position of the output $\mathbf {y}$ to represent the entire sequence $[\mathbf {x}, \mathbf {y}]$. This representation is then mapped to a scalar through a linear transformation, which serves as the reward for the output. }}{16}{figure.9}\protected@file@percent }
\newlabel{fig:reward-model}{{9}{16}{Architecture of the reward model based on an LLM. Unlike the value model depicted in Figure \ref {fig:architecture-value-function}, this model extracts the representation from the last position of the output $\mathbf {y}$ to represent the entire sequence $[\mathbf {x}, \mathbf {y}]$. This representation is then mapped to a scalar through a linear transformation, which serves as the reward for the output}{figure.9}{}}
\citation{wu-etal:2023fine}
\citation{lightman-etal:2023let}
\def\@LN@column{1}
\@writefile{lof}{\contentsline {figure}{\numberline {10}{\ignorespaces An example of loss computation using the Bradley-Terry model. Given an input $\mathbf {x}$ and two outputs $(\mathbf {y}_{a}, \mathbf {y}_{b})$, where $\mathbf {y}_a \succ \mathbf {y}_b$, we initially obtain rewards $R(\mathbf {x}, \mathbf {y}_a)$ and $R(\mathbf {x}, \mathbf {y}_b)$ for two sequences $\mathrm {seq}_{\mathbf {x},\mathbf {y}_{a}}$ and $\mathrm {seq}_{\mathbf {x},\mathbf {y}_{b}}$. These rewards are subsequently used to compute the loss as described in Eq. (\ref {eq:pairwise-reward-loss-expectation}). Note that, for efficiency, both sequences are typically processed in a single batch during one forward pass to simultaneously obtain $R(\mathbf {x}, \mathbf {y}_a)$ and $R(\mathbf {x}, \mathbf {y}_b)$. In fact, this training approach is also common in Siamese networks \citep {bromley-etal:1993signature}. }}{17}{figure.10}\protected@file@percent }
\newlabel{fig:training-reward-models}{{10}{17}{An example of loss computation using the Bradley-Terry model. Given an input $\mathbf {x}$ and two outputs $(\mathbf {y}_{a}, \mathbf {y}_{b})$, where $\mathbf {y}_a \succ \mathbf {y}_b$, we initially obtain rewards $R(\mathbf {x}, \mathbf {y}_a)$ and $R(\mathbf {x}, \mathbf {y}_b)$ for two sequences $\mathrm {seq}_{\mathbf {x},\mathbf {y}_{a}}$ and $\mathrm {seq}_{\mathbf {x},\mathbf {y}_{b}}$. These rewards are subsequently used to compute the loss as described in Eq. (\ref {eq:pairwise-reward-loss-expectation}). Note that, for efficiency, both sequences are typically processed in a single batch during one forward pass to simultaneously obtain $R(\mathbf {x}, \mathbf {y}_a)$ and $R(\mathbf {x}, \mathbf {y}_b)$. In fact, this training approach is also common in Siamese networks \citep {bromley-etal:1993signature}}{figure.10}{}}
\newlabel{eq:pairwise-reward-loss-expectation}{{30}{17}{Training Reward Models}{equation.30}{}}
\def\@LN@column{1}
\@writefile{lof}{\contentsline {figure}{\numberline {11}{\ignorespaces Illustration of PPO-based optimization workflow. Given an optimized reward model and a reference model, we proceed to train both the policy and the value model. At each prediction step, we compute the sum of the PPO-based loss and update the policy parameters. This requires access to the reward model, the reference model, and the value model at hand. At the same time, we update the parameters of the value model. It is worth noting that steps 2, 3, and 4 do not follow a fixed order, and they can be executed simultaneously or in any order. }}{18}{figure.11}\protected@file@percent }
\newlabel{fig:ppo-rlhf}{{11}{18}{Illustration of PPO-based optimization workflow. Given an optimized reward model and a reference model, we proceed to train both the policy and the value model. At each prediction step, we compute the sum of the PPO-based loss and update the policy parameters. This requires access to the reward model, the reference model, and the value model at hand. At the same time, we update the parameters of the value model. It is worth noting that steps 2, 3, and 4 do not follow a fixed order, and they can be executed simultaneously or in any order}{figure.11}{}}
\@writefile{toc}{\contentsline {subsection}{\numberline {3.7}A Complete Workflow}{18}{subsection.3.7}\protected@file@percent }
\citation{wang-etal:2024hybrid}
\citation{gao-etal:2023scaling}
\citation{xiao-etal:2023introduction}
\citation{wang-etal:2024esrl}
\citation{shao-etal:2024deepseekmath}
\def\@LN@column{1}
\def\@LN@column{1}
\@writefile{toc}{\contentsline {section}{\numberline {4}Improved Reinforcement Learning for LLMs}{20}{section.4}\protected@file@percent }
\@writefile{toc}{\contentsline {subsection}{\numberline {4.1}Advanced Reward Models}{20}{subsection.4.1}\protected@file@percent }
\@writefile{toc}{\contentsline {subsubsection}{\numberline {4.1.1}Automatic Preference Data Generation}{20}{subsubsection.4.1.1}\protected@file@percent }
\newlabel{sec:automatic-preference-data-generation}{{4.1.1}{20}{Automatic Preference Data Generation}{subsubsection.4.1.1}{}}
\citation{zheng-etal:2023judging}
\citation{liu-etal:2023GEval}
\citation{cui-etal:2024ultra}
\citation{dubois-etal:2024alpacafarm}
\citation{kumar-etal:2024training}
\def\@LN@column{1}
\@writefile{toc}{\contentsline {subsubsection}{\numberline {4.1.2}Reward Shaping}{21}{subsubsection.4.1.2}\protected@file@percent }
\newlabel{sec:reward-shaping}{{4.1.2}{21}{Reward Shaping}{subsubsection.4.1.2}{}}
\citation{bahdanau-etal:2016actor,wu-etal:2023fine}
\citation{Sutton-and-Barto:2018RL}
\def\@LN@column{1}
\@writefile{lof}{\contentsline {figure}{\numberline {12}{\ignorespaces An example of reward shaping. Here, length-based shaping rewards are implemented, e.g., awarding a +1 intermediate reward for every 10 tokens generated. Additionally, a delayed reward from the reward model evaluates the overall quality of the output at the end. }}{22}{figure.12}\protected@file@percent }
\newlabel{fig:example-shaping-rewards}{{12}{22}{An example of reward shaping. Here, length-based shaping rewards are implemented, e.g., awarding a +1 intermediate reward for every 10 tokens generated. Additionally, a delayed reward from the reward model evaluates the overall quality of the output at the end}{figure.12}{}}
\newlabel{eq:transformed-reward-function}{{33}{22}{Reward Shaping}{equation.33}{}}
\newlabel{eq:shaping-reward-function}{{34}{22}{Reward Shaping}{equation.34}{}}
\citation{gao-etal:2023scaling,eisenstein-etal:2023helping}
\citation{coste-etal:2024reward}
\def\@LN@column{1}
\@writefile{lof}{\contentsline {figure}{\numberline {13}{\ignorespaces When training reward models, we can use parameter freezing and regularization methods to preserve the features of the LLM and thereby improve reward generalization. }}{23}{figure.13}\protected@file@percent }
\newlabel{fig:improve-reward-generalization}{{13}{23}{When training reward models, we can use parameter freezing and regularization methods to preserve the features of the LLM and thereby improve reward generalization}{figure.13}{}}
\@writefile{toc}{\contentsline {subsubsection}{\numberline {4.1.3}Improved Reward Generalization}{23}{subsubsection.4.1.3}\protected@file@percent }
\newlabel{sec:improved-reward-generalization}{{4.1.3}{23}{Improved Reward Generalization}{subsubsection.4.1.3}{}}
\citation{miettinen:1999nonlinear,Bishop:2006}
\citation{yang-etal:2024regularizing}
\citation{zhang-etal:zhang2024generative,wang-etal:wang2025gram}
\def\@LN@column{1}
\@writefile{lof}{\contentsline {figure}{\numberline {14}{\ignorespaces In generative reward models, we use an LLM to predict the label token given a prompt, an input, and a pair of outputs. This model can be trained in the same way as standard LLMs.}}{24}{figure.14}\protected@file@percent }
\newlabel{fig:generative-reward-model-architecture}{{14}{24}{In generative reward models, we use an LLM to predict the label token given a prompt, an input, and a pair of outputs. This model can be trained in the same way as standard LLMs}{figure.14}{}}
\@writefile{toc}{\contentsline {subsubsection}{\numberline {4.1.4}Generative Reward Models}{24}{subsubsection.4.1.4}\protected@file@percent }
\newlabel{sec:generative-reward-models}{{4.1.4}{24}{Generative Reward Models}{subsubsection.4.1.4}{}}
\citation{wang-etal:2023large}
\citation{zhang-etal:2024generative,wang-etal:wang2026gram}
\citation{zhang-etal:2024generative,mahan-etal:2024generative}
\def\@LN@column{1}
\newlabel{eq:gen-reward-modeling}{{39}{25}{Generative Reward Models}{equation.39}{}}
\newlabel{eq:apply-generative-rm}{{40}{25}{Generative Reward Models}{equation.40}{}}
\citation{sutton-and-richard:1988learning}
\citation{schulman-etal:2015high}
\def\@LN@column{1}
\@writefile{toc}{\contentsline {subsection}{\numberline {4.2}Better Advantage Estimation}{26}{subsection.4.2}\protected@file@percent }
\@writefile{toc}{\contentsline {subsubsection}{\numberline {4.2.1}Temporal Difference-based Advantage Estimation}{26}{subsubsection.4.2.1}\protected@file@percent }
\newlabel{eq:advantage-estimation-monte-carlo}{{42}{26}{Temporal Difference-based Advantage Estimation}{equation.42}{}}
\newlabel{eq:td-advantage-estimation}{{43}{26}{Temporal Difference-based Advantage Estimation}{equation.43}{}}
\@writefile{toc}{\contentsline {subsubsection}{\numberline {4.2.2}Generalized Advantage Estimation}{26}{subsubsection.4.2.2}\protected@file@percent }
\def\@LN@column{1}
\@writefile{lof}{\contentsline {figure}{\numberline {15}{\ignorespaces Illustration of the issue of high variance in Monte Carlo-based advantage estimation, using a length-based reward function as described in Eq. (\ref {eq:langth-based-reward-function}). Typically, if the value model is well-trained, it would predict an average $V_{t}$ of 6 at time step $t$, based on most outputs having around 60 tokens in length. However, for outlier outputs like $\mathbf {y}_{d,t:T}$, which might extend up to 300 tokens, the advantage estimation can exhibit high variance. For instance, the advantage for such an outlier could be computed as $V_{t}(\mathbf {x},\mathbf {y}_{d,<t},y_{d,t}) = \DOTSB \sum@ \slimits@ _{k=t}^T r_{k} - V_t = 30 - 6 = 24$, a stark contrast to another output, say $\mathbf {y}_{1,t:T}$, which is closer to the average length, where the advantage might be $V_{t}(\mathbf {x},\mathbf {y}_{1,<t},y_{1,t}) = -1$. This discrepancy leads to high gradient variance, potentially destabilizing the learning process. }}{27}{figure.15}\protected@file@percent }
\newlabel{fig:monte-carlo-advantage-issue}{{15}{27}{Illustration of the issue of high variance in Monte Carlo-based advantage estimation, using a length-based reward function as described in Eq. (\ref {eq:langth-based-reward-function}). Typically, if the value model is well-trained, it would predict an average $V_{t}$ of 6 at time step $t$, based on most outputs having around 60 tokens in length. However, for outlier outputs like $\mathbf {y}_{d,t:T}$, which might extend up to 300 tokens, the advantage estimation can exhibit high variance. For instance, the advantage for such an outlier could be computed as $V_{t}(\mathbf {x},\mathbf {y}_{d,<t},y_{d,t}) = \sum _{k=t}^T r_{k} - V_t = 30 - 6 = 24$, a stark contrast to another output, say $\mathbf {y}_{1,t:T}$, which is closer to the average length, where the advantage might be $V_{t}(\mathbf {x},\mathbf {y}_{1,<t},y_{1,t}) = -1$. This discrepancy leads to high gradient variance, potentially destabilizing the learning process}{figure.15}{}}
\newlabel{eq:GAE-advantage-estimation}{{46}{27}{Generalized Advantage Estimation}{equation.46}{}}
\@writefile{toc}{\contentsline {subsubsection}{\numberline {4.2.3}Group Relative Policy Optimization}{27}{subsubsection.4.2.3}\protected@file@percent }
\citation{shao-etal:2024deepseekmath}
\def\@LN@column{1}
\@writefile{lof}{\contentsline {figure}{\numberline {16}{\ignorespaces Group relative policy optimization. Compared to PPO, it foregoes the value model by estimating advantages directly from the rewards of a group of outputs. This method can simplify the training process and reduce the computational resources. }}{28}{figure.16}\protected@file@percent }
\newlabel{fig:workflow-grpo}{{16}{28}{Group relative policy optimization. Compared to PPO, it foregoes the value model by estimating advantages directly from the rewards of a group of outputs. This method can simplify the training process and reduce the computational resources}{figure.16}{}}
\newlabel{eq:advantage-grpo}{{47}{28}{Group Relative Policy Optimization}{equation.47}{}}
\newlabel{eq:grpo-loss}{{48}{28}{Group Relative Policy Optimization}{equation.48}{}}
\citation{li-etal:2023remax}
\citation{hu:2025reinforce++}
\citation{guo:2025deepseek}
\citation{pope-etal:2023efficiently}
\citation{zhao-etal:2024atom}
\citation{chen-etal:2023accelerating}
\citation{Sutton-and-Barto:2018RL}
\def\@LN@column{1}
\@writefile{toc}{\contentsline {subsection}{\numberline {4.3}Efficient RL Methods}{29}{subsection.4.3}\protected@file@percent }
\@writefile{toc}{\contentsline {subsubsection}{\numberline {4.3.1}Dynamic Sampling}{29}{subsubsection.4.3.1}\protected@file@percent }
\citation{wang-etal:2024esrl}
\citation{kimi-team:2025kimi}
\citation{li-etal:2025limr}
\citation{wang-etal:2021selective}
\citation{chen-etal:2023alpagasus}
\citation{gao-etal:2023scaling}
\def\@LN@column{1}
\@writefile{toc}{\contentsline {subsubsection}{\numberline {4.3.2}Lightweight Reward Methods}{30}{subsubsection.4.3.2}\protected@file@percent }
\newlabel{sec:lightweight-reward-methods}{{4.3.2}{30}{Lightweight Reward Methods}{subsubsection.4.3.2}{}}
\citation{havrilla-etal:2024teaching,shao-etal:2024deepseekmath}
\citation{guo:2025deepseek,xie-etal:2025logic}
\citation{masoudnia-etal:2014mixture}
\citation{sun-etal:2023simple}
\citation{wang-etal:2023learning}
\def\@LN@column{1}
\citation{rafailov:2023direct}
\def\@LN@column{1}
\@writefile{lof}{\contentsline {figure}{\numberline {17}{\ignorespaces Standard PPO vs. DPO in training LLMs. In PPO, the human preference data is used to train a reward model, which is then employed to train the policy and the value function. In DPO, the use of human preference data is more direct, and the policy is trained on this data without the need for reward model training.}}{32}{figure.17}\protected@file@percent }
\newlabel{fig:comparsion-rl-dpo}{{17}{32}{Standard PPO vs. DPO in training LLMs. In PPO, the human preference data is used to train a reward model, which is then employed to train the policy and the value function. In DPO, the use of human preference data is more direct, and the policy is trained on this data without the need for reward model training}{figure.17}{}}
\@writefile{toc}{\contentsline {subsection}{\numberline {4.4}Direct Preference Optimization}{32}{subsection.4.4}\protected@file@percent }
\newlabel{eq:dpo-loss}{{50}{32}{Direct Preference Optimization}{equation.50}{}}
\citation{cui-etal:2023ultrafeedback,morimura-etal:2024filtered}
\citation{azar:2024general}
\citation{xu:2024contrastive}
\citation{ethayarajh:2024kto}
\citation{hong:2024orpo}
\citation{gallego:2024refined}
\citation{zhou:2024prior}
\citation{meng:2025simpo}
\citation{shao:2025earlier}
\def\@LN@column{1}
\@writefile{lof}{\contentsline {figure}{\numberline {18}{\ignorespaces Illustration of the optimization objective of DPO from a hypothesis space perspective. The right-top chart shows the probability distributions over different hypotheses before optimization, where $\mathbf {y}_{a}$ is the preferred output and $\mathbf {y}_{b}$ is the dispreferred output. The right-bottom chart shows the probabilities after applying DPO, where the likelihood of the preferred output increases and that of dispreferred outputs decreases. Note that during this optimization process, outputs similar to the preferred output $\mathbf {y}_{a}$ (such as $\mathbf {y}_{1}$) will obtain an increased probability, while those similar to the less preferred output $\mathbf {y}_{b}$ (such as $\mathbf {y}_{2}$) will experience a decreased probability, due to the generalization of the model. }}{33}{figure.18}\protected@file@percent }
\newlabel{fig:understand-dpo-optimization-objective}{{18}{33}{Illustration of the optimization objective of DPO from a hypothesis space perspective. The right-top chart shows the probability distributions over different hypotheses before optimization, where $\mathbf {y}_{a}$ is the preferred output and $\mathbf {y}_{b}$ is the dispreferred output. The right-bottom chart shows the probabilities after applying DPO, where the likelihood of the preferred output increases and that of dispreferred outputs decreases. Note that during this optimization process, outputs similar to the preferred output $\mathbf {y}_{a}$ (such as $\mathbf {y}_{1}$) will obtain an increased probability, while those similar to the less preferred output $\mathbf {y}_{b}$ (such as $\mathbf {y}_{2}$) will experience a decreased probability, due to the generalization of the model}{figure.18}{}}
\citation{singhal-etal:2023long,wang-etal:2023far}
\citation{silver-etal:2016mastering}
\citation{vinyals-rtal:2019grandmaster}
\def\@LN@column{1}
\@writefile{lot}{\contentsline {table}{\numberline {1}{\ignorespaces DPO variants and their optimization objectives.}}{34}{table.1}\protected@file@percent }
\newlabel{tab:dpo_varients}{{1}{34}{DPO variants and their optimization objectives}{table.1}{}}
\@writefile{toc}{\contentsline {section}{\numberline {5}Reinforcement Learning for LLM Reasoning}{34}{section.5}\protected@file@percent }
\citation{rafailov:2023direct,gao-etal:2023scaling}
\def\@LN@column{1}
\@writefile{toc}{\contentsline {subsection}{\numberline {5.1}Test-time Scaling}{35}{subsection.5.1}\protected@file@percent }
\@writefile{toc}{\contentsline {subsubsection}{\numberline {5.1.1}Best-of-N Sampling}{35}{subsubsection.5.1.1}\protected@file@percent }
\newlabel{sec:BoN-sampling}{{5.1.1}{35}{Best-of-N Sampling}{subsubsection.5.1.1}{}}
\citation{nakano-etal:2021webgpt,touvron-etal:2023llama2}
\citation{setlur-etal:2024rewarding}
\def\@LN@column{1}
\@writefile{lof}{\contentsline {figure}{\numberline {19}{\ignorespaces Best-of-n sampling. Given an input, multiple candidate outputs are sampled, and a reward model is used to select the best output. A majority vote can also determine the final output in scenarios without a reward model. For example, in this figure, most of the outputs suggest an answer of 290. Therefore, we can consider any one of the outputs resulting in 290 to be the final output. }}{36}{figure.19}\protected@file@percent }
\newlabel{fig:bon-sampling}{{19}{36}{Best-of-n sampling. Given an input, multiple candidate outputs are sampled, and a reward model is used to select the best output. A majority vote can also determine the final output in scenarios without a reward model. For example, in this figure, most of the outputs suggest an answer of 290. Therefore, we can consider any one of the outputs resulting in 290 to be the final output}{figure.19}{}}
\citation{lightman-etal:2024lets}
\citation{wang-etal:2023math}
\citation{zhang-etal:2024generative}
\citation{yao-etal:2023tree}
\citation{silver-etal:2016mastering}
\def\@LN@column{1}
\@writefile{toc}{\contentsline {subsubsection}{\numberline {5.1.2}Step-by-step Verification}{37}{subsubsection.5.1.2}\protected@file@percent }
\newlabel{sec:step-by-step-verification}{{5.1.2}{37}{Step-by-step Verification}{subsubsection.5.1.2}{}}
\def\@LN@column{1}
\@writefile{lof}{\contentsline {figure}{\numberline {20}{\ignorespaces Step-by-step verification with greedy search. This process can be readily extended to beam search by retaining the top steps at each step from all candidate paths for further scaling up test-time computation. As the blue line indicates, the reasoning steps can be refined using verification feedback, such as rewards, to enhance accuracy. }}{38}{figure.20}\protected@file@percent }
\newlabel{fig:step-by-step-with-prm}{{20}{38}{Step-by-step verification with greedy search. This process can be readily extended to beam search by retaining the top steps at each step from all candidate paths for further scaling up test-time computation. As the blue line indicates, the reasoning steps can be refined using verification feedback, such as rewards, to enhance accuracy}{figure.20}{}}
\@writefile{toc}{\contentsline {subsubsection}{\numberline {5.1.3}Monte Carlo Tree Search}{38}{subsubsection.5.1.3}\protected@file@percent }
\def\@LN@column{1}
\@writefile{lof}{\contentsline {figure}{\numberline {21}{\ignorespaces Illustration of step-by-step verification with MCTS. The process consists of four phases: (1) Selection, where the algorithm navigates the decision tree to select the most promising node based on strategies like UCT; (2) Expansion, where new potential nodes are added to the tree for further exploration; (3) Simulation, where each new node is evaluated by simulating possible outcomes (i.e., reasoning paths); and (4) Backpropagation, where the outcomes from the simulations are used to refine node values, such as UCT values, thus enhancing the decision-making for subsequent explorations. }}{39}{figure.21}\protected@file@percent }
\newlabel{fig:monte-carlo-tree-search}{{21}{39}{Illustration of step-by-step verification with MCTS. The process consists of four phases: (1) Selection, where the algorithm navigates the decision tree to select the most promising node based on strategies like UCT; (2) Expansion, where new potential nodes are added to the tree for further exploration; (3) Simulation, where each new node is evaluated by simulating possible outcomes (i.e., reasoning paths); and (4) Backpropagation, where the outcomes from the simulations are used to refine node values, such as UCT values, thus enhancing the decision-making for subsequent explorations}{figure.21}{}}
\citation{yuan-etal:2023scaling,zeng-etal:2025revisiting}
\citation{openai:2024learning}
\citation{guo:2025deepseek}
\citation{kimi-team:2025kimi}
\citation{guo:2025deepseek}
\citation{yuan-etal:2024selfrewarding}
\def\@LN@column{1}
\newlabel{eq:uct}{{55}{40}{Monte Carlo Tree Search}{equation.55}{}}
\@writefile{toc}{\contentsline {subsection}{\numberline {5.2}Large-scale Reinforcement Learning}{40}{subsection.5.2}\protected@file@percent }
\newlabel{sec:large-sclae-rl}{{5.2}{40}{Large-scale Reinforcement Learning}{subsection.5.2}{}}
\citation{gorbatovski-etal:2024learn}
\citation{guo:2025deepseek}
\citation{guo:2025deepseek}
\citation{guo:2025deepseek}
\def\@LN@column{1}
\@writefile{toc}{\contentsline {subsection}{\numberline {5.3}Iterative Reinforcement Learning}{41}{subsection.5.3}\protected@file@percent }
\citation{wang-etal:2024hybrid}
\citation{miettinen-etal:2008introduction,deb-etal:2016multi}
\def\@LN@column{1}
\pgfsyspdfmark {pgfid55}{16583363}{28184036}
\pgfsyspdfmark {pgfid56}{16582437}{32334819}
\pgfsyspdfmark {pgfid57}{16543961}{38716143}
\pgfsyspdfmark {pgfid58}{16582437}{43575321}
\pgfsyspdfmark {pgfid54}{9837915}{22821523}
\@writefile{lof}{\contentsline {figure}{\numberline {22}{\ignorespaces Illustration of iterative RL in DeepSeek-R1 \citep {guo:2025deepseek}. This method maintains multi-phase RL to develop a robust reasoning LLM with a GRPO algorithm. Initially, a pre-trained LLM is fine-tuned using a small set of high-quality labeled reasoning data with SFT as a cold start. Then, large-scale RL is applied specifically to enhance reasoning capabilities. After that, the model undergoes reject sampling across multiple tasks to refine output quality. In the final phase, the model is further trained using RL to enhance generalization across various tasks. }}{42}{figure.22}\protected@file@percent }
\newlabel{fig:iterative-rl}{{22}{42}{Illustration of iterative RL in DeepSeek-R1 \citep {guo:2025deepseek}. This method maintains multi-phase RL to develop a robust reasoning LLM with a GRPO algorithm. Initially, a pre-trained LLM is fine-tuned using a small set of high-quality labeled reasoning data with SFT as a cold start. Then, large-scale RL is applied specifically to enhance reasoning capabilities. After that, the model undergoes reject sampling across multiple tasks to refine output quality. In the final phase, the model is further trained using RL to enhance generalization across various tasks}{figure.22}{}}
\citation{baltruvsaitis-etal:2018multimodal}
\citation{radford-etal:2021learning}
\citation{xu-etal:2025qwen2,zhang-etal:2024mm}
\citation{qwenTeam:2025qwen2.5-VL}
\citation{grattafiori-etal:2024llama}
\citation{yu-etal:2024rlhf,wang-etal:2024rovrm,zang2025internlm,ji2025safe}
\citation{yu-etal:2024rlaif}
\citation{wang-etal:2024rovrm}
\def\@LN@column{1}
\@writefile{toc}{\contentsline {section}{\numberline {6}Reinforcement Learning for LLM-based Agents}{43}{section.6}\protected@file@percent }
\@writefile{toc}{\contentsline {section}{\numberline {7}Reinforcement Learning for Multimodal Models}{43}{section.7}\protected@file@percent }
\@writefile{toc}{\contentsline {subsection}{\numberline {7.1}Visual Language Models}{43}{subsection.7.1}\protected@file@percent }
\citation{xia-etal:2024less}
\citation{cheng-etal:2023everyone,wu-etal:2024reuse}
\citation{wang2026msrl}
\citation{sun-etal:2023aligning}
\def\@LN@column{1}
\@writefile{lof}{\contentsline {figure}{\numberline {23}{\ignorespaces RoVRM}}{44}{figure.23}\protected@file@percent }
\newlabel{fig:preference-transfer}{{23}{44}{RoVRM}{figure.23}{}}
\def\@LN@column{1}
\@writefile{toc}{\contentsline {subsection}{\numberline {7.2}Speech Generation Models}{45}{subsection.7.2}\protected@file@percent }
\@writefile{toc}{\contentsline {subsection}{\numberline {7.3}Diffusion Models}{45}{subsection.7.3}\protected@file@percent }
\@writefile{toc}{\contentsline {section}{\numberline {8}Summary}{45}{section.8}\protected@file@percent }
\def\@LN@column{1}
\@writefile{toc}{\contentsline {section}{\numberline {9}Systems and Datasets}{46}{section.9}\protected@file@percent }
\@writefile{lot}{\contentsline {table}{\numberline {2}{\ignorespaces datasets}}{46}{table.2}\protected@file@percent }
\newlabel{tab:dataset}{{2}{46}{datasets}{table.2}{}}
\@writefile{lot}{\contentsline {table}{\numberline {3}{\ignorespaces systems}}{46}{table.3}\protected@file@percent }
\newlabel{tab:system}{{3}{46}{systems}{table.3}{}}
\bibdata{rl-introduction}
\bibcite{azar:2024general}{{1}{2024}{{Azar et~al.}}{{Azar, Guo, Piot, Munos, Rowland, Valko, and Calandriello}}}
\bibcite{bahdanau-etal:2016actor}{{2}{2017}{{Bahdanau et~al.}}{{Bahdanau, Brakel, Xu, Goyal, Lowe, Pineau, Courville, and Bengio}}}
\bibcite{baltruvsaitis-etal:2018multimodal}{{3}{2018}{{Baltru{\v {s}}aitis et~al.}}{{Baltru{\v {s}}aitis, Ahuja, and Morency}}}
\bibcite{Bishop:2006}{{4}{2006}{{Bishop}}{{}}}
\bibcite{bradley-and-terry:rank}{{5}{1952}{{Bradley \& Terry}}{{Bradley and Terry}}}
\bibcite{bromley-etal:1993signature}{{6}{1993}{{Bromley et~al.}}{{Bromley, Guyon, LeCun, S{\"a}ckinger, and Shah}}}
\bibcite{chen-etal:2023accelerating}{{7}{2023}{{Chen et~al.}}{{Chen, Borgeaud, Irving, Lespiau, Sifre, and Jumper}}}
\bibcite{chen-etal:2023alpagasus}{{8}{2024}{{Chen et~al.}}{{Chen, Li, Yan, Wang, Gunaratna, Yadav, Tang, Srinivasan, Zhou, Huang, and Jin}}}
\bibcite{cheng-etal:2023everyone}{{9}{2023}{{Cheng et~al.}}{{Cheng, Xie, Bai, Dai, and Du}}}
\bibcite{christiano-etal:2017deep}{{10}{2017}{{Christiano et~al.}}{{Christiano, Leike, Brown, Martic, Legg, and Amodei}}}
\bibcite{coste-etal:2024reward}{{11}{2024}{{Coste et~al.}}{{Coste, Anwar, Kirk, and Krueger}}}
\bibcite{cui-etal:2023ultrafeedback}{{12}{2024{a}}{{Cui et~al.}}{{Cui, Yuan, Ding, Yao, He, Zhu, Ni, Xie, Xie, Lin, Liu, and Sun}}}
\bibcite{cui-etal:2024ultra}{{13}{2024{b}}{{Cui et~al.}}{{Cui, Yuan, Ding, Yao, He, Zhu, Ni, Xie, Xie, Lin, Liu, and Sun}}}
\bibcite{deb-etal:2016multi}{{14}{2016}{{Deb et~al.}}{{Deb, Sindhya, and Hakanen}}}
\def\@LN@column{1}
\bibcite{deepseek:2025r1}{{15}{2025}{{Deepseek}}{{}}}
\bibcite{dubois-etal:2024alpacafarm}{{16}{2023}{{Dubois et~al.}}{{Dubois, Li, Taori, Zhang, Gulrajani, Ba, Guestrin, Liang, and Hashimoto}}}
\bibcite{eisenstein-etal:2023helping}{{17}{2023}{{Eisenstein et~al.}}{{Eisenstein, Nagpal, Agarwal, Beirami, D'Amour, Dvijotham, Fisch, Heller, Pfohl, Ramachandran, et~al.}}}
\bibcite{ethayarajh:2024kto}{{18}{2024}{{Ethayarajh et~al.}}{{Ethayarajh, Xu, Muennighoff, Jurafsky, and Kiela}}}
\bibcite{gallego:2024refined}{{19}{2024}{{Gallego}}{{}}}
\bibcite{gao-etal:2023scaling}{{20}{2023}{{Gao et~al.}}{{Gao, Schulman, and Hilton}}}
\bibcite{gorbatovski-etal:2024learn}{{21}{2024}{{Gorbatovski et~al.}}{{Gorbatovski, Shaposhnikov, Malakhov, Surnachev, Aksenov, Maksimov, Balagansky, and Gavrilov}}}
\bibcite{grattafiori-etal:2024llama}{{22}{2024}{{Grattafiori et~al.}}{{Grattafiori, Dubey, Jauhri, Pandey, Kadian, Al-Dahle, Letman, Mathur, Schelten, Vaughan, et~al.}}}
\bibcite{guo:2025deepseek}{{23}{2025}{{Guo et~al.}}{{Guo, Yang, Zhang, Song, Zhang, Xu, Zhu, Ma, Wang, Bi, et~al.}}}
\bibcite{havrilla-etal:2024teaching}{{24}{2024}{{Havrilla et~al.}}{{Havrilla, Du, Raparthy, Nalmpantis, Dwivedi-Yu, Zhuravinskyi, Hambro, Sukhbaatar, and Raileanu}}}
\bibcite{hong:2024orpo}{{25}{2024}{{Hong et~al.}}{{Hong, Lee, and Thorne}}}
\bibcite{hu:2025reinforce++}{{26}{2025}{{Hu}}{{}}}
\bibcite{ji2025safe}{{27}{2025}{{Ji et~al.}}{{Ji, Chen, Pan, Zhu, Zhang, Li, Hong, Chen, Zhou, Wang, et~al.}}}
\bibcite{kumar-etal:2024training}{{28}{2024}{{Kumar et~al.}}{{Kumar, Zhuang, Agarwal, Su, Co-Reyes, Singh, Baumli, Iqbal, Bishop, Roelofs, et~al.}}}
\def\@LN@column{1}
\bibcite{li-etal:2025limr}{{29}{2025}{{Li et~al.}}{{Li, Zou, and Liu}}}
\bibcite{li-etal:2023remax}{{30}{2024}{{Li et~al.}}{{Li, Xu, Zhang, Lin, Yu, Sun, and Luo}}}
\bibcite{lightman-etal:2023let}{{31}{2024{a}}{{Lightman et~al.}}{{Lightman, Kosaraju, Burda, Edwards, Baker, Lee, Leike, Schulman, Sutskever, and Cobbe}}}
\bibcite{lightman-etal:2024lets}{{32}{2024{b}}{{Lightman et~al.}}{{Lightman, Kosaraju, Burda, Edwards, Baker, Lee, Leike, Schulman, Sutskever, and Cobbe}}}
\bibcite{liu-etal:2023GEval}{{33}{2023}{{Liu et~al.}}{{Liu, Iter, Xu, Wang, Xu, and Zhu}}}
\bibcite{mahan-etal:2024generative}{{34}{2024}{{Mahan et~al.}}{{Mahan, Van~Phung, Rafailov, Blagden, Lile, Castricato, Fr{\"a}nken, Finn, and Albalak}}}
\bibcite{masoudnia-etal:2014mixture}{{35}{2014}{{Masoudnia \& Ebrahimpour}}{{Masoudnia and Ebrahimpour}}}
\bibcite{meng:2025simpo}{{36}{2024}{{Meng et~al.}}{{Meng, Xia, and Chen}}}
\bibcite{miettinen:1999nonlinear}{{37}{1999}{{Miettinen}}{{}}}
\bibcite{miettinen-etal:2008introduction}{{38}{2008}{{Miettinen et~al.}}{{Miettinen, Ruiz, and Wierzbicki}}}
\bibcite{mnih-etal:2016asynchronous}{{39}{2016}{{Mnih et~al.}}{{Mnih, Badia, Mirza, Graves, Lillicrap, Harley, Silver, and Kavukcuoglu}}}
\bibcite{morimura-etal:2024filtered}{{40}{2024}{{Morimura et~al.}}{{Morimura, Sakamoto, Jinnai, Abe, and Ariu}}}
\bibcite{nakano-etal:2021webgpt}{{41}{2021}{{Nakano et~al.}}{{Nakano, Hilton, Balaji, Wu, Ouyang, Kim, Hesse, Jain, Kosaraju, Saunders, Jiang, Cobbe, Eloundou, Krueger, Button, Knight, Chess, and Schulman}}}
\def\@LN@column{1}
\bibcite{openai:2024learning}{{42}{2024}{{OpenAI}}{{}}}
\bibcite{ouyang:2022training}{{43}{2022}{{Ouyang et~al.}}{{Ouyang, Wu, Jiang, Almeida, Wainwright, Mishkin, Zhang, Agarwal, Slama, Ray, Schulman, Hilton, Kelton, Miller, Simens, Askell, Welinder, Christiano, Leike, and Lowe}}}
\bibcite{pope-etal:2023efficiently}{{44}{2023}{{Pope et~al.}}{{Pope, Douglas, Chowdhery, Devlin, Bradbury, Heek, Xiao, Agrawal, and Dean}}}
\bibcite{radford-etal:2021learning}{{45}{2021}{{Radford et~al.}}{{Radford, Kim, Hallacy, Ramesh, Goh, Agarwal, Sastry, Askell, Mishkin, Clark, Krueger, and Sutskever}}}
\bibcite{rafailov:2023direct}{{46}{2023}{{Rafailov et~al.}}{{Rafailov, Sharma, Mitchell, Manning, Ermon, and Finn}}}
\bibcite{schulman-etal:2015trust}{{47}{2015}{{Schulman et~al.}}{{Schulman, Levine, Abbeel, Jordan, and Moritz}}}
\bibcite{schulman-etal:2015high}{{48}{2016}{{Schulman et~al.}}{{Schulman, Moritz, Levine, Jordan, and Abbeel}}}
\bibcite{schulman-etal:2017proximal}{{49}{2017}{{Schulman et~al.}}{{Schulman, Wolski, Dhariwal, Radford, and Klimov}}}
\bibcite{setlur-etal:2024rewarding}{{50}{2024}{{Setlur et~al.}}{{Setlur, Nagpal, Fisch, Geng, Eisenstein, Agarwal, Agarwal, Berant, and Kumar}}}
\bibcite{shao:2025earlier}{{51}{2025}{{Shao et~al.}}{{Shao, Li, Liu, Chen, Zhou, Wang, Cai, and Li}}}
\bibcite{shao-etal:2024deepseekmath}{{52}{2024}{{Shao et~al.}}{{Shao, Wang, Zhu, Xu, Song, Bi, Zhang, Zhang, Li, Wu, et~al.}}}
\bibcite{silver-etal:2016mastering}{{53}{2016}{{Silver et~al.}}{{Silver, Huang, Maddison, Guez, Sifre, Van Den~Driessche, Schrittwieser, Antonoglou, Panneershelvam, Lanctot, et~al.}}}
\def\@LN@column{1}
\bibcite{singhal-etal:2023long}{{54}{2023}{{Singhal et~al.}}{{Singhal, Goyal, Xu, and Durrett}}}
\bibcite{sun-etal:2023simple}{{55}{2024}{{Sun et~al.}}{{Sun, Liu, Bair, and Kolter}}}
\bibcite{sun-etal:2023aligning}{{56}{2023}{{Sun et~al.}}{{Sun, Shen, Cao, Liu, Li, Shen, Gan, Gui, Wang, Yang, et~al.}}}
\bibcite{sutton-and-richard:1988learning}{{57}{1988}{{Sutton}}{{}}}
\bibcite{Sutton-and-Barto:2018RL}{{58}{2018}{{Sutton \& Barto}}{{Sutton and Barto}}}
\bibcite{szepesvari:2010algorithms}{{59}{2010}{{Szepesv{\'a}ri}}{{}}}
\bibcite{kimi-team:2025kimi}{{60}{2025}{{Team et~al.}}{{Team, Du, Gao, Xing, Jiang, Chen, Li, Xiao, Du, Liao, et~al.}}}
\bibcite{qwenTeam:2025qwen2.5-VL}{{61}{2025}{{Team}}{{}}}
\bibcite{touvron-etal:2023llama2}{{62}{2023}{{Touvron et~al.}}{{Touvron, Martin, Stone, Albert, Almahairi, Babaei, Bashlykov, Batra, Bhargava, Bhosale, Bikel, Blecher, Ferrer, Chen, Cucurull, Esiobu, Fernandes, Fu, Fu, Fuller, Gao, Goswami, Goyal, Hartshorn, Hosseini, Hou, Inan, Kardas, Kerkez, Khabsa, Kloumann, Korenev, Koura, Lachaux, Lavril, Lee, Liskovich, Lu, Mao, Martinet, Mihaylov, Mishra, Molybog, Nie, Poulton, Reizenstein, Rungta, Saladi, Schelten, Silva, Smith, Subramanian, Tan, Tang, Taylor, Williams, Kuan, Xu, Yan, Zarov, Zhang, Fan, Kambadur, Narang, Rodriguez, Stojnic, Edunov, and Scialom}}}
\bibcite{vinyals-rtal:2019grandmaster}{{63}{2019}{{Vinyals et~al.}}{{Vinyals, Babuschkin, Czarnecki, Mathieu, Dudzik, Chung, Choi, Powell, Ewalds, Georgiev, et~al.}}}
\bibcite{wang-etal:2023learning}{{64}{2023{a}}{{Wang et~al.}}{{Wang, Zhou, Chang, Liu, Zhang, Du, Xiao, and Zhu}}}
\bibcite{wang-etal:2024rovrm}{{65}{2024{a}}{{Wang et~al.}}{{Wang, Gan, Huo, Mu, Yang, He, Xiao, Zhang, Liu, Du, et~al.}}}
\bibcite{wang-etal:2024hybrid}{{66}{2024{b}}{{Wang et~al.}}{{Wang, Zhou, Chang, Li, Mu, Xiao, Liu, and Zhu}}}
\bibcite{wang-etal:2024esrl}{{67}{2024{c}}{{Wang et~al.}}{{Wang, Zhou, Hu, Huo, Li, Liu, Xiao, and Zhu}}}
\def\@LN@column{1}
\bibcite{wang-etal:wang2025gram}{{68}{2025}{{Wang et~al.}}{{Wang, Gan, Huo, Mu, He, Yang, Li, Xiao, Zhang, Liu, et~al.}}}
\bibcite{wang2026msrl}{{69}{2026{a}}{{Wang et~al.}}{{Wang, Huo, Gan, He, Meng, Li, Wang, Liu, Zhou, Zhu, et~al.}}}
\bibcite{wang-etal:wang2026gram}{{70}{2026{b}}{{Wang et~al.}}{{Wang, Mu, Zhou, Huo, Zhu, Zeng, Yang, Li, Hao, Zhang, et~al.}}}
\bibcite{wang-etal:2021selective}{{71}{2021}{{Wang et~al.}}{{Wang, Yan, Meng, and Zhou}}}
\bibcite{wang-etal:2023large}{{72}{2023{b}}{{Wang et~al.}}{{Wang, Li, Chen, Cai, Zhu, Lin, Cao, Liu, Liu, and Sui}}}
\bibcite{wang-etal:2023math}{{73}{2023{c}}{{Wang et~al.}}{{Wang, Li, Shao, Xu, Dai, Li, Chen, Wu, and Sui}}}
\bibcite{wang-and-zhou:2024chain}{{74}{2024}{{Wang \& Zhou}}{{Wang and Zhou}}}
\bibcite{wang-etal:2023far}{{75}{2023{d}}{{Wang et~al.}}{{Wang, Ivison, Dasigi, Hessel, Khot, Chandu, Wadden, MacMillan, Smith, Beltagy, and Hajishirzi}}}
\bibcite{wei-etal:2022finetuned}{{76}{2022}{{Wei et~al.}}{{Wei, Bosma, Zhao, Guu, Yu, Lester, Du, Dai, and Le}}}
\bibcite{wu-etal:2023fine}{{77}{2023}{{Wu et~al.}}{{Wu, Hu, Shi, Dziri, Suhr, Ammanabrolu, Smith, Ostendorf, and Hajishirzi}}}
\def\@LN@column{1}
\bibcite{wu-etal:2024reuse}{{78}{2024}{{Wu et~al.}}{{Wu, Balashankar, Kim, Eisenstein, and Beirami}}}
\bibcite{xia-etal:2024less}{{79}{2024}{{Xia et~al.}}{{Xia, Malladi, Gururangan, Arora, and Chen}}}
\bibcite{xiao-etal:2023introduction}{{80}{2023}{{Xiao \& Zhu}}{{Xiao and Zhu}}}
\bibcite{xiao-and-zhu:2025foundations}{{81}{2025}{{Xiao \& Zhu}}{{Xiao and Zhu}}}
\bibcite{xie-etal:2025logic}{{82}{2025}{{Xie et~al.}}{{Xie, Gao, Ren, Luo, Hong, Dai, Zhou, Qiu, Wu, and Luo}}}
\bibcite{xu:2024contrastive}{{83}{2024}{{Xu et~al.}}{{Xu, Sharaf, Chen, Tan, Shen, Durme, Murray, and Kim}}}
\bibcite{xu-etal:2025qwen2}{{84}{2025}{{Xu et~al.}}{{Xu, Guo, He, Hu, He, Bai, Chen, Wang, Fan, Dang, et~al.}}}
\bibcite{yang-etal:2024regularizing}{{85}{2024}{{Yang et~al.}}{{Yang, Ding, Lin, Zhang, and Zhang}}}
\bibcite{yao-etal:2023tree}{{86}{2023}{{Yao et~al.}}{{Yao, Yu, Zhao, Shafran, Griffiths, Cao, and Narasimhan}}}
\bibcite{yu-etal:2024rlhf}{{87}{2024{a}}{{Yu et~al.}}{{Yu, Yao, Zhang, He, Han, Cui, Hu, Liu, Zheng, Sun, et~al.}}}
\bibcite{yu-etal:2024rlaif}{{88}{2024{b}}{{Yu et~al.}}{{Yu, Zhang, Yao, Dang, Chen, Lu, Cui, He, Liu, Chua, et~al.}}}
\bibcite{yuan-etal:2024selfrewarding}{{89}{2024}{{Yuan et~al.}}{{Yuan, Pang, Cho, Li, Sukhbaatar, Xu, and Weston}}}
\bibcite{yuan-etal:2023scaling}{{90}{2023}{{Yuan et~al.}}{{Yuan, Yuan, Li, Dong, Lu, Tan, Zhou, and Zhou}}}
\def\@LN@column{1}
\bibcite{zang2025internlm}{{91}{2025}{{Zang et~al.}}{{Zang, Dong, Zhang, Cao, Liu, Ding, Wu, Ma, Duan, Zhang, et~al.}}}
\bibcite{zeng-etal:2025revisiting}{{92}{2025}{{Zeng et~al.}}{{Zeng, Cheng, Yin, Zhou, and Qiu}}}
\bibcite{zhang-etal:2024mm}{{93}{2024{a}}{{Zhang et~al.}}{{Zhang, Yu, Dong, Li, Su, Chu, and Yu}}}
\bibcite{zhang-etal:2024generative}{{94}{2024{b}}{{Zhang et~al.}}{{Zhang, Hosseini, Bansal, Kazemi, Kumar, and Agarwal}}}
\bibcite{zhang-etal:zhang2024generative}{{95}{2024{c}}{{Zhang et~al.}}{{Zhang, Hosseini, Bansal, Kazemi, Kumar, and Agarwal}}}
\bibcite{zhao-etal:2024atom}{{96}{2024}{{Zhao et~al.}}{{Zhao, Lin, Zhu, Ye, Chen, Zheng, Ceze, Krishnamurthy, Chen, and Kasikci}}}
\bibcite{zheng-etal:2023judging}{{97}{2023}{{Zheng et~al.}}{{Zheng, Chiang, Sheng, Zhuang, Wu, Zhuang, Lin, Li, Li, Xing, Zhang, Gonzalez, and Stoica}}}
\bibcite{zhou:2024prior}{{98}{2024}{{Zhou et~al.}}{{Zhou, Wang, Hu, Xiao, Zhang, and Zhu}}}
\bibstyle{rl-introduction}
\def\@LN@column{1}
\gdef \@abspage@last{54}
rl-introduction.bbl 0 → 100644
\begin{thebibliography}{98}
\providecommand{\natexlab}[1]{#1}
\providecommand{\url}[1]{\texttt{#1}}
\expandafter\ifx\csname urlstyle\endcsname\relax
\providecommand{\doi}[1]{doi: #1}\else
\providecommand{\doi}{doi: \begingroup \urlstyle{rm}\Url}\fi
\bibitem[Azar et~al.(2024)Azar, Guo, Piot, Munos, Rowland, Valko, and Calandriello]{azar:2024general}
Mohammad~Gheshlaghi Azar, Zhaohan~Daniel Guo, Bilal Piot, R{\'{e}}mi Munos, Mark Rowland, Michal Valko, and Daniele Calandriello.
\newblock A general theoretical paradigm to understand learning from human preferences.
\newblock In Sanjoy Dasgupta, Stephan Mandt, and Yingzhen Li (eds.), \emph{International Conference on Artificial Intelligence and Statistics, 2-4 May 2024, Palau de Congressos, Valencia, Spain}, volume 238 of \emph{Proceedings of Machine Learning Research}, pp.\ 4447--4455. {PMLR}, 2024.
\newblock URL \url{https://proceedings.mlr.press/v238/gheshlaghi-azar24a.html}.
\bibitem[Bahdanau et~al.(2017)Bahdanau, Brakel, Xu, Goyal, Lowe, Pineau, Courville, and Bengio]{bahdanau-etal:2016actor}
Dzmitry Bahdanau, Philemon Brakel, Kelvin Xu, Anirudh Goyal, Ryan Lowe, Joelle Pineau, Aaron~C. Courville, and Yoshua Bengio.
\newblock An actor-critic algorithm for sequence prediction.
\newblock In \emph{5th International Conference on Learning Representations, {ICLR} 2017, Toulon, France, April 24-26, 2017, Conference Track Proceedings}. OpenReview.net, 2017.
\newblock URL \url{https://openreview.net/forum?id=SJDaqqveg}.
\bibitem[Baltru{\v{s}}aitis et~al.(2018)Baltru{\v{s}}aitis, Ahuja, and Morency]{baltruvsaitis-etal:2018multimodal}
Tadas Baltru{\v{s}}aitis, Chaitanya Ahuja, and Louis-Philippe Morency.
\newblock Multimodal machine learning: A survey and taxonomy.
\newblock \emph{IEEE transactions on pattern analysis and machine intelligence}, 41\penalty0 (2):\penalty0 423--443, 2018.
\bibitem[Bishop(2006)]{Bishop:2006}
Christopher~M. Bishop.
\newblock \emph{Pattern Recognition and Machine Learning}.
\newblock Springer, 2006.
\bibitem[Bradley \& Terry(1952)Bradley and Terry]{bradley-and-terry:rank}
Ralph~Allan Bradley and Milton~E. Terry.
\newblock Rank analysis of incomplete block designs: I. the method of paired comparisons.
\newblock \emph{Biometrika}, 39\penalty0 (3/4):\penalty0 324--345, 1952.
\bibitem[Bromley et~al.(1993)Bromley, Guyon, LeCun, S{\"a}ckinger, and Shah]{bromley-etal:1993signature}
Jane Bromley, Isabelle Guyon, Yann LeCun, Eduard S{\"a}ckinger, and Roopak Shah.
\newblock Signature verification using a" siamese" time delay neural network.
\newblock \emph{Advances in neural information processing systems}, 6, 1993.
\bibitem[Chen et~al.(2023)Chen, Borgeaud, Irving, Lespiau, Sifre, and Jumper]{chen-etal:2023accelerating}
Charlie Chen, Sebastian Borgeaud, Geoffrey Irving, Jean-Baptiste Lespiau, Laurent Sifre, and John Jumper.
\newblock Accelerating large language model decoding with speculative sampling.
\newblock \emph{ArXiv preprint}, abs/2302.01318, 2023.
\newblock URL \url{https://arxiv.org/abs/2302.01318}.
\bibitem[Chen et~al.(2024)Chen, Li, Yan, Wang, Gunaratna, Yadav, Tang, Srinivasan, Zhou, Huang, and Jin]{chen-etal:2023alpagasus}
Lichang Chen, Shiyang Li, Jun Yan, Hai Wang, Kalpa Gunaratna, Vikas Yadav, Zheng Tang, Vijay Srinivasan, Tianyi Zhou, Heng Huang, and Hongxia Jin.
\newblock Alpagasus: Training a better alpaca with fewer data.
\newblock In \emph{The Twelfth International Conference on Learning Representations, {ICLR} 2024, Vienna, Austria, May 7-11, 2024}. OpenReview.net, 2024.
\newblock URL \url{https://openreview.net/forum?id=FdVXgSJhvz}.
\bibitem[Cheng et~al.(2023)Cheng, Xie, Bai, Dai, and Du]{cheng-etal:2023everyone}
Pengyu Cheng, Jiawen Xie, Ke~Bai, Yong Dai, and Nan Du.
\newblock Everyone deserves a reward: Learning customized human preferences.
\newblock \emph{arXiv preprint arXiv:2309.03126}, 2023.
\bibitem[Christiano et~al.(2017)Christiano, Leike, Brown, Martic, Legg, and Amodei]{christiano-etal:2017deep}
Paul~F. Christiano, Jan Leike, Tom~B. Brown, Miljan Martic, Shane Legg, and Dario Amodei.
\newblock Deep reinforcement learning from human preferences.
\newblock In Isabelle Guyon, Ulrike von Luxburg, Samy Bengio, Hanna~M. Wallach, Rob Fergus, S.~V.~N. Vishwanathan, and Roman Garnett (eds.), \emph{Advances in Neural Information Processing Systems 30: Annual Conference on Neural Information Processing Systems 2017, December 4-9, 2017, Long Beach, CA, {USA}}, pp.\ 4299--4307, 2017.
\newblock URL \url{https://proceedings.neurips.cc/paper/2017/hash/d5e2c0adad503c91f91df240d0cd4e49-Abstract.html}.
\bibitem[Coste et~al.(2024)Coste, Anwar, Kirk, and Krueger]{coste-etal:2024reward}
Thomas Coste, Usman Anwar, Robert Kirk, and David Krueger.
\newblock Reward model ensembles help mitigate overoptimization.
\newblock In \emph{The Twelfth International Conference on Learning Representations, {ICLR} 2024, Vienna, Austria, May 7-11, 2024}. OpenReview.net, 2024.
\newblock URL \url{https://openreview.net/forum?id=dcjtMYkpXx}.
\bibitem[Cui et~al.(2024{\natexlab{a}})Cui, Yuan, Ding, Yao, He, Zhu, Ni, Xie, Xie, Lin, Liu, and Sun]{cui-etal:2023ultrafeedback}
Ganqu Cui, Lifan Yuan, Ning Ding, Guanming Yao, Bingxiang He, Wei Zhu, Yuan Ni, Guotong Xie, Ruobing Xie, Yankai Lin, Zhiyuan Liu, and Maosong Sun.
\newblock {ULTRAFEEDBACK:} boosting language models with scaled {AI} feedback.
\newblock In \emph{Forty-first International Conference on Machine Learning, {ICML} 2024, Vienna, Austria, July 21-27, 2024}. OpenReview.net, 2024{\natexlab{a}}.
\newblock URL \url{https://openreview.net/forum?id=BOorDpKHiJ}.
\bibitem[Cui et~al.(2024{\natexlab{b}})Cui, Yuan, Ding, Yao, He, Zhu, Ni, Xie, Xie, Lin, Liu, and Sun]{cui-etal:2024ultra}
Ganqu Cui, Lifan Yuan, Ning Ding, Guanming Yao, Bingxiang He, Wei Zhu, Yuan Ni, Guotong Xie, Ruobing Xie, Yankai Lin, Zhiyuan Liu, and Maosong Sun.
\newblock {ULTRAFEEDBACK:} boosting language models with scaled {AI} feedback.
\newblock In \emph{Forty-first International Conference on Machine Learning, {ICML} 2024, Vienna, Austria, July 21-27, 2024}. OpenReview.net, 2024{\natexlab{b}}.
\newblock URL \url{https://openreview.net/forum?id=BOorDpKHiJ}.
\bibitem[Deb et~al.(2016)Deb, Sindhya, and Hakanen]{deb-etal:2016multi}
Kalyanmoy Deb, Karthik Sindhya, and Jussi Hakanen.
\newblock Multi-objective optimization.
\newblock In \emph{Decision sciences}, pp.\ 161--200. CRC Press, 2016.
\bibitem[Deepseek(2025)]{deepseek:2025r1}
Deepseek.
\newblock Deepseek-r1: Incentivizing reasoning capability in llms via reinforcement learning.
\newblock \emph{arXiv preprint arXiv:2501.12948}, 2025.
\bibitem[Dubois et~al.(2023)Dubois, Li, Taori, Zhang, Gulrajani, Ba, Guestrin, Liang, and Hashimoto]{dubois-etal:2024alpacafarm}
Yann Dubois, Chen~Xuechen Li, Rohan Taori, Tianyi Zhang, Ishaan Gulrajani, Jimmy Ba, Carlos Guestrin, Percy Liang, and Tatsunori~B. Hashimoto.
\newblock Alpacafarm: {A} simulation framework for methods that learn from human feedback.
\newblock In Alice Oh, Tristan Naumann, Amir Globerson, Kate Saenko, Moritz Hardt, and Sergey Levine (eds.), \emph{Advances in Neural Information Processing Systems 36: Annual Conference on Neural Information Processing Systems 2023, NeurIPS 2023, New Orleans, LA, USA, December 10 - 16, 2023}, 2023.
\newblock URL \url{http://papers.nips.cc/paper\_files/paper/2023/hash/5fc47800ee5b30b8777fdd30abcaaf3b-Abstract-Conference.html}.
\bibitem[Eisenstein et~al.(2023)Eisenstein, Nagpal, Agarwal, Beirami, D'Amour, Dvijotham, Fisch, Heller, Pfohl, Ramachandran, et~al.]{eisenstein-etal:2023helping}
Jacob Eisenstein, Chirag Nagpal, Alekh Agarwal, Ahmad Beirami, Alex D'Amour, DJ~Dvijotham, Adam Fisch, Katherine Heller, Stephen Pfohl, Deepak Ramachandran, et~al.
\newblock Helping or herding? reward model ensembles mitigate but do not eliminate reward hacking.
\newblock \emph{ArXiv preprint}, abs/2312.09244, 2023.
\newblock URL \url{https://arxiv.org/abs/2312.09244}.
\bibitem[Ethayarajh et~al.(2024)Ethayarajh, Xu, Muennighoff, Jurafsky, and Kiela]{ethayarajh:2024kto}
Kawin Ethayarajh, Winnie Xu, Niklas Muennighoff, Dan Jurafsky, and Douwe Kiela.
\newblock Kto: Model alignment as prospect theoretic optimization.
\newblock \emph{ArXiv preprint}, abs/2402.01306, 2024.
\newblock URL \url{https://arxiv.org/abs/2402.01306}.
\bibitem[Gallego(2024)]{gallego:2024refined}
V{\'\i}ctor Gallego.
\newblock Refined direct preference optimization with synthetic data for behavioral alignment of llms.
\newblock \emph{ArXiv preprint}, abs/2402.08005, 2024.
\newblock URL \url{https://arxiv.org/abs/2402.08005}.
\bibitem[Gao et~al.(2023)Gao, Schulman, and Hilton]{gao-etal:2023scaling}
Leo Gao, John Schulman, and Jacob Hilton.
\newblock Scaling laws for reward model overoptimization.
\newblock In Andreas Krause, Emma Brunskill, Kyunghyun Cho, Barbara Engelhardt, Sivan Sabato, and Jonathan Scarlett (eds.), \emph{International Conference on Machine Learning, {ICML} 2023, 23-29 July 2023, Honolulu, Hawaii, {USA}}, volume 202 of \emph{Proceedings of Machine Learning Research}, pp.\ 10835--10866. {PMLR}, 2023.
\newblock URL \url{https://proceedings.mlr.press/v202/gao23h.html}.
\bibitem[Gorbatovski et~al.(2024)Gorbatovski, Shaposhnikov, Malakhov, Surnachev, Aksenov, Maksimov, Balagansky, and Gavrilov]{gorbatovski-etal:2024learn}
Alexey Gorbatovski, Boris Shaposhnikov, Alexey Malakhov, Nikita Surnachev, Yaroslav Aksenov, Ian Maksimov, Nikita Balagansky, and Daniil Gavrilov.
\newblock Learn your reference model for real good alignment.
\newblock \emph{ArXiv preprint}, abs/2404.09656, 2024.
\newblock URL \url{https://arxiv.org/abs/2404.09656}.
\bibitem[Grattafiori et~al.(2024)Grattafiori, Dubey, Jauhri, Pandey, Kadian, Al-Dahle, Letman, Mathur, Schelten, Vaughan, et~al.]{grattafiori-etal:2024llama}
Aaron Grattafiori, Abhimanyu Dubey, Abhinav Jauhri, Abhinav Pandey, Abhishek Kadian, Ahmad Al-Dahle, Aiesha Letman, Akhil Mathur, Alan Schelten, Alex Vaughan, et~al.
\newblock The llama 3 herd of models.
\newblock \emph{arXiv preprint arXiv:2407.21783}, 2024.
\bibitem[Guo et~al.(2025)Guo, Yang, Zhang, Song, Zhang, Xu, Zhu, Ma, Wang, Bi, et~al.]{guo:2025deepseek}
Daya Guo, Dejian Yang, Haowei Zhang, Junxiao Song, Ruoyu Zhang, Runxin Xu, Qihao Zhu, Shirong Ma, Peiyi Wang, Xiao Bi, et~al.
\newblock Deepseek-r1: Incentivizing reasoning capability in llms via reinforcement learning.
\newblock \emph{ArXiv preprint}, abs/2501.12948, 2025.
\newblock URL \url{https://arxiv.org/abs/2501.12948}.
\bibitem[Havrilla et~al.(2024)Havrilla, Du, Raparthy, Nalmpantis, Dwivedi-Yu, Zhuravinskyi, Hambro, Sukhbaatar, and Raileanu]{havrilla-etal:2024teaching}
Alex Havrilla, Yuqing Du, Sharath~Chandra Raparthy, Christoforos Nalmpantis, Jane Dwivedi-Yu, Maksym Zhuravinskyi, Eric Hambro, Sainbayar Sukhbaatar, and Roberta Raileanu.
\newblock Teaching large language models to reason with reinforcement learning.
\newblock \emph{ArXiv preprint}, abs/2403.04642, 2024.
\newblock URL \url{https://arxiv.org/abs/2403.04642}.
\bibitem[Hong et~al.(2024)Hong, Lee, and Thorne]{hong:2024orpo}
Jiwoo Hong, Noah Lee, and James Thorne.
\newblock Orpo: Monolithic preference optimization without reference model.
\newblock \emph{ArXiv preprint}, abs/2403.07691, 2024.
\newblock URL \url{https://arxiv.org/abs/2403.07691}.
\bibitem[Hu(2025)]{hu:2025reinforce++}
Jian Hu.
\newblock Reinforce++: A simple and efficient approach for aligning large language models.
\newblock \emph{ArXiv preprint}, abs/2501.03262, 2025.
\newblock URL \url{https://arxiv.org/abs/2501.03262}.
\bibitem[Ji et~al.(2025)Ji, Chen, Pan, Zhu, Zhang, Li, Hong, Chen, Zhou, Wang, et~al.]{ji2025safe}
Jiaming Ji, Xinyu Chen, Rui Pan, Han Zhu, Conghui Zhang, Jiahao Li, Donghai Hong, Boyuan Chen, Jiayi Zhou, Kaile Wang, et~al.
\newblock Safe rlhf-v: Safe reinforcement learning from human feedback in multimodal large language models.
\newblock \emph{arXiv preprint arXiv:2503.17682}, 2025.
\bibitem[Kumar et~al.(2024)Kumar, Zhuang, Agarwal, Su, Co-Reyes, Singh, Baumli, Iqbal, Bishop, Roelofs, et~al.]{kumar-etal:2024training}
Aviral Kumar, Vincent Zhuang, Rishabh Agarwal, Yi~Su, John~D Co-Reyes, Avi Singh, Kate Baumli, Shariq Iqbal, Colton Bishop, Rebecca Roelofs, et~al.
\newblock Training language models to self-correct via reinforcement learning.
\newblock \emph{ArXiv preprint}, abs/2409.12917, 2024.
\newblock URL \url{https://arxiv.org/abs/2409.12917}.
\bibitem[Li et~al.(2025)Li, Zou, and Liu]{li-etal:2025limr}
Xuefeng Li, Haoyang Zou, and Pengfei Liu.
\newblock Limr: Less is more for rl scaling.
\newblock \emph{ArXiv preprint}, abs/2502.11886, 2025.
\newblock URL \url{https://arxiv.org/abs/2502.11886}.
\bibitem[Li et~al.(2024)Li, Xu, Zhang, Lin, Yu, Sun, and Luo]{li-etal:2023remax}
Ziniu Li, Tian Xu, Yushun Zhang, Zhihang Lin, Yang Yu, Ruoyu Sun, and Zhi{-}Quan Luo.
\newblock Remax: {A} simple, effective, and efficient reinforcement learning method for aligning large language models.
\newblock In \emph{Forty-first International Conference on Machine Learning, {ICML} 2024, Vienna, Austria, July 21-27, 2024}. OpenReview.net, 2024.
\newblock URL \url{https://openreview.net/forum?id=Stn8hXkpe6}.
\bibitem[Lightman et~al.(2024{\natexlab{a}})Lightman, Kosaraju, Burda, Edwards, Baker, Lee, Leike, Schulman, Sutskever, and Cobbe]{lightman-etal:2023let}
Hunter Lightman, Vineet Kosaraju, Yuri Burda, Harrison Edwards, Bowen Baker, Teddy Lee, Jan Leike, John Schulman, Ilya Sutskever, and Karl Cobbe.
\newblock Let's verify step by step.
\newblock In \emph{The Twelfth International Conference on Learning Representations, {ICLR} 2024, Vienna, Austria, May 7-11, 2024}. OpenReview.net, 2024{\natexlab{a}}.
\newblock URL \url{https://openreview.net/forum?id=v8L0pN6EOi}.
\bibitem[Lightman et~al.(2024{\natexlab{b}})Lightman, Kosaraju, Burda, Edwards, Baker, Lee, Leike, Schulman, Sutskever, and Cobbe]{lightman-etal:2024lets}
Hunter Lightman, Vineet Kosaraju, Yuri Burda, Harrison Edwards, Bowen Baker, Teddy Lee, Jan Leike, John Schulman, Ilya Sutskever, and Karl Cobbe.
\newblock Let's verify step by step.
\newblock In \emph{The Twelfth International Conference on Learning Representations, {ICLR} 2024, Vienna, Austria, May 7-11, 2024}. OpenReview.net, 2024{\natexlab{b}}.
\newblock URL \url{https://openreview.net/forum?id=v8L0pN6EOi}.
\bibitem[Liu et~al.(2023)Liu, Iter, Xu, Wang, Xu, and Zhu]{liu-etal:2023GEval}
Yang Liu, Dan Iter, Yichong Xu, Shuohang Wang, Ruochen Xu, and Chenguang Zhu.
\newblock {G}-eval: {NLG} evaluation using gpt-4 with better human alignment.
\newblock In Houda Bouamor, Juan Pino, and Kalika Bali (eds.), \emph{Proceedings of the 2023 Conference on Empirical Methods in Natural Language Processing}, pp.\ 2511--2522, Singapore, 2023. Association for Computational Linguistics.
\newblock \doi{10.18653/v1/2023.emnlp-main.153}.
\newblock URL \url{https://aclanthology.org/2023.emnlp-main.153}.
\bibitem[Mahan et~al.(2024)Mahan, Van~Phung, Rafailov, Blagden, Lile, Castricato, Fr{\"a}nken, Finn, and Albalak]{mahan-etal:2024generative}
Dakota Mahan, Duy Van~Phung, Rafael Rafailov, Chase Blagden, Nathan Lile, Louis Castricato, Jan-Philipp Fr{\"a}nken, Chelsea Finn, and Alon Albalak.
\newblock Generative reward models.
\newblock \emph{ArXiv preprint}, abs/2410.12832, 2024.
\newblock URL \url{https://arxiv.org/abs/2410.12832}.
\bibitem[Masoudnia \& Ebrahimpour(2014)Masoudnia and Ebrahimpour]{masoudnia-etal:2014mixture}
Saeed Masoudnia and Reza Ebrahimpour.
\newblock Mixture of experts: a literature survey.
\newblock \emph{Artificial Intelligence Review}, 42:\penalty0 275--293, 2014.
\bibitem[Meng et~al.(2024)Meng, Xia, and Chen]{meng:2025simpo}
Yu~Meng, Mengzhou Xia, and Danqi Chen.
\newblock Simpo: Simple preference optimization with a reference-free reward.
\newblock In Amir Globersons, Lester Mackey, Danielle Belgrave, Angela Fan, Ulrich Paquet, Jakub~M. Tomczak, and Cheng Zhang (eds.), \emph{Advances in Neural Information Processing Systems 38: Annual Conference on Neural Information Processing Systems 2024, NeurIPS 2024, Vancouver, BC, Canada, December 10 - 15, 2024}, 2024.
\newblock URL \url{http://papers.nips.cc/paper\_files/paper/2024/hash/e099c1c9699814af0be873a175361713-Abstract-Conference.html}.
\bibitem[Miettinen(1999)]{miettinen:1999nonlinear}
Kaisa Miettinen.
\newblock \emph{Nonlinear multiobjective optimization}, volume~12.
\newblock Springer Science \& Business Media, 1999.
\bibitem[Miettinen et~al.(2008)Miettinen, Ruiz, and Wierzbicki]{miettinen-etal:2008introduction}
Kaisa Miettinen, Francisco Ruiz, and Andrzej~P Wierzbicki.
\newblock Introduction to multiobjective optimization: interactive approaches.
\newblock In \emph{Multiobjective optimization: interactive and evolutionary approaches}, pp.\ 27--57. Springer, 2008.
\bibitem[Mnih et~al.(2016)Mnih, Badia, Mirza, Graves, Lillicrap, Harley, Silver, and Kavukcuoglu]{mnih-etal:2016asynchronous}
Volodymyr Mnih, Adri{\`{a}}~Puigdom{\`{e}}nech Badia, Mehdi Mirza, Alex Graves, Timothy~P. Lillicrap, Tim Harley, David Silver, and Koray Kavukcuoglu.
\newblock Asynchronous methods for deep reinforcement learning.
\newblock In Maria{-}Florina Balcan and Kilian~Q. Weinberger (eds.), \emph{Proceedings of the 33nd International Conference on Machine Learning, {ICML} 2016, New York City, NY, USA, June 19-24, 2016}, volume~48 of \emph{{JMLR} Workshop and Conference Proceedings}, pp.\ 1928--1937. JMLR.org, 2016.
\newblock URL \url{http://proceedings.mlr.press/v48/mniha16.html}.
\bibitem[Morimura et~al.(2024)Morimura, Sakamoto, Jinnai, Abe, and Ariu]{morimura-etal:2024filtered}
Tetsuro Morimura, Mitsuki Sakamoto, Yuu Jinnai, Kenshi Abe, and Kaito Ariu.
\newblock Filtered direct preference optimization.
\newblock \emph{ArXiv preprint}, abs/2404.13846, 2024.
\newblock URL \url{https://arxiv.org/abs/2404.13846}.
\bibitem[Nakano et~al.(2021)Nakano, Hilton, Balaji, Wu, Ouyang, Kim, Hesse, Jain, Kosaraju, Saunders, Jiang, Cobbe, Eloundou, Krueger, Button, Knight, Chess, and Schulman]{nakano-etal:2021webgpt}
Reiichiro Nakano, Jacob Hilton, Suchir Balaji, Jeff Wu, Long Ouyang, Christina Kim, Christopher Hesse, Shantanu Jain, Vineet Kosaraju, William Saunders, Xu~Jiang, Karl Cobbe, Tyna Eloundou, Gretchen Krueger, Kevin Button, Matthew Knight, Benjamin Chess, and John Schulman.
\newblock Webgpt: Browser-assisted question-answering with human feedback.
\newblock \emph{ArXiv preprint}, abs/2112.09332, 2021.
\newblock URL \url{https://arxiv.org/abs/2112.09332}.
\bibitem[OpenAI(2024)]{openai:2024learning}
OpenAI.
\newblock Learning to reason with llms, September 2024.
\newblock URL \url{https://openai.com/index/learning-to-reason-with-llms/}.
\bibitem[Ouyang et~al.(2022)Ouyang, Wu, Jiang, Almeida, Wainwright, Mishkin, Zhang, Agarwal, Slama, Ray, Schulman, Hilton, Kelton, Miller, Simens, Askell, Welinder, Christiano, Leike, and Lowe]{ouyang:2022training}
Long Ouyang, Jeffrey Wu, Xu~Jiang, Diogo Almeida, Carroll~L. Wainwright, Pamela Mishkin, Chong Zhang, Sandhini Agarwal, Katarina Slama, Alex Ray, John Schulman, Jacob Hilton, Fraser Kelton, Luke Miller, Maddie Simens, Amanda Askell, Peter Welinder, Paul~F. Christiano, Jan Leike, and Ryan Lowe.
\newblock Training language models to follow instructions with human feedback.
\newblock In Sanmi Koyejo, S.~Mohamed, A.~Agarwal, Danielle Belgrave, K.~Cho, and A.~Oh (eds.), \emph{Advances in Neural Information Processing Systems 35: Annual Conference on Neural Information Processing Systems 2022, NeurIPS 2022, New Orleans, LA, USA, November 28 - December 9, 2022}, 2022.
\newblock URL \url{http://papers.nips.cc/paper\_files/paper/2022/hash/b1efde53be364a73914f58805a001731-Abstract-Conference.html}.
\bibitem[Pope et~al.(2023)Pope, Douglas, Chowdhery, Devlin, Bradbury, Heek, Xiao, Agrawal, and Dean]{pope-etal:2023efficiently}
Reiner Pope, Sholto Douglas, Aakanksha Chowdhery, Jacob Devlin, James Bradbury, Jonathan Heek, Kefan Xiao, Shivani Agrawal, and Jeff Dean.
\newblock Efficiently scaling transformer inference.
\newblock \emph{Proceedings of Machine Learning and Systems}, 5:\penalty0 606--624, 2023.
\bibitem[Radford et~al.(2021)Radford, Kim, Hallacy, Ramesh, Goh, Agarwal, Sastry, Askell, Mishkin, Clark, Krueger, and Sutskever]{radford-etal:2021learning}
Alec Radford, Jong~Wook Kim, Chris Hallacy, Aditya Ramesh, Gabriel Goh, Sandhini Agarwal, Girish Sastry, Amanda Askell, Pamela Mishkin, Jack Clark, Gretchen Krueger, and Ilya Sutskever.
\newblock Learning transferable visual models from natural language supervision, 2021.
\newblock URL \url{https://arxiv.org/abs/2103.00020}.
\bibitem[Rafailov et~al.(2023)Rafailov, Sharma, Mitchell, Manning, Ermon, and Finn]{rafailov:2023direct}
Rafael Rafailov, Archit Sharma, Eric Mitchell, Christopher~D. Manning, Stefano Ermon, and Chelsea Finn.
\newblock Direct preference optimization: Your language model is secretly a reward model.
\newblock In Alice Oh, Tristan Naumann, Amir Globerson, Kate Saenko, Moritz Hardt, and Sergey Levine (eds.), \emph{Advances in Neural Information Processing Systems 36: Annual Conference on Neural Information Processing Systems 2023, NeurIPS 2023, New Orleans, LA, USA, December 10 - 16, 2023}, 2023.
\newblock URL \url{http://papers.nips.cc/paper\_files/paper/2023/hash/a85b405ed65c6477a4fe8302b5e06ce7-Abstract-Conference.html}.
\bibitem[Schulman et~al.(2015)Schulman, Levine, Abbeel, Jordan, and Moritz]{schulman-etal:2015trust}
John Schulman, Sergey Levine, Pieter Abbeel, Michael~I. Jordan, and Philipp Moritz.
\newblock Trust region policy optimization.
\newblock In Francis~R. Bach and David~M. Blei (eds.), \emph{Proceedings of the 32nd International Conference on Machine Learning, {ICML} 2015, Lille, France, 6-11 July 2015}, volume~37 of \emph{{JMLR} Workshop and Conference Proceedings}, pp.\ 1889--1897. JMLR.org, 2015.
\newblock URL \url{http://proceedings.mlr.press/v37/schulman15.html}.
\bibitem[Schulman et~al.(2016)Schulman, Moritz, Levine, Jordan, and Abbeel]{schulman-etal:2015high}
John Schulman, Philipp Moritz, Sergey Levine, Michael~I. Jordan, and Pieter Abbeel.
\newblock High-dimensional continuous control using generalized advantage estimation.
\newblock In Yoshua Bengio and Yann LeCun (eds.), \emph{4th International Conference on Learning Representations, {ICLR} 2016, San Juan, Puerto Rico, May 2-4, 2016, Conference Track Proceedings}, 2016.
\newblock URL \url{http://arxiv.org/abs/1506.02438}.
\bibitem[Schulman et~al.(2017)Schulman, Wolski, Dhariwal, Radford, and Klimov]{schulman-etal:2017proximal}
John Schulman, Filip Wolski, Prafulla Dhariwal, Alec Radford, and Oleg Klimov.
\newblock Proximal policy optimization algorithms.
\newblock \emph{ArXiv preprint}, abs/1707.06347, 2017.
\newblock URL \url{https://arxiv.org/abs/1707.06347}.
\bibitem[Setlur et~al.(2024)Setlur, Nagpal, Fisch, Geng, Eisenstein, Agarwal, Agarwal, Berant, and Kumar]{setlur-etal:2024rewarding}
Amrith Setlur, Chirag Nagpal, Adam Fisch, Xinyang Geng, Jacob Eisenstein, Rishabh Agarwal, Alekh Agarwal, Jonathan Berant, and Aviral Kumar.
\newblock Rewarding progress: Scaling automated process verifiers for llm reasoning.
\newblock \emph{ArXiv preprint}, abs/2410.08146, 2024.
\newblock URL \url{https://arxiv.org/abs/2410.08146}.
\bibitem[Shao et~al.(2025)Shao, Li, Liu, Chen, Zhou, Wang, Cai, and Li]{shao:2025earlier}
Ruichen Shao, Bei Li, Gangao Liu, Yang Chen, Xiang Zhou, Jingang Wang, Xunliang Cai, and Peng Li.
\newblock Earlier tokens contribute more: Learning direct preference optimization from temporal decay perspective.
\newblock \emph{ArXiv preprint}, abs/2502.14340, 2025.
\newblock URL \url{https://arxiv.org/abs/2502.14340}.
\bibitem[Shao et~al.(2024)Shao, Wang, Zhu, Xu, Song, Bi, Zhang, Zhang, Li, Wu, et~al.]{shao-etal:2024deepseekmath}
Zhihong Shao, Peiyi Wang, Qihao Zhu, Runxin Xu, Junxiao Song, Xiao Bi, Haowei Zhang, Mingchuan Zhang, YK~Li, Y~Wu, et~al.
\newblock Deepseekmath: Pushing the limits of mathematical reasoning in open language models.
\newblock \emph{ArXiv preprint}, abs/2402.03300, 2024.
\newblock URL \url{https://arxiv.org/abs/2402.03300}.
\bibitem[Silver et~al.(2016)Silver, Huang, Maddison, Guez, Sifre, Van Den~Driessche, Schrittwieser, Antonoglou, Panneershelvam, Lanctot, et~al.]{silver-etal:2016mastering}
David Silver, Aja Huang, Chris~J Maddison, Arthur Guez, Laurent Sifre, George Van Den~Driessche, Julian Schrittwieser, Ioannis Antonoglou, Veda Panneershelvam, Marc Lanctot, et~al.
\newblock Mastering the game of go with deep neural networks and tree search.
\newblock \emph{nature}, 529\penalty0 (7587):\penalty0 484--489, 2016.
\bibitem[Singhal et~al.(2023)Singhal, Goyal, Xu, and Durrett]{singhal-etal:2023long}
Prasann Singhal, Tanya Goyal, Jiacheng Xu, and Greg Durrett.
\newblock A long way to go: Investigating length correlations in rlhf.
\newblock \emph{ArXiv preprint}, abs/2310.03716, 2023.
\newblock URL \url{https://arxiv.org/abs/2310.03716}.
\bibitem[Sun et~al.(2024)Sun, Liu, Bair, and Kolter]{sun-etal:2023simple}
Mingjie Sun, Zhuang Liu, Anna Bair, and J.~Zico Kolter.
\newblock A simple and effective pruning approach for large language models.
\newblock In \emph{The Twelfth International Conference on Learning Representations, {ICLR} 2024, Vienna, Austria, May 7-11, 2024}. OpenReview.net, 2024.
\newblock URL \url{https://openreview.net/forum?id=PxoFut3dWW}.
\bibitem[Sun et~al.(2023)Sun, Shen, Cao, Liu, Li, Shen, Gan, Gui, Wang, Yang, et~al.]{sun-etal:2023aligning}
Zhiqing Sun, Sheng Shen, Shengcao Cao, Haotian Liu, Chunyuan Li, Yikang Shen, Chuang Gan, Liang-Yan Gui, Yu-Xiong Wang, Yiming Yang, et~al.
\newblock Aligning large multimodal models with factually augmented rlhf.
\newblock \emph{arXiv preprint arXiv:2309.14525}, 2023.
\bibitem[Sutton(1988)]{sutton-and-richard:1988learning}
Richard~S Sutton.
\newblock Learning to predict by the methods of temporal differences.
\newblock \emph{Machine learning}, 3:\penalty0 9--44, 1988.
\bibitem[Sutton \& Barto(2018)Sutton and Barto]{Sutton-and-Barto:2018RL}
Richard~S. Sutton and Andrew~G. Barto.
\newblock \emph{Reinforcement Learning: An Introduction (2nd ed.)}.
\newblock The MIT Press, 2018.
\bibitem[Szepesv{\'a}ri(2010)]{szepesvari:2010algorithms}
Csaba Szepesv{\'a}ri.
\newblock Algorithms for reinforcement learning.
\newblock \emph{Synthesis Lectures on Artificial Intelligence and Machine Learning}, 4\penalty0 (1):\penalty0 1--103, 2010.
\bibitem[Team et~al.(2025)Team, Du, Gao, Xing, Jiang, Chen, Li, Xiao, Du, Liao, et~al.]{kimi-team:2025kimi}
Kimi Team, Angang Du, Bofei Gao, Bowei Xing, Changjiu Jiang, Cheng Chen, Cheng Li, Chenjun Xiao, Chenzhuang Du, Chonghua Liao, et~al.
\newblock Kimi k1. 5: Scaling reinforcement learning with llms.
\newblock \emph{ArXiv preprint}, abs/2501.12599, 2025.
\newblock URL \url{https://arxiv.org/abs/2501.12599}.
\bibitem[Team(2025)]{qwenTeam:2025qwen2.5-VL}
Qwen Team.
\newblock Qwen2.5-vl, January 2025.
\newblock URL \url{https://qwenlm.github.io/blog/qwen2.5-vl/}.
\bibitem[Touvron et~al.(2023)Touvron, Martin, Stone, Albert, Almahairi, Babaei, Bashlykov, Batra, Bhargava, Bhosale, Bikel, Blecher, Ferrer, Chen, Cucurull, Esiobu, Fernandes, Fu, Fu, Fuller, Gao, Goswami, Goyal, Hartshorn, Hosseini, Hou, Inan, Kardas, Kerkez, Khabsa, Kloumann, Korenev, Koura, Lachaux, Lavril, Lee, Liskovich, Lu, Mao, Martinet, Mihaylov, Mishra, Molybog, Nie, Poulton, Reizenstein, Rungta, Saladi, Schelten, Silva, Smith, Subramanian, Tan, Tang, Taylor, Williams, Kuan, Xu, Yan, Zarov, Zhang, Fan, Kambadur, Narang, Rodriguez, Stojnic, Edunov, and Scialom]{touvron-etal:2023llama2}
Hugo Touvron, Louis Martin, Kevin Stone, Peter Albert, Amjad Almahairi, Yasmine Babaei, Nikolay Bashlykov, Soumya Batra, Prajjwal Bhargava, Shruti Bhosale, Dan Bikel, Lukas Blecher, Cristian~Canton Ferrer, Moya Chen, Guillem Cucurull, David Esiobu, Jude Fernandes, Jeremy Fu, Wenyin Fu, Brian Fuller, Cynthia Gao, Vedanuj Goswami, Naman Goyal, Anthony Hartshorn, Saghar Hosseini, Rui Hou, Hakan Inan, Marcin Kardas, Viktor Kerkez, Madian Khabsa, Isabel Kloumann, Artem Korenev, Punit~Singh Koura, Marie-Anne Lachaux, Thibaut Lavril, Jenya Lee, Diana Liskovich, Yinghai Lu, Yuning Mao, Xavier Martinet, Todor Mihaylov, Pushkar Mishra, Igor Molybog, Yixin Nie, Andrew Poulton, Jeremy Reizenstein, Rashi Rungta, Kalyan Saladi, Alan Schelten, Ruan Silva, Eric~Michael Smith, Ranjan Subramanian, Xiaoqing~Ellen Tan, Binh Tang, Ross Taylor, Adina Williams, Jian~Xiang Kuan, Puxin Xu, Zheng Yan, Iliyan Zarov, Yuchen Zhang, Angela Fan, Melanie Kambadur, Sharan Narang, Aurelien Rodriguez, Robert Stojnic, Sergey Edunov, and Thomas Scialom.
\newblock Llama 2: Open foundation and fine-tuned chat models.
\newblock \emph{ArXiv preprint}, abs/2307.09288, 2023.
\newblock URL \url{https://arxiv.org/abs/2307.09288}.
\bibitem[Vinyals et~al.(2019)Vinyals, Babuschkin, Czarnecki, Mathieu, Dudzik, Chung, Choi, Powell, Ewalds, Georgiev, et~al.]{vinyals-rtal:2019grandmaster}
Oriol Vinyals, Igor Babuschkin, Wojciech~M Czarnecki, Micha{\"e}l Mathieu, Andrew Dudzik, Junyoung Chung, David~H Choi, Richard Powell, Timo Ewalds, Petko Georgiev, et~al.
\newblock Grandmaster level in starcraft ii using multi-agent reinforcement learning.
\newblock \emph{nature}, 575\penalty0 (7782):\penalty0 350--354, 2019.
\bibitem[Wang et~al.(2023{\natexlab{a}})Wang, Zhou, Chang, Liu, Zhang, Du, Xiao, and Zhu]{wang-etal:2023learning}
Chenglong Wang, Hang Zhou, Kaiyan Chang, Tongran Liu, Chunliang Zhang, Quan Du, Tong Xiao, and Jingbo Zhu.
\newblock Learning evaluation models from large language models for sequence generation.
\newblock \emph{ArXiv preprint}, abs/2308.04386, 2023{\natexlab{a}}.
\newblock URL \url{https://arxiv.org/abs/2308.04386}.
\bibitem[Wang et~al.(2024{\natexlab{a}})Wang, Gan, Huo, Mu, Yang, He, Xiao, Zhang, Liu, Du, et~al.]{wang-etal:2024rovrm}
Chenglong Wang, Yang Gan, Yifu Huo, Yongyu Mu, Murun Yang, Qiaozhi He, Tong Xiao, Chunliang Zhang, Tongran Liu, Quan Du, et~al.
\newblock Rovrm: A robust visual reward model optimized via auxiliary textual preference data.
\newblock \emph{arXiv preprint arXiv:2408.12109}, 2024{\natexlab{a}}.
\bibitem[Wang et~al.(2024{\natexlab{b}})Wang, Zhou, Chang, Li, Mu, Xiao, Liu, and Zhu]{wang-etal:2024hybrid}
Chenglong Wang, Hang Zhou, Kaiyan Chang, Bei Li, Yongyu Mu, Tong Xiao, Tongran Liu, and Jingbo Zhu.
\newblock Hybrid alignment training for large language models.
\newblock \emph{ArXiv preprint}, abs/2406.15178, 2024{\natexlab{b}}.
\newblock URL \url{https://arxiv.org/abs/2406.15178}.
\bibitem[Wang et~al.(2024{\natexlab{c}})Wang, Zhou, Hu, Huo, Li, Liu, Xiao, and Zhu]{wang-etal:2024esrl}
Chenglong Wang, Hang Zhou, Yimin Hu, Yifu Huo, Bei Li, Tongran Liu, Tong Xiao, and Jingbo Zhu.
\newblock {ESRL:} efficient sampling-based reinforcement learning for sequence generation.
\newblock In Michael~J. Wooldridge, Jennifer~G. Dy, and Sriraam Natarajan (eds.), \emph{Thirty-Eighth {AAAI} Conference on Artificial Intelligence, {AAAI} 2024, Thirty-Sixth Conference on Innovative Applications of Artificial Intelligence, {IAAI} 2024, Fourteenth Symposium on Educational Advances in Artificial Intelligence, {EAAI} 2014, February 20-27, 2024, Vancouver, Canada}, pp.\ 19107--19115. {AAAI} Press, 2024{\natexlab{c}}.
\newblock \doi{10.1609/AAAI.V38I17.29878}.
\newblock URL \url{https://doi.org/10.1609/aaai.v38i17.29878}.
\bibitem[Wang et~al.(2025)Wang, Gan, Huo, Mu, He, Yang, Li, Xiao, Zhang, Liu, et~al.]{wang-etal:wang2025gram}
Chenglong Wang, Yang Gan, Yifu Huo, Yongyu Mu, Qiaozhi He, Murun Yang, Bei Li, Tong Xiao, Chunliang Zhang, Tongran Liu, et~al.
\newblock Gram: A generative foundation reward model for reward generalization.
\newblock \emph{arXiv preprint arXiv:2506.14175}, 2025.
\bibitem[Wang et~al.(2026{\natexlab{a}})Wang, Huo, Gan, He, Meng, Li, Wang, Liu, Zhou, Zhu, et~al.]{wang2026msrl}
Chenglong Wang, Yifu Huo, Yang Gan, Qiaozhi He, Qi~Meng, Bei Li, Yan Wang, Junfu Liu, Tianhua Zhou, Jingbo Zhu, et~al.
\newblock Msrl: Scaling generative multimodal reward modeling via multi-stage reinforcement learning.
\newblock \emph{arXiv preprint arXiv:2603.25108}, 2026{\natexlab{a}}.
\bibitem[Wang et~al.(2026{\natexlab{b}})Wang, Mu, Zhou, Huo, Zhu, Zeng, Yang, Li, Hao, Zhang, et~al.]{wang-etal:wang2026gram}
Chenglong Wang, Yongyu Mu, Hang Zhou, Yifu Huo, Ziming Zhu, Jiali Zeng, Murun Yang, Bei Li, Xiaoyang Hao, Chunliang Zhang, et~al.
\newblock Gram-r$^2$: Self-training generative foundation reward models for reward reasoning.
\newblock In \emph{Proceedings of the AAAI Conference on Artificial Intelligence}, volume~40, pp.\ 33395--33403, 2026{\natexlab{b}}.
\bibitem[Wang et~al.(2021)Wang, Yan, Meng, and Zhou]{wang-etal:2021selective}
Fusheng Wang, Jianhao Yan, Fandong Meng, and Jie Zhou.
\newblock Selective knowledge distillation for neural machine translation.
\newblock In Chengqing Zong, Fei Xia, Wenjie Li, and Roberto Navigli (eds.), \emph{Proceedings of the 59th Annual Meeting of the Association for Computational Linguistics and the 11th International Joint Conference on Natural Language Processing (Volume 1: Long Papers)}, pp.\ 6456--6466, Online, 2021. Association for Computational Linguistics.
\newblock \doi{10.18653/v1/2021.acl-long.504}.
\newblock URL \url{https://aclanthology.org/2021.acl-long.504}.
\bibitem[Wang et~al.(2023{\natexlab{b}})Wang, Li, Chen, Cai, Zhu, Lin, Cao, Liu, Liu, and Sui]{wang-etal:2023large}
Peiyi Wang, Lei Li, Liang Chen, Zefan Cai, Dawei Zhu, Binghuai Lin, Yunbo Cao, Qi~Liu, Tianyu Liu, and Zhifang Sui.
\newblock Large language models are not fair evaluators.
\newblock \emph{ArXiv preprint}, 2023{\natexlab{b}}.
\bibitem[Wang et~al.(2023{\natexlab{c}})Wang, Li, Shao, Xu, Dai, Li, Chen, Wu, and Sui]{wang-etal:2023math}
Peiyi Wang, Lei Li, Zhihong Shao, RX~Xu, Damai Dai, Yifei Li, Deli Chen, Yu~Wu, and Zhifang Sui.
\newblock Math-shepherd: Verify and reinforce llms step-by-step without human annotations.
\newblock \emph{ArXiv preprint}, abs/2312.08935, 2023{\natexlab{c}}.
\newblock URL \url{https://arxiv.org/abs/2312.08935}.
\bibitem[Wang \& Zhou(2024)Wang and Zhou]{wang-and-zhou:2024chain}
Xuezhi Wang and Denny Zhou.
\newblock Chain-of-thought reasoning without prompting.
\newblock In Amir Globersons, Lester Mackey, Danielle Belgrave, Angela Fan, Ulrich Paquet, Jakub~M. Tomczak, and Cheng Zhang (eds.), \emph{Advances in Neural Information Processing Systems 38: Annual Conference on Neural Information Processing Systems 2024, NeurIPS 2024, Vancouver, BC, Canada, December 10 - 15, 2024}, 2024.
\newblock URL \url{http://papers.nips.cc/paper\_files/paper/2024/hash/7a8e7fd295aa04eac4b470ae27f8785c-Abstract-Conference.html}.
\bibitem[Wang et~al.(2023{\natexlab{d}})Wang, Ivison, Dasigi, Hessel, Khot, Chandu, Wadden, MacMillan, Smith, Beltagy, and Hajishirzi]{wang-etal:2023far}
Yizhong Wang, Hamish Ivison, Pradeep Dasigi, Jack Hessel, Tushar Khot, Khyathi Chandu, David Wadden, Kelsey MacMillan, Noah~A. Smith, Iz~Beltagy, and Hannaneh Hajishirzi.
\newblock How far can camels go? exploring the state of instruction tuning on open resources.
\newblock In Alice Oh, Tristan Naumann, Amir Globerson, Kate Saenko, Moritz Hardt, and Sergey Levine (eds.), \emph{Advances in Neural Information Processing Systems 36: Annual Conference on Neural Information Processing Systems 2023, NeurIPS 2023, New Orleans, LA, USA, December 10 - 16, 2023}, 2023{\natexlab{d}}.
\newblock URL \url{http://papers.nips.cc/paper\_files/paper/2023/hash/ec6413875e4ab08d7bc4d8e225263398-Abstract-Datasets\_and\_Benchmarks.html}.
\bibitem[Wei et~al.(2022)Wei, Bosma, Zhao, Guu, Yu, Lester, Du, Dai, and Le]{wei-etal:2022finetuned}
Jason Wei, Maarten Bosma, Vincent~Y. Zhao, Kelvin Guu, Adams~Wei Yu, Brian Lester, Nan Du, Andrew~M. Dai, and Quoc~V. Le.
\newblock Finetuned language models are zero-shot learners.
\newblock In \emph{The Tenth International Conference on Learning Representations, {ICLR} 2022, Virtual Event, April 25-29, 2022}. OpenReview.net, 2022.
\newblock URL \url{https://openreview.net/forum?id=gEZrGCozdqR}.
\bibitem[Wu et~al.(2023)Wu, Hu, Shi, Dziri, Suhr, Ammanabrolu, Smith, Ostendorf, and Hajishirzi]{wu-etal:2023fine}
Zeqiu Wu, Yushi Hu, Weijia Shi, Nouha Dziri, Alane Suhr, Prithviraj Ammanabrolu, Noah~A. Smith, Mari Ostendorf, and Hannaneh Hajishirzi.
\newblock Fine-grained human feedback gives better rewards for language model training.
\newblock In Alice Oh, Tristan Naumann, Amir Globerson, Kate Saenko, Moritz Hardt, and Sergey Levine (eds.), \emph{Advances in Neural Information Processing Systems 36: Annual Conference on Neural Information Processing Systems 2023, NeurIPS 2023, New Orleans, LA, USA, December 10 - 16, 2023}, 2023.
\newblock URL \url{http://papers.nips.cc/paper\_files/paper/2023/hash/b8c90b65739ae8417e61eadb521f63d5-Abstract-Conference.html}.
\bibitem[Wu et~al.(2024)Wu, Balashankar, Kim, Eisenstein, and Beirami]{wu-etal:2024reuse}
Zhaofeng Wu, Ananth Balashankar, Yoon Kim, Jacob Eisenstein, and Ahmad Beirami.
\newblock Reuse your rewards: Reward model transfer for zero-shot cross-lingual alignment.
\newblock \emph{arXiv preprint arXiv:2404.12318}, 2024.
\bibitem[Xia et~al.(2024)Xia, Malladi, Gururangan, Arora, and Chen]{xia-etal:2024less}
Mengzhou Xia, Sadhika Malladi, Suchin Gururangan, Sanjeev Arora, and Danqi Chen.
\newblock Less: Selecting influential data for targeted instruction tuning.
\newblock \emph{arXiv preprint arXiv:2402.04333}, 2024.
\bibitem[Xiao \& Zhu(2023)Xiao and Zhu]{xiao-etal:2023introduction}
Tong Xiao and Jingbo Zhu.
\newblock Introduction to transformers: an nlp perspective.
\newblock \emph{ArXiv preprint}, abs/2311.17633, 2023.
\newblock URL \url{https://arxiv.org/abs/2311.17633}.
\bibitem[Xiao \& Zhu(2025)Xiao and Zhu]{xiao-and-zhu:2025foundations}
Tong Xiao and Jingbo Zhu.
\newblock Foundations of large language models.
\newblock \emph{ArXiv preprint}, abs/2501.09223, 2025.
\newblock URL \url{https://arxiv.org/abs/2501.09223}.
\bibitem[Xie et~al.(2025)Xie, Gao, Ren, Luo, Hong, Dai, Zhou, Qiu, Wu, and Luo]{xie-etal:2025logic}
Tian Xie, Zitian Gao, Qingnan Ren, Haoming Luo, Yuqian Hong, Bryan Dai, Joey Zhou, Kai Qiu, Zhirong Wu, and Chong Luo.
\newblock Logic-rl: Unleashing llm reasoning with rule-based reinforcement learning.
\newblock \emph{ArXiv preprint}, abs/2502.14768, 2025.
\newblock URL \url{https://arxiv.org/abs/2502.14768}.
\bibitem[Xu et~al.(2024)Xu, Sharaf, Chen, Tan, Shen, Durme, Murray, and Kim]{xu:2024contrastive}
Haoran Xu, Amr Sharaf, Yunmo Chen, Weiting Tan, Lingfeng Shen, Benjamin~Van Durme, Kenton Murray, and Young~Jin Kim.
\newblock Contrastive preference optimization: Pushing the boundaries of {LLM} performance in machine translation.
\newblock In \emph{Forty-first International Conference on Machine Learning, {ICML} 2024, Vienna, Austria, July 21-27, 2024}. OpenReview.net, 2024.
\newblock URL \url{https://openreview.net/forum?id=51iwkioZpn}.
\bibitem[Xu et~al.(2025)Xu, Guo, He, Hu, He, Bai, Chen, Wang, Fan, Dang, et~al.]{xu-etal:2025qwen2}
Jin Xu, Zhifang Guo, Jinzheng He, Hangrui Hu, Ting He, Shuai Bai, Keqin Chen, Jialin Wang, Yang Fan, Kai Dang, et~al.
\newblock Qwen2. 5-omni technical report.
\newblock \emph{arXiv preprint arXiv:2503.20215}, 2025.
\bibitem[Yang et~al.(2024)Yang, Ding, Lin, Zhang, and Zhang]{yang-etal:2024regularizing}
Rui Yang, Ruomeng Ding, Yong Lin, Huan Zhang, and Tong Zhang.
\newblock Regularizing hidden states enables learning generalizable reward model for llms.
\newblock In Amir Globersons, Lester Mackey, Danielle Belgrave, Angela Fan, Ulrich Paquet, Jakub~M. Tomczak, and Cheng Zhang (eds.), \emph{Advances in Neural Information Processing Systems 38: Annual Conference on Neural Information Processing Systems 2024, NeurIPS 2024, Vancouver, BC, Canada, December 10 - 15, 2024}, 2024.
\newblock URL \url{http://papers.nips.cc/paper\_files/paper/2024/hash/71f7154547c748c8041505521ca433ab-Abstract-Conference.html}.
\bibitem[Yao et~al.(2023)Yao, Yu, Zhao, Shafran, Griffiths, Cao, and Narasimhan]{yao-etal:2023tree}
Shunyu Yao, Dian Yu, Jeffrey Zhao, Izhak Shafran, Tom Griffiths, Yuan Cao, and Karthik Narasimhan.
\newblock Tree of thoughts: Deliberate problem solving with large language models.
\newblock In Alice Oh, Tristan Naumann, Amir Globerson, Kate Saenko, Moritz Hardt, and Sergey Levine (eds.), \emph{Advances in Neural Information Processing Systems 36: Annual Conference on Neural Information Processing Systems 2023, NeurIPS 2023, New Orleans, LA, USA, December 10 - 16, 2023}, 2023.
\newblock URL \url{http://papers.nips.cc/paper\_files/paper/2023/hash/271db9922b8d1f4dd7aaef84ed5ac703-Abstract-Conference.html}.
\bibitem[Yu et~al.(2024{\natexlab{a}})Yu, Yao, Zhang, He, Han, Cui, Hu, Liu, Zheng, Sun, et~al.]{yu-etal:2024rlhf}
Tianyu Yu, Yuan Yao, Haoye Zhang, Taiwen He, Yifeng Han, Ganqu Cui, Jinyi Hu, Zhiyuan Liu, Hai-Tao Zheng, Maosong Sun, et~al.
\newblock Rlhf-v: Towards trustworthy mllms via behavior alignment from fine-grained correctional human feedback.
\newblock In \emph{Proceedings of the IEEE/CVF Conference on Computer Vision and Pattern Recognition}, pp.\ 13807--13816, 2024{\natexlab{a}}.
\bibitem[Yu et~al.(2024{\natexlab{b}})Yu, Zhang, Yao, Dang, Chen, Lu, Cui, He, Liu, Chua, et~al.]{yu-etal:2024rlaif}
Tianyu Yu, Haoye Zhang, Yuan Yao, Yunkai Dang, Da~Chen, Xiaoman Lu, Ganqu Cui, Taiwen He, Zhiyuan Liu, Tat-Seng Chua, et~al.
\newblock Rlaif-v: Aligning mllms through open-source ai feedback for super gpt-4v trustworthiness.
\newblock \emph{arXiv preprint arXiv:2405.17220}, 2024{\natexlab{b}}.
\bibitem[Yuan et~al.(2024)Yuan, Pang, Cho, Li, Sukhbaatar, Xu, and Weston]{yuan-etal:2024selfrewarding}
Weizhe Yuan, Richard~Yuanzhe Pang, Kyunghyun Cho, Xian Li, Sainbayar Sukhbaatar, Jing Xu, and Jason Weston.
\newblock Self-rewarding language models.
\newblock In \emph{Forty-first International Conference on Machine Learning, {ICML} 2024, Vienna, Austria, July 21-27, 2024}. OpenReview.net, 2024.
\newblock URL \url{https://openreview.net/forum?id=0NphYCmgua}.
\bibitem[Yuan et~al.(2023)Yuan, Yuan, Li, Dong, Lu, Tan, Zhou, and Zhou]{yuan-etal:2023scaling}
Zheng Yuan, Hongyi Yuan, Chengpeng Li, Guanting Dong, Keming Lu, Chuanqi Tan, Chang Zhou, and Jingren Zhou.
\newblock Scaling relationship on learning mathematical reasoning with large language models.
\newblock \emph{ArXiv preprint}, abs/2308.01825, 2023.
\newblock URL \url{https://arxiv.org/abs/2308.01825}.
\bibitem[Zang et~al.(2025)Zang, Dong, Zhang, Cao, Liu, Ding, Wu, Ma, Duan, Zhang, et~al.]{zang2025internlm}
Yuhang Zang, Xiaoyi Dong, Pan Zhang, Yuhang Cao, Ziyu Liu, Shengyuan Ding, Shenxi Wu, Yubo Ma, Haodong Duan, Wenwei Zhang, et~al.
\newblock Internlm-xcomposer2. 5-reward: A simple yet effective multi-modal reward model.
\newblock \emph{arXiv preprint arXiv:2501.12368}, 2025.
\bibitem[Zeng et~al.(2025)Zeng, Cheng, Yin, Zhou, and Qiu]{zeng-etal:2025revisiting}
Zhiyuan Zeng, Qinyuan Cheng, Zhangyue Yin, Yunhua Zhou, and Xipeng Qiu.
\newblock Revisiting the test-time scaling of o1-like models: Do they truly possess test-time scaling capabilities?
\newblock \emph{ArXiv preprint}, abs/2502.12215, 2025.
\newblock URL \url{https://arxiv.org/abs/2502.12215}.
\bibitem[Zhang et~al.(2024{\natexlab{a}})Zhang, Yu, Dong, Li, Su, Chu, and Yu]{zhang-etal:2024mm}
Duzhen Zhang, Yahan Yu, Jiahua Dong, Chenxing Li, Dan Su, Chenhui Chu, and Dong Yu.
\newblock Mm-llms: Recent advances in multimodal large language models.
\newblock \emph{arXiv preprint arXiv:2401.13601}, 2024{\natexlab{a}}.
\bibitem[Zhang et~al.(2024{\natexlab{b}})Zhang, Hosseini, Bansal, Kazemi, Kumar, and Agarwal]{zhang-etal:2024generative}
Lunjun Zhang, Arian Hosseini, Hritik Bansal, Mehran Kazemi, Aviral Kumar, and Rishabh Agarwal.
\newblock Generative verifiers: Reward modeling as next-token prediction.
\newblock \emph{ArXiv preprint}, abs/2408.15240, 2024{\natexlab{b}}.
\newblock URL \url{https://arxiv.org/abs/2408.15240}.
\bibitem[Zhang et~al.(2024{\natexlab{c}})Zhang, Hosseini, Bansal, Kazemi, Kumar, and Agarwal]{zhang-etal:zhang2024generative}
Lunjun Zhang, Arian Hosseini, Hritik Bansal, Mehran Kazemi, Aviral Kumar, and Rishabh Agarwal.
\newblock Generative verifiers: Reward modeling as next-token prediction.
\newblock \emph{arXiv preprint arXiv:2408.15240}, 2024{\natexlab{c}}.
\bibitem[Zhao et~al.(2024)Zhao, Lin, Zhu, Ye, Chen, Zheng, Ceze, Krishnamurthy, Chen, and Kasikci]{zhao-etal:2024atom}
Yilong Zhao, Chien-Yu Lin, Kan Zhu, Zihao Ye, Lequn Chen, Size Zheng, Luis Ceze, Arvind Krishnamurthy, Tianqi Chen, and Baris Kasikci.
\newblock Atom: Low-bit quantization for efficient and accurate llm serving.
\newblock \emph{Proceedings of Machine Learning and Systems}, 6:\penalty0 196--209, 2024.
\bibitem[Zheng et~al.(2023)Zheng, Chiang, Sheng, Zhuang, Wu, Zhuang, Lin, Li, Li, Xing, Zhang, Gonzalez, and Stoica]{zheng-etal:2023judging}
Lianmin Zheng, Wei{-}Lin Chiang, Ying Sheng, Siyuan Zhuang, Zhanghao Wu, Yonghao Zhuang, Zi~Lin, Zhuohan Li, Dacheng Li, Eric~P. Xing, Hao Zhang, Joseph~E. Gonzalez, and Ion Stoica.
\newblock Judging llm-as-a-judge with mt-bench and chatbot arena.
\newblock In Alice Oh, Tristan Naumann, Amir Globerson, Kate Saenko, Moritz Hardt, and Sergey Levine (eds.), \emph{Advances in Neural Information Processing Systems 36: Annual Conference on Neural Information Processing Systems 2023, NeurIPS 2023, New Orleans, LA, USA, December 10 - 16, 2023}, 2023.
\newblock URL \url{http://papers.nips.cc/paper\_files/paper/2023/hash/91f18a1287b398d378ef22505bf41832-Abstract-Datasets\_and\_Benchmarks.html}.
\bibitem[Zhou et~al.(2024)Zhou, Wang, Hu, Xiao, Zhang, and Zhu]{zhou:2024prior}
Hang Zhou, Chenglong Wang, Yimin Hu, Tong Xiao, Chunliang Zhang, and Jingbo Zhu.
\newblock Prior constraints-based reward model training for aligning large language models.
\newblock In \emph{China National Conference on Chinese Computational Linguistics}, pp.\ 555--570. Springer, 2024.
\end{thebibliography}
rl-introduction.bib
@article{wang2026msrl,
title={MSRL: Scaling Generative Multimodal Reward Modeling via Multi-Stage Reinforcement Learning},
author={Wang, Chenglong and Huo, Yifu and Gan, Yang and He, Qiaozhi and Meng, Qi and Li, Bei and Wang, Yan and Liu, Junfu and Zhou, Tianhua and Zhu, Jingbo and others},
journal={arXiv preprint arXiv:2603.25108},
year={2026}
}
@inproceedings{wang-etal:wang2026gram,
title={Gram-r$^2$: Self-training generative foundation reward models for reward reasoning},
author={Wang, Chenglong and Mu, Yongyu and Zhou, Hang and Huo, Yifu and Zhu, Ziming and Zeng, Jiali and Yang, Murun and Li, Bei and Hao, Xiaoyang and Zhang, Chunliang and others},
booktitle={Proceedings of the AAAI Conference on Artificial Intelligence},
volume={40},
number={39},
pages={33395--33403},
year={2026}
}
@article{wang-etal:wang2025gram,
title={Gram: A generative foundation reward model for reward generalization},
author={Wang, Chenglong and Gan, Yang and Huo, Yifu and Mu, Yongyu and He, Qiaozhi and Yang, Murun and Li, Bei and Xiao, Tong and Zhang, Chunliang and Liu, Tongran and others},
journal={arXiv preprint arXiv:2506.14175},
year={2025}
}
@article{zhang-etal:zhang2024generative,
title={Generative verifiers: Reward modeling as next-token prediction},
author={Zhang, Lunjun and Hosseini, Arian and Bansal, Hritik and Kazemi, Mehran and Kumar, Aviral and Agarwal, Rishabh},
journal={arXiv preprint arXiv:2408.15240},
year={2024}
}
@article{sun-etal:2023aligning, @article{sun-etal:2023aligning,
title={Aligning large multimodal models with factually augmented rlhf}, title={Aligning large multimodal models with factually augmented rlhf},
author={Sun, Zhiqing and Shen, Sheng and Cao, Shengcao and Liu, Haotian and Li, Chunyuan and Shen, Yikang and Gan, Chuang and Gui, Liang-Yan and Wang, Yu-Xiong and Yang, Yiming and others}, author={Sun, Zhiqing and Shen, Sheng and Cao, Shengcao and Liu, Haotian and Li, Chunyuan and Shen, Yikang and Gan, Chuang and Gui, Liang-Yan and Wang, Yu-Xiong and Yang, Yiming and others},
......
rl-introduction.blg 0 → 100644
This is BibTeX, Version 0.99e (TeX Live 2026)
Capacity: max_strings=200000, hash_size=200000, hash_prime=170003
The top-level auxiliary file: rl-introduction.aux
The style file: rl-introduction.bst
Database file #1: rl-introduction.bib
Warning--can't use both volume and number fields in wang-etal:wang2026gram
You've used 98 entries,
2773 wiz_defined-function locations,
1106 strings with 33536 characters,
and the built_in function-call counts, 71100 in all, are:
= -- 5525
> -- 7412
< -- 34
+ -- 2481
- -- 2372
* -- 6800
:= -- 11734
add.period$ -- 380
call.type$ -- 98
change.case$ -- 1032
chr.to.int$ -- 87
cite$ -- 197
duplicate$ -- 2238
empty$ -- 3704
format.name$ -- 2482
if$ -- 14216
int.to.chr$ -- 12
int.to.str$ -- 1
missing$ -- 98
newline$ -- 562
num.names$ -- 430
pop$ -- 2052
preamble$ -- 1
purify$ -- 937
quote$ -- 0
skip$ -- 2138
stack$ -- 0
substring$ -- 926
swap$ -- 304
text.length$ -- 19
text.prefix$ -- 0
top$ -- 0
type$ -- 1063
warning$ -- 1
while$ -- 370
width$ -- 0
write$ -- 1394
(There was 1 warning)
rl-introduction.fdb_latexmk 0 → 100644
# Fdb version 4
["bibtex rl-introduction"] 1782404668.09764 "rl-introduction.aux" "rl-introduction.bbl" "rl-introduction" 1782404678.41302 0
"./rl-introduction.bib" 1782375814 53633 feb038f8d5b946d311e7b0083d4cd3bb ""
"./rl-introduction.bst" 1782375814 26973 af983813ca1a1414954dbceae7baa2a1 ""
"rl-introduction.aux" 1782404676 50785 2348d3be4ba13df8c287520ce3a2118d "pdflatex"
(generated)
"rl-introduction.bbl"
"rl-introduction.blg"
(rewritten before read)
["pdflatex"] 1782404669.00136 "c:/Users/Lenovo/Desktop/rl_introduction/rl-introduction.tex" "rl-introduction.pdf" "rl-introduction" 1782404678.41404 0
"c:/Users/Lenovo/Desktop/rl_introduction/rl-introduction.tex" 1782375814 9653 2303ab2f9125c255cbcda33bd6a948f0 ""
"d:/software/texlive/2026/texmf-dist/fonts/enc/dvips/base/8r.enc" 1782316766 4850 80dc9bab7f31fb78a000ccfed0e27cab ""
"d:/software/texlive/2026/texmf-dist/fonts/enc/dvips/fontawesome5/fa5free0.enc" 1782317297 3083 a707712f77b073b3833e4b1334a30e99 ""
"d:/software/texlive/2026/texmf-dist/fonts/enc/dvips/fontawesome5/fa5free1.enc" 1782317297 3042 0e2b14b9be7d01075a4c4591d5d1781b ""
"d:/software/texlive/2026/texmf-dist/fonts/enc/dvips/fontawesome5/fa5free3.enc" 1782317297 2905 1556c73dc99638e0dfff2f132c244f30 ""
"d:/software/texlive/2026/texmf-dist/fonts/enc/dvips/inconsolata/i4-ot1-0.enc" 1782317916 2496 4d35740f3a177992ff7b134746c0a1db ""
"d:/software/texlive/2026/texmf-dist/fonts/map/fontname/texfonts.map" 1782317312 3524 cb3e574dea2d1052e39280babc910dc8 ""
"d:/software/texlive/2026/texmf-dist/fonts/tfm/adobe/palatino/pplb7t.tfm" 1782319461 1616 371d1ee60c04b008c8fd760060cd34f1 ""
"d:/software/texlive/2026/texmf-dist/fonts/tfm/adobe/palatino/pplb8r.tfm" 1782319461 2532 9ad73cf4dd2173a847f2a5f5608e0b9a ""
"d:/software/texlive/2026/texmf-dist/fonts/tfm/adobe/palatino/pplbi7t.tfm" 1782319461 1816 bb70e8d3600ea56abb211fdcc6909bf3 ""
"d:/software/texlive/2026/texmf-dist/fonts/tfm/adobe/palatino/pplbi8r.tfm" 1782319461 2788 e071e05f6ae1a90dbbbc13a193c3a135 ""
"d:/software/texlive/2026/texmf-dist/fonts/tfm/adobe/palatino/pplr7t.tfm" 1782319461 1804 7b5f73028f6509167f47ace9d69509ed ""
"d:/software/texlive/2026/texmf-dist/fonts/tfm/adobe/palatino/pplr8c.tfm" 1782319461 1348 0e8eb69b4437626c0f6c9bed4a4e373c ""
"d:/software/texlive/2026/texmf-dist/fonts/tfm/adobe/palatino/pplr8r.tfm" 1782319461 2796 d37c29814a6717720ee1a7c9a0b2c3b8 ""
"d:/software/texlive/2026/texmf-dist/fonts/tfm/adobe/palatino/pplri7t.tfm" 1782319461 1732 3671a0e54806deb8b706eb5e53aba0a5 ""
"d:/software/texlive/2026/texmf-dist/fonts/tfm/adobe/palatino/pplri8r.tfm" 1782319461 2720 70000d5e623e601132eab3cded5b819b ""
"d:/software/texlive/2026/texmf-dist/fonts/tfm/jknappen/ec/ectt1000.tfm" 1782316809 1536 06717a2b50de47d4087ac0e6cd759455 ""
"d:/software/texlive/2026/texmf-dist/fonts/tfm/public/amsfonts/cmextra/cmbsy7.tfm" 1782315199 1120 7f9f170e8aa57527ad6c49feafd45d54 ""
"d:/software/texlive/2026/texmf-dist/fonts/tfm/public/amsfonts/euler/euex10.tfm" 1782315199 808 f0482638b09e07d31277eba36400f7dc ""
"d:/software/texlive/2026/texmf-dist/fonts/tfm/public/amsfonts/euler/euex7.tfm" 1782315199 808 ffd475d90f7bb606e739dfb01f8e0300 ""
"d:/software/texlive/2026/texmf-dist/fonts/tfm/public/amsfonts/euler/euex8.tfm" 1782315199 808 f7a165c5327ac5bfc6623d1288e07ad3 ""
"d:/software/texlive/2026/texmf-dist/fonts/tfm/public/amsfonts/euler/euex9.tfm" 1782315199 812 fe49bb17bbad97dedaadd4d7bd3439ec ""
"d:/software/texlive/2026/texmf-dist/fonts/tfm/public/amsfonts/symbols/msam10.tfm" 1782315199 916 f87d7c45f9c908e672703b83b72241a3 ""
"d:/software/texlive/2026/texmf-dist/fonts/tfm/public/amsfonts/symbols/msbm10.tfm" 1782315199 908 2921f8a10601f252058503cc6570e581 ""
"d:/software/texlive/2026/texmf-dist/fonts/tfm/public/bbding/bbding10.tfm" 1782315508 584 35c3c56c2ca57268b3e46133b2da0876 ""
"d:/software/texlive/2026/texmf-dist/fonts/tfm/public/cm/cmbx10.tfm" 1782316134 1328 c834bbb027764024c09d3d2bf908b5f0 ""
"d:/software/texlive/2026/texmf-dist/fonts/tfm/public/cm/cmex10.tfm" 1782316134 992 662f679a0b3d2d53c1b94050fdaa3f50 ""
"d:/software/texlive/2026/texmf-dist/fonts/tfm/public/cm/cmmi10.tfm" 1782316134 1528 abec98dbc43e172678c11b3b9031252a ""
"d:/software/texlive/2026/texmf-dist/fonts/tfm/public/cm/cmr10.tfm" 1782316134 1296 45809c5a464d5f32c8f98ba97c1bb47f ""
"d:/software/texlive/2026/texmf-dist/fonts/tfm/public/cm/cmsy10.tfm" 1782316134 1124 6c73e740cf17375f03eec0ee63599741 ""
"d:/software/texlive/2026/texmf-dist/fonts/tfm/public/cm/cmsy9.tfm" 1782316134 1116 25a7bf822c58caf309a702ef79f4afbb ""
"d:/software/texlive/2026/texmf-dist/fonts/tfm/public/fontawesome5/fa5free0solid.tfm" 1782317297 1328 8843c76048ec1e1c4b2d1007558b8833 ""
"d:/software/texlive/2026/texmf-dist/fonts/tfm/public/fontawesome5/fa5free1solid.tfm" 1782317297 1316 1c2ee668ea88be9bd4f39690662cc344 ""
"d:/software/texlive/2026/texmf-dist/fonts/tfm/public/fontawesome5/fa5free3solid.tfm" 1782317297 1236 01f4577264ada23bfb25b6d983302f6b ""
"d:/software/texlive/2026/texmf-dist/fonts/tfm/public/inconsolata/ot1-zi4r-0.tfm" 1782317916 912 a20aa2a2ed86fa42e5ec387898cd02d6 ""
"d:/software/texlive/2026/texmf-dist/fonts/tfm/public/latex-fonts/lasy6.tfm" 1782318280 520 4889cce2180234b97cad636b6039c722 ""
"d:/software/texlive/2026/texmf-dist/fonts/tfm/public/mathpazo/fplmb.tfm" 1782318806 984 3dcde7360ef3d34302e591b511cba83c ""
"d:/software/texlive/2026/texmf-dist/fonts/tfm/public/mathpazo/fplmbb.tfm" 1782318806 564 cb1a5fc3f0326dd000c6e6c0fce59d7c ""
"d:/software/texlive/2026/texmf-dist/fonts/tfm/public/mathpazo/fplmr.tfm" 1782318806 1032 7fa31d93ecd4cbdfac02c7a1ebc6facc ""
"d:/software/texlive/2026/texmf-dist/fonts/tfm/public/mathpazo/fplmri.tfm" 1782318806 1148 56b0fdf7ca3f800294e968f64cfdcb87 ""
"d:/software/texlive/2026/texmf-dist/fonts/tfm/public/mathpazo/zplmb7t.tfm" 1782318806 1640 c14e51df62d6cb0dd79e4ca3b91ca7c3 ""
"d:/software/texlive/2026/texmf-dist/fonts/tfm/public/mathpazo/zplmr7m.tfm" 1782318806 2080 40a71f65088bdd047622ce295520749e ""
"d:/software/texlive/2026/texmf-dist/fonts/tfm/public/mathpazo/zplmr7t.tfm" 1782318806 1828 bb8add3bd19426549f9267c88b0cb8bd ""
"d:/software/texlive/2026/texmf-dist/fonts/tfm/public/mathpazo/zplmr7v.tfm" 1782318806 1012 955c4ca523d7827d33db91a33412b048 ""
"d:/software/texlive/2026/texmf-dist/fonts/tfm/public/mathpazo/zplmr7y.tfm" 1782318806 1316 b4037e9c09961a72f8476628774e1ec1 ""
"d:/software/texlive/2026/texmf-dist/fonts/tfm/public/tex-gyre/rm-qplr.tfm" 1782320936 11860 c3b4aba4cfa63b7a6c3688dc5ab4931d ""
"d:/software/texlive/2026/texmf-dist/fonts/type1/public/amsfonts/cm/cmex10.pfb" 1782315199 30251 6afa5cb1d0204815a708a080681d4674 ""
"d:/software/texlive/2026/texmf-dist/fonts/type1/public/amsfonts/cm/cmmi10.pfb" 1782315199 36299 5f9df58c2139e7edcf37c8fca4bd384d ""
"d:/software/texlive/2026/texmf-dist/fonts/type1/public/amsfonts/cm/cmr10.pfb" 1782315199 35752 024fb6c41858982481f6968b5fc26508 ""
"d:/software/texlive/2026/texmf-dist/fonts/type1/public/amsfonts/cm/cmsy10.pfb" 1782315199 32569 5e5ddc8df908dea60932f3c484a54c0d ""
"d:/software/texlive/2026/texmf-dist/fonts/type1/public/amsfonts/cm/cmsy9.pfb" 1782315199 32442 c975af247b6702f7ca0c299af3616b80 ""
"d:/software/texlive/2026/texmf-dist/fonts/type1/public/amsfonts/cmextra/cmbsy7.pfb" 1782315199 30068 eae4ba75635647d82fb0b7d7bccc4748 ""
"d:/software/texlive/2026/texmf-dist/fonts/type1/public/fontawesome5/FontAwesome5Free-Solid.pfb" 1782317297 295323 07c90be888af62cddcbea2d451a520d3 ""
"d:/software/texlive/2026/texmf-dist/fonts/type1/public/inconsolata/Inconsolata-zi4r.pfb" 1782317916 55526 866691d4af8dece5424ad744e8c0133d ""
"d:/software/texlive/2026/texmf-dist/fonts/type1/public/mathpazo/fplmbb.pfb" 1782318806 12410 7504dd653d0e7cf266cb973aeafd597c ""
"d:/software/texlive/2026/texmf-dist/fonts/type1/public/mathpazo/fplmr.pfb" 1782318806 9356 3e1f7eacbb57d05d0b682da8d83eb600 ""
"d:/software/texlive/2026/texmf-dist/fonts/type1/public/mathpazo/fplmri.pfb" 1782318806 15213 d999bbe8bd9c8f231f3f66a737c76478 ""
"d:/software/texlive/2026/texmf-dist/fonts/type1/public/niceframe-type1/bbding10.pfb" 1782319241 55242 bbd814637d6ccdbd30ac6740e6b060c9 ""
"d:/software/texlive/2026/texmf-dist/fonts/type1/urw/palatino/uplb8a.pfb" 1782319461 52406 dad2f72863a03727d5f536c64a69c452 ""
"d:/software/texlive/2026/texmf-dist/fonts/type1/urw/palatino/uplbi8a.pfb" 1782319461 51285 a23fbd59aeaa61f5a504e70284f9de4f ""
"d:/software/texlive/2026/texmf-dist/fonts/type1/urw/palatino/uplr8a.pfb" 1782319461 52665 661b1e6b26edb5f50dd491f8a701cb57 ""
"d:/software/texlive/2026/texmf-dist/fonts/type1/urw/palatino/uplri8a.pfb" 1782319461 50022 90249cba7e3e4e9845f80328d6f9bd13 ""
"d:/software/texlive/2026/texmf-dist/fonts/vf/adobe/palatino/pplb7t.vf" 1782319461 1380 798ef26de8f0a21f37c9b1147f4fe5d8 ""
"d:/software/texlive/2026/texmf-dist/fonts/vf/adobe/palatino/pplbi7t.vf" 1782319461 1388 a313cf7cb3917eab429671fbba73c499 ""
"d:/software/texlive/2026/texmf-dist/fonts/vf/adobe/palatino/pplr7t.vf" 1782319461 1380 037725697594405988999f61172f52de ""
"d:/software/texlive/2026/texmf-dist/fonts/vf/adobe/palatino/pplr8c.vf" 1782319461 3504 f65f653e1ad318a9d01b4af15300683a ""
"d:/software/texlive/2026/texmf-dist/fonts/vf/adobe/palatino/pplri7t.vf" 1782319461 1376 a5f9301e24d8e27c2286a3b415778d2d ""
"d:/software/texlive/2026/texmf-dist/fonts/vf/public/mathpazo/zplmb7t.vf" 1782318806 1536 03ea9eccf9f5341331bed4d35eb9a3d8 ""
"d:/software/texlive/2026/texmf-dist/fonts/vf/public/mathpazo/zplmr7m.vf" 1782318806 1812 de7624743c8bafe2ce204bec2f72ce7e ""
"d:/software/texlive/2026/texmf-dist/fonts/vf/public/mathpazo/zplmr7t.vf" 1782318806 1532 0ba0b316cfeb3df86c9701756c9a4b71 ""
"d:/software/texlive/2026/texmf-dist/fonts/vf/public/mathpazo/zplmr7v.vf" 1782318806 1436 3cc3caf644d79936444909aaff6786a8 ""
"d:/software/texlive/2026/texmf-dist/fonts/vf/public/mathpazo/zplmr7y.vf" 1782318806 1372 31129ac1f6b6f62aaec24c8fed41b02b ""
"d:/software/texlive/2026/texmf-dist/tex/context/base/mkii/supp-pdf.mkii" 1782319094 71627 94eb9990bed73c364d7f53f960cc8c5b ""
"d:/software/texlive/2026/texmf-dist/tex/generic/bigintcalc/bigintcalc.sty" 1782315757 40635 c40361e206be584d448876bba8a64a3b ""
"d:/software/texlive/2026/texmf-dist/tex/generic/bitset/bitset.sty" 1782315770 33961 6b5c75130e435b2bfdb9f480a09a39f9 ""
"d:/software/texlive/2026/texmf-dist/tex/generic/etexcmds/etexcmds.sty" 1782317003 7734 b98cbb34c81f667027c1e3ebdbfce34b ""
"d:/software/texlive/2026/texmf-dist/tex/generic/gettitlestring/gettitlestring.sty" 1782317471 8371 9d55b8bd010bc717624922fb3477d92e ""
"d:/software/texlive/2026/texmf-dist/tex/generic/iftex/iftex.sty" 1782317891 7984 7dbb9280f03c0a315425f1b4f35d43ee ""
"d:/software/texlive/2026/texmf-dist/tex/generic/infwarerr/infwarerr.sty" 1782317925 8356 7bbb2c2373aa810be568c29e333da8ed ""
"d:/software/texlive/2026/texmf-dist/tex/generic/intcalc/intcalc.sty" 1782317950 31769 002a487f55041f8e805cfbf6385ffd97 ""
"d:/software/texlive/2026/texmf-dist/tex/generic/kvdefinekeys/kvdefinekeys.sty" 1782318192 5412 d5a2436094cd7be85769db90f29250a6 ""
"d:/software/texlive/2026/texmf-dist/tex/generic/ltxcmds/ltxcmds.sty" 1782318580 17865 1a9bd36b4f98178fa551aca822290953 ""
"d:/software/texlive/2026/texmf-dist/tex/generic/pdfescape/pdfescape.sty" 1782319537 19007 15924f7228aca6c6d184b115f4baa231 ""
"d:/software/texlive/2026/texmf-dist/tex/generic/pdftexcmds/pdftexcmds.sty" 1782319567 20089 80423eac55aa175305d35b49e04fe23b ""
"d:/software/texlive/2026/texmf-dist/tex/generic/pgf/basiclayer/pgfcore.code.tex" 1782319601 1016 1c2b89187d12a2768764b83b4945667c ""
"d:/software/texlive/2026/texmf-dist/tex/generic/pgf/basiclayer/pgfcorearrows.code.tex" 1782319601 43906 06058dc09064474303f3b5dd62d982c0 ""
"d:/software/texlive/2026/texmf-dist/tex/generic/pgf/basiclayer/pgfcoreexternal.code.tex" 1782319601 19324 f4e4c6403dd0f1605fd20ed22fa79dea ""
"d:/software/texlive/2026/texmf-dist/tex/generic/pgf/basiclayer/pgfcoregraphicstate.code.tex" 1782319601 6038 ccb406740cc3f03bbfb58ad504fe8c27 ""
"d:/software/texlive/2026/texmf-dist/tex/generic/pgf/basiclayer/pgfcoreimage.code.tex" 1782319601 6911 f6d4cf5a3fef5cc879d668b810e82868 ""
"d:/software/texlive/2026/texmf-dist/tex/generic/pgf/basiclayer/pgfcorelayers.code.tex" 1782319601 4883 42daaf41e27c3735286e23e48d2d7af9 ""
"d:/software/texlive/2026/texmf-dist/tex/generic/pgf/basiclayer/pgfcoreobjects.code.tex" 1782319601 2544 8c06d2a7f0f469616ac9e13db6d2f842 ""
"d:/software/texlive/2026/texmf-dist/tex/generic/pgf/basiclayer/pgfcorepathconstruct.code.tex" 1782319601 44195 5e390c414de027626ca5e2df888fa68d ""
"d:/software/texlive/2026/texmf-dist/tex/generic/pgf/basiclayer/pgfcorepathprocessing.code.tex" 1782319601 17311 e001219836e75b16c4af9a112785f30a ""
"d:/software/texlive/2026/texmf-dist/tex/generic/pgf/basiclayer/pgfcorepathusage.code.tex" 1782319601 21302 788a79944eb22192a4929e46963a3067 ""
"d:/software/texlive/2026/texmf-dist/tex/generic/pgf/basiclayer/pgfcorepatterns.code.tex" 1782319601 9691 3d42d89522f4650c2f3dc616ca2b925e ""
"d:/software/texlive/2026/texmf-dist/tex/generic/pgf/basiclayer/pgfcorepoints.code.tex" 1782319601 33335 dd1fa4814d4e51f18be97d88bf0da60c ""
"d:/software/texlive/2026/texmf-dist/tex/generic/pgf/basiclayer/pgfcorequick.code.tex" 1782319601 2965 4c2b1f4e0826925746439038172e5d6f ""
"d:/software/texlive/2026/texmf-dist/tex/generic/pgf/basiclayer/pgfcorerdf.code.tex" 1782319601 5196 2cc249e0ee7e03da5f5f6589257b1e5b ""
"d:/software/texlive/2026/texmf-dist/tex/generic/pgf/basiclayer/pgfcorescopes.code.tex" 1782319601 20821 7579108c1e9363e61a0b1584778804aa ""
"d:/software/texlive/2026/texmf-dist/tex/generic/pgf/basiclayer/pgfcoreshade.code.tex" 1782319601 35251 5ff5b5b310c5ac882610e0ccc99095e7 ""
"d:/software/texlive/2026/texmf-dist/tex/generic/pgf/basiclayer/pgfcoretransformations.code.tex" 1782319601 22012 81b34a0aa8fa1a6158cc6220b00e4f10 ""
"d:/software/texlive/2026/texmf-dist/tex/generic/pgf/basiclayer/pgfcoretransparency.code.tex" 1782319601 8893 e851de2175338fdf7c17f3e091d94618 ""
"d:/software/texlive/2026/texmf-dist/tex/generic/pgf/frontendlayer/tikz/libraries/tikzlibraryangles.code.tex" 1782319601 3644 e9b9c24022f5a439aa80f5032fb9b6c6 ""
"d:/software/texlive/2026/texmf-dist/tex/generic/pgf/frontendlayer/tikz/libraries/tikzlibrarybackgrounds.code.tex" 1782319601 4572 4a19637ef65ce88ad2f2d5064b69541d ""
"d:/software/texlive/2026/texmf-dist/tex/generic/pgf/frontendlayer/tikz/libraries/tikzlibrarybending.code.tex" 1782319601 345 9efe6b557b64ab0401dc06a4ef8ca04c ""
"d:/software/texlive/2026/texmf-dist/tex/generic/pgf/frontendlayer/tikz/libraries/tikzlibrarycalc.code.tex" 1782319601 15929 463535aa2c4268fead6674a75c0e8266 ""
"d:/software/texlive/2026/texmf-dist/tex/generic/pgf/frontendlayer/tikz/libraries/tikzlibrarydecorations.code.tex" 1782319601 5650 0ccd824135f363f45d6a97602e59d55a ""
"d:/software/texlive/2026/texmf-dist/tex/generic/pgf/frontendlayer/tikz/libraries/tikzlibrarydecorations.pathmorphing.code.tex" 1782319601 321 cdd11262840e01e25374a2d458f15e99 ""
"d:/software/texlive/2026/texmf-dist/tex/generic/pgf/frontendlayer/tikz/libraries/tikzlibrarydecorations.pathreplacing.code.tex" 1782319601 1319 0b2de5126c6cbc295f0eb77f7344b34d ""
"d:/software/texlive/2026/texmf-dist/tex/generic/pgf/frontendlayer/tikz/libraries/tikzlibraryfadings.code.tex" 1782319601 1179 5483d86c1582c569e665c74efab6281f ""
"d:/software/texlive/2026/texmf-dist/tex/generic/pgf/frontendlayer/tikz/libraries/tikzlibraryfit.code.tex" 1782319601 3626 2d87dc681257fa32d07a8b3934b10f88 ""
"d:/software/texlive/2026/texmf-dist/tex/generic/pgf/frontendlayer/tikz/libraries/tikzlibrarymatrix.code.tex" 1782319601 4228 c39d423dc1a80da31c8c67bf1067f384 ""
"d:/software/texlive/2026/texmf-dist/tex/generic/pgf/frontendlayer/tikz/libraries/tikzlibraryplotmarks.code.tex" 1782319601 325 36322b0789619b270aec5993d5a9ed08 ""
"d:/software/texlive/2026/texmf-dist/tex/generic/pgf/frontendlayer/tikz/libraries/tikzlibrarypositioning.code.tex" 1782319601 3937 3f208572dd82c71103831da976d74f1a ""
"d:/software/texlive/2026/texmf-dist/tex/generic/pgf/frontendlayer/tikz/libraries/tikzlibraryquotes.code.tex" 1782319601 3970 c199216b190c70e45bff870bacf0da00 ""
"d:/software/texlive/2026/texmf-dist/tex/generic/pgf/frontendlayer/tikz/libraries/tikzlibraryshadows.code.tex" 1782319601 2889 d698e3a959304efa342d47e3bb86da5b ""
"d:/software/texlive/2026/texmf-dist/tex/generic/pgf/frontendlayer/tikz/libraries/tikzlibrarytopaths.code.tex" 1782319601 11518 738408f795261b70ce8dd47459171309 ""
"d:/software/texlive/2026/texmf-dist/tex/generic/pgf/frontendlayer/tikz/tikz.code.tex" 1782319601 186859 0445d9a41a87648b4723e04765409541 ""
"d:/software/texlive/2026/texmf-dist/tex/generic/pgf/libraries/decorations/pgflibrarydecorations.pathmorphing.code.tex" 1782319601 8843 5533436db3e30fbad1e0440db6027dac ""
"d:/software/texlive/2026/texmf-dist/tex/generic/pgf/libraries/decorations/pgflibrarydecorations.pathreplacing.code.tex" 1782319601 7474 f05a7223b140f230922562ac6a9fede5 ""
"d:/software/texlive/2026/texmf-dist/tex/generic/pgf/libraries/pgflibraryarrows.meta.code.tex" 1782319601 58801 1e750fb0692eb99aaac45698bbec96b1 ""
"d:/software/texlive/2026/texmf-dist/tex/generic/pgf/libraries/pgflibrarycurvilinear.code.tex" 1782319601 14117 ce877c5ca2b43f31c2cd81e74cc26bd9 ""
"d:/software/texlive/2026/texmf-dist/tex/generic/pgf/libraries/pgflibraryfadings.code.tex" 1782319601 2563 d5b174eb7709fd6bdcc2f70953dbdf8e ""
"d:/software/texlive/2026/texmf-dist/tex/generic/pgf/libraries/pgflibraryfpu.code.tex" 1782319601 85937 c349c2b4a923dd367213cf12546baa15 ""
"d:/software/texlive/2026/texmf-dist/tex/generic/pgf/libraries/pgflibraryplothandlers.code.tex" 1782319601 32995 ac577023e12c0e4bd8aa420b2e852d1a ""
"d:/software/texlive/2026/texmf-dist/tex/generic/pgf/libraries/pgflibraryplotmarks.code.tex" 1782319601 14526 4bc184b12436aa7f6490b2d2036870ef ""
"d:/software/texlive/2026/texmf-dist/tex/generic/pgf/math/pgfint.code.tex" 1782319601 3063 8c415c68a0f3394e45cfeca0b65f6ee6 ""
"d:/software/texlive/2026/texmf-dist/tex/generic/pgf/math/pgfmath.code.tex" 1782319601 949 cea70942e7b7eddabfb3186befada2e6 ""
"d:/software/texlive/2026/texmf-dist/tex/generic/pgf/math/pgfmathcalc.code.tex" 1782319601 13272 7777a64fbd07131a37d276b131c17ee2 ""
"d:/software/texlive/2026/texmf-dist/tex/generic/pgf/math/pgfmathfloat.code.tex" 1782319601 104717 9b2393fbf004a0ce7fa688dbce423848 ""
"d:/software/texlive/2026/texmf-dist/tex/generic/pgf/math/pgfmathfunctions.base.code.tex" 1782319601 10165 cec5fa73d49da442e56efc2d605ef154 ""
"d:/software/texlive/2026/texmf-dist/tex/generic/pgf/math/pgfmathfunctions.basic.code.tex" 1782319601 28178 41c17713108e0795aac6fef3d275fbca ""
"d:/software/texlive/2026/texmf-dist/tex/generic/pgf/math/pgfmathfunctions.code.tex" 1782319601 9649 85779d3d8d573bfd2cd4137ba8202e60 ""
"d:/software/texlive/2026/texmf-dist/tex/generic/pgf/math/pgfmathfunctions.comparison.code.tex" 1782319601 3865 ac538ab80c5cf82b345016e474786549 ""
"d:/software/texlive/2026/texmf-dist/tex/generic/pgf/math/pgfmathfunctions.integerarithmetics.code.tex" 1782319601 3177 27d85c44fbfe09ff3b2cf2879e3ea434 ""
"d:/software/texlive/2026/texmf-dist/tex/generic/pgf/math/pgfmathfunctions.misc.code.tex" 1782319601 11024 0179538121bc2dba172013a3ef89519f ""
"d:/software/texlive/2026/texmf-dist/tex/generic/pgf/math/pgfmathfunctions.random.code.tex" 1782319601 7889 d0e193914ddc35444510f5b569e26b3d ""
"d:/software/texlive/2026/texmf-dist/tex/generic/pgf/math/pgfmathfunctions.round.code.tex" 1782319601 3379 781797a101f647bab82741a99944a229 ""
"d:/software/texlive/2026/texmf-dist/tex/generic/pgf/math/pgfmathfunctions.trigonometric.code.tex" 1782319601 92405 f515f31275db273f97b9d8f52e1b0736 ""
"d:/software/texlive/2026/texmf-dist/tex/generic/pgf/math/pgfmathparser.code.tex" 1782319601 37733 0fe471ac50324723cf6ab693e5c0916c ""
"d:/software/texlive/2026/texmf-dist/tex/generic/pgf/math/pgfmathutil.code.tex" 1782319601 8471 c2883569d03f69e8e1cabfef4999cfd7 ""
"d:/software/texlive/2026/texmf-dist/tex/generic/pgf/modules/pgfmodulebending.code.tex" 1782319601 10901 2a1622fae7fe4eb4574c13e264019b2f ""
"d:/software/texlive/2026/texmf-dist/tex/generic/pgf/modules/pgfmoduledecorations.code.tex" 1782319601 71742 3da44a8be6626eef1c400c68776c7a0f ""
"d:/software/texlive/2026/texmf-dist/tex/generic/pgf/modules/pgfmodulematrix.code.tex" 1782319601 21211 1e73ec76bd73964d84197cc3d2685b01 ""
"d:/software/texlive/2026/texmf-dist/tex/generic/pgf/modules/pgfmodulenonlineartransformations.code.tex" 1782319601 12243 a8b138086af750a4e0d253790eb930d8 ""
"d:/software/texlive/2026/texmf-dist/tex/generic/pgf/modules/pgfmoduleplot.code.tex" 1782319601 16218 98503859deba28f16813029fd927ed8e ""
"d:/software/texlive/2026/texmf-dist/tex/generic/pgf/modules/pgfmoduleshapes.code.tex" 1782319601 44792 c4a5a3feba777682c1d16420f2f01a5b ""
"d:/software/texlive/2026/texmf-dist/tex/generic/pgf/pgf.revision.tex" 1782319601 116 760d50e6a16543bf6edb475635793673 ""
"d:/software/texlive/2026/texmf-dist/tex/generic/pgf/systemlayer/pgf.cfg" 1782319601 926 2963ea0dcf6cc6c0a770b69ec46a477b ""
"d:/software/texlive/2026/texmf-dist/tex/generic/pgf/systemlayer/pgfsys-common-pdf.def" 1782319601 5542 32f75a31ea6c3a7e1148cd6d5e93dbb7 ""
"d:/software/texlive/2026/texmf-dist/tex/generic/pgf/systemlayer/pgfsys-luatex.def" 1782319601 13367 8af38d54c970a326e8ce3a60c0557d2c ""
"d:/software/texlive/2026/texmf-dist/tex/generic/pgf/systemlayer/pgfsys-pdftex.def" 1782319601 12612 7774ba67bfd72e593c4436c2de6201e3 ""
"d:/software/texlive/2026/texmf-dist/tex/generic/pgf/systemlayer/pgfsys.code.tex" 1782319601 61355 39904e7552da3800a6838d41440943a5 ""
"d:/software/texlive/2026/texmf-dist/tex/generic/pgf/systemlayer/pgfsysprotocol.code.tex" 1782319601 1896 b8e0ca0ac371d74c0ca05583f6313c91 ""
"d:/software/texlive/2026/texmf-dist/tex/generic/pgf/systemlayer/pgfsyssoftpath.code.tex" 1782319601 7778 53c8b5623d80238f6a20aa1df1868e63 ""
"d:/software/texlive/2026/texmf-dist/tex/generic/pgf/utilities/pgffor.code.tex" 1782319601 24033 d8893a1ec4d1bfa101b172754743d340 ""
"d:/software/texlive/2026/texmf-dist/tex/generic/pgf/utilities/pgfkeys.code.tex" 1782319601 39784 414c54e866ebab4b801e2ad81d9b21d8 ""
"d:/software/texlive/2026/texmf-dist/tex/generic/pgf/utilities/pgfkeyslibraryfiltered.code.tex" 1782319601 37436 50ba7794827e363eec9ea3467c15c6d7 ""
"d:/software/texlive/2026/texmf-dist/tex/generic/pgf/utilities/pgfrcs.code.tex" 1782319601 4385 510565c2f07998c8a0e14f0ec07ff23c ""
"d:/software/texlive/2026/texmf-dist/tex/generic/pgf/utilities/pgfutil-common.tex" 1782319601 30029 c49ea8f95207c46731469c614daf4e33 ""
"d:/software/texlive/2026/texmf-dist/tex/generic/pgf/utilities/pgfutil-latex.def" 1782319601 7067 11553488d1600cac6a0cfca012fca111 ""
"d:/software/texlive/2026/texmf-dist/tex/generic/pgfplots/libs/pgflibrarypgfplots.surfshading.code.tex" 1782319631 22701 5fab7b8ebb90b053dc067d1bd37e43c2 ""
"d:/software/texlive/2026/texmf-dist/tex/generic/pgfplots/libs/pgfplotslibrary.code.tex" 1782319631 3064 7406f3330a999363276dc290a7d7921f ""
"d:/software/texlive/2026/texmf-dist/tex/generic/pgfplots/libs/tikzlibrarypgfplots.contourlua.code.tex" 1782319631 2931 5d52092da9e839accd7c9026062fe5c3 ""
"d:/software/texlive/2026/texmf-dist/tex/generic/pgfplots/liststructure/pgfplotsarray.code.tex" 1782319631 23291 da0665683d9fcee6858b5099aa0cc376 ""
"d:/software/texlive/2026/texmf-dist/tex/generic/pgfplots/liststructure/pgfplotsdeque.code.tex" 1782319631 4288 b8d6247899b21e3bb66bb11b24d30f2c ""
"d:/software/texlive/2026/texmf-dist/tex/generic/pgfplots/liststructure/pgfplotsliststructure.code.tex" 1782319631 13828 11d1b09335a4a8baa693dd1e6cac3edf ""
"d:/software/texlive/2026/texmf-dist/tex/generic/pgfplots/liststructure/pgfplotsliststructureext.code.tex" 1782319631 24372 46183a992ed6ed4c75dc31b0e6970711 ""
"d:/software/texlive/2026/texmf-dist/tex/generic/pgfplots/liststructure/pgfplotsmatrix.code.tex" 1782319631 18861 7dc35832c8ccea3aa73cdcd75ec0a60b ""
"d:/software/texlive/2026/texmf-dist/tex/generic/pgfplots/numtable/pgfplotstableshared.code.tex" 1782319631 83466 9293553c8041928a731c3eaa456db6e4 ""
"d:/software/texlive/2026/texmf-dist/tex/generic/pgfplots/oldpgfcompatib/pgfplotsoldpgfsupp_loader.code.tex" 1782319631 12293 cd32853e5dbfc25ab862747a810dc816 ""
"d:/software/texlive/2026/texmf-dist/tex/generic/pgfplots/oldpgfcompatib/pgfplotsoldpgfsupp_pgfutil-common-lists.tex" 1782319631 8008 3bb2d07671e6afab7dcb90dfaec572a4 ""
"d:/software/texlive/2026/texmf-dist/tex/generic/pgfplots/pgfplots.code.tex" 1782319631 486348 c0cdf121072f3e4771fa41ef805bb926 ""
"d:/software/texlive/2026/texmf-dist/tex/generic/pgfplots/pgfplots.errorbars.code.tex" 1782319631 22428 72578a4c9324bc5dfafe23fe64f64024 ""
"d:/software/texlive/2026/texmf-dist/tex/generic/pgfplots/pgfplots.markers.code.tex" 1782319631 12489 859c23df41fb9067128ef5a64b01c0a4 ""
"d:/software/texlive/2026/texmf-dist/tex/generic/pgfplots/pgfplots.paths.code.tex" 1782319631 3533 973f376afa5a4526f16b11630b9931b4 ""
"d:/software/texlive/2026/texmf-dist/tex/generic/pgfplots/pgfplots.revision.tex" 1782319631 520 c40870755df65a4602f92875a20f3f8b ""
"d:/software/texlive/2026/texmf-dist/tex/generic/pgfplots/pgfplots.scaling.code.tex" 1782319631 123682 2117ef1f3ff6c8dd2e20b95369835332 ""
"d:/software/texlive/2026/texmf-dist/tex/generic/pgfplots/pgfplotscoordprocessing.code.tex" 1782319631 367036 1887cd74da095a2fa2609244cacf15ee ""
"d:/software/texlive/2026/texmf-dist/tex/generic/pgfplots/pgfplotscore.code.tex" 1782319631 19946 c57cf876c3bc8927342e35d115623398 ""
"d:/software/texlive/2026/texmf-dist/tex/generic/pgfplots/pgfplotsmeshplothandler.code.tex" 1782319631 133878 296c82ec25456366ff106f38f2e89700 ""
"d:/software/texlive/2026/texmf-dist/tex/generic/pgfplots/pgfplotsmeshplotimage.code.tex" 1782319631 25240 07e0b7af672c4a9b5c1a6979f1f295f7 ""
"d:/software/texlive/2026/texmf-dist/tex/generic/pgfplots/pgfplotsplothandlers.code.tex" 1782319631 120955 b66d7a3b9be47a014528f75db9740ee4 ""
"d:/software/texlive/2026/texmf-dist/tex/generic/pgfplots/pgfplotsstackedplots.code.tex" 1782319631 26394 1b5cec392133f8a87b22191451a4d180 ""
"d:/software/texlive/2026/texmf-dist/tex/generic/pgfplots/pgfplotsticks.code.tex" 1782319631 91243 7139c903319539c8ca6ba39a74ea2a67 ""
"d:/software/texlive/2026/texmf-dist/tex/generic/pgfplots/sys/pgflibrarypgfplots.surfshading.pgfsys-pdftex.def" 1782319631 5907 9dc460712c23e5b3338820499d47608c ""
"d:/software/texlive/2026/texmf-dist/tex/generic/pgfplots/sys/pgfplotssysgeneric.code.tex" 1782319631 3095 c82d281b748902a65be2ccca97360b11 ""
"d:/software/texlive/2026/texmf-dist/tex/generic/pgfplots/util/pgfplotsbinary.code.tex" 1782319631 23051 dc88e35196d9435fb503dbdfcff2224d ""
"d:/software/texlive/2026/texmf-dist/tex/generic/pgfplots/util/pgfplotsbinary.data.code.tex" 1782319631 616 3153b8daca68537d23ba64df38c053cc ""
"d:/software/texlive/2026/texmf-dist/tex/generic/pgfplots/util/pgfplotscolor.code.tex" 1782319631 23961 1ba722368c50b46d34a3c5d74971b44e ""
"d:/software/texlive/2026/texmf-dist/tex/generic/pgfplots/util/pgfplotscolormap.code.tex" 1782319631 88958 d1c10d9003595d2dbfdbbb48a3af878f ""
"d:/software/texlive/2026/texmf-dist/tex/generic/pgfplots/util/pgfplotsutil.code.tex" 1782319631 71792 5ccb3da6fae1ab597e321b15e8375f9c ""
"d:/software/texlive/2026/texmf-dist/tex/generic/pgfplots/util/pgfplotsutil.verb.code.tex" 1782319631 3286 c17079ba50483e1ac1721268ea016041 ""
"d:/software/texlive/2026/texmf-dist/tex/generic/soul/soul-ori.sty" 1782320624 25493 086d8ed68c4b31878876ccaedeef8ef5 ""
"d:/software/texlive/2026/texmf-dist/tex/generic/soul/soul.sty" 1782320624 16881 47a5b499d83fa8230ef9439152a7dfaa ""
"d:/software/texlive/2026/texmf-dist/tex/generic/stringenc/stringenc.sty" 1782320750 21514 b7557edcee22835ef6b03ede1802dad4 ""
"d:/software/texlive/2026/texmf-dist/tex/generic/ulem/ulem.sty" 1782321487 15682 94f55b803e160cf7fb6e4d77d07cfe1d ""
"d:/software/texlive/2026/texmf-dist/tex/generic/uniquecounter/uniquecounter.sty" 1782321542 7008 f92eaa0a3872ed622bbf538217cd2ab7 ""
"d:/software/texlive/2026/texmf-dist/tex/generic/xkeyval/xkeyval.tex" 1782321841 19299 c7c03646b32f5a7caaa137673b7e76d2 ""
"d:/software/texlive/2026/texmf-dist/tex/generic/xkeyval/xkvutils.tex" 1782321841 7787 7dfc2f158253fc82fd655393220e6620 ""
"d:/software/texlive/2026/texmf-dist/tex/latex/algorithmicx/algorithmicx.sty" 1782315161 26750 ce139c05a983e19ddca355b43e29c395 ""
"d:/software/texlive/2026/texmf-dist/tex/latex/algorithmicx/algpseudocode.sty" 1782315161 3457 d9077efe6b74c5a094199256af8d7d9a ""
"d:/software/texlive/2026/texmf-dist/tex/latex/algorithms/algorithm.sty" 1782315162 3249 15763257e50278eef5db1952ccde229c ""
"d:/software/texlive/2026/texmf-dist/tex/latex/amsfonts/amsfonts.sty" 1782315199 5949 3f3fd50a8cc94c3d4cbf4fc66cd3df1c ""
"d:/software/texlive/2026/texmf-dist/tex/latex/amsfonts/amssymb.sty" 1782315199 13829 94730e64147574077f8ecfea9bb69af4 ""
"d:/software/texlive/2026/texmf-dist/tex/latex/amsfonts/ueuex.fd" 1782315199 1001 ebfbbf5441dcc5764bd08a9636bca400 ""
"d:/software/texlive/2026/texmf-dist/tex/latex/amsmath/amsbsy.sty" 1782315204 2222 19e175673ef879f3f232f36ac15767a8 ""
"d:/software/texlive/2026/texmf-dist/tex/latex/amsmath/amsgen.sty" 1782315204 4173 c36014b45333fb59a0b1c1ba84df10f4 ""
"d:/software/texlive/2026/texmf-dist/tex/latex/amsmath/amsmath.sty" 1782315204 88826 df33b75c2b510b1365372c880e3e43b1 ""
"d:/software/texlive/2026/texmf-dist/tex/latex/amsmath/amsopn.sty" 1782315204 4474 e8e78ae89fe9e8de1decac472cd058c2 ""
"d:/software/texlive/2026/texmf-dist/tex/latex/amsmath/amstext.sty" 1782315204 2444 7f0b8a693414aa18acf9a58b6c630be5 ""
"d:/software/texlive/2026/texmf-dist/tex/latex/base/article.cls" 1782318246 20144 e6010938fb17dfbae90a243ad7cf351d ""
"d:/software/texlive/2026/texmf-dist/tex/latex/base/ifthen.sty" 1782318246 5525 17697d373c4ff479956bc7a9290f9c25 ""
"d:/software/texlive/2026/texmf-dist/tex/latex/base/size10.clo" 1782318246 8448 f7d8bfe7b7a3986fbf9022b21b7800cc ""
"d:/software/texlive/2026/texmf-dist/tex/latex/base/textcomp.sty" 1782318246 2846 d96ef9e68c7fc3c6fd92cf2c01b553f8 ""
"d:/software/texlive/2026/texmf-dist/tex/latex/bbding/Uding.fd" 1782315508 855 c7d367b8febde87c7a39615c0372cee3 ""
"d:/software/texlive/2026/texmf-dist/tex/latex/bbding/bbding.sty" 1782315508 7225 9b2f9a70c30e88d8bba0d20fcca8b94e ""
"d:/software/texlive/2026/texmf-dist/tex/latex/booktabs/booktabs.sty" 1782315815 6078 f1cb470c9199e7110a27851508ed7a5c ""
"d:/software/texlive/2026/texmf-dist/tex/latex/cjk/texinput/CJK.enc" 1782316097 29591 d67c9aa515f4260cdd60fa9f76d96368 ""
"d:/software/texlive/2026/texmf-dist/tex/latex/cjk/texinput/CJK.sty" 1782316097 33241 3ee25f21824a4a347866900e5037d6f3 ""
"d:/software/texlive/2026/texmf-dist/tex/latex/cjk/texinput/mule/MULEenc.sty" 1782316097 12177 48c95ebf85a580326918e5eb490b477b ""
"d:/software/texlive/2026/texmf-dist/tex/latex/environ/environ.sty" 1782316930 4378 f429f0da968c278653359293040a8f52 ""
"d:/software/texlive/2026/texmf-dist/tex/latex/epstopdf-pkg/epstopdf-base.sty" 1782316953 13886 d1306dcf79a944f6988e688c1785f9ce ""
"d:/software/texlive/2026/texmf-dist/tex/latex/eso-pic/eso-pic.sty" 1782316988 12026 69122f61f5f226a39624c40c33a6cfa5 ""
"d:/software/texlive/2026/texmf-dist/tex/latex/etoolbox/etoolbox.sty" 1782317013 46885 8953c67ffba03252c6090aa19568b8ba ""
"d:/software/texlive/2026/texmf-dist/tex/latex/float/float.sty" 1782317263 6749 16d2656a1984957e674b149555f1ea1d ""
"d:/software/texlive/2026/texmf-dist/tex/latex/fontawesome5/fontawesome5-generic-helper.sty" 1782317297 1796 6d3390e7a3f2a0f63ce89532aa30ac20 ""
"d:/software/texlive/2026/texmf-dist/tex/latex/fontawesome5/fontawesome5-mapping.def" 1782317297 107808 07e47606922c76b1824e149c54155c4c ""
"d:/software/texlive/2026/texmf-dist/tex/latex/fontawesome5/fontawesome5.sty" 1782317297 8112 16dbb19fca3f2df864fea49874356f35 ""
"d:/software/texlive/2026/texmf-dist/tex/latex/fontawesome5/ufontawesomefree0.fd" 1782317297 980 f00df3fe21ca0934b441dacef142037f ""
"d:/software/texlive/2026/texmf-dist/tex/latex/fontawesome5/ufontawesomefree1.fd" 1782317297 980 1b42613a5245ed252c04030af746d737 ""
"d:/software/texlive/2026/texmf-dist/tex/latex/fontawesome5/ufontawesomefree3.fd" 1782317297 980 2697c93582246b8da75635f956e941ff ""
"d:/software/texlive/2026/texmf-dist/tex/latex/graphics-cfg/color.cfg" 1782317580 1213 620bba36b25224fa9b7e1ccb4ecb76fd ""
"d:/software/texlive/2026/texmf-dist/tex/latex/graphics-cfg/graphics.cfg" 1782317580 1224 978390e9c2234eab29404bc21b268d1e ""
"d:/software/texlive/2026/texmf-dist/tex/latex/graphics-def/pdftex.def" 1782317580 19626 23e2822b9b2b5005f4c549ca98b9334d ""
"d:/software/texlive/2026/texmf-dist/tex/latex/graphics/color.sty" 1782317578 7256 1856cdb64afc68f9b9347da073bcd2e4 ""
"d:/software/texlive/2026/texmf-dist/tex/latex/graphics/graphics.sty" 1782317578 18585 02393ee85dc80955e05e568e02814a7c ""
"d:/software/texlive/2026/texmf-dist/tex/latex/graphics/graphicx.sty" 1782317578 8118 42e441807705be09edaacda384cc0b03 ""
"d:/software/texlive/2026/texmf-dist/tex/latex/graphics/keyval.sty" 1782317578 2671 22a90a26d1edef7d1936f15a91d387af ""
"d:/software/texlive/2026/texmf-dist/tex/latex/graphics/mathcolor.ltx" 1782317578 2885 9c645d672ae17285bba324998918efd8 ""
"d:/software/texlive/2026/texmf-dist/tex/latex/graphics/trig.sty" 1782317578 4023 5a4fcfe68d7c6df02f94d7bf578e99fd ""
"d:/software/texlive/2026/texmf-dist/tex/latex/hycolor/hycolor.sty" 1782317802 17914 4c28a13fc3d975e6e81c9bea1d697276 ""
"d:/software/texlive/2026/texmf-dist/tex/latex/hyperref/hpdftex.def" 1782317812 48140 472fdaac0a206974ae3b63f92447d038 ""
"d:/software/texlive/2026/texmf-dist/tex/latex/hyperref/hyperref.sty" 1782317812 223061 069921b261ccff1fe3645959b07c7d04 ""
"d:/software/texlive/2026/texmf-dist/tex/latex/hyperref/nameref.sty" 1782317812 11459 697f11f6c439d25d39d2674b99566af4 ""
"d:/software/texlive/2026/texmf-dist/tex/latex/hyperref/pd1enc.def" 1782317812 14249 9f74ed7af1e113c8740376df8af31aa7 ""
"d:/software/texlive/2026/texmf-dist/tex/latex/hyperref/puenc.def" 1782317812 117118 c21654d7e82a631b4af46ab92e8ebca3 ""
"d:/software/texlive/2026/texmf-dist/tex/latex/inconsolata/inconsolata.sty" 1782317916 4327 344b968526dbd149d23066f8b2160aa4 ""
"d:/software/texlive/2026/texmf-dist/tex/latex/inconsolata/ot1zi4.fd" 1782317916 1033 63d23fc5e9712c8312d76d4376f88831 ""
"d:/software/texlive/2026/texmf-dist/tex/latex/kvoptions/kvoptions.sty" 1782318195 22555 6d8e155cfef6d82c3d5c742fea7c992e ""
"d:/software/texlive/2026/texmf-dist/tex/latex/kvsetkeys/kvsetkeys.sty" 1782318196 13815 760b0c02f691ea230f5359c4e1de23a7 ""
"d:/software/texlive/2026/texmf-dist/tex/latex/l3backend/l3backend-pdftex.def" 1782318204 30909 7dedb28cb21a1e79c47a35a35ae3c507 ""
"d:/software/texlive/2026/texmf-dist/tex/latex/l3kernel/expl3.sty" 1782318211 6558 6ea5ca1cec1bef01add19741f77b03bd ""
"d:/software/texlive/2026/texmf-dist/tex/latex/l3packages/l3keys2e/l3keys2e.sty" 1782318219 4674 ff006c14a0adf55e7318570f962bcbbb ""
"d:/software/texlive/2026/texmf-dist/tex/latex/l3packages/xparse/xparse.sty" 1782318219 10176 c2fc1c663b3f3899ddac6e6ed3fb1d7e ""
"d:/software/texlive/2026/texmf-dist/tex/latex/latexconfig/epstopdf-sys.cfg" 1782318335 678 4792914a8f45be57bb98413425e4c7af ""
"d:/software/texlive/2026/texmf-dist/tex/latex/lineno/lineno.sty" 1782318454 156060 5215e5556723efde95042c27de7c8740 ""
"d:/software/texlive/2026/texmf-dist/tex/latex/makecell/makecell.sty" 1782318736 15773 2dd7dde1ec1c2a3d0c85bc3b273e04d8 ""
"d:/software/texlive/2026/texmf-dist/tex/latex/microtype/microtype-pdftex.def" 1782318936 49656 1c61dbb6f95479ba3d6c0033b53590e2 ""
"d:/software/texlive/2026/texmf-dist/tex/latex/microtype/microtype.cfg" 1782318936 27642 f0cea12315babf4d40608f4eeb5c8458 ""
"d:/software/texlive/2026/texmf-dist/tex/latex/microtype/microtype.sty" 1782318936 102845 043c56602c7d94c8a716319df1af5479 ""
"d:/software/texlive/2026/texmf-dist/tex/latex/microtype/mt-msa.cfg" 1782318936 5929 2b35ae0f0fb46984dfffa2bc9d09de5c ""
"d:/software/texlive/2026/texmf-dist/tex/latex/microtype/mt-msb.cfg" 1782318936 5594 992ef5c3f8fd1168bb7101a957333065 ""
"d:/software/texlive/2026/texmf-dist/tex/latex/microtype/mt-ppl.cfg" 1782318936 6374 f178b3503ad84f93c65c2e44f7dd5a67 ""
"d:/software/texlive/2026/texmf-dist/tex/latex/multirow/multirow.sty" 1782319129 6696 886c9f3087d0b973ed2c19aa79cb3023 ""
"d:/software/texlive/2026/texmf-dist/tex/latex/pgf/basiclayer/pgf.sty" 1782319601 1090 bae35ef70b3168089ef166db3e66f5b2 ""
"d:/software/texlive/2026/texmf-dist/tex/latex/pgf/basiclayer/pgfcore.sty" 1782319601 373 00b204b1d7d095b892ad31a7494b0373 ""
"d:/software/texlive/2026/texmf-dist/tex/latex/pgf/compatibility/pgfcomp-version-0-65.sty" 1782319601 21013 f4ff83d25bb56552493b030f27c075ae ""
"d:/software/texlive/2026/texmf-dist/tex/latex/pgf/compatibility/pgfcomp-version-1-18.sty" 1782319601 989 c49c8ae06d96f8b15869da7428047b1e ""
"d:/software/texlive/2026/texmf-dist/tex/latex/pgf/frontendlayer/tikz.sty" 1782319601 339 c2e180022e3afdb99c7d0ea5ce469b7d ""
"d:/software/texlive/2026/texmf-dist/tex/latex/pgf/math/pgfmath.sty" 1782319601 306 c56a323ca5bf9242f54474ced10fca71 ""
"d:/software/texlive/2026/texmf-dist/tex/latex/pgf/systemlayer/pgfsys.sty" 1782319601 443 8c872229db56122037e86bcda49e14f3 ""
"d:/software/texlive/2026/texmf-dist/tex/latex/pgf/utilities/pgffor.sty" 1782319601 348 ee405e64380c11319f0e249fed57e6c5 ""
"d:/software/texlive/2026/texmf-dist/tex/latex/pgf/utilities/pgfkeys.sty" 1782319601 274 5ae372b7df79135d240456a1c6f2cf9a ""
"d:/software/texlive/2026/texmf-dist/tex/latex/pgf/utilities/pgfrcs.sty" 1782319601 325 f9f16d12354225b7dd52a3321f085955 ""
"d:/software/texlive/2026/texmf-dist/tex/latex/pgfplots/pgfplots.sty" 1782319631 4904 ee78b44e85d6fccf08cd99370557481e ""
"d:/software/texlive/2026/texmf-dist/tex/latex/psnfss/mathpazo.sty" 1782319855 8976 3f18c815295e2a2949e87fa743bcb489 ""
"d:/software/texlive/2026/texmf-dist/tex/latex/psnfss/omlzplm.fd" 1782319855 638 2349dd185efcefd0c3380a2601df0cee ""
"d:/software/texlive/2026/texmf-dist/tex/latex/psnfss/omsppl.fd" 1782319855 576 be451a105739548c4929bd91be18fae6 ""
"d:/software/texlive/2026/texmf-dist/tex/latex/psnfss/omszplm.fd" 1782319855 455 c09241d92b40b1b84eb2bb5776606aea ""
"d:/software/texlive/2026/texmf-dist/tex/latex/psnfss/omxzplm.fd" 1782319855 322 fb88cb3e5f25cf1596ba8826c4210e0e ""
"d:/software/texlive/2026/texmf-dist/tex/latex/psnfss/ot1ppl.fd" 1782319855 961 06b773644d960aac68add40fcb596208 ""
"d:/software/texlive/2026/texmf-dist/tex/latex/psnfss/ot1zplm.fd" 1782319855 436 5e7baa1a259bab50a2bdffad6426f38c ""
"d:/software/texlive/2026/texmf-dist/tex/latex/psnfss/ts1ppl.fd" 1782319855 619 35f1cf70bf09b6e2d22ee0902cfb1a8b ""
"d:/software/texlive/2026/texmf-dist/tex/latex/psnfss/ufplmbb.fd" 1782319855 321 077609d8fc08b1f1181795378b603500 ""
"d:/software/texlive/2026/texmf-dist/tex/latex/refcount/refcount.sty" 1782320235 9878 9e94e8fa600d95f9c7731bb21dfb67a4 ""
"d:/software/texlive/2026/texmf-dist/tex/latex/rerunfilecheck/rerunfilecheck.sty" 1782320265 9684 a33a14b82ce60d6e77cb9be689d79ee6 ""
"d:/software/texlive/2026/texmf-dist/tex/latex/setspace/setspace.sty" 1782320498 22490 8cac309b79a4c53a4ffce4b1b07aead0 ""
"d:/software/texlive/2026/texmf-dist/tex/latex/tcolorbox/tcolorbox.sty" 1782320889 111306 872e63fa92fff8693b07d290f4f5e62f ""
"d:/software/texlive/2026/texmf-dist/tex/latex/tex-gyre/ot1qpl.fd" 1782320936 1660 e9b22d666df8ddf20235e469d85dea27 ""
"d:/software/texlive/2026/texmf-dist/tex/latex/tex-gyre/tgpagella.sty" 1782320936 2230 fc853fd117e6b73f5d6716214d3abb3c ""
"d:/software/texlive/2026/texmf-dist/tex/latex/tikz-qtree/pgfsubpic.sty" 1782321190 54 f9439ec673a174422959417bbf935693 ""
"d:/software/texlive/2026/texmf-dist/tex/latex/tikz-qtree/pgfsubpic.tex" 1782321190 8570 d1bab08986c0118f4c006d2f8c250422 ""
"d:/software/texlive/2026/texmf-dist/tex/latex/tikz-qtree/pgftree.sty" 1782321190 103 1ec4da228521932da66ce8de6a762d94 ""
"d:/software/texlive/2026/texmf-dist/tex/latex/tikz-qtree/pgftree.tex" 1782321190 7300 d92acc9c8a60dbc36f92a0414bc55881 ""
"d:/software/texlive/2026/texmf-dist/tex/latex/tikz-qtree/tikz-qtree.sty" 1782321190 80 0f0f922a334da96eeeca4db3f8b5834a ""
"d:/software/texlive/2026/texmf-dist/tex/latex/tikz-qtree/tikz-qtree.tex" 1782321190 7407 bee44709234c7b6ab03b4bbe0f0eb1cf ""
"d:/software/texlive/2026/texmf-dist/tex/latex/tools/array.sty" 1782321329 17611 1fdd60d12fed6aebd140f4d060571642 ""
"d:/software/texlive/2026/texmf-dist/tex/latex/tools/verbatim.sty" 1782321329 7532 f4ccdd43fe3dcc1fb2fb9431b10aa336 ""
"d:/software/texlive/2026/texmf-dist/tex/latex/trimspaces/trimspaces.sty" 1782321377 1380 971a51b00a14503ddf754cab24c3f209 ""
"d:/software/texlive/2026/texmf-dist/tex/latex/upquote/upquote.sty" 1782321574 1048 517e01cde97c1c0baf72e69d43aa5a2e ""
"d:/software/texlive/2026/texmf-dist/tex/latex/url/url.sty" 1782321584 12796 8edb7d69a20b857904dd0ea757c14ec9 ""
"d:/software/texlive/2026/texmf-dist/tex/latex/varwidth/varwidth.sty" 1782321623 10894 d359a13923460b2a73d4312d613554c8 ""
"d:/software/texlive/2026/texmf-dist/tex/latex/xcolor/xcolor.sty" 1782321765 55384 b454dec21c2d9f45ec0b793f0995b992 ""
"d:/software/texlive/2026/texmf-dist/tex/latex/xkeyval/xkeyval.sty" 1782321841 5006 d7e899bb5d9717c7f214e9bc0284de7b ""
"d:/software/texlive/2026/texmf-dist/web2c/texmf.cnf" 1782315048 43630 e9fa012910491fe5ae08a72c3c2997a7 ""
"d:/software/texlive/2026/texmf-var/fonts/map/pdftex/updmap/pdftex.map" 1782321996 5619420 fb59592dcd176eaaceeb94ae4751f97d ""
"d:/software/texlive/2026/texmf-var/web2c/pdftex/pdflatex.fmt" 1782322062 3656600 2f2edbb6b9e15cf204b23395d90b6d26 ""
"d:/software/texlive/2026/texmf.cnf" 1782321978 713 e69b156964470283e0530f5060668171 ""
"fancyhdr.sty" 1782375814 20523 b9ce13e6c2356567d83cbc56b64436e0 ""
"natbib.sty" 1782375814 45154 40088ce024445d7ee2bca59e704cdd01 ""
"rl-introduction.aux" 1782404676 50785 2348d3be4ba13df8c287520ce3a2118d "pdflatex"
"rl-introduction.bbl" 1782404668 45895 aaba085a3f3cdac8e9edff56f6659c1c "bibtex rl-introduction"
"rl-introduction.out" 1782404676 8324 2060c280c04e7a328244371d66772709 "pdflatex"
"rl-introduction.sty" 1782375814 7779 e52e2d3c711eddb17f4fcb6fa19d966d ""
"rl-introduction.tex" 1782375814 9653 2303ab2f9125c255cbcda33bd6a948f0 ""
"rl-introduction.toc" 1782404676 3977 90bc0179786a4281a2911d2abe1ff412 "pdflatex"
"section2/Figures/rl_framework.tex" 1782375814 3578 25dae09e9664cebc5e75c18f0cc9f80b ""
"section2/section2.tex" 1782375814 9029 65f9912edaadc95e15e852e632c847c3 ""
"section3/Figures/figure-an-overlap-example.tex" 1782375814 1759 1ae318c8e98195f54a48b7ae05d7b15d ""
"section3/Figures/figure-architecture-value-function.tex" 1782375814 2967 65f4ab0705f35edcd999fa5bf4d21b47 ""
"section3/Figures/figure-importance-sampling-bad-sampling.tex" 1782375814 3372 ee6430065020ed120e24187966232846 ""
"section3/Figures/figure-importance-sampling.tex" 1782375814 2868 c79ce5dd3a283dcde3130d718dc7b458 ""
"section3/Figures/figure-one-example.tex" 1782375814 4983 133fe77db807f8f5276d4bac169a85b2 ""
"section3/Figures/figure-pairwise-reward-loss-expectation.tex" 1782375814 3533 91c7f696436f510d7e151ddc35998430 ""
"section3/Figures/figure-ppo-clip.tex" 1782375814 2650 4ff1265029030c7683b60d7ff3e63817 ""
"section3/Figures/figure-reward-model.tex" 1782375814 2099 764880c6d79ffc6b956a5e013c1b0637 ""
"section3/Figures/figure-rlhf.tex" 1782375814 5390 be073578864d3d57b18eb5b304e2ff19 ""
"section3/Figures/figure-update-with-policy-gradient.tex" 1782375814 5105 04e24abbe0ea85bf3f84b02001165c42 ""
"section3/section3.tex" 1782375814 48235 16b1d8948fc57ac465d991b70ecf0fe7 ""
"section4/Figures/figure-an-overlap-example-dense-reward-model.tex" 1782375814 1872 d80bb05758a9e84c9de290ecc23e5f50 ""
"section4/Figures/figure-dpo.tex" 1782375814 5275 a305f854536de85d060e389ca75f061b ""
"section4/Figures/figure-examples-of-rule-based-reward-length.tex" 1782375814 2368 3a17d2fe54cac2cacc42770932463e00 ""
"section4/Figures/figure-examples-of-rule-based-reward-math.tex" 1782375814 2044 aef9a2c1f50989db4a9476b948f9983a ""
"section4/Figures/figure-generative-reward-model-architecture.tex" 1782375814 2523 a62805e0ed51b78b16f2df86d5c10740 ""
"section4/Figures/figure-grpo.tex" 1782375814 5008 62b7a5db1ce5d0c15ec9741d2f499937 ""
"section4/Figures/figure-issue-monte-carlo.tex" 1782375814 4027 d056bb2dd145b974bf31cd774dbbe276 ""
"section4/Figures/figure-methods-improving-reward-generalization.tex" 1782375814 7863 81b2b323c89755bd980587135716ce40 ""
"section4/Figures/figure-rlhf-vs-dpo.tex" 1782375814 3833 6a25ecb5fa334441b88a96f967eb10c9 ""
"section4/Tables/dpo_variants.tex" 1782375814 3582 96eacf2221d8c24ed0189208aad447dd ""
"section4/section4.tex" 1782375814 47598 635267cf6d3c9470f5d92bf7b7f52e85 ""
"section5/Figures/figure-bon-sampling.tex" 1782375814 3263 b4305dd28224afa0e16db7e03ec520ea ""
"section5/Figures/figure-iterative-rl.tex" 1782375814 7368 9120e3d496537072c7cd5759b816b0e0 ""
"section5/Figures/figure-mcts.tex" 1782375814 8161 bff6d7ae6fb58e9e64b7a5b062e1f4de ""
"section5/Figures/figure-step-by-step-with-prm.tex" 1782375814 6527 294aea74afccfc7075d703b0aa14ecc3 ""
"section5/section5.tex" 1782375814 27281 b316d4b13186bbffcd976da0be043376 ""
"section6/section6.tex" 1782375814 55 0b6bbd0928084a63e3dea1fb13ead5bd ""
"section7/section7.tex" 1782375814 9297 ae45328eea02fe015e44bc8b7b1ebba8 ""
"section8/section8.tex" 1782375814 116 8616535f61a4ddedaec5663eeed7b1b5 ""
"section9/section9.tex" 1782375814 342 bcce0141b44885aef98d03bc557843a1 ""
"section9/tables/dataset.tex" 1782375814 983 6e17450eb6f0f9d6ffd4ddeed20be381 ""
"section9/tables/systems.tex" 1782375814 1415 392dea0c8541d33ab7ca3cc242351b05 ""
(generated)
"rl-introduction.aux"
"rl-introduction.log"
"rl-introduction.out"
"rl-introduction.pdf"
"rl-introduction.toc"
(rewritten before read)
rl-introduction.fls 0 → 100644
PWD c:/Users/Lenovo/Desktop/rl_introduction
INPUT d:/software/texlive/2026/texmf.cnf
INPUT d:/software/texlive/2026/texmf-dist/web2c/texmf.cnf
INPUT d:/software/texlive/2026/texmf-var/web2c/pdftex/pdflatex.fmt
INPUT c:/Users/Lenovo/Desktop/rl_introduction/rl-introduction.tex
OUTPUT rl-introduction.log
INPUT d:/software/texlive/2026/texmf-dist/tex/latex/base/article.cls
INPUT d:/software/texlive/2026/texmf-dist/tex/latex/base/article.cls
INPUT d:/software/texlive/2026/texmf-dist/tex/latex/base/size10.clo
INPUT d:/software/texlive/2026/texmf-dist/tex/latex/base/size10.clo
INPUT d:/software/texlive/2026/texmf-dist/tex/latex/base/size10.clo
INPUT ./rl-introduction.sty
INPUT rl-introduction.sty
INPUT d:/software/texlive/2026/texmf-dist/tex/latex/tex-gyre/tgpagella.sty
INPUT d:/software/texlive/2026/texmf-dist/tex/latex/tex-gyre/tgpagella.sty
INPUT d:/software/texlive/2026/texmf-dist/tex/latex/kvoptions/kvoptions.sty
INPUT d:/software/texlive/2026/texmf-dist/tex/latex/kvoptions/kvoptions.sty
INPUT d:/software/texlive/2026/texmf-dist/tex/latex/graphics/keyval.sty
INPUT d:/software/texlive/2026/texmf-dist/tex/latex/graphics/keyval.sty
INPUT d:/software/texlive/2026/texmf-dist/tex/generic/ltxcmds/ltxcmds.sty
INPUT d:/software/texlive/2026/texmf-dist/tex/generic/ltxcmds/ltxcmds.sty
INPUT d:/software/texlive/2026/texmf-dist/tex/latex/kvsetkeys/kvsetkeys.sty
INPUT d:/software/texlive/2026/texmf-dist/tex/latex/kvsetkeys/kvsetkeys.sty
INPUT d:/software/texlive/2026/texmf-dist/tex/latex/psnfss/mathpazo.sty
INPUT d:/software/texlive/2026/texmf-dist/tex/latex/psnfss/mathpazo.sty
INPUT d:/software/texlive/2026/texmf-dist/tex/latex/inconsolata/inconsolata.sty
INPUT d:/software/texlive/2026/texmf-dist/tex/latex/inconsolata/inconsolata.sty
INPUT d:/software/texlive/2026/texmf-dist/tex/latex/base/textcomp.sty
INPUT d:/software/texlive/2026/texmf-dist/tex/latex/base/textcomp.sty
INPUT d:/software/texlive/2026/texmf-dist/tex/latex/xkeyval/xkeyval.sty
INPUT d:/software/texlive/2026/texmf-dist/tex/latex/xkeyval/xkeyval.sty
INPUT d:/software/texlive/2026/texmf-dist/tex/generic/xkeyval/xkeyval.tex
INPUT d:/software/texlive/2026/texmf-dist/tex/generic/xkeyval/xkvutils.tex
INPUT d:/software/texlive/2026/texmf-dist/tex/latex/eso-pic/eso-pic.sty
INPUT d:/software/texlive/2026/texmf-dist/tex/latex/eso-pic/eso-pic.sty
INPUT d:/software/texlive/2026/texmf-dist/tex/latex/xcolor/xcolor.sty
INPUT d:/software/texlive/2026/texmf-dist/tex/latex/xcolor/xcolor.sty
INPUT d:/software/texlive/2026/texmf-dist/tex/latex/xcolor/xcolor.sty
INPUT d:/software/texlive/2026/texmf-dist/tex/latex/graphics-cfg/color.cfg
INPUT d:/software/texlive/2026/texmf-dist/tex/latex/graphics-cfg/color.cfg
INPUT d:/software/texlive/2026/texmf-dist/tex/latex/graphics-cfg/color.cfg
INPUT d:/software/texlive/2026/texmf-dist/tex/latex/graphics-def/pdftex.def
INPUT d:/software/texlive/2026/texmf-dist/tex/latex/graphics-def/pdftex.def
INPUT d:/software/texlive/2026/texmf-dist/tex/latex/graphics-def/pdftex.def
INPUT d:/software/texlive/2026/texmf-dist/tex/latex/graphics/mathcolor.ltx
INPUT d:/software/texlive/2026/texmf-dist/tex/latex/graphics/mathcolor.ltx
INPUT d:/software/texlive/2026/texmf-dist/tex/latex/graphics/mathcolor.ltx
INPUT ./fancyhdr.sty
INPUT fancyhdr.sty
INPUT ./natbib.sty
INPUT natbib.sty
INPUT d:/software/texlive/2026/texmf-dist/tex/latex/microtype/microtype.sty
INPUT d:/software/texlive/2026/texmf-dist/tex/latex/microtype/microtype.sty
INPUT d:/software/texlive/2026/texmf-dist/tex/latex/etoolbox/etoolbox.sty
INPUT d:/software/texlive/2026/texmf-dist/tex/latex/etoolbox/etoolbox.sty
INPUT d:/software/texlive/2026/texmf-dist/tex/latex/microtype/microtype-pdftex.def
INPUT d:/software/texlive/2026/texmf-dist/tex/latex/microtype/microtype-pdftex.def
INPUT d:/software/texlive/2026/texmf-dist/tex/latex/microtype/microtype-pdftex.def
INPUT d:/software/texlive/2026/texmf-dist/tex/latex/microtype/microtype.cfg
INPUT d:/software/texlive/2026/texmf-dist/tex/latex/microtype/microtype.cfg
INPUT d:/software/texlive/2026/texmf-dist/tex/latex/microtype/microtype.cfg
INPUT d:/software/texlive/2026/texmf-dist/tex/latex/hyperref/hyperref.sty
INPUT d:/software/texlive/2026/texmf-dist/tex/latex/hyperref/hyperref.sty
INPUT d:/software/texlive/2026/texmf-dist/tex/generic/iftex/iftex.sty
INPUT d:/software/texlive/2026/texmf-dist/tex/generic/iftex/iftex.sty
INPUT d:/software/texlive/2026/texmf-dist/tex/generic/kvdefinekeys/kvdefinekeys.sty
INPUT d:/software/texlive/2026/texmf-dist/tex/generic/kvdefinekeys/kvdefinekeys.sty
INPUT d:/software/texlive/2026/texmf-dist/tex/generic/pdfescape/pdfescape.sty
INPUT d:/software/texlive/2026/texmf-dist/tex/generic/pdfescape/pdfescape.sty
INPUT d:/software/texlive/2026/texmf-dist/tex/generic/pdftexcmds/pdftexcmds.sty
INPUT d:/software/texlive/2026/texmf-dist/tex/generic/pdftexcmds/pdftexcmds.sty
INPUT d:/software/texlive/2026/texmf-dist/tex/generic/infwarerr/infwarerr.sty
INPUT d:/software/texlive/2026/texmf-dist/tex/generic/infwarerr/infwarerr.sty
INPUT d:/software/texlive/2026/texmf-dist/tex/latex/hycolor/hycolor.sty
INPUT d:/software/texlive/2026/texmf-dist/tex/latex/hycolor/hycolor.sty
INPUT d:/software/texlive/2026/texmf-dist/tex/latex/hyperref/nameref.sty
INPUT d:/software/texlive/2026/texmf-dist/tex/latex/hyperref/nameref.sty
INPUT d:/software/texlive/2026/texmf-dist/tex/latex/refcount/refcount.sty
INPUT d:/software/texlive/2026/texmf-dist/tex/latex/refcount/refcount.sty
INPUT d:/software/texlive/2026/texmf-dist/tex/generic/gettitlestring/gettitlestring.sty
INPUT d:/software/texlive/2026/texmf-dist/tex/generic/gettitlestring/gettitlestring.sty
INPUT d:/software/texlive/2026/texmf-dist/tex/generic/stringenc/stringenc.sty
INPUT d:/software/texlive/2026/texmf-dist/tex/generic/stringenc/stringenc.sty
INPUT d:/software/texlive/2026/texmf-dist/tex/latex/hyperref/pd1enc.def
INPUT d:/software/texlive/2026/texmf-dist/tex/latex/hyperref/pd1enc.def
INPUT d:/software/texlive/2026/texmf-dist/tex/latex/hyperref/pd1enc.def
INPUT d:/software/texlive/2026/texmf-dist/tex/generic/intcalc/intcalc.sty
INPUT d:/software/texlive/2026/texmf-dist/tex/generic/intcalc/intcalc.sty
INPUT d:/software/texlive/2026/texmf-dist/tex/latex/hyperref/puenc.def
INPUT d:/software/texlive/2026/texmf-dist/tex/latex/hyperref/puenc.def
INPUT d:/software/texlive/2026/texmf-dist/tex/latex/hyperref/puenc.def
INPUT d:/software/texlive/2026/texmf-dist/tex/latex/url/url.sty
INPUT d:/software/texlive/2026/texmf-dist/tex/latex/url/url.sty
INPUT d:/software/texlive/2026/texmf-dist/tex/generic/bitset/bitset.sty
INPUT d:/software/texlive/2026/texmf-dist/tex/generic/bitset/bitset.sty
INPUT d:/software/texlive/2026/texmf-dist/tex/generic/bigintcalc/bigintcalc.sty
INPUT d:/software/texlive/2026/texmf-dist/tex/generic/bigintcalc/bigintcalc.sty
INPUT d:/software/texlive/2026/texmf-dist/tex/latex/hyperref/hpdftex.def
INPUT d:/software/texlive/2026/texmf-dist/tex/latex/hyperref/hpdftex.def
INPUT d:/software/texlive/2026/texmf-dist/tex/latex/hyperref/hpdftex.def
INPUT d:/software/texlive/2026/texmf-dist/tex/latex/rerunfilecheck/rerunfilecheck.sty
INPUT d:/software/texlive/2026/texmf-dist/tex/latex/rerunfilecheck/rerunfilecheck.sty
INPUT d:/software/texlive/2026/texmf-dist/tex/generic/uniquecounter/uniquecounter.sty
INPUT d:/software/texlive/2026/texmf-dist/tex/generic/uniquecounter/uniquecounter.sty
INPUT d:/software/texlive/2026/texmf-dist/tex/latex/booktabs/booktabs.sty
INPUT d:/software/texlive/2026/texmf-dist/tex/latex/booktabs/booktabs.sty
INPUT d:/software/texlive/2026/texmf-dist/tex/latex/fontawesome5/fontawesome5.sty
INPUT d:/software/texlive/2026/texmf-dist/tex/latex/fontawesome5/fontawesome5.sty
INPUT d:/software/texlive/2026/texmf-dist/tex/latex/l3kernel/expl3.sty
INPUT d:/software/texlive/2026/texmf-dist/tex/latex/l3kernel/expl3.sty
INPUT d:/software/texlive/2026/texmf-dist/tex/latex/l3backend/l3backend-pdftex.def
INPUT d:/software/texlive/2026/texmf-dist/tex/latex/l3backend/l3backend-pdftex.def
INPUT d:/software/texlive/2026/texmf-dist/tex/latex/l3packages/l3keys2e/l3keys2e.sty
INPUT d:/software/texlive/2026/texmf-dist/tex/latex/l3packages/l3keys2e/l3keys2e.sty
INPUT d:/software/texlive/2026/texmf-dist/tex/latex/l3packages/xparse/xparse.sty
INPUT d:/software/texlive/2026/texmf-dist/tex/latex/l3packages/xparse/xparse.sty
INPUT d:/software/texlive/2026/texmf-dist/tex/latex/fontawesome5/fontawesome5-generic-helper.sty
INPUT d:/software/texlive/2026/texmf-dist/tex/latex/fontawesome5/fontawesome5-generic-helper.sty
INPUT d:/software/texlive/2026/texmf-dist/tex/latex/fontawesome5/fontawesome5-mapping.def
INPUT d:/software/texlive/2026/texmf-dist/tex/latex/fontawesome5/fontawesome5-mapping.def
INPUT d:/software/texlive/2026/texmf-dist/tex/latex/lineno/lineno.sty
INPUT d:/software/texlive/2026/texmf-dist/tex/latex/lineno/lineno.sty
INPUT d:/software/texlive/2026/texmf-dist/tex/latex/bbding/bbding.sty
INPUT d:/software/texlive/2026/texmf-dist/tex/latex/bbding/bbding.sty
INPUT d:/software/texlive/2026/texmf-dist/tex/latex/cjk/texinput/CJK.sty
INPUT d:/software/texlive/2026/texmf-dist/tex/latex/cjk/texinput/CJK.sty
INPUT d:/software/texlive/2026/texmf-dist/tex/latex/cjk/texinput/mule/MULEenc.sty
INPUT d:/software/texlive/2026/texmf-dist/tex/latex/cjk/texinput/mule/MULEenc.sty
INPUT d:/software/texlive/2026/texmf-dist/tex/latex/cjk/texinput/CJK.enc
INPUT d:/software/texlive/2026/texmf-dist/tex/latex/algorithms/algorithm.sty
INPUT d:/software/texlive/2026/texmf-dist/tex/latex/algorithms/algorithm.sty
INPUT d:/software/texlive/2026/texmf-dist/tex/latex/float/float.sty
INPUT d:/software/texlive/2026/texmf-dist/tex/latex/float/float.sty
INPUT d:/software/texlive/2026/texmf-dist/tex/latex/base/ifthen.sty
INPUT d:/software/texlive/2026/texmf-dist/tex/latex/base/ifthen.sty
INPUT d:/software/texlive/2026/texmf-dist/tex/latex/algorithmicx/algpseudocode.sty
INPUT d:/software/texlive/2026/texmf-dist/tex/latex/algorithmicx/algpseudocode.sty
INPUT d:/software/texlive/2026/texmf-dist/tex/latex/algorithmicx/algorithmicx.sty
INPUT d:/software/texlive/2026/texmf-dist/tex/latex/algorithmicx/algorithmicx.sty
INPUT d:/software/texlive/2026/texmf-dist/tex/latex/tikz-qtree/tikz-qtree.sty
INPUT d:/software/texlive/2026/texmf-dist/tex/latex/tikz-qtree/tikz-qtree.sty
INPUT d:/software/texlive/2026/texmf-dist/tex/latex/pgf/frontendlayer/tikz.sty
INPUT d:/software/texlive/2026/texmf-dist/tex/latex/pgf/frontendlayer/tikz.sty
INPUT d:/software/texlive/2026/texmf-dist/tex/latex/pgf/basiclayer/pgf.sty
INPUT d:/software/texlive/2026/texmf-dist/tex/latex/pgf/basiclayer/pgf.sty
INPUT d:/software/texlive/2026/texmf-dist/tex/latex/pgf/utilities/pgfrcs.sty
INPUT d:/software/texlive/2026/texmf-dist/tex/latex/pgf/utilities/pgfrcs.sty
INPUT d:/software/texlive/2026/texmf-dist/tex/generic/pgf/utilities/pgfutil-common.tex
INPUT d:/software/texlive/2026/texmf-dist/tex/generic/pgf/utilities/pgfutil-latex.def
INPUT d:/software/texlive/2026/texmf-dist/tex/generic/pgf/utilities/pgfrcs.code.tex
INPUT d:/software/texlive/2026/texmf-dist/tex/generic/pgf/utilities/pgfrcs.code.tex
INPUT d:/software/texlive/2026/texmf-dist/tex/generic/pgf/utilities/pgfrcs.code.tex
INPUT d:/software/texlive/2026/texmf-dist/tex/generic/pgf/pgf.revision.tex
INPUT d:/software/texlive/2026/texmf-dist/tex/generic/pgf/pgf.revision.tex
INPUT d:/software/texlive/2026/texmf-dist/tex/latex/pgf/basiclayer/pgfcore.sty
INPUT d:/software/texlive/2026/texmf-dist/tex/latex/pgf/basiclayer/pgfcore.sty
INPUT d:/software/texlive/2026/texmf-dist/tex/latex/graphics/graphicx.sty
INPUT d:/software/texlive/2026/texmf-dist/tex/latex/graphics/graphicx.sty
INPUT d:/software/texlive/2026/texmf-dist/tex/latex/graphics/graphics.sty
INPUT d:/software/texlive/2026/texmf-dist/tex/latex/graphics/graphics.sty
INPUT d:/software/texlive/2026/texmf-dist/tex/latex/graphics/trig.sty
INPUT d:/software/texlive/2026/texmf-dist/tex/latex/graphics/trig.sty
INPUT d:/software/texlive/2026/texmf-dist/tex/latex/graphics-cfg/graphics.cfg
INPUT d:/software/texlive/2026/texmf-dist/tex/latex/graphics-cfg/graphics.cfg
INPUT d:/software/texlive/2026/texmf-dist/tex/latex/graphics-cfg/graphics.cfg
INPUT d:/software/texlive/2026/texmf-dist/tex/latex/pgf/systemlayer/pgfsys.sty
INPUT d:/software/texlive/2026/texmf-dist/tex/latex/pgf/systemlayer/pgfsys.sty
INPUT d:/software/texlive/2026/texmf-dist/tex/generic/pgf/systemlayer/pgfsys.code.tex
INPUT d:/software/texlive/2026/texmf-dist/tex/generic/pgf/systemlayer/pgfsys.code.tex
INPUT d:/software/texlive/2026/texmf-dist/tex/generic/pgf/systemlayer/pgfsys.code.tex
INPUT d:/software/texlive/2026/texmf-dist/tex/generic/pgf/utilities/pgfkeys.code.tex
INPUT d:/software/texlive/2026/texmf-dist/tex/generic/pgf/utilities/pgfkeyslibraryfiltered.code.tex
INPUT d:/software/texlive/2026/texmf-dist/tex/generic/pgf/systemlayer/pgf.cfg
INPUT d:/software/texlive/2026/texmf-dist/tex/generic/pgf/systemlayer/pgfsys-pdftex.def
INPUT d:/software/texlive/2026/texmf-dist/tex/generic/pgf/systemlayer/pgfsys-pdftex.def
INPUT d:/software/texlive/2026/texmf-dist/tex/generic/pgf/systemlayer/pgfsys-common-pdf.def
INPUT d:/software/texlive/2026/texmf-dist/tex/generic/pgf/systemlayer/pgfsyssoftpath.code.tex
INPUT d:/software/texlive/2026/texmf-dist/tex/generic/pgf/systemlayer/pgfsyssoftpath.code.tex
INPUT d:/software/texlive/2026/texmf-dist/tex/generic/pgf/systemlayer/pgfsyssoftpath.code.tex
INPUT d:/software/texlive/2026/texmf-dist/tex/generic/pgf/systemlayer/pgfsysprotocol.code.tex
INPUT d:/software/texlive/2026/texmf-dist/tex/generic/pgf/systemlayer/pgfsysprotocol.code.tex
INPUT d:/software/texlive/2026/texmf-dist/tex/generic/pgf/systemlayer/pgfsysprotocol.code.tex
INPUT d:/software/texlive/2026/texmf-dist/tex/generic/pgf/basiclayer/pgfcore.code.tex
INPUT d:/software/texlive/2026/texmf-dist/tex/generic/pgf/basiclayer/pgfcore.code.tex
INPUT d:/software/texlive/2026/texmf-dist/tex/generic/pgf/basiclayer/pgfcore.code.tex
INPUT d:/software/texlive/2026/texmf-dist/tex/generic/pgf/math/pgfmath.code.tex
INPUT d:/software/texlive/2026/texmf-dist/tex/generic/pgf/math/pgfmathutil.code.tex
INPUT d:/software/texlive/2026/texmf-dist/tex/generic/pgf/math/pgfmathparser.code.tex
INPUT d:/software/texlive/2026/texmf-dist/tex/generic/pgf/math/pgfmathfunctions.code.tex
INPUT d:/software/texlive/2026/texmf-dist/tex/generic/pgf/math/pgfmathfunctions.basic.code.tex
INPUT d:/software/texlive/2026/texmf-dist/tex/generic/pgf/math/pgfmathfunctions.trigonometric.code.tex
INPUT d:/software/texlive/2026/texmf-dist/tex/generic/pgf/math/pgfmathfunctions.random.code.tex
INPUT d:/software/texlive/2026/texmf-dist/tex/generic/pgf/math/pgfmathfunctions.comparison.code.tex
INPUT d:/software/texlive/2026/texmf-dist/tex/generic/pgf/math/pgfmathfunctions.base.code.tex
INPUT d:/software/texlive/2026/texmf-dist/tex/generic/pgf/math/pgfmathfunctions.round.code.tex
INPUT d:/software/texlive/2026/texmf-dist/tex/generic/pgf/math/pgfmathfunctions.misc.code.tex
INPUT d:/software/texlive/2026/texmf-dist/tex/generic/pgf/math/pgfmathfunctions.integerarithmetics.code.tex
INPUT d:/software/texlive/2026/texmf-dist/tex/generic/pgf/math/pgfmathcalc.code.tex
INPUT d:/software/texlive/2026/texmf-dist/tex/generic/pgf/math/pgfmathfloat.code.tex
INPUT d:/software/texlive/2026/texmf-dist/tex/generic/pgf/math/pgfint.code.tex
INPUT d:/software/texlive/2026/texmf-dist/tex/generic/pgf/basiclayer/pgfcorepoints.code.tex
INPUT d:/software/texlive/2026/texmf-dist/tex/latex/tex-gyre/ot1qpl.fd
INPUT d:/software/texlive/2026/texmf-dist/tex/latex/tex-gyre/ot1qpl.fd
INPUT d:/software/texlive/2026/texmf-dist/tex/latex/tex-gyre/ot1qpl.fd
INPUT d:/software/texlive/2026/texmf-dist/fonts/map/fontname/texfonts.map
INPUT d:/software/texlive/2026/texmf-dist/fonts/tfm/public/tex-gyre/rm-qplr.tfm
INPUT d:/software/texlive/2026/texmf-dist/tex/generic/pgf/basiclayer/pgfcorepathconstruct.code.tex
INPUT d:/software/texlive/2026/texmf-dist/tex/generic/pgf/basiclayer/pgfcorepathusage.code.tex
INPUT d:/software/texlive/2026/texmf-dist/tex/generic/pgf/basiclayer/pgfcorescopes.code.tex
INPUT d:/software/texlive/2026/texmf-dist/tex/generic/pgf/basiclayer/pgfcoregraphicstate.code.tex
INPUT d:/software/texlive/2026/texmf-dist/tex/generic/pgf/basiclayer/pgfcoretransformations.code.tex
INPUT d:/software/texlive/2026/texmf-dist/tex/generic/pgf/basiclayer/pgfcorequick.code.tex
INPUT d:/software/texlive/2026/texmf-dist/tex/generic/pgf/basiclayer/pgfcoreobjects.code.tex
INPUT d:/software/texlive/2026/texmf-dist/tex/generic/pgf/basiclayer/pgfcorepathprocessing.code.tex
INPUT d:/software/texlive/2026/texmf-dist/tex/generic/pgf/basiclayer/pgfcorearrows.code.tex
INPUT d:/software/texlive/2026/texmf-dist/tex/generic/pgf/basiclayer/pgfcoreshade.code.tex
INPUT d:/software/texlive/2026/texmf-dist/tex/generic/pgf/basiclayer/pgfcoreimage.code.tex
INPUT d:/software/texlive/2026/texmf-dist/tex/generic/pgf/basiclayer/pgfcoreexternal.code.tex
INPUT d:/software/texlive/2026/texmf-dist/tex/generic/pgf/basiclayer/pgfcorelayers.code.tex
INPUT d:/software/texlive/2026/texmf-dist/tex/generic/pgf/basiclayer/pgfcoretransparency.code.tex
INPUT d:/software/texlive/2026/texmf-dist/tex/generic/pgf/basiclayer/pgfcorepatterns.code.tex
INPUT d:/software/texlive/2026/texmf-dist/tex/generic/pgf/basiclayer/pgfcorerdf.code.tex
INPUT d:/software/texlive/2026/texmf-dist/tex/generic/pgf/modules/pgfmoduleshapes.code.tex
INPUT d:/software/texlive/2026/texmf-dist/tex/generic/pgf/modules/pgfmoduleplot.code.tex
INPUT d:/software/texlive/2026/texmf-dist/tex/latex/pgf/compatibility/pgfcomp-version-0-65.sty
INPUT d:/software/texlive/2026/texmf-dist/tex/latex/pgf/compatibility/pgfcomp-version-0-65.sty
INPUT d:/software/texlive/2026/texmf-dist/tex/latex/pgf/compatibility/pgfcomp-version-1-18.sty
INPUT d:/software/texlive/2026/texmf-dist/tex/latex/pgf/compatibility/pgfcomp-version-1-18.sty
INPUT d:/software/texlive/2026/texmf-dist/tex/latex/pgf/utilities/pgffor.sty
INPUT d:/software/texlive/2026/texmf-dist/tex/latex/pgf/utilities/pgffor.sty
INPUT d:/software/texlive/2026/texmf-dist/tex/latex/pgf/utilities/pgfkeys.sty
INPUT d:/software/texlive/2026/texmf-dist/tex/latex/pgf/utilities/pgfkeys.sty
INPUT d:/software/texlive/2026/texmf-dist/tex/generic/pgf/utilities/pgfkeys.code.tex
INPUT d:/software/texlive/2026/texmf-dist/tex/generic/pgf/utilities/pgfkeys.code.tex
INPUT d:/software/texlive/2026/texmf-dist/tex/generic/pgf/utilities/pgfkeys.code.tex
INPUT d:/software/texlive/2026/texmf-dist/tex/latex/pgf/math/pgfmath.sty
INPUT d:/software/texlive/2026/texmf-dist/tex/latex/pgf/math/pgfmath.sty
INPUT d:/software/texlive/2026/texmf-dist/tex/generic/pgf/math/pgfmath.code.tex
INPUT d:/software/texlive/2026/texmf-dist/tex/generic/pgf/math/pgfmath.code.tex
INPUT d:/software/texlive/2026/texmf-dist/tex/generic/pgf/math/pgfmath.code.tex
INPUT d:/software/texlive/2026/texmf-dist/tex/generic/pgf/utilities/pgffor.code.tex
INPUT d:/software/texlive/2026/texmf-dist/tex/generic/pgf/utilities/pgffor.code.tex
INPUT d:/software/texlive/2026/texmf-dist/tex/generic/pgf/utilities/pgffor.code.tex
INPUT d:/software/texlive/2026/texmf-dist/tex/generic/pgf/frontendlayer/tikz/tikz.code.tex
INPUT d:/software/texlive/2026/texmf-dist/tex/generic/pgf/frontendlayer/tikz/tikz.code.tex
INPUT d:/software/texlive/2026/texmf-dist/tex/generic/pgf/frontendlayer/tikz/tikz.code.tex
INPUT d:/software/texlive/2026/texmf-dist/tex/generic/pgf/libraries/pgflibraryplothandlers.code.tex
INPUT d:/software/texlive/2026/texmf-dist/tex/generic/pgf/libraries/pgflibraryplothandlers.code.tex
INPUT d:/software/texlive/2026/texmf-dist/tex/generic/pgf/modules/pgfmodulematrix.code.tex
INPUT d:/software/texlive/2026/texmf-dist/tex/generic/pgf/frontendlayer/tikz/libraries/tikzlibrarytopaths.code.tex
INPUT d:/software/texlive/2026/texmf-dist/tex/generic/pgf/frontendlayer/tikz/libraries/tikzlibrarytopaths.code.tex
INPUT d:/software/texlive/2026/texmf-dist/tex/latex/tikz-qtree/pgftree.sty
INPUT d:/software/texlive/2026/texmf-dist/tex/latex/tikz-qtree/pgftree.sty
INPUT d:/software/texlive/2026/texmf-dist/tex/latex/tikz-qtree/pgfsubpic.sty
INPUT d:/software/texlive/2026/texmf-dist/tex/latex/tikz-qtree/pgfsubpic.sty
INPUT d:/software/texlive/2026/texmf-dist/tex/latex/tikz-qtree/pgfsubpic.tex
INPUT d:/software/texlive/2026/texmf-dist/tex/latex/tikz-qtree/pgfsubpic.tex
INPUT d:/software/texlive/2026/texmf-dist/tex/latex/tikz-qtree/pgfsubpic.tex
INPUT d:/software/texlive/2026/texmf-dist/tex/latex/tikz-qtree/pgftree.tex
INPUT d:/software/texlive/2026/texmf-dist/tex/latex/tikz-qtree/pgftree.tex
INPUT d:/software/texlive/2026/texmf-dist/tex/latex/tikz-qtree/pgftree.tex
INPUT d:/software/texlive/2026/texmf-dist/tex/latex/tikz-qtree/tikz-qtree.tex
INPUT d:/software/texlive/2026/texmf-dist/tex/latex/tikz-qtree/tikz-qtree.tex
INPUT d:/software/texlive/2026/texmf-dist/tex/latex/tikz-qtree/tikz-qtree.tex
INPUT d:/software/texlive/2026/texmf-dist/tex/latex/tcolorbox/tcolorbox.sty
INPUT d:/software/texlive/2026/texmf-dist/tex/latex/tcolorbox/tcolorbox.sty
INPUT d:/software/texlive/2026/texmf-dist/tex/latex/tools/verbatim.sty
INPUT d:/software/texlive/2026/texmf-dist/tex/latex/tools/verbatim.sty
INPUT d:/software/texlive/2026/texmf-dist/tex/latex/environ/environ.sty
INPUT d:/software/texlive/2026/texmf-dist/tex/latex/environ/environ.sty
INPUT d:/software/texlive/2026/texmf-dist/tex/latex/trimspaces/trimspaces.sty
INPUT d:/software/texlive/2026/texmf-dist/tex/latex/trimspaces/trimspaces.sty
INPUT d:/software/texlive/2026/texmf-dist/tex/generic/pgf/libraries/pgflibraryarrows.meta.code.tex
INPUT d:/software/texlive/2026/texmf-dist/tex/generic/pgf/libraries/pgflibraryarrows.meta.code.tex
INPUT d:/software/texlive/2026/texmf-dist/tex/generic/pgf/libraries/pgflibraryarrows.meta.code.tex
INPUT d:/software/texlive/2026/texmf-dist/tex/generic/pgf/frontendlayer/tikz/libraries/tikzlibrarybending.code.tex
INPUT d:/software/texlive/2026/texmf-dist/tex/generic/pgf/frontendlayer/tikz/libraries/tikzlibrarybending.code.tex
INPUT d:/software/texlive/2026/texmf-dist/tex/generic/pgf/modules/pgfmodulebending.code.tex
INPUT d:/software/texlive/2026/texmf-dist/tex/generic/pgf/modules/pgfmodulenonlineartransformations.code.tex
INPUT d:/software/texlive/2026/texmf-dist/tex/generic/pgf/libraries/pgflibrarycurvilinear.code.tex
INPUT d:/software/texlive/2026/texmf-dist/tex/generic/pgf/libraries/pgflibrarycurvilinear.code.tex
INPUT d:/software/texlive/2026/texmf-dist/tex/generic/pgf/frontendlayer/tikz/libraries/tikzlibrarypositioning.code.tex
INPUT d:/software/texlive/2026/texmf-dist/tex/generic/pgf/frontendlayer/tikz/libraries/tikzlibrarypositioning.code.tex
INPUT d:/software/texlive/2026/texmf-dist/tex/generic/pgf/frontendlayer/tikz/libraries/tikzlibraryangles.code.tex
INPUT d:/software/texlive/2026/texmf-dist/tex/generic/pgf/frontendlayer/tikz/libraries/tikzlibraryangles.code.tex
INPUT d:/software/texlive/2026/texmf-dist/tex/generic/pgf/frontendlayer/tikz/libraries/tikzlibraryquotes.code.tex
INPUT d:/software/texlive/2026/texmf-dist/tex/generic/pgf/frontendlayer/tikz/libraries/tikzlibraryquotes.code.tex
INPUT d:/software/texlive/2026/texmf-dist/tex/generic/pgf/frontendlayer/tikz/libraries/tikzlibraryfit.code.tex
INPUT d:/software/texlive/2026/texmf-dist/tex/generic/pgf/frontendlayer/tikz/libraries/tikzlibraryfit.code.tex
INPUT d:/software/texlive/2026/texmf-dist/tex/generic/pgf/frontendlayer/tikz/libraries/tikzlibrarybackgrounds.code.tex
INPUT d:/software/texlive/2026/texmf-dist/tex/generic/pgf/frontendlayer/tikz/libraries/tikzlibrarybackgrounds.code.tex
INPUT d:/software/texlive/2026/texmf-dist/tex/generic/pgf/frontendlayer/tikz/libraries/tikzlibraryshadows.code.tex
INPUT d:/software/texlive/2026/texmf-dist/tex/generic/pgf/frontendlayer/tikz/libraries/tikzlibraryshadows.code.tex
INPUT d:/software/texlive/2026/texmf-dist/tex/generic/pgf/frontendlayer/tikz/libraries/tikzlibraryfadings.code.tex
INPUT d:/software/texlive/2026/texmf-dist/tex/generic/pgf/frontendlayer/tikz/libraries/tikzlibraryfadings.code.tex
INPUT d:/software/texlive/2026/texmf-dist/tex/generic/pgf/libraries/pgflibraryfadings.code.tex
INPUT d:/software/texlive/2026/texmf-dist/tex/generic/pgf/libraries/pgflibraryfadings.code.tex
OUTPUT rl-introduction.pdf
INPUT d:/software/texlive/2026/texmf-dist/tex/generic/pgf/frontendlayer/tikz/libraries/tikzlibrarycalc.code.tex
INPUT d:/software/texlive/2026/texmf-dist/tex/generic/pgf/frontendlayer/tikz/libraries/tikzlibrarycalc.code.tex
INPUT d:/software/texlive/2026/texmf-dist/tex/generic/pgf/frontendlayer/tikz/libraries/tikzlibrarymatrix.code.tex
INPUT d:/software/texlive/2026/texmf-dist/tex/generic/pgf/frontendlayer/tikz/libraries/tikzlibrarymatrix.code.tex
INPUT d:/software/texlive/2026/texmf-dist/tex/latex/pgfplots/pgfplots.sty
INPUT d:/software/texlive/2026/texmf-dist/tex/latex/pgfplots/pgfplots.sty
INPUT d:/software/texlive/2026/texmf-dist/tex/generic/pgfplots/pgfplots.revision.tex
INPUT d:/software/texlive/2026/texmf-dist/tex/generic/pgfplots/pgfplots.revision.tex
INPUT d:/software/texlive/2026/texmf-dist/tex/generic/pgfplots/pgfplots.revision.tex
INPUT d:/software/texlive/2026/texmf-dist/tex/generic/pgf/systemlayer/pgfsys-luatex.def
INPUT d:/software/texlive/2026/texmf-dist/tex/generic/pgf/systemlayer/pgfsys-luatex.def
INPUT d:/software/texlive/2026/texmf-dist/tex/generic/pgfplots/pgfplots.code.tex
INPUT d:/software/texlive/2026/texmf-dist/tex/generic/pgfplots/pgfplotscore.code.tex
INPUT d:/software/texlive/2026/texmf-dist/tex/generic/pgfplots/sys/pgfplotssysgeneric.code.tex
INPUT d:/software/texlive/2026/texmf-dist/tex/generic/pgfplots/libs/pgfplotslibrary.code.tex
INPUT d:/software/texlive/2026/texmf-dist/tex/generic/pgfplots/oldpgfcompatib/pgfplotsoldpgfsupp_loader.code.tex
INPUT d:/software/texlive/2026/texmf-dist/tex/generic/pgf/libraries/pgflibraryfpu.code.tex
INPUT d:/software/texlive/2026/texmf-dist/tex/generic/pgf/libraries/pgflibraryfpu.code.tex
INPUT d:/software/texlive/2026/texmf-dist/tex/generic/pgf/libraries/pgflibraryfpu.code.tex
INPUT d:/software/texlive/2026/texmf-dist/tex/generic/pgfplots/oldpgfcompatib/pgfplotsoldpgfsupp_pgfutil-common-lists.tex
INPUT d:/software/texlive/2026/texmf-dist/tex/generic/pgfplots/util/pgfplotsutil.code.tex
INPUT d:/software/texlive/2026/texmf-dist/tex/generic/pgfplots/liststructure/pgfplotsliststructure.code.tex
INPUT d:/software/texlive/2026/texmf-dist/tex/generic/pgfplots/liststructure/pgfplotsliststructureext.code.tex
INPUT d:/software/texlive/2026/texmf-dist/tex/generic/pgfplots/liststructure/pgfplotsarray.code.tex
INPUT d:/software/texlive/2026/texmf-dist/tex/generic/pgfplots/liststructure/pgfplotsmatrix.code.tex
INPUT d:/software/texlive/2026/texmf-dist/tex/generic/pgfplots/numtable/pgfplotstableshared.code.tex
INPUT d:/software/texlive/2026/texmf-dist/tex/generic/pgfplots/liststructure/pgfplotsdeque.code.tex
INPUT d:/software/texlive/2026/texmf-dist/tex/generic/pgfplots/util/pgfplotsbinary.code.tex
INPUT d:/software/texlive/2026/texmf-dist/tex/generic/pgfplots/util/pgfplotsbinary.data.code.tex
INPUT d:/software/texlive/2026/texmf-dist/tex/generic/pgfplots/util/pgfplotsutil.verb.code.tex
INPUT d:/software/texlive/2026/texmf-dist/tex/generic/pgfplots/libs/pgflibrarypgfplots.surfshading.code.tex
INPUT d:/software/texlive/2026/texmf-dist/tex/generic/pgfplots/sys/pgflibrarypgfplots.surfshading.pgfsys-pdftex.def
INPUT d:/software/texlive/2026/texmf-dist/tex/generic/pgfplots/sys/pgflibrarypgfplots.surfshading.pgfsys-pdftex.def
INPUT d:/software/texlive/2026/texmf-dist/tex/generic/pgfplots/util/pgfplotscolormap.code.tex
INPUT d:/software/texlive/2026/texmf-dist/tex/generic/pgfplots/util/pgfplotscolor.code.tex
INPUT d:/software/texlive/2026/texmf-dist/tex/generic/pgfplots/pgfplotsstackedplots.code.tex
INPUT d:/software/texlive/2026/texmf-dist/tex/generic/pgfplots/pgfplotsplothandlers.code.tex
INPUT d:/software/texlive/2026/texmf-dist/tex/generic/pgfplots/pgfplotsmeshplothandler.code.tex
INPUT d:/software/texlive/2026/texmf-dist/tex/generic/pgfplots/pgfplotsmeshplotimage.code.tex
INPUT d:/software/texlive/2026/texmf-dist/tex/generic/pgfplots/pgfplots.scaling.code.tex
INPUT d:/software/texlive/2026/texmf-dist/tex/generic/pgfplots/pgfplotscoordprocessing.code.tex
INPUT d:/software/texlive/2026/texmf-dist/tex/generic/pgfplots/pgfplots.errorbars.code.tex
INPUT d:/software/texlive/2026/texmf-dist/tex/generic/pgfplots/pgfplots.markers.code.tex
INPUT d:/software/texlive/2026/texmf-dist/tex/generic/pgfplots/pgfplotsticks.code.tex
INPUT d:/software/texlive/2026/texmf-dist/tex/generic/pgfplots/pgfplots.paths.code.tex
INPUT d:/software/texlive/2026/texmf-dist/tex/generic/pgf/frontendlayer/tikz/libraries/tikzlibrarydecorations.code.tex
INPUT d:/software/texlive/2026/texmf-dist/tex/generic/pgf/frontendlayer/tikz/libraries/tikzlibrarydecorations.code.tex
INPUT d:/software/texlive/2026/texmf-dist/tex/generic/pgf/modules/pgfmoduledecorations.code.tex
INPUT d:/software/texlive/2026/texmf-dist/tex/generic/pgf/frontendlayer/tikz/libraries/tikzlibrarydecorations.pathmorphing.code.tex
INPUT d:/software/texlive/2026/texmf-dist/tex/generic/pgf/frontendlayer/tikz/libraries/tikzlibrarydecorations.pathmorphing.code.tex
INPUT d:/software/texlive/2026/texmf-dist/tex/generic/pgf/libraries/decorations/pgflibrarydecorations.pathmorphing.code.tex
INPUT d:/software/texlive/2026/texmf-dist/tex/generic/pgf/libraries/decorations/pgflibrarydecorations.pathmorphing.code.tex
INPUT d:/software/texlive/2026/texmf-dist/tex/generic/pgf/frontendlayer/tikz/libraries/tikzlibrarydecorations.pathreplacing.code.tex
INPUT d:/software/texlive/2026/texmf-dist/tex/generic/pgf/frontendlayer/tikz/libraries/tikzlibrarydecorations.pathreplacing.code.tex
INPUT d:/software/texlive/2026/texmf-dist/tex/generic/pgf/libraries/decorations/pgflibrarydecorations.pathreplacing.code.tex
INPUT d:/software/texlive/2026/texmf-dist/tex/generic/pgf/libraries/decorations/pgflibrarydecorations.pathreplacing.code.tex
INPUT d:/software/texlive/2026/texmf-dist/tex/generic/pgfplots/libs/tikzlibrarypgfplots.contourlua.code.tex
INPUT d:/software/texlive/2026/texmf-dist/tex/generic/pgfplots/libs/tikzlibrarypgfplots.contourlua.code.tex
INPUT d:/software/texlive/2026/texmf-dist/tex/generic/pgf/frontendlayer/tikz/libraries/tikzlibraryplotmarks.code.tex
INPUT d:/software/texlive/2026/texmf-dist/tex/generic/pgf/frontendlayer/tikz/libraries/tikzlibraryplotmarks.code.tex
INPUT d:/software/texlive/2026/texmf-dist/tex/generic/pgf/libraries/pgflibraryplotmarks.code.tex
INPUT d:/software/texlive/2026/texmf-dist/tex/generic/pgf/libraries/pgflibraryplotmarks.code.tex
INPUT d:/software/texlive/2026/texmf-dist/tex/generic/soul/soul.sty
INPUT d:/software/texlive/2026/texmf-dist/tex/generic/soul/soul.sty
INPUT d:/software/texlive/2026/texmf-dist/tex/generic/soul/soul-ori.sty
INPUT d:/software/texlive/2026/texmf-dist/tex/generic/soul/soul-ori.sty
INPUT d:/software/texlive/2026/texmf-dist/fonts/tfm/jknappen/ec/ectt1000.tfm
INPUT d:/software/texlive/2026/texmf-dist/tex/generic/etexcmds/etexcmds.sty
INPUT d:/software/texlive/2026/texmf-dist/tex/generic/etexcmds/etexcmds.sty
INPUT d:/software/texlive/2026/texmf-dist/tex/latex/makecell/makecell.sty
INPUT d:/software/texlive/2026/texmf-dist/tex/latex/makecell/makecell.sty
INPUT d:/software/texlive/2026/texmf-dist/tex/latex/tools/array.sty
INPUT d:/software/texlive/2026/texmf-dist/tex/latex/tools/array.sty
INPUT d:/software/texlive/2026/texmf-dist/tex/generic/ulem/ulem.sty
INPUT d:/software/texlive/2026/texmf-dist/tex/generic/ulem/ulem.sty
INPUT d:/software/texlive/2026/texmf-dist/fonts/tfm/public/latex-fonts/lasy6.tfm
INPUT d:/software/texlive/2026/texmf-dist/tex/latex/multirow/multirow.sty
INPUT d:/software/texlive/2026/texmf-dist/tex/latex/multirow/multirow.sty
INPUT d:/software/texlive/2026/texmf-dist/tex/latex/setspace/setspace.sty
INPUT d:/software/texlive/2026/texmf-dist/tex/latex/setspace/setspace.sty
INPUT d:/software/texlive/2026/texmf-dist/tex/latex/varwidth/varwidth.sty
INPUT d:/software/texlive/2026/texmf-dist/tex/latex/varwidth/varwidth.sty
INPUT d:/software/texlive/2026/texmf-dist/tex/latex/amsmath/amsmath.sty
INPUT d:/software/texlive/2026/texmf-dist/tex/latex/amsmath/amsmath.sty
INPUT d:/software/texlive/2026/texmf-dist/tex/latex/amsmath/amsopn.sty
INPUT d:/software/texlive/2026/texmf-dist/tex/latex/amsmath/amstext.sty
INPUT d:/software/texlive/2026/texmf-dist/tex/latex/amsmath/amstext.sty
INPUT d:/software/texlive/2026/texmf-dist/tex/latex/amsmath/amsgen.sty
INPUT d:/software/texlive/2026/texmf-dist/tex/latex/amsmath/amsgen.sty
INPUT d:/software/texlive/2026/texmf-dist/tex/latex/amsmath/amsbsy.sty
INPUT d:/software/texlive/2026/texmf-dist/tex/latex/amsmath/amsbsy.sty
INPUT d:/software/texlive/2026/texmf-dist/tex/latex/amsmath/amsopn.sty
INPUT d:/software/texlive/2026/texmf-dist/tex/latex/amsfonts/amssymb.sty
INPUT d:/software/texlive/2026/texmf-dist/tex/latex/amsfonts/amssymb.sty
INPUT d:/software/texlive/2026/texmf-dist/tex/latex/amsfonts/amsfonts.sty
INPUT d:/software/texlive/2026/texmf-dist/tex/latex/amsfonts/amsfonts.sty
INPUT d:/software/texlive/2026/texmf-dist/tex/latex/psnfss/ot1ppl.fd
INPUT d:/software/texlive/2026/texmf-dist/tex/latex/psnfss/ot1ppl.fd
INPUT d:/software/texlive/2026/texmf-dist/tex/latex/psnfss/ot1ppl.fd
INPUT d:/software/texlive/2026/texmf-dist/fonts/tfm/adobe/palatino/pplr7t.tfm
INPUT ./rl-introduction.aux
INPUT ./rl-introduction.aux
INPUT rl-introduction.aux
OUTPUT rl-introduction.aux
INPUT d:/software/texlive/2026/texmf-dist/tex/latex/upquote/upquote.sty
INPUT d:/software/texlive/2026/texmf-dist/tex/latex/upquote/upquote.sty
INPUT d:/software/texlive/2026/texmf-dist/tex/context/base/mkii/supp-pdf.mkii
INPUT d:/software/texlive/2026/texmf-dist/tex/context/base/mkii/supp-pdf.mkii
INPUT d:/software/texlive/2026/texmf-dist/tex/context/base/mkii/supp-pdf.mkii
INPUT d:/software/texlive/2026/texmf-dist/tex/latex/epstopdf-pkg/epstopdf-base.sty
INPUT d:/software/texlive/2026/texmf-dist/tex/latex/epstopdf-pkg/epstopdf-base.sty
INPUT d:/software/texlive/2026/texmf-dist/tex/latex/latexconfig/epstopdf-sys.cfg
INPUT d:/software/texlive/2026/texmf-dist/tex/latex/latexconfig/epstopdf-sys.cfg
INPUT d:/software/texlive/2026/texmf-dist/tex/latex/latexconfig/epstopdf-sys.cfg
INPUT d:/software/texlive/2026/texmf-dist/tex/latex/microtype/mt-ppl.cfg
INPUT d:/software/texlive/2026/texmf-dist/tex/latex/microtype/mt-ppl.cfg
INPUT d:/software/texlive/2026/texmf-dist/tex/latex/microtype/mt-ppl.cfg
INPUT d:/software/texlive/2026/texmf-dist/tex/latex/graphics/color.sty
INPUT ./rl-introduction.out
INPUT ./rl-introduction.out
INPUT rl-introduction.out
INPUT rl-introduction.out
INPUT ./rl-introduction.out
INPUT ./rl-introduction.out
OUTPUT rl-introduction.out
INPUT d:/software/texlive/2026/texmf-dist/fonts/tfm/adobe/palatino/pplr7t.tfm
INPUT d:/software/texlive/2026/texmf-dist/fonts/tfm/adobe/palatino/pplb7t.tfm
INPUT d:/software/texlive/2026/texmf-dist/fonts/tfm/adobe/palatino/pplb7t.tfm
INPUT d:/software/texlive/2026/texmf-dist/tex/latex/inconsolata/ot1zi4.fd
INPUT d:/software/texlive/2026/texmf-dist/tex/latex/inconsolata/ot1zi4.fd
INPUT d:/software/texlive/2026/texmf-dist/tex/latex/inconsolata/ot1zi4.fd
INPUT d:/software/texlive/2026/texmf-dist/fonts/tfm/public/inconsolata/ot1-zi4r-0.tfm
INPUT d:/software/texlive/2026/texmf-dist/fonts/tfm/adobe/palatino/pplr7t.tfm
INPUT d:/software/texlive/2026/texmf-dist/fonts/tfm/adobe/palatino/pplb7t.tfm
INPUT ./rl-introduction.toc
INPUT ./rl-introduction.toc
INPUT rl-introduction.toc
INPUT d:/software/texlive/2026/texmf-dist/fonts/tfm/adobe/palatino/pplr7t.tfm
INPUT d:/software/texlive/2026/texmf-dist/fonts/tfm/adobe/palatino/pplr7t.tfm
INPUT d:/software/texlive/2026/texmf-dist/tex/latex/psnfss/omlzplm.fd
INPUT d:/software/texlive/2026/texmf-dist/tex/latex/psnfss/omlzplm.fd
INPUT d:/software/texlive/2026/texmf-dist/tex/latex/psnfss/omlzplm.fd
INPUT d:/software/texlive/2026/texmf-dist/fonts/tfm/public/mathpazo/zplmr7m.tfm
INPUT d:/software/texlive/2026/texmf-dist/fonts/tfm/public/mathpazo/zplmr7m.tfm
INPUT d:/software/texlive/2026/texmf-dist/fonts/tfm/public/mathpazo/zplmr7m.tfm
INPUT d:/software/texlive/2026/texmf-dist/tex/latex/psnfss/omszplm.fd
INPUT d:/software/texlive/2026/texmf-dist/tex/latex/psnfss/omszplm.fd
INPUT d:/software/texlive/2026/texmf-dist/tex/latex/psnfss/omszplm.fd
INPUT d:/software/texlive/2026/texmf-dist/fonts/tfm/public/mathpazo/zplmr7y.tfm
INPUT d:/software/texlive/2026/texmf-dist/fonts/tfm/public/mathpazo/zplmr7y.tfm
INPUT d:/software/texlive/2026/texmf-dist/fonts/tfm/public/mathpazo/zplmr7y.tfm
INPUT d:/software/texlive/2026/texmf-dist/tex/latex/psnfss/omxzplm.fd
INPUT d:/software/texlive/2026/texmf-dist/tex/latex/psnfss/omxzplm.fd
INPUT d:/software/texlive/2026/texmf-dist/tex/latex/psnfss/omxzplm.fd
INPUT d:/software/texlive/2026/texmf-dist/fonts/tfm/public/mathpazo/zplmr7v.tfm
INPUT d:/software/texlive/2026/texmf-dist/fonts/tfm/public/mathpazo/zplmr7v.tfm
INPUT d:/software/texlive/2026/texmf-dist/fonts/tfm/public/mathpazo/zplmr7v.tfm
INPUT d:/software/texlive/2026/texmf-dist/tex/latex/psnfss/ot1zplm.fd
INPUT d:/software/texlive/2026/texmf-dist/tex/latex/psnfss/ot1zplm.fd
INPUT d:/software/texlive/2026/texmf-dist/tex/latex/psnfss/ot1zplm.fd
INPUT d:/software/texlive/2026/texmf-dist/fonts/tfm/public/mathpazo/zplmr7t.tfm
INPUT d:/software/texlive/2026/texmf-dist/fonts/tfm/public/mathpazo/zplmr7t.tfm
INPUT d:/software/texlive/2026/texmf-dist/fonts/tfm/public/mathpazo/zplmr7t.tfm
INPUT d:/software/texlive/2026/texmf-dist/fonts/tfm/public/amsfonts/symbols/msam10.tfm
INPUT d:/software/texlive/2026/texmf-dist/tex/latex/microtype/mt-msa.cfg
INPUT d:/software/texlive/2026/texmf-dist/tex/latex/microtype/mt-msa.cfg
INPUT d:/software/texlive/2026/texmf-dist/tex/latex/microtype/mt-msa.cfg
INPUT d:/software/texlive/2026/texmf-dist/fonts/tfm/public/amsfonts/symbols/msam10.tfm
INPUT d:/software/texlive/2026/texmf-dist/fonts/tfm/public/amsfonts/symbols/msam10.tfm
INPUT d:/software/texlive/2026/texmf-dist/fonts/tfm/public/amsfonts/symbols/msbm10.tfm
INPUT d:/software/texlive/2026/texmf-dist/tex/latex/microtype/mt-msb.cfg
INPUT d:/software/texlive/2026/texmf-dist/tex/latex/microtype/mt-msb.cfg
INPUT d:/software/texlive/2026/texmf-dist/tex/latex/microtype/mt-msb.cfg
INPUT d:/software/texlive/2026/texmf-dist/fonts/tfm/public/amsfonts/symbols/msbm10.tfm
INPUT d:/software/texlive/2026/texmf-dist/fonts/tfm/public/amsfonts/symbols/msbm10.tfm
INPUT d:/software/texlive/2026/texmf-dist/tex/latex/amsfonts/ueuex.fd
INPUT d:/software/texlive/2026/texmf-dist/tex/latex/amsfonts/ueuex.fd
INPUT d:/software/texlive/2026/texmf-dist/tex/latex/amsfonts/ueuex.fd
INPUT d:/software/texlive/2026/texmf-dist/fonts/tfm/public/amsfonts/euler/euex10.tfm
INPUT d:/software/texlive/2026/texmf-dist/fonts/tfm/public/amsfonts/euler/euex8.tfm
INPUT d:/software/texlive/2026/texmf-dist/fonts/tfm/public/amsfonts/euler/euex7.tfm
INPUT d:/software/texlive/2026/texmf-dist/fonts/vf/adobe/palatino/pplb7t.vf
INPUT d:/software/texlive/2026/texmf-dist/fonts/tfm/adobe/palatino/pplb8r.tfm
INPUT d:/software/texlive/2026/texmf-var/fonts/map/pdftex/updmap/pdftex.map
INPUT d:/software/texlive/2026/texmf-dist/fonts/enc/dvips/base/8r.enc
INPUT d:/software/texlive/2026/texmf-dist/fonts/vf/adobe/palatino/pplb7t.vf
INPUT d:/software/texlive/2026/texmf-dist/fonts/tfm/adobe/palatino/pplb8r.tfm
INPUT d:/software/texlive/2026/texmf-dist/fonts/vf/adobe/palatino/pplr7t.vf
INPUT d:/software/texlive/2026/texmf-dist/fonts/tfm/adobe/palatino/pplr8r.tfm
INPUT d:/software/texlive/2026/texmf-dist/fonts/enc/dvips/inconsolata/i4-ot1-0.enc
INPUT d:/software/texlive/2026/texmf-dist/fonts/vf/adobe/palatino/pplb7t.vf
INPUT d:/software/texlive/2026/texmf-dist/fonts/tfm/adobe/palatino/pplb8r.tfm
OUTPUT rl-introduction.toc
INPUT ./section2/section2.tex
INPUT ./section2/section2.tex
INPUT ./section2/section2.tex
INPUT ./section2/section2.tex
INPUT section2/section2.tex
INPUT d:/software/texlive/2026/texmf-dist/fonts/tfm/adobe/palatino/pplr7t.tfm
INPUT d:/software/texlive/2026/texmf-dist/fonts/tfm/adobe/palatino/pplr7t.tfm
INPUT d:/software/texlive/2026/texmf-dist/fonts/tfm/adobe/palatino/pplr7t.tfm
INPUT d:/software/texlive/2026/texmf-dist/fonts/tfm/public/mathpazo/zplmr7m.tfm
INPUT d:/software/texlive/2026/texmf-dist/fonts/tfm/public/mathpazo/zplmr7m.tfm
INPUT d:/software/texlive/2026/texmf-dist/fonts/tfm/public/mathpazo/zplmr7m.tfm
INPUT d:/software/texlive/2026/texmf-dist/fonts/tfm/public/mathpazo/zplmr7y.tfm
INPUT d:/software/texlive/2026/texmf-dist/fonts/tfm/public/mathpazo/zplmr7y.tfm
INPUT d:/software/texlive/2026/texmf-dist/fonts/tfm/public/mathpazo/zplmr7y.tfm
INPUT d:/software/texlive/2026/texmf-dist/fonts/tfm/public/mathpazo/zplmr7v.tfm
INPUT d:/software/texlive/2026/texmf-dist/fonts/tfm/public/mathpazo/zplmr7v.tfm
INPUT d:/software/texlive/2026/texmf-dist/fonts/tfm/public/mathpazo/zplmr7v.tfm
INPUT d:/software/texlive/2026/texmf-dist/fonts/tfm/public/mathpazo/zplmr7t.tfm
INPUT d:/software/texlive/2026/texmf-dist/fonts/tfm/public/mathpazo/zplmr7t.tfm
INPUT d:/software/texlive/2026/texmf-dist/fonts/tfm/public/mathpazo/zplmr7t.tfm
INPUT d:/software/texlive/2026/texmf-dist/fonts/tfm/public/amsfonts/symbols/msam10.tfm
INPUT d:/software/texlive/2026/texmf-dist/fonts/tfm/public/amsfonts/symbols/msam10.tfm
INPUT d:/software/texlive/2026/texmf-dist/fonts/tfm/public/amsfonts/symbols/msam10.tfm
INPUT d:/software/texlive/2026/texmf-dist/fonts/tfm/public/amsfonts/symbols/msbm10.tfm
INPUT d:/software/texlive/2026/texmf-dist/fonts/tfm/public/amsfonts/symbols/msbm10.tfm
INPUT d:/software/texlive/2026/texmf-dist/fonts/tfm/public/amsfonts/symbols/msbm10.tfm
INPUT d:/software/texlive/2026/texmf-dist/fonts/tfm/public/amsfonts/euler/euex9.tfm
INPUT d:/software/texlive/2026/texmf-dist/fonts/tfm/public/amsfonts/euler/euex7.tfm
INPUT d:/software/texlive/2026/texmf-dist/fonts/tfm/public/amsfonts/euler/euex7.tfm
INPUT d:/software/texlive/2026/texmf-dist/fonts/tfm/adobe/palatino/pplri7t.tfm
INPUT d:/software/texlive/2026/texmf-dist/fonts/tfm/public/mathpazo/zplmb7t.tfm
INPUT d:/software/texlive/2026/texmf-dist/fonts/tfm/public/mathpazo/zplmb7t.tfm
INPUT d:/software/texlive/2026/texmf-dist/fonts/tfm/public/mathpazo/zplmb7t.tfm
INPUT d:/software/texlive/2026/texmf-dist/fonts/tfm/adobe/palatino/pplbi7t.tfm
INPUT ./section2/Figures/rl_framework.tex
INPUT ./section2/Figures/rl_framework.tex
INPUT ./section2/Figures/rl_framework.tex
INPUT ./section2/Figures/rl_framework.tex
INPUT section2/Figures/rl_framework.tex
INPUT d:/software/texlive/2026/texmf-dist/fonts/tfm/public/mathpazo/zplmr7m.tfm
INPUT d:/software/texlive/2026/texmf-dist/fonts/tfm/public/mathpazo/zplmr7y.tfm
INPUT d:/software/texlive/2026/texmf-dist/fonts/tfm/public/mathpazo/zplmr7v.tfm
INPUT d:/software/texlive/2026/texmf-dist/fonts/tfm/public/mathpazo/zplmb7t.tfm
INPUT d:/software/texlive/2026/texmf-dist/fonts/tfm/public/mathpazo/zplmb7t.tfm
INPUT d:/software/texlive/2026/texmf-dist/fonts/tfm/public/mathpazo/zplmb7t.tfm
INPUT d:/software/texlive/2026/texmf-dist/fonts/tfm/public/mathpazo/zplmr7t.tfm
INPUT d:/software/texlive/2026/texmf-dist/fonts/tfm/public/amsfonts/symbols/msam10.tfm
INPUT d:/software/texlive/2026/texmf-dist/fonts/tfm/public/amsfonts/symbols/msbm10.tfm
INPUT d:/software/texlive/2026/texmf-dist/fonts/tfm/public/amsfonts/euler/euex10.tfm
INPUT d:/software/texlive/2026/texmf-dist/tex/latex/psnfss/ufplmbb.fd
INPUT d:/software/texlive/2026/texmf-dist/tex/latex/psnfss/ufplmbb.fd
INPUT d:/software/texlive/2026/texmf-dist/tex/latex/psnfss/ufplmbb.fd
INPUT d:/software/texlive/2026/texmf-dist/fonts/tfm/public/mathpazo/fplmbb.tfm
INPUT d:/software/texlive/2026/texmf-dist/fonts/tfm/public/mathpazo/fplmbb.tfm
INPUT d:/software/texlive/2026/texmf-dist/fonts/tfm/public/mathpazo/fplmbb.tfm
INPUT d:/software/texlive/2026/texmf-dist/fonts/vf/adobe/palatino/pplr7t.vf
INPUT d:/software/texlive/2026/texmf-dist/fonts/tfm/adobe/palatino/pplr8r.tfm
INPUT d:/software/texlive/2026/texmf-dist/fonts/vf/public/mathpazo/zplmb7t.vf
INPUT d:/software/texlive/2026/texmf-dist/fonts/tfm/public/mathpazo/fplmb.tfm
INPUT d:/software/texlive/2026/texmf-dist/fonts/tfm/public/cm/cmbx10.tfm
INPUT d:/software/texlive/2026/texmf-dist/fonts/vf/public/mathpazo/zplmr7t.vf
INPUT d:/software/texlive/2026/texmf-dist/fonts/tfm/public/mathpazo/fplmr.tfm
INPUT d:/software/texlive/2026/texmf-dist/fonts/tfm/public/cm/cmr10.tfm
INPUT d:/software/texlive/2026/texmf-dist/fonts/vf/public/mathpazo/zplmr7m.vf
INPUT d:/software/texlive/2026/texmf-dist/fonts/tfm/public/mathpazo/fplmri.tfm
INPUT d:/software/texlive/2026/texmf-dist/fonts/tfm/public/cm/cmmi10.tfm
INPUT d:/software/texlive/2026/texmf-dist/fonts/tfm/adobe/palatino/pplri8r.tfm
INPUT d:/software/texlive/2026/texmf-dist/fonts/vf/public/mathpazo/zplmr7m.vf
INPUT d:/software/texlive/2026/texmf-dist/fonts/tfm/public/mathpazo/fplmri.tfm
INPUT d:/software/texlive/2026/texmf-dist/fonts/tfm/public/cm/cmmi10.tfm
INPUT d:/software/texlive/2026/texmf-dist/fonts/tfm/adobe/palatino/pplri8r.tfm
INPUT d:/software/texlive/2026/texmf-dist/fonts/vf/public/mathpazo/zplmr7t.vf
INPUT d:/software/texlive/2026/texmf-dist/fonts/tfm/public/mathpazo/fplmr.tfm
INPUT d:/software/texlive/2026/texmf-dist/fonts/tfm/public/cm/cmr10.tfm
INPUT d:/software/texlive/2026/texmf-dist/fonts/vf/public/mathpazo/zplmr7t.vf
INPUT d:/software/texlive/2026/texmf-dist/fonts/tfm/public/mathpazo/fplmr.tfm
INPUT d:/software/texlive/2026/texmf-dist/fonts/tfm/adobe/palatino/pplr8r.tfm
INPUT d:/software/texlive/2026/texmf-dist/fonts/tfm/public/cm/cmr10.tfm
INPUT d:/software/texlive/2026/texmf-dist/fonts/vf/public/mathpazo/zplmr7v.vf
INPUT d:/software/texlive/2026/texmf-dist/fonts/tfm/public/cm/cmex10.tfm
INPUT d:/software/texlive/2026/texmf-dist/fonts/tfm/public/cm/cmex10.tfm
INPUT d:/software/texlive/2026/texmf-dist/fonts/tfm/public/mathpazo/fplmr.tfm
INPUT d:/software/texlive/2026/texmf-dist/fonts/vf/public/mathpazo/zplmb7t.vf
INPUT d:/software/texlive/2026/texmf-dist/fonts/tfm/public/mathpazo/fplmb.tfm
INPUT d:/software/texlive/2026/texmf-dist/fonts/tfm/adobe/palatino/pplb8r.tfm
INPUT d:/software/texlive/2026/texmf-dist/fonts/tfm/public/cm/cmbx10.tfm
INPUT d:/software/texlive/2026/texmf-dist/fonts/vf/public/mathpazo/zplmr7y.vf
INPUT d:/software/texlive/2026/texmf-dist/fonts/tfm/public/cm/cmsy10.tfm
INPUT d:/software/texlive/2026/texmf-dist/fonts/vf/public/mathpazo/zplmr7y.vf
INPUT d:/software/texlive/2026/texmf-dist/fonts/tfm/public/cm/cmsy10.tfm
INPUT d:/software/texlive/2026/texmf-dist/fonts/vf/adobe/palatino/pplbi7t.vf
INPUT d:/software/texlive/2026/texmf-dist/fonts/tfm/adobe/palatino/pplbi8r.tfm
INPUT d:/software/texlive/2026/texmf-dist/fonts/vf/adobe/palatino/pplr7t.vf
INPUT d:/software/texlive/2026/texmf-dist/fonts/tfm/adobe/palatino/pplr8r.tfm
INPUT d:/software/texlive/2026/texmf-dist/fonts/vf/adobe/palatino/pplr7t.vf
INPUT d:/software/texlive/2026/texmf-dist/fonts/tfm/adobe/palatino/pplr8r.tfm
INPUT d:/software/texlive/2026/texmf-dist/fonts/vf/adobe/palatino/pplri7t.vf
INPUT d:/software/texlive/2026/texmf-dist/fonts/tfm/adobe/palatino/pplri8r.tfm
INPUT d:/software/texlive/2026/texmf-dist/fonts/tfm/adobe/palatino/pplri7t.tfm
INPUT d:/software/texlive/2026/texmf-dist/tex/latex/psnfss/ts1ppl.fd
INPUT d:/software/texlive/2026/texmf-dist/tex/latex/psnfss/ts1ppl.fd
INPUT d:/software/texlive/2026/texmf-dist/tex/latex/psnfss/ts1ppl.fd
INPUT d:/software/texlive/2026/texmf-dist/fonts/tfm/adobe/palatino/pplr8c.tfm
INPUT d:/software/texlive/2026/texmf-dist/fonts/vf/adobe/palatino/pplr7t.vf
INPUT d:/software/texlive/2026/texmf-dist/fonts/tfm/adobe/palatino/pplr8r.tfm
INPUT d:/software/texlive/2026/texmf-dist/fonts/vf/public/mathpazo/zplmr7m.vf
INPUT d:/software/texlive/2026/texmf-dist/fonts/tfm/public/mathpazo/fplmri.tfm
INPUT d:/software/texlive/2026/texmf-dist/fonts/tfm/public/cm/cmmi10.tfm
INPUT d:/software/texlive/2026/texmf-dist/fonts/tfm/adobe/palatino/pplri8r.tfm
INPUT d:/software/texlive/2026/texmf-dist/fonts/vf/public/mathpazo/zplmr7m.vf
INPUT d:/software/texlive/2026/texmf-dist/fonts/tfm/public/mathpazo/fplmri.tfm
INPUT d:/software/texlive/2026/texmf-dist/fonts/tfm/public/cm/cmmi10.tfm
INPUT d:/software/texlive/2026/texmf-dist/fonts/vf/adobe/palatino/pplri7t.vf
INPUT d:/software/texlive/2026/texmf-dist/fonts/vf/adobe/palatino/pplr8c.vf
INPUT ./section3/section3.tex
INPUT ./section3/section3.tex
INPUT ./section3/section3.tex
INPUT ./section3/section3.tex
INPUT section3/section3.tex
INPUT d:/software/texlive/2026/texmf-dist/fonts/tfm/public/mathpazo/zplmb7t.tfm
INPUT d:/software/texlive/2026/texmf-dist/fonts/tfm/public/mathpazo/fplmbb.tfm
INPUT d:/software/texlive/2026/texmf-dist/fonts/tfm/public/mathpazo/fplmbb.tfm
INPUT d:/software/texlive/2026/texmf-dist/fonts/tfm/public/mathpazo/fplmbb.tfm
INPUT ./section3/Figures/figure-update-with-policy-gradient.tex
INPUT ./section3/Figures/figure-update-with-policy-gradient.tex
INPUT section3/Figures/figure-update-with-policy-gradient.tex
INPUT d:/software/texlive/2026/texmf-dist/tex/latex/fontawesome5/ufontawesomefree3.fd
INPUT d:/software/texlive/2026/texmf-dist/tex/latex/fontawesome5/ufontawesomefree3.fd
INPUT d:/software/texlive/2026/texmf-dist/tex/latex/fontawesome5/ufontawesomefree3.fd
INPUT d:/software/texlive/2026/texmf-dist/fonts/tfm/public/fontawesome5/fa5free3solid.tfm
INPUT d:/software/texlive/2026/texmf-dist/tex/latex/fontawesome5/ufontawesomefree1.fd
INPUT d:/software/texlive/2026/texmf-dist/tex/latex/fontawesome5/ufontawesomefree1.fd
INPUT d:/software/texlive/2026/texmf-dist/tex/latex/fontawesome5/ufontawesomefree1.fd
INPUT d:/software/texlive/2026/texmf-dist/fonts/tfm/public/fontawesome5/fa5free1solid.tfm
INPUT d:/software/texlive/2026/texmf-dist/fonts/tfm/adobe/palatino/pplb7t.tfm
INPUT d:/software/texlive/2026/texmf-dist/fonts/tfm/adobe/palatino/pplb7t.tfm
INPUT d:/software/texlive/2026/texmf-dist/fonts/vf/public/mathpazo/zplmr7m.vf
INPUT d:/software/texlive/2026/texmf-dist/fonts/tfm/public/mathpazo/fplmri.tfm
INPUT d:/software/texlive/2026/texmf-dist/fonts/tfm/public/cm/cmmi10.tfm
INPUT d:/software/texlive/2026/texmf-dist/fonts/tfm/adobe/palatino/pplri8r.tfm
INPUT d:/software/texlive/2026/texmf-dist/fonts/vf/public/mathpazo/zplmr7m.vf
INPUT d:/software/texlive/2026/texmf-dist/fonts/tfm/public/mathpazo/fplmri.tfm
INPUT d:/software/texlive/2026/texmf-dist/fonts/tfm/public/cm/cmmi10.tfm
INPUT d:/software/texlive/2026/texmf-dist/fonts/tfm/adobe/palatino/pplri8r.tfm
INPUT d:/software/texlive/2026/texmf-dist/fonts/vf/public/mathpazo/zplmr7t.vf
INPUT d:/software/texlive/2026/texmf-dist/fonts/tfm/public/mathpazo/fplmr.tfm
INPUT d:/software/texlive/2026/texmf-dist/fonts/tfm/public/cm/cmr10.tfm
INPUT d:/software/texlive/2026/texmf-dist/fonts/vf/public/mathpazo/zplmr7y.vf
INPUT d:/software/texlive/2026/texmf-dist/fonts/tfm/public/cm/cmsy10.tfm
INPUT d:/software/texlive/2026/texmf-dist/fonts/vf/public/mathpazo/zplmb7t.vf
INPUT d:/software/texlive/2026/texmf-dist/fonts/tfm/public/mathpazo/fplmb.tfm
INPUT d:/software/texlive/2026/texmf-dist/fonts/tfm/adobe/palatino/pplb8r.tfm
INPUT d:/software/texlive/2026/texmf-dist/fonts/tfm/public/cm/cmbx10.tfm
INPUT ./section3/Figures/figure-one-example.tex
INPUT ./section3/Figures/figure-one-example.tex
INPUT ./section3/Figures/figure-one-example.tex
INPUT ./section3/Figures/figure-one-example.tex
INPUT section3/Figures/figure-one-example.tex
INPUT d:/software/texlive/2026/texmf-dist/fonts/tfm/adobe/palatino/pplb7t.tfm
INPUT d:/software/texlive/2026/texmf-dist/tex/latex/psnfss/omsppl.fd
INPUT d:/software/texlive/2026/texmf-dist/tex/latex/psnfss/omsppl.fd
INPUT d:/software/texlive/2026/texmf-dist/tex/latex/psnfss/omsppl.fd
INPUT d:/software/texlive/2026/texmf-dist/fonts/tfm/public/amsfonts/cmextra/cmbsy7.tfm
INPUT d:/software/texlive/2026/texmf-dist/tex/latex/fontawesome5/ufontawesomefree0.fd
INPUT d:/software/texlive/2026/texmf-dist/tex/latex/fontawesome5/ufontawesomefree0.fd
INPUT d:/software/texlive/2026/texmf-dist/tex/latex/fontawesome5/ufontawesomefree0.fd
INPUT d:/software/texlive/2026/texmf-dist/fonts/tfm/public/fontawesome5/fa5free0solid.tfm
INPUT d:/software/texlive/2026/texmf-dist/fonts/enc/dvips/fontawesome5/fa5free3.enc
INPUT d:/software/texlive/2026/texmf-dist/fonts/enc/dvips/fontawesome5/fa5free1.enc
INPUT ./section3/Figures/figure-an-overlap-example.tex
INPUT ./section3/Figures/figure-an-overlap-example.tex
INPUT ./section3/Figures/figure-an-overlap-example.tex
INPUT ./section3/Figures/figure-an-overlap-example.tex
INPUT section3/Figures/figure-an-overlap-example.tex
INPUT d:/software/texlive/2026/texmf-dist/fonts/vf/adobe/palatino/pplb7t.vf
INPUT d:/software/texlive/2026/texmf-dist/fonts/tfm/adobe/palatino/pplb8r.tfm
INPUT d:/software/texlive/2026/texmf-dist/fonts/enc/dvips/fontawesome5/fa5free0.enc
INPUT d:/software/texlive/2026/texmf-dist/fonts/vf/public/mathpazo/zplmr7v.vf
INPUT d:/software/texlive/2026/texmf-dist/fonts/tfm/public/cm/cmex10.tfm
INPUT d:/software/texlive/2026/texmf-dist/fonts/tfm/public/cm/cmex10.tfm
INPUT d:/software/texlive/2026/texmf-dist/fonts/tfm/public/cm/cmex10.tfm
INPUT d:/software/texlive/2026/texmf-dist/fonts/tfm/public/mathpazo/fplmr.tfm
INPUT d:/software/texlive/2026/texmf-dist/fonts/vf/public/mathpazo/zplmr7t.vf
INPUT d:/software/texlive/2026/texmf-dist/fonts/tfm/public/mathpazo/fplmr.tfm
INPUT d:/software/texlive/2026/texmf-dist/fonts/tfm/public/cm/cmr10.tfm
INPUT ./section3/Figures/figure-architecture-value-function.tex
INPUT ./section3/Figures/figure-architecture-value-function.tex
INPUT ./section3/Figures/figure-architecture-value-function.tex
INPUT ./section3/Figures/figure-architecture-value-function.tex
INPUT section3/Figures/figure-architecture-value-function.tex
INPUT d:/software/texlive/2026/texmf-dist/fonts/vf/adobe/palatino/pplr7t.vf
INPUT ./section3/Figures/figure-importance-sampling.tex
INPUT ./section3/Figures/figure-importance-sampling.tex
INPUT ./section3/Figures/figure-importance-sampling.tex
INPUT ./section3/Figures/figure-importance-sampling.tex
INPUT section3/Figures/figure-importance-sampling.tex
INPUT d:/software/texlive/2026/texmf-dist/fonts/tfm/public/mathpazo/fplmbb.tfm
INPUT d:/software/texlive/2026/texmf-dist/fonts/vf/adobe/palatino/pplr7t.vf
INPUT d:/software/texlive/2026/texmf-dist/fonts/tfm/adobe/palatino/pplr8r.tfm
INPUT d:/software/texlive/2026/texmf-dist/fonts/vf/public/mathpazo/zplmr7m.vf
INPUT d:/software/texlive/2026/texmf-dist/fonts/tfm/public/mathpazo/fplmri.tfm
INPUT d:/software/texlive/2026/texmf-dist/fonts/tfm/public/cm/cmmi10.tfm
INPUT d:/software/texlive/2026/texmf-dist/fonts/tfm/adobe/palatino/pplri8r.tfm
INPUT ./section3/Figures/figure-importance-sampling-bad-sampling.tex
INPUT ./section3/Figures/figure-importance-sampling-bad-sampling.tex
INPUT ./section3/Figures/figure-importance-sampling-bad-sampling.tex
INPUT ./section3/Figures/figure-importance-sampling-bad-sampling.tex
INPUT section3/Figures/figure-importance-sampling-bad-sampling.tex
INPUT d:/software/texlive/2026/texmf-dist/fonts/vf/public/mathpazo/zplmb7t.vf
INPUT d:/software/texlive/2026/texmf-dist/fonts/tfm/public/mathpazo/fplmb.tfm
INPUT d:/software/texlive/2026/texmf-dist/fonts/tfm/public/cm/cmbx10.tfm
INPUT d:/software/texlive/2026/texmf-dist/fonts/vf/public/mathpazo/zplmr7y.vf
INPUT d:/software/texlive/2026/texmf-dist/fonts/tfm/public/cm/cmsy10.tfm
INPUT d:/software/texlive/2026/texmf-dist/fonts/tfm/public/mathpazo/fplmr.tfm
INPUT ./section3/Figures/figure-ppo-clip.tex
INPUT ./section3/Figures/figure-ppo-clip.tex
INPUT ./section3/Figures/figure-ppo-clip.tex
INPUT ./section3/Figures/figure-ppo-clip.tex
INPUT section3/Figures/figure-ppo-clip.tex
INPUT ./section3/Figures/figure-reward-model.tex
INPUT ./section3/Figures/figure-reward-model.tex
INPUT ./section3/Figures/figure-reward-model.tex
INPUT ./section3/Figures/figure-reward-model.tex
INPUT section3/Figures/figure-reward-model.tex
INPUT d:/software/texlive/2026/texmf-dist/fonts/vf/public/mathpazo/zplmr7y.vf
INPUT d:/software/texlive/2026/texmf-dist/fonts/tfm/public/cm/cmsy10.tfm
INPUT d:/software/texlive/2026/texmf-dist/fonts/vf/public/mathpazo/zplmb7t.vf
INPUT d:/software/texlive/2026/texmf-dist/fonts/tfm/public/mathpazo/fplmb.tfm
INPUT d:/software/texlive/2026/texmf-dist/fonts/tfm/public/cm/cmbx10.tfm
INPUT ./section3/Figures/figure-pairwise-reward-loss-expectation.tex
INPUT ./section3/Figures/figure-pairwise-reward-loss-expectation.tex
INPUT ./section3/Figures/figure-pairwise-reward-loss-expectation.tex
INPUT ./section3/Figures/figure-pairwise-reward-loss-expectation.tex
INPUT section3/Figures/figure-pairwise-reward-loss-expectation.tex
INPUT ./section3/Figures/figure-rlhf.tex
INPUT ./section3/Figures/figure-rlhf.tex
INPUT ./section3/Figures/figure-rlhf.tex
INPUT ./section3/Figures/figure-rlhf.tex
INPUT section3/Figures/figure-rlhf.tex
INPUT ./section4/section4.tex
INPUT ./section4/section4.tex
INPUT ./section4/section4.tex
INPUT ./section4/section4.tex
INPUT section4/section4.tex
INPUT ./section4/Figures/figure-an-overlap-example-dense-reward-model.tex
INPUT ./section4/Figures/figure-an-overlap-example-dense-reward-model.tex
INPUT ./section4/Figures/figure-an-overlap-example-dense-reward-model.tex
INPUT ./section4/Figures/figure-an-overlap-example-dense-reward-model.tex
INPUT section4/Figures/figure-an-overlap-example-dense-reward-model.tex
INPUT ./section4/Figures/figure-methods-improving-reward-generalization.tex
INPUT ./section4/Figures/figure-methods-improving-reward-generalization.tex
INPUT ./section4/Figures/figure-methods-improving-reward-generalization.tex
INPUT ./section4/Figures/figure-methods-improving-reward-generalization.tex
INPUT section4/Figures/figure-methods-improving-reward-generalization.tex
INPUT d:/software/texlive/2026/texmf-dist/fonts/tfm/public/fontawesome5/fa5free1solid.tfm
INPUT ./section4/Figures/figure-generative-reward-model-architecture.tex
INPUT ./section4/Figures/figure-generative-reward-model-architecture.tex
INPUT ./section4/Figures/figure-generative-reward-model-architecture.tex
INPUT ./section4/Figures/figure-generative-reward-model-architecture.tex
INPUT section4/Figures/figure-generative-reward-model-architecture.tex
INPUT ./section4/Figures/figure-issue-monte-carlo.tex
INPUT ./section4/Figures/figure-issue-monte-carlo.tex
INPUT ./section4/Figures/figure-issue-monte-carlo.tex
INPUT ./section4/Figures/figure-issue-monte-carlo.tex
INPUT section4/Figures/figure-issue-monte-carlo.tex
INPUT d:/software/texlive/2026/texmf-dist/fonts/tfm/adobe/palatino/pplr7t.tfm
INPUT d:/software/texlive/2026/texmf-dist/fonts/tfm/public/mathpazo/zplmr7m.tfm
INPUT d:/software/texlive/2026/texmf-dist/fonts/tfm/public/mathpazo/zplmr7y.tfm
INPUT d:/software/texlive/2026/texmf-dist/fonts/tfm/public/mathpazo/zplmr7v.tfm
INPUT d:/software/texlive/2026/texmf-dist/fonts/tfm/public/mathpazo/zplmb7t.tfm
INPUT d:/software/texlive/2026/texmf-dist/fonts/tfm/public/mathpazo/zplmr7t.tfm
INPUT d:/software/texlive/2026/texmf-dist/fonts/tfm/public/amsfonts/symbols/msam10.tfm
INPUT d:/software/texlive/2026/texmf-dist/fonts/tfm/public/amsfonts/symbols/msbm10.tfm
INPUT d:/software/texlive/2026/texmf-dist/fonts/tfm/public/amsfonts/euler/euex10.tfm
INPUT d:/software/texlive/2026/texmf-dist/fonts/tfm/public/mathpazo/fplmbb.tfm
INPUT d:/software/texlive/2026/texmf-dist/fonts/vf/public/mathpazo/zplmr7y.vf
INPUT d:/software/texlive/2026/texmf-dist/fonts/tfm/public/cm/cmsy10.tfm
INPUT d:/software/texlive/2026/texmf-dist/fonts/tfm/public/mathpazo/fplmr.tfm
INPUT d:/software/texlive/2026/texmf-dist/fonts/tfm/adobe/palatino/pplr8r.tfm
INPUT ./section4/Figures/figure-grpo.tex
INPUT ./section4/Figures/figure-grpo.tex
INPUT ./section4/Figures/figure-grpo.tex
INPUT ./section4/Figures/figure-grpo.tex
INPUT section4/Figures/figure-grpo.tex
INPUT ./section4/Figures/figure-examples-of-rule-based-reward-length.tex
INPUT ./section4/Figures/figure-examples-of-rule-based-reward-length.tex
INPUT ./section4/Figures/figure-examples-of-rule-based-reward-length.tex
INPUT ./section4/Figures/figure-examples-of-rule-based-reward-length.tex
INPUT section4/Figures/figure-examples-of-rule-based-reward-length.tex
INPUT d:/software/texlive/2026/texmf-dist/fonts/tfm/adobe/palatino/pplb7t.tfm
INPUT d:/software/texlive/2026/texmf-dist/fonts/tfm/public/cm/cmsy9.tfm
INPUT ./section4/Figures/figure-examples-of-rule-based-reward-math.tex
INPUT ./section4/Figures/figure-examples-of-rule-based-reward-math.tex
INPUT ./section4/Figures/figure-examples-of-rule-based-reward-math.tex
INPUT ./section4/Figures/figure-examples-of-rule-based-reward-math.tex
INPUT section4/Figures/figure-examples-of-rule-based-reward-math.tex
INPUT d:/software/texlive/2026/texmf-dist/fonts/tfm/adobe/palatino/pplr8c.tfm
INPUT d:/software/texlive/2026/texmf-dist/fonts/vf/adobe/palatino/pplb7t.vf
INPUT d:/software/texlive/2026/texmf-dist/fonts/vf/adobe/palatino/pplr8c.vf
INPUT ./section4/Figures/figure-rlhf-vs-dpo.tex
INPUT ./section4/Figures/figure-rlhf-vs-dpo.tex
INPUT ./section4/Figures/figure-rlhf-vs-dpo.tex
INPUT ./section4/Figures/figure-rlhf-vs-dpo.tex
INPUT section4/Figures/figure-rlhf-vs-dpo.tex
INPUT d:/software/texlive/2026/texmf-dist/fonts/tfm/adobe/palatino/pplr7t.tfm
INPUT d:/software/texlive/2026/texmf-dist/fonts/tfm/public/mathpazo/zplmr7m.tfm
INPUT d:/software/texlive/2026/texmf-dist/fonts/tfm/public/mathpazo/zplmr7m.tfm
INPUT d:/software/texlive/2026/texmf-dist/fonts/tfm/public/mathpazo/zplmr7y.tfm
INPUT d:/software/texlive/2026/texmf-dist/fonts/tfm/public/mathpazo/zplmr7y.tfm
INPUT d:/software/texlive/2026/texmf-dist/fonts/tfm/public/mathpazo/zplmr7v.tfm
INPUT d:/software/texlive/2026/texmf-dist/fonts/tfm/public/mathpazo/zplmr7v.tfm
INPUT d:/software/texlive/2026/texmf-dist/fonts/tfm/public/mathpazo/zplmb7t.tfm
INPUT d:/software/texlive/2026/texmf-dist/fonts/tfm/public/mathpazo/zplmb7t.tfm
INPUT d:/software/texlive/2026/texmf-dist/fonts/tfm/public/mathpazo/zplmr7t.tfm
INPUT d:/software/texlive/2026/texmf-dist/fonts/tfm/public/mathpazo/zplmr7t.tfm
INPUT d:/software/texlive/2026/texmf-dist/fonts/tfm/public/amsfonts/symbols/msam10.tfm
INPUT d:/software/texlive/2026/texmf-dist/fonts/tfm/public/amsfonts/symbols/msam10.tfm
INPUT d:/software/texlive/2026/texmf-dist/fonts/tfm/public/amsfonts/symbols/msbm10.tfm
INPUT d:/software/texlive/2026/texmf-dist/fonts/tfm/public/amsfonts/symbols/msbm10.tfm
INPUT d:/software/texlive/2026/texmf-dist/fonts/tfm/public/amsfonts/euler/euex10.tfm
INPUT d:/software/texlive/2026/texmf-dist/fonts/tfm/public/amsfonts/euler/euex8.tfm
INPUT d:/software/texlive/2026/texmf-dist/fonts/tfm/public/mathpazo/fplmbb.tfm
INPUT d:/software/texlive/2026/texmf-dist/fonts/tfm/public/mathpazo/fplmbb.tfm
INPUT d:/software/texlive/2026/texmf-dist/fonts/vf/public/mathpazo/zplmb7t.vf
INPUT d:/software/texlive/2026/texmf-dist/fonts/tfm/public/mathpazo/fplmb.tfm
INPUT d:/software/texlive/2026/texmf-dist/fonts/tfm/public/cm/cmbx10.tfm
INPUT d:/software/texlive/2026/texmf-dist/fonts/vf/public/mathpazo/zplmr7y.vf
INPUT d:/software/texlive/2026/texmf-dist/fonts/tfm/public/cm/cmsy10.tfm
INPUT d:/software/texlive/2026/texmf-dist/fonts/tfm/public/mathpazo/fplmr.tfm
INPUT d:/software/texlive/2026/texmf-dist/fonts/tfm/adobe/palatino/pplr8r.tfm
INPUT ./section4/Figures/figure-dpo.tex
INPUT ./section4/Figures/figure-dpo.tex
INPUT ./section4/Figures/figure-dpo.tex
INPUT ./section4/Figures/figure-dpo.tex
INPUT section4/Figures/figure-dpo.tex
INPUT ./section4/Tables/dpo_variants.tex
INPUT ./section4/Tables/dpo_variants.tex
INPUT ./section4/Tables/dpo_variants.tex
INPUT ./section4/Tables/dpo_variants.tex
INPUT section4/Tables/dpo_variants.tex
INPUT d:/software/texlive/2026/texmf-dist/fonts/vf/public/mathpazo/zplmr7v.vf
INPUT d:/software/texlive/2026/texmf-dist/fonts/tfm/public/cm/cmex10.tfm
INPUT d:/software/texlive/2026/texmf-dist/fonts/tfm/public/cm/cmex10.tfm
INPUT d:/software/texlive/2026/texmf-dist/fonts/tfm/public/cm/cmex10.tfm
INPUT d:/software/texlive/2026/texmf-dist/fonts/tfm/public/mathpazo/fplmr.tfm
INPUT d:/software/texlive/2026/texmf-dist/fonts/vf/public/mathpazo/zplmr7t.vf
INPUT d:/software/texlive/2026/texmf-dist/fonts/tfm/public/mathpazo/fplmr.tfm
INPUT d:/software/texlive/2026/texmf-dist/fonts/tfm/public/cm/cmr10.tfm
INPUT d:/software/texlive/2026/texmf-dist/fonts/vf/public/mathpazo/zplmb7t.vf
INPUT d:/software/texlive/2026/texmf-dist/fonts/tfm/public/mathpazo/fplmb.tfm
INPUT d:/software/texlive/2026/texmf-dist/fonts/tfm/adobe/palatino/pplb8r.tfm
INPUT d:/software/texlive/2026/texmf-dist/fonts/tfm/public/cm/cmbx10.tfm
INPUT d:/software/texlive/2026/texmf-dist/fonts/vf/public/mathpazo/zplmr7y.vf
INPUT d:/software/texlive/2026/texmf-dist/fonts/tfm/public/cm/cmsy10.tfm
INPUT ./section5/section5.tex
INPUT ./section5/section5.tex
INPUT ./section5/section5.tex
INPUT ./section5/section5.tex
INPUT section5/section5.tex
INPUT d:/software/texlive/2026/texmf-dist/fonts/vf/public/mathpazo/zplmr7y.vf
INPUT d:/software/texlive/2026/texmf-dist/fonts/tfm/public/cm/cmsy10.tfm
INPUT d:/software/texlive/2026/texmf-dist/fonts/vf/public/mathpazo/zplmb7t.vf
INPUT d:/software/texlive/2026/texmf-dist/fonts/tfm/public/mathpazo/fplmb.tfm
INPUT d:/software/texlive/2026/texmf-dist/fonts/tfm/adobe/palatino/pplb8r.tfm
INPUT d:/software/texlive/2026/texmf-dist/fonts/tfm/public/cm/cmbx10.tfm
INPUT d:/software/texlive/2026/texmf-dist/fonts/vf/adobe/palatino/pplb7t.vf
INPUT ./section5/Figures/figure-bon-sampling.tex
INPUT ./section5/Figures/figure-bon-sampling.tex
INPUT ./section5/Figures/figure-bon-sampling.tex
INPUT ./section5/Figures/figure-bon-sampling.tex
INPUT section5/Figures/figure-bon-sampling.tex
INPUT d:/software/texlive/2026/texmf-dist/fonts/tfm/adobe/palatino/pplri7t.tfm
INPUT d:/software/texlive/2026/texmf-dist/fonts/tfm/adobe/palatino/pplri7t.tfm
INPUT ./section5/Figures/figure-step-by-step-with-prm.tex
INPUT ./section5/Figures/figure-step-by-step-with-prm.tex
INPUT section5/Figures/figure-step-by-step-with-prm.tex
INPUT ./section5/Figures/figure-mcts.tex
INPUT ./section5/Figures/figure-mcts.tex
INPUT ./section5/Figures/figure-mcts.tex
INPUT ./section5/Figures/figure-mcts.tex
INPUT section5/Figures/figure-mcts.tex
INPUT d:/software/texlive/2026/texmf-dist/fonts/tfm/adobe/palatino/pplr8c.tfm
INPUT d:/software/texlive/2026/texmf-dist/fonts/vf/adobe/palatino/pplr8c.vf
INPUT ./section5/Figures/figure-iterative-rl.tex
INPUT ./section5/Figures/figure-iterative-rl.tex
INPUT ./section5/Figures/figure-iterative-rl.tex
INPUT ./section5/Figures/figure-iterative-rl.tex
INPUT section5/Figures/figure-iterative-rl.tex
INPUT d:/software/texlive/2026/texmf-dist/fonts/tfm/adobe/palatino/pplb7t.tfm
INPUT d:/software/texlive/2026/texmf-dist/fonts/tfm/public/fontawesome5/fa5free0solid.tfm
INPUT ./section6/section6.tex
INPUT ./section6/section6.tex
INPUT ./section6/section6.tex
INPUT ./section6/section6.tex
INPUT section6/section6.tex
INPUT ./section7/section7.tex
INPUT ./section7/section7.tex
INPUT ./section7/section7.tex
INPUT ./section7/section7.tex
INPUT section7/section7.tex
INPUT ./section8/section8.tex
INPUT ./section8/section8.tex
INPUT ./section8/section8.tex
INPUT ./section8/section8.tex
INPUT section8/section8.tex
INPUT ./section9/section9.tex
INPUT ./section9/section9.tex
INPUT ./section9/section9.tex
INPUT ./section9/section9.tex
INPUT section9/section9.tex
INPUT ./section9/tables/dataset.tex
INPUT ./section9/tables/dataset.tex
INPUT ./section9/tables/dataset.tex
INPUT ./section9/tables/dataset.tex
INPUT section9/tables/dataset.tex
INPUT ./section9/tables/systems.tex
INPUT ./section9/tables/systems.tex
INPUT ./section9/tables/systems.tex
INPUT ./section9/tables/systems.tex
INPUT section9/tables/systems.tex
INPUT d:/software/texlive/2026/texmf-dist/tex/latex/bbding/Uding.fd
INPUT d:/software/texlive/2026/texmf-dist/tex/latex/bbding/Uding.fd
INPUT d:/software/texlive/2026/texmf-dist/tex/latex/bbding/Uding.fd
INPUT d:/software/texlive/2026/texmf-dist/fonts/tfm/public/bbding/bbding10.tfm
INPUT ./rl-introduction.bbl
INPUT ./rl-introduction.bbl
INPUT rl-introduction.bbl
INPUT d:/software/texlive/2026/texmf-dist/fonts/tfm/public/inconsolata/ot1-zi4r-0.tfm
INPUT d:/software/texlive/2026/texmf-dist/fonts/tfm/public/inconsolata/ot1-zi4r-0.tfm
INPUT rl-introduction.aux
INPUT ./rl-introduction.out
INPUT ./rl-introduction.out
INPUT d:/software/texlive/2026/texmf-dist/fonts/type1/public/fontawesome5/FontAwesome5Free-Solid.pfb
INPUT d:/software/texlive/2026/texmf-dist/fonts/type1/public/inconsolata/Inconsolata-zi4r.pfb
INPUT d:/software/texlive/2026/texmf-dist/fonts/type1/public/niceframe-type1/bbding10.pfb
INPUT d:/software/texlive/2026/texmf-dist/fonts/type1/public/amsfonts/cmextra/cmbsy7.pfb
INPUT d:/software/texlive/2026/texmf-dist/fonts/type1/public/amsfonts/cm/cmex10.pfb
INPUT d:/software/texlive/2026/texmf-dist/fonts/type1/public/amsfonts/cm/cmmi10.pfb
INPUT d:/software/texlive/2026/texmf-dist/fonts/type1/public/amsfonts/cm/cmr10.pfb
INPUT d:/software/texlive/2026/texmf-dist/fonts/type1/public/amsfonts/cm/cmsy10.pfb
INPUT d:/software/texlive/2026/texmf-dist/fonts/type1/public/amsfonts/cm/cmsy9.pfb
INPUT d:/software/texlive/2026/texmf-dist/fonts/type1/public/mathpazo/fplmbb.pfb
INPUT d:/software/texlive/2026/texmf-dist/fonts/type1/public/mathpazo/fplmr.pfb
INPUT d:/software/texlive/2026/texmf-dist/fonts/type1/public/mathpazo/fplmri.pfb
INPUT d:/software/texlive/2026/texmf-dist/fonts/type1/urw/palatino/uplb8a.pfb
INPUT d:/software/texlive/2026/texmf-dist/fonts/type1/urw/palatino/uplbi8a.pfb
INPUT d:/software/texlive/2026/texmf-dist/fonts/type1/urw/palatino/uplr8a.pfb
INPUT d:/software/texlive/2026/texmf-dist/fonts/type1/urw/palatino/uplri8a.pfb
rl-introduction.log 0 → 100644
This is pdfTeX, Version 3.141592653-2.6-1.40.29 (TeX Live 2026) (preloaded format=pdflatex 2026.6.25) 26 JUN 2026 00:24
entering extended mode
restricted \write18 enabled.
file:line:error style messages enabled.
%&-line parsing enabled.
**c:/Users/Lenovo/Desktop/rl_introduction/rl-introduction.tex
(c:/Users/Lenovo/Desktop/rl_introduction/rl-introduction.tex
LaTeX2e <2026-06-01>
L3 programming layer <2026-06-18>
(d:/software/texlive/2026/texmf-dist/tex/latex/base/article.cls
Document Class: article 2025/01/22 v1.4n Standard LaTeX document class
(d:/software/texlive/2026/texmf-dist/tex/latex/base/size10.clo
File: size10.clo 2025/01/22 v1.4n Standard LaTeX file (size option)
)
\c@part=\count276
\c@section=\count277
\c@subsection=\count278
\c@subsubsection=\count279
\c@paragraph=\count280
\c@subparagraph=\count281
\c@figure=\count282
\c@table=\count283
\abovecaptionskip=\skip49
\belowcaptionskip=\skip50
\bibindent=\dimen150
) (./rl-introduction.sty (d:/software/texlive/2026/texmf-dist/tex/latex/tex-gyre/tgpagella.sty
Package: tgpagella 2009/09/27 v1.2 TeX Gyre Pagella as default roman family
(d:/software/texlive/2026/texmf-dist/tex/latex/kvoptions/kvoptions.sty
Package: kvoptions 2022-06-15 v3.15 Key value format for package options (HO)
(d:/software/texlive/2026/texmf-dist/tex/latex/graphics/keyval.sty
Package: keyval 2026-05-17 v1.15 key=value parser (DPC)
\KV@toks@=\toks17
) (d:/software/texlive/2026/texmf-dist/tex/generic/ltxcmds/ltxcmds.sty
Package: ltxcmds 2023-12-04 v1.26 LaTeX kernel commands for general use (HO)
) (d:/software/texlive/2026/texmf-dist/tex/latex/kvsetkeys/kvsetkeys.sty
Package: kvsetkeys 2022-10-05 v1.19 Key value parser (HO)
))) (d:/software/texlive/2026/texmf-dist/tex/latex/psnfss/mathpazo.sty
Package: mathpazo 2020/03/25 PSNFSS-v9.3 Palatino w/ Pazo Math (D.Puga, WaS)
\symupright=\mathgroup4
) (d:/software/texlive/2026/texmf-dist/tex/latex/inconsolata/inconsolata.sty
Package: inconsolata 2019/05/17 v1.12
`inconsolata-zi4' v1.12, 2019/05/17 Text macros for Inconsolata (msharpe) (d:/software/texlive/2026/texmf-dist/tex/latex/base/textcomp.sty
Package: textcomp 2024/04/24 v2.1b Standard LaTeX package
) (d:/software/texlive/2026/texmf-dist/tex/latex/xkeyval/xkeyval.sty
Package: xkeyval 2025/11/04 v2.10 package option processing (HA)
(d:/software/texlive/2026/texmf-dist/tex/generic/xkeyval/xkeyval.tex (d:/software/texlive/2026/texmf-dist/tex/generic/xkeyval/xkvutils.tex
\XKV@toks=\toks18
\XKV@tempa@toks=\toks19
\XKV@tempb@toks=\toks20
)
\XKV@depth=\count284
File: xkeyval.tex 2014/12/03 v2.7a key=value parser (HA)
))
\zifour@ocount=\count285
) (d:/software/texlive/2026/texmf-dist/tex/latex/eso-pic/eso-pic.sty
Package: eso-pic 2026/05/31 v3.0e eso-pic (RN)
\ESO@tempdima=\dimen151
\ESO@tempdimb=\dimen152
(d:/software/texlive/2026/texmf-dist/tex/latex/xcolor/xcolor.sty
Package: xcolor 2024/09/29 v3.02 LaTeX color extensions (UK)
(d:/software/texlive/2026/texmf-dist/tex/latex/graphics-cfg/color.cfg
File: color.cfg 2016/01/02 v1.6 sample color configuration
)
Package xcolor Info: Driver file: pdftex.def on input line 274.
(d:/software/texlive/2026/texmf-dist/tex/latex/graphics-def/pdftex.def
File: pdftex.def 2025/09/29 v1.2d Graphics/color driver for pdftex
) (d:/software/texlive/2026/texmf-dist/tex/latex/graphics/mathcolor.ltx)
Package xcolor Info: Model `cmy' substituted by `cmy0' on input line 1349.
Package xcolor Info: Model `hsb' substituted by `rgb' on input line 1353.
Package xcolor Info: Model `RGB' extended on input line 1365.
Package xcolor Info: Model `HTML' substituted by `rgb' on input line 1367.
Package xcolor Info: Model `Hsb' substituted by `hsb' on input line 1368.
Package xcolor Info: Model `tHsb' substituted by `hsb' on input line 1369.
Package xcolor Info: Model `HSB' substituted by `hsb' on input line 1370.
Package xcolor Info: Model `Gray' substituted by `gray' on input line 1371.
Package xcolor Info: Model `wave' substituted by `hsb' on input line 1372.
)) (./fancyhdr.sty
\fancy@headwidth=\skip51
\f@ncyO@elh=\skip52
\f@ncyO@erh=\skip53
\f@ncyO@olh=\skip54
\f@ncyO@orh=\skip55
\f@ncyO@elf=\skip56
\f@ncyO@erf=\skip57
\f@ncyO@olf=\skip58
\f@ncyO@orf=\skip59
) (./natbib.sty
Package: natbib 2009/07/16 8.31 (PWD, AO)
\bibhang=\skip60
\bibsep=\skip61
LaTeX Info: Redefining \cite on input line 694.
\c@NAT@ctr=\count286
)) (d:/software/texlive/2026/texmf-dist/tex/latex/microtype/microtype.sty
Package: microtype 2026/03/01 v3.2d Micro-typographical refinements (RS)
(d:/software/texlive/2026/texmf-dist/tex/latex/etoolbox/etoolbox.sty
Package: etoolbox 2025/10/02 v2.5m e-TeX tools for LaTeX (JAW)
\etb@tempcnta=\count287
)
\MT@toks=\toks21
\MT@tempbox=\box53
\MT@count=\count288
LaTeX Info: Redefining \noprotrusionifhmode on input line 1084.
LaTeX Info: Redefining \leftprotrusion on input line 1085.
\MT@prot@toks=\toks22
LaTeX Info: Redefining \rightprotrusion on input line 1104.
LaTeX Info: Redefining \textls on input line 1449.
\MT@outer@kern=\dimen153
LaTeX Info: Redefining \microtypecontext on input line 2053.
LaTeX Info: Redefining \textmicrotypecontext on input line 2070.
\MT@listname@count=\count289
(d:/software/texlive/2026/texmf-dist/tex/latex/microtype/microtype-pdftex.def
File: microtype-pdftex.def 2026/03/01 v3.2d Definitions specific to pdftex (RS)
LaTeX Info: Redefining \lsstyle on input line 944.
LaTeX Info: Redefining \lslig on input line 944.
\MT@outer@space=\skip62
)
Package microtype Info: Loading configuration file microtype.cfg.
(d:/software/texlive/2026/texmf-dist/tex/latex/microtype/microtype.cfg
File: microtype.cfg 2026/03/01 v3.2d microtype main configuration file (RS)
)
LaTeX Info: Redefining \microtypesetup on input line 3065.
) (d:/software/texlive/2026/texmf-dist/tex/latex/hyperref/hyperref.sty
Package: hyperref 2026-06-17 v7.01r Hypertext links for LaTeX
(d:/software/texlive/2026/texmf-dist/tex/generic/iftex/iftex.sty
Package: iftex 2024/12/12 v1.0g TeX engine tests
) (d:/software/texlive/2026/texmf-dist/tex/generic/kvdefinekeys/kvdefinekeys.sty
Package: kvdefinekeys 2019-12-19 v1.6 Define keys (HO)
) (d:/software/texlive/2026/texmf-dist/tex/generic/pdfescape/pdfescape.sty
Package: pdfescape 2019/12/09 v1.15 Implements pdfTeX's escape features (HO)
(d:/software/texlive/2026/texmf-dist/tex/generic/pdftexcmds/pdftexcmds.sty
Package: pdftexcmds 2020-06-27 v0.33 Utility functions of pdfTeX for LuaTeX (HO)
(d:/software/texlive/2026/texmf-dist/tex/generic/infwarerr/infwarerr.sty
Package: infwarerr 2019/12/03 v1.5 Providing info/warning/error messages (HO)
)
Package pdftexcmds Info: \pdf@primitive is available.
Package pdftexcmds Info: \pdf@ifprimitive is available.
Package pdftexcmds Info: \pdfdraftmode found.
)) (d:/software/texlive/2026/texmf-dist/tex/latex/hycolor/hycolor.sty
Package: hycolor 2020-01-27 v1.10 Color options for hyperref/bookmark (HO)
) (d:/software/texlive/2026/texmf-dist/tex/latex/hyperref/nameref.sty
Package: nameref 2026-01-29 v2.58 Cross-referencing by name of section
(d:/software/texlive/2026/texmf-dist/tex/latex/refcount/refcount.sty
Package: refcount 2019/12/15 v3.6 Data extraction from label references (HO)
) (d:/software/texlive/2026/texmf-dist/tex/generic/gettitlestring/gettitlestring.sty
Package: gettitlestring 2019/12/15 v1.6 Cleanup title references (HO)
)
\c@section@level=\count290
) (d:/software/texlive/2026/texmf-dist/tex/generic/stringenc/stringenc.sty
Package: stringenc 2019/11/29 v1.12 Convert strings between diff. encodings (HO)
)
\@linkdim=\dimen154
\Hy@linkcounter=\count291
\Hy@pagecounter=\count292
(d:/software/texlive/2026/texmf-dist/tex/latex/hyperref/pd1enc.def
File: pd1enc.def 2026-06-17 v7.01r Hyperref: PDFDocEncoding definition (HO)
Now handling font encoding PD1 ...
... no UTF-8 mapping file for font encoding PD1
) (d:/software/texlive/2026/texmf-dist/tex/generic/intcalc/intcalc.sty
Package: intcalc 2019/12/15 v1.3 Expandable calculations with integers (HO)
)
\Hy@SavedSpaceFactor=\count293
(d:/software/texlive/2026/texmf-dist/tex/latex/hyperref/puenc.def
File: puenc.def 2026-06-17 v7.01r Hyperref: PDF Unicode definition (HO)
Now handling font encoding PU ...
... no UTF-8 mapping file for font encoding PU
)
Package hyperref Info: Hyper figures OFF on input line 4199.
Package hyperref Info: Link nesting OFF on input line 4204.
Package hyperref Info: Hyper index ON on input line 4207.
Package hyperref Info: Plain pages OFF on input line 4214.
Package hyperref Info: Backreferencing OFF on input line 4219.
Package hyperref Info: Implicit mode ON; LaTeX internals redefined.
Package hyperref Info: Bookmarks ON on input line 4466.
\c@Hy@tempcnt=\count294
(d:/software/texlive/2026/texmf-dist/tex/latex/url/url.sty
\Urlmuskip=\muskip17
Package: url 2013/09/16 ver 3.4 Verb mode for urls, etc.
)
LaTeX Info: Redefining \url on input line 4805.
\XeTeXLinkMargin=\dimen155
(d:/software/texlive/2026/texmf-dist/tex/generic/bitset/bitset.sty
Package: bitset 2019/12/09 v1.3 Handle bit-vector datatype (HO)
(d:/software/texlive/2026/texmf-dist/tex/generic/bigintcalc/bigintcalc.sty
Package: bigintcalc 2019/12/15 v1.5 Expandable calculations on big integers (HO)
))
\Fld@menulength=\count295
\Field@Width=\dimen156
\Fld@charsize=\dimen157
Package hyperref Info: Hyper figures OFF on input line 6091.
Package hyperref Info: Link nesting OFF on input line 6096.
Package hyperref Info: Hyper index ON on input line 6099.
Package hyperref Info: backreferencing OFF on input line 6106.
Package hyperref Info: Link coloring OFF on input line 6111.
Package hyperref Info: Link coloring with OCG OFF on input line 6116.
Package hyperref Info: PDF/A mode OFF on input line 6121.
\Hy@abspage=\count296
\c@Item=\count297
\c@Hfootnote=\count298
)
Package hyperref Info: Driver (autodetected): hpdftex.
(d:/software/texlive/2026/texmf-dist/tex/latex/hyperref/hpdftex.def
File: hpdftex.def 2026-06-17 v7.01r Hyperref driver for pdfTeX
\Fld@listcount=\count299
\c@bookmark@seq@number=\count300
(d:/software/texlive/2026/texmf-dist/tex/latex/rerunfilecheck/rerunfilecheck.sty
Package: rerunfilecheck 2025-06-21 v1.11 Rerun checks for auxiliary files (HO)
(d:/software/texlive/2026/texmf-dist/tex/generic/uniquecounter/uniquecounter.sty
Package: uniquecounter 2019/12/15 v1.4 Provide unlimited unique counter (HO)
)
Package uniquecounter Info: New unique counter `rerunfilecheck' on input line 284.
)
\Hy@SectionHShift=\skip63
) (d:/software/texlive/2026/texmf-dist/tex/latex/booktabs/booktabs.sty
Package: booktabs 2020/01/12 v1.61803398 Publication quality tables
\heavyrulewidth=\dimen158
\lightrulewidth=\dimen159
\cmidrulewidth=\dimen160
\belowrulesep=\dimen161
\belowbottomsep=\dimen162
\aboverulesep=\dimen163
\abovetopsep=\dimen164
\cmidrulesep=\dimen165
\cmidrulekern=\dimen166
\defaultaddspace=\dimen167
\@cmidla=\count301
\@cmidlb=\count302
\@aboverulesep=\dimen168
\@belowrulesep=\dimen169
\@thisruleclass=\count303
\@lastruleclass=\count304
\@thisrulewidth=\dimen170
) (d:/software/texlive/2026/texmf-dist/tex/latex/fontawesome5/fontawesome5.sty (d:/software/texlive/2026/texmf-dist/tex/latex/l3kernel/expl3.sty
Package: expl3 2026-06-18 L3 programming layer (loader)
(d:/software/texlive/2026/texmf-dist/tex/latex/l3backend/l3backend-pdftex.def
File: l3backend-pdftex.def 2026-02-18 L3 backend support: PDF output (pdfTeX)
\l__color_backend_stack_int=\count305
))
Package: fontawesome5 2022/05/02 v5.15.4 Font Awesome 5
(d:/software/texlive/2026/texmf-dist/tex/latex/l3packages/l3keys2e/l3keys2e.sty
Package: l3keys2e 2025-10-09 LaTeX2e option processing using LaTeX3 keys
) (d:/software/texlive/2026/texmf-dist/tex/latex/l3packages/xparse/xparse.sty
Package: xparse 2025-10-09 L3 Experimental document command parser
) (d:/software/texlive/2026/texmf-dist/tex/latex/fontawesome5/fontawesome5-generic-helper.sty
Package: fontawesome5-generic-helper 2022/05/02 v5.15.4 non-uTeX helper for fontawesome5
(d:/software/texlive/2026/texmf-dist/tex/latex/fontawesome5/fontawesome5-mapping.def))) (d:/software/texlive/2026/texmf-dist/tex/latex/lineno/lineno.sty
Package: lineno 2026/03/08 line numbers on paragraphs v5.9
\linenopenalty=\count306
\output=\toks23
\linenoprevgraf=\count307
\linenumbersep=\dimen171
\linenumberwidth=\dimen172
\c@linenumber=\count308
\c@pagewiselinenumber=\count309
\c@LN@truepage=\count310
\c@internallinenumber=\count311
\c@internallinenumbers=\count312
\quotelinenumbersep=\dimen173
\bframerule=\dimen174
\bframesep=\dimen175
\bframebox=\box54
) (d:/software/texlive/2026/texmf-dist/tex/latex/bbding/bbding.sty
Package: bbding 1999/04/15 v1.01 Dingbats symbols
) (d:/software/texlive/2026/texmf-dist/tex/latex/cjk/texinput/CJK.sty
Package: CJK 2021/10/16 4.8.5
(d:/software/texlive/2026/texmf-dist/tex/latex/cjk/texinput/mule/MULEenc.sty
Package: MULEenc 2021/10/16 4.8.5
) (d:/software/texlive/2026/texmf-dist/tex/latex/cjk/texinput/CJK.enc
File: CJK.enc 2021/10/16 4.8.5
Now handling font encoding C00 ...
... no UTF-8 mapping file for font encoding C00
Now handling font encoding C05 ...
... no UTF-8 mapping file for font encoding C05
Now handling font encoding C09 ...
... no UTF-8 mapping file for font encoding C09
Now handling font encoding C10 ...
... no UTF-8 mapping file for font encoding C10
Now handling font encoding C20 ...
... no UTF-8 mapping file for font encoding C20
Now handling font encoding C19 ...
... no UTF-8 mapping file for font encoding C19
Now handling font encoding C40 ...
... no UTF-8 mapping file for font encoding C40
Now handling font encoding C42 ...
... no UTF-8 mapping file for font encoding C42
Now handling font encoding C43 ...
... no UTF-8 mapping file for font encoding C43
Now handling font encoding C50 ...
... no UTF-8 mapping file for font encoding C50
Now handling font encoding C52 ...
... no UTF-8 mapping file for font encoding C52
Now handling font encoding C49 ...
... no UTF-8 mapping file for font encoding C49
Now handling font encoding C60 ...
... no UTF-8 mapping file for font encoding C60
Now handling font encoding C61 ...
... no UTF-8 mapping file for font encoding C61
Now handling font encoding C63 ...
... no UTF-8 mapping file for font encoding C63
Now handling font encoding C64 ...
... no UTF-8 mapping file for font encoding C64
Now handling font encoding C65 ...
... no UTF-8 mapping file for font encoding C65
Now handling font encoding C70 ...
... no UTF-8 mapping file for font encoding C70
Now handling font encoding C31 ...
... no UTF-8 mapping file for font encoding C31
Now handling font encoding C32 ...
... no UTF-8 mapping file for font encoding C32
Now handling font encoding C33 ...
... no UTF-8 mapping file for font encoding C33
Now handling font encoding C34 ...
... no UTF-8 mapping file for font encoding C34
Now handling font encoding C35 ...
... no UTF-8 mapping file for font encoding C35
Now handling font encoding C36 ...
... no UTF-8 mapping file for font encoding C36
Now handling font encoding C37 ...
... no UTF-8 mapping file for font encoding C37
Now handling font encoding C80 ...
... no UTF-8 mapping file for font encoding C80
Now handling font encoding C81 ...
... no UTF-8 mapping file for font encoding C81
Now handling font encoding C01 ...
... no UTF-8 mapping file for font encoding C01
Now handling font encoding C11 ...
... no UTF-8 mapping file for font encoding C11
Now handling font encoding C21 ...
... no UTF-8 mapping file for font encoding C21
Now handling font encoding C41 ...
... no UTF-8 mapping file for font encoding C41
Now handling font encoding C62 ...
... no UTF-8 mapping file for font encoding C62
)
\CJK@indent=\box55
) (d:/software/texlive/2026/texmf-dist/tex/latex/algorithms/algorithm.sty
Package: algorithm 2009/08/24 v0.1 Document Style `algorithm' - floating environment
(d:/software/texlive/2026/texmf-dist/tex/latex/float/float.sty
Package: float 2001/11/08 v1.3d Float enhancements (AL)
\c@float@type=\count313
\float@exts=\toks24
\float@box=\box56
\@float@everytoks=\toks25
\@floatcapt=\box57
) (d:/software/texlive/2026/texmf-dist/tex/latex/base/ifthen.sty
Package: ifthen 2024/03/16 v1.1e Standard LaTeX ifthen package (DPC)
)
\@float@every@algorithm=\toks26
\c@algorithm=\count314
) (d:/software/texlive/2026/texmf-dist/tex/latex/algorithmicx/algpseudocode.sty
Package: algpseudocode
(d:/software/texlive/2026/texmf-dist/tex/latex/algorithmicx/algorithmicx.sty
Package: algorithmicx 2005/04/27 v1.2 Algorithmicx
Document Style algorithmicx 1.2 - a greatly improved `algorithmic' style
\c@ALG@line=\count315
\c@ALG@rem=\count316
\c@ALG@nested=\count317
\ALG@tlm=\skip64
\ALG@thistlm=\skip65
\c@ALG@Lnr=\count318
\c@ALG@blocknr=\count319
\c@ALG@storecount=\count320
\c@ALG@tmpcounter=\count321
\ALG@tmplength=\skip66
)
Document Style - pseudocode environments for use with the `algorithmicx' style
) (d:/software/texlive/2026/texmf-dist/tex/latex/tikz-qtree/tikz-qtree.sty (d:/software/texlive/2026/texmf-dist/tex/latex/pgf/frontendlayer/tikz.sty (d:/software/texlive/2026/texmf-dist/tex/latex/pgf/basiclayer/pgf.sty (d:/software/texlive/2026/texmf-dist/tex/latex/pgf/utilities/pgfrcs.sty (d:/software/texlive/2026/texmf-dist/tex/generic/pgf/utilities/pgfutil-common.tex
\pgfutil@everybye=\toks27
\pgfutil@tempdima=\dimen176
\pgfutil@tempdimb=\dimen177
) (d:/software/texlive/2026/texmf-dist/tex/generic/pgf/utilities/pgfutil-latex.def
\pgfutil@abb=\box58
) (d:/software/texlive/2026/texmf-dist/tex/generic/pgf/utilities/pgfrcs.code.tex (d:/software/texlive/2026/texmf-dist/tex/generic/pgf/pgf.revision.tex)
Package: pgfrcs 2025-08-29 v3.1.11a (3.1.11a)
))
Package: pgf 2025-08-29 v3.1.11a (3.1.11a)
(d:/software/texlive/2026/texmf-dist/tex/latex/pgf/basiclayer/pgfcore.sty (d:/software/texlive/2026/texmf-dist/tex/latex/graphics/graphicx.sty
Package: graphicx 2026-05-17 v1.2e Enhanced LaTeX Graphics (DPC,SPQR)
(d:/software/texlive/2026/texmf-dist/tex/latex/graphics/graphics.sty
Package: graphics 2026-05-17 v1.4h Standard LaTeX Graphics (DPC,SPQR)
(d:/software/texlive/2026/texmf-dist/tex/latex/graphics/trig.sty
Package: trig 2023/12/02 v1.11 sin cos tan (DPC)
) (d:/software/texlive/2026/texmf-dist/tex/latex/graphics-cfg/graphics.cfg
File: graphics.cfg 2016/06/04 v1.11 sample graphics configuration
)
Package graphics Info: Driver file: pdftex.def on input line 106.
)
\Gin@req@height=\dimen178
\Gin@req@width=\dimen179
) (d:/software/texlive/2026/texmf-dist/tex/latex/pgf/systemlayer/pgfsys.sty (d:/software/texlive/2026/texmf-dist/tex/generic/pgf/systemlayer/pgfsys.code.tex
Package: pgfsys 2025-08-29 v3.1.11a (3.1.11a)
(d:/software/texlive/2026/texmf-dist/tex/generic/pgf/utilities/pgfkeys.code.tex
\pgfkeys@pathtoks=\toks28
\pgfkeys@temptoks=\toks29
(d:/software/texlive/2026/texmf-dist/tex/generic/pgf/utilities/pgfkeyslibraryfiltered.code.tex
\pgfkeys@tmptoks=\toks30
))
\pgf@x=\dimen180
\pgf@y=\dimen181
\pgf@xa=\dimen182
\pgf@ya=\dimen183
\pgf@xb=\dimen184
\pgf@yb=\dimen185
\pgf@xc=\dimen186
\pgf@yc=\dimen187
\pgf@xd=\dimen188
\pgf@yd=\dimen189
\w@pgf@writea=\write3
\r@pgf@reada=\read2
\c@pgf@counta=\count322
\c@pgf@countb=\count323
\c@pgf@countc=\count324
\c@pgf@countd=\count325
\t@pgf@toka=\toks31
\t@pgf@tokb=\toks32
\t@pgf@tokc=\toks33
\pgf@sys@id@count=\count326
(d:/software/texlive/2026/texmf-dist/tex/generic/pgf/systemlayer/pgf.cfg
File: pgf.cfg 2025-08-29 v3.1.11a (3.1.11a)
)
Driver file for pgf: pgfsys-pdftex.def
(d:/software/texlive/2026/texmf-dist/tex/generic/pgf/systemlayer/pgfsys-pdftex.def
File: pgfsys-pdftex.def 2025-08-29 v3.1.11a (3.1.11a)
(d:/software/texlive/2026/texmf-dist/tex/generic/pgf/systemlayer/pgfsys-common-pdf.def
File: pgfsys-common-pdf.def 2025-08-29 v3.1.11a (3.1.11a)
))) (d:/software/texlive/2026/texmf-dist/tex/generic/pgf/systemlayer/pgfsyssoftpath.code.tex
File: pgfsyssoftpath.code.tex 2025-08-29 v3.1.11a (3.1.11a)
\pgfsyssoftpath@smallbuffer@items=\count327
\pgfsyssoftpath@bigbuffer@items=\count328
) (d:/software/texlive/2026/texmf-dist/tex/generic/pgf/systemlayer/pgfsysprotocol.code.tex
File: pgfsysprotocol.code.tex 2025-08-29 v3.1.11a (3.1.11a)
)) (d:/software/texlive/2026/texmf-dist/tex/generic/pgf/basiclayer/pgfcore.code.tex
Package: pgfcore 2025-08-29 v3.1.11a (3.1.11a)
(d:/software/texlive/2026/texmf-dist/tex/generic/pgf/math/pgfmath.code.tex (d:/software/texlive/2026/texmf-dist/tex/generic/pgf/math/pgfmathutil.code.tex) (d:/software/texlive/2026/texmf-dist/tex/generic/pgf/math/pgfmathparser.code.tex
\pgfmath@dimen=\dimen190
\pgfmath@count=\count329
\pgfmath@box=\box59
\pgfmath@toks=\toks34
\pgfmath@stack@operand=\toks35
\pgfmath@stack@operation=\toks36
) (d:/software/texlive/2026/texmf-dist/tex/generic/pgf/math/pgfmathfunctions.code.tex) (d:/software/texlive/2026/texmf-dist/tex/generic/pgf/math/pgfmathfunctions.basic.code.tex) (d:/software/texlive/2026/texmf-dist/tex/generic/pgf/math/pgfmathfunctions.trigonometric.code.tex) (d:/software/texlive/2026/texmf-dist/tex/generic/pgf/math/pgfmathfunctions.random.code.tex) (d:/software/texlive/2026/texmf-dist/tex/generic/pgf/math/pgfmathfunctions.comparison.code.tex) (d:/software/texlive/2026/texmf-dist/tex/generic/pgf/math/pgfmathfunctions.base.code.tex) (d:/software/texlive/2026/texmf-dist/tex/generic/pgf/math/pgfmathfunctions.round.code.tex) (d:/software/texlive/2026/texmf-dist/tex/generic/pgf/math/pgfmathfunctions.misc.code.tex) (d:/software/texlive/2026/texmf-dist/tex/generic/pgf/math/pgfmathfunctions.integerarithmetics.code.tex) (d:/software/texlive/2026/texmf-dist/tex/generic/pgf/math/pgfmathcalc.code.tex) (d:/software/texlive/2026/texmf-dist/tex/generic/pgf/math/pgfmathfloat.code.tex
\c@pgfmathroundto@lastzeros=\count330
)) (d:/software/texlive/2026/texmf-dist/tex/generic/pgf/math/pgfint.code.tex) (d:/software/texlive/2026/texmf-dist/tex/generic/pgf/basiclayer/pgfcorepoints.code.tex
File: pgfcorepoints.code.tex 2025-08-29 v3.1.11a (3.1.11a)
\pgf@picminx=\dimen191
\pgf@picmaxx=\dimen192
\pgf@picminy=\dimen193
\pgf@picmaxy=\dimen194
\pgf@pathminx=\dimen195
\pgf@pathmaxx=\dimen196
\pgf@pathminy=\dimen197
\pgf@pathmaxy=\dimen198
\pgf@xx=\dimen199
\pgf@xy=\dimen256
\pgf@yx=\dimen257
\pgf@yy=\dimen258
\pgf@zx=\dimen259
\pgf@zy=\dimen260
LaTeX Font Info: Trying to load font information for OT1+qpl on input line 926.
(d:/software/texlive/2026/texmf-dist/tex/latex/tex-gyre/ot1qpl.fd
File: ot1qpl.fd 2009/09/25 v1.2 font definition file for OT1/qpl
)) (d:/software/texlive/2026/texmf-dist/tex/generic/pgf/basiclayer/pgfcorepathconstruct.code.tex
File: pgfcorepathconstruct.code.tex 2025-08-29 v3.1.11a (3.1.11a)
\pgf@path@lastx=\dimen261
\pgf@path@lasty=\dimen262
) (d:/software/texlive/2026/texmf-dist/tex/generic/pgf/basiclayer/pgfcorepathusage.code.tex
File: pgfcorepathusage.code.tex 2025-08-29 v3.1.11a (3.1.11a)
\pgf@shorten@end@additional=\dimen263
\pgf@shorten@start@additional=\dimen264
) (d:/software/texlive/2026/texmf-dist/tex/generic/pgf/basiclayer/pgfcorescopes.code.tex
File: pgfcorescopes.code.tex 2025-08-29 v3.1.11a (3.1.11a)
\pgfpic=\box60
\pgf@hbox=\box61
\pgf@layerbox@main=\box62
\pgf@picture@serial@count=\count331
) (d:/software/texlive/2026/texmf-dist/tex/generic/pgf/basiclayer/pgfcoregraphicstate.code.tex
File: pgfcoregraphicstate.code.tex 2025-08-29 v3.1.11a (3.1.11a)
\pgflinewidth=\dimen265
) (d:/software/texlive/2026/texmf-dist/tex/generic/pgf/basiclayer/pgfcoretransformations.code.tex
File: pgfcoretransformations.code.tex 2025-08-29 v3.1.11a (3.1.11a)
\pgf@pt@x=\dimen266
\pgf@pt@y=\dimen267
\pgf@pt@temp=\dimen268
) (d:/software/texlive/2026/texmf-dist/tex/generic/pgf/basiclayer/pgfcorequick.code.tex
File: pgfcorequick.code.tex 2025-08-29 v3.1.11a (3.1.11a)
) (d:/software/texlive/2026/texmf-dist/tex/generic/pgf/basiclayer/pgfcoreobjects.code.tex
File: pgfcoreobjects.code.tex 2025-08-29 v3.1.11a (3.1.11a)
) (d:/software/texlive/2026/texmf-dist/tex/generic/pgf/basiclayer/pgfcorepathprocessing.code.tex
File: pgfcorepathprocessing.code.tex 2025-08-29 v3.1.11a (3.1.11a)
) (d:/software/texlive/2026/texmf-dist/tex/generic/pgf/basiclayer/pgfcorearrows.code.tex
File: pgfcorearrows.code.tex 2025-08-29 v3.1.11a (3.1.11a)
\pgfarrowsep=\dimen269
) (d:/software/texlive/2026/texmf-dist/tex/generic/pgf/basiclayer/pgfcoreshade.code.tex
File: pgfcoreshade.code.tex 2025-08-29 v3.1.11a (3.1.11a)
\pgf@max=\dimen270
\pgf@sys@shading@range@num=\count332
\pgf@shadingcount=\count333
) (d:/software/texlive/2026/texmf-dist/tex/generic/pgf/basiclayer/pgfcoreimage.code.tex
File: pgfcoreimage.code.tex 2025-08-29 v3.1.11a (3.1.11a)
) (d:/software/texlive/2026/texmf-dist/tex/generic/pgf/basiclayer/pgfcoreexternal.code.tex
File: pgfcoreexternal.code.tex 2025-08-29 v3.1.11a (3.1.11a)
\pgfexternal@startupbox=\box63
) (d:/software/texlive/2026/texmf-dist/tex/generic/pgf/basiclayer/pgfcorelayers.code.tex
File: pgfcorelayers.code.tex 2025-08-29 v3.1.11a (3.1.11a)
) (d:/software/texlive/2026/texmf-dist/tex/generic/pgf/basiclayer/pgfcoretransparency.code.tex
File: pgfcoretransparency.code.tex 2025-08-29 v3.1.11a (3.1.11a)
) (d:/software/texlive/2026/texmf-dist/tex/generic/pgf/basiclayer/pgfcorepatterns.code.tex
File: pgfcorepatterns.code.tex 2025-08-29 v3.1.11a (3.1.11a)
) (d:/software/texlive/2026/texmf-dist/tex/generic/pgf/basiclayer/pgfcorerdf.code.tex
File: pgfcorerdf.code.tex 2025-08-29 v3.1.11a (3.1.11a)
))) (d:/software/texlive/2026/texmf-dist/tex/generic/pgf/modules/pgfmoduleshapes.code.tex
File: pgfmoduleshapes.code.tex 2025-08-29 v3.1.11a (3.1.11a)
\pgfnodeparttextbox=\box64
) (d:/software/texlive/2026/texmf-dist/tex/generic/pgf/modules/pgfmoduleplot.code.tex
File: pgfmoduleplot.code.tex 2025-08-29 v3.1.11a (3.1.11a)
) (d:/software/texlive/2026/texmf-dist/tex/latex/pgf/compatibility/pgfcomp-version-0-65.sty
Package: pgfcomp-version-0-65 2025-08-29 v3.1.11a (3.1.11a)
\pgf@nodesepstart=\dimen271
\pgf@nodesepend=\dimen272
) (d:/software/texlive/2026/texmf-dist/tex/latex/pgf/compatibility/pgfcomp-version-1-18.sty
Package: pgfcomp-version-1-18 2025-08-29 v3.1.11a (3.1.11a)
)) (d:/software/texlive/2026/texmf-dist/tex/latex/pgf/utilities/pgffor.sty (d:/software/texlive/2026/texmf-dist/tex/latex/pgf/utilities/pgfkeys.sty (d:/software/texlive/2026/texmf-dist/tex/generic/pgf/utilities/pgfkeys.code.tex)) (d:/software/texlive/2026/texmf-dist/tex/latex/pgf/math/pgfmath.sty (d:/software/texlive/2026/texmf-dist/tex/generic/pgf/math/pgfmath.code.tex)) (d:/software/texlive/2026/texmf-dist/tex/generic/pgf/utilities/pgffor.code.tex
Package: pgffor 2025-08-29 v3.1.11a (3.1.11a)
\pgffor@iter=\dimen273
\pgffor@skip=\dimen274
\pgffor@stack=\toks37
\pgffor@toks=\toks38
)) (d:/software/texlive/2026/texmf-dist/tex/generic/pgf/frontendlayer/tikz/tikz.code.tex
Package: tikz 2025-08-29 v3.1.11a (3.1.11a)
(d:/software/texlive/2026/texmf-dist/tex/generic/pgf/libraries/pgflibraryplothandlers.code.tex
File: pgflibraryplothandlers.code.tex 2025-08-29 v3.1.11a (3.1.11a)
\pgf@plot@mark@count=\count334
\pgfplotmarksize=\dimen275
)
\tikz@lastx=\dimen276
\tikz@lasty=\dimen277
\tikz@lastxsaved=\dimen278
\tikz@lastysaved=\dimen279
\tikz@lastmovetox=\dimen280
\tikz@lastmovetoy=\dimen281
\tikzleveldistance=\dimen282
\tikzsiblingdistance=\dimen283
\tikz@figbox=\box65
\tikz@figbox@bg=\box66
\tikz@tempbox=\box67
\tikz@tempbox@bg=\box68
\tikztreelevel=\count335
\tikznumberofchildren=\count336
\tikznumberofcurrentchild=\count337
\tikz@fig@count=\count338
(d:/software/texlive/2026/texmf-dist/tex/generic/pgf/modules/pgfmodulematrix.code.tex
File: pgfmodulematrix.code.tex 2025-08-29 v3.1.11a (3.1.11a)
\pgfmatrixcurrentrow=\count339
\pgfmatrixcurrentcolumn=\count340
\pgf@matrix@numberofcolumns=\count341
)
\tikz@expandcount=\count342
(d:/software/texlive/2026/texmf-dist/tex/generic/pgf/frontendlayer/tikz/libraries/tikzlibrarytopaths.code.tex
File: tikzlibrarytopaths.code.tex 2025-08-29 v3.1.11a (3.1.11a)
))) (d:/software/texlive/2026/texmf-dist/tex/latex/tikz-qtree/pgftree.sty (d:/software/texlive/2026/texmf-dist/tex/latex/tikz-qtree/pgfsubpic.sty (d:/software/texlive/2026/texmf-dist/tex/latex/tikz-qtree/pgfsubpic.tex
\pgf@subpicminx=\dimen284
\pgf@subpicminy=\dimen285
\pgf@subpicmaxx=\dimen286
\pgf@subpicmaxy=\dimen287
)) (d:/software/texlive/2026/texmf-dist/tex/latex/tikz-qtree/pgftree.tex
\levelsep=\dimen288
\subtreesep=\dimen289
\smuggle@levelsep=\dimen290
\smuggle@subtreesep=\dimen291
\this@levelsep=\dimen292
\this@subtreesep=\dimen293
\pgftree@childx=\dimen294
\pgftree@savechildx=\dimen295
\pgftree@childy=\dimen296
\pgftree@savechildy=\dimen297
\pgftree@childi=\count343
\pgftree@savechildi=\count344
\pgftree@level=\count345
\pgftree@depth=\dimen298
\pgf@subpic@hbox@children=\box69
\pgf@subpic@minx@children=\dimen299
\pgf@subpic@miny@children=\dimen300
\pgf@subpic@maxx@children=\dimen301
\pgf@subpic@maxy@children=\dimen302
\pgftree@lastchildx=\dimen303
\pgftree@lastchildy=\dimen304
)) (d:/software/texlive/2026/texmf-dist/tex/latex/tikz-qtree/tikz-qtree.tex
\@result=\toks39
\child@list=\toks40
\root@node=\toks41
)) (d:/software/texlive/2026/texmf-dist/tex/latex/tcolorbox/tcolorbox.sty
Package: tcolorbox 2026/05/28 version 6.10.0 text color boxes
(d:/software/texlive/2026/texmf-dist/tex/latex/tools/verbatim.sty
Package: verbatim 2024-01-22 v1.5x LaTeX2e package for verbatim enhancements
\every@verbatim=\toks42
\verbatim@line=\toks43
\verbatim@in@stream=\read3
) (d:/software/texlive/2026/texmf-dist/tex/latex/environ/environ.sty
Package: environ 2014/05/04 v0.3 A new way to define environments
(d:/software/texlive/2026/texmf-dist/tex/latex/trimspaces/trimspaces.sty
Package: trimspaces 2009/09/17 v1.1 Trim spaces around a token list
)
\@envbody=\toks44
)
\tcb@titlebox=\box70
\tcb@upperbox=\box71
\tcb@lowerbox=\box72
\tcb@phantombox=\box73
\c@tcbbreakpart=\count346
\c@tcblayer=\count347
\c@tcolorbox@number=\count348
\l__tcobox_tmpa_box=\box74
\l__tcobox_tmpa_dim=\dimen305
\tcb@temp=\box75
\tcb@temp=\box76
\tcb@temp=\box77
\tcb@temp=\box78
) (d:/software/texlive/2026/texmf-dist/tex/generic/pgf/libraries/pgflibraryarrows.meta.code.tex
File: pgflibraryarrows.meta.code.tex 2025-08-29 v3.1.11a (3.1.11a)
\pgfarrowinset=\dimen306
\pgfarrowlength=\dimen307
\pgfarrowwidth=\dimen308
\pgfarrowlinewidth=\dimen309
) (d:/software/texlive/2026/texmf-dist/tex/generic/pgf/frontendlayer/tikz/libraries/tikzlibrarybending.code.tex
File: tikzlibrarybending.code.tex 2025-08-29 v3.1.11a (3.1.11a)
(d:/software/texlive/2026/texmf-dist/tex/generic/pgf/modules/pgfmodulebending.code.tex
File: pgfmodulebending.code.tex 2025-08-29 v3.1.11a (3.1.11a)
(d:/software/texlive/2026/texmf-dist/tex/generic/pgf/modules/pgfmodulenonlineartransformations.code.tex
File: pgfmodulenonlineartransformations.code.tex 2025-08-29 v3.1.11a (3.1.11a)
\pgftransformnonlinearflatness=\dimen310
) (d:/software/texlive/2026/texmf-dist/tex/generic/pgf/libraries/pgflibrarycurvilinear.code.tex
File: pgflibrarycurvilinear.code.tex 2025-08-29 v3.1.11a (3.1.11a)
\pgf@curvilinear@time@a=\dimen311
\pgf@curvilinear@length@a=\dimen312
\pgf@curvilinear@length@b=\dimen313
\pgf@curvilinear@length@c=\dimen314
\pgf@curvilinear@length@d=\dimen315
)
\pgf@arrows@the@rigidity=\dimen316
)) (d:/software/texlive/2026/texmf-dist/tex/generic/pgf/frontendlayer/tikz/libraries/tikzlibrarypositioning.code.tex
File: tikzlibrarypositioning.code.tex 2025-08-29 v3.1.11a (3.1.11a)
) (d:/software/texlive/2026/texmf-dist/tex/generic/pgf/frontendlayer/tikz/libraries/tikzlibraryangles.code.tex
File: tikzlibraryangles.code.tex 2025-08-29 v3.1.11a (3.1.11a)
) (d:/software/texlive/2026/texmf-dist/tex/generic/pgf/frontendlayer/tikz/libraries/tikzlibraryquotes.code.tex
File: tikzlibraryquotes.code.tex 2025-08-29 v3.1.11a (3.1.11a)
) (d:/software/texlive/2026/texmf-dist/tex/generic/pgf/frontendlayer/tikz/libraries/tikzlibraryfit.code.tex
File: tikzlibraryfit.code.tex 2025-08-29 v3.1.11a (3.1.11a)
) (d:/software/texlive/2026/texmf-dist/tex/generic/pgf/frontendlayer/tikz/libraries/tikzlibrarybackgrounds.code.tex
File: tikzlibrarybackgrounds.code.tex 2025-08-29 v3.1.11a (3.1.11a)
\pgf@layerbox@background=\box79
\pgf@layerboxsaved@background=\box80
) (d:/software/texlive/2026/texmf-dist/tex/generic/pgf/frontendlayer/tikz/libraries/tikzlibraryshadows.code.tex
File: tikzlibraryshadows.code.tex 2025-08-29 v3.1.11a (3.1.11a)
(d:/software/texlive/2026/texmf-dist/tex/generic/pgf/frontendlayer/tikz/libraries/tikzlibraryfadings.code.tex
File: tikzlibraryfadings.code.tex 2025-08-29 v3.1.11a (3.1.11a)
(d:/software/texlive/2026/texmf-dist/tex/generic/pgf/libraries/pgflibraryfadings.code.tex
File: pgflibraryfadings.code.tex 2025-08-29 v3.1.11a (3.1.11a)
))) (d:/software/texlive/2026/texmf-dist/tex/generic/pgf/frontendlayer/tikz/libraries/tikzlibrarycalc.code.tex
File: tikzlibrarycalc.code.tex 2025-08-29 v3.1.11a (3.1.11a)
) (d:/software/texlive/2026/texmf-dist/tex/generic/pgf/frontendlayer/tikz/libraries/tikzlibrarymatrix.code.tex
File: tikzlibrarymatrix.code.tex 2025-08-29 v3.1.11a (3.1.11a)
) (d:/software/texlive/2026/texmf-dist/tex/latex/pgfplots/pgfplots.sty (d:/software/texlive/2026/texmf-dist/tex/generic/pgfplots/pgfplots.revision.tex)
Package: pgfplots 2025/08/14 v1.18.2 Data Visualization (1.18.2)
(d:/software/texlive/2026/texmf-dist/tex/generic/pgfplots/pgfplots.code.tex (d:/software/texlive/2026/texmf-dist/tex/generic/pgfplots/pgfplotscore.code.tex
\t@pgfplots@toka=\toks45
\t@pgfplots@tokb=\toks46
\t@pgfplots@tokc=\toks47
\pgfplots@tmpa=\dimen317
\c@pgfplots@coordindex=\count349
\c@pgfplots@scanlineindex=\count350
(d:/software/texlive/2026/texmf-dist/tex/generic/pgfplots/sys/pgfplotssysgeneric.code.tex)) (d:/software/texlive/2026/texmf-dist/tex/generic/pgfplots/libs/pgfplotslibrary.code.tex) (d:/software/texlive/2026/texmf-dist/tex/generic/pgfplots/oldpgfcompatib/pgfplotsoldpgfsupp_loader.code.tex (d:/software/texlive/2026/texmf-dist/tex/generic/pgf/libraries/pgflibraryfpu.code.tex) (d:/software/texlive/2026/texmf-dist/tex/generic/pgfplots/oldpgfcompatib/pgfplotsoldpgfsupp_pgfutil-common-lists.tex)) (d:/software/texlive/2026/texmf-dist/tex/generic/pgfplots/util/pgfplotsutil.code.tex (d:/software/texlive/2026/texmf-dist/tex/generic/pgfplots/liststructure/pgfplotsliststructure.code.tex) (d:/software/texlive/2026/texmf-dist/tex/generic/pgfplots/liststructure/pgfplotsliststructureext.code.tex) (d:/software/texlive/2026/texmf-dist/tex/generic/pgfplots/liststructure/pgfplotsarray.code.tex
\c@pgfplotsarray@tmp=\count351
) (d:/software/texlive/2026/texmf-dist/tex/generic/pgfplots/liststructure/pgfplotsmatrix.code.tex) (d:/software/texlive/2026/texmf-dist/tex/generic/pgfplots/numtable/pgfplotstableshared.code.tex
\c@pgfplotstable@counta=\count352
\t@pgfplotstable@a=\toks48
) (d:/software/texlive/2026/texmf-dist/tex/generic/pgfplots/liststructure/pgfplotsdeque.code.tex) (d:/software/texlive/2026/texmf-dist/tex/generic/pgfplots/util/pgfplotsbinary.code.tex (d:/software/texlive/2026/texmf-dist/tex/generic/pgfplots/util/pgfplotsbinary.data.code.tex)) (d:/software/texlive/2026/texmf-dist/tex/generic/pgfplots/util/pgfplotsutil.verb.code.tex) (d:/software/texlive/2026/texmf-dist/tex/generic/pgfplots/libs/pgflibrarypgfplots.surfshading.code.tex
\c@pgfplotslibrarysurf@no=\count353
(d:/software/texlive/2026/texmf-dist/tex/generic/pgfplots/sys/pgflibrarypgfplots.surfshading.pgfsys-pdftex.def))) (d:/software/texlive/2026/texmf-dist/tex/generic/pgfplots/util/pgfplotscolormap.code.tex (d:/software/texlive/2026/texmf-dist/tex/generic/pgfplots/util/pgfplotscolor.code.tex)) (d:/software/texlive/2026/texmf-dist/tex/generic/pgfplots/pgfplotsstackedplots.code.tex) (d:/software/texlive/2026/texmf-dist/tex/generic/pgfplots/pgfplotsplothandlers.code.tex (d:/software/texlive/2026/texmf-dist/tex/generic/pgfplots/pgfplotsmeshplothandler.code.tex (d:/software/texlive/2026/texmf-dist/tex/generic/pgfplots/pgfplotsmeshplotimage.code.tex))) (d:/software/texlive/2026/texmf-dist/tex/generic/pgfplots/pgfplots.scaling.code.tex) (d:/software/texlive/2026/texmf-dist/tex/generic/pgfplots/pgfplotscoordprocessing.code.tex) (d:/software/texlive/2026/texmf-dist/tex/generic/pgfplots/pgfplots.errorbars.code.tex) (d:/software/texlive/2026/texmf-dist/tex/generic/pgfplots/pgfplots.markers.code.tex) (d:/software/texlive/2026/texmf-dist/tex/generic/pgfplots/pgfplotsticks.code.tex) (d:/software/texlive/2026/texmf-dist/tex/generic/pgfplots/pgfplots.paths.code.tex) (d:/software/texlive/2026/texmf-dist/tex/generic/pgf/frontendlayer/tikz/libraries/tikzlibrarydecorations.code.tex (d:/software/texlive/2026/texmf-dist/tex/generic/pgf/modules/pgfmoduledecorations.code.tex
\pgfdecoratedcompleteddistance=\dimen318
\pgfdecoratedremainingdistance=\dimen319
\pgfdecoratedinputsegmentcompleteddistance=\dimen320
\pgfdecoratedinputsegmentremainingdistance=\dimen321
\pgf@decorate@distancetomove=\dimen322
\pgf@decorate@repeatstate=\count354
\pgfdecorationsegmentamplitude=\dimen323
\pgfdecorationsegmentlength=\dimen324
)
\tikz@lib@dec@box=\box81
) (d:/software/texlive/2026/texmf-dist/tex/generic/pgf/frontendlayer/tikz/libraries/tikzlibrarydecorations.pathmorphing.code.tex (d:/software/texlive/2026/texmf-dist/tex/generic/pgf/libraries/decorations/pgflibrarydecorations.pathmorphing.code.tex)) (d:/software/texlive/2026/texmf-dist/tex/generic/pgf/frontendlayer/tikz/libraries/tikzlibrarydecorations.pathreplacing.code.tex (d:/software/texlive/2026/texmf-dist/tex/generic/pgf/libraries/decorations/pgflibrarydecorations.pathreplacing.code.tex)) (d:/software/texlive/2026/texmf-dist/tex/generic/pgfplots/libs/tikzlibrarypgfplots.contourlua.code.tex)
\pgfplots@numplots=\count355
\pgfplots@xmin@reg=\dimen325
\pgfplots@xmax@reg=\dimen326
\pgfplots@ymin@reg=\dimen327
\pgfplots@ymax@reg=\dimen328
\pgfplots@zmin@reg=\dimen329
\pgfplots@zmax@reg=\dimen330
) (d:/software/texlive/2026/texmf-dist/tex/generic/pgf/frontendlayer/tikz/libraries/tikzlibraryplotmarks.code.tex
File: tikzlibraryplotmarks.code.tex 2025-08-29 v3.1.11a (3.1.11a)
(d:/software/texlive/2026/texmf-dist/tex/generic/pgf/libraries/pgflibraryplotmarks.code.tex
File: pgflibraryplotmarks.code.tex 2025-08-29 v3.1.11a (3.1.11a)
))) (d:/software/texlive/2026/texmf-dist/tex/generic/soul/soul.sty
Package: soul 2026-03-06 v3.2 Permit use of UTF-8 characters in soul (HO)
(d:/software/texlive/2026/texmf-dist/tex/generic/soul/soul-ori.sty
Package: soul-ori 2026-03-06 v3.2 letterspacing/underlining (mf)
\SOUL@word=\toks49
\SOUL@lasttoken=\toks50
\SOUL@syllable=\toks51
\SOUL@cmds=\toks52
\SOUL@buffer=\toks53
\SOUL@token=\toks54
\SOUL@syllgoal=\dimen331
\SOUL@syllwidth=\dimen332
\SOUL@charkern=\dimen333
\SOUL@hyphkern=\dimen334
\SOUL@dimen=\dimen335
\SOUL@dimeni=\dimen336
\SOUL@minus=\count356
\SOUL@comma=\count357
\SOUL@apo=\count358
\SOUL@grave=\count359
\SOUL@spaceskip=\skip67
\SOUL@ttwidth=\dimen337
\SOUL@uldp=\dimen338
\SOUL@ulht=\dimen339
) (d:/software/texlive/2026/texmf-dist/tex/generic/etexcmds/etexcmds.sty
Package: etexcmds 2019/12/15 v1.7 Avoid name clashes with e-TeX commands (HO)
)) (d:/software/texlive/2026/texmf-dist/tex/latex/makecell/makecell.sty
Package: makecell 2009/08/03 V0.1e Managing of Tab Column Heads and Cells
(d:/software/texlive/2026/texmf-dist/tex/latex/tools/array.sty
Package: array 2026/02/24 v2.7b Tabular extension package (FMi)
\col@sep=\dimen340
\ar@mcellbox=\box82
\extrarowheight=\dimen341
\NC@list=\toks55
\extratabsurround=\skip68
\backup@length=\skip69
\ar@cellbox=\box83
)
\rotheadsize=\dimen342
\c@nlinenum=\count360
\TeXr@lab=\toks56
) (d:/software/texlive/2026/texmf-dist/tex/generic/ulem/ulem.sty
\UL@box=\box84
\UL@hyphenbox=\box85
\UL@skip=\skip70
\UL@hook=\toks57
\UL@height=\dimen343
\UL@pe=\count361
\UL@pixel=\dimen344
\ULC@box=\box86
Package: ulem 2019/11/18
\ULdepth=\dimen345
) (d:/software/texlive/2026/texmf-dist/tex/latex/multirow/multirow.sty
Package: multirow 2024/11/12 v2.9 Span multiple rows of a table
\multirow@colwidth=\skip71
\multirow@cntb=\count362
\multirow@dima=\skip72
\bigstrutjot=\dimen346
) (d:/software/texlive/2026/texmf-dist/tex/latex/setspace/setspace.sty
Package: setspace 2022/12/04 v6.7b set line spacing
) (d:/software/texlive/2026/texmf-dist/tex/latex/varwidth/varwidth.sty
Package: varwidth 2009/03/30 ver 0.92; Variable-width minipages
\@vwid@box=\box87
\sift@deathcycles=\count363
\@vwid@loff=\dimen347
\@vwid@roff=\dimen348
)
==> First Aid for varwidth.sty applied!
(d:/software/texlive/2026/texmf-dist/tex/latex/amsmath/amsmath.sty
Package: amsmath 2026/05/19 v2.18d AMS math features
\@mathmargin=\skip73
For additional information on amsmath, use the `?' option.
(d:/software/texlive/2026/texmf-dist/tex/latex/amsmath/amstext.sty
Package: amstext 2024/11/17 v2.01 AMS text
(d:/software/texlive/2026/texmf-dist/tex/latex/amsmath/amsgen.sty
File: amsgen.sty 1999/11/30 v2.0 generic functions
\@emptytoks=\toks58
\ex@=\dimen349
)) (d:/software/texlive/2026/texmf-dist/tex/latex/amsmath/amsbsy.sty
Package: amsbsy 1999/11/29 v1.2d Bold Symbols
\pmbraise@=\dimen350
) (d:/software/texlive/2026/texmf-dist/tex/latex/amsmath/amsopn.sty
Package: amsopn 2022/04/08 v2.04 operator names
)
\inf@bad=\count364
LaTeX Info: Redefining \frac on input line 233.
\uproot@=\count365
\leftroot@=\count366
LaTeX Info: Redefining \overline on input line 398.
LaTeX Info: Redefining \colon on input line 409.
\classnum@=\count367
\DOTSCASE@=\count368
LaTeX Info: Redefining \ldots on input line 495.
LaTeX Info: Redefining \dots on input line 498.
LaTeX Info: Redefining \cdots on input line 619.
\Mathstrutbox@=\box88
\strutbox@=\box89
LaTeX Info: Redefining \big on input line 721.
LaTeX Info: Redefining \Big on input line 722.
LaTeX Info: Redefining \bigg on input line 723.
LaTeX Info: Redefining \Bigg on input line 724.
\big@size=\dimen351
LaTeX Font Info: Redeclaring font encoding OML on input line 742.
LaTeX Font Info: Redeclaring font encoding OMS on input line 743.
\macc@depth=\count369
LaTeX Info: Redefining \bmod on input line 904.
LaTeX Info: Redefining \pmod on input line 909.
LaTeX Info: Redefining \smash on input line 939.
LaTeX Info: Redefining \relbar on input line 969.
LaTeX Info: Redefining \Relbar on input line 970.
\c@MaxMatrixCols=\count370
\dotsspace@=\muskip18
\c@parentequation=\count371
\dspbrk@lvl=\count372
\tag@help=\toks59
\row@=\count373
\column@=\count374
\maxfields@=\count375
\andhelp@=\toks60
\eqnshift@=\dimen352
\alignsep@=\dimen353
\tagshift@=\dimen354
\tagwidth@=\dimen355
\totwidth@=\dimen356
\lineht@=\dimen357
\@envbody=\toks61
\multlinegap=\skip74
\multlinetaggap=\skip75
\mathdisplay@stack=\toks62
LaTeX Info: Redefining \[ on input line 2965.
LaTeX Info: Redefining \] on input line 2966.
)
\@LN@amsmath@ams@eqpen=\count376
(d:/software/texlive/2026/texmf-dist/tex/latex/amsfonts/amssymb.sty
Package: amssymb 2013/01/14 v3.01 AMS font symbols
(d:/software/texlive/2026/texmf-dist/tex/latex/amsfonts/amsfonts.sty
Package: amsfonts 2013/01/14 v3.01 Basic AMSFonts support
\symAMSa=\mathgroup5
\symAMSb=\mathgroup6
LaTeX Font Info: Redeclaring math symbol \hbar on input line 98.
LaTeX Font Info: Overwriting math alphabet `\mathfrak' in version `bold'
(Font) U/euf/m/n --> U/euf/b/n on input line 106.
))
\symEulerExtension=\mathgroup7
Package hyperref Info: Option `colorlinks' set `true' on input line 110.
LaTeX Font Info: Trying to load font information for OT1+ppl on input line 146.
(d:/software/texlive/2026/texmf-dist/tex/latex/psnfss/ot1ppl.fd
File: ot1ppl.fd 2001/06/04 font definitions for OT1/ppl.
) (./rl-introduction.aux)
\openout1 = `rl-introduction.aux'.
LaTeX Font Info: Checking defaults for OML/cmm/m/it on input line 146.
LaTeX Font Info: ... okay on input line 146.
LaTeX Font Info: Checking defaults for OMS/cmsy/m/n on input line 146.
LaTeX Font Info: ... okay on input line 146.
LaTeX Font Info: Checking defaults for OT1/cmr/m/n on input line 146.
LaTeX Font Info: ... okay on input line 146.
LaTeX Font Info: Checking defaults for T1/cmr/m/n on input line 146.
LaTeX Font Info: ... okay on input line 146.
LaTeX Font Info: Checking defaults for TS1/cmr/m/n on input line 146.
LaTeX Font Info: ... okay on input line 146.
LaTeX Font Info: Checking defaults for OMX/cmex/m/n on input line 146.
LaTeX Font Info: ... okay on input line 146.
LaTeX Font Info: Checking defaults for U/cmr/m/n on input line 146.
LaTeX Font Info: ... okay on input line 146.
LaTeX Font Info: Checking defaults for PD1/pdf/m/n on input line 146.
LaTeX Font Info: ... okay on input line 146.
LaTeX Font Info: Checking defaults for PU/pdf/m/n on input line 146.
LaTeX Font Info: ... okay on input line 146.
LaTeX Font Info: Checking defaults for C00/song/m/n on input line 146.
LaTeX Font Info: ... okay on input line 146.
LaTeX Font Info: Checking defaults for C05/song/m/n on input line 146.
LaTeX Font Info: ... okay on input line 146.
LaTeX Font Info: Checking defaults for C09/song/m/n on input line 146.
LaTeX Font Info: ... okay on input line 146.
LaTeX Font Info: Checking defaults for C10/song/m/n on input line 146.
LaTeX Font Info: ... okay on input line 146.
LaTeX Font Info: Checking defaults for C20/song/m/n on input line 146.
LaTeX Font Info: ... okay on input line 146.
LaTeX Font Info: Checking defaults for C19/song/m/n on input line 146.
LaTeX Font Info: ... okay on input line 146.
LaTeX Font Info: Checking defaults for C40/song/m/n on input line 146.
LaTeX Font Info: ... okay on input line 146.
LaTeX Font Info: Checking defaults for C42/song/m/n on input line 146.
LaTeX Font Info: ... okay on input line 146.
LaTeX Font Info: Checking defaults for C43/song/m/n on input line 146.
LaTeX Font Info: ... okay on input line 146.
LaTeX Font Info: Checking defaults for C50/song/m/n on input line 146.
LaTeX Font Info: ... okay on input line 146.
LaTeX Font Info: Checking defaults for C52/song/m/n on input line 146.
LaTeX Font Info: ... okay on input line 146.
LaTeX Font Info: Checking defaults for C49/song/m/n on input line 146.
LaTeX Font Info: ... okay on input line 146.
LaTeX Font Info: Checking defaults for C60/mj/m/n on input line 146.
LaTeX Font Info: ... okay on input line 146.
LaTeX Font Info: Checking defaults for C61/mj/m/n on input line 146.
LaTeX Font Info: ... okay on input line 146.
LaTeX Font Info: Checking defaults for C63/mj/m/n on input line 146.
LaTeX Font Info: ... okay on input line 146.
LaTeX Font Info: Checking defaults for C64/mj/m/n on input line 146.
LaTeX Font Info: ... okay on input line 146.
LaTeX Font Info: Checking defaults for C65/mj/m/n on input line 146.
LaTeX Font Info: ... okay on input line 146.
LaTeX Font Info: Checking defaults for C70/song/m/n on input line 146.
LaTeX Font Info: ... okay on input line 146.
LaTeX Font Info: Checking defaults for C31/song/m/n on input line 146.
LaTeX Font Info: ... okay on input line 146.
LaTeX Font Info: Checking defaults for C32/song/m/n on input line 146.
LaTeX Font Info: ... okay on input line 146.
LaTeX Font Info: Checking defaults for C33/song/m/n on input line 146.
LaTeX Font Info: ... okay on input line 146.
LaTeX Font Info: Checking defaults for C34/song/m/n on input line 146.
LaTeX Font Info: ... okay on input line 146.
LaTeX Font Info: Checking defaults for C35/song/m/n on input line 146.
LaTeX Font Info: ... okay on input line 146.
LaTeX Font Info: Checking defaults for C36/song/m/n on input line 146.
LaTeX Font Info: ... okay on input line 146.
LaTeX Font Info: Checking defaults for C37/song/m/n on input line 146.
LaTeX Font Info: ... okay on input line 146.
LaTeX Font Info: Checking defaults for C80/song/m/n on input line 146.
LaTeX Font Info: ... okay on input line 146.
LaTeX Font Info: Checking defaults for C81/song/m/n on input line 146.
LaTeX Font Info: ... okay on input line 146.
LaTeX Font Info: Checking defaults for C01/song/m/n on input line 146.
LaTeX Font Info: ... okay on input line 146.
LaTeX Font Info: Checking defaults for C11/song/m/n on input line 146.
LaTeX Font Info: ... okay on input line 146.
LaTeX Font Info: Checking defaults for C21/song/m/n on input line 146.
LaTeX Font Info: ... okay on input line 146.
LaTeX Font Info: Checking defaults for C41/song/m/n on input line 146.
LaTeX Font Info: ... okay on input line 146.
LaTeX Font Info: Checking defaults for C62/song/m/n on input line 146.
LaTeX Font Info: ... okay on input line 146.
(d:/software/texlive/2026/texmf-dist/tex/latex/upquote/upquote.sty
Package: upquote 2012/04/19 v1.3 upright-quote and grave-accent glyphs in verbatim
) (d:/software/texlive/2026/texmf-dist/tex/context/base/mkii/supp-pdf.mkii
[Loading MPS to PDF converter (version 2006.09.02).]
\scratchcounter=\count377
\scratchdimen=\dimen358
\scratchbox=\box90
\nofMPsegments=\count378
\nofMParguments=\count379
\everyMPshowfont=\toks63
\MPscratchCnt=\count380
\MPscratchDim=\dimen359
\MPnumerator=\count381
\makeMPintoPDFobject=\count382
\everyMPtoPDFconversion=\toks64
) (d:/software/texlive/2026/texmf-dist/tex/latex/epstopdf-pkg/epstopdf-base.sty
Package: epstopdf-base 2020-01-24 v2.11 Base part for package epstopdf
Package epstopdf-base Info: Redefining graphics rule for `.eps' on input line 485.
(d:/software/texlive/2026/texmf-dist/tex/latex/latexconfig/epstopdf-sys.cfg
File: epstopdf-sys.cfg 2010/07/13 v1.3 Configuration of (r)epstopdf for TeX Live
))
Package microtype Info: Patching varwidth to enable character protrusion.
\MT@vwid@leftmargin=\dimen360
\MT@vwid@rightmargin=\dimen361
LaTeX Info: Redefining \microtypecontext on input line 146.
Package microtype Info: Applying patch `item' on input line 146.
Package microtype Info: Applying patch `toc' on input line 146.
Package microtype Info: Applying patch `eqnum' on input line 146.
Package microtype Warning: Unable to apply patch `footnote' on input line 146.
Package microtype Info: Applying patch `verbatim' on input line 146.
LaTeX Info: Redefining \microtypesetup on input line 146.
Package microtype Info: Generating PDF output.
Package microtype Info: Character protrusion enabled (level 2).
Package microtype Info: Using default protrusion set `alltext'.
Package microtype Info: Automatic font expansion enabled (level 2),
(microtype) stretch: 20, shrink: 20, step: 1, non-selected.
Package microtype Info: Using default expansion set `alltext-nott'.
Package microtype Info: Patching command `\showhyphens'.
Package microtype Info: No adjustment of tracking.
Package microtype Info: No adjustment of interword spacing.
Package microtype Info: No adjustment of character kerning.
(d:/software/texlive/2026/texmf-dist/tex/latex/microtype/mt-ppl.cfg
File: mt-ppl.cfg 2005/11/16 v1.6 microtype config. file: Palatino (RS)
)
Package hyperref Info: Link coloring ON on input line 146.
(./rl-introduction.out) (./rl-introduction.out)
\@outlinefile=\write4
\openout4 = `rl-introduction.out'.
Package pgfplots Warning: running in backwards compatibility mode (unsuitable tick labels; missing features). Consider writing \pgfplotsset{compat=1.18} into your preamble.
on input line 146.
LaTeX Font Info: Trying to load font information for OT1+zi4 on input line 153.
(d:/software/texlive/2026/texmf-dist/tex/latex/inconsolata/ot1zi4.fd
File: ot1zi4.fd 2018/01/14 OT1/zi4 (Inconsolata)
)
LaTeX Font Info: Font shape `OT1/zi4/m/n' will be
(Font) scaled to size 10.0pt on input line 153.
LaTeX Font Info: Trying to load font information for OMS+zi4 on input line 153.
LaTeX Font Info: No file OMSzi4.fd. on input line 153.
LaTeX Font Warning: Font shape `OMS/zi4/m/n' undefined
(Font) using `OMS/cmsy/m/n' instead
(Font) for symbol `textbraceleft' on input line 153.
Underfull \hbox (badness 10000) in paragraph at lines 153--153
[][]
[]
(./rl-introduction.toc
LaTeX Font Info: Trying to load font information for OML+zplm on input line 3.
(d:/software/texlive/2026/texmf-dist/tex/latex/psnfss/omlzplm.fd
File: omlzplm.fd 2002/09/08 Fontinst v1.914 font definitions for OML/zplm.
)
LaTeX Font Info: Trying to load font information for OMS+zplm on input line 3.
(d:/software/texlive/2026/texmf-dist/tex/latex/psnfss/omszplm.fd
File: omszplm.fd 2002/09/08 Fontinst v1.914 font definitions for OMS/zplm.
)
LaTeX Font Info: Trying to load font information for OMX+zplm on input line 3.
(d:/software/texlive/2026/texmf-dist/tex/latex/psnfss/omxzplm.fd
File: omxzplm.fd 2002/09/08 Fontinst v1.914 font definitions for OMX/zplm.
)
LaTeX Font Info: Trying to load font information for OT1+zplm on input line 3.
(d:/software/texlive/2026/texmf-dist/tex/latex/psnfss/ot1zplm.fd
File: ot1zplm.fd 2002/09/08 Fontinst v1.914 font definitions for OT1/zplm.
)
Package microtype Info: Loading generic protrusion settings for font family
(microtype) `zplm' (encoding: OT1).
(microtype) For optimal results, create family-specific settings.
(microtype) See the microtype manual for details.
LaTeX Font Info: Font shape `U/msa/m/n' will be
(Font) scaled to size 10.42007pt on input line 3.
(d:/software/texlive/2026/texmf-dist/tex/latex/microtype/mt-msa.cfg
File: mt-msa.cfg 2006/02/04 v1.1 microtype config. file: AMS symbols (a) (RS)
)
LaTeX Font Info: Font shape `U/msa/m/n' will be
(Font) scaled to size 7.91925pt on input line 3.
LaTeX Font Info: Font shape `U/msa/m/n' will be
(Font) scaled to size 6.25204pt on input line 3.
LaTeX Font Info: Font shape `U/msb/m/n' will be
(Font) scaled to size 10.42007pt on input line 3.
(d:/software/texlive/2026/texmf-dist/tex/latex/microtype/mt-msb.cfg
File: mt-msb.cfg 2005/06/01 v1.0 microtype config. file: AMS symbols (b) (RS)
)
LaTeX Font Info: Font shape `U/msb/m/n' will be
(Font) scaled to size 7.91925pt on input line 3.
LaTeX Font Info: Font shape `U/msb/m/n' will be
(Font) scaled to size 6.25204pt on input line 3.
LaTeX Font Info: Trying to load font information for U+euex on input line 3.
(d:/software/texlive/2026/texmf-dist/tex/latex/amsfonts/ueuex.fd
File: ueuex.fd 2013/01/14 v3.01 Euler extra symbols
) [1
{d:/software/texlive/2026/texmf-var/fonts/map/pdftex/updmap/pdftex.map}{d:/software/texlive/2026/texmf-dist/fonts/enc/dvips/base/8r.enc}{d:/software/texlive/2026/texmf-dist/fonts/enc/dvips/inconsolata/i4-ot1-0.enc}])
\tf@toc=\write5
\openout5 = `rl-introduction.toc'.
[2] (./section2/section2.tex
LaTeX Font Info: Font shape `U/msa/m/n' will be
(Font) scaled to size 9.37807pt on input line 9.
LaTeX Font Info: Font shape `U/msa/m/n' will be
(Font) scaled to size 7.29405pt on input line 9.
LaTeX Font Info: Font shape `U/msa/m/n' will be
(Font) scaled to size 5.21004pt on input line 9.
LaTeX Font Info: Font shape `U/msb/m/n' will be
(Font) scaled to size 9.37807pt on input line 9.
LaTeX Font Info: Font shape `U/msb/m/n' will be
(Font) scaled to size 7.29405pt on input line 9.
LaTeX Font Info: Font shape `U/msb/m/n' will be
(Font) scaled to size 5.21004pt on input line 9.
[3
] (./section2/Figures/rl_framework.tex
LaTeX Font Info: Font shape `U/msa/m/n' will be
(Font) scaled to size 12.50409pt on input line 59.
LaTeX Font Info: Font shape `U/msb/m/n' will be
(Font) scaled to size 12.50409pt on input line 59.
)
LaTeX Font Info: Trying to load font information for U+fplmbb on input line 45.
(d:/software/texlive/2026/texmf-dist/tex/latex/psnfss/ufplmbb.fd
File: ufplmbb.fd 2003/10/30 Fontinst v1.914 font definitions for U/fplmbb.
) [4]
LaTeX Font Info: Trying to load font information for TS1+ppl on input line 54.
(d:/software/texlive/2026/texmf-dist/tex/latex/psnfss/ts1ppl.fd
File: ts1ppl.fd 2001/06/04 font definitions for TS1/ppl.
) [5
pdfTeX warning (ext4): destination with the same identifier (name{figure.1}) has been already used, duplicate ignored
<argument> ...shipout:D \box_use:N \l_shipout_box
\__shipout_drop_firstpage_...
l.73 \end{equation}
]) (./section3/section3.tex [6] (./section3/Figures/figure-update-with-policy-gradient.tex
Missing character: There is no ; in font nullfont!
LaTeX Font Info: Trying to load font information for U+fontawesomefree3 on input line 38.
(d:/software/texlive/2026/texmf-dist/tex/latex/fontawesome5/ufontawesomefree3.fd)
LaTeX Font Info: Trying to load font information for U+fontawesomefree1 on input line 63.
(d:/software/texlive/2026/texmf-dist/tex/latex/fontawesome5/ufontawesomefree1.fd)) [7] (./section3/Figures/figure-one-example.tex
Underfull \hbox (badness 10000) in paragraph at lines 9--10
[]\OT1/ppl/m/n/7 (+20) Give me three tips to im-prove my ac-cu-racy in solv-ing math prob-
[]
LaTeX Font Info: Trying to load font information for OMS+ppl on input line 17.
(d:/software/texlive/2026/texmf-dist/tex/latex/psnfss/omsppl.fd
File: omsppl.fd
)
LaTeX Font Info: Font shape `OMS/ppl/b/n' in size <7> not available
(Font) Font shape `OMS/cmsy/b/n' tried instead on input line 17.
LaTeX Font Info: Trying to load font information for U+fontawesomefree0 on input line 106.
(d:/software/texlive/2026/texmf-dist/tex/latex/fontawesome5/ufontawesomefree0.fd)) [8{d:/software/texlive/2026/texmf-dist/fonts/enc/dvips/fontawesome5/fa5free3.enc}{d:/software/texlive/2026/texmf-dist/fonts/enc/dvips/fontawesome5/fa5free1.enc}
pdfTeX warning (ext4): destination with the same identifier (name{figure.2}) has been already used, duplicate ignored
<argument> ...shipout:D \box_use:N \l_shipout_box
\__shipout_drop_firstpage_...
l.97 \subsection
{Temporal Decomposition}] (./section3/Figures/figure-an-overlap-example.tex) [9{d:/software/texlive/2026/texmf-dist/fonts/enc/dvips/fontawesome5/fa5free0.enc}
pdfTeX warning (ext4): destination with the same identifier (name{figure.3}) has been already used, duplicate ignored
<argument> ...shipout:D \box_use:N \l_shipout_box
\__shipout_drop_firstpage_...
l.119 \begin{eqnarray}
]
Underfull \vbox (badness 1028) has occurred while \output is active []
Underfull \vbox (badness 1028) has occurred while \output is active []
[10
pdfTeX warning (ext4): destination with the same identifier (name{figure.4}) has been already used, duplicate ignored
<argument> ...shipout:D \box_use:N \l_shipout_box
\__shipout_drop_firstpage_...
l.137 \end{eqnarray}
] (./section3/Figures/figure-architecture-value-function.tex
Missing character: There is no ; in font nullfont!
) [11] (./section3/Figures/figure-importance-sampling.tex) [12
pdfTeX warning (ext4): destination with the same identifier (name{figure.5}) has been already used, duplicate ignored
<argument> ...shipout:D \box_use:N \l_shipout_box
\__shipout_drop_firstpage_...
l.210
] (./section3/Figures/figure-importance-sampling-bad-sampling.tex) [13
pdfTeX warning (ext4): destination with the same identifier (name{figure.6}) has been already used, duplicate ignored
<argument> ...shipout:D \box_use:N \l_shipout_box
\__shipout_drop_firstpage_...
l.235 W
e can incorporate this penalty into the Eq. (\ref{eq:loss-function-tr...] (./section3/Figures/figure-ppo-clip.tex) [14
pdfTeX warning (ext4): destination with the same identifier (name{figure.7}) has been already used, duplicate ignored
<argument> ...shipout:D \box_use:N \l_shipout_box
\__shipout_drop_firstpage_...
l.273
] (./section3/Figures/figure-reward-model.tex
Missing character: There is no ; in font nullfont!
) [15
pdfTeX warning (ext4): destination with the same identifier (name{figure.8}) has been already used, duplicate ignored
<argument> ...shipout:D \box_use:N \l_shipout_box
\__shipout_drop_firstpage_...
l.293 \end{figure*}
] (./section3/Figures/figure-pairwise-reward-loss-expectation.tex
Missing character: There is no ; in font nullfont!
) [16
pdfTeX warning (ext4): destination with the same identifier (name{figure.9}) has been already used, duplicate ignored
<argument> ...shipout:D \box_use:N \l_shipout_box
\__shipout_drop_firstpage_...
l.327 \end{eqnarray}
] (./section3/Figures/figure-rlhf.tex) [17
pdfTeX warning (ext4): destination with the same identifier (name{figure.10}) has been already used, duplicate ignored
<argument> ...shipout:D \box_use:N \l_shipout_box
\__shipout_drop_firstpage_...
l.355 \end{figure}
] [18
pdfTeX warning (ext4): destination with the same identifier (name{figure.11}) has been already used, duplicate ignored
<argument> ...shipout:D \box_use:N \l_shipout_box
\__shipout_drop_firstpage_...
l.367 \end{itemize}
]) (./section4/section4.tex
Underfull \vbox (badness 10000) has occurred while \output is active []
Underfull \vbox (badness 10000) has occurred while \output is active []
[19]
Underfull \vbox (badness 10000) has occurred while \output is active []
Underfull \vbox (badness 10000) has occurred while \output is active []
[20] (./section4/Figures/figure-an-overlap-example-dense-reward-model.tex)
Underfull \vbox (badness 10000) has occurred while \output is active []
Underfull \vbox (badness 10000) has occurred while \output is active []
[21] [22
pdfTeX warning (ext4): destination with the same identifier (name{figure.12}) has been already used, duplicate ignored
<argument> ...shipout:D \box_use:N \l_shipout_box
\__shipout_drop_firstpage_...
l.113 \subsubsection
{Improved Reward Generalization}] (./section4/Figures/figure-methods-improving-reward-generalization.tex
Missing character: There is no ; in font nullfont!
) [23
pdfTeX warning (ext4): destination with the same identifier (name{figure.13}) has been already used, duplicate ignored
<argument> ...shipout:D \box_use:N \l_shipout_box
\__shipout_drop_firstpage_...
l.134 \end{eqnarray}
] (./section4/Figures/figure-generative-reward-model-architecture.tex) [24
pdfTeX warning (ext4): destination with the same identifier (name{figure.14}) has been already used, duplicate ignored
<argument> ...shipout:D \box_use:N \l_shipout_box
\__shipout_drop_firstpage_...
l.167 \end{tcolorbox}
] [25] (./section4/Figures/figure-issue-monte-carlo.tex
LaTeX Font Info: Font shape `U/msa/m/n' will be
(Font) scaled to size 18.00587pt on input line 29.
LaTeX Font Info: Font shape `U/msb/m/n' will be
(Font) scaled to size 18.00587pt on input line 29.
) [26] [27
pdfTeX warning (ext4): destination with the same identifier (name{figure.15}) has been already used, duplicate ignored
<argument> ...shipout:D \box_use:N \l_shipout_box
\__shipout_drop_firstpage_...
l.269
] (./section4/Figures/figure-grpo.tex) [28
pdfTeX warning (ext4): destination with the same identifier (name{figure.16}) has been already used, duplicate ignored
<argument> ...shipout:D \box_use:N \l_shipout_box
\__shipout_drop_firstpage_...
l.294 M
oreover, there are other methods like GRPO that design advantage esti...] [29] (./section4/Figures/figure-examples-of-rule-based-reward-length.tex
LaTeX Font Info: Font shape `OMS/ppl/m/n' in size <9> not available
(Font) Font shape `OMS/cmsy/m/n' tried instead on input line 16.
[30]) (./section4/Figures/figure-examples-of-rule-based-reward-math.tex) [31] (./section4/Figures/figure-rlhf-vs-dpo.tex
Missing character: There is no ; in font nullfont!
LaTeX Font Info: Font shape `U/msa/m/n' will be
(Font) scaled to size 15.0049pt on input line 14.
LaTeX Font Info: Font shape `U/msa/m/n' will be
(Font) scaled to size 8.33606pt on input line 14.
LaTeX Font Info: Font shape `U/msb/m/n' will be
(Font) scaled to size 15.0049pt on input line 14.
LaTeX Font Info: Font shape `U/msb/m/n' will be
(Font) scaled to size 8.33606pt on input line 14.
) [32
pdfTeX warning (ext4): destination with the same identifier (name{figure.17}) has been already used, duplicate ignored
<argument> ...shipout:D \box_use:N \l_shipout_box
\__shipout_drop_firstpage_...
l.365 \end{eqnarray}
] (./section4/Figures/figure-dpo.tex
Underfull \hbox (badness 10000) in paragraph at lines 9--10
[]\OT1/ppl/m/n/7 (+20) Give me three tips to im-prove my ac-cu-racy in solv-ing math prob-
[]
) (./section4/Tables/dpo_variants.tex) [33
pdfTeX warning (ext4): destination with the same identifier (name{figure.18}) has been already used, duplicate ignored
<argument> ...shipout:D \box_use:N \l_shipout_box
\__shipout_drop_firstpage_...
l.386 \end{table*}
]) (./section5/section5.tex [34
pdfTeX warning (ext4): destination with the same identifier (name{table.1}) has been already used, duplicate ignored
<argument> ...shipout:D \box_use:N \l_shipout_box
\__shipout_drop_firstpage_...
l.6
] (./section5/Figures/figure-bon-sampling.tex) [35]
Underfull \vbox (badness 10000) has occurred while \output is active []
Underfull \vbox (badness 10000) has occurred while \output is active []
[36
pdfTeX warning (ext4): destination with the same identifier (name{figure.19}) has been already used, duplicate ignored
<argument> ...shipout:D \box_use:N \l_shipout_box
\__shipout_drop_firstpage_...
l.43
] (./section5/Figures/figure-step-by-step-with-prm.tex
Overfull \hbox (3.52284pt too wide) in paragraph at lines 61--62
[]
[]
) [37] (./section5/Figures/figure-mcts.tex)
Underfull \vbox (badness 10000) has occurred while \output is active []
Underfull \vbox (badness 10000) has occurred while \output is active []
[38
pdfTeX warning (ext4): destination with the same identifier (name{figure.20}) has been already used, duplicate ignored
<argument> ...shipout:D \box_use:N \l_shipout_box
\__shipout_drop_firstpage_...
l.75 \end{figure*}
]
Underfull \vbox (badness 10000) has occurred while \output is active []
Underfull \vbox (badness 10000) has occurred while \output is active []
[39
pdfTeX warning (ext4): destination with the same identifier (name{figure.21}) has been already used, duplicate ignored
<argument> ...shipout:D \box_use:N \l_shipout_box
\__shipout_drop_firstpage_...
l.83 \end{eqnarray}
] [40] (./section5/Figures/figure-iterative-rl.tex) [41]
Underfull \vbox (badness 10000) has occurred while \output is active []
Underfull \vbox (badness 10000) has occurred while \output is active []
[42
pdfTeX warning (ext4): destination with the same identifier (name{figure.22}) has been already used, duplicate ignored
<argument> ...shipout:D \box_use:N \l_shipout_box
\__shipout_drop_firstpage_...
l.129
]) (./section6/section6.tex) (./section7/section7.tex [43]) (./section8/section8.tex) (./section9/section9.tex [44
pdfTeX warning (ext4): destination with the same identifier (name{figure.23}) has been already used, duplicate ignored
<argument> ...shipout:D \box_use:N \l_shipout_box
\__shipout_drop_firstpage_...
l.2 \clearpage
] [45] (./section9/tables/dataset.tex) (./section9/tables/systems.tex
LaTeX Font Info: Trying to load font information for U+ding on input line 5.
(d:/software/texlive/2026/texmf-dist/tex/latex/bbding/Uding.fd
File: Uding.fd 1996/02/01 v1.0 BBding font definitions
)
Package microtype Info: Fixing zero \fontdimen 6 for font `U/ding/m/n/10'
(microtype) (new value: 10.0pt) on input line 5.
)) [46
pdfTeX warning (ext4): destination with the same identifier (name{table.2}) has been already used, duplicate ignored
<argument> ...shipout:D \box_use:N \l_shipout_box
\__shipout_drop_firstpage_...
l.208 \clearpage
pdfTeX warning (ext4): destination with the same identifier (name{table.3}) has been already used, duplicate ignored
<argument> ...shipout:D \box_use:N \l_shipout_box
\__shipout_drop_firstpage_...
l.208 \clearpage
] (./rl-introduction.bbl [47
]
LaTeX Font Info: Font shape `OT1/zi4/m/n' will be
(Font) scaled to size 7.6pt on input line 95.
LaTeX Font Info: Font shape `OT1/zi4/m/n' will be
(Font) scaled to size 6.0pt on input line 95.
[48] [49]
Underfull \hbox (badness 2635) in paragraph at lines 244--247
[]\OT1/ppl/m/n/10 (+20) OpenAI. Learn-ing to rea-son with llms, Septem-ber 2024. URL [][]$\OT1/zi4/m/n/10 https : / / openai . com / index /
[]
[50] [51] [52] [53]) [54] (./rl-introduction.aux)
***********
LaTeX2e <2026-06-01>
L3 programming layer <2026-06-18>
***********
LaTeX Font Warning: Some font shapes were not available, defaults substituted.
Package rerunfilecheck Info: File `rl-introduction.out' has not changed.
(rerunfilecheck) Checksum: 2060C280C04E7A328244371D66772709;8324.
)
Here is how much of TeX's memory you used:
46397 strings out of 467168
1044286 string characters out of 5408944
1549794 words of memory out of 5000000
73904 multiletter control sequences out of 15000+600000
724575 words of font info for 584 fonts, out of 8000000 for 9000
1141 hyphenation exceptions out of 8191
117i,19n,121p,2153b,2508s stack positions out of 10000i,1000n,20000p,200000b,200000s
pdfTeX warning (dest): name{Hfootnote.10} has been referenced but does not exist, replaced by a fixed one
pdfTeX warning (dest): name{Hfootnote.9} has been referenced but does not exist, replaced by a fixed one
pdfTeX warning (dest): name{Hfootnote.8} has been referenced but does not exist, replaced by a fixed one
pdfTeX warning (dest): name{Hfootnote.7} has been referenced but does not exist, replaced by a fixed one
pdfTeX warning (dest): name{Hfootnote.6} has been referenced but does not exist, replaced by a fixed one
pdfTeX warning (dest): name{Hfootnote.5} has been referenced but does not exist, replaced by a fixed one
pdfTeX warning (dest): name{Hfootnote.4} has been referenced but does not exist, replaced by a fixed one
pdfTeX warning (dest): name{Hfootnote.3} has been referenced but does not exist, replaced by a fixed one
pdfTeX warning (dest): name{Hfootnote.2} has been referenced but does not exist, replaced by a fixed one
pdfTeX warning (dest): name{Hfootnote.1} has been referenced but does not exist, replaced by a fixed one
<d:/software/texlive/2026/texmf-dist/fonts/type1/public/fontawesome5/FontAwesome5Free-Solid.pfb><d:/software/texlive/2026/texmf-dist/fonts/type1/public/inconsolata/Inconsolata-zi4r.pfb><d:/software/texlive/2026/texmf-dist/fonts/type1/public/niceframe-type1/bbding10.pfb><d:/software/texlive/2026/texmf-dist/fonts/type1/public/amsfonts/cmextra/cmbsy7.pfb><d:/software/texlive/2026/texmf-dist/fonts/type1/public/amsfonts/cm/cmex10.pfb><d:/software/texlive/2026/texmf-dist/fonts/type1/public/amsfonts/cm/cmmi10.pfb><d:/software/texlive/2026/texmf-dist/fonts/type1/public/amsfonts/cm/cmr10.pfb><d:/software/texlive/2026/texmf-dist/fonts/type1/public/amsfonts/cm/cmsy10.pfb><d:/software/texlive/2026/texmf-dist/fonts/type1/public/amsfonts/cm/cmsy9.pfb><d:/software/texlive/2026/texmf-dist/fonts/type1/public/mathpazo/fplmbb.pfb><d:/software/texlive/2026/texmf-dist/fonts/type1/public/mathpazo/fplmr.pfb><d:/software/texlive/2026/texmf-dist/fonts/type1/public/mathpazo/fplmri.pfb><d:/software/texlive/2026/texmf-dist/fonts/type1/urw/palatino/uplb8a.pfb><d:/software/texlive/2026/texmf-dist/fonts/type1/urw/palatino/uplbi8a.pfb><d:/software/texlive/2026/texmf-dist/fonts/type1/urw/palatino/uplr8a.pfb><d:/software/texlive/2026/texmf-dist/fonts/type1/urw/palatino/uplri8a.pfb>
Output written on rl-introduction.pdf (54 pages, 486374 bytes).
PDF statistics:
1263 PDF objects out of 1440 (max. 8388607)
1137 compressed objects within 12 object streams
288 named destinations out of 1000 (max. 500000)
104381 words of extra memory for PDF output out of 106986 (max. 10000000)
rl-introduction.out 0 → 100644
\BOOKMARK [1][-]{section.1}{\376\377\000I\000n\000t\000r\000o\000d\000u\000c\000t\000i\000o\000n}{}% 1
\BOOKMARK [1][-]{section.2}{\376\377\000P\000r\000e\000l\000i\000m\000i\000n\000a\000r\000y}{}% 2
\BOOKMARK [2][-]{subsection.2.1}{\376\377\000F\000u\000n\000d\000a\000m\000e\000n\000t\000a\000l\000s\000\040\000o\000f\000\040\000L\000a\000r\000g\000e\000\040\000L\000a\000n\000g\000u\000a\000g\000e\000\040\000M\000o\000d\000e\000l\000s}{section.2}% 3
\BOOKMARK [2][-]{subsection.2.2}{\376\377\000F\000u\000n\000d\000a\000m\000e\000n\000t\000a\000l\000s\000\040\000o\000f\000\040\000R\000e\000i\000n\000f\000o\000r\000c\000e\000m\000e\000n\000t\000\040\000L\000e\000a\000r\000n\000i\000n\000g}{section.2}% 4
\BOOKMARK [3][-]{subsubsection.2.2.1}{\376\377\000G\000e\000n\000e\000r\000a\000l\000\040\000F\000r\000a\000m\000e\000w\000o\000r\000k\000\040\000o\000f\000\040\000R\000e\000i\000n\000f\000o\000r\000c\000e\000m\000e\000n\000t\000\040\000L\000e\000a\000r\000n\000i\000n\000g}{subsection.2.2}% 5
\BOOKMARK [3][-]{subsubsection.2.2.2}{\376\377\000K\000e\000y\000\040\000E\000l\000e\000m\000e\000n\000t\000s}{subsection.2.2}% 6
\BOOKMARK [1][-]{section.3}{\376\377\000A\000n\000\040\000E\000x\000a\000m\000p\000l\000e\000\040\000o\000f\000\040\000U\000s\000i\000n\000g\000\040\000R\000e\000i\000n\000f\000o\000r\000c\000e\000m\000e\000n\000t\000\040\000L\000e\000a\000r\000n\000i\000n\000g\000\040\000t\000o\000\040\000T\000r\000a\000i\000n\000\040\000L\000L\000M\000s}{}% 7
\BOOKMARK [2][-]{subsection.3.1}{\376\377\000P\000o\000l\000i\000c\000y\000\040\000G\000r\000a\000d\000i\000e\000n\000t}{section.3}% 8
\BOOKMARK [2][-]{subsection.3.2}{\376\377\000T\000e\000m\000p\000o\000r\000a\000l\000\040\000D\000e\000c\000o\000m\000p\000o\000s\000i\000t\000i\000o\000n}{section.3}% 9
\BOOKMARK [2][-]{subsection.3.3}{\376\377\000R\000e\000d\000u\000c\000i\000n\000g\000\040\000G\000r\000a\000d\000i\000e\000n\000t\000\040\000V\000a\000r\000i\000a\000n\000c\000e}{section.3}% 10
\BOOKMARK [2][-]{subsection.3.4}{\376\377\000I\000m\000p\000o\000r\000t\000a\000n\000c\000e\000\040\000S\000a\000m\000p\000l\000i\000n\000g}{section.3}% 11
\BOOKMARK [2][-]{subsection.3.5}{\376\377\000P\000r\000o\000x\000i\000m\000a\000l\000\040\000P\000o\000l\000i\000c\000y\000\040\000O\000p\000t\000i\000m\000i\000z\000a\000t\000i\000o\000n}{section.3}% 12
\BOOKMARK [2][-]{subsection.3.6}{\376\377\000T\000r\000a\000i\000n\000i\000n\000g\000\040\000R\000e\000w\000a\000r\000d\000\040\000M\000o\000d\000e\000l\000s}{section.3}% 13
\BOOKMARK [2][-]{subsection.3.7}{\376\377\000A\000\040\000C\000o\000m\000p\000l\000e\000t\000e\000\040\000W\000o\000r\000k\000f\000l\000o\000w}{section.3}% 14
\BOOKMARK [1][-]{section.4}{\376\377\000I\000m\000p\000r\000o\000v\000e\000d\000\040\000R\000e\000i\000n\000f\000o\000r\000c\000e\000m\000e\000n\000t\000\040\000L\000e\000a\000r\000n\000i\000n\000g\000\040\000f\000o\000r\000\040\000L\000L\000M\000s}{}% 15
\BOOKMARK [2][-]{subsection.4.1}{\376\377\000A\000d\000v\000a\000n\000c\000e\000d\000\040\000R\000e\000w\000a\000r\000d\000\040\000M\000o\000d\000e\000l\000s}{section.4}% 16
\BOOKMARK [3][-]{subsubsection.4.1.1}{\376\377\000A\000u\000t\000o\000m\000a\000t\000i\000c\000\040\000P\000r\000e\000f\000e\000r\000e\000n\000c\000e\000\040\000D\000a\000t\000a\000\040\000G\000e\000n\000e\000r\000a\000t\000i\000o\000n}{subsection.4.1}% 17
\BOOKMARK [3][-]{subsubsection.4.1.2}{\376\377\000R\000e\000w\000a\000r\000d\000\040\000S\000h\000a\000p\000i\000n\000g}{subsection.4.1}% 18
\BOOKMARK [3][-]{subsubsection.4.1.3}{\376\377\000I\000m\000p\000r\000o\000v\000e\000d\000\040\000R\000e\000w\000a\000r\000d\000\040\000G\000e\000n\000e\000r\000a\000l\000i\000z\000a\000t\000i\000o\000n}{subsection.4.1}% 19
\BOOKMARK [3][-]{subsubsection.4.1.4}{\376\377\000G\000e\000n\000e\000r\000a\000t\000i\000v\000e\000\040\000R\000e\000w\000a\000r\000d\000\040\000M\000o\000d\000e\000l\000s}{subsection.4.1}% 20
\BOOKMARK [2][-]{subsection.4.2}{\376\377\000B\000e\000t\000t\000e\000r\000\040\000A\000d\000v\000a\000n\000t\000a\000g\000e\000\040\000E\000s\000t\000i\000m\000a\000t\000i\000o\000n}{section.4}% 21
\BOOKMARK [3][-]{subsubsection.4.2.1}{\376\377\000T\000e\000m\000p\000o\000r\000a\000l\000\040\000D\000i\000f\000f\000e\000r\000e\000n\000c\000e\000-\000b\000a\000s\000e\000d\000\040\000A\000d\000v\000a\000n\000t\000a\000g\000e\000\040\000E\000s\000t\000i\000m\000a\000t\000i\000o\000n}{subsection.4.2}% 22
\BOOKMARK [3][-]{subsubsection.4.2.2}{\376\377\000G\000e\000n\000e\000r\000a\000l\000i\000z\000e\000d\000\040\000A\000d\000v\000a\000n\000t\000a\000g\000e\000\040\000E\000s\000t\000i\000m\000a\000t\000i\000o\000n}{subsection.4.2}% 23
\BOOKMARK [3][-]{subsubsection.4.2.3}{\376\377\000G\000r\000o\000u\000p\000\040\000R\000e\000l\000a\000t\000i\000v\000e\000\040\000P\000o\000l\000i\000c\000y\000\040\000O\000p\000t\000i\000m\000i\000z\000a\000t\000i\000o\000n}{subsection.4.2}% 24
\BOOKMARK [2][-]{subsection.4.3}{\376\377\000E\000f\000f\000i\000c\000i\000e\000n\000t\000\040\000R\000L\000\040\000M\000e\000t\000h\000o\000d\000s}{section.4}% 25
\BOOKMARK [3][-]{subsubsection.4.3.1}{\376\377\000D\000y\000n\000a\000m\000i\000c\000\040\000S\000a\000m\000p\000l\000i\000n\000g}{subsection.4.3}% 26
\BOOKMARK [3][-]{subsubsection.4.3.2}{\376\377\000L\000i\000g\000h\000t\000w\000e\000i\000g\000h\000t\000\040\000R\000e\000w\000a\000r\000d\000\040\000M\000e\000t\000h\000o\000d\000s}{subsection.4.3}% 27
\BOOKMARK [2][-]{subsection.4.4}{\376\377\000D\000i\000r\000e\000c\000t\000\040\000P\000r\000e\000f\000e\000r\000e\000n\000c\000e\000\040\000O\000p\000t\000i\000m\000i\000z\000a\000t\000i\000o\000n}{section.4}% 28
\BOOKMARK [1][-]{section.5}{\376\377\000R\000e\000i\000n\000f\000o\000r\000c\000e\000m\000e\000n\000t\000\040\000L\000e\000a\000r\000n\000i\000n\000g\000\040\000f\000o\000r\000\040\000L\000L\000M\000\040\000R\000e\000a\000s\000o\000n\000i\000n\000g}{}% 29
\BOOKMARK [2][-]{subsection.5.1}{\376\377\000T\000e\000s\000t\000-\000t\000i\000m\000e\000\040\000S\000c\000a\000l\000i\000n\000g}{section.5}% 30
\BOOKMARK [3][-]{subsubsection.5.1.1}{\376\377\000B\000e\000s\000t\000-\000o\000f\000-\000N\000\040\000S\000a\000m\000p\000l\000i\000n\000g}{subsection.5.1}% 31
\BOOKMARK [3][-]{subsubsection.5.1.2}{\376\377\000S\000t\000e\000p\000-\000b\000y\000-\000s\000t\000e\000p\000\040\000V\000e\000r\000i\000f\000i\000c\000a\000t\000i\000o\000n}{subsection.5.1}% 32
\BOOKMARK [3][-]{subsubsection.5.1.3}{\376\377\000M\000o\000n\000t\000e\000\040\000C\000a\000r\000l\000o\000\040\000T\000r\000e\000e\000\040\000S\000e\000a\000r\000c\000h}{subsection.5.1}% 33
\BOOKMARK [2][-]{subsection.5.2}{\376\377\000L\000a\000r\000g\000e\000-\000s\000c\000a\000l\000e\000\040\000R\000e\000i\000n\000f\000o\000r\000c\000e\000m\000e\000n\000t\000\040\000L\000e\000a\000r\000n\000i\000n\000g}{section.5}% 34
\BOOKMARK [2][-]{subsection.5.3}{\376\377\000I\000t\000e\000r\000a\000t\000i\000v\000e\000\040\000R\000e\000i\000n\000f\000o\000r\000c\000e\000m\000e\000n\000t\000\040\000L\000e\000a\000r\000n\000i\000n\000g}{section.5}% 35
\BOOKMARK [1][-]{section.6}{\376\377\000R\000e\000i\000n\000f\000o\000r\000c\000e\000m\000e\000n\000t\000\040\000L\000e\000a\000r\000n\000i\000n\000g\000\040\000f\000o\000r\000\040\000L\000L\000M\000-\000b\000a\000s\000e\000d\000\040\000A\000g\000e\000n\000t\000s}{}% 36
\BOOKMARK [1][-]{section.7}{\376\377\000R\000e\000i\000n\000f\000o\000r\000c\000e\000m\000e\000n\000t\000\040\000L\000e\000a\000r\000n\000i\000n\000g\000\040\000f\000o\000r\000\040\000M\000u\000l\000t\000i\000m\000o\000d\000a\000l\000\040\000M\000o\000d\000e\000l\000s}{}% 37
\BOOKMARK [2][-]{subsection.7.1}{\376\377\000V\000i\000s\000u\000a\000l\000\040\000L\000a\000n\000g\000u\000a\000g\000e\000\040\000M\000o\000d\000e\000l\000s}{section.7}% 38
\BOOKMARK [2][-]{subsection.7.2}{\376\377\000S\000p\000e\000e\000c\000h\000\040\000G\000e\000n\000e\000r\000a\000t\000i\000o\000n\000\040\000M\000o\000d\000e\000l\000s}{section.7}% 39
\BOOKMARK [2][-]{subsection.7.3}{\376\377\000D\000i\000f\000f\000u\000s\000i\000o\000n\000\040\000M\000o\000d\000e\000l\000s}{section.7}% 40
\BOOKMARK [1][-]{section.8}{\376\377\000S\000u\000m\000m\000a\000r\000y}{}% 41
\BOOKMARK [1][-]{section.9}{\376\377\000S\000y\000s\000t\000e\000m\000s\000\040\000a\000n\000d\000\040\000D\000a\000t\000a\000s\000e\000t\000s}{}% 42
rl-introduction.tex
...@@ -45,6 +45,7 @@ ...@@ -45,6 +45,7 @@
\usepackage{fontawesome5} \usepackage{fontawesome5}
\usepackage{varwidth} \usepackage{varwidth}
\usetikzlibrary{shadows} \usetikzlibrary{shadows}
\usetikzlibrary{arrows.meta,positioning,calc,fit,backgrounds}
\definecolor{ocrebase}{RGB}{243,102,25} \definecolor{ocrebase}{RGB}{243,102,25}
\definecolor{ocre}{RGB}{0,0,0} \definecolor{ocre}{RGB}{0,0,0}
\definecolor{amber}{rgb}{1.0, 0.75, 0.0} \definecolor{amber}{rgb}{1.0, 0.75, 0.0}
...@@ -167,16 +168,16 @@ However, RL was not so popular in the long history of NLP, and the field has jus ...@@ -167,16 +168,16 @@ However, RL was not so popular in the long history of NLP, and the field has jus
Of course, we'd like to learn about RL, which appears straightforward. However, common RL textbooks, such as \citet{Sutton-and-Barto:2018RL}'s book, are mostly based on classic robotics or control problems. This makes it difficult to align the concepts of RL with those of LLMs. On the other hand, most LLM literature lacks an in-depth discussion of RL techniques, often omitting related details. As a result, we find ourselves in an awkward middle ground between RL and LLMs, struggling to bridge the two fields. Of course, we'd like to learn about RL, which appears straightforward. However, common RL textbooks, such as \citet{Sutton-and-Barto:2018RL}'s book, are mostly based on classic robotics or control problems. This makes it difficult to align the concepts of RL with those of LLMs. On the other hand, most LLM literature lacks an in-depth discussion of RL techniques, often omitting related details. As a result, we find ourselves in an awkward middle ground between RL and LLMs, struggling to bridge the two fields.
The aim of this paper is to provide a comprehensive introduction to RL from the perspective of LLMs. We begin by introducing basic concepts and algorithms of RL using the LLM language. In particular, we illustrate their application through an example of training LLMs, making the concepts more accessible. We then discuss a series of refinements to the basic RL framework for LLM alignment, including advanced reward modeling, improved advantage estimation, efficient sampling, and direct LLM optimization without RL. Furthermore, we discuss how to apply RL techniques to LLM reasoning. These methods are closely related to recent LLMs, such as OpenAI o1/o3 and Deepseek R1, which adopt large-scale RL and test-time scaling to significantly advance their reasoning abilities. In addition, we discuss applications of RL to multimodal LLMs to demonstrate how RL can be adapted to various problems. Finally, we conclude by outlining promising future research directions and discussing relevant systems and datasets. The aim of this paper is to provide a comprehensive introduction to RL from the perspective of LLMs. We begin by introducing basic concepts and algorithms of RL using the LLM language. In particular, we illustrate their application through an example of training LLMs, making the concepts more accessible. We then discuss a series of refinements to the basic RL framework for LLM alignment, including advanced reward modeling, improved advantage estimation, efficient sampling, and direct LLM optimization without RL. Furthermore, we discuss how to apply RL techniques to LLM reasoning. These methods are closely related to recent LLMs, such as OpenAI o1/o3 and DeepSeek R1, which adopt large-scale RL and test-time scaling to significantly advance their reasoning abilities. In addition, we discuss applications of RL to multimodal LLMs to demonstrate how RL can be adapted to various problems. Finally, we conclude by outlining promising future research directions and discussing relevant systems and datasets.
% \clearpage % \clearpage
% New section: training LLMs with reinforcement learning % New section: training LLMs with reinforcement learning
% \input{section2/section2} \input{section2/section2}
% \clearpage % \clearpage
% New section: training LLMs with reinforcement learning % New section: training LLMs with reinforcement learning
% \input{section3/section3} \input{section3/section3}
% \clearpage % \clearpage
% New section: improved reinforcement learning for LLMs % New section: improved reinforcement learning for LLMs
...@@ -185,19 +186,23 @@ The aim of this paper is to provide a comprehensive introduction to RL from the ...@@ -185,19 +186,23 @@ The aim of this paper is to provide a comprehensive introduction to RL from the
% \clearpage % \clearpage
% New section: reinforcement learning for LLM inference % New section: reinforcement learning for LLM inference
% \input{section5/section5} \input{section5/section5}
% \clearpage % \clearpage
% New section: reinforcement learning for MLLMs % New section: reinforcement learning for MLLMs
% \input{section6/section6} \input{section6/section6}
% \clearpage
% New section: reinforcement learning for MLLMs
\input{section7/section7}
% \clearpage % \clearpage
% conclusion % conclusion
% \input{section7/section7} \input{section8/section8}
% \clearpage % \clearpage
% systems & datasets todo: ganyang % systems & datasets todo: ganyang
% \input{section8/section8} \input{section9/section9}
\clearpage \clearpage
......
rl-introduction.toc 0 → 100644
\contentsline {section}{\numberline {1}Introduction}{3}{section.1}%
\contentsline {section}{\numberline {2}Preliminary}{3}{section.2}%
\contentsline {subsection}{\numberline {2.1}Fundamentals of Large Language Models}{3}{subsection.2.1}%
\contentsline {subsection}{\numberline {2.2}Fundamentals of Reinforcement Learning}{4}{subsection.2.2}%
\contentsline {subsubsection}{\numberline {2.2.1}General Framework of Reinforcement Learning}{4}{subsubsection.2.2.1}%
\contentsline {subsubsection}{\numberline {2.2.2}Key Elements}{5}{subsubsection.2.2.2}%
\contentsline {section}{\numberline {3}An Example of Using Reinforcement Learning to Train LLMs}{6}{section.3}%
\contentsline {subsection}{\numberline {3.1}Policy Gradient}{7}{subsection.3.1}%
\contentsline {subsection}{\numberline {3.2}Temporal Decomposition}{9}{subsection.3.2}%
\contentsline {subsection}{\numberline {3.3}Reducing Gradient Variance}{10}{subsection.3.3}%
\contentsline {subsection}{\numberline {3.4}Importance Sampling}{12}{subsection.3.4}%
\contentsline {subsection}{\numberline {3.5}Proximal Policy Optimization}{14}{subsection.3.5}%
\contentsline {subsection}{\numberline {3.6}Training Reward Models}{15}{subsection.3.6}%
\contentsline {subsection}{\numberline {3.7}A Complete Workflow}{18}{subsection.3.7}%
\contentsline {section}{\numberline {4}Improved Reinforcement Learning for LLMs}{20}{section.4}%
\contentsline {subsection}{\numberline {4.1}Advanced Reward Models}{20}{subsection.4.1}%
\contentsline {subsubsection}{\numberline {4.1.1}Automatic Preference Data Generation}{20}{subsubsection.4.1.1}%
\contentsline {subsubsection}{\numberline {4.1.2}Reward Shaping}{21}{subsubsection.4.1.2}%
\contentsline {subsubsection}{\numberline {4.1.3}Improved Reward Generalization}{23}{subsubsection.4.1.3}%
\contentsline {subsubsection}{\numberline {4.1.4}Generative Reward Models}{24}{subsubsection.4.1.4}%
\contentsline {subsection}{\numberline {4.2}Better Advantage Estimation}{26}{subsection.4.2}%
\contentsline {subsubsection}{\numberline {4.2.1}Temporal Difference-based Advantage Estimation}{26}{subsubsection.4.2.1}%
\contentsline {subsubsection}{\numberline {4.2.2}Generalized Advantage Estimation}{26}{subsubsection.4.2.2}%
\contentsline {subsubsection}{\numberline {4.2.3}Group Relative Policy Optimization}{27}{subsubsection.4.2.3}%
\contentsline {subsection}{\numberline {4.3}Efficient RL Methods}{29}{subsection.4.3}%
\contentsline {subsubsection}{\numberline {4.3.1}Dynamic Sampling}{29}{subsubsection.4.3.1}%
\contentsline {subsubsection}{\numberline {4.3.2}Lightweight Reward Methods}{30}{subsubsection.4.3.2}%
\contentsline {subsection}{\numberline {4.4}Direct Preference Optimization}{32}{subsection.4.4}%
\contentsline {section}{\numberline {5}Reinforcement Learning for LLM Reasoning}{34}{section.5}%
\contentsline {subsection}{\numberline {5.1}Test-time Scaling}{35}{subsection.5.1}%
\contentsline {subsubsection}{\numberline {5.1.1}Best-of-N Sampling}{35}{subsubsection.5.1.1}%
\contentsline {subsubsection}{\numberline {5.1.2}Step-by-step Verification}{37}{subsubsection.5.1.2}%
\contentsline {subsubsection}{\numberline {5.1.3}Monte Carlo Tree Search}{38}{subsubsection.5.1.3}%
\contentsline {subsection}{\numberline {5.2}Large-scale Reinforcement Learning}{40}{subsection.5.2}%
\contentsline {subsection}{\numberline {5.3}Iterative Reinforcement Learning}{41}{subsection.5.3}%
\contentsline {section}{\numberline {6}Reinforcement Learning for LLM-based Agents}{43}{section.6}%
\contentsline {section}{\numberline {7}Reinforcement Learning for Multimodal Models}{43}{section.7}%
\contentsline {subsection}{\numberline {7.1}Visual Language Models}{43}{subsection.7.1}%
\contentsline {subsection}{\numberline {7.2}Speech Generation Models}{45}{subsection.7.2}%
\contentsline {subsection}{\numberline {7.3}Diffusion Models}{45}{subsection.7.3}%
\contentsline {section}{\numberline {8}Summary}{45}{section.8}%
\contentsline {section}{\numberline {9}Systems and Datasets}{46}{section.9}%
section2/Figures/rl_framework.tex 0 → 100644
\begin{center}
\begin{tikzpicture}[
font=\small,
% line width=0.9pt,
box/.style={
draw,
rounded corners=3pt,
minimum width=3.1cm,
minimum height=1.35cm,
align=center,
font=\normalsize
},
neuron/.style={
circle,
draw=black!55,
minimum size=7mm,
inner sep=0pt
},
conn/.style={
draw=black!65,
line width=0.5pt
},
flow/.style={
draw=black,
-Latex,
line width=0.9pt
}
]
% =========================================================
% Left part: Environment-Agent interaction
% =========================================================
\node[
anchor=center,
draw,
thick,
minimum width=3.0cm,
minimum height=1.0cm,
fill=white,
drop shadow={
fill=gray!40,
shadow xshift=0.6ex,
shadow yshift=-0.6ex
}
] (env) at (0,0) {\large Environment};
\node[anchor=center,draw, minimum width=3.0cm, minimum height=1.0cm, fill=white, drop shadow={
fill=gray!40,
shadow xshift=0.6ex,
shadow yshift=-0.6ex
}
] (agent) at (5.2,0) {\large Agent};
% top arc: state
\draw[->]
($(env.north west)+(1.6,0.05)$)
to[out=45,in=135]
node[pos=0.15, above, font=\large] {$S_t$}
($(agent.north east)+(-1.6,0.05)$);
% middle arc: reward
\draw[->]
($(env.north east)+(0.02,0.10)$)
to[out=20,in=160]
node[midway, above=2pt] {\large $r_t$}
($(agent.north west)+(-0.02,0.10)$);
% bottom arc: action
\draw[->]
($(agent.south west)+(-0.02,-0.10)$)
to[out=200,in=340]
node[midway, below=2pt] {\large $a_t$}
($(env.south east)+(0.02,-0.10)$);
% connector to policy net
\draw[dashed, line width=0.9pt, ->]
($(agent.east)+(0.10,0.00)$)
.. controls +(1.0,0.45) and +(-0.9,0.25)
.. ($(8.6,0.15)$);
% =========================================================
% Right part: Policy network
% =========================================================
% input layer
\foreach \i/\x in {1/9.3,2/10.1,3/10.9,4/11.7}
{
\node[neuron, fill=gray!25, minimum size=4mm] (I\i) at (\x,-1.15) {};
}
% hidden layer
\foreach \i/\x in {1/9.1,2/9.8,3/10.5,4/11.2,5/11.9}
{
\node[neuron, fill=blue!30, minimum size=4mm] (H\i) at (\x,0.05) {};
}
% output layer
\foreach \i/\x in {2/9.8,4/11.2}
{
\node[neuron, fill=yellow!45, minimum size=4mm] (O\i) at (\x,1.35) {};
}
% connections: input -> hidden
\foreach \i in {1,2,3,4}
{
\foreach \j in {1,2,3,4,5}
{
\draw[conn] (I\i) -- (H\j);
}
}
% connections: hidden -> output
\foreach \i in {1,2,3,4,5}
{
\foreach \j in {2,4}
{
\draw[conn] (H\i) -- (O\j);
}
}
% arrows below input layer
\foreach \i in {1,2,3,4}
{
\draw [->] ($(I\i.south)+(0,-0.36)$) -- ($(I\i.south)+(0,-0.02)$);
}
% \draw [->] ([yshift=0.2cm]\x.center) -- ([yshift=0.55cm]\x.center);
% arrows above output layer
\foreach \i in {2,4}
{
\draw[->] ($(O\i.north)+(0,0.02)$) -- ($(O\i.north)+(0,0.42)$);
}
% dashed rounded frame around policy net
\node[
draw=gray!70,
dashed,
rounded corners=5pt,
minimum width=5.0cm,
minimum height=4.0cm,
inner xsep=0.85cm,
inner ysep=1.00cm
] (policybox) at (11.2, 0.2){};
% labels
\node[anchor=west] at (10.6, -2.2) {Policy};
\node[anchor=center, font=\small] at ($(policybox.east)+(-0.9, 1.15)$) {Output};
\node[anchor=center, font=\small] at ($(policybox.east)+(-0.9, -0.1)$) {Hidden};
\node[anchor=center, font=\small] at ($(policybox.east)+(-0.9, -1.4)$) {Input};
\end{tikzpicture}
\end{center}
\ No newline at end of file
section2/section2.tex
\section{Preliminary} \section{Preliminary}
In this section, we introduce the fundamentals of supervised fine-tuning for LLMs and focus on presenting the key concepts and nations necessary for the discussions in future sections.
\subsection{Fundamentals of Large Language Models}
In this subsection, we introduce the fundamentals of supervised fine-tuning for LLMs and focus on presenting the key concepts and nations necessary for the discussions in future sections.
Although pre-trained LLMs possess a vast amount of general knowledge, they are typically limited to tasks related to language modeling. Our ultimate goal is to enable them to perform various specific tasks, such as summarization, question-answer, and machine translation. Although pre-trained LLMs possess a vast amount of general knowledge, they are typically limited to tasks related to language modeling. Our ultimate goal is to enable them to perform various specific tasks, such as summarization, question-answer, and machine translation.
One straightforward method to achieve this goal is supervised fine-tuning (SFT), in which the pre-trained LLM is further trained on a dataset comprising task-specific input (i.e., instruction+user input)\footnote{In this paper, the \textit{instruction} represents the description of a specific task, e.g., ``Summarize the following article''; the \textit{user input} represents the extra content to the instruction, e.g., ``Article: In recent years, solar energy has seen a significant increase in the amount of energy available to the public. recent years, solar energy has seen unprecedented growth, becoming the fastest-growing ...''} paired with their expected outputs \citep{ouyang:2022training,wei-etal:2022finetuned}. One straightforward method to achieve this goal is supervised fine-tuning (SFT), in which the pre-trained LLM is further trained on a dataset comprising task-specific input (i.e., instruction+user input)\footnote{In this paper, the \textit{instruction} represents the description of a specific task, e.g., ``Summarize the following article''; the \textit{user input} represents the extra content to the instruction, e.g., ``Article: In recent years, solar energy has seen a significant increase in the amount of energy available to the public. recent years, solar energy has seen unprecedented growth, becoming the fastest-growing ...''} paired with their expected outputs \citep{ouyang:2022training,wei-etal:2022finetuned}.
...@@ -21,10 +23,67 @@ The objective function $\log \mathrm{Pr}_{\theta}(y_i|\mathbf{x},\mathbf{y}_{<i} ...@@ -21,10 +23,67 @@ The objective function $\log \mathrm{Pr}_{\theta}(y_i|\mathbf{x},\mathbf{y}_{<i}
This formulation is equivalent to minimizing the cross-entropy loss. In the following section, we refer to the LLM after SFT as the ``SFT LLM'' for short. Beyond SFT, the model architecture design and the pre-training process are also fundamentals of building an LLM. However, we will not discuss these topics. Interested readers can find further details in \citet{xiao-and-zhu:2025foundations}'s book. This formulation is equivalent to minimizing the cross-entropy loss. In the following section, we refer to the LLM after SFT as the ``SFT LLM'' for short. Beyond SFT, the model architecture design and the pre-training process are also fundamentals of building an LLM. However, we will not discuss these topics. Interested readers can find further details in \citet{xiao-and-zhu:2025foundations}'s book.
%%%%%%%%%% Preliminary修改建议 %%%%%%%%%% \subsection{Fundamentals of Reinforcement Learning}
% 1. 再加一些强化学习的基础知识 RL, supported by a well-established theoretical framework, has been widely applied across a broad range of domains. In particular, with the rapid advancement of LLMs, RL has become a standard approach in the post-training stage. Before exploring the interplay between RL and LLMs, we first provide a brief overview of deep reinforcement learning in this subsection. We then introduce the general formulation of RL, along with key terminologies and notations that are essential for understanding RL.
% 包含对环境、动作、policy这些术语的定义，方面后面对应起来的好定义。
% 如果修改这一章节 那么3.1这部分要对应起来，有些需要删除掉。 \subsubsection{General Framework of Reinforcement Learning}
The general framework of RL is illustrated in Figure~\ref{fig:rl_framework}. An RL system primarily consists of two components: an agent and an environment. At each timestep $t$, the agent observes a state $s_t$ from the environment and selects an action $a_t$ according to a policy $\pi$, which is typically parameterized by a neural network. After executing the action, the environment transitions to a new state $s_{t+1}$ and returns a reward $r_t$ corresponding to the taken action.
\begin{figure}[!t]
\centering
\input{section2/Figures/rl_framework}
\caption{
A general framework of deep reinforcement learning. The policy is parameterized by neural networks, and the agent learns an optimal policy through interactions with the environment to maximize cumulative rewards.
}
\label{fig:rl_framework}
\end{figure}
This interaction process is commonly modeled as a Markov Decision Process (MDP), where the agent interacts with the environment over discrete timesteps \citep{Sutton-and-Barto:2018RL}. A sequence of states and actions forms a trajectory, denoted as $\tau = (s_0, a_0, s_1, a_1, \cdots, s_{H-1}, a_{H-1})$, where $H$ represents the trajectory length. Each trajectory accumulates rewards from the environment.
The objective of RL is to learn an optimal policy $\pi^*$ that maximizes the expected cumulative reward over trajectories, which can be formulated as:
\begin{eqnarray}
\pi^{*} = \mathop{\arg\max} \limits_{\pi} \mathbb{E}\left[\sum_{t=0}^{H-1}r_{t} \mid \pi \right]
\end{eqnarray}
In practice, the agent improves its policy through repeated interactions with the environment by observing states and receiving feedback in the form of rewards. Each such interaction sequence constitutes a trajectory $\tau$. The process of generating one or more trajectories under a given policy is commonly referred to as \textit{sampling} in the RL literature.
\subsubsection{Key Elements}
To better understand the application of reinforcement learning to large language models (LLMs), we summarize the key elements of the RL framework and reinterpret them in the context of language modeling.
\begin{itemize}
\item \textbf{Agent.}
The agent is the learner or decision-maker in reinforcement learning. In the context of LLMs, the agent corresponds to the language model itself, which generates tokens sequentially and updates its behavior based on feedback signals.
\item \textbf{Environment.}
The environment comprises everything external to the agent with which it interacts. Unlike traditional RL settings that involve physical or simulated environments, the environment in LLM-based RL is typically abstract, consisting of the training framework that provides feedback (e.g., reward models, human annotations, or evaluation metrics) for generated outputs.
\item \textbf{State ($S$).}
A state represents the current situation of the environment. For language modeling, the state at timestep $t$ can be defined as the sequence of observed tokens up to that point, i.e., the context used to predict the next token. Formally, the state can be represented as $S = (x, y_{<t})$, where $x$ denotes the input prompt and $y_{<t}$ denotes the previously generated tokens.
\item \textbf{Action ($a$).}
An action corresponds to a decision made by the agent. In LLMs, actions are naturally defined as selecting the next token from the vocabulary, i.e., $a = y_t$.
\item \textbf{Reward ($r$).}
The reward provides feedback from the environment to evaluate the quality of an action. In general, the reward function can be defined as $r(s, a, s')$, representing the feedback received when the agent transitions from state $s$ to $s'$ by taking action $a$. At timestep $t$, this can be written as $r_t = r(s_t, a_t, s_{t+1})$. In deterministic settings, where the next state is uniquely determined by $(s_t, a_t)$, the reward can be simplified as $r(s_t, a_t)$.
\item \textbf{Policy ($\pi$).}
The policy defines the agent’s behavior, i.e., the probability of taking an action given a state. For LLMs, the policy corresponds to the conditional probability distribution over the next token given the context:
\begin{equation}
\pi(a \mid s) = \Pr(y_t \mid x, y_{<t})
\end{equation}
where $a = y_t$ and $s = (x, y_{<t})$. Under this formulation, an LLM can be naturally interpreted as a parameterized policy.
\item \textbf{Value Function ($V$ and $Q$).}
The value function estimates the expected cumulative reward when following a policy. The state-value function $V(s)$ measures the expected discounted return starting from state $s$:
\begin{equation}
V(s) = \mathbb{E} \left[ \sum_{t=0}^{\infty} \gamma^t r_t \middle| s_0 = s, \pi \right]
\end{equation}
where $\gamma \in [0,1]$ is the discount factor. The action-value function $Q(s,a)$ further conditions on the initial action:
\begin{equation}
Q(s,a) = \mathbb{E} \left[ \sum_{t=0}^{\infty} \gamma^t r_t \middle| s_0 = s, a_0 = a, \pi \right]
\end{equation}
\end{itemize}
By this point, the reader should have developed a foundational understanding of the core concepts and notations in RL. In contrast to conventional RL literature, which typically presents algorithms in the context of classical robotics or control tasks, \textit{we adopt an NLP perspective to introduce RL in the following section}.
section3/section3.tex
...@@ -3,7 +3,7 @@ ...@@ -3,7 +3,7 @@
\section{An Example of Using Reinforcement Learning to Train LLMs} \section{An Example of Using Reinforcement Learning to Train LLMs}
\label{sec:example-using-rl-training-llms} \label{sec:example-using-rl-training-llms}
To explain how RL can be applied to train LLMs, we begin by considering a practical scenario: using an LLM as a homework assistant. In this context, we aim to improve the capability of an SFT LLM to handle education-related inputs more effectively. Suppose we have a homework assistant powered by an SFT LLM, and a student types the input ``Give me three tips to improve my accuracy in solving math problems.'' A typical SFT LLM might generate a short output, such as We begin by considering a practical scenario: using an LLM as a homework assistant. In this context, we aim to improve the capability of an SFT LLM to handle education-related inputs more effectively. Suppose we have a homework assistant powered by an SFT LLM, and a student types the input ``Give me three tips to improve my accuracy in solving math problems.'' A typical SFT LLM might generate a short output, such as
\vspace{0.1cm} \vspace{0.1cm}
\begin{tcolorbox}[frame empty] \begin{tcolorbox}[frame empty]
...@@ -23,7 +23,7 @@ There are three tips for improving accuracy in solving math problems: \\ [1mm] ...@@ -23,7 +23,7 @@ There are three tips for improving accuracy in solving math problems: \\ [1mm]
Although this output adheres to the given input, it is very short and not sufficiently detailed or comprehensive for this scenario. In practice, the ``short'' feature in the generated output stems from the training approach of the SFT LLM. During SFT, the LLM learns from a large set of labeled samples that typically contain concise and factual answers to common inputs. These samples guide the LLM to prioritize brevity and clarity, so it often generates short outputs, like the one shown above. Of course, we could annotate enough additional data to fine-tune the LLM further and adjust it to generate the desired outputs. However, this approach is limited in its ability to scale. For example, in this scenario, the input involves a variety of potential answers, and the expectations of the student can be pretty diverse. Therefore, describing the ``ideal'' output would require immense annotation effort. Consequently, collecting or annotating fine-tuning data is not as straightforward as it is with SFT, particularly when attempting to cover the breadth of possible student inputs and outputs. Instead, we can use RL to enable the model to discern outputs that better align with human preferences, such as generating more detailed and comprehensive content that not only adheres to the given input but also meets the expectations of the student in this scenario. Although this output adheres to the given input, it is very short and not sufficiently detailed or comprehensive for this scenario. In practice, the ``short'' feature in the generated output stems from the training approach of the SFT LLM. During SFT, the LLM learns from a large set of labeled samples that typically contain concise and factual answers to common inputs. These samples guide the LLM to prioritize brevity and clarity, so it often generates short outputs, like the one shown above. Of course, we could annotate enough additional data to fine-tune the LLM further and adjust it to generate the desired outputs. However, this approach is limited in its ability to scale. For example, in this scenario, the input involves a variety of potential answers, and the expectations of the student can be pretty diverse. Therefore, describing the ``ideal'' output would require immense annotation effort. Consequently, collecting or annotating fine-tuning data is not as straightforward as it is with SFT, particularly when attempting to cover the breadth of possible student inputs and outputs. Instead, we can use RL to enable the model to discern outputs that better align with human preferences, such as generating more detailed and comprehensive content that not only adheres to the given input but also meets the expectations of the student in this scenario.
In the following sections, we will demonstrate how to use RL to train LLMs through a specific example—enabling the LLM to generate a longer output in the context of a homework assistant. Note that the techniques discussed are not limited to this single case. Rather, they can be broadly applied to align LLM with any set of expectations. In the following sections, we will demonstrate how to use RL to train LLMs through a specific example—enabling the LLM to generate a longer output in the context of a homework assistant. Throughout this example, we introduce RL, including key algorithms and their improvements.
% policy gradient % policy gradient
\subsection{Policy Gradient} \subsection{Policy Gradient}
......
section4/section4.tex
...@@ -151,7 +151,7 @@ where $\alpha$ is a balancing factor, we further illustrate the freezing paramet ...@@ -151,7 +151,7 @@ where $\alpha$ is a balancing factor, we further illustrate the freezing paramet
\subsubsection{Generative Reward Models} \subsubsection{Generative Reward Models}
\label{sec:generative-reward-models} \label{sec:generative-reward-models}
Reward models are typically trained as discriminative models to assign numerical rewards to outputs and classify them as preferred or dispreferred. However, this method does not leverage the text-generation capabilities for which LLMs are fundamentally designed. For example, the discriminative reward model can not perform CoT reasoning. To address this, an LLM can alternatively be employed as a reward model, thus endowing it with the ability to engage in text generation and reasoning, as depicted in Figure \ref{fig:generative-reward-model-architecture}. This model works as follows. First, we input a prompt $\mathbf{c}$, along with the tuple $(\mathbf{x},\mathbf{y}_{a},\mathbf{y}_{b})$, to the LLM. The prompt is a description of the task, as demonstrated in the example below. Reward models are typically trained as discriminative models to assign numerical rewards to outputs and classify them as preferred or dispreferred. However, this method does not leverage the text-generation capabilities for which LLMs are fundamentally designed \citep{zhang-etal:zhang2024generative,wang-etal:wang2025gram}. For example, the discriminative reward model can not perform CoT reasoning. To address this, an LLM can alternatively be employed as a reward model, thus endowing it with the ability to engage in text generation and reasoning, as depicted in Figure \ref{fig:generative-reward-model-architecture}. This model works as follows. First, we input a prompt $\mathbf{c}$, along with the tuple $(\mathbf{x},\mathbf{y}_{a},\mathbf{y}_{b})$, to the LLM. The prompt is a description of the task, as demonstrated in the example below.
\vspace{0.5em} \vspace{0.5em}
\begin{tcolorbox}[frame empty] \begin{tcolorbox}[frame empty]
...@@ -187,7 +187,7 @@ r_{\phi}(\mathbf{x}', \mathbf{y}') & = & \frac{\mathrm{Pr}_{\theta}(w=\text{A}|\ ...@@ -187,7 +187,7 @@ r_{\phi}(\mathbf{x}', \mathbf{y}') & = & \frac{\mathrm{Pr}_{\theta}(w=\text{A}|\
where the reward ranges from 0 to 1. where the reward ranges from 0 to 1.
To further improve the generative reward model, we can label the relevant explanation to enable the generative reward model to produce a CoT rationale \citep{zhang-etal:2024generative}. In this case, we use a prompt $\mathbf{c}$ with a CoT rationale generation instruction, as shown below. To further improve the generative reward model, we can label the relevant explanation to enable the generative reward model to produce a CoT rationale \citep{zhang-etal:2024generative,wang-etal:wang2026gram}. In this case, we use a prompt $\mathbf{c}$ with a CoT rationale generation instruction, as shown below.
\vspace{0.5em} \vspace{0.5em}
\begin{tcolorbox}[frame empty] \begin{tcolorbox}[frame empty]
......
section6/section6.tex
\section{Reinforcement Learning for Multimodal Models} \section{Reinforcement Learning for LLM-based Agents}
Multimodal learning involves models that process and relate information from multiple data modalities (e.g. vision, text, speech), enabling more comprehensive understanding than single-modality systems. By combining modalities, models can make more robust predictions and capture complementary information that one modality alone might miss. Such multimodal approaches have benefits across various applications, such as image caption and image generation. Given these advantages, multimodal learning has become increasingly significant as AI systems aim to perceive and reason more like humans, who naturally integrate sight, sound, and language \citep{baltruvsaitis-etal:2018multimodal}. \ No newline at end of file
In the era of LLM, we can extend their capabilities into the multimodal domain by training them with diverse data types. For example, we can develop a visual language model by training an LLM with image-text pairs using SFT. Here, we return to the issue of aligning models with human preferences. Ideally, once aligned with human preferences through RL, the LLM would also generalize the target modality well by the SFT. However, the reality does not always meet expectations. Although an LLM may align well with human preferences in one aspect, it often struggles to generalize this alignment across a different modality. Thus, to align multimodal models with human preferences, we perform RL to enhance their performance further within the specific modality.
In this section, we use the visual language, speech generation, and diffusion models to discuss the application of RL in multimodal models.
\subsection{Visual Language Models}
While RL is commonly used to train LLMs, its application to other domains has been a prominent research topic. In multimodal language models\footnote{A multimodal language model is defined as a model that integrates an LLM with a multimodal encoder, such as CLIP \citep{radford-etal:2021learning}, allowing the LLM to process inputs beyond text, such as images. Recent literature has also introduced the use of LLMs for generating outputs in non-text modalities, such as images and speech \citep{xu-etal:2025qwen2,zhang-etal:2024mm}. However, in this section, we focus on the former definition of multimodal language models.}, for example, a notable trend is to perform RL training to improve their trustworthiness and helpfulness. This section considers Visual Language Models (VLMs), which connect a visual encoder to an LLM through a linear projector, facilitating general-purpose visual and language understanding. VLMs are currently the most explored extension in multimodal language research, and they form the foundation for many open-source multimodal language models, such as Qwen2.5-VL \citep{qwenTeam:2025qwen2.5-VL} and LLaMA-3.2-11B-Vision \citep{grattafiori-etal:2024llama}.
Training LLMs and VLMs with RL exhibits only minimal differences, primarily related to the input content. Unlike the textual input used for LLMs, the input for VLMs typically comprises a combination of one or multiple images and an instruction, denoted by $(\mathrm{\mathbf{I}}, \mathrm{\mathbf{x}})$, where $\mathrm{\mathbf{I}}$ represents the input images. These images are encoded into representations that are either concatenated with instruction embeddings or integrated through cross-attention mechanisms into the LLM. In practice, this subtle difference does not significantly affect the applicability of RL algorithms to VLMs. As a result, the RL training process for LLMs, as described in Section \ref{sec:example-using-rl-training-llms}, can be seamlessly adapted to train VLMs without major improvements \citep{yu-etal:2024rlhf,wang-etal:2024rovrm,zang2025internlm,ji2025safe}.
\begin{figure*}[!t]
\centering
\caption{RoVRM}
\label{fig:preference-transfer}
\end{figure*}
However, training VLMs with RL is not a low-hanging fruit in practical applications. This is because it typically encounters the challenge of training a visual reward model due to the scarcity of high-quality visual preference data. One straightforward approach to address this issue is to generate visual preference data through the automatic preference data generation method described in Section \ref{sec:automatic-preference-data-generation} \citep{yu-etal:2024rlaif}. Another alternative is a multi-stage training approach for the visual reward model, motivated by a simple idea: human preferences are well captured in text, and these preferences can be transferred across modalities \citep{wang-etal:2024rovrm}. By leveraging textual preference data, this approach reduces the dependence on visual preference data in training a visual reward model. More specifically, as illustrated in Figure \ref{fig:preference-transfer}, we can train a visual reward model in the following three stages:
\begin{itemize}
\item Stage 1: pre-training with large-scale textual preference data. Given the transferability of human preferences across different modalities, we can begin by using large-scale textual preference data to pre-train the visual reward model. Note that the projector parameters are frozen without images. This stage can be considered as providing a stronger starting point for training the visual reward model, as it enables the model to pre-learn general human preferences.
\item Stage 2: fine-tuning with image caption-based preference data. Pre-learned human preferences cannot be directly applied to vision tasks due to both \textit{task gap} and \textit{modality gap}. To bridge the task gap, we fine-tune the reward model using image caption-based preference data in this stage. The rationale behind this approach is that general textual preference data does not cover vision-specific tasks, such as ``Please describe the content in this image''. We use such data to fine-tune the model, adapting it to vision-specific tasks. The projector parameters are frozen at this stage.
\item Stage 3: fine-tuning with small-scale visual preference data. To further bridge the modality gap, we use visual preference data in the final stage to train the model. Note that in this phase, we also train the projector parameters.
\end{itemize}
In this process, not all preference data may align with the preferences used in subsequent phases, potentially leading to preference conflicts. We can enhance the visual reward model through data selection techniques, such as LESS \citep{xia-etal:2024less} to address this. In fact, preference transfer is effective across modalities and has also been shown to work across different tasks and languages \citep{cheng-etal:2023everyone,wu-etal:2024reuse}. Interested readers can refer to these papers for more detailed discussions of these topics.
Apart from preference data, another approach to improving the visual model is to integrate additional image content, such as image captions, into the reward model \citep{sun-etal:2023aligning}. This approach aims to achieve factually augmented reward prediction, addressing reward hacking. Specifically, in the original setup, the reward model predicts a reward based solely on the image, input, and output; that is, the reward model’s input is $[\mathbf{I}, \mathbf{x}, \mathbf{y}]$. In the factually augmented setup, the reward model also receives additional input in the form of the textual image caption $\mathbf{C}$, resulting in an input of $[\mathbf{I}, \mathbf{C}, \mathbf{x}, \mathbf{y}]$. The basic idea is that the backbone of the visual reward model remains a well-trained LLM with an in-context learning ability. With this ability, we can provide additional content to help the model predict rewards more accurately.
While our discussion primarily focuses on models for visual inputs in the context of visual language models, the RL techniques are adaptable across inputs of various modalities, such as video and audio.
% visual language model as reward model
% 视觉奖励模型训练大致思路
% image reward
% 融合文本数据进行提升
% 视觉奖励模型应用场景---对齐图生文模型 文生图评估模型
% 视觉语言模型的定义
% 框架
% Rlhf-v: Towards trustworthy mllms via behavior alignment from fine-grained correctional human feedback
% Aligning Large Multimodal Models with Factually Augmented RLHF
% 偏好数据
% RLAIF-V
% Silkie: Preference Distillation for Large Visual Language Models
% RoVRM
\subsection{Speech Generation Models}
% SpeechAlign: Aligning Speech Generation to Human Preferences
\subsection{Diffusion Models}
% Image Reward
% 为什么要进行强化学习
% 大体训练思路和流程
% DDPO (denoising diffusion preference optimization model)
section7/section7.tex
\section{Summary}
% future work \section{Reinforcement Learning for Multimodal Models}
% 强化学习作为一种预训练方式 Multimodal learning involves models that process and relate information from multiple data modalities (e.g. vision, text, speech), enabling more comprehensive understanding than single-modality systems. By combining modalities, models can make more robust predictions and capture complementary information that one modality alone might miss. Such multimodal approaches have benefits across various applications, such as image caption and image generation. Given these advantages, multimodal learning has become increasingly significant as AI systems aim to perceive and reason more like humans, who naturally integrate sight, sound, and language \citep{baltruvsaitis-etal:2018multimodal}.
% 高效强化学习方法
% 全模态RL In the era of LLM, we can extend their capabilities into the multimodal domain by training them with diverse data types. For example, we can develop a visual language model by training an LLM with image-text pairs using SFT. Here, we return to the issue of aligning models with human preferences. Ideally, once aligned with human preferences through RL, the LLM would also generalize the target modality well by the SFT. However, the reality does not always meet expectations. Although an LLM may align well with human preferences in one aspect, it often struggles to generalize this alignment across a different modality. Thus, to align multimodal models with human preferences, we perform RL to enhance their performance further within the specific modality.
\ No newline at end of file
In this section, we use the visual language, speech generation, and diffusion models to discuss the application of RL in multimodal models.
\subsection{Visual Language Models}
While RL is commonly used to train LLMs, its application to other domains has been a prominent research topic. In multimodal language models\footnote{A multimodal language model is defined as a model that integrates an LLM with a multimodal encoder, such as CLIP \citep{radford-etal:2021learning}, allowing the LLM to process inputs beyond text, such as images. Recent literature has also introduced the use of LLMs for generating outputs in non-text modalities, such as images and speech \citep{xu-etal:2025qwen2,zhang-etal:2024mm}. However, in this section, we focus on the former definition of multimodal language models.}, for example, a notable trend is to perform RL training to improve their trustworthiness and helpfulness. This section considers Visual Language Models (VLMs), which connect a visual encoder to an LLM through a linear projector, facilitating general-purpose visual and language understanding. VLMs are currently the most explored extension in multimodal language research, and they form the foundation for many open-source multimodal language models, such as Qwen2.5-VL \citep{qwenTeam:2025qwen2.5-VL} and LLaMA-3.2-11B-Vision \citep{grattafiori-etal:2024llama}.
Training LLMs and VLMs with RL exhibits only minimal differences, primarily related to the input content. Unlike the textual input used for LLMs, the input for VLMs typically comprises a combination of one or multiple images and an instruction, denoted by $(\mathrm{\mathbf{I}}, \mathrm{\mathbf{x}})$, where $\mathrm{\mathbf{I}}$ represents the input images. These images are encoded into representations that are either concatenated with instruction embeddings or integrated through cross-attention mechanisms into the LLM. In practice, this subtle difference does not significantly affect the applicability of RL algorithms to VLMs. As a result, the RL training process for LLMs, as described in Section \ref{sec:example-using-rl-training-llms}, can be seamlessly adapted to train VLMs without major improvements \citep{yu-etal:2024rlhf,wang-etal:2024rovrm,zang2025internlm,ji2025safe}.
However, training VLMs with RL is not a low-hanging fruit in practical applications. This is because it typically encounters the challenge of training a visual reward model due to the scarcity of high-quality visual preference data. One straightforward approach to address this issue is to generate visual preference data through the automatic preference data generation method described in Section \ref{sec:automatic-preference-data-generation} \citep{yu-etal:2024rlaif}. Another alternative is a multi-stage training approach for the visual reward model, motivated by a simple idea: human preferences are well captured in text, and these preferences can be transferred across modalities \citep{wang-etal:2024rovrm}. By leveraging textual preference data, this approach reduces the dependence on visual preference data in training a visual reward model. More specifically, as illustrated in Figure \ref{fig:preference-transfer}, we can train a visual reward model in the following three stages:
\begin{itemize}
\item Stage 1: pre-training with large-scale textual preference data. Given the transferability of human preferences across different modalities, we can begin by using large-scale textual preference data to pre-train the visual reward model. Note that the projector parameters are frozen without images. This stage can be considered as providing a stronger starting point for training the visual reward model, as it enables the model to pre-learn general human preferences.
\item Stage 2: fine-tuning with image caption-based preference data. Pre-learned human preferences cannot be directly applied to vision tasks due to both \textit{task gap} and \textit{modality gap}. To bridge the task gap, we fine-tune the reward model using image caption-based preference data in this stage. The rationale behind this approach is that general textual preference data does not cover vision-specific tasks, such as ``Please describe the content in this image''. We use such data to fine-tune the model, adapting it to vision-specific tasks. The projector parameters are frozen at this stage.
\item Stage 3: fine-tuning with small-scale visual preference data. To further bridge the modality gap, we use visual preference data in the final stage to train the model. Note that in this phase, we also train the projector parameters.
\end{itemize}
In this process, not all preference data may align with the preferences used in subsequent phases, potentially leading to preference conflicts. We can enhance the visual reward model through data selection techniques, such as LESS \citep{xia-etal:2024less} to address this. In fact, preference transfer is effective across modalities and has also been shown to work across different tasks and languages \citep{cheng-etal:2023everyone,wu-etal:2024reuse}. Interested readers can refer to these papers for more detailed discussions of these topics.
As discussed in Section~\ref{sec:generative-reward-models}, reward reasoning models have demonstrated superior performance in reward prediction. A natural question then arises: \textit{can reward reasoning capabilities also be transferred from text to multimodal settings?} Recent work by \cite{wang2026msrl} provides empirical evidence supporting this hypothesis. By following a similar multi-stage training paradigm, they show that reward reasoning capabilities can indeed be effectively transferred across modalities, further enhancing the performance of visual reward models.
\begin{figure*}[!t]
\centering
\caption{RoVRM}
\label{fig:preference-transfer}
\end{figure*}
Apart from preference data, another approach to improving the visual model is to integrate additional image content, such as image captions, into the reward model \citep{sun-etal:2023aligning}. This approach aims to achieve factually augmented reward prediction, addressing reward hacking. Specifically, in the original setup, the reward model predicts a reward based solely on the image, input, and output; that is, the reward model’s input is $[\mathbf{I}, \mathbf{x}, \mathbf{y}]$. In the factually augmented setup, the reward model also receives additional input in the form of the textual image caption $\mathbf{C}$, resulting in an input of $[\mathbf{I}, \mathbf{C}, \mathbf{x}, \mathbf{y}]$. The basic idea is that the backbone of the visual reward model remains a well-trained LLM with an in-context learning ability. With this ability, we can provide additional content to help the model predict rewards more accurately.
While our discussion primarily focuses on models for visual inputs in the context of visual language models, the RL techniques are adaptable across inputs of various modalities, such as video and audio.
% visual language model as reward model
% 视觉奖励模型训练大致思路
% image reward
% 融合文本数据进行提升
% 视觉奖励模型应用场景---对齐图生文模型 文生图评估模型
% 视觉语言模型的定义
% 框架
% Rlhf-v: Towards trustworthy mllms via behavior alignment from fine-grained correctional human feedback
% Aligning Large Multimodal Models with Factually Augmented RLHF
% 偏好数据
% RLAIF-V
% Silkie: Preference Distillation for Large Visual Language Models
% RoVRM
\subsection{Speech Generation Models}
% SpeechAlign: Aligning Speech Generation to Human Preferences
\subsection{Diffusion Models}
% Image Reward
% 为什么要进行强化学习
% 大体训练思路和流程
% DDPO (denoising diffusion preference optimization model)
section8/section8.tex
\clearpage \section{Summary}
\section{Systems and Datasets}
% future work
\begin{table}[h] % 强化学习作为一种预训练方式
\centering % 高效强化学习方法
\scalebox{0.88}{ % 全模态RL
\input{section8/tables/dataset}} \ No newline at end of file
\caption{datasets}
\label{tab:dataset}
\end{table}
\begin{table}[h]
\centering
\scalebox{0.88}{
\input{section8/tables/systems}}
\caption{systems}
\label{tab:system}
\end{table}
\ No newline at end of file
section9/section9.tex 0 → 100644
\clearpage
\section{Systems and Datasets}
\begin{table}[h]
\centering
\scalebox{0.88}{
\input{section9/tables/dataset}}
\caption{datasets}
\label{tab:dataset}
\end{table}
\begin{table}[h]
\centering
\scalebox{0.88}{
\input{section9/tables/systems}}
\caption{systems}
\label{tab:system}
\end{table}
\ No newline at end of file
section9/tables/dataset.tex 0 → 100644
\begin{tabular}{lccccc}
\toprule[1.1pt]
\multirow{2}{*}{Dataset Name} & \multirow{2}{*}{\begin{tabular}[c]{@{}c@{}}Sample\\Size\end{tabular}} & \multirow{2}{*}{\begin{tabular}[c]{@{}c@{}}Response\\Size\end{tabular}} & \multirow{2}{*}{Modality} & \multicolumn{2}{c}{Feedback} \\ \cmidrule(l){5-6}
& & & & Source & Category \\ \midrule
\href{https://url}{HuggingFaceH4/stack-exchange-preferences}
&10,000 &1 & \faIcon{file-alt} & Human & Score \\
\href{https://url}{Skywork/Skywork-Reward-Preference-80K-v0.2} &10,000 &2 & \faIcon{images} & AI & Ranking \\
\href{https://huggingface.co/datasets/openbmb/UltraFeedback}{openbmb/UltraFeedback} &10,000 &3 & \faIcon{video} & Rule & \\
&10,000 &4 & \faIcon{volume-up} & & \\
& & &\faIcon{file-alt} \faIcon{images} & & \\
& & & & & \\
& & & & & \\
& & & & & \\
\bottomrule[1.1pt]
\end{tabular}
\ No newline at end of file
section9/tables/systems.tex 0 → 100644
\begin{tabular}{lccccc}
\toprule[1.1pt]
\multirow{2}{*}{System Name} & \multicolumn{4}{c}{Supported Modality} & \multirow{2}{*}{Supported Training Approaches} \\ \cmidrule(r){2-5}
& Text & Image & Video & Audio & \\ \midrule
\href{https://github.com/huggingface/trl}{TRL} & \CheckmarkBold & \CheckmarkBold &\XSolidBrush & \CheckmarkBold & Reward Modeling, PPO, GRPO, DPO, Online-DPO, etc. \\
\href{https://github.com/OpenRLHF/OpenRLHF}{OpenRLHF}& \CheckmarkBold & \XSolidBrush & \XSolidBrush & \XSolidBrush & Sft, Reject Sampling, PPO, GRPO, DPO, KTO, etc.\\
\href{https://github.com/hiyouga/EasyR1}{EasyR1}& \CheckmarkBold & \CheckmarkBold & \XSolidBrush & \XSolidBrush & GRPO, Reinforce++, Remax, RLOO, etc. \\
\href{https://github.com/volcengine/verl}{veRL}& \CheckmarkBold & \XSolidBrush & \XSolidBrush & \XSolidBrush & GRPO, PPO, Remax, RLOO, SFT, etc. \\
\href{https://github.com/OpenRLHF/OpenRLHF-M}{OpenRLHF-M}& \XSolidBrush & \CheckmarkBold & \XSolidBrush & \XSolidBrush & PPO, GRPO, RLOO, Online-RLHF, Reject-Sampling, etc. \\
\href{https://github.com/PKU-Alignment/align-anything}{Align-Anything}& \CheckmarkBold & \CheckmarkBold & \CheckmarkBold & \CheckmarkBold &PPO, GRPO, DPO,KTO, ORPO, etc. \\
\bottomrule[1.1pt]
\end{tabular}
\ No newline at end of file
Markdown 格式
0% 或
您添加了 0 人 到此讨论。请谨慎行事。
请先完成此评论的编辑！
请 注册 或者 后发表评论