update.

rl-introduction.bib

@@ -2,6 +2,34 @@
 
 
 
 
 
 
+@inproceedings{melnyk:2024distributional,
+  title={Distributional Preference Alignment of LLMs via Optimal Transport},
+  author={Melnyk, Igor and Mroueh, Youssef and Belgodere, Brian and Rigotti, Mattia and Nitsure, Apoorva and Yurochkin, Mikhail and Greenewald, Kristjan and Navratil, Jiri and Ross, Jerret},
+  booktitle={Advances in Neural Information Processing Systems},
+  year={2024}
+}
+
+@inproceedings{zeng:2024token,
+  title={Token-level Direct Preference Optimization},
+  author={Zeng, Yongcheng and Liu, Guoqing and Ma, Weiyu and Yang, Ning and Zhang, Haifeng and Wang, Jun},
+  booktitle={Proceedings of the 41st International Conference on Machine Learning},
+  year={2024}
+}
+
+@inproceedings{xiao:2024cal,
+  title={Cal-DPO: Calibrated Direct Preference Optimization for Language Model Alignment},
+  author={Xiao, Teng and Yuan, Yige and Zhu, Huaisheng and Li, Mingxiao and Honavar, Vasant G.},
+  booktitle={The Thirty-eighth Annual Conference on Neural Information Processing Systems},
+  year={2024}
+}
+
+@article{amini:2024direct,
+  title={Direct Preference Optimization with an Offset},
+  author={Amini, Afra and Vieira, Tim and Cotterell, Ryan},
+  journal={arXiv preprint arXiv:2402.10571},
+  year={2024}
+}
+
 @article{qian-etal:toolrl,
 @article{qian-etal:toolrl,
   title={Toolrl: Reward is all tool learning needs},
   title={Toolrl: Reward is all tool learning needs},
   author={Qian, Cheng and Acikgoz, Emre Can and He, Qi and Wang, Hongru and Chen, Xiusi and Hakkani-Tur, Dilek and Tur, Gokhan and Ji, Heng},
   author={Qian, Cheng and Acikgoz, Emre Can and He, Qi and Wang, Hongru and Chen, Xiusi and Hakkani-Tur, Dilek and Tur, Gokhan and Ji, Heng},

section4/Tables/dpo_variants.tex

-  DPO \citep{rafailov:2023direct} 
-  & $-\log \textrm{Sigmoid} \left(
-  \beta \log \frac{\mathrm{Pr}_\theta(\mathbf{y}_a|\mathbf{x})}{\mathrm{Pr}_{\theta_\mathrm{ref}}(\mathbf{y}_a|\mathbf{x})}
-  -
-  \beta \log \frac{\mathrm{Pr}_\theta(\mathbf{y}_b|\mathbf{x})}{\mathrm{Pr}_{\theta_\mathrm{ref}}(\mathbf{y}_b|\mathbf{x})}
-  \right)$ \\ \midrule
-
+\begin{tabular}{ll}
+  \toprule[1.1pt]
+  \textbf{Method} & \textbf{Objective} \\ \midrule
+  IPO \citep{azar:2024general} & $ \left( \log \frac{\mathrm{Pr}_\theta(\mathbf{y}_a|\mathbf{x})}{\mathrm{Pr}_{\theta_\mathrm{ref}}(\mathbf{y}_a|\mathbf{x})} - \log \frac{\mathrm{Pr}_\theta(\mathbf{y}_b|\mathbf{x})}{\mathrm{Pr}_{\theta_\mathrm{ref}}(\mathbf{y}_b|\mathbf{x})} - \frac{1}{2\tau} \right)^2$ \\ \midrule
+  CPO \citep{xu:2024contrastive} &  $-\log \textrm{Sigmoid}  \left(\beta \log \mathrm{Pr}_\theta(\mathbf{y}_a|\mathbf{x}) - \beta \log \mathrm{Pr}_\theta(\mathbf{y}_b|\mathbf{x}) \right) - \lambda \log \mathrm{Pr}_\theta (\mathbf{y}_a|\mathbf{x})$  \\ \midrule
+  \multirow{2}{*}{KTO \citep{ethayarajh:2024kto}} & $-\lambda_a \textrm{Sigmoid} \left( \beta \log \frac{\mathrm{Pr}_\theta(\mathbf{y}_a|\mathbf{x})}{\mathrm{Pr}_{\theta_\mathrm{ref}}(\mathbf{y}_a|\mathbf{x})} - z_{\theta_\mathrm{ref}} \right) +  \lambda_b \textrm{Sigmoid} \left( z_{\theta_\mathrm{ref}} - \beta \log \frac{\mathrm{Pr}_\theta(\mathbf{y}_b|\mathbf{x})}{\mathrm{Pr}_{\theta_\mathrm{ref}}(\mathbf{y}_b|\mathbf{x})} \right)\,$ \\
+  & $\text{where} \,\, z_{\theta_\mathrm{ref}} = \mathbb{E}_{(\mathbf{x}, \mathbf{y}) \sim \mathcal{S}} \left[\beta \text{KL}\left( \mathrm{Pr}_\theta(\mathbf{y}|\mathbf{x}) || \mathrm{Pr}_{\theta_\mathrm{ref}}(\mathbf{y}|\mathbf{x}) \right)  \right]$ \\ \midrule
+  \multirow{2}{*}{ORPO \citep{hong:2024orpo}} & $-\log p_\theta(\mathbf{y}_a|\mathbf{x}) - \lambda  \log \textrm{Sigmoid} \left(\log \frac{p_\theta(\mathbf{y}_a|\mathbf{x})}{1 - p_\theta(\mathbf{y}_a|\mathbf{x})} - \log \frac{p_\theta(\mathbf{y}_b|\mathbf{x})}{1 - p_\theta(\mathbf{y}_b|\mathbf{x})}  \right)\,$ \\
+ & $\text{where} \,\, p_\theta(\mathbf{y}|\mathbf{x}) = e^{ \frac{1}{|\mathbf{y}|} \log \mathrm{Pr}_\theta(\mathbf{y}|\mathbf{x})}$ \\  \midrule
+  R-DPO \citep{gallego:2024refined} & $-\log \textrm{Sigmoid} \left( \beta \log \frac{\mathrm{Pr}_\theta(\mathbf{y}_a|\mathbf{x})}{\mathrm{Pr}_{\theta_\mathrm{ref}}(\mathbf{y}_a|\mathbf{x})} - \beta \log \frac{\mathrm{Pr}_\theta(\mathbf{y}_b|\mathbf{x})}{\mathrm{Pr}_{\theta_\mathrm{ref}}(\mathbf{y}_b|\mathbf{x})} + \left(\alpha |\mathbf{y}_a| - \alpha |\mathbf{y}_b| \right) \right)$  \\ \midrule
+  \multirow{2}{*}{PCDPO \citep{zhou:2024prior}} & $-\log \textrm{Sigmoid} \left(\Delta^* - \Delta_{\mathrm{Pr}_{\theta}}\right) -\log \textrm{Sigmoid} \Delta_{\mathrm{Pr}_{\theta}} \,,$ \\ 
+  & $\text{where} \,\, \Delta_{\mathrm{Pr}_{\theta}}=  \beta \log \frac{\mathrm{Pr}_\theta(\mathbf{y}_a|\mathbf{x})}{\mathrm{Pr}_{\theta_\mathrm{ref}}(\mathbf{y}_a|\mathbf{x})} - \beta \log \frac{\mathrm{Pr}_\theta(\mathbf{y}_b|\mathbf{x})}{\mathrm{Pr}_{\theta_\mathrm{ref}}(\mathbf{y}_b|\mathbf{x})},\,\, \Delta^* = \frac{\beta_1}{\textrm{Sim}(\textbf{y}_a,\textbf{y}_b|\textbf{x})+\beta_2}+\beta_3 $ \\ \midrule
+  SimPO \citep{meng:2025simpo} & $-\log \textrm{Sigmoid}  \left( \frac{\beta}{|\mathbf{y}_a|} \log \mathrm{Pr}_\theta(\mathbf{y}_a|\mathbf{x}) - \frac{\beta}{|\mathbf{y}_b|} \log \mathrm{Pr}_\theta(\mathbf{y}_b|\mathbf{x}) - \gamma \right)$ \\ \midrule
   ODPO \citep{amini:2024direct} 
   ODPO \citep{amini:2024direct} 
   & $-\log \textrm{Sigmoid} \left(
   & $-\log \textrm{Sigmoid} \left(
   \beta \log \frac{\mathrm{Pr}_\theta(\mathbf{y}_a|\mathbf{x})}{\mathrm{Pr}_{\theta_\mathrm{ref}}(\mathbf{y}_a|\mathbf{x})}
   \beta \log \frac{\mathrm{Pr}_\theta(\mathbf{y}_a|\mathbf{x})}{\mathrm{Pr}_{\theta_\mathrm{ref}}(\mathbf{y}_a|\mathbf{x})}
@@ -48,3 +54,6 @@
   \beta \log \frac{\mathrm{Pr}_\theta(\mathbf{y}|\mathbf{x})}
   \beta \log \frac{\mathrm{Pr}_\theta(\mathbf{y}|\mathbf{x})}
   {\mathrm{Pr}_{\theta_\mathrm{ref}}(\mathbf{y}|\mathbf{x})},
   {\mathrm{Pr}_{\theta_\mathrm{ref}}(\mathbf{y}|\mathbf{x})},
   \text{ and } \mathcal{L}_{\mathrm{OT}} \text{ aligns preference distributions via optimal transport.}$ \\ \midrule
   \text{ and } \mathcal{L}_{\mathrm{OT}} \text{ aligns preference distributions via optimal transport.}$ \\ \midrule
+  D$^2$PO \citep{shao:2025earlier} &  $-\log \textrm{Sigmoid} \left( \sum_{t=0}^{T}\gamma^t\beta \log \frac{\mathrm{Pr}_\theta(\mathbf{y}_a^t|\mathbf{x},\mathbf{y}_{a,<t})}{\mathrm{Pr}_{\theta_\mathrm{ref}}(\mathbf{y}_{a,t}|\mathbf{x},\mathbf{y}_{a,<t})} - \sum_{t=0}^{T}\gamma^t\beta \log \frac{\mathrm{Pr}_\theta(\mathbf{y}_{b,t}|\mathbf{x},\mathbf{y}_{b,<t})}{\mathrm{Pr}_{\theta_\mathrm{ref}}(\mathbf{y}_{b,t}|\mathbf{x},\mathbf{y}_{b,<t})}\right)$ \\ 
+\bottomrule[1.1pt]
+\end{tabular}

section6/section6.tex

@@ -169,7 +169,11 @@ Tool using is another fundamental capability of LLM-based agents. Tools extend t
 \subsection{Improved Environments}
 \subsection{Improved Environments}
 
 
 
 
-\subsubsection{Scaling Up }
+% large-scale environment construct
+
+
+
+\subsection{Self-Evolving Agents}