\begin{tabular}{ll}
  \toprule[1.1pt]
  \textbf{Method} & \textbf{Objective} \\ \midrule
  IPO \citep{azar:2024general} & $ \left( \log \frac{\mathrm{Pr}_\theta(\mathbf{y}_a|\mathbf{x})}{\mathrm{Pr}_{\theta_\mathrm{ref}}(\mathbf{y}_a|\mathbf{x})} - \log \frac{\mathrm{Pr}_\theta(\mathbf{y}_b|\mathbf{x})}{\mathrm{Pr}_{\theta_\mathrm{ref}}(\mathbf{y}_b|\mathbf{x})} - \frac{1}{2\tau} \right)^2$ \\ \midrule
  CPO \citep{xu:2024contrastive} &  $-\log \textrm{Sigmoid}  \left(\beta \log \mathrm{Pr}_\theta(\mathbf{y}_a|\mathbf{x}) - \beta \log \mathrm{Pr}_\theta(\mathbf{y}_b|\mathbf{x}) \right) - \lambda \log \mathrm{Pr}_\theta (\mathbf{y}_a|\mathbf{x})$  \\ \midrule
  \multirow{2}{*}{KTO \citep{ethayarajh:2024kto}} & $-\lambda_a \textrm{Sigmoid} \left( \beta \log \frac{\mathrm{Pr}_\theta(\mathbf{y}_a|\mathbf{x})}{\mathrm{Pr}_{\theta_\mathrm{ref}}(\mathbf{y}_a|\mathbf{x})} - z_{\theta_\mathrm{ref}} \right) +  \lambda_b \textrm{Sigmoid} \left( z_{\theta_\mathrm{ref}} - \beta \log \frac{\mathrm{Pr}_\theta(\mathbf{y}_b|\mathbf{x})}{\mathrm{Pr}_{\theta_\mathrm{ref}}(\mathbf{y}_b|\mathbf{x})} \right)\,$ \\
  & $\text{where} \,\, z_{\theta_\mathrm{ref}} = \mathbb{E}_{(\mathbf{x}, \mathbf{y}) \sim \mathcal{S}} \left[\beta \text{KL}\left( \mathrm{Pr}_\theta(\mathbf{y}|\mathbf{x}) || \mathrm{Pr}_{\theta_\mathrm{ref}}(\mathbf{y}|\mathbf{x}) \right)  \right]$ \\ \midrule
  \multirow{2}{*}{ORPO \citep{hong:2024orpo}} & $-\log p_\theta(\mathbf{y}_a|\mathbf{x}) - \lambda  \log \textrm{Sigmoid} \left(\log \frac{p_\theta(\mathbf{y}_a|\mathbf{x})}{1 - p_\theta(\mathbf{y}_a|\mathbf{x})} - \log \frac{p_\theta(\mathbf{y}_b|\mathbf{x})}{1 - p_\theta(\mathbf{y}_b|\mathbf{x})}  \right)\,$ \\
 & $\text{where} \,\, p_\theta(\mathbf{y}|\mathbf{x}) = e^{ \frac{1}{|\mathbf{y}|} \log \mathrm{Pr}_\theta(\mathbf{y}|\mathbf{x})}$ \\  \midrule
  R-DPO \citep{gallego:2024refined} & $-\log \textrm{Sigmoid} \left( \beta \log \frac{\mathrm{Pr}_\theta(\mathbf{y}_a|\mathbf{x})}{\mathrm{Pr}_{\theta_\mathrm{ref}}(\mathbf{y}_a|\mathbf{x})} - \beta \log \frac{\mathrm{Pr}_\theta(\mathbf{y}_b|\mathbf{x})}{\mathrm{Pr}_{\theta_\mathrm{ref}}(\mathbf{y}_b|\mathbf{x})} + \left(\alpha |\mathbf{y}_a| - \alpha |\mathbf{y}_b| \right) \right)$  \\ \midrule
  \multirow{2}{*}{PCDPO \citep{zhou:2024prior}} & $-\log \textrm{Sigmoid} \left(\Delta^* - \Delta_{\mathrm{Pr}_{\theta}}\right) -\log \textrm{Sigmoid} \Delta_{\mathrm{Pr}_{\theta}} \,,$ \\ 
  & $\text{where} \,\, \Delta_{\mathrm{Pr}_{\theta}}=  \beta \log \frac{\mathrm{Pr}_\theta(\mathbf{y}_a|\mathbf{x})}{\mathrm{Pr}_{\theta_\mathrm{ref}}(\mathbf{y}_a|\mathbf{x})} - \beta \log \frac{\mathrm{Pr}_\theta(\mathbf{y}_b|\mathbf{x})}{\mathrm{Pr}_{\theta_\mathrm{ref}}(\mathbf{y}_b|\mathbf{x})},\,\, \Delta^* = \frac{\beta_1}{\textrm{Sim}(\textbf{y}_a,\textbf{y}_b|\textbf{x})+\beta_2}+\beta_3 $ \\ \midrule
  SimPO \citep{meng:2025simpo} & $-\log \textrm{Sigmoid}  \left( \frac{\beta}{|\mathbf{y}_a|} \log \mathrm{Pr}_\theta(\mathbf{y}_a|\mathbf{x}) - \frac{\beta}{|\mathbf{y}_b|} \log \mathrm{Pr}_\theta(\mathbf{y}_b|\mathbf{x}) - \gamma \right)$ \\ \midrule
  ODPO \citep{amini:2024direct} 
  & $-\log \textrm{Sigmoid} \left(
  \beta \log \frac{\mathrm{Pr}_\theta(\mathbf{y}_a|\mathbf{x})}{\mathrm{Pr}_{\theta_\mathrm{ref}}(\mathbf{y}_a|\mathbf{x})}
  -
  \beta \log \frac{\mathrm{Pr}_\theta(\mathbf{y}_b|\mathbf{x})}{\mathrm{Pr}_{\theta_\mathrm{ref}}(\mathbf{y}_b|\mathbf{x})}
  - \delta(\mathbf{x},\mathbf{y}_a,\mathbf{y}_b)
  \right)$ \\ \midrule

  \multirow{2}{*}{Cal-DPO \citep{xiao:2024cal}} 
  & $-\log \textrm{Sigmoid} \left(\Delta_{\mathrm{Pr}_\theta}\right)
  + \lambda \mathcal{L}_{\mathrm{cal}}$ \\
  & $\text{where} \,\, 
  \Delta_{\mathrm{Pr}_\theta}
  =
  \beta \log \frac{\mathrm{Pr}_\theta(\mathbf{y}_a|\mathbf{x})}{\mathrm{Pr}_{\theta_\mathrm{ref}}(\mathbf{y}_a|\mathbf{x})}
  -
  \beta \log \frac{\mathrm{Pr}_\theta(\mathbf{y}_b|\mathbf{x})}{\mathrm{Pr}_{\theta_\mathrm{ref}}(\mathbf{y}_b|\mathbf{x})},
  \,\, \mathcal{L}_{\mathrm{cal}} \text{ calibrates implicit rewards.}$ \\ \midrule

  \multirow{2}{*}{TDPO \citep{zeng:2024token}} 
  & $-\log \textrm{Sigmoid} \left(
  \sum_{t=1}^{T_a} \beta \log 
  \frac{\mathrm{Pr}_\theta(\mathbf{y}_{a,t}|\mathbf{x},\mathbf{y}_{a,<t})}
  {\mathrm{Pr}_{\theta_\mathrm{ref}}(\mathbf{y}_{a,t}|\mathbf{x},\mathbf{y}_{a,<t})}
  -
  \sum_{t=1}^{T_b} \beta \log 
  \frac{\mathrm{Pr}_\theta(\mathbf{y}_{b,t}|\mathbf{x},\mathbf{y}_{b,<t})}
  {\mathrm{Pr}_{\theta_\mathrm{ref}}(\mathbf{y}_{b,t}|\mathbf{x},\mathbf{y}_{b,<t})}
  \right)
  + \lambda \sum_t \mathrm{KL}_t$ \\
  & $\text{where} \,\, \mathrm{KL}_t \text{ denotes the token-level KL regularization term.}$ \\ \midrule

  \multirow{2}{*}{AOT \citep{melnyk:2024distributional}} 
  & $\mathcal{L}_{\mathrm{OT}}\left(
  \{r_\theta(\mathbf{x},\mathbf{y}_a)\},
  \{r_\theta(\mathbf{x},\mathbf{y}_b)\}
  \right)$ \\
  & $\text{where} \,\,
  r_\theta(\mathbf{x},\mathbf{y})
  =
  \beta \log \frac{\mathrm{Pr}_\theta(\mathbf{y}|\mathbf{x})}
  {\mathrm{Pr}_{\theta_\mathrm{ref}}(\mathbf{y}|\mathbf{x})},
  \text{ and } \mathcal{L}_{\mathrm{OT}} \text{ aligns preference distributions via optimal transport.}$ \\ \midrule
  D$^2$PO \citep{shao:2025earlier} &  $-\log \textrm{Sigmoid} \left( \sum_{t=0}^{T}\gamma^t\beta \log \frac{\mathrm{Pr}_\theta(\mathbf{y}_a^t|\mathbf{x},\mathbf{y}_{a,<t})}{\mathrm{Pr}_{\theta_\mathrm{ref}}(\mathbf{y}_{a,t}|\mathbf{x},\mathbf{y}_{a,<t})} - \sum_{t=0}^{T}\gamma^t\beta \log \frac{\mathrm{Pr}_\theta(\mathbf{y}_{b,t}|\mathbf{x},\mathbf{y}_{b,<t})}{\mathrm{Pr}_{\theta_\mathrm{ref}}(\mathbf{y}_{b,t}|\mathbf{x},\mathbf{y}_{b,<t})}\right)$ \\ 
\bottomrule[1.1pt]
\end{tabular}
