Skip to content

R
rl-introduction
Commit 80e68ccc by 周航
Options

Update the figure of section4

parent 9bf1efa9
正在显示 包含 68 行增加 和 181 行删除
section4/Figures/figure-rm-evaluation.tex
\begin{tikzpicture}[ \begin{center}
box/.style={ \begin{tikzpicture}[
draw, box/.style={draw, rounded corners, rectangle, minimum height=0.85cm, minimum width=1.65cm, align=center, font=\scriptsize, anchor=west},
rectangle, smallbox/.style={minimum height=0.6cm, minimum width=1.2cm, align=center, font=\scriptsize, anchor=west,},
minimum height=0.85cm, resultbox/.style={box, fill=rmgreen, text width=2.9cm, minimum width=3.15cm, font=\scriptsize},
minimum width=1.65cm, arrow/.style={->,},
align=center, arrowlabel/.style={midway, above, fill=white, inner sep=1pt, font=\scriptsize},
font=\small, subtitle/.style={font=\bfseries\small}
thick ]
},
smallbox/.style={
draw,
rectangle,
minimum height=0.6cm,
minimum width=1.2cm,
align=center,
font=\scriptsize,
thick
},
resultbox/.style={
box,
fill=rmgreen,
text width=2.9cm,
minimum width=3.15cm,
font=\scriptsize
},
arrow/.style={
-{Stealth},
thick
},
arrowlabel/.style={
midway,
above,
fill=white,
inner sep=1pt,
font=\scriptsize
},
subtitle/.style={
font=\bfseries\small
}
]
% colors
\definecolor{rmblue}{RGB}{190,210,240}
\definecolor{rmpink}{RGB}{245,190,200}
\definecolor{rmyellow}{RGB}{250,235,180}
\definecolor{rmgreen}{RGB}{190,230,190}
\definecolor{rmpurple}{RGB}{220,200,240}
\definecolor{rmblue}{RGB}{190,210,240}
\definecolor{rmpink}{RGB}{245,190,200}
\definecolor{rmyellow}{RGB}{250,235,180}
\definecolor{rmgreen}{RGB}{190,230,190}
\definecolor{rmpurple}{RGB}{220,200,240}
% ====================== \scriptsize
% (a) RL-based Evaluation
% ======================
\node[box, fill=rmpurple] (rm) \begin{scope}[local bounding box=pa]
at (-5.25,4.35) {Reward\\Model};
\node[box, fill=rmblue] (rl) \node[text width=8cm] (patitle) at (0,0)
at (-1.75,4.35) {RL\\Training}; {blabla blabla blabla blabla blabla blabla blabla blabla blabla blabla blabla blabla blabla blabla blabla blabla};
\node[box, fill=rmpurple!60,anchor=north west] (rma) at ([yshift=-0.1cm]patitle.south west) {Reward \\ Model A};
\node[box, fill=rmpurple!60,anchor=north] (rmb) at ([yshift=-0.2cm]rma.south) {Reward \\ Model B};
\node[box, fill=rmyellow] \node[box, fill=rmblue!60] (rl) at ([xshift=2cm]$1/2*(rma.east)+1/2*(rmb.east)$) {RL\\Training};
(policy) \node[box, fill=rmyellow!60] (policy) at ([xshift=3cm]rl) {Policy\\LLM};
at (1.75,4.35) {Policy\\LLM};
\node[box, fill=rmgreen] \node[box, fill=rmgreen!60,anchor=north west] (bencha) at ([xshift=2cm]rma.north-|policy.east) {Score of \\ Reward Model A};
(bench) \node[box, fill=rmgreen!60,anchor=north west] (benchb) at ([xshift=2cm]rmb.north-|policy.east) {Score of \\ Reward Model B};
at (5.25,4.35) {Benchmark}; \node[anchor=south] at ([yshift=0.1cm]bencha.north) {\textbf{Evaluation Results}};
\path[draw,dashed,rounded corners] ([xshift=1.75cm]rma.north east) rectangle ([xshift=-1.75cm]benchb.south west);
\draw[arrow] (rm)--(rl) \draw[arrow] (rma.east) -- node[below,align=center]{reward signal} +(1.75cm,0);
node[arrowlabel]{reward signal}; \draw[arrow] (rmb.east) -- +(1.75cm,0);
\draw[arrow] (rl)--(policy) node[arrowlabel]{RL training};
\draw[arrow] ([xshift=-1.75cm]bencha.west) -- node[below,align=center]{benchmark} +(1.75cm,0);
\draw[arrow] ([xshift=-1.75cm]benchb.west) -- +(1.75cm,0);
\draw[arrow] (rl)--(policy) \node[subtitle,anchor=north] at ([yshift=-0.1cm]$1/2*(rmb.south west)+1/2*(benchb.south east)$) {(a) RL-based Evaluation};
node[arrowlabel]{RL training};
\draw[arrow] (policy)--(bench) \end{scope}
node[arrowlabel]{benchmark};
\begin{scope}[shift={({$(pa)-(8,0)$}|-(0,-5.5))},local bounding box=pb]
\node[smallbox] (pair-input) at (0,0) {$(x,y^+,y^-)$};
\node[subtitle] at (0,3.55) {(a) RL-based Evaluation}; \node[box,fill=rmblue!60] (rma) at ([xshift=1.5cm,yshift=.75cm]pair-input.east) {Reward\\Model A};
\node[box,fill=rmpink!60] (rmb) at ([xshift=1.5cm,yshift=-.75cm]pair-input.east) {Reward\\Model B};
\node[smallbox] (a-result) at ([xshift=1.5cm]rma.east) {$y^+>y^-$};
\node[smallbox] (b-result) at ([xshift=1.5cm]rmb.east) {$y^+<y^-$};
\node[anchor=south,align=center] at ([yshift=0.1cm]a-result.north) {\textbf{Evaluation}\\\textbf{Results}};
\node[text width=5cm,anchor=south west] (pair-eval) at ([yshift=0.1cm]pair-input.west|-rma.north)
{Evaluation Result Human prefers $y^+$ Model A correct};
% ====================== \draw[arrow] (pair-input)--(rma);
% (b) Pairwise \draw[arrow] (pair-input)--(rmb);
% ====================== \draw[arrow] (rma)--(a-result);
\draw[arrow] (rmb)--(b-result);
\node[smallbox] (pair-input) \node[subtitle,anchor=north,yshift=-0.25cm] (subb) at ({$1/2*(pair-input.west)+1/2*(b-result.east)$}|-rmb.south)
at (-6.85,-0.45) {(b) Pairwise Ranking Evaluation};
{$(x,y^+,y^-)$};
\end{scope}
\node[box,fill=rmblue] \begin{scope}[shift={({$2*(pa)-(pb.east)$}|-(0,-5.5))}]
(rma)
at (-5.05,0.25)
{Reward\\Model A};
\node[box,fill=rmpink]
(rmb)
at (-5.05,-1.1)
{Reward\\Model B};
\node[smallbox] (candidate) at (0,0) {$(x, y_1, y_2, y_3, \cdots)$};
\node[smallbox] \node[box,fill=rmblue!60] (lrma) at ([xshift=1cm,yshift=.75cm]candidate.east) {Reward\\Model A};
(a-result) \node[box,fill=rmpink!60] (lrmb) at ([xshift=1cm,yshift=-.75cm]candidate.east) {Reward\\Model B};
at (-3.3,0.25)
{$y^+>y^-$};
\node[smallbox]
(b-result)
at (-3.3,-1.1)
{$y^+<y^-$};
\node[smallbox] (best-a) at ([xshift=1cm]lrma.east) {Best is $y_3$};
\node[smallbox] (best-b) at ([xshift=1cm]lrmb.east) {Best is $y_4$};
\node[anchor=south,align=center] at ([yshift=0.1cm]best-a.north) {\textbf{Evaluation}\\\textbf{Results}};
\node[resultbox] \node[text width=5cm,anchor=south west] (list-eval) at ([yshift=0.1cm]candidate.west|-lrma.north)
(pair-eval) {Evaluation Result Human selects $y_3$ RM-A better};
at (-4.05,1.45)
{Evaluation\\Result\\
Human prefers $y^+$\\
Model A correct};
\draw[arrow] (candidate)--(lrma);
\draw[arrow] (candidate)--(lrmb);
\draw[arrow] \draw[arrow] (lrma)--(best-a);
(pair-input)--(rma); \draw[arrow] (lrmb)--(best-b);
\draw[arrow] \node[subtitle,anchor=north] at ({$1/2*(candidate.west)+1/2*(best-b.east)$}|-subb.north)
(pair-input)--(rmb); {(c) Listwise Ranking Evaluation};
\draw[arrow] \end{scope}
(rma)--(a-result);
\draw[arrow]
(rmb)--(b-result);
\draw[arrow]
(a-result.north)--(pair-eval.south east);
\draw[arrow] \end{tikzpicture}
(b-result.north)--(pair-eval.south); \end{center}
\node[subtitle] at (-4.05,-1.85)
{(b) Pairwise Ranking Evaluation};
% ======================
% (c) Listwise
% ======================
\node[smallbox]
(candidate)
at (1.15,-0.45)
{$y_1$\\$y_2$\\$y_3$\\$\cdots$};
\node[box,fill=rmblue]
(lrma)
at (3.0,0.25)
{Reward\\Model A};
\node[box,fill=rmpink]
(lrmb)
at (3.0,-1.1)
{Reward\\Model B};
\node[smallbox]
(best-a)
at (4.7,0.25)
{Best is\\$y_3$};
\node[smallbox]
(best-b)
at (4.7,-1.1)
{Best is\\$y_4$};
\node[resultbox]
(list-eval)
at (4.0,1.45)
{Evaluation\\Result\\
Human selects $y_3$\\
RM-A better};
\draw[arrow]
(candidate)--(lrma);
\draw[arrow]
(candidate)--(lrmb);
\draw[arrow]
(lrma)--(best-a);
\draw[arrow]
(lrmb)--(best-b);
\draw[arrow]
(best-a.north)--(list-eval.south east);
\draw[arrow]
(best-b.north)--(list-eval.south);
\node[subtitle] at (4.0,-1.85)
{(c) Listwise Ranking Evaluation};
\end{tikzpicture}
section4/section4.tex
...@@ -220,8 +220,7 @@ After training a reward model, an important question is \textit{how to evaluate ...@@ -220,8 +220,7 @@ After training a reward model, an important question is \textit{how to evaluate
\begin{figure*}[!t] \begin{figure*}[!t]
\centering \centering
\resizebox{\linewidth}{!}{ \input{section4/Figures/figure-rm-evaluation.tex}
\input{section4/Figures/figure-rm-evaluation.tex}}
\caption{ \caption{
Commonly used reward model evaluation approaches. Commonly used reward model evaluation approaches.
For the RL-based evaluation approach, the reward model is used to provide reward signals for policy optimization, and its quality is measured by the downstream performance of the optimized policy. For the RL-based evaluation approach, the reward model is used to provide reward signals for policy optimization, and its quality is measured by the downstream performance of the optimized policy.
......
Markdown 格式
0% 或
您添加了 0 人 到此讨论。请谨慎行事。
请先完成此评论的编辑！
请 注册 或者 后发表评论