update.

section4/Figures/figure-rm-evaluation.tex

+\begin{tikzpicture}[
+    box/.style={
+        draw,
+        rectangle,
+        minimum height=0.85cm,
+        minimum width=1.65cm,
+        align=center,
+        font=\small,
+        thick
+    },
+    smallbox/.style={
+        draw,
+        rectangle,
+        minimum height=0.6cm,
+        minimum width=1.2cm,
+        align=center,
+        font=\scriptsize,
+        thick
+    },
+    resultbox/.style={
+        box,
+        fill=rmgreen,
+        text width=2.9cm,
+        minimum width=3.15cm,
+        font=\scriptsize
+    },
+    arrow/.style={
+        -{Stealth},
+        thick
+    },
+    arrowlabel/.style={
+        midway,
+        above,
+        fill=white,
+        inner sep=1pt,
+        font=\scriptsize
+    },
+    subtitle/.style={
+        font=\bfseries\small
+    }
+]
+
+% colors
+\definecolor{rmblue}{RGB}{190,210,240}
+\definecolor{rmpink}{RGB}{245,190,200}
+\definecolor{rmyellow}{RGB}{250,235,180}
+\definecolor{rmgreen}{RGB}{190,230,190}
+\definecolor{rmpurple}{RGB}{220,200,240}
+
+
+% ======================
+% (a) RL-based Evaluation
+% ======================
+
+\node[box, fill=rmpurple] (rm)
+at (-5.25,4.35) {Reward\\Model};
+
+\node[box, fill=rmblue] (rl)
+at (-1.75,4.35) {RL\\Training};
+
+\node[box, fill=rmyellow]
+(policy)
+at (1.75,4.35) {Policy\\LLM};
+
+\node[box, fill=rmgreen]
+(bench)
+at (5.25,4.35) {Benchmark};
+
+
+\draw[arrow] (rm)--(rl)
+node[arrowlabel]{reward signal};
+
+\draw[arrow] (rl)--(policy)
+node[arrowlabel]{RL training};
+
+\draw[arrow] (policy)--(bench)
+node[arrowlabel]{benchmark};
+
+
+\node[subtitle] at (0,3.55) {(a) RL-based Evaluation};
+
+
+
+% ======================
+% (b) Pairwise
+% ======================
+
+\node[smallbox] (pair-input)
+at (-6.85,-0.45)
+{$(x,y^+,y^-)$};
+
+
+\node[box,fill=rmblue]
+(rma)
+at (-5.05,0.25)
+{Reward\\Model A};
+
+\node[box,fill=rmpink]
+(rmb)
+at (-5.05,-1.1)
+{Reward\\Model B};
+
+
+\node[smallbox]
+(a-result)
+at (-3.3,0.25)
+{$y^+>y^-$};
+
+\node[smallbox]
+(b-result)
+at (-3.3,-1.1)
+{$y^+<y^-$};
+
+
+\node[resultbox]
+(pair-eval)
+at (-4.05,1.45)
+{Evaluation\\Result\\
+Human prefers $y^+$\\
+Model A correct};
+
+
+\draw[arrow]
+(pair-input)--(rma);
+
+\draw[arrow]
+(pair-input)--(rmb);
+
+\draw[arrow]
+(rma)--(a-result);
+
+\draw[arrow]
+(rmb)--(b-result);
+
+\draw[arrow]
+(a-result.north)--(pair-eval.south east);
+
+\draw[arrow]
+(b-result.north)--(pair-eval.south);
+
+
+\node[subtitle] at (-4.05,-1.85)
+{(b) Pairwise Ranking Evaluation};
+
+
+
+% ======================
+% (c) Listwise
+% ======================
+
+\node[smallbox]
+(candidate)
+at (1.15,-0.45)
+{$y_1$\\$y_2$\\$y_3$\\$\cdots$};
+
+
+\node[box,fill=rmblue]
+(lrma)
+at (3.0,0.25)
+{Reward\\Model A};
+
+\node[box,fill=rmpink]
+(lrmb)
+at (3.0,-1.1)
+{Reward\\Model B};
+
+
+\node[smallbox]
+(best-a)
+at (4.7,0.25)
+{Best is\\$y_3$};
+
+\node[smallbox]
+(best-b)
+at (4.7,-1.1)
+{Best is\\$y_4$};
+
+
+\node[resultbox]
+(list-eval)
+at (4.0,1.45)
+{Evaluation\\Result\\
+Human selects $y_3$\\
+RM-A better};
+
+
+\draw[arrow]
+(candidate)--(lrma);
+
+\draw[arrow]
+(candidate)--(lrmb);
+
+\draw[arrow]
+(lrma)--(best-a);
+
+\draw[arrow]
+(lrmb)--(best-b);
+
+\draw[arrow]
+(best-a.north)--(list-eval.south east);
+
+\draw[arrow]
+(best-b.north)--(list-eval.south);
+
+
+\node[subtitle] at (4.0,-1.85)
+{(c) Listwise Ranking Evaluation};
+
+
+\end{tikzpicture}