update.

section4/Figures/figure-rm-evaluation.tex

 \begin{center}
 \begin{center}
   \begin{tikzpicture}[
   \begin{tikzpicture}[
-    box/.style={draw, rounded corners, rectangle, minimum height=0.85cm, minimum width=1.65cm, align=center, font=\scriptsize, anchor=west},
+    box/.style={draw, rectangle, minimum height=0.85cm, minimum width=1.65cm, align=center, font=\scriptsize, anchor=west},
     smallbox/.style={minimum height=0.6cm, minimum width=1.2cm, align=center, font=\scriptsize, anchor=west,},
     smallbox/.style={minimum height=0.6cm, minimum width=1.2cm, align=center, font=\scriptsize, anchor=west,},
     resultbox/.style={box, fill=rmgreen, text width=2.9cm, minimum width=3.15cm, font=\scriptsize},
     resultbox/.style={box, fill=rmgreen, text width=2.9cm, minimum width=3.15cm, font=\scriptsize},
     arrow/.style={->,},
     arrow/.style={->,},
@@ -16,44 +16,44 @@
 
 
     \scriptsize
     \scriptsize
 
 
-    \begin{scope}[local bounding box=pa]
+    \begin{scope}[yshift=0.35cm,local bounding box=pa]
 
 
-      \node[text width=8cm] (patitle) at (0,0)
-      {blabla blabla blabla blabla blabla blabla blabla blabla blabla blabla blabla blabla blabla blabla blabla blabla};
-      \node[box, fill=rmpurple!60,anchor=north west] (rma) at ([yshift=-0.1cm]patitle.south west) {Reward \\ Model A};
-      \node[box, fill=rmpurple!60,anchor=north] (rmb) at ([yshift=-0.2cm]rma.south) {Reward \\ Model B};
+      \node[text width=12.5cm] (patitle) at (0,0.25)
+      {Use each reward model to provide reward signals for RL training, and evaluate the trained policy on a benchmark. A higher benchmark score indicates that the corresponding reward model is better aligned with human preferences.};
+      \node[box, fill=rmblue!60,anchor=north west] (rma) at ([yshift=-0.35cm]patitle.south west) {Reward \\ Model A};
+      \node[box, fill=rmpink!60,anchor=north] (rmb) at ([yshift=-0.2cm]rma.south) {Reward \\ Model B};
 
 
-      \node[box, fill=rmblue!60] (rl) at ([xshift=2cm]$1/2*(rma.east)+1/2*(rmb.east)$) {RL\\Training};
-      \node[box, fill=rmyellow!60] (policy) at ([xshift=3cm]rl) {Policy\\LLM};
+      \node[box] (rl) at ([xshift=2cm]$1/2*(rma.east)+1/2*(rmb.east)$) {Reward \\ Scores};
+      \node[box] (policy) at ([xshift=3cm]rl) {Policy\\LLM};
 
 
-      \node[box, fill=rmgreen!60,anchor=north west] (bencha) at ([xshift=2cm]rma.north-|policy.east) {Score of \\ Reward Model A};
-      \node[box, fill=rmgreen!60,anchor=north west] (benchb) at ([xshift=2cm]rmb.north-|policy.east) {Score of \\ Reward Model B};
+      \node[box, fill=rmblue!30,anchor=north west] (bencha) at ([xshift=4.0cm]rma.north-|policy.east) {Score of \\ Reward Model A};
+      \node[box, fill=rmpink!30,anchor=north west] (benchb) at ([xshift=4.0cm]rmb.north-|policy.east) {Score of \\ Reward Model B};
       \node[anchor=south] at ([yshift=0.1cm]bencha.north) {\textbf{Evaluation Results}};
       \node[anchor=south] at ([yshift=0.1cm]bencha.north) {\textbf{Evaluation Results}};
 
 
-      \path[draw,dashed,rounded corners] ([xshift=1.75cm]rma.north east) rectangle ([xshift=-1.75cm]benchb.south west);
+      \path[draw,dashed] ([xshift=1.75cm]rma.north east) rectangle ([xshift=-3.75cm]benchb.south west);
 
 
-      \draw[arrow] (rma.east) -- node[below,align=center]{reward signal} +(1.75cm,0);
+      \draw[arrow] (rma.east) -- node[below,align=center]{} +(1.75cm,0);
       \draw[arrow] (rmb.east) -- +(1.75cm,0);
       \draw[arrow] (rmb.east) -- +(1.75cm,0);
       \draw[arrow] (rl)--(policy) node[arrowlabel]{RL training};
       \draw[arrow] (rl)--(policy) node[arrowlabel]{RL training};
-      \draw[arrow] ([xshift=-1.75cm]bencha.west) -- node[below,align=center]{benchmark} +(1.75cm,0);
-      \draw[arrow] ([xshift=-1.75cm]benchb.west) -- +(1.75cm,0);
+      \draw[arrow] ([xshift=-3.75cm]bencha.west) -- node[below,yshift=-0.16cm,align=center]{Evaluating the Trained Policy \\ Using a Benchmark} +(3.75cm,0);
+      \draw[arrow] ([xshift=-3.75cm]benchb.west) -- +(3.75cm,0);
 
 
       \node[subtitle,anchor=north] at ([yshift=-0.1cm]$1/2*(rmb.south west)+1/2*(benchb.south east)$) {(a) RL-based Evaluation};
       \node[subtitle,anchor=north] at ([yshift=-0.1cm]$1/2*(rmb.south west)+1/2*(benchb.south east)$) {(a) RL-based Evaluation};
 
 
     \end{scope}
     \end{scope}
 
 
     \begin{scope}[shift={({$(pa)-(8,0)$}|-(0,-5.5))},local bounding box=pb]
     \begin{scope}[shift={({$(pa)-(8,0)$}|-(0,-5.5))},local bounding box=pb]
-      \node[smallbox] (pair-input) at (0,0) {$(x,y^+,y^-)$};
+      \node[smallbox] (pair-input) at (0,0) {$(\textbf{x},\textbf{y}^+,\textbf{y}^-)$};
 
 
       \node[box,fill=rmblue!60] (rma) at ([xshift=1.5cm,yshift=.75cm]pair-input.east) {Reward\\Model A};
       \node[box,fill=rmblue!60] (rma) at ([xshift=1.5cm,yshift=.75cm]pair-input.east) {Reward\\Model A};
       \node[box,fill=rmpink!60] (rmb) at ([xshift=1.5cm,yshift=-.75cm]pair-input.east) {Reward\\Model B};
       \node[box,fill=rmpink!60] (rmb) at ([xshift=1.5cm,yshift=-.75cm]pair-input.east) {Reward\\Model B};
 
 
-      \node[smallbox] (a-result) at ([xshift=1.5cm]rma.east) {$y^+>y^-$};
-      \node[smallbox] (b-result) at ([xshift=1.5cm]rmb.east) {$y^+<y^-$};
+      \node[smallbox] (a-result) at ([xshift=1.5cm]rma.east) {$\textbf{y}^+>\textbf{y}^-$};
+      \node[smallbox] (b-result) at ([xshift=1.5cm]rmb.east) {$\textbf{y}^+<\textbf{y}^-$};
       \node[anchor=south,align=center] at ([yshift=0.1cm]a-result.north) {\textbf{Evaluation}\\\textbf{Results}};
       \node[anchor=south,align=center] at ([yshift=0.1cm]a-result.north) {\textbf{Evaluation}\\\textbf{Results}};
 
 
-      \node[text width=5cm,anchor=south west] (pair-eval) at ([yshift=0.1cm]pair-input.west|-rma.north)
-      {Evaluation Result Human prefers $y^+$ Model A correct};
+      \node[text width=5.5cm,anchor=south west] (pair-eval) at ([yshift=0.1cm]pair-input.west|-rma.north)
+      {Human prefers $\mathbf{y}^{+}$, so Reward Model A better aligns with human preferences.};
 
 
       \draw[arrow] (pair-input)--(rma);
       \draw[arrow] (pair-input)--(rma);
       \draw[arrow] (pair-input)--(rmb);
       \draw[arrow] (pair-input)--(rmb);
@@ -68,18 +68,18 @@
     \begin{scope}[shift={({$2*(pa)-(pb.east)$}|-(0,-5.5))}]
     \begin{scope}[shift={({$2*(pa)-(pb.east)$}|-(0,-5.5))}]
 
 
 
 
-      \node[smallbox] (candidate) at (0,0) {$(x, y_1, y_2, y_3, \cdots)$};
+      \node[smallbox] (candidate) at (0,0) {$(\textbf{x}, \textbf{y}_1, \textbf{y}_2, \textbf{y}_3, \cdots)$};
 
 
       \node[box,fill=rmblue!60] (lrma) at ([xshift=1cm,yshift=.75cm]candidate.east) {Reward\\Model A};
       \node[box,fill=rmblue!60] (lrma) at ([xshift=1cm,yshift=.75cm]candidate.east) {Reward\\Model A};
       \node[box,fill=rmpink!60] (lrmb) at ([xshift=1cm,yshift=-.75cm]candidate.east) {Reward\\Model B};
       \node[box,fill=rmpink!60] (lrmb) at ([xshift=1cm,yshift=-.75cm]candidate.east) {Reward\\Model B};
 
 
 
 
-      \node[smallbox] (best-a) at ([xshift=1cm]lrma.east) {Best is $y_3$};
-      \node[smallbox] (best-b) at ([xshift=1cm]lrmb.east) {Best is $y_4$};
+      \node[smallbox] (best-a) at ([xshift=1cm]lrma.east) {Best is $\textbf{y}_3$};
+      \node[smallbox] (best-b) at ([xshift=1cm]lrmb.east) {Best is $\textbf{y}_4$};
       \node[anchor=south,align=center] at ([yshift=0.1cm]best-a.north) {\textbf{Evaluation}\\\textbf{Results}};
       \node[anchor=south,align=center] at ([yshift=0.1cm]best-a.north) {\textbf{Evaluation}\\\textbf{Results}};
 
 
-      \node[text width=5cm,anchor=south west] (list-eval) at ([yshift=0.1cm]candidate.west|-lrma.north)
-      {Evaluation Result Human selects $y_3$ RM-A better};
+      \node[text width=5.5cm,anchor=south west] (list-eval) at ([yshift=0.1cm]candidate.west|-lrma.north)
+      {Human preference selects $\textbf{y}_3$ as the best output, so Reward Model A better aligns with human preferences.};
 
 
       \draw[arrow] (candidate)--(lrma);
       \draw[arrow] (candidate)--(lrma);
       \draw[arrow] (candidate)--(lrmb);
       \draw[arrow] (candidate)--(lrmb);