\begin{center}
  \begin{tikzpicture}[
    box/.style={draw, rectangle, minimum height=0.85cm, minimum width=1.65cm, align=center, font=\scriptsize, anchor=west},
    smallbox/.style={minimum height=0.6cm, minimum width=1.2cm, align=center, font=\scriptsize, anchor=west,},
    resultbox/.style={box, fill=rmgreen, text width=2.9cm, minimum width=3.15cm, font=\scriptsize},
    arrow/.style={->,},
    arrowlabel/.style={midway, above, fill=white, inner sep=1pt, font=\scriptsize},
    subtitle/.style={font=\bfseries\small}
    ]

    \definecolor{rmblue}{RGB}{190,210,240}
    \definecolor{rmpink}{RGB}{245,190,200}
    \definecolor{rmyellow}{RGB}{250,235,180}
    \definecolor{rmgreen}{RGB}{190,230,190}
    \definecolor{rmpurple}{RGB}{220,200,240}

    \scriptsize

    \begin{scope}[yshift=0.35cm,local bounding box=pa]

      \node[text width=12.5cm] (patitle) at (0,0.25)
      {Use each reward model to provide reward signals for RL training, and evaluate the trained policy on a benchmark. A higher benchmark score indicates that the corresponding reward model is better aligned with human preferences.};
      \node[box, fill=rmblue!60,anchor=north west] (rma) at ([yshift=-0.35cm]patitle.south west) {Reward \\ Model A};
      \node[box, fill=rmpink!60,anchor=north] (rmb) at ([yshift=-0.2cm]rma.south) {Reward \\ Model B};

      \node[box] (rl) at ([xshift=2cm]$1/2*(rma.east)+1/2*(rmb.east)$) {Reward \\ Scores};
      \node[box] (policy) at ([xshift=3cm]rl) {Policy\\LLM};

      \node[box, fill=rmblue!20,anchor=north west] (bencha) at ([xshift=4.0cm]rma.north-|policy.east) {Score of \\ Reward Model A};
      \node[box, fill=rmpink!20,anchor=north west] (benchb) at ([xshift=4.0cm]rmb.north-|policy.east) {Score of \\ Reward Model B};
      \node[anchor=south] at ([yshift=0.1cm]bencha.north) {\textbf{Evaluation Results}};

      \path[draw,dashed] ([xshift=1.75cm]rma.north east) rectangle ([xshift=-3.75cm]benchb.south west);

      \draw[arrow] (rma.east) -- node[below,align=center]{} +(1.75cm,0);
      \draw[arrow] (rmb.east) -- +(1.75cm,0);
      \draw[arrow] (rl)--(policy) node[arrowlabel]{RL training};
      \draw[arrow] ([xshift=-3.75cm]bencha.west) -- node[below,yshift=-0.16cm,align=center]{Evaluating the Trained Policy \\ Using a Benchmark} +(3.75cm,0);
      \draw[arrow] ([xshift=-3.75cm]benchb.west) -- +(3.75cm,0);

      \node[subtitle,anchor=north] at ([yshift=-0.1cm]$1/2*(rmb.south west)+1/2*(benchb.south east)$) {(a) RL-based Evaluation};

    \end{scope}

    \begin{scope}[shift={({$(pa)-(8,0)$}|-(0,-5.5))},local bounding box=pb]
      \node[smallbox] (pair-input) at (0,0) {$(\textbf{x},\textbf{y}^+,\textbf{y}^-)$};

      \node[box,fill=rmblue!60] (rma) at ([xshift=1.5cm,yshift=.75cm]pair-input.east) {Reward\\Model A};
      \node[box,fill=rmpink!60] (rmb) at ([xshift=1.5cm,yshift=-.75cm]pair-input.east) {Reward\\Model B};

      \node[smallbox] (a-result) at ([xshift=1.5cm]rma.east) {$\textbf{y}^+>\textbf{y}^-$};
      \node[smallbox] (b-result) at ([xshift=1.5cm]rmb.east) {$\textbf{y}^+<\textbf{y}^-$};
      \node[anchor=south,align=center] at ([yshift=0.1cm]a-result.north) {\textbf{Evaluation}\\\textbf{Results}};

      \node[text width=5.5cm,anchor=south west] (pair-eval) at ([yshift=0.1cm]pair-input.west|-rma.north)
      {Human prefers $\mathbf{y}^{+}$, so Reward Model A better aligns with human preferences.};

      \draw[arrow] (pair-input)--(rma);
      \draw[arrow] (pair-input)--(rmb);
      \draw[arrow] (rma)--(a-result);
      \draw[arrow] (rmb)--(b-result);

      \node[subtitle,anchor=north,yshift=-0.25cm] (subb) at ({$1/2*(pair-input.west)+1/2*(b-result.east)$}|-rmb.south)
      {(b) Pairwise Ranking Evaluation};

    \end{scope}

    \begin{scope}[shift={({$2*(pa)-(pb.east)$}|-(0,-5.5))}]


      \node[smallbox] (candidate) at (0,0) {$(\textbf{x}, \textbf{y}_1, \textbf{y}_2, \textbf{y}_3, \cdots)$};

      \node[box,fill=rmblue!60] (lrma) at ([xshift=1cm,yshift=.75cm]candidate.east) {Reward\\Model A};
      \node[box,fill=rmpink!60] (lrmb) at ([xshift=1cm,yshift=-.75cm]candidate.east) {Reward\\Model B};


      \node[smallbox] (best-a) at ([xshift=1cm]lrma.east) {Best is $\textbf{y}_3$};
      \node[smallbox] (best-b) at ([xshift=1cm]lrmb.east) {Best is $\textbf{y}_4$};
      \node[anchor=south,align=center] at ([yshift=0.1cm]best-a.north) {\textbf{Evaluation}\\\textbf{Results}};

      \node[text width=5.5cm,anchor=south west] (list-eval) at ([yshift=0.1cm]candidate.west|-lrma.north)
      {Human preference selects $\textbf{y}_3$ as the best output, so Reward Model A better aligns with human preferences.};

      \draw[arrow] (candidate)--(lrma);
      \draw[arrow] (candidate)--(lrmb);

      \draw[arrow] (lrma)--(best-a);
      \draw[arrow] (lrmb)--(best-b);

      \node[subtitle,anchor=north] at ({$1/2*(candidate.west)+1/2*(best-b.east)$}|-subb.north)
      {(c) Listwise Ranking Evaluation};

    \end{scope}



  \end{tikzpicture}
\end{center}
