update.

section4/Figures/figure-rubric-based-reward-modeling.tex

+% Required packages:
+% \usepackage{tikz}
+% \usetikzlibrary{positioning,arrows.meta,calc}
+
+\begin{tikzpicture}[
+    >=Latex,
+    node distance=7mm and 11mm,
+    box/.style={
+        draw,
+        align=left,
+        inner sep=6pt,
+        font=\small
+    },
+    model/.style={
+        draw,
+        align=center,
+        minimum width=2.5cm,
+        minimum height=1.0cm,
+        inner sep=5pt,
+        font=\small
+    },
+    reward model/.style={model, fill=blue!12},
+    rubric generator/.style={model, fill=green!15},
+    arrow/.style={->, line width=0.7pt},
+    every node/.style={font=\small}
+]
+
+% ============================================================
+% (a) External LLM generates the rubric
+% ============================================================
+
+\begin{scope}[xshift=-8mm]
+
+\node[box, text width=5.2cm] (inputA) {
+    \makebox[\linewidth][c]{\textbf{Reward Model Input}}\\[2pt]
+    \textit{Input:} Give me three tips to improve my accuracy in solving math problems.\\[2pt]
+    \textit{Output A:} Improving math accuracy requires careful attention to detail and consistent practice. Here are three tips: practice regularly, understand the concepts, and \ldots\\[2pt]
+    \textit{Output B:} Practice more, read carefully, and check your answers. \ldots
+};
+
+\node[rubric generator, right=13mm of inputA.north east, anchor=north west] (llm) {
+    Rubric Generator\\
+    (an LLM)
+};
+
+\node[box, text width=4.3cm, below=8mm of llm] (rubricA) {
+    \makebox[\linewidth][c]{\textbf{Generated Rubric}}\\[2pt]
+    Check whether each output satisfies:\\
+    1. Provides exactly three tips\\
+    2. Gives math-relevant tips\\
+    3. Explains the suggested tips\\
+    4. Is clear and easy to read
+};
+
+\node[reward model, below=12mm of inputA] (grmA) {
+    Generative\\
+    Reward Model
+};
+
+\node[box, text width=5.2cm, below=12mm of grmA] (outputA) {
+    \makebox[\linewidth][c]{\textbf{Reward Model Output}}\\[2pt]
+    Based on the checklist:\\
+    Output A: Pass, Pass, Pass, Pass.\\
+    Output B: Pass, Pass, Fail, Pass. \ldots\\
+    Since Output A satisfies all checklist items while Output B does not explain the tips, Output A receives the higher reward.\\[2pt]
+    Preference: Output A
+};
+
+% Arrows
+\draw[arrow]
+    ($(inputA.east)+(0,10mm)$)
+    .. controls ($(inputA.east)+(9mm,10mm)$) and ($(llm.west)+(-12mm,0)$) ..
+    (llm.west);
+\draw[arrow] (llm.south) -- (rubricA.north);
+
+\draw[arrow] (inputA.south) -- (grmA.north);
+
+\draw[arrow]
+    (rubricA.south)
+    |-
+    ($(grmA.east)+(7mm,0)$)
+    -- (grmA.east);
+
+\draw[arrow] (grmA.south) -- (outputA.north);
+
+\coordinate (captionAcenter) at ($(inputA.west)!0.5!(rubricA.east)$);
+\node[font=\bfseries, anchor=north] at ($(captionAcenter |- outputA.south)+(0,-5mm)$) {
+    (a) Using an external LLM to generate the rubric
+};
+
+\end{scope}
+
+
+% ============================================================
+% (b) GRM self-generates the rubric
+% ============================================================
+
+\node[box, text width=5.2cm, right=60mm of inputA] (inputB) {
+    \makebox[\linewidth][c]{\textbf{Reward Model Input}}\\[2pt]
+    \textit{Input:} Give me three tips to improve my accuracy in solving math problems.\\[2pt]
+    \textit{Output A:} Improving math accuracy requires careful attention to detail and consistent practice. Here are three tips: practice regularly, understand the concepts, and \ldots\\[2pt]
+    \textit{Output B:} Practice more, read carefully, and check your answers. \ldots
+};
+
+\node[reward model, below=12mm of inputB] (grmB) {
+    Generative\\
+    Reward Model
+};
+
+\node[box, text width=5.6cm, below=8mm of grmB] (outputB) {
+    \makebox[\linewidth][c]{\textbf{Reward Model Output}}\\[2pt]
+    We should first use the following checklist:\\
+    1. Provides exactly three tips\\
+    2. Gives math-relevant tips\\
+    3. Explains the suggested tips\\
+    4. Is clear and easy to read\\[3pt]
+    Based on the checklist:\\
+    Output A: Pass, Pass, Pass, Pass.\\
+    Output B: Pass, Pass, Fail, Pass. \ldots\\[2pt]
+    Preference: Output A
+};
+
+% Arrows
+\draw[arrow] (inputB.south) -- (grmB.north);
+\draw[arrow] (grmB.south) -- (outputB.north);
+
+\node[font=\bfseries, below=5mm of outputB] {
+    (b) GRM self-generates the rubric
+};
+
+\end{tikzpicture}