% !Mode:: "TeX:UTF-8"
% !TEX encoding = UTF-8 Unicode

\definecolor{color_255_204_204}{RGB}{255,204,204}
\definecolor{color_153_204_255}{RGB}{153,204,255}

\begin{center}
  \begin{tikzpicture}

    \def\ssep{.7cm}
    \def\nsize{0.3cm}

    \tikzstyle{lnode} = [minimum width=7cm,minimum height=1.1cm,inner sep=2pt,draw,thick,fill=white];
    \begin{scope}

      \node [draw,thick,minimum width=5.2cm,minimum height=1.3cm,fill=white] (llm) at (0,0) {Reward Model (LLM)};
      \node [anchor=south east] (oy last) at ([yshift=.5cm,xshift=0cm]llm.north east) {\textcolor{color_153_204_255}{$R_{\phi}(\mathbf{x},\mathbf{y}_a)$}};
      \draw [->] ([yshift=.1cm]llm.north-|oy last.south) -- (oy last.south);

      \node [draw,rounded corners=2pt,inner sep=2pt,text width=5.5em, anchor=north west,font=\linespread{0.8}\selectfont, minimum height=9ex] (x) at ([yshift=-1cm,xshift=-1cm]llm.south west) {\scriptsize{Give me three tips to improve my accuracy in solving math problems.\\}};
      \node [draw,rounded corners=2pt,inner sep=2pt,text width=13.5em, anchor=north east,font=\linespread{0.8}\selectfont, minimum height=9ex] (y) at ([yshift=-1cm,xshift=1cm]llm.south east) {\scriptsize{Improving math accuracy requires careful attention to detail, ... \\
          1.Practice regularly: ... \\
          ... , anyone can significantly enhance their math problem-solving abilities.\\}};
      \node [anchor=north] at ([yshift=-.2cm]x.south) {$\mathbf{x}$};
      \node [anchor=north] at ([yshift=-.2cm]y.south) {$\mathbf{y}_a$};
      \draw [<-] ([yshift=0cm,xshift=-.5cm]llm.south) .. controls +(0, -.95cm) and +(0,+.95cm) .. +(-2cm, -.95cm);
      \draw [<-] ([yshift=0cm,xshift=.5cm]llm.south) .. controls +(0, -.95cm) and +(0,+.95cm) .. +(+2cm, -.95cm);

      %% 
      \node [draw,thick,minimum width=5.2cm,minimum height=1.3cm,fill=white] (llm) at (+8cm,0) {Reward Model (LLM)};
      \node [anchor=south east] (oy last1) at ([yshift=.5cm,xshift=0cm]llm.north east) {\textcolor{color_255_204_204}{$R_{\phi}(\mathbf{x},\mathbf{y}_b)$}};
      \draw [->] ([yshift=.1cm]llm.north-|oy last1.south) -- (oy last1.south);

      \node [draw,rounded corners=2pt,inner sep=2pt,text width=5.5em, anchor=north west,font=\linespread{0.8}\selectfont, minimum height=9ex] (x) at ([yshift=-1cm,xshift=-1cm]llm.south west) {\scriptsize{Give me three tips to improve my accuracy in solving math problems.\\}};
      \node [draw,rounded corners=2pt,inner sep=2pt,text width=13.5em, anchor=north east,font=\linespread{0.8}\selectfont, minimum height=9ex] (y) at ([yshift=-1cm,xshift=1cm]llm.south east) {\scriptsize{There are three tips for improving accuracy in solving math problems: \\
          1.Practice regularly. \\
          2.Understand the concepts.\\
          3.Double-check your work.\\}};
      \node [anchor=north] at ([yshift=-.2cm]x.south) {$\mathbf{x}$};
      \node [anchor=north] at ([yshift=-.2cm]y.south) {$\mathbf{y}_b$};
      \draw [<-] ([yshift=0cm,xshift=-.5cm]llm.south) .. controls +(0, -.95cm) and +(0,+.95cm) .. +(-2cm, -.95cm);
      \draw [<-] ([yshift=0cm,xshift=.5cm]llm.south) .. controls +(0, -.95cm) and +(0,+.95cm) .. +(+2cm, -.95cm);

      \node at (4.65cm,3.3cm) {$- \log \mathrm{Sigmoid}($\textcolor{color_153_204_255}{$R_{\phi}(\mathbf{x},\mathbf{y}_a)$}$-$\textcolor{color_255_204_204}{$R_{\phi}(\mathbf{x},\mathbf{y}_b)$}$)$};
      \draw [->] (oy last.north) .. controls +(0, 1cm) and +(0, -1cm) .. ([xshift=3.1cm, yshift=1.25cm]oy last.north);
      \draw [->] (oy last1.north) .. controls +(0, 1cm) and +(0, -1cm) .. ([xshift=-3.1cm, yshift=1.25cm]oy last1.north);

    \end{scope}

  \end{tikzpicture}
\end{center}
