% !Mode:: "TeX:UTF-8"
% !TEX encoding = UTF-8 Unicode

\definecolor{color_255_204_204}{RGB}{255,204,204}
\definecolor{color_153_204_255}{RGB}{153,204,255}

\begin{center}
  \begin{tikzpicture}

    \def\ssep{.7cm}
    \def\nsize{0.3cm}

    \tikzstyle{lnode} = [minimum width=7cm,minimum height=1.1cm,inner sep=2pt,draw,thick,fill=white];
    \begin{scope}
      \node [anchor=west] (x0) at (0,0) {\footnotesize{$x_0$}};
      \node [anchor=center] (x cdots) at ([xshift=\ssep]x0.center) {};
      \node [anchor=center] (x m) at ([xshift=\ssep]x cdots.center) {};
      \node [anchor=center] (y1) at ([xshift=\ssep]x m.center) {\footnotesize{$x_{m}$}};
      \node (x real cdots) at ($(x cdots)!.5!(x m)$) {\footnotesize{$\cdots$}};
      \node [anchor=center] (y2) at ([xshift=\ssep]y1.center) {};
      \node [anchor=center] (y cdots) at ([xshift=\ssep]y2.center) {};
      \node [anchor=center] (y n) at ([xshift=\ssep]y cdots.center) {};

      \foreach \x in {x0,y1}
      \draw [->] ([yshift=0.2cm]\x.center) -- ([yshift=0.55cm]\x.center);

      \node [anchor=center] (oy1) at ([yshift=2.8cm]y1.center) {\footnotesize{$y_1$}};
      \node [anchor=center] (oy2) at ([yshift=2.8cm]y2.center) {};
      \node [anchor=center] (oy cdots) at ([yshift=2.8cm]y cdots.center) {};
      \node [anchor=center] (oy last) at ([yshift=2.8cm]y n.center) {\footnotesize{$y_T$}};
      \node (oy real cdots) at ($(oy cdots)!.5!(oy2)$) {\footnotesize{$\cdots$}};

      \draw [decorate, decoration={brace,amplitude=.2cm},thick] (oy1.north west) -- (oy last.north east);
      
      \node [anchor=center] (oy) at ([yshift=.7cm]$(oy2)!.5!(oy cdots)$) {\footnotesize{$\mathbf{y}$}};

      \foreach \x in {oy1, oy last}
      \draw [<-] ([yshift=-0.25cm]\x.center) -- ([yshift=-0.8cm]\x.center);

      \node [anchor=south,draw,thick,minimum width=5.2cm,minimum height=1.3cm,fill=white,drop shadow={fill=gray,shadow xshift=.6ex,shadow yshift=-.6ex}] (llm) at ([yshift=.6cm]y1.center) {Policy (LLM)};
      \node [anchor=center] (freeze) at ([xshift=-.25cm,yshift=.25cm]llm.south east) {\textcolor{cyan!60}{ \faSnowflake}};

      \node (rside) at ([xshift=0.5cm,yshift=-1.2cm]oy last.north east|-llm.east) {};
      \node (lside) at ([yshift=-1.2cm]llm.west|-y n.east) {};
      \node at ($(lside)!.5!(rside)$) {\footnotesize{(a)~sample an output $\mathbf{y}$}};

      \node [anchor=west] (x0) at (\textwidth/2,0) {\footnotesize{$x_0$}};
      \node [anchor=center] (x cdots) at ([xshift=\ssep]x0.center) {};
      \node [anchor=center] (x m) at ([xshift=\ssep]x cdots.center) {};
      \node [anchor=center] (y1) at ([xshift=\ssep]x m.center) {\footnotesize{$x_{m}$}};
      \node (x real cdots) at ($(x cdots)!.5!(x m)$) {\footnotesize{$\cdots$}};
      \node [anchor=center] (y2) at ([xshift=\ssep]y1.center) {};
      \node [anchor=center] (y cdots) at ([xshift=\ssep]y2.center) {};
      \node [anchor=center] (y n) at ([xshift=\ssep]y cdots.center) {$y_{T-1}$};
      \node (y real cdots) at ($(y2)!.5!(y cdots)$) {\footnotesize{$\cdots$}};

      \foreach \x in {x0,y1, y n}
      \draw [->] ([yshift=0.2cm]\x.center) -- ([yshift=0.55cm]\x.center);

      \node [anchor=center] (oy1) at ([yshift=2.8cm]y1.center) {};
      \node [anchor=center] (oy2) at ([yshift=2.8cm]y2.center) {};
      \node [anchor=center] (oy cdots) at ([yshift=2.8cm]y cdots.center) {};
      \node [anchor=center] (oy last) at ([yshift=2.8cm]y n.center) {};

      \node [anchor=south,draw,thick,minimum width=5.2cm,minimum height=1.3cm,fill=white,drop shadow={fill=gray,shadow xshift=.6ex,shadow yshift=-.6ex}] (llm) at ([yshift=.6cm]y1.center) {Policy (LLM)};
      \node [anchor=center] (freeze) at ([xshift=-.25cm,yshift=.25cm]llm.south east) {\textcolor{red!60}{ \faFire}};
      \node [anchor=east,inner sep=0] (prob) at ([xshift=1em]oy last.east-|llm.east) {\textcolor{color_255_204_204}{$\log\left(\textrm{Pr}_\theta (y_1|\textbf{x})\times\cdots\times\textrm{Pr}_\theta (y_T|\textbf{y}_{<T},\textbf{x})\right)$}};
      \node [anchor=west,inner sep=0] (reward) at ([xshift=.2cm]prob.east) {\textcolor{color_153_204_255}{$R(\mathbf{y})$}};
      \node [anchor=center] at ($(prob.east)!.5!(reward.west)$) {$\times$};

      \node (rside) at ([xshift=0.5cm,yshift=-1.2cm]oy last.north east|-llm.east) {};
      \node (lside) at ([yshift=-1.2cm]llm.west|-y n.east) {};
      \node at ($(lside)!.5!(rside)$) {\footnotesize{(b)~train the LLM via policy gradient}};

      \draw [<-] ([yshift=-.25cm,xshift=-1.35cm]oy1.center) .. controls ([yshift=-.525cm,xshift=-1.35cm]oy1.center) .. ([yshift=-.525cm,xshift=-1cm]oy1.center) -- ([yshift=-.525cm,xshift=-.35cm]oy1.center) .. controls ([yshift=-.525cm]oy1.center) .. ([yshift=-.8cm]oy1.center);
      \draw [<-] ([yshift=-.25cm, xshift=-.7cm]oy last.center) .. controls ([yshift=-.525cm,xshift=-.7cm]oy last.center) .. ([yshift=-.525cm,xshift=-.35cm]oy last.center) .. controls ([yshift=-.525cm]oy last.center) .. ([yshift=-.8cm]oy last.center);

      \draw [->,thick] (oy.north) -- +(0,.6cm) -- ([yshift=.6cm]oy.north-|reward) node [midway,above] {compute the reward: \textcolor{color_153_204_255}{$R(\mathbf{y})$}} -- ([yshift=.2cm]reward.north);
    \end{scope}

  \end{tikzpicture}
\end{center}
