% !Mode:: "TeX:UTF-8"
% !TEX encoding = UTF-8 Unicode

\begin{center}
  \begin{tikzpicture}
    \tikzset{
      block/.style = {
        minimum width=3.5cm,minimum height=1.5cm,
        draw=#1,fill=white,line width=2pt,
        drop shadow={fill=gray,shadow xshift=.6ex,shadow yshift=-.6ex}
      },
      block/.default=black
    }
    \def\sep{1.3cm}
    \begin{scope}
      \node [block=lolblue] (llm) at (0,0) {Policy (LLM)};
      \node [anchor=south, block=lolred] (old llm) at ([yshift=1cm]llm.north) {Reference Policy (LLM)};

      \draw [->] ([yshift=.1cm]llm.north) -- node [midway,right] {copy} ([yshift=-.1cm]old llm.south);

      \node [align=left] (step) at ([yshift=-.6cm]llm.south) {\scriptsize{Step (1): resync the reference policy}};

      %% 
      \node [block=lolred,anchor=west] (old llm1) at ([xshift=\sep]$(old llm.east)!.5!(llm.east)$) {Reference Policy (LLM)};
      \node [anchor=north] (x) at ([yshift=-.5cm]old llm1.south) {\large{$\mathbf{x}$}};
      \node [anchor=south] (y) at ([yshift=.5cm]old llm1.north) {\large{$\mathbf{y}$}};
      \draw [->] (x.north) -- ([yshift=-.1cm]old llm1.south);
      \draw [<-] (y.south) -- ([yshift=.1cm]old llm1.north);
      
      \node [anchor=north, align=left] (step) at (old llm1.south |- step.north) {\scriptsize{Step (2): sample an output from}\\ \scriptsize{the reference policy}};

      %%
      \node [block=lolblue,anchor=west] (llm2) at ([xshift=\sep]old llm1.east) {Policy (LLM)};
      \node [anchor=north] (x) at ([xshift=-1.5cm, yshift=-.5cm]llm2.south) {\large{$\mathbf{x}$}};
      \node [anchor=north] (y) at ([xshift=-1.1cm, yshift=-.5cm]llm2.south) {\large{$\mathbf{y}$}};
      \node [align=center] at ([yshift=-.3cm]$(x.south)!.5!(y.south)$) {\scriptsize{1st time}\\[-2pt]\scriptsize{update}};
      \draw [->] (x.north) -- ([yshift=-.1cm]x.north|-llm2.south);
      \draw [->] (y.north) -- ([yshift=-.1cm]y.north|-llm2.south);
      \node [anchor=north] (x) at ([xshift=-.2cm, yshift=-.5cm]llm2.south) {\large\textcolor{black!40!white}{$\mathbf{x}$}};
      \node [anchor=north] (y) at ([xshift=.2cm, yshift=-.5cm]llm2.south) {\large\textcolor{black!40!white}{$\mathbf{y}$}};
      \node [align=center] at ([yshift=-.3cm]$(x.south)!.5!(y.south)$) {\scriptsize{2nd time}\\[-2pt]\scriptsize{update}};
      \draw [->,black!40!white] (x.north) -- ([yshift=-.1cm]x.north|-llm2.south);
      \draw [->,black!40!white] (y.north) -- ([yshift=-.1cm]y.north|-llm2.south);
      \node [anchor=north] (cdots) at ([xshift=1.3cm, yshift=-.5cm]llm2.south) {\large$\cdots$};
      \node [align=center] at ([yshift=-.3cm]cdots.south) {\large$\cdots$};   
      
      \node [anchor=north,align=left] (step) at (llm2.south |- step.north) {\scriptsize{Step (3): update the policy multiple}\\ \scriptsize{times and go to Step (1)}};
      
    \end{scope}
  \end{tikzpicture}
\end{center}
