% !Mode:: "TeX:UTF-8"
% !TEX encoding = UTF-8 Unicode

\begin{center}
    \begin{tikzpicture}
      \tikzstyle{model} = [minimum width=2cm, minimum height=1.2cm,align=center,inner sep=2pt,draw,thick,drop shadow={fill=gray,shadow xshift=.6ex,shadow yshift=-.6ex},fill=white]
      \tikzstyle{label} = [minimum width=2ex, minimum height=1.5ex,draw,scale=.75,fill=white]
      \tikzstyle{circled} = [shape=circle,draw,inner sep=2pt]
      \begin{scope}
        \node [anchor=west] (x) at (-0.2cm,0) {$\mathbf{x}$};
        \node [anchor=west,model] (policy model) at ([xshift=.8cm]x) {Policy\\Model};
        \node [anchor=north west,label] at ([xshift=-.1cm,yshift=.15cm]policy model.north west) {To Learn};
        \node [anchor=west] (y) at ([xshift=1.2cm]policy model.east) {$\mathbf{y}$};
        \node [anchor=west,model] (reward model) at ([xshift=1.4cm]y.east) {Reward\\Model};
        \node [anchor=north west,label] at ([xshift=-.1cm,yshift=.15cm]reward model.north west) {Fixed};
        \node [anchor=south,model] (reference model) at ([yshift=.4cm]reward model.north) {Reference\\Model};
        \node [anchor=north west,label] at ([xshift=-.1cm,yshift=.15cm]reference model.north west) {Fixed};
        \node [anchor=north,model] (value model) at ([yshift=-.4cm]reward model.south) {Value\\Model};
        \node [anchor=north west,label] at ([xshift=-.1cm,yshift=.15cm]value model.north west) {To Learn};

        \node [anchor=west] (reward) at ([xshift=0.1cm]reward model.east) {$R_\phi(\mathbf{x}, \mathbf{y})$};
        \node [anchor=west] (value) at ([xshift=0.1cm]value model.east) {$\{V_{\omega,1}, \cdots, V_{\omega,T}\}$};
        \node [anchor=west] (reference) at ([xshift=0.1cm]reference model.east) {Penalty};

        \node [anchor=west,align=center,rounded corners=5pt,inner sep=12pt,fill=lightgray] (advantage estimation) at ([xshift=1.5cm]reward) {Advantage\\Estimation};
        \node [anchor=west] (advantage estimation out) at (advantage estimation.east) {$\{A_{1}, \cdots, A_{T}\}$};
        

        \node (left bottom) at (x.west|-value model.south) {};
        \node (right bottom) at (advantage estimation out.east|-value model.south) {};
        \node [anchor=north,minimum width=13.6cm, minimum height=2cm,rounded corners=8pt,draw=lightgray,thick] (note) at ($(left bottom)!.5!(right bottom)+(0, -1cm)$) {};
        \matrix [anchor=center, every even column/.style={column sep=1cm}] at (note.center) {
          \node {\circled{1}}; & \node [anchor=west, inner sep=0pt] {sample an output $\mathbf{y}$ from $\mathrm{Pr}_{\theta_{\mathrm{ref}}}(\cdot|\mathbf{x})$} ; & \node {\circled{2}}; & \node [anchor=west, inner sep=0pt] {compute the $\mathrm{Penalty}$} ; \\
          \node {\circled{3}}; & \node [anchor=west, inner sep=0pt] {compute the reward for $\mathrm{seq}_{\mathbf{x}, \mathbf{y}}$ } ; & \node {\circled{4}}; & \node [anchor=west, inner sep=0pt] {predict the value for $\mathrm{seq}_{\mathbf{x}, \mathbf{y}}$} ; \\
          \node {\circled{5}}; & \node [anchor=west, inner sep=0pt] {optimize the value model $V_{\omega}(\cdot)$} ; & \node {\circled{6}}; & \node [anchor=west, inner sep=0pt] {optimize the policy model} ; \\
        };
        
        \draw [->,thick] (x.east) -- (policy model.west);
        \draw [->,thick] (policy model.east) node [xshift=.5cm,yshift=.0cm,above,align=center] {\circled{1}} -- (y.west);
        \draw [->,thick] (y.east)  .. controls (reward model.west) and ($(y.east |- reference model.west)$) .. (reference model.west);
        \draw [->,thick] (y.east) .. controls (reward model.west) and ($(y.east |- value model.west)$) .. (value model.west);
        \draw [->,thick] (y.east) -- (reward model.west);

        \draw [->,thick] (reference.east) node [above,xshift=.8cm] {\circled{2}} .. controls (reference.east -| advantage estimation.north) .. (advantage estimation.north);
        \draw [->,thick] (reward.east) -- node [above,xshift=0.0cm] {\circled{3}} (advantage estimation.west);
        \draw [->,thick] (value.east) .. controls (value.east -| advantage estimation.south) .. node [above,xshift=-.2cm] {\circled{4}}(advantage estimation.south);

        % update model
        \node (policy model bottom) at ($(right bottom -| policy model)+(0,-.6cm)$) {};
        \node [anchor=north] (policy model bottom1) at (policy model bottom |- advantage estimation out.north) {};
        \node (value model bottom) at ($(right bottom -| value model)+(0,-.6cm)$) {};
        \node (advantage estimation out bottom) at ($(advantage estimation out |- right bottom)+(0,-.6cm)$) {};
        \draw [->,thick,dashed] (advantage estimation out) -- ($(advantage estimation out)!.5!(advantage estimation out bottom)$) .. controls (advantage estimation out bottom) .. ($(advantage estimation out bottom.center)!.5!(policy model bottom.center)+(5em,0)$) -- ($(advantage estimation out bottom.center)!.5!(policy model bottom.center)-(5em,0)$)  .. controls (policy model bottom.center) .. ($(policy model bottom1.center)!.5!(policy model bottom)$) -- node [above,xshift=0.3cm,yshift=-.2cm,solid] {\circled{6}}(policy model.south);
        \draw [->,thick,dashed] ($(advantage estimation out bottom.center)!.5!(policy model bottom.center)+(2.89em,0)$) node [above,xshift=-1.5cm,solid] {\circled{5}}.. controls (value model bottom) .. (value model.south);
      \end{scope}
    \end{tikzpicture}    
\end{center}
