% !Mode:: "TeX:UTF-8"
% !TEX encoding = UTF-8 Unicode

\begin{center}
  \begin{tikzpicture}[remember picture]
    \tikzset {
      block/.style={draw,inner sep=0pt,fill=white, minimum width=4cm, minimum height=1.5cm, align=center},
      miniblock/.style={draw,inner sep=0pt,fill=white, minimum width=1.5cm, minimum height=.8cm, align=center, rounded corners=2pt, text width=1.4cm},
      linetext/.style={fill=#1, minimum height=1.5pt, minimum width=.8cm, inner sep=0},
    }
    \def\sep{1.5cm}
    \def\ssep{1cm}
    \def\vsep{.2cm}
    \begin{scope}
      \node [anchor=center, block] (b1) at (0,0) {Pre-training an LLM};
      \node [anchor=south, block] (b2) at ([yshift=\sep]b1.north) {\footnotesize{Cold Start} \\ \footnotesize{for Reasoning}};
      \node [anchor=south, block] (b3) at ([yshift=\sep]b2.north) {\footnotesize{Large-scale Reinforcement}\\ \footnotesize{Learning for Reasoning}};
      \node [anchor=south, block] (b4) at ([yshift=\sep]b3.north) {\footnotesize{Rejection Sampling} \\ \footnotesize{for Multi-task}};
      \node [anchor=south, block] (b5) at ([yshift=\sep]b4.north) {\footnotesize{Reinforcement Learning} \\ \footnotesize{for Multi-task}};
      % \node [anchor=south] (b6) at ([yshift=\sep]b5.north) {\LARGE{$\cdots$}};
      \draw [->] (b1.north) -- (b2.south);
      \draw [->] (b2.north) -- (b3.south);
      \draw [->] (b3.north) -- (b4.south);
      \draw [->] (b4.north) -- (b5.south);
      % \draw [->] (b5.north) -- (b6.south);

      \def\notesep{10cm}
      %% 
      \node [anchor=north west, text width=\notesep] (n1) at ([xshift=\ssep]b1.north east) {\scriptsize{Starting with a pre-trained model, the iterative process incorporates a series of strategically designed phases to enhance the reasoning capabilities of the LLM.\\}};
      %% 
      \def\hsep{2.4cm}
      \node [anchor=north west, text width=\notesep] (n2) at ([xshift=\ssep]b2.north east) {\scriptsize{This phase involves collecting high-quality data from various reasoning tasks, which is then used to fine-tune the pre-trained LLM, providing a cold start.\\}};
      \node [anchor=north] at (n2.south) {
        \tikz{
          \node [anchor=north west] (m1-1) at (0,0) {\scriptsize{$\textbf{x}$}};
          \node [anchor=north west] (m1-2) at ([yshift=-\vsep]m1-1.south west) {\scriptsize{$\textbf{y}$}};
          \node [anchor=west, linetext=lolred] (m1-3) at (m1-1.east) {};
          \node [anchor=west, linetext=lolblue] (m1-4) at (m1-2.east) {};
          \node [anchor=west, miniblock] (m1-5) at ([xshift=\hsep]$(m1-3.east)!.5!(m1-4.east)$) {\scriptsize{Pre-trained \\ LLM \\}};
          \draw [->] ([xshift=-\hsep]m1-5.west) -- (m1-5.west);
          \node [anchor=west, miniblock] (m1-6) at ([xshift=\hsep]m1-5.east) {\scriptsize{SFT\\LLM\\}};
          \draw [->] ([xshift=-\hsep]m1-6.west) -- node [midway, above] {\scriptsize{SFT}} (m1-6.west);
        }
      };
      %% 
      \def\hsep{.95cm}
      \node [anchor=north west, text width=\notesep] (n3) at ([xshift=\ssep]b3.north east) {\scriptsize{Following the cold start, the model undergoes large-scale RL, leveraging rule-based rewards to enhance reasoning capabilities.\\}};
      \node [anchor=north] at (n3.south){
        \tikz{
          \node [anchor=north west] (m2-1) at (0,0) {\scriptsize{$\textbf{x}$}};
          \node [anchor=west, linetext=lolred] (m2-2) at (m2-1.east) {};
          \node [anchor=west,miniblock] (m2-3) at ([xshift=\hsep]m2-2.east) {\scriptsize{SFT\\LLM\\}};
          \matrix [matrix anchor=west,nodes={inner sep=0,anchor=center},column sep={.1cm,between borders},row sep={.2cm,between borders}] (m2-4) at ([xshift=\hsep]m2-3.east) {
            \node{\scriptsize{$\mathbf{y}_1$}}; & \node [linetext=lolblue] {}; & \node (t1){\scriptsize{$6$}}; \\
            \node{\scriptsize{$\cdots$}}; & \node{\scriptsize{$\cdots$}}; & \node{\scriptsize{$\cdots$}}; \\
            \node{\scriptsize{$\mathbf{y}_G$}}; & \node [linetext=lolblue] {}; & \node{\scriptsize{$12$}}; \\};
          \node [anchor=south] at ([yshift=.1cm]t1.north) {\scriptsize{Rewards}};
          \node [anchor=west,miniblock] (m2-5) at ([xshift=\hsep]m2-4.east) {\scriptsize{Reinforced \\ LLM\\}};
          \draw [->] ([xshift=.1cm]m2-2.east) -- (m2-3.west);
          \draw [->] (m2-3.east) -- (m2-4.west);
          \draw [->] (m2-4.east) -- node [midway, above] {\scriptsize{GRPO}} (m2-5.west);
        }        
      };
      %% 
      \def\hsep{1cm}
      \node [anchor=north west, text width=\notesep] (n4) at ([xshift=\ssep]b4.north east) {\scriptsize{This phase performs multi-task rejection sampling to further enhance the readability of outputs. \\}};
      \node[anchor=north] at (n4.south) {
        \tikz{
          \node [anchor=north west] (m3-1) at (0,0) {\scriptsize{$\textbf{x}$}};
          \node [anchor=west, linetext=lolred] (m3-2) at (m3-1.east) {};
          \node [anchor=west,miniblock] (m3-3) at ([xshift=\hsep]m3-2.east) {\scriptsize{Reinforced\\LLM\\}};
          \matrix [matrix anchor=west,nodes={inner sep=0,anchor=center},column sep={.1cm,between borders},row sep={.2cm,between borders}] (m3-4) at ([xshift=\hsep]m3-3.east) {
            \node{\scriptsize{$\mathbf{y}_1$}}; & \node [linetext=lolblue] {}; & \\
            \node{\scriptsize{$\cdots$}}; & \node{\scriptsize{$\cdots$}}; &  \\
            \node{\scriptsize{$\mathbf{y}_N$}}; & \node [linetext=lolblue] {}; & \node {\textcolor{ugreen}{\scriptsize{\faCheck}}}; \\ };
          \node [anchor=west,miniblock] (m3-5) at ([xshift=\hsep]m3-4.east) {\scriptsize{SFT\\LLM\\}};
          \draw [->] ([xshift=.1cm]m3-2.east) -- (m3-3.west);
          \draw [->] (m3-3.east) -- (m3-4.west);
          \draw [->] (m3-4.east) -- node [midway, above] {\scriptsize{SFT}} (m3-5.west);
        }
      };
      %% 
      \def\hsep{.95cm}
      \node [anchor=north west, text width=\notesep] (n5) at ([xshift=\ssep]b5.north east) {\scriptsize{In the final phase, multi-task RL is implemented to ensure broader generalization and maintain high performance across tasks beyond reasoning.\\}};
      \node [anchor=north] at (n5.south){
        \tikz{
          \node [anchor=north west] (m2-1) at (0,0) {\scriptsize{$\textbf{x}$}};
          \node [anchor=west, linetext=lolred] (m2-2) at (m2-1.east) {};
          \node [anchor=west,miniblock] (m2-3) at ([xshift=\hsep]m2-2.east) {\scriptsize{SFT\\LLM\\}};
          \matrix [matrix anchor=west,nodes={inner sep=0,anchor=center},column sep={.1cm,between borders},row sep={.2cm,between borders}] (m2-4) at ([xshift=\hsep]m2-3.east) {
            \node{\scriptsize{$\mathbf{y}_1$}}; & \node [linetext=lolblue] {}; & \node (t1){\scriptsize{$0.6$}}; \\
            \node{\scriptsize{$\cdots$}}; & \node{\scriptsize{$\cdots$}}; & \node{\scriptsize{$\cdots$}}; \\
            \node{\scriptsize{$\mathbf{y}_G$}}; & \node [linetext=lolblue] {}; & \node{\scriptsize{$0.7$}}; \\};
          \node [anchor=south] at ([yshift=.1cm]t1.north) {\scriptsize{Rewards}};
          \node [anchor=west,miniblock] (m2-5) at ([xshift=\hsep]m2-4.east) {\scriptsize{Reinforced\\LLM\\}};
          \draw [->] ([xshift=.1cm]m2-2.east) -- (m2-3.west);
          \draw [->] (m2-3.east) -- (m2-4.west);
          \draw [->] (m2-4.east) -- node [midway, above] {\scriptsize{GRPO}} (m2-5.west);
        }        
      };
      %% 
    \end{scope}
  \end{tikzpicture}
\end{center}
