Fix compiling errors

section3/Figures/figure-pairwise-reward-loss-expectation.tex

 % !Mode:: "TeX:UTF-8"
 % !Mode:: "TeX:UTF-8"
 % !TEX encoding = UTF-8 Unicode
 % !TEX encoding = UTF-8 Unicode
 
 
+\definecolor{color_255_204_204}{RGB}{255,204,204}
+\definecolor{color_153_204_255}{RGB}{153,204,255}
+
 \begin{center}
 \begin{center}
   \begin{tikzpicture}
   \begin{tikzpicture}
 
 
@@ -11,7 +14,7 @@
     \begin{scope}
     \begin{scope}
 
 
       \node [draw,thick,minimum width=5.2cm,minimum height=1.3cm,fill=white] (llm) at (0,0) {Reward Model (LLM)};
       \node [draw,thick,minimum width=5.2cm,minimum height=1.3cm,fill=white] (llm) at (0,0) {Reward Model (LLM)};
-      \node [anchor=south east] (oy last) at ([yshift=.5cm,xshift=0cm]llm.north east) {\ctext[RGB]{153,204,255}{$R_{\phi}(\mathbf{x},\mathbf{y}_a)$}};
+      \node [anchor=south east] (oy last) at ([yshift=.5cm,xshift=0cm]llm.north east) {\textcolor{color_153_204_255}{$R_{\phi}(\mathbf{x},\mathbf{y}_a)$}};
       \draw [->] ([yshift=.1cm]llm.north-|oy last.south) -- (oy last.south);
       \draw [->] ([yshift=.1cm]llm.north-|oy last.south) -- (oy last.south);
 
 
       \node [draw,rounded corners=2pt,inner sep=2pt,text width=5.5em, anchor=north west,font=\linespread{0.8}\selectfont, minimum height=9ex] (x) at ([yshift=-1cm,xshift=-1cm]llm.south west) {\scriptsize{Give me three tips to improve my accuracy in solving math problems.\\}};
       \node [draw,rounded corners=2pt,inner sep=2pt,text width=5.5em, anchor=north west,font=\linespread{0.8}\selectfont, minimum height=9ex] (x) at ([yshift=-1cm,xshift=-1cm]llm.south west) {\scriptsize{Give me three tips to improve my accuracy in solving math problems.\\}};
@@ -25,7 +28,7 @@
 
 
       %% 
       %% 
       \node [draw,thick,minimum width=5.2cm,minimum height=1.3cm,fill=white] (llm) at (+8cm,0) {Reward Model (LLM)};
       \node [draw,thick,minimum width=5.2cm,minimum height=1.3cm,fill=white] (llm) at (+8cm,0) {Reward Model (LLM)};
-      \node [anchor=south east] (oy last1) at ([yshift=.5cm,xshift=0cm]llm.north east) {\ctext[RGB]{255,204,204}{$R_{\phi}(\mathbf{x},\mathbf{y}_b)$}};
+      \node [anchor=south east] (oy last1) at ([yshift=.5cm,xshift=0cm]llm.north east) {\textcolor{color_255_204_204}{$R_{\phi}(\mathbf{x},\mathbf{y}_b)$}};
       \draw [->] ([yshift=.1cm]llm.north-|oy last1.south) -- (oy last1.south);
       \draw [->] ([yshift=.1cm]llm.north-|oy last1.south) -- (oy last1.south);
 
 
       \node [draw,rounded corners=2pt,inner sep=2pt,text width=5.5em, anchor=north west,font=\linespread{0.8}\selectfont, minimum height=9ex] (x) at ([yshift=-1cm,xshift=-1cm]llm.south west) {\scriptsize{Give me three tips to improve my accuracy in solving math problems.\\}};
       \node [draw,rounded corners=2pt,inner sep=2pt,text width=5.5em, anchor=north west,font=\linespread{0.8}\selectfont, minimum height=9ex] (x) at ([yshift=-1cm,xshift=-1cm]llm.south west) {\scriptsize{Give me three tips to improve my accuracy in solving math problems.\\}};
@@ -38,7 +41,7 @@
       \draw [<-] ([yshift=0cm,xshift=-.5cm]llm.south) .. controls +(0, -.95cm) and +(0,+.95cm) .. +(-2cm, -.95cm);
       \draw [<-] ([yshift=0cm,xshift=-.5cm]llm.south) .. controls +(0, -.95cm) and +(0,+.95cm) .. +(-2cm, -.95cm);
       \draw [<-] ([yshift=0cm,xshift=.5cm]llm.south) .. controls +(0, -.95cm) and +(0,+.95cm) .. +(+2cm, -.95cm);
       \draw [<-] ([yshift=0cm,xshift=.5cm]llm.south) .. controls +(0, -.95cm) and +(0,+.95cm) .. +(+2cm, -.95cm);
 
 
-      \node at (4.65cm,3.3cm) {$- \log \mathrm{Sigmoid}($\ctext[RGB]{153,204,255}{$R_{\phi}(\mathbf{x},\mathbf{y}_a)$}$-$\ctext[RGB]{255,204,204}{$R_{\phi}(\mathbf{x},\mathbf{y}_b)$}$)$};
+      \node at (4.65cm,3.3cm) {$- \log \mathrm{Sigmoid}($\textcolor{color_153_204_255}{$R_{\phi}(\mathbf{x},\mathbf{y}_a)$}$-$\textcolor{color_255_204_204}{$R_{\phi}(\mathbf{x},\mathbf{y}_b)$}$)$};
       \draw [->] (oy last.north) .. controls +(0, 1cm) and +(0, -1cm) .. ([xshift=3.1cm, yshift=1.25cm]oy last.north);
       \draw [->] (oy last.north) .. controls +(0, 1cm) and +(0, -1cm) .. ([xshift=3.1cm, yshift=1.25cm]oy last.north);
       \draw [->] (oy last1.north) .. controls +(0, 1cm) and +(0, -1cm) .. ([xshift=-3.1cm, yshift=1.25cm]oy last1.north);
       \draw [->] (oy last1.north) .. controls +(0, 1cm) and +(0, -1cm) .. ([xshift=-3.1cm, yshift=1.25cm]oy last1.north);
 
 

section3/Figures/figure-update-with-policy-gradient.tex

 % !Mode:: "TeX:UTF-8"
 % !Mode:: "TeX:UTF-8"
 % !TEX encoding = UTF-8 Unicode
 % !TEX encoding = UTF-8 Unicode
 
 
+\definecolor{color_255_204_204}{RGB}{255,204,204}
+\definecolor{color_153_204_255}{RGB}{153,204,255}
+
 \begin{center}
 \begin{center}
   \begin{tikzpicture}
   \begin{tikzpicture}
 
 
@@ -61,8 +64,8 @@
 
 
       \node [anchor=south,draw,thick,minimum width=5.2cm,minimum height=1.3cm,fill=white,drop shadow={fill=gray,shadow xshift=.6ex,shadow yshift=-.6ex}] (llm) at ([yshift=.6cm]y1.center) {Policy (LLM)};
       \node [anchor=south,draw,thick,minimum width=5.2cm,minimum height=1.3cm,fill=white,drop shadow={fill=gray,shadow xshift=.6ex,shadow yshift=-.6ex}] (llm) at ([yshift=.6cm]y1.center) {Policy (LLM)};
       \node [anchor=center] (freeze) at ([xshift=-.25cm,yshift=.25cm]llm.south east) {\textcolor{red!60}{ \faFire}};
       \node [anchor=center] (freeze) at ([xshift=-.25cm,yshift=.25cm]llm.south east) {\textcolor{red!60}{ \faFire}};
-      \node [anchor=east,inner sep=0] (prob) at ([xshift=1em]oy last.east-|llm.east) {\ctext[RGB]{255,204,204}{$\log\left(\textrm{Pr}_\theta (y_1|\textbf{x})\times\cdots\times\textrm{Pr}_\theta (y_T|\textbf{y}_{<T},\textbf{x})\right)$}};
-      \node [anchor=west,inner sep=0] (reward) at ([xshift=.2cm]prob.east) {\ctext[RGB]{153,204,255}{$R(\mathbf{y})$}};
+      \node [anchor=east,inner sep=0] (prob) at ([xshift=1em]oy last.east-|llm.east) {\textcolor{color_255_204_204}{$\log\left(\textrm{Pr}_\theta (y_1|\textbf{x})\times\cdots\times\textrm{Pr}_\theta (y_T|\textbf{y}_{<T},\textbf{x})\right)$}};
+      \node [anchor=west,inner sep=0] (reward) at ([xshift=.2cm]prob.east) {\textcolor{color_153_204_255}{$R(\mathbf{y})$}};
       \node [anchor=center] at ($(prob.east)!.5!(reward.west)$) {$\times$};
       \node [anchor=center] at ($(prob.east)!.5!(reward.west)$) {$\times$};
 
 
       \node (rside) at ([xshift=0.5cm,yshift=-1.2cm]oy last.north east|-llm.east) {};
       \node (rside) at ([xshift=0.5cm,yshift=-1.2cm]oy last.north east|-llm.east) {};
@@ -72,7 +75,7 @@
       \draw [<-] ([yshift=-.25cm,xshift=-1.35cm]oy1.center) .. controls ([yshift=-.525cm,xshift=-1.35cm]oy1.center) .. ([yshift=-.525cm,xshift=-1cm]oy1.center) -- ([yshift=-.525cm,xshift=-.35cm]oy1.center) .. controls ([yshift=-.525cm]oy1.center) .. ([yshift=-.8cm]oy1.center);
       \draw [<-] ([yshift=-.25cm,xshift=-1.35cm]oy1.center) .. controls ([yshift=-.525cm,xshift=-1.35cm]oy1.center) .. ([yshift=-.525cm,xshift=-1cm]oy1.center) -- ([yshift=-.525cm,xshift=-.35cm]oy1.center) .. controls ([yshift=-.525cm]oy1.center) .. ([yshift=-.8cm]oy1.center);
       \draw [<-] ([yshift=-.25cm, xshift=-.7cm]oy last.center) .. controls ([yshift=-.525cm,xshift=-.7cm]oy last.center) .. ([yshift=-.525cm,xshift=-.35cm]oy last.center) .. controls ([yshift=-.525cm]oy last.center) .. ([yshift=-.8cm]oy last.center);
       \draw [<-] ([yshift=-.25cm, xshift=-.7cm]oy last.center) .. controls ([yshift=-.525cm,xshift=-.7cm]oy last.center) .. ([yshift=-.525cm,xshift=-.35cm]oy last.center) .. controls ([yshift=-.525cm]oy last.center) .. ([yshift=-.8cm]oy last.center);
 
 
-      \draw [->,thick] (oy.north) -- +(0,.6cm) -- ([yshift=.6cm]oy.north-|reward) node [midway,above] {compute the reward: \ctext[RGB]{153,204,255}{$R(\mathbf{y})$}} -- ([yshift=.2cm]reward.north);
+      \draw [->,thick] (oy.north) -- +(0,.6cm) -- ([yshift=.6cm]oy.north-|reward) node [midway,above] {compute the reward: \textcolor{color_153_204_255}{$R(\mathbf{y})$}} -- ([yshift=.2cm]reward.north);
     \end{scope}
     \end{scope}
 
 
   \end{tikzpicture}
   \end{tikzpicture}