Skip to content

R
rl-introduction
Commit 53ac0b50 by wangchenglong
Options

Merge branch 'master' of 47.105.50.196:wangchenglong/rl-introduction

parents 829e3e6d da5f0a7b
正在显示 包含 244 行增加 和 191 行删除
section6/Figures/memory-and-retrieval.tex
% Required: % Required:
% \usetikzlibrary{positioning,calc,shadows} % \usetikzlibrary{positioning,calc,shadows}
\begin{tikzpicture}[ \begin{center}
box/.style={
\begin{tikzpicture}[
box/.style={
draw, draw,
fill=white, fill=white,
drop shadow={shadow xshift=0.08cm, shadow yshift=-0.08cm}, drop shadow={shadow xshift=0.08cm, shadow yshift=-0.08cm},
font=\small, align=center,
align=center rounded corners,
}, },
smallmsg/.style={ smallmsg/.style={
draw, draw,
fill=white, fill=white,
font=\small, font=\scriptsize,
align=left, align=left,
minimum height=0.9cm, text width=3cm,
text width=4.2cm rounded corners=4pt,
}, minimum height=0.5cm,
memitem/.style={ },
memitem/.style={
draw, draw,
fill=white, fill=white,
font=\small, font=\scriptsize,
align=left, align=left,
minimum height=0.7cm, minimum height=0.5cm,
text width=3.2cm text width=3.8cm,
}, rounded corners
opseg/.style={ },
opseg/.style={
draw, draw,
fill=white, fill=white,
font=\small\bfseries, font=\scriptsize\bfseries,
align=center, align=center,
minimum height=1.1cm, minimum height=0.6cm,
minimum width=2.5cm text width=1.3cm,
}, rounded corners,
arrow/.style={ },
arrow/.style={
->, ->,
thick thick
} }
] ]
% ======================= \node[box, minimum width=5.6cm, minimum height=5.6cm] (conv) at (0,0) {};
% Conversation container \node[font=\scriptsize\bfseries, rotate=90,anchor=north east] at (conv.north east) {Conversation};
% =======================
\node[box, minimum width=7.6cm, minimum height=6.5cm] (conv) at (0,0) {}; \path
\node[font=\small\bfseries, rotate=90] at ($(conv.east)+(0.35,0)$) {Conversation}; ([yshift=-0.3cm]conv.north west) node [anchor=north west,text width=0.4cm] (historyicon) {\faIcon{history}}
(historyicon.north east) node[smallmsg, anchor=north west,text width=] (histmsg) {\textbf{[History Messages]}};
\node[smallmsg, anchor=north] (usermsg) at ($(conv.north)+(0,-0.8)$) {
\textbf{User message:}\\ \path
Plan a trip to Kyoto next week. ([yshift=-0.3cm]histmsg.south-|conv.east) node [anchor=north east,text width=0.4cm,align=right] (usericon) {\faIcon{user}}
}; (usericon.north west) node[smallmsg, anchor=north east] (usermsg) {Plan a trip to Kyoto next week.};
\node[smallmsg, anchor=north] (assistantmsg) at ($(usermsg.south)+(0,-0.65)$) { \path
\textbf{Assistant:}\\ ([yshift=-0.3cm]usermsg.south-|conv.west) node [anchor=north west,text width=0.4cm] (roboticon) {\faIcon{robot}}
I will check your travel preferences and past choices. (roboticon.north east) node[smallmsg, anchor=north west] (robotmsg0) {I will check your travel preferences and past choices.};
};
\path
\node[smallmsg, anchor=north] (usedmem) at ($(assistantmsg.south)+(0,-0.95)$) { ([yshift=-0.3cm]robotmsg0.south-|conv.east) node [anchor=north east,text width=0.4cm,align=right] (infoicon) {\faIcon{info-circle}}
\textbf{Used memories:}\\ (infoicon.north west) node[smallmsg, anchor=north east] (infomsg)
$\bullet$ prefers window seat\\ {$\bullet$ prefers window seat\\ $\bullet$ vegetarian meals\\ $\bullet$ budget hotel};
$\bullet$ vegetarian meals\\
$\bullet$ budget hotel \path
}; ([yshift=-0.3cm]infomsg.south-|conv.west) node [anchor=north west,text width=0.4cm] (roboticon) {\faIcon{robot}}
(roboticon.north east) node[smallmsg, anchor=north west] (robotmsg) {...};
% =======================
% Memory bank \node[box, minimum width=5cm, minimum height=4.8cm] (memory) at ([xshift=8cm]conv) {};
% ======================= \node[font=\small\bfseries] at ($(memory.north)+(0,-0.35)$) {Memory Bank};
\node[box, minimum width=4.8cm, minimum height=4.1cm] (memory) at (8.2,-0.3) {};
\node[font=\small\bfseries] at ($(memory.north)+(0,-0.35)$) {Memory Bank}; \node[memitem, anchor=north,align=center] (m1) at ([yshift=-0.9cm]memory.north) {\textbf{Key 1}\\ prefers vegetarian meals};
\node[memitem, anchor=north,align=center] (m2) at ([yshift=-0.2cm]m1.south) {\textbf{Key 2}\\ prefers window seat};
\node[memitem, anchor=north] (m1) at ($(memory.north)+(0,-0.9)$) {Key 1: prefers vegetarian meals}; \node[memitem, anchor=north,align=center] (m3) at ([yshift=-0.2cm]m2.south) {\textbf{Key 3}\\ usually books budget hotels};
\node[memitem, anchor=north] (m2) at ($(m1.south)+(0,-0.15)$) {Key 2: prefers window seat}; \node[memitem, anchor=north,align=center] (m4) at ([yshift=-0.2cm]m3.south) {\textbf{Key ...}\\ ...};
\node[memitem, anchor=north] (m3) at ($(m2.south)+(0,-0.15)$) {Key 3: usually books budget hotels};
\draw[arrow] (conv.east|-robotmsg0.east) -- node[above, font=\scriptsize,align=center] {retrieve \\ relevant memories} (memory.west|-robotmsg0.east);
% ======================= \draw[arrow] (memory.west|-infomsg.east) -- node[above, font=\scriptsize,align=center] {useful memories} (conv.east|-infomsg.east);
% Operation box
% ======================= \scriptsize
\node[opseg] (add) at (1.4,-5.0) {ADD}; \node[box, text width=4cm, minimum height=2cm,anchor=north west,align=left] (memext) at ([yshift=-.5cm]conv.south west)
\node[opseg, right=0cm of add] (update) {UPDATE}; {\centering\textbf{\small Extract Memories} \\[.2cm] Integrate the newly extracted information with existing memories};
\node[opseg, right=0cm of update] (delete) {DELETE};
\node[opseg, right=0cm of delete] (noop) {NOOP}; \node[box, text width=7.5cm, minimum height=2cm,anchor=north east,align=center] (llmupdatemem) at ([yshift=-.5cm]conv.south-|memory.south east)
{\textbf{\small Memory Manager} \vspace{1.cm}};
% outer frame for the operation bar
\draw[thick] \node[opseg,anchor=south west] (add) at ([xshift=.35cm,yshift=.3cm]llmupdatemem.south west) {ADD};
($(add.north west)+(-0.08,0.08)$) rectangle \node[opseg, right=.3cm of add] (update) {UPDATE};
($(noop.south east)+(0.08,-0.08)$); \node[opseg, right=.3cm of update] (delete) {DELETE};
\node[opseg, right=.3cm of delete] (noop) {NOOP};
% =======================
% Arrows and labels \draw[] ([xshift=-.1cm,yshift=.1cm]add.north west) rectangle ([xshift=.1cm,yshift=-.1cm]noop.south east);
% =======================
\draw[arrow] (memext.center|-conv.south) -- node[right, font=\scriptsize] {new or updated memories} (memext.north);
% retrieval: conversation -> memory \draw[arrow] (memext) -- (llmupdatemem);
\draw[arrow] ($(conv.east)+(0,-0.25)$) -- node[above, font=\small] {retrieve relevant memories} ($(memory.west)+(0,0.45)$); \draw[arrow] (memory.south|-llmupdatemem.north) -- node[right,font=\scriptsize] {operate} (memory.south);
% use retrieved memory: memory -> conversation \end{tikzpicture}
\draw[arrow] ($(memory.west)+(0,-0.45)$) -- node[below, font=\small] {use memories} ($(conv.east)+(0,-1.1)$);
\end{center}
% new / updated memory from conversation -> operations
\draw[arrow]
($(conv.south)+(-2.0,0)$) -- ++(0,-0.75) -- ++(0,-0.7)
-- node[above, font=\small] {new or updated memories} ($(add.west)+(-0.65,0)$);
% operation -> memory
\draw[arrow]
($(update.north)+(0,0.08)$) -- ++(0,1.55)
-- node[right, font=\small] {update} ($(memory.south)+(0,-0.05)$);
\end{tikzpicture}
\ No newline at end of file
section6/Figures/reinfroced-memory-manager.tex
\begin{tikzpicture}[
box/.style={
\begin{center}
\begin{tikzpicture}[
box/.style={
draw, draw,
fill=white, fill=white,
drop shadow={shadow xshift=0.08cm, shadow yshift=-0.08cm}, drop shadow={shadow xshift=0.08cm, shadow yshift=-0.08cm},
font=\small, font=\scriptsize,
align=center, align=center,
minimum height=1.2cm minimum height=1.2cm
}, },
arrow/.style={ arrow/.style={
->, ->,
thick thick
}, }
reward/.style={ ]
<->,
thick,
dashed
}
]
% Extracted memory
\node[box, text width=2.8cm] (extracted) {
\textbf{Extracted Memory}\\[0.1cm]
$x^{\mathrm{mem}}$
};
% Memory manager
\node[box, text width=3.2cm, right=1.5cm of extracted] (manager) {
\textbf{Memory Manager}\\[0.1cm]
$\pi_\theta$\\
Select memory operation
};
% Memory bank % Extracted memory
\node[box, text width=3.2cm, below=0.8cm of manager] (bank) { \node[box, text width=3cm,align=left] (extracted)
\textbf{Memory Bank}\\[0.1cm] {\begin{center} \textbf{Extracted Memory} \end{center} \\[0.1cm]
$\mathcal{M}_{old} $\bullet$ prefers window seat\\ $\bullet$ vegetarian meals\\ $\bullet$ budget hotel};
\rightarrow
\mathcal{M}_{new}$
};
% Agent % Memory manager
\node[box, text width=2.8cm, right=2.0cm of manager] (agent) { \node[box, text width=3cm, right=1.5cm of extracted] (manager) {
\textbf{Agent}\\[0.1cm] \textbf{Memory Manager}
Downstream task solving };
};
% Reward % Memory bank
\node[box, text width=2.8cm, below=1.4cm of agent] (rewardbox) { \node[box, text width=3cm, below=0.8cm of manager] (bank) {
\textbf{Task Feedback}\\[0.1cm] \textbf{Memory Bank}\\[0.1cm]
Reward $R$ $\mathcal{M}_{old}
}; \rightarrow
\mathcal{M}_{new}$
};
% Dashed container for memory manager % Agent
\node[ \node[box, text width=2.8cm, right=2.0cm of bank.east] (agent) {
draw, \textbf{Agent}\\[0.1cm]
dashed, Downstream task solving
rounded corners, };
inner sep=0.35cm,
fit=(manager)(bank),
label={[font=\small]above:Learnable Memory Management}
] (managerbox) {};
% Arrows % Reward
\draw[arrow] (extracted) -- node[above,font=\small] {input} (manager); \node[box, text width=2.8cm, right=2.0cm of manager.east] (rewardbox) {
\textbf{Task Feedback}\\[0.1cm]
Reward $R$
};
\draw[arrow] (manager) -- node[right,font=\small] {operation} (bank); % Dashed container for memory manager
\node[
\draw[arrow] (bank.east) -- ++(0.8,0) draw,
|- node[pos=0.25,above,font=\small] {retrieve} dashed,
(agent.west); rounded corners,
inner sep=0.35cm,
\draw[arrow] (agent.west) -- ++(-0.8,0) fit=(manager)(bank),
|- node[pos=0.25,below,font=\small] {updated memory} label={[font=\scriptsize]above:Learnable Memory Management}
(bank.east); ] (managerbox) {};
\draw[reward] (agent.south) -- node[right,font=\small] {feedback} (rewardbox.north);
\draw[reward] (rewardbox.west) -- ++(-1.5,0) % Arrows
|- node[pos=0.25,left,font=\small] {RL optimization} \draw[arrow] (extracted) -- node[xshift=-.2cm,above,font=\scriptsize] {input} (manager);
(manager.south); \draw[arrow] (manager) -- node[xshift=0.1cm,center,font=\scriptsize] {sampling operations} (bank);
\draw[arrow] ([yshift=0.2cm]bank.east) -- node[above,font=\scriptsize] {retrieve} ([yshift=0.2cm]agent.west);
\draw[arrow] ([yshift=-0.2cm]agent.west) -- node[below,font=\scriptsize,align=right] {updated\\ memory} ([yshift=-0.2cm]bank.east);
\draw[arrow] (agent.north) -- node[right,font=\scriptsize] {feedback} (rewardbox.south);
\draw[arrow] (rewardbox.west) -- node[xshift=0.2cm,font=\scriptsize,align=center] {RL \\optimization} (manager.east);
\end{tikzpicture} \end{tikzpicture}
\ No newline at end of file \end{center}
section6/Figures/skillrl_illustration.tex 0 → 100644
% !Mode:: "TeX:UTF-8"
% !TEX encoding = UTF-8 Unicode
\begin{center}
\begin{tikzpicture}[remember picture]
\tikzset {
block/.style={draw,inner sep=0pt,fill=white, minimum width=5cm, minimum height=1.5cm, align=center},
miniblock/.style={draw,inner sep=0pt,fill=white, minimum width=1.5cm, minimum height=.8cm, align=center, rounded corners=2pt, text width=1.4cm},
linetext/.style={fill=#1, minimum height=1.5pt, minimum width=.8cm, inner sep=0},
}
\def\sep{1cm}
\def\ssep{1cm}
\begin{scope}
\node [block, anchor=north] (b1) at (0,0) {\footnotesize{Experience Collecting}};
\node [block, anchor=north] (b2) at ([yshift=-\sep]b1.south) {\footnotesize{Skill Generation}};
\node [block, anchor=north] (b3) at ([yshift=-\sep]b2.south) {\footnotesize{Skill Bank}};
\node [block, anchor=north] (b4) at ([yshift=-\sep]b3.south) {\footnotesize{Skill-Augmented RL Training}};
\node [block, anchor=north] (b5) at ([yshift=-\sep]b4.south) {\footnotesize{Recursive Skill Evolution}};
% \node [anchor=south] (b6) at ([yshift=\sep]b5.north) {\LARGE{$\cdots$}};
\draw [->] (b1.south) -- (b2.north);
\draw [->] (b2.south) -- (b3.north);
\draw [->] (b3.south) -- (b4.north);
\draw [->] (b4.south) -- (b5.north);
% \draw [->] (b5.north) -- (b6.south);
\scriptsize
\node [anchor=north west, text width=7.6cm] (n1) at ([xshift=\ssep]b1.north east) {Collect interaction trajectories from the enrionment.\\};
\node [anchor=north west, text width=3.5cm,draw,rounded corners,minimum height=1.5cm] (n12) at ([yshift=-0.1cm]n1.south west) {} ;
\node [anchor=west, text width=3.5cm,draw,rounded corners,minimum height=1.5cm] (n13) at ([xshift=0.3cm]n12.east) {} ;
\draw [dashed] ([xshift=0.8cm,yshift=0.1cm]b2.north east) -- ([xshift=9cm,yshift=0.1cm]b2.north east);
\node [anchor=north west, text width=7.6cm] (n2) at ([xshift=\ssep]b2.north east) {Generate reusable skills from experiences.\\};
\node [anchor=north west, text width=] (n21) at (n2.south west) {Successful trajectories};
\node [anchor=west, text width=] (n22) at ([xshift=.2cm]n21.east) {$\rightarrow$~~Skill Principles};
\node [anchor=north west, text width=] (n23) at (n21.south west) {Failed trajectories};
\node [anchor=north west, text width=] (n24) at (n22.south west) {$\rightarrow$~~Failed Lessons};
\draw [dashed] ([xshift=0.8cm,yshift=0.1cm]b3.north east) -- ([xshift=9cm,yshift=0.1cm]b3.north east);
\node [anchor=north west, text width=7.6cm] (n3) at ([xshift=\ssep]b3.north east) {Construct a hierarchical skill library.};
\node [anchor=north west, text width=7.6cm,draw,rounded corners,minimum height=0.8cm,align=left] (n31) at (n3.south west) {};
\node [anchor=west,text width=3cm,align=center] (n311) at ([xshift=0.2cm]n31.west) {\textbf{\small{General Skills}}};
\node [anchor=west,text width=4cm,align=left] (n312) at ([xshift=0.2cm]n311.east) {bla bla bla bla bla bla bla bla bla bla bla bla bla bla };
\node [anchor=north west, text width=7.6cm,draw,rounded corners,minimum height=0.8cm,align=left] (n32) at ([yshift=-0.1cm]n31.south west) {};
\node [anchor=west,text width=3cm,align=center] (n321) at ([xshift=0.2cm]n32.west) {\textbf{\small{Task-specific Skils}}};
\node [anchor=west,text width=4cm,align=left] (n322) at ([xshift=0.2cm]n321.east) {bla bla bla bla bla bla bla bla bla bla bla bla bla bla };
\draw [decorate,decoration={brace,mirror,raise=5pt,amplitude=6pt}] (n31.north west) -- (n32.south west);
\draw [dashed] ([xshift=0.8cm,yshift=0.1cm]b4.north east) -- ([xshift=9cm,yshift=0.1cm]b4.north east);
\node [anchor=north west, text width=7.6cm,align=left] (n4) at ([xshift=\ssep]b4.north east) {Retrieve relevant skills then provide retrieved skills to the agent.
Optimize the skill-augmented policy with RL.};
\draw ([yshift=-0.1cm]n4.south west) node (n41) [anchor=north west, text width=1.2cm,draw,rounded corners,minimum height=0.8cm,align=center] {Task}
([xshift=0.8cm]n41.east) node (n42) [anchor=west, text width=1.2cm,draw,rounded corners,minimum height=0.8cm,align=center] {Retrieve \\ Skills}
([xshift=0.8cm]n42.east) node (n43) [anchor=west, text width=1.2cm,draw,rounded corners,minimum height=0.8cm,align=center] {Agent}
([xshift=0.8cm]n43.east) node (n44) [anchor=west, text width=1.2cm,draw,rounded corners,minimum height=0.8cm,align=center] {Reward};
\draw[->] (n41.east) -- (n42.west);
\draw[->] (n42.east) -- (n43.west);
\draw[->] (n43.east) -- (n44.west);
\draw [dashed] ([xshift=0.8cm,yshift=0.1cm]b5.north east) -- ([xshift=9cm,yshift=0.1cm]b5.north east);
\node [anchor=north west, text width=7.6cm,align=left] (n5) at ([xshift=\ssep]b5.north east) {Analyze failure trajectories during training.
Generate new skills or refine existing skills.};
\node [anchor=north west, text width=6cm,draw,dashed,rounded corners,minimum height=1cm,align=left] (n51) at (n5.south west) {};
\node [anchor=west,text width=2cm,align=center] (n52) at ([xshift=0.2cm]n51.west) {\textbf{\small{Outcome}}};
\path (n52.east) node(n53) [anchor=west, text width=1.3cm,draw,rounded corners,minimum height=0.6cm,align=center] {new skill}
([xshift=0.5cm]n53.east) node(n54) [anchor=west, text width=1.3cm,draw,rounded corners,minimum height=0.6cm,align=center] {old skill};
\draw ([xshift=0.1cm]n53.south east) -- ([xshift=-0.1cm]n54.north west);
\end{scope}
\end{tikzpicture}
\end{center}
section6/section6.tex
...@@ -340,8 +340,7 @@ Memory management is a direct way for agents to learn from agentic experience. D ...@@ -340,8 +340,7 @@ Memory management is a direct way for agents to learn from agentic experience. D
\begin{figure}[!t] \begin{figure}[!t]
\centering \centering
\resizebox{\linewidth}{!}{ \input{section6/Figures/memory-and-retrieval.tex}
\input{section6/Figures/memory-and-retrieval.tex}}
\caption{ \caption{
An overview of the memory system. An overview of the memory system.
} }
...@@ -354,8 +353,7 @@ In learning from experience, it is easy to observe that the performance of memor ...@@ -354,8 +353,7 @@ In learning from experience, it is easy to observe that the performance of memor
\begin{figure}[!t] \begin{figure}[!t]
\centering \centering
\resizebox{\linewidth}{!}{ \input{section6/Figures/reinfroced-memory-manager.tex}
\input{section6/Figures/reinfroced-memory-manager.tex}}
\caption{Overview of training a memory manager with RL.} \caption{Overview of training a memory manager with RL.}
\label{fig:reinforced-memory} \label{fig:reinforced-memory}
\end{figure} \end{figure}
...@@ -457,7 +455,8 @@ The above way of generating skills often suffer from quality issues. First, a ge ...@@ -457,7 +455,8 @@ The above way of generating skills often suffer from quality issues. First, a ge
% \resizebox{\linewidth}{!}{ % \resizebox{\linewidth}{!}{
% \input{section6/Figures/skillrl.tex} % \input{section6/Figures/skillrl.tex}
% } % }
\includegraphics[width=0.5\textwidth]{section6/Figures/skillrl_draft.png} % \includegraphics[width=0.5\textwidth]{section6/Figures/skillrl_draft.png}
\input{section6/Figures/skillrl_illustration.tex}
\caption{ \caption{
Illustration of SkillRL \citep{xia-etal:skillrl}. SkillRL maintains a pool of agentic trajectories and initially abstracts reusable skills from these trajectories. The extracted skills are then used to guide RL training. As the agent improves through RL training, it generates higher-quality trajectories, which are further used to refine the skill bank. Illustration of SkillRL \citep{xia-etal:skillrl}. SkillRL maintains a pool of agentic trajectories and initially abstracts reusable skills from these trajectories. The extracted skills are then used to guide RL training. As the agent improves through RL training, it generates higher-quality trajectories, which are further used to refine the skill bank.
} }
......
section7/Figures/rovrm.tex
...@@ -3,51 +3,58 @@ ...@@ -3,51 +3,58 @@
\definecolor{temp4}{RGB}{250,235,243} \definecolor{temp4}{RGB}{250,235,243}
\usetikzlibrary {shapes.arrows} \usetikzlibrary {shapes.arrows}
\begin{tikzpicture} [ \begin{center}
\begin{tikzpicture} [
box/.style={rounded corners=6pt, align=center, minimum height=1.2cm,anchor=center}, box/.style={rounded corners=6pt, align=center, minimum height=1.2cm,anchor=center},
phasebox/.style={sharp corners,draw,thick,align=center,minimum height=1.2cm,anchor=center,text width=5.8cm,fill=white,drop shadow={fill=gray,shadow xshift=.6ex,shadow yshift=-.6ex}}, phasebox/.style={sharp corners,draw,thick,align=center,minimum height=0.8cm,anchor=center,text width=4.2cm,fill=white,drop shadow={fill=gray,shadow xshift=.6ex,shadow yshift=-.6ex}},
box1/.style={box,sharp corners,align=left, minimum height=4cm,anchor=center,text width=6.5cm,fill=white}, box1/.style={box,sharp corners,align=left, minimum height=3.5cm,anchor=center,text width=4.6cm,fill=white},
box2/.style={box,rounded corners=10pt,align=left, minimum height=4.4cm,anchor=center,minimum width=7.1cm,fill=temp2} box2/.style={box,rounded corners=6pt,align=left, minimum height=3.8cm,anchor=center,minimum width=5cm,fill=temp2}
] ]
\path[use as bounding box] (-11.4cm,-6.4cm) rectangle (11.4cm,1.25cm); \begin{scope}
\node [phasebox] (phase-1) at (-7.5,0) {\textit{\textbf{Stage 1: Pre-training with \\ Textual Preference Data}}}; % \path[use as bounding box] (-11.4cm,-6.4cm) rectangle (11.4cm,1.25cm);
\node [phasebox] (phase-2) at ([xshift=7.5cm]phase-1.center) {\textit{\textbf{Stage 2: Fine-tuning with Image \\ Caption-based Preference Data}}}; \scriptsize
\node [phasebox] (phase-3) at ([xshift=7.5cm]phase-2.center) {\textit{\textbf{Stage 3: Fine-tuning with \\ Visual Preference Data}}}; \node [phasebox] (phase-1) at (0,0) {\textit{\textbf{Stage 1: Pre-training with \\ Textual Preference Data}}};
\node [phasebox] (phase-2) at ([xshift=5.5cm]phase-1.center) {\textit{\textbf{Stage 2: Fine-tuning with Image \\ Caption-based Preference Data}}};
\node [phasebox] (phase-3) at ([xshift=5.5cm]phase-2.center) {\textit{\textbf{Stage 3: Fine-tuning with \\ Visual Preference Data}}};
\node [box2] (box-1) at ([yshift=-4cm]phase-1.center) {}; \node [box2] (box-1) at ([yshift=-3.6cm]phase-1.center) {};
\node [box1] (box-2) at (box-1) \node [box1] (box-2) at (box-1)
{\textit{\underline{Instruction}}: Can you determine the missing number in the sequence: 2, 6, 14, 30, 62, \_\_? \\ ~ \\ {\textit{\underline{Instruction}}: Can you determine the missing number in the sequence: 2, 6, 14, 30, 62, \_\_? \\ ~ \\
\textit{\underline{Chosen Response}}: The missing number in the sequence is 126.\\ \textit{\underline{Rejected Response}}: The sequence is 6.}; \textit{\underline{Chosen Response}}: The missing number in the sequence is 126.\\ \textit{\underline{Rejected Response}}: The sequence is 6.};
\node [box2] (box-3) at ([yshift=-4cm]phase-2.center) {}; \node [box2] (box-3) at ([yshift=-3.6cm]phase-2.center) {};
\node [box1] (box-4) at (box-3) \node [box1] (box-4) at (box-3)
{\textit{\underline{Instruction}}: Describe the image in detail.\\ \textit{\underline{Image Caption}}: There are four mangosteens, {\textit{\underline{Instruction}}: Describe the image in detail.\\ \textit{\underline{Image Caption}}: There are four mangosteens,
one of which has been cut open.\\ \textit{\underline{Chosen Response}}: There are several \textcolor{temp3}{mangosteens} on a \textcolor{temp3}{wooden table}.\\ one of which has been cut open.\\ \textit{\underline{Chosen Response}}: There are several \textcolor{temp3}{mangosteens} on a \textcolor{temp3}{wooden table}.\\
\textit{\underline{Rejected Response}}: This is a dark fruit, possibly a variety of \textcolor{red}{pitaya}.\\}; \textit{\underline{Rejected Response}}: This is a dark fruit, possibly a variety of \textcolor{red}{pitaya}.\\};
\node [box2] (box-5) at ([yshift=-4cm]phase-3.center) {}; \node [box2] (box-5) at ([yshift=-3.6cm]phase-3.center) {};
\node [box1] (box-6) at (box-5) \node [box1] (box-6) at (box-5)
{\includegraphics{section7/Figures/mangosteens.jpg}\\\textit{\underline{Chosen Response}}: There are several \textcolor{temp3}{mangosteens} on a \textcolor{temp3}{wooden table}.\\ {\includegraphics{section7/Figures/mangosteens.jpg}\\\textit{\underline{Chosen Response}}: There are several \textcolor{temp3}{mangosteens} on a \textcolor{temp3}{wooden table}.\\
\textit{\underline{Rejected Response}}: This is a dark fruit, possibly a variety of \textcolor{red}{pitaya}.\\}; \textit{\underline{Rejected Response}}: This is a dark fruit, possibly a variety of \textcolor{red}{pitaya}.\\};
\node [text width=4.5cm,anchor=north,align=left] at ([xshift=1.2cm,yshift=-.5cm]box-6.north) \node [text width=2.5cm,anchor=north,align=left] at ([xshift=1.1cm,yshift=-.5cm]box-6.north)
{\textit{\underline{Instruction}}: Describe the image in detail.}; {\textit{\underline{Instruction}}: Describe the image in detail.};
\node[draw,fill=temp4,minimum height=1cm,minimum width=0.75cm,single arrow,anchor=south,shape border rotate=90,single arrow head extend=0.12cm,] (arrow) at ([yshift=0.1cm]box-1.north) {}; \node[draw,fill=temp4,minimum height=1cm,minimum width=0.75cm,single arrow,anchor=south,shape border rotate=90,single arrow head extend=0.12cm,] (arrow) at ([yshift=0.1cm]box-1.north) {};
\node[draw,fill=temp4,minimum height=1cm,minimum width=0.75cm,single arrow,anchor=south,shape border rotate=90,single arrow head extend=0.12cm,] (arrow) at ([yshift=0.1cm]box-3.north) {}; \node[draw,fill=temp4,minimum height=1cm,minimum width=0.75cm,single arrow,anchor=south,shape border rotate=90,single arrow head extend=0.12cm,] (arrow) at ([yshift=0.1cm]box-3.north) {};
\node[draw,fill=temp4,minimum height=1cm,minimum width=0.75cm,single arrow,anchor=south,shape border rotate=90,single arrow head extend=0.12cm,] (arrow) at ([yshift=0.1cm]box-5.north) {}; \node[draw,fill=temp4,minimum height=1cm,minimum width=0.75cm,single arrow,anchor=south,shape border rotate=90,single arrow head extend=0.12cm,] (arrow) at ([yshift=0.1cm]box-5.north) {};
\draw[line width=0.5mm,arrows = {-Stealth[inset=0pt, length=0.2cm, angle'=45]}] (phase-1.east) to ([xshift=0.3cm]phase-1.east) \draw[line width=0.35mm,arrows = {-Stealth[inset=0pt, length=0.2cm, angle'=45]}] (phase-1.east) to ([xshift=0.18cm]phase-1.east)
to [out=90,in=180] ($1/2*(phase-1.east)+1/2*(phase-2.west)+(-0.1cm,0.5cm)$) to [out=0,in=90] ([xshift=-0.5cm]phase-2.west) to (phase-2.west); to [out=90,in=180] ($1/2*(phase-1.east)+1/2*(phase-2.west)+(-0.1cm,0.35cm)$) to [out=0,in=90] ([xshift=-0.38cm]phase-2.west) to (phase-2.west);
\draw[line width=0.2mm] ($1/2*(phase-1.east)+1/2*(phase-2.west)+(-0.1cm,0.8cm)$) .. controls +(-0.5cm,-1.4cm) and +(+0.5cm,1.4cm) .. ($1/2*(phase-1.east)+1/2*(phase-2.west)+(-0.3cm,-0.4cm)$); \draw[line width=0.2mm] ($1/2*(phase-1.east)+1/2*(phase-2.west)+(-0.1cm,0.6cm)$) .. controls +(-0.5cm,-1.4cm) and +(+0.5cm,1.4cm) .. ($1/2*(phase-1.east)+1/2*(phase-2.west)+(-0.3cm,-0.4cm)$);
\draw[line width=0.2mm] ($1/2*(phase-1.east)+1/2*(phase-2.west)+(0.1cm,0.8cm)$) .. controls +(-0.5cm,-1.4cm) and +(+0.5cm,1.4cm) .. ($1/2*(phase-1.east)+1/2*(phase-2.west)+(-0.1cm,-0.4cm)$); \draw[line width=0.2mm] ($1/2*(phase-1.east)+1/2*(phase-2.west)+(0.1cm,0.6cm)$) .. controls +(-0.5cm,-1.4cm) and +(+0.5cm,1.4cm) .. ($1/2*(phase-1.east)+1/2*(phase-2.west)+(-0.1cm,-0.4cm)$);
\node at ([yshift=-0.9cm]$1/2*(phase-1.east)+1/2*(phase-2.west)$) {Bridging Task Gap}; \node at ([yshift=-0.7cm]$1/2*(phase-1.east)+1/2*(phase-2.west)$) {Bridging Task Gap};
\draw[line width=0.5mm,arrows = {-Stealth[inset=0pt, length=0.2cm, angle'=45]}] (phase-2.east) to ([xshift=0.3cm]phase-2.east) \draw[line width=0.35mm,arrows = {-Stealth[inset=0pt, length=0.2cm, angle'=45]}] (phase-2.east) to ([xshift=0.18cm]phase-2.east)
to [out=90,in=180] ($1/2*(phase-2.east)+1/2*(phase-3.west)+(-0.1cm,0.5cm)$) to [out=0,in=90] ([xshift=-0.5cm]phase-3.west) to (phase-3.west); to [out=90,in=180] ($1/2*(phase-2.east)+1/2*(phase-3.west)+(-0.1cm,0.35cm)$) to [out=0,in=90] ([xshift=-0.38cm]phase-3.west) to (phase-3.west);
\draw[line width=0.2mm] ($1/2*(phase-2.east)+1/2*(phase-3.west)+(-0.1cm,0.8cm)$) .. controls +(-0.5cm,-1.4cm) and +(+0.5cm,1.4cm) .. ($1/2*(phase-2.east)+1/2*(phase-3.west)+(-0.3cm,-0.4cm)$); \draw[line width=0.2mm] ($1/2*(phase-2.east)+1/2*(phase-3.west)+(-0.1cm,0.6cm)$) .. controls +(-0.5cm,-1.4cm) and +(+0.5cm,1.4cm) .. ($1/2*(phase-2.east)+1/2*(phase-3.west)+(-0.3cm,-0.4cm)$);
\draw[line width=0.2mm] ($1/2*(phase-2.east)+1/2*(phase-3.west)+(0.1cm,0.8cm)$) .. controls +(-0.5cm,-1.4cm) and +(+0.5cm,1.4cm) .. ($1/2*(phase-2.east)+1/2*(phase-3.west)+(-0.1cm,-0.4cm)$); \draw[line width=0.2mm] ($1/2*(phase-2.east)+1/2*(phase-3.west)+(0.1cm,0.6cm)$) .. controls +(-0.5cm,-1.4cm) and +(+0.5cm,1.4cm) .. ($1/2*(phase-2.east)+1/2*(phase-3.west)+(-0.1cm,-0.4cm)$);
\node at ([yshift=-0.9cm]$1/2*(phase-2.east)+1/2*(phase-3.west)$) {Bridging Modality Gap }; \node at ([yshift=-0.7cm]$1/2*(phase-2.east)+1/2*(phase-3.west)$) {Bridging Modality Gap };
\end{tikzpicture} \end{scope}
\ No newline at end of file
\end{tikzpicture}
\end{center}
section7/section7.tex
...@@ -15,7 +15,8 @@ Training LLMs and VLMs with RL exhibits only minimal differences, primarily rela ...@@ -15,7 +15,8 @@ Training LLMs and VLMs with RL exhibits only minimal differences, primarily rela
\begin{figure*}[!t] \begin{figure*}[!t]
\centering \centering
\includegraphics[width=0.98\textwidth]{section7/Figures/rovrm.pdf} %% \includegraphics[width=0.98\textwidth]{section7/Figures/rovrm.pdf}
\input{section7/Figures/rovrm.tex}
\caption{An overview of the multi-stage training approach for visual reward models.} \caption{An overview of the multi-stage training approach for visual reward models.}
\label{fig:preference-transfer} \label{fig:preference-transfer}
\end{figure*} \end{figure*}
......
Markdown 格式
0% 或
您添加了 0 人 到此讨论。请谨慎行事。
请先完成此评论的编辑！
请 注册 或者 后发表评论