Skip to content

R
rl-introduction
Commit 53ac0b50 by wangchenglong
Options

Merge branch 'master' of 47.105.50.196:wangchenglong/rl-introduction

parents 829e3e6d da5f0a7b
正在显示 包含 244 行增加 和 191 行删除
section6/Figures/memory-and-retrieval.tex
% Required:
% \usetikzlibrary{positioning,calc,shadows}
\begin{tikzpicture}[
box/.style={
\begin{center}
\begin{tikzpicture}[
box/.style={
draw,
fill=white,
drop shadow={shadow xshift=0.08cm, shadow yshift=-0.08cm},
font=\small,
align=center
},
smallmsg/.style={
align=center,
rounded corners,
},
smallmsg/.style={
draw,
fill=white,
font=\small,
font=\scriptsize,
align=left,
minimum height=0.9cm,
text width=4.2cm
},
memitem/.style={
text width=3cm,
rounded corners=4pt,
minimum height=0.5cm,
},
memitem/.style={
draw,
fill=white,
font=\small,
font=\scriptsize,
align=left,
minimum height=0.7cm,
text width=3.2cm
},
opseg/.style={
minimum height=0.5cm,
text width=3.8cm,
rounded corners
},
opseg/.style={
draw,
fill=white,
font=\small\bfseries,
font=\scriptsize\bfseries,
align=center,
minimum height=1.1cm,
minimum width=2.5cm
},
arrow/.style={
minimum height=0.6cm,
text width=1.3cm,
rounded corners,
},
arrow/.style={
->,
thick
}
]
% =======================
% Conversation container
% =======================
\node[box, minimum width=7.6cm, minimum height=6.5cm] (conv) at (0,0) {};
\node[font=\small\bfseries, rotate=90] at ($(conv.east)+(0.35,0)$) {Conversation};
\node[smallmsg, anchor=north] (usermsg) at ($(conv.north)+(0,-0.8)$) {
\textbf{User message:}\\
Plan a trip to Kyoto next week.
};
\node[smallmsg, anchor=north] (assistantmsg) at ($(usermsg.south)+(0,-0.65)$) {
\textbf{Assistant:}\\
I will check your travel preferences and past choices.
};
\node[smallmsg, anchor=north] (usedmem) at ($(assistantmsg.south)+(0,-0.95)$) {
\textbf{Used memories:}\\
$\bullet$ prefers window seat\\
$\bullet$ vegetarian meals\\
$\bullet$ budget hotel
};
% =======================
% Memory bank
% =======================
\node[box, minimum width=4.8cm, minimum height=4.1cm] (memory) at (8.2,-0.3) {};
\node[font=\small\bfseries] at ($(memory.north)+(0,-0.35)$) {Memory Bank};
\node[memitem, anchor=north] (m1) at ($(memory.north)+(0,-0.9)$) {Key 1: prefers vegetarian meals};
\node[memitem, anchor=north] (m2) at ($(m1.south)+(0,-0.15)$) {Key 2: prefers window seat};
\node[memitem, anchor=north] (m3) at ($(m2.south)+(0,-0.15)$) {Key 3: usually books budget hotels};
% =======================
% Operation box
% =======================
\node[opseg] (add) at (1.4,-5.0) {ADD};
\node[opseg, right=0cm of add] (update) {UPDATE};
\node[opseg, right=0cm of update] (delete) {DELETE};
\node[opseg, right=0cm of delete] (noop) {NOOP};
% outer frame for the operation bar
\draw[thick]
($(add.north west)+(-0.08,0.08)$) rectangle
($(noop.south east)+(0.08,-0.08)$);
% =======================
% Arrows and labels
% =======================
% retrieval: conversation -> memory
\draw[arrow] ($(conv.east)+(0,-0.25)$) -- node[above, font=\small] {retrieve relevant memories} ($(memory.west)+(0,0.45)$);
% use retrieved memory: memory -> conversation
\draw[arrow] ($(memory.west)+(0,-0.45)$) -- node[below, font=\small] {use memories} ($(conv.east)+(0,-1.1)$);
% new / updated memory from conversation -> operations
\draw[arrow]
($(conv.south)+(-2.0,0)$) -- ++(0,-0.75) -- ++(0,-0.7)
-- node[above, font=\small] {new or updated memories} ($(add.west)+(-0.65,0)$);
% operation -> memory
\draw[arrow]
($(update.north)+(0,0.08)$) -- ++(0,1.55)
-- node[right, font=\small] {update} ($(memory.south)+(0,-0.05)$);
\end{tikzpicture}
\ No newline at end of file
}
]
\node[box, minimum width=5.6cm, minimum height=5.6cm] (conv) at (0,0) {};
\node[font=\scriptsize\bfseries, rotate=90,anchor=north east] at (conv.north east) {Conversation};
\path
([yshift=-0.3cm]conv.north west) node [anchor=north west,text width=0.4cm] (historyicon) {\faIcon{history}}
(historyicon.north east) node[smallmsg, anchor=north west,text width=] (histmsg) {\textbf{[History Messages]}};
\path
([yshift=-0.3cm]histmsg.south-|conv.east) node [anchor=north east,text width=0.4cm,align=right] (usericon) {\faIcon{user}}
(usericon.north west) node[smallmsg, anchor=north east] (usermsg) {Plan a trip to Kyoto next week.};
\path
([yshift=-0.3cm]usermsg.south-|conv.west) node [anchor=north west,text width=0.4cm] (roboticon) {\faIcon{robot}}
(roboticon.north east) node[smallmsg, anchor=north west] (robotmsg0) {I will check your travel preferences and past choices.};
\path
([yshift=-0.3cm]robotmsg0.south-|conv.east) node [anchor=north east,text width=0.4cm,align=right] (infoicon) {\faIcon{info-circle}}
(infoicon.north west) node[smallmsg, anchor=north east] (infomsg)
{$\bullet$ prefers window seat\\ $\bullet$ vegetarian meals\\ $\bullet$ budget hotel};
\path
([yshift=-0.3cm]infomsg.south-|conv.west) node [anchor=north west,text width=0.4cm] (roboticon) {\faIcon{robot}}
(roboticon.north east) node[smallmsg, anchor=north west] (robotmsg) {...};
\node[box, minimum width=5cm, minimum height=4.8cm] (memory) at ([xshift=8cm]conv) {};
\node[font=\small\bfseries] at ($(memory.north)+(0,-0.35)$) {Memory Bank};
\node[memitem, anchor=north,align=center] (m1) at ([yshift=-0.9cm]memory.north) {\textbf{Key 1}\\ prefers vegetarian meals};
\node[memitem, anchor=north,align=center] (m2) at ([yshift=-0.2cm]m1.south) {\textbf{Key 2}\\ prefers window seat};
\node[memitem, anchor=north,align=center] (m3) at ([yshift=-0.2cm]m2.south) {\textbf{Key 3}\\ usually books budget hotels};
\node[memitem, anchor=north,align=center] (m4) at ([yshift=-0.2cm]m3.south) {\textbf{Key ...}\\ ...};
\draw[arrow] (conv.east|-robotmsg0.east) -- node[above, font=\scriptsize,align=center] {retrieve \\ relevant memories} (memory.west|-robotmsg0.east);
\draw[arrow] (memory.west|-infomsg.east) -- node[above, font=\scriptsize,align=center] {useful memories} (conv.east|-infomsg.east);
\scriptsize
\node[box, text width=4cm, minimum height=2cm,anchor=north west,align=left] (memext) at ([yshift=-.5cm]conv.south west)
{\centering\textbf{\small Extract Memories} \\[.2cm] Integrate the newly extracted information with existing memories};
\node[box, text width=7.5cm, minimum height=2cm,anchor=north east,align=center] (llmupdatemem) at ([yshift=-.5cm]conv.south-|memory.south east)
{\textbf{\small Memory Manager} \vspace{1.cm}};
\node[opseg,anchor=south west] (add) at ([xshift=.35cm,yshift=.3cm]llmupdatemem.south west) {ADD};
\node[opseg, right=.3cm of add] (update) {UPDATE};
\node[opseg, right=.3cm of update] (delete) {DELETE};
\node[opseg, right=.3cm of delete] (noop) {NOOP};
\draw[] ([xshift=-.1cm,yshift=.1cm]add.north west) rectangle ([xshift=.1cm,yshift=-.1cm]noop.south east);
\draw[arrow] (memext.center|-conv.south) -- node[right, font=\scriptsize] {new or updated memories} (memext.north);
\draw[arrow] (memext) -- (llmupdatemem);
\draw[arrow] (memory.south|-llmupdatemem.north) -- node[right,font=\scriptsize] {operate} (memory.south);
\end{tikzpicture}
\end{center}
section6/Figures/reinfroced-memory-manager.tex
\begin{tikzpicture}[
box/.style={
\begin{center}
\begin{tikzpicture}[
box/.style={
draw,
fill=white,
drop shadow={shadow xshift=0.08cm, shadow yshift=-0.08cm},
font=\small,
font=\scriptsize,
align=center,
minimum height=1.2cm
},
arrow/.style={
},
arrow/.style={
->,
thick
},
reward/.style={
<->,
thick,
dashed
}
]
% Extracted memory
\node[box, text width=2.8cm] (extracted) {
\textbf{Extracted Memory}\\[0.1cm]
$x^{\mathrm{mem}}$
};
% Memory manager
\node[box, text width=3.2cm, right=1.5cm of extracted] (manager) {
\textbf{Memory Manager}\\[0.1cm]
$\pi_\theta$\\
Select memory operation
};
}
]
% Memory bank
\node[box, text width=3.2cm, below=0.8cm of manager] (bank) {
\textbf{Memory Bank}\\[0.1cm]
$\mathcal{M}_{old}
\rightarrow
\mathcal{M}_{new}$
};
% Extracted memory
\node[box, text width=3cm,align=left] (extracted)
{\begin{center} \textbf{Extracted Memory} \end{center} \\[0.1cm]
$\bullet$ prefers window seat\\ $\bullet$ vegetarian meals\\ $\bullet$ budget hotel};
% Agent
\node[box, text width=2.8cm, right=2.0cm of manager] (agent) {
\textbf{Agent}\\[0.1cm]
Downstream task solving
};
% Memory manager
\node[box, text width=3cm, right=1.5cm of extracted] (manager) {
\textbf{Memory Manager}
};
% Reward
\node[box, text width=2.8cm, below=1.4cm of agent] (rewardbox) {
\textbf{Task Feedback}\\[0.1cm]
Reward $R$
};
% Memory bank
\node[box, text width=3cm, below=0.8cm of manager] (bank) {
\textbf{Memory Bank}\\[0.1cm]
$\mathcal{M}_{old}
\rightarrow
\mathcal{M}_{new}$
};
% Dashed container for memory manager
\node[
draw,
dashed,
rounded corners,
inner sep=0.35cm,
fit=(manager)(bank),
label={[font=\small]above:Learnable Memory Management}
] (managerbox) {};
% Agent
\node[box, text width=2.8cm, right=2.0cm of bank.east] (agent) {
\textbf{Agent}\\[0.1cm]
Downstream task solving
};
% Arrows
\draw[arrow] (extracted) -- node[above,font=\small] {input} (manager);
% Reward
\node[box, text width=2.8cm, right=2.0cm of manager.east] (rewardbox) {
\textbf{Task Feedback}\\[0.1cm]
Reward $R$
};
\draw[arrow] (manager) -- node[right,font=\small] {operation} (bank);
\draw[arrow] (bank.east) -- ++(0.8,0)
|- node[pos=0.25,above,font=\small] {retrieve}
(agent.west);
\draw[arrow] (agent.west) -- ++(-0.8,0)
|- node[pos=0.25,below,font=\small] {updated memory}
(bank.east);
\draw[reward] (agent.south) -- node[right,font=\small] {feedback} (rewardbox.north);
% Dashed container for memory manager
\node[
draw,
dashed,
rounded corners,
inner sep=0.35cm,
fit=(manager)(bank),
label={[font=\scriptsize]above:Learnable Memory Management}
] (managerbox) {};
\draw[reward] (rewardbox.west) -- ++(-1.5,0)
|- node[pos=0.25,left,font=\small] {RL optimization}
(manager.south);
% Arrows
\draw[arrow] (extracted) -- node[xshift=-.2cm,above,font=\scriptsize] {input} (manager);
\draw[arrow] (manager) -- node[xshift=0.1cm,center,font=\scriptsize] {sampling operations} (bank);
\draw[arrow] ([yshift=0.2cm]bank.east) -- node[above,font=\scriptsize] {retrieve} ([yshift=0.2cm]agent.west);
\draw[arrow] ([yshift=-0.2cm]agent.west) -- node[below,font=\scriptsize,align=right] {updated\\ memory} ([yshift=-0.2cm]bank.east);
\draw[arrow] (agent.north) -- node[right,font=\scriptsize] {feedback} (rewardbox.south);
\draw[arrow] (rewardbox.west) -- node[xshift=0.2cm,font=\scriptsize,align=center] {RL \\optimization} (manager.east);
\end{tikzpicture}
\ No newline at end of file
\end{tikzpicture}
\end{center}
section6/Figures/skillrl_illustration.tex 0 → 100644
% !Mode:: "TeX:UTF-8"
% !TEX encoding = UTF-8 Unicode
\begin{center}
\begin{tikzpicture}[remember picture]
\tikzset {
block/.style={draw,inner sep=0pt,fill=white, minimum width=5cm, minimum height=1.5cm, align=center},
miniblock/.style={draw,inner sep=0pt,fill=white, minimum width=1.5cm, minimum height=.8cm, align=center, rounded corners=2pt, text width=1.4cm},
linetext/.style={fill=#1, minimum height=1.5pt, minimum width=.8cm, inner sep=0},
}
\def\sep{1cm}
\def\ssep{1cm}
\begin{scope}
\node [block, anchor=north] (b1) at (0,0) {\footnotesize{Experience Collecting}};
\node [block, anchor=north] (b2) at ([yshift=-\sep]b1.south) {\footnotesize{Skill Generation}};
\node [block, anchor=north] (b3) at ([yshift=-\sep]b2.south) {\footnotesize{Skill Bank}};
\node [block, anchor=north] (b4) at ([yshift=-\sep]b3.south) {\footnotesize{Skill-Augmented RL Training}};
\node [block, anchor=north] (b5) at ([yshift=-\sep]b4.south) {\footnotesize{Recursive Skill Evolution}};
% \node [anchor=south] (b6) at ([yshift=\sep]b5.north) {\LARGE{$\cdots$}};
\draw [->] (b1.south) -- (b2.north);
\draw [->] (b2.south) -- (b3.north);
\draw [->] (b3.south) -- (b4.north);
\draw [->] (b4.south) -- (b5.north);
% \draw [->] (b5.north) -- (b6.south);
\scriptsize
\node [anchor=north west, text width=7.6cm] (n1) at ([xshift=\ssep]b1.north east) {Collect interaction trajectories from the enrionment.\\};
\node [anchor=north west, text width=3.5cm,draw,rounded corners,minimum height=1.5cm] (n12) at ([yshift=-0.1cm]n1.south west) {} ;
\node [anchor=west, text width=3.5cm,draw,rounded corners,minimum height=1.5cm] (n13) at ([xshift=0.3cm]n12.east) {} ;
\draw [dashed] ([xshift=0.8cm,yshift=0.1cm]b2.north east) -- ([xshift=9cm,yshift=0.1cm]b2.north east);
\node [anchor=north west, text width=7.6cm] (n2) at ([xshift=\ssep]b2.north east) {Generate reusable skills from experiences.\\};
\node [anchor=north west, text width=] (n21) at (n2.south west) {Successful trajectories};
\node [anchor=west, text width=] (n22) at ([xshift=.2cm]n21.east) {$\rightarrow$~~Skill Principles};
\node [anchor=north west, text width=] (n23) at (n21.south west) {Failed trajectories};
\node [anchor=north west, text width=] (n24) at (n22.south west) {$\rightarrow$~~Failed Lessons};
\draw [dashed] ([xshift=0.8cm,yshift=0.1cm]b3.north east) -- ([xshift=9cm,yshift=0.1cm]b3.north east);
\node [anchor=north west, text width=7.6cm] (n3) at ([xshift=\ssep]b3.north east) {Construct a hierarchical skill library.};
\node [anchor=north west, text width=7.6cm,draw,rounded corners,minimum height=0.8cm,align=left] (n31) at (n3.south west) {};
\node [anchor=west,text width=3cm,align=center] (n311) at ([xshift=0.2cm]n31.west) {\textbf{\small{General Skills}}};
\node [anchor=west,text width=4cm,align=left] (n312) at ([xshift=0.2cm]n311.east) {bla bla bla bla bla bla bla bla bla bla bla bla bla bla };
\node [anchor=north west, text width=7.6cm,draw,rounded corners,minimum height=0.8cm,align=left] (n32) at ([yshift=-0.1cm]n31.south west) {};
\node [anchor=west,text width=3cm,align=center] (n321) at ([xshift=0.2cm]n32.west) {\textbf{\small{Task-specific Skils}}};
\node [anchor=west,text width=4cm,align=left] (n322) at ([xshift=0.2cm]n321.east) {bla bla bla bla bla bla bla bla bla bla bla bla bla bla };
\draw [decorate,decoration={brace,mirror,raise=5pt,amplitude=6pt}] (n31.north west) -- (n32.south west);
\draw [dashed] ([xshift=0.8cm,yshift=0.1cm]b4.north east) -- ([xshift=9cm,yshift=0.1cm]b4.north east);
\node [anchor=north west, text width=7.6cm,align=left] (n4) at ([xshift=\ssep]b4.north east) {Retrieve relevant skills then provide retrieved skills to the agent.
Optimize the skill-augmented policy with RL.};
\draw ([yshift=-0.1cm]n4.south west) node (n41) [anchor=north west, text width=1.2cm,draw,rounded corners,minimum height=0.8cm,align=center] {Task}
([xshift=0.8cm]n41.east) node (n42) [anchor=west, text width=1.2cm,draw,rounded corners,minimum height=0.8cm,align=center] {Retrieve \\ Skills}
([xshift=0.8cm]n42.east) node (n43) [anchor=west, text width=1.2cm,draw,rounded corners,minimum height=0.8cm,align=center] {Agent}
([xshift=0.8cm]n43.east) node (n44) [anchor=west, text width=1.2cm,draw,rounded corners,minimum height=0.8cm,align=center] {Reward};
\draw[->] (n41.east) -- (n42.west);
\draw[->] (n42.east) -- (n43.west);
\draw[->] (n43.east) -- (n44.west);
\draw [dashed] ([xshift=0.8cm,yshift=0.1cm]b5.north east) -- ([xshift=9cm,yshift=0.1cm]b5.north east);
\node [anchor=north west, text width=7.6cm,align=left] (n5) at ([xshift=\ssep]b5.north east) {Analyze failure trajectories during training.
Generate new skills or refine existing skills.};
\node [anchor=north west, text width=6cm,draw,dashed,rounded corners,minimum height=1cm,align=left] (n51) at (n5.south west) {};
\node [anchor=west,text width=2cm,align=center] (n52) at ([xshift=0.2cm]n51.west) {\textbf{\small{Outcome}}};
\path (n52.east) node(n53) [anchor=west, text width=1.3cm,draw,rounded corners,minimum height=0.6cm,align=center] {new skill}
([xshift=0.5cm]n53.east) node(n54) [anchor=west, text width=1.3cm,draw,rounded corners,minimum height=0.6cm,align=center] {old skill};
\draw ([xshift=0.1cm]n53.south east) -- ([xshift=-0.1cm]n54.north west);
\end{scope}
\end{tikzpicture}
\end{center}
section6/section6.tex
......@@ -340,8 +340,7 @@ Memory management is a direct way for agents to learn from agentic experience. D
\begin{figure}[!t]
\centering
\resizebox{\linewidth}{!}{
\input{section6/Figures/memory-and-retrieval.tex}}
\input{section6/Figures/memory-and-retrieval.tex}
\caption{
An overview of the memory system.
}
......@@ -354,8 +353,7 @@ In learning from experience, it is easy to observe that the performance of memor
\begin{figure}[!t]
\centering
\resizebox{\linewidth}{!}{
\input{section6/Figures/reinfroced-memory-manager.tex}}
\input{section6/Figures/reinfroced-memory-manager.tex}
\caption{Overview of training a memory manager with RL.}
\label{fig:reinforced-memory}
\end{figure}
......@@ -457,7 +455,8 @@ The above way of generating skills often suffer from quality issues. First, a ge
% \resizebox{\linewidth}{!}{
% \input{section6/Figures/skillrl.tex}
% }
\includegraphics[width=0.5\textwidth]{section6/Figures/skillrl_draft.png}
% \includegraphics[width=0.5\textwidth]{section6/Figures/skillrl_draft.png}
\input{section6/Figures/skillrl_illustration.tex}
\caption{
Illustration of SkillRL \citep{xia-etal:skillrl}. SkillRL maintains a pool of agentic trajectories and initially abstracts reusable skills from these trajectories. The extracted skills are then used to guide RL training. As the agent improves through RL training, it generates higher-quality trajectories, which are further used to refine the skill bank.
}
......
section7/Figures/rovrm.tex
......@@ -3,51 +3,58 @@
\definecolor{temp4}{RGB}{250,235,243}
\usetikzlibrary {shapes.arrows}
\begin{tikzpicture} [
\begin{center}
\begin{tikzpicture} [
box/.style={rounded corners=6pt, align=center, minimum height=1.2cm,anchor=center},
phasebox/.style={sharp corners,draw,thick,align=center,minimum height=1.2cm,anchor=center,text width=5.8cm,fill=white,drop shadow={fill=gray,shadow xshift=.6ex,shadow yshift=-.6ex}},
box1/.style={box,sharp corners,align=left, minimum height=4cm,anchor=center,text width=6.5cm,fill=white},
box2/.style={box,rounded corners=10pt,align=left, minimum height=4.4cm,anchor=center,minimum width=7.1cm,fill=temp2}
phasebox/.style={sharp corners,draw,thick,align=center,minimum height=0.8cm,anchor=center,text width=4.2cm,fill=white,drop shadow={fill=gray,shadow xshift=.6ex,shadow yshift=-.6ex}},
box1/.style={box,sharp corners,align=left, minimum height=3.5cm,anchor=center,text width=4.6cm,fill=white},
box2/.style={box,rounded corners=6pt,align=left, minimum height=3.8cm,anchor=center,minimum width=5cm,fill=temp2}
]
\path[use as bounding box] (-11.4cm,-6.4cm) rectangle (11.4cm,1.25cm);
\begin{scope}
\node [phasebox] (phase-1) at (-7.5,0) {\textit{\textbf{Stage 1: Pre-training with \\ Textual Preference Data}}};
\node [phasebox] (phase-2) at ([xshift=7.5cm]phase-1.center) {\textit{\textbf{Stage 2: Fine-tuning with Image \\ Caption-based Preference Data}}};
\node [phasebox] (phase-3) at ([xshift=7.5cm]phase-2.center) {\textit{\textbf{Stage 3: Fine-tuning with \\ Visual Preference Data}}};
% \path[use as bounding box] (-11.4cm,-6.4cm) rectangle (11.4cm,1.25cm);
\scriptsize
\node [phasebox] (phase-1) at (0,0) {\textit{\textbf{Stage 1: Pre-training with \\ Textual Preference Data}}};
\node [phasebox] (phase-2) at ([xshift=5.5cm]phase-1.center) {\textit{\textbf{Stage 2: Fine-tuning with Image \\ Caption-based Preference Data}}};
\node [phasebox] (phase-3) at ([xshift=5.5cm]phase-2.center) {\textit{\textbf{Stage 3: Fine-tuning with \\ Visual Preference Data}}};
\node [box2] (box-1) at ([yshift=-4cm]phase-1.center) {};
\node [box2] (box-1) at ([yshift=-3.6cm]phase-1.center) {};
\node [box1] (box-2) at (box-1)
{\textit{\underline{Instruction}}: Can you determine the missing number in the sequence: 2, 6, 14, 30, 62, \_\_? \\ ~ \\
\textit{\underline{Chosen Response}}: The missing number in the sequence is 126.\\ \textit{\underline{Rejected Response}}: The sequence is 6.};
\node [box2] (box-3) at ([yshift=-4cm]phase-2.center) {};
\node [box2] (box-3) at ([yshift=-3.6cm]phase-2.center) {};
\node [box1] (box-4) at (box-3)
{\textit{\underline{Instruction}}: Describe the image in detail.\\ \textit{\underline{Image Caption}}: There are four mangosteens,
one of which has been cut open.\\ \textit{\underline{Chosen Response}}: There are several \textcolor{temp3}{mangosteens} on a \textcolor{temp3}{wooden table}.\\
\textit{\underline{Rejected Response}}: This is a dark fruit, possibly a variety of \textcolor{red}{pitaya}.\\};
\node [box2] (box-5) at ([yshift=-4cm]phase-3.center) {};
\node [box2] (box-5) at ([yshift=-3.6cm]phase-3.center) {};
\node [box1] (box-6) at (box-5)
{\includegraphics{section7/Figures/mangosteens.jpg}\\\textit{\underline{Chosen Response}}: There are several \textcolor{temp3}{mangosteens} on a \textcolor{temp3}{wooden table}.\\
\textit{\underline{Rejected Response}}: This is a dark fruit, possibly a variety of \textcolor{red}{pitaya}.\\};
\node [text width=4.5cm,anchor=north,align=left] at ([xshift=1.2cm,yshift=-.5cm]box-6.north)
\node [text width=2.5cm,anchor=north,align=left] at ([xshift=1.1cm,yshift=-.5cm]box-6.north)
{\textit{\underline{Instruction}}: Describe the image in detail.};
\node[draw,fill=temp4,minimum height=1cm,minimum width=0.75cm,single arrow,anchor=south,shape border rotate=90,single arrow head extend=0.12cm,] (arrow) at ([yshift=0.1cm]box-1.north) {};
\node[draw,fill=temp4,minimum height=1cm,minimum width=0.75cm,single arrow,anchor=south,shape border rotate=90,single arrow head extend=0.12cm,] (arrow) at ([yshift=0.1cm]box-3.north) {};
\node[draw,fill=temp4,minimum height=1cm,minimum width=0.75cm,single arrow,anchor=south,shape border rotate=90,single arrow head extend=0.12cm,] (arrow) at ([yshift=0.1cm]box-5.north) {};
\draw[line width=0.5mm,arrows = {-Stealth[inset=0pt, length=0.2cm, angle'=45]}] (phase-1.east) to ([xshift=0.3cm]phase-1.east)
to [out=90,in=180] ($1/2*(phase-1.east)+1/2*(phase-2.west)+(-0.1cm,0.5cm)$) to [out=0,in=90] ([xshift=-0.5cm]phase-2.west) to (phase-2.west);
\draw[line width=0.2mm] ($1/2*(phase-1.east)+1/2*(phase-2.west)+(-0.1cm,0.8cm)$) .. controls +(-0.5cm,-1.4cm) and +(+0.5cm,1.4cm) .. ($1/2*(phase-1.east)+1/2*(phase-2.west)+(-0.3cm,-0.4cm)$);
\draw[line width=0.2mm] ($1/2*(phase-1.east)+1/2*(phase-2.west)+(0.1cm,0.8cm)$) .. controls +(-0.5cm,-1.4cm) and +(+0.5cm,1.4cm) .. ($1/2*(phase-1.east)+1/2*(phase-2.west)+(-0.1cm,-0.4cm)$);
\node at ([yshift=-0.9cm]$1/2*(phase-1.east)+1/2*(phase-2.west)$) {Bridging Task Gap};
\draw[line width=0.35mm,arrows = {-Stealth[inset=0pt, length=0.2cm, angle'=45]}] (phase-1.east) to ([xshift=0.18cm]phase-1.east)
to [out=90,in=180] ($1/2*(phase-1.east)+1/2*(phase-2.west)+(-0.1cm,0.35cm)$) to [out=0,in=90] ([xshift=-0.38cm]phase-2.west) to (phase-2.west);
\draw[line width=0.2mm] ($1/2*(phase-1.east)+1/2*(phase-2.west)+(-0.1cm,0.6cm)$) .. controls +(-0.5cm,-1.4cm) and +(+0.5cm,1.4cm) .. ($1/2*(phase-1.east)+1/2*(phase-2.west)+(-0.3cm,-0.4cm)$);
\draw[line width=0.2mm] ($1/2*(phase-1.east)+1/2*(phase-2.west)+(0.1cm,0.6cm)$) .. controls +(-0.5cm,-1.4cm) and +(+0.5cm,1.4cm) .. ($1/2*(phase-1.east)+1/2*(phase-2.west)+(-0.1cm,-0.4cm)$);
\node at ([yshift=-0.7cm]$1/2*(phase-1.east)+1/2*(phase-2.west)$) {Bridging Task Gap};
\draw[line width=0.5mm,arrows = {-Stealth[inset=0pt, length=0.2cm, angle'=45]}] (phase-2.east) to ([xshift=0.3cm]phase-2.east)
to [out=90,in=180] ($1/2*(phase-2.east)+1/2*(phase-3.west)+(-0.1cm,0.5cm)$) to [out=0,in=90] ([xshift=-0.5cm]phase-3.west) to (phase-3.west);
\draw[line width=0.2mm] ($1/2*(phase-2.east)+1/2*(phase-3.west)+(-0.1cm,0.8cm)$) .. controls +(-0.5cm,-1.4cm) and +(+0.5cm,1.4cm) .. ($1/2*(phase-2.east)+1/2*(phase-3.west)+(-0.3cm,-0.4cm)$);
\draw[line width=0.2mm] ($1/2*(phase-2.east)+1/2*(phase-3.west)+(0.1cm,0.8cm)$) .. controls +(-0.5cm,-1.4cm) and +(+0.5cm,1.4cm) .. ($1/2*(phase-2.east)+1/2*(phase-3.west)+(-0.1cm,-0.4cm)$);
\node at ([yshift=-0.9cm]$1/2*(phase-2.east)+1/2*(phase-3.west)$) {Bridging Modality Gap };
\draw[line width=0.35mm,arrows = {-Stealth[inset=0pt, length=0.2cm, angle'=45]}] (phase-2.east) to ([xshift=0.18cm]phase-2.east)
to [out=90,in=180] ($1/2*(phase-2.east)+1/2*(phase-3.west)+(-0.1cm,0.35cm)$) to [out=0,in=90] ([xshift=-0.38cm]phase-3.west) to (phase-3.west);
\draw[line width=0.2mm] ($1/2*(phase-2.east)+1/2*(phase-3.west)+(-0.1cm,0.6cm)$) .. controls +(-0.5cm,-1.4cm) and +(+0.5cm,1.4cm) .. ($1/2*(phase-2.east)+1/2*(phase-3.west)+(-0.3cm,-0.4cm)$);
\draw[line width=0.2mm] ($1/2*(phase-2.east)+1/2*(phase-3.west)+(0.1cm,0.6cm)$) .. controls +(-0.5cm,-1.4cm) and +(+0.5cm,1.4cm) .. ($1/2*(phase-2.east)+1/2*(phase-3.west)+(-0.1cm,-0.4cm)$);
\node at ([yshift=-0.7cm]$1/2*(phase-2.east)+1/2*(phase-3.west)$) {Bridging Modality Gap };
\end{tikzpicture}
\ No newline at end of file
\end{scope}
\end{tikzpicture}
\end{center}
section7/section7.tex
......@@ -15,7 +15,8 @@ Training LLMs and VLMs with RL exhibits only minimal differences, primarily rela
\begin{figure*}[!t]
\centering
\includegraphics[width=0.98\textwidth]{section7/Figures/rovrm.pdf}
%% \includegraphics[width=0.98\textwidth]{section7/Figures/rovrm.pdf}
\input{section7/Figures/rovrm.tex}
\caption{An overview of the multi-stage training approach for visual reward models.}
\label{fig:preference-transfer}
\end{figure*}
......
Markdown 格式
0% 或
您添加了 0 人 到此讨论。请谨慎行事。
请先完成此评论的编辑！
请 注册 或者 后发表评论