Skip to content

R
rl-introduction
Commit 9e484ac7 by wangchenglong
Options

update.

parent d6804a6b
正在显示 包含 38 行增加 和 20 行删除
section4/section4.tex
......@@ -255,7 +255,7 @@ where $v_i=1$ indicates that the output is correct. If the output selected by th
\frac{1}{N}
\sum_{j=1}^{N}
\mathbb{I}
(\mathrm{Verify}(\mathbf{x}_j,\mathbf{y}_{\mathrm{best}}^j)=1),
(\mathrm{Verify}(\mathbf{x}_j,\mathbf{y}_{\mathrm{best}}^j)=1)
\end{eqnarray}
where $N$ denotes the number of evaluation instances.
......@@ -263,7 +263,7 @@ where $N$ denotes the number of evaluation instances.
\end{itemize}
-> probing approach?
......
section6/Figures/memory-and-retrieval.tex
......@@ -96,7 +96,7 @@
\path
([yshift=-0.25cm]newmsg.south-|conv.west) node [anchor=north west,text width=0.4cm] (roboticon2) {\faIcon{robot}}
(roboticon2.north east) node[smallmsg, anchor=north west] (robotmsg) {$\cdots \cdots$};
(roboticon2.north east) node[smallmsg, anchor=north west] (robotmsg) {$\cdots$};
\node[box, minimum width=5cm, minimum height=5.8cm] (memory) at ([xshift=8cm]conv) {};
\node[font=\small\bfseries] at ($(memory.north)+(0,-0.35)$) {Memory Bank};
......
section6/Figures/skillrl_illustration.tex
......@@ -6,6 +6,7 @@
\tikzset {
block/.style={draw,inner sep=0pt,fill=white, minimum width=5cm, minimum height=1.5cm, align=center},
miniblock/.style={draw,inner sep=0pt,fill=white, minimum width=1.5cm, minimum height=.8cm, align=center, rounded corners=2pt, text width=1.4cm},
trajblock/.style={draw,rounded corners,minimum height=1.5cm,text width=3.35cm,align=left,font=\tiny,inner sep=3pt},
linetext/.style={fill=#1, minimum height=1.5pt, minimum width=.8cm, inner sep=0},
}
\def\sep{1cm}
......@@ -25,12 +26,27 @@
% \draw [->] (b5.north) -- (b6.south);
\scriptsize
\node [anchor=north west, text width=7.6cm] (n1) at ([xshift=\ssep]b1.north east) {Collect interaction trajectories from the enrionment.\\};
\node [anchor=north west, text width=3.5cm,draw,rounded corners,minimum height=1.5cm] (n12) at ([yshift=-0.1cm]n1.south west) {} ;
\node [anchor=west, text width=3.5cm,draw,rounded corners,minimum height=1.5cm] (n13) at ([xshift=0.3cm]n12.east) {} ;
\node [anchor=north west, text width=7.6cm] (n1) at ([xshift=\ssep]b1.north east) {Collect interaction trajectories from the environment.\\};
\node [anchor=north west,trajblock] (n12) at ([yshift=-0.1cm]n1.south west) {
\textbf{Successful trajectory}\\
Task: solve $2x+3=11$.\\
Thought: subtract $3$, then divide by $2$.\\
$\cdots$ \\
Answer: $x=4$.
};
\node [anchor=west,font=\small] (n1dots) at ([xshift=0.15cm]n12.east) {$\cdots$};
\node [anchor=west,trajblock] (n13) at ([xshift=0.15cm]n1dots.east) {
\textbf{Failed trajectory}\\
Task: solve $2x+3=11$.\\
Thought: divide by $2$, then subtract $3$.\\
$\cdots$ \\
Answer: $x=2.5$.
};
\node [anchor=south east,font=\large,text=green!55!black] at ([xshift=-0.05cm,yshift=0.02cm]n12.south east) {$\checkmark$};
\node [anchor=south east,font=\large,text=red!65!black] at ([xshift=-0.05cm,yshift=0.02cm]n13.south east) {$\times$};
\draw [dashed] ([xshift=0.8cm,yshift=0.1cm]b2.north east) -- ([xshift=9cm,yshift=0.1cm]b2.north east);
\node [anchor=north west, text width=7.6cm] (n2) at ([xshift=\ssep]b2.north east) {Generate reusable skills from experiences.\\};
\node [anchor=north west, text width=7.6cm] (n2) at ([xshift=\ssep]b2.north east) {Generate reusable skills from collected trajectories.\\};
\node [anchor=north west, text width=] (n21) at (n2.south west) {Successful trajectories};
\node [anchor=west, text width=] (n22) at ([xshift=.2cm]n21.east) {$\rightarrow$~~Skill Principles};
\node [anchor=north west, text width=] (n23) at (n21.south west) {Failed trajectories};
......@@ -38,30 +54,32 @@
\draw [dashed] ([xshift=0.8cm,yshift=0.1cm]b3.north east) -- ([xshift=9cm,yshift=0.1cm]b3.north east);
\node [anchor=north west, text width=7.6cm] (n3) at ([xshift=\ssep]b3.north east) {Construct a hierarchical skill library.};
\node [anchor=north west, text width=7.6cm,draw,rounded corners,minimum height=0.8cm,align=left] (n31) at (n3.south west) {};
\node [anchor=west,text width=3cm,align=center] (n311) at ([xshift=0.2cm]n31.west) {\textbf{\small{General Skills}}};
\node [anchor=west,text width=4cm,align=left] (n312) at ([xshift=0.2cm]n311.east) {bla bla bla bla bla bla bla bla bla bla bla bla bla bla };
\node [anchor=north west, text width=7.6cm,draw,rounded corners,minimum height=0.8cm,align=left] (n32) at ([yshift=-0.1cm]n31.south west) {};
\node [anchor=west,text width=3cm,align=center] (n321) at ([xshift=0.2cm]n32.west) {\textbf{\small{Task-specific Skils}}};
\node [anchor=west,text width=4cm,align=left] (n322) at ([xshift=0.2cm]n321.east) {bla bla bla bla bla bla bla bla bla bla bla bla bla bla };
\draw [decorate,decoration={brace,mirror,raise=5pt,amplitude=6pt}] (n31.north west) -- (n32.south west);
\node [anchor=north west, text width=7.7cm,draw,rounded corners,minimum height=0.72cm,align=left] (n31) at (n3.south west) {};
\node [anchor=west,text width=2.55cm,align=center] (n311) at ([xshift=0.15cm]n31.west) {\textbf{General Skills}};
\node [anchor=west,text width=4.8cm,align=left,font=\tiny] (n312) at ([xshift=0.15cm]n311.east) {Use algebraic invariants to transform equations step by step.};
\node [anchor=north west, text width=7.7cm,draw,rounded corners,minimum height=0.72cm,align=left] (n32) at ([yshift=-0.1cm]n31.south west) {};
\node [anchor=west,text width=2.55cm,align=center] (n321) at ([xshift=0.15cm]n32.west) {\textbf{Task-specific Skills}};
\node [anchor=west,text width=4.8cm,align=left,font=\tiny] (n322) at ([xshift=0.15cm]n321.east) {For linear equations, isolate the variable before computing its value.};
\draw [dashed] ([xshift=0.8cm,yshift=0.1cm]b4.north east) -- ([xshift=9cm,yshift=0.1cm]b4.north east);
\node [anchor=north west, text width=7.6cm,align=left] (n4) at ([xshift=\ssep]b4.north east) {Retrieve relevant skills then provide retrieved skills to the agent.
Optimize the skill-augmented policy with RL.};
\draw ([yshift=-0.1cm]n4.south west) node (n41) [anchor=north west, text width=1.2cm,draw,rounded corners,minimum height=0.8cm,align=center] {Task}
([xshift=0.8cm]n41.east) node (n42) [anchor=west, text width=1.2cm,draw,rounded corners,minimum height=0.8cm,align=center] {Retrieve \\ Skills}
([xshift=0.8cm]n42.east) node (n43) [anchor=west, text width=1.2cm,draw,rounded corners,minimum height=0.8cm,align=center] {Agent}
([xshift=0.8cm]n43.east) node (n44) [anchor=west, text width=1.2cm,draw,rounded corners,minimum height=0.8cm,align=center] {Reward};
\draw ([yshift=-0.1cm]n4.south west) node (n41) [anchor=north west, text width=1.3cm,draw,rounded corners,minimum height=0.8cm,align=center] {Task}
([xshift=0.60cm]n41.east) node (n42) [anchor=west, text width=1.3cm,draw,rounded corners,minimum height=0.8cm,align=center] {Retrieving \\ Skills}
([xshift=0.60cm]n42.east) node (n43) [anchor=west, text width=1.3cm,draw,rounded corners,minimum height=0.8cm,align=center] {Agent}
([xshift=0.60cm]n43.east) node (n44) [anchor=west, text width=1.45cm,draw,rounded corners,minimum height=0.8cm,align=center] {Computing Rewards};
\draw[->] (n41.east) -- (n42.west);
\draw[->] (n42.east) -- (n43.west);
\draw[->] (n43.east) -- (n44.west);
\draw[->,dashed]
([yshift=-0.08cm]n44.south) .. controls +(down:0.35cm) and +(down:0.35cm) ..
([yshift=-0.08cm]n43.south);
\draw [dashed] ([xshift=0.8cm,yshift=0.1cm]b5.north east) -- ([xshift=9cm,yshift=0.1cm]b5.north east);
\node [anchor=north west, text width=7.6cm,align=left] (n5) at ([xshift=\ssep]b5.north east) {Analyze failure trajectories during training.
Generate new skills or refine existing skills.};
\node [anchor=north west, text width=6cm,draw,dashed,rounded corners,minimum height=1cm,align=left] (n51) at (n5.south west) {};
\node [anchor=west,text width=2cm,align=center] (n52) at ([xshift=0.2cm]n51.west) {\textbf{\small{Outcome}}};
\node [anchor=west,text width=2cm,align=center] (n52) at ([xshift=0.2cm]n51.west) {\textbf{Outcome}};
\path (n52.east) node(n53) [anchor=west, text width=1.3cm,draw,rounded corners,minimum height=0.6cm,align=center] {new skill}
([xshift=0.5cm]n53.east) node(n54) [anchor=west, text width=1.3cm,draw,rounded corners,minimum height=0.6cm,align=center] {old skill};
\draw ([xshift=0.1cm]n53.south east) -- ([xshift=-0.1cm]n54.north west);
......
section6/section6.tex
......@@ -342,7 +342,7 @@ Memory management is a direct way for agents to learn from agentic experience. D
\centering
\input{section6/Figures/memory-and-retrieval.tex}
\caption{
An overview of a memory system for agentic interaction. We take a travel-planning scenario as an example, where the user asks the agent to plan a trip to Kyoto. The agent first retrieves relevant user preferences from the memory bank, such as vegetarian meals, window seats, and budget hotels, and uses them to support the current conversation. During the interaction, the user provides new information, such as a preference for quiet ryokans near Kyoto Station. This new information is extracted as memory and passed to a memory manager, which updates the memory bank by selecting an operation such as \texttt{ADD}. Blue highlights indicate retrieved memories, while red highlights indicate newly extracted memories and their update path back to the memory bank.
An overview of a memory system for agentic interaction. We take a travel-planning scenario as an example, where the user asks the agent to plan a trip to Kyoto. The agent first retrieves relevant user preferences from the memory bank, such as vegetarian meals, window seats, and budget hotels, and uses them to support the current conversation. During the interaction, the user provides new information, such as a preference for quiet ryokans near Kyoto Station. This new information is extracted as memory and passed to a memory manager, which updates the memory bank by selecting an operation such as \texttt{ADD}. \colorbox{blue!30}{Blue} highlights denote retrieved memories, while \colorbox{red!10}{read} highlights denote newly extracted memories and their update path back to the memory bank.
}
\label{fig:memory-and-retrieve}
\end{figure}
......
Markdown 格式
0% 或
您添加了 0 人 到此讨论。请谨慎行事。
请先完成此评论的编辑！
请 注册 或者 后发表评论