Skip to content

R
rl-introduction
Commit 122319ad by wangchenglong
Options

update.

parent 53ac0b50
正在显示 包含 70 行增加 和 32 行删除
rl-introduction.pdf
No preview for this file type
rl-introduction.tex
...@@ -11,6 +11,8 @@ ...@@ -11,6 +11,8 @@
\usepackage{bbding} \usepackage{bbding}
% Write outlines on Chinese % Write outlines on Chinese
\usepackage{CJK} \usepackage{CJK}
\def\bibdata#1{}
\def\bibstyle#1{}
%%%%%%%%%%%%%%%%%%%%%% algorithm %%%%%%%%%%%%%%%%%%%%%%% %%%%%%%%%%%%%%%%%%%%%% algorithm %%%%%%%%%%%%%%%%%%%%%%%
\usepackage{algorithm} % http://ctan.org/pkg/algorithms \usepackage{algorithm} % http://ctan.org/pkg/algorithms
......
section6/Figures/memory-and-retrieval.tex
...@@ -29,6 +29,21 @@ ...@@ -29,6 +29,21 @@
text width=3.8cm, text width=3.8cm,
rounded corners rounded corners
}, },
retrievedmem/.style={
memitem,
fill=blue!20,
draw=blue!55
},
newmem/.style={
memitem,
fill=red!8,
draw=red!55!black
},
usedmem/.style={
smallmsg,
fill=blue!20,
draw=blue!55
},
opseg/.style={ opseg/.style={
draw, draw,
fill=white, fill=white,
...@@ -41,10 +56,20 @@ ...@@ -41,10 +56,20 @@
arrow/.style={ arrow/.style={
->, ->,
thick thick
},
retrarrow/.style={
->,
thick,
blue!55!black
},
updatearrow/.style={
->,
thick,
red!65!black
} }
] ]
\node[box, minimum width=5.6cm, minimum height=5.6cm] (conv) at (0,0) {}; \node[box, minimum width=5.6cm, minimum height=6.8cm] (conv) at (0,0) {};
\node[font=\scriptsize\bfseries, rotate=90,anchor=north east] at (conv.north east) {Conversation}; \node[font=\scriptsize\bfseries, rotate=90,anchor=north east] at (conv.north east) {Conversation};
\path \path
...@@ -61,41 +86,52 @@ ...@@ -61,41 +86,52 @@
\path \path
([yshift=-0.3cm]robotmsg0.south-|conv.east) node [anchor=north east,text width=0.4cm,align=right] (infoicon) {\faIcon{info-circle}} ([yshift=-0.3cm]robotmsg0.south-|conv.east) node [anchor=north east,text width=0.4cm,align=right] (infoicon) {\faIcon{info-circle}}
(infoicon.north west) node[smallmsg, anchor=north east] (infomsg) (infoicon.north west) node[usedmem, anchor=north east] (infomsg)
{$\bullet$ prefers window seat\\ $\bullet$ vegetarian meals\\ $\bullet$ budget hotel}; {$\bullet$ prefers window seat\\ $\bullet$ vegetarian meals\\ $\bullet$ budget hotel};
\path \path
([yshift=-0.3cm]infomsg.south-|conv.west) node [anchor=north west,text width=0.4cm] (roboticon) {\faIcon{robot}} ([yshift=-0.25cm]infomsg.south-|conv.east) node [anchor=north east,text width=0.4cm,align=right] (usericon2) {\faIcon{user}}
(roboticon.north east) node[smallmsg, anchor=north west] (robotmsg) {...}; (usericon2.north west) node[smallmsg, anchor=north east,fill=red!8,draw=red!55!black] (newmsg)
{I also prefer a quiet\\ ryokan near Kyoto\\ Station.};
\path
([yshift=-0.25cm]newmsg.south-|conv.west) node [anchor=north west,text width=0.4cm] (roboticon2) {\faIcon{robot}}
(roboticon2.north east) node[smallmsg, anchor=north west] (robotmsg) {$\cdots \cdots$};
\node[box, minimum width=5cm, minimum height=4.8cm] (memory) at ([xshift=8cm]conv) {}; \node[box, minimum width=5cm, minimum height=5.8cm] (memory) at ([xshift=8cm]conv) {};
\node[font=\small\bfseries] at ($(memory.north)+(0,-0.35)$) {Memory Bank}; \node[font=\small\bfseries] at ($(memory.north)+(0,-0.35)$) {Memory Bank};
\node[memitem, anchor=north,align=center] (m1) at ([yshift=-0.9cm]memory.north) {\textbf{Key 1}\\ prefers vegetarian meals}; \node[retrievedmem, anchor=north,align=center] (m1) at ([yshift=-0.9cm]memory.north) {\textbf{Memory 1}\\ prefers vegetarian meals};
\node[memitem, anchor=north,align=center] (m2) at ([yshift=-0.2cm]m1.south) {\textbf{Key 2}\\ prefers window seat}; \node[retrievedmem, anchor=north,align=center] (m2) at ([yshift=-0.2cm]m1.south) {\textbf{Memory 2}\\ prefers window seat};
\node[memitem, anchor=north,align=center] (m3) at ([yshift=-0.2cm]m2.south) {\textbf{Key 3}\\ usually books budget hotels}; \node[retrievedmem, anchor=north,align=center] (m3) at ([yshift=-0.2cm]m2.south) {\textbf{Memory 3}\\ usually books budget hotels};
\node[memitem, anchor=north,align=center] (m4) at ([yshift=-0.2cm]m3.south) {\textbf{Key ...}\\ ...}; \node[newmem, anchor=north,align=center] (m4) at ([yshift=-0.2cm]m3.south) {\textbf{Memory 4}\\ prefers quiet ryokans\\ near Kyoto Station};
\node[memitem, anchor=north,align=center] (m5) at ([yshift=-0.2cm]m4.south) {$\cdots$};
\draw[arrow] (conv.east|-robotmsg0.east) -- node[above, font=\scriptsize,align=center] {retrieve \\ relevant memories} (memory.west|-robotmsg0.east); \draw[retrarrow] (conv.east|-robotmsg0.east) -- node[above, font=\scriptsize,align=center,text=blue!55!black] {retrieving \\ relevant memories} (memory.west|-robotmsg0.east);
\draw[arrow] (memory.west|-infomsg.east) -- node[above, font=\scriptsize,align=center] {useful memories} (conv.east|-infomsg.east); \draw[retrarrow] (memory.west|-infomsg.east) -- node[above, font=\scriptsize,align=center,text=blue!55!black] {useful memories} (conv.east|-infomsg.east);
\scriptsize \scriptsize
\node[box, text width=4cm, minimum height=2cm,anchor=north west,align=left] (memext) at ([yshift=-.5cm]conv.south west) \node[box, text width=4.5cm, minimum height=2cm,anchor=north west,align=left] (memext) at ([yshift=-.5cm]conv.south west)
{\centering\textbf{\small Extract Memories} \\[.2cm] Integrate the newly extracted information with existing memories}; {\centering\textbf{\small Extract Memories}\\[.2cm]
\raggedright
\textbf{New memory:}\\
user prefers quiet ryokans\\
user prefers the Kyoto Station area};
\node[box, text width=7.5cm, minimum height=2cm,anchor=north east,align=center] (llmupdatemem) at ([yshift=-.5cm]conv.south-|memory.south east) \node[box, text width=7.5cm, minimum height=2cm,anchor=north east,align=center] (llmupdatemem) at ([yshift=-.5cm]conv.south-|memory.south east)
{\textbf{\small Memory Manager} \vspace{1.cm}}; {\textbf{\small Memory Manager} \vspace{1.cm}};
\node[opseg,anchor=south west] (add) at ([xshift=.35cm,yshift=.3cm]llmupdatemem.south west) {ADD}; \node[opseg,anchor=south west] (add) at ([xshift=.4cm,yshift=.3cm]llmupdatemem.south west) {ADD};
\node[opseg, right=.3cm of add] (update) {UPDATE}; \node[opseg, right=.3cm of add] (update) {UPDATE};
\node[opseg, right=.3cm of update] (delete) {DELETE}; \node[opseg, right=.3cm of update] (delete) {DELETE};
\node[opseg, right=.3cm of delete] (noop) {NOOP}; \node[opseg, right=.3cm of delete] (noop) {NOOP};
\draw[] ([xshift=-.1cm,yshift=.1cm]add.north west) rectangle ([xshift=.1cm,yshift=-.1cm]noop.south east); \draw[] ([xshift=-.1cm,yshift=.1cm]add.north west) rectangle ([xshift=.1cm,yshift=-.1cm]noop.south east);
\draw[arrow] (memext.center|-conv.south) -- node[right, font=\scriptsize] {new or updated memories} (memext.north); \draw[updatearrow] (memext.north|-conv.south) -- node[right, font=\scriptsize,text=red!65!black] {new memories} (memext.north);
\draw[arrow] (memext) -- (llmupdatemem); \draw[updatearrow] (memext) -- (llmupdatemem);
\draw[arrow] (memory.south|-llmupdatemem.north) -- node[right,font=\scriptsize] {operate} (memory.south); \draw[updatearrow] (memory.south|-llmupdatemem.north) -- node[right,font=\scriptsize,text=red!65!black] {ADD} (memory.south);
\end{tikzpicture} \end{tikzpicture}
......
section6/Figures/reinfroced-memory-manager.tex
...@@ -18,8 +18,10 @@ ...@@ -18,8 +18,10 @@
% Extracted memory % Extracted memory
\node[box, text width=3cm,align=left] (extracted) \node[box, text width=3cm,align=left] (extracted)
{\begin{center} \textbf{Extracted Memory} \end{center} \\[0.1cm] {\makebox[\linewidth][c]{\textbf{Extracted Memory}}\\[0.1cm]
$\bullet$ prefers window seat\\ $\bullet$ vegetarian meals\\ $\bullet$ budget hotel}; $\bullet$ prefers window seat\\
$\bullet$ vegetarian meals\\
$\bullet$ budget hotel};
% Memory manager % Memory manager
\node[box, text width=3cm, right=1.5cm of extracted] (manager) { \node[box, text width=3cm, right=1.5cm of extracted] (manager) {
...@@ -29,21 +31,19 @@ ...@@ -29,21 +31,19 @@
% Memory bank % Memory bank
\node[box, text width=3cm, below=0.8cm of manager] (bank) { \node[box, text width=3cm, below=0.8cm of manager] (bank) {
\textbf{Memory Bank}\\[0.1cm] \textbf{Memory Bank}\\[0.1cm]
$\mathcal{M}_{old} updating memory
\rightarrow
\mathcal{M}_{new}$
}; };
% Agent % Agent
\node[box, text width=2.8cm, right=2.0cm of bank.east] (agent) { \node[box, text width=2.8cm, right=3.0cm of bank.east] (agent) {
\textbf{Agent}\\[0.1cm] \textbf{Agent}\\[0.1cm]
Downstream task solving solving tasks
}; };
% Reward % Reward
\node[box, text width=2.8cm, right=2.0cm of manager.east] (rewardbox) { \node[box, text width=2.8cm, right=3.0cm of manager.east] (rewardbox) {
\textbf{Task Feedback}\\[0.1cm] \textbf{Task Feedback}\\[0.1cm]
Reward $R$ computing rewards
}; };
% Dashed container for memory manager % Dashed container for memory manager
...@@ -58,11 +58,11 @@ ...@@ -58,11 +58,11 @@
% Arrows % Arrows
\draw[arrow] (extracted) -- node[xshift=-.2cm,above,font=\scriptsize] {input} (manager); \draw[arrow] (extracted) -- node[xshift=-.2cm,above,font=\scriptsize] {input} (manager);
\draw[arrow] (manager) -- node[xshift=0.1cm,center,font=\scriptsize] {sampling operations} (bank); \draw[arrow] (manager) -- node[xshift=0.1cm,align=center,font=\scriptsize] {sampling operations} (bank);
\draw[arrow] ([yshift=0.2cm]bank.east) -- node[above,font=\scriptsize] {retrieve} ([yshift=0.2cm]agent.west); \draw[arrow] ([yshift=0.2cm]bank.east) -- node[midway,above,font=\scriptsize,text width=1.7cm,align=center] {retrieving} ([yshift=0.2cm]agent.west);
\draw[arrow] ([yshift=-0.2cm]agent.west) -- node[below,font=\scriptsize,align=right] {updated\\ memory} ([yshift=-0.2cm]bank.east); \draw[arrow] ([yshift=-0.2cm]agent.west) -- node[midway,below,font=\scriptsize,text width=1.7cm,align=center] {useful memory} ([yshift=-0.2cm]bank.east);
\draw[arrow] (agent.north) -- node[right,font=\scriptsize] {feedback} (rewardbox.south); \draw[arrow] (agent.north) -- node[right,font=\scriptsize] { trajectory} (rewardbox.south);
\draw[arrow] (rewardbox.west) -- node[xshift=0.2cm,font=\scriptsize,align=center] {RL \\optimization} (manager.east); \draw[arrow] (rewardbox.west) -- node[midway,above,yshift=0.08cm,inner sep=1pt,font=\scriptsize,text width=1.7cm,align=center] {RL training} (manager.east);
\end{tikzpicture} \end{tikzpicture}
\end{center} \end{center}
section6/section6.tex
...@@ -342,7 +342,7 @@ Memory management is a direct way for agents to learn from agentic experience. D ...@@ -342,7 +342,7 @@ Memory management is a direct way for agents to learn from agentic experience. D
\centering \centering
\input{section6/Figures/memory-and-retrieval.tex} \input{section6/Figures/memory-and-retrieval.tex}
\caption{ \caption{
An overview of the memory system. An overview of a memory system for agentic interaction. We take a travel-planning scenario as an example, where the user asks the agent to plan a trip to Kyoto. The agent first retrieves relevant user preferences from the memory bank, such as vegetarian meals, window seats, and budget hotels, and uses them to support the current conversation. During the interaction, the user provides new information, such as a preference for quiet ryokans near Kyoto Station. This new information is extracted as memory and passed to a memory manager, which updates the memory bank by selecting an operation such as \texttt{ADD}. Blue highlights indicate retrieved memories, while red highlights indicate newly extracted memories and their update path back to the memory bank.
} }
\label{fig:memory-and-retrieve} \label{fig:memory-and-retrieve}
\end{figure} \end{figure}
......
Markdown 格式
0% 或
您添加了 0 人 到此讨论。请谨慎行事。
请先完成此评论的编辑！
请 注册 或者 后发表评论