update.

rl-introduction.tex

@@ -11,6 +11,8 @@
 \usepackage{bbding}
 % Write outlines on Chinese
 \usepackage{CJK}
+\def\bibdata#1{}
+\def\bibstyle#1{}
 
 %%%%%%%%%%%%%%%%%%%%%% algorithm %%%%%%%%%%%%%%%%%%%%%%%
 \usepackage{algorithm} % http://ctan.org/pkg/algorithms

section6/Figures/memory-and-retrieval.tex

@@ -29,6 +29,21 @@
       text width=3.8cm,
       rounded corners
     },
+    retrievedmem/.style={
+      memitem,
+      fill=blue!20,
+      draw=blue!55
+    },
+    newmem/.style={
+      memitem,
+      fill=red!8,
+      draw=red!55!black
+    },
+    usedmem/.style={
+      smallmsg,
+      fill=blue!20,
+      draw=blue!55
+    },
     opseg/.style={
       draw,
       fill=white,
@@ -41,10 +56,20 @@
     arrow/.style={
       ->,
       thick
+    },
+    retrarrow/.style={
+      ->,
+      thick,
+      blue!55!black
+    },
+    updatearrow/.style={
+      ->,
+      thick,
+      red!65!black
     }
     ]
 
-      \node[box, minimum width=5.6cm, minimum height=5.6cm] (conv) at (0,0) {};
+      \node[box, minimum width=5.6cm, minimum height=6.8cm] (conv) at (0,0) {};
       \node[font=\scriptsize\bfseries, rotate=90,anchor=north east] at (conv.north east) {Conversation};
 
       \path
@@ -61,41 +86,52 @@
 
       \path
       ([yshift=-0.3cm]robotmsg0.south-|conv.east) node [anchor=north east,text width=0.4cm,align=right] (infoicon) {\faIcon{info-circle}}
-      (infoicon.north west) node[smallmsg, anchor=north east] (infomsg)
+      (infoicon.north west) node[usedmem, anchor=north east] (infomsg)
       {$\bullet$ prefers window seat\\ $\bullet$ vegetarian meals\\ $\bullet$ budget hotel};
 
       \path
-      ([yshift=-0.3cm]infomsg.south-|conv.west) node [anchor=north west,text width=0.4cm] (roboticon) {\faIcon{robot}}
-      (roboticon.north east) node[smallmsg, anchor=north west] (robotmsg) {...};
+      ([yshift=-0.25cm]infomsg.south-|conv.east) node [anchor=north east,text width=0.4cm,align=right] (usericon2) {\faIcon{user}}
+      (usericon2.north west) node[smallmsg, anchor=north east,fill=red!8,draw=red!55!black] (newmsg)
+      {I also prefer a quiet\\ ryokan near Kyoto\\ Station.};
 
-      \node[box, minimum width=5cm, minimum height=4.8cm] (memory) at ([xshift=8cm]conv) {};
+      \path
+      ([yshift=-0.25cm]newmsg.south-|conv.west) node [anchor=north west,text width=0.4cm] (roboticon2) {\faIcon{robot}}
+      (roboticon2.north east) node[smallmsg, anchor=north west] (robotmsg) {$\cdots \cdots$};
+
+      \node[box, minimum width=5cm, minimum height=5.8cm] (memory) at ([xshift=8cm]conv) {};
       \node[font=\small\bfseries] at ($(memory.north)+(0,-0.35)$) {Memory Bank};
 
-      \node[memitem, anchor=north,align=center] (m1) at ([yshift=-0.9cm]memory.north) {\textbf{Key 1}\\ prefers vegetarian meals};
-      \node[memitem, anchor=north,align=center] (m2) at ([yshift=-0.2cm]m1.south) {\textbf{Key 2}\\ prefers window seat};
-      \node[memitem, anchor=north,align=center] (m3) at ([yshift=-0.2cm]m2.south) {\textbf{Key 3}\\ usually books budget hotels};
-      \node[memitem, anchor=north,align=center] (m4) at ([yshift=-0.2cm]m3.south) {\textbf{Key ...}\\ ...};
+      \node[retrievedmem, anchor=north,align=center] (m1) at ([yshift=-0.9cm]memory.north) {\textbf{Memory 1}\\ prefers vegetarian meals};
+      \node[retrievedmem, anchor=north,align=center] (m2) at ([yshift=-0.2cm]m1.south) {\textbf{Memory 2}\\ prefers window seat};
+      \node[retrievedmem, anchor=north,align=center] (m3) at ([yshift=-0.2cm]m2.south) {\textbf{Memory 3}\\ usually books budget hotels};
+      \node[newmem, anchor=north,align=center] (m4) at ([yshift=-0.2cm]m3.south) {\textbf{Memory 4}\\ prefers quiet ryokans\\ near Kyoto Station};
+      \node[memitem, anchor=north,align=center] (m5) at ([yshift=-0.2cm]m4.south) {$\cdots$};
+      
 
-      \draw[arrow] (conv.east|-robotmsg0.east) -- node[above, font=\scriptsize,align=center] {retrieve \\ relevant memories} (memory.west|-robotmsg0.east);
-      \draw[arrow] (memory.west|-infomsg.east) -- node[above, font=\scriptsize,align=center] {useful memories} (conv.east|-infomsg.east);
+      \draw[retrarrow] (conv.east|-robotmsg0.east) -- node[above, font=\scriptsize,align=center,text=blue!55!black] {retrieving \\ relevant memories} (memory.west|-robotmsg0.east);
+      \draw[retrarrow] (memory.west|-infomsg.east) -- node[above, font=\scriptsize,align=center,text=blue!55!black] {useful memories} (conv.east|-infomsg.east);
 
       \scriptsize
-      \node[box, text width=4cm, minimum height=2cm,anchor=north west,align=left] (memext) at ([yshift=-.5cm]conv.south west)
-      {\centering\textbf{\small Extract Memories} \\[.2cm] Integrate the newly extracted information with existing memories};
+      \node[box, text width=4.5cm, minimum height=2cm,anchor=north west,align=left] (memext) at ([yshift=-.5cm]conv.south west)
+      {\centering\textbf{\small Extract Memories}\\[.2cm]
+      \raggedright
+      \textbf{New memory:}\\
+      user prefers quiet ryokans\\
+      user prefers the Kyoto Station area};
 
       \node[box, text width=7.5cm, minimum height=2cm,anchor=north east,align=center] (llmupdatemem) at ([yshift=-.5cm]conv.south-|memory.south east)
       {\textbf{\small Memory Manager} \vspace{1.cm}};
 
-      \node[opseg,anchor=south west] (add) at ([xshift=.35cm,yshift=.3cm]llmupdatemem.south west) {ADD};
+      \node[opseg,anchor=south west] (add) at ([xshift=.4cm,yshift=.3cm]llmupdatemem.south west) {ADD};
       \node[opseg, right=.3cm of add] (update) {UPDATE};
       \node[opseg, right=.3cm of update] (delete) {DELETE};
       \node[opseg, right=.3cm of delete] (noop) {NOOP};
 
       \draw[] ([xshift=-.1cm,yshift=.1cm]add.north west) rectangle ([xshift=.1cm,yshift=-.1cm]noop.south east);
 
-      \draw[arrow] (memext.center|-conv.south) -- node[right, font=\scriptsize] {new or updated memories} (memext.north);
-      \draw[arrow] (memext) -- (llmupdatemem);
-      \draw[arrow] (memory.south|-llmupdatemem.north) -- node[right,font=\scriptsize] {operate} (memory.south);
+      \draw[updatearrow] (memext.north|-conv.south) -- node[right, font=\scriptsize,text=red!65!black] {new memories} (memext.north);
+      \draw[updatearrow] (memext) -- (llmupdatemem);
+      \draw[updatearrow] (memory.south|-llmupdatemem.north) -- node[right,font=\scriptsize,text=red!65!black] {ADD} (memory.south);
 
   \end{tikzpicture}
 

section6/Figures/reinfroced-memory-manager.tex

@@ -18,8 +18,10 @@
 
     % Extracted memory
     \node[box, text width=3cm,align=left] (extracted)
-    {\begin{center} \textbf{Extracted Memory} \end{center} \\[0.1cm]
-      $\bullet$ prefers window seat\\ $\bullet$ vegetarian meals\\ $\bullet$ budget hotel};
+    {\makebox[\linewidth][c]{\textbf{Extracted Memory}}\\[0.1cm]
+      $\bullet$ prefers window seat\\
+      $\bullet$ vegetarian meals\\
+      $\bullet$ budget hotel};
 
     % Memory manager
     \node[box, text width=3cm, right=1.5cm of extracted] (manager) {
@@ -29,21 +31,19 @@
     % Memory bank
     \node[box, text width=3cm, below=0.8cm of manager] (bank) {
       \textbf{Memory Bank}\\[0.1cm]
-      $\mathcal{M}_{old}
-      \rightarrow
-      \mathcal{M}_{new}$
+      updating memory
     };
 
     % Agent
-    \node[box, text width=2.8cm, right=2.0cm of bank.east] (agent) {
+    \node[box, text width=2.8cm, right=3.0cm of bank.east] (agent) {
       \textbf{Agent}\\[0.1cm]
-      Downstream task solving
+      solving tasks
     };
 
     % Reward
-    \node[box, text width=2.8cm, right=2.0cm of manager.east] (rewardbox) {
+    \node[box, text width=2.8cm, right=3.0cm of manager.east] (rewardbox) {
       \textbf{Task Feedback}\\[0.1cm]
-      Reward $R$
+    computing rewards
     };
 
     % Dashed container for memory manager
@@ -58,11 +58,11 @@
 
     % Arrows
     \draw[arrow] (extracted) -- node[xshift=-.2cm,above,font=\scriptsize] {input} (manager);
-    \draw[arrow] (manager) -- node[xshift=0.1cm,center,font=\scriptsize] {sampling operations} (bank);
-    \draw[arrow] ([yshift=0.2cm]bank.east) -- node[above,font=\scriptsize] {retrieve} ([yshift=0.2cm]agent.west);
-    \draw[arrow] ([yshift=-0.2cm]agent.west) -- node[below,font=\scriptsize,align=right] {updated\\ memory} ([yshift=-0.2cm]bank.east);
-    \draw[arrow] (agent.north) -- node[right,font=\scriptsize] {feedback} (rewardbox.south);
-    \draw[arrow] (rewardbox.west) -- node[xshift=0.2cm,font=\scriptsize,align=center] {RL \\optimization} (manager.east);
+    \draw[arrow] (manager) -- node[xshift=0.1cm,align=center,font=\scriptsize] {sampling operations} (bank);
+    \draw[arrow] ([yshift=0.2cm]bank.east) -- node[midway,above,font=\scriptsize,text width=1.7cm,align=center] {retrieving} ([yshift=0.2cm]agent.west);
+    \draw[arrow] ([yshift=-0.2cm]agent.west) -- node[midway,below,font=\scriptsize,text width=1.7cm,align=center] {useful memory} ([yshift=-0.2cm]bank.east);
+    \draw[arrow] (agent.north) -- node[right,font=\scriptsize] { trajectory} (rewardbox.south);
+    \draw[arrow] (rewardbox.west) -- node[midway,above,yshift=0.08cm,inner sep=1pt,font=\scriptsize,text width=1.7cm,align=center] {RL training} (manager.east);
 
   \end{tikzpicture}  
 \end{center}

section6/section6.tex

@@ -342,7 +342,7 @@ Memory management is a direct way for agents to learn from agentic experience. D
     \centering
     \input{section6/Figures/memory-and-retrieval.tex}
     \caption{
-    An overview of the memory system.
+    An overview of a memory system for agentic interaction. We take a travel-planning scenario as an example, where the user asks the agent to plan a trip to Kyoto. The agent first retrieves relevant user preferences from the memory bank, such as vegetarian meals, window seats, and budget hotels, and uses them to support the current conversation. During the interaction, the user provides new information, such as a preference for quiet ryokans near Kyoto Station. This new information is extracted as memory and passed to a memory manager, which updates the memory bank by selecting an operation such as \texttt{ADD}. Blue highlights indicate retrieved memories, while red highlights indicate newly extracted memories and their update path back to the memory bank.
     }
     \label{fig:memory-and-retrieve}
 \end{figure}