page of beam search

Section06-Neural-Machine-Translation/section06-test.tex

@@ -145,174 +145,165 @@
 \subsection{注意力机制}
 
 %%%------------------------------------------------------------------------------------------------------------
-%%% 解码
-\begin{frame}{推断}
+%%% 解码 - beam search
+\begin{frame}{推断 - Beam Search}
 \begin{itemize}
-\item 使用NMT时，对于源语言句子$\textbf{x}$，需要得到最优译文$\hat{\textbf{y}}$
-
-\vspace{-1.5em}
-\begin{displaymath}
-\hat{\textbf{y}} = \argmax_{\textbf{y}} \log\textrm{P}(\textbf{y}|\textbf{x}) = \argmax_{\textbf{y}} \sum_{j=1}^{n} \log\textrm{P}(y_j|\textbf{y}_{<j}, \textbf{x})
-\end{displaymath}
-
-\item<2-> 由于$y_i$的生成需要依赖$y_{i-1}$，因此无法同时生成$\{y_1,...,y_n\}$。常用的方法是自左向右逐个单词生成
-
+\item \textbf{Greedy Search}: 目标语每一个位置，输出层的Softmax可以得到所有单词的概率，然后选择一个概率最大单词输出，下一个位置的预测就基于这一步输出的单词
+\item \textbf{Beach Search}: 为了避免贪婪方法造成的错误累加，可以每次对$b$个单词进行扩展，而不是只使用一个单词，其中$b$称做束的宽度 - 这样可以搜索更多可能的译文
 \end{itemize}
 
-\visible<3->{
-\vspace{-0.5em}
+\vspace{-0.3em}
+\visible<2->{
 \begin{center}
 \begin{tikzpicture}
 \begin{scope}
-\tikzstyle{rnnnode} = [minimum height=1.1em,minimum width=2.1em,inner sep=2pt,rounded corners=1pt,draw,fill=red!20];
-
-\node [rnnnode,anchor=west] (h1) at (0,0) {\tiny{$\textbf{h}_1$}};
-\node [anchor=west] (h2) at ([xshift=1em]h1.east) {\tiny{...}};
-\node [rnnnode,anchor=west] (h3) at ([xshift=1em]h2.east) {\tiny{$\textbf{h}_m$}};
-\node [rnnnode,anchor=north,fill=green!20] (e1) at ([yshift=-1em]h1.south) {\tiny{$e_x()$}};
-\node [anchor=west] (e2) at ([xshift=1em]e1.east) {\tiny{...}};
-\node [rnnnode,anchor=west,fill=green!20] (e3) at ([xshift=1em]e2.east) {\tiny{$e_x()$}};
-\node [anchor=north,inner sep=2pt] (w1) at ([yshift=-0.6em]e1.south) {\tiny{你}};
-\node [anchor=north,inner sep=2pt] (w2) at ([yshift=-0.8em]e2.south) {\tiny{...}};
-\node [anchor=north,inner sep=2pt] (w3) at ([yshift=-0.6em]e3.south) {\tiny{EOS}};
-
-\draw [->] (w1.north) -- ([yshift=-0.1em]e1.south);
-\draw [->] (w3.north) -- ([yshift=-0.1em]e3.south);
-\draw [->] ([yshift=0.1em]e1.north) -- ([yshift=-0.1em]h1.south);
-\draw [->] ([yshift=0.1em]e3.north) -- ([yshift=-0.1em]h3.south);
-\draw [->] ([xshift=0.1em]h1.east) -- ([xshift=-0.1em]h2.west);
-\draw [->] ([xshift=0.1em]h2.east) -- ([xshift=-0.1em]h3.west);
-\draw [->] ([xshift=-0.8em]h1.west) -- ([xshift=-0.1em]h1.west) node [pos=0,left,inner sep=2pt] {\tiny{0}};
-\node [anchor=south] (encoder) at ([xshift=-0.2em]h1.north west) {\scriptsize{\textbf{编码器}}};
+\tikzstyle{rnnnode} = [minimum height=1.1em,minimum width=3.5em,inner sep=2pt,rounded corners=1pt,draw,fill=red!20];
+\tikzstyle{wnode} = [minimum height=1.0em,minimum width=3.5em,inner sep=2pt,rounded corners=1pt,draw,fill=white];
 
-\visible<5->{
-\node [rnnnode,anchor=west,fill=green!20] (t1) at ([xshift=3em]h3.east) {\tiny{$e_y()$}};
+
+\visible<3->{
+\node [rnnnode,anchor=west,fill=green!20] (t1) at (0,0) {\tiny{$e_y()$}};
 }
-\visible<8->{
-\node [rnnnode,anchor=west,fill=green!20] (t2) at ([xshift=1.5em]t1.east) {\tiny{$e_y()$}};
+\visible<7->{
+\node [rnnnode,anchor=west,fill=green!20] (t2) at ([xshift=2.2em]t1.east) {\tiny{$e_y()$ ($\times 3$)}};
 }
-\visible<9->{
-\node [rnnnode,anchor=west,fill=green!20] (t3) at ([xshift=1.5em]t2.east) {\tiny{$e_y()$}};
-\node [rnnnode,anchor=west,fill=green!20] (t4) at ([xshift=1.5em]t3.east) {\tiny{$e_y()$}};
-\node [anchor=west,inner sep=2pt] (t5) at ([xshift=0.3em]t4.east) {\tiny{...}};
+\visible<8->{
+\node [rnnnode,anchor=west,fill=green!20] (t3) at ([xshift=2.2em]t2.east) {\tiny{$e_y()$ ($\times 3$)}};
+\node [anchor=west,inner sep=2pt] (t4) at ([xshift=0.3em]t3.east) {\tiny{...}};
 }
-\visible<5->{
+\visible<3->{
 \node [rnnnode,anchor=south] (s1) at ([yshift=1em]t1.north) {\tiny{$\textbf{s}_1$}};
 }
-\visible<8->{
-\node [rnnnode,anchor=south] (s2) at ([yshift=1em]t2.north) {\tiny{$\textbf{s}_2$}};
+\visible<7->{
+\node [rnnnode,anchor=south] (s2) at ([yshift=1em]t2.north) {\tiny{$\textbf{s}_2$ ($\times 3$)}};
 }
-\visible<9->{
-\node [rnnnode,anchor=south] (s3) at ([yshift=1em]t3.north) {\tiny{$\textbf{s}_3$}};
-\node [rnnnode,anchor=south] (s4) at ([yshift=1em]t4.north) {\tiny{$\textbf{s}_4$}};
-\node [anchor=west,inner sep=2pt] (s5) at ([xshift=0.3em]s4.east) {\tiny{...}};
+\visible<8->{
+\node [rnnnode,anchor=south] (s3) at ([yshift=1em]t3.north) {\tiny{$\textbf{s}_3$ ($\times 3$)}};
+\node [anchor=west,inner sep=2pt] (s4) at ([xshift=0.3em]s3.east) {\tiny{...}};
 }
-\visible<5->{
+\visible<3->{
 \node [rnnnode,anchor=south,fill=blue!20] (o1) at ([yshift=1em]s1.north) {\tiny{softmax}};
-\node [anchor=east] (decoder) at ([xshift=-0.5em]o1.north west) {\scriptsize{\textbf{解码器}}};
-}
-\visible<8->{
-\node [rnnnode,anchor=south,fill=blue!20] (o2) at ([yshift=1em]s2.north) {\tiny{softmax}};
 }
-\visible<9->{
-\node [rnnnode,anchor=south,fill=blue!20] (o3) at ([yshift=1em]s3.north) {\tiny{softmax}};
-\node [rnnnode,anchor=south,fill=blue!20] (o4) at ([yshift=1em]s4.north) {\tiny{softmax}};
-\node [anchor=west,inner sep=2pt] (o5) at ([xshift=0.3em]o4.east) {\tiny{...}};
+\visible<7->{
+\node [rnnnode,anchor=south,fill=blue!20] (o2) at ([yshift=1em]s2.north) {\tiny{softmax ($\times 3$)}};
 }
-\visible<4->{
-\node [anchor=north,inner sep=2pt] (wt1) at ([yshift=-0.6em]t1.south) {\tiny{EOS}};
+\visible<8->{
+\node [rnnnode,anchor=south,fill=blue!20] (o3) at ([yshift=1em]s3.north) {\tiny{softmax ($\times 3$)}};
+\node [anchor=west,inner sep=2pt] (o4) at ([xshift=0.3em]o3.east) {\tiny{...}};
 }
-\visible<7->{
-\node [anchor=north,inner sep=2pt] (wt2) at ([yshift=-0.6em]t2.south) {\tiny{Have}};
+
+\node [wnode,anchor=north] (wt1) at ([yshift=-0.8em]t1.south) {\tiny{EOS}};
+
+\visible<6->{
+\node [wnode,anchor=north] (wt2) at ([yshift=-0.8em]t2.south) {\tiny{Have}};
+\node [wnode,anchor=north] (wt2copy1) at ([xshift=-0.2em,yshift=-0.2em]wt2.north) {\tiny{Have}};
+\node [wnode,anchor=north] (wt2copy2) at ([xshift=-0.4em,yshift=-0.4em]wt2.north) {\tiny{Have}};
 }
-\visible<9->{
-\node [anchor=north,inner sep=2pt] (wt3) at ([yshift=-0.8em]t3.south) {\tiny{you}};
-\node [anchor=north,inner sep=2pt] (wt4) at ([yshift=-0.6em]t4.south) {\tiny{learned}};
+
+\visible<8->{
+\node [wnode,anchor=north,inner sep=2pt] (wt3) at ([yshift=-0.8em]t3.south) {\tiny{you}};
+\node [wnode,anchor=north] (wt3copy1) at ([xshift=-0.2em,yshift=-0.2em]wt3.north) {\tiny{you}};
+\node [wnode,anchor=north] (wt3copy2) at ([xshift=-0.4em,yshift=-0.4em]wt3.north) {\tiny{you}};
 }
+
 \visible<5->{
-\node [anchor=center,inner sep=2pt] (wo1) at ([yshift=1.2em]o1.north) {\tiny{Have}};
-\node [anchor=south,inner sep=2pt] (wos1) at (wo1.north) {\tiny{\textbf{[step 1]}}};
+\node [wnode,anchor=center,inner sep=2pt] (wo1) at ([xshift=0.4em,yshift=1.8em]o1.north) {\tiny{Have}};
+\node [wnode,anchor=north] (wo1copy1) at ([xshift=-0.2em,yshift=-0.2em]wo1.north) {\tiny{Have}};
+\node [wnode,anchor=north] (wo1copy2) at ([xshift=-0.4em,yshift=-0.4em]wo1.north) {\tiny{Have}};
 }
+
 \visible<8->{
-\node [anchor=center,inner sep=2pt] (wo2) at ([yshift=1.2em]o2.north) {\tiny{you}};
-\node [anchor=south,inner sep=2pt] (wos2) at (wo2.north) {\tiny{\textbf{[step 2]}}};
+\node [wnode,anchor=center,inner sep=2pt] (wo2) at ([xshift=0.4em,yshift=1.8em]o2.north) {\tiny{you}};
+\node [wnode,anchor=north] (wo2copy1) at ([xshift=-0.2em,yshift=-0.2em]wo2.north) {\tiny{you}};
+\node [wnode,anchor=north] (wo2copy2) at ([xshift=-0.4em,yshift=-0.4em]wo2.north) {\tiny{you}};
 }
-\visible<9->{
-\node [anchor=center,inner sep=2pt] (wo3) at ([yshift=1.2em]o3.north) {\tiny{learned}};
-\node [anchor=south,inner sep=2pt] (wos3) at (wo3.north) {\tiny{\textbf{[step 3]}}};
-\node [anchor=center,inner sep=2pt] (wo4) at ([yshift=1.2em]o4.north) {\tiny{nothing}};
-\node [anchor=south,inner sep=2pt] (wos4) at (wo4.north) {\tiny{\textbf{[step 4]}}};
+
+\visible<8->{
+\node [wnode,anchor=center,inner sep=2pt] (wo3) at ([xshift=0.4em,yshift=1.8em]o3.north) {\tiny{learned}};
+\node [wnode,anchor=north] (wo3copy1) at ([xshift=-0.2em,yshift=-0.2em]wo3.north) {\tiny{learned}};
+\node [wnode,anchor=north] (wo3copy2) at ([xshift=-0.4em,yshift=-0.4em]wo3.north) {\tiny{learned}};
 }
 
-\visible<5->{
+\visible<3->{
 \foreach \x in {1}{
     \draw [->] ([yshift=-0.7em]t\x.south) -- ([yshift=-0.1em]t\x.south);
     \draw [->] ([yshift=0.1em]t\x.north) -- ([yshift=-0.1em]s\x.south);
     \draw [->] ([yshift=0.1em]s\x.north) -- ([yshift=-0.1em]o\x.south);
-    \draw [->] ([yshift=0.1em]o\x.north) -- ([yshift=0.8em]o\x.north) node [pos=0.5,right] {\tiny{top1}};
+   
 }
 }
 
-\visible<8->{
+\visible<5->{
+ \draw [->] ([yshift=0.1em]o1.north) -- ([yshift=0.8em]o1.north) node [pos=0.5,right] {\tiny{top-3}};
+ }
+
+\visible<7->{
 \foreach \x in {2}{
     \draw [->] ([yshift=-0.7em]t\x.south) -- ([yshift=-0.1em]t\x.south);
     \draw [->] ([yshift=0.1em]t\x.north) -- ([yshift=-0.1em]s\x.south);
     \draw [->] ([yshift=0.1em]s\x.north) -- ([yshift=-0.1em]o\x.south);
-    \draw [->] ([yshift=0.1em]o\x.north) -- ([yshift=0.8em]o\x.north) node [pos=0.5,right] {\tiny{top1}};
+    \draw [->] ([yshift=0.1em]o\x.north) -- ([yshift=0.8em]o\x.north) node [pos=0.5,right] {\tiny{top-3}};
 }
 }
 
-\visible<9->{
-\foreach \x in {3,4}{
+\visible<8->{
+\foreach \x in {3}{
     \draw [->] ([yshift=-0.7em]t\x.south) -- ([yshift=-0.1em]t\x.south);
     \draw [->] ([yshift=0.1em]t\x.north) -- ([yshift=-0.1em]s\x.south);
     \draw [->] ([yshift=0.1em]s\x.north) -- ([yshift=-0.1em]o\x.south);
-    \draw [->] ([yshift=0.1em]o\x.north) -- ([yshift=0.8em]o\x.north) node [pos=0.5,right] {\tiny{top1}};
+    \draw [->] ([yshift=0.1em]o\x.north) -- ([yshift=0.8em]o\x.north) node [pos=0.5,right] {\tiny{top-3}};
 }
 }
 
-\visible<5->{
-\draw [->] ([xshift=-0.8em]s1.west) -- ([xshift=-0.1em]s1.west) node [pos=0,left] {\tiny{0}};
+\visible<3->{
+\draw [->] ([xshift=-0.5em]s1.west) -- ([xshift=-0.1em]s1.west) node [pos=0,left,inner sep=1pt] {\tiny{0}};
 }
-\visible<8->{
+\visible<7->{
 \draw [->] ([xshift=0.1em]s1.east) -- ([xshift=-0.1em]s2.west);
 }
-\visible<9->{
+\visible<8->{
 \draw [->] ([xshift=0.1em]s2.east) -- ([xshift=-0.1em]s3.west);
-\draw [->] ([xshift=0.1em]s3.east) -- ([xshift=-0.1em]s4.west);
 }
 
-\visible<7->{
-\draw [->,thick,dotted] (wo1.east) .. controls +(east:1.1) and +(west:1.0) ..(wt2.west);
+\visible<6->{
+\draw [->,very thick,dotted] (wo1.east) .. controls +(east:0.6) and +(west:0.8) ..(wt2copy2.west);
 }
-\visible<9->{
-\draw [->,thick,dotted] (wo2.east) .. controls +(east:1.3) and +(west:1.1) ..(wt3.west);
-\draw [->,thick,dotted] (wo3.east) .. controls +(east:0.9) and +(west:0.9) ..(wt4.west);
+\visible<8->{
+\draw [->,very thick,dotted] (wo2.east) .. controls +(east:0.6) and +(west:0.8) ..(wt3copy2.west);
 }
 
-\visible<6->{
-\node [circle,draw,anchor=south,inner sep=3pt,fill=orange!20] (c2) at ([yshift=2em]h2.north) {\tiny{$\textbf{C}_2$}};
-\node [anchor=south] (c2more) at ([yshift=-1.5em]c2.south) {\tiny{...}};
-\draw [->] (h1.north) .. controls +(north:0.6) and +(250:0.9) .. (c2.250);
-\draw [->] (h3.north) .. controls +(north:0.6) and +(290:0.9) .. (c2.290);
-\draw [->] ([yshift=-0.3em]s1.west) .. controls +(west:2) and +(-50:0.3) .. (c2.-40);
+\visible<7->{
+\node [circle,draw,anchor=north,inner sep=2pt,fill=orange!20] (c2) at ([yshift=-2.5em]t1.south) {\tiny{$\textbf{C}_2$}};
+\node [circle,draw,inner sep=2pt,fill=orange!20] (c2copy1) at ([yshift=-0.1em,xshift=-0.1em]c2) {\tiny{$\textbf{C}_2$}};
+\node [circle,draw,inner sep=2pt,fill=orange!20] (c2copy2) at ([yshift=-0.2em,xshift=-0.2em]c2) {\tiny{$\textbf{C}_2$}};
+\draw [->] ([xshift=-0.9em]c2.west) -- ([xshift=-0.3em]c2.west);
+\draw [->] ([xshift=0.1em]c2.east) .. controls +(east:1.5) and +(west:0.8) ..([yshift=-0.3em,xshift=-0.1em]s2.west);
 }
+
 \visible<8->{
-\draw [->] (c2.0) -- ([xshift=1.358in]c2.0) -- ([yshift=0.3em,xshift=-1.2em]s2.west) -- ([yshift=0.3em,xshift=-0.1em]s2.west);
+\node [circle,draw,anchor=north,inner sep=2pt,fill=orange!20] (c3) at ([yshift=-2.5em]t2.south) {\tiny{$\textbf{C}_3$}};
+\node [circle,draw,inner sep=2pt,fill=orange!20] (c3copy1) at ([yshift=-0.1em,xshift=-0.1em]c3) {\tiny{$\textbf{C}_3$}};
+\node [circle,draw,inner sep=2pt,fill=orange!20] (c3copy2) at ([yshift=-0.2em,xshift=-0.2em]c3) {\tiny{$\textbf{C}_3$}};
+\draw [->] ([xshift=-0.9em]c3.west) -- ([xshift=-0.3em]c3.west);
+\draw [->] ([xshift=0.1em]c3.east) .. controls +(east:1.5) and +(west:0.8) ..([yshift=-0.3em,xshift=-0.1em]s3.west);
 }
 
-\visible<9->{
-\node [circle,draw,anchor=north,inner sep=3pt,fill=orange!20] (c3) at ([yshift=-2em]t2.south) {\tiny{$\textbf{C}_3$}};
-\draw [->] ([xshift=-0.7em]c3.west) -- ([xshift=-0.1em]c3.west);
-\draw [->] ([xshift=0.1em]c3.east) .. controls +(east:0.6) and +(west:0.8) ..([yshift=-0.3em,xshift=-0.1em]s3.west);
+\visible<3->{
+\node [anchor=east] (vocab) at ([xshift=-5em]s1.west) {\tiny{$\begin{bmatrix} \textrm{Have} & 0.50 \\ \textrm{I} & 0.02 \\ \textrm{it} & 0.03 \\ \textrm{has} & 0.30 \\ \textrm{you} & 0.01 \\ \textrm{the} & 0.01 \\ \textrm{a} & 0.01 \\ \textrm{an} & 0.02 \\ \textrm{he} & 0.03 \\ \textrm{she} & 0.01 \\ \textrm{are} & 0.00 \\ \textrm{am} & 0.01 \\ ... & ... \end{bmatrix}$}};
+\node [anchor=south] (vocablabel) at (vocab.north) {\tiny{单词的概率分布}};
+\draw [->,red,very thick,dotted] (o1.west) .. controls +(west:1) and +(east:2) .. ([yshift=1em]vocab.south east); 
 }
 
-\visible<9->{
-\node [circle,draw,anchor=north,inner sep=3pt,fill=orange!20] (c4) at ([yshift=-2em]t3.south) {\tiny{$\textbf{C}_4$}};
-\draw [->] ([xshift=-0.7em]c4.west) -- ([xshift=-0.1em]c4.west);
-\draw [->] ([xshift=0.1em]c4.east) .. controls +(east:0.6) and +(west:0.8) ..([yshift=-0.3em,xshift=-0.1em]s4.west);
+\visible<4->{
+\node [anchor=east,inner sep=1pt] (vocabtopn) at ([xshift=-0.5em,yshift=-0.5em]wo1.west) {\tiny{$\begin{bmatrix} \textrm{Have} \\ \textrm{has} \\ \textrm{it} \end{bmatrix}$}};
+\draw [->] ([yshift=-1.6em,xshift=-0.4em]vocab.north east) .. controls +(east:1) and +(west:1) ..  ([xshift=0.1em,yshift=0.4em]vocabtopn.west) node [pos=0.3,below] (topnlabel) {\tiny{top-3}};
+
+\visible<4->{
+\node [anchor=north] (cap) at (vocab.south east) {\scriptsize{\textbf{束搜索($b=3$)}}};
+}
 }
 
+
 \end{scope}
 \end{tikzpicture}
 \end{center}

Section06-Neural-Machine-Translation/section06.tex

@@ -508,12 +508,12 @@ NLP问题的隐含结构假设 & 无隐含结构假设，端到端学习 \\
                 
                 \node[rnnnode,fill=blue!30!white,right=\base of rnn3] (rnn4) {};
                 \node[rnnnode,fill=green!30!white,below=\base of rnn4] (emb4) {};
-                \node[wordnode,below=0pt of emb4] (word4) {$\langle$eos$\rangle$};
+                \node[wordnode,below=0pt of emb4] (word4) {EOS};
                 \draw[-latex'] (emb4.north) to (rnn4.south);
                 \draw[-latex'] (rnn3.east) to (rnn4.west);
             }
             \visible<4->{
-                \draw[decoration={mirror,brace},decorate] (word1.south west) to node [auto,anchor=north,align=center] {编码器} (word4.south east);
+                \draw[decoration={mirror,brace},decorate] (word1.south west) to node [auto,anchor=north,align=center] {编码器} ([yshift=-0.2em]word4.south east);
             }
             \visible<5->{
                 \node[rnnnode,fill=purple] (repr) at (rnn4) {};
@@ -523,7 +523,7 @@ NLP问题的隐含结构假设 & 无隐含结构假设，端到端学习 \\
             \visible<6->{
                 \node[rnnnode,fill=blue!30!white,right=\base of rnn4] (rnn5) {};
                 \node[rnnnode,fill=green!30!white,below=\base of rnn5] (emb5) {};
-                \node[wordnode,below=0pt of emb5] (word5) {$\langle$sos$\rangle$};
+                \node[wordnode,below=0pt of emb5] (word5) {SOS};
                 \draw[-latex'] (emb5.north) to (rnn5.south);
                 \draw[-latex'] (rnn4.east) to (rnn5.west);
                 \node[rnnnode,fill=red!30!white,above=\base of rnn5] (softmax1) {};
@@ -566,7 +566,7 @@ NLP问题的隐含结构假设 & 无隐含结构假设，端到端学习 \\
                 \node[wordnode,anchor=base] (word8) at (\XCoord,\YCoord) {fine};
                 \ExtractX{$(emb8)$}
                 \ExtractY{$(out1.base)$}
-                \node[wordnode,anchor=base] (out4) at (\XCoord,\YCoord) {$\langle$eos$\rangle$};
+                \node[wordnode,anchor=base] (out4) at (\XCoord,\YCoord) {EOS};
                 \draw[-latex'] (emb8.north) to (rnn8.south);
                 \draw[-latex'] (rnn7.east) to (rnn8.west);
                 \draw[-latex'] (rnn8.north) to (softmax4.south);
@@ -574,7 +574,7 @@ NLP问题的隐含结构假设 & 无隐含结构假设，端到端学习 \\
             \visible<9->{
                 \ExtractX{$(word8.east)$}
                 \ExtractY{$(word5.south)$}
-                \draw[decoration={mirror,brace},decorate] (word5.south west) to node [auto,anchor=north,align=center] {解码器} (\XCoord,\YCoord);
+                \draw[decoration={mirror,brace},decorate] ([yshift=-0.2em]word5.south west) to node [auto,anchor=north,align=center] {解码器} (\XCoord,\YCoord-0.2em);
             }
         \end{scope}
     \end{tikzpicture}
@@ -1218,7 +1218,7 @@ NLP问题的隐含结构假设 & 无隐含结构假设，端到端学习 \\
                 \node[rnnnode,minimum height=0.5\base,fill=red!30!white,anchor=south] (softmax\x) at ([yshift=0.5\base]dec\x.north) {};
 
             % Decoder input words
-            \node[wordnode,below=0pt of demb1] (decwordin) {$\langle$sos$\rangle$};
+            \node[wordnode,below=0pt of demb1] (decwordin) {EOS};
             \ExtractX{$(demb2.south)$}
             \ExtractY{$(decwordin.base)$}
             \node[wordnode,anchor=base] () at (\XCoord,\YCoord) {Do};
@@ -1275,7 +1275,7 @@ NLP问题的隐含结构假设 & 无隐含结构假设，端到端学习 \\
             \node[wordnode,anchor=base] () at (\XCoord,\YCoord) {Station};
             \ExtractX{$(softmax10.north)$}
             \ExtractY{$(decwordout.base)$}
-            \node[wordnode,anchor=base] () at (\XCoord,\YCoord) {$\langle$eos$\rangle$};
+            \node[wordnode,anchor=base] () at (\XCoord,\YCoord) {EOS};
 
             % Connections
             \draw[-latex'] (init.east) to (enc1.west);
@@ -1356,7 +1356,7 @@ NLP问题的隐含结构假设 & 无隐含结构假设，端到端学习 \\
             \node[wordnode,below=0pt of eemb7] () {怎么};
             \node[wordnode,below=0pt of eemb8] () {走};
             \node[wordnode,below=0pt of eemb9] () {吗};
-            \node[wordnode,below=0pt of eemb10] () {$\langle$eos$\rangle$};
+            \node[wordnode,below=0pt of eemb10] () {EOS};
 
             % RNN Decoder
             \foreach \x in {1,2,...,10}
@@ -1426,7 +1426,7 @@ NLP问题的隐含结构假设 & 无隐含结构假设，端到端学习 \\
             \node[wordnode,anchor=base] () at (\XCoord,\YCoord) {Station};
             \ExtractX{$(softmax10.north)$}
             \ExtractY{$(decwordout.base)$}
-            \node[wordnode,anchor=base] () at (\XCoord,\YCoord) {$\langle$eos$\rangle$};
+            \node[wordnode,anchor=base] () at (\XCoord,\YCoord) {EOS};
 
             % Connections
             \draw[-latex'] (init1.east) to (enc11.west);
@@ -2091,8 +2091,351 @@ $\textrm{``you''} = \argmax_{y} \textrm{P}(y|\textbf{s}_1, \alert{\textbf{C}})$
 %%%------------------------------------------------------------------------------------------------------------
 %%% 解码
 \begin{frame}{推断}
+\begin{itemize}
+\item 使用NMT时，对于源语言句子$\textbf{x}$，需要得到最优译文$\hat{\textbf{y}}$
+
+\vspace{-1.5em}
+\begin{displaymath}
+\hat{\textbf{y}} = \argmax_{\textbf{y}} \log\textrm{P}(\textbf{y}|\textbf{x}) = \argmax_{\textbf{y}} \sum_{j=1}^{n} \log\textrm{P}(y_j|\textbf{y}_{<j}, \textbf{x})
+\end{displaymath}
+
+\item<2-> 由于$y_i$的生成需要依赖$y_{i-1}$，因此无法同时生成$\{y_1,...,y_n\}$。常用的方法是自左向右逐个单词生成
+
+\end{itemize}
+
+\vspace{-0.8em}
+\visible<3->{
+\vspace{-0.5em}
+\begin{center}
+\begin{tikzpicture}
+\begin{scope}
+\tikzstyle{rnnnode} = [minimum height=1.1em,minimum width=2.1em,inner sep=2pt,rounded corners=1pt,draw,fill=red!20];
+
+\node [rnnnode,anchor=west] (h1) at (0,0) {\tiny{$\textbf{h}_1$}};
+\node [anchor=west] (h2) at ([xshift=1em]h1.east) {\tiny{...}};
+\node [rnnnode,anchor=west] (h3) at ([xshift=1em]h2.east) {\tiny{$\textbf{h}_m$}};
+\node [rnnnode,anchor=north,fill=green!20] (e1) at ([yshift=-1em]h1.south) {\tiny{$e_x()$}};
+\node [anchor=west] (e2) at ([xshift=1em]e1.east) {\tiny{...}};
+\node [rnnnode,anchor=west,fill=green!20] (e3) at ([xshift=1em]e2.east) {\tiny{$e_x()$}};
+\node [anchor=north,inner sep=2pt] (w1) at ([yshift=-0.6em]e1.south) {\tiny{你}};
+\node [anchor=north,inner sep=2pt] (w2) at ([yshift=-0.8em]e2.south) {\tiny{...}};
+\node [anchor=north,inner sep=2pt] (w3) at ([yshift=-0.6em]e3.south) {\tiny{EOS}};
+
+\draw [->] (w1.north) -- ([yshift=-0.1em]e1.south);
+\draw [->] (w3.north) -- ([yshift=-0.1em]e3.south);
+\draw [->] ([yshift=0.1em]e1.north) -- ([yshift=-0.1em]h1.south);
+\draw [->] ([yshift=0.1em]e3.north) -- ([yshift=-0.1em]h3.south);
+\draw [->] ([xshift=0.1em]h1.east) -- ([xshift=-0.1em]h2.west);
+\draw [->] ([xshift=0.1em]h2.east) -- ([xshift=-0.1em]h3.west);
+\draw [->] ([xshift=-0.8em]h1.west) -- ([xshift=-0.1em]h1.west) node [pos=0,left,inner sep=2pt] {\tiny{0}};
+\node [anchor=south] (encoder) at ([xshift=-0.2em]h1.north west) {\scriptsize{\textbf{编码器}}};
+
+\visible<5->{
+\node [rnnnode,anchor=west,fill=green!20] (t1) at ([xshift=3em]h3.east) {\tiny{$e_y()$}};
+}
+\visible<8->{
+\node [rnnnode,anchor=west,fill=green!20] (t2) at ([xshift=1.5em]t1.east) {\tiny{$e_y()$}};
+}
+\visible<9->{
+\node [rnnnode,anchor=west,fill=green!20] (t3) at ([xshift=1.5em]t2.east) {\tiny{$e_y()$}};
+\node [rnnnode,anchor=west,fill=green!20] (t4) at ([xshift=1.5em]t3.east) {\tiny{$e_y()$}};
+\node [anchor=west,inner sep=2pt] (t5) at ([xshift=0.3em]t4.east) {\tiny{...}};
+}
+\visible<5->{
+\node [rnnnode,anchor=south] (s1) at ([yshift=1em]t1.north) {\tiny{$\textbf{s}_1$}};
+}
+\visible<8->{
+\node [rnnnode,anchor=south] (s2) at ([yshift=1em]t2.north) {\tiny{$\textbf{s}_2$}};
+}
+\visible<9->{
+\node [rnnnode,anchor=south] (s3) at ([yshift=1em]t3.north) {\tiny{$\textbf{s}_3$}};
+\node [rnnnode,anchor=south] (s4) at ([yshift=1em]t4.north) {\tiny{$\textbf{s}_4$}};
+\node [anchor=west,inner sep=2pt] (s5) at ([xshift=0.3em]s4.east) {\tiny{...}};
+}
+\visible<5->{
+\node [rnnnode,anchor=south,fill=blue!20] (o1) at ([yshift=1em]s1.north) {\tiny{softmax}};
+\node [anchor=east] (decoder) at ([xshift=-0.5em]o1.north west) {\scriptsize{\textbf{解码器}}};
+}
+\visible<8->{
+\node [rnnnode,anchor=south,fill=blue!20] (o2) at ([yshift=1em]s2.north) {\tiny{softmax}};
+}
+\visible<9->{
+\node [rnnnode,anchor=south,fill=blue!20] (o3) at ([yshift=1em]s3.north) {\tiny{softmax}};
+\node [rnnnode,anchor=south,fill=blue!20] (o4) at ([yshift=1em]s4.north) {\tiny{softmax}};
+\node [anchor=west,inner sep=2pt] (o5) at ([xshift=0.3em]o4.east) {\tiny{...}};
+}
+\visible<4->{
+\node [anchor=north,inner sep=2pt] (wt1) at ([yshift=-0.6em]t1.south) {\tiny{EOS}};
+}
+\visible<7->{
+\node [anchor=north,inner sep=2pt] (wt2) at ([yshift=-0.6em]t2.south) {\tiny{Have}};
+}
+\visible<9->{
+\node [anchor=north,inner sep=2pt] (wt3) at ([yshift=-0.8em]t3.south) {\tiny{you}};
+\node [anchor=north,inner sep=2pt] (wt4) at ([yshift=-0.6em]t4.south) {\tiny{learned}};
+}
+\visible<5->{
+\node [anchor=center,inner sep=2pt] (wo1) at ([yshift=1.2em]o1.north) {\tiny{Have}};
+}
+\visible<4->{
+\node [anchor=south,inner sep=2pt] (wos1) at (wo1.north) {\tiny{\textbf{[step 1]}}};
+}
+\visible<8->{
+\node [anchor=center,inner sep=2pt] (wo2) at ([yshift=1.2em]o2.north) {\tiny{you}};
+}
+\visible<7->{
+\node [anchor=south,inner sep=2pt] (wos2) at (wo2.north) {\tiny{\textbf{[step 2]}}};
+}
+\visible<9->{
+\node [anchor=center,inner sep=2pt] (wo3) at ([yshift=1.2em]o3.north) {\tiny{learned}};
+\node [anchor=south,inner sep=2pt] (wos3) at (wo3.north) {\tiny{\textbf{[step 3]}}};
+\node [anchor=center,inner sep=2pt] (wo4) at ([yshift=1.2em]o4.north) {\tiny{nothing}};
+\node [anchor=south,inner sep=2pt] (wos4) at (wo4.north) {\tiny{\textbf{[step 4]}}};
+}
+
+\visible<5->{
+\foreach \x in {1}{
+    \draw [->] ([yshift=-0.7em]t\x.south) -- ([yshift=-0.1em]t\x.south);
+    \draw [->] ([yshift=0.1em]t\x.north) -- ([yshift=-0.1em]s\x.south);
+    \draw [->] ([yshift=0.1em]s\x.north) -- ([yshift=-0.1em]o\x.south);
+    \draw [->] ([yshift=0.1em]o\x.north) -- ([yshift=0.8em]o\x.north) node [pos=0.5,right] {\tiny{top1}};
+}
+}
+
+\visible<8->{
+\foreach \x in {2}{
+    \draw [->] ([yshift=-0.7em]t\x.south) -- ([yshift=-0.1em]t\x.south);
+    \draw [->] ([yshift=0.1em]t\x.north) -- ([yshift=-0.1em]s\x.south);
+    \draw [->] ([yshift=0.1em]s\x.north) -- ([yshift=-0.1em]o\x.south);
+    \draw [->] ([yshift=0.1em]o\x.north) -- ([yshift=0.8em]o\x.north) node [pos=0.5,right] {\tiny{top1}};
+}
+}
+
+\visible<9->{
+\foreach \x in {3,4}{
+    \draw [->] ([yshift=-0.7em]t\x.south) -- ([yshift=-0.1em]t\x.south);
+    \draw [->] ([yshift=0.1em]t\x.north) -- ([yshift=-0.1em]s\x.south);
+    \draw [->] ([yshift=0.1em]s\x.north) -- ([yshift=-0.1em]o\x.south);
+    \draw [->] ([yshift=0.1em]o\x.north) -- ([yshift=0.8em]o\x.north) node [pos=0.5,right] {\tiny{top1}};
+}
+}
+
+\visible<5->{
+\draw [->] ([xshift=-0.8em]s1.west) -- ([xshift=-0.1em]s1.west) node [pos=0,left] {\tiny{0}};
+}
+\visible<8->{
+\draw [->] ([xshift=0.1em]s1.east) -- ([xshift=-0.1em]s2.west);
+}
+\visible<9->{
+\draw [->] ([xshift=0.1em]s2.east) -- ([xshift=-0.1em]s3.west);
+\draw [->] ([xshift=0.1em]s3.east) -- ([xshift=-0.1em]s4.west);
+}
+
+\visible<7->{
+\draw [->,thick,dotted] (wo1.east) .. controls +(east:1.0) and +(west:1.0) ..(wt2.west);
+}
+\visible<9->{
+\draw [->,thick,dotted] (wo2.east) .. controls +(east:1.3) and +(west:1.1) ..(wt3.west);
+\draw [->,thick,dotted] (wo3.east) .. controls +(east:0.9) and +(west:0.9) ..(wt4.west);
+}
+
+\visible<6->{
+\node [circle,draw,anchor=south,inner sep=3pt,fill=orange!20] (c2) at ([yshift=2em]h2.north) {\tiny{$\textbf{C}_2$}};
+\node [anchor=south] (c2label) at (c2.north) {\tiny{\textbf{注意力机制：上下文}}};
+\node [anchor=south] (c2more) at ([yshift=-1.5em]c2.south) {\tiny{...}};
+\draw [->] (h1.north) .. controls +(north:0.6) and +(250:0.9) .. (c2.250);
+\draw [->] (h3.north) .. controls +(north:0.6) and +(290:0.9) .. (c2.290);
+\draw [->] ([yshift=-0.3em]s1.west) .. controls +(west:2) and +(-50:0.3) .. (c2.-40);
+}
+\visible<8->{
+\draw [->] (c2.0) -- ([xshift=1.358in]c2.0) -- ([yshift=0.3em,xshift=-1.2em]s2.west) -- ([yshift=0.3em,xshift=-0.1em]s2.west);
+}
+
+\visible<9->{
+\node [circle,draw,anchor=north,inner sep=3pt,fill=orange!20] (c3) at ([yshift=-2em]t2.south) {\tiny{$\textbf{C}_3$}};
+\draw [->] ([xshift=-0.7em]c3.west) -- ([xshift=-0.1em]c3.west);
+\draw [->] ([xshift=0.1em]c3.east) .. controls +(east:0.6) and +(west:0.8) ..([yshift=-0.3em,xshift=-0.1em]s3.west);
+}
+
+\visible<9->{
+\node [circle,draw,anchor=north,inner sep=3pt,fill=orange!20] (c4) at ([yshift=-2em]t3.south) {\tiny{$\textbf{C}_4$}};
+\draw [->] ([xshift=-0.7em]c4.west) -- ([xshift=-0.1em]c4.west);
+\draw [->] ([xshift=0.1em]c4.east) .. controls +(east:0.6) and +(west:0.8) ..([yshift=-0.3em,xshift=-0.1em]s4.west);
+}
+
+\end{scope}
+\end{tikzpicture}
+\end{center}
+}
+
 \end{frame}
 
+%%%------------------------------------------------------------------------------------------------------------
+%%% 解码 - beam search
+\begin{frame}{推断 - Beam Search}
+\begin{itemize}
+\item \textbf{Greedy Search}: 目标语每一个位置，输出层的Softmax可以得到所有单词的概率，然后选择一个概率最大单词输出，下一个位置的预测就基于这一步输出的单词
+\item \textbf{Beach Search}: 为了避免贪婪方法造成的错误累加，可以每次对$b$个单词进行扩展，而不是只使用一个单词，其中$b$称做束的宽度 - 这样可以搜索更多可能的译文
+\end{itemize}
+
+\vspace{-0.3em}
+\visible<2->{
+\begin{center}
+\begin{tikzpicture}
+\begin{scope}
+\tikzstyle{rnnnode} = [minimum height=1.1em,minimum width=3.5em,inner sep=2pt,rounded corners=1pt,draw,fill=red!20];
+\tikzstyle{wnode} = [minimum height=1.0em,minimum width=3.5em,inner sep=2pt,rounded corners=1pt,draw,fill=white];
+
+
+\visible<3->{
+\node [rnnnode,anchor=west,fill=green!20] (t1) at (0,0) {\tiny{$e_y()$}};
+}
+\visible<7->{
+\node [rnnnode,anchor=west,fill=green!20] (t2) at ([xshift=2.2em]t1.east) {\tiny{$e_y()$ ($\times 3$)}};
+}
+\visible<8->{
+\node [rnnnode,anchor=west,fill=green!20] (t3) at ([xshift=2.2em]t2.east) {\tiny{$e_y()$ ($\times 3$)}};
+\node [anchor=west,inner sep=2pt] (t4) at ([xshift=0.3em]t3.east) {\tiny{...}};
+}
+\visible<3->{
+\node [rnnnode,anchor=south] (s1) at ([yshift=1em]t1.north) {\tiny{$\textbf{s}_1$}};
+}
+\visible<7->{
+\node [rnnnode,anchor=south] (s2) at ([yshift=1em]t2.north) {\tiny{$\textbf{s}_2$ ($\times 3$)}};
+}
+\visible<8->{
+\node [rnnnode,anchor=south] (s3) at ([yshift=1em]t3.north) {\tiny{$\textbf{s}_3$ ($\times 3$)}};
+\node [anchor=west,inner sep=2pt] (s4) at ([xshift=0.3em]s3.east) {\tiny{...}};
+}
+\visible<3->{
+\node [rnnnode,anchor=south,fill=blue!20] (o1) at ([yshift=1em]s1.north) {\tiny{softmax}};
+}
+\visible<7->{
+\node [rnnnode,anchor=south,fill=blue!20] (o2) at ([yshift=1em]s2.north) {\tiny{softmax ($\times 3$)}};
+}
+\visible<8->{
+\node [rnnnode,anchor=south,fill=blue!20] (o3) at ([yshift=1em]s3.north) {\tiny{softmax ($\times 3$)}};
+\node [anchor=west,inner sep=2pt] (o4) at ([xshift=0.3em]o3.east) {\tiny{...}};
+}
+
+\node [wnode,anchor=north] (wt1) at ([yshift=-0.8em]t1.south) {\tiny{EOS}};
+
+\visible<6->{
+\node [wnode,anchor=north] (wt2) at ([yshift=-0.8em]t2.south) {\tiny{Have}};
+\node [wnode,anchor=north] (wt2copy1) at ([xshift=-0.2em,yshift=-0.2em]wt2.north) {\tiny{Have}};
+\node [wnode,anchor=north] (wt2copy2) at ([xshift=-0.4em,yshift=-0.4em]wt2.north) {\tiny{Have}};
+}
+
+\visible<8->{
+\node [wnode,anchor=north,inner sep=2pt] (wt3) at ([yshift=-0.8em]t3.south) {\tiny{you}};
+\node [wnode,anchor=north] (wt3copy1) at ([xshift=-0.2em,yshift=-0.2em]wt3.north) {\tiny{you}};
+\node [wnode,anchor=north] (wt3copy2) at ([xshift=-0.4em,yshift=-0.4em]wt3.north) {\tiny{you}};
+}
+
+\visible<5->{
+\node [wnode,anchor=center,inner sep=2pt] (wo1) at ([xshift=0.4em,yshift=1.8em]o1.north) {\tiny{Have}};
+\node [wnode,anchor=north] (wo1copy1) at ([xshift=-0.2em,yshift=-0.2em]wo1.north) {\tiny{Have}};
+\node [wnode,anchor=north] (wo1copy2) at ([xshift=-0.4em,yshift=-0.4em]wo1.north) {\tiny{Have}};
+}
+
+\visible<8->{
+\node [wnode,anchor=center,inner sep=2pt] (wo2) at ([xshift=0.4em,yshift=1.8em]o2.north) {\tiny{you}};
+\node [wnode,anchor=north] (wo2copy1) at ([xshift=-0.2em,yshift=-0.2em]wo2.north) {\tiny{you}};
+\node [wnode,anchor=north] (wo2copy2) at ([xshift=-0.4em,yshift=-0.4em]wo2.north) {\tiny{you}};
+}
+
+\visible<8->{
+\node [wnode,anchor=center,inner sep=2pt] (wo3) at ([xshift=0.4em,yshift=1.8em]o3.north) {\tiny{learned}};
+\node [wnode,anchor=north] (wo3copy1) at ([xshift=-0.2em,yshift=-0.2em]wo3.north) {\tiny{learned}};
+\node [wnode,anchor=north] (wo3copy2) at ([xshift=-0.4em,yshift=-0.4em]wo3.north) {\tiny{learned}};
+}
+
+\visible<3->{
+\foreach \x in {1}{
+    \draw [->] ([yshift=-0.7em]t\x.south) -- ([yshift=-0.1em]t\x.south);
+    \draw [->] ([yshift=0.1em]t\x.north) -- ([yshift=-0.1em]s\x.south);
+    \draw [->] ([yshift=0.1em]s\x.north) -- ([yshift=-0.1em]o\x.south);
+   
+}
+}
+
+\visible<5->{
+ \draw [->] ([yshift=0.1em]o1.north) -- ([yshift=0.8em]o1.north) node [pos=0.5,right] {\tiny{top-3}};
+ }
+
+\visible<7->{
+\foreach \x in {2}{
+    \draw [->] ([yshift=-0.7em]t\x.south) -- ([yshift=-0.1em]t\x.south);
+    \draw [->] ([yshift=0.1em]t\x.north) -- ([yshift=-0.1em]s\x.south);
+    \draw [->] ([yshift=0.1em]s\x.north) -- ([yshift=-0.1em]o\x.south);
+    \draw [->] ([yshift=0.1em]o\x.north) -- ([yshift=0.8em]o\x.north) node [pos=0.5,right] {\tiny{top-3}};
+}
+}
+
+\visible<8->{
+\foreach \x in {3}{
+    \draw [->] ([yshift=-0.7em]t\x.south) -- ([yshift=-0.1em]t\x.south);
+    \draw [->] ([yshift=0.1em]t\x.north) -- ([yshift=-0.1em]s\x.south);
+    \draw [->] ([yshift=0.1em]s\x.north) -- ([yshift=-0.1em]o\x.south);
+    \draw [->] ([yshift=0.1em]o\x.north) -- ([yshift=0.8em]o\x.north) node [pos=0.5,right] {\tiny{top-3}};
+}
+}
+
+\visible<3->{
+\draw [->] ([xshift=-0.5em]s1.west) -- ([xshift=-0.1em]s1.west) node [pos=0,left,inner sep=1pt] {\tiny{0}};
+}
+\visible<7->{
+\draw [->] ([xshift=0.1em]s1.east) -- ([xshift=-0.1em]s2.west);
+}
+\visible<8->{
+\draw [->] ([xshift=0.1em]s2.east) -- ([xshift=-0.1em]s3.west);
+}
+
+\visible<6->{
+\draw [->,very thick,dotted] (wo1.east) .. controls +(east:0.6) and +(west:0.8) ..(wt2copy2.west);
+}
+\visible<8->{
+\draw [->,very thick,dotted] (wo2.east) .. controls +(east:0.6) and +(west:0.8) ..(wt3copy2.west);
+}
+
+\visible<7->{
+\node [circle,draw,anchor=north,inner sep=2pt,fill=orange!20] (c2) at ([yshift=-2.5em]t1.south) {\tiny{$\textbf{C}_2$}};
+\node [circle,draw,inner sep=2pt,fill=orange!20] (c2copy1) at ([yshift=-0.1em,xshift=-0.1em]c2) {\tiny{$\textbf{C}_2$}};
+\node [circle,draw,inner sep=2pt,fill=orange!20] (c2copy2) at ([yshift=-0.2em,xshift=-0.2em]c2) {\tiny{$\textbf{C}_2$}};
+\draw [->] ([xshift=-0.9em]c2.west) -- ([xshift=-0.3em]c2.west);
+\draw [->] ([xshift=0.1em]c2.east) .. controls +(east:1.5) and +(west:0.8) ..([yshift=-0.3em,xshift=-0.1em]s2.west);
+}
+
+\visible<8->{
+\node [circle,draw,anchor=north,inner sep=2pt,fill=orange!20] (c3) at ([yshift=-2.5em]t2.south) {\tiny{$\textbf{C}_3$}};
+\node [circle,draw,inner sep=2pt,fill=orange!20] (c3copy1) at ([yshift=-0.1em,xshift=-0.1em]c3) {\tiny{$\textbf{C}_3$}};
+\node [circle,draw,inner sep=2pt,fill=orange!20] (c3copy2) at ([yshift=-0.2em,xshift=-0.2em]c3) {\tiny{$\textbf{C}_3$}};
+\draw [->] ([xshift=-0.9em]c3.west) -- ([xshift=-0.3em]c3.west);
+\draw [->] ([xshift=0.1em]c3.east) .. controls +(east:1.5) and +(west:0.8) ..([yshift=-0.3em,xshift=-0.1em]s3.west);
+}
+
+\visible<3->{
+\node [anchor=east] (vocab) at ([xshift=-5em]s1.west) {\tiny{$\begin{bmatrix} \textrm{Have} & 0.50 \\ \textrm{I} & 0.02 \\ \textrm{it} & 0.03 \\ \textrm{has} & 0.30 \\ \textrm{you} & 0.01 \\ \textrm{the} & 0.01 \\ \textrm{a} & 0.01 \\ \textrm{an} & 0.02 \\ \textrm{he} & 0.03 \\ \textrm{she} & 0.01 \\ \textrm{are} & 0.00 \\ \textrm{am} & 0.01 \\ ... & ... \end{bmatrix}$}};
+\node [anchor=south] (vocablabel) at (vocab.north) {\tiny{单词的概率分布}};
+\draw [->,red,very thick,dotted] (o1.west) .. controls +(west:1) and +(east:2) .. ([yshift=1em]vocab.south east); 
+}
+
+\visible<4->{
+\node [anchor=east,inner sep=1pt] (vocabtopn) at ([xshift=-0.5em,yshift=-0.5em]wo1.west) {\tiny{$\begin{bmatrix} \textrm{Have} \\ \textrm{has} \\ \textrm{it} \end{bmatrix}$}};
+\draw [->] ([yshift=-1.6em,xshift=-0.4em]vocab.north east) .. controls +(east:1) and +(west:1) ..  ([xshift=0.1em,yshift=0.4em]vocabtopn.west) node [pos=0.3,below] (topnlabel) {\tiny{top-3}};
+
+\visible<4->{
+\node [anchor=north] (cap) at (vocab.south east) {\scriptsize{\textbf{束搜索($b=3$)}}};
+}
+}
+
+
+\end{scope}
+\end{tikzpicture}
+\end{center}
+}
+
+\end{frame}
 
 %%%------------------------------------------------------------------------------------------------------------
 %%% 实验结果