new pages

Section05-Neural-Networks-and-Language-Modeling/section05-test.tex

@@ -120,17 +120,103 @@
 \subsection{词嵌入}
 \subsection{词嵌入}
 
 
 %%%------------------------------------------------------------------------------------------------------------
 %%%------------------------------------------------------------------------------------------------------------
-%%% 用实例理解词的分布式表示
-\begin{frame}{分布式表示的可视化}
+%%% 上下文表示模型
+\begin{frame}{表示更长的片段 - 上下文表示模型}
 \begin{itemize}
 \begin{itemize}
-\item \textbf{一个著名的例子}：国王 $\to$ 王后\\
-    \begin{displaymath}
-    \vv{\textrm{国王}} - \vv{\textrm{男人}} + \vv{\textrm{女人}} = \vv{\textrm{王后}}
-    \end{displaymath}
-    这里，$\vv{\textrm{word}}$表示单词的分布式向量表示
-\item 更多的词的可视化：相似的词聚在一起
-    
+\item 在语言模型中已经包含了每个位置的上下文表示信息
+    \begin{itemize}
+    \item 以RNN LM为例，位置$i$的隐层输出就是一种$w_1...w_i$的表示
+    \end{itemize}
 \end{itemize}
 \end{itemize}
+
+\vspace{-0.5em}
+\begin{center}
+\begin{tikzpicture}
+\begin{scope}
+\tikzstyle{rnnnode} = [draw,inner sep=5pt,minimum width=4em,minimum height=1.5em,fill=green!30!white,blur shadow={shadow xshift=1pt,shadow yshift=-1pt}]
+\node [anchor=west,rnnnode] (node11) at (0,0) {\scriptsize{RNN Cell}};
+\node [anchor=west,rnnnode] (node12) at ([xshift=2em]node11.east) {\scriptsize{RNN Cell}};
+\node [anchor=west,rnnnode] (node13) at ([xshift=2em]node12.east) {\scriptsize{RNN Cell}};
+\node [anchor=west,rnnnode] (node14) at ([xshift=2em]node13.east) {\scriptsize{RNN Cell}};
+
+\node [anchor=north,rnnnode,fill=red!30!white] (e1) at ([yshift=-1.2em]node11.south) {\scriptsize{embedding}};
+\node [anchor=north,rnnnode,fill=red!30!white] (e2) at ([yshift=-1.2em]node12.south) {\scriptsize{embedding}};
+\node [anchor=north,rnnnode,fill=red!30!white] (e3) at ([yshift=-1.2em]node13.south) {\scriptsize{embedding}};
+\node [anchor=north,rnnnode,fill=red!30!white] (e4) at ([yshift=-1.2em]node14.south) {\scriptsize{embedding}};
+\node [anchor=north] (w1) at ([yshift=-1em]e1.south) {\footnotesize{乔布斯}};
+\node [anchor=north] (w2) at ([yshift=-1em]e2.south) {\footnotesize{任职}};
+\node [anchor=north] (w3) at ([yshift=-1em]e3.south) {\footnotesize{于}};
+\node [anchor=north] (w4) at ([yshift=-1em]e4.south) {\footnotesize{苹果}};
+
+\draw [->,thick] ([yshift=0.1em]w1.north)--([yshift=-0.1em]e1.south);
+\draw [->,thick] ([yshift=0.1em]w2.north)--([yshift=-0.1em]e2.south);
+\draw [->,thick] ([yshift=0.1em]w3.north)--([yshift=-0.1em]e3.south);
+\draw [->,thick] ([yshift=0.1em]w4.north)--([yshift=-0.1em]e4.south);
+
+\draw [->,thick] ([yshift=0.1em]e1.north)--([yshift=-0.1em]node11.south);
+\draw [->,thick] ([yshift=0.1em]e2.north)--([yshift=-0.1em]node12.south);
+\draw [->,thick] ([yshift=0.1em]e3.north)--([yshift=-0.1em]node13.south);
+\draw [->,thick] ([yshift=0.1em]e4.north)--([yshift=-0.1em]node14.south);
+
+\node [anchor=south,rnnnode] (node21) at ([yshift=1.5em]node11.north) {\scriptsize{RNN Cell}};
+\node [anchor=south,rnnnode] (node22) at ([yshift=1.5em]node12.north) {\scriptsize{RNN Cell}};
+\node [anchor=south,rnnnode] (node23) at ([yshift=1.5em]node13.north) {\scriptsize{RNN Cell}};
+\node [anchor=south,rnnnode] (node24) at ([yshift=1.5em]node14.north) {\scriptsize{RNN Cell}};
+
+\node [anchor=south] (node31) at ([yshift=1.0em]node21.north) {\scriptsize{的表示}};
+\node [anchor=south west] (node31new) at ([yshift=-0.3em]node31.north west) {\scriptsize{``乔布斯''}};
+\node [anchor=south] (node32) at ([yshift=1.0em]node22.north) {\scriptsize{的表示\ \ \ }};
+\node [anchor=south west] (node32new) at ([yshift=-0.3em]node32.north west) {\scriptsize{``乔布斯 任职''}};
+\node [anchor=south] (node33) at ([yshift=1.0em]node23.north) {\scriptsize{的表示\ \ \ \ \ \ \ \ }};
+\node [anchor=south west] (node33new) at ([yshift=-0.3em]node33.north west) {\scriptsize{``乔布斯 任职 于''}};
+\node [anchor=south] (node34) at ([yshift=1.0em]node24.north) {\scriptsize{的表示\ \ \ \ \ \ \ \ }};
+\node [anchor=south west] (node34new) at ([yshift=-0.3em]node34.north west) {\scriptsize{``乔布斯 任职 于 苹果''}};
+
+\draw [->,thick] ([yshift=0.1em]node21.north)--([yshift=-0.1em]node31.south);
+\draw [->,thick] ([yshift=0.1em]node22.north)--([yshift=-0.1em]node32.south);
+\draw [->,thick] ([yshift=0.1em]node23.north)--([yshift=-0.1em]node33.south);
+\draw [->,thick] ([yshift=0.1em]node24.north)--([yshift=-0.1em]node34.south);
+
+\draw [->,thick] ([xshift=-1em]node21.west)--([xshift=-0.1em]node21.west);
+\draw [->,thick] ([xshift=0.1em]node21.east)--([xshift=-0.1em]node22.west);
+\draw [->,thick] ([xshift=0.1em]node22.east)--([xshift=-0.1em]node23.west);
+\draw [->,thick] ([xshift=0.1em]node23.east)--([xshift=-0.1em]node24.west);
+\draw [->,thick] ([xshift=0.1em]node24.east)--([xshift=1em]node24.east);
+
+\draw [->,thick] ([yshift=0.1em]node11.north)--([yshift=-0.1em]node21.south);
+\draw [->,thick] ([yshift=0.1em]node12.north)--([yshift=-0.1em]node22.south);
+\draw [->,thick] ([yshift=0.1em]node13.north)--([yshift=-0.1em]node23.south);
+\draw [->,thick] ([yshift=0.1em]node14.north)--([yshift=-0.1em]node24.south);
+
+\draw [->,thick] ([xshift=-1em]node11.west)--([xshift=-0.1em]node11.west);
+\draw [->,thick] ([xshift=0.1em]node11.east)--([xshift=-0.1em]node12.west);
+\draw [->,thick] ([xshift=0.1em]node12.east)--([xshift=-0.1em]node13.west);
+\draw [->,thick] ([xshift=0.1em]node13.east)--([xshift=-0.1em]node14.west);
+\draw [->,thick] ([xshift=0.1em]node14.east)--([xshift=1em]node14.east);
+
+\visible<2->{
+\node [anchor=south] (toplabel1) at ([yshift=2em,xshift=-2em]node32new.north) {\footnotesize{``苹果''的表示：}};
+\node [anchor=west,fill=blue!20!white,minimum width=3em] (toplabel2) at (toplabel1.east) {\footnotesize{上下文}};
+}
+\visible<3->{
+\node [anchor=west,fill=red!20!white,minimum width=3em] (toplabel3) at (toplabel2.east) {\footnotesize{词}};
+}
+
+\begin{pgfonlayer}{background}
+\visible<3->{
+\node [rectangle,inner sep=2pt,draw,thick,dashed,red] [fit = (e4)] (r2) {};
+\draw [->,thick,red] (r2.west) .. controls +(west:0.8) and +(south:2) .. ([xshift=1.3em]toplabel3.south);
+}
+\visible<2->{
+\node [rectangle,inner sep=2pt,draw,thick,dashed,ublue,fill=white] [fit = (node33) (node33new)] (r1) {};
+\draw [->,thick,ublue] ([xshift=-2em]r1.north) .. controls +(north:0.7) and +(south:0.7) .. ([xshift=-0.5em]toplabel2.south);
+}
+\end{pgfonlayer}
+
+\end{scope}
+\end{tikzpicture}
+\end{center}
+
 \end{frame}
 \end{frame}
 
 
 %%%------------------------------------------------------------------------------------------------------------
 %%%------------------------------------------------------------------------------------------------------------

Section05-Neural-Networks-and-Language-Modeling/section05.tex

@@ -4454,9 +4454,208 @@ $\textbf{V}, \textbf{U}, \textbf{W}$: 参数
 \end{itemize}
 \end{itemize}
 \end{frame}
 \end{frame}
 
 
+%%%------------------------------------------------------------------------------------------------------------
+%%% 神经语言模型中的词嵌入
+\begin{frame}{神经语言模型中的词嵌入}
+\begin{itemize}
+\item 在神经语言模型中，需要把词表示成它的分布式表示
+    \begin{itemize}
+    \item<2-> 其中$\textbf{C}$是词嵌入矩阵，每一行对应一个词的分布式表示
+    \item<3-> $\textbf{C}$可以用语言模型训练，也可以利用其它模型训练，固定词嵌入，让语言模型专注长片段的学习
+    \end{itemize}
+\end{itemize}
+
+\vspace{-0.5em}
+\begin{center}
+\begin{tikzpicture}
+\begin{scope}
+\node [anchor=center,inner sep=2pt] (e) at (0,0) {\small{$e=w$}};
+\node [anchor=west,inner sep=2pt] (c) at (e.east) {\small{$\textbf{C}$}};
+
+\begin{pgfonlayer}{background}
+\node [rectangle,inner sep=0.4em,draw,fill=blue!20!white] [fit = (e) (c)] (box) {};
+\end{pgfonlayer}
+
+\draw [->,thick] ([yshift=-1em]box.south)--([yshift=-0.1em]box.south) node [pos=0,below] (bottom1) {\small{单词$w$}};
+\draw [->,thick] ([yshift=0.1em]box.north)--([yshift=1em]box.north) node [pos=1,above] (top1) {\scriptsize{$e$=(8,.2,-1,.9,...,1)}};
+\node [anchor=north] (bottom2) at ([yshift=0.3em]bottom1.south) {\scriptsize{$w$=(0,0,1,0,...,0)}};
+\node [anchor=south] (top2) at ([yshift=-0.3em]top1.north) {\small{$w$的分布式表示}};
+
+\visible<2->{
+\node [anchor=north west,fill=red!20!white] (cmatrix) at ([xshift=3em,yshift=1.0em]c.north east) {\scriptsize{$\begin{pmatrix} 1 & .2 & -.2 & 8 & ... & 0 \\ .6 & .8 & -2 & 1 & ... & -.2 \\ 8 & .2 & -1 & .9 & ... & 2.3 \\ 1 & 1.2 & -.9 & 3 & ... & .2 \\ ... & ... & ... & ... & ... & ... \\ 1 & .3 & 3 & .9 & ... & 5.1 \end{pmatrix}$}};
+\node [anchor=west,inner sep=2pt,fill=red!30!white] (c) at (e.east) {\small{$\textbf{C}$}};
+\draw [<-,thick] (c.east) -- ([xshift=3em]c.east);
+}
+
+\visible<3->{
+\node [anchor=south,draw,fill=green!20!white] (e2) at ([yshift=1.5em]cmatrix.north) {\scriptsize{外部词嵌入系统得到的$\textbf{C}$}};
+\draw [->,very thick,dashed] (e2.south) -- (cmatrix.north);
+}
+
+\end{scope}
+\end{tikzpicture}
+\end{center}
+
+\vspace{-1.0em}
+
+\begin{itemize}
+\item<4-> 词嵌入如何学习得到？
+    \begin{itemize}
+    \item 可以和语言模型的其它部分一起训练，不过速度较慢
+    \item 也可以考虑使用效率更高的外部模型，如word2vec、 Glove等，这样可以使用更大规模的数据
+    \end{itemize}
+\end{itemize}
+
+\end{frame}
 
 
 %%%------------------------------------------------------------------------------------------------------------
 %%%------------------------------------------------------------------------------------------------------------
 \subsection{句子表示模型及预训练}
 \subsection{句子表示模型及预训练}
 
 
+%%%------------------------------------------------------------------------------------------------------------
+%%% 词嵌入的问题
+\begin{frame}{不仅如``词''}
+\begin{itemize}
+\item 词嵌入已经成为诸多NLP系统的标配，当然也衍生出各种花式玩法，甚至有``embed everything''的口号，但是词嵌入也有问题
+    \begin{itemize}
+    \item 每个词都对应唯一的向量表示，但是对于一词多义现象，词义需要通过上下文进行区分。一个著名的例子：
+    \end{itemize}
+    \vspace{0.3em}
+    \hspace{6em} Jobs was the CEO of \alert{\underline{apple}}.\\
+    \hspace{6em} He finally ate the \alert{\underline{apple}}.
+\item<2-> 引入上下文信息
+    \begin{itemize}
+    \item 上述问题引发了新的思考：不能简单地考虑词的表示，应同时考虑其上下文信息
+    \item 对于句子中的一个词(或者位置)，同时表示词和上下文
+    \end{itemize}
+\end{itemize}
+
+\visible<2->{
+\begin{center}
+\begin{tikzpicture}
+\begin{scope}
+\node [anchor=west] (node1) at (0,0) {\footnotesize{Jobs was the CEO of}};
+\node [anchor=west] (node2) at ([xshift=-0.2em,yshift=-0.05em]node1.east) {\footnotesize{\alert{\underline{apple}}}};
+\node [anchor=west] (node3) at ([xshift=-0.2em,yshift=-0.1em]node2.east) {\footnotesize{.}};
+\node [anchor=south,inner sep=2pt,minimum width=2.4em,fill=red!20!white] (node4) at ([yshift=1.5em]node2.north) {\scriptsize{词}};
+\node [anchor=north] (label) at ([xshift=1em]node1.south) {\scriptsize{\textbf{词表示模型}}};
+\draw [->,thick] (node2.north) -- (node4.south);
+\end{scope}
+\begin{scope}[xshift=2in]
+\node [anchor=west] (node1) at (0,0) {\footnotesize{Jobs was the CEO of}};
+\node [anchor=west] (node2) at ([xshift=-0.2em,yshift=-0.05em]node1.east) {\footnotesize{\alert{\underline{apple}}}};
+\node [anchor=west] (node3) at ([xshift=-0.2em,yshift=-0.1em]node2.east) {\footnotesize{.}};
+\node [anchor=south,inner sep=2pt,minimum width=2.4em,fill=red!20!white] (node4) at ([yshift=1.5em]node2.north) {\scriptsize{词}};
+\node [anchor=south,inner sep=2pt,minimum width=2.4em,fill=blue!20!white] (node5) at (node4.north) {\scriptsize{上下文}};
+\node [anchor=north] (label) at ([xshift=1em]node1.south) {\scriptsize{\textbf{词+上下文表示模型}}};
+\draw [->,thick] (node2.north) -- (node4.south);
+\draw [->] ([xshift=1em]node1.north west) .. controls +(north:1) and +(west:2) .. ([yshift=0.2em]node5.west);
+\draw [->] ([xshift=3em]node1.north west) .. controls +(north:0.8) and +(west:1.5) .. ([yshift=-0.2em]node5.west);
+\node [anchor=east] (morelines) at ([xshift=-1.5em]node4.west) {...};
+\end{scope}
+\end{tikzpicture}
+\end{center}
+}
+\end{frame}
+
+%%%------------------------------------------------------------------------------------------------------------
+%%% 上下文表示模型
+\begin{frame}{表示更长的片段 - 上下文表示模型}
+\begin{itemize}
+\item 在语言模型中已经包含了每个位置的上下文表示信息
+    \begin{itemize}
+    \item 以RNN LM为例，位置$i$的隐层输出就是一种$w_1...w_i$的表示
+    \end{itemize}
+\end{itemize}
+
+\vspace{-0.5em}
+\begin{center}
+\begin{tikzpicture}
+\begin{scope}
+\tikzstyle{rnnnode} = [draw,inner sep=5pt,minimum width=4em,minimum height=1.5em,fill=green!30!white,blur shadow={shadow xshift=1pt,shadow yshift=-1pt}]
+\node [anchor=west,rnnnode] (node11) at (0,0) {\scriptsize{RNN Cell}};
+\node [anchor=west,rnnnode] (node12) at ([xshift=2em]node11.east) {\scriptsize{RNN Cell}};
+\node [anchor=west,rnnnode] (node13) at ([xshift=2em]node12.east) {\scriptsize{RNN Cell}};
+\node [anchor=west,rnnnode] (node14) at ([xshift=2em]node13.east) {\scriptsize{RNN Cell}};
+
+\node [anchor=north,rnnnode,fill=red!30!white] (e1) at ([yshift=-1.2em]node11.south) {\scriptsize{embedding}};
+\node [anchor=north,rnnnode,fill=red!30!white] (e2) at ([yshift=-1.2em]node12.south) {\scriptsize{embedding}};
+\node [anchor=north,rnnnode,fill=red!30!white] (e3) at ([yshift=-1.2em]node13.south) {\scriptsize{embedding}};
+\node [anchor=north,rnnnode,fill=red!30!white] (e4) at ([yshift=-1.2em]node14.south) {\scriptsize{embedding}};
+\node [anchor=north] (w1) at ([yshift=-1em]e1.south) {\footnotesize{乔布斯}};
+\node [anchor=north] (w2) at ([yshift=-1em]e2.south) {\footnotesize{任职}};
+\node [anchor=north] (w3) at ([yshift=-1em]e3.south) {\footnotesize{于}};
+\node [anchor=north] (w4) at ([yshift=-1em]e4.south) {\footnotesize{苹果}};
+
+\draw [->,thick] ([yshift=0.1em]w1.north)--([yshift=-0.1em]e1.south);
+\draw [->,thick] ([yshift=0.1em]w2.north)--([yshift=-0.1em]e2.south);
+\draw [->,thick] ([yshift=0.1em]w3.north)--([yshift=-0.1em]e3.south);
+\draw [->,thick] ([yshift=0.1em]w4.north)--([yshift=-0.1em]e4.south);
+
+\draw [->,thick] ([yshift=0.1em]e1.north)--([yshift=-0.1em]node11.south);
+\draw [->,thick] ([yshift=0.1em]e2.north)--([yshift=-0.1em]node12.south);
+\draw [->,thick] ([yshift=0.1em]e3.north)--([yshift=-0.1em]node13.south);
+\draw [->,thick] ([yshift=0.1em]e4.north)--([yshift=-0.1em]node14.south);
+
+\node [anchor=south,rnnnode] (node21) at ([yshift=1.5em]node11.north) {\scriptsize{RNN Cell}};
+\node [anchor=south,rnnnode] (node22) at ([yshift=1.5em]node12.north) {\scriptsize{RNN Cell}};
+\node [anchor=south,rnnnode] (node23) at ([yshift=1.5em]node13.north) {\scriptsize{RNN Cell}};
+\node [anchor=south,rnnnode] (node24) at ([yshift=1.5em]node14.north) {\scriptsize{RNN Cell}};
+
+\node [anchor=south] (node31) at ([yshift=1.0em]node21.north) {\scriptsize{的表示}};
+\node [anchor=south west] (node31new) at ([yshift=-0.3em]node31.north west) {\scriptsize{``乔布斯''}};
+\node [anchor=south] (node32) at ([yshift=1.0em]node22.north) {\scriptsize{的表示\ \ \ }};
+\node [anchor=south west] (node32new) at ([yshift=-0.3em]node32.north west) {\scriptsize{``乔布斯 任职''}};
+\node [anchor=south] (node33) at ([yshift=1.0em]node23.north) {\scriptsize{的表示\ \ \ \ \ \ \ \ }};
+\node [anchor=south west] (node33new) at ([yshift=-0.3em]node33.north west) {\scriptsize{``乔布斯 任职 于''}};
+\node [anchor=south] (node34) at ([yshift=1.0em]node24.north) {\scriptsize{的表示\ \ \ \ \ \ \ \ }};
+\node [anchor=south west] (node34new) at ([yshift=-0.3em]node34.north west) {\scriptsize{``乔布斯 任职 于 苹果''}};
+
+\draw [->,thick] ([yshift=0.1em]node21.north)--([yshift=-0.1em]node31.south);
+\draw [->,thick] ([yshift=0.1em]node22.north)--([yshift=-0.1em]node32.south);
+\draw [->,thick] ([yshift=0.1em]node23.north)--([yshift=-0.1em]node33.south);
+\draw [->,thick] ([yshift=0.1em]node24.north)--([yshift=-0.1em]node34.south);
+
+\draw [->,thick] ([xshift=-1em]node21.west)--([xshift=-0.1em]node21.west);
+\draw [->,thick] ([xshift=0.1em]node21.east)--([xshift=-0.1em]node22.west);
+\draw [->,thick] ([xshift=0.1em]node22.east)--([xshift=-0.1em]node23.west);
+\draw [->,thick] ([xshift=0.1em]node23.east)--([xshift=-0.1em]node24.west);
+\draw [->,thick] ([xshift=0.1em]node24.east)--([xshift=1em]node24.east);
+
+\draw [->,thick] ([yshift=0.1em]node11.north)--([yshift=-0.1em]node21.south);
+\draw [->,thick] ([yshift=0.1em]node12.north)--([yshift=-0.1em]node22.south);
+\draw [->,thick] ([yshift=0.1em]node13.north)--([yshift=-0.1em]node23.south);
+\draw [->,thick] ([yshift=0.1em]node14.north)--([yshift=-0.1em]node24.south);
+
+\draw [->,thick] ([xshift=-1em]node11.west)--([xshift=-0.1em]node11.west);
+\draw [->,thick] ([xshift=0.1em]node11.east)--([xshift=-0.1em]node12.west);
+\draw [->,thick] ([xshift=0.1em]node12.east)--([xshift=-0.1em]node13.west);
+\draw [->,thick] ([xshift=0.1em]node13.east)--([xshift=-0.1em]node14.west);
+\draw [->,thick] ([xshift=0.1em]node14.east)--([xshift=1em]node14.east);
+
+\visible<2->{
+\node [anchor=south] (toplabel1) at ([yshift=2em,xshift=-2em]node32new.north) {\footnotesize{``苹果''的表示：}};
+\node [anchor=west,fill=blue!20!white,minimum width=3em] (toplabel2) at (toplabel1.east) {\footnotesize{上下文}};
+}
+\visible<3->{
+\node [anchor=west,fill=red!20!white,minimum width=3em] (toplabel3) at (toplabel2.east) {\footnotesize{词}};
+}
+
+\begin{pgfonlayer}{background}
+\visible<3->{
+\node [rectangle,inner sep=2pt,draw,thick,dashed,red] [fit = (e4)] (r2) {};
+\draw [->,thick,red] (r2.west) .. controls +(west:0.8) and +(south:2) .. ([xshift=1.3em]toplabel3.south);
+}
+\visible<2->{
+\node [rectangle,inner sep=2pt,draw,thick,dashed,ublue,fill=white] [fit = (node33) (node33new)] (r1) {};
+\draw [->,thick,ublue] ([xshift=-2em]r1.north) .. controls +(north:0.7) and +(south:0.7) .. ([xshift=-0.5em]toplabel2.south);
+}
+\end{pgfonlayer}
+
+\end{scope}
+\end{tikzpicture}
+\end{center}
+
+\end{frame}
+
 \end{CJK}
 \end{CJK}
 \end{document}
 \end{document}