% !Mode:: "TeX:UTF-8"
% !TEX encoding = UTF-8 Unicode

\def\CTeXPreproc{Created by ctex v0.2.13, don't edit!}
\documentclass[cjk,t,compress,12pt]{beamer}
\usepackage{pstricks}
\usepackage{etex}
\usepackage{eso-pic,graphicx}
\usepackage{fancybox}
\usepackage{amsmath,amssymb}
\usepackage{setspace}
\usepackage{xcolor}
\usepackage{array,multirow}
\usepackage{CJK}
\usepackage{tikz}
\usepackage{tikz-qtree}
\usepackage{hyperref}
\usepackage{changepage}
\usepackage{pgfplots}
\usepackage{subfigure}
\usepackage{tikz-3dplot}
\usepackage{esvect}
\usepackage{CJKulem}
\usepackage{booktabs}

\usepackage{tcolorbox}
\tcbuselibrary{skins}

\usetikzlibrary{calc,intersections}
\usetikzlibrary{matrix}
\usetikzlibrary{patterns}
\usetikzlibrary{arrows,decorations.pathreplacing}
\usetikzlibrary{shadows} % LATEX and plain TEX when using Tik Z
\usetikzlibrary{shadows.blur}

\usepgflibrary{arrows} % LATEX and plain TEX and pure pgf
\usetikzlibrary{arrows} % LATEX and plain TEX when using Tik Z
\usetikzlibrary{decorations}
\usetikzlibrary{arrows,shapes}

\usetikzlibrary{positioning,fit,calc}

\usetikzlibrary{mindmap,backgrounds} % mind map

\DeclareMathOperator*{\argmax}{arg\,max}
\DeclareMathOperator*{\argmin}{arg\,min}

\setbeamertemplate{items}[ball]
\usefonttheme[onlymath]{serif}  % fout of math

\definecolor{ugreen}{rgb}{0,0.5,0}
\definecolor{lgreen}{rgb}{0.9,1,0.8}
\definecolor{xtgreen1}{rgb}{0.824,0.898,0.8}
\definecolor{xtgreen}{rgb}{0.914,0.945,0.902}
\definecolor{lightgray}{gray}{0.85}

\setbeamercolor{uppercol}{fg=white,bg=ugreen}
\setbeamercolor{lowercol}{fg=black,bg=xtgreen}

\definecolor{ublue}{rgb}{0.152,0.250,0.545}
\setbeamercolor{uppercolblue}{fg=white,bg=ublue}
\setbeamercolor{lowercolblue}{fg=black,bg=blue!10}


%\usetheme{default}
%\usetheme{Darmstadt}
%\usetheme{Madrid}
%\usetheme{Frankfurt}
%\usetheme{Dresden}
%\usetheme{Boadilla}
%\usecolortheme{dolphin}

% not compatible with [scale=?]
\newdimen\XCoord
\newdimen\YCoord
\newdimen\TMP
\newcommand*{\ExtractCoordinate}[1]{\path (#1); \pgfgetlastxy{\XCoord}{\YCoord};}%
\newcommand*{\ExtractX}[1]{\path (#1); \pgfgetlastxy{\XCoord}{\TMP};}%
\newcommand*{\ExtractY}[1]{\path (#1); \pgfgetlastxy{\TMP}{\YCoord};}%

\newcounter{mycount1}
\newcounter{mycount2}
\newcounter{mycount3}
\newcounter{mycount4}
\newlength{\mystep}
\newlength{\base}
\newlength{\wseg}
\newlength{\hseg}
\newlength{\wnode}
\newlength{\hnode}


\usefonttheme[onlylarge]{structurebold}

\IfFileExists{C:/WINDOWS/win.ini}
{\newcommand{\mycfont}{you}}
{\newcommand{\mycfont}{gbsn}}

\begin{CJK}{UTF8}{\mycfont}
\end{CJK}

\setbeamerfont*{frametitle}{size=\large,series=\bfseries}
\setbeamertemplate{navigation symbols}{\begin{CJK}{UTF8}{\mycfont} 第六章 神经机器翻译 \hspace*{2em} 肖桐\&朱靖波 \end{CJK} \hspace*{2em} \today \hspace*{2em} \insertframenumber{}/\inserttotalframenumber}

\setbeamertemplate{itemize items}[circle] % if you want a circle
\setbeamertemplate{itemize subitem}[triangle] % if you wnat a triangle
\setbeamertemplate{itemize subsubitem}[ball] % if you want a ball

\begin{document}

\begin{CJK}{UTF8}{\mycfont}

\title{\Large{神经机器翻译}}
\author{\large{\textbf{肖桐\ \ 朱靖波}}}
\institute{
\blue{\url{xiaotong@mail.neu.edu.cn}} \black{} \\
\blue{\url{zhujingbo@mail.neu.edu.cn}} \black{} \\
\vspace{1.0em}
东北大学 自然语言处理实验室 \\
\blue{\underline{\url{http://www.nlplab.com}}} \black{} \\
\vspace{0.2cm}
\hspace{0.1cm} \includegraphics[scale=0.1]{../Figures/logo.pdf}
}
\date{}

\maketitle

\setlength{\leftmargini}{1em}
\setlength{\leftmarginii}{1em}

%%%------------------------------------------------------------------------------------------------------------
\section{编码器-解码器框架}

%%%------------------------------------------------------------------------------------------------------------
%%% landscape
\begin{frame}{Landscape}

\vspace{-1em}
\begin{center}
\begin{tikzpicture}

\node [] (history) at (0,0) {\includegraphics[scale=0.29]{./Figures/mt-history.png}};

\draw[-,thick] ([xshift=-1.1in]history.south) coordinate (top01) -- ([yshift=-2em]top01);
\draw[-,thick] ([xshift=1in]history.south) coordinate (top02) -- ([yshift=-2em]top02);
\draw[-,thick] ([xshift=-0.3in]history.south east) coordinate (top03) -- ([yshift=-2em]top03);
\draw[<->,thick] ([yshift=-1em]top01) -- ([yshift=-1em]top02) node [pos=0.5,below,align=left,yshift=-0.5em] {\footnotesize{\textbf{Sections 3-4:}}\\\footnotesize{统计机器翻译}};
\draw[<->,thick] ([yshift=-1em]top02) -- ([yshift=-1em]top03) node [pos=0.5,below,align=left,yshift=-0.5em] {\footnotesize{\textbf{Sections 5-6:}}\\\footnotesize{神经机器翻译}};

\visible<2->{
\draw[<-,thick] ([xshift=0.9in,yshift=-0.2in]history.north) coordinate (label01) -- ([yshift=1.5em,xshift=-1.5em]label01) coordinate (label02);
\node[anchor=south west,align=left] (label) at ([xshift=-5em]label02) {\textbf{\alert{本章内容（Section 6）：}}\\神经机器翻译建模及实现};
}

\end{tikzpicture}
\end{center}

\end{frame}

%%%------------------------------------------------------------------------------------------------------------
%%% 译文对比
\begin{frame}{机器翻译今天的水平}

\begin{tcolorbox}[enhanced,frame engine=empty,boxrule=0.1mm,size=title,colback=red!10!white]
\begin{flushleft}

\begin{spacing}{0.9}
\scriptsize{\textbf{原文(英语)}: During Soviet times, if a city's population topped one million, it would become eligible for its own metro. Planners wanted to brighten the lives of everyday Soviet citizens, and saw the metros, with their tens of thousands of daily passengers, as a singular opportunity to do so. In 1977, Tashkent, the capital of Uzbekistan, became the seventh Soviet city to have a metro built. Grand themes celebrating the history of Uzbekistan and the Soviet Union were brought to life, as art was commissioned and designers set to work. The stations reflected different themes, some with domed ceilings and painted tiles reminiscent of Uzbekistan's Silk Road mosques, while others ... }
\end{spacing}
\end{flushleft}
\end{tcolorbox}

\begin{minipage}[t]{0.47\textwidth}
\begin{beamerboxesrounded}[upper=uppercolblue,lower=lowercolblue,shadow=true]{\scriptsize{\textbf{译文\visible<2->{(统计机器翻译)}}}}
{\scriptsize
\begin{spacing}{0.9}
在苏联时代，如果一个城市的人口突破一百万，这将成为合资格为自己的地铁。规划者想去照亮每天的苏联公民的生命，看到地铁，与他们的数十每天数千乘客，作为一个独特的机会来这样做。1977年，塔什干，乌兹别克斯坦的首都，成了苏联第七城市建有地铁。宏大主题，庆祝乌兹别克斯坦和苏联的历史被带到生活，因为艺术是委托和设计师开始工作。车站反映了不同的主题，有的圆顶天花板和绘瓷砖让人想起乌兹别克斯坦是丝绸之路的清真寺，而另一些则装饰着...
\end{spacing}
}
\end{beamerboxesrounded}
\end{minipage}
\hfill
\begin{minipage}[t]{0.47\textwidth}
\begin{beamerboxesrounded}[upper=uppercolblue,lower=lowercolblue,shadow=true]{\scriptsize{\textbf{译文\visible<2->{(神经机器翻译 - 很流畅！)}}}}
{\scriptsize
\begin{spacing}{0.9}
在苏联时期，如果一个城市的人口超过一百万，它就有资格拥有自己的地铁。 规划者想要照亮日常苏联公民的生活，并把拥有数万名每日乘客的地铁看作是这样做的一个绝佳机会。 1977年，乌兹别克斯坦首都塔什干成为苏联第七个修建地铁的城市。 随着艺术的委托和设计师们的工作，乌兹别克斯坦和苏联历史的宏伟主题被赋予了生命力。 这些电台反映了不同的主题，有的有穹顶和彩砖，让人想起乌兹别克斯坦的丝绸之路清真寺，有的则用...
\end{spacing}
}
\end{beamerboxesrounded}
\end{minipage}

\end{frame}

%%%------------------------------------------------------------------------------------------------------------
%%% 神经机器翻译的性能增长
\begin{frame}{神经机器翻译的进展}
\begin{itemize}
\item 想当年，08年NIST举办的汉英机器翻译评测，BLEU能突破30已经是巨猛无比的结果，而现在的神经机器翻译轻松突破45！
\item 再比如，机器翻译的旗舰评测比赛WMT(Workshop of Machine Translation)已经被神经机器翻译刷榜
    \begin{itemize}
    \item 现在冠军系统几乎没有纯统计机器翻译系统
    \end{itemize}
\end{itemize}
%%% 秀smt nmt的blue随着年代的变化
\begin{center}
    \begin{tikzpicture}
        \begin{scope}[local bounding box=WMT]
            \draw[->,thick] (0.5,0) to (10,0);
            \draw[->,thick] (0.5,-0) to (0.5,3.5);

            % 2015
            \node[minimum width=0.5cm,thick,minimum height=7*0.2cm,draw,fill=blue!30!white,inner sep=0pt,outer sep=0pt,anchor=south west] (smt2015) at (1.5,0.5pt) {};
            \node[minimum width=0.5cm,thick,minimum height=2*0.2cm,draw,fill=red!30!white,inner sep=0pt,outer sep=0pt,anchor=south west] (nmt2015) at (smt2015.south east) {};
            \node[font=\normalsize,anchor=north] () at (smt2015.south east) {2015};
            % 2016
            \node[minimum width=0.5cm,thick,minimum height=3*0.2cm,draw,fill=blue!30!white,inner sep=0pt,outer sep=0pt,anchor=south west] (smt2016) at ($(nmt2015.south east)+(1,0)$) {};
            \node[minimum width=0.5cm,thick,minimum height=8*0.2cm,draw,fill=red!30!white,inner sep=0pt,outer sep=0pt,anchor=south west] (nmt2016) at (smt2016.south east) {};
            \node[font=\normalsize,anchor=north] () at (smt2016.south east) {2016};
            % 2017
            \node[minimum width=0.5cm,thick,minimum height=3*0.2cm,draw,fill=blue!30!white,inner sep=0pt,outer sep=0pt,anchor=south west] (smt2017) at ($(nmt2016.south east)+(1,0)$) {};
            \node[minimum width=0.5cm,thick,minimum height=13*0.2cm,draw,fill=red!30!white,inner sep=0pt,outer sep=0pt,anchor=south west] (nmt2017) at (smt2017.south east) {};
            \node[font=\normalsize,anchor=north] () at (smt2017.south east) {2017};
            % 2018
            \node[minimum width=0.5cm,thick,minimum height=0cm,draw,fill=blue!30!white,inner sep=0pt,outer sep=0pt,anchor=south west] (smt2018) at ($(nmt2017.south east)+(1,0)$) {};
            \node[minimum width=0.5cm,thick,minimum height=14*0.2cm,draw,fill=red!30!white,inner sep=0pt,outer sep=0pt,anchor=south west] (nmt2018) at (smt2018.south east) {};
            \node[font=\normalsize,anchor=north] () at (smt2018.south east) {2018};
        \end{scope}

        % legend
        \ExtractX{$(nmt2015.west)$}
        \ExtractY{$(WMT.north)$}
        \node[minimum width=0.5cm,rectangle,draw,fill=blue!30!white,anchor=north west,label={[label distance=1pt,font=\scriptsize]0:统计机器翻译}] () at (\XCoord,\YCoord) {};
        \ExtractX{$(nmt2017.west)$}
        \node[minimum width=0.5cm,rectangle,draw,fill=red!30!white,anchor=north west,label={[label distance=1pt,font=\scriptsize]0:神经机器翻译}] () at (\XCoord,\YCoord) {};

        \node[font=\normalsize,below=0pt of WMT] () {WMT冠军系统};
        \node[font=\normalsize,rotate=90] () at ([xshift=-1em]WMT.west) {数量};
    \end{tikzpicture}
\end{center}
\end{frame}

%%%------------------------------------------------------------------------------------------------------------
%%% 神经机器翻译的性能增长
\begin{frame}{神经机器翻译的进展(续)}
\begin{itemize}
\item 神经机器翻译在很多场景下已经超越统计机器翻译
{
    \footnotesize
    \begin{center}
        \setlength{\tabcolsep}{3pt}
        \renewcommand\arraystretch{1}
        \begin{tabular}{l}
            \begin{tabular}{lcccl}
                \specialrule{1pt}{1pt}{1pt}
                \multirow{2}{*}{\#} & \multicolumn{3}{c}{自动评价} & \multirow{2}{*}{系统} \\
                \cline{2-4}
                & BLEU & HTER & mTER & \\
                \specialrule{0.6pt}{1pt}{1pt}
                统计机器翻译 & 25.3 & 28.0 & 21.8 & PBSY \\
                & 24.6 & 29.9 & 23.4 & HPB \\
                & 25.8 & 29.0 & 22.7 & SPB \\
                \specialrule{0.6pt}{1pt}{1pt}
                神经机器翻译 & \textbf{31.1} & \textbf{21.1} & \textbf{16.2} & NMT \\
                \specialrule{1pt}{1pt}{1pt}
            \end{tabular}\\
            \addlinespace[-0.3ex]
            \tiny *Neural versus Phrase-Based Machine Translation Quality: a Case Study\\
        \end{tabular}
    \end{center}
}
\item 微软的报道：在部分场景下机器翻译质量已经接近甚至超过人工翻译
{
    \footnotesize
    \begin{center}
        \renewcommand\arraystretch{1}
        \begin{tabular}{l}
            \begin{tabular}{lrl}
                \specialrule{1pt}{1pt}{1pt}
                \# & 人工评价 & 系统 \\
                \specialrule{0.6pt}{1pt}{1pt}
                机器翻译 & \textbf{69.9} & COMBO-6 \\
                & 69.8 & COMBO-4 \\
                & \textbf{69.9} & COMBO-5 \\
                \specialrule{0.6pt}{1pt}{1pt}
                人工翻译 & 68.6 & REFERENCE-HT \\
                & 67.6 & REFERENCE-PE \\
                \specialrule{1pt}{1pt}{1pt}
            \end{tabular}\\
            \addlinespace[-0.3ex]
            \tiny *Achieving Human Parity on Automatic Chinese to English News Translation\\
        \end{tabular}
    \end{center}
}
\end{itemize}
%%% 进一步找一些证据说明NMT的进展，很大google、microsoft的文章，可以看看提升了多少性能等等
\end{frame}

%%%------------------------------------------------------------------------------------------------------------
%%% 传统方法 vs 神经网络方法
\begin{frame}{神经机器翻译的优势}
\begin{itemize}
\item 神经网络方法带来了新的思路，还是同样的表再看一遍
\end{itemize}

\begin{tabular} {l | l}
\textbf{传统基于统计的方法} & \textbf{深度学习方法} \\
\textbf{(统计机器翻译)} & \textbf{(神经机器翻译)} \\ \hline
基于离散空间的表示模型 & 基于连续空间的表示模型 \\
NLP问题的隐含结构假设 & 无隐含结构假设，端到端学习 \\
特征工程为主 & 无显性特征，但需要设计网络 \\
特征、规则的存储耗资源 & 模型存储相对小，但计算慢
\end{tabular}

\begin{itemize}
\item<2-> \only<2>{在统计机器翻译时代，系统依赖很多模块，比如}\only<3->{\sout{在统计机器翻译时代，系统依赖很多模块，比如}}
    \begin{itemize}
    \item \only<2>{\textbf{词对齐}：双语句子之间词和词的对应关系}\only<3->{\sout{\textbf{词对齐}：双语句子之间词和词的对应关系}}
    \item \only<2>{\textbf{短语(规则)表}：原文和译文互译的片段}\only<3->{\sout{\textbf{短语(规则)表}：原文和译文互译的片段}}
    \item \only<2>{\textbf{特征}：人工设计的用于建模翻译的各种各样的子模型}\only<3->{\sout{\textbf{特征}：人工设计的用于建模翻译的各种各样的子模型}}
    \item \only<2>{\textbf{目标语语言模型}：大量单语数据训练的目标语建模模块}\only<3->{\sout{\textbf{目标语语言模型}：大量单语数据训练的目标语建模模块}}
    \item \only<2>{...}\only<3->{\sout{...}}
    \end{itemize}
\item<3-> 在\alert{神经机器翻译}时代，以上均不是必要的，而仅仅需要一个神经网络进行端到端建模
\end{itemize}

\end{frame}

%%%------------------------------------------------------------------------------------------------------------
%%% 编码器-解码器框架
\begin{frame}{编码器-解码器框架}

\begin{itemize}
\item 神经机器翻译遵循一种叫\alert{``编码器-解码器''}的结构
    \begin{itemize}
    \item<2-> \textbf{编码器}负责把源语言编码成一种句子表示形式(如：向量)
    \item<3-> \textbf{解码器}负责利用这种表示逐词把目标语句子生成处理
    \item<4-> 编码器和解码器之间的向量就是连接这两部分的句子表示
    \end{itemize}
\end{itemize}

\vspace{0.5em}
\begin{tikzpicture}

\begin{scope}
\small{
\node [anchor=south west,minimum width=15em] (source) at (0,0) {\textbf{source}: 我\ \ \ \ 对\ \ \ \ 你\ \ \ \ 感到\ \ \ \ 满意};
\visible<3->{
\node [anchor=south west,minimum width=15em] (target) at ([yshift=12em]source.north west) {\textbf{target}: I\ \ am\ \ \ satisfied\ \ \ with\ \ \ you};
}
\visible<2->{
\node [anchor=center,minimum width=9.6em,minimum height=1.8em,draw,rounded corners=0.3em] (hidden) at ([yshift=6em]source.north) {};
\node [anchor=west,minimum width=1.5em,minimum size=1.5em,fill=ugreen!20] (cell01) at ([xshift=0.2em]hidden.west) {\footnotesize{.2}};
\node [anchor=west,minimum width=1.5em,minimum size=1.5em,fill=ugreen!10] (cell02) at (cell01.east) {\footnotesize{-1}};
\node [anchor=west,minimum width=1.5em,minimum size=1.5em,fill=ugreen!70] (cell03) at (cell02.east) {\footnotesize{6}};
\node [anchor=west,minimum width=1.5em,minimum size=1.5em,fill=ugreen!50] (cell04) at (cell03.east) {\footnotesize{5}};
\node [anchor=west,minimum width=1.5em,minimum size=1.5em,fill=ugreen!30] (cell05) at (cell04.east) {\footnotesize{.7}};
\node [anchor=west,minimum width=1.5em,minimum size=1.5em,fill=ugreen!10] (cell06) at (cell05.east) {\footnotesize{-2}};
}

\visible<2->{
\filldraw [fill=red!20,draw=white] (source.north west) -- (source.north east) -- ([xshift=-0.2em,yshift=-0.1em]hidden.south east) -- ([xshift=0.2em,yshift=-0.1em]hidden.south west);
}
\visible<3->{
\filldraw [fill=blue!20,draw=white] (target.south west) -- (target.south east) -- ([xshift=-0.2em,yshift=0.1em]hidden.north east) -- ([xshift=0.2em,yshift=0.1em]hidden.north west);
}
\visible<2->{
\draw [->,thick] (source.north west) -- ([xshift=0.2em,yshift=-0.1em]hidden.south west);
\draw [->,thick] (source.north east) -- ([xshift=-0.2em,yshift=-0.1em]hidden.south east);
}
\visible<3->{
\draw [->,thick] ([xshift=0.2em,yshift=0.1em]hidden.north west) -- (target.south west);
\draw [->,thick] ([xshift=-0.2em,yshift=0.1em]hidden.north east) -- (target.south east);
}
}

\visible<2->{
\node [anchor=south] (enclabel) at ([yshift=2em]source.north) {\large{\textbf{Encoder}}};
}
\visible<3->{
\node [anchor=north] (declabel) at ([yshift=-2em]target.south) {\large{\textbf{Decoder}}};
}

\visible<4->{
\node [anchor=west] (representation) at ([xshift=3.5em,yshift=3em]hidden.east) {\footnotesize{\textbf{句子的表示:}}};
\node [anchor=north west] (rline1) at (representation.south west) {\footnotesize{源语言句子被表示成一个实数}};
\node [anchor=north west] (rline2) at ([yshift=0.2em]rline1.south west) {\footnotesize{向量}\footnotesize{$(0.2,-1,6,5,0.7,-2)$}};
\node [anchor=north west] (rline3) at ([yshift=0.2em]rline2.south west) {\footnotesize{不要问0.2是什么意思，因为}};
\node [anchor=north west] (rline4) at ([yshift=0.2em]rline3.south west) {\footnotesize{只有系统自己才知道 :)}};

\begin{pgfonlayer}{background}
\visible<4->{
\node [rectangle,inner sep=0.4em,rounded corners=1pt,fill=ugreen!10,drop shadow] [fit = (representation) (rline1) (rline2) (rline3) (rline4)] (rlabel) {};
}
\end{pgfonlayer}

\draw[->,very thick,dashed] (hidden.east) .. controls +(east:1) and +(west:1) .. ([yshift=-1em]rlabel.north west);
}

\end{scope}

\end{tikzpicture}

\end{frame}

%%%------------------------------------------------------------------------------------------------------------
%%% 表示模型
\begin{frame}{基于连续空间表示模型的方法}
\begin{itemize}
\item 编码器-解码器框架的革命在于：\alert{它把传统基于符号的离散型知识转化为基于表示的连续型知识}
    \begin{itemize}
    \item 比如，对于一个短语，它可以对应一个文法规则的使用过程，而规则都是由离散的符号表示
    \item 它也可以被表示为一种更加抽象的形式，通过向量记录短语的各个``属性''
    \end{itemize}
\item<2-> 这种表示形式上的创新，让我们不再依赖离散符号系统的各种分解、组合，而直接在连续空间下学习这种表示
    \begin{itemize}
    \item 所谓``端到端''只是这种表示模型下的一种自然的学习范式
    \end{itemize}
\end{itemize}

\visible<3->{
\begin{center}
\begin{tikzpicture}

\begin{scope}
\node [anchor=south west,draw,thick,red,minimum width=0.9in,minimum height=0.7in] (space1) at (0,0) {};
\node [anchor=south west,fill=blue,minimum width=0.1in,minimum height=0.1in] (unit1) at (0.2,0.8) {};
\node [anchor=south west,fill=ugreen,minimum width=0.1in,minimum height=0.1in] (unit2) at (0.7,0.3) {};
\node [anchor=south west,fill=blue,minimum width=0.1in,minimum height=0.1in] (unit3) at (1.3,1.3) {};
\node [anchor=south west,fill=ugreen,minimum width=0.1in,minimum height=0.1in] (unit4) at ([xshift=0.1em]unit3.south east) {};

\begin{pgfonlayer}{background}
\node [rectangle,inner sep=2pt,rounded corners=1pt,draw,thick] [fit = (unit3) (unit4)] (unitbox) {};
\end{pgfonlayer}

\draw [->] ([yshift=1pt]unit1.north) .. controls +(north:0.23) and +(west:0.2) .. ([yshift=0.2em,xshift=-1pt]unitbox.west);
\draw [->] ([xshift=1pt]unit2.east) .. controls +(east:0.5) and +(south:0.2) .. ([xshift=0.2em,yshift=-1pt]unitbox.south);

\node [anchor=south] (spacelabel1) at (space1.north) {\scriptsize{离散表示空间}};
\node [anchor=north] (captain1) at ([yshift=-0.5em]space1.south) {\scriptsize{(a) \textbf{统计机器翻译}}};

\end{scope}

\begin{scope}[xshift=1.3in]
\node [anchor=south west,draw,thick,red,minimum width=0.9in,minimum height=0.7in] (space1) at (0,0) {};
\node [anchor=south west,fill=blue,minimum width=0.1in,minimum height=0.1in] (unit1) at (0.2,0.8) {};
\node [anchor=south west,fill=ugreen,minimum width=0.1in,minimum height=0.1in] (unit2) at (0.7,0.3) {};

\node [anchor=south west,draw,thick,red,minimum width=0.9in,minimum height=0.7in] (space2) at (1.1in,0) {};
\node [anchor=south west,circle,fill=orange,minimum width=0.1in,minimum height=0.1in] (unit3) at (1.5in,1.3) {};

\draw [->] ([yshift=1pt]unit1.north) .. controls +(north:0.4) and +(west:2) .. ([yshift=0.0em,xshift=-1pt]unit3.west);
\draw [->] ([xshift=1pt]unit2.east) .. controls +(east:1.5) and +(south:1) .. ([xshift=0.0em,yshift=-1pt]unit3.south);

\node [anchor=south] (spacelabel1) at (space1.north) {\scriptsize{离散表示空间}};
\node [anchor=south] (spacelabel2) at (space2.north) {\scriptsize{连续表示空间}};
\node [anchor=north] (captain1) at ([yshift=-0.5em,xshift=1em]space1.south east) {\scriptsize{(b) \textbf{神经机器翻译}}};

\end{scope}

\end{tikzpicture}
\end{center}
}

\end{frame}

%%%------------------------------------------------------------------------------------------------------------
%%% 一个NMT运行的简单例子
\begin{frame}{简单的运行实例}
\begin{itemize}
\item 一个简单的例子：基于循环神经网络的翻译过程
        \begin{itemize}
        \item<1-> \textbf{编码器}顺序处理源语言单词
        \item<5-> 源语言句子信息被表示在最后一个循环单元的输出中
        \item<6-> \textbf{解码器}利用源语言句子信息逐词生成目标语译文
        \end{itemize}
\end{itemize}
%%% 运行实例的图
\begin{center}
    \begin{tikzpicture}
        \setlength{\base}{0.6cm}

        \tikzstyle{rnnnode} = [minimum size=\base,inner sep=0pt,rounded corners=1pt,draw]
        \tikzstyle{wordnode} = [font=\normalsize,align=center]

        \begin{scope}
            \visible<1->{
                \node[wordnode] (init) at (0,0) {$0$};
                \node[rnnnode,fill=blue!30!white,right=\base of init] (rnn1) {};
                \node[rnnnode,fill=green!30!white,below=\base of rnn1] (emb1) {};
                \node[wordnode,below=0pt of emb1] (word1) {我};
                \draw[-latex'] (emb1.north) to (rnn1.south);
                \draw[-latex'] (init.east) to (rnn1.west);
            }
            \visible<2->{
                \node[rnnnode,fill=blue!30!white,right=\base of rnn1] (rnn2) {};
                \node[rnnnode,fill=green!30!white,below=\base of rnn2] (emb2) {};
                \node[wordnode,below=0pt of emb2] (word2) {很};
                \draw[-latex'] (emb2.north) to (rnn2.south);
                \draw[-latex'] (rnn1.east) to (rnn2.west);
            }
            \visible<3->{
                \node[rnnnode,fill=blue!30!white,right=\base of rnn2] (rnn3) {};
                \node[rnnnode,fill=green!30!white,below=\base of rnn3] (emb3) {};
                \node[wordnode,below=0pt of emb3] (word3) {好};
                \draw[-latex'] (emb3.north) to (rnn3.south);
                \draw[-latex'] (rnn2.east) to (rnn3.west);

                \node[rnnnode,fill=blue!30!white,right=\base of rnn3] (rnn4) {};
                \node[rnnnode,fill=green!30!white,below=\base of rnn4] (emb4) {};
                \node[wordnode,below=0pt of emb4] (word4) {EOS};
                \draw[-latex'] (emb4.north) to (rnn4.south);
                \draw[-latex'] (rnn3.east) to (rnn4.west);
            }
            \visible<4->{
                \draw[decoration={mirror,brace},decorate] (word1.south west) to node [auto,anchor=north,align=center] {编码器} ([yshift=-0.2em]word4.south east);
            }
            \visible<5->{
                \node[rnnnode,fill=purple] (repr) at (rnn4) {};
                \node[wordnode,above=\base of rnn2] (label) {源语言句子信息};
                \draw[->,dashed,thick] (label.east) .. controls +(east:\base) and +(north:\base) .. (rnn4.north);
            }
            \visible<6->{
                \node[rnnnode,fill=blue!30!white,right=\base of rnn4] (rnn5) {};
                \node[rnnnode,fill=green!30!white,below=\base of rnn5] (emb5) {};
                \node[wordnode,below=0pt of emb5] (word5) {SOS};
                \draw[-latex'] (emb5.north) to (rnn5.south);
                \draw[-latex'] (rnn4.east) to (rnn5.west);
                \node[rnnnode,fill=red!30!white,above=\base of rnn5] (softmax1) {};
                \node[wordnode,above=0pt of softmax1] (out1) {I};
                \draw[-latex'] (rnn5.north) to (softmax1.south);
            }
            \visible<7->{
                \node[rnnnode,fill=blue!30!white,right=\base of rnn5] (rnn6) {};
                \node[rnnnode,fill=green!30!white,below=\base of rnn6] (emb6) {};
                \node[rnnnode,fill=red!30!white,above=\base of rnn6] (softmax2) {};
                \ExtractX{$(emb6)$}
                \ExtractY{$(word4.base)$}
                \node[wordnode,anchor=base] (word6) at (\XCoord,\YCoord) {I};
                \ExtractX{$(emb6)$}
                \ExtractY{$(out1.base)$}
                \node[wordnode,anchor=base] (out2) at (\XCoord,\YCoord) {am};
                \draw[-latex'] (emb6.north) to (rnn6.south);
                \draw[-latex'] (rnn5.east) to (rnn6.west);
                \draw[-latex'] (rnn6.north) to (softmax2.south);
            }
            \visible<8->{
                \node[rnnnode,fill=blue!30!white,right=\base of rnn6] (rnn7) {};
                \node[rnnnode,fill=green!30!white,below=\base of rnn7] (emb7) {};
                \node[rnnnode,fill=red!30!white,above=\base of rnn7] (softmax3) {};
                \ExtractX{$(emb7)$}
                \ExtractY{$(word4.base)$}
                \node[wordnode,anchor=base] (word7) at (\XCoord,\YCoord) {am};
                \ExtractX{$(emb7)$}
                \ExtractY{$(out1.base)$}
                \node[wordnode,anchor=base] (out3) at (\XCoord,\YCoord) {fine};
                \draw[-latex'] (emb7.north) to (rnn7.south);
                \draw[-latex'] (rnn6.east) to (rnn7.west);
                \draw[-latex'] (rnn7.north) to (softmax3.south);

                \node[rnnnode,fill=blue!30!white,right=\base of rnn7] (rnn8) {};
                \node[rnnnode,fill=green!30!white,below=\base of rnn8] (emb8) {};
                \node[rnnnode,fill=red!30!white,above=\base of rnn8] (softmax4) {};
                \ExtractX{$(emb8)$}
                \ExtractY{$(word4.base)$}
                \node[wordnode,anchor=base] (word8) at (\XCoord,\YCoord) {fine};
                \ExtractX{$(emb8)$}
                \ExtractY{$(out1.base)$}
                \node[wordnode,anchor=base] (out4) at (\XCoord,\YCoord) {EOS};
                \draw[-latex'] (emb8.north) to (rnn8.south);
                \draw[-latex'] (rnn7.east) to (rnn8.west);
                \draw[-latex'] (rnn8.north) to (softmax4.south);
            }
            \visible<9->{
                \ExtractX{$(word8.east)$}
                \ExtractY{$(word5.south)$}
                \draw[decoration={mirror,brace},decorate] ([yshift=-0.2em]word5.south west) to node [auto,anchor=north,align=center] {解码器} (\XCoord,\YCoord-0.2em);
            }
        \end{scope}
    \end{tikzpicture}
\end{center}
\end{frame}

%%%------------------------------------------------------------------------------------------------------------
%%% NMT是魔法？
\begin{frame}{神经机器翻译会``魔法''?}
\begin{minipage}[c][6.5cm][t]{0.46\textwidth}
\alert{统计机器翻译}仍然依赖人工定义特征和翻译单元
\begin{itemize}
\item 相对符合人类的理解
\item 具有一定可解释性
\end{itemize}
\vspace{1em}
\includegraphics[width=\textwidth,height=3.5cm]{./Figures/whiteboard-math.jpg}
\end{minipage}
\hfill
\begin{minipage}[c][6.5cm][t]{0.48\textwidth}
\alert{神经机器翻译}把所有工作都交给神经网络
\begin{itemize}
\item 不需要先验假设
\item 语言的``表示''给机器看的
\end{itemize}
\vspace{1em}
\includegraphics[width=\textwidth,height=3.5cm]{./Figures/expelliarmus.jpg}
\end{minipage}

\begin{itemize}
\item 神经机器翻译在使用魔法？\\
        - No! No! No! 变化的只有人类使用知识的形式
\end{itemize}

\end{frame}

%%%------------------------------------------------------------------------------------------------------------
%%% NMT所对应的范式变换
\begin{frame}{神经机器翻译所带来的范式更迭}
\begin{itemize}
\item 不同机器翻译时代，人类知识的使用方式
\end{itemize}

\begin{center}
\begin{tabular}{l | l}
机器翻译方法 & 人类参与方式 \\ \hline
基于规则的方法 & 设计翻译规则 \\
传统统计方法 & 设计翻译特征 \\
神经网络方法 & 设计网络架构
 \end{tabular}
 \end{center}

 \visible<2->{
 \begin{tcolorbox}[enhanced,size=normal,left=2mm,right=1mm,colback=blue!5!white,colframe=blue!75!black,drop fuzzy shadow]
{\Large
\textbf{人类在机器翻译中将扮演什么角色？}
}
\end{tcolorbox}

\begin{itemize}
\item<3-> 机器翻译会逐渐替代人？\visible<4->{\alert{- 不会，爹是不能被儿子替代的}}
\item<3-> 还需要人的语言学知识吗？\visible<4->{\alert{- 会，但是需要新的思路}}
\item<3-> 就连神经网络也可以通过结构搜索自动学习，人类是不是就什么也不用干了？\visible<4->{\alert{- 结构搜索又是谁设计的，人至少需要敲下键盘吧，哈哈哈哈}}
\end{itemize}
}

\end{frame}

%%%------------------------------------------------------------------------------------------------------------
%%% 本章的内容
\begin{frame}{Outline}

\vspace{1.0em}
 \begin{tcolorbox}[enhanced,size=normal,left=2mm,right=1mm,colback=red!5!white,colframe=red!75!black,drop fuzzy shadow]
{\large
\textbf{入门：循环网络翻译模型及注意力机制} \\
\small{1. 起源} \\
\small{2. 模型结构} \\
\small{3. 注意力机制} \\
\small{4. 训练和推断}
}
\end{tcolorbox}

\vspace{0.2em}

 \begin{tcolorbox}[enhanced,size=normal,left=2mm,right=1mm,colback=red!5!white,colframe=red!75!black,drop fuzzy shadow]
{\large
\textbf{热门：Transformer} \\
\small{1. 自注意力模型} \\
\small{2. 多头注意力和层正则化} \\
\small{3. 更深、更宽的模型}
}
\end{tcolorbox}

\vspace{0.2em}

\begin{tcolorbox}[enhanced,size=normal,left=2mm,right=1mm,colback=red!5!white,colframe=red!75!black,drop fuzzy shadow]
{\large
\textbf{其它：一些有趣的神经机器翻译应用}
}
\end{tcolorbox}

\end{frame}

%%%------------------------------------------------------------------------------------------------------------
\section{循环神经网络翻译模型及注意力机制}


%%%------------------------------------------------------------------------------------------------------------
%%% 第一部分
\begin{frame}{首先}

\vspace{5.0em}
 \begin{tcolorbox}[enhanced,size=normal,left=2mm,right=1mm,colback=red!5!white,colframe=red!75!black,drop fuzzy shadow]
{\Large
\textbf{循环网络翻译模型及注意力机制}
}
\end{tcolorbox}

\begin{center}
\begin{tikzpicture}

\begin{scope}[scale=0.7]
\tikzstyle{rnnnode} = [draw,inner sep=5pt,minimum width=3em,minimum height=0.8em,fill=green!30!white,blur shadow={shadow xshift=1pt,shadow yshift=-1pt}]
\node [anchor=west,rnnnode] (node11) at (0,0) {\tiny{RNN Cell}};
\node [anchor=west,rnnnode] (node12) at ([xshift=1em]node11.east) {\tiny{RNN Cell}};
\node [anchor=west,rnnnode] (node13) at ([xshift=1em]node12.east) {\tiny{RNN Cell}};
\node [anchor=west,rnnnode] (node14) at ([xshift=1em]node13.east) {\tiny{RNN Cell}};

\node [anchor=north,rnnnode,fill=blue!30!white] (e1) at ([yshift=-1em]node11.south) {\tiny{}};
\node [anchor=north,rnnnode,fill=blue!30!white] (e2) at ([yshift=-1em]node12.south) {\tiny{}};
\node [anchor=north,rnnnode,fill=blue!30!white] (e3) at ([yshift=-1em]node13.south) {\tiny{}};
\node [anchor=north,rnnnode,fill=blue!30!white] (e4) at ([yshift=-1em]node14.south) {\tiny{}};
\node [anchor=north,inner sep=2pt] (w1) at ([yshift=-1em]e1.south) {\tiny{$<$s$>$}};
\node [anchor=north,inner sep=2pt] (w2) at ([yshift=-1em]e2.south) {\tiny{让}};
\node [anchor=north,inner sep=2pt] (w3) at ([yshift=-1em]e3.south) {\tiny{我们}};
\node [anchor=north,inner sep=2pt] (w4) at ([yshift=-1em]e4.south) {\tiny{开始}};

\draw [->,thick] ([yshift=0.1em]w1.north)--([yshift=-0.1em]e1.south);
\draw [->,thick] ([yshift=0.1em]w2.north)--([yshift=-0.1em]e2.south);
\draw [->,thick] ([yshift=0.1em]w3.north)--([yshift=-0.1em]e3.south);
\draw [->,thick] ([yshift=0.1em]w4.north)--([yshift=-0.1em]e4.south);

\draw [->,thick] ([yshift=0.1em]e1.north)--([yshift=-0.1em]node11.south);
\draw [->,thick] ([yshift=0.1em]e2.north)--([yshift=-0.1em]node12.south);
\draw [->,thick] ([yshift=0.1em]e3.north)--([yshift=-0.1em]node13.south);
\draw [->,thick] ([yshift=0.1em]e4.north)--([yshift=-0.1em]node14.south);

\node [anchor=south,rnnnode,fill=blue!30!white] (node21) at ([yshift=1.0em]node11.north) {\tiny{}};
\node [anchor=south,rnnnode,fill=blue!30!white] (node22) at ([yshift=1.0em]node12.north) {\tiny{}};
\node [anchor=south,rnnnode,fill=blue!30!white] (node23) at ([yshift=1.0em]node13.north) {\tiny{}};
\node [anchor=south,rnnnode,fill=blue!30!white] (node24) at ([yshift=1.0em]node14.north) {\tiny{}};

\node [anchor=south] (output1) at ([yshift=1em]node21.north) {\Large{\textbf{}}};
\node [anchor=south] (output2) at ([yshift=1em]node22.north) {\Large{\textbf{}}};
\node [anchor=south] (output3) at ([yshift=1em]node23.north) {\Large{\textbf{}}};
\node [anchor=south] (output4) at ([yshift=1em]node24.north) {\Large{\textbf{}}};

\draw [->,thick] ([yshift=0.1em]node21.north)--([yshift=-0.1em]output1.south);
\draw [->,thick] ([yshift=0.1em]node22.north)--([yshift=-0.1em]output2.south);
\draw [->,thick] ([yshift=0.1em]node23.north)--([yshift=-0.1em]output3.south);
\draw [->,thick] ([yshift=0.1em]node24.north)--([yshift=-0.1em]output4.south);

\draw [->,thick] ([yshift=0.1em]node11.north)--([yshift=-0.1em]node21.south);
\draw [->,thick] ([yshift=0.1em]node12.north)--([yshift=-0.1em]node22.south);
\draw [->,thick] ([yshift=0.1em]node13.north)--([yshift=-0.1em]node23.south);
\draw [->,thick] ([yshift=0.1em]node14.north)--([yshift=-0.1em]node24.south);

\draw [->,thick] ([xshift=-1em]node11.west)--([xshift=-0.1em]node11.west);
\draw [->,thick] ([xshift=0.1em]node11.east)--([xshift=-0.1em]node12.west);
\draw [->,thick] ([xshift=0.1em]node12.east)--([xshift=-0.1em]node13.west);
\draw [->,thick] ([xshift=0.1em]node13.east)--([xshift=-0.1em]node14.west);
\draw [->,thick] ([xshift=0.1em]node14.east)--([xshift=1em]node14.east);
\end{scope}

\end{tikzpicture}
\end{center}

\end{frame}

%%%------------------------------------------------------------------------------------------------------------
\subsection{起源}

%%%------------------------------------------------------------------------------------------------------------
%%% 神经机器翻译的历史
\begin{frame}{最初的神经机器翻译}

\begin{itemize}
\item 神经网络的在机器翻译中并不新鲜，在很多模块中早有实现，比如，翻译候选打分、语言模型等
    \begin{itemize}
    \item 但是，整个框架仍然是统计机器翻译
    \end{itemize}
\item<2-> 基于神经元网络的端到端建模出现在2013-2015，被称为\alert{Neural Machine Translation (NMT)}，一些代表性工作：
\end{itemize}

\visible<2->{
\begin{center}
{\footnotesize
\begin{tabular}{l | l | l}
\textbf{时间} & \textbf{作者} & \textbf{论文} \\ \hline
2013 & Kalchbrenner和 & Recurrent Continuous Translation Models \\
     & Blunsom        & \\
2014 & Sutskever等 & Sequence to Sequence Learning with \\
     &             & neural networks \\
2014 & Bahdanau等 & Neural Machine Translation by Jointly  \\
     &       & Learning to Align and Translate \\
2014 & Cho等 & On the Properties of Neural Machine \\
     &       & Translation \\
2015 & Jean等 & On Using Very Large Target Vocabulary \\
     &        & for Neural Machine Translation \\
2015 & Luong等 & Effective Approaches to Attention-based \\
         &               & Neural Machine Translation
\end{tabular}
}
\end{center}
}

\end{frame}

%%%------------------------------------------------------------------------------------------------------------
%%% 逐渐崛起的NMT
\begin{frame}{崛起}
\begin{itemize}
\item 2015年前统计机器翻译(SMT)在NLP是具有统治力的
    \begin{itemize}
    \item 当时的NMT系统还很初级，被SMT碾压
    \item 大多数的认知还没有进化到NMT时代，甚至Kalchbrenner等人早期的报告也被人质疑
    \end{itemize}
\item 2016年情况大有改变，当时非常受关注的一项工作是Google上线了神经机器翻译系统GNMT
    \begin{itemize}
    \item 在GNMT前后，百度、微软、小牛翻译等也分别推出了自己的神经机器翻译系统，出现了百花齐放的局面
    \end{itemize}
\end{itemize}
\begin{center}
\includegraphics[scale=0.35]{./Figures/google-news.png}
\end{center}
\end{frame}

%%%------------------------------------------------------------------------------------------------------------
\subsection{模型结构}

%%%------------------------------------------------------------------------------------------------------------
%%% 模型结构
\begin{frame}{基于循环神经网络的翻译模型}
\begin{itemize}
\item 一种简单的模型：用循环神经网络进行编码和解码
    \begin{itemize}
    \item 编码端是一个RNN，最后一个隐层状态被看做句子表示
    \item 解码端也是一个RNN，利用编码结果逐词解码出译文
    \end{itemize}
\end{itemize}

\vspace{-0.5em}
\begin{center}
    \begin{tikzpicture}
        \setlength{\base}{0.9cm}

        \tikzstyle{rnnnode} = [rounded corners=1pt,minimum size=0.5\base,draw,inner sep=0pt,outer sep=0pt]
        \tikzstyle{wordnode} = [font=\tiny]

        % RNN translation model
        \begin{scope}[local bounding box=RNNMT]
            % RNN Encoder
            \coordinate (eemb0) at (0,0);
            \foreach \x [count=\y from 0] in {1,2,...,10}
                \node[rnnnode,minimum height=0.5\base,fill=green!30!white,anchor=west] (eemb\x) at ([xshift=0.4\base]eemb\y.east) {};
            \foreach \x in {1,2,...,10}
                \node[rnnnode,fill=blue!30!white,anchor=south] (enc\x) at ([yshift=0.5\base]eemb\x.north) {};
            \node[wordnode,left=0.4\base of enc1] (init) {$0$};

            \node[wordnode,below=0pt of eemb1] () {你};
            \node[wordnode,below=0pt of eemb2] () {知道};
            \node[wordnode,below=0pt of eemb3] () {去};
            \node[wordnode,below=0pt of eemb4] () {北京站};
            \node[wordnode,below=0pt of eemb5] () {的};
            \node[wordnode,below=0pt of eemb6] () {路};
            \node[wordnode,below=0pt of eemb7] () {怎么};
            \node[wordnode,below=0pt of eemb8] () {走};
            \node[wordnode,below=0pt of eemb9] () {吗};
            \node[wordnode,below=0pt of eemb10] () {$\langle$eos$\rangle$};

            % RNN Decoder
            \foreach \x in {1,2,...,10}
                \node[rnnnode,minimum height=0.5\base,fill=green!30!white,anchor=south] (demb\x) at ([yshift=2\base]enc\x.north) {};
            \foreach \x in {1,2,...,10}
                \node[rnnnode,fill=blue!30!white,anchor=south] (dec\x) at ([yshift=0.5\base]demb\x.north) {};
            \foreach \x in {1,2,...,10}
                \node[rnnnode,minimum height=0.5\base,fill=red!30!white,anchor=south] (softmax\x) at ([yshift=0.5\base]dec\x.north) {};

            % Decoder input words
            \node[wordnode,below=0pt of demb1] (decwordin) {$\langle$sos$\rangle$};
            \ExtractX{$(demb2.south)$}
            \ExtractY{$(decwordin.base)$}
            \node[wordnode,anchor=base] () at (\XCoord,\YCoord) {Do};
            \ExtractX{$(demb3.south)$}
            \ExtractY{$(decwordin.base)$}
            \node[wordnode,anchor=base] () at (\XCoord,\YCoord) {you};
            \ExtractX{$(demb4.south)$}
            \ExtractY{$(decwordin.base)$}
            \node[wordnode,anchor=base] () at (\XCoord,\YCoord) {know};
            \ExtractX{$(demb5.south)$}
            \ExtractY{$(decwordin.base)$}
            \node[wordnode,anchor=base] () at (\XCoord,\YCoord) {the};
            \ExtractX{$(demb6.south)$}
            \ExtractY{$(decwordin.base)$}
            \node[wordnode,anchor=base] () at (\XCoord,\YCoord) {way};
            \ExtractX{$(demb7.south)$}
            \ExtractY{$(decwordin.base)$}
            \node[wordnode,anchor=base] () at (\XCoord,\YCoord) {to};
            \ExtractX{$(demb8.south)$}
            \ExtractY{$(decwordin.base)$}
            \node[wordnode,anchor=base] () at (\XCoord,\YCoord) {Beijing};
            \ExtractX{$(demb9.south)$}
            \ExtractY{$(decwordin.base)$}
            \node[wordnode,anchor=base] () at (\XCoord,\YCoord) {Railway};
            \ExtractX{$(demb10.south)$}
            \ExtractY{$(decwordin.base)$}
            \node[wordnode,anchor=base] () at (\XCoord,\YCoord) {Station};

            % Decoder output words
            \node[wordnode,above=0pt of softmax1] (decwordout) {Do};
            \ExtractX{$(softmax2.north)$}
            \ExtractY{$(decwordout.base)$}
            \node[wordnode,anchor=base] () at (\XCoord,\YCoord) {you};
            \ExtractX{$(softmax3.north)$}
            \ExtractY{$(decwordout.base)$}
            \node[wordnode,anchor=base] () at (\XCoord,\YCoord) {know};
            \ExtractX{$(softmax4.north)$}
            \ExtractY{$(decwordout.base)$}
            \node[wordnode,anchor=base] () at (\XCoord,\YCoord) {the};
            \ExtractX{$(softmax5.north)$}
            \ExtractY{$(decwordout.base)$}
            \node[wordnode,anchor=base] () at (\XCoord,\YCoord) {way};
            \ExtractX{$(softmax6.north)$}
            \ExtractY{$(decwordout.base)$}
            \node[wordnode,anchor=base] () at (\XCoord,\YCoord) {to};
            \ExtractX{$(softmax7.north)$}
            \ExtractY{$(decwordout.base)$}
            \node[wordnode,anchor=base] () at (\XCoord,\YCoord) {Beijing};
            \ExtractX{$(softmax8.north)$}
            \ExtractY{$(decwordout.base)$}
            \node[wordnode,anchor=base] () at (\XCoord,\YCoord) {Railway};
            \ExtractX{$(softmax9.north)$}
            \ExtractY{$(decwordout.base)$}
            \node[wordnode,anchor=base] () at (\XCoord,\YCoord) {Station};
            \ExtractX{$(softmax10.north)$}
            \ExtractY{$(decwordout.base)$}
            \node[wordnode,anchor=base] () at (\XCoord,\YCoord) {$\langle$eos$\rangle$};

            % Connections
            \draw[-latex'] (init.east) to (enc1.west);
            \foreach \x in {1,2,...,10}
                \draw[-latex'] (eemb\x) to (enc\x);
            \foreach \x in {1,2,...,10}
                \draw[-latex'] (demb\x) to (dec\x);
            \foreach \x in {1,2,...,10}
                \draw[-latex'] (dec\x.north) to ([yshift=0.5\base]dec\x.north);
            \foreach \x [count=\y from 2] in {1,2,...,9}
            {
                \draw[-latex'] (enc\x.east) to (enc\y.west);
                \draw[-latex'] (dec\x.east) to (dec\y.west);
            }
            \coordinate (bridge) at ([yshift=-1.2\base]demb2);
            \draw[-latex'] (enc10.north) .. controls +(north:\base) and +(east:1.5\base) .. (bridge) .. controls +(west:2.5\base) and +(west:0.6\base) .. (dec1.west);
        \end{scope}

        % legend
        \begin{scope}[shift={(10\base,2.5\base)}]
            \node[rnnnode,minimum height=0.5\base,fill=green!30!white,label={[label distance=3pt,font=\scriptsize]0:词嵌入层}] (emb) at (0,0) {};
            \node[rnnnode,fill=blue!30!white,anchor=north west,label={[label distance=3pt,font=\scriptsize]0:循环单元}] (rnn) at ([yshift=2\base]emb.south west) {};
            \node[rnnnode,minimum height=0.5\base,fill=red!30!white,anchor=north west,label={[label distance=3pt,font=\scriptsize]0:输出层}] (softmax) at ([yshift=2\base]rnn.south west) {};
            \node [anchor=north west] (softmax2) at ([xshift=0.6\base]softmax.south west) {\scriptsize{Softmax}};
            \node [anchor=north west] (rnn2) at ([xshift=0.6\base]rnn.south west) {\scriptsize{LSTM}};

            \node [anchor=west] (reprlabel) at ([xshift=1em]enc10.east) {\scriptsize{句子表示}};
            \draw [->,dashed] (reprlabel.west) -- ([xshift=0.1em]enc10.east);
            \node [rnnnode,fill=purple!30!white] at (enc10) {};
        \end{scope}
    \end{tikzpicture}
\end{center}
\end{frame}

%%%------------------------------------------------------------------------------------------------------------
%%% NMT的数学描述
\begin{frame}{数学建模}
\begin{itemize}
\item 对于源语言序列$\textbf{x} = \{x_1,x_2,...,x_m\}$，生成目标语序列$\textbf{y} = \{y_1,y_2,...,y_n\}$的概率可以被描述为

\begin{displaymath}
\log\textrm{P}(\textbf{y}|\textbf{x}) = \sum_{j=1}^{n} \log\textrm{P}(y_j|\textbf{y}_{<j}, \textbf{x})
\end{displaymath}

根据源于句子$\textbf{x}$和已生成的译文$\textbf{y}_{<j} = \{y_1,y_2,...,y_{j-1}\}$生成第$j$个译文$y_j$

\item<2-> \textbf{核心}：如何求解$\textrm{P}(y_j|\textbf{y}_{<j}, \textbf{x})$。在这个循环神经网络模型中，有三个步骤
    \begin{enumerate}
    \item 输入的单词用分布式表示，如$\textbf{x}$被表示为词向量序列$e_x(\textbf{x})$，同理$\textbf{y}_{<j}$被表示为$e_y(\textbf{y}_{<j})$
    \item 源语言句子被一个RNN编码为一个表示$\textbf{C}$，如前面的例子中是一个实数向量
    \item 目标端解码用另一个RNN，因此生成$y_j$时只考虑前一个状态$\textbf{s}_{j-1}$（这里，$\textbf{s}_{j-1}$表示RNN第$j-1$步骤的隐层状态）
    \end{enumerate}

\end{itemize}
\end{frame}

%%%------------------------------------------------------------------------------------------------------------
%%% 各部分的解释
\begin{frame}{数学建模(续)}

\vspace{-1.5em}
	\begin{center}
		% \hspace*{-1.5cm}
		\begin{tikzpicture}
			\setlength{\base}{0.9cm}
	
			\tikzstyle{rnnnode} = [rounded corners=1pt,minimum height=0.5\base,minimum width=1\base,draw,inner sep=0pt,outer sep=0pt]
			\tikzstyle{wordnode} = [font=\tiny]
	
			% RNN translation model
			\begin{scope}[local bounding box=RNNMT]
				% RNN Encoder
				\coordinate (eemb0) at (0,0);
				\foreach \x [count=\y from 0] in {1,2,...,3}
					\node[rnnnode,minimum height=0.5\base,fill=green!30!white,anchor=west] (eemb\x) at ([xshift=0.4\base]eemb\y.east) {\tiny{$e_x()$}};
				\foreach \x in {1,2,...,3}
					\node[rnnnode,fill=blue!30!white,anchor=south] (enc\x) at ([yshift=0.3\base]eemb\x.north) {};
			        \node[] (enclabel1) at (enc1) {\tiny{$\textbf{h}_{m-2}$}};
			        \node[] (enclabel2) at (enc2) {\tiny{$\textbf{h}_{m-1}$}};
			        \node[rnnnode,fill=purple!30!white] (enclabel3) at (enc3) {\tiny{$\textbf{h}_{m}$}};
				\node[wordnode,left=0.4\base of enc1] (init1) {$\cdots$};
				\node[wordnode,left=0.4\base of eemb1] (init2) {$\cdots$};

				\node[wordnode,below=0pt of eemb1] () {走};
				\node[wordnode,below=0pt of eemb2] () {吗};
				\node[wordnode,below=0pt of eemb3] () {$\langle$eos$\rangle$};
	
				% RNN Decoder
				\foreach \x in {1,2,...,3}
					\node[rnnnode,minimum height=0.5\base,fill=green!30!white,anchor=south] (demb\x) at ([yshift=\base]enc\x.north) {\tiny{$e_y()$}};
				\foreach \x in {1,2,...,3}
					\node[rnnnode,fill=blue!30!white,anchor=south] (dec\x) at ([yshift=0.3\base]demb\x.north) {{\tiny{$\textbf{s}_\x$}}};
				\foreach \x in {1,2,...,3}
					\node[rnnnode,minimum height=0.5\base,fill=red!30!white,anchor=south] (softmax\x) at ([yshift=0.3\base]dec\x.north) {\tiny{Softmax}};
				\node[wordnode,right=0.4\base of demb3] (end1) {$\cdots$};
				\node[wordnode,right=0.4\base of dec3] (end2) {$\cdots$};
				\node[wordnode,right=0.4\base of softmax3] (end3) {$\cdots$};

				% Decoder input words
				\node[wordnode,below=0pt of demb1] (decwordin) {$\langle$sos$\rangle$};
				\ExtractX{$(demb2.south)$}
				\ExtractY{$(decwordin.base)$}
				\node[wordnode,anchor=base] () at (\XCoord,\YCoord) {Do};
				\ExtractX{$(demb3.south)$}
				\ExtractY{$(decwordin.base)$}
				\node[wordnode,anchor=base] () at (\XCoord,\YCoord) {you};

				% Decoder output words
				\node[wordnode,above=0pt of softmax1] (decwordout) {Do};
				\ExtractX{$(softmax2.north)$}
				\ExtractY{$(decwordout.base)$}
				\node[wordnode,anchor=base] () at (\XCoord,\YCoord) {you};
				\ExtractX{$(softmax3.north)$}
				\ExtractY{$(decwordout.base)$}
				\node[wordnode,anchor=base] () at (\XCoord,\YCoord) {know};
	
				% Connections
				\draw[-latex'] (init1.east) to (enc1.west);
				\draw[-latex'] (dec3.east) to (end2.west);
				\foreach \x in {1,2,...,3}
					\draw[-latex'] (eemb\x) to (enc\x);
				\foreach \x in {1,2,...,3}
					\draw[-latex'] (demb\x) to (dec\x);
				\foreach \x in {1,2,...,3}
					\draw[-latex'] (dec\x.north) to (softmax\x.south);
				\foreach \x [count=\y from 2] in {1,2}
				{
					\draw[-latex'] (enc\x.east) to (enc\y.west);
					\draw[-latex'] (dec\x.east) to (dec\y.west);
				}
	
				\coordinate (bridge) at ([yshift=0.4\base]enc2.north west);
				\draw[-latex'] (enc3.north) .. controls +(north:0.3\base) and +(east:\base) .. (bridge) .. controls +(west:2.7\base) and +(west:0.3\base) .. (dec1.west);
				
				\visible<2->{
				\node [anchor=east] (line1) at ([xshift=-3em,yshift=0.5em]softmax1.west) {\scriptsize{基于RNN的隐层状态$\textbf{s}_i$}};
				\node [anchor=north west] (line2) at ([yshift=0.3em]line1.south west) {\scriptsize{预测目标词的概率}};
				\node [anchor=north west] (line3) at ([yshift=0.3em]line2.south west) {\scriptsize{通常，用Softmax函数}};
				\node [anchor=north west] (line4) at ([yshift=0.3em]line3.south west) {\scriptsize{实现 $\textrm{P}(y_i|...)$}};
				}
				
				\visible<3->{
				\node [anchor=north west] (line11) at ([yshift=-1.8em]line4.west) {\scriptsize{每个词的one-hot}};
				\node [anchor=north west] (line12) at ([yshift=0.3em]line11.south west) {\scriptsize{离散化表示都被转化为}};
				\node [anchor=north west] (line13) at ([yshift=0.3em]line12.south west) {\scriptsize{实数向量，即词嵌入}};
				\node [anchor=north west] (line14) at ([yshift=0.3em]line13.south west) {\scriptsize{($e_x()$和$e_y()$函数)}};
				}
				
				\visible<4->{
				\node [anchor=west] (line21) at ([xshift=1.3em,yshift=1.5em]enc3.east)  {\scriptsize{源语编码器最后一个}};
				\node [anchor=north west] (line22) at ([yshift=0.3em]line21.south west) {\scriptsize{循环单元的输出被}};
				\node [anchor=north west] (line23) at ([yshift=0.3em]line22.south west) {\scriptsize{看作是句子的表示,}};
				\node [anchor=north west] (line24) at ([yshift=0.3em]line23.south west) {\scriptsize{记为$\textbf{C}$}};
				}
				
				\begin{pgfonlayer}{background}
				\visible<2->{
				\node [rectangle,inner sep=0.2em,rounded corners=1pt,fill=red!10,drop shadow,draw=red] [fit = (line1) (line2) (line3) (line4)] (box1) {};
				\node [rectangle,inner sep=0.2em,rounded corners=1pt,very thick,dotted,draw=red] [fit = (softmax1) (softmax2) (softmax3)] (box4) {};
				\draw [->,dotted,very thick,red] ([yshift=1em,xshift=2.5em]box1.east) -- ([yshift=1em,xshift=0.1em]box1.east);
				}
				
				\visible<3->{
				\node [rectangle,inner sep=0.2em,rounded corners=1pt,fill=green!10,drop shadow,draw=ugreen] [fit = (line11) (line12) (line13) (line14)] (box2) {};
				\node [rectangle,inner sep=0.2em,rounded corners=1pt,very thick,dotted,draw=ugreen] [fit = (eemb1) (eemb2) (eemb3)] (box5) {};
				\node [rectangle,inner sep=0.2em,rounded corners=1pt,very thick,dotted,draw=ugreen] [fit = (demb1) (demb2) (demb3)] (box6) {};
				\draw [->,dotted,very thick,ugreen] ([yshift=-1.3em,xshift=2.5em]box2.east) -- ([yshift=-1.3em,xshift=0.1em]box2.east);
				\draw [->,dotted,very thick,ugreen] ([xshift=0.1em]box6.west) .. controls +(west:1) and +(east:1) .. ([yshift=1.0em]box2.east) ;
				}
				
				\visible<4->{
				\node [rectangle,inner sep=0.2em,rounded corners=1pt,fill=purple!10,drop shadow,draw=purple] [fit = (line21) (line22) (line23) (line24)] (box3) {};
				\node [rectangle,inner sep=0.2em,rounded corners=1pt,very thick,dotted,draw=purple] [fit = (enc3)] (box7) {};
				\draw [->,dotted,very thick,purple] ([xshift=0.1em]box7.east) -- ([xshift=0.8em]box7.east) ;
				}
							
				\end{pgfonlayer}
	
			\end{scope}
		\end{tikzpicture}
	\end{center}

\visible<5->{
\vspace{-1.5em}
\begin{itemize}
\item 可以重新定义\\
\vspace{-0.8em}
\begin{displaymath}
\textrm{P}(y_j|\textbf{y}_{<j}, \textbf{x}) \triangleq \textrm{P}(y_j|\textbf{s}_{j-1}, \textbf{C})
\end{displaymath}

对于上图中的模型，进一步化简为：\\
\vspace{-0.3em}

\begin{displaymath}
\textrm{P}(y_j|\textbf{y}_{<j}, \textbf{x}) \triangleq \left\{
    \begin{matrix}
        \textrm{P}(y_j|\textbf{C})\ \ \ \  & j = 1 \\
        \textrm{P}(y_j|\textbf{s}_{j-1}) & j > 1
    \end{matrix} \right.
\end{displaymath}

\end{itemize}
}

\end{frame}
%%%------------------------------------------------------------------------------------------------------------
%%% 词嵌入
\begin{frame}{模块1：词嵌入层}
\begin{itemize}
\item 词嵌入
\end{itemize}
%%% 图
\end{frame}

%%%------------------------------------------------------------------------------------------------------------
%%% 输出
\begin{frame}{模块2：输出层}
\begin{itemize}
\item Softmax
\end{itemize}
%%% 图
\end{frame}

%%%------------------------------------------------------------------------------------------------------------
%%% LSTM
\begin{frame}{模块3：循环单元 - 长短时记忆模型(LSTM) (2页？)}
\begin{itemize}
\item LSTM
\end{itemize}
%%% 图
\end{frame}

%%%------------------------------------------------------------------------------------------------------------
%%% GRU
\begin{frame}{改进 - 门循环单元(GRU)}
\begin{itemize}
\item GRU
\end{itemize}
%%% 图
\end{frame}

%%%------------------------------------------------------------------------------------------------------------
%%% 一些变种
\begin{frame}{改进 - 双向模型}
\begin{itemize}
\item 自左向右的模型只考虑了左侧的上下文，因此可以用自右向左的模型对右侧上下文建模
	\begin{itemize}
	\item 最终将两个模型融合同时送给编码端
	\end{itemize}
\end{itemize}

\vspace{-0.5em}
\begin{center}
    \begin{tikzpicture}
        \setlength{\base}{0.9cm}

        \tikzstyle{rnnnode} = [rounded corners=1pt,minimum size=0.5\base,draw,inner sep=0pt,outer sep=0pt]
        \tikzstyle{wordnode} = [font=\tiny]

        % RNN translation model
        \begin{scope}[local bounding box=RNNMT]
            % RNN Encoder
            \coordinate (eemb0) at (0,0);
            \foreach \x [count=\y from 0] in {1,2,...,10}
                \node[rnnnode,minimum height=0.5\base,fill=green!30!white,anchor=west] (eemb\x) at ([xshift=0.4\base]eemb\y.east) {};
            \foreach \x in {1,2,...,10}
                \node[rnnnode,fill=blue!30!white,anchor=south] (backenc\x) at ([yshift=0.5\base]eemb\x.north) {};
            \foreach \x in {1,2,...,10}
                \node[rnnnode,fill=blue!30!white,anchor=south] (enc\x) at ([yshift=0.5\base]backenc\x.north) {};
            \node[wordnode,left=0.4\base of enc1] (init) {$0$};
            \node[wordnode,right=0.4\base of backenc10] (backinit) {$0$};
            \node [rnnnode,fill=purple!30!white] at (enc10) {};
            \node [rnnnode,fill=purple!30!white] at (backenc1) {};

            \node[wordnode,below=0pt of eemb1] () {你};
            \node[wordnode,below=0pt of eemb2] () {知道};
            \node[wordnode,below=0pt of eemb3] () {去};
            \node[wordnode,below=0pt of eemb4] () {北京站};
            \node[wordnode,below=0pt of eemb5] () {的};
            \node[wordnode,below=0pt of eemb6] () {路};
            \node[wordnode,below=0pt of eemb7] () {怎么};
            \node[wordnode,below=0pt of eemb8] () {走};
            \node[wordnode,below=0pt of eemb9] () {吗};
            \node[wordnode,below=0pt of eemb10] () {$\langle$eos$\rangle$};

            % RNN Decoder
            \foreach \x in {1,2,...,10}
                \node[rnnnode,minimum height=0.5\base,fill=green!30!white,anchor=south] (demb\x) at ([yshift=1.5\base]enc\x.north) {};
            \foreach \x in {1,2,...,10}
                \node[rnnnode,fill=blue!30!white,anchor=south] (dec\x) at ([yshift=0.5\base]demb\x.north) {};
            \foreach \x in {1,2,...,10}
                \node[rnnnode,minimum height=0.5\base,fill=red!30!white,anchor=south] (softmax\x) at ([yshift=0.5\base]dec\x.north) {};

            % Decoder input words
            \node[wordnode,below=0pt of demb1] (decwordin) {EOS};
            \ExtractX{$(demb2.south)$}
            \ExtractY{$(decwordin.base)$}
            \node[wordnode,anchor=base] () at (\XCoord,\YCoord) {Do};
            \ExtractX{$(demb3.south)$}
            \ExtractY{$(decwordin.base)$}
            \node[wordnode,anchor=base] () at (\XCoord,\YCoord) {you};
            \ExtractX{$(demb4.south)$}
            \ExtractY{$(decwordin.base)$}
            \node[wordnode,anchor=base] () at (\XCoord,\YCoord) {know};
            \ExtractX{$(demb5.south)$}
            \ExtractY{$(decwordin.base)$}
            \node[wordnode,anchor=base] () at (\XCoord,\YCoord) {the};
            \ExtractX{$(demb6.south)$}
            \ExtractY{$(decwordin.base)$}
            \node[wordnode,anchor=base] () at (\XCoord,\YCoord) {way};
            \ExtractX{$(demb7.south)$}
            \ExtractY{$(decwordin.base)$}
            \node[wordnode,anchor=base] () at (\XCoord,\YCoord) {to};
            \ExtractX{$(demb8.south)$}
            \ExtractY{$(decwordin.base)$}
            \node[wordnode,anchor=base] () at (\XCoord,\YCoord) {Beijing};
            \ExtractX{$(demb9.south)$}
            \ExtractY{$(decwordin.base)$}
            \node[wordnode,anchor=base] () at (\XCoord,\YCoord) {Railway};
            \ExtractX{$(demb10.south)$}
            \ExtractY{$(decwordin.base)$}
            \node[wordnode,anchor=base] () at (\XCoord,\YCoord) {Station};

            % Decoder output words
            \node[wordnode,above=0pt of softmax1] (decwordout) {Do};
            \ExtractX{$(softmax2.north)$}
            \ExtractY{$(decwordout.base)$}
            \node[wordnode,anchor=base] () at (\XCoord,\YCoord) {you};
            \ExtractX{$(softmax3.north)$}
            \ExtractY{$(decwordout.base)$}
            \node[wordnode,anchor=base] () at (\XCoord,\YCoord) {know};
            \ExtractX{$(softmax4.north)$}
            \ExtractY{$(decwordout.base)$}
            \node[wordnode,anchor=base] () at (\XCoord,\YCoord) {the};
            \ExtractX{$(softmax5.north)$}
            \ExtractY{$(decwordout.base)$}
            \node[wordnode,anchor=base] () at (\XCoord,\YCoord) {way};
            \ExtractX{$(softmax6.north)$}
            \ExtractY{$(decwordout.base)$}
            \node[wordnode,anchor=base] () at (\XCoord,\YCoord) {to};
            \ExtractX{$(softmax7.north)$}
            \ExtractY{$(decwordout.base)$}
            \node[wordnode,anchor=base] () at (\XCoord,\YCoord) {Beijing};
            \ExtractX{$(softmax8.north)$}
            \ExtractY{$(decwordout.base)$}
            \node[wordnode,anchor=base] () at (\XCoord,\YCoord) {Railway};
            \ExtractX{$(softmax9.north)$}
            \ExtractY{$(decwordout.base)$}
            \node[wordnode,anchor=base] () at (\XCoord,\YCoord) {Station};
            \ExtractX{$(softmax10.north)$}
            \ExtractY{$(decwordout.base)$}
            \node[wordnode,anchor=base] () at (\XCoord,\YCoord) {EOS};

            % Connections
            \draw[-latex'] (init.east) to (enc1.west);
            \draw[-latex'] (backinit.west) to (backenc10.east);
            \foreach \x in {1,2,...,10}
                \draw[-latex'] (eemb\x) to (backenc\x);
            \foreach \x in {1,2,...,10}
                \draw[-latex'] (eemb\x.north) to [out=15,in=-15] (enc\x.south);
            \foreach \x in {1,2,...,10}
                \draw[-latex'] (demb\x) to (dec\x);
            \foreach \x in {1,2,...,10}
                \draw[-latex'] (dec\x.north) to ([yshift=0.5\base]dec\x.north);
            \foreach \x [count=\y from 2] in {1,2,...,9}
            {
                \draw[-latex'] (enc\x.east) to (enc\y.west);
                \draw[-latex'] (dec\x.east) to (dec\y.west);
            }
            \def\y{0}
            \foreach \x in {10,9,...,2}
            {
                \pgfmathtruncatemacro{\y}{\x - 1}
                \draw[-latex'] (backenc\x.west) to (backenc\y.east);
            }
            \coordinate (bridge) at ([yshift=-1.2\base]demb2);
            \draw[-latex'] (enc10.north) .. controls +(north:0.7\base) and +(east:1.5\base) .. (bridge) .. controls +(west:2.5\base) and +(west:0.6\base) .. (dec1.west);
            \draw[-latex'] (backenc1) to [out=180,in=180] (dec1.west);

            % Backward RNN
            \begin{pgfonlayer}{background}
                \node[draw=red,thick,densely dashed,inner sep=5pt] [fit = (backinit) (backenc1) (backenc10)] (backrnn) {};
            \end{pgfonlayer}
            \node[font=\scriptsize,anchor=south] (backrnnlabel) at ([xshift=-0.5\base,yshift=\base]backrnn.north east) {反向RNN};
            \draw[->,dashed] (backrnnlabel.south) to ([xshift=-0.5\base]backrnn.north east);
        \end{scope}
    \end{tikzpicture}
\end{center}
\end{frame}

%%%------------------------------------------------------------------------------------------------------------
%%% 一些变种
\begin{frame}{改进 - 多层网络}

\begin{itemize}
\item 堆叠更多层的网络，可以提升模型的表示能力
    \begin{itemize}
    \item 常见的NMT系统有2-8层
    \end{itemize}
\end{itemize}

\vspace*{-0.6cm}
\begin{center}
    \hspace*{-2.5cm}
    \begin{tikzpicture}
        \setlength{\base}{0.9cm}

        \tikzstyle{rnnnode} = [rounded corners=1pt,minimum size=0.5\base,draw,inner sep=0pt,outer sep=0pt]
        \tikzstyle{wordnode} = [font=\tiny]

        % RNN translation model
        \begin{scope}[local bounding box=RNNMT]
            % RNN Encoder
            \coordinate (eemb0) at (0,0);
            \foreach \x [count=\y from 0] in {1,2,...,10}
                \node[rnnnode,minimum height=0.5\base,fill=green!30!white,anchor=west] (eemb\x) at ([xshift=0.4\base]eemb\y.east) {};
            \foreach \x in {1,2,...,10}
                \node[rnnnode,fill=blue!30!white,anchor=south] (enc1\x) at ([yshift=0.3\base]eemb\x.north) {};
            \foreach \x in {1,2,...,10}
                \node[rnnnode,fill=blue!30!white,anchor=south] (enc2\x) at ([yshift=0.5\base]enc1\x.north) {};
            \node[wordnode,left=0.4\base of enc11] (init1) {$0$};
            \node[wordnode,left=0.4\base of enc21] (init2) {$0$};

            \node[wordnode,below=0pt of eemb1] () {你};
            \node[wordnode,below=0pt of eemb2] () {知道};
            \node[wordnode,below=0pt of eemb3] () {去};
            \node[wordnode,below=0pt of eemb4] () {北京站};
            \node[wordnode,below=0pt of eemb5] () {的};
            \node[wordnode,below=0pt of eemb6] () {路};
            \node[wordnode,below=0pt of eemb7] () {怎么};
            \node[wordnode,below=0pt of eemb8] () {走};
            \node[wordnode,below=0pt of eemb9] () {吗};
            \node[wordnode,below=0pt of eemb10] () {EOS};

            % RNN Decoder
            \foreach \x in {1,2,...,10}
                \node[rnnnode,minimum height=0.5\base,fill=green!30!white,anchor=south] (demb\x) at ([yshift=1.5\base]enc2\x.north) {};
            \foreach \x in {1,2,...,10}
                \node[rnnnode,fill=blue!30!white,anchor=south] (dec1\x) at ([yshift=0.3\base]demb\x.north) {};
            \foreach \x in {1,2,...,10}
                \node[rnnnode,fill=blue!30!white,anchor=south] (dec2\x) at ([yshift=0.5\base]dec1\x.north) {};
            \foreach \x in {1,2,...,10}
                \node[rnnnode,minimum height=0.5\base,fill=red!30!white,anchor=south] (softmax\x) at ([yshift=0.5\base]dec2\x.north) {};

            % Decoder input words
            \node[wordnode,below=0pt of demb1] (decwordin) {$\langle$sos$\rangle$};
            \ExtractX{$(demb2.south)$}
            \ExtractY{$(decwordin.base)$}
            \node[wordnode,anchor=base] () at (\XCoord,\YCoord) {Do};
            \ExtractX{$(demb3.south)$}
            \ExtractY{$(decwordin.base)$}
            \node[wordnode,anchor=base] () at (\XCoord,\YCoord) {you};
            \ExtractX{$(demb4.south)$}
            \ExtractY{$(decwordin.base)$}
            \node[wordnode,anchor=base] () at (\XCoord,\YCoord) {know};
            \ExtractX{$(demb5.south)$}
            \ExtractY{$(decwordin.base)$}
            \node[wordnode,anchor=base] () at (\XCoord,\YCoord) {the};
            \ExtractX{$(demb6.south)$}
            \ExtractY{$(decwordin.base)$}
            \node[wordnode,anchor=base] () at (\XCoord,\YCoord) {way};
            \ExtractX{$(demb7.south)$}
            \ExtractY{$(decwordin.base)$}
            \node[wordnode,anchor=base] () at (\XCoord,\YCoord) {to};
            \ExtractX{$(demb8.south)$}
            \ExtractY{$(decwordin.base)$}
            \node[wordnode,anchor=base] () at (\XCoord,\YCoord) {Beijing};
            \ExtractX{$(demb9.south)$}
            \ExtractY{$(decwordin.base)$}
            \node[wordnode,anchor=base] () at (\XCoord,\YCoord) {Railway};
            \ExtractX{$(demb10.south)$}
            \ExtractY{$(decwordin.base)$}
            \node[wordnode,anchor=base] () at (\XCoord,\YCoord) {Station};

            % Decoder output words
            \node[wordnode,above=0pt of softmax1] (decwordout) {Do};
            \ExtractX{$(softmax2.north)$}
            \ExtractY{$(decwordout.base)$}
            \node[wordnode,anchor=base] () at (\XCoord,\YCoord) {you};
            \ExtractX{$(softmax3.north)$}
            \ExtractY{$(decwordout.base)$}
            \node[wordnode,anchor=base] () at (\XCoord,\YCoord) {know};
            \ExtractX{$(softmax4.north)$}
            \ExtractY{$(decwordout.base)$}
            \node[wordnode,anchor=base] () at (\XCoord,\YCoord) {the};
            \ExtractX{$(softmax5.north)$}
            \ExtractY{$(decwordout.base)$}
            \node[wordnode,anchor=base] () at (\XCoord,\YCoord) {way};
            \ExtractX{$(softmax6.north)$}
            \ExtractY{$(decwordout.base)$}
            \node[wordnode,anchor=base] () at (\XCoord,\YCoord) {to};
            \ExtractX{$(softmax7.north)$}
            \ExtractY{$(decwordout.base)$}
            \node[wordnode,anchor=base] () at (\XCoord,\YCoord) {Beijing};
            \ExtractX{$(softmax8.north)$}
            \ExtractY{$(decwordout.base)$}
            \node[wordnode,anchor=base] () at (\XCoord,\YCoord) {Railway};
            \ExtractX{$(softmax9.north)$}
            \ExtractY{$(decwordout.base)$}
            \node[wordnode,anchor=base] () at (\XCoord,\YCoord) {Station};
            \ExtractX{$(softmax10.north)$}
            \ExtractY{$(decwordout.base)$}
            \node[wordnode,anchor=base] () at (\XCoord,\YCoord) {EOS};

            % Connections
            \draw[-latex'] (init1.east) to (enc11.west);
            \draw[-latex'] (init2.east) to (enc21.west);
            \foreach \x in {1,2,...,10}
                \draw[-latex'] (eemb\x) to (enc1\x);
            \foreach \x in {1,2,...,10}
                \draw[-latex'] (enc1\x) to (enc2\x);
            \foreach \x in {1,2,...,10}
                \draw[-latex'] (demb\x) to (dec1\x);
            \foreach \x in {1,2,...,10}
                \draw[-latex'] (dec1\x) to (dec2\x);
            \foreach \x in {1,2,...,10}
                \draw[-latex'] (dec2\x.north) to ([yshift=0.5\base]dec2\x.north);
            \foreach \x [count=\y from 2] in {1,2,...,9}
            {
                \draw[-latex'] (enc1\x.east) to (enc1\y.west);
                \draw[-latex'] (enc2\x.east) to (enc2\y.west);
                \draw[-latex'] (dec1\x.east) to (dec1\y.west);
                \draw[-latex'] (dec2\x.east) to (dec2\y.west);
            }

            \coordinate (bridge) at ([yshift=1.4\base]enc16.north west);
            \draw[-latex'] (enc210.north) .. controls +(north:0.4\base) and +(east:1.5\base) .. (bridge) .. controls +(west:9.5\base) and +(west:0.6\base) .. (dec21.west);

            \coordinate (bridge) at ([yshift=1.6\base]enc16.north west);
            \draw[-latex'] (enc110.east) .. controls +(east:0.5\base) and +(east:8\base) .. (bridge) .. controls +(west:8\base) and +(west:0.3\base) .. (dec11.west);

            % stack RNN
            \begin{pgfonlayer}{background}
                \node[draw=red,thick,densely dashed,inner sep=5pt] [fit = (init2) (enc21) (enc210)] (enc2) {};
                \node[draw=red,thick,densely dashed,inner sep=5pt] [fit = (dec21) (dec210)] (dec2) {};
            \end{pgfonlayer}
            \node[font=\scriptsize,anchor=west] (label) at ([xshift=0.2\base]demb10.east) {堆叠RNN};
            \draw[->,dashed] (label.north) to (dec2.east);
            \draw[->,dashed] (label.south) to (enc2.east);
        \end{scope}
    \end{tikzpicture}
\end{center}
\end{frame}

%%%------------------------------------------------------------------------------------------------------------
\subsection{注意力机制}

%%%------------------------------------------------------------------------------------------------------------
%%% 简单的编码-解码框架的问题
\begin{frame}{简单的编码器-解码器就足够了？}
\begin{itemize}
\item 将源语言句子编码为一个实数向量确实很神奇，但是也有明显问题
    \begin{itemize}
    \item 整个句子编码到一个向量里可能会有信息丢失
    \item 缺少源语单词与目标语单词之间的对应。某种意义上讲，一个目标语单词的生成无法区分不同源语单词的贡献
    \end{itemize}
\item<2-> 但是，翻译是具有很强的\alert{局部性}的，有些词之间会有更紧密的关系
    \begin{itemize}
    \item 源语词和目标语词的对应并不是均匀的，甚至非常稀疏
    \item 比如，一些短语的生成仅依赖于源文中的少数词
    \item<3-> 这些关系可以在表示模型中考虑
    \end{itemize}
\end{itemize}

\begin{center}
\begin{tikzpicture}
\begin{scope}
\setlength{\mystep}{1.6em}

\foreach \x in {1,2,...,6}
    \node[] (s\x) at (\x * \mystep,0) {};

\node [] (ws1) at (s1) {\scriptsize{这}};
\node [] (ws2) at (s2) {\scriptsize{是}};
\node [] (ws3) at (s3) {\scriptsize{个}};
\node [] (ws4) at (s4) {\scriptsize{很长}};
\node [] (ws5) at (s5) {\scriptsize{的}};
\node [] (ws6) at (s6) {\scriptsize{句子}};

\foreach \x in {1,2,...,6}
    \node[] (t\x) at (\x * \mystep + 2.4in,0) {};

\node [] (wt1) at (t1) {\scriptsize{This}};
\node [] (wt2) at (t2) {\scriptsize{is}};
\node [] (wt3) at ([yshift=-1pt]t3) {\scriptsize{a}};
\node [] (wt4) at ([yshift=-0.1em]t4) {\scriptsize{very}};
\node [] (wt5) at (t5) {\scriptsize{long}};
\node [] (wt6) at ([xshift=1em]t6) {\scriptsize{sentence}};

\node [anchor=south west,fill=red!30,minimum width=1.6in,minimum height=1.5em] (encoder) at ([yshift=1.0em]ws1.north west) {\footnotesize{Encoder}};
\node [anchor=west,fill=blue!30,minimum width=1.9in,minimum height=1.5em] (decoder) at ([xshift=4.5em]encoder.east) {\footnotesize{Decoder}};
\node [anchor=west,fill=green!30,minimum height=1.5em] (representation) at ([xshift=1em]encoder.east) {\footnotesize{表示}};
\draw [->,thick] ([xshift=1pt]encoder.east)--([xshift=-1pt]representation.west);
\draw [->,thick] ([xshift=1pt]representation.east)--([xshift=-1pt]decoder.west);

\foreach \x in {1,2,...,6}
    \draw[->] ([yshift=0.1em]s\x.north) -- ([yshift=1.2em]s\x.north);

\foreach \x in {1,2,...,5}
    \draw[<-] ([yshift=0.1em]t\x.north) -- ([yshift=1.2em]t\x.north);

\draw[<-] ([yshift=0.1em,xshift=1em]t6.north) -- ([yshift=1.2em,xshift=1em]t6.north);

\visible<2->{
\draw [<->,ublue,thick] ([xshift=0.3em]ws4.south) .. controls +(-60:1) and +(south:1) .. (wt4.south);
\draw [<->,ublue,thick] (ws4.south) .. controls +(south:1.0) and +(south:1.5) .. (wt5.south);
}

\visible<3->{
\node [anchor=north,fill=green!30,draw=ublue] (attentionlabel) at ([yshift=-3.4em]representation.south) {\footnotesize{词语的关注度}};
\draw [->,dotted,very thick,ublue] ([yshift=0.1em]attentionlabel.north)--([yshift=-0.1em]representation.south);
}

\end{scope}
\end{tikzpicture}
\end{center}

\end{frame}

%%%------------------------------------------------------------------------------------------------------------
%%% 注意力机制
\begin{frame}{注意力机制}
\begin{itemize}
\item 关注的``局部性''在图像处理、语音识别等领域也有广泛讨论，比如，对于下图
    \begin{itemize}
    \item 关注的顺序：大狗的帽子 $\to$ 大狗 $\to$ 小狗的帽子 $\to$ 小狗
    \end{itemize}
\item 人往往不是``均匀地''看图像中的所有区域，翻译是一个道理，生成一个目标语单词时参考的源语单词不会太多
\end{itemize}

\begin{center}
\includegraphics[scale=0.20]{./Figures/dog-hat.jpg}
\end{center}

\begin{itemize}
\item<2-> \alert{注意力机制}在机器翻译中已经成功应用，经典的论文\\
\textbf{Neural Machine Translation by Jointly Learning to Align and Translate}\\
\textbf{Bahdanau et al., 2015, In Proc of ICLR}
\end{itemize}

\end{frame}

%%%------------------------------------------------------------------------------------------------------------
%%% 注意力机制的简单示例
\begin{frame}{神经机器翻译的注意力机制}
\begin{itemize}
\item 在注意力机制中，每个目标语单词的生成会使用一个动态的源语表示，而非一个统一的固定表示
    \begin{itemize}
    \item 这里$\textbf{C}_i$表示第$i$个目标语单词所使用的源语表示
    \end{itemize}
\end{itemize}

\vspace{0.4em}
\begin{center}
\begin{tikzpicture}

\setlength{\mystep}{1.6em}

%%% a simple encoder-decoder model
\begin{scope}
\foreach \x in {1,2,...,6}
    \node[] (s\x) at (\x * \mystep,0) {};

\node [] (ws1) at (s1) {\scriptsize{这}};
\node [] (ws2) at (s2) {\scriptsize{是}};
\node [] (ws3) at (s3) {\scriptsize{个}};
\node [] (ws4) at (s4) {\scriptsize{很长}};
\node [] (ws5) at (s5) {\scriptsize{的}};
\node [] (ws6) at (s6) {\scriptsize{句子}};

\foreach \x in {1,2,...,6}
    \node[] (t\x) at (\x * \mystep + 2.4in,0) {};

\node [] (wt1) at (t1) {\scriptsize{This}};
\node [] (wt2) at (t2) {\scriptsize{is}};
\node [] (wt3) at ([yshift=-1pt]t3) {\scriptsize{a}};
\node [] (wt4) at ([yshift=-0.1em]t4) {\scriptsize{very}};
\node [] (wt5) at (t5) {\scriptsize{long}};
\node [] (wt6) at ([xshift=1em]t6) {\scriptsize{sentence}};

\node [anchor=south west,fill=red!30,minimum width=1.6in,minimum height=1.5em] (encoder) at ([yshift=1.0em]ws1.north west) {\footnotesize{Encoder}};
\node [anchor=west,fill=blue!30,minimum width=1.9in,minimum height=1.5em] (decoder) at ([xshift=4.5em]encoder.east) {\footnotesize{Decoder}};
\node [anchor=west,fill=green!30,minimum height=1.5em] (representation) at ([xshift=1em]encoder.east) {\footnotesize{表示}};
\draw [->,thick] ([xshift=1pt]encoder.east)--([xshift=-1pt]representation.west);
\draw [->,thick] ([xshift=1pt]representation.east)--([xshift=-1pt]decoder.west);

\foreach \x in {1,2,...,6}
    \draw[->] ([yshift=0.1em]s\x.north) -- ([yshift=1.2em]s\x.north);

\foreach \x in {1,2,...,5}
    \draw[<-] ([yshift=0.1em]t\x.north) -- ([yshift=1.2em]t\x.north);

\draw[<-] ([yshift=0.1em,xshift=1em]t6.north) -- ([yshift=1.2em,xshift=1em]t6.north);
\node [anchor=north] (cap) at ([xshift=2em,yshift=-2.5em]encoder.south east) {\scriptsize{(a) 简单的编码器-解码器框架}};

\end{scope}

%%% a encoder-decoder model with attention
\begin{scope}[yshift=-1.7in]
\foreach \x in {1,2,...,6}
    \node[] (s\x) at (\x * \mystep,0) {};

\node [] (ws1) at (s1) {\scriptsize{这}};
\node [] (ws2) at (s2) {\scriptsize{是}};
\node [] (ws3) at (s3) {\scriptsize{个}};
\node [] (ws4) at (s4) {\scriptsize{很长}};
\node [] (ws5) at (s5) {\scriptsize{的}};
\node [] (ws6) at (s6) {\scriptsize{句子}};

\foreach \x in {1,2,...,6}
    \node[] (t\x) at (\x * \mystep + 2.4in,0) {};

\node [] (wt1) at (t1) {\scriptsize{This}};
\node [] (wt2) at (t2) {\scriptsize{is}};
\node [] (wt3) at ([yshift=-1pt]t3) {\scriptsize{a}};
\node [] (wt4) at ([yshift=-0.1em]t4) {\scriptsize{very}};
\node [] (wt5) at (t5) {\scriptsize{long}};
\node [] (wt6) at ([xshift=1em]t6) {\scriptsize{sentence}};

\node [anchor=south west,fill=red!30,minimum width=1.6in,minimum height=1.5em] (encoder) at ([yshift=1.0em]ws1.north west) {\footnotesize{Encoder}};
\node [anchor=west,fill=blue!30,minimum width=1.9in,minimum height=1.5em] (decoder) at ([xshift=4.5em]encoder.east) {\footnotesize{Decoder}};

\foreach \x in {1,2,...,6}
    \draw[->] ([yshift=0.1em]s\x.north) -- ([yshift=1.2em]s\x.north);

\foreach \x in {1,2,...,5}
    \draw[<-] ([yshift=0.1em]t\x.north) -- ([yshift=1.2em]t\x.north);

\draw[<-] ([yshift=0.1em,xshift=1em]t6.north) -- ([yshift=1.2em,xshift=1em]t6.north);

\draw [->] ([yshift=3em]s6.north) -- ([yshift=4em]s6.north) -- ([yshift=4em]t1.north) node [pos=0.5,fill=green!30,inner sep=2pt] (c1) {\scriptsize{表示$\textbf{C}_1$}} -- ([yshift=3em]t1.north) ;
\draw [->] ([yshift=3em]s5.north) -- ([yshift=5.3em]s5.north) -- ([yshift=5.3em]t2.north) node [pos=0.5,fill=green!30,inner sep=2pt] (c2) {\scriptsize{表示$\textbf{C}_2$}} -- ([yshift=3em]t2.north) ;
\draw [->] ([yshift=3.5em]s3.north) -- ([yshift=6.6em]s3.north) -- ([yshift=6.6em]t4.north) node [pos=0.5,fill=green!30,inner sep=2pt] (c3) {\scriptsize{表示$\textbf{C}_i$}} -- ([yshift=3.5em]t4.north) ;
\node [anchor=north] (smore) at ([yshift=3.5em]s3.north) {...};
\node [anchor=north] (tmore) at ([yshift=3.5em]t4.north) {...};

\node [anchor=north] (cap) at ([xshift=2em,yshift=-2.5em]encoder.south east) {\scriptsize{(b) 引入注意力机制的编码器-解码器框架}};

\end{scope}

\end{tikzpicture}
\end{center}

\end{frame}

%%%------------------------------------------------------------------------------------------------------------
%%% C_i的定义
\begin{frame}{上下文向量$\textbf{C}_i$}
\begin{itemize}
\item 对于目标语位置$i$，$\textbf{C}_i$是目标语$i$使用的上下文向量
	\begin{itemize}
	\item $\textbf{h}_j$表示编码器第$j$个位置的隐层状态
	\item $\textbf{s}_i$表示解码器第$i$个位置的隐层状态
	\item<2-> $\alpha_{i,j}$表示注意力权重，表示目标语第$i$个位置与源语第$j$个位置之间的相关性大小
	\item<2-> $a(\cdot)$表示注意力函数，计算$\textbf{s}_{i-1}$和$\textbf{h}_j$之间的相关性
	\item<3-> $\textbf{C}_i$是所有源语编码表示$\{\textbf{h}_j\}$的加权求和，权重为$\{\alpha_{i,j}\}$
	\end{itemize}
\end{itemize}

\begin{center}
\begin{tikzpicture}

\begin{scope}

\node [anchor=west,draw,fill=red!20!white,inner sep=3pt,minimum width=2em,minimum height=1.2em] (h1) at (0,0) {\scriptsize{$\textbf{h}_1$}};
\node [anchor=west,draw,fill=red!20!white,inner sep=3pt,minimum width=2em,minimum height=1.2em] (h2) at ([xshift=1em]h1.east) {\scriptsize{$\textbf{h}_2$}};
\node [anchor=west,inner sep=0pt,minimum width=3em] (h3) at ([xshift=0.5em]h2.east) {\scriptsize{...}};
\node [anchor=west,draw,fill=red!20!white,inner sep=3pt,minimum width=2em,minimum height=1.2em] (h4) at ([xshift=0.5em]h3.east) {\scriptsize{$\textbf{h}_n$}};

\node [anchor=south,circle,minimum size=1.0em,draw,ublue,thick] (sum) at ([yshift=2em]h2.north east) {};
\draw [thick,-,ublue] (sum.north) -- (sum.south);
\draw [thick,-,ublue] (sum.west) -- (sum.east);

\node [anchor=south,draw,fill=green!20!white,inner sep=3pt,minimum width=2em,minimum height=1.2em] (th1) at ([yshift=2em,xshift=-1em]sum.north west) {\scriptsize{$\textbf{s}_{i-1}$}};
\node [anchor=west,draw,fill=green!20!white,inner sep=3pt,minimum width=2em,minimum height=1.2em] (th2) at ([xshift=2em]th1.east) {\scriptsize{$\textbf{s}_{i}$}};

\draw [->] (h1.north) .. controls +(north:0.8) and +(west:1) ..  (sum.190) node [pos=0.3,left] {\tiny{$\alpha_{i,1}$}};
\draw [->] (h2.north) .. controls +(north:0.6) and +(220:0.2) ..  (sum.220) node [pos=0.2,right] {\tiny{$\alpha_{i,2}$}};
\draw [->] (h4.north) .. controls +(north:0.8) and +(east:1) ..  (sum.-10) node [pos=0.1,left] (alphan) {\tiny{$\alpha_{i,n}$}};

\draw [->] ([xshift=-1.5em]th1.west) -- ([xshift=-0.1em]th1.west);
\draw [->] ([xshift=0.1em]th1.east) -- ([xshift=-0.1em]th2.west);
\draw [->] ([xshift=0.1em]th2.east) -- ([xshift=1.5em]th2.east);
\draw [->] (sum.north) .. controls +(north:0.8) and +(west:0.2) ..  ([yshift=-0.4em,xshift=-0.1em]th2.west) node [pos=0.2,right] (ci) {\scriptsize{$\textbf{C}_{i}$}};

\node [anchor=south,inner sep=1pt] (output) at ([yshift=0.8em]th2.north) {\tiny{输出层}};
\draw [->] ([yshift=0.1em]th2.north) -- ([yshift=-0.1em]output.south);
\node [anchor=north] (enc1) at (h1.south) {\tiny{编码器输出}};
\node [anchor=north] (enc12) at ([yshift=0.5em]enc1.south) {\tiny{(位置$1$)}};
\node [anchor=north] (enc2) at (h2.south) {\tiny{编码器输出}};
\node [anchor=north] (enc22) at ([yshift=0.5em]enc2.south) {\tiny{(位置$2$)}};
\node [anchor=north] (enc4) at (h4.south) {\tiny{编码器输出}};
\node [anchor=north] (enc42) at ([yshift=0.5em]enc4.south) {\tiny{(位置$4$)}};

\visible<2->{
\node [anchor=west] (math1) at ([xshift=5em,yshift=1em]th2.east) {$\textbf{C}_i = \sum_{j} \alpha_{i,j} \textbf{h}_j \ \ $};
}
\visible<3->{
\node [anchor=north west] (math2) at ([yshift=-2em]math1.south west) {$\alpha_{i,j} = \frac{\exp(\beta_{i,j})}{\sum_{j'} \exp(\beta_{i,j'})}$};
\node [anchor=north west] (math3) at ([yshift=-0em]math2.south west) {$\beta_{i,j} = a(\textbf{s}_{i-1}, \textbf{h}_j)$};
}

\begin{pgfonlayer}{background}
\visible<2->{
\node [rectangle,inner sep=0.4em,rounded corners=1pt,fill=blue!10,drop shadow] [fit = (math1)] (box1) {};
}
\visible<3->{
\node [rectangle,inner sep=0.4em,rounded corners=1pt,fill=orange!10,drop shadow] [fit = (math2) (math3)] (box2) {};
}
\end{pgfonlayer}

\visible<2->{
\draw [->,dotted,thick,blue] (box1.west) .. controls +(west:1.2) and +(east:2.0) .. ([xshift=-0.3em]ci.east);
}
\visible<3->{
\draw [->,dotted,thick,orange] ([yshift=1em]box2.west) .. controls +(west:1.2) and +(east:1.0) .. ([xshift=-0.35em]alphan.east);
}

\end{scope}

\end{tikzpicture}
\end{center}

\end{frame}

%%%------------------------------------------------------------------------------------------------------------
%%% 注意力权重的可视化
\begin{frame}{注意力权重$\alpha_{ij}$}

\begin{itemize}
\item 注意力权重$\alpha_{ij}$的可视化
\end{itemize}

\vspace{-1.5em}

\begin{center}
\begin{tikzpicture}

\setlength{\wseg}{1.5cm}
\setlength{\hseg}{1.0cm}
\setlength{\wnode}{3.75cm}
\setlength{\hnode}{1.0cm}

\tikzstyle{elementnode} = [rectangle,text=white,anchor=center]
\tikzstyle{srcnode} = [rotate=45,font=\small,anchor=south west]
\tikzstyle{tgtnode} = [left,font=\small,anchor=north east]
\tikzstyle{alignmentnode} = [rectangle,draw,minimum height=3.6\hnode,minimum width=0.36\hnode]
\tikzstyle{probnode} = [fill=blue!30,minimum width=0.4\hnode]
\tikzstyle{labelnode} = [above]

% alignment matrix
\begin{scope}[scale=0.9,yshift=0.12in]
\foreach \i / \j / \c in
    {0/7/0.2, 1/7/0.45, 2/7/0.15, 3/7/0.15, 4/7/0.15, 5/7/0.15,
    0/6/0.35, 1/6/0.45, 2/6/0.15, 3/6/0.15, 4/6/0.15, 5/6/0.15,
    0/5/0.25, 1/5/0.15, 2/5/0.15, 3/5/0.35, 4/5/0.15, 5/5/0.15,
    0/4/0.15, 1/4/0.25, 2/4/0.2, 3/4/0.30, 4/4/0.15, 5/4/0.15,
    0/3/0.15, 1/3/0.15, 2/3/0.8, 3/3/0.25, 4/3/0.15, 5/3/0.25,
    0/2/0.15, 1/2/0.15, 2/2/0.15, 3/2/0.15, 4/2/0.25, 5/2/0.3,
    0/1/0.15, 1/1/0.15, 2/1/0.15, 3/1/0.15, 4/1/0.8, 5/1/0.15,
    0/0/0.15, 1/0/0.15, 2/0/0.15, 3/0/0.15, 4/0/0.25, 5/0/0.60}
    \node[elementnode,minimum size=0.6*\hnode*\c,inner sep=0.1pt,fill=blue] (a\i\j) at (0.5*\hnode*\i-5.4*0.5*\hnode,0.5*\hnode*\j-1.05*\hnode) {};

%attention score labels
\node[align=center] (l17) at (a17) {\scriptsize{{\color{white} .4}}};
\node[align=center] (l26) at (a06) {\scriptsize{{\color{white} .3}}};
\node[align=center] (l26) at (a16) {\scriptsize{{\color{white} .4}}};
\node[align=center] (l17) at (a35) {\scriptsize{{\color{white} .3}}};
\node[align=center] (l17) at (a34) {\tiny{{\color{white} .3}}};
\node[align=center] (l17) at (a23) {\small{{\color{white} .8}}};
\node[align=center] (l17) at (a41) {\small{{\color{white} .8}}};
\node[align=center] (l17) at (a50) {\small{{\color{white} .7}}};

% source
\node[srcnode] (src1) at (-5.4*0.5*\hnode,-1.05*\hnode+7.5*0.5*\hnode) {\scriptsize{Have}};
\node[srcnode] (src2) at ([xshift=0.5\hnode]src1.south west) {\scriptsize{you}};
\node[srcnode] (src3) at ([xshift=0.5\hnode]src2.south west) {\scriptsize{learned}};
\node[srcnode] (src4) at ([xshift=0.5\hnode]src3.south west) {\scriptsize{nothing}};
\node[srcnode] (src5) at ([xshift=0.5\hnode]src4.south west) {\scriptsize{?}};
\node[srcnode] (src6) at ([xshift=0.5\hnode]src5.south west) {\scriptsize{EOS}};

% target
\node[tgtnode] (tgt1) at (-6.0*0.5*\hnode,-1.05*\hnode+7.5*0.5*\hnode) {\scriptsize{你}};
\node[tgtnode] (tgt2) at ([yshift=-0.5\hnode]tgt1.north east) {\scriptsize{什么}};
\node[tgtnode] (tgt3) at ([yshift=-0.5\hnode]tgt2.north east) {\scriptsize{都}};
\node[tgtnode] (tgt4) at ([yshift=-0.5\hnode]tgt3.north east) {\scriptsize{没}};
\node[tgtnode] (tgt5) at ([yshift=-0.5\hnode]tgt4.north east) {\scriptsize{学}};
\node[tgtnode] (tgt6) at ([yshift=-0.5\hnode]tgt5.north east) {\scriptsize{到}};
\node[tgtnode] (tgt7) at ([yshift=-0.5\hnode]tgt6.north east) {\scriptsize{?}};
\node[tgtnode] (tgt8) at ([yshift=-0.5\hnode]tgt7.north east) {\scriptsize{EOS}};

\end{scope}

\visible<2->{
% alignment rectangle 2
\node[alignmentnode, ugreen, anchor=north west] (alignment1) at ([xshift=-0.3em,yshift=0.4em]a07.north west) {};
}

\visible<3->{
% alignment rectangle 1
\node[alignmentnode, red, anchor=north west] (alignment2) at ([xshift=-0.1em,yshift=0.2em]a17.north west) {};
}

\visible<3->{
% alignment bars 2
\node[probnode,anchor=south west,minimum height=0.4\hnode,inner sep=0.1pt,fill=red!40,label=below:\scriptsize{$0.4$}] (attn21) at ([xshift=2.3\hnode,yshift=0.5\hnode]alignment2.east) {};
\node[probnode,anchor=south west,minimum height=0.4\hnode,inner sep=0.1pt,fill=red!40,label=below:\scriptsize{$0.4$}] (attn22) at ([xshift=1pt]attn21.south east) {};
\node[probnode,anchor=south west,minimum height=0.05\hnode,inner sep=0.1pt,fill=red!40,label=below:\scriptsize{$0$}] (attn23) at ([xshift=1pt]attn22.south east) {};
\node[probnode,anchor=south west,minimum height=0.1\hnode,inner sep=0.1pt,fill=red!40,label=below:\scriptsize{$0.1$}] (attn24) at ([xshift=1pt]attn23.south east) {};
\node[probnode,anchor=south west,minimum height=0.05\hnode,inner sep=0.1pt,fill=red!40,label=below:\scriptsize{$0$}] (attn25) at ([xshift=1pt]attn24.south east) {};
\node[probnode,anchor=south west,minimum height=0.05\hnode,inner sep=0.1pt,fill=red!40,label=below:\scriptsize{$...$}] (attn26) at ([xshift=1pt]attn25.south east) {};
}

\visible<2->{
% alignment bars 1
\node[probnode,anchor=south,minimum height=0.2\hnode,inner sep=0.1pt,fill=ugreen!40,label=below:\scriptsize{$0.2$}] (attn11) at ([xshift=2.5\hnode,yshift=-1em]alignment2.north east) {};
\node[probnode,anchor=south west,minimum height=0.3\hnode,inner sep=0.1pt,fill=ugreen!40,label=below:\scriptsize{$0.3$}] (attn12) at ([xshift=1pt]attn11.south east) {};
\node[probnode,anchor=south west,minimum height=0.2\hnode,inner sep=0.1pt,fill=ugreen!40,label=below:\scriptsize{$0.2$}] (attn13) at ([xshift=1pt]attn12.south east) {};
\node[probnode,anchor=south west,minimum height=0.05\hnode,inner sep=0.1pt,fill=ugreen!40,label=below:\scriptsize{$0$}] (attn14) at ([xshift=1pt]attn13.south east) {};
\node[probnode,anchor=south west,minimum height=0.05\hnode,inner sep=0.1pt,fill=ugreen!40,label=below:\scriptsize{$0$}] (attn15) at ([xshift=1pt]attn14.south east) {};
\node[probnode,anchor=south west,minimum height=0.05\hnode,inner sep=0.1pt,fill=ugreen!40,label=below:\scriptsize{$...$}] (attn16) at ([xshift=1pt]attn15.south east) {};
}

\visible<3->{
% coverage score formula node
\node [anchor=north west] (formula) at ([xshift=-0.3\hnode,yshift=-1.5\hnode]attn11.south) {\small{不同$\textbf{C}_i$所对应的源语言词的权重是不同的}};
\node [anchor=north west] (example) at (formula.south west) {\footnotesize{$\textbf{C}_2=0.4 \times \textbf{h}(\textrm{``你''}) + 0.4 \times \textbf{h}(\textrm{``什么''}) +$}};
\node [anchor=north west] (example2) at ([yshift=0.4em]example.south west) {\footnotesize{$\ \ \ \ \ \ \ \ 0 \times \textbf{h}(\textrm{``都''}) + 0.1 \times \textbf{h}(\textrm{``没''}) + ..$}};
}

\visible<3->{
% matrix -> attn2
\draw[->,red] ([xshift=0.1em,yshift=2.3em]alignment2.east).. controls +(east:1.9cm) and +(west:1.0cm) ..([xshift=-0.15\hnode,yshift=-1em]attn21.north west);
}

\visible<2->{
\draw[->,ugreen] ([xshift=0.1em,yshift=-1.2em]alignment1.north east)--([xshift=2.2\hnode,yshift=-1.2em]alignment2.north east);
}

\visible<3->{
% attn2 -> cov2
\draw[->] ([xshift=0.2\hnode,yshift=0.0\hnode]attn26.east)--([xshift=0.7\hnode,yshift=0]attn26.east) node[pos=0.5,above] (sum2) {\small{$\sum$}}; % 0.3 - 0.5 height of the
}

\visible<2->{
% attn1 -> cov1
\draw[->] ([xshift=0.2\hnode]attn16.east)--([xshift=0.7\hnode]attn16.east) node[pos=0.5,above] (sum1) {\small{$\sum$}};
}

% coverage score for each source word
\visible<2->{
\node[anchor=west] (sc1) at ([xshift=0.9\hnode]attn16.east) {$\textbf{C}_1 = \sum_{i=1}^{8} \alpha_{i1} \textbf{h}_{i}$};
}

\visible<3->{
\node[anchor=west] (sc2) at ([xshift=0.9\hnode,yshift=0.0\hnode]attn26.east) {$\textbf{C}_2 = \sum_{i=1}^{8} \alpha_{i2} \textbf{h}_{i}$};
}

\end{tikzpicture}
\end{center}

\visible<4->{
\begin{itemize}
\item 对比
\end{itemize}

\begin{center}
{\small
\begin{tabular}{l | l}
引入注意力机制以前 & 引入注意力机制以后 \\ \hline
$\textrm{``Have''} = \argmax_{y} \textrm{P}(y|0, \alert{\textbf{C}})$ & $\textrm{``Have''} = \argmax_{y} \textrm{P}(y|0, \alert{\textbf{C}_1})$ \\
$\textrm{``you''} = \argmax_{y} \textrm{P}(y|\textbf{s}_1, \alert{\textbf{C}})$ & $\textrm{``you''} = \argmax_{y} \textrm{P}(y|\textbf{s}_1, \alert{\textbf{C}_2})$
\end{tabular}
}
\end{center}
}

\end{frame}

%%%------------------------------------------------------------------------------------------------------------
%%% 如何定义注意力函数
\begin{frame}{计算注意力权重 - 注意力函数}
\begin{itemize}
\item 再来看一下注意力权重的定义。这个过程实际上是对$a(\cdot,\cdot)$做指数归一化：\\
\vspace{-0.3em}
\begin{displaymath}
\alpha_{i,j} = \frac{\exp(a(\textbf{s}_{i-1}, \textbf{h}_j))}{\sum_{j'} \exp(a(\textbf{s}_{i-1}, \textbf{h}_{j'}))}
\end{displaymath}

\item<2-> 注意力函数$a(\textbf{s},\textbf{h})$的目的是捕捉$\textbf{s}$和$\textbf{h}$之间的\alert{相似性}，这也可以被看作是目标语表示和源语言表示的一种``统一化''，即把源语言和目标语表示在同一个语义空间，进而语义相近的内容有更大的相似性。\visible<3->{定义$a(\textbf{s},\textbf{h})$的方式：}
    \vspace{-1em}

    \visible<3->{
    \begin{displaymath}
    a(\textbf{s},\textbf{h}) =  \left\{ \begin{array}{ll}
    \textbf{s} \textbf{h}^{\textrm{T}} & \textrm{向量乘} \\
    \textrm{cos}(\textbf{s}, \textbf{h}) & \textrm{向量夹角} \\
    \textbf{s} \textbf{W} \textbf{h}^{\textrm{T}} & \textrm{线性模型} \\
    \textrm{TanH}(\textbf{W}[\textbf{s},\textbf{h}])\textbf{v}^{\textrm{T}} & \textrm{拼接}[\textbf{s},\textbf{h}]+\textrm{单层网络}
    \end{array}
    \right.
    \end{displaymath}
    \vspace{-0.3em}
    $\textbf{W}$和$\textbf{v}$是可学习参数
    }
\end{itemize}
\end{frame}

%%%------------------------------------------------------------------------------------------------------------
%%% 注意力模型的效果 - 热图
\begin{frame}{真实的实例}
\begin{itemize}
\item 注意力的权重符合双语对应的规律
\end{itemize}
\end{frame}

%%%------------------------------------------------------------------------------------------------------------
%%% 如何进一步理解注意力机制
\begin{frame}{重新解释注意力机制}
\begin{itemize}
\item 换一个问题，假设有若干key-value单元，其中key是这个单元的索引表示，value是这个单元的值。对于任意一个query，可以找到匹配的key，并输出其对应的value
\end{itemize}

\vspace{-0.8em}
\begin{center}
\begin{tikzpicture}
\begin{scope}

\tikzstyle{rnode} = [draw,minimum width=3em,minimum height=1.2em]

\node [rnode,anchor=south west,fill=blue!20!white] (value1) at (0,0) {\scriptsize{value$_1$}};
\node [rnode,anchor=south west,fill=blue!20!white] (value2) at ([xshift=1em]value1.south east) {\scriptsize{value$_2$}};
\node [rnode,anchor=south west,fill=red!20!white] (value3) at ([xshift=1em]value2.south east) {\scriptsize{value$_3$}};
\node [rnode,anchor=south west,fill=blue!20!white] (value4) at ([xshift=1em]value3.south east) {\scriptsize{value$_4$}};

\node [rnode,anchor=south west,pattern=north east lines] (key1) at ([yshift=0.2em]value1.north west) {};
\node [rnode,anchor=south west,pattern=dots] (key2) at ([yshift=0.2em]value2.north west) {};
\node [rnode,anchor=south west,pattern=horizontal lines] (key3) at ([yshift=0.2em]value3.north west) {};
\node [rnode,anchor=south west,pattern=crosshatch dots] (key4) at ([yshift=0.2em]value4.north west) {};
\node [fill=white,inner sep=1pt] (key1label) at (key1) {\scriptsize{key$_1$}};
\node [fill=white,inner sep=1pt] (key1label) at (key2) {\scriptsize{key$_2$}};
\node [fill=white,inner sep=1pt] (key1label) at (key3) {\scriptsize{key$_3$}};
\node [fill=white,inner sep=1pt] (key1label) at (key4) {\scriptsize{key$_4$}};

\node [rnode,anchor=east,pattern=horizontal lines] (query) at ([xshift=-3em]key1.west) {};
\node [anchor=east] (querylabel) at ([xshift=-0.2em]query.west) {\scriptsize{query}};

\draw [->] ([yshift=1pt]query.north) .. controls +(90:2em) and +(90:2em) .. ([yshift=1pt]key3.north) node [pos=0.5,below,yshift=0.2em] {\scriptsize{匹配}};

\node [anchor=north] (result) at (value3.south) {\scriptsize{\alert{返回结果}}};

\end{scope}
\end{tikzpicture}
\end{center}

\vspace{-0.7em}

\begin{itemize}
\item<2-> 注意力机制也可以被看做对key-value单元的查询，但是所有key和query之间都有一种匹配程度，返回结果是对所有value的加权
\end{itemize}

\visible<2->{
\vspace{-0.5em}
\begin{center}
\begin{tikzpicture}
\begin{scope}

\tikzstyle{rnode} = [draw,minimum width=3em,minimum height=1.2em]

\node [rnode,anchor=south west,fill=red!20!white] (value1) at (0,0) {\scriptsize{value$_1$}};
\node [rnode,anchor=south west,fill=red!20!white] (value2) at ([xshift=1em]value1.south east) {\scriptsize{value$_2$}};
\node [rnode,anchor=south west,fill=red!20!white] (value3) at ([xshift=1em]value2.south east) {\scriptsize{value$_3$}};
\node [rnode,anchor=south west,fill=red!20!white] (value4) at ([xshift=1em]value3.south east) {\scriptsize{value$_4$}};

\node [rnode,anchor=south west,pattern=north east lines] (key1) at ([yshift=0.2em]value1.north west) {};
\node [rnode,anchor=south west,pattern=dots] (key2) at ([yshift=0.2em]value2.north west) {};
\node [rnode,anchor=south west,pattern=horizontal lines] (key3) at ([yshift=0.2em]value3.north west) {};
\node [rnode,anchor=south west,pattern=crosshatch dots] (key4) at ([yshift=0.2em]value4.north west) {};
\node [fill=white,inner sep=1pt] (key1label) at (key1) {\scriptsize{key$_1$}};
\node [fill=white,inner sep=1pt] (key1label) at (key2) {\scriptsize{key$_2$}};
\node [fill=white,inner sep=1pt] (key1label) at (key3) {\scriptsize{key$_3$}};
\node [fill=white,inner sep=1pt] (key1label) at (key4) {\scriptsize{key$_4$}};

\node [rnode,anchor=east,pattern=vertical lines] (query) at ([xshift=-3em]key1.west) {};
\node [anchor=east] (querylabel) at ([xshift=-0.2em]query.west) {\scriptsize{query}};

\draw [->] ([yshift=1pt,xshift=6pt]query.north) .. controls +(90:1em) and +(90:1em) .. ([yshift=1pt]key1.north);
\draw [->] ([yshift=1pt,xshift=3pt]query.north) .. controls +(90:1.5em) and +(90:1.5em) .. ([yshift=1pt]key2.north);
\draw [->] ([yshift=1pt]query.north) .. controls +(90:2em) and +(90:2em) .. ([yshift=1pt]key3.north);
\draw [->] ([yshift=1pt,xshift=-3pt]query.north) .. controls +(90:2.5em) and +(90:2.5em) .. ([yshift=1pt]key4.north);
\node [anchor=south east] (alpha1) at (key1.north east) {\scriptsize{$\alpha_1$}};
\node [anchor=south east] (alpha2) at (key2.north east) {\scriptsize{$\alpha_2$}};
\node [anchor=south east] (alpha3) at (key3.north east) {\scriptsize{$\alpha_3$}};
\node [anchor=south east] (alpha4) at (key4.north east) {\scriptsize{$\alpha_4$}};

\node [anchor=north] (result) at ([xshift=-1.5em]value2.south east) {\scriptsize{\alert{返回结果}=$\alpha_1 \cdot \textrm{value}_1 + \alpha_2 \cdot \textrm{value}_2 + \alpha_3 \cdot \textrm{value}_3 + \alpha_4 \cdot \textrm{value}_4$}};

\end{scope}
\end{tikzpicture}
\end{center}
}

\end{frame}

%%%------------------------------------------------------------------------------------------------------------
%%% 如何进一步理解注意力机制 - 回到机器翻译任务
\begin{frame}{重新解释注意力机制(续)}
\begin{itemize}
\item 回到机器翻译，如果把目标语状态$\textbf{s}_{i-1}$看做query，而把源语言所有位置的最上层RNN表示$\textbf{h}_{j}$看做{\color{ugreen} \textbf{key}}和{\color{red} \textbf{value}}
\end{itemize}

\vspace{-1.5em}
\begin{center}
\begin{tikzpicture}
\begin{scope}

\tikzstyle{rnode} = [draw,minimum width=3.5em,minimum height=1.2em]

\node [rnode,anchor=south west,fill=red!20!white] (value1) at (0,0) {\scriptsize{$\textbf{h}(\textrm{``你''})$}};
\node [rnode,anchor=south west,fill=red!20!white] (value2) at ([xshift=1em]value1.south east) {\scriptsize{$\textbf{h}(\textrm{``什么''})$}};
\node [rnode,anchor=south west,fill=red!20!white] (value3) at ([xshift=1em]value2.south east) {\scriptsize{$\textbf{h}(\textrm{``也''})$}};
\node [rnode,anchor=south west,fill=red!20!white] (value4) at ([xshift=1em]value3.south east) {\scriptsize{$\textbf{h}(\textrm{``没''})$}};

\node [rnode,anchor=south west,fill=green!20!white] (key1) at ([yshift=0.2em]value1.north west) {\scriptsize{$\textbf{h}(\textrm{``你''})$}};
\node [rnode,anchor=south west,fill=green!20!white] (key2) at ([yshift=0.2em]value2.north west) {\scriptsize{$\textbf{h}(\textrm{``什么''})$}};
\node [rnode,anchor=south west,fill=green!20!white] (key3) at ([yshift=0.2em]value3.north west) {\scriptsize{$\textbf{h}(\textrm{``也''})$}};
\node [rnode,anchor=south west,fill=green!20!white] (key4) at ([yshift=0.2em]value4.north west) {\scriptsize{$\textbf{h}(\textrm{``没''})$}};

\node [rnode,anchor=east] (query) at ([xshift=-2em]key1.west) {\scriptsize{$\textbf{s}(\textrm{``you''})$}};
\node [anchor=east] (querylabel) at ([xshift=-0.2em]query.west) {\scriptsize{query}};

\draw [->] ([yshift=1pt,xshift=6pt]query.north) .. controls +(90:1em) and +(90:1em) .. ([yshift=1pt]key1.north);
\draw [->] ([yshift=1pt,xshift=3pt]query.north) .. controls +(90:1.5em) and +(90:1.5em) .. ([yshift=1pt]key2.north);
\draw [->] ([yshift=1pt]query.north) .. controls +(90:2em) and +(90:2em) .. ([yshift=1pt]key3.north);
\draw [->] ([yshift=1pt,xshift=-3pt]query.north) .. controls +(90:2.5em) and +(90:2.5em) .. ([yshift=1pt]key4.north);
\node [anchor=south east] (alpha1) at ([xshift=1em]key1.north east) {\scriptsize{$\alpha_1=.4$}};
\node [anchor=south east] (alpha2) at ([xshift=1em]key2.north east) {\scriptsize{$\alpha_2=.4$}};
\node [anchor=south east] (alpha3) at ([xshift=1em]key3.north east) {\scriptsize{$\alpha_3=0$}};
\node [anchor=south east] (alpha4) at ([xshift=1em]key4.north east) {\scriptsize{$\alpha_4=.1$}};

\end{scope}
\end{tikzpicture}
\end{center}

\vspace{-2.5em}
\begin{eqnarray}
\textbf{C}_3 & = & 0.4 \times \textbf{h}(\textrm{``什么''}) + 0.4 \times \textbf{h}(\textrm{``也''}) + \nonumber \\
             &   & 0 \times \textbf{h}(\textrm{``没''}) + 0.1 \times \textbf{h}(\textrm{``学''}) \nonumber
\end{eqnarray}

\vspace{-0.5em}
\begin{itemize}
\item<2-> 注意力机制也可以被看做是一个重新生成value的过程：对于一组value值，注意力模型对他们加权求和，并得到一个新的value。而这个新的value实际上就是query所对应查询结果，在机器翻译中被看做是目标语所对应的源语言上下文表示。
\end{itemize}

\end{frame}

\subsection{训练及推断}

%%%------------------------------------------------------------------------------------------------------------
%%% 训练
\begin{frame}{训练}
\end{frame}

%%%------------------------------------------------------------------------------------------------------------
%%% 解码
\begin{frame}{推断}
\begin{itemize}
\item 使用NMT时，对于源语言句子$\textbf{x}$，需要得到最优译文$\hat{\textbf{y}}$

\vspace{-1.5em}
\begin{displaymath}
\hat{\textbf{y}} = \argmax_{\textbf{y}} \log\textrm{P}(\textbf{y}|\textbf{x}) = \argmax_{\textbf{y}} \sum_{j=1}^{n} \log\textrm{P}(y_j|\textbf{y}_{<j}, \textbf{x})
\end{displaymath}

\item<2-> 由于$y_i$的生成需要依赖$y_{i-1}$，因此无法同时生成$\{y_1,...,y_n\}$。常用的方法是自左向右逐个单词生成

\end{itemize}

\vspace{-0.8em}
\visible<3->{
\vspace{-0.5em}
\begin{center}
\begin{tikzpicture}
\begin{scope}
\tikzstyle{rnnnode} = [minimum height=1.1em,minimum width=2.1em,inner sep=2pt,rounded corners=1pt,draw,fill=red!20];

\node [rnnnode,anchor=west] (h1) at (0,0) {\tiny{$\textbf{h}_1$}};
\node [anchor=west] (h2) at ([xshift=1em]h1.east) {\tiny{...}};
\node [rnnnode,anchor=west] (h3) at ([xshift=1em]h2.east) {\tiny{$\textbf{h}_m$}};
\node [rnnnode,anchor=north,fill=green!20] (e1) at ([yshift=-1em]h1.south) {\tiny{$e_x()$}};
\node [anchor=west] (e2) at ([xshift=1em]e1.east) {\tiny{...}};
\node [rnnnode,anchor=west,fill=green!20] (e3) at ([xshift=1em]e2.east) {\tiny{$e_x()$}};
\node [anchor=north,inner sep=2pt] (w1) at ([yshift=-0.6em]e1.south) {\tiny{你}};
\node [anchor=north,inner sep=2pt] (w2) at ([yshift=-0.8em]e2.south) {\tiny{...}};
\node [anchor=north,inner sep=2pt] (w3) at ([yshift=-0.6em]e3.south) {\tiny{EOS}};

\draw [->] (w1.north) -- ([yshift=-0.1em]e1.south);
\draw [->] (w3.north) -- ([yshift=-0.1em]e3.south);
\draw [->] ([yshift=0.1em]e1.north) -- ([yshift=-0.1em]h1.south);
\draw [->] ([yshift=0.1em]e3.north) -- ([yshift=-0.1em]h3.south);
\draw [->] ([xshift=0.1em]h1.east) -- ([xshift=-0.1em]h2.west);
\draw [->] ([xshift=0.1em]h2.east) -- ([xshift=-0.1em]h3.west);
\draw [->] ([xshift=-0.8em]h1.west) -- ([xshift=-0.1em]h1.west) node [pos=0,left,inner sep=2pt] {\tiny{0}};
\node [anchor=south] (encoder) at ([xshift=-0.2em]h1.north west) {\scriptsize{\textbf{编码器}}};

\visible<5->{
\node [rnnnode,anchor=west,fill=green!20] (t1) at ([xshift=3em]h3.east) {\tiny{$e_y()$}};
}
\visible<8->{
\node [rnnnode,anchor=west,fill=green!20] (t2) at ([xshift=1.5em]t1.east) {\tiny{$e_y()$}};
}
\visible<9->{
\node [rnnnode,anchor=west,fill=green!20] (t3) at ([xshift=1.5em]t2.east) {\tiny{$e_y()$}};
\node [rnnnode,anchor=west,fill=green!20] (t4) at ([xshift=1.5em]t3.east) {\tiny{$e_y()$}};
\node [anchor=west,inner sep=2pt] (t5) at ([xshift=0.3em]t4.east) {\tiny{...}};
}
\visible<5->{
\node [rnnnode,anchor=south] (s1) at ([yshift=1em]t1.north) {\tiny{$\textbf{s}_1$}};
}
\visible<8->{
\node [rnnnode,anchor=south] (s2) at ([yshift=1em]t2.north) {\tiny{$\textbf{s}_2$}};
}
\visible<9->{
\node [rnnnode,anchor=south] (s3) at ([yshift=1em]t3.north) {\tiny{$\textbf{s}_3$}};
\node [rnnnode,anchor=south] (s4) at ([yshift=1em]t4.north) {\tiny{$\textbf{s}_4$}};
\node [anchor=west,inner sep=2pt] (s5) at ([xshift=0.3em]s4.east) {\tiny{...}};
}
\visible<5->{
\node [rnnnode,anchor=south,fill=blue!20] (o1) at ([yshift=1em]s1.north) {\tiny{softmax}};
\node [anchor=east] (decoder) at ([xshift=-0.5em]o1.north west) {\scriptsize{\textbf{解码器}}};
}
\visible<8->{
\node [rnnnode,anchor=south,fill=blue!20] (o2) at ([yshift=1em]s2.north) {\tiny{softmax}};
}
\visible<9->{
\node [rnnnode,anchor=south,fill=blue!20] (o3) at ([yshift=1em]s3.north) {\tiny{softmax}};
\node [rnnnode,anchor=south,fill=blue!20] (o4) at ([yshift=1em]s4.north) {\tiny{softmax}};
\node [anchor=west,inner sep=2pt] (o5) at ([xshift=0.3em]o4.east) {\tiny{...}};
}
\visible<4->{
\node [anchor=north,inner sep=2pt] (wt1) at ([yshift=-0.6em]t1.south) {\tiny{EOS}};
}
\visible<7->{
\node [anchor=north,inner sep=2pt] (wt2) at ([yshift=-0.6em]t2.south) {\tiny{Have}};
}
\visible<9->{
\node [anchor=north,inner sep=2pt] (wt3) at ([yshift=-0.8em]t3.south) {\tiny{you}};
\node [anchor=north,inner sep=2pt] (wt4) at ([yshift=-0.6em]t4.south) {\tiny{learned}};
}
\visible<5->{
\node [anchor=center,inner sep=2pt] (wo1) at ([yshift=1.2em]o1.north) {\tiny{Have}};
}
\visible<4->{
\node [anchor=south,inner sep=2pt] (wos1) at (wo1.north) {\tiny{\textbf{[step 1]}}};
}
\visible<8->{
\node [anchor=center,inner sep=2pt] (wo2) at ([yshift=1.2em]o2.north) {\tiny{you}};
}
\visible<7->{
\node [anchor=south,inner sep=2pt] (wos2) at (wo2.north) {\tiny{\textbf{[step 2]}}};
}
\visible<9->{
\node [anchor=center,inner sep=2pt] (wo3) at ([yshift=1.2em]o3.north) {\tiny{learned}};
\node [anchor=south,inner sep=2pt] (wos3) at (wo3.north) {\tiny{\textbf{[step 3]}}};
\node [anchor=center,inner sep=2pt] (wo4) at ([yshift=1.2em]o4.north) {\tiny{nothing}};
\node [anchor=south,inner sep=2pt] (wos4) at (wo4.north) {\tiny{\textbf{[step 4]}}};
}

\visible<5->{
\foreach \x in {1}{
    \draw [->] ([yshift=-0.7em]t\x.south) -- ([yshift=-0.1em]t\x.south);
    \draw [->] ([yshift=0.1em]t\x.north) -- ([yshift=-0.1em]s\x.south);
    \draw [->] ([yshift=0.1em]s\x.north) -- ([yshift=-0.1em]o\x.south);
    \draw [->] ([yshift=0.1em]o\x.north) -- ([yshift=0.8em]o\x.north) node [pos=0.5,right] {\tiny{top1}};
}
}

\visible<8->{
\foreach \x in {2}{
    \draw [->] ([yshift=-0.7em]t\x.south) -- ([yshift=-0.1em]t\x.south);
    \draw [->] ([yshift=0.1em]t\x.north) -- ([yshift=-0.1em]s\x.south);
    \draw [->] ([yshift=0.1em]s\x.north) -- ([yshift=-0.1em]o\x.south);
    \draw [->] ([yshift=0.1em]o\x.north) -- ([yshift=0.8em]o\x.north) node [pos=0.5,right] {\tiny{top1}};
}
}

\visible<9->{
\foreach \x in {3,4}{
    \draw [->] ([yshift=-0.7em]t\x.south) -- ([yshift=-0.1em]t\x.south);
    \draw [->] ([yshift=0.1em]t\x.north) -- ([yshift=-0.1em]s\x.south);
    \draw [->] ([yshift=0.1em]s\x.north) -- ([yshift=-0.1em]o\x.south);
    \draw [->] ([yshift=0.1em]o\x.north) -- ([yshift=0.8em]o\x.north) node [pos=0.5,right] {\tiny{top1}};
}
}

\visible<5->{
\draw [->] ([xshift=-0.8em]s1.west) -- ([xshift=-0.1em]s1.west) node [pos=0,left] {\tiny{0}};
}
\visible<8->{
\draw [->] ([xshift=0.1em]s1.east) -- ([xshift=-0.1em]s2.west);
}
\visible<9->{
\draw [->] ([xshift=0.1em]s2.east) -- ([xshift=-0.1em]s3.west);
\draw [->] ([xshift=0.1em]s3.east) -- ([xshift=-0.1em]s4.west);
}

\visible<7->{
\draw [->,thick,dotted] (wo1.east) .. controls +(east:1.0) and +(west:1.0) ..(wt2.west);
}
\visible<9->{
\draw [->,thick,dotted] (wo2.east) .. controls +(east:1.3) and +(west:1.1) ..(wt3.west);
\draw [->,thick,dotted] (wo3.east) .. controls +(east:0.9) and +(west:0.9) ..(wt4.west);
}

\visible<6->{
\node [circle,draw,anchor=south,inner sep=3pt,fill=orange!20] (c2) at ([yshift=2em]h2.north) {\tiny{$\textbf{C}_2$}};
\node [anchor=south] (c2label) at (c2.north) {\tiny{\textbf{注意力机制：上下文}}};
\node [anchor=south] (c2more) at ([yshift=-1.5em]c2.south) {\tiny{...}};
\draw [->] (h1.north) .. controls +(north:0.6) and +(250:0.9) .. (c2.250);
\draw [->] (h3.north) .. controls +(north:0.6) and +(290:0.9) .. (c2.290);
\draw [->] ([yshift=-0.3em]s1.west) .. controls +(west:2) and +(-50:0.3) .. (c2.-40);
}
\visible<8->{
\draw [->] (c2.0) -- ([xshift=1.358in]c2.0) -- ([yshift=0.3em,xshift=-1.2em]s2.west) -- ([yshift=0.3em,xshift=-0.1em]s2.west);
}

\visible<9->{
\node [circle,draw,anchor=north,inner sep=3pt,fill=orange!20] (c3) at ([yshift=-2em]t2.south) {\tiny{$\textbf{C}_3$}};
\draw [->] ([xshift=-0.7em]c3.west) -- ([xshift=-0.1em]c3.west);
\draw [->] ([xshift=0.1em]c3.east) .. controls +(east:0.6) and +(west:0.8) ..([yshift=-0.3em,xshift=-0.1em]s3.west);
}

\visible<9->{
\node [circle,draw,anchor=north,inner sep=3pt,fill=orange!20] (c4) at ([yshift=-2em]t3.south) {\tiny{$\textbf{C}_4$}};
\draw [->] ([xshift=-0.7em]c4.west) -- ([xshift=-0.1em]c4.west);
\draw [->] ([xshift=0.1em]c4.east) .. controls +(east:0.6) and +(west:0.8) ..([yshift=-0.3em,xshift=-0.1em]s4.west);
}

\end{scope}
\end{tikzpicture}
\end{center}
}

\end{frame}

%%%------------------------------------------------------------------------------------------------------------
%%% 解码 - beam search
\begin{frame}{推断 - Beam Search}
\begin{itemize}
\item \textbf{Greedy Search}: 目标语每一个位置，输出层的Softmax可以得到所有单词的概率，然后选择一个概率最大单词输出，下一个位置的预测就基于这一步输出的单词
\item \textbf{Beach Search}: 为了避免贪婪方法造成的错误累加，可以每次对$b$个单词进行扩展，而不是只使用一个单词，其中$b$称做束的宽度 - 这样可以搜索更多可能的译文
\end{itemize}

\vspace{-0.3em}
\visible<2->{
\begin{center}
\begin{tikzpicture}
\begin{scope}
\tikzstyle{rnnnode} = [minimum height=1.1em,minimum width=3.5em,inner sep=2pt,rounded corners=1pt,draw,fill=red!20];
\tikzstyle{wnode} = [minimum height=1.0em,minimum width=3.5em,inner sep=2pt,rounded corners=1pt,draw,fill=white];


\visible<3->{
\node [rnnnode,anchor=west,fill=green!20] (t1) at (0,0) {\tiny{$e_y()$}};
}
\visible<7->{
\node [rnnnode,anchor=west,fill=green!20] (t2) at ([xshift=2.2em]t1.east) {\tiny{$e_y()$ ($\times 3$)}};
}
\visible<8->{
\node [rnnnode,anchor=west,fill=green!20] (t3) at ([xshift=2.2em]t2.east) {\tiny{$e_y()$ ($\times 3$)}};
\node [anchor=west,inner sep=2pt] (t4) at ([xshift=0.3em]t3.east) {\tiny{...}};
}
\visible<3->{
\node [rnnnode,anchor=south] (s1) at ([yshift=1em]t1.north) {\tiny{$\textbf{s}_1$}};
}
\visible<7->{
\node [rnnnode,anchor=south] (s2) at ([yshift=1em]t2.north) {\tiny{$\textbf{s}_2$ ($\times 3$)}};
}
\visible<8->{
\node [rnnnode,anchor=south] (s3) at ([yshift=1em]t3.north) {\tiny{$\textbf{s}_3$ ($\times 3$)}};
\node [anchor=west,inner sep=2pt] (s4) at ([xshift=0.3em]s3.east) {\tiny{...}};
}
\visible<3->{
\node [rnnnode,anchor=south,fill=blue!20] (o1) at ([yshift=1em]s1.north) {\tiny{softmax}};
}
\visible<7->{
\node [rnnnode,anchor=south,fill=blue!20] (o2) at ([yshift=1em]s2.north) {\tiny{softmax ($\times 3$)}};
}
\visible<8->{
\node [rnnnode,anchor=south,fill=blue!20] (o3) at ([yshift=1em]s3.north) {\tiny{softmax ($\times 3$)}};
\node [anchor=west,inner sep=2pt] (o4) at ([xshift=0.3em]o3.east) {\tiny{...}};
}

\node [wnode,anchor=north] (wt1) at ([yshift=-0.8em]t1.south) {\tiny{EOS}};

\visible<6->{
\node [wnode,anchor=north] (wt2) at ([yshift=-0.8em]t2.south) {\tiny{Have}};
\node [wnode,anchor=north] (wt2copy1) at ([xshift=-0.2em,yshift=-0.2em]wt2.north) {\tiny{Have}};
\node [wnode,anchor=north] (wt2copy2) at ([xshift=-0.4em,yshift=-0.4em]wt2.north) {\tiny{Have}};
}

\visible<8->{
\node [wnode,anchor=north,inner sep=2pt] (wt3) at ([yshift=-0.8em]t3.south) {\tiny{you}};
\node [wnode,anchor=north] (wt3copy1) at ([xshift=-0.2em,yshift=-0.2em]wt3.north) {\tiny{you}};
\node [wnode,anchor=north] (wt3copy2) at ([xshift=-0.4em,yshift=-0.4em]wt3.north) {\tiny{you}};
}

\visible<5->{
\node [wnode,anchor=center,inner sep=2pt] (wo1) at ([xshift=0.4em,yshift=1.8em]o1.north) {\tiny{Have}};
\node [wnode,anchor=north] (wo1copy1) at ([xshift=-0.2em,yshift=-0.2em]wo1.north) {\tiny{Have}};
\node [wnode,anchor=north] (wo1copy2) at ([xshift=-0.4em,yshift=-0.4em]wo1.north) {\tiny{Have}};
}

\visible<8->{
\node [wnode,anchor=center,inner sep=2pt] (wo2) at ([xshift=0.4em,yshift=1.8em]o2.north) {\tiny{you}};
\node [wnode,anchor=north] (wo2copy1) at ([xshift=-0.2em,yshift=-0.2em]wo2.north) {\tiny{you}};
\node [wnode,anchor=north] (wo2copy2) at ([xshift=-0.4em,yshift=-0.4em]wo2.north) {\tiny{you}};
}

\visible<8->{
\node [wnode,anchor=center,inner sep=2pt] (wo3) at ([xshift=0.4em,yshift=1.8em]o3.north) {\tiny{learned}};
\node [wnode,anchor=north] (wo3copy1) at ([xshift=-0.2em,yshift=-0.2em]wo3.north) {\tiny{learned}};
\node [wnode,anchor=north] (wo3copy2) at ([xshift=-0.4em,yshift=-0.4em]wo3.north) {\tiny{learned}};
}

\visible<3->{
\foreach \x in {1}{
    \draw [->] ([yshift=-0.7em]t\x.south) -- ([yshift=-0.1em]t\x.south);
    \draw [->] ([yshift=0.1em]t\x.north) -- ([yshift=-0.1em]s\x.south);
    \draw [->] ([yshift=0.1em]s\x.north) -- ([yshift=-0.1em]o\x.south);

}
}

\visible<5->{
 \draw [->] ([yshift=0.1em]o1.north) -- ([yshift=0.8em]o1.north) node [pos=0.5,right] {\tiny{top-3}};
 }

\visible<7->{
\foreach \x in {2}{
    \draw [->] ([yshift=-0.7em]t\x.south) -- ([yshift=-0.1em]t\x.south);
    \draw [->] ([yshift=0.1em]t\x.north) -- ([yshift=-0.1em]s\x.south);
    \draw [->] ([yshift=0.1em]s\x.north) -- ([yshift=-0.1em]o\x.south);
    \draw [->] ([yshift=0.1em]o\x.north) -- ([yshift=0.8em]o\x.north) node [pos=0.5,right] {\tiny{top-3}};
}
}

\visible<8->{
\foreach \x in {3}{
    \draw [->] ([yshift=-0.7em]t\x.south) -- ([yshift=-0.1em]t\x.south);
    \draw [->] ([yshift=0.1em]t\x.north) -- ([yshift=-0.1em]s\x.south);
    \draw [->] ([yshift=0.1em]s\x.north) -- ([yshift=-0.1em]o\x.south);
    \draw [->] ([yshift=0.1em]o\x.north) -- ([yshift=0.8em]o\x.north) node [pos=0.5,right] {\tiny{top-3}};
}
}

\visible<3->{
\draw [->] ([xshift=-0.5em]s1.west) -- ([xshift=-0.1em]s1.west) node [pos=0,left,inner sep=1pt] {\tiny{0}};
}
\visible<7->{
\draw [->] ([xshift=0.1em]s1.east) -- ([xshift=-0.1em]s2.west);
}
\visible<8->{
\draw [->] ([xshift=0.1em]s2.east) -- ([xshift=-0.1em]s3.west);
}

\visible<6->{
\draw [->,very thick,dotted] (wo1.east) .. controls +(east:0.6) and +(west:0.8) ..(wt2copy2.west);
}
\visible<8->{
\draw [->,very thick,dotted] (wo2.east) .. controls +(east:0.6) and +(west:0.8) ..(wt3copy2.west);
}

\visible<7->{
\node [circle,draw,anchor=north,inner sep=2pt,fill=orange!20] (c2) at ([yshift=-2.5em]t1.south) {\tiny{$\textbf{C}_2$}};
\node [circle,draw,inner sep=2pt,fill=orange!20] (c2copy1) at ([yshift=-0.1em,xshift=-0.1em]c2) {\tiny{$\textbf{C}_2$}};
\node [circle,draw,inner sep=2pt,fill=orange!20] (c2copy2) at ([yshift=-0.2em,xshift=-0.2em]c2) {\tiny{$\textbf{C}_2$}};
\draw [->] ([xshift=-0.9em]c2.west) -- ([xshift=-0.3em]c2.west);
\draw [->] ([xshift=0.1em]c2.east) .. controls +(east:1.5) and +(west:0.8) ..([yshift=-0.3em,xshift=-0.1em]s2.west);
}

\visible<8->{
\node [circle,draw,anchor=north,inner sep=2pt,fill=orange!20] (c3) at ([yshift=-2.5em]t2.south) {\tiny{$\textbf{C}_3$}};
\node [circle,draw,inner sep=2pt,fill=orange!20] (c3copy1) at ([yshift=-0.1em,xshift=-0.1em]c3) {\tiny{$\textbf{C}_3$}};
\node [circle,draw,inner sep=2pt,fill=orange!20] (c3copy2) at ([yshift=-0.2em,xshift=-0.2em]c3) {\tiny{$\textbf{C}_3$}};
\draw [->] ([xshift=-0.9em]c3.west) -- ([xshift=-0.3em]c3.west);
\draw [->] ([xshift=0.1em]c3.east) .. controls +(east:1.5) and +(west:0.8) ..([yshift=-0.3em,xshift=-0.1em]s3.west);
}

\visible<3->{
\node [anchor=east] (vocab) at ([xshift=-5em]s1.west) {\tiny{$\begin{bmatrix} \textrm{Have} & 0.50 \\ \textrm{I} & 0.02 \\ \textrm{it} & 0.03 \\ \textrm{has} & 0.30 \\ \textrm{you} & 0.01 \\ \textrm{the} & 0.01 \\ \textrm{a} & 0.01 \\ \textrm{an} & 0.02 \\ \textrm{he} & 0.03 \\ \textrm{she} & 0.01 \\ \textrm{are} & 0.00 \\ \textrm{am} & 0.01 \\ ... & ... \end{bmatrix}$}};
\node [anchor=south] (vocablabel) at (vocab.north) {\tiny{单词的概率分布}};
\draw [->,red,very thick,dotted] (o1.west) .. controls +(west:1) and +(east:2) .. ([yshift=1em]vocab.south east);
}

\visible<4->{
\node [anchor=east,inner sep=1pt] (vocabtopn) at ([xshift=-0.5em,yshift=-0.5em]wo1.west) {\tiny{$\begin{bmatrix} \textrm{Have} \\ \textrm{has} \\ \textrm{it} \end{bmatrix}$}};
\draw [->] ([yshift=-1.6em,xshift=-0.4em]vocab.north east) .. controls +(east:1) and +(west:1) ..  ([xshift=0.1em,yshift=0.4em]vocabtopn.west) node [pos=0.3,below] (topnlabel) {\tiny{top-3}};

\visible<4->{
\node [anchor=north] (cap) at (vocab.south east) {\scriptsize{\textbf{束搜索($b=3$)}}};
}
}


\end{scope}
\end{tikzpicture}
\end{center}
}

\end{frame}

%%%------------------------------------------------------------------------------------------------------------
%%% 解码 - 长度惩罚和覆盖度
\begin{frame}{推断 - 其它特征}
\begin{itemize}
\item 直接用$\textrm{P}(\textbf{y}|\textbf{x})$进行解码，面临两方面问题
    \begin{itemize}
    \item 对$\textrm{P}(y_j|\textbf{y}_{<j},\textbf{x})$进行乘积会导致长句的概率很低
    \item 模型本身并没有考虑每个源语言单词被使用的程度，比如一个单词可能会被翻译了很多``次''
    \end{itemize}
\item<2-> 因此，解码时会使用其它特征与$\textrm{P}(\textbf{y}|\textbf{x})$一起组成模型得分$score(\textbf{y},\textbf{x})$，$score(\textbf{y},\textbf{x})$也作为beam search的排序依据
    \begin{eqnarray}
    score(\textbf{y},\textbf{x}) & = & \textrm{P}(\textbf{y}|\textbf{x})/\textrm{lp}(\textbf{y}) + \textrm{cp}(\textbf{y},\textbf{x}) \nonumber \\
    \textrm{lp}(\textbf{y})      & = & \frac{(5 + |\textbf{y}|)^\alpha}{(5 + 1)^\alpha} \nonumber \\
    \textrm{cp}(\textbf{y},\textbf{x}) & = & \beta \cdot \sum\nolimits_{i=1}^{|\textbf{x}|} \log (\min(\sum\nolimits_{j}^{|\textbf{y}|} a_{ij}, 1))) \nonumber
    \end{eqnarray}

    \vspace{-0.5em}
    \begin{itemize}
    \item lp会惩罚译文过短的结果(长度惩罚)；cp会惩罚把某些源语单词对应到很多目标语单词的情况(覆盖度)，被覆盖的程度用$\sum\nolimits_{j}^{|\textbf{y}|} a_{ij}$度量；$\alpha$和$\beta$是超参，需要经验性设置
    \end{itemize}
\end{itemize}

\end{frame}

%%%------------------------------------------------------------------------------------------------------------
%%% 实验结果
\begin{frame}{效果}
%% 实用注意力机制带来的提升
%% 个大评测比赛没有不使用注意力机制的系统，已经成为标配
\end{frame}

%%%------------------------------------------------------------------------------------------------------------
%%% GNMT
\begin{frame}{成功案例 - GNMT}
%% GNMT的图和几句话说它多牛
\end{frame}

%%%------------------------------------------------------------------------------------------------------------
\section{Transformer}

%%%------------------------------------------------------------------------------------------------------------
\subsection{多头自注意力模型}

%%%------------------------------------------------------------------------------------------------------------
\subsection{训练}

%%%------------------------------------------------------------------------------------------------------------
\subsection{推断}

%%%------------------------------------------------------------------------------------------------------------
\subsection{深层网络}

%%%------------------------------------------------------------------------------------------------------------
\section{其它应用}

\end{CJK}
\end{document}
