% !Mode:: "TeX:UTF-8"
% !TEX encoding = UTF-8 Unicode

\def\CTeXPreproc{Created by ctex v0.2.13, don't edit!}
\documentclass[cjk,t,compress,12pt]{beamer}
\usepackage{pstricks}
\usepackage{etex}
\usepackage{eso-pic,graphicx}
\usepackage{fancybox}
\usepackage{amsmath,amssymb}
\usepackage{setspace}
\usepackage{xcolor}
\usepackage{array,multirow}
\usepackage{CJK}
\usepackage{tikz}
\usepackage{tikz-qtree}
\usepackage{hyperref}
\usepackage{changepage}
\usepackage{pgfplots}
\usepackage{subfigure}
\usepackage{tikz-3dplot}
\usepackage{esvect}
\usepackage{CJKulem}
\usepackage{booktabs}

\usepackage{tcolorbox}
\tcbuselibrary{skins}

\usetikzlibrary{calc,intersections}
\usetikzlibrary{matrix}
\usetikzlibrary{arrows,decorations.pathreplacing}
\usetikzlibrary{shadows} % LATEX and plain TEX when using Tik Z
\usetikzlibrary{shadows.blur}

\usepgflibrary{arrows} % LATEX and plain TEX and pure pgf
\usetikzlibrary{arrows} % LATEX and plain TEX when using Tik Z
\usetikzlibrary{decorations}
\usetikzlibrary{arrows,shapes}

\usetikzlibrary{positioning,fit,calc}

\usetikzlibrary{mindmap,backgrounds} % mind map

\DeclareMathOperator*{\argmax}{arg\,max}
\DeclareMathOperator*{\argmin}{arg\,min}

\setbeamertemplate{items}[ball]
\usefonttheme[onlymath]{serif}  % fout of math

\definecolor{ugreen}{rgb}{0,0.5,0}
\definecolor{lgreen}{rgb}{0.9,1,0.8}
\definecolor{xtgreen1}{rgb}{0.824,0.898,0.8}
\definecolor{xtgreen}{rgb}{0.914,0.945,0.902}
\definecolor{lightgray}{gray}{0.85}

\setbeamercolor{uppercol}{fg=white,bg=ugreen}
\setbeamercolor{lowercol}{fg=black,bg=xtgreen}

\definecolor{ublue}{rgb}{0.152,0.250,0.545}
\setbeamercolor{uppercolblue}{fg=white,bg=ublue}
\setbeamercolor{lowercolblue}{fg=black,bg=blue!10}


%\usetheme{default}
%\usetheme{Darmstadt}
%\usetheme{Madrid}
%\usetheme{Frankfurt}
%\usetheme{Dresden}
%\usetheme{Boadilla}
%\usecolortheme{dolphin}

\newdimen\XCoord
\newdimen\YCoord
\newdimen\TMP
\newcommand*{\ExtractCoordinate}[1]{\path (#1); \pgfgetlastxy{\XCoord}{\YCoord};}%
\newcommand*{\ExtractX}[1]{\path (#1); \pgfgetlastxy{\XCoord}{\TMP};}%
\newcommand*{\ExtractY}[1]{\path (#1); \pgfgetlastxy{\TMP}{\YCoord};}%

\newcounter{mycount1}
\newcounter{mycount2}
\newcounter{mycount3}
\newcounter{mycount4}

\usefonttheme[onlylarge]{structurebold}

\IfFileExists{C:/WINDOWS/win.ini}
{\newcommand{\mycfont}{you}}
{\newcommand{\mycfont}{gbsn}}

\begin{CJK}{UTF8}{\mycfont}
\end{CJK}

\setbeamerfont*{frametitle}{size=\large,series=\bfseries}
\setbeamertemplate{navigation symbols}{\begin{CJK}{UTF8}{\mycfont} 第六章 神经机器翻译 \hspace*{2em} 肖桐\&朱靖波 \end{CJK} \hspace*{2em} \today \hspace*{2em} \insertframenumber{}/\inserttotalframenumber}

\setbeamertemplate{itemize items}[circle] % if you want a circle
\setbeamertemplate{itemize subitem}[triangle] % if you wnat a triangle
\setbeamertemplate{itemize subsubitem}[ball] % if you want a ball

\begin{document}

\begin{CJK}{UTF8}{\mycfont}

\title{\Large{神经机器翻译}}
\author{\large{\textbf{肖桐\ \ 朱靖波}}}
\institute{
\blue{\url{xiaotong@mail.neu.edu.cn}} \black{} \\
\blue{\url{zhujingbo@mail.neu.edu.cn}} \black{} \\
\vspace{1.0em}
东北大学 自然语言处理实验室 \\
\blue{\underline{\url{http://www.nlplab.com}}} \black{} \\
\vspace{0.2cm}
\hspace{0.1cm} \includegraphics[scale=0.1]{../Figures/logo.pdf}
}
\date{}

\maketitle

\setlength{\leftmargini}{1em}
\setlength{\leftmarginii}{1em}

%%%------------------------------------------------------------------------------------------------------------
\section{编码器-解码器框架}

%%%------------------------------------------------------------------------------------------------------------
%%% landscape
\begin{frame}{Landscape}

\vspace{-1em}
\begin{center}
\begin{tikzpicture}

\node [] (history) at (0,0) {\includegraphics[scale=0.29]{./Figures/mt-history.png}};

\draw[-,thick] ([xshift=-1.1in]history.south) coordinate (top01) -- ([yshift=-2em]top01);
\draw[-,thick] ([xshift=1in]history.south) coordinate (top02) -- ([yshift=-2em]top02);
\draw[-,thick] ([xshift=-0.3in]history.south east) coordinate (top03) -- ([yshift=-2em]top03);
\draw[<->,thick] ([yshift=-1em]top01) -- ([yshift=-1em]top02) node [pos=0.5,below,align=left,yshift=-0.5em] {\footnotesize{\textbf{Sections 3-4:}}\\\footnotesize{统计机器翻译}};
\draw[<->,thick] ([yshift=-1em]top02) -- ([yshift=-1em]top03) node [pos=0.5,below,align=left,yshift=-0.5em] {\footnotesize{\textbf{Sections 5-6:}}\\\footnotesize{神经机器翻译}};

\visible<2->{
\draw[<-,thick] ([xshift=0.9in,yshift=-0.2in]history.north) coordinate (label01) -- ([yshift=1.5em,xshift=-1.5em]label01) coordinate (label02);
\node[anchor=south west,align=left] (label) at ([xshift=-5em]label02) {\textbf{\alert{本章内容（Section 6）：}}\\神经机器翻译建模及实现};
}

\end{tikzpicture}
\end{center}

\end{frame}

%%%------------------------------------------------------------------------------------------------------------
%%% 译文对比
\begin{frame}{机器翻译今天的水平}

\begin{tcolorbox}[enhanced,frame engine=empty,boxrule=0.1mm,size=title,colback=red!10!white]
\begin{flushleft}

\begin{spacing}{0.9}
\scriptsize{\textbf{原文(英语)}: During Soviet times, if a city's population topped one million, it would become eligible for its own metro. Planners wanted to brighten the lives of everyday Soviet citizens, and saw the metros, with their tens of thousands of daily passengers, as a singular opportunity to do so. In 1977, Tashkent, the capital of Uzbekistan, became the seventh Soviet city to have a metro built. Grand themes celebrating the history of Uzbekistan and the Soviet Union were brought to life, as art was commissioned and designers set to work. The stations reflected different themes, some with domed ceilings and painted tiles reminiscent of Uzbekistan's Silk Road mosques, while others ... }
\end{spacing}
\end{flushleft}
\end{tcolorbox}

\begin{minipage}[t]{0.47\textwidth}
\begin{beamerboxesrounded}[upper=uppercolblue,lower=lowercolblue,shadow=true]{\scriptsize{\textbf{译文\visible<2->{(统计机器翻译)}}}}
{\scriptsize
\begin{spacing}{0.9}
在苏联时代，如果一个城市的人口突破一百万，这将成为合资格为自己的地铁。规划者想去照亮每天的苏联公民的生命，看到地铁，与他们的数十每天数千乘客，作为一个独特的机会来这样做。1977年，塔什干，乌兹别克斯坦的首都，成了苏联第七城市建有地铁。宏大主题，庆祝乌兹别克斯坦和苏联的历史被带到生活，因为艺术是委托和设计师开始工作。车站反映了不同的主题，有的圆顶天花板和绘瓷砖让人想起乌兹别克斯坦是丝绸之路的清真寺，而另一些则装饰着...
\end{spacing}
}
\end{beamerboxesrounded}
\end{minipage}
\hfill
\begin{minipage}[t]{0.47\textwidth}
\begin{beamerboxesrounded}[upper=uppercolblue,lower=lowercolblue,shadow=true]{\scriptsize{\textbf{译文\visible<2->{(神经机器翻译 - 很流畅！)}}}}
{\scriptsize
\begin{spacing}{0.9}
在苏联时期，如果一个城市的人口超过一百万，它就有资格拥有自己的地铁。 规划者想要照亮日常苏联公民的生活，并把拥有数万名每日乘客的地铁看作是这样做的一个绝佳机会。 1977年，乌兹别克斯坦首都塔什干成为苏联第七个修建地铁的城市。 随着艺术的委托和设计师们的工作，乌兹别克斯坦和苏联历史的宏伟主题被赋予了生命力。 这些电台反映了不同的主题，有的有穹顶和彩砖，让人想起乌兹别克斯坦的丝绸之路清真寺，有的则用...
\end{spacing}
}
\end{beamerboxesrounded}
\end{minipage}

\end{frame}

%%%------------------------------------------------------------------------------------------------------------
%%% 神经机器翻译的性能增长
\begin{frame}{神经机器翻译的进展}
\begin{itemize}
\item 想当年，08年NIST举办的汉英机器翻译评测，BLEU能突破30已经是巨猛无比的结果，而现在的神经机器翻译轻松突破45！
\item WMT的结果？？
\end{itemize}
%%% 秀smt nmt的blue随着年代的变化
\end{frame}

%%%------------------------------------------------------------------------------------------------------------
%%% 神经机器翻译的性能增长
\begin{frame}{神经机器翻译的进展(续)}
\begin{itemize}
\item 性能提升
\only<1>{
    \begin{center}
        \begin{tabular}{r}
            \begin{tabular}{lcccl}
                \toprule
                \multirow{2}{*}{\#} & \multicolumn{3}{c}{自动评价} & \multirow{2}{*}{系统} \\
                \cline{2-4}
                & \footnotesize BLEU & \footnotesize HTER & \footnotesize mTER & \\
                \midrule
                统计机器翻译 & 25.3 & 28.0 & 21.8 & PBSY \\
                & 24.6 & 29.9 & 23.4 & HPB \\
                & 25.8 & 29.0 & 22.7 & SPB \\
                \midrule
                神经机器翻译 & \textbf{31.1} & \textbf{21.1} & \textbf{16.2} & NMT \\
                \bottomrule
            \end{tabular}\\[4pt]
            \tiny *Neural versus Phrase-Based Machine Translation Quality: a Case Study\\
        \end{tabular}
    \end{center}
}
\item Human party MT
\only<2>{
    \begin{center}
        \begin{tabular}{r}
            \begin{tabular}{lrl}
                \toprule
                \# & 人工评价 & 系统 \\
                \midrule
                机器翻译 & \textbf{69.9} & COMBO-6 \\
                & 69.8 & COMBO-4 \\
                & \textbf{69.9} & COMBO-5 \\
                \midrule
                人工翻译 & 68.6 & REFERENCE-HT \\
                & 67.6 & REFERENCE-PE \\
                & 62.1 & REFERENCE-WMT \\
                \bottomrule
            \end{tabular}\\[4pt]
            \tiny *Achieving Human Parity on Automatic Chinese to English News Translation\\
        \end{tabular}
    \end{center}
}
\end{itemize}
%%% 进一步找一些证据说明NMT的进展，很大google、microsoft的文章，可以看看提升了多少性能等等
\end{frame}

%%%------------------------------------------------------------------------------------------------------------
%%% 传统方法 vs 神经网络方法
\begin{frame}{神经机器翻译的优势}
\begin{itemize}
\item 神经网络方法带来了新的思路，还是同样的表再看一遍
\end{itemize}

\begin{tabular} {l | l}
\textbf{传统基于统计的方法} & \textbf{深度学习方法} \\
\textbf{(统计机器翻译)} & \textbf{(神经机器翻译)} \\ \hline
基于离散空间的表示模型 & 基于连续空间的表示模型 \\
NLP问题的隐含结构假设 & 无隐含结构假设，端到端学习 \\
特征工程为主 & 无显性特征，但需要设计网络 \\
特征、规则的存储耗资源 & 模型存储相对小，但计算慢
\end{tabular}

\begin{itemize}
\item<2-> \only<2>{在统计机器翻译时代，系统依赖很多模块，比如}\only<3->{\sout{在统计机器翻译时代，系统依赖很多模块，比如}}
    \begin{itemize}
    \item \only<2>{\textbf{词对齐}：双语句子之间词和词的对应关系}\only<3->{\sout{\textbf{词对齐}：双语句子之间词和词的对应关系}}
    \item \only<2>{\textbf{短语(规则)表}：原文和译文互译的片段}\only<3->{\sout{\textbf{短语(规则)表}：原文和译文互译的片段}}
    \item \only<2>{\textbf{特征}：人工设计的用于建模翻译的各种各样的子模型}\only<3->{\sout{\textbf{特征}：人工设计的用于建模翻译的各种各样的子模型}}
    \item \only<2>{\textbf{目标语语言模型}：大量单语数据训练的目标语建模模块}\only<3->{\sout{\textbf{目标语语言模型}：大量单语数据训练的目标语建模模块}}
    \item \only<2>{...}\only<3->{\sout{...}}
    \end{itemize}
\item<3-> 在\alert{神经机器翻译}时代，以上均不是必要的，而仅仅需要一个神经网络进行端到端建模
\end{itemize}

\end{frame}

%%%------------------------------------------------------------------------------------------------------------
%%% 编码器-解码器框架
\begin{frame}{编码器-解码器框架}

\begin{itemize}
\item 神经机器翻译遵循一种叫\alert{``编码器-解码器''}的结构
    \begin{itemize}
    \item<2-> \textbf{编码器}负责把源语言编码成一种句子表示形式(如：向量)
    \item<3-> \textbf{解码器}负责利用这种表示逐词把目标语句子生成处理
    \item<4-> 编码器和解码器之间的向量就是连接这两部分的句子表示
    \end{itemize}
\end{itemize}

\vspace{0.5em}
\begin{tikzpicture}

\begin{scope}
\small{
\node [anchor=south west,minimum width=15em] (source) at (0,0) {\textbf{source}: 我\ \ \ \ 对\ \ \ \ 你\ \ \ \ 感到\ \ \ \ 满意};
\visible<3->{
\node [anchor=south west,minimum width=15em] (target) at ([yshift=12em]source.north west) {\textbf{target}: I\ \ am\ \ \ satisfied\ \ \ with\ \ \ you};
}
\visible<2->{
\node [anchor=center,minimum width=9.6em,minimum height=1.8em,draw,rounded corners=0.3em] (hidden) at ([yshift=6em]source.north) {};
\node [anchor=west,minimum width=1.5em,minimum size=1.5em,fill=ugreen!20] (cell01) at ([xshift=0.2em]hidden.west) {\footnotesize{.2}};
\node [anchor=west,minimum width=1.5em,minimum size=1.5em,fill=ugreen!10] (cell02) at (cell01.east) {\footnotesize{-1}};
\node [anchor=west,minimum width=1.5em,minimum size=1.5em,fill=ugreen!70] (cell03) at (cell02.east) {\footnotesize{6}};
\node [anchor=west,minimum width=1.5em,minimum size=1.5em,fill=ugreen!50] (cell04) at (cell03.east) {\footnotesize{5}};
\node [anchor=west,minimum width=1.5em,minimum size=1.5em,fill=ugreen!30] (cell05) at (cell04.east) {\footnotesize{.7}};
\node [anchor=west,minimum width=1.5em,minimum size=1.5em,fill=ugreen!10] (cell06) at (cell05.east) {\footnotesize{-2}};
}

\visible<2->{
\filldraw [fill=red!20,draw=white] (source.north west) -- (source.north east) -- ([xshift=-0.2em,yshift=-0.1em]hidden.south east) -- ([xshift=0.2em,yshift=-0.1em]hidden.south west);
}
\visible<3->{
\filldraw [fill=blue!20,draw=white] (target.south west) -- (target.south east) -- ([xshift=-0.2em,yshift=0.1em]hidden.north east) -- ([xshift=0.2em,yshift=0.1em]hidden.north west);
}
\visible<2->{
\draw [->,thick] (source.north west) -- ([xshift=0.2em,yshift=-0.1em]hidden.south west);
\draw [->,thick] (source.north east) -- ([xshift=-0.2em,yshift=-0.1em]hidden.south east);
}
\visible<3->{
\draw [->,thick] ([xshift=0.2em,yshift=0.1em]hidden.north west) -- (target.south west);
\draw [->,thick] ([xshift=-0.2em,yshift=0.1em]hidden.north east) -- (target.south east);
}
}

\visible<2->{
\node [anchor=south] (enclabel) at ([yshift=2em]source.north) {\large{\textbf{Encoder}}};
}
\visible<3->{
\node [anchor=north] (declabel) at ([yshift=-2em]target.south) {\large{\textbf{Decoder}}};
}

\visible<4->{
\node [anchor=west] (representation) at ([xshift=3.5em,yshift=3em]hidden.east) {\footnotesize{\textbf{句子的表示:}}};
\node [anchor=north west] (rline1) at (representation.south west) {\footnotesize{源语言句子被表示成一个实数}};
\node [anchor=north west] (rline2) at ([yshift=0.2em]rline1.south west) {\footnotesize{向量}\footnotesize{$(0.2,-1,6,5,0.7,-2)$}};
\node [anchor=north west] (rline3) at ([yshift=0.2em]rline2.south west) {\footnotesize{不要问0.2是什么意思，因为}};
\node [anchor=north west] (rline4) at ([yshift=0.2em]rline3.south west) {\footnotesize{只有系统自己才知道 :)}};

\begin{pgfonlayer}{background}
\visible<4->{
\node [rectangle,inner sep=0.4em,rounded corners=1pt,fill=ugreen!10,drop shadow] [fit = (representation) (rline1) (rline2) (rline3) (rline4)] (rlabel) {};
}
\end{pgfonlayer}

\draw[->,very thick,dashed] (hidden.east) .. controls +(east:1) and +(west:1) .. ([yshift=-1em]rlabel.north west);
}

\end{scope}

\end{tikzpicture}

\end{frame}

%%%------------------------------------------------------------------------------------------------------------
%%% 表示模型
\begin{frame}{基于连续空间表示模型的方法}
\begin{itemize}
\item 编码器-解码器框架的革命在于：\alert{它把传统基于符号的离散型知识转化为基于表示的连续型知识}
    \begin{itemize}
    \item 比如，对于一个短语，它可以对应一个文法规则的使用过程，而规则都是由离散的符号表示
    \item 它也可以被表示为一种更加抽象的形式，通过向量记录短语的各个``属性''
    \end{itemize}
\item<2-> 这种表示形式上的创新，让我们不再依赖离散符号系统的各种分解、组合，而直接在连续空间下学习这种表示
    \begin{itemize}
    \item 所谓``端到端''只是这种表示模型下的一种自然的学习范式
    \end{itemize}
\end{itemize}

\visible<3->{
\begin{center}
\begin{tikzpicture}

\begin{scope}
\node [anchor=south west,draw,thick,red,minimum width=0.9in,minimum height=0.7in] (space1) at (0,0) {};
\node [anchor=south west,fill=blue,minimum width=0.1in,minimum height=0.1in] (unit1) at (0.2,0.8) {};
\node [anchor=south west,fill=ugreen,minimum width=0.1in,minimum height=0.1in] (unit2) at (0.7,0.3) {};
\node [anchor=south west,fill=blue,minimum width=0.1in,minimum height=0.1in] (unit3) at (1.3,1.3) {};
\node [anchor=south west,fill=ugreen,minimum width=0.1in,minimum height=0.1in] (unit4) at ([xshift=0.1em]unit3.south east) {};

\begin{pgfonlayer}{background}
\node [rectangle,inner sep=2pt,rounded corners=1pt,draw,thick] [fit = (unit3) (unit4)] (unitbox) {};
\end{pgfonlayer}

\draw [->] ([yshift=1pt]unit1.north) .. controls +(north:0.23) and +(west:0.2) .. ([yshift=0.2em,xshift=-1pt]unitbox.west);
\draw [->] ([xshift=1pt]unit2.east) .. controls +(east:0.5) and +(south:0.2) .. ([xshift=0.2em,yshift=-1pt]unitbox.south);

\node [anchor=south] (spacelabel1) at (space1.north) {\scriptsize{离散空间模型}};
\node [anchor=north] (captain1) at ([yshift=-0.5em]space1.south) {\scriptsize{(a) \textbf{统计机器翻译}}};

\end{scope}

\begin{scope}[xshift=1.3in]
\node [anchor=south west,draw,thick,red,minimum width=0.9in,minimum height=0.7in] (space1) at (0,0) {};
\node [anchor=south west,fill=blue,minimum width=0.1in,minimum height=0.1in] (unit1) at (0.2,0.8) {};
\node [anchor=south west,fill=ugreen,minimum width=0.1in,minimum height=0.1in] (unit2) at (0.7,0.3) {};

\node [anchor=south west,draw,thick,red,minimum width=0.9in,minimum height=0.7in] (space2) at (1.1in,0) {};
\node [anchor=south west,circle,fill=orange,minimum width=0.1in,minimum height=0.1in] (unit3) at (1.5in,1.3) {};

\draw [->] ([yshift=1pt]unit1.north) .. controls +(north:0.4) and +(west:2) .. ([yshift=0.0em,xshift=-1pt]unit3.west);
\draw [->] ([xshift=1pt]unit2.east) .. controls +(east:1.5) and +(south:1) .. ([xshift=0.0em,yshift=-1pt]unit3.south);

\node [anchor=south] (spacelabel1) at (space1.north) {\scriptsize{离散空间模型}};
\node [anchor=south] (spacelabel2) at (space2.north) {\scriptsize{连续空间模型}};
\node [anchor=north] (captain1) at ([yshift=-0.5em,xshift=1em]space1.south east) {\scriptsize{(b) \textbf{神经机器翻译}}};

\end{scope}

\end{tikzpicture}
\end{center}
}

\end{frame}

%%%------------------------------------------------------------------------------------------------------------
%%% 一个NMT运行的简单例子
\begin{frame}{简单的运行实例}
\begin{itemize}
\item 一个简单的例子：基于循环神经网络的翻译过程
        \begin{itemize}
        \item 顺序处理源语言单词
        \item 源语言句子信息被表示再最后一个循环单元的输出中
        \item 逐词生成目标语译文
        \end{itemize}
\end{itemize}
%%% 运行实例的图
\end{frame}

%%%------------------------------------------------------------------------------------------------------------
%%% NMT是魔法？
\begin{frame}{神经机器翻译会``魔法''?}
\begin{minipage}[c][6.5cm][t]{0.46\textwidth}
\alert{统计机器翻译}仍然依赖人工定义特征和翻译单元
\begin{itemize}
\item 相对符合人类的理解
\item 具有一定可解释性
\end{itemize}
\vspace{1em}
\includegraphics[width=\textwidth,height=3.5cm]{./Figures/whiteboard-math.jpg}
\end{minipage}
\hfill
\begin{minipage}[c][6.5cm][t]{0.48\textwidth}
\alert{神经机器翻译}把所有工作都交给神经网络
\begin{itemize}
\item 不需要先验假设
\item 语言的``表示''给机器看的
\end{itemize}
\vspace{1em}
\includegraphics[width=\textwidth,height=3.5cm]{./Figures/expelliarmus.jpg}
\end{minipage}

\begin{itemize}
\item 神经机器翻译在使用魔法？\\
        - No! No! No! 变化的只有人类使用知识的形式
\end{itemize}

\end{frame}

%%%------------------------------------------------------------------------------------------------------------
%%% NMT所对应的范式变换
\begin{frame}{神经机器翻译所带来的范式更迭}
\begin{itemize}
\item 不同机器翻译时代，人类知识的使用方式
\end{itemize}

\begin{center}
\begin{tabular}{l | l}
机器翻译方法 & 人类参与方式 \\ \hline
基于规则的方法 & 设计翻译规则 \\
传统统计方法 & 设计翻译特征 \\
神经网络方法 & 设计网络架构
 \end{tabular}
 \end{center}

 \visible<2->{
 \begin{tcolorbox}[enhanced,size=normal,left=2mm,right=1mm,colback=blue!5!white,colframe=blue!75!black,drop fuzzy shadow]
{\Large
\textbf{人类在机器翻译中将扮演什么角色？}
}
\end{tcolorbox}

\begin{itemize}
\item<3-> 机器翻译会逐渐替代人？\visible<4->{\alert{- 不会，爹是不能被儿子替代的}}
\item<3-> 还需要人的语言学知识吗？\visible<4->{\alert{- 会，但是需要新的思路}}
\item<3-> 就连神经网络也可以通过结构搜索自动学习，人类是不是就什么也不用干了？\visible<4->{\alert{- 结构搜索又是谁设计的，人至少需要敲下键盘吧，哈哈哈哈}}
\end{itemize}
}

\end{frame}

%%%------------------------------------------------------------------------------------------------------------
%%% 本章的内容
\begin{frame}{Outline}

\vspace{1.0em}
 \begin{tcolorbox}[enhanced,size=normal,left=2mm,right=1mm,colback=red!5!white,colframe=red!75!black,drop fuzzy shadow]
{\large
\textbf{入门：循环网络翻译模型及注意力机制} \\
\small{1. 起源} \\
\small{2. 模型结构} \\
\small{3. 注意力机制}
}
\end{tcolorbox}

\vspace{0.5em}

 \begin{tcolorbox}[enhanced,size=normal,left=2mm,right=1mm,colback=red!5!white,colframe=red!75!black,drop fuzzy shadow]
{\large
\textbf{热门：Transformer} \\
\small{1. 多头自注意力模型} \\
\small{2. 训练及推断} \\
\small{3. 深层网络翻译模型}
}
\end{tcolorbox}

\vspace{0.5em}

\begin{tcolorbox}[enhanced,size=normal,left=2mm,right=1mm,colback=red!5!white,colframe=red!75!black,drop fuzzy shadow]
{\large
\textbf{其它：一些有趣的神经机器翻译应用}
}
\end{tcolorbox}

\end{frame}

%%%------------------------------------------------------------------------------------------------------------
\section{循环神经网络翻译模型及注意力机制}


%%%------------------------------------------------------------------------------------------------------------
%%% 第一部分
\begin{frame}{首先}

\vspace{5.0em}
 \begin{tcolorbox}[enhanced,size=normal,left=2mm,right=1mm,colback=red!5!white,colframe=red!75!black,drop fuzzy shadow]
{\Large
\textbf{循环网络翻译模型及注意力机制}
}
\end{tcolorbox}

\begin{center}
\begin{tikzpicture}

\begin{scope}[scale=0.7]
\tikzstyle{rnnnode} = [draw,inner sep=5pt,minimum width=3em,minimum height=0.8em,fill=green!30!white,blur shadow={shadow xshift=1pt,shadow yshift=-1pt}]
\node [anchor=west,rnnnode] (node11) at (0,0) {\tiny{RNN Cell}};
\node [anchor=west,rnnnode] (node12) at ([xshift=1em]node11.east) {\tiny{RNN Cell}};
\node [anchor=west,rnnnode] (node13) at ([xshift=1em]node12.east) {\tiny{RNN Cell}};
\node [anchor=west,rnnnode] (node14) at ([xshift=1em]node13.east) {\tiny{RNN Cell}};

\node [anchor=north,rnnnode,fill=blue!30!white] (e1) at ([yshift=-1em]node11.south) {\tiny{}};
\node [anchor=north,rnnnode,fill=blue!30!white] (e2) at ([yshift=-1em]node12.south) {\tiny{}};
\node [anchor=north,rnnnode,fill=blue!30!white] (e3) at ([yshift=-1em]node13.south) {\tiny{}};
\node [anchor=north,rnnnode,fill=blue!30!white] (e4) at ([yshift=-1em]node14.south) {\tiny{}};
\node [anchor=north,inner sep=2pt] (w1) at ([yshift=-1em]e1.south) {\tiny{$<$s$>$}};
\node [anchor=north,inner sep=2pt] (w2) at ([yshift=-1em]e2.south) {\tiny{让}};
\node [anchor=north,inner sep=2pt] (w3) at ([yshift=-1em]e3.south) {\tiny{我们}};
\node [anchor=north,inner sep=2pt] (w4) at ([yshift=-1em]e4.south) {\tiny{开始}};

\draw [->,thick] ([yshift=0.1em]w1.north)--([yshift=-0.1em]e1.south);
\draw [->,thick] ([yshift=0.1em]w2.north)--([yshift=-0.1em]e2.south);
\draw [->,thick] ([yshift=0.1em]w3.north)--([yshift=-0.1em]e3.south);
\draw [->,thick] ([yshift=0.1em]w4.north)--([yshift=-0.1em]e4.south);

\draw [->,thick] ([yshift=0.1em]e1.north)--([yshift=-0.1em]node11.south);
\draw [->,thick] ([yshift=0.1em]e2.north)--([yshift=-0.1em]node12.south);
\draw [->,thick] ([yshift=0.1em]e3.north)--([yshift=-0.1em]node13.south);
\draw [->,thick] ([yshift=0.1em]e4.north)--([yshift=-0.1em]node14.south);

\node [anchor=south,rnnnode,fill=blue!30!white] (node21) at ([yshift=1.0em]node11.north) {\tiny{}};
\node [anchor=south,rnnnode,fill=blue!30!white] (node22) at ([yshift=1.0em]node12.north) {\tiny{}};
\node [anchor=south,rnnnode,fill=blue!30!white] (node23) at ([yshift=1.0em]node13.north) {\tiny{}};
\node [anchor=south,rnnnode,fill=blue!30!white] (node24) at ([yshift=1.0em]node14.north) {\tiny{}};

\node [anchor=south] (output1) at ([yshift=1em]node21.north) {\Large{\textbf{}}};
\node [anchor=south] (output2) at ([yshift=1em]node22.north) {\Large{\textbf{}}};
\node [anchor=south] (output3) at ([yshift=1em]node23.north) {\Large{\textbf{}}};
\node [anchor=south] (output4) at ([yshift=1em]node24.north) {\Large{\textbf{}}};

\draw [->,thick] ([yshift=0.1em]node21.north)--([yshift=-0.1em]output1.south);
\draw [->,thick] ([yshift=0.1em]node22.north)--([yshift=-0.1em]output2.south);
\draw [->,thick] ([yshift=0.1em]node23.north)--([yshift=-0.1em]output3.south);
\draw [->,thick] ([yshift=0.1em]node24.north)--([yshift=-0.1em]output4.south);

\draw [->,thick] ([yshift=0.1em]node11.north)--([yshift=-0.1em]node21.south);
\draw [->,thick] ([yshift=0.1em]node12.north)--([yshift=-0.1em]node22.south);
\draw [->,thick] ([yshift=0.1em]node13.north)--([yshift=-0.1em]node23.south);
\draw [->,thick] ([yshift=0.1em]node14.north)--([yshift=-0.1em]node24.south);

\draw [->,thick] ([xshift=-1em]node11.west)--([xshift=-0.1em]node11.west);
\draw [->,thick] ([xshift=0.1em]node11.east)--([xshift=-0.1em]node12.west);
\draw [->,thick] ([xshift=0.1em]node12.east)--([xshift=-0.1em]node13.west);
\draw [->,thick] ([xshift=0.1em]node13.east)--([xshift=-0.1em]node14.west);
\draw [->,thick] ([xshift=0.1em]node14.east)--([xshift=1em]node14.east);
\end{scope}

\end{tikzpicture}
\end{center}

\end{frame}

%%%------------------------------------------------------------------------------------------------------------
\subsection{起源}

%%%------------------------------------------------------------------------------------------------------------
%%% 神经机器翻译的历史
\begin{frame}{最初的神经机器翻译}

\begin{itemize}
\item 神经网络的在机器翻译中并不新鲜，在很多模块中早有实现，比如，翻译候选打分、语言模型等
    \begin{itemize}
    \item 但是，整个框架仍然是统计机器翻译
    \end{itemize}
\item<2-> 基于神经元网络的端到端建模出现在2013-2015，被称为\alert{Neural Machine Translation (NMT)}，一些代表性工作：
\end{itemize}

\visible<2->{
\begin{center}
{\footnotesize
\begin{tabular}{l | l | l}
\textbf{时间} & \textbf{作者} & \textbf{论文} \\ \hline
2013 & Kalchbrenner和 & Recurrent Continuous Translation Models \\
     & Blunsom        & \\
2014 & Sutskever等 & Sequence to Sequence Learning with \\
     &             & neural networks \\
2014 & Bahdanau等 & Neural Machine Translation by Jointly  \\
     &       & Learning to Align and Translate \\
2014 & Cho等 & On the Properties of Neural Machine \\
     &       & Translation \\
2015 & Jean等 & On Using Very Large Target Vocabulary \\
     &        & for Neural Machine Translation \\
2015 & Luong等 & Effective Approaches to Attention-based \\
         &               & Neural Machine Translation
\end{tabular}
}
\end{center}
}

\end{frame}

%%%------------------------------------------------------------------------------------------------------------
%%% 逐渐崛起的NMT
\begin{frame}{崛起}
\begin{itemize}
\item 2015年前统计机器翻译(SMT)在NLP是具有统治力的
    \begin{itemize}
    \item 当时NMT的系统还很初级，被SMT碾压
    \item 大多数的认知还没有进化到NMT时代，甚至Kalchbrenner等人早期的报告也被人质疑
    \end{itemize}
\item 2016年情况大有改变，当时非常受关注的一项工作是Google上线了神经机器翻译系统GNMT
    \begin{itemize}
    \item 在GNMT前后，百度、微软、小牛翻译等也分别推出了自己的神经机器翻译系统，出现了百花齐放的局面
    \end{itemize}
\end{itemize}
\begin{center}
\includegraphics[scale=0.35]{./Figures/google-news.png}
\end{center}
\end{frame}

%%%------------------------------------------------------------------------------------------------------------
%%% 模型结构
\begin{frame}{基于循环神经网络的翻译模型}
\begin{itemize}
\item 一种简单的模型：用循环神经网络进行编码和解码
    \begin{itemize}
    \item 编码端是一个RNN，最后一个隐层状态被看做句子表示
    \item 解码端也是一个RNN，利用编码结果逐词解码出译文
    \end{itemize}
\end{itemize}
%%% 图
\begin{center}
    \begin{tikzpicture}
        \newlength{\base}
        \setlength{\base}{0.9cm}

        \begin{scope}
            \tikzstyle{rnnnode} = [rounded corners=2pt,minimum width=0.5\base,minimum height=0.5\base,draw]
            \tikzstyle{wordnode} = [font=\tiny]

            % RNN Encoder
            \coordinate (eemb0) at (0,0);
            \foreach \x [count=\y from 0] in {1,2,...,10}
                \node[rnnnode,fill=green!30!white,anchor=west] (eemb\x) at ([xshift=0.5\base]eemb\y.east) {};
            \foreach \x in {1,2,...,10}
                \node[rnnnode,fill=blue!30!white,anchor=south] (enc\x) at ([yshift=0.5\base]eemb\x.north) {};
            \node[wordnode,left=0.5\base of enc1] (init) {$0$};
            
            \node[wordnode,below=0pt of eemb1] () {你};
            \node[wordnode,below=0pt of eemb2] () {知道};
            \node[wordnode,below=0pt of eemb3] () {去};
            \node[wordnode,below=0pt of eemb4] () {北京站};
            \node[wordnode,below=0pt of eemb5] () {的};
            \node[wordnode,below=0pt of eemb6] () {路};
            \node[wordnode,below=0pt of eemb7] () {怎么};
            \node[wordnode,below=0pt of eemb8] () {走};
            \node[wordnode,below=0pt of eemb9] () {吗};
            \node[wordnode,below=0pt of eemb10] () {$\langle$eos$\rangle$};
            
            % RNN Decoder
            \foreach \x in {1,2,...,10}
                \node[rnnnode,fill=green!30!white,anchor=south] (demb\x) at ([yshift=1.5\base]enc\x.north) {};
            \foreach \x in {1,2,...,10}
                \node[rnnnode,fill=blue!30!white,anchor=south] (dec\x) at ([yshift=0.5\base]demb\x.north) {};
            
            % Decoder input words
            \node[wordnode,below=0pt of demb1] (decwordin) {$\langle$eos$\rangle$};
            \ExtractX{$(demb2.south)$}
            \ExtractY{$(decwordin.base)$}
            \node[wordnode,anchor=base] () at (\XCoord,\YCoord) {Do};
            \ExtractX{$(demb3.south)$}
            \ExtractY{$(decwordin.base)$}
            \node[wordnode,anchor=base] () at (\XCoord,\YCoord) {you};
            \ExtractX{$(demb4.south)$}
            \ExtractY{$(decwordin.base)$}
            \node[wordnode,anchor=base] () at (\XCoord,\YCoord) {know};
            \ExtractX{$(demb5.south)$}
            \ExtractY{$(decwordin.base)$}
            \node[wordnode,anchor=base] () at (\XCoord,\YCoord) {the};
            \ExtractX{$(demb6.south)$}
            \ExtractY{$(decwordin.base)$}
            \node[wordnode,anchor=base] () at (\XCoord,\YCoord) {way};
            \ExtractX{$(demb7.south)$}
            \ExtractY{$(decwordin.base)$}
            \node[wordnode,anchor=base] () at (\XCoord,\YCoord) {to};
            \ExtractX{$(demb8.south)$}
            \ExtractY{$(decwordin.base)$}
            \node[wordnode,anchor=base] () at (\XCoord,\YCoord) {Beijing};
            \ExtractX{$(demb9.south)$}
            \ExtractY{$(decwordin.base)$}
            \node[wordnode,anchor=base] () at (\XCoord,\YCoord) {Railway};
            \ExtractX{$(demb10.south)$}
            \ExtractY{$(decwordin.base)$}
            \node[wordnode,anchor=base] () at (\XCoord,\YCoord) {Station};
            
            % Decoder output words
            \node[wordnode,above=0.5\base of dec1] (decwordout) {Do};
            \ExtractX{$(dec2.north)$}
            \ExtractY{$(decwordout.base)$}
            \node[wordnode,anchor=base] () at (\XCoord,\YCoord) {you};
            \ExtractX{$(dec3.north)$}
            \ExtractY{$(decwordout.base)$}
            \node[wordnode,anchor=base] () at (\XCoord,\YCoord) {know};
            \ExtractX{$(dec4.north)$}
            \ExtractY{$(decwordout.base)$}
            \node[wordnode,anchor=base] () at (\XCoord,\YCoord) {the};
            \ExtractX{$(dec5.north)$}
            \ExtractY{$(decwordout.base)$}
            \node[wordnode,anchor=base] () at (\XCoord,\YCoord) {way};
            \ExtractX{$(dec6.north)$}
            \ExtractY{$(decwordout.base)$}
            \node[wordnode,anchor=base] () at (\XCoord,\YCoord) {to};
            \ExtractX{$(dec7.north)$}
            \ExtractY{$(decwordout.base)$}
            \node[wordnode,anchor=base] () at (\XCoord,\YCoord) {Beijing};
            \ExtractX{$(dec8.north)$}
            \ExtractY{$(decwordout.base)$}
            \node[wordnode,anchor=base] () at (\XCoord,\YCoord) {Railway};
            \ExtractX{$(dec9.north)$}
            \ExtractY{$(decwordout.base)$}
            \node[wordnode,anchor=base] () at (\XCoord,\YCoord) {Station};
            \ExtractX{$(dec10.north)$}
            \ExtractY{$(decwordout.base)$}
            \node[wordnode,anchor=base] () at (\XCoord,\YCoord) {$\langle$eos$\rangle$};
            
            % Connections
            \draw[-latex'] (init.east) to (enc1.west);
            \foreach \x in {1,2,...,10}
                \draw[-latex'] (eemb\x) to (enc\x);
            \foreach \x in {1,2,...,10}
                \draw[-latex'] (demb\x) to (dec\x);
            \foreach \x in {1,2,...,10}
                \draw[-latex'] (dec\x.north) to ([yshift=0.5\base]dec\x.north);
            \foreach \x [count=\y from 2] in {1,2,...,9}
            {
                \draw[-latex'] (enc\x.east) to (enc\y.west);
                \draw[-latex'] (dec\x.east) to (dec\y.west);
            }
            \coordinate (bridge1) at ([yshift=-\base]demb5);
            \coordinate (bridge2) at ([yshift=-\base]demb2.west);
            \draw[-latex'] (enc10.north) .. controls +(north:0.8\base) and +(east:\base) .. (bridge1) to (bridge2) .. controls +(west:2.5\base) and +(west:0.6\base) .. (dec1.west);
        \end{scope}
    \end{tikzpicture}
\end{center}
\end{frame}

%%%------------------------------------------------------------------------------------------------------------
%%% 词嵌入
\begin{frame}{模块1：词嵌入层}
\begin{itemize}
\item 词嵌入
\end{itemize}
%%% 图
\end{frame}

%%%------------------------------------------------------------------------------------------------------------
%%% 输出
\begin{frame}{模块2：输出层}
\begin{itemize}
\item Softmax
\end{itemize}
%%% 图
\end{frame}

%%%------------------------------------------------------------------------------------------------------------
%%% LSTM
\begin{frame}{模块3：循环单元 - 长短时记忆模型(LSTM) (2页？)}
\begin{itemize}
\item LSTM
\end{itemize}
%%% 图
\end{frame}

%%%------------------------------------------------------------------------------------------------------------
%%% GRU
\begin{frame}{改进 - 门循环单元(GRU)}
\begin{itemize}
\item GRU
\end{itemize}
%%% 图
\end{frame}

%%%------------------------------------------------------------------------------------------------------------
%%% 一些变种
\begin{frame}{改进 - 双向模型}
%%% 图
\end{frame}

%%%------------------------------------------------------------------------------------------------------------
%%% 一些变种
\begin{frame}{改进 - 多层网络}
%%% 图
\end{frame}

%%%------------------------------------------------------------------------------------------------------------
%%% 一些变种
\begin{frame}{改进 - fine-tuning}
%%% 图
\end{frame}

%%%------------------------------------------------------------------------------------------------------------
\subsection{模型结构}

%%%------------------------------------------------------------------------------------------------------------
\subsection{注意力机制}

%%%------------------------------------------------------------------------------------------------------------
\section{Transformer}

%%%------------------------------------------------------------------------------------------------------------
\subsection{多头自注意力模型}

%%%------------------------------------------------------------------------------------------------------------
\subsection{训练}

%%%------------------------------------------------------------------------------------------------------------
\subsection{推断}

%%%------------------------------------------------------------------------------------------------------------
\subsection{深层网络}

%%%------------------------------------------------------------------------------------------------------------
\section{其它应用}

\end{CJK}
\end{document}
