13章

Chapter13/Figures/figure-bpe.tex

 \begin{tikzpicture}
-\tikzstyle{tnode} = [rectangle,inner sep=0em,minimum width=8em,minimum height=6.6em,rounded corners=5pt,fill=ugreen!5,drop shadow]
+\tikzstyle{tnode} = [rectangle,inner sep=0em,minimum width=8em,minimum height=6.6em,rounded corners=5pt,fill=green!5,drop shadow]
 \tikzstyle{pnode} = [rectangle,inner sep=0em,minimum width=8em,minimum height=6.6em,rounded corners=5pt,fill=yellow!5,drop shadow]
 \tikzstyle{mnode} = [rectangle,inner sep=0em,minimum width=8em,minimum height=6.6em,rounded corners=5pt,fill=red!5,drop shadow]
 \tikzstyle{wnode} = [inner sep=0em,minimum height=1.5em]
@@ -19,7 +19,7 @@
 
 
 \begin{pgfonlayer}{background}
-\node [rectangle,inner sep=0.7em,draw,ugreen!40,dashed,very thick,rounded corners=7pt] [fit = (n1) (n4)] (box1) {};
+\node [rectangle,inner sep=0.7em,draw,ugreen!60,dashed,very thick,rounded corners=7pt] [fit = (n1) (n4)] (box1) {};
 \end{pgfonlayer}
 
 \node [anchor=west,align=left,font=\footnotesize] (nt1) at ([xshift=0.1em,yshift=0em]n2.east) {统计词表和\\[0.5ex]词频};
@@ -75,7 +75,7 @@
 \node [anchor=east,ublue,align=left,font=\footnotesize] (l3) at ([xshift=-0.5em,yshift=0em]cd.west) {直至达到设定的符号合\\并表大小或无法合并};
 
 \begin{pgfonlayer}{background}
-\node [rectangle,inner sep=0.7em,draw,yellow!40,dashed,very thick,rounded corners=7pt] [fit = (n5) (n8) (l3) (cd)] (box2) {};
+\node [rectangle,inner sep=0.7em,draw,orange!40,dashed,very thick,rounded corners=7pt] [fit = (n5) (n8) (l3) (cd)] (box2) {};
 \end{pgfonlayer}
 
 %第五排

Chapter13/Figures/figure-exposure-bias.tex

@@ -18,11 +18,11 @@
 \node [anchor=west,rnnnode] (n3) at ([xshift=2em,yshift=0em]n2.east) {$\mathbi{h}_{j-1}$};
 \node [anchor=west,rnnnode] (n4) at ([xshift=2em,yshift=0em]n3.east) {$\mathbi{h}_{j}$};
 \node [anchor=south,snode,font=\footnotesize] (n5) at ([xshift=0em,yshift=1em]n3.north) {Softmax};
-\node [anchor=south,ynode] (n6) at ([xshift=0em,yshift=1em]n5.north) {$\tilde{{y}}_{j-1}$};
+\node [anchor=south,ynode] (n6) at ([xshift=0em,yshift=1em]n5.north) {$\hat{{y}}_{j-1}$};
 \node [anchor=south,snode,font=\footnotesize] (n7) at ([xshift=0em,yshift=1em]n4.north) {Softmax};
-\node [anchor=south,ynode] (n8) at ([xshift=0em,yshift=1em]n7.north) {$\tilde{{y}}_{j}$};
+\node [anchor=south,ynode] (n8) at ([xshift=0em,yshift=1em]n7.north) {$\hat{{y}}_{j}$};
 \node [anchor=south,snode,font=\footnotesize] (n13) at ([xshift=0em,yshift=1em]n1.north) {Softmax};
-\node [anchor=south,ynode] (n14) at ([xshift=0em,yshift=1em]n13.north) {$\tilde{{y}}_{1}$};
+\node [anchor=south,ynode] (n14) at ([xshift=0em,yshift=1em]n13.north) {$\hat{{y}}_{1}$};
 
 \node [anchor=north] (n11) at ([xshift=0em,yshift=-1em]n3.south) {${{y}}_{j-2}$};
 \node [anchor=north] (n12) at ([xshift=0em,yshift=-1em]n4.south) {${{y}}_{j-1}$};
@@ -132,16 +132,16 @@
 \node [anchor=west,rnnnode] (n3) at ([xshift=2em,yshift=0em]n2.east) {$\mathbi{h}_{j-1}$};
 \node [anchor=west,rnnnode] (n4) at ([xshift=2em,yshift=0em]n3.east) {$\mathbi{h}_{j}$};
 \node [anchor=south,snode,font=\footnotesize] (n5) at ([xshift=0em,yshift=1em]n3.north) {Softmax};
-\node [anchor=south,ynode] (n6) at ([xshift=0em,yshift=1em]n5.north) {$\tilde{{y}}_{j-1}$};
+\node [anchor=south,ynode] (n6) at ([xshift=0em,yshift=1em]n5.north) {$\hat{{y}}_{j-1}$};
 \node [anchor=south,snode,font=\footnotesize] (n7) at ([xshift=0em,yshift=1em]n4.north) {Softmax};
-\node [anchor=south,ynode] (n8) at ([xshift=0em,yshift=1em]n7.north) {$\tilde{{y}}_{j}$};
+\node [anchor=south,ynode] (n8) at ([xshift=0em,yshift=1em]n7.north) {$\hat{{y}}_{j}$};
 \node [anchor=south,snode,font=\footnotesize] (n13) at ([xshift=0em,yshift=1em]n1.north) {Softmax};
-\node [anchor=south,ynode] (n14) at ([xshift=0em,yshift=1em]n13.north) {$\tilde{{y}}_{1}$};
+\node [anchor=south,ynode] (n14) at ([xshift=0em,yshift=1em]n13.north) {$\hat{{y}}_{1}$};
 
 \node [anchor=north] (n9) at ([xshift=0em,yshift=-3em]n4.south) {\small{(b) 推断阶段}};
 
-\node [anchor=north] (n11) at ([xshift=0em,yshift=-1em]n3.south) {$\tilde{{y}}_{j-2}$};
-\node [anchor=north] (n12) at ([xshift=0em,yshift=-1em]n4.south) {$\tilde{{y}}_{j-1}$};
+\node [anchor=north] (n11) at ([xshift=0em,yshift=-1em]n3.south) {$\hat{{y}}_{j-2}$};
+\node [anchor=north] (n12) at ([xshift=0em,yshift=-1em]n4.south) {$\hat{{y}}_{j-1}$};
 
 \node [anchor=north] (x1) at ([xshift=0em,yshift=-1em]n1.south) {$\langle$sos$\rangle$};
 
@@ -172,13 +172,13 @@
 \draw [->,thick] ([xshift=-0em,yshift=0em]n11.north)--([xshift=-0em,yshift=0em]n3.south);
 \draw [->,thick] ([xshift=0em,yshift=0em]n12.north)--([xshift=-0em,yshift=-0em]n4.south);
 
-\node [anchor=west] (n10) at ([xshift=4em,yshift=0em]n8.east) {模型输出$\tilde{{y}}$：};
+\node [anchor=west] (n10) at ([xshift=4em,yshift=0em]n8.east) {模型输出$\hat{{y}}$：};
 
-\node [anchor=north west] (y1) at ([xshift=0.5em,yshift=-7em]n10.south west) {$\tilde{y}_{1}$};
-\node [anchor=west] (y2) at ([xshift=1em,yshift=0em]y1.east) {$\tilde{y}_{2}$};
-\node [anchor=west] (y3) at ([xshift=1em,yshift=0em]y2.east) {$\tilde{y}_{3}$};
+\node [anchor=north west] (y1) at ([xshift=0.5em,yshift=-7em]n10.south west) {$\hat{y}_{1}$};
+\node [anchor=west] (y2) at ([xshift=1em,yshift=0em]y1.east) {$\hat{y}_{2}$};
+\node [anchor=west] (y3) at ([xshift=1em,yshift=0em]y2.east) {$\hat{y}_{3}$};
 \node [anchor=west] (y4) at ([xshift=0.5em,yshift=0em]y3.east) {$\ldots$};
-\node [anchor=west] (y5) at ([xshift=0.5em,yshift=0em]y4.east) {$\tilde{y}_{n}$};
+\node [anchor=west] (y5) at ([xshift=0.5em,yshift=0em]y4.east) {$\hat{y}_{n}$};
 
 
 \node [anchor=center,prob,minimum size=0.3em] (label11) at ([xshift=-0.1em,yshift=1em]y1.north) {};

Chapter13/Figures/figure-framework-of-Adversarial-Neural-machine-translation.tex

 
-
 %------------------------------------------------------------
 
 \begin{tikzpicture}
 
-\tikzstyle{rnnnode} = [draw,inner sep=4pt,minimum width=2em,minimum height=2em,rounded corners=1pt,fill=yellow!15]
+\tikzstyle{rnnnode} = [draw,inner sep=4pt,minimum width=2em,minimum height=2em,rounded corners=1pt,fill=green!15]
 \tikzstyle{snode} = [draw,inner sep=4pt,minimum width=2em,minimum height=2em,rounded corners=1pt,fill=red!15]
 \tikzstyle{wode} = [inner sep=0pt,minimum width=2em,minimum height=2em,rounded corners=0pt]
 
 \node [anchor=west,wode] (n1) at (0,0) {$y$};
 \node [anchor=north west,wode] (n2) at ([xshift=3em,yshift=-2.5em]n1.south east) {$x$};
 \node [anchor=south west,rnnnode] (n3) at ([xshift=8em,yshift=0.5em]n2.north east) {生成模型$G$};
-\node [anchor=south east,wode] (n4) at ([xshift=-2em,yshift=0em]n3.north west) {$\tilde{y}$};
+\node [anchor=south east,wode] (n4) at ([xshift=-2em,yshift=0em]n3.north west) {$\hat{y}$};
 \node [anchor=south,snode] (n5) at ([xshift=0em,yshift=6em]n2.north) {判别网络$D$};
-\node [anchor=west,align=left,font=\small] (n6) at ([xshift=15em,yshift=-3em]n5.east) {根据$(\seq{x},\seq{\tilde{y}})$生\\成奖励信号};
+\node [anchor=west,align=left,font=\small] (n6) at ([xshift=15em,yshift=-3em]n5.east) {根据$(\seq{x},\seq{\hat{y}})$生\\成奖励信号};
 
 
 \draw [->,thick] ([xshift=0em,yshift=-0.3em]n1.north)--([xshift=-0.3em,yshift=-0.1em]n5.south);

Chapter13/Figures/figure-label-smoothing.tex

@@ -38,20 +38,20 @@
     \node [anchor=south,font=\scriptsize] (one_hot_w7) at (one_hot_label7.north) {$0$};
 
     %label smoothing
-	\node [anchor=west,minimum width=1.2em,minimum height=0.2em,fill=red!50,inner sep=0pt] (label1) at ([xshift=1.5em,yshift=-4.4em]model.east) {};
+	\node [anchor=west,minimum width=1.2em,minimum height=0.2em,fill=orange!50,inner sep=0pt] (label1) at ([xshift=1.5em,yshift=-4.4em]model.east) {};
     \node [anchor=south,font=\scriptsize] (w1) at (label1.north) {$0.1$};
-    \node [anchor=south west,minimum width=1.2em,minimum height=0.2em,fill=red!50,inner sep=0pt] (label2) at (label1.south east) {};
+    \node [anchor=south west,minimum width=1.2em,minimum height=0.2em,fill=orange!50,inner sep=0pt] (label2) at (label1.south east) {};
     \node [anchor=south,font=\scriptsize] (w2) at (label2.north) {$0.1$};
-    \node [anchor=south west,minimum width=1.2em,minimum height=0.8em,fill=red!50,inner sep=0pt] (label3) at (label2.south east) {};
+    \node [anchor=south west,minimum width=1.2em,minimum height=0.8em,fill=orange!50,inner sep=0pt] (label3) at (label2.south east) {};
     \node [anchor=south,font=\scriptsize] (w3) at (label3.north) {{\color{red} $0.4$}};
-    \node [anchor=south west,minimum width=1.2em,minimum height=0.2em,fill=red!50,inner sep=0pt] (label4) at (label3.south east) {};
+    \node [anchor=south west,minimum width=1.2em,minimum height=0.2em,fill=orange!50,inner sep=0pt] (label4) at (label3.south east) {};
     \node [anchor=south,font=\scriptsize] (w5) at (label4.north) {$0.1$};
 
-    \node [anchor=south west,minimum width=1.2em,minimum height=0.2em,fill=red!50,inner sep=0pt] (label5) at (label4.south east) {};
+    \node [anchor=south west,minimum width=1.2em,minimum height=0.2em,fill=orange!50,inner sep=0pt] (label5) at (label4.south east) {};
     \node [anchor=south,font=\scriptsize] (w6) at (label5.north) {$0.1$};
-    \node [anchor=south west,minimum width=1.2em,minimum height=0.2em,fill=red!50,inner sep=0pt] (label6) at (label5.south east) {};
+    \node [anchor=south west,minimum width=1.2em,minimum height=0.2em,fill=orange!50,inner sep=0pt] (label6) at (label5.south east) {};
     \node [anchor=south,font=\scriptsize] (w7) at (label6.north) {$0.1$};
-    \node [anchor=south west,minimum width=1.2em,minimum height=0.2em,fill=red!50,inner sep=0pt] (label7) at (label6.south east) {};
+    \node [anchor=south west,minimum width=1.2em,minimum height=0.2em,fill=orange!50,inner sep=0pt] (label7) at (label6.south east) {};
     \node [anchor=south,font=\scriptsize] (w8) at (label7.north) {$0.1$};
     
 

Chapter13/Figures/figure-of-scheduling-sampling-method.tex

@@ -14,18 +14,18 @@
 \node [anchor=west,rnnnode] (n3) at ([xshift=3em,yshift=0em]n2.east) {$\mathbi{h}_{j-1}$};
 \node [anchor=west,rnnnode] (n4) at ([xshift=3em,yshift=0em]n3.east) {$\mathbi{h}_{j}$};
 \node [anchor=south,snode] (n5) at ([xshift=0em,yshift=1em]n3.north) {Softmax};
-\node [anchor=south,ynode] (n6) at ([xshift=0em,yshift=1em]n5.north) {$\tilde{{y}}_{j-1}$};
+\node [anchor=south,ynode] (n6) at ([xshift=0em,yshift=1em]n5.north) {$\hat{{y}}_{j-1}$};
 \node [anchor=south,snode] (n7) at ([xshift=0em,yshift=1em]n4.north) {Softmax};
-\node [anchor=south,ynode] (n8) at ([xshift=0em,yshift=1em]n7.north) {$\tilde{{y}}_{j}$};
+\node [anchor=south,ynode] (n8) at ([xshift=0em,yshift=1em]n7.north) {$\hat{{y}}_{j}$};
 
 \node [anchor=south,snode,font=\footnotesize] (n13) at ([xshift=0em,yshift=1em]n1.north) {Softmax};
-\node [anchor=south,ynode] (n14) at ([xshift=0em,yshift=1em]n13.north) {$\tilde{{y}}_{1}$};
+\node [anchor=south,ynode] (n14) at ([xshift=0em,yshift=1em]n13.north) {$\hat{{y}}_{1}$};
 
 \node [anchor=north] (x1) at ([xshift=0em,yshift=-1em]n1.south) {$\langle$sos$\rangle$};
-\node [anchor=north,font=\small] (x2) at ([xshift=-1.3em,yshift=-2.3em]n3.south) {$\tilde{{y}}_{j-2}$};
+\node [anchor=north,font=\small] (x2) at ([xshift=-1.3em,yshift=-2.3em]n3.south) {$\hat{{y}}_{j-2}$};
 \node [anchor=north,font=\small] (x3) at ([xshift=1.3em,yshift=-2.5em]n3.south) {${y}_{j-2}$};
 \node [anchor=north,font=\small] (x4) at ([xshift=1.3em,yshift=-2.5em]n4.south) {${y}_{j-1}$};
-\node [anchor=north,font=\small] (x5) at ([xshift=-1.3em,yshift=-2.3em]n4.south) {$\tilde{{y}}_{j-1}$};
+\node [anchor=north,font=\small] (x5) at ([xshift=-1.3em,yshift=-2.3em]n4.south) {$\hat{{y}}_{j-1}$};
 
 \node [anchor=south,inner sep=2pt] (st1) at (n6.north) {\scriptsize{\textbf{[step $j-1$]}}};
 \node [anchor=south,inner sep=2pt] (st2) at (n8.north) {\scriptsize{\textbf{[step $j$]}}};

bibliography.bib

@@ -9243,6 +9243,24 @@ author    = {Zhuang Liu and
   publisher = {Asian Conference on Machine Learning},
   year      = {2018}
 }
+@inproceedings{DBLP:journals/corr/abs-1810-02525,
+  author    = {Peter Henderson and
+               Joshua Romoff and
+               Joelle Pineau},
+  title     = {Where Did My Optimum Go?: An Empirical Analysis of Gradient Descent
+               Optimization in Policy Gradient Methods},
+  publisher   = {CoRR},
+  volume    = {abs/1810.02525},
+  year      = {2018}
+}
+
+@inproceedings{DBLP:conf/nips/Kakade01,
+  author    = {Sham M. Kakade},
+  title     = {A Natural Policy Gradient},
+  pages     = {1531--1538},
+  publisher = {Advances in Neural Information Processing Systems},
+  year      = {2001}
+}
 %%%%% chapter 15------------------------------------------------------
 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%