13.4

Chapter13/Figures/figure-exposure-bias.tex

+
+
+%------------------------------------------------------------
+
+\begin{tikzpicture}
+
+\tikzstyle{rnnnode} = [draw,inner sep=2pt,minimum width=4em,minimum height=2em,rounded corners=1pt,fill=red!20]
+\tikzstyle{snode} = [draw,inner sep=2pt,minimum width=4em,minimum height=2em,rounded corners=1pt,fill=blue!20]
+\tikzstyle{ynode} = [inner sep=2pt,minimum width=4em,minimum height=2em,rounded corners=1pt]
+
+
+\node [anchor=west,rnnnode] (n1) at (0,0) {$\mathbi{h}_{1}$};
+\node [anchor=west] (n2) at ([xshift=3em,yshift=0em]n1.east) {$\cdots$};
+\node [anchor=west,rnnnode] (n3) at ([xshift=3em,yshift=0em]n2.east) {$\mathbi{h}_{j-1}$};
+\node [anchor=west,rnnnode] (n4) at ([xshift=3em,yshift=0em]n3.east) {$\mathbi{h}_{j}$};
+\node [anchor=south,snode] (n5) at ([xshift=0em,yshift=1em]n3.north) {Softmax};
+\node [anchor=south,ynode] (n6) at ([xshift=0em,yshift=1em]n5.north) {$\tilde{\mathbi{y}}_{j-1}$};
+\node [anchor=south,snode] (n7) at ([xshift=0em,yshift=1em]n4.north) {Softmax};
+\node [anchor=south,ynode] (n8) at ([xshift=0em,yshift=1em]n7.north) {$\tilde{\mathbi{y}}_{j}$};
+
+\node [anchor=north] (x1) at ([xshift=0em,yshift=-1em]n1.south) {$\seq{x}$};
+\node [anchor=north,font=\small,align=left] (x2) at ([xshift=-4em,yshift=-1.7em]n3.south) {采样出\\的$\tilde{\mathbi{y}}_{j-2}$};
+\node [anchor=north,font=\small,align=left] (x3) at ([xshift=2em,yshift=-2.5em]n3.south) {真实答\\案$\mathbi{y}_{j-2}$};
+\node [anchor=north,font=\small,align=left] (x4) at ([xshift=2em,yshift=-2.5em]n4.south) {真实答\\案$\mathbi{y}_{j-1}$};
+
+\node [anchor=south,inner sep=2pt] (st1) at (n6.north) {\scriptsize{\textbf{[step $j-1$]}}};
+\node [anchor=south,inner sep=2pt] (st2) at (n8.north) {\scriptsize{\textbf{[step $j$]}}};
+
+\node [anchor=north,font=\scriptsize] (e1) at ([xshift=-3em,yshift=-0em]n3.south) {$\funp{P}={(1-\epsilon_i)}^2$};
+\node [anchor=north,font=\scriptsize] (e2) at ([xshift=2em,yshift=-0.1em]n3.south) {$\funp{P}=\epsilon_i$};
+\node [anchor=north,font=\scriptsize] (e3) at ([xshift=-2em,yshift=-1em]n4.south) {$\funp{P}={(1-\epsilon_i)}^2$};
+\node [anchor=north,font=\scriptsize] (e4) at ([xshift=2em,yshift=-0.1em]n4.south) {$\funp{P}=\epsilon_i$};
+
+\node [anchor=south east,font=\small] (l1) at ([xshift=-1em,yshift=0.5em]n5.north west) {Loss};
+\node [anchor=south west,font=\small] (l2) at ([xshift=1em,yshift=0.5em]n7.north east) {Loss};
+
+\draw [->,thick] ([xshift=0em,yshift=0em]x1.north)--([xshift=0em,yshift=0em]n1.south);
+\draw [->,thick] ([xshift=0em,yshift=0em]n1.east)--([xshift=0em,yshift=0em]n2.west);
+\draw [->,thick] ([xshift=0em,yshift=0em]n2.east)--([xshift=0em,yshift=0em]n3.west);
+\draw [->,thick] ([xshift=0em,yshift=0em]n3.east)--([xshift=0em,yshift=0em]n4.west);
+\draw [->,thick] ([xshift=0em,yshift=0em]n4.east)--([xshift=3em,yshift=0em]n4.east);
+
+
+\draw [->,thick] ([xshift=0em,yshift=0em]n3.north)--([xshift=0em,yshift=0em]n5.south);
+\draw [->,thick] ([xshift=0em,yshift=0em]n5.north)--([xshift=0em,yshift=0em]n6.south);
+\draw [->,thick] ([xshift=0em,yshift=0em]n4.north)--([xshift=0em,yshift=0em]n7.south);
+\draw [->,thick] ([xshift=0em,yshift=0em]n7.north)--([xshift=0em,yshift=0em]n8.south);
+
+\draw [->,thick] ([xshift=0em,yshift=0em]l1.south) .. controls +(south:1em) and +(west:0.1em) .. ([xshift=0em,yshift=0em]n5.west);
+\draw [->,thick] ([xshift=0em,yshift=0em]l2.south) .. controls +(south:1em) and +(east:0.1em) .. ([xshift=0em,yshift=0em]n7.east);
+
+\draw [->,thick,dotted] ([xshift=0em,yshift=-0.5em]x2.north east) .. controls +(east:1.5em) and +(south:0.2em) .. ([xshift=-0.5em,yshift=0em]n3.south);
+\draw [->,thick] ([xshift=0em,yshift=0em]x3.north) .. controls +(north:1em) and +(south:2em) .. ([xshift=0em,yshift=0em]n3.south);
+\draw [->,thick,dotted] ([xshift=0em,yshift=0em]n6.east) .. controls ([xshift=2em,yshift=1em]n6.east) and ([xshift=-2em,yshift=-5em]n4.south west) .. ([xshift=-0.5em,yshift=-0em]n4.south);
+\draw [->,thick] ([xshift=0em,yshift=0em]x4.north) .. controls +(north:1em) and +(south:2em) .. ([xshift=0em,yshift=0em]n4.south);
+
+\end{tikzpicture}

Chapter13/Figures/figure-of-scheduling-sampling-method.tex

+
+
+%------------------------------------------------------------
+
+\begin{tikzpicture}
+
+\tikzstyle{rnnnode} = [draw,inner sep=2pt,minimum width=4em,minimum height=2em,rounded corners=1pt,fill=red!20]
+\tikzstyle{snode} = [draw,inner sep=2pt,minimum width=4em,minimum height=2em,rounded corners=1pt,fill=blue!20]
+\tikzstyle{ynode} = [inner sep=2pt,minimum width=4em,minimum height=2em,rounded corners=1pt]
+
+
+\node [anchor=west,rnnnode] (n1) at (0,0) {$\mathbi{h}_{1}$};
+\node [anchor=west] (n2) at ([xshift=3em,yshift=0em]n1.east) {$\cdots$};
+\node [anchor=west,rnnnode] (n3) at ([xshift=3em,yshift=0em]n2.east) {$\mathbi{h}_{j-1}$};
+\node [anchor=west,rnnnode] (n4) at ([xshift=3em,yshift=0em]n3.east) {$\mathbi{h}_{j}$};
+\node [anchor=south,snode] (n5) at ([xshift=0em,yshift=1em]n3.north) {Softmax};
+\node [anchor=south,ynode] (n6) at ([xshift=0em,yshift=1em]n5.north) {$\tilde{\mathbi{y}}_{j-1}$};
+\node [anchor=south,snode] (n7) at ([xshift=0em,yshift=1em]n4.north) {Softmax};
+\node [anchor=south,ynode] (n8) at ([xshift=0em,yshift=1em]n7.north) {$\tilde{\mathbi{y}}_{j}$};
+
+\node [anchor=north] (x1) at ([xshift=0em,yshift=-1em]n1.south) {$\seq{x}$};
+\node [anchor=north,font=\small,align=left] (x2) at ([xshift=-4em,yshift=-1.7em]n3.south) {采样出\\的$\tilde{\mathbi{y}}_{j-2}$};
+\node [anchor=north,font=\small,align=left] (x3) at ([xshift=2em,yshift=-2.5em]n3.south) {真实答\\案$\mathbi{y}_{j-2}$};
+\node [anchor=north,font=\small,align=left] (x4) at ([xshift=2em,yshift=-2.5em]n4.south) {真实答\\案$\mathbi{y}_{j-1}$};
+
+\node [anchor=south,inner sep=2pt] (st1) at (n6.north) {\scriptsize{\textbf{[step $j-1$]}}};
+\node [anchor=south,inner sep=2pt] (st2) at (n8.north) {\scriptsize{\textbf{[step $j$]}}};
+
+\node [anchor=north,font=\scriptsize] (e1) at ([xshift=-3em,yshift=-0em]n3.south) {$\funp{P}={(1-\epsilon_i)}^2$};
+\node [anchor=north,font=\scriptsize] (e2) at ([xshift=2em,yshift=-0.1em]n3.south) {$\funp{P}=\epsilon_i$};
+\node [anchor=north,font=\scriptsize] (e3) at ([xshift=-2em,yshift=-1em]n4.south) {$\funp{P}={(1-\epsilon_i)}^2$};
+\node [anchor=north,font=\scriptsize] (e4) at ([xshift=2em,yshift=-0.1em]n4.south) {$\funp{P}=\epsilon_i$};
+
+\node [anchor=south east,font=\small] (l1) at ([xshift=-1em,yshift=0.5em]n5.north west) {Loss};
+\node [anchor=south west,font=\small] (l2) at ([xshift=1em,yshift=0.5em]n7.north east) {Loss};
+
+\draw [->,thick] ([xshift=0em,yshift=0em]x1.north)--([xshift=0em,yshift=0em]n1.south);
+\draw [->,thick] ([xshift=0em,yshift=0em]n1.east)--([xshift=0em,yshift=0em]n2.west);
+\draw [->,thick] ([xshift=0em,yshift=0em]n2.east)--([xshift=0em,yshift=0em]n3.west);
+\draw [->,thick] ([xshift=0em,yshift=0em]n3.east)--([xshift=0em,yshift=0em]n4.west);
+\draw [->,thick] ([xshift=0em,yshift=0em]n4.east)--([xshift=3em,yshift=0em]n4.east);
+
+
+\draw [->,thick] ([xshift=0em,yshift=0em]n3.north)--([xshift=0em,yshift=0em]n5.south);
+\draw [->,thick] ([xshift=0em,yshift=0em]n5.north)--([xshift=0em,yshift=0em]n6.south);
+\draw [->,thick] ([xshift=0em,yshift=0em]n4.north)--([xshift=0em,yshift=0em]n7.south);
+\draw [->,thick] ([xshift=0em,yshift=0em]n7.north)--([xshift=0em,yshift=0em]n8.south);
+
+\draw [->,thick] ([xshift=0em,yshift=0em]l1.south) .. controls +(south:1em) and +(west:0.1em) .. ([xshift=0em,yshift=0em]n5.west);
+\draw [->,thick] ([xshift=0em,yshift=0em]l2.south) .. controls +(south:1em) and +(east:0.1em) .. ([xshift=0em,yshift=0em]n7.east);
+
+\draw [->,thick,dotted] ([xshift=0em,yshift=-0.5em]x2.north east) .. controls +(east:1.5em) and +(south:0.2em) .. ([xshift=-0.5em,yshift=0em]n3.south);
+\draw [->,thick] ([xshift=0em,yshift=0em]x3.north) .. controls +(north:1em) and +(south:2em) .. ([xshift=0em,yshift=0em]n3.south);
+\draw [->,thick,dotted] ([xshift=0em,yshift=0em]n6.east) .. controls ([xshift=2em,yshift=1em]n6.east) and ([xshift=-2em,yshift=-5em]n4.south west) .. ([xshift=-0.5em,yshift=-0em]n4.south);
+\draw [->,thick] ([xshift=0em,yshift=0em]x4.north) .. controls +(north:1em) and +(south:2em) .. ([xshift=0em,yshift=0em]n4.south);
+
+\end{tikzpicture}

Chapter13/Figures/figure-reinforcement-learning-method-based-on-actor-critic.tex

@@ -12,17 +12,23 @@
 	\node[anchor=south,inner sep=0mm,font=\small] (c1) at ([xshift=0em,yshift=1em]n2.north) {评论家$Q$};
 	\node[anchor=north,inner sep=0mm] (c2) at ([xshift=0em,yshift=-1em]n2.south) {$\mathbi{y}_1,\mathbi{y}_2,\ldots,\mathbi{y}_J$};
 
-	\node[anchor=west,inner sep=0mm] (n3) at ([xshift=2.1em,yshift=2em]n1.east) {$Q_1,Q_2,\ldots,Q_J$};
-	\node[anchor=west,inner sep=0mm] (n4) at ([xshift=2.9em,yshift=-0.4em]n1.east) {$\hat{\mathbi{y}}_1,\hat{\mathbi{y}}_2,\ldots,\hat{\mathbi{y}}_J$};
-	\node[anchor=west,inner sep=0mm,font=\small] (n5) at ([xshift=3em,yshift=-3em]n1.east) {演员状态};
+%	\node[anchor=west,inner sep=0mm] (n3) at ([xshift=2.1em,yshift=2em]n1.east) {$Q_1,Q_2,\ldots,Q_J$};
+%	\node[anchor=west,inner sep=0mm] (n4) at ([xshift=2.9em,yshift=-0.4em]n1.east) {$\hat{\mathbi{y}}_1,\hat{\mathbi{y}}_2,\ldots,\hat{\mathbi{y}}_J$};
+%	\node[anchor=west,inner sep=0mm,font=\small] (n5) at ([xshift=3em,yshift=-3em]n1.east) {演员状态};
 
 \draw [-,thick] ([xshift=0em,yshift=0em]n1.west) -- ([xshift=0em,yshift=0em]n1.east);
 \draw [-,thick] ([xshift=0em,yshift=0em]n2.west) -- ([xshift=0em,yshift=0em]n2.east);
 
-\draw [->,thick] ([xshift=0em,yshift=1em]n2.west) -- ([xshift=0em,yshift=1em]n1.east);
-\draw [->,thick] ([xshift=0em,yshift=0.5em]n1.east) -- ([xshift=0em,yshift=0.5em]n2.west);
+%\draw [->,thick] ([xshift=0em,yshift=1em]n2.west) -- ([xshift=0em,yshift=1em]n1.east);
+%\draw [->,thick] ([xshift=0em,yshift=0.5em]n1.east) -- ([xshift=0em,yshift=0.5em]n2.west);
 
-\draw [->,dotted,very thick] ([xshift=0em,yshift=0em]n1.east)  .. controls ([xshift=3em,yshift=-1em]n1.-90) and ([xshift=-3em,yshift=-1em]n2.-90) .. (n2.west);
+%\draw [->,dotted,very thick] ([xshift=0em,yshift=0em]n1.east)  .. controls ([xshift=3em,yshift=-1em]n1.-90) and ([xshift=-3em,yshift=-1em]n2.-90) .. (n2.west);
+
+	\node[anchor=west,inner sep=0mm] (n3) at ([xshift=2.1em,yshift=1em]n1.east) {$Q_1,Q_2,\ldots,Q_J$};
+	\node[anchor=west,inner sep=0mm] (n4) at ([xshift=2.9em,yshift=-1em]n1.east) {$\tilde{\mathbi{y}}_1,\tilde{\mathbi{y}}_2,\ldots,\tilde{\mathbi{y}}_J$};
+
+\draw [->,thick] ([xshift=0em,yshift=0.2em]n2.west) -- ([xshift=0em,yshift=0.2em]n1.east);
+\draw [->,thick] ([xshift=0em,yshift=-0.2em]n1.east) -- ([xshift=0em,yshift=-0.2em]n2.west);
 
 
 \end{tikzpicture}
\ No newline at end of file

Chapter13/Figures/figure-word-root.tex

@@ -11,8 +11,8 @@
 \node[anchor = north] (new_root) at ([yshift = -1.5em]newer.south) {new};
 \draw [->] ([yshift=0.2em]do_root.north) .. controls +(north:0.4) and +(south:0.6) ..(do.south);
 \draw [->] (do_root.north) -- (does.south);
-\draw [->] ([yshift=0.2em]do_root.north) .. controls +(north:0.4) and +(south:0.6) ..([yshift=0.1em]doing.south);
+\draw [->] ([yshift=0.2em]do_root.north) .. controls +(north:0.4) and +(south:0.6) ..(doing.south);
 \draw [->] ([yshift=0.2em]new_root.north) .. controls +(north:0.4) and +(south:0.6) ..(new.south);
 \draw [->] (new_root.north) -- (newer.south);
-\draw [->] ([yshift=0.2em]new_root.north) .. controls +(north:0.4) and +(south:0.6) ..([yshift=0.08em]newest.south);
+\draw [->] ([yshift=0.2em]new_root.north) .. controls +(north:0.4) and +(south:0.6) ..(newest.south);
 \end{tikzpicture}
\ No newline at end of file