Skip to content

T
Toy-MT-Introduction
Commit 54be304a by xiaotong
Options

Merge branch 'master' of 47.105.50.196:NiuTrans/Toy-MT-Introduction

parents 1d54dd5e ada1321c
正在显示 包含 4341 行增加 和 6 行删除
Book/Chapter5/Figures/fig-4-gram.tex 0 → 100644
%%%------------------------------------------------------------------------------------------------------------
\begin{tikzpicture}
\begin{scope}
\node [anchor=west] (w0) at (0,0) {\footnotesize{$w_{i-3}$}};
\node [anchor=west] (w1) at ([xshift=2em]w0.east) {\footnotesize{$w_{i-2}$}};
\node [anchor=west] (w2) at ([xshift=2em]w1.east) {\footnotesize{$w_{i-1}$}};
\node [anchor=north] (index0) at ([yshift=0.5em]w0.south) {\tiny(index)};
\node [anchor=north] (index1) at ([yshift=0.5em]w1.south) {\tiny(index)};
\node [anchor=north] (index2) at ([yshift=0.5em]w2.south) {\tiny(index)};
\node [anchor=south,draw,inner sep=3pt] (e0) at ([yshift=1em]w0.north) {\tiny{$\textbf{e}_0=w_{i-3} \textbf{C}$}};
\node [anchor=south,draw,inner sep=3pt] (e1) at ([yshift=1em]w1.north) {\tiny{$\textbf{e}_1=w_{i-2} \textbf{C}$}};
\node [anchor=south,draw,inner sep=3pt] (e2) at ([yshift=1em]w2.north) {\tiny{$\textbf{e}_2=w_{i-1} \textbf{C}$}};
\node [anchor=south,draw,minimum width=9em,inner sep=3pt] (h0) at ([yshift=1.5em]e1.north) {\tiny{$\textbf{h}_0=\textrm{Tanh}([\textbf{e}_0,\textbf{e}_1,\textbf{e}_2] \textbf{H} + \textbf{d})$}};
\node [anchor=south,draw,minimum width=9em,inner sep=3pt] (h1) at ([yshift=1.5em]h0.north) {\tiny{$\textbf{y}=\textrm{Softmax}(\textbf{h}_0 \textbf{U})$}};
\node [anchor=south] (ylabel) at ([yshift=1em]h1.north) {\footnotesize{$\textrm{P}(w_i|w_{i-3}w_{i-2}w_{i-1})$}};
\draw [->] ([yshift=0.1em]w0.north) -- ([yshift=-0.1em]e0.south);
\draw [->] ([yshift=0.1em]w1.north) -- ([yshift=-0.1em]e1.south);
\draw [->] ([yshift=0.1em]w2.north) -- ([yshift=-0.1em]e2.south);
\draw [->] ([yshift=0.1em]e0.north) -- ([xshift=-2em,yshift=-0.1em]h0.south);
\draw [->] ([yshift=0.1em]e1.north) -- ([yshift=-0.1em]h0.south);
\draw [->] ([yshift=0.1em]e2.north) -- ([xshift=2em,yshift=-0.1em]h0.south);
\draw [->] ([yshift=0.1em]h0.north) -- ([yshift=-0.1em]h1.south);
\draw [->] ([yshift=0.1em]h1.north) -- ([yshift=-0.1em]ylabel.south);
{
\draw [->,dashed,red,thick] ([xshift=1em,yshift=0.1em]e1.north) -- ([xshift=1em,yshift=-0.1em]h1.south);
\draw [->,dashed,red,thick] ([xshift=-1em,yshift=0.1em]e0.north) .. controls +(north:2) and +(south:1) .. ([xshift=-3em,yshift=-0.1em]h1.south);
\draw [->,dashed,red,thick] ([xshift=1em,yshift=0.1em]e2.north) .. controls +(north:2) and +(south:1) .. ([xshift=3em,yshift=-0.1em]h1.south);
}
\begin{pgfonlayer}{background}
{
\node [rectangle,inner sep=0.1em,fill=ugreen!20!white] [fit = (w0) (index0)] (wordbox0) {};
\node [rectangle,inner sep=0.1em,fill=ugreen!20!white] [fit = (w1) (index1)] (wordbox1) {};
\node [rectangle,inner sep=0.1em,fill=ugreen!20!white] [fit = (w2) (index2)] (wordbox2) {};
}
\end{pgfonlayer}
{
\node [anchor=south,draw,inner sep=3pt,fill=blue!20!white] (e0) at ([yshift=1em]w0.north) {\tiny{$\textbf{e}_0=w_{i-3} \textbf{C}$}};
\node [anchor=south,draw,inner sep=3pt,fill=blue!20!white] (e1) at ([yshift=1em]w1.north) {\tiny{$\textbf{e}_1=w_{i-2} \textbf{C}$}};
\node [anchor=south,draw,inner sep=3pt,fill=blue!20!white] (e2) at ([yshift=1em]w2.north) {\tiny{$\textbf{e}_2=w_{i-1} \textbf{C}$}};
}
{
\node [anchor=south,draw,minimum width=9em,inner sep=3pt,fill=orange!20!white] (h0) at ([yshift=1.5em]e1.north) {\tiny{$\textbf{h}_0=\textrm{Tanh}([\textbf{e}_0,\textbf{e}_1,\textbf{e}_2] \textbf{H} + \textbf{d})$}};
\node [anchor=south,draw,minimum width=9em,inner sep=3pt,fill=orange!20!white] (h1) at ([yshift=1.5em]h0.north) {\tiny{$\textbf{y}=\textrm{Softmax}(\textbf{h}_0 \textbf{U})$}};
}
\end{scope}
\end{tikzpicture}
%%%------------------------------------------------------------------------------------------------------------
Book/Chapter5/Figures/fig-absolute-loss.tex 0 → 100644
%%%------------------------------------------------------------------------------------------------------------
\begin{tikzpicture}
\begin{scope}[yscale=0.2,xscale=0.8]
\draw[-,very thick,ublue,domain=-4.2:3.5,samples=100] plot (\x,{ - 1/14 * (\x + 4) * (\x + 1) * (\x - 1) * (\x - 3)});
{
\draw[-,very thick,ugreen,domain=-3.8:3.0,samples=100] plot (\x,{ - 1/14 * (4*\x*\x*\x + 3*\x*\x - 26*\x - 1)});
}
\draw[->,thick] (-6,0) -- (5,0);
\draw[->,thick] (-5,-4) -- (-5,5);
\draw [<-] (-2.5,4) -- (-2,5) node [pos=1,right,inner sep=2pt] {\footnotesize{答案$\tilde{\textbf{y}}_i$}};
{
\draw [<-] (-3,-3) -- (-2.5,-2) node [pos=0,left,inner sep=2pt] {\footnotesize{预测$\textbf{y}_i$}};}
{
\draw [<-] (2.3,1) -- (3.3,2) node [pos=1,right,inner sep=2pt] {\footnotesize{偏差$|\tilde{\textbf{y}}_i - \textbf{y}_i|$}};
\foreach \x in {-3.8,-3.7,...,3.0}{
\pgfmathsetmacro{\p}{- 1/14 * (\x + 4) * (\x + 1) * (\x - 1) * (\x - 3)};
\pgfmathsetmacro{\q}{- 1/14 * (4*\x*\x*\x + 3*\x*\x - 26*\x - 1)};
\draw [-] (\x,\p) -- (\x, \q);
}
}
\end{scope}
\end{tikzpicture}
%%%------------------------------------------------------------------------------------------------------------
Book/Chapter5/Figures/fig-activation.tex 0 → 100644
%%%------------------------------------------------------------------------------------------------------------
\begin{figure}
\subfigure[softplus]{
\centering
\begin{minipage}{.2\textwidth}
\begin{tikzpicture}
\draw[->](-1.2,0)--(1.2,0)node[left,below,font=\tiny]{$x$};
\draw[->](0,-1.2)--(0,1.2)node[right,font=\tiny]{$y$};
\foreach \x in {-1.0,-0.5,0.0,0.5,1.0}{\draw(\x,0)--(\x,0.05)node[below,outer sep=2pt,font=\tiny]at(\x,0){\x};}
\foreach \y in {1.0,0.5}{\draw(0,\y)--(0.05,\y)node[left,outer sep=2pt,font=\tiny]at(0,\y){\y};}
\draw[color=red ,domain=-1.2:1]plot(\x,{ln(1+(exp(\x))});
\node[black,anchor=south] at (0,1.2) {\small $y = ln(1+e^x)$};
\end{tikzpicture}
\end{minipage}%
}
\hfill
\subfigure[sigmoid]{
\centering
\begin{minipage}{.2\textwidth}
\begin{tikzpicture}
\draw[->](-1.2,0)--(1.2,0)node[left,below,font=\tiny]{$x$};
\draw[->](0,-1.2)--(0,1.2)node[right,font=\tiny]{$y$};
\draw[dashed](-1.2,1)--(1.2,1);
\foreach \x in {-1,-0.5,0,0.5,1}{\draw(\x,0)--(\x,0.05)node[below,outer sep=2pt,font=\tiny]at(\x,0){
\pgfmathparse{(\x)*5}
\pgfmathresult};}
\foreach \y in {0.5,1.0}{\draw(0,\y)--(0.05,\y)node[left,outer sep=2pt,font=\tiny]at(0,\y){\y};}
\draw[color=red,domain=-1.2:1.2]plot(\x,{1/(1+(exp(-5*\x)))});
\node[black,anchor=south] at (0,1.2) {\small $y = \frac{1}{1+e^{-x}}$};
\end{tikzpicture}
\end{minipage}%
}
\hfill
\subfigure[tanh]{
\centering
\begin{minipage}{.2\textwidth}
\begin{tikzpicture}
\draw[->](-1.2,0)--(1.2,0)node[left,below,font=\tiny]{$x$};
\draw[->](0,-1.2)--(0,1.2)node[right,font=\tiny]{$y$};
\draw[dashed](-1.2,1)--(1.2,1);
\draw[dashed](-1.2,-1)--(1.2,-1);
\foreach \x in {-1.0,-0.5,0.0,0.5,1.0}{\draw(\x,0)--(\x,0.05)node[below,outer sep=2pt,font=\tiny]at(\x,0){\x};}
\foreach \y in {0.5,1.0}{\draw(0,\y)--(0.05,\y)node[left,outer sep=2pt,font=\tiny]at(0,\y){\y};}
\draw[color=red ,domain=-1.2:1.2]plot(\x,{tanh(\x)});
\node[black,anchor=south] at (0,1.2) {\small $y = \frac{e^{x}-e^{-x}}{e^{x}+e^{-x}}$};
\end{tikzpicture}
\end{minipage}
}\\ \vspace{-1em}
\subfigure[relu]{
\centering
\begin{minipage}{.2\textwidth}
\begin{tikzpicture}
\draw[->](-1.2,0)--(1.2,0)node[left,below,font=\tiny]{$x$};
\draw[->](0,-1.2)--(0,1.2)node[right,font=\tiny]{$y$};
\draw[dashed](-1.2,1)--(1.2,1);
\draw[dashed](-1.2,-1)--(1.2,-1);
\foreach \x in {-1.0,-0.5,0.0,0.5,1.0}{\draw(\x,0)--(\x,0.05)node[below,outer sep=2pt,font=\tiny]at(\x,0){\x};}
\foreach \y in {0.5,1.0}{\draw(0,\y)--(0.05,\y)node[left,outer sep=2pt,font=\tiny]at(0,\y){\y};}
\draw[color=red ,domain=-1.2:1.2]plot(\x,{max(\x,0)});
\node[black,anchor=south] at (0,1.2) {\small $y =\max (0, x)$};
\end{tikzpicture}
\end{minipage}%
}
\hfill
\subfigure[gaussian]{
\centering
\begin{minipage}{.2\textwidth}
\begin{tikzpicture}
\draw[->](-1.2,0)--(1.2,0)node[left,below,font=\tiny]{$x$};
\draw[->](0,-1.2)--(0,1.2)node[right,font=\tiny]{$y$};
\draw[dashed](-1.2,1)--(1.2,1);
\foreach \x in {-1.0,-0.5,0.0,0.5,1.0}{\draw(\x,0)--(\x,0.05)node[below,outer sep=2pt,font=\tiny]at(\x,0){\x};}
\foreach \y in {0.5,1.0}{\draw(0,\y)--(0.05,\y)node[left,outer sep=2pt,font=\tiny]at(0,\y){\y};}
\draw[color=red ,domain=-1.2:1.2]plot(\x,{exp(-1*((\x)^2))});
\node[black,anchor=south] at (0,1.2) {\small $y =e^{-x^2}$};
\end{tikzpicture}
\end{minipage}%
}
\hfill
\subfigure[identity]{
\centering
\begin{minipage}{.2\textwidth}
\begin{tikzpicture}
\draw[->](-1.2,0)--(1.2,0)node[left,below,font=\tiny]{$x$};
\draw[->](0,-1.2)--(0,1.2)node[right,font=\tiny]{$y$};
\foreach \x in {-1.0,-0.5,0.0,0.5,1.0}{\draw(\x,0)--(\x,0.05)node[below,outer sep=2pt,font=\tiny]at(\x,0){\x};}
\foreach \y in {0.5,1.0}{\draw(0,\y)--(0.05,\y)node[left,outer sep=2pt,font=\tiny]at(0,\y){\y};}
\draw[color=red ,domain=-1:1]plot(\x,\x);
\node[black,anchor=south] at (0,1.2) {\small $y =x$};
\end{tikzpicture}
\end{minipage}
}
\end{figure}
%%%------------------------------------------------------------------------------------------------------------
Book/Chapter5/Figures/fig-artificial-neuron.tex 0 → 100644
%%%------------------------------------------------------------------------------------------------------------
\begin{tikzpicture}
\begin{scope}
\node [anchor=center,circle,draw,ublue,very thick,minimum size=3.5em,fill=white,drop shadow={shadow xshift=0.1em,shadow yshift=-0.1em}] (neuron) at (0,0) {};
\node [anchor=east] (x1) at ([xshift=-6em]neuron.west) {\Large{$x_1$}};
\node [anchor=center] (x0) at ([yshift=3em]x1.center) {\Large{$x_0$}};
\node [anchor=center] (x2) at ([yshift=-3em]x1.center) {\Large{$b$}};
\node [anchor=west] (y) at ([xshift=6em]neuron.east) {\Large{$y$}};
\node [anchor=center] (neuronmath) at (neuron.center) {\Large{$f$}};
\draw [->,thick] (x0.east) -- (neuron.150) node [pos=0.5,above] {$w_0$};
\draw [->,thick] (x1.east) -- (neuron.180) node [pos=0.5,above] {$w_1$};
\draw [->,thick] (x2.east) -- (neuron.210) node [pos=0.5,above] {$$};
\draw [->,thick] (neuron.east) -- (y.west);
\end{scope}
\end{tikzpicture}
%
%%%------------------------------------------------------------------------------------------------------------
Book/Chapter5/Figures/fig-back-propagation-hid.tex 0 → 100644
%%%------------------------------------------------------------------------------------------------------------
\begin{tikzpicture}
\begin{scope}
\node [anchor=center,draw,fill=red!20,minimum height=1.8em,minimum width=2.5em] (h) at (0,0) {$\textbf{h}^{k-1}$};
\node [anchor=west,draw,fill=blue!20,minimum height=1.8em,minimum width=2.5em] (s) at ([xshift=6em]h.east) {$\textbf{s}^{k}$};
\node [anchor=west,draw,fill=green!20,minimum height=1.8em,minimum width=2.5em] (h2) at ([xshift=6em]s.east) {$\textbf{h}^{k}$};
\node [anchor=east] (prev) at ([xshift=-2em]h.west) {...};
\node [anchor=west] (next) at ([xshift=2em]h2.east) {...};
\draw [->,thick] ([xshift=0.1em]prev.east) -- ([xshift=-0.1em]h.west);
\draw [->,thick] ([xshift=0.1em]h.east) -- ([xshift=-0.1em]s.west) node [pos=0.5,below] {\tiny{$\textbf{s}^k = \textbf{h}^{k-1}\textbf{w}^k$}};
\draw [->,thick] ([xshift=0.1em]s.east) -- ([xshift=-0.1em]h2.west) node [pos=0.5,below] {\tiny{$\textbf{h}^k = f(\textbf{s}^{k})$}};
\draw [->,thick] ([xshift=0.1em]h2.east) -- ([xshift=-0.1em]next.west);
{
\draw [<-,thick,red] ([xshift=0.1em,yshift=0.4em]h2.east) -- ([xshift=-0.1em,yshift=0.4em]next.west) node [pos=0.8,above] {\tiny{反向传播}};
}
{
\draw [<-,thick,red] ([xshift=0.1em,yshift=0.4em]s.east) -- ([xshift=-0.1em,yshift=0.4em]h2.west) node [pos=0.5,above] {\tiny{反向传播}};
}
{
\draw [<-,thick,red] ([xshift=0.1em,yshift=0.4em]h.east) -- ([xshift=-0.1em,yshift=0.4em]s.west) node [pos=0.5,above] {\tiny{反向传播}};
}
{
\draw [->,thick,red,dashed] ([yshift=-0.1em]h.south) -- ([yshift=-1em]h.south) -- ([yshift=-1em]h2.south) -- ([yshift=-0.1em]h2.south);
\node [anchor=north,red] (recur) at ([yshift=-1em]s.south) {\scriptsize{$k=k-1$重复上述过程}};
}
{
\node [anchor=south] (h2label) at (h2.north) {$\frac{\partial L}{\partial \textbf{h}^{k}}$};
}
{
\node [anchor=south] (slabel) at (s.north) {$\pi^k = \frac{\partial L}{\partial \textbf{s}^{k}}$};
}
{
\node [anchor=south] (hlabel) at (h.north) {$\frac{\partial L}{\partial \textbf{h}^{k-1}}$, $\frac{\partial L}{\partial \textbf{w}^{k}}$};
}
\end{scope}
\end{tikzpicture}
%%%------------------------------------------------------------------------------------------------------------
Book/Chapter5/Figures/fig-back-propagation-output1.tex 0 → 100644
%%%------------------------------------------------------------------------------------------------------------
\begin{tikzpicture}
\begin{scope}
\node [anchor=west,minimum height=1.7em,fill=blue!20,draw] (s) at (0,0) {$\textbf{s}^{K}$};
\node [anchor=west,minimum height=1.7em,fill=green!20,draw] (h2) at ([xshift=5.5em]s.east) {$\textbf{h}^{K}$};
\node [anchor=west,minimum height=1.7em,fill=orange!20,draw] (l) at ([xshift=5.5em]h2.east) {$L$};
\draw [->] (s.east) -- (h2.west);
\draw [->] (h2.east) -- (l.west);
\draw [->,very thick,red] ([yshift=1em,xshift=-0.1em]l.north) -- ([yshift=1em,xshift=0.1em]h2.north) node [pos=0.5,above] {\tiny{求梯度{$\frac{\partial L}{\partial \textbf{h}^K} = ?$}}};
\draw [->,very thick,red] ([yshift=1em,xshift=-0.1em]h2.north) -- ([yshift=1em,xshift=0.1em]s.north) node [pos=0.5,above] {\tiny{求梯度{$\frac{\partial f^K(\textbf{s}^K)}{\partial \textbf{s}^K} = ?$}}};
\draw [-,very thick,red] ([yshift=0.5em]l.north) -- ([yshift=1.5em]l.north);
\draw [-,very thick,red] ([yshift=0.5em]h2.north) -- ([yshift=1.5em]h2.north);
\draw [-,very thick,red] ([yshift=0.5em]s.north) -- ([yshift=1.5em]s.north);
\end{scope}
\end{tikzpicture}
%%%------------------------------------------------------------------------------------------------------------
Book/Chapter5/Figures/fig-back-propagation-output2.tex 0 → 100644
%%%------------------------------------------------------------------------------------------------------------
\begin{tikzpicture}
\begin{scope}
\node [anchor=center,minimum height=1.7em,fill=yellow!20,draw] (h) at (0,0) {$\textbf{h}^{K-1}$};
\node [anchor=west,minimum height=1.7em,fill=blue!20,draw] (s) at ([xshift=5.5em]h.east) {$\textbf{s}^{K}$};
\draw [->] (h.east) -- (s.west);
\node [anchor=south west,inner sep=2pt] (step100) at ([xshift=0.5em,yshift=-0.8em]h.north east) {\tiny{$\textbf{s}^K = \textbf{h}^{K-1} \textbf{w}^K$}};
\node [anchor=south west] (slabel) at ([yshift=1em,xshift=0.3em]s.north) {\scriptsize{\red{\textbf{{已经得到：$\pi^K = \frac{\partial L}{\partial \textbf{s}^K}$}}}}};
\draw [->,red] ([yshift=0.3em]slabel.south) .. controls +(south:0.5) and +(north:0.5) .. ([xshift=0.5em]s.north);
{
\draw [->,very thick,red] ([yshift=1em,xshift=-0.1em]s.north) -- ([yshift=1em,xshift=0.1em]h.north) node [pos=0.5,above] {\tiny{{$\frac{\partial L}{\partial \textbf{w}^K} = ?$, $\frac{\partial L}{\partial \textbf{h}^{K-1}} = ?$}}};
\draw [-,very thick,red] ([yshift=0.5em]h.north) -- ([yshift=1.5em]h.north);
\draw [-,very thick,red] ([yshift=0.5em]s.north) -- ([yshift=1.5em]s.north);
}
\end{scope}
\end{tikzpicture}
%%%------------------------------------------------------------------------------------------------------------
Book/Chapter5/Figures/fig-back-propagation.tex 0 → 100644
%%%------------------------------------------------------------------------------------------------------------
\begin{tikzpicture}
\begin{scope}
\tikzstyle{layernode} = [draw,thick,fill=ugreen!30!white,blur shadow={shadow xshift=1pt,shadow yshift=-1pt}];
\node [anchor=center,layernode,minimum height=4em,minimum width=1em] (layer01) at (0,0) {};
\node [anchor=north west,layernode,minimum height=3em,minimum width=1em] (layer02) at ([xshift=3em]layer01.north east) {};
\node [anchor=south west,layernode,minimum height=3em,minimum width=1em] (layer03) at ([xshift=7em]layer01.south east) {};
\node [anchor=south west,layernode,minimum height=4em,minimum width=1em] (layer04) at ([xshift=11em]layer01.south east) {};
\node [anchor=south west,layernode,minimum height=4em,minimum width=1em] (layer05) at ([xshift=3em]layer04.south east) {};
\node [anchor=east] (input) at ([xshift=-1em]layer01.west){\scriptsize{输入}};
\node [anchor=west] (output) at ([xshift=1em]layer05.east){\scriptsize{输出}};
{
\draw [<-,very thick,red] ([xshift=-1em,yshift=-0.3em]layer01.west) -- ([xshift=-0.1em,yshift=-0.3em]layer01.west)node [pos=0.5,above] {\tiny{⑦}};
\draw [<-,very thick,red] ([xshift=0.1em,yshift=-0.8em]layer01.north east) -- ([xshift=-0.1em,yshift=-0.8em]layer02.north west)node [pos=0.5,above] {\tiny{⑥}};
\draw [<-,very thick,red] ([xshift=0.1em,yshift=0.2em]layer01.south east) -- ([xshift=-0.1em,yshift=0.2em]layer03.south west)node [pos=0.5,below] {\tiny{⑤}};
\draw [<-,very thick,red] ([xshift=0.1em,yshift=-0.8em]layer02.north east) -- ([xshift=-0.1em,yshift=-0.8em]layer04.north west)node [pos=0.5,above] {\tiny{④}};
\draw [<-,very thick,red] ([xshift=0.1em,yshift=0.2em]layer03.south east) -- ([xshift=-0.1em,yshift=0.2em]layer04.south west)node [pos=0.5,below] {\tiny{③}};
\draw [<-,very thick,red] ([xshift=0.1em,yshift=-0.3em]layer04.east) -- ([xshift=-0.1em,yshift=-0.3em]layer05.west)node [pos=0.5,above] {\tiny{②}};
\draw [<-,very thick,red] ([xshift=0.1em,yshift=-0.3em]layer05.east) -- ([xshift=1.0em,yshift=-0.3em]layer05.east)node [pos=0.5,above] {\tiny{①}};
}
{
\draw [<-,thin] ([xshift=0.3em,yshift=-0.7em]layer04.east) .. controls +(-35:1) and +(145:1) .. ([xshift=-2em,yshift=-0.3em]layer05.south west) node [pos=1,below] {\scriptsize{反向：$h_{i}$ 处的梯度$\frac{\partial L}{\partial h_i}$}};
}
\end{scope}
\end{tikzpicture}
%%%------------------------------------------------------------------------------------------------------------
Book/Chapter5/Figures/fig-bert.tex 0 → 100644
%%%------------------------------------------------------------------------------------------------------------
\begin{tikzpicture}
\begin{scope}[scale=1.2]
\node [anchor=west,draw,inner sep=4pt,fill=blue!20!white,minimum width=3em] (Trm0) at (0,0) {\scriptsize{TRM}};
\node [anchor=west,draw,inner sep=4pt,fill=blue!20!white,minimum width=3em] (Trm1) at ([xshift=1em]Trm0.east) {\scriptsize{TRM}};
\node [anchor=west,draw,inner sep=4pt,fill=blue!20!white,minimum width=3em] (Trm2) at ([xshift=1em]Trm1.east) {\scriptsize{TRM}};
\node [anchor=west,draw,inner sep=4pt,fill=blue!20!white,minimum width=3em] (Trm3) at ([xshift=1em]Trm2.east) {\scriptsize{TRM}};
\node [anchor=west,inner sep=4pt] (sep) at ([xshift=1em]Trm3.east) {\scriptsize{...}};
\node [anchor=west,draw,inner sep=4pt,fill=blue!20!white,minimum width=3em] (Trm4) at ([xshift=1em]sep.east) {\scriptsize{TRM}};
\node [anchor=south,draw,inner sep=4pt,fill=blue!20!white,minimum width=3em] (Trm5) at ([yshift=1em]Trm0.north) {\scriptsize{TRM}};
\node [anchor=west,draw,inner sep=4pt,fill=blue!20!white,minimum width=3em] (Trm6) at ([xshift=1em]Trm5.east) {\scriptsize{TRM}};
\node [anchor=west,draw,inner sep=4pt,fill=blue!20!white,minimum width=3em] (Trm7) at ([xshift=1em]Trm6.east) {\scriptsize{TRM}};
\node [anchor=west,draw,inner sep=4pt,fill=blue!20!white,minimum width=3em] (Trm8) at ([xshift=1em]Trm7.east) {\scriptsize{TRM}};
\node [anchor=west,inner sep=4pt] (sep1) at ([xshift=1em]Trm8.east) {\scriptsize{...}};
\node [anchor=west,draw,inner sep=4pt,fill=blue!20!white,minimum width=3em] (Trm9) at ([xshift=1em]sep1.east) {\scriptsize{TRM}};
\node [rectangle,rounded corners,draw=black!50,densely dashed,inner sep=0.4em] [fit = (Trm0) (Trm4) (Trm5) (Trm9)] (inputshadow) {};
\node [anchor=north,draw,inner sep=4pt,fill=ugreen!20!white,minimum width=2em] (e1) at ([yshift=-1em]Trm0.south) {\scriptsize{$\textbf{e}_1$}};
\node [anchor=north,draw,inner sep=4pt,fill=ugreen!20!white,minimum width=2em] (e2) at ([yshift=-1em]Trm1.south) {\scriptsize{$\textbf{e}_2$}};
\node [anchor=north,draw,inner sep=4pt,fill=ugreen!20!white,minimum width=2em] (e3) at ([yshift=-1em]Trm2.south) {\scriptsize{$\textbf{e}_3$}};
\node [anchor=north,draw,inner sep=4pt,fill=ugreen!20!white,minimum width=2em] (e4) at ([yshift=-1em]Trm3.south) {\scriptsize{$\textbf{e}_4$}};
\node [anchor=north,inner sep=4pt] (sep5) at ([yshift=-1em]sep.south) {\scriptsize{...}};
\node [anchor=north,draw,inner sep=4pt,fill=ugreen!20!white,minimum width=2em] (e5) at ([yshift=-1em]Trm4.south) {\scriptsize{$\textbf{e}_m$}};
\node [anchor=south,draw,inner sep=4pt,fill=yellow!30,minimum width=2em] (t1) at ([yshift=1em]Trm5.north) {\scriptsize{$\textbf{h}_1$}};
\node [anchor=south,draw,inner sep=4pt,fill=yellow!30,minimum width=2em] (t2) at ([yshift=1em]Trm6.north) {\scriptsize{$\textbf{h}_2$}};
\node [anchor=south,draw,inner sep=4pt,fill=yellow!30,minimum width=2em] (t3) at ([yshift=1em]Trm7.north) {\scriptsize{$\textbf{h}_3$}};
\node [anchor=south,draw,inner sep=4pt,fill=yellow!30,minimum width=2em] (t4) at ([yshift=1em]Trm8.north) {\scriptsize{$\textbf{h}_4$}};
\node [anchor=south,inner sep=4pt] (sep6) at ([yshift=1em]sep1.north) {\scriptsize{...}};
\node [anchor=south,draw,inner sep=4pt,fill=yellow!30,minimum width=2em] (t5) at ([yshift=1em]Trm9.north) {\scriptsize{$\textbf{h}_m$}};
\node [anchor=west,draw,inner sep=3pt,fill=blue!20!white,minimum width=1em] (Lt1) at ([yshift=1.5em]t1.west) {\tiny{TRM}};
\node [anchor=west] (Lt2) at ([xshift=-0.1em]Lt1.east) {\tiny{: Transformer}};
\draw [->] ([yshift=0.1em]e1.north) -- ([yshift=-0.1em]Trm0.south);
\draw [->] ([yshift=0.1em]e1.north) -- ([yshift=-0.1em]Trm1.south);
\draw [->] ([yshift=0.1em]e1.north) -- ([yshift=-0.1em]Trm2.south);
\draw [->] ([yshift=0.1em]e1.north) -- ([yshift=-0.1em]Trm3.south);
\draw [->] ([yshift=0.1em]e1.north) -- ([yshift=-0.1em]Trm4.south);
\draw [->] ([yshift=0.1em]e2.north) -- ([yshift=-0.1em]Trm1.south);
\draw [->] ([yshift=0.1em]e3.north) -- ([yshift=-0.1em]Trm0.south);
\draw [->] ([yshift=0.1em]e3.north) -- ([yshift=-0.1em]Trm1.south);
\draw [->] ([yshift=0.1em]e3.north) -- ([yshift=-0.1em]Trm2.south);
\draw [->] ([yshift=0.1em]e3.north) -- ([yshift=-0.1em]Trm3.south);
\draw [->] ([yshift=0.1em]e3.north) -- ([yshift=-0.1em]Trm4.south);
\draw [->] ([yshift=0.1em]e4.north) -- ([yshift=-0.1em]Trm3.south);
\draw [->] ([yshift=0.1em]e5.north) -- ([yshift=-0.1em]Trm4.south);
\draw [->] ([yshift=0.1em]Trm0.north) -- ([yshift=-0.1em]Trm5.south);
\draw [->] ([yshift=0.1em]Trm0.north) -- ([yshift=-0.1em]Trm6.south);
\draw [->] ([yshift=0.1em]Trm0.north) -- ([yshift=-0.1em]Trm7.south);
\draw [->] ([yshift=0.1em]Trm0.north) -- ([yshift=-0.1em]Trm8.south);
\draw [->] ([yshift=0.1em]Trm0.north) -- ([yshift=-0.1em]Trm9.south);
\draw [->] ([yshift=0.1em]Trm1.north) -- ([yshift=-0.1em]Trm6.south);
\draw [->] ([yshift=0.1em]Trm2.north) -- ([yshift=-0.1em]Trm5.south);
\draw [->] ([yshift=0.1em]Trm2.north) -- ([yshift=-0.1em]Trm6.south);
\draw [->] ([yshift=0.1em]Trm2.north) -- ([yshift=-0.1em]Trm7.south);
\draw [->] ([yshift=0.1em]Trm2.north) -- ([yshift=-0.1em]Trm8.south);
\draw [->] ([yshift=0.1em]Trm2.north) -- ([yshift=-0.1em]Trm9.south);
\draw [->] ([yshift=0.1em]Trm3.north) -- ([yshift=-0.1em]Trm8.south);
\draw [->] ([yshift=0.1em]Trm4.north) -- ([yshift=-0.1em]Trm9.south);
\draw [->] ([yshift=0.1em]Trm5.north) -- ([yshift=-0.1em]t1.south);
\draw [->] ([yshift=0.1em]Trm6.north) -- ([yshift=-0.1em]t2.south);
\draw [->] ([yshift=0.1em]Trm7.north) -- ([yshift=-0.1em]t3.south);
\draw [->] ([yshift=0.1em]Trm8.north) -- ([yshift=-0.1em]t4.south);
\draw [->] ([yshift=0.1em]Trm9.north) -- ([yshift=-0.1em]t5.south);
\end{scope}
\end{tikzpicture}
%%%------------------------------------------------------------------------------------------------------------
Book/Chapter5/Figures/fig-bias.tex 0 → 100644
%%%------------------------------------------------------------------------------------------------------------
\begin{tikzpicture}
%% a two-layer neural network
\begin{scope}
{
\draw [->,thick] (-1.8,0) -- (1.8,0);
\draw [->,thick] (0,0) -- (0,2);
\draw [-] (-0.05,1) -- (0.05,1);
\node [anchor=east,inner sep=1pt] (label1) at (0,1) {\tiny{1}};
\node [anchor=south east,inner sep=1pt] (label2) at (0,0) {\tiny{0}};
\node [anchor=south east,inner sep=1pt] (labela) at (0.2,-0.5) {\footnotesize{(a)}};
}
{\node [anchor=north west,align=left] (wblabel) at (-1.8,2) {{\scriptsize{$w_1=100$}}\\[-0ex] \scriptsize{\ $b_1=0$}};}
{\draw [-,very thick,ublue,rounded corners=0.1em] (-1.5,0) -- (0,0) -- (0,1) -- (1.5,1);}
\end{scope}
%---------------------------------------------------------------------------------------------
\begin{scope}[xshift=1.6in]
{
\draw [->,thick] (-1.8,0) -- (1.8,0);
\draw [->,thick] (0,0) -- (0,2);
\draw [-] (-0.05,1) -- (0.05,1);
\node [anchor=east,inner sep=1pt] (label1) at (0,1) {\tiny{1}};
\node [anchor=south east,inner sep=1pt] (label2) at (0,0) {\tiny{0}};
\node [anchor=south east,inner sep=1pt] (labelb) at (0.2,-0.5) {\footnotesize{(b)}};
}
{\node [anchor=north west,align=left] (wblabel) at (-1.8,2) {\scriptsize{$w_1=100$}\\[-0ex] {\scriptsize{\ $b_1=-2$}}};}
{\draw [-,very thick,ublue,rounded corners=0.1em] (-1.5,0) -- (0.25,0) -- (0.25,1) -- (1.5,1);}
\end{scope}
%-----------------------------------------------------------------------------------------------
\begin{scope}[xshift=3.2in]
{
\draw [->,thick] (-1.8,0) -- (1.8,0);
\draw [->,thick] (0,0) -- (0,2);
\draw [-] (-0.05,1) -- (0.05,1);
\node [anchor=east,inner sep=1pt] (label1) at (0,1) {\tiny{1}};
\node [anchor=south east,inner sep=1pt] (label2) at (0,0) {\tiny{0}};
\node [anchor=south east,inner sep=1pt] (labelc) at (0.2,-0.5) {\footnotesize{(c)}};
}
{\node [anchor=north west,align=left] (wblabel) at (-1.8,2) {\scriptsize{$w_1=100$}\\[-0ex] {\scriptsize{\ $b_1=-4$}}};}
{\draw [-,very thick,ublue,rounded corners=0.1em] (-1.5,0) -- (0.5,0) -- (0.5,1) -- (1.5,1);}
\end{scope}
\end{tikzpicture}
%%%------------------------------------------------------------------------------------------------------------
Book/Chapter5/Figures/fig-broadcast.tex 0 → 100644
%%%------------------------------------------------------------------------------------------------------------
\begin{tikzpicture}
\begin{scope}[xshift=0.6in]
\setcounter{mycount1}{1}
\draw[step=0.5cm,color=orange,thick] (-1,-0.5) grid (1,0.5);
\foreach \y in {+0.25,-0.25}
\foreach \x in {-0.75,-0.25,0.25,0.75}{
\node [fill=orange!20,inner sep=0pt,minimum height=0.49cm,minimum width=0.49cm] at (\x,\y) {$\number\value{mycount1}$};
\addtocounter{mycount1}{1};
}
\node [anchor=south] (varlabel) at (0,0.6) {$\textbf{s}$};
\node [anchor=north] (labelc) at (0,-0.7) {\footnotesize{(a)}};
\end{scope}
\begin{scope}[xshift=2.1in]
\setcounter{mycount1}{1}
\draw[step=0.5cm,color=ugreen,thick] (-1,-0.5) grid (1,0);
\foreach \y in {-0.25}
\foreach \x in {-0.75,-0.25,0.25,0.75}{
\node [fill=green!20,inner sep=0pt,minimum height=0.48cm,minimum width=0.48cm] at (\x,\y) {$1$};
\addtocounter{mycount1}{1};
}
\node [anchor=south] (varlabel) at (0,0.1) {$\textbf{b}$};
\node [anchor=north] (labelc) at (0,-0.7) {\footnotesize{(b)}};
\end{scope}
\begin{scope}[yshift=-1in]
\setcounter{mycount1}{1}
\draw[step=0.5cm,color=orange,thick] (-1,-0.5) grid (1,0.5);
\foreach \y in {+0.25,-0.25}
\foreach \x in {-0.75,-0.25,0.25,0.75}{
\node [fill=orange!20,inner sep=0pt,minimum height=0.49cm,minimum width=0.49cm] at (\x,\y) {$\number\value{mycount1}$};
\addtocounter{mycount1}{1};
}
\node [anchor=south] (varlabel) at (0,0.6) {$\textbf{s}$};
\end{scope}
\begin{scope}[yshift=-1in,xshift=1.5in]
\setcounter{mycount1}{1}
\draw[step=0.5cm,color=ugreen,thick] (-1,-0.5) grid (1,0.5);
\foreach \y in {+0.25}
\foreach \x in {-0.75,-0.25,0.25,0.75}{
\node [fill=green!20,inner sep=0pt,minimum height=0.49cm,minimum width=0.49cm] at (\x,\y) {$1$};
\addtocounter{mycount1}{1};
}
\foreach \y in {-0.25}
\foreach \x in {-0.75,-0.25,0.25,0.75}{
\node [fill=purple!20,inner sep=0pt,minimum height=0.49cm,minimum width=0.49cm] at (\x,\y) {$1$};
\addtocounter{mycount1}{1};
}
\node [anchor=center] (plabel) at (-4.5em,0) {\huge{$\textbf{+}$}};
\node [anchor=south] (varlabel) at (0,0.6) {$\textbf{b}$};
\node [anchor=north] (labelc) at (0,-0.7) {\footnotesize{(c)}};
\end{scope}
\begin{scope}[yshift=-1in,xshift=3in]
\setcounter{mycount1}{2}
\draw[step=0.5cm,color=orange,thick] (-1,-0.5) grid (1,0.5);
\foreach \y in {+0.25,-0.25}
\foreach \x in {-0.75,-0.25,0.25,0.75}{
\node [fill=orange!20,inner sep=0pt,minimum height=0.49cm,minimum width=0.49cm] at (\x,\y) {$\number\value{mycount1}$};
\addtocounter{mycount1}{1};
}
\node [anchor=center] (plabel) at (-4.5em,0) {\huge{$\textbf{=}$}};
\node [anchor=south] (varlabel) at (0,0.6) {$\textbf{s+b}$};
\end{scope}
\end{tikzpicture}
%%%------------------------------------------------------------------------------------------------------------
Book/Chapter5/Figures/fig-code-FNNLM.tex 0 → 100644
%%%------------------------------------------------------------------------------------------------------------
\begin{tcolorbox}
[bicolor,sidebyside,width=12cm,righthand width=4cm,size=title,frame engine=empty,
colback=blue!10!white,colbacklower=black!5!white]
{\scriptsize
\begin{tabbing}
\texttt{XTensor w[3], e[3], h0, y;} \\
\texttt{XTensor C, H, d, U;} \\
\texttt{...}\\
\texttt{} \\
\texttt{for(unsigned i = 0; i < 3; i++)} \\
\texttt{\ \ \ \ e[i] = MMul(w[i], C);}\\
\texttt{e01 = Concatenate(e[0], e[1], -1);}\\
\texttt{e = Concatenate(e01, e[2], -1);}\\
\texttt{} \\
\texttt{h0 = TanH(MMul(e, H) + d);}\\
\texttt{y = Softmax(MMul(h0, U));}\\
\texttt{} \\
\texttt{for(unsigned k = 0; k < size; k++)\{} \\
\texttt{} \ \ \ \ ... // {\texttt{y}}的第$k$元素表示 $\textrm{P}(w|...)$\\
\texttt{} \ \ \ \ ... // $w$为词汇表里第$k$个词\\
\texttt{\}}
\end{tabbing}
}
\tcblower
\begin{center}
\begin{tikzpicture}
\begin{scope}
\node [anchor=west] (w0) at (0,0) {\scriptsize{$w_{i-3}$}};
\node [anchor=west] (w1) at ([xshift=0.5em]w0.east) {\scriptsize{$w_{i-2}$}};
\node [anchor=west] (w2) at ([xshift=0.5em]w1.east) {\scriptsize{$w_{i-1}$}};
\node [anchor=north] (index0) at ([yshift=0.5em]w0.south) {\tiny(index)};
\node [anchor=north] (index1) at ([yshift=0.5em]w1.south) {\tiny(index)};
\node [anchor=north] (index2) at ([yshift=0.5em]w2.south) {\tiny(index)};
\node [anchor=south,draw,inner sep=3pt,align=left] (e0) at ([yshift=1.0em]w0.north) {\tiny{$e_0:$}\\\tiny{$w_{i-3} \textbf{C}$}};
\node [anchor=south,draw,inner sep=3pt,align=left] (e1) at ([yshift=1.0em]w1.north) {\tiny{$e_1:$}\\\tiny{$w_{i-2} \textbf{C}$}};
\node [anchor=south,draw,inner sep=3pt,align=left] (e2) at ([yshift=1.0em]w2.north) {\tiny{$e_2:$}\\\tiny{$w_{i-1} \textbf{C}$}};
\node [anchor=south,draw,minimum width=9em,inner sep=3pt] (h0) at ([yshift=1.5em]e1.north) {\tiny{$h_0=\textrm{Tanh}([e_0,e_1,e_2] \textbf{H} + \textbf{d})$}};
\node [anchor=south,draw,minimum width=9em,inner sep=3pt] (h1) at ([yshift=1.5em]h0.north) {\tiny{$y=\textrm{Softmax}(h_0 \textbf{U})$}};
\node [anchor=south] (ylabel) at ([yshift=1em]h1.north) {\scriptsize{$\textrm{P}(w_i|w_{i-3}w_{i-2}w_{i-1})$}};
\draw [->] ([yshift=0.1em]w0.north) -- ([yshift=-0.1em]e0.south);
\draw [->] ([yshift=0.1em]w1.north) -- ([yshift=-0.1em]e1.south);
\draw [->] ([yshift=0.1em]w2.north) -- ([yshift=-0.1em]e2.south);
\draw [->] ([yshift=0.1em]e0.north) -- ([xshift=-2em,yshift=-0.1em]h0.south);
\draw [->] ([yshift=0.1em]e1.north) -- ([yshift=-0.1em]h0.south);
\draw [->] ([yshift=0.1em]e2.north) -- ([xshift=2em,yshift=-0.1em]h0.south);
\draw [->] ([yshift=0.1em]h0.north) -- ([yshift=-0.1em]h1.south);
\draw [->] ([yshift=0.1em]h1.north) -- ([yshift=-0.1em]ylabel.south);
\end{scope}
\end{tikzpicture}
\end{center}
\end{tcolorbox}
%%%------------------------------------------------------------------------------------------------------------
Book/Chapter5/Figures/fig-code-back-propagation-1.tex 0 → 100644
%%%------------------------------------------------------------------------------------------------------------
\begin{tcolorbox}
[bicolor,sidebyside,width=12cm,righthand width=4cm,size=title,frame engine=empty,
colback=blue!10!white,colbacklower=black!5!white]
{\scriptsize
\begin{tabbing}
\texttt{XTensor x, y, gold, h[5], w[5], s[5];} \\
\texttt{XTensor dh[5], dw[5], ds[5];} \\
\texttt{...} // 前向过程 \\
\texttt{h[0] = x;} \\
\texttt{y = h[4];} \\
\texttt{} \\
\texttt{CrossEntropyBackward(dh[4], y, gold);} \\
\texttt{SoftmaxBackward(y, s[4], dh[4], ds[4]);}\\
\texttt{MMul(h[3], {\tiny X\_TRANS}, ds[4], {\tiny X\_NOTRANS}, dw[4]);}\\
\texttt{MMul(ds[4], {\tiny X\_NOTRANS}, w[4], {\tiny X\_RANS}, dh[3]);}\\
\texttt{} \\
\texttt{dh[2] = dh[3];}\\
\texttt{ReluBackward(h[2], s[2], dh[2], ds[2]);}\\
\texttt{MMul(h[1], {\tiny X\_TRANS}, ds[2], {\tiny X\_NOTRANS}, dw[2]);}\\
\texttt{MMul(ds[2], {\tiny X\_NOTRANS}, w[2], {\tiny X\_TRANS}, dh[2]);}\\
\texttt{} \\
\texttt{dh[1] = dh[1] + dh[3];}\\
\texttt{...} // 继续反向传播 \\
\texttt{} \\
\texttt{for(unsigned i = 0; i < 5; i++)\{} \\
\texttt{} \ \ \ \ ... // 通过{\texttt{dw[i]}}访问参数的梯度\\
\texttt{\}}
\end{tabbing}
}
\tcblower
\begin{center}
\begin{tikzpicture}
\node [anchor=south,draw,rounded corners,inner sep=2pt,minimum width=8em,minimum height=1.2em,fill=red!30!white,blur shadow={shadow xshift=1pt,shadow yshift=-1pt}] (h1) at (0,0) {\tiny{x (input)}};
\node [anchor=south,draw,rounded corners,inner sep=2pt,minimum width=8em,minimum height=1.2em,fill=green!30!white,blur shadow={shadow xshift=1pt,shadow yshift=-1pt}] (h2) at ([yshift=1.5em]h1.north) {\tiny{h1 = Relu(x * w1)}};
\node [anchor=south,draw,rounded corners,inner sep=2pt,minimum width=8em,minimum height=1.2em,fill=green!30!white,blur shadow={shadow xshift=1pt,shadow yshift=-1pt}] (h3) at ([yshift=1.5em]h2.north) {\tiny{h2 = Relu(h1 * w2)}};
\node [anchor=south,draw,rounded corners,inner sep=2pt,minimum width=8em,minimum height=1.2em,fill=green!30!white,blur shadow={shadow xshift=1pt,shadow yshift=-1pt}] (h4) at ([yshift=1.5em]h3.north) {\tiny{h3 = h2 + h1}};
{\draw [->,thick] (h1.north) -- (h2.south);}
{\draw [->,thick] (h2.north) -- (h3.south);}
{\draw [->,thick] (h3.north) -- (h4.south);}
{\draw [->,thick,rounded corners] (h2.east) -- ([xshift=0.5em]h2.east) -- ([xshift=0.5em,yshift=0.5em]h3.north east) -- ([xshift=-2em,yshift=0.5em]h3.north east) -- ([xshift=-2em,yshift=1.5em]h3.north east);}
{\draw [<-,very thick,red] (h1.north) -- (h2.south);}
{\draw [<-,very thick,red] (h2.north) -- (h3.south);}
{\draw [<-,very thick,red] (h3.north) -- (h4.south);}
{\draw [<-,very thick,red,rounded corners] (h2.east) -- ([xshift=0.5em]h2.east) -- ([xshift=0.5em,yshift=0.5em]h3.north east) -- ([xshift=-2em,yshift=0.5em]h3.north east) -- ([xshift=-2em,yshift=1.5em]h3.north east);}
\node [anchor=south,draw,rounded corners,inner sep=2pt,minimum width=8.0em,minimum height=1.2em,fill=red!30!white,blur shadow={shadow xshift=1pt,shadow yshift=-1pt}] (slayer) at ([yshift=1.5em]h4.north) {\tiny{h4 = Softmax(h3 * w4) (output)}};
\node [anchor=south] (losslabel) at (slayer.north) {\scriptsize{\textbf{Cross Entropy Loss}}};
{\draw [->,thick] (h4.north) -- (slayer.south);}
{\draw [<-,very thick,red] (h4.north) -- (slayer.south);}
\end{tikzpicture}
\end{center}
\end{tcolorbox}
%%%------------------------------------------------------------------------------------------------------------
Book/Chapter5/Figures/fig-code-back-propagation-2.tex 0 → 100644
%%%------------------------------------------------------------------------------------------------------------
\begin{tcolorbox}
[bicolor,sidebyside,width=12cm,righthand width=4cm,size=title,frame engine=empty,
colback=blue!10!white,colbacklower=black!5!white]
{\scriptsize
\begin{tabbing}
\texttt{XTensor x, y, gold, h[5], w[5], s[5];} \\
\texttt{XTensor dh[5], dw[5], ds[5];} \\
\texttt{...} // 前向过程 \\
\texttt{h[0] = x;} \\
\texttt{y = h[4];} \\
\texttt{} \\
\texttt{CrossEntropyBackward(dh[4], y, gold);} \\
\texttt{SoftmaxBackward(y, s[4], dh[4], ds[4]);}\\
\texttt{MMul(h[3], {\tiny X\_TRANS}, ds[4], {\tiny X\_NOTRANS}, dw[4]);}\\
\texttt{MMul(ds[4], {\tiny X\_NOTRANS}, w[4], {\tiny X\_RANS}, dh[3]);}\\
\texttt{} \\
\texttt{dh[2] = dh[3];}\\
\texttt{ReluBackward(h[2], s[2], dh[2], ds[2]);}\\
\texttt{MMul(h[1], {\tiny X\_TRANS}, ds[2], {\tiny X\_NOTRANS}, dw[2]);}\\
\texttt{MMul(ds[2], {\tiny X\_NOTRANS}, w[2], {\tiny X\_TRANS}, dh[2]);}\\
\texttt{} \\
\texttt{dh[1] = dh[1] + dh[3];}\\
\texttt{...} // 继续反向传播 \\
\texttt{} \\
\texttt{for(unsigned i = 0; i < 5; i++)\{} \\
\texttt{} \ \ \ \ ... // 通过{\texttt{dw[i]}}访问参数的梯度\\
\texttt{\}}
\end{tabbing}
}
\tcblower
\begin{center}
\begin{tikzpicture}
\node [anchor=south,draw,rounded corners,inner sep=2pt,minimum width=8em,minimum height=1.2em,fill=red!30!white,blur shadow={shadow xshift=1pt,shadow yshift=-1pt}] (h1) at (0,0) {\tiny{x (input)}};
\node [anchor=south,draw,rounded corners,inner sep=2pt,minimum width=8em,minimum height=1.2em,fill=green!30!white,blur shadow={shadow xshift=1pt,shadow yshift=-1pt}] (h2) at ([yshift=1.5em]h1.north) {\tiny{h1 = Relu(x * w1)}};
\node [anchor=south,draw,rounded corners,inner sep=2pt,minimum width=8em,minimum height=1.2em,fill=green!30!white,blur shadow={shadow xshift=1pt,shadow yshift=-1pt}] (h3) at ([yshift=1.5em]h2.north) {\tiny{h2 = Relu(h1 * w2)}};
\node [anchor=south,draw,rounded corners,inner sep=2pt,minimum width=8em,minimum height=1.2em,fill=green!30!white,blur shadow={shadow xshift=1pt,shadow yshift=-1pt}] (h4) at ([yshift=1.5em]h3.north) {\tiny{h3 = h2 + h1}};
{\draw [->,thick] (h1.north) -- (h2.south);}
{\draw [->,thick] (h2.north) -- (h3.south);}
{\draw [->,thick] (h3.north) -- (h4.south);}
{\draw [->,thick,rounded corners] (h2.east) -- ([xshift=0.5em]h2.east) -- ([xshift=0.5em,yshift=0.5em]h3.north east) -- ([xshift=-2em,yshift=0.5em]h3.north east) -- ([xshift=-2em,yshift=1.5em]h3.north east);}
{\draw [<-,very thick,red] (h1.north) -- (h2.south);}
{\draw [<-,very thick,red] (h2.north) -- (h3.south);}
{\draw [<-,very thick,red] (h3.north) -- (h4.south);}
{\draw [<-,very thick,red,rounded corners] (h2.east) -- ([xshift=0.5em]h2.east) -- ([xshift=0.5em,yshift=0.5em]h3.north east) -- ([xshift=-2em,yshift=0.5em]h3.north east) -- ([xshift=-2em,yshift=1.5em]h3.north east);}
\node [anchor=south,draw,rounded corners,inner sep=2pt,minimum width=8.0em,minimum height=1.2em,fill=red!30!white,blur shadow={shadow xshift=1pt,shadow yshift=-1pt}] (slayer) at ([yshift=1.5em]h4.north) {\tiny{h4 = Softmax(h3 * w4) (output)}};
\node [anchor=south] (losslabel) at (slayer.north) {\scriptsize{\textbf{Cross Entropy Loss}}};
{\draw [->,thick] (h4.north) -- (slayer.south);}
{\draw [<-,very thick,red] (h4.north) -- (slayer.south);}
\end{tikzpicture}
\end{center}
\end{tcolorbox}
%%%------------------------------------------------------------------------------------------------------------
Book/Chapter5/Figures/fig-code-niutensor-one.tex 0 → 100644
%%%------------------------------------------------------------------------------------------------------------
\begin{tcolorbox}
[bicolor,sidebyside,width=11cm,righthand width=4cm,size=title,frame engine=empty,
colback=blue!10!white,colbacklower=black!5!white]
{\scriptsize
\begin{tabbing}
\texttt{XTensor x, y, w, b;} \\
\texttt{InitTensor3D(\&x, 3, 4, 5);} \\
\texttt{InitTensor2D(\&w, 5, 3);} \\
\texttt{InitTensor1D(\&b, 3);} \\
\texttt{...} \\
\texttt{y = Sigmoid(MMul(x, w) + b);}
\end{tabbing}
}
\tcblower
\begin{center}
\begin{tikzpicture}
\node [draw,circle,inner sep=2pt,fill=red!30!white,blur shadow={shadow xshift=1pt,shadow yshift=-1pt}] (x) at (0,0) {\footnotesize{$\textrm{x}$}};
\node [anchor=south,draw,rounded corners,inner sep=2pt,minimum width=4em,fill=green!30!white,blur shadow={shadow xshift=1pt,shadow yshift=-1pt}] (layer) at ([yshift=0.7em]x.north) {\scriptsize{layer}};
\node [anchor=south,draw,circle,inner sep=2pt,fill=red!30!white,blur shadow={shadow xshift=1pt,shadow yshift=-1pt}] (y) at ([yshift=0.7em]layer.north) {\scriptsize{$\textrm{y}$}};
\draw [thick,->] (x.north) -- (layer.south);
\draw [thick,->] (layer.north) -- (y.south);
\node [anchor=west,align=left] (xshape) at (x.east) {\tiny{shape: 3*4*5}};
\node [anchor=west,align=left] (yshape) at (y.east) {\tiny{shape: 3*4*3}};
\end{tikzpicture}
\end{center}
\end{tcolorbox}
%%%------------------------------------------------------------------------------------------------------------
Book/Chapter5/Figures/fig-code-niutensor-rnn.tex 0 → 100644
%%%------------------------------------------------------------------------------------------------------------
\begin{tcolorbox}
[bicolor,sidebyside,width=11cm,righthand width=4cm,size=title,frame engine=empty,
colback=blue!10!white,colbacklower=black!5!white]
{\scriptsize
\begin{tabbing}
\texttt{XTensor x[3], y[3], r, wh;} \\
\texttt{XTensor h1, h2, w1, b1, h3, h4;} \\
\texttt{XList splits;} \\
\texttt{...} \\
\texttt{for(unsigned i = 0; i < 3; i++)\{} \\
\texttt{\hspace{2em}r = Concatenate(x[i] + r) * wh;}\\
\texttt{\hspace{2em}splits.Add(\&r);}\\
\texttt{\}}\\
\texttt{} \\
\texttt{h1 = Merge(splits, 0);}\\
\texttt{h2 = Relu(h1 * w1 + b1);}\\
\texttt{h3 = h1 + h2;} \\
\texttt{h4 = Softmax(h3);} \\
\texttt{} \\
\texttt{Split(h4, splits, 0);} \\
\texttt{} \\
\texttt{for(unsigned i = 0; i < 3; i++)\{} \\
\texttt{\hspace{2em}y[i] = *(XTensor*)splits.Get(i);}\\
\texttt{\hspace{2em}y[i].Dump(stdout);}\\
\texttt{\}}
\end{tabbing}
}
\tcblower
\begin{center}
\begin{tikzpicture}
\node [draw,circle,inner sep=1pt,fill=red!30!white,blur shadow={shadow xshift=1pt,shadow yshift=-1pt}] (x1) at (0,0) {\footnotesize{$\textrm{x}_1$}};
\node [anchor=west,draw,circle,inner sep=1pt,fill=red!30!white,blur shadow={shadow xshift=1pt,shadow yshift=-1pt}] (x2) at ([xshift=2em]x1.east) {\footnotesize{$\textrm{x}_2$}};
\node [anchor=west,draw,circle,inner sep=1pt,fill=red!30!white,blur shadow={shadow xshift=1pt,shadow yshift=-1pt}] (x3) at ([xshift=2em]x2.east) {\footnotesize{$\textrm{x}_3$}};
\node [anchor=south,draw,rounded corners,inner sep=2pt,minimum width=2.5em,fill=green!30!white,blur shadow={shadow xshift=1pt,shadow yshift=-1pt}] (rlayer1) at ([yshift=1em]x1.north) {\tiny{rlayer}};
\node [anchor=south,draw,rounded corners,inner sep=2pt,minimum width=2.5em,fill=green!30!white,blur shadow={shadow xshift=1pt,shadow yshift=-1pt}] (rlayer2) at ([yshift=1em]x2.north) {\tiny{rlayer}};
\node [anchor=south,draw,rounded corners,inner sep=2pt,minimum width=2.5em,fill=green!30!white,blur shadow={shadow xshift=1pt,shadow yshift=-1pt}] (rlayer3) at ([yshift=1em]x3.north) {\tiny{rlayer}};
\draw [->,thick] (x1.north) -- (rlayer1.south);
\draw [->,thick] (x2.north) -- (rlayer2.south);
\draw [->,thick] (x3.north) -- (rlayer3.south);
\draw [->,thick] (rlayer1.east) -- (rlayer2.west);
\draw [->,thick] (rlayer2.east) -- (rlayer3.west);
\draw [->,thick] (rlayer1.north) -- ([yshift=1em]rlayer1.north);
\draw [->,thick] (rlayer2.north) -- ([yshift=1em]rlayer2.north);
\draw [->,thick] (rlayer3.north) -- ([yshift=1em]rlayer3.north);
{
\node [anchor=south,draw,rounded corners,inner sep=2pt,minimum width=9.4em,minimum height=1.0em,fill=green!30!white,blur shadow={shadow xshift=1pt,shadow yshift=-1pt}] (h1) at ([yshift=1em]rlayer2.north) {\tiny{h1 = Merge($\cdot$)}};
\node [anchor=south,draw,rounded corners,inner sep=2pt,minimum width=9.4em,minimum height=1.0em,fill=green!30!white,blur shadow={shadow xshift=1pt,shadow yshift=-1pt}] (h2) at ([yshift=1em]h1.north) {\tiny{h2 = Relu($\cdot$)}};
\node [anchor=south,draw,rounded corners,inner sep=2pt,minimum width=9.4em,minimum height=1.0em,fill=green!30!white,blur shadow={shadow xshift=1pt,shadow yshift=-1pt}] (h3) at ([yshift=1em]h2.north) {\tiny{h3 = Sum($\cdot$)}};
\node [anchor=south,draw,rounded corners,inner sep=2pt,minimum width=9.4em,minimum height=1.0em,fill=green!30!white,blur shadow={shadow xshift=1pt,shadow yshift=-1pt}] (h4) at ([yshift=1em]h3.north) {\tiny{h4 = Softmax($\cdot$)}};
\draw [->,thick] (h1.north) -- (h2.south);
\draw [->,thick] (h2.north) -- (h3.south);
\draw [->,thick] (h3.north) -- (h4.south);
\draw [->,thick,rounded corners] (h1.east) -- ([xshift=0.5em]h1.east) -- ([xshift=0.5em,yshift=0.5em]h2.north east) -- ([xshift=-2em,yshift=0.5em]h2.north east) -- ([xshift=-2em,yshift=1em]h2.north east);
}
{
\node [anchor=south,draw,rounded corners,inner sep=2pt,minimum width=9.4em,minimum height=1.0em,fill=green!30!white,blur shadow={shadow xshift=1pt,shadow yshift=-1pt}] (slayer) at ([yshift=1em]h4.north) {\tiny{Split($\cdot$)}};
\node [anchor=south,draw,circle,inner sep=1pt,fill=red!30!white,blur shadow={shadow xshift=1pt,shadow yshift=-1pt}] (y2) at ([yshift=1em]slayer.north) {\footnotesize{$\textrm{y}_2$}};
\node [anchor=east,draw,circle,inner sep=1pt,fill=red!30!white,blur shadow={shadow xshift=1pt,shadow yshift=-1pt}] (y1) at ([xshift=-2em]y2.west) {\footnotesize{$\textrm{y}_1$}};
\node [anchor=west,draw,circle,inner sep=1pt,fill=red!30!white,blur shadow={shadow xshift=1pt,shadow yshift=-1pt}] (y3) at ([xshift=2em]y2.east) {\footnotesize{$\textrm{y}_3$}};
\draw [<-,thick] (y1.south) -- ([yshift=-1em]y1.south);
\draw [<-,thick] (y2.south) -- ([yshift=-1em]y2.south);
\draw [<-,thick] (y3.south) -- ([yshift=-1em]y3.south);
}
{
\draw [->,thick] (h4.north) -- (slayer.south);
}
\end{tikzpicture}
\end{center}
\end{tcolorbox}
%%%------------------------------------------------------------------------------------------------------------
Book/Chapter5/Figures/fig-code-niutensor-three.tex 0 → 100644
%%%------------------------------------------------------------------------------------------------------------
\begin{tcolorbox}
[bicolor,sidebyside,width=11cm,righthand width=4cm,size=title,frame engine=empty,
colback=blue!10!white,colbacklower=black!5!white]
{\scriptsize
\begin{tabbing}
\texttt{XTensor x, y, h1, h2;} \\
\texttt{XTensor w1, b1, w2, w3;} \\
\texttt{InitTensor3D(\&x, 3, 4, 5);} \\
\texttt{InitTensor2D(\&w1, 5, 3);} \\
\texttt{InitTensor1D(\&b1, 3);} \\
\texttt{InitTensor2D(\&w2, 3, 6);} \\
\texttt{InitTensor2D(\&w3, 6, 4);} \\
\texttt{...} \\
\texttt{h1 = Sigmoid(MMul(x, w1) + b1);} \\
\texttt{h2 = HandTanH(MMul(h1, w2));} \\
\texttt{y = Relu(MMul(h2, w3));}
\end{tabbing}
}
\tcblower
\begin{center}
\begin{tikzpicture}
\node [draw,circle,inner sep=2pt,fill=red!30!white,blur shadow={shadow xshift=1pt,shadow yshift=-1pt}] (x) at (0,0) {\footnotesize{$\textrm{x}$}};
\node [anchor=south,draw,rounded corners,inner sep=2pt,minimum width=4em,fill=green!30!white,blur shadow={shadow xshift=1pt,shadow yshift=-1pt}] (layer1) at ([yshift=0.7em]x.north) {\scriptsize{layer1}};
\node [anchor=south,draw,rounded corners,inner sep=2pt,minimum width=4em,fill=green!30!white,blur shadow={shadow xshift=1pt,shadow yshift=-1pt}] (layer2) at ([yshift=1.0em]layer1.north) {\scriptsize{layer2}};
\node [anchor=south,draw,rounded corners,inner sep=2pt,minimum width=4em,fill=green!30!white,blur shadow={shadow xshift=1pt,shadow yshift=-1pt}] (layer3) at ([yshift=1.0em]layer2.north) {\scriptsize{layer3}};
\node [anchor=south,draw,circle,inner sep=2pt,fill=red!30!white,blur shadow={shadow xshift=1pt,shadow yshift=-1pt}] (y) at ([yshift=0.7em]layer3.north) {\scriptsize{$\textrm{y}$}};
\draw [thick,->] (x.north) -- (layer1.south);
\draw [thick,->] (layer1.north) -- (layer2.south);
\draw [thick,->] (layer2.north) -- (layer3.south);
\draw [thick,->] (layer3.north) -- (y.south);
\node [anchor=west,align=left] (xshape) at (x.east) {\tiny{shape: 3*4*5}};
\node [anchor=west,align=left] (yshape) at (y.east) {\tiny{shape: 3*4*4}};
\node [anchor=south west,align=left,inner sep=2pt] (l1shape) at (layer1.north) {\tiny{shape: 3*4*3}};
\node [anchor=south west,align=left,inner sep=2pt] (l2shape) at (layer2.north) {\tiny{shape: 3*4*6}};
\end{tikzpicture}
\end{center}
\end{tcolorbox}
%%%------------------------------------------------------------------------------------------------------------
Book/Chapter5/Figures/fig-code-out.tex 0 → 100644
%%%------------------------------------------------------------------------------------------------------------
\begin{tcolorbox}[enhanced,width=11cm,frame engine=empty,boxrule=0.1mm,size=title,colback=black!10!white]
\begin{flushleft}
{\scriptsize
\begin{tabbing}
\texttt{order=2 dimsize=2,2 dtype=X\_FLOAT dense=1.000000} \\
\texttt{3.605762e-001 2.992340e-001 1.393780e-001 7.301248e-001}
\end{tabbing}
}
\end{flushleft}
\end{tcolorbox}
%%%------------------------------------------------------------------------------------------------------------
Book/Chapter5/Figures/fig-code-tensor-define-2.tex 0 → 100644
%%%------------------------------------------------------------------------------------------------------------
\begin{tcolorbox}[enhanced,width=11cm,frame engine=empty,boxrule=0.1mm,size=title,colback=blue!10!white]
\begin{flushleft}
{\scriptsize
\begin{tabbing}
\texttt{XTensor tensor;} \hspace{12em} \= // 声明张量tensor \\
\texttt{int sizes[6] = \{2,3,4,2,3,4\};} \> // 张量的形状为2*3*4*2*3*4 \\
\texttt{InitTensor(\&tensor, 6, sizes, X\_FLOAT);} \> // 定义形状为sizes的6阶张量
\end{tabbing}
}
\end{flushleft}
\end{tcolorbox}
%%%------------------------------------------------------------------------------------------------------------
Book/Chapter5/Figures/fig-code-tensor-define-3.tex 0 → 100644
%%%------------------------------------------------------------------------------------------------------------
\begin{tcolorbox}[enhanced,width=11cm,frame engine=empty,boxrule=0.1mm,size=title,colback=blue!10!white]
\begin{flushleft}
{\scriptsize
\begin{tabbing}
\texttt{XTensor a, b, c;} \hspace{11.5em} \= // 声明张量tensor \\
\texttt{InitTensor1D(\&a, 10, X\_INT);} \> // 10维的整数型向量\\
\texttt{InitTensor1D(\&b, 10);} \> // 10维的向量，缺省类型(浮点)\\
\texttt{InitTensor4D(\&c, 10, 20, 30, 40);} \> // 10*20*30*40的4阶张量(浮点)
\end{tabbing}
}
\end{flushleft}
\end{tcolorbox}
%%%------------------------------------------------------------------------------------------------------------
Book/Chapter5/Figures/fig-code-tensor-define-GPU.tex 0 → 100644
%%%------------------------------------------------------------------------------------------------------------
\begin{tcolorbox}[enhanced,width=11cm,frame engine=empty,boxrule=0.1mm,size=title,colback=blue!10!white]
\begin{flushleft}
{\scriptsize
\begin{tabbing}
\texttt{XTensor tensorGPU;} \hspace{10.5em} \= // 声明张量tensor \\
\texttt{InitTensor2D(\&tensorGPU, 10, 20,} $\backslash$ \> // 在编号为0的GPU上定义张量 \\
\hspace{6.7em} \texttt{X\_FLOAT, 0);}
\end{tabbing}
}
\end{flushleft}
\end{tcolorbox}
%%%------------------------------------------------------------------------------------------------------------
Book/Chapter5/Figures/fig-code-tensor-define.tex 0 → 100644
%------------------------------------------------------------------------------------------------------------
\begin{tcolorbox}[enhanced,width=11cm,frame engine=empty,boxrule=0.1mm,size=title,colback=blue!10!white]
\begin{flushleft}
{\scriptsize
\begin{tabbing}
\texttt{\#include "source/tensor/XTensor.h"} \hspace{4em} \= // 引用XTensor定义的头文件 \\
\texttt{using namespace nts;} \> // 引用nts命名空间 \\
\ \\
\texttt{int main(int argc, const char ** argv)\{} \\
\ \ \ \ \texttt{XTensor tensor;} \> // 声明张量tensor \\
\ \ \ \ \texttt{InitTensor2D(\&tensor, 2, 2, X\_FLOAT);} \> // 定义张量为2*2的矩阵 \\
\ \ \ \ \texttt{tensor.SetDataRand();} \> // [0,1]均匀分布初始化张量 \\
\ \ \ \ \texttt{tensor.Dump(stdout);} \> // 输出张量内容 \\
\ \ \ \ \texttt{return 0;}\\
\texttt{\}}
\end{tabbing}
}
\end{flushleft}
\end{tcolorbox}
%%%------------------------------------------------------------------------------------------------------------
Book/Chapter5/Figures/fig-code-tensor-mul.tex 0 → 100644
%%%------------------------------------------------------------------------------------------------------------
\begin{tcolorbox}[enhanced,width=11cm,frame engine=empty,boxrule=0.1mm,size=title,colback=blue!10!white]
\begin{flushleft}
{\scriptsize
\begin{tabbing}
\texttt{XTensor a, b, c;} \hspace{10.0em} \= // 声明张量tensor \\
\texttt{InitTensor4D(\&a, 2, 2, 3, 4);} \> // a为2*2*3*4的4阶张量 \\
\texttt{InitTensor2D(\&b, 4, 5);} \> // b为4*5的矩阵 \\
\texttt{a.SetDataRand();} \> // 随机初始化a \\
\texttt{b.SetDataRand();} \> // 随机初始化b \\
\texttt{c = MMul(a, b);} \> // 矩阵乘的结果为2*2*3*5的4阶张量
\end{tabbing}
}
\end{flushleft}
\end{tcolorbox}
%%%------------------------------------------------------------------------------------------------------------
Book/Chapter5/Figures/fig-code-tensor-operation.tex 0 → 100644
%%%------------------------------------------------------------------------------------------------------------
\begin{tcolorbox}[enhanced,width=11cm,frame engine=empty,boxrule=0.1mm,size=title,colback=blue!10!white]
\begin{flushleft}
{\scriptsize
\begin{tabbing}
\texttt{XTensor a, b, c, d, e;} \hspace{7em} \= // 声明张量tensor \\
\texttt{InitTensor3D(\&a, 2, 3, 4);} \> // a为2*3*4的3阶张量 \\
\texttt{InitTensor3D(\&b, 2, 3, 4);} \> // b为2*3*4的3阶张量 \\
\texttt{InitTensor3D(\&c, 2, 3, 4);} \> // c为2*3*4的3阶张量 \\
\texttt{a.SetDataRand();} \> // 随机初始化a \\
\texttt{b.SetDataRand();} \> // 随机初始化b \\
\texttt{c.SetDataRand();} \> // 随机初始化c \\
\texttt{d = a + b * c;} \> // d被赋值为 a + b * c \\
\texttt{d = ((a + b) * d - b / c ) * d;} \> // d可以被嵌套使用 \\
\texttt{e = Sigmoid(d);} \> // d经过激活函数Sigmoid赋值给e
\end{tabbing}
}
\end{flushleft}
\end{tcolorbox}
%%%------------------------------------------------------------------------------------------------------------
Book/Chapter5/Figures/fig-corresponence-between-matrix-element-and-output.tex 0 → 100644
%%%------------------------------------------------------------------------------------------------------------
\begin{tikzpicture}
\begin{scope}
\tikzstyle{neuronnode} = [minimum size=1.5em,circle,draw,ublue,very thick,fill=white,drop shadow={shadow xshift=0.1em,shadow yshift=-0.1em}]
\node [anchor=center,neuronnode] (neuron00) at (0,0) {};
\node [anchor=center,neuronnode] (neuron01) at ([yshift=-3em]neuron00) {};
\node [anchor=center,neuronnode] (neuron02) at ([yshift=-3em]neuron01) {};
\node [anchor=east] (x0) at ([xshift=-6em]neuron00.west) {$x_0$};
\node [anchor=east] (x1) at ([xshift=-6em]neuron01.west) {$x_1$};
\node [anchor=east] (x2) at ([xshift=-6em]neuron02.west) {$b$};
\node [anchor=west] (y0) at ([xshift=4em]neuron00.east) {$y_0$：\scriptsize{温度}};
\draw [->,red!50,line width=0.4mm] (x0.east) -- (neuron00.180) node [pos=0.1,above] {\tiny{$w_{00}$}};
\draw [->,red!50,line width=0.4mm] (x1.east) -- (neuron00.200) node [pos=0.1,above] {\tiny{$w_{10}$}};
\draw [->,red!50,line width=0.4mm] (x2.east) -- (neuron00.220) node [pos=0.05,above,yshift=0.3em] {\tiny{$b_{0}$}};
\draw [->,red!30,line width=0.4mm] (neuron00.east) -- (y0.west);
\node [anchor=west] (y1) at ([xshift=4em]neuron01.east) {$y_1$：\scriptsize{湿度}};
\draw [->,blue!50,line width=0.4mm] (x0.east) -- (neuron01.160) node [pos=0.4,above] {\tiny{$w_{01}$}};
\draw [->,blue!50,line width=0.4mm] (x1.east) -- (neuron01.180) node [pos=0.35,above,yshift=-0.2em] {\tiny{$w_{11}$}};
\draw [->,blue!50,line width=0.4mm] (x2.east) -- (neuron01.200) node [pos=0.3,below,yshift=0.2em] {\tiny{$b_{1}$}};
\draw [->,blue!30,line width=0.4mm] (neuron01.east) -- (y1.west);
\node [anchor=west] (y2) at ([xshift=4em]neuron02.east) {$y_2$：\scriptsize{风力}};
\draw [->,purple!40,line width=0.4mm] (x0.east) -- (neuron02.140) node [pos=0.1,below,yshift=-0.2em] {\tiny{$w_{02}$}};
\draw [->,purple!40,line width=0.4mm] (x1.east) -- (neuron02.160) node [pos=0.1,below] {\tiny{$w_{12}$}};
\draw [->,purple!40,line width=0.4mm] (x2.east) -- (neuron02.180) node [pos=0.3,below] {\tiny{$b_{2}$}};
\draw [->,purple!30,line width=0.4mm] (neuron02.east) -- (y2.west);
\end{scope}
\end{tikzpicture}
%%%------------------------------------------------------------------------------------------------------------
Book/Chapter5/Figures/fig-derivative1.tex 0 → 100644
%%%------------------------------------------------------------------------------------------------------------
\begin{tikzpicture}
\begin{axis}[
name=sigmoid,
width=6cm, height=4.5cm,
xtick={-5,-2.5,...,6},
ytick={0,0.5,1.0},
xlabel={\textbf{x}},
ylabel={\textbf{y}},
xlabel style={xshift=2.4cm,yshift=0.7cm},
axis y line=middle,%y轴居中
ylabel style={xshift=0.1cm,yshift=0cm},
x axis line style={->},%x轴箭头
axis line style={very thick},
axis x line*=bottom,%将上面的x轴去掉，仅保留下面的x轴
xmin=-6,
xmax=6,
ymin=0,
ymax=1.4]
\addplot[draw=red,thick]{1/(1 + exp( -x))};
\end{axis}
\node [anchor=north] (labelc) at (2.3,-0.5) {\footnotesize{(a)}};
\begin{axis}[
at={(sigmoid.east)},
anchor=east,
xshift=6cm,
yshift=0cm,
width=6cm, height=4.5cm,
xtick={-4,-2,...,4},
ytick={0,0.1,0.2},
xlabel={\textbf{x}},
ylabel={\textbf{y}},
xlabel style={xshift=2.4cm,yshift=0.7cm},
axis y line=middle,%y轴居中
ylabel style={xshift=0.1cm,yshift=0cm},
x axis line style={->},%x轴箭头
axis line style={very thick},
axis x line*=bottom,%将上面的x轴去掉，仅保留下面的x轴
xmin=-6,
xmax=6,
ymin=0,
ymax=0.3]
\addplot[draw=ublue,thick]{(1/(1 + exp( -x)))*(1-(1/(1 + exp( -x))))};
\end{axis}
\node [anchor=north] (labelc) at (8.2,-0.5) {\footnotesize{(b)}};
\end{tikzpicture}
%%%------------------------------------------------------------------------------------------------------------
Book/Chapter5/Figures/fig-derivative2.tex 0 → 100644
%%%------------------------------------------------------------------------------------------------------------
\begin{tikzpicture}
\begin{axis}[
name=tanh,
width=6cm, height=4.5cm,
xtick={-5,-2.5,...,6},
ytick={-1.0,-0.5,0,0.5,1.0},
xlabel={\textbf{x}},
ylabel={\textbf{y}},
xlabel style={xshift=2.4cm,yshift=2.5cm},
axis y line=middle,%y轴居中
ylabel style={xshift=0.1cm,yshift=0cm},
x axis line style={->},%x轴箭头
axis line style={very thick},
axis x line*=middle,%将上面的x轴去掉，仅保留下面的x轴
xmin=-6,
xmax=6,
ymin=-1.2,
ymax=1.2]
\addplot[draw=red,thick]{tanh(x)};
\end{axis}
\node [anchor=north] (labelc) at (2.3,-0.5) {\footnotesize{(a)}};
\begin{axis}[
at={(tanh.east)},
anchor=east,
xshift=6cm,
yshift=0cm,
width=6cm, height=4.5cm,
xtick={-10,-5,...,10},
ytick={0,0.5,1},
xlabel={\textbf{x}},
ylabel={\textbf{y}},
xlabel style={xshift=2.4cm,yshift=0.7cm},
axis y line=middle,%y轴居中
ylabel style={xshift=0.1cm,yshift=0cm},
x axis line style={->},%x轴箭头
axis line style={very thick},
axis x line*=bottom,%将上面的x轴去掉，仅保留下面的x轴
xmin=-10,
xmax=10,
ymin=0,
ymax=1.2]
\addplot[draw=ublue,thick]{1-tanh(x)*tanh(x)};
\end{axis}
\node [anchor=north] (labelc) at (8.2,-0.5) {\footnotesize{(b)}};
\end{tikzpicture}
%%%------------------------------------------------------------------------------------------------------------
Book/Chapter5/Figures/fig-derivative3.tex 0 → 100644
%%%------------------------------------------------------------------------------------------------------------
\begin{tikzpicture}
\begin{axis}[
name=relu,
width=6cm, height=4.5cm,
xtick={-1.0,-0.5,...,1.0},
ytick={0.5,1.0},
xlabel={\textbf{x}},
ylabel={\textbf{y}},
xlabel style={xshift=2.4cm,yshift=0.7cm},
axis y line=middle,%y轴居中
ylabel style={xshift=0.1cm,yshift=0cm},
x axis line style={->},%x轴箭头
axis line style={very thick},
axis x line*=middle,%将上面的x轴去掉，仅保留下面的x轴
xmin=-1.2,
xmax=1.2,
ymin=0,
ymax=1.2]
\addplot[domain=-1:0,draw=red,thick]{0};
\addplot[domain=0:1,draw=red,thick]{x};
\end{axis}
\node [anchor=north] (labelc) at (2.3,-0.5) {\footnotesize{(a)}};
\begin{axis}[
at={(relu.east)},
anchor=east,
xshift=6cm,
yshift=0cm,
width=6cm, height=4.5cm,
xtick={-4,-2,...,4},
ytick={0,0.2,...,1.0},
xlabel={\textbf{x}},
ylabel={\textbf{y}},
xlabel style={xshift=2.4cm,yshift=0.7cm},
axis y line=middle,%y轴居中
ylabel style={xshift=0.1cm,yshift=0cm},
x axis line style={->},%x轴箭头
axis line style={very thick},
axis x line*=middle,%将上面的x轴去掉，仅保留下面的x轴
xmin=-5,
xmax=5,
ymin=0,
ymax=1.2]
\addplot[domain=-4:0,draw=ublue,thick]{0};
\addplot[domain=0:4,draw=ublue,thick]{1};
\end{axis}
\node [anchor=north] (labelc) at (8.2,-0.5) {\footnotesize{(b)}};
\end{tikzpicture}
%%%------------------------------------------------------------------------------------------------------------
Book/Chapter5/Figures/fig-different-forms-of-neuronal-input.tex 0 → 100644
%%%------------------------------------------------------------------------------------------------------------
\begin{tikzpicture}
\begin{scope}
\draw [->,thick] (0,0) -- (2.5,0);
\draw [->,thick] (0,0) -- (0, 1.5);
\draw [-,very thick,ublue,domain=0.6:2,samples=100] plot (\x,{ 1/\x - 0.2});
\node [anchor=east] (ylabel) at (0, 3.2em) {\footnotesize{$x_0$}};
\node [anchor=north] (xlabel) at (5em, 0em) {\scriptsize{距离(km)}};
\end{scope}
\begin{scope}[xshift=9em]
\draw [->,thick] (0,0) -- (2.5,0);
\draw [->,thick] (0,0) -- (0, 1.5);
\draw [-,very thick,ublue,domain=0.4:2,samples=100] plot (\x,{ 0.5/\x});
\node [anchor=east] (ylabel) at (0, 3.2em) {\footnotesize{$x_1$}};
\node [anchor=north] (xlabel) at (5em, 0em) {\scriptsize{票价(元)}};
\end{scope}
\begin{scope}[xshift=18em]
\draw [->,thick] (0,0) -- (2.5,0);
\draw [->,thick] (0,0) -- (0, 1.5);
\node [anchor=east] (ylabel) at (0, 3.2em) {\footnotesize{$x_2$}};
\node [anchor=south, fill=ublue, minimum width=1.5em, minimum height=0.1em, inner sep=0] (histogram1) at (1.5em, 0) {};
\node [anchor=south, fill=ublue, minimum width=1.5em, minimum height=3em, inner sep=0] (histogram2) at (4.0em, 0) {};
\node [anchor=north] (hlabel1) at (histogram1.south) {\tiny{女友不去}};
\node [anchor=north] (hlabel2) at (histogram2.south) {\tiny{女友去}};
\end{scope}
\end{tikzpicture}
%%%------------------------------------------------------------------------------------------------------------
Book/Chapter5/Figures/fig-elmo.tex 0 → 100644
%%%------------------------------------------------------------------------------------------------------------
\begin{tikzpicture}
\begin{scope}[scale=1.2]
\node [anchor=west,draw,inner sep=4pt,fill=blue!20!white] (Lstm0) at (0,0) {\scriptsize{LSTM}};
\node [anchor=west,draw,inner sep=4pt,fill=blue!20!white] (Lstm1) at ([xshift=1em]Lstm0.east) {\scriptsize{LSTM}};
\node [anchor=west,inner sep=4pt] (sep) at ([xshift=0.5em]Lstm1.east) {\scriptsize{...}};
\node [anchor=west,draw,inner sep=4pt,fill=blue!20!white] (Lstm2) at ([xshift=0.5em]sep.east) {\scriptsize{LSTM}};
\node [anchor=south,draw,inner sep=4pt,fill=blue!20!white] (Lstm3) at ([yshift=1em]Lstm0.north) {\scriptsize{LSTM}};
\node [anchor=west,draw,inner sep=4pt,fill=blue!20!white] (Lstm4) at ([xshift=1em]Lstm3.east) {\scriptsize{LSTM}};
\node [anchor=west,inner sep=4pt] (sep1) at ([xshift=0.5em]Lstm4.east) {\scriptsize{...}};
\node [anchor=west,draw,inner sep=4pt,fill=blue!20!white] (Lstm5) at ([xshift=0.5em]sep1.east) {\scriptsize{LSTM}};
\node [rectangle,rounded corners,draw=black!50,densely dashed,inner sep=0.4em] [fit = (Lstm0) (Lstm2) (Lstm3) (Lstm5)] (inputshadow) {};
\node [anchor=north,draw,inner sep=4pt,fill=ugreen!20!white,minimum width=2em] (e1) at ([xshift=-2em,yshift=-1em]Lstm2.south) {\scriptsize{$\textbf{e}_1$}};
\node [anchor=west,draw,inner sep=4pt,fill=ugreen!20!white,minimum width=2em] (e2) at ([xshift=1em]e1.east) {\scriptsize{$\textbf{e}_2$}};
\node [anchor=west,inner sep=4pt] (sep5) at ([xshift=1em]e2.east) {\scriptsize{...}};
\node [anchor=west,draw,inner sep=4pt,fill=ugreen!20!white,minimum width=2em] (e3) at ([xshift=1em]sep5.east) {\scriptsize{$\textbf{e}_m$}};
\node [anchor=south,draw,inner sep=4pt,fill=yellow!30,minimum width=2em] (t1) at ([xshift=-2em,yshift=1em]Lstm5.north) {\scriptsize{$\textbf{h}_1$}};
\node [anchor=west,draw,inner sep=4pt,fill=yellow!30,minimum width=2em] (t2) at ([xshift=1em]t1.east) {\scriptsize{$\textbf{h}_2$}};
\node [anchor=west,inner sep=4pt] (sep6) at ([xshift=1em]t2.east) {\scriptsize{...}};
\node [anchor=west,draw,inner sep=4pt,fill=yellow!30,minimum width=2em] (t3) at ([xshift=1em]sep6.east) {\scriptsize{$\textbf{h}_m$}};
\node [anchor=west,draw,inner sep=4pt,fill=blue!20!white] (Lstm6) at ([xshift=1.5em]Lstm2.east) {\scriptsize{LSTM}};
\node [anchor=west,draw,inner sep=4pt,fill=blue!20!white] (Lstm7) at ([xshift=1em]Lstm6.east) {\scriptsize{LSTM}};
\node [anchor=west,inner sep=4pt] (sep3) at ([xshift=0.5em]Lstm7.east) {\scriptsize{...}};
\node [anchor=west,draw,inner sep=4pt,fill=blue!20!white] (Lstm8) at ([xshift=0.5em]sep3.east) {\scriptsize{LSTM}};
\node [anchor=south,draw,inner sep=4pt,fill=blue!20!white] (Lstm9) at ([yshift=1em]Lstm6.north) {\scriptsize{LSTM}};
\node [anchor=west,draw,inner sep=4pt,fill=blue!20!white] (Lstm10) at ([xshift=1em]Lstm9.east) {\scriptsize{LSTM}};
\node [anchor=west,inner sep=4pt] (sep4) at ([xshift=0.5em]Lstm10.east) {\scriptsize{...}};
\node [anchor=west,draw,inner sep=4pt,fill=blue!20!white] (Lstm11) at ([xshift=0.5em]sep4.east) {\scriptsize{LSTM}};
\node [rectangle,rounded corners,draw=black!50,densely dashed,inner sep=0.4em] [fit = (Lstm6) (Lstm8) (Lstm9) (Lstm11)] (inputshadow) {};
\draw [->] ([xshift=0.1em]Lstm0.east) -- ([xshift=-0.1em]Lstm1.west);
\draw [->] ([xshift=0.1em]Lstm1.east) -- ([xshift=0.1em]sep.west);
\draw [->] ([xshift=-0.1em]sep.east) -- ([xshift=-0.1em]Lstm2.west);
\draw [->] ([xshift=0.1em]Lstm3.east) -- ([xshift=-0.1em]Lstm4.west);
\draw [->] ([xshift=0.1em]Lstm4.east) -- ([xshift=0.1em]sep1.west);
\draw [->] ([xshift=-0.1em]sep1.east) -- ([xshift=-0.1em]Lstm5.west);
\draw [->] ([yshift=0.1em]Lstm0.north) -- ([yshift=-0.1em]Lstm3.south);
\draw [->] ([yshift=0.1em]Lstm1.north) -- ([yshift=-0.1em]Lstm4.south);
\draw [->] ([yshift=0.1em]Lstm2.north) -- ([yshift=-0.1em]Lstm5.south);
\draw [->] ([xshift=0.1em]Lstm6.east) -- ([xshift=-0.1em]Lstm7.west);
\draw [->] ([xshift=0.1em]Lstm7.east) -- ([xshift=0.1em]sep3.west);
\draw [->] ([xshift=-0.1em]sep3.east) -- ([xshift=-0.1em]Lstm8.west);
\draw [->] ([xshift=0.1em]Lstm9.east) -- ([xshift=-0.1em]Lstm10.west);
\draw [->] ([xshift=0.1em]Lstm10.east) -- ([xshift=0.1em]sep4.west);
\draw [->] ([xshift=-0.1em]sep4.east) -- ([xshift=-0.1em]Lstm11.west);
\draw [->] ([yshift=0.1em]Lstm6.north) -- ([yshift=-0.1em]Lstm9.south);
\draw [->] ([yshift=0.1em]Lstm7.north) -- ([yshift=-0.1em]Lstm10.south);
\draw [->] ([yshift=0.1em]Lstm8.north) -- ([yshift=-0.1em]Lstm11.south);
\draw [->] ([yshift=0.1em]e1.north) -- ([yshift=-0.1em]Lstm0.south);
\draw [->] ([yshift=0.1em]e1.north) -- ([yshift=-0.1em]Lstm6.south);
\draw [->] ([yshift=0.1em]e2.north) -- ([yshift=-0.1em]Lstm1.south);
\draw [->] ([yshift=0.1em]e2.north) -- ([yshift=-0.1em]Lstm7.south);
\draw [->] ([yshift=0.1em]e3.north) -- ([yshift=-0.1em]Lstm2.south);
\draw [->] ([yshift=0.1em]e3.north) -- ([yshift=-0.1em]Lstm8.south);
\draw [->] ([yshift=0.1em]Lstm3.north) -- ([xshift=-0.05em,yshift=-0.1em]t1.south);
\draw [->] ([yshift=0.1em]Lstm9.north) -- ([yshift=-0.1em]t1.south);
\draw [->] ([yshift=0.1em]Lstm4.north) -- ([xshift=-0.05em,yshift=-0.1em]t2.south);
\draw [->] ([yshift=0.1em]Lstm10.north) -- ([yshift=-0.1em]t2.south);
\draw [->] ([yshift=0.1em]Lstm5.north) -- ([xshift=-0.05em,yshift=-0.1em]t3.south);
\draw [->] ([yshift=0.1em]Lstm11.north) -- ([yshift=-0.1em]t3.south);
\end{scope}
\end{tikzpicture}
%%%------------------------------------------------------------------------------------------------------------
Book/Chapter5/Figures/fig-embedding-matrix.tex 0 → 100644
%%%------------------------------------------------------------------------------------------------------------
\begin{tikzpicture}
\begin{scope}
\node [anchor=center,inner sep=2pt] (e) at (0,0) {\small{$e=w$}};
\node [anchor=west,inner sep=2pt] (c) at (e.east) {\small{$\textbf{C}$}};
\begin{pgfonlayer}{background}
\node [rectangle,inner sep=0.4em,draw,fill=blue!20!white] [fit = (e) (c)] (box) {};
\end{pgfonlayer}
\draw [->,thick] ([yshift=-1em]box.south)--([yshift=-0.1em]box.south) node [pos=0,below] (bottom1) {\small{单词$w$}};
\draw [->,thick] ([yshift=0.1em]box.north)--([yshift=1em]box.north) node [pos=1,above] (top1) {\scriptsize{$e$=(8,.2,-1,.9,...,1)}};
\node [anchor=north] (bottom2) at ([yshift=0.3em]bottom1.south) {\scriptsize{$w$=(0,0,1,0,...,0)}};
\node [anchor=south] (top2) at ([yshift=-0.3em]top1.north) {\small{$w$的分布式表示}};
{
\node [anchor=north west,fill=red!20!white] (cmatrix) at ([xshift=3em,yshift=1.0em]c.north east) {\scriptsize{$\begin{pmatrix} 1 & .2 & -.2 & 8 & ... & 0 \\ .6 & .8 & -2 & 1 & ... & -.2 \\ 8 & .2 & -1 & .9 & ... & 2.3 \\ 1 & 1.2 & -.9 & 3 & ... & .2 \\ ... & ... & ... & ... & ... & ... \\ 1 & .3 & 3 & .9 & ... & 5.1 \end{pmatrix}$}};
\node [anchor=west,inner sep=2pt,fill=red!30!white] (c) at (e.east) {\small{$\textbf{C}$}};
\draw [<-,thick] (c.east) -- ([xshift=3em]c.east);
}
{
\node [anchor=south,draw,fill=green!20!white] (e2) at ([yshift=1.5em]cmatrix.north) {\scriptsize{外部词嵌入系统得到的$\textbf{C}$}};
\draw [->,very thick,dashed] (e2.south) -- (cmatrix.north);
}
\end{scope}
\end{tikzpicture}
%%%------------------------------------------------------------------------------------------------------------
Book/Chapter5/Figures/fig-embedding.tex 0 → 100644
%%%------------------------------------------------------------------------------------------------------------
\begin{tikzpicture}
{
\begin{scope}[xshift=2in]
\node [anchor=north west] (o1) at (0,0) {\footnotesize{$\begin{bmatrix} .1 \\ -1 \\ 2 \\ ... \\ 0 \end{bmatrix}$}};
\node [anchor=north west] (o2) at ([xshift=1em]o1.north east) {\footnotesize{$\begin{bmatrix} 1 \\ 2 \\ .2 \\ ... \\ -1 \end{bmatrix}$}};
\node [anchor=north east] (v) at ([xshift=-0em]o1.north west) {\footnotesize{$\begin{matrix} \textrm{\ \ \ 属性}_1 \\ \textrm{\ \ \ 属性}_2 \\ \textrm{\ \ \ 属性}_3 \\ ... \\ \textrm{属性}_{512} \end{matrix}$}};
\node [anchor=south] (w1) at (o1.north) {\footnotesize{桌子}};
\node [anchor=south] (w2) at (o2.north) {\footnotesize{椅子}};
{
\node [anchor=south,fill=red!20!white] (cosine) at (w1.north) {\footnotesize{$\textrm{cosine}(\textrm{`桌子'},\textrm{`椅子'})=0.5$}};
}
\end{scope}
}
\end{tikzpicture}
%%%------------------------------------------------------------------------------------------------------------
Book/Chapter5/Figures/fig-fit.tex 0 → 100644
%%%------------------------------------------------------------------------------------------------------------
\begin{tikzpicture}
%% a two-layer neural network
\begin{scope}
\tikzstyle{neuronnode} = [minimum size=1.7em,circle,draw,ublue,very thick,inner sep=1pt, fill=white,align=center,drop shadow={shadow xshift=0.1em,shadow yshift=-0.1em}]
%% input and hidden layers
\node [neuronnode] (n10) at (0,0) {\tiny{$f$}\\[-1ex] \tiny{$\sum$}};
\node [neuronnode] (n11) at (1.4,0) {\tiny{$f$}\\[-1ex] \tiny{$\sum$}};
\draw [-,ublue] (n10.west) -- (n10.east);
\draw [-,ublue] (n11.west) -- (n11.east);
\node [anchor=north] (x1) at ([yshift=-6em]n11.south) {$x_1$};
\node [anchor=north] (labela) at ([xshift=3.5em,yshift=-0.5em]x1.south) {\footnotesize{(a)}};
\node [anchor=north] (b) at ([yshift=-6em]n10.south) {$b$};
{
\draw [->,thick,red] (b.north) -- ([yshift=-0.1em]n10.south);
\draw [->,thick,ugreen] (x1.north) -- ([yshift=-0.1em]n10.290);
}
{
\draw [->,thick,blue] (b.north) -- ([yshift=-0.1em]n11.250);
\draw [->,thick,purple] (x1.north) -- ([yshift=-0.1em]n11.south);
}
%% output layers
\node [neuronnode] (n20) at (0.7,5em) {\scriptsize{$\sum$}};
{\draw [->,thick,brown] ([yshift=0.1em]n10.north) -- ([yshift=-0.1em]n20.250);}
{\draw [->,thick,orange] ([yshift=0.1em]n11.north) -- ([yshift=-0.1em]n20.290);}
\node [] (y) at ([yshift=3em]n20.north) {$y$};
\draw [->,thick] ([yshift=0.1em]n20.north) -- (y.south);
%% weight and bias
{\node [anchor=center,rotate=90,fill=white,inner sep=1pt] (b0) at ([yshift=3em,xshift=-0.5em]b.north) {\tiny{$b=-6$}};}
{\node [anchor=center,rotate=-59,fill=white,inner sep=1pt] (w2) at ([yshift=1.2em,xshift=-1.2em]x1.north) {\tiny{$w=100$}};}
{\node [anchor=center,rotate=62,fill=white,inner sep=1pt] (w21) at ([yshift=2em,xshift=0.5em]n10.north) {\tiny{$w'=-0.7$}};}
{\node [anchor=center,rotate=-62,fill=white,inner sep=1pt] (w22) at ([yshift=2em,xshift=-0.5em]n11.north) {\tiny{$w'=0.7$}};}
{\node [anchor=center,rotate=59,fill=white,inner sep=1pt] (b1) at ([yshift=4.9em,xshift=2.2em]b.north) {\tiny{$b=-4$}};}
{\node [anchor=center,rotate=90,fill=white,inner sep=1pt] (w1) at ([yshift=3em,xshift=0.5em]x1.north) {\tiny{$w=100$}};}
%% sigmoid box
\begin{scope}
{
\node [anchor=west] (flabel) at ([xshift=0.5in]y.east) {\scriptsize{sigmoid:}};
\node [anchor=north east] (slabel) at ([xshift=0]flabel.south east) {\scriptsize{sum:}};
\node [anchor=west,inner sep=2pt] (flabel2) at (flabel.east) {\scriptsize{$f(s)=1/(1+e^{-s})$}};
\node [anchor=west,inner sep=2pt] (flabel3) at (slabel.east) {\scriptsize{$s=x_1 \cdot w + b$}};
\draw [->,thick,dotted] ([yshift=-0.3em,xshift=-0.1em]n11.60) .. controls +(east:1) and +(west:2) .. ([xshift=-0.2em]flabel.west) ;
\begin{pgfonlayer}{background}
{
\node [rectangle,inner sep=0.2em,fill=blue!20,drop shadow={shadow xshift=0.1em,shadow yshift=-0.1em}] [fit = (flabel) (flabel2) (flabel3)] (funcbox) {};
}
\end{pgfonlayer}
}
\end{scope}
%% output illustration
\begin{scope}[xshift=1.6in,yshift=0.1in]
{
\draw [->,thick] (-1.6,0) -- (1.6,0);
\draw [->,thick] (0,0) -- (0,2);
\draw [-] (-0.05,1) -- (0.05,1);
\node [anchor=east,inner sep=1pt] (label1) at (0,1) {\tiny{1}};
\node [anchor=south east,inner sep=1pt] (label2) at (0,0) {\tiny{0}};
}
{\draw [->,dashed] (0.6,-0.05) -- (0.6,-0.75in);}
{\draw [-,very thick,ublue,rounded corners=0.1em] (-1.5,0) -- (0.5,0) -- (0.5,0.7) -- (0.7,0.7) -- (0.7,0) -- (1.5,0);}
\end{scope}
\begin{scope}[xshift=1.6in,yshift=-1.0in]
{
\draw [->,thick] (-1.6,0) -- (1.6,0);
\draw [->,thick] (0,0) -- (0,2);
\draw [-,very thick,red,domain=-1.5:1.5,samples=100] plot (\x,{0.2 * (\x +0.4)^3 + 1.2 - 0.3 *(\x + 0.8)^2});
}
{
\foreach \n in {0.5}{
\pgfmathsetmacro{\result}{0.2 * (\n + 0.1 + 0.4)^3 + 1.2 - 0.3 *(\n + 0.1 + 0.8)^2};
\draw [-,ublue,thick] (\n,0) -- (\n, \result) -- (\n + 0.2, \result) -- (\n + 0.2, 0);
}
}
\end{scope}
\end{scope}
%----------------------------------------------------------------------------------------
\begin{scope}[xshift=2.6in]
\tikzstyle{neuronnode} = [minimum size=1.7em,circle,draw,ublue,very thick,inner sep=1pt, fill=white,align=center,drop shadow={shadow xshift=0.1em,shadow yshift=-0.1em}]
%% input and hidden layers
\node [neuronnode] (n10) at (0,0) {\tiny{$f$}\\[-1ex] \tiny{$\sum$}};
\node [neuronnode] (n11) at (1.4,0) {\tiny{$f$}\\[-1ex] \tiny{$\sum$}};
\draw [-,ublue] (n10.west) -- (n10.east);
\draw [-,ublue] (n11.west) -- (n11.east);
\node [anchor=north] (x1) at ([yshift=-6em]n11.south) {$x_1$};
\node [anchor=north] (labelb) at ([xshift=6em,yshift=-0.5em]x1.south) {\footnotesize{(b)}};
\node [anchor=north] (b) at ([yshift=-6em]n10.south) {$b$};
{
\draw [->,thick,red] (b.north) -- ([yshift=-0.1em]n10.south);
\draw [->,thick,ugreen] (x1.north) -- ([yshift=-0.1em]n10.290);
}
{
\draw [->,thick,blue] (b.north) -- ([yshift=-0.1em]n11.250);
\draw [->,thick,purple] (x1.north) -- ([yshift=-0.1em]n11.south);
}
{
\node [neuronnode] (n12) at (2.5,0) {\tiny{$f$}\\[-1ex] \tiny{$\sum$}};
\node [neuronnode] (n13) at (3.4,0) {\tiny{$f$}\\[-1ex] \tiny{$\sum$}};
\draw [-,ublue] (n12.west) -- (n12.east);
\draw [-,ublue] (n13.west) -- (n13.east);
\draw [->,thick] (b.north) -- ([yshift=-0.1em]n12.250);
\draw [->,thick] (x1.north) -- ([yshift=-0.1em]n12.270);
\draw [->,thick] (b.north) -- ([yshift=-0.1em]n13.230);
\draw [->,thick] (x1.north) -- ([yshift=-0.1em]n13.250);
}
%% output layers
\node [neuronnode] (n20) at (0.7,5em) {\scriptsize{$\sum$}};
{\draw [->,thick,brown] ([yshift=0.1em]n10.north) -- ([yshift=-0.1em]n20.250);}
{\draw [->,thick,orange] ([yshift=0.1em]n11.north) -- ([yshift=-0.1em]n20.290);}
\node [] (y) at ([yshift=3em]n20.north) {$y$};
\draw [->,thick] ([yshift=0.1em]n20.north) -- (y.south);
{
\draw [->,thick] ([yshift=0.1em]n12.north) -- ([yshift=-0.1em]n20.310);
\draw [->,thick] ([yshift=0.1em]n13.north) -- ([yshift=-0.1em]n20.330);
}
%% weight and bias
{\node [anchor=center,rotate=90,fill=white,inner sep=1pt] (b0) at ([yshift=3em,xshift=-0.5em]b.north) {\tiny{$b=-6$}};}
{\node [anchor=center,rotate=-59,fill=white,inner sep=1pt] (w2) at ([yshift=1.2em,xshift=-1.2em]x1.north) {\tiny{$w=100$}};}
{\node [anchor=center,rotate=59,fill=white,inner sep=1pt] (b1) at ([yshift=4.9em,xshift=2.2em]b.north) {\tiny{$b=-4$}};}
{\node [anchor=center,rotate=90,fill=white,inner sep=1pt] (w1) at ([yshift=3em,xshift=0.5em]x1.north) {\tiny{$w=100$}};}
{\node [anchor=center,rotate=62,fill=white,inner sep=1pt] (w21) at ([yshift=2em,xshift=0.5em]n10.north) {\tiny{$w'=-0.7$}};}
{\node [anchor=center,rotate=-62,fill=white,inner sep=1pt] (w22) at ([yshift=2em,xshift=-0.5em]n11.north) {\tiny{$w'=0.7$}};}
%% sigmoid box
\begin{scope}
{
\node [anchor=west] (flabel) at ([xshift=0.8in]y.east) {\scriptsize{sigmoid:}};
\node [anchor=north east] (slabel) at ([xshift=0]flabel.south east) {\scriptsize{sum:}};
\node [anchor=west,inner sep=2pt] (flabel2) at (flabel.east) {\scriptsize{$f(s)=1/(1+e^{-s})$}};
\node [anchor=west,inner sep=2pt] (flabel3) at (slabel.east) {\scriptsize{$s=x_1 \cdot w + b$}};
\draw [->,thick,dotted] ([yshift=-0.3em,xshift=-0.1em]n11.60) .. controls +(east:1) and +(west:2) .. ([xshift=-0.2em]flabel.west) ;
\begin{pgfonlayer}{background}
{
\node [rectangle,inner sep=0.2em,fill=blue!20,drop shadow={shadow xshift=0.1em,shadow yshift=-0.1em}] [fit = (flabel) (flabel2) (flabel3)] (funcbox) {};
}
\end{pgfonlayer}
}
\end{scope}
%% output illustration
\begin{scope}[xshift=2.1in,yshift=0.1in]
\draw [->,thick] (-1.6,0) -- (1.6,0);
\draw [->,thick] (0,0) -- (0,2);
\draw [-] (-0.05,1) -- (0.05,1);
{\draw [-,very thick,ublue,rounded corners=0.1em] (-1.5,0) -- (0.5,0) -- (0.5,0.7) -- (0.7,0.7) -- (0.7,0) -- (1.5,0);}
{\draw [-,very thick,ublue,rounded corners=0.1em] (-1.5,0) -- (0.7,0) -- (0.7,0.6) -- (0.9,0.6) -- (0.9,0) -- (1.5,0);}
{\draw [->,dashed] (0.8,-0.05) -- (0.8,-0.78in);}
\end{scope}
\begin{scope}[xshift=2.1in,yshift=-1.0in]
{
\draw [->,thick] (-1.6,0) -- (1.6,0);
\draw [->,thick] (0,0) -- (0,2);
\draw [-,very thick,red,domain=-1.5:1.5,samples=100] plot (\x,{0.2 * (\x +0.4)^3 + 1.2 - 0.3 *(\x + 0.8)^2});
}
\foreach \n in {0.5}{
\pgfmathsetmacro{\result}{0.2 * (\n + 0.1 + 0.4)^3 + 1.2 - 0.3 *(\n + 0.1 + 0.8)^2};
\draw [-,ublue,thick] (\n,0) -- (\n, \result) -- (\n + 0.2, \result) -- (\n + 0.2, 0);
}
{
\foreach \n in {0.7}{
\pgfmathsetmacro{\result}{0.2 * (\n + 0.1 + 0.4)^3 + 1.2 - 0.3 *(\n + 0.1 + 0.8)^2};
\draw [-,ublue,thick] (\n,0) -- (\n, \result) -- (\n + 0.2, \result) -- (\n + 0.2, 0);
}
}
\end{scope}
\end{scope}
\end{tikzpicture}
%%%------------------------------------------------------------------------------------------------------------
Book/Chapter5/Figures/fig-forward-propagation-hid.tex 0 → 100644
%%%------------------------------------------------------------------------------------------------------------
\begin{tikzpicture}
\begin{scope}
\node [anchor=center,draw,fill=red!20,minimum height=1.8em,minimum width=2.5em] (h) at (0,0) {$\textbf{h}^{k-1}$};
\node [anchor=west,draw,fill=blue!20,minimum height=1.8em,minimum width=2.5em] (s) at ([xshift=6em]h.east) {$\textbf{s}^{k}$};
\node [anchor=west,draw,fill=green!20,minimum height=1.8em,minimum width=2.5em] (h2) at ([xshift=6em]s.east) {$\textbf{h}^{k}$};
\node [anchor=east] (prev) at ([xshift=-2em]h.west) {...};
\node [anchor=west] (next) at ([xshift=2em]h2.east) {...};
\draw [->,thick] ([xshift=0.1em]prev.east) -- ([xshift=-0.1em]h.west);
\draw [->,thick] ([xshift=0.1em]h.east) -- ([xshift=-0.1em]s.west) node [pos=0.5,below] {\tiny{$\textbf{s}^k = \textbf{h}^{k-1}\textbf{w}^k$}};
\draw [->,thick] ([xshift=0.1em]s.east) -- ([xshift=-0.1em]h2.west) node [pos=0.5,below] {\tiny{$\textbf{h}^k = f(\textbf{s}^{k})$}};
\draw [->,thick] ([xshift=0.1em]h2.east) -- ([xshift=-0.1em]next.west);
\end{scope}
\end{tikzpicture}
%%%------------------------------------------------------------------------------------------------------------
Book/Chapter5/Figures/fig-forward-propagation-output.tex 0 → 100644
%%%------------------------------------------------------------------------------------------------------------
\begin{tikzpicture}
\begin{scope}
\node [anchor=center,minimum height=1.7em,fill=yellow!20,draw] (h) at (0,0) {$\textbf{h}^{K-1}$};
\node [anchor=west,minimum height=1.7em,fill=blue!20,draw] (s) at ([xshift=5.5em]h.east) {$\textbf{s}^{K}$};
\node [anchor=west,minimum height=1.7em,fill=green!20,draw] (h2) at ([xshift=5.5em]s.east) {$\textbf{h}^{K}$};
\node [anchor=west,minimum height=1.7em,fill=orange!20,draw] (l) at ([xshift=5.5em]h2.east) {$L$};
\draw [->] (h.east) -- (s.west);
\draw [->] (s.east) -- (h2.west);
\draw [->] (h2.east) -- (l.west) node [pos=0.5,above] {\tiny{损失}};
\node [anchor=south west,inner sep=2pt] (step100) at ([xshift=0.5em,yshift=-0.8em]h.north east) {\tiny{$\textbf{s}^K = \textbf{h}^{K-1} \textbf{w}^K$}};
\node [anchor=south west,inner sep=2pt] (step101) at (step100.north west) {\tiny{线性变换}};
\node [anchor=south west,inner sep=2pt] (step200) at ([xshift=0.5em,yshift=-0.8em]s.north east) {\tiny{$\textbf{h}^K = f^K(\textbf{s}^K)$}};
\node [anchor=south west,inner sep=2pt] (step201) at (step200.north west) {\tiny{激活函数}};
\node [anchor=south,inner sep=1pt] (outputlabel) at ([yshift=0.0em]h2.north) {\tiny{\textbf{输出层}}};
{
\draw[decorate,thick,decoration={brace,mirror,raise=0.4em,amplitude=2mm}] (h.south west) -- (s.south west) node [pos=0.5,below,yshift=-1em] {\scriptsize{\textbf{第一阶段：线性变换}}};
}
{
\draw[decorate,thick,decoration={brace,mirror,raise=0.4em,amplitude=2mm}] ([xshift=0.2em]s.south west) -- (l.south east) node [pos=0.5,below,yshift=-1em] (step2) {\scriptsize{\textbf{第二阶段：激活函数+损失函数}}};
}
\end{scope}
\end{tikzpicture}
%%%------------------------------------------------------------------------------------------------------------
Book/Chapter5/Figures/fig-forward-propagation.tex 0 → 100644
%%%------------------------------------------------------------------------------------------------------------
\begin{tikzpicture}
\begin{scope}
\tikzstyle{layernode} = [draw,thick,fill=ugreen!30!white,blur shadow={shadow xshift=1pt,shadow yshift=-1pt}];
\node [anchor=center,layernode,minimum height=4em,minimum width=1em] (layer01) at (0,0) {};
\node [anchor=north west,layernode,minimum height=3em,minimum width=1em] (layer02) at ([xshift=3em]layer01.north east) {};
\node [anchor=south west,layernode,minimum height=3em,minimum width=1em] (layer03) at ([xshift=7em]layer01.south east) {};
\node [anchor=south west,layernode,minimum height=4em,minimum width=1em] (layer04) at ([xshift=11em]layer01.south east) {};
\node [anchor=south west,layernode,minimum height=4em,minimum width=1em] (layer05) at ([xshift=3em]layer04.south east) {};
\node [anchor=east] (input) at ([xshift=-1em]layer01.west){\scriptsize{输入}};
\node [anchor=west] (output) at ([xshift=1em]layer05.east){\scriptsize{输出}};
{
\draw [->,very thick,ublue] ([xshift=-1em]layer01.west) -- ([xshift=-0.1em]layer01.west)node [pos=0.5,above] {\tiny{①}};;
}
{
\draw [->,very thick,ublue] ([xshift=0.1em,yshift=-0.5em]layer01.north east) -- ([xshift=-0.1em,yshift=-0.5em]layer02.north west)node [pos=0.5,above] {\tiny{②}};
}
{
\draw [->,very thick,ublue] ([xshift=0.1em,yshift=0.5em]layer01.south east) -- ([xshift=-0.1em,yshift=0.5em]layer03.south west)node [pos=0.5,below] {\tiny{③}};
}
{
\draw [->,very thick,ublue] ([xshift=0.1em,yshift=-0.5em]layer02.north east) -- ([xshift=-0.1em,yshift=-0.5em]layer04.north west)node [pos=0.5,above] {\tiny{④}};
\draw [->,very thick,ublue] ([xshift=0.1em,yshift=0.5em]layer03.south east) -- ([xshift=-0.1em,yshift=0.5em]layer04.south west)node [pos=0.5,below] {\tiny{⑤}};
\draw [->,very thick,ublue] ([xshift=0.1em]layer04.east) -- ([xshift=-0.1em]layer05.west)node [pos=0.5,above] {\tiny{⑥}};
\draw [->,very thick,ublue] ([xshift=0.1em]layer05.east) -- ([xshift=1.0em]layer05.east)node [pos=0.5,above] {\tiny{⑦}};
}
{
\draw [<-,thin] ([xshift=0.3em,yshift=0.3em]layer04.east) .. controls +(35:1) and +(215:1) .. ([xshift=-2em,yshift=0.3em]layer05.north west) node [pos=1,above] {\scriptsize{前向：层$i$ 的输出$h_{i}$}};
}
\end{scope}
\end{tikzpicture}
%%%------------------------------------------------------------------------------------------------------------
Book/Chapter5/Figures/fig-four-layers-of-neural-network.tex 0 → 100644
%%%------------------------------------------------------------------------------------------------------------
\begin{tikzpicture}
\begin{scope}[]
\def\neuronsep{1.3}
\tikzstyle{neuronnode} = [minimum size=1.7em,circle,draw,ublue,very thick,inner sep=1pt, fill=white,align=center,drop shadow={shadow xshift=0.1em,shadow yshift=-0.1em}]
%%% layer 1
\foreach \n in {1,...,5}{
\node [neuronnode] (neuron0\n) at (\n * \neuronsep,0) {\tiny{$f_1$}\\[-1ex] \tiny{$\sum$}};
\draw [-,ublue] (neuron0\n.east) -- (neuron0\n.west);
}
\foreach \n in {1,...,5}{
\foreach \m in {1,...,5}{
\draw [<-] (neuron0\m.south) -- ([yshift=-1.8em]neuron0\n.south);
}
\node [anchor=north] (x\n) at ([yshift=-1.8em]neuron0\n.south) {$x_\n$};
}
\node [anchor=west] (w1label) at ([xshift=-0.5em,yshift=0.5em]x5.north east) {$\textbf{w}_1$};
\begin{pgfonlayer}{background}
\node [rectangle,inner sep=0.2em,fill=red!20] [fit = (neuron01) (neuron05)] (layer01) {};
\end{pgfonlayer}
\node [anchor=west] (layer00label) at ([xshift=1.25em]x5.east) {\footnotesize{\red{{输入层}}}};
{
\node [anchor=west] (layer01label) at ([xshift=1em]layer01.east) {\footnotesize{第二层}};
}
{
\node [anchor=west] (layer01label2) at (layer01label.east) {\footnotesize{\red{({隐层})}}};
}
%%% layer 2
{
\foreach \n in {2,...,4}{
\node [neuronnode] (neuron1\n) at (\n * \neuronsep,3.5em) {\tiny{$f_2$}\\[-1ex] \tiny{$\sum$}};
\draw [-,ublue] (neuron1\n.east) -- (neuron1\n.west);
}
\foreach \n in {2,...,4}{
\foreach \m in {1,...,5}{
\draw [<-] (neuron1\n.south) -- (neuron0\m.north);
}
}
\node [anchor=west] (w2label) at ([xshift=-2.5em,yshift=4.6em]x5.north east) {$\textbf{w}_2$};
\begin{pgfonlayer}{background}
{
\node [rectangle,inner sep=0.2em,fill=ugreen!20] [fit = (neuron12) (neuron14)] (layer02) {};
}
\end{pgfonlayer}
\node [anchor=west] (layer02label) at ([xshift=4.5em]layer02.east) {\footnotesize{第三层}};
{
\node [anchor=west] (layer02label2) at (layer02label.east) {\footnotesize{\red{({隐层})}}};
}
}
%%% layer 3
{
\foreach \n in {1,...,5}{
\node [neuronnode] (neuron2\n) at (\n * \neuronsep,7em) {\tiny{$f_3$}\\[-1ex] \tiny{$\sum$}};
\draw [-,ublue] (neuron2\n.east) -- (neuron2\n.west);
}
\foreach \n in {1,...,5}{
\foreach \m in {2,...,4}{
\draw [<-] (neuron2\n.south) -- (neuron1\m.north);
}
\node [anchor=south] (y\n) at ([yshift=1.2em]neuron2\n.north) {$y_\n$};
\draw [<-,thick] ([yshift=1.1em]neuron2\n.north) -- (neuron2\n.north);
}
\node [anchor=west] (w3label) at ([xshift=-2.5em,yshift=7.5em]x5.north east) {$\textbf{w}_3$};
\begin{pgfonlayer}{background}
{
\node [rectangle,inner sep=0.2em,fill=blue!20] [fit = (neuron21) (neuron25)] (layer03) {};
}
\end{pgfonlayer}
\node [anchor=west] (layer03label) at ([xshift=1em]layer03.east) {\footnotesize{第四层}};
{
\node [anchor=west] (layer03label2) at (layer03label.east) {\footnotesize{\red{({输出层})}}};
}
}
\end{scope}
\end{tikzpicture}
%%%------------------------------------------------------------------------------------------------------------
Book/Chapter5/Figures/fig-gaussian.tex 0 → 100644
%%%------------------------------------------------------------------------------------------------------------
\begin{tikzpicture}
\draw[->](-1.2,0)--(1.2,0)node[left,below,font=\tiny]{$x$};
\draw[->](0,-1.2)--(0,1.2)node[right,font=\tiny]{$y$};
\draw[dashed](-1.2,1)--(1.2,1);
\foreach \x in {-1.0,-0.5,0.0,0.5,1.0}{\draw(\x,0)--(\x,0.05)node[below,outer sep=2pt,font=\tiny]at(\x,0){\x};}
\foreach \y in {0.5,1.0}{\draw(0,\y)--(0.05,\y)node[left,outer sep=2pt,font=\tiny]at(0,\y){\y};}
\draw[color=red ,domain=-1.2:1.2]plot(\x,{exp(-1*((\x)^2))});
\node[black,anchor=south] at (0,1.2) {\small $y =e^{-x^2}$};
\end{tikzpicture}
%%%------------------------------------------------------------------------------------------------------------
Book/Chapter5/Figures/fig-gpt.tex 0 → 100644
%%%------------------------------------------------------------------------------------------------------------
\begin{tikzpicture}
\begin{scope}[scale=1.2]
\node [anchor=west,draw,inner sep=4pt,fill=blue!20!white,minimum width=3em] (Trm0) at (0,0) {\scriptsize{TRM}};
\node [anchor=west,draw,inner sep=4pt,fill=blue!20!white,minimum width=3em] (Trm1) at ([xshift=1em]Trm0.east) {\scriptsize{TRM}};
\node [anchor=west,draw,inner sep=4pt,fill=blue!20!white,minimum width=3em] (Trm2) at ([xshift=1em]Trm1.east) {\scriptsize{TRM}};
\node [anchor=west,draw,inner sep=4pt,fill=blue!20!white,minimum width=3em] (Trm3) at ([xshift=1em]Trm2.east) {\scriptsize{TRM}};
\node [anchor=west,inner sep=4pt] (sep) at ([xshift=1em]Trm3.east) {\scriptsize{...}};
\node [anchor=west,draw,inner sep=4pt,fill=blue!20!white,minimum width=3em] (Trm4) at ([xshift=1em]sep.east) {\scriptsize{TRM}};
\node [anchor=south,draw,inner sep=4pt,fill=blue!20!white,minimum width=3em] (Trm5) at ([yshift=1em]Trm0.north) {\scriptsize{TRM}};
\node [anchor=west,draw,inner sep=4pt,fill=blue!20!white,minimum width=3em] (Trm6) at ([xshift=1em]Trm5.east) {\scriptsize{TRM}};
\node [anchor=west,draw,inner sep=4pt,fill=blue!20!white,minimum width=3em] (Trm7) at ([xshift=1em]Trm6.east) {\scriptsize{TRM}};
\node [anchor=west,draw,inner sep=4pt,fill=blue!20!white,minimum width=3em] (Trm8) at ([xshift=1em]Trm7.east) {\scriptsize{TRM}};
\node [anchor=west,inner sep=4pt] (sep1) at ([xshift=1em]Trm8.east) {\scriptsize{...}};
\node [anchor=west,draw,inner sep=4pt,fill=blue!20!white,minimum width=3em] (Trm9) at ([xshift=1em]sep1.east) {\scriptsize{TRM}};
\node [rectangle,rounded corners,draw=black!50,densely dashed,inner sep=0.4em] [fit = (Trm0) (Trm4) (Trm5) (Trm9)] (inputshadow) {};
\node [anchor=north,draw,inner sep=4pt,fill=ugreen!20!white,minimum width=2em] (e1) at ([yshift=-1em]Trm0.south) {\scriptsize{$\textbf{e}_1$}};
\node [anchor=north,draw,inner sep=4pt,fill=ugreen!20!white,minimum width=2em] (e2) at ([yshift=-1em]Trm1.south) {\scriptsize{$\textbf{e}_2$}};
\node [anchor=north,draw,inner sep=4pt,fill=ugreen!20!white,minimum width=2em] (e3) at ([yshift=-1em]Trm2.south) {\scriptsize{$\textbf{e}_3$}};
\node [anchor=north,draw,inner sep=4pt,fill=ugreen!20!white,minimum width=2em] (e4) at ([yshift=-1em]Trm3.south) {\scriptsize{$\textbf{e}_4$}};
\node [anchor=north,inner sep=4pt] (sep5) at ([yshift=-1em]sep.south) {\scriptsize{...}};
\node [anchor=north,draw,inner sep=4pt,fill=ugreen!20!white,minimum width=2em] (e5) at ([yshift=-1em]Trm4.south) {\scriptsize{$\textbf{e}_m$}};
\node [anchor=south,draw,inner sep=4pt,fill=yellow!30,minimum width=2em] (t1) at ([yshift=1em]Trm5.north) {\scriptsize{$\textbf{h}_1$}};
\node [anchor=south,draw,inner sep=4pt,fill=yellow!30,minimum width=2em] (t2) at ([yshift=1em]Trm6.north) {\scriptsize{$\textbf{h}_2$}};
\node [anchor=south,draw,inner sep=4pt,fill=yellow!30,minimum width=2em] (t3) at ([yshift=1em]Trm7.north) {\scriptsize{$\textbf{h}_3$}};
\node [anchor=south,draw,inner sep=4pt,fill=yellow!30,minimum width=2em] (t4) at ([yshift=1em]Trm8.north) {\scriptsize{$\textbf{h}_4$}};
\node [anchor=south,inner sep=4pt] (sep6) at ([yshift=1em]sep1.north) {\scriptsize{...}};
\node [anchor=south,draw,inner sep=4pt,fill=yellow!30,minimum width=2em] (t5) at ([yshift=1em]Trm9.north) {\scriptsize{$\textbf{h}_m$}};
\node [anchor=west,draw,inner sep=3pt,fill=blue!20!white,minimum width=1em] (Lt1) at ([yshift=1.5em]t1.west) {\tiny{TRM}};
\node [anchor=west] (Lt2) at ([xshift=-0.1em]Lt1.east) {\tiny{: Transformer}};
\draw [->] ([yshift=0.1em]e1.north) -- ([yshift=-0.1em]Trm0.south);
\draw [->] ([yshift=0.1em]e1.north) -- ([yshift=-0.1em]Trm1.south);
\draw [->] ([yshift=0.1em]e1.north) -- ([yshift=-0.1em]Trm2.south);
\draw [->] ([yshift=0.1em]e1.north) -- ([yshift=-0.1em]Trm3.south);
\draw [->] ([yshift=0.1em]e1.north) -- ([yshift=-0.1em]Trm4.south);
\draw [->] ([yshift=0.1em]e2.north) -- ([yshift=-0.1em]Trm1.south);
\draw [->] ([yshift=0.1em]e3.north) -- ([yshift=-0.1em]Trm2.south);
\draw [->] ([yshift=0.1em]e3.north) -- ([yshift=-0.1em]Trm3.south);
\draw [->] ([yshift=0.1em]e3.north) -- ([yshift=-0.1em]Trm4.south);
\draw [->] ([yshift=0.1em]e4.north) -- ([yshift=-0.1em]Trm3.south);
\draw [->] ([yshift=0.1em]e5.north) -- ([yshift=-0.1em]Trm4.south);
\draw [->] ([yshift=0.1em]Trm0.north) -- ([yshift=-0.1em]Trm5.south);
\draw [->] ([yshift=0.1em]Trm0.north) -- ([yshift=-0.1em]Trm6.south);
\draw [->] ([yshift=0.1em]Trm0.north) -- ([yshift=-0.1em]Trm7.south);
\draw [->] ([yshift=0.1em]Trm0.north) -- ([yshift=-0.1em]Trm8.south);
\draw [->] ([yshift=0.1em]Trm0.north) -- ([yshift=-0.1em]Trm9.south);
\draw [->] ([yshift=0.1em]Trm1.north) -- ([yshift=-0.1em]Trm6.south);
\draw [->] ([yshift=0.1em]Trm2.north) -- ([yshift=-0.1em]Trm7.south);
\draw [->] ([yshift=0.1em]Trm2.north) -- ([yshift=-0.1em]Trm8.south);
\draw [->] ([yshift=0.1em]Trm2.north) -- ([yshift=-0.1em]Trm9.south);
\draw [->] ([yshift=0.1em]Trm3.north) -- ([yshift=-0.1em]Trm8.south);
\draw [->] ([yshift=0.1em]Trm4.north) -- ([yshift=-0.1em]Trm9.south);
\draw [->] ([yshift=0.1em]Trm5.north) -- ([yshift=-0.1em]t1.south);
\draw [->] ([yshift=0.1em]Trm6.north) -- ([yshift=-0.1em]t2.south);
\draw [->] ([yshift=0.1em]Trm7.north) -- ([yshift=-0.1em]t3.south);
\draw [->] ([yshift=0.1em]Trm8.north) -- ([yshift=-0.1em]t4.south);
\draw [->] ([yshift=0.1em]Trm9.north) -- ([yshift=-0.1em]t5.south);
\end{scope}
\end{tikzpicture}
%%%------------------------------------------------------------------------------------------------------------
Book/Chapter5/Figures/fig-gradient-descent.tex 0 → 100644
%%%------------------------------------------------------------------------------------------------------------
\pgfplotsset{
colormap={whitered}{color(-1cm)=(orange!75!red);color(1cm)=(white)}
}
\begin{tikzpicture}[
declare function = {mu1=1;},
declare function = {mu2=2;},
declare function = {sigma1=0.5;},
declare function = {sigma2=1;},
declare function = {normal(\m,\s)=1/(2*\s*sqrt(pi))*exp(-(x-\m)^2/(2*\s^2));},
declare function = {bivar(\ma,\sa,\mb,\sb)=1/(2*pi*\sa*\sb) * exp(-((x-\ma)^2/\sa^2 + (y-\mb)^2/\sb^2))/2;}]
\footnotesize{
{
\begin{scope}
\begin{axis}[
colormap name = whitered,
width = 8cm,
height = 5cm,
view = {20}{45},
enlargelimits = false,
grid = major,
domain = -1:3,
y domain = 0:4,
samples = 30,
xlabel = $\textbf{w}^{[1]}$,
ylabel = $\textbf{w}^{[2]}$,
xlabel style = {xshift=0em,yshift=0.8em},
ylabel style = {xshift=0.2em,yshift=0.8em},
zlabel = {$J(\textbf{w})$},
ztick = {-0.1},
colorbar,
colorbar style = {
at = {(1.2,0.5)},
anchor = north west,
ytick = {0,-0.1},
height = 0.25*\pgfkeysvalueof{/pgfplots/parent axis height},
title = {}
}
]
\addplot3 [surf] {-bivar(mu1,sigma1,mu2,sigma2)};
\node [circle,fill=red,minimum size=3pt,inner sep=1.5pt] () at (axis cs:0.5,2,-0.01) {};
\draw [->,very thick,ublue] (axis cs:0.5,2,-0.01) -- (axis cs:0.8,1.6,-0.03) node [pos=1,right,inner sep=2pt] {\tiny{-$\frac{\partial J(\textbf{w})}{\partial \textbf{w}}$}};
\draw [->,very thick,dotted] (axis cs:0.5,2,-0.01) -- (axis cs:0.2,1.5,-0.03);
\draw [->,very thick,dotted] (axis cs:0.5,2,-0.01) -- (axis cs:0.2,3.5,-0.03);
%\draw [black!50] (axis cs:0,-1,0) -- (axis cs:0,4,0);
\end{axis}
\end{scope}
}
}
\end{tikzpicture}
%%%------------------------------------------------------------------------------------------------------------
Book/Chapter5/Figures/fig-identity.tex 0 → 100644
%%%------------------------------------------------------------------------------------------------------------
\begin{tikzpicture}
\draw[->](-1.2,0)--(1.2,0)node[left,below,font=\tiny]{$x$};
\draw[->](0,-1.2)--(0,1.2)node[right,font=\tiny]{$y$};
\foreach \x in {-1.0,-0.5,0.0,0.5,1.0}{\draw(\x,0)--(\x,0.05)node[below,outer sep=2pt,font=\tiny]at(\x,0){\x};}
\foreach \y in {0.5,1.0}{\draw(0,\y)--(0.05,\y)node[left,outer sep=2pt,font=\tiny]at(0,\y){\y};}
\draw[color=red ,domain=-1:1]plot(\x,\x);
\node[black,anchor=south] at (0,1.2) {\small $y =x$};
\end{tikzpicture}
%%%------------------------------------------------------------------------------------------------------------
Book/Chapter5/Figures/fig-linear-transformation.tex 0 → 100644
%%%------------------------------------------------------------------------------------------------------------
\begin{footnotesize}
$$
\begin{smallmatrix} \underbrace{
\left\{
\begin{smallmatrix}
\left[
\begin{array}{cccc}
1& 0 &0 \\
0& 1 &0 \\
0& 0 &1
\end{array}
\right ]
\cdots
\left[
\begin{array}{cccc}
1& 0 &0 \\
0& 1 &0 \\
0& 0 &1
\end{array}
\right]
\end{smallmatrix}
\right\}
}\\5
\end{smallmatrix}
\times
\begin{smallmatrix}
\left[
\begin{array}{cccc}
1\\
1\\
1
\end{array}
\right ]
\end{smallmatrix}
=
\begin{smallmatrix} \underbrace{
\left\{
\begin{smallmatrix}
\left[
\begin{array}{cccc}
1 \\
1 \\
1
\end{array}
\right ]
\cdots
\left[
\begin{array}{cccc}
1 \\
1 \\
1
\end{array}
\right]
\end{smallmatrix}
\right\}
}\\5
\end{smallmatrix}
$$
\end{footnotesize}
%%%------------------------------------------------------------------------------------------------------------
Book/Chapter5/Figures/fig-model-training.tex 0 → 100644
%%%------------------------------------------------------------------------------------------------------------
\begin{tikzpicture}
\begin{scope}
\node [anchor=west,draw,thick,minimum width=4em,minimum height=1.7em,fill=blue!20] (encoder) at (0,0) {模块};
\node [anchor=south,minimum width=4em,minimum height=1.7em] (space) at ([yshift=0.3em]encoder.north) {\footnotesize{目标系统}};
\begin{pgfonlayer}{background}
\node [rectangle,draw,thick,fill=red!20] [fit = (encoder) (space)] (system) {};
\end{pgfonlayer}
\node [anchor=north] (data) at ([yshift=-1em]system.south) {\scriptsize{\textbf{目标任务有标注数据}}};
\draw [->,thick] (data.north) -- ([yshift=-0.1em]system.south);
\node [anchor=north] (label) at ([yshift=-0em]data.south) {\scriptsize{(a) standard method}};
\end{scope}
\begin{scope}[xshift=2.8in]
\node [anchor=west,draw,dashed,thick,minimum width=4em,minimum height=1.7em,fill=blue!20] (encoder) at (0,0) {模块};
\node [anchor=south,minimum width=4em,minimum height=1.7em] (space) at ([yshift=0.3em]encoder.north) {\footnotesize{目标系统}};
\node [anchor=center,draw,thick,minimum width=4em,minimum height=1.7em,fill=green!20] (encoderpre) at ([xshift=-7em]encoder.center) {\footnotesize{语言模型}};
\draw [->,thick] (encoderpre.east) -- (encoder.west);
\begin{pgfonlayer}{background}
\node [rectangle,draw,thick,fill=red!20] [fit = (encoder) (space)] (system) {};
\end{pgfonlayer}
\node [anchor=north] (data) at ([yshift=-1em]system.south) {\scriptsize{\textbf{目标任务有标注数据}}};
\draw [->,thick] (data.north) -- ([yshift=-0.1em]system.south);
\node [anchor=north] (data2) at ([yshift=-1em,xshift=-7em]system.south) {\scriptsize{\textbf{大规模无标注数据}}};
\draw [->,thick] (data2.north) -- ([yshift=-0.1em]encoderpre.south);
\node [anchor=north] (label) at ([yshift=-0em,xshift=-4em]data.south) {\scriptsize{(b) pre-training + fine-tuning}};
\end{scope}
\end{tikzpicture}
%%%------------------------------------------------------------------------------------------------------------
Book/Chapter5/Figures/fig-more-layers.tex 0 → 100644
%%%------------------------------------------------------------------------------------------------------------
\begin{tikzpicture}
\begin{scope}[]
\def\neuronsep{1.3}
\tikzstyle{neuronnode} = [minimum size=1.7em,circle,draw,ublue,very thick,inner sep=1pt, fill=white,align=center,drop shadow={shadow xshift=0.1em,shadow yshift=-0.1em}]
%%% layer 1
\foreach \n in {1,...,5}{
\node [neuronnode] (neuron0\n) at (\n * \neuronsep,0) {\tiny{$f_1$}\\[-1ex] \tiny{$\sum$}};
\draw [-,ublue] (neuron0\n.east) -- (neuron0\n.west);
}
\foreach \n in {1,...,5}{
\foreach \m in {1,...,5}{
\draw [<-] (neuron0\m.south) -- ([yshift=-1.8em]neuron0\n.south);
}
\node [anchor=north] (x\n) at ([yshift=-1.8em]neuron0\n.south) {$x_\n$};
{
\draw [<-,thick] ([yshift=1.1em]neuron0\n.north) -- (neuron0\n.north);
\node [anchor=south] (y\n) at ([yshift=1.2em]neuron0\n.north) {$y_\n$};
}
}
\node [anchor=west] (w1label) at ([xshift=-0.5em,yshift=0.5em]x5.north east) {$\textbf{w}_1$};
\begin{pgfonlayer}{background}
\node [rectangle,inner sep=0.2em,fill=red!20] [fit = (neuron01) (neuron05)] (layer01) {};
\end{pgfonlayer}
{
\node [anchor=west] (layer01label) at ([xshift=2em]layer01.east) {\footnotesize{单层神经网络}};
}
\node[anchor=north] (arrow1) at ([yshift=0em]x3.south) {};
\draw[fill=blue!20,draw=blue!30]([xshift=-0.20em]arrow1.north west)--([xshift=0.20em]arrow1.north east)--([yshift=0.15em,xshift=0.20em]arrow1.south east)--([yshift=0.15em,xshift=0.8em]arrow1.south east) --([yshift=-0.8em]arrow1.south)--([yshift=0.15em,xshift=-0.8em]arrow1.south west)--([yshift=0.15em,xshift=-0.20em]arrow1.south west)--([xshift=-0.20em]arrow1.north west);
\end{scope}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\begin{scope}[xshift=0.0em,yshift=-12.7em]
\def\neuronsep{1.3}
\tikzstyle{neuronnode} = [minimum size=1.7em,circle,draw,ublue,very thick,inner sep=1pt, fill=white,align=center,drop shadow={shadow xshift=0.1em,shadow yshift=-0.1em}]
%%% layer 1
\foreach \n in {1,...,5}{
\node [neuronnode] (neuron0\n) at (\n * \neuronsep,0) {\tiny{$f_1$}\\[-1ex] \tiny{$\sum$}};
\draw [-,ublue] (neuron0\n.east) -- (neuron0\n.west);
}
\foreach \n in {1,...,5}{
\foreach \m in {1,...,5}{
\draw [<-] (neuron0\m.south) -- ([yshift=-1.8em]neuron0\n.south);
}
\node [anchor=north] (x\n) at ([yshift=-1.8em]neuron0\n.south) {$x_\n$};
}
\node [anchor=west] (w1label) at ([xshift=-0.5em,yshift=0.5em]x5.north east) {$\textbf{w}_1$};
\begin{pgfonlayer}{background}
\node [rectangle,inner sep=0.2em,fill=red!20] [fit = (neuron01) (neuron05)] (layer01) {};
\end{pgfonlayer}
%%% layer 2
\foreach \n in {2,...,4}{
\node [neuronnode] (neuron1\n) at (\n * \neuronsep,3.5em) {\tiny{$f_2$}\\[-1ex] \tiny{$\sum$}};
\draw [-,ublue] (neuron1\n.east) -- (neuron1\n.west);
}
\foreach \n in {2,...,4}{
\foreach \m in {1,...,5}{
\draw [<-] (neuron1\n.south) -- (neuron0\m.north);
}
\draw [<-,thick] ([yshift=1.1em]neuron1\n.north) -- (neuron1\n.north);
\node [anchor=south] (y\n) at ([yshift=1.25em]neuron1\n.north) {$y_\n$};
}
\node [anchor=west] (w2label) at ([xshift=-2.5em,yshift=4.6em]x5.north east) {$\textbf{w}_2$};
\node [anchor=west] (layer02label) at ([xshift=2.7em]w2label.east) {\footnotesize{两层神经网络}};
\begin{pgfonlayer}{background}
{
\node [rectangle,inner sep=0.2em,fill=ugreen!20] [fit = (neuron12) (neuron14)] (layer02) {};
}
\end{pgfonlayer}
\node[anchor=north] (arrow1) at ([yshift=0em]x3.south) {};
\draw[fill=blue!20,draw=blue!30]([xshift=-0.20em]arrow1.north west)--([xshift=0.20em]arrow1.north east)--([yshift=0.15em,xshift=0.20em]arrow1.south east)--([yshift=0.15em,xshift=0.8em]arrow1.south east) --([yshift=-0.8em]arrow1.south)--([yshift=0.15em,xshift=-0.8em]arrow1.south west)--([yshift=0.15em,xshift=-0.20em]arrow1.south west)--([xshift=-0.20em]arrow1.north west);
\end{scope}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\begin{scope}[xshift=0.0em,yshift=-29em]
\def\neuronsep{1.3}
\tikzstyle{neuronnode} = [minimum size=1.7em,circle,draw,ublue,very thick,inner sep=1pt, fill=white,align=center,drop shadow={shadow xshift=0.1em,shadow yshift=-0.1em}]
%%% layer 1
\foreach \n in {1,...,5}{
\node [neuronnode] (neuron0\n) at (\n * \neuronsep,0) {\tiny{$f_1$}\\[-1ex] \tiny{$\sum$}};
\draw [-,ublue] (neuron0\n.east) -- (neuron0\n.west);
}
\foreach \n in {1,...,5}{
\foreach \m in {1,...,5}{
\draw [<-] (neuron0\m.south) -- ([yshift=-1.8em]neuron0\n.south);
}
\node [anchor=north] (x\n) at ([yshift=-1.8em]neuron0\n.south) {$x_\n$};
}
\node [anchor=west] (w1label) at ([xshift=-0.5em,yshift=0.5em]x5.north east) {$\textbf{w}_1$};
\begin{pgfonlayer}{background}
\node [rectangle,inner sep=0.2em,fill=red!20] [fit = (neuron01) (neuron05)] (layer01) {};
\end{pgfonlayer}
%%% layer 2
{
\foreach \n in {2,...,4}{
\node [neuronnode] (neuron1\n) at (\n * \neuronsep,3.5em) {\tiny{$f_2$}\\[-1ex] \tiny{$\sum$}};
\draw [-,ublue] (neuron1\n.east) -- (neuron1\n.west);
}
\foreach \n in {2,...,4}{
\foreach \m in {1,...,5}{
\draw [<-] (neuron1\n.south) -- (neuron0\m.north);
}
}
\node [anchor=west] (w2label) at ([xshift=-2.5em,yshift=4.6em]x5.north east) {$\textbf{w}_2$};
\begin{pgfonlayer}{background}
{
\node [rectangle,inner sep=0.2em,fill=ugreen!20] [fit = (neuron12) (neuron14)] (layer02) {};
}
\end{pgfonlayer}
\node [anchor=west] (layer02label) at ([xshift=5.2em]layer02.east) {\footnotesize{三层神经网络}};
}
%%% layer 3
{
\foreach \n in {1,...,5}{
\node [neuronnode] (neuron2\n) at (\n * \neuronsep,7em) {\tiny{$f_3$}\\[-1ex] \tiny{$\sum$}};
\draw [-,ublue] (neuron2\n.east) -- (neuron2\n.west);
}
\foreach \n in {1,...,5}{
\foreach \m in {2,...,4}{
\draw [<-] (neuron2\n.south) -- (neuron1\m.north);
}
\node [anchor=south] (y\n) at ([yshift=1.25em]neuron2\n.north) {$y_\n$};
\draw [<-,thick] ([yshift=1.1em]neuron2\n.north) -- (neuron2\n.north);
}
\node [anchor=west] (w3label) at ([xshift=-2.5em,yshift=7.5em]x5.north east) {$\textbf{w}_3$};
\begin{pgfonlayer}{background}
{
\node [rectangle,inner sep=0.2em,fill=blue!20] [fit = (neuron21) (neuron25)] (layer03) {};
}
\end{pgfonlayer}
}
\end{scope}
\end{tikzpicture}
%%%------------------------------------------------------------------------------------------------------------
Book/Chapter5/Figures/fig-multilayer-neural-network-example.tex 0 → 100644
%%%------------------------------------------------------------------------------------------------------------
\begin{tikzpicture}
\begin{scope}
\def\neuronsep{1}
\tikzstyle{neuronnode} = [minimum size=1.2em,circle,draw,ublue,very thick,inner sep=1pt, fill=white,align=center,drop shadow={shadow xshift=0.1em,shadow yshift=-0.1em}];
%%% layer 1
\foreach \n in {1,...,4}{
\node [neuronnode] (neuron0\n) at (\n * \neuronsep,0) {};
\draw [->] ([yshift=-0.8em]neuron0\n.south) -- ([yshift=-0.1em]neuron0\n.south) node [pos=0,below] {\tiny{...}};
}
\begin{pgfonlayer}{background}
\node [rectangle,inner sep=0.2em,fill=red!20] [fit = (neuron01) (neuron04)] (layer01) {};
\node [anchor=east] (layer01label) at (layer01.west) {\scriptsize{层$k-1$}};
\end{pgfonlayer}
%%% layer 2
\foreach \n in {1,...,4}{
\node [neuronnode] (neuron1\n) at (\n * \neuronsep,3em) {};
}
\foreach \n in {1,...,4}{
\foreach \m in {1,...,4}{
\draw [<-] (neuron1\n.south) -- (neuron0\m.north);
}
}
\begin{pgfonlayer}{background}
\node [rectangle,inner sep=0.2em,fill=ugreen!20] [fit = (neuron11) (neuron14)] (layer02) {};
\node [anchor=east] (layer02label) at (layer02.west) {\scriptsize{层$k$}};
\end{pgfonlayer}
%%% layer 3
\foreach \n in {1,...,4}{
\node [neuronnode] (neuron2\n) at (\n * \neuronsep,6em) {};
\draw [<-] ([yshift=0.8em]neuron2\n.north) -- ([yshift=0.0em]neuron2\n.north) node [pos=0,above] {\tiny{...}};
}
\foreach \n in {1,...,4}{
\foreach \m in {1,...,4}{
\draw [<-] (neuron2\n.south) -- (neuron1\m.north);
}
}
\begin{pgfonlayer}{background}
\node [rectangle,inner sep=0.2em,fill=blue!20] [fit = (neuron21) (neuron24)] (layer03) {};
\node [anchor=east] (layer03label) at (layer03.west) {\scriptsize{层$k+1$}};
\end{pgfonlayer}
%%% output layer
\foreach \n in {1,...,4}{
\node [neuronnode] (neuron3\n) at (\n * \neuronsep,9.4em) {};
{
\draw [<-] ([yshift=0.6em]neuron3\n.north) -- ([yshift=0.0em]neuron3\n.north) node [pos=0,above] {\tiny{output}};
}
{
\draw [<-,very thick] ([yshift=0.6em]neuron3\n.north) -- ([yshift=0.0em]neuron3\n.north) node [pos=0,above] {\tiny{output}};
}
\draw [->] ([yshift=-0.6em]neuron3\n.south) -- ([yshift=0.0em]neuron3\n.south);
}
\begin{pgfonlayer}{background}
\node [rectangle,inner sep=0.2em,fill=ugreen!20] [fit = (neuron31) (neuron34)] (layer04) {};
\node [anchor=east] (layer04label) at (layer04.west) {\scriptsize{层$K$(输出)}};
\end{pgfonlayer}
{
\node [neuronnode,draw=red,fill=red!20!white,inner sep=1pt] (neuron12new) at (2 * \neuronsep,3em) {};
\node [anchor=east] (neuronsamplelabel) at ([yshift=-1em]layer02label.south east) {{\textbf{\tiny{第$k$层, 第$i$个神经元}}}};
\draw [->,dashed,very thick,red] ([xshift=-0.2em,yshift=0.2em]neuronsamplelabel.east) .. controls +(30:1) and +(220:1) .. ([xshift=-0em,yshift=-0em]neuron12new.210);
}
{
\foreach \n in {1,...,4}{
\draw [<-,thick,red] (neuron2\n.south) -- (neuron12.north);
}
}
{
\draw [<-,thick,red] (neuron14.south) -- (neuron04.north);
\node [anchor=north] (wlabel) at (layer02.south east) {{\scriptsize{$w_{4,4}^{k}$}}};
}
\end{scope}
\end{tikzpicture}
%%%------------------------------------------------------------------------------------------------------------
Book/Chapter5/Figures/fig-one-hot.tex 0 → 100644
%%%------------------------------------------------------------------------------------------------------------
\begin{tikzpicture}
\begin{scope}
\node [anchor=north west] (o1) at (0,0) {\footnotesize{$\begin{bmatrix} 0 \\ 1 \\ 0 \\ 0 \\ 0 \\ ... \\ 0 \end{bmatrix}$}};
\node [anchor=north west] (o2) at ([xshift=1em]o1.north east) {\footnotesize{$\begin{bmatrix} 0 \\ 0 \\ 0 \\ 1 \\ 0 \\ ... \\ 0 \end{bmatrix}$}};
\node [anchor=north east] (v) at ([xshift=-0em]o1.north west) {\footnotesize{$\begin{matrix} \textrm{\ \ \ \ \ 你}_1 \\ \textrm{\ \ 桌子}_2 \\ \textrm{\ \ \ \ \ 他}_3 \\ \textrm{\ \ 椅子}_4 \\ \textrm{\ \ 我们}_5 \\ ... \\ \textrm{你好}_{10k} \end{matrix}$}};
\node [anchor=south] (w1) at (o1.north) {\footnotesize{桌子}};
\node [anchor=south] (w2) at (o2.north) {\footnotesize{椅子}};
{
\node [anchor=south,fill=red!20!white] (cosine) at (w1.north) {\footnotesize{$\textrm{cosine}(\textrm{`桌子'},\textrm{`椅子'})=0$}};
}
\end{scope}
\end{tikzpicture}
%%%------------------------------------------------------------------------------------------------------------
Book/Chapter5/Figures/fig-parallel.tex 0 → 100644
%%%------------------------------------------------------------------------------------------------------------
\begin{tikzpicture}
%%%%%%%%%%%%%%%%
% parameter server + processor
\begin{scope}[]
{\scriptsize
\tikzstyle{parametershard} = [draw,thick,minimum width=4em,align=left,rounded corners=2pt]
{
\node[parametershard,anchor=west,fill=yellow!10] (param1) at (0,0) {$W_o$, $b_o$};
\node (param2) at ([xshift=1em]param1.east) {};
\node[parametershard,anchor=west,fill=red!10] (param3) at ([xshift=1em]param2.east) {$W_h$, $b_h$};
\node[anchor=south,inner sep=1pt] (serverlabel) at ([yshift=0.2em]param2.north) {\footnotesize{\textbf{parameter server}: $\mathbf w_{new} = \mathbf w - \alpha\cdot \frac{\partial L}{\partial \mathbf w}$}};
}
\begin{pgfonlayer}{background}
{
\node[rectangle,draw,thick,inner sep=2pt,fill=gray!20] [fit = (param1) (param2) (param3) (serverlabel)] (serverbox) {};
}
\end{pgfonlayer}
\tikzstyle{processor} = [draw,thick,fill=orange!20,minimum width=4em,align=left,rounded corners=2pt]
{
\node [processor,anchor=north,align=center] (processor2) at ([yshift=-1.2in]serverlabel.south) {\scriptsize{Processor 2}\\\scriptsize{on GPU2 (G2)}};
\node [anchor=north] (labela) at ([xshift=4em,yshift=-1em]processor2.south) {\footnotesize {(a)同步更新}};
\node [processor,anchor=east,align=center] (processor1) at ([xshift=-1em]processor2.west) {\scriptsize{Processor 1}\\\scriptsize{on GPU1 (G1)}};
\node [processor,anchor=west,align=center] (processor3) at ([xshift=1em]processor2.east) {\scriptsize{Processor 3}\\\scriptsize{on GPU3 (G3)}};
}
{
\draw[->,very thick,red] ([xshift=-0.5em,yshift=2pt]processor2.north) -- ([xshift=-0.5em,yshift=-2pt]serverbox.south) node [pos=0.5,align=right,xshift=-2em] (pushlabel) {\scriptsize{$\frac{\partial L}{\partial \mathbf w}$}};;
\draw[<-,very thick,blue] ([xshift=0.5em,yshift=2pt]processor2.north) -- ([xshift=0.5em,yshift=-2pt]serverbox.south) node [pos=0.5,align=left,xshift=2.2em] (fetchlabel) {\scriptsize{$\mathbf w_{new}$}};;;
\draw[->,very thick,red] ([xshift=-0.5em,yshift=2pt]processor3.north) --
([xshift=3em,yshift=-2pt]serverbox.south);
\draw[<-,very thick,blue] ([xshift=0.5em,yshift=2pt]processor3.north) -- ([xshift=4em,yshift=-2pt]serverbox.south) node [pos=0.5,align=left,xshift=2.2em] (fetchlabel) {\scriptsize{fetch (F)}};
\draw[->,very thick,red] ([xshift=-0.5em,yshift=2pt]processor1.north) -- ([xshift=-4em,yshift=-2pt]serverbox.south) node [pos=0.5,align=right,xshift=-2em] (pushlabel) {\scriptsize{push (P)}};
\draw[<-,very thick,blue] ([xshift=0.5em,yshift=2pt]processor1.north) -- ([xshift=-3em,yshift=-2pt]serverbox.south);
}
%%%%%%%%%%%
% synchronous mode
\tikzstyle{job} = [draw,rotate=90,minimum height=0.25in]
\scriptsize{
{
\node[job,anchor=south west,fill=blue!50] (fetch11) at ([xshift=6em,yshift=1em]processor3.east) {\textbf{F}};
\node[job,anchor=west,fill=orange!30] (minibatch11) at ([yshift=1pt]fetch11.east) {\tiny{minibatch3}};
\node[job,anchor=west,fill=red!50] (push11) at ([yshift=1pt]minibatch11.east) {\textbf{P}};
\node[job,anchor=north west,fill=blue!50] (fetch12) at ([xshift=0.8em]fetch11.south west) {\textbf{F}};
\node[job,anchor=west,fill=orange!30] (minibatch12) at ([yshift=1pt]fetch12.east) {\tiny{minibatch2}};
\node[job,anchor=west,fill=red!50] (push12) at ([yshift=1pt]minibatch12.east) {\textbf{P}};
\node[job,anchor=north west,fill=blue!50] (fetch13) at ([xshift=0.8em]fetch12.south west) {\textbf{F}};
\node[job,anchor=west,fill=orange!30,minimum width=8em] (minibatch13) at ([yshift=1pt]fetch13.east) {\scriptsize{minibatch1}};
\node[job,anchor=west,fill=red!50] (push13) at ([yshift=1pt]minibatch13.east) {\textbf{P}};
\node[anchor=south west,draw,fill=gray!20,minimum width=8.0em] (update11) at ([yshift=4.0em]push11.north east) {Update};
\node[anchor=north] (G11) at (fetch11.west) {\small{G3}};
\node[anchor=north] (G12) at (fetch12.west) {\small{G2}};
\node[anchor=north] (G13) at (fetch13.west) {\small{G1}};
\node[anchor=north,align=center] (synlabel) at (G12.south) {\small{\textbf{Synchronous}}\\\small{\textbf{Training}}};
\draw[->,thick] ([xshift=1em]G13.east) -- ([xshift=1em,yshift=1.4in]G13.east) node [pos=0.5,rotate=90,yshift=-1em] {\small{time line}};
}
}
{
\draw [<->,thin,dotted] ([xshift=-1pt]minibatch11.north) .. controls +(west:3em) and +(east:3em) .. ([xshift=1pt]processor3.east);
\draw [<->,thin,dotted] ([xshift=-1pt]fetch11.north) .. controls +(west:4em) and +(east:4em) .. ([xshift=-0.5em,yshift=0.3in]processor3.north);
\draw [<->,thin,dotted] ([xshift=-1pt]push11.north) -- ([xshift=-4em,yshift=0.8in]processor3.north);
}
{
\draw [<->,thin,dotted] ([xshift=-1pt]update11.west) -- ([xshift=1pt,yshift=-1.5em]serverbox.north east);
}
}
\end{scope}
\begin{scope}[yshift=-2.5in]
{\scriptsize
\tikzstyle{parametershard} = [draw,thick,minimum width=4em,align=left,rounded corners=2pt]
{
\node[parametershard,anchor=west,fill=yellow!10] (param1) at (0,0) {$W_o$, $b_o$};
\node (param2) at ([xshift=1em]param1.east) {};
\node[parametershard,anchor=west,fill=red!10] (param3) at ([xshift=1em]param2.east) {$W_h$, $b_h$};
\node[anchor=south,inner sep=1pt] (serverlabel) at ([yshift=0.2em]param2.north) {\footnotesize{\textbf{parameter server}: $\mathbf w_{new} = \mathbf w - \alpha\cdot \frac{\partial L}{\partial \mathbf w}$}};
}
\begin{pgfonlayer}{background}
{
\node[rectangle,draw,thick,inner sep=2pt,fill=gray!20] [fit = (param1) (param2) (param3) (serverlabel)] (serverbox) {};
}
\end{pgfonlayer}
\tikzstyle{processor} = [draw,thick,fill=orange!20,minimum width=4em,align=left,rounded corners=2pt]
{
\node [processor,anchor=north,align=center] (processor2) at ([yshift=-1.2in]serverlabel.south) {\scriptsize{Processor 2}\\\scriptsize{on GPU2 (G2)}};
\node [anchor=north] (label) at ([xshift=4em,yshift=-1em]processor2.south) {\footnotesize {(b)异步更新}};
\node [processor,anchor=east,align=center] (processor1) at ([xshift=-1em]processor2.west) {\scriptsize{Processor 1}\\\scriptsize{on GPU1 (G1)}};
\node [processor,anchor=west,align=center] (processor3) at ([xshift=1em]processor2.east) {\scriptsize{Processor 3}\\\scriptsize{on GPU3 (G3)}};
}
{
\draw[->,very thick,red] ([xshift=-0.5em,yshift=2pt]processor2.north) -- ([xshift=-0.5em,yshift=-2pt]serverbox.south) node [pos=0.5,align=right,xshift=-2em] (pushlabel) {\scriptsize{$\frac{\partial L}{\partial \mathbf w}$}};;
\draw[<-,very thick,blue] ([xshift=0.5em,yshift=2pt]processor2.north) -- ([xshift=0.5em,yshift=-2pt]serverbox.south) node [pos=0.5,align=left,xshift=2.2em] (fetchlabel) {\scriptsize{$\mathbf w_{new}$}};;;
\draw[->,very thick,red] ([xshift=-0.5em,yshift=2pt]processor3.north) --
([xshift=3em,yshift=-2pt]serverbox.south);
\draw[<-,very thick,blue] ([xshift=0.5em,yshift=2pt]processor3.north) -- ([xshift=4em,yshift=-2pt]serverbox.south) node [pos=0.5,align=left,xshift=2.2em] (fetchlabel) {\scriptsize{fetch (F)}};
\draw[->,very thick,red] ([xshift=-0.5em,yshift=2pt]processor1.north) -- ([xshift=-4em,yshift=-2pt]serverbox.south) node [pos=0.5,align=right,xshift=-2em] (pushlabel) {\scriptsize{push (P)}};
\draw[<-,very thick,blue] ([xshift=0.5em,yshift=2pt]processor1.north) -- ([xshift=-3em,yshift=-2pt]serverbox.south);
}
%%%%%%%%%%%
% synchronous mode
\tikzstyle{job} = [draw,rotate=90,minimum height=0.25in]
\scriptsize{
{
\node[job,anchor=south west,fill=blue!50] (fetch21) at ([xshift=6em,yshift=1em]processor3.east) {\textbf{F}};
\node[job,anchor=west,fill=orange!30] (minibatch21) at ([yshift=1pt]fetch21.east) {\tiny{minibatch3}};
\node[job,anchor=west,fill=red!50] (push21) at ([yshift=1pt]minibatch21.east) {\textbf{P}};
\node[job,anchor=north west,fill=blue!50] (fetch22) at ([xshift=0.8em]fetch21.south west) {\textbf{F}};
\node[job,anchor=west,fill=orange!30] (minibatch22) at ([yshift=1pt]fetch22.east) {\tiny{minibatch2}};
\node[job,anchor=west,fill=red!50] (push22) at ([yshift=1pt]minibatch22.east) {\textbf{P}};
\node[job,anchor=north west,fill=blue!50] (fetch23) at ([xshift=0.8em]fetch22.south west) {\textbf{F}};
\node[job,anchor=west,fill=orange!30,minimum width=8em] (minibatch23) at ([yshift=1pt]fetch23.east) {\scriptsize{minibatch1}};
\node[job,anchor=west,fill=red!50] (push23) at ([yshift=1pt]minibatch23.east) {\textbf{P}};
\node[anchor=south west,draw,fill=gray!20,minimum width=0.59in] (update21) at ([yshift=2pt]push21.north east) {Update};
\node[anchor=south west,draw,fill=gray!20,minimum width=0.25in] (update22) at ([yshift=2pt]push23.north east) {\tiny{Upd.}};
\node[anchor=north] (G21) at (fetch21.west) {\small{G3}};
\node[anchor=north] (G22) at (fetch22.west) {\small{G2}};
\node[anchor=north] (G23) at (fetch23.west) {\small{G1}};
\node[anchor=north,align=center] (synlabel) at (G22.south) {\small{\textbf{Asynchronous}}\\\small{\textbf{Training}}};
\draw[->,thick] ([xshift=1em]G23.east) -- ([xshift=1em,yshift=1.4in]G23.east) node [pos=0.5,rotate=90,yshift=-1em] {\small{time line}};
\draw [<->,thin,dotted] ([xshift=-1pt]update21.west) -- ([xshift=1pt,yshift=-1.5em]serverbox.north east);
\draw [<->,thin,dotted] ([xshift=-1pt]update22.west) -- ([xshift=1pt,yshift=-1.5em]serverbox.north east);
}
}
}
\end{scope}
\end{tikzpicture}
%%%------------------------------------------------------------------------------------------------------------
Book/Chapter5/Figures/fig-parameter-space.tex 0 → 100644
%%%------------------------------------------------------------------------------------------------------------
\begin{tikzpicture}
\draw[name path=ellipse,thick] (0,0) circle[x radius = 2, y radius = 1];
\node[rectangle,minimum size=0.1em,inner sep=2pt,fill=red] (p1) at (0.2,0.5) {};
\node[rectangle,minimum size=0.1em,inner sep=2pt,fill=red] (p2) at (0.3,0.6) {};
\node[rectangle,minimum size=0.1em,inner sep=2pt,fill=red] (p3) at (0.1,-0.1) {};
\node[rectangle,minimum size=0.1em,inner sep=2pt,fill=red] (p4) at (0.4,0) {};
\node[rectangle,minimum size=0.1em,inner sep=2pt,fill=red] (p5) at (0.5,0.3) {};
\node[rectangle,minimum size=0.1em,inner sep=2pt,fill=red] (p6) at (0.6,0.1) {};
\node[rectangle,minimum size=0.1em,inner sep=2pt,fill=red] (p7) at (0.7,-0.1) {};
\node[rectangle,minimum size=0.1em,inner sep=2pt,fill=red] (p8) at (-1.2,0.4) {};
\node[rectangle,minimum size=0.1em,inner sep=2pt,fill=red] (p9) at (-1.0,-0.3) {};
\node[rectangle,minimum size=0.1em,inner sep=2pt,fill=red] (p10) at (-0.1,-0.8) {};
\begin{pgfonlayer}{background}
{
\node [rectangle,inner sep=0.4em,draw,blue] [fit = (p1) (p2) (p3) (p4) (p5) (p6)] (area) {};
}
\end{pgfonlayer}
\draw [->] (2.5,-0.7) -- (1.8,-0.5) node [pos=0,right] {\scriptsize{模型参数解空间}};
{
\draw [->] (2.0,0.7) -- (area.20) node [pos=0,right] {\scriptsize{优质解高密度区域(预训练)}};
}
{
\draw [->] (-2.0,0.7) -- (p8.west) node [pos=0,left] {\scriptsize{游离的解}};
}
\end{tikzpicture}
%%%------------------------------------------------------------------------------------------------------------
Book/Chapter5/Figures/fig-perceptron mode.tex 0 → 100644
%%%------------------------------------------------------------------------------------------------------------
\begin{tikzpicture}
\begin{scope}
\node [anchor=center,circle,draw,ublue,very thick,minimum size=3.5em,fill=white,drop shadow={shadow xshift=0.1em,shadow yshift=-0.1em}] (neuron) at (0,0) {};
\node [anchor=east] (x1) at ([xshift=-6em]neuron.west) {\Large{$x_1$}};
\node [anchor=center] (x0) at ([yshift=3em]x1.center) {\Large{$x_0$}};
\node [anchor=center] (x2) at ([yshift=-3em]x1.center) {\Large{$x_2$}};
\node [anchor=west] (y) at ([xshift=6em]neuron.east) {\Large{$y$}};
\node [anchor=center] (neuronmath) at (neuron.center) {\red{\small{$\sum \ge \sigma$}}};
\draw [->,thick] (x0.east) -- (neuron.150) node [pos=0.5,above] {$w_0$};
\draw [->,thick] (x1.east) -- (neuron.180) node [pos=0.5,above] {$w_1$};
\draw [->,thick] (x2.east) -- (neuron.210) node [pos=0.5,above] {$w_2$};
\draw [->,thick] (neuron.east) -- (y.west);
\end{scope}
\end{tikzpicture}
%%%------------------------------------------------------------------------------------------------------------
Book/Chapter5/Figures/fig-perceptron-to-predict-1.tex 0 → 100644
%%%------------------------------------------------------------------------------------------------------------
\begin{tikzpicture}
\begin{scope}
\node [anchor=center,circle,draw,ublue,very thick,minimum size=3.5em,fill=white,drop shadow={shadow xshift=0.1em,shadow yshift=-0.1em}] (neuron) at (0,0) {};
\node [anchor=east] (x1) at ([xshift=-6em]neuron.west) {$x_1$:票价够低？};
\node [anchor=center] (x0) at ([yshift=3em]x1.center) {$x_0$:距离够近？};
\node [anchor=center] (x2) at ([yshift=-3em]x1.center) {$x_2$:女友喜欢？};
\node [anchor=west] (y) at ([xshift=2em]neuron.east) {$y$:去？还是不去？};
{
\draw [->,thick] (x0.east) -- (neuron.150) node [pos=0.5,above,yshift=0.2em] {\small{$w_0=1$}};
\draw [->,thick] (x1.east) -- (neuron.180) node [pos=0.5,above,yshift=-0.1em] {\small{$w_1=1$}};
\draw [->,thick] (x2.east) -- (neuron.210) node [pos=0.5,above,yshift=0.1em] {\small{$w_2=1$}};
}
\draw [->,thick] (neuron.east) -- (y.west);
\node [anchor=center] (neuronmath) at (neuron.center) {\small{$\sum \ge \sigma$}};
\end{scope}
\end{tikzpicture}
%%%------------------------------------------------------------------------------------------------------------
Book/Chapter5/Figures/fig-perceptron-to-predict-2.tex 0 → 100644
%%%------------------------------------------------------------------------------------------------------------
\begin{tikzpicture}
\begin{scope}
\node [anchor=center,circle,draw,ublue,very thick,minimum size=3.5em,fill=white,drop shadow={shadow xshift=0.1em,shadow yshift=-0.1em}] (neuron) at (0,0) {};
\node [anchor=east] (x1) at ([xshift=-6em]neuron.west) {$x_1$:票价够低？};
\node [anchor=center] (x0) at ([yshift=3em]x1.center) {$x_0$:距离够近？};
\node [anchor=center] (x2) at ([yshift=-3em]x1.center) {$x_2$:女友喜欢？};
\node [anchor=west] (y) at ([xshift=2em]neuron.east) {$y$:去？还是不去？};
\draw [->,thin,red] (x0.east) -- (neuron.150) node [pos=0.5,above,yshift=0.2em] {\small{$w_0=.5$}};
\draw [->,line width=0.5mm,red] (x1.east) -- (neuron.180) node [pos=0.5,above,yshift=-0.1em] {\textbf{\small{$w_1=2$}}};
\draw [->,thin,red] (x2.east) -- (neuron.210) node [pos=0.5,above,yshift=0.1em] {\small{$w_2=.5$}};
\draw [->,thick] (neuron.east) -- (y.west);
\node [anchor=center] (neuronmath) at (neuron.center) {\small{$\sum \ge \sigma$}};
\node [anchor=south] (ylabel) at (y.north) {\red{\textbf{不去了！}}};
\end{scope}
\end{tikzpicture}
%%%------------------------------------------------------------------------------------------------------------
Book/Chapter5/Figures/fig-perceptron-to-predict-3.tex 0 → 100644
%%%------------------------------------------------------------------------------------------------------------
\begin{tikzpicture}
\begin{scope}
\node [anchor=center,circle,draw,ublue,very thick,minimum size=3.5em,fill=white,drop shadow={shadow xshift=0.1em,shadow yshift=-0.1em}] (neuron) at (0,0) {};
\node [anchor=east] (x1) at ([xshift=-6em]neuron.west) {$x_1$:便宜程度\ \ \ \ };
\node [anchor=center] (x0) at ([yshift=3em]x1.center) {$x_0$:远近程度\ \ \ \ };
\node [anchor=center] (x2) at ([yshift=-3em]x1.center) {$x_2$:女友喜欢？};
\node [anchor=west] (y) at ([xshift=2em]neuron.east) {$y$:去？还是不去？};
\draw [->,thick] (neuron.east) -- (y.west);
\node [anchor=center] (neuronmath) at (neuron.center) {\small{$\sum \ge \sigma$}};
{
\draw [->,dotted] (x0.east) -- (neuron.150) node [pos=0.5,above,yshift=0.2em] {\small{$w_0=0$}};
\draw [->,dotted] (x1.east) -- (neuron.180) node [pos=0.5,above,yshift=-0.1em] {\textbf{\small{$w_1=0$}}};
\draw [->,line width=0.5mm] (x2.east) -- (neuron.210) node [pos=0.5,above,yshift=0.1em] {\small{$w_2=10$}};
}
\end{scope}
\end{tikzpicture}
%%%------------------------------------------------------------------------------------------------------------
Book/Chapter5/Figures/fig-piecewise.tex 0 → 100644
%%%------------------------------------------------------------------------------------------------------------
\begin{tikzpicture}
%% a two-layer neural network
\begin{scope}
\tikzstyle{neuronnode} = [minimum size=1.7em,circle,draw,ublue,very thick,inner sep=1pt, fill=white,align=center,drop shadow={shadow xshift=0.1em,shadow yshift=-0.1em}]
%% output illustration
\begin{scope}[xshift=2.8in,yshift=0.1in]
{
\draw [->,thick] (-2.2,0) -- (2.2,0);
\draw [->,thick] (0,0) -- (0,2);
\draw [-] (-0.05,1) -- (0.05,1);
\node [anchor=north,inner sep=1pt] (labela) at (0,-0.2) {\footnotesize{(a)}};
}
{
\draw [->,thick] (-2.2,0) -- (2.2,0);
\draw [->,thick] (0,0) -- (0,2);
\draw [-,very thick,red,domain=-1.98:2,samples=100] plot (\x,{0.2 * (\x +0.4)^3 + 1.2 - 0.3 *(\x + 0.8)^2});
}
\foreach \n in {-1.9,-1.7,...,1.9}{
\pgfmathsetmacro{\result}{0.2 * (\n + 0.1 + 0.4)^3 + 1.2 - 0.3 *(\n + 0.1 + 0.8)^2};
\draw [-,ublue,thick] (\n,0) -- (\n, \result) -- (\n + 0.2, \result) -- (\n + 0.2, 0);
}
\end{scope}
\end{scope}
%% a two-layer neural network
\begin{scope}[xshift=2in]
\tikzstyle{neuronnode} = [minimum size=1.7em,circle,draw,ublue,very thick,inner sep=1pt, fill=white,align=center,drop shadow={shadow xshift=0.1em,shadow yshift=-0.1em}]
%% output illustration
\begin{scope}[xshift=2.8in,yshift=0.1in]
{
\draw [->,thick] (-2.2,0) -- (2.2,0);
\draw [->,thick] (0,0) -- (0,2);
\draw [-] (-0.05,1) -- (0.05,1);
\node [anchor=east,inner sep=1pt] (label1) at (0,1) {\tiny{1}};
\node [anchor=south east,inner sep=1pt] (label2) at (0,0) {\tiny{0}};
\node [anchor=north,inner sep=1pt] (labelb) at (0,-0.2) {\footnotesize{(b)}};
}
{
\draw [->,thick] (-2.2,0) -- (2.2,0);
\draw [->,thick] (0,0) -- (0,2);
\draw [-,very thick,red,domain=-1.98:2,samples=100] plot (\x,{0.2 * (\x +0.4)^3 + 1.2 - 0.3 *(\x + 0.8)^2});
}
\end{scope}
\end{scope}
\end{tikzpicture}
%%%------------------------------------------------------------------------------------------------------------
Book/Chapter5/Figures/fig-relu.tex 0 → 100644
%%%------------------------------------------------------------------------------------------------------------
\begin{tikzpicture}
\draw[->](-1.2,0)--(1.2,0)node[left,below,font=\tiny]{$x$};
\draw[->](0,-1.2)--(0,1.2)node[right,font=\tiny]{$y$};
\draw[dashed](-1.2,1)--(1.2,1);
\draw[dashed](-1.2,-1)--(1.2,-1);
\foreach \x in {-1.0,-0.5,0.0,0.5,1.0}{\draw(\x,0)--(\x,0.05)node[below,outer sep=2pt,font=\tiny]at(\x,0){\x};}
\foreach \y in {0.5,1.0}{\draw(0,\y)--(0.05,\y)node[left,outer sep=2pt,font=\tiny]at(0,\y){\y};}
\draw[color=red ,domain=-1.2:1.2]plot(\x,{max(\x,0)});
\node[black,anchor=south] at (0,1.2) {\small $y =\max (0, x)$};
\end{tikzpicture}
%%%------------------------------------------------------------------------------------------------------------
Book/Chapter5/Figures/fig-residual-structure.tex 0 → 100644
%%%------------------------------------------------------------------------------------------------------------
\begin{tikzpicture}
\begin{scope}
\node [anchor=center] (node1) at (0,0) {};
\node [anchor=north,draw,thick](node2)at ([yshift=-1.5em]node1.south){\small{weight layer}};
\draw[->,thick](node1.south)--(node2.north);
\node [anchor=north](node3)at ([yshift=-1.2em]node2.south){$\bigoplus$};
\draw[->,thick](node2.south)--([yshift=-0.3em]node3.north);
\node [anchor=east](node4) at ([xshift=0.0em]node2.west) {$\textrm{F}(\mathbf{x})$};
\node [anchor=east](node5) at ([xshift=0.3em]node3.west) {$\textrm{F}(\mathbf{x})+\mathbf{x}$};
\node [anchor=east](node1-1) at ([xshift=0.5em,yshift=-1.0em]node1.west) {$\mathbf{x}$};
\draw[->,thick]([xshift=-0.1em]node1-1.east)--([xshift=4.0em]node1-1.east)--([xshift=4.0em,yshift=-4.45em]node1-1.east)--([xshift=-0.35em]node3.east);
\node[anchor=west](node2-1) at ([xshift=2.3em]node2.east) {$\mathbf{x}$};
\node[anchor=north](node2-2) at ([xshift=0.2em,yshift=-0.5em]node2-1.south) {\footnotesize{$\rm{identity}$}};
\node [anchor=north](node6) at ([yshift=-1.2em]node3.south) {};
\draw[->,thick]([yshift=0.3em]node3.south)--([yshift=0.0em]node6.north);
\node[anchor=west](node6-1) at ([xshift=-0.2em,yshift=0.6em]node6.east) {\footnotesize{$\rm{Relu}$}};
\end{scope}
\end{tikzpicture}
%%%------------------------------------------------------------------------------------------------------------
Book/Chapter5/Figures/fig-rnn-LM.tex 0 → 100644
%%%------------------------------------------------------------------------------------------------------------
\begin{tikzpicture}
\begin{scope}
\tikzstyle{rnnnode} = [draw,inner sep=5pt,minimum width=4em,minimum height=1.5em,fill=green!30!white,blur shadow={shadow xshift=1pt,shadow yshift=-1pt}]
{
\node [anchor=west,rnnnode] (node11) at (0,0) {\scriptsize{RNN Cell}};
\node [anchor=west,rnnnode] (node12) at ([xshift=2em]node11.east) {\scriptsize{RNN Cell}};
\node [anchor=west,rnnnode] (node13) at ([xshift=2em]node12.east) {\scriptsize{RNN Cell}};
\node [anchor=west,rnnnode] (node14) at ([xshift=2em]node13.east) {\scriptsize{RNN Cell}};
}
\node [anchor=north,rnnnode,fill=red!30!white] (e1) at ([yshift=-1.2em]node11.south) {\tiny{$e_1=w_1\textbf{C}$}};
\node [anchor=north,rnnnode,fill=red!30!white] (e2) at ([yshift=-1.2em]node12.south) {\tiny{$e_2=w_2\textbf{C}$}};
\node [anchor=north,rnnnode,fill=red!30!white] (e3) at ([yshift=-1.2em]node13.south) {\tiny{$e_3=w_3\textbf{C}$}};
\node [anchor=north,rnnnode,fill=red!30!white] (e4) at ([yshift=-1.2em]node14.south) {\tiny{$e_4=w_4\textbf{C}$}};
\node [anchor=north] (w1) at ([yshift=-1em]e1.south) {\footnotesize{$w_1$}};
\node [anchor=north] (w2) at ([yshift=-1em]e2.south) {\footnotesize{$w_2$}};
\node [anchor=north] (w3) at ([yshift=-1em]e3.south) {\footnotesize{$w_3$}};
\node [anchor=north] (w4) at ([yshift=-1em]e4.south) {\footnotesize{$w_4$}};
\draw [->,thick] ([yshift=0.1em]w1.north)--([yshift=-0.1em]e1.south);
\draw [->,thick] ([yshift=0.1em]w2.north)--([yshift=-0.1em]e2.south);
\draw [->,thick] ([yshift=0.1em]w3.north)--([yshift=-0.1em]e3.south);
\draw [->,thick] ([yshift=0.1em]w4.north)--([yshift=-0.1em]e4.south);
\draw [->,thick] ([yshift=0.1em]e1.north)--([yshift=-0.1em]node11.south);
\draw [->,thick] ([yshift=0.1em]e2.north)--([yshift=-0.1em]node12.south);
\draw [->,thick] ([yshift=0.1em]e3.north)--([yshift=-0.1em]node13.south);
\draw [->,thick] ([yshift=0.1em]e4.north)--([yshift=-0.1em]node14.south);
{
\node [anchor=south,rnnnode] (node21) at ([yshift=1.5em]node11.north) {\scriptsize{RNN Cell}};
\node [anchor=south,rnnnode] (node22) at ([yshift=1.5em]node12.north) {\scriptsize{RNN Cell}};
\node [anchor=south,rnnnode] (node23) at ([yshift=1.5em]node13.north) {\scriptsize{RNN Cell}};
\node [anchor=south,rnnnode] (node24) at ([yshift=1.5em]node14.north) {\scriptsize{RNN Cell}};
\node [anchor=south,rnnnode,fill=blue!30!white] (node31) at ([yshift=1.5em]node21.north) {\scriptsize{Softmax($\cdot$)}};
\node [anchor=south,rnnnode,fill=blue!30!white] (node32) at ([yshift=1.5em]node22.north) {\scriptsize{Softmax($\cdot$)}};
\node [anchor=south,rnnnode,fill=blue!30!white] (node33) at ([yshift=1.5em]node23.north) {\scriptsize{Softmax($\cdot$)}};
\node [anchor=south,rnnnode,fill=blue!30!white] (node34) at ([yshift=1.5em]node24.north) {\scriptsize{Softmax($\cdot$)}};
}
{
\node [anchor=south,rnnnode,fill=blue!30!white] (node21) at ([yshift=1.5em]node11.north) {\scriptsize{Softmax($\cdot$)}};
\node [anchor=south,rnnnode,fill=blue!30!white] (node22) at ([yshift=1.5em]node12.north) {\scriptsize{Softmax($\cdot$)}};
\node [anchor=south,rnnnode,fill=blue!30!white] (node23) at ([yshift=1.5em]node13.north) {\scriptsize{Softmax($\cdot$)}};
\node [anchor=south,rnnnode,fill=blue!30!white] (node24) at ([yshift=1.5em]node14.north) {\scriptsize{Softmax($\cdot$)}};
}
{
\draw [->,thick] ([yshift=0.1em]node31.north)--([yshift=1em]node31.north) node[pos=1,above] {\scriptsize{$\textrm{P}(w_2)$}};
\draw [->,thick] ([yshift=0.1em]node32.north)--([yshift=1em]node32.north) node[pos=1,above] {\scriptsize{$\textrm{P}(w_3|w_2)$}};
\draw [->,thick] ([yshift=0.1em]node33.north)--([yshift=1em]node33.north) node[pos=1,above] {\scriptsize{$\textrm{P}(w_4|w_2 w_3)$}};
\draw [->,thick] ([yshift=0.1em]node34.north)--([yshift=1em]node34.north) node[pos=1,above] {\scriptsize{$\textrm{P}(w_5|w_2 w_3 w_4)$}};
\draw [->,thick] ([yshift=0.1em]node21.north)--([yshift=-0.1em]node31.south);
\draw [->,thick] ([yshift=0.1em]node22.north)--([yshift=-0.1em]node32.south);
\draw [->,thick] ([yshift=0.1em]node23.north)--([yshift=-0.1em]node33.south);
\draw [->,thick] ([yshift=0.1em]node24.north)--([yshift=-0.1em]node34.south);
\draw [->,thick] ([xshift=-1em]node21.west)--([xshift=-0.1em]node21.west);
\draw [->,thick] ([xshift=0.1em]node21.east)--([xshift=-0.1em]node22.west);
\draw [->,thick] ([xshift=0.1em]node22.east)--([xshift=-0.1em]node23.west);
\draw [->,thick] ([xshift=0.1em]node23.east)--([xshift=-0.1em]node24.west);
\draw [->,thick] ([xshift=0.1em]node24.east)--([xshift=1em]node24.east);
}
{
\draw [->,thick] ([yshift=0.1em]node11.north)--([yshift=-0.1em]node21.south);
\draw [->,thick] ([yshift=0.1em]node12.north)--([yshift=-0.1em]node22.south);
\draw [->,thick] ([yshift=0.1em]node13.north)--([yshift=-0.1em]node23.south);
\draw [->,thick] ([yshift=0.1em]node14.north)--([yshift=-0.1em]node24.south);
\draw [->,thick] ([xshift=-1em]node11.west)--([xshift=-0.1em]node11.west);
\draw [->,thick] ([xshift=0.1em]node11.east)--([xshift=-0.1em]node12.west);
\draw [->,thick] ([xshift=0.1em]node12.east)--([xshift=-0.1em]node13.west);
\draw [->,thick] ([xshift=0.1em]node13.east)--([xshift=-0.1em]node14.west);
\draw [->,thick] ([xshift=0.1em]node14.east)--([xshift=1em]node14.east);
}
\end{scope}
\end{tikzpicture}
%%%------------------------------------------------------------------------------------------------------------
Book/Chapter5/Figures/fig-rnn-model.tex 0 → 100644
%%%------------------------------------------------------------------------------------------------------------
\begin{tikzpicture}
\begin{scope}
\tikzstyle{rnnnode} = [draw,inner sep=5pt,minimum width=4em,minimum height=1.5em,fill=green!30!white,blur shadow={shadow xshift=1pt,shadow yshift=-1pt}]
\node [anchor=west,rnnnode] (node11) at (0,0) {\scriptsize{RNN Cell}};
\node [anchor=west,rnnnode] (node12) at ([xshift=2em]node11.east) {\scriptsize{RNN Cell}};
\node [anchor=west,rnnnode] (node13) at ([xshift=2em]node12.east) {\scriptsize{RNN Cell}};
\node [anchor=west,rnnnode] (node14) at ([xshift=2em]node13.east) {\scriptsize{RNN Cell}};
\node [anchor=north,rnnnode,fill=red!30!white] (e1) at ([yshift=-1.2em]node11.south) {\scriptsize{embedding}};
\node [anchor=north,rnnnode,fill=red!30!white] (e2) at ([yshift=-1.2em]node12.south) {\scriptsize{embedding}};
\node [anchor=north,rnnnode,fill=red!30!white] (e3) at ([yshift=-1.2em]node13.south) {\scriptsize{embedding}};
\node [anchor=north,rnnnode,fill=red!30!white] (e4) at ([yshift=-1.2em]node14.south) {\scriptsize{embedding}};
\node [anchor=north] (w1) at ([yshift=-1em]e1.south) {\footnotesize{乔布斯}};
\node [anchor=north] (w2) at ([yshift=-1em]e2.south) {\footnotesize{任职}};
\node [anchor=north] (w3) at ([yshift=-1em]e3.south) {\footnotesize{于}};
\node [anchor=north] (w4) at ([yshift=-1em]e4.south) {\footnotesize{苹果}};
\draw [->,thick] ([yshift=0.1em]w1.north)--([yshift=-0.1em]e1.south);
\draw [->,thick] ([yshift=0.1em]w2.north)--([yshift=-0.1em]e2.south);
\draw [->,thick] ([yshift=0.1em]w3.north)--([yshift=-0.1em]e3.south);
\draw [->,thick] ([yshift=0.1em]w4.north)--([yshift=-0.1em]e4.south);
\draw [->,thick] ([yshift=0.1em]e1.north)--([yshift=-0.1em]node11.south);
\draw [->,thick] ([yshift=0.1em]e2.north)--([yshift=-0.1em]node12.south);
\draw [->,thick] ([yshift=0.1em]e3.north)--([yshift=-0.1em]node13.south);
\draw [->,thick] ([yshift=0.1em]e4.north)--([yshift=-0.1em]node14.south);
\node [anchor=south,rnnnode] (node21) at ([yshift=1.5em]node11.north) {\scriptsize{RNN Cell}};
\node [anchor=south,rnnnode] (node22) at ([yshift=1.5em]node12.north) {\scriptsize{RNN Cell}};
\node [anchor=south,rnnnode] (node23) at ([yshift=1.5em]node13.north) {\scriptsize{RNN Cell}};
\node [anchor=south,rnnnode] (node24) at ([yshift=1.5em]node14.north) {\scriptsize{RNN Cell}};
\node [anchor=south] (node31) at ([yshift=1.0em]node21.north) {\scriptsize{的表示}};
\node [anchor=south west] (node31new) at ([yshift=-0.3em]node31.north west) {\scriptsize{``乔布斯''}};
\node [anchor=south] (node32) at ([yshift=1.0em]node22.north) {\scriptsize{的表示\ \ \ }};
\node [anchor=south west] (node32new) at ([yshift=-0.3em]node32.north west) {\scriptsize{``乔布斯 任职''}};
\node [anchor=south] (node33) at ([yshift=1.0em]node23.north) {\scriptsize{的表示\ \ \ \ \ \ \ \ }};
\node [anchor=south west] (node33new) at ([yshift=-0.3em]node33.north west) {\scriptsize{``乔布斯 任职 于''}};
\node [anchor=south] (node34) at ([yshift=1.0em]node24.north) {\scriptsize{的表示\ \ \ \ \ \ \ \ }};
\node [anchor=south west] (node34new) at ([yshift=-0.3em]node34.north west) {\scriptsize{``乔布斯 任职 于 苹果''}};
\draw [->,thick] ([yshift=0.1em]node21.north)--([yshift=-0.1em]node31.south);
\draw [->,thick] ([yshift=0.1em]node22.north)--([yshift=-0.1em]node32.south);
\draw [->,thick] ([yshift=0.1em]node23.north)--([yshift=-0.1em]node33.south);
\draw [->,thick] ([yshift=0.1em]node24.north)--([yshift=-0.1em]node34.south);
\draw [->,thick] ([xshift=-1em]node21.west)--([xshift=-0.1em]node21.west);
\draw [->,thick] ([xshift=0.1em]node21.east)--([xshift=-0.1em]node22.west);
\draw [->,thick] ([xshift=0.1em]node22.east)--([xshift=-0.1em]node23.west);
\draw [->,thick] ([xshift=0.1em]node23.east)--([xshift=-0.1em]node24.west);
\draw [->,thick] ([xshift=0.1em]node24.east)--([xshift=1em]node24.east);
\draw [->,thick] ([yshift=0.1em]node11.north)--([yshift=-0.1em]node21.south);
\draw [->,thick] ([yshift=0.1em]node12.north)--([yshift=-0.1em]node22.south);
\draw [->,thick] ([yshift=0.1em]node13.north)--([yshift=-0.1em]node23.south);
\draw [->,thick] ([yshift=0.1em]node14.north)--([yshift=-0.1em]node24.south);
\draw [->,thick] ([xshift=-1em]node11.west)--([xshift=-0.1em]node11.west);
\draw [->,thick] ([xshift=0.1em]node11.east)--([xshift=-0.1em]node12.west);
\draw [->,thick] ([xshift=0.1em]node12.east)--([xshift=-0.1em]node13.west);
\draw [->,thick] ([xshift=0.1em]node13.east)--([xshift=-0.1em]node14.west);
\draw [->,thick] ([xshift=0.1em]node14.east)--([xshift=1em]node14.east);
{
\node [anchor=south] (toplabel1) at ([yshift=2em,xshift=-2em]node32new.north) {\footnotesize{``苹果''的表示：}};
\node [anchor=west,fill=blue!20!white,minimum width=3em] (toplabel2) at (toplabel1.east) {\footnotesize{上下文}};
}
{
\node [anchor=west,fill=red!20!white,minimum width=3em] (toplabel3) at (toplabel2.east) {\footnotesize{词}};
}
\begin{pgfonlayer}{background}
{
\node [rectangle,inner sep=2pt,draw,thick,dashed,red] [fit = (e4)] (r2) {};
\draw [->,thick,red] (r2.west) .. controls +(west:0.8) and +(south:2) .. ([xshift=1.3em]toplabel3.south);
}
{
\node [rectangle,inner sep=2pt,draw,thick,dashed,ublue,fill=white] [fit = (node33) (node33new)] (r1) {};
\draw [->,thick,ublue] ([xshift=-2em]r1.north) .. controls +(north:0.7) and +(south:0.7) .. ([xshift=-0.5em]toplabel2.south);
}
\end{pgfonlayer}
\end{scope}
\end{tikzpicture}
%%%------------------------------------------------------------------------------------------------------------
Book/Chapter5/Figures/fig-rotation.tex 0 → 100644
%%%------------------------------------------------------------------------------------------------------------
\begin{tikzpicture}
\tikzstyle{neuron} = [rectangle,draw,thick,fill=red!30,red!35,minimum height=2em,minimum width=2em,font=\small]
\node[neuron,anchor=north] (a1) at (0,0) {};
\draw[->,thick] ([xshift=-2em,yshift=0em]a1.south) to ([xshift=3em,yshift=0em]a1.south);
\draw[->,thick] ([xshift=0em,yshift=-4em]a1.west) to ([xshift=0em,yshift=2em]a1.west);
\node[below] at ([xshift=0.5em,yshift=-1em]a1.west){0};
\node[below] at ([xshift=2em,yshift=-1em]a1.west){1};
\node[below] at ([xshift=-0.5em,yshift=2em]a1.west){1};
\node [anchor=west] (x) at ([xshift=-0.7em,yshift=1em]a1.south) {\Large{$\textbf{F}$}};
{
\tikzstyle{neuron} = [rectangle,draw,thick,fill=red!30,red!35,minimum height=2em,minimum width=2em,font=\small]
\node[neuron,anchor=north] (a2) at ([xshift=10em,yshift=0em]a1.south) {};
\draw[->,thick] ([xshift=-2em,yshift=0em]a2.north) to ([xshift=3em,yshift=0em]a2.north);
\draw[->,thick] ([xshift=0em,yshift=-2em]a2.west) to ([xshift=0em,yshift=4em]a2.west);
\node[above] at ([xshift=0.5em,yshift=1em]a2.west){0};
\node[above] at ([xshift=2em,yshift=1em]a2.west){1};
\node[below] at ([xshift=-0.5em,yshift=0em]a2.west){-1};
\node [anchor=west] (x) at ([xshift=-3.5cm,yshift=2em]a2.north) {\scriptsize{
$w=\begin{bmatrix}
1&0&0\\
0&-1&0\\
0&0&1
\end{bmatrix}$}
};
\node [anchor=west,rotate = 180] (x) at ([xshift=0.7em,yshift=1em]a2.south) {\Large{$\textbf{F}$}};
\draw[-stealth, line width=2pt,dashed] ([xshift=4em,yshift=0em]a1.south) to ([xshift=-3em,yshift=0em]a2.north);
}
\end{tikzpicture}
%%%------------------------------------------------------------------------------------------------------------
Book/Chapter5/Figures/fig-save.tex 0 → 100644
%%%------------------------------------------------------------------------------------------------------------
\begin{tikzpicture}
\begin{scope}
\setcounter{mycount1}{1}
\draw[step=0.5cm,thick] (0,-0) grid (1.5,0.5);
\foreach \x in {0.25,0.75,1.25}{
\node [fill=green!20,inner sep=0pt,minimum height=0.49cm,minimum width=0.49cm](vector1) at (\x,0.25) {$\number\value{mycount1}$};
\addtocounter{mycount1}{1};
}
\node [anchor=north] (labela) at ([xshift=-1.2em,yshift=-0em]vector1.south) {\footnotesize{(a) }};
\end{scope}
\begin{scope}[xshift=1.2in]
\draw[step=0.5cm,thick] (0,-0) grid (3.0,0.5);
\setcounter{mycount2}{1}
\foreach \x in {0.25,0.75,1.25}{
\node [fill=green!20,inner sep=0pt,minimum height=0.49cm,minimum width=0.49cm] (vector2)at (\x,0.25) {$\number\value{mycount2}$};
\addtocounter{mycount2}{1};
}
\foreach \x in {1.75,2.25,2.75}{
\node [fill=red!20,inner sep=0pt,minimum height=0.49cm,minimum width=0.49cm] at (\x,0.25) {$\number\value{mycount2}$};
\addtocounter{mycount2}{1};
}
\node [anchor=north] (labelb) at ([xshift=0.3em,yshift=-0em]vector2.south) {\footnotesize{(b) }};
\end{scope}
\begin{scope}[yshift=-0.6in]
\draw[step=0.5cm,thick] (0,-0) grid (6.0,0.5);
\setcounter{mycount3}{1}
\foreach \x in {0.25,0.75,1.25}{
\node [fill=green!20,inner sep=0pt,minimum height=0.49cm,minimum width=0.49cm] at (\x,0.25) {$\number\value{mycount3}$};
\addtocounter{mycount3}{1};
}
\foreach \x in {1.75,2.25,2.75}{
\node [fill=red!20,inner sep=0pt,minimum height=0.49cm,minimum width=0.49cm] at (\x,0.25) {$\number\value{mycount3}$};
\addtocounter{mycount3}{1};
}
\foreach \x in {3.25,3.75,4.25}{
\node [fill=green!20,inner sep=0pt,minimum height=0.49cm,minimum width=0.49cm] at (\x,0.25) {$\number\value{mycount3}$};
\addtocounter{mycount3}{1};
}
\foreach \x in {4.75,5.25,5.75}{
\node [fill=red!20,inner sep=0pt,minimum height=0.49cm,minimum width=0.49cm] at (\x,0.25) {$\number\value{mycount3}$};
\addtocounter{mycount3}{1};
}
\draw[decorate,thick,decoration={brace,mirror,raise=0.2em}] (0,-0.2) -- (2.95,-0.2);
\draw[decorate,thick,decoration={brace,mirror,raise=0.2em}] (3.05,-0.2) -- (6,-0.2);
\node [anchor=north] (subtensor1) at (1.5,-0.4) {\footnotesize{$3 \times 2$ sub-tensor}};
\node [anchor=north] (subtensor1) at (4.5,-0.4) {\footnotesize{$3 \times 2$ sub-tensor}};
\node [anchor=north] (labelc) at (3,-0.8) {\footnotesize{(c)}};
\end{scope}
\end{tikzpicture}
%%%------------------------------------------------------------------------------------------------------------
Book/Chapter5/Figures/fig-sawtooth.tex 0 → 100644
%%%------------------------------------------------------------------------------------------------------------
\begin{tikzpicture}
\begin{scope}
\node [anchor=center,color=red] (node1) at (0,0) {};
\node [anchor=north,color=red] (node4) at ([xshift=9.0em,yshift=0.88em]node1.south) {\large{$\bullet$}};
\node [anchor=north,color=red] (node5) at ([xshift=6.35em,yshift=-2.5em]node1.south) {\large{$\bullet$}};
\node [anchor=north,color=red] (node6) at ([xshift=3.8em,yshift=-0.5em]node1.south) {\large{$\bullet$}};
\node [anchor=north,color=red] (node7) at ([xshift=2.6em,yshift=-2.0em]node1.south) {\large{$\bullet$}};
\node [anchor=north,color=red] (node8) at ([xshift=1.22em,yshift=-0.91em]node1.south) {\large{$\bullet$}};
\node [anchor=north,color=red] (node9) at ([xshift=0.6em,yshift=-1.7em]node1.south) {\large{$\bullet$}};
\node [anchor=north,color=red] (node10) at ([xshift=0.0em,yshift=-1.2em]node1.south) {\large{$\bullet$}};
\draw[-,ublue]([xshift=0.5em,yshift=0.46em]node4.south west)--([xshift=-0.5em,yshift=-0.4em]node5.north east);
\draw[-,ublue]([xshift=-0.45em,yshift=0.52em]node6.south east)--([xshift=0.47em,yshift=-0.43em]node5.north west);
\draw[-,ublue]([xshift=0.5em,yshift=0.46em]node6.south west)--([xshift=-0.5em,yshift=-0.4em]node7.north east);
\draw[-,ublue]([xshift=-0.45em,yshift=0.52em]node8.south east)--([xshift=0.47em,yshift=-0.43em]node7.north west);
\draw[-,ublue]([xshift=0.5em,yshift=0.46em]node8.south west)--([xshift=-0.5em,yshift=-0.4em]node9.north east);
\draw[-,ublue]([xshift=-0.78em,yshift=0.77em]node9.south east)--([xshift=0.78em,yshift=-0.68em]node10.north west);
\draw [-,ublue] (0,0) .. controls (2,0) and (3,-1.0)..(3,-1.5) .. controls (3,-2.2) and (2,-1.75)..(1.5,-1.65)..controls (1.5,-1.65) and (0.5,-1.45)..(0,-1.45)..controls (-0.5,-1.45) and (-1.5,-1.65)..(-1.5,-1.65)..controls (-2,-1.75)and (-3,-2.2).. (-3,-1.5)..controls (-3,-1.0) and (-2,0)..(0,0);
\draw [-,ublue] (0,0.5)..controls (2,0.5) and (4,-1.0).. (4,-1.7)..controls(4,-2.6)and (3,-2.3)..(2,-2.05)..controls (2,-2.05) and (1,-1.80)..(0,-1.80)..controls (-1,-1.80)and (-2,-2.05)..(-2,-2.05)..controls(-3,-2.3)and(-4,-2.6)..(-4,-1.7)..controls(-4,-1.0)and (-2,0.5)..(0,0.5);
\draw[-,ublue](0,1.0)..controls(3,1.0) and (5,-1.0)..(5,-1.9)..controls (5,-3.2)and (4,-2.7)..(3,-2.5)..controls (3,-2.5) and (2,-2.20)..(0,-2.15)..controls (-2,-2.20)and (-3,-2.5)..(-3,-2.5)..controls (-4,-2.7) and (-5,-3.2) ..(-5,-1.9)..controls (-5,-1.0) and (-3,1.0)..(0,1.0);
\draw[-,ublue] (0,-0.3)..controls (1.5,-0.3)and (2.5,-1.0)..(2.5,-1.4)..controls(2.5,-1.8)and (2,-1.55)..(1.5,-1.45) ..controls (1.5,-1.45) and (0.5,-1.25)..(0,-1.25) .. controls(-0.5,-1.25)and (-1.5,-1.45)..(-1.5,-1.45)..controls(-2,-1.55)and (-2.5,-1.8) ..(-2.5,-1.4)..controls(-2.5,-1.0) and (-1.5,-0.3)..(0,-0.3);
\draw[-,ublue](0,-0.5)..controls (1.0,-0.5) and (1.9,-0.8)..(1.9,-1.3)..controls(1.9,-1.5)and (1.5,-1.3)..(1.0,-1.2) ..controls(1.0,-1.2) and (0.5,-1.1)..(0,-1.1)..controls(-0.5,-1.1) and (-1.0,-1.2)..(-1.0,-1.2)..controls (-1.5,-1.3)and (-1.9,-1.5)..(-1.9,-1.3) ..controls(-1.9,-0.8)and (-1.0,-0.5) ..(0,-0.5);
\draw[-,ublue](0,-0.7)..controls(1.0,-0.7) and (1.4,-0.9)..(1.4,-1.1) .. controls(1.4,-1.25) and (1.2,-1.15)..(1.0,-1.1)..controls(1.0,-1.1) and (0.5,-0.95)..(0,-0.95)..controls(-0.5,-0.95)and (-1.0,-1.1) ..(-1.0,-1.1)..controls(-1.2,-1.15) and (-1.4,-1.25)..(-1.4,-1.1)..controls(-1.4,-0.9) and (-1.0,-0.7)..(0,-0.7);
\draw[-,ublue](0,-0.75)..controls(0.7,-0.75)and (1.0,-0.9)..(1.0,-1.0)..controls(1.0,-1.05) and (0.9,-1.05)..(0.7,-1.0)..controls(0.5,-0.95)and (0.3,-0.9)..(0,-0.9)..controls(-0.3,-0.9)and (-0.5,-0.95)..(-0.7,-1.0)..controls(-0.9,-1.05)and (-1.0,-1.05)..(-1.0,-1.0) ..controls(-1.0,-0.9)and (-0.7,-0.75)..(0,-0.75);
\draw[-,ublue](0,-0.8)..controls(0.5,-0.8) and (0.6,-0.85)..(0.6,-0.9)..controls(0.6,-0.93)and (0.5,-0.91)..(0.3,-0.88)..controls(0.2,-0.87)and (0.1,-0.86)..(0,-0.86)..controls(-0.1,-0.86)and(-0.2,-0.87)..(-0.3,-0.88)..controls(-0.5,-0.91) and(-0.6,-0.93) ..(-0.6,-0.9)..controls(-0.6,-0.85)and (-0.5,-0.8)..(0,-0.8);
\end{scope}
\end{tikzpicture}
%%%------------------------------------------------------------------------------------------------------------
Book/Chapter5/Figures/fig-sigmoid.tex 0 → 100644
%%%------------------------------------------------------------------------------------------------------------
\begin{tikzpicture}
\draw[->](-1.2,0)--(1.2,0)node[left,below,font=\tiny]{$x$};
\draw[->](0,-1.2)--(0,1.2)node[right,font=\tiny]{$y$};
\draw[dashed](-1.2,1)--(1.2,1);
\foreach \x in {-1,-0.5,0,0.5,1}{\draw(\x,0)--(\x,0.05)node[below,outer sep=2pt,font=\tiny]at(\x,0){
\pgfmathparse{(\x)*5}
\pgfmathresult};}
\foreach \y in {0.5,1.0}{\draw(0,\y)--(0.05,\y)node[left,outer sep=2pt,font=\tiny]at(0,\y){\y};}
\draw[color=red,domain=-1.2:1.2]plot(\x,{1/(1+(exp(-5*\x)))});
\node[black,anchor=south] at (0,1.2) {\small $y = \frac{1}{1+e^{-x}}$};
\end{tikzpicture}
%%%------------------------------------------------------------------------------------------------------------
Book/Chapter5/Figures/fig-single-layer-of-neural-network-for-weather-prediction.tex 0 → 100644
%%%------------------------------------------------------------------------------------------------------------
\begin{tikzpicture}
\begin{scope}
\tikzstyle{neuronnode} = [minimum size=1.5em,circle,draw,ublue,very thick,fill=white,drop shadow={shadow xshift=0.1em,shadow yshift=-0.1em}]
\node [anchor=center,neuronnode] (neuron00) at (0,0) {};
\node [anchor=center,neuronnode] (neuron01) at ([yshift=-3em]neuron00) {};
\node [anchor=center,neuronnode] (neuron02) at ([yshift=-3em]neuron01) {};
\node [anchor=east] (x0) at ([xshift=-6em]neuron00.west) {$x_0$};
\node [anchor=east] (x1) at ([xshift=-6em]neuron01.west) {$x_1$};
\node [anchor=east] (x2) at ([xshift=-6em]neuron02.west) {$b$};
\node [anchor=west] (y0) at ([xshift=4em]neuron00.east) {$y_0$};
\draw [->] (x0.east) -- (neuron00.180) node [pos=0.1,above] {\tiny{$w_{00}$}};
\draw [->] (x1.east) -- (neuron00.200) node [pos=0.1,above] {\tiny{$w_{10}$}};
\draw [->] (x2.east) -- (neuron00.220) node [pos=0.05,above,yshift=0.3em] {\tiny{$b_{0}$}};
\draw [->] (neuron00.east) -- (y0.west);
\node [anchor=west] (y1) at ([xshift=4em]neuron01.east) {$y_1$};
\draw [->] (x0.east) -- (neuron01.160) node [pos=0.4,above] {\tiny{$w_{01}$}};
\draw [->] (x1.east) -- (neuron01.180) node [pos=0.35,above,yshift=-0.2em] {\tiny{$w_{11}$}};
\draw [->] (x2.east) -- (neuron01.200) node [pos=0.3,below,yshift=0.2em] {\tiny{$b_{1}$}};
\draw [->] (neuron01.east) -- (y1.west);
\node [anchor=west] (y2) at ([xshift=4em]neuron02.east) {$y_2$};
\draw [->] (x0.east) -- (neuron02.140) node [pos=0.1,below,yshift=-0.2em] {\tiny{$w_{02}$}};
\draw [->] (x1.east) -- (neuron02.160) node [pos=0.1,below] {\tiny{$w_{12}$}};
\draw [->] (x2.east) -- (neuron02.180) node [pos=0.3,below] {\tiny{$b_{2}$}};
\draw [->] (neuron02.east) -- (y2.west);
\node [anchor=east,align=left] (inputlabel) at ([xshift=-0.1em]x1.west) {\scriptsize{输入向量}:\\\small{$\textbf{x}=(x_0,x_1)$}};
\node [anchor=west,align=left] (outputlabel) at ([xshift=0.1em]y1.east) {\scriptsize{输出向量}:\\\small{$\textbf{y}=(y_0,y_1,y_2)$}};
\begin{pgfonlayer}{background}
\node [rectangle,inner sep=0.4em,fill=red!20] [fit = (neuron00) (neuron01) (neuron02)] (layer) {};
\node [anchor=south] (layerlabel) at ([yshift=0.2em]layer.north) {\scriptsize{一层神经元}};
\node [rectangle,inner sep=0.1em,fill=ugreen!20] [fit = (x0) (x1)] (inputshadow) {};
\node [rectangle,inner sep=0.1em,fill=blue!20] [fit = (y0) (y1) (y2)] (outputshadow) {};
\end{pgfonlayer}
\end{scope}
\end{tikzpicture}
%%%------------------------------------------------------------------------------------------------------------
Book/Chapter5/Figures/fig-softplus.tex 0 → 100644
%%%------------------------------------------------------------------------------------------------------------
\begin{tikzpicture}
\draw[->](-1.2,0)--(1.2,0)node[left,below,font=\tiny]{$x$};
\draw[->](0,-1.2)--(0,1.2)node[right,font=\tiny]{$y$};
\foreach \x in {-1.0,-0.5,0.0,0.5,1.0}{\draw(\x,0)--(\x,0.05)node[below,outer sep=2pt,font=\tiny]at(\x,0){\x};}
\foreach \y in {1.0,0.5}{\draw(0,\y)--(0.05,\y)node[left,outer sep=2pt,font=\tiny]at(0,\y){\y};}
\draw[color=red ,domain=-1.2:1]plot(\x,{ln(1+(exp(\x))});
\node[black,anchor=south] at (0,1.2) {\small $y = ln(1+e^x)$};
\end{tikzpicture}
%%%------------------------------------------------------------------------------------------------------------
Book/Chapter5/Figures/fig-tanh.tex 0 → 100644
%%%------------------------------------------------------------------------------------------------------------
\begin{tikzpicture}
\draw[->](-1.2,0)--(1.2,0)node[left,below,font=\tiny]{$x$};
\draw[->](0,-1.2)--(0,1.2)node[right,font=\tiny]{$y$};
\draw[dashed](-1.2,1)--(1.2,1);
\draw[dashed](-1.2,-1)--(1.2,-1);
\foreach \x in {-1.0,-0.5,0.0,0.5,1.0}{\draw(\x,0)--(\x,0.05)node[below,outer sep=2pt,font=\tiny]at(\x,0){\x};}
\foreach \y in {0.5,1.0}{\draw(0,\y)--(0.05,\y)node[left,outer sep=2pt,font=\tiny]at(0,\y){\y};}
\draw[color=red ,domain=-1.2:1.2]plot(\x,{tanh(\x)});
\node[black,anchor=south] at (0,1.2) {\small $y = \frac{e^{x}-e^{-x}}{e^{x}+e^{-x}}$};
\end{tikzpicture}
%%%------------------------------------------------------------------------------------------------------------
Book/Chapter5/Figures/fig-tensor-mul.tex 0 → 100644
%%%------------------------------------------------------------------------------------------------------------
\begin{tikzpicture}
\begin{scope}[yshift=6.5em,xshift=1em]
{
\setcounter{mycount1}{1}
\draw[step=0.5cm,color=orange,thick] (-1,-1) grid (1,1);
\foreach \y in {+0.75,+0.25,-0.25,-0.75}
\foreach \x in {-0.75,-0.25,0.25,0.75}{
\node [fill=orange!20,inner sep=0pt,minimum height=0.49cm,minimum width=0.49cm] at (\x,\y) {$\number\value{mycount1}$};
\addtocounter{mycount1}{1};
}
}
\node [anchor=south west] (label11) at (-1.3,0.9) {\footnotesize{①}};
\end{scope}
\begin{scope}[yshift=6em,xshift=0.5em]
{
\setcounter{mycount2}{2}
\draw[step=0.5cm,color=blue,thick] (-1,-1) grid (1,1);
\foreach \y in {+0.75,+0.25,-0.25,-0.75}
\foreach \x in {-0.75,-0.25,0.25,0.75}{
\node [fill=blue!20,inner sep=0pt,minimum height=0.49cm,minimum width=0.49cm] at (\x,\y) {$\number\value{mycount2}$};
\addtocounter{mycount2}{1};
}
}
\node [anchor=south west] (label12) at (-1.3,0.9) {\footnotesize{②}};
\end{scope}
\begin{scope}[yshift=5.5em,xshift=0em]
{
\setcounter{mycount3}{3}
\draw[step=0.5cm,color=ugreen,thick] (-1,-1) grid (1,1);
\foreach \y in {+0.75,+0.25,-0.25,-0.75}
\foreach \x in {-0.75,-0.25,0.25,0.75}{
\node [fill=green!20,inner sep=0pt,minimum height=0.49cm,minimum width=0.49cm] at (\x,\y) {$\number\value{mycount3}$};
\addtocounter{mycount3}{1};
}
}
\node [anchor=south west] (label13) at (-1.3,0.9) {\footnotesize{③}};
\end{scope}
\begin{scope}[yshift=5em,xshift=-0.5em]
{
\setcounter{mycount4}{4}
\draw[step=0.5cm,color=red,thick] (-1,-1) grid (1,1);
\foreach \y in {+0.75,+0.25,-0.25,-0.75}
\foreach \x in {-0.75,-0.25,0.25,0.75}{
\node [fill=red!20,inner sep=0pt,minimum height=0.49cm,minimum width=0.49cm] at (\x,\y) {$\number\value{mycount4}$};
\addtocounter{mycount4}{1};
}
\node [anchor=north] (xlabel) at (0,-1.2) {$\textbf{x}$};
}
\node [anchor=south west] (label14) at (-1.3,0.9) {\footnotesize{④}};
\end{scope}
\begin{scope}[yshift=5em,xshift=1.5in]
{
\draw[step=0.5cm,thick] (-0.5,-1) grid (0.5,1.0);
\node [fill=black!20,inner sep=0pt,minimum height=0.49cm,minimum width=0.49cm] at (-0.25,0.75) {\small{$-1$}};
\node [fill=black!20,inner sep=0pt,minimum height=0.49cm,minimum width=0.49cm] at (-0.25,0.25) {$0$};
\node [fill=black!20,inner sep=0pt,minimum height=0.49cm,minimum width=0.49cm] at (-0.25,-0.25) {$1$};
\node [fill=black!20,inner sep=0pt,minimum height=0.49cm,minimum width=0.49cm] at (-0.25,-0.75) {$0$};
\node [fill=black!20,inner sep=0pt,minimum height=0.49cm,minimum width=0.49cm] at (0.25,0.75) {$0$};
\node [fill=black!20,inner sep=0pt,minimum height=0.49cm,minimum width=0.49cm] at (0.25,0.25) {\small{$-1$}};
\node [fill=black!20,inner sep=0pt,minimum height=0.49cm,minimum width=0.49cm] at (0.25,-0.25) {$1$};
\node [fill=black!20,inner sep=0pt,minimum height=0.49cm,minimum width=0.49cm] at (0.25,-0.75) {$0$};
\node [anchor=north] (xlabel) at (0,-1.2) {$\textbf{w}$};
}
{\draw [->,thick,dashed] (-1.5in+2em+1.5em,-0.3) .. controls +(east:2) and +(west:1) .. (-0.55,0.8) node [pos=0.5,left] {\scriptsize{\textbf{矩阵乘}}};}
{\draw [->,thick,dashed] (-1.5in+2em+1.0em,-0.5) .. controls +(east:2) and +(west:1) .. (-0.55,0.8) ;}
{\draw [->,thick,dashed] (-1.5in+2em+0.5em,-0.7) .. controls +(east:2.5) and +(west:1) .. (-0.55,0.8) ;}
{\draw [->,thick,dashed] (-1.5in+2em,-0.9) .. controls +(east:3) and +(west:1) .. (-0.55,0.8);}
\end{scope}
\begin{scope}[yshift=6.5em,xshift=1em+3in]
{
\draw[step=0.5cm,color=orange,thick] (-0.5,-1) grid (0.5,1.0);
\foreach \y in {+0.75,+0.25,-0.25,-0.75}{
\setcounter{mycount1}{2}
\foreach \x in {-0.25,0.25}{
\node [fill=orange!20,inner sep=0pt,minimum height=0.49cm,minimum width=0.49cm] at (\x,\y) {$\number\value{mycount1}$};
\addtocounter{mycount1}{-1};
}
}
}
\node [anchor=south west] (label21) at (-0.8,0.9) {\footnotesize{①}};
\end{scope}
\begin{scope}[yshift=6em,xshift=0.5em+3in]
{
\draw[step=0.5cm,color=blue,thick] (-0.5,-1) grid (0.5,1.0);
\foreach \y in {+0.75,+0.25,-0.25,-0.75}{
\setcounter{mycount1}{2}
\foreach \x in {-0.25,0.25}{
\node [fill=blue!20,inner sep=0pt,minimum height=0.49cm,minimum width=0.49cm] at (\x,\y) {$\number\value{mycount1}$};
\addtocounter{mycount1}{-1};
}
}
}
\node [anchor=south west] (label22) at (-0.8,0.9) {\footnotesize{②}};
\end{scope}
\begin{scope}[yshift=5.5em,xshift=0em+3in]
{
\draw[step=0.5cm,color=ugreen,thick] (-0.5,-1) grid (0.5,1.0);
\foreach \y in {+0.75,+0.25,-0.25,-0.75}{
\setcounter{mycount1}{2}
\foreach \x in {-0.25,0.25}{
\node [fill=green!20,inner sep=0pt,minimum height=0.49cm,minimum width=0.49cm] at (\x,\y) {$\number\value{mycount1}$};
\addtocounter{mycount1}{-1};
}
}
}
\node [anchor=south west] (label23) at (-0.8,0.9) {\footnotesize{③}};
\end{scope}
\begin{scope}[yshift=5.0em,xshift=-0.5em+3in]
{
\draw[step=0.5cm,color=red,thick] (-0.5,-1) grid (0.5,1.0);
\foreach \y in {+0.75,+0.25,-0.25,-0.75}{
\setcounter{mycount1}{2}
\foreach \x in {-0.25,0.25}{
\node [fill=red!20,inner sep=0pt,minimum height=0.49cm,minimum width=0.49cm] at (\x,\y) {$\number\value{mycount1}$};
\addtocounter{mycount1}{-1};
}
}
}
\node [anchor=south west] (label24) at (-0.8,0.9) {\footnotesize{④}};
{
\node [anchor=north] (xlabel) at (0,-1.2) {$\textbf{x} \cdot \textbf{w}$};
\node [anchor=center] (elabel) at (-0.7in,0) {\Huge{$\textbf{=}$}};
}
\end{scope}
\end{tikzpicture}
%%------------------------------------------------------------------------------------------------------------
Book/Chapter5/Figures/fig-tensor-sample.tex 0 → 100644
%%%------------------------------------------------------------------------------------------------------------
\newcounter{mycount1}
\newcounter{mycount2}
\newcounter{mycount3}
\newcounter{mycount4}
\begin{tikzpicture}
\begin{scope}[yshift=6.5em,xshift=1em]
\setcounter{mycount1}{1}
\draw[step=0.5cm,color=orange,line width=0.2mm] (-2,-2) grid (1,1);
\foreach \y in {+0.5,-0.5,-1.5}
\foreach \x in {-1.5,-0.5,0.5}{
\node [fill=orange!20,inner sep=0pt,minimum height=0.98cm,minimum width=0.98cm] at (\x,\y) {\number\value{mycount1}};
\addtocounter{mycount1}{1};
}
\end{scope}
\begin{scope}[yshift=5.5em,xshift=0em]
\setcounter{mycount2}{2}
\draw[step=0.5cm,color=blue,line width=0.2mm] (-2,-2) grid (1,1);
\foreach \y in {+0.5,-0.5,-1.5}
\foreach \x in {-1.5,-0.5,0.5}{
\node [fill=blue!20,inner sep=0pt,minimum height=0.98cm,minimum width=0.98cm] at (\x,\y) {\number\value{mycount2}};
\addtocounter{mycount2}{1};
}
\end{scope}
\begin{scope}[yshift=4.5em,xshift=-1em]
\setcounter{mycount3}{3}
\draw[step=0.5cm,color=ugreen,line width=0.2mm] (-2,-2) grid (1,1);
\foreach \y in {+0.5,-0.5,-1.5}
\foreach \x in {-1.5,-0.5,0.5}{
\node [fill=green!20,inner sep=0pt,minimum height=0.98cm,minimum width=0.98cm] at (\x,\y) {\number\value{mycount3}};
\addtocounter{mycount3}{1};
}
\end{scope}
\begin{scope}[yshift=3.5em,xshift=-2em]
\setcounter{mycount4}{4}
\draw[step=0.5cm,color=red,line width=0.2mm] (-2,-2) grid (1,1);
\foreach \y in {+0.5,-0.5,-1.5}
\foreach \x in {-1.5,-0.5,0.5}{
\node [fill=red!20,inner sep=0pt,minimum height=0.98cm,minimum width=0.98cm] at (\x,\y) {\number\value{mycount4}};
\addtocounter{mycount4}{1};
}
\end{scope}
\end{tikzpicture}
%%%------------------------------------------------------------------------------------------------------------
\ No newline at end of file
Book/Chapter5/Figures/fig-tensor.tex 0 → 100644
%%%------------------------------------------------------------------------------------------------------------
\tdplotsetmaincoords{50}{140}
\begin{tikzpicture}[scale=2,tdplot_main_coords]
{
\draw[thick,->] (0,0,0) -- (1,0,0) node[anchor=north east]{$a$};
\draw[thick,->] (0,0,0) -- (0,1,0) node[anchor=north west]{$b$};
\draw[thick,->] (0,0,0) -- (0,0,1) node[anchor=south]{$c$};
}
\pgfmathsetmacro{\ax}{2}
\pgfmathsetmacro{\ay}{2}
\pgfmathsetmacro{\az}{1}
\tdplotsetrotatedcoords{20}{40}{00}
{
\draw[thick,color=red,tdplot_rotated_coords,->] (0,0,0)
-- (.7,0,0) node[anchor=east]{$a'$};
\draw[thick,color=green!50!black,tdplot_rotated_coords,->] (0,0,0)
-- (0,.7,0) node[anchor=west]{$b'$};
\draw[thick,color=blue,tdplot_rotated_coords,->] (0,0,0)
-- (0,0,.7) node[anchor=south]{$c'$};
}
\tdplottransformmainrot{\ax}{\ay}{\az}
\end{tikzpicture}
%%------------------------------------------------------------------------------------------------------------
Book/Chapter5/Figures/fig-the-amount-of-data-in-a-bilingual-corpus.tex 0 → 100644
%%%------------------------------------------------------------------------------------------------------------
\begin{tikzpicture}
\scriptsize{
\begin{semilogyaxis}[
width=.75\textwidth,
height=.30\textwidth,
yticklabel style={/pgf/number format/precision=1,/pgf/number format/fixed zerofill},
xticklabel style={/pgf/number format/1000 sep=},
xlabel style={yshift=0.5em},
xlabel={\footnotesize{Year}},ylabel={\footnotesize{\# of sents.}},
ymin=1,ymax=1000000000000,
xmin=1999,xmax=2020,xtick={2000,2005,2010,2015,2020},
legend style={yshift=-5em,xshift=0em,legend cell align=left,legend plot pos=right}
]
\addplot[purple,mark=square,mark=star,very thick] coordinates {(2001,10000) (2005,2000000) (2008,8000000) (2009,9000000) (2011,10000000) (2012,12000000) (2014,20000000) (2016,30000000) (2018,40000000) };
\addlegendentry{\tiny{Bi-text used in MT papers}\ \ \ \ \ \ \ \ \ \ }
{
\addplot[ublue,mark=otimes*,very thick] coordinates {(2005,10000000) (2008,100000000) (2012,3000000000) (2016,5000000000) (2019,10000000000) };
\addlegendentry{\tiny{Bi-text used in practical systems}}
}
\end{semilogyaxis}
}
\end{tikzpicture}
%%%------------------------------------------------------------------------------------------------------------
Book/Chapter5/Figures/fig-transformer-LM.tex 0 → 100644
%%%------------------------------------------------------------------------------------------------------------
\begin{tikzpicture}
\begin{scope}
\node [anchor=west] (w0) at (0,0) {\footnotesize{$w_{0}$}};
\node [anchor=west] (w1) at ([xshift=5em]w0.east) {\footnotesize{$w_{1}$}};
\node [anchor=west] (w2) at ([xshift=5em]w1.east) {\footnotesize{$w_{2}$}};
\node [anchor=west] (w3) at ([xshift=5em]w2.east) {\footnotesize{$w_{3}$}};
\node [anchor=north] (index0) at ([yshift=0.5em]w0.south) {\tiny(index)};
\node [anchor=north] (index1) at ([yshift=0.5em]w1.south) {\tiny(index)};
\node [anchor=north] (index2) at ([yshift=0.5em]w2.south) {\tiny(index)};
\node [anchor=north] (index3) at ([yshift=0.5em]w3.south) {\tiny(index)};
\node [anchor=south,draw,inner sep=3pt] (e0) at ([yshift=1em]w0.north) {\tiny{$\textbf{e}_0=w_{0} \textbf{C} + \textrm{PE}(0)$}};
\node [anchor=south,draw,inner sep=3pt] (e1) at ([yshift=1em]w1.north) {\tiny{$\textbf{e}_1=w_{1} \textbf{C} + \textrm{PE}(1)$}};
\node [anchor=south,draw,inner sep=3pt] (e2) at ([yshift=1em]w2.north) {\tiny{$\textbf{e}_2=w_{2} \textbf{C} + \textrm{PE}(2)$}};
\node [anchor=south,draw,inner sep=3pt] (e3) at ([yshift=1em]w3.north) {\tiny{$\textbf{e}_3=w_{3} \textbf{C} + \textrm{PE}(3)$}};
\node [anchor=south,draw,inner sep=3pt] (h0) at ([xshift=-0.5em, yshift=1.5em]e0.north) {\tiny{$\textbf{h}_{0}=\textrm{SelfAtt}(\textbf{e}_0,\textbf{e}_3)$}};
\node [anchor=south,draw,inner sep=3pt] (h1) at ([xshift=0.5em, yshift=1.5em]e1.north) {\tiny{$\textbf{h}_{1}=\textrm{SelfAtt}(\textbf{e}_1,\textbf{e}_3)$}};
\node [anchor=south,draw,inner sep=3pt] (h2) at ([xshift=1.5em, yshift=1.5em]e2.north) {\tiny{$\textbf{h}_{2}=\textrm{SelfAtt}(\textbf{e}_2,\textbf{e}_3)$}};
\node [anchor=south,draw,minimum width=9em,inner sep=3pt] (f1) at ([xshift=0.5em, yshift=1.5em]h2.north) {\tiny{$\textbf{f}_3=\textrm{FNN}([\textbf{h}_0,\textbf{h}_1,\textbf{h}_2,\textbf{e}_3])$}};
\node [anchor=south,draw,minimum width=9em,inner sep=3pt] (o1) at ([yshift=1em]f1.north) {\tiny{$\textbf{y}=\textrm{Softmax}(f_3 \textbf{U})$}};
\node [anchor=south] (ylabel) at ([yshift=1em]o1.north) {\footnotesize{$\textrm{P}(w_4|w_{0}w_{1}w_{2}w_{3})$}};
\draw [->] ([yshift=0.1em]w0.north) -- ([yshift=-0.1em]e0.south);
\draw [->] ([yshift=0.1em]w1.north) -- ([yshift=-0.1em]e1.south);
\draw [->] ([yshift=0.1em]w2.north) -- ([yshift=-0.1em]e2.south);
\draw [->] ([yshift=0.1em]w3.north) -- ([yshift=-0.1em]e3.south);
\draw [->] ([yshift=0.1em]e0.north) -- ([xshift=0em,yshift=-0.1em]h0.south);
\draw [->] ([yshift=0.1em]e1.north) -- ([xshift=-0.5em,yshift=-0.1em]h1.south);
\draw [->] ([yshift=0.1em]e2.north) -- ([xshift=-1em,yshift=-0.1em]h2.south);
\draw [->] ([yshift=0.1em]e3.north) -- ([xshift=1em,yshift=-0.1em]h0.south);
\draw [->] ([yshift=0.1em]e3.north) -- ([xshift=1em,yshift=-0.1em]h1.south);
\draw [->] ([yshift=0.1em]e3.north) -- ([xshift=1em,yshift=-0.1em]h2.south);
\draw [->] ([yshift=0.1em]h0.north) -- ([xshift=-2em,yshift=-0.1em]f1.south);
\draw [->] ([yshift=0.1em]e3.north) -- ([xshift=2em,yshift=-0.1em]f1.south);
\draw [->] ([yshift=0.1em]h1.north) -- ([xshift=-1em,yshift=-0.1em]f1.south);
\draw [->] ([yshift=0.1em]h2.north) -- ([xshift=0em,yshift=-0.1em]f1.south);
\draw [->] ([yshift=0.1em]f1.north) -- ([yshift=-0.1em]o1.south);
\draw [->] ([yshift=0.1em]o1.north) -- ([yshift=-0.1em]ylabel.south);
{
\node [anchor=south,draw,inner sep=3pt,fill=blue!20!white] (e0) at ([yshift=1em]w0.north) {\tiny{$\textbf{e}_0=w_{0} \textbf{C} + \textrm{PE}(0)$}};
\node [anchor=south,draw,inner sep=3pt,fill=blue!20!white] (e1) at ([yshift=1em]w1.north) {\tiny{$\textbf{e}_1=w_{1} \textbf{C} + \textrm{PE}(1)$}};
\node [anchor=south,draw,inner sep=3pt,fill=blue!20!white] (e2) at ([yshift=1em]w2.north) {\tiny{$\textbf{e}_2=w_{2} \textbf{C} + \textrm{PE}(2)$}};
\node [anchor=south,draw,inner sep=3pt,fill=blue!20!white] (e3) at ([yshift=1em]w3.north) {\tiny{$\textbf{e}_3=w_{3} \textbf{C} + \textrm{PE}(3)$}};
}
{
\node [anchor=south,draw,inner sep=3pt,fill=ugreen!20!white] (h0) at ([xshift=-0.5em, yshift=1.5em]e0.north) {\tiny{$\textbf{h}_{0}=\textrm{SelfAtt}(\textbf{e}_0,\textbf{e}_3)$}};
\node [anchor=south,draw,inner sep=3pt,fill=ugreen!20!white] (h1) at ([xshift=0.5em, yshift=1.5em]e1.north) {\tiny{$\textbf{h}_{1}=\textrm{SelfAtt}(\textbf{e}_1,\textbf{e}_3)$}};
\node [anchor=south,draw,inner sep=3pt,fill=ugreen!20!white] (h2) at ([xshift=1.5em, yshift=1.5em]e2.north) {\tiny{$\textbf{h}_{2}=\textrm{SelfAtt}(\textbf{e}_2,\textbf{e}_3)$}};
}
{
\node [anchor=south,draw,minimum width=9em,inner sep=3pt,fill=orange!20!white] (f1) at ([xshift=0.5em, yshift=1.5em]h2.north) {\tiny{$\textbf{f}_3=\textrm{FNN}([\textbf{h}_0,\textbf{h}_1,\textbf{h}_2,\textbf{e}_3])$}};
\node [anchor=south,draw,minimum width=9em,inner sep=3pt,fill=orange!20!white] (o1) at ([yshift=1em]f1.north) {\tiny{$\textbf{y}=\textrm{Softmax}(f_3 \textbf{U})$}};
}
\end{scope}
\end{tikzpicture}
%%%------------------------------------------------------------------------------------------------------------
Book/Chapter5/Figures/fig-translation.tex 0 → 100644
%%%------------------------------------------------------------------------------------------------------------
\begin{tikzpicture}
\tikzstyle{neuron} = [rectangle,draw,thick,fill=red!30,red!35,minimum height=2em,minimum width=2em,font=\small]
\node[neuron,anchor=north] (a1) at (0,0) {};
\draw[->,thick] ([xshift=-2em,yshift=0em]a1.south) to ([xshift=3em,yshift=0em]a1.south);
\draw[->,thick] ([xshift=0em,yshift=-4em]a1.west) to ([xshift=0em,yshift=2em]a1.west);
\node[below] at ([xshift=0.5em,yshift=-1em]a1.west){0};
\node[below] at ([xshift=2em,yshift=-1em]a1.west){1};
\node[below] at ([xshift=-0.5em,yshift=2em]a1.west){1};
\node [anchor=west] (x) at ([xshift=-0.7em,yshift=1em]a1.south) {\Large{$\textbf{F}$}};
{
\tikzstyle{neuron} = [rectangle,draw,thick,fill=red!30,red!35,minimum height=2em,minimum width=2em,font=\small]
\node[neuron,anchor=north] (a2) at ([xshift=10em,yshift=0em]a1.south) {};
\draw[->,thick] ([xshift=-2em,yshift=0em]a2.north) to ([xshift=3em,yshift=0em]a2.north);
\draw[->,thick] ([xshift=0em,yshift=-2em]a2.west) to ([xshift=0em,yshift=4em]a2.west);
\node[above] at ([xshift=0.5em,yshift=1em]a2.west){0};
\node[above] at ([xshift=2em,yshift=1em]a2.west){1};
\node[below] at ([xshift=-0.5em,yshift=0em]a2.west){-1};
\node [anchor=west] (x) at ([xshift=-3.5cm,yshift=2em]a2.north) {\scriptsize{
$w=\begin{bmatrix}
1&0&0\\
0&-1&0\\
0&0&1
\end{bmatrix}$}
};
\node [anchor=west,rotate = 180] (x) at ([xshift=0.7em,yshift=1em]a2.south) {\Large{$\textbf{F}$}};
\draw[-stealth, line width=2pt,dashed] ([xshift=4em,yshift=0em]a1.south) to ([xshift=-3em,yshift=0em]a2.north);
}
{
\tikzstyle{neuron} = [rectangle,draw,thick,fill=red!30,red!35,minimum height=2em,minimum width=2em,font=\small]
\node[neuron,anchor=north] (a3) at ([xshift=11em,yshift=2.05em]a2.south) {};
\draw[->,thick] ([xshift=-3em,yshift=0em]a3.north) to ([xshift=2em,yshift=0em]a3.north);
\draw[->,thick] ([xshift=-1em,yshift=-2em]a3.west) to ([xshift=-1em,yshift=4em]a3.west);
\node[above] at ([xshift=-0.5em,yshift=1em]a3.west){0};
\node[above] at ([xshift=1em,yshift=1em]a3.west){1};
\node[left] at ([xshift=-0.75em,yshift=-0.5em]a3.west){-1};
\node [anchor=west,rotate = 180] (x) at ([xshift=0.7em,yshift=1em]a3.south) {\Large{$\textbf{F}$}};
\node [anchor=west] (x) at ([xshift=-4cm,yshift=2em]a3.north) {\scriptsize{
$b=\begin{bmatrix}
0.5&0&0\\
0&0&0\\
0&0&0
\end{bmatrix}$}
};
\draw[-stealth, line width=2pt,dashed] ([xshift=3em,yshift=1em]a2.east) to ([xshift=-3em,yshift=1em]a3.west);
}
\end{tikzpicture}
%%%------------------------------------------------------------------------------------------------------------
Book/Chapter5/Figures/fig-two-layer-neural-network.tex 0 → 100644
%%%------------------------------------------------------------------------------------------------------------
\begin{tikzpicture}
%% a two-layer neural network
\begin{scope}
\tikzstyle{neuronnode} = [minimum size=1.7em,circle,draw,ublue,very thick,inner sep=1pt, fill=white,align=center,drop shadow={shadow xshift=0.1em,shadow yshift=-0.1em}]
%% input and hidden layers
\node [neuronnode] (n10) at (0,0) {\tiny{$f$}\\[-1ex] \tiny{$\sum$}};
\node [neuronnode] (n11) at (1.0,0) {\tiny{$f$}\\[-1ex] \tiny{$\sum$}};
\draw [-,ublue] (n10.west) -- (n10.east);
\draw [-,ublue] (n11.west) -- (n11.east);
\node [anchor=north] (x1) at ([yshift=-4em]n11.south) {$x_1$};
\node [anchor=north] (b) at ([yshift=-4em]n10.south) {$b$};
{
\draw [->,thick] (b.north) -- ([yshift=-0.1em]n10.south);
\draw [->,thick] (x1.north) -- ([yshift=-0.1em]n10.290);
}
{
\draw [->,thick] (b.north) -- ([yshift=-0.1em]n11.250);
\draw [->,thick] (x1.north) -- ([yshift=-0.1em]n11.south);
}
{
\draw [->,thick,blue] (b.north) -- ([yshift=-0.1em]n11.250);
\draw [->,thick,purple] (x1.north) -- ([yshift=-0.1em]n11.south);
}
%% output layers
\node [neuronnode] (n20) at (0.5,4em) {\scriptsize{$\sum$}};
{\draw [->,thick,brown] ([yshift=0.1em]n10.north) -- ([yshift=-0.1em]n20.250);}
{\draw [->,thick,orange] ([yshift=0.1em]n11.north) -- ([yshift=-0.1em]n20.290);}
\node [] (y) at ([yshift=2.5em]n20.north) {$y$};
\draw [->,thick] ([yshift=0.1em]n20.north) -- (y.south);
%% sigmoid box
\begin{scope}
{
\node [anchor=west] (flabel) at ([xshift=1in]y.east) {\footnotesize{sigmoid:}};
\node [anchor=north east] (slabel) at ([xshift=0]flabel.south east) {\footnotesize{sum:}};
\node [anchor=west,inner sep=2pt] (flabel2) at (flabel.east) {\footnotesize{$f(s)=1/(1+e^{-s})$}};
\node [anchor=west,inner sep=2pt] (flabel3) at (slabel.east) {\footnotesize{$s=x_1 \cdot w + b$}};
\draw [->,thick,dotted] ([yshift=-0.3em,xshift=-0.1em]n11.60) .. controls +(east:1) and +(west:2) .. ([xshift=-0.2em]flabel.west) ;
\begin{pgfonlayer}{background}
{
\node [rectangle,inner sep=0.2em,fill=blue!20,drop shadow={shadow xshift=0.1em,shadow yshift=-0.1em}] [fit = (flabel) (flabel2) (flabel3)] (funcbox) {};
}
\end{pgfonlayer}
}
\end{scope}
\end{scope}
\end{tikzpicture}
%%%------------------------------------------------------------------------------------------------------------
Book/Chapter5/Figures/fig-w1.tex 0 → 100644
%%%------------------------------------------------------------------------------------------------------------
\begin{tikzpicture}
%% a two-layer neural network
\begin{scope}
{
\draw [->,thick] (-1.8,0) -- (1.8,0);
\draw [->,thick] (0,0) -- (0,2);
\draw [-] (-0.05,1) -- (0.05,1);
\node [anchor=east,inner sep=1pt] (label1) at (0,1) {\tiny{1}};
\node [anchor=south east,inner sep=1pt] (label2) at (0,0) {\tiny{0}};
\node [anchor=south east,inner sep=1pt] (labela) at (0.2,-0.5) {\footnotesize{(a)}};
}
{\node [anchor=north west,align=left] (wblabel) at (-1.8,2) {\scriptsize{$w_1=100$}\\[-0ex] {\scriptsize{\ $b_1=-4$}}};}
{\draw [-,very thick,ublue,rounded corners=0.1em] (-1.5,0) -- (0.5,0) -- (0.5,1) -- (1.5,1);}
\end{scope}
%---------------------------------------------------------------------------------------------
\begin{scope}[xshift=1.6in]
{
\draw [->,thick] (-1.8,0) -- (1.8,0);
\draw [->,thick] (0,0) -- (0,2);
\draw [-] (-0.05,1) -- (0.05,1);
\node [anchor=east,inner sep=1pt] (label1) at (0,1) {\tiny{1}};
\node [anchor=south east,inner sep=1pt] (label2) at (0,0) {\tiny{0}};
\node [anchor=south east,inner sep=1pt] (labelb) at (0.2,-0.5) {\footnotesize{(b)}};
}
{\node [anchor=north west,align=left] (wblabel) at (-1.8,2) {{\scriptsize{$w'_1=0.9$}}};}
{\draw [-,very thick,ublue,rounded corners=0.1em] (-1.8,0) -- (0.5,0) -- (0.5,0.9) -- (1.8,0.9);}
\end{scope}
%-----------------------------------------------------------------------------------------------
\begin{scope}[xshift=3.2in]
{
\draw [->,thick] (-1.8,0) -- (1.8,0);
\draw [->,thick] (0,0) -- (0,2);
\draw [-] (-0.05,1) -- (0.05,1);
\node [anchor=east,inner sep=1pt] (label1) at (0,1) {\tiny{1}};
\node [anchor=south east,inner sep=1pt] (label2) at (0,0) {\tiny{0}};
\node [anchor=south east,inner sep=1pt] (labelc) at (0.2,-0.5) {\footnotesize{(c)}};
}
{\node [anchor=north west,align=left] (wblabel) at (-1.8,2) {{\scriptsize{$w'_1=0.7$}}};}
{\draw [-,very thick,ublue,rounded corners=0.1em] (-1.5,0) -- (0.5,0) -- (0.5,0.7) -- (1.5,0.7);}
\end{scope}
\end{tikzpicture}
%%%------------------------------------------------------------------------------------------------------------
Book/Chapter5/Figures/fig-w2.tex 0 → 100644
%%%------------------------------------------------------------------------------------------------------------
\begin{tikzpicture}
%% a two-layer neural network
\begin{scope}
{
\draw [->,thick] (-1.8,0) -- (1.8,0);
\draw [->,thick] (0,0) -- (0,2);
\draw [-] (-0.05,1) -- (0.05,1);
\node [anchor=east,inner sep=1pt] (label1) at (0,1) {\tiny{1}};
\node [anchor=south east,inner sep=1pt] (label2) at (0,0) {\tiny{0}};
\node [anchor=south east,inner sep=1pt] (labela) at (0.2,-0.5) {\footnotesize{(a)}};
}
{\node [anchor=north west,align=left] (wblabel) at (-1.8,2) {{\scriptsize{$w'_1=0.7$}}};}
{\draw [-,very thick,ublue,rounded corners=0.1em] (-1.5,0) -- (0.5,0) -- (0.5,0.7) -- (1.5,0.7);}
\end{scope}
%---------------------------------------------------------------------------------------------
\begin{scope}[xshift=1.6in]
{
\draw [->,thick] (-1.8,0) -- (1.8,0);
\draw [->,thick] (0,0) -- (0,2);
\draw [-] (-0.05,1) -- (0.05,1);
\node [anchor=east,inner sep=1pt] (label1) at (0,1) {\tiny{1}};
\node [anchor=south east,inner sep=1pt] (label2) at (0,0) {\tiny{0}};
\node [anchor=south east,inner sep=1pt] (labelb) at (0.2,-0.5) {\footnotesize{(b)}};
}
{\node [anchor=north west,align=left] (wblabel) at (-1.8,2) {{\scriptsize{$w_2=100$}}\\[-0ex] {\scriptsize{\ $b_2=-6$}}\\[-0ex] {\scriptsize{\ $w'_2=0.7$}}};}
{\draw [-,very thick,ublue,rounded corners=0.1em] (-1.5,0) -- (0.5,0) -- (0.5,0.7) -- (0.7,0.7) -- (0.7,1.4) -- (1.5,1.4);}
\end{scope}
%-----------------------------------------------------------------------------------------------
\begin{scope}[xshift=3.2in]
{
\draw [->,thick] (-1.8,0) -- (1.8,0);
\draw [->,thick] (0,0) -- (0,2);
\draw [-] (-0.05,1) -- (0.05,1);
\node [anchor=east,inner sep=1pt] (label1) at (0,1) {\tiny{1}};
\node [anchor=south east,inner sep=1pt] (label2) at (0,0) {\tiny{0}};
\node [anchor=south east,inner sep=1pt] (labelc) at (0.2,-0.5) {\footnotesize{(c)}};
}
{\node [anchor=north west,align=left] (wblabel) at (-1.8,2) {\scriptsize{$w_2=100$}\\[-0ex] \scriptsize{\ $b_2=-6$}\\[-0ex] {\scriptsize{\ $w'_2=-0.7$}}};}
{\draw [-,very thick,ublue,rounded corners=0.1em] (-1.5,0) -- (0.5,0) -- (0.5,0.7) -- (0.7,0.7) -- (0.7,0) -- (1.5,0);}
\end{scope}
\end{tikzpicture}
%%%------------------------------------------------------------------------------------------------------------
Book/Chapter5/Figures/fig-weather-2layer.tex 0 → 100644
%%%------------------------------------------------------------------------------------------------------------
\begin{tikzpicture}
%左
\node [anchor=west,draw=ublue,minimum width=1.5em,minimum height=1.5em] (part1-1) at (0,0) {\small{$x_1$}};
\node [anchor=north,draw=ublue,minimum width=1.5em,minimum height=1.5em] (part1-2) at ([yshift=-2.5em]part1-1.south) {\small {$x_2$}};
\node [anchor=north,draw=ublue,minimum width=1.5em,minimum height=1.5em] (part1-3) at ([yshift=-2.5em]part1-2.south) {\small {$x_3$}};
\node [anchor=north,minimum width=3.0em] (part1-4) at ([yshift=-1.0em]part1-3.south) {\small {输入层}};
\node[anchor=south,minimum height=13em,minimum width=4.0em,draw=ublue,dotted,thick] (part1out) at ([xshift=0.0em,yshift=-8.5em]part1-2.north) {};
%中
\node [circle,anchor=west,draw=ublue,minimum width=2.0em] (part2-1) at ([xshift=4.6em,yshift=2.0em]part1-2.east) {\small {$a_1$}};
\node [circle,anchor=west,draw=ublue,minimum width=2.0em] (part2-2) at ([xshift=4.6em,yshift=-2.0em]part1-2.east) {\small {$a_2$}};
\node [anchor=north,minimum width=3.0em] (part2-3) at ([xshift=0.0em,yshift=-2.79em]part2-2.south) {\small {隐藏层}};
\node[anchor=south,minimum height=13em,minimum width=4.0em,draw=ublue,dotted,thick] (part2out) at ([xshift=6.4em,yshift=-8.5em]part1-2.north) {};
%右
\node [circle,anchor=west,draw=ublue,minimum width=2.0em] (part3-1) at ([xshift=10.5em,yshift=0.0em]part1-2.east) {\small {$y$}};
\node [anchor=west,draw=ublue,minimum width=1.5em,minimum height=1.5em] (part3-2) at ([xshift=1.2em]part3-1.east) {\small {$y$}};
\node [anchor=north,minimum width=3.0em] (part3-3) at ([xshift=1.4em,yshift=-4.79em]part3-1.south) {\small {输出层}};
\node[anchor=south,minimum height=13em,minimum width=6.0em,draw=ublue,dotted,thick] (part3out) at ([xshift=13.8em,yshift=-8.5em]part1-2.north) {};
%连线
\draw [->,thick,ublue](part1-1.east)--(part2-1.west);
\draw [->,thick,ublue](part1-1.east)--(part2-2.west);
\draw [->,thick,ublue](part1-2.east)--(part2-1.west);
\draw [->,thick,ublue](part1-2.east)--(part2-2.west);
\draw [->,thick,ublue](part1-3.east)--(part2-1.west);
\draw [->,thick,ublue](part1-3.east)--(part2-2.west);
\draw [->,thick,ublue](part2-1.east)--(part3-1.west);
\draw [->,thick,ublue](part2-2.east)--(part3-1.west);
\draw [->,thick,ublue](part3-1.east)--(part3-2.west);
\end{tikzpicture}
%%------------------------------------------------------------------------------------------------------------
Book/Chapter5/Figures/fig-weather-forward.tex 0 → 100644
%%%------------------------------------------------------------------------------------------------------------
\begin{tikzpicture}
\node [anchor=west,minimum width=1.5em,minimum height=1.5em] (part1) at (0,0) {\footnotesize{$y$}};
\node [anchor=north,minimum width=1.5em,minimum height=1.5em] (part1-2) at ([xshift=-1.6em,yshift=-0.3em]part1.south) {\scriptsize {$\rm {shape(1)}$}};
\node [anchor=north,draw,minimum width=4.0em,minimum height=1.5em] (part2) at ([yshift=-1.5em]part1.south) {\footnotesize {$\rm{Sigmoid}$}};
\draw [-,thick](part1.south)--(part2.north);
\node [anchor=north,minimum width=1.5em,minimum height=1.5em] (part2-2) at ([xshift=-1.6em,yshift=-0.3em]part2.south) {\scriptsize {$\rm{shape(1)}$}};
\node [anchor=north,draw,minimum width=4.0em,minimum height=1.5em] (part3) at ([yshift=-1.5em]part2.south) {\footnotesize {$\rm{ADD}$}};
\draw [-,thick](part2.south)--(part3.north);
\node [anchor=north,minimum width=1.5em,minimum height=1.5em] (part3-2) at ([xshift=-1.6em,yshift=-0.3em]part3.south) {\scriptsize {$\rm {shape(1)}$}};
\node [anchor=north,draw,minimum width=4.0em,minimum height=1.5em] (part4) at ([yshift=-1.5em]part3.south) {\footnotesize {$\rm{MUL}$}};
\draw [-,thick](part3.south)--(part4.north);
\node [anchor=north,minimum width=1.5em,minimum height=1.5em] (part4-2) at ([xshift=-1.6em,yshift=-0.2em]part4.south) {\scriptsize {$\rm {shape(2)}$}};
\node [anchor=north,minimum width=4.0em,minimum height=1.5em] (part5) at ([yshift=-1.4em]part4.south) {\footnotesize {$\mathbf a$}};
\draw [-,thick](part4.south)--([yshift=-0.1em]part5.north);
%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\node [anchor=west,minimum width=2.0em,minimum height=1.5em,draw] (part5-3) at ([xshift=0.0em,yshift=0.1em]part5.east) {\footnotesize {$\mathbf w^2$}};
\node [anchor=west,minimum width=2.0em,minimum height=1.5em,draw] (part5-4) at ([xshift=2.0em,yshift=0.0em]part5-3.east) {\footnotesize {$\mathbf b^2$}};
\draw[-,thick](part4.south)--(part5-3.north);
\draw[-,thick](part3.south)--(part5-4.north);
\node [anchor=south,minimum width=1.5em,minimum height=1.5em] (part5-3-1) at ([xshift=1.3em,yshift=-0.45em]part5-3.north) {\scriptsize {$\rm{shape(2)}$}};
\node [anchor=south,minimum width=1.5em,minimum height=1.5em] (part5-4-1) at ([xshift=1.3em,yshift=-0.45em]part5-4.north) {\scriptsize {$\rm{shape(1)}$}};
%%%%%%%%%%%%%%%%%%%%%%%%%%
\node [anchor=north,minimum width=1.5em,minimum height=1.5em] (part5-2) at ([xshift=-1.6em,yshift=-0.2em]part5.south) {\scriptsize {$\rm{shape(2)}$}};
\node [anchor=north,draw,minimum width=4.0em,minimum height=1.5em] (part6) at ([yshift=-1.4em]part5.south) {\footnotesize {$\rm{Tanh}$}};
\draw [-,thick]([yshift=0.1em]part5.south)--(part6.north);
\node [anchor=north,minimum width=1.5em,minimum height=1.5em] (part6-2) at ([xshift=-1.6em,yshift=-0.3em]part6.south) {\scriptsize {$\rm{shape(2)}$}};
\node [anchor=north,draw,minimum width=4.0em,minimum height=1.5em] (part7) at ([yshift=-1.5em]part6.south) {\footnotesize {$\rm{ADD}$}};
\draw [-,thick](part6.south)--(part7.north);
\node [anchor=north,minimum width=1.5em,minimum height=1.5em] (part7-2) at ([xshift=-1.6em,yshift=-0.3em]part7.south) {\scriptsize {$\rm{shape(2)}$}};
\node [anchor=north,draw,minimum width=4.0em,minimum height=1.5em] (part8) at ([yshift=-1.5em]part7.south) {\footnotesize {$\rm{MUL}$}};
\draw [-,thick](part7.south)--(part8.north);
\node [anchor=north,minimum width=1.5em,minimum height=1.5em] (part8-2) at ([xshift=-1.6em,yshift=-0.2em]part8.south) {\scriptsize{$\rm{shape(2)}$}};
\node [anchor=north,minimum width=4.0em,minimum height=1.5em] (part9) at ([yshift=-1.4em]part8.south) {\footnotesize {$\mathbf x$}};
\draw [-,thick](part8.south)--([yshift=-0.1em]part9.north);
%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\node [anchor=west,minimum width=2.0em,minimum height=1.5em,draw] (part9-3) at ([xshift=0.0em,yshift=0.1em]part9.east) {\footnotesize {$\mathbf w’$}};
\node [anchor=west,minimum width=2.0em,minimum height=1.5em,draw] (part9-4) at ([xshift=2.0em,yshift=0.0em]part9-3.east) {\footnotesize {$\mathbf b’$}};
\draw[-,thick](part8.south)--(part9-3.north);
\draw[-,thick](part7.south)--(part9-4.north);
\node [anchor=south,minimum width=1.5em,minimum height=1.5em] (part9-3-1) at ([xshift=1.5em,yshift=-0.45em]part9-3.north) {\scriptsize {$\rm{shape(3,2)}$}};
\node [anchor=south,minimum width=1.5em,minimum height=1.5em] (part9-4-1) at ([xshift=1.3em,yshift=-0.45em]part9-4.north) {\scriptsize {$\rm{shape(2)}$}};
%%%%%%%%%%%%%%%%%%%%%%%%%%
\end{tikzpicture}
%%------------------------------------------------------------------------------------------------------------
Book/Chapter5/Figures/fig-weather-model.tex 0 → 100644
%%%------------------------------------------------------------------------------------------------------------
\begin{tikzpicture}
%左
\node [anchor=west,draw=ublue,minimum width=3.0em] (part1-1) at (0,0) {\small{天空状况}};
\node [anchor=north,draw=ublue,minimum width=3.0em] (part1-2) at ([yshift=-2.5em]part1-1.south) {\small {低空气温}};
\node [anchor=north,draw=ublue,minimum width=3.0em] (part1-3) at ([yshift=-2.5em]part1-2.south) {\small {水平气压}};
\node [anchor=north,minimum width=3.0em] (part1-4) at ([yshift=-1.0em]part1-3.south) {\small {输入层}};
\node[anchor=south,minimum height=13em,minimum width=6.0em,draw=ublue,dotted,thick] (part1out) at ([xshift=0.0em,yshift=-8.5em]part1-2.north) {};
%中
\node [circle,anchor=west,draw=ublue,minimum width=2.2em] (part2-1) at ([xshift=4.0em,yshift=2.0em]part1-2.east) {\footnotesize {温度}};
\node [circle,anchor=west,draw=ublue,minimum width=2.2em] (part2-2) at ([xshift=4.0em,yshift=-2.0em]part1-2.east) {\footnotesize {风速}};
\node [anchor=north,minimum width=3.0em] (part2-3) at ([xshift=0.0em,yshift=-2.42em]part2-2.south) {\small {隐藏层}};
\node[anchor=south,minimum height=13em,minimum width=4.0em,draw=ublue,dotted,thick] (part2out) at ([xshift=7.4em,yshift=-8.5em]part1-2.north) {};
%右
\node [anchor=west,draw=ublue,minimum width=3.0em] (part3-1) at ([xshift=10.0em,yshift=0.0em]part1-2.east) {\small {穿衣指数}};
\node [anchor=north,minimum width=3.0em] (part3-2) at ([yshift=-5.05em]part3-1.south) {\small {输出层}};
\node[anchor=south,minimum height=13em,minimum width=6.0em,draw=ublue,dotted,thick] (part3out) at ([xshift=14.4em,yshift=-8.5em]part1-2.north) {};
%连线
\draw [->,thick,ublue](part1-1.east)--(part2-1.west);
\draw [->,thick,ublue](part1-1.east)--(part2-2.west);
\draw [->,thick,ublue](part1-2.east)--(part2-1.west);
\draw [->,thick,ublue](part1-2.east)--(part2-2.west);
\draw [->,thick,ublue](part1-3.east)--(part2-1.west);
\draw [->,thick,ublue](part1-3.east)--(part2-2.west);
\draw [->,thick,ublue](part2-1.east)--(part3-1.west);
\draw [->,thick,ublue](part2-2.east)--(part3-1.west);
\end{tikzpicture}
%%%------------------------------------------------------------------------------------------------------------
Book/Chapter5/Figures/fig-weather.tex 0 → 100644
%%%------------------------------------------------------------------------------------------------------------
\begin{tikzpicture}
\begin{scope}
%左
\node [anchor=west,draw=ublue,minimum width=2.5em] (part1-1) at (0,0) {\footnotesize{天空状况}};
\node [anchor=north,draw=ublue,minimum width=2.5em] (part1-2) at ([yshift=-2em]part1-1.south) {\footnotesize {低空气温}};
\node [anchor=north,draw=ublue,minimum width=2.5em] (part1-3) at ([yshift=-2em]part1-2.south) {\footnotesize {水平气压}};
\node [anchor=north,minimum width=2.5em] (part1-4) at ([yshift=-1.0em]part1-3.south) {\footnotesize {输入层}};
\node[anchor=south,minimum height=12em,minimum width=5.0em,draw=ublue,dotted,thick] (part1out) at ([xshift=0.0em,yshift=-8em]part1-2.north) {};
%中
\node [circle,anchor=west,draw=ublue,minimum width=2.2em] (part2-1) at ([xshift=2.0em,yshift=1.5em]part1-2.east) {\footnotesize {温度}};
\node [circle,anchor=west,draw=ublue,minimum width=2.2em] (part2-2) at ([xshift=2.0em,yshift=-1.5em]part1-2.east) {\footnotesize {风速}};
\node [anchor=north,minimum width=3.0em] (part2-3) at ([xshift=0.0em,yshift=-2.42em]part2-2.south) {\footnotesize{隐藏层}};
\node [anchor=north] (labela) at ([xshift=0.0em,yshift=-1em]part2-3.south) {\footnotesize {(a)}};
\node[anchor=south,minimum height=12em,minimum width=4.0em,draw=ublue,dotted,thick] (part2out) at ([xshift=5.2em,yshift=-8em]part1-2.north) {};
%右
\node [anchor=west,draw=ublue,minimum width=3.0em] (part3-1) at ([xshift=6.5em,yshift=0.0em]part1-2.east) {\footnotesize {穿衣指数}};
\node [anchor=north,minimum width=3.0em] (part3-2) at ([yshift=-4.65em]part3-1.south) {\footnotesize{输出层}};
\node[anchor=south,minimum height=12em,minimum width=5em,draw=ublue,dotted,thick] (part3out) at ([xshift=10.5em,yshift=-8em]part1-2.north) {};
%连线
\draw [->,thick,ublue](part1-1.east)--(part2-1.west);
\draw [->,thick,ublue](part1-1.east)--(part2-2.west);
\draw [->,thick,ublue](part1-2.east)--(part2-1.west);
\draw [->,thick,ublue](part1-2.east)--(part2-2.west);
\draw [->,thick,ublue](part1-3.east)--(part2-1.west);
\draw [->,thick,ublue](part1-3.east)--(part2-2.west);
\draw [->,thick,ublue](part2-1.east)--(part3-1.west);
\draw [->,thick,ublue](part2-2.east)--(part3-1.west);
\end{scope}
\begin{scope}[xshift=3in]
%左
\node [anchor=west,draw=ublue,minimum width=1.5em,minimum height=1.5em] (part1-1) at (0,0) {\footnotesize{$x_1$}};
\node [anchor=north,draw=ublue,minimum width=1.5em,minimum height=1.5em] (part1-2) at ([yshift=-2em]part1-1.south) {\footnotesize{$x_2$}};
\node [anchor=north,draw=ublue,minimum width=1.5em,minimum height=1.5em] (part1-3) at ([yshift=-2em]part1-2.south) {\footnotesize{$x_3$}};
\node [anchor=north,minimum width=3.0em] (part1-4) at ([yshift=-1.0em]part1-3.south) {\footnotesize {输入层}};
\node[anchor=south,minimum height=12em,minimum width=3.4em,draw=ublue,dotted,thick] (part1out) at ([xshift=0.0em,yshift=-8em]part1-2.north) {};
%中
\node [circle,anchor=west,draw=ublue,minimum width=2.0em] (part2-1) at ([xshift=2.2em,yshift=1.5em]part1-2.east) {\footnotesize{$a_1$}};
\node [circle,anchor=west,draw=ublue,minimum width=2.0em] (part2-2) at ([xshift=2.2em,yshift=-1.5em]part1-2.east) {\footnotesize {$a_2$}};
\node [anchor=north,minimum width=3.0em] (part2-3) at ([xshift=0.0em,yshift=-2.79em]part2-2.south) {\footnotesize {隐藏层}};
\node [anchor=north] (labelb) at ([xshift=1em,yshift=-1em]part2-3.south) {\footnotesize {(b)}};
\node[anchor=south,minimum height=12em,minimum width=3.4em,draw=ublue,dotted,thick] (part2out) at ([xshift=4em,yshift=-8em]part1-2.north) {};
%右
\node [circle,anchor=west,draw=ublue,minimum width=2.0em] (part3-1) at ([xshift=6.2em,yshift=0.0em]part1-2.east) {\footnotesize{$y$}};
\node [anchor=west,draw=ublue,minimum width=1.5em,minimum height=1.5em] (part3-2) at ([xshift=1.2em]part3-1.east) {\footnotesize {$y$}};
\node [anchor=north,minimum width=3.0em] (part3-3) at ([xshift=1.4em,yshift=-4.3em]part3-1.south) {\footnotesize {输出层}};
\node[anchor=south,minimum height=12em,minimum width=5.5em,draw=ublue,dotted,thick] (part3out) at ([xshift=9.2em,yshift=-8em]part1-2.north) {};
%连线
\draw [->,thick,ublue](part1-1.east)--(part2-1.west);
\draw [->,thick,ublue](part1-1.east)--(part2-2.west);
\draw [->,thick,ublue](part1-2.east)--(part2-1.west);
\draw [->,thick,ublue](part1-2.east)--(part2-2.west);
\draw [->,thick,ublue](part1-3.east)--(part2-1.west);
\draw [->,thick,ublue](part1-3.east)--(part2-2.west);
\draw [->,thick,ublue](part2-1.east)--(part3-1.west);
\draw [->,thick,ublue](part2-2.east)--(part3-1.west);
\draw [->,thick,ublue](part3-1.east)--(part3-2.west);
\end{scope}
\end{tikzpicture}
%%------------------------------------------------------------------------------------------------------------
Book/Chapter5/Figures/fig-weight.tex 0 → 100644
%%%------------------------------------------------------------------------------------------------------------
\begin{tikzpicture}
%% a two-layer neural network
\begin{scope}
{
\draw [->,thick] (-1.8,0) -- (1.8,0);
\draw [->,thick] (0,0) -- (0,2);
\draw [-] (-0.05,1) -- (0.05,1);
\node [anchor=east,inner sep=1pt] (label1) at (0,1) {\tiny{1}};
\node [anchor=south east,inner sep=1pt] (label2) at (0,0) {\tiny{0}};
\node [anchor=south east,inner sep=1pt] (labela) at (0.2,-0.5) {\footnotesize{(a)}};
}
{\node [anchor=north west,align=left] (wblabel) at (-1.8,2) {\scriptsize{$w_1=1$}\\[-0ex] \scriptsize{\ $b_1=0$}};}
{\draw [-,very thick,ublue,domain=-1.5:1.5,samples=100] plot (\x,{1/(1+exp(-2*\x))});}
\end{scope}
%---------------------------------------------------------------------------------------------
\begin{scope}[xshift=1.6in]
{
\draw [->,thick] (-1.8,0) -- (1.8,0);
\draw [->,thick] (0,0) -- (0,2);
\draw [-] (-0.05,1) -- (0.05,1);
\node [anchor=east,inner sep=1pt] (label1) at (0,1) {\tiny{1}};
\node [anchor=south east,inner sep=1pt] (label2) at (0,0) {\tiny{0}};
\node [anchor=south east,inner sep=1pt] (labelb) at (0.2,-0.5) {\footnotesize{(b)}};
}
{\node [anchor=north west,align=left] (wblabel) at (-1.8,2) {{\scriptsize{$w_1=10$}}\\[-0ex] \scriptsize{\ $b_1=0$}};}
{\draw [-,very thick,ublue,domain=-1.5:1.5,samples=100] plot (\x,{1/(1+exp(-4*\x))});}
\end{scope}
%-----------------------------------------------------------------------------------------------
\begin{scope}[xshift=3.2in]
{
\draw [->,thick] (-1.8,0) -- (1.8,0);
\draw [->,thick] (0,0) -- (0,2);
\draw [-] (-0.05,1) -- (0.05,1);
\node [anchor=east,inner sep=1pt] (label1) at (0,1) {\tiny{1}};
\node [anchor=south east,inner sep=1pt] (label2) at (0,0) {\tiny{0}};
\node [anchor=south east,inner sep=1pt] (labelc) at (0.2,-0.5) {\footnotesize{(c)}};
}
{\node [anchor=north west,align=left] (wblabel) at (-1.8,2) {{\scriptsize{$w_1=100$}}\\[-0ex] \scriptsize{\ $b_1=0$}};}
{\draw [-,very thick,ublue,rounded corners=0.1em] (-1.5,0) -- (0,0) -- (0,1) -- (1.5,1);}
\end{scope}
\end{tikzpicture}
%%%------------------------------------------------------------------------------------------------------------
Book/Chapter5/chapter5.tex 0 → 100644
This source diff could not be displayed because it is too large. You can view the blob instead.
Book/bibliography.bib
......@@ -1331,3 +1331,540 @@ url = {http://papers.nips.cc/paper/5346-sequence-to-sequence-learning-with-neura
year={2017},
}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%%chapter5------------------------------------------------------------
@article{baydin2017automatic,
title={Automatic differentiation in machine learning: a survey},
author={Baydin, At{\i}l{\i}m G{\"u}nes and Pearlmutter, Barak A and Radul, Alexey Andreyevich and Siskind, Jeffrey Mark},
journal={The Journal of Machine Learning Research},
volume={18},
number={1},
pages={5595--5637},
year={2017},
publisher={JMLR. org}
}
@article{qian1999momentum,
title={On the momentum term in gradient descent learning algorithms},
author={Qian, Ning},
journal={Neural networks},
volume={12},
number={1},
pages={145--151},
year={1999},
publisher={Elsevier}
}
@article{duchi2011adaptive,
title={Adaptive subgradient methods for online learning and stochastic optimization},
author={Duchi, John and Hazan, Elad and Singer, Yoram},
journal={Journal of machine learning research},
volume={12},
number={Jul},
pages={2121--2159},
year={2011}
}
@article{tieleman2012rmsprop,
title={Rmsprop: Divide the gradient by a running average of its recent magnitude. coursera: Neural networks for machine learning},
author={Tieleman, Tijmen and Hinton, Geoffrey},
journal={COURSERA Neural Networks Mach. Learn},
year={2012}
}
@article{kingma2014adam,
title={Adam: A method for stochastic optimization},
author={Kingma, Diederik P and Ba, Jimmy},
journal={arXiv preprint arXiv:1412.6980},
year={2014}
}
@inproceedings{xiao2017fast,
title={Fast Parallel Training of Neural Language Models.},
author={Xiao, Tong and Zhu, Jingbo and Liu, Tongran and Zhang, Chunliang},
booktitle={IJCAI},
pages={4193--4199},
year={2017}
}
@article{ioffe2015batch,
title={Batch normalization: Accelerating deep network training by reducing internal covariate shift},
author={Ioffe, Sergey and Szegedy, Christian},
journal={arXiv preprint arXiv:1502.03167},
year={2015}
}
@article{ba2016layer,
title={Layer normalization},
author={Ba, Jimmy Lei and Kiros, Jamie Ryan and Hinton, Geoffrey E},
journal={arXiv preprint arXiv:1607.06450},
year={2016}
}
@inproceedings{he2016deep,
title={Deep residual learning for image recognition},
author={He, Kaiming and Zhang, Xiangyu and Ren, Shaoqing and Sun, Jian},
booktitle={Proceedings of the IEEE conference on computer vision and pattern recognition},
pages={770--778},
year={2016}
}
@article{bengio2003neural,
title={A neural probabilistic language model},
author={Bengio, Yoshua and Ducharme, R{\'e}jean and Vincent, Pascal and Jauvin, Christian},
journal={Journal of machine learning research},
volume={3},
number={Feb},
pages={1137--1155},
year={2003}
}
@inproceedings{mikolov2010recurrent,
title={Recurrent neural network based language model},
author={Mikolov, Tom{\'a}{\v{s}} and Karafi{\'a}t, Martin and Burget, Luk{\'a}{\v{s}} and {\v{C}}ernock{\`y}, Jan and Khudanpur, Sanjeev},
booktitle={Eleventh annual conference of the international speech communication association},
year={2010}
}
@inproceedings{vaswani2017attention,
title={Attention is all you need},
author={Vaswani, Ashish and Shazeer, Noam and Parmar, Niki and Uszkoreit, Jakob and Jones, Llion and Gomez, Aidan N and Kaiser, {\L}ukasz and Polosukhin, Illia},
booktitle={Advances in neural information processing systems},
pages={5998--6008},
year={2017}
}
@inproceedings{mikolov2013distributed,
title={Distributed representations of words and phrases and their compositionality},
author={Mikolov, Tomas and Sutskever, Ilya and Chen, Kai and Corrado, Greg S and Dean, Jeff},
booktitle={Advances in neural information processing systems},
pages={3111--3119},
year={2013}
}
@inproceedings{pennington2014glove,
title={Glove: Global vectors for word representation},
author={Pennington, Jeffrey and Socher, Richard and Manning, Christopher D},
booktitle={Proceedings of the 2014 conference on empirical methods in natural language processing (EMNLP)},
pages={1532--1543},
year={2014}
}
@article{peters2018deep,
title={Deep contextualized word representations},
author={Peters, Matthew E and Neumann, Mark and Iyyer, Mohit and Gardner, Matt and Clark, Christopher and Lee, Kenton and Zettlemoyer, Luke},
journal={arXiv preprint arXiv:1802.05365},
year={2018}
}
@article{radford2018improving,
title={Improving language understanding by generative pre-training},
author={Radford, Alec and Narasimhan, Karthik and Salimans, Tim and Sutskever, Ilya},
journal={URL https://s3-us-west-2. amazonaws. com/openai-assets/researchcovers/languageunsupervised/language understanding paper. pdf},
year={2018}
}
@article{devlin2018bert,
title={Bert: Pre-training of deep bidirectional transformers for language understanding},
author={Devlin, Jacob and Chang, Ming-Wei and Lee, Kenton and Toutanova, Kristina},
journal={arXiv preprint arXiv:1810.04805},
year={2018}
}
@article{goldberg2017neural,
title={Neural network methods for natural language processing},
author={Goldberg, Yoav},
journal={Synthesis Lectures on Human Language Technologies},
volume={10},
number={1},
pages={1--309},
year={2017},
publisher={Morgan \& Claypool Publishers}
}
@article{lecun2015deep,
title={Deep learning},
author={LeCun, Yann and Bengio, Yoshua and Hinton, Geoffrey},
journal={nature},
volume={521},
number={7553},
pages={436--444},
year={2015},
publisher={Nature Publishing Group}
}
@article{guidotti2018survey,
title={A survey of methods for explaining black box models},
author={Guidotti, Riccardo and Monreale, Anna and Ruggieri, Salvatore and Turini, Franco and Giannotti, Fosca and Pedreschi, Dino},
journal={ACM computing surveys (CSUR)},
volume={51},
number={5},
pages={1--42},
year={2018},
publisher={ACM New York, NY, USA}
}
@inproceedings{koh2017understanding,
title={Understanding black-box predictions via influence functions},
author={Koh, Pang Wei and Liang, Percy},
booktitle={Proceedings of the 34th International Conference on Machine Learning-Volume 70},
pages={1885--1894},
year={2017},
organization={JMLR. org}
}
@article{arthur2016incorporating,
title={Incorporating discrete translation lexicons into neural machine translation},
author={Arthur, Philip and Neubig, Graham and Nakamura, Satoshi},
journal={arXiv preprint arXiv:1606.02006},
year={2016}
}
@article{zhang2018prior,
title={Prior knowledge integration for neural machine translation using posterior regularization},
author={Zhang, Jiacheng and Liu, Yang and Luan, Huanbo and Xu, Jingfang and Sun, Maosong},
journal={arXiv preprint arXiv:1811.01100},
year={2018}
}
@inproceedings{zollmann2006syntax,
title={Syntax augmented machine translation via chart parsing},
author={Zollmann, Andreas and Venugopal, Ashish},
booktitle={Proceedings of the Workshop on Statistical Machine Translation},
pages={138--141},
year={2006},
organization={Association for Computational Linguistics}
}
@inproceedings{charniak2003syntax,
title={Syntax-based language models for statistical machine translation},
author={Charniak, Eugene and Knight, Kevin and Yamada, Kenji},
booktitle={Proceedings of MT Summit IX},
pages={40--46},
year={2003}
}
@article{stahlberg2016syntactically,
title={Syntactically guided neural machine translation},
author={Stahlberg, Felix and Hasler, Eva and Waite, Aurelien and Byrne, Bill},
journal={arXiv preprint arXiv:1605.04569},
year={2016}
}
@inproceedings{plank2013embedding,
title={Embedding semantic similarity in tree kernels for domain adaptation of relation extraction},
author={Plank, Barbara and Moschitti, Alessandro},
booktitle={Proceedings of the 51st Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers)},
pages={1498--1507},
year={2013}
}
@inproceedings{perozzi2014deepwalk,
title={Deepwalk: Online learning of social representations},
author={Perozzi, Bryan and Al-Rfou, Rami and Skiena, Steven},
booktitle={Proceedings of the 20th ACM SIGKDD international conference on Knowledge discovery and data mining},
pages={701--710},
year={2014}
}
@article{collobert2011natural,
title={Natural language processing (almost) from scratch},
author={Collobert, Ronan and Weston, Jason and Bottou, L{\'e}on and Karlen, Michael and Kavukcuoglu, Koray and Kuksa, Pavel},
journal={Journal of machine learning research},
volume={12},
number={Aug},
pages={2493--2537},
year={2011}
}
@article{mikolov2013efficient,
title={Efficient estimation of word representations in vector space},
author={Mikolov, Tomas and Chen, Kai and Corrado, Greg and Dean, Jeffrey},
journal={arXiv preprint arXiv:1301.3781},
year={2013}
}
@inproceedings{mccann2017learned,
title={Learned in translation: Contextualized word vectors},
author={McCann, Bryan and Bradbury, James and Xiong, Caiming and Socher, Richard},
booktitle={Advances in Neural Information Processing Systems},
pages={6294--6305},
year={2017}
}
@article{radford2019language,
title={Language models are unsupervised multitask learners},
author={Radford, Alec and Wu, Jeffrey and Child, Rewon and Luan, David and Amodei, Dario and Sutskever, Ilya},
journal={OpenAI Blog},
volume={1},
number={8},
pages={9},
year={2019}
}
@article{lample2019cross,
title={Cross-lingual language model pretraining},
author={Lample, Guillaume and Conneau, Alexis},
journal={arXiv preprint arXiv:1901.07291},
year={2019}
}
@article{song2019mass,
title={Mass: Masked sequence to sequence pre-training for language generation},
author={Song, Kaitao and Tan, Xu and Qin, Tao and Lu, Jianfeng and Liu, Tie-Yan},
journal={arXiv preprint arXiv:1905.02450},
year={2019}
}
@inproceedings{yang2019xlnet,
title={Xlnet: Generalized autoregressive pretraining for language understanding},
author={Yang, Zhilin and Dai, Zihang and Yang, Yiming and Carbonell, Jaime and Salakhutdinov, Russ R and Le, Quoc V},
booktitle={Advances in neural information processing systems},
pages={5754--5764},
year={2019}
}
@article{lan2019albert,
title={Albert: A lite bert for self-supervised learning of language representations},
author={Lan, Zhenzhong and Chen, Mingda and Goodman, Sebastian and Gimpel, Kevin and Sharma, Piyush and Soricut, Radu},
journal={arXiv preprint arXiv:1909.11942},
year={2019}
}
%%-----------------------------------------------------------------------------------------
%%%%%chapter5------------------------------------------------------------
@article{baydin2017automatic,
title={Automatic differentiation in machine learning: a survey},
author={Baydin, At{\i}l{\i}m G{\"u}nes and Pearlmutter, Barak A and Radul, Alexey Andreyevich and Siskind, Jeffrey Mark},
journal={The Journal of Machine Learning Research},
volume={18},
number={1},
pages={5595--5637},
year={2017},
publisher={JMLR. org}
}
@article{qian1999momentum,
title={On the momentum term in gradient descent learning algorithms},
author={Qian, Ning},
journal={Neural networks},
volume={12},
number={1},
pages={145--151},
year={1999},
publisher={Elsevier}
}
@article{duchi2011adaptive,
title={Adaptive subgradient methods for online learning and stochastic optimization},
author={Duchi, John and Hazan, Elad and Singer, Yoram},
journal={Journal of machine learning research},
volume={12},
number={Jul},
pages={2121--2159},
year={2011}
}
@article{tieleman2012rmsprop,
title={Rmsprop: Divide the gradient by a running average of its recent magnitude. coursera: Neural networks for machine learning},
author={Tieleman, Tijmen and Hinton, Geoffrey},
journal={COURSERA Neural Networks Mach. Learn},
year={2012}
}
@article{kingma2014adam,
title={Adam: A method for stochastic optimization},
author={Kingma, Diederik P and Ba, Jimmy},
journal={arXiv preprint arXiv:1412.6980},
year={2014}
}
@inproceedings{xiao2017fast,
title={Fast Parallel Training of Neural Language Models.},
author={Xiao, Tong and Zhu, Jingbo and Liu, Tongran and Zhang, Chunliang},
booktitle={IJCAI},
pages={4193--4199},
year={2017}
}
@article{ioffe2015batch,
title={Batch normalization: Accelerating deep network training by reducing internal covariate shift},
author={Ioffe, Sergey and Szegedy, Christian},
journal={arXiv preprint arXiv:1502.03167},
year={2015}
}
@article{ba2016layer,
title={Layer normalization},
author={Ba, Jimmy Lei and Kiros, Jamie Ryan and Hinton, Geoffrey E},
journal={arXiv preprint arXiv:1607.06450},
year={2016}
}
@inproceedings{he2016deep,
title={Deep residual learning for image recognition},
author={He, Kaiming and Zhang, Xiangyu and Ren, Shaoqing and Sun, Jian},
booktitle={Proceedings of the IEEE conference on computer vision and pattern recognition},
pages={770--778},
year={2016}
}
@article{bengio2003neural,
title={A neural probabilistic language model},
author={Bengio, Yoshua and Ducharme, R{\'e}jean and Vincent, Pascal and Jauvin, Christian},
journal={Journal of machine learning research},
volume={3},
number={Feb},
pages={1137--1155},
year={2003}
}
@inproceedings{mikolov2010recurrent,
title={Recurrent neural network based language model},
author={Mikolov, Tom{\'a}{\v{s}} and Karafi{\'a}t, Martin and Burget, Luk{\'a}{\v{s}} and {\v{C}}ernock{\`y}, Jan and Khudanpur, Sanjeev},
booktitle={Eleventh annual conference of the international speech communication association},
year={2010}
}
@inproceedings{vaswani2017attention,
title={Attention is all you need},
author={Vaswani, Ashish and Shazeer, Noam and Parmar, Niki and Uszkoreit, Jakob and Jones, Llion and Gomez, Aidan N and Kaiser, {\L}ukasz and Polosukhin, Illia},
booktitle={Advances in neural information processing systems},
pages={5998--6008},
year={2017}
}
@inproceedings{mikolov2013distributed,
title={Distributed representations of words and phrases and their compositionality},
author={Mikolov, Tomas and Sutskever, Ilya and Chen, Kai and Corrado, Greg S and Dean, Jeff},
booktitle={Advances in neural information processing systems},
pages={3111--3119},
year={2013}
}
@inproceedings{pennington2014glove,
title={Glove: Global vectors for word representation},
author={Pennington, Jeffrey and Socher, Richard and Manning, Christopher D},
booktitle={Proceedings of the 2014 conference on empirical methods in natural language processing (EMNLP)},
pages={1532--1543},
year={2014}
}
@article{peters2018deep,
title={Deep contextualized word representations},
author={Peters, Matthew E and Neumann, Mark and Iyyer, Mohit and Gardner, Matt and Clark, Christopher and Lee, Kenton and Zettlemoyer, Luke},
journal={arXiv preprint arXiv:1802.05365},
year={2018}
}
@article{radford2018improving,
title={Improving language understanding by generative pre-training},
author={Radford, Alec and Narasimhan, Karthik and Salimans, Tim and Sutskever, Ilya},
journal={URL https://s3-us-west-2. amazonaws. com/openai-assets/researchcovers/languageunsupervised/language understanding paper. pdf},
year={2018}
}
@article{devlin2018bert,
title={Bert: Pre-training of deep bidirectional transformers for language understanding},
author={Devlin, Jacob and Chang, Ming-Wei and Lee, Kenton and Toutanova, Kristina},
journal={arXiv preprint arXiv:1810.04805},
year={2018}
}
@article{goldberg2017neural,
title={Neural network methods for natural language processing},
author={Goldberg, Yoav},
journal={Synthesis Lectures on Human Language Technologies},
volume={10},
number={1},
pages={1--309},
year={2017},
publisher={Morgan \& Claypool Publishers}
}
@article{lecun2015deep,
title={Deep learning},
author={LeCun, Yann and Bengio, Yoshua and Hinton, Geoffrey},
journal={nature},
volume={521},
number={7553},
pages={436--444},
year={2015},
publisher={Nature Publishing Group}
}
@article{guidotti2018survey,
title={A survey of methods for explaining black box models},
author={Guidotti, Riccardo and Monreale, Anna and Ruggieri, Salvatore and Turini, Franco and Giannotti, Fosca and Pedreschi, Dino},
journal={ACM computing surveys (CSUR)},
volume={51},
number={5},
pages={1--42},
year={2018},
publisher={ACM New York, NY, USA}
}
@inproceedings{koh2017understanding,
title={Understanding black-box predictions via influence functions},
author={Koh, Pang Wei and Liang, Percy},
booktitle={Proceedings of the 34th International Conference on Machine Learning-Volume 70},
pages={1885--1894},
year={2017},
organization={JMLR. org}
}
@article{arthur2016incorporating,
title={Incorporating discrete translation lexicons into neural machine translation},
author={Arthur, Philip and Neubig, Graham and Nakamura, Satoshi},
journal={arXiv preprint arXiv:1606.02006},
year={2016}
}
@article{zhang2018prior,
title={Prior knowledge integration for neural machine translation using posterior regularization},
author={Zhang, Jiacheng and Liu, Yang and Luan, Huanbo and Xu, Jingfang and Sun, Maosong},
journal={arXiv preprint arXiv:1811.01100},
year={2018}
}
@inproceedings{zollmann2006syntax,
title={Syntax augmented machine translation via chart parsing},
author={Zollmann, Andreas and Venugopal, Ashish},
booktitle={Proceedings of the Workshop on Statistical Machine Translation},
pages={138--141},
year={2006},
organization={Association for Computational Linguistics}
}
@inproceedings{charniak2003syntax,
title={Syntax-based language models for statistical machine translation},
author={Charniak, Eugene and Knight, Kevin and Yamada, Kenji},
booktitle={Proceedings of MT Summit IX},
pages={40--46},
year={2003}
}
@article{stahlberg2016syntactically,
title={Syntactically guided neural machine translation},
author={Stahlberg, Felix and Hasler, Eva and Waite, Aurelien and Byrne, Bill},
journal={arXiv preprint arXiv:1605.04569},
year={2016}
}
@inproceedings{plank2013embedding,
title={Embedding semantic similarity in tree kernels for domain adaptation of relation extraction},
author={Plank, Barbara and Moschitti, Alessandro},
booktitle={Proceedings of the 51st Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers)},
pages={1498--1507},
year={2013}
}
@inproceedings{perozzi2014deepwalk,
title={Deepwalk: Online learning of social representations},
author={Perozzi, Bryan and Al-Rfou, Rami and Skiena, Steven},
booktitle={Proceedings of the 20th ACM SIGKDD international conference on Knowledge discovery and data mining},
pages={701--710},
year={2014}
}
@article{collobert2011natural,
title={Natural language processing (almost) from scratch},
author={Collobert, Ronan and Weston, Jason and Bottou, L{\'e}on and Karlen, Michael and Kavukcuoglu, Koray and Kuksa, Pavel},
journal={Journal of machine learning research},
volume={12},
number={Aug},
pages={2493--2537},
year={2011}
}
@article{mikolov2013efficient,
title={Efficient estimation of word representations in vector space},
author={Mikolov, Tomas and Chen, Kai and Corrado, Greg and Dean, Jeffrey},
journal={arXiv preprint arXiv:1301.3781},
year={2013}
}
@inproceedings{mccann2017learned,
title={Learned in translation: Contextualized word vectors},
author={McCann, Bryan and Bradbury, James and Xiong, Caiming and Socher, Richard},
booktitle={Advances in Neural Information Processing Systems},
pages={6294--6305},
year={2017}
}
@article{radford2019language,
title={Language models are unsupervised multitask learners},
author={Radford, Alec and Wu, Jeffrey and Child, Rewon and Luan, David and Amodei, Dario and Sutskever, Ilya},
journal={OpenAI Blog},
volume={1},
number={8},
pages={9},
year={2019}
}
@article{lample2019cross,
title={Cross-lingual language model pretraining},
author={Lample, Guillaume and Conneau, Alexis},
journal={arXiv preprint arXiv:1901.07291},
year={2019}
}
@article{song2019mass,
title={Mass: Masked sequence to sequence pre-training for language generation},
author={Song, Kaitao and Tan, Xu and Qin, Tao and Lu, Jianfeng and Liu, Tie-Yan},
journal={arXiv preprint arXiv:1905.02450},
year={2019}
}
@inproceedings{yang2019xlnet,
title={Xlnet: Generalized autoregressive pretraining for language understanding},
author={Yang, Zhilin and Dai, Zihang and Yang, Yiming and Carbonell, Jaime and Salakhutdinov, Russ R and Le, Quoc V},
booktitle={Advances in neural information processing systems},
pages={5754--5764},
year={2019}
}
@article{lan2019albert,
title={Albert: A lite bert for self-supervised learning of language representations},
author={Lan, Zhenzhong and Chen, Mingda and Goodman, Sebastian and Gimpel, Kevin and Sharma, Piyush and Soricut, Radu},
journal={arXiv preprint arXiv:1909.11942},
year={2019}
}
%%-----------------------------------------------------------------------------------------
Book/mt-book-xelatex.tex
......@@ -67,10 +67,11 @@
%----------------------------------------------------------------------------------------
% CHAPTERS
%----------------------------------------------------------------------------------------
\include{Chapter1/chapter1}
\include{Chapter2/chapter2}
\include{Chapter3/chapter3}
\include{Chapter6/chapter6}
%\include{Chapter1/chapter1}
%\include{Chapter2/chapter2}
%\include{Chapter3/chapter3}
\include{Chapter5/chapter5}
%\include{Chapter6/chapter6}
......
Book/mt-book.bbl
This source diff could not be displayed because it is too large. You can view the blob instead.
Book/structure.tex
......@@ -580,8 +580,15 @@ addtohook={%
%\usetikzlibrary{decorations}
\usetikzlibrary{arrows,shapes}
%%%%%%%%%%%chapter5图片等---------------------------------------
\usepackage{tikz-3dplot}
\tcbuselibrary{skins}
\definecolor{ublue}{rgb}{0.152,0.250,0.545}
\definecolor{ugreen}{rgb}{0,0.5,0}
\definecolor{lgreen}{rgb}{0.9,1,0.8}
\definecolor{xtgreen1}{rgb}{0.824,0.898,0.8}
\definecolor{xtgreen}{rgb}{0.914,0.945,0.902}
\definecolor{lightgray}{gray}{0.85}
......
Markdown 格式
0% 或
您添加了 0 人 到此讨论。请谨慎行事。
请先完成此评论的编辑！
请 注册 或者 后发表评论