a new page of complex networks

Section05-Neural-Networks-and-Language-Modeling/section05-test.tex

@@ -116,10 +116,10 @@
 \newcounter{mycount4}
 
 %%%------------------------------------------------------------------------------------------------------------
-%%% 利用XTensor构建神经网络
-\begin{frame}{构建神经网络}
+%%% 利用XTensor构建更复杂的神经网络
+\begin{frame}{更复杂一点的例子}
 \begin{itemize}
-\item 可以很方便的构建一个单层网络
+\item 任何网络都可以构建，比如RNN、Transformer 等
 \end{itemize}
 
 \begin{tcolorbox}
@@ -127,71 +127,78 @@
  colback=blue!10!white,colbacklower=black!5!white]
  {\scriptsize
 \begin{tabbing}
-\texttt{XTensor x, y, w, b;} \\
-\texttt{InitTensor3D(\&x, 3, 4, 5);} \\
-\texttt{InitTensor2D(\&w, 5, 3);} \\
-\texttt{InitTensor1D(\&b, 3);} \\
+\texttt{XTensor x[3], y[3], r, wh;} \\
+\texttt{XTensor h1, h2, w1, b1, h3, h4;} \\
+\texttt{XList splits;} \\
 \texttt{...} \\
-\texttt{y = Sigmoid(MMul(x, w) + b);}
+\texttt{for(unsigned i = 0; i < 3; i++)\{} \\
+\texttt{\hspace{2em}r = Concatenate(x[i] + r) * wh;}\\
+\texttt{\hspace{2em}splits.Add(\&r);}\\
+\texttt{\}}\\
+\visible<2->{
+\texttt{} \\
+\texttt{h1 = Merge(splits, 0);}\\
+\texttt{h2 = Relu(h1 * w1 + b1);}\\
+\texttt{h3 = h1 + h2;} \\
+\texttt{h4 = Softmax(h3);} \\
+}
+\visible<3->{
+\texttt{} \\
+\texttt{Split(h4, splits, 0);} \\
+\texttt{} \\
+\texttt{for(unsigned i = 0; i < 3; i++)\{} \\
+\texttt{\hspace{2em}y[i] = *(XTensor*)splits.Get(i);}\\
+\texttt{\hspace{2em}y[i].Dump(stdout);}\\
+\texttt{\}}
+}
 \end{tabbing}
 }
 \tcblower
 \begin{center}
 \begin{tikzpicture}
-\node [draw,circle,inner sep=2pt,fill=red!30!white,blur shadow={shadow xshift=1pt,shadow yshift=-1pt}] (x) at (0,0) {\footnotesize{$\textrm{x}$}};
-\node [anchor=south,draw,rounded corners,inner sep=2pt,minimum width=4em,fill=green!30!white,blur shadow={shadow xshift=1pt,shadow yshift=-1pt}] (layer) at ([yshift=0.7em]x.north) {\scriptsize{layer}};
-\node [anchor=south,draw,circle,inner sep=2pt,fill=red!30!white,blur shadow={shadow xshift=1pt,shadow yshift=-1pt}] (y) at ([yshift=0.7em]layer.north) {\scriptsize{$\textrm{y}$}};
-\draw [thick,->] (x.north) -- (layer.south);
-\draw [thick,->] (layer.north) -- (y.south);
-\node [anchor=west,align=left] (xshape) at (x.east) {\tiny{shape: 3*4*5}};
-\node [anchor=west,align=left] (yshape) at (y.east) {\tiny{shape: 3*4*3}};
-\end{tikzpicture}
-\end{center}
-\end{tcolorbox}
+\node [draw,circle,inner sep=1pt,fill=red!30!white,blur shadow={shadow xshift=1pt,shadow yshift=-1pt}] (x1) at (0,0) {\footnotesize{$\textrm{x}_1$}};
+\node [anchor=west,draw,circle,inner sep=1pt,fill=red!30!white,blur shadow={shadow xshift=1pt,shadow yshift=-1pt}] (x2) at ([xshift=2em]x1.east) {\footnotesize{$\textrm{x}_2$}};
+\node [anchor=west,draw,circle,inner sep=1pt,fill=red!30!white,blur shadow={shadow xshift=1pt,shadow yshift=-1pt}] (x3) at ([xshift=2em]x2.east) {\footnotesize{$\textrm{x}_3$}};
+\node [anchor=south,draw,rounded corners,inner sep=2pt,minimum width=2.5em,fill=green!30!white,blur shadow={shadow xshift=1pt,shadow yshift=-1pt}] (rlayer1) at ([yshift=1em]x1.north) {\tiny{rlayer}};
+\node [anchor=south,draw,rounded corners,inner sep=2pt,minimum width=2.5em,fill=green!30!white,blur shadow={shadow xshift=1pt,shadow yshift=-1pt}] (rlayer2) at ([yshift=1em]x2.north) {\tiny{rlayer}};
+\node [anchor=south,draw,rounded corners,inner sep=2pt,minimum width=2.5em,fill=green!30!white,blur shadow={shadow xshift=1pt,shadow yshift=-1pt}] (rlayer3) at ([yshift=1em]x3.north) {\tiny{rlayer}};
+\draw [->,thick] (x1.north) -- (rlayer1.south);
+\draw [->,thick] (x2.north) -- (rlayer2.south);
+\draw [->,thick] (x3.north) -- (rlayer3.south);
+\draw [->,thick] (rlayer1.east) -- (rlayer2.west);
+\draw [->,thick] (rlayer2.east) -- (rlayer3.west);
+\draw [->,thick] (rlayer1.north) -- ([yshift=1em]rlayer1.north);
+\draw [->,thick] (rlayer2.north) -- ([yshift=1em]rlayer2.north);
+\draw [->,thick] (rlayer3.north) -- ([yshift=1em]rlayer3.north);
 
 \visible<2->{
-\begin{itemize}
-\item 一个多层网络
-\end{itemize}
+\node [anchor=south,draw,rounded corners,inner sep=2pt,minimum width=9.4em,minimum height=1.0em,fill=green!30!white,blur shadow={shadow xshift=1pt,shadow yshift=-1pt}] (h1) at ([yshift=1em]rlayer2.north) {\tiny{h1 = Merge($\cdot$)}};
+\node [anchor=south,draw,rounded corners,inner sep=2pt,minimum width=9.4em,minimum height=1.0em,fill=green!30!white,blur shadow={shadow xshift=1pt,shadow yshift=-1pt}] (h2) at ([yshift=1em]h1.north) {\tiny{h2 = Relu($\cdot$)}};
+\node [anchor=south,draw,rounded corners,inner sep=2pt,minimum width=9.4em,minimum height=1.0em,fill=green!30!white,blur shadow={shadow xshift=1pt,shadow yshift=-1pt}] (h3) at ([yshift=1em]h2.north) {\tiny{h3 = Sum($\cdot$)}};
+\node [anchor=south,draw,rounded corners,inner sep=2pt,minimum width=9.4em,minimum height=1.0em,fill=green!30!white,blur shadow={shadow xshift=1pt,shadow yshift=-1pt}] (h4) at ([yshift=1em]h3.north) {\tiny{h4 = Softmax($\cdot$)}};
+\draw [->,thick] (h1.north) -- (h2.south);
+\draw [->,thick] (h2.north) -- (h3.south);
+\draw [->,thick] (h3.north) -- (h4.south);
+\draw [->,thick,rounded corners] (h1.east) -- ([xshift=0.5em]h1.east) -- ([xshift=0.5em,yshift=0.5em]h2.north east) -- ([xshift=-2em,yshift=0.5em]h2.north east) -- ([xshift=-2em,yshift=1em]h2.north east);
+}
 
-\begin{tcolorbox}
-[bicolor,sidebyside,righthand width=4cm,size=title,frame engine=empty,
- colback=blue!10!white,colbacklower=black!5!white]
- {\scriptsize
-\begin{tabbing}
-\texttt{XTensor x, y, h1, h2;} \\
-\texttt{XTensor w1, b1, w2, w3;} \\
-\texttt{InitTensor3D(\&x, 3, 4, 5);} \\
-\texttt{InitTensor2D(\&w1, 5, 3);} \\
-\texttt{InitTensor1D(\&b1, 3);} \\
-\texttt{InitTensor2D(\&w2, 3, 6);} \\
-\texttt{InitTensor2D(\&w3, 6, 4);} \\
-\texttt{...} \\
-\texttt{h1 = Sigmoid(MMul(x, w1) + b1);} \\
-\texttt{h2 = HandTanH(MMul(h1, w2));} \\
-\texttt{y = Relu(MMul(h2, w3));}
-\end{tabbing}
+\visible<3->{
+\node [anchor=south,draw,rounded corners,inner sep=2pt,minimum width=9.4em,minimum height=1.0em,fill=green!30!white,blur shadow={shadow xshift=1pt,shadow yshift=-1pt}] (slayer) at ([yshift=1em]h4.north) {\tiny{Split($\cdot$)}};
+\node [anchor=south,draw,circle,inner sep=1pt,fill=red!30!white,blur shadow={shadow xshift=1pt,shadow yshift=-1pt}] (y2) at ([yshift=1em]slayer.north) {\footnotesize{$\textrm{y}_2$}};
+\node [anchor=east,draw,circle,inner sep=1pt,fill=red!30!white,blur shadow={shadow xshift=1pt,shadow yshift=-1pt}] (y1) at ([xshift=-2em]y2.west) {\footnotesize{$\textrm{y}_1$}};
+\node [anchor=west,draw,circle,inner sep=1pt,fill=red!30!white,blur shadow={shadow xshift=1pt,shadow yshift=-1pt}] (y3) at ([xshift=2em]y2.east) {\footnotesize{$\textrm{y}_3$}};
+\draw [<-,thick] (y1.south) -- ([yshift=-1em]y1.south);
+\draw [<-,thick] (y2.south) -- ([yshift=-1em]y2.south);
+\draw [<-,thick] (y3.south) -- ([yshift=-1em]y3.south);
 }
-\tcblower
-\begin{center}
-\begin{tikzpicture}
-\node [draw,circle,inner sep=2pt,fill=red!30!white,blur shadow={shadow xshift=1pt,shadow yshift=-1pt}] (x) at (0,0) {\footnotesize{$\textrm{x}$}};
-\node [anchor=south,draw,rounded corners,inner sep=2pt,minimum width=4em,fill=green!30!white,blur shadow={shadow xshift=1pt,shadow yshift=-1pt}] (layer1) at ([yshift=0.7em]x.north) {\scriptsize{layer1}};
-\node [anchor=south,draw,rounded corners,inner sep=2pt,minimum width=4em,fill=green!30!white,blur shadow={shadow xshift=1pt,shadow yshift=-1pt}] (layer2) at ([yshift=1.0em]layer.north) {\scriptsize{layer2}};
-\node [anchor=south,draw,rounded corners,inner sep=2pt,minimum width=4em,fill=green!30!white,blur shadow={shadow xshift=1pt,shadow yshift=-1pt}] (layer3) at ([yshift=1.0em]layer2.north) {\scriptsize{layer3}};
-\node [anchor=south,draw,circle,inner sep=2pt,fill=red!30!white,blur shadow={shadow xshift=1pt,shadow yshift=-1pt}] (y) at ([yshift=0.7em]layer3.north) {\scriptsize{$\textrm{y}$}};
-\draw [thick,->] (x.north) -- (layer1.south);
-\draw [thick,->] (layer1.north) -- (layer2.south);
-\draw [thick,->] (layer2.north) -- (layer3.south);
-\draw [thick,->] (layer3.north) -- (y.south);
-\node [anchor=west,align=left] (xshape) at (x.east) {\tiny{shape: 3*4*5}};
-\node [anchor=west,align=left] (yshape) at (y.east) {\tiny{shape: 3*4*4}};
-\node [anchor=south west,align=left,inner sep=2pt] (l1shape) at (layer1.north) {\tiny{shape: 3*4*3}};
-\node [anchor=south west,align=left,inner sep=2pt] (l2shape) at (layer2.north) {\tiny{shape: 3*4*6}};
+
+\visible<2->{
+\draw [->,thick] (h4.north) -- (slayer.south);
+}
+
 \end{tikzpicture}
 \end{center}
 \end{tcolorbox}
-}
 
 \end{frame}