Skip to content

T
Toy-MT-Introduction
Commit 63f6e3b1 by xiaotong
Options

new pages

parent bb9ddf8f
正在显示 包含 148 行增加 和 55 行删除
Section05-Neural-Networks-and-Language-Modeling/section05-test.tex
......@@ -116,72 +116,84 @@
\subsection{参数学习 - 反向传播}
%%%------------------------------------------------------------------------------------------------------------
%%% 如何计算梯度 - 符号微分
\begin{frame}{符号微分}
%%% 反向传播
\begin{frame}{反向传播}
\begin{itemize}
\item \textbf{符号微分}：类似于手写出微分表达式，最后带入变量的值，得到微分结果。比如，对于如下表达式
\begin{displaymath}
L(\textbf{w}) = \textbf{x} \cdot \textbf{w} + 2 \textbf{w}^2
\end{displaymath}
\visible<2->{
\vspace{0.5em}
可以手动推导出微分表达式
\begin{displaymath}
\frac{\partial L(\textbf{w})}{\partial \textbf{w}} = \textbf{x} + 4 \textbf{w}
\end{displaymath}
\item 反向传播
\end{itemize}
\begin{center}
\begin{tikzpicture}
\begin{scope}
\def\neuronsep{1}
\tikzstyle{neuronnode} = [minimum size=1.2em,circle,draw,ublue,very thick,inner sep=1pt, fill=white,align=center,drop shadow={shadow xshift=0.1em,shadow yshift=-0.1em}];
%%% layer 1
\foreach \n in {1,...,4}{
\node [neuronnode] (neuron0\n) at (\n * \neuronsep,0) {};
\draw [->] ([yshift=-0.8em]neuron0\n.south) -- ([yshift=-0.1em]neuron0\n.south) node [pos=0,below] {\tiny{...}};
}
\visible<3->{
\vspace{0.5em}
最后，带入$\textbf{x} = \begin{pmatrix} 2 \\ -3 \end{pmatrix}$和$\textbf{w} = \begin{pmatrix} -1 \\ 1 \end{pmatrix}$，得到微分结果\\
\vspace{1em}
\begin{pgfonlayer}{background}
\node [rectangle,inner sep=0.2em,fill=red!20] [fit = (neuron01) (neuron04)] (layer01) {};
\node [anchor=east] (layer01label) at (layer01.west) {\scriptsize{层$i-1$}};
\end{pgfonlayer}
\begin{displaymath}
\frac{\partial L(\textbf{w})}{\partial \textbf{w}} = \begin{pmatrix} 2 \\ -3 \end{pmatrix} + 4 \begin{pmatrix} -1 \\ 1 \end{pmatrix} = \begin{pmatrix} -2 \\ 1 \end{pmatrix}
\end{displaymath}
%%% layer 2
\foreach \n in {1,...,4}{
\node [neuronnode] (neuron1\n) at (\n * \neuronsep,3em) {};
}
\end{itemize}
\foreach \n in {1,...,4}{
\foreach \m in {1,...,4}{
\draw [<-] (neuron1\n.south) -- (neuron0\m.north);
}
}
\end{frame}
\begin{pgfonlayer}{background}
\node [rectangle,inner sep=0.2em,fill=ugreen!20] [fit = (neuron11) (neuron14)] (layer02) {};
\node [anchor=east] (layer02label) at (layer02.west) {\scriptsize{层$i$}};
\end{pgfonlayer}
%%%------------------------------------------------------------------------------------------------------------
%%% 符号微分的膨胀问题
\begin{frame}{符号微分的膨胀问题}
%%% layer 3
\foreach \n in {1,...,4}{
\node [neuronnode] (neuron2\n) at (\n * \neuronsep,6em) {};
\draw [<-] ([yshift=0.8em]neuron2\n.north) -- ([yshift=0.0em]neuron2\n.north) node [pos=0,above] {\tiny{...}};
}
\begin{itemize}
\item \textbf{Expression Swell}：深层函数的微分表达式会非常复杂
\begin{itemize}
\item 表达式冗长不易存储和管理
\item 真正需要的是微分结果，而不是微分表达式
\end{itemize}
\end{itemize}
\foreach \n in {1,...,4}{
\foreach \m in {1,...,4}{
\draw [<-] (neuron2\n.south) -- (neuron1\m.north);
}
}
\begin{pgfonlayer}{background}
\node [rectangle,inner sep=0.2em,fill=blue!20] [fit = (neuron21) (neuron24)] (layer03) {};
\node [anchor=east] (layer03label) at (layer03.west) {\scriptsize{层$i+1$}};
\end{pgfonlayer}
\vspace{0.5em}
{\small
\begin{tabular} {l | l | l}
函数 & 微分表达式 & 化简的微分表达式 \\ \hline
$x$ & $1$ & $1$ \\ \hline
$x(x+1)$ & $(x+1)+x$ & $2x + 1$ \\ \hline
$x(x+1)$ & $(x+1)(x^2+x+1)$ & $4x^3+6x^2$ \\
$(x^2+x+1)$ & $+x(x^2+x+1)$ & $+4x+1$ \\
& $+x(x+1)(2x+1)$ & \\ \hline
$(x^2+x)$ & $(2x+1)(x^2+x+1)$ & $8x^7+28x^6$ \\
$(x^2+x+1)$ & $(x^4+2x^3+2x^2+x+1)$ & $+48x^5+50x^4$ \\
$(x^4+2x^3$ & $+(2x+1)(x^2+x)$ & $+36x^3+18x^2$ \\
$+2x^2+x+1)$ & \ \ $(x^4+2x^3+2x^2+x+1)$ & $+6x+1$ \\
& $+(x^2+x)(x^2+x+1)$ & \\
& \ \ $(4x^3+6x^2+4x+1)$ & \\
\end{tabular}
\node [neuronnode,draw=red,fill=red!20!white,inner sep=1pt] (neuron12new) at (2 * \neuronsep,3em) {};
\node [anchor=east] (neuronsamplelabel) at ([yshift=-1em]layer02label.south east) {\alert{\textbf{\tiny{第$l$层, 第$i$个神经元}}}};
\draw [->,dashed,very thick,red] ([xshift=-0.2em,yshift=0.2em]neuronsamplelabel.east) .. controls +(40:1) and +(220:1) .. ([xshift=-0em,yshift=-0em]neuron12new.210);
\foreach \n in {1,...,4}{
\draw [<-,thick,red] (neuron2\n.south) -- (neuron12.north);
}
\draw [<-,thick,red] (neuron24.south) -- (neuron14.north);
\node [anchor=north] (wlabel) at (layer03.south east) {\alert{\scriptsize{$w_{4,4}^{2}$}}};
\node [anchor=west,align=left] (line01) at ([xshift=2em]layer03.east) {\footnotesize{$h_{i}^{l}$：第$l$层, 第$i$个神经元的输出}};
\node [anchor=north west,align=left] (line02) at (line01.south west) {\footnotesize{$\textbf{h}^{l}$：第$l$层的输出}};
\node [anchor=north west,align=left] (line03) at (line02.south west) {\footnotesize{$w_{j,i}^{l}$：第$l-1$层神经元$j$与}\\\footnotesize{第$l+1$层神经元$i$的连接权重}};
\end{scope}
\end{tikzpicture}
\end{center}
\end{frame}
\end{CJK}
......
Section05-Neural-Networks-and-Language-Modeling/section05.tex
......@@ -2702,7 +2702,7 @@ cycle}
\vspace{2em}
\begin{center}
\begin{tikzpicture}
\node [fill=blue!10] (label) at (0,0) {\LARGE{$\frac{\partial \textbf{E}}{\partial \textbf{w}} = $ ? }};
\node [fill=blue!10] (label) at (0,0) {\LARGE{$\frac{\partial \textbf{L}}{\partial \textbf{w}} = $ ? }};
\end{tikzpicture}
\end{center}
......@@ -3101,7 +3101,7 @@ L(\textbf{w}) = \textbf{x} \cdot \textbf{w} + 2 \textbf{w}^2
\item \textbf{Expression Swell}：深层函数的微分表达式会非常复杂
\begin{itemize}
\item 表达式冗长不易存储和管理
\item 真正需要的是微分\alert{结果}，而不是微分表达式
\item 真正需要的是微分的\alert{结果值}，而不是微分表达式
\end{itemize}
\end{itemize}
......@@ -3112,7 +3112,7 @@ L(\textbf{w}) = \textbf{x} \cdot \textbf{w} + 2 \textbf{w}^2
函数 & 微分表达式 & 化简的微分表达式 \\ \hline
$x$ & $1$ & $1$ \\ \hline
$x(x+1)$ & $(x+1)+x$ & $2x + 1$ \\ \hline
$x(x+1)$ & $(x+1)(x^2+x+1)$ & $4x^3+6x^2$ \\
$x(x+1)$ & $(x+1)(x^2+x+1)$ & $4x^3+6x^2$ \\
$(x^2+x+1)$ & $+x(x^2+x+1)$ & $+4x+1$ \\
& $+x(x+1)(2x+1)$ & \\ \hline
$(x^2+x)$ & $(2x+1)(x^2+x+1)$ & $8x^7+28x^6$ \\
......@@ -3128,5 +3128,86 @@ $+2x^2+x+1)$ & \ \ $(x^4+2x^3+2x^2+x+1)$ & $+6x+1$ \\
\end{frame}
%%%------------------------------------------------------------------------------------------------------------
%%% 自动微分
\begin{frame}{自动微分}
\begin{itemize}
\item \textbf{自动微分}：复杂的微分变成简单的步骤，这些步骤完全自动化，而且容易进行存储、计算。这可以用一种反向模式进行描述（也就是\alert{反向传播}思想），包括两步
\begin{enumerate}
\item \textbf{前向计算}：从神经网络的输入，逐层计算每层网络的输出值，这也是神经网络的标准使用方式
\item \textbf{反向计算}：从神经网络的输出，逆向逐层计算每层网络输入（输出）所对应的微分
\end{enumerate}
\end{itemize}
\visible<2->{
\vspace{-1em}
\begin{center}
\begin{tikzpicture}
\begin{scope}
\tikzstyle{layernode} = [draw,thick,fill=ugreen!30!white,blur shadow={shadow xshift=1pt,shadow yshift=-1pt}];
\node [anchor=center,layernode,minimum height=4em,minimum width=1em] (layer01) at (0,0) {};
\node [anchor=north west,layernode,minimum height=3em,minimum width=1em] (layer02) at ([xshift=3em]layer01.north east) {};
\node [anchor=south west,layernode,minimum height=3em,minimum width=1em] (layer03) at ([xshift=7em]layer01.south east) {};
\node [anchor=south west,layernode,minimum height=4em,minimum width=1em] (layer04) at ([xshift=11em]layer01.south east) {};
\node [anchor=south west,layernode,minimum height=4em,minimum width=1em] (layer05) at ([xshift=3em]layer04.south east) {};
\node [anchor=east] (input) at ([xshift=-1em]layer01.west){\scriptsize{输入}};
\node [anchor=west] (output) at ([xshift=1em]layer05.east){\scriptsize{输出}};
\draw [->] ([xshift=-1em]layer01.west) -- ([xshift=-0.1em]layer01.west);
\draw [->] ([xshift=0.1em,yshift=-0.5em]layer01.north east) -- ([xshift=-0.1em,yshift=-0.5em]layer02.north west);
\draw [->] ([xshift=0.1em,yshift=0.5em]layer01.south east) -- ([xshift=-0.1em,yshift=0.5em]layer03.south west);
\draw [->] ([xshift=0.1em,yshift=-0.5em]layer02.north east) -- ([xshift=-0.1em,yshift=-0.5em]layer04.north west);
\draw [->] ([xshift=0.1em,yshift=0.5em]layer03.south east) -- ([xshift=-0.1em,yshift=0.5em]layer04.south west);
\draw [->] ([xshift=0.1em]layer04.east) -- ([xshift=-0.1em]layer05.west);
\draw [->] ([xshift=0.1em]layer05.east) -- ([xshift=1.0em]layer05.east);
\visible<3->{
\draw [->,very thick,ublue] ([xshift=-1em]layer01.west) -- ([xshift=-0.1em]layer01.west);
}
\visible<4->{
\draw [->,very thick,ublue] ([xshift=0.1em,yshift=-0.5em]layer01.north east) -- ([xshift=-0.1em,yshift=-0.5em]layer02.north west);
}
\visible<5->{
\draw [->,very thick,ublue] ([xshift=0.1em,yshift=0.5em]layer01.south east) -- ([xshift=-0.1em,yshift=0.5em]layer03.south west);
}
\visible<6->{
\draw [->,very thick,ublue] ([xshift=0.1em,yshift=-0.5em]layer02.north east) -- ([xshift=-0.1em,yshift=-0.5em]layer04.north west);
\draw [->,very thick,ublue] ([xshift=0.1em,yshift=0.5em]layer03.south east) -- ([xshift=-0.1em,yshift=0.5em]layer04.south west);
\draw [->,very thick,ublue] ([xshift=0.1em]layer04.east) -- ([xshift=-0.1em]layer05.west);
\draw [->,very thick,ublue] ([xshift=0.1em]layer05.east) -- ([xshift=1.0em]layer05.east);
}
\visible<8->{
\draw [<-,very thick,red] ([xshift=-1em,yshift=-0.3em]layer01.west) -- ([xshift=-0.1em,yshift=-0.3em]layer01.west);
\draw [<-,very thick,red] ([xshift=0.1em,yshift=-0.8em]layer01.north east) -- ([xshift=-0.1em,yshift=-0.8em]layer02.north west);
\draw [<-,very thick,red] ([xshift=0.1em,yshift=0.2em]layer01.south east) -- ([xshift=-0.1em,yshift=0.2em]layer03.south west);
\draw [<-,very thick,red] ([xshift=0.1em,yshift=-0.8em]layer02.north east) -- ([xshift=-0.1em,yshift=-0.8em]layer04.north west);
\draw [<-,very thick,red] ([xshift=0.1em,yshift=0.2em]layer03.south east) -- ([xshift=-0.1em,yshift=0.2em]layer04.south west);
\draw [<-,very thick,red] ([xshift=0.1em,yshift=-0.3em]layer04.east) -- ([xshift=-0.1em,yshift=-0.3em]layer05.west);
\draw [<-,very thick,red] ([xshift=0.1em,yshift=-0.3em]layer05.east) -- ([xshift=1.0em,yshift=-0.3em]layer05.east);
}
\visible<7->{
\draw [-,thin] ([xshift=0.3em,yshift=0.3em]layer04.east) .. controls +(35:1) and +(215:1) .. ([xshift=-2em,yshift=0.3em]layer05.north west) node [pos=1,above] {\scriptsize{前向：层$i$ 的输出$h_{i}$}};
}
\visible<9->{
\draw [-,thin] ([xshift=0.3em,yshift=-0.7em]layer04.east) .. controls +(-35:1) and +(145:1) .. ([xshift=-2em,yshift=-0.3em]layer05.south west) node [pos=1,below] {\scriptsize{反向：$h_{i}$ 处的梯度$\frac{\partial L}{\partial h_i}$}};
}
\end{scope}
\end{tikzpicture}
\end{center}
}
\vspace{-1em}
\begin{itemize}
\item<10-> 自动微分可以用\alert{计算图}实现(TensorFlow、 NiuTensor等)，不过计算图超出了课程的范围，建议大家自行学习
\end{itemize}
\end{frame}
\end{CJK}
\end{document}
Markdown 格式
0% 或
您添加了 0 人 到此讨论。请谨慎行事。
请先完成此评论的编辑！
请 注册 或者 后发表评论