new pages

Section05-Neural-Networks-and-Language-Modeling/section05-test.tex

@@ -113,152 +113,66 @@
 
 
 
 
 %%%------------------------------------------------------------------------------------------------------------
 %%%------------------------------------------------------------------------------------------------------------
-%%% 张量矩阵乘法
-\begin{frame}{张量的矩阵乘法}
+%%% 张量的单元操作
+\begin{frame}{张量的单元操作}
 \begin{itemize}
 \begin{itemize}
-\item 对于神经网络$\textbf{y}=f(\textbf{x}\cdot \textbf{w} + \textbf{b})$，$\textbf{x} \cdot \textbf{w}$或$\textbf{x} \times \textbf{w}$是线性变换，其中$\textbf{x}$是输入张量，$\textbf{w}$是一个矩阵
+\item 神经网络$\textbf{y}=f(\textbf{x}\cdot \textbf{w} + \textbf{b})$也包括一些张量的单元操作（element-wise opertation）
 	\begin{itemize}
 	\begin{itemize}
-    \item $\textbf{x} \cdot \textbf{w}$表示的是矩阵乘法（或记为$\times$）
-    \item 注意这里不是张量乘法，它还有其它定义
-    \item $\textbf{w}$是$n \times m$的矩阵，$\textbf{x}$的形状是$... \times n$，即$\textbf{x}$的第一维度需要和$\textbf{w}$的行数\\
-    \vspace{0.5em}
-    $\textbf{x}(1:2,1:2,\alert{1:3}) \times \textbf{w}(\alert{1:3},1:2) = \textbf{s}(1:2,1:2,1:2)$
+	\item 加法：$\textbf{s}+\textbf{b}$，其中$\textbf{s} = \textbf{x}\cdot \textbf{w}$
+	\item 激活函数：$f(\cdot)$
+	\end{itemize}
+\item<2-> \textbf{单元加}就是对张量中的每个位置都进行加法
+	\begin{itemize}
+	\item 一般要求两个张量的形状是一样的
+	\item<3-> 不过，这里可以使用加法的\textbf{广播}，重复利用一个张量进行加法，并不要求两个张量形状相同
 	\end{itemize}
 	\end{itemize}
 \end{itemize}
 \end{itemize}
 
 
+\vspace{-1.5em}
 \begin{center}
 \begin{center}
 \begin{tikzpicture}
 \begin{tikzpicture}
-
-
-\begin{scope}[yshift=6.5em,xshift=1em]
-\visible<2->{
+\visible<3->{
+\begin{scope}
 \setcounter{mycount1}{1}
 \setcounter{mycount1}{1}
-\draw[step=0.5cm,color=orange,thick] (-1,-1) grid (1,1);
-\foreach \y in {+0.75,+0.25,-0.25,-0.75}
+\draw[step=0.5cm,color=orange,thick] (-1,-0.5) grid (1,0.5);
+\foreach \y in {+0.25,-0.25}
   \foreach \x in {-0.75,-0.25,0.25,0.75}{
   \foreach \x in {-0.75,-0.25,0.25,0.75}{
     \node [fill=orange!20,inner sep=0pt,minimum height=0.49cm,minimum width=0.49cm] at (\x,\y) {\number\value{mycount1}};
     \node [fill=orange!20,inner sep=0pt,minimum height=0.49cm,minimum width=0.49cm] at (\x,\y) {\number\value{mycount1}};
     \addtocounter{mycount1}{1};
     \addtocounter{mycount1}{1};
   }
   }
-}
-\end{scope}
-
-\begin{scope}[yshift=6em,xshift=0.5em]
-\visible<2->{
-\setcounter{mycount2}{2}
-\draw[step=0.5cm,color=blue,thick] (-1,-1) grid (1,1);
-\foreach \y in {+0.75,+0.25,-0.25,-0.75}
-  \foreach \x in {-0.75,-0.25,0.25,0.75}{
-    \node [fill=blue!20,inner sep=0pt,minimum height=0.49cm,minimum width=0.49cm] at (\x,\y) {\number\value{mycount2}};
-    \addtocounter{mycount2}{1};
-  }
-}
+\node [anchor=south] (varlabel) at (0,0.6) {$\textbf{s}$};
 \end{scope}
 \end{scope}
-
-\begin{scope}[yshift=5.5em,xshift=0em]
-\visible<2->{
-\setcounter{mycount3}{3}
-\draw[step=0.5cm,color=ugreen,thick] (-1,-1) grid (1,1);
-\foreach \y in {+0.75,+0.25,-0.25,-0.75}
+\begin{scope}[xshift=1.5in]
+\setcounter{mycount1}{1}
+\draw[step=0.5cm,color=ugreen,thick] (-1,-0) grid (1,0.5);
+\foreach \y in {+0.25}
   \foreach \x in {-0.75,-0.25,0.25,0.75}{
   \foreach \x in {-0.75,-0.25,0.25,0.75}{
-    \node [fill=green!20,inner sep=0pt,minimum height=0.49cm,minimum width=0.49cm] at (\x,\y) {\number\value{mycount3}};
-    \addtocounter{mycount3}{1};
+    \node [fill=green!20,inner sep=0pt,minimum height=0.49cm,minimum width=0.49cm] at (\x,\y) {1};
+    \addtocounter{mycount1}{1};
   }
   }
-}
+\node [anchor=center] (plabel) at (-4.5em,0) {\huge{\textbf{$+$}}};
+\node [anchor=south] (varlabel) at (0,0.6) {$\textbf{b}$};
 \end{scope}
 \end{scope}
-
-\begin{scope}[yshift=5em,xshift=-0.5em]
-\visible<2->{
-\setcounter{mycount4}{4}
-\draw[step=0.5cm,color=red,thick] (-1,-1) grid (1,1);
-\foreach \y in {+0.75,+0.25,-0.25,-0.75}
+\begin{scope}[xshift=3in]
+\setcounter{mycount1}{2}
+\draw[step=0.5cm,color=orange,thick] (-1,-0.5) grid (1,0.5);
+\foreach \y in {+0.25,-0.25}
   \foreach \x in {-0.75,-0.25,0.25,0.75}{
   \foreach \x in {-0.75,-0.25,0.25,0.75}{
-    \node [fill=red!20,inner sep=0pt,minimum height=0.49cm,minimum width=0.49cm] at (\x,\y) {\number\value{mycount4}};
-    \addtocounter{mycount4}{1};
-  }
-\node [anchor=north] (xlabel) at (0,-1.2) {$\textbf{x}$};
-}
-\end{scope}
-
-\begin{scope}[yshift=5em,xshift=1.5in]
-\visible<2->{
-\draw[step=0.5cm,thick] (-0.5,-1) grid (0.5,1.0);
-\node [fill=black!20,inner sep=0pt,minimum height=0.49cm,minimum width=0.49cm] at (-0.25,0.75) {-1};
-\node [fill=black!20,inner sep=0pt,minimum height=0.49cm,minimum width=0.49cm] at (-0.25,0.25) {0};
-\node [fill=black!20,inner sep=0pt,minimum height=0.49cm,minimum width=0.49cm] at (-0.25,-0.25) {1};
-\node [fill=black!20,inner sep=0pt,minimum height=0.49cm,minimum width=0.49cm] at (-0.25,-0.75) {0};
-\node [fill=black!20,inner sep=0pt,minimum height=0.49cm,minimum width=0.49cm] at (0.25,0.75) {0};
-\node [fill=black!20,inner sep=0pt,minimum height=0.49cm,minimum width=0.49cm] at (0.25,0.25) {-1};
-\node [fill=black!20,inner sep=0pt,minimum height=0.49cm,minimum width=0.49cm] at (0.25,-0.25) {1};
-\node [fill=black!20,inner sep=0pt,minimum height=0.49cm,minimum width=0.49cm] at (0.25,-0.75) {0};
-\node [anchor=north] (xlabel) at (0,-1.2) {$\textbf{w}$};
-}
-
-\visible<3>{\draw [->,thick] (-1.5in+2em+1.5em,-0.3) .. controls +(east:2) and +(west:1) .. (-0.55,0.8) node [pos=0.5,left] {\textbf{矩阵乘}};}
-\visible<4>{\draw [->,thick] (-1.5in+2em+1.0em,-0.5) .. controls +(east:2) and +(west:1) .. (-0.55,0.8) node [pos=0.5,left] {\textbf{$\times$}};}
-\visible<5>{\draw [->,thick] (-1.5in+2em+0.5em,-0.7) .. controls +(east:2) and +(west:1) .. (-0.55,0.8) node [pos=0.5,left] {\textbf{$\times$}};}
-\visible<6->{\draw [->,thick] (-1.5in+2em,-0.9) .. controls +(east:2) and +(west:1) .. (-0.55,0.8) node [pos=0.5,left] {\textbf{$\times$}};}
-\end{scope}
-
-\begin{scope}[yshift=6.5em,xshift=1em+3in]
-\visible<3->{
-\draw[step=0.5cm,color=orange,thick] (-0.5,-1) grid (0.5,1.0);
-\foreach \y in {+0.75,+0.25,-0.25,-0.75}{
-  \setcounter{mycount1}{2}
-  \foreach \x in {-0.25,0.25}{
     \node [fill=orange!20,inner sep=0pt,minimum height=0.49cm,minimum width=0.49cm] at (\x,\y) {\number\value{mycount1}};
     \node [fill=orange!20,inner sep=0pt,minimum height=0.49cm,minimum width=0.49cm] at (\x,\y) {\number\value{mycount1}};
-    \addtocounter{mycount1}{-1};
-  }
-}
-}
-\end{scope}
-
-\begin{scope}[yshift=6em,xshift=0.5em+3in]
-\visible<4->{
-\draw[step=0.5cm,color=blue,thick] (-0.5,-1) grid (0.5,1.0);
-\foreach \y in {+0.75,+0.25,-0.25,-0.75}{
-  \setcounter{mycount1}{2}
-  \foreach \x in {-0.25,0.25}{
-    \node [fill=blue!20,inner sep=0pt,minimum height=0.49cm,minimum width=0.49cm] at (\x,\y) {\number\value{mycount1}};
-    \addtocounter{mycount1}{-1};
-  }
-}
-}
-\end{scope}
-
-\begin{scope}[yshift=5.5em,xshift=0em+3in]
-\visible<5->{
-\draw[step=0.5cm,color=ugreen,thick] (-0.5,-1) grid (0.5,1.0);
-\foreach \y in {+0.75,+0.25,-0.25,-0.75}{
-  \setcounter{mycount1}{2}
-  \foreach \x in {-0.25,0.25}{
-    \node [fill=green!20,inner sep=0pt,minimum height=0.49cm,minimum width=0.49cm] at (\x,\y) {\number\value{mycount1}};
-    \addtocounter{mycount1}{-1};
+    \addtocounter{mycount1}{1};
   }
   }
-}
-}
+\node [anchor=center] (plabel) at (-4.5em,0) {\huge{\textbf{$=$}}};
+\node [anchor=south] (varlabel) at (0,0.6) {$\textbf{s+b}$};
 \end{scope}
 \end{scope}
-
-\begin{scope}[yshift=5.0em,xshift=-0.5em+3in]
-\visible<6->{
-\draw[step=0.5cm,color=red,thick] (-0.5,-1) grid (0.5,1.0);
-\foreach \y in {+0.75,+0.25,-0.25,-0.75}{
-  \setcounter{mycount1}{2}
-  \foreach \x in {-0.25,0.25}{
-    \node [fill=red!20,inner sep=0pt,minimum height=0.49cm,minimum width=0.49cm] at (\x,\y) {\number\value{mycount1}};
-    \addtocounter{mycount1}{-1};
-  }
-}
-}
-
-\visible<3->{
-\node [anchor=north] (xlabel) at (0,-1.2) {$\textbf{x} \cdot \textbf{w}$};
-\node [anchor=center] (elabel) at (-0.7in,0) {\Huge{$=$}};
 }
 }
-\end{scope}
 
 
 \end{tikzpicture}
 \end{tikzpicture}
 \end{center}
 \end{center}
 
 
+\begin{itemize}
+\item<4-> 类似的，我们可以对所有位置做加法、乘法等等一系列算数操作，也包括激活函数
+\end{itemize}
+
 \end{frame}
 \end{frame}
 
 
 %%%------------------------------------------------------------------------------------------------------------
 %%%------------------------------------------------------------------------------------------------------------

Section05-Neural-Networks-and-Language-Modeling/section05.tex

@@ -1942,6 +1942,218 @@ cycle}
 \end{frame}
 \end{frame}
 
 
 %%%------------------------------------------------------------------------------------------------------------
 %%%------------------------------------------------------------------------------------------------------------
+%%% 张量矩阵乘法
+\begin{frame}{张量的矩阵乘法}
+\begin{itemize}
+\item 对于神经网络$\textbf{y}=f(\textbf{x}\cdot \textbf{w} + \textbf{b})$，$\textbf{x} \cdot \textbf{w}$或$\textbf{x} \times \textbf{w}$是线性变换，其中$\textbf{x}$是输入张量，$\textbf{w}$是一个矩阵
+    \begin{itemize}
+    \item $\textbf{x} \cdot \textbf{w}$表示的是矩阵乘法（或记为$\times$）
+    \item 注意，这里不是张量乘法，因为张量乘法还有其它定义
+    \item $\textbf{w}$是$n \times m$的矩阵，$\textbf{x}$的形状是$... \times n$，即$\textbf{x}$的第一维度需要和$\textbf{w}$的行数\\
+    \vspace{0.5em}
+    $\textbf{x}(1:2,1:2,\alert{1:3}) \times \textbf{w}(\alert{1:3},1:2) = \textbf{s}(1:2,1:2,1:2)$
+    \end{itemize}
+\end{itemize}
+
+\begin{center}
+\begin{tikzpicture}
+
+
+\begin{scope}[yshift=6.5em,xshift=1em]
+\visible<2->{
+\setcounter{mycount1}{1}
+\draw[step=0.5cm,color=orange,thick] (-1,-1) grid (1,1);
+\foreach \y in {+0.75,+0.25,-0.25,-0.75}
+  \foreach \x in {-0.75,-0.25,0.25,0.75}{
+    \node [fill=orange!20,inner sep=0pt,minimum height=0.49cm,minimum width=0.49cm] at (\x,\y) {\number\value{mycount1}};
+    \addtocounter{mycount1}{1};
+  }
+}
+\end{scope}
+
+\begin{scope}[yshift=6em,xshift=0.5em]
+\visible<2->{
+\setcounter{mycount2}{2}
+\draw[step=0.5cm,color=blue,thick] (-1,-1) grid (1,1);
+\foreach \y in {+0.75,+0.25,-0.25,-0.75}
+  \foreach \x in {-0.75,-0.25,0.25,0.75}{
+    \node [fill=blue!20,inner sep=0pt,minimum height=0.49cm,minimum width=0.49cm] at (\x,\y) {\number\value{mycount2}};
+    \addtocounter{mycount2}{1};
+  }
+}
+\end{scope}
+
+\begin{scope}[yshift=5.5em,xshift=0em]
+\visible<2->{
+\setcounter{mycount3}{3}
+\draw[step=0.5cm,color=ugreen,thick] (-1,-1) grid (1,1);
+\foreach \y in {+0.75,+0.25,-0.25,-0.75}
+  \foreach \x in {-0.75,-0.25,0.25,0.75}{
+    \node [fill=green!20,inner sep=0pt,minimum height=0.49cm,minimum width=0.49cm] at (\x,\y) {\number\value{mycount3}};
+    \addtocounter{mycount3}{1};
+  }
+}
+\end{scope}
+
+\begin{scope}[yshift=5em,xshift=-0.5em]
+\visible<2->{
+\setcounter{mycount4}{4}
+\draw[step=0.5cm,color=red,thick] (-1,-1) grid (1,1);
+\foreach \y in {+0.75,+0.25,-0.25,-0.75}
+  \foreach \x in {-0.75,-0.25,0.25,0.75}{
+    \node [fill=red!20,inner sep=0pt,minimum height=0.49cm,minimum width=0.49cm] at (\x,\y) {\number\value{mycount4}};
+    \addtocounter{mycount4}{1};
+  }
+\node [anchor=north] (xlabel) at (0,-1.2) {$\textbf{x}$};
+}
+\end{scope}
+
+\begin{scope}[yshift=5em,xshift=1.5in]
+\visible<2->{
+\draw[step=0.5cm,thick] (-0.5,-1) grid (0.5,1.0);
+\node [fill=black!20,inner sep=0pt,minimum height=0.49cm,minimum width=0.49cm] at (-0.25,0.75) {-1};
+\node [fill=black!20,inner sep=0pt,minimum height=0.49cm,minimum width=0.49cm] at (-0.25,0.25) {0};
+\node [fill=black!20,inner sep=0pt,minimum height=0.49cm,minimum width=0.49cm] at (-0.25,-0.25) {1};
+\node [fill=black!20,inner sep=0pt,minimum height=0.49cm,minimum width=0.49cm] at (-0.25,-0.75) {0};
+\node [fill=black!20,inner sep=0pt,minimum height=0.49cm,minimum width=0.49cm] at (0.25,0.75) {0};
+\node [fill=black!20,inner sep=0pt,minimum height=0.49cm,minimum width=0.49cm] at (0.25,0.25) {-1};
+\node [fill=black!20,inner sep=0pt,minimum height=0.49cm,minimum width=0.49cm] at (0.25,-0.25) {1};
+\node [fill=black!20,inner sep=0pt,minimum height=0.49cm,minimum width=0.49cm] at (0.25,-0.75) {0};
+\node [anchor=north] (xlabel) at (0,-1.2) {$\textbf{w}$};
+}
+
+\visible<3>{\draw [->,thick] (-1.5in+2em+1.5em,-0.3) .. controls +(east:2) and +(west:1) .. (-0.55,0.8) node [pos=0.5,left] {\scriptsize{\textbf{矩阵乘}}};}
+\visible<4>{\draw [->,thick] (-1.5in+2em+1.0em,-0.5) .. controls +(east:2) and +(west:1) .. (-0.55,0.8) node [pos=0.5,left] {\scriptsize{\textbf{矩阵乘}}};}
+\visible<5>{\draw [->,thick] (-1.5in+2em+0.5em,-0.7) .. controls +(east:2.5) and +(west:1) .. (-0.55,0.8) node [pos=0.5,left] {\scriptsize{\textbf{矩阵乘}}};}
+\visible<6->{\draw [->,thick] (-1.5in+2em,-0.9) .. controls +(east:3) and +(west:1) .. (-0.55,0.8) node [pos=0.5,left] {\scriptsize{\textbf{矩阵乘}}};}
+\end{scope}
+
+\begin{scope}[yshift=6.5em,xshift=1em+3in]
+\visible<3->{
+\draw[step=0.5cm,color=orange,thick] (-0.5,-1) grid (0.5,1.0);
+\foreach \y in {+0.75,+0.25,-0.25,-0.75}{
+  \setcounter{mycount1}{2}
+  \foreach \x in {-0.25,0.25}{
+    \node [fill=orange!20,inner sep=0pt,minimum height=0.49cm,minimum width=0.49cm] at (\x,\y) {\number\value{mycount1}};
+    \addtocounter{mycount1}{-1};
+  }
+}
+}
+\end{scope}
+
+\begin{scope}[yshift=6em,xshift=0.5em+3in]
+\visible<4->{
+\draw[step=0.5cm,color=blue,thick] (-0.5,-1) grid (0.5,1.0);
+\foreach \y in {+0.75,+0.25,-0.25,-0.75}{
+  \setcounter{mycount1}{2}
+  \foreach \x in {-0.25,0.25}{
+    \node [fill=blue!20,inner sep=0pt,minimum height=0.49cm,minimum width=0.49cm] at (\x,\y) {\number\value{mycount1}};
+    \addtocounter{mycount1}{-1};
+  }
+}
+}
+\end{scope}
+
+\begin{scope}[yshift=5.5em,xshift=0em+3in]
+\visible<5->{
+\draw[step=0.5cm,color=ugreen,thick] (-0.5,-1) grid (0.5,1.0);
+\foreach \y in {+0.75,+0.25,-0.25,-0.75}{
+  \setcounter{mycount1}{2}
+  \foreach \x in {-0.25,0.25}{
+    \node [fill=green!20,inner sep=0pt,minimum height=0.49cm,minimum width=0.49cm] at (\x,\y) {\number\value{mycount1}};
+    \addtocounter{mycount1}{-1};
+  }
+}
+}
+\end{scope}
+
+\begin{scope}[yshift=5.0em,xshift=-0.5em+3in]
+\visible<6->{
+\draw[step=0.5cm,color=red,thick] (-0.5,-1) grid (0.5,1.0);
+\foreach \y in {+0.75,+0.25,-0.25,-0.75}{
+  \setcounter{mycount1}{2}
+  \foreach \x in {-0.25,0.25}{
+    \node [fill=red!20,inner sep=0pt,minimum height=0.49cm,minimum width=0.49cm] at (\x,\y) {\number\value{mycount1}};
+    \addtocounter{mycount1}{-1};
+  }
+}
+}
+
+\visible<3->{
+\node [anchor=north] (xlabel) at (0,-1.2) {$\textbf{x} \cdot \textbf{w}$};
+\node [anchor=center] (elabel) at (-0.7in,0) {\Huge{$\textbf{=}$}};
+}
+\end{scope}
+
+\end{tikzpicture}
+\end{center}
+
+\end{frame}
+
+%%%------------------------------------------------------------------------------------------------------------
+%%% 张量的单元操作
+\begin{frame}{张量的单元操作}
+\begin{itemize}
+\item 神经网络$\textbf{y}=f(\textbf{x}\cdot \textbf{w} + \textbf{b})$也包括一些张量的单元操作（element-wise opertation）
+	\begin{itemize}
+	\item 加法：$\textbf{s}+\textbf{b}$，其中$\textbf{s} = \textbf{x}\cdot \textbf{w}$
+	\item 激活函数：$f(\cdot)$
+	\end{itemize}
+\item<2-> \textbf{单元加}就是对张量中的每个位置都进行加法
+	\begin{itemize}
+	\item 一般要求两个张量的形状是一样的
+	\item<3-> 不过，这里可以使用加法的\textbf{广播}，重复利用一个张量进行加法，并不要求两个张量形状相同
+	\end{itemize}
+\end{itemize}
+
+\vspace{-1.5em}
+\begin{center}
+\begin{tikzpicture}
+\visible<3->{
+\begin{scope}
+\setcounter{mycount1}{1}
+\draw[step=0.5cm,color=orange,thick] (-1,-0.5) grid (1,0.5);
+\foreach \y in {+0.25,-0.25}
+  \foreach \x in {-0.75,-0.25,0.25,0.75}{
+    \node [fill=orange!20,inner sep=0pt,minimum height=0.49cm,minimum width=0.49cm] at (\x,\y) {\number\value{mycount1}};
+    \addtocounter{mycount1}{1};
+  }
+\node [anchor=south] (varlabel) at (0,0.6) {$\textbf{s}$};
+\end{scope}
+\begin{scope}[xshift=1.5in]
+\setcounter{mycount1}{1}
+\draw[step=0.5cm,color=ugreen,thick] (-1,-0) grid (1,0.5);
+\foreach \y in {+0.25}
+  \foreach \x in {-0.75,-0.25,0.25,0.75}{
+    \node [fill=green!20,inner sep=0pt,minimum height=0.49cm,minimum width=0.49cm] at (\x,\y) {1};
+    \addtocounter{mycount1}{1};
+  }
+\node [anchor=center] (plabel) at (-4.5em,0) {\huge{$\textbf{+}$}};
+\node [anchor=south] (varlabel) at (0,0.6) {$\textbf{b}$};
+\end{scope}
+\begin{scope}[xshift=3in]
+\setcounter{mycount1}{2}
+\draw[step=0.5cm,color=orange,thick] (-1,-0.5) grid (1,0.5);
+\foreach \y in {+0.25,-0.25}
+  \foreach \x in {-0.75,-0.25,0.25,0.75}{
+    \node [fill=orange!20,inner sep=0pt,minimum height=0.49cm,minimum width=0.49cm] at (\x,\y) {\number\value{mycount1}};
+    \addtocounter{mycount1}{1};
+  }
+\node [anchor=center] (plabel) at (-4.5em,0) {\huge{$\textbf{=}$}};
+\node [anchor=south] (varlabel) at (0,0.6) {$\textbf{s+b}$};
+\end{scope}
+}
+
+\end{tikzpicture}
+\end{center}
+
+\begin{itemize}
+\item<4-> 类似的，我们可以对所有位置做加法、乘法等等一系列算数操作，也包括激活函数
+\end{itemize}
+
+\end{frame}
+
+%%%------------------------------------------------------------------------------------------------------------
 \subsection{参数学习 - 反向传播}
 \subsection{参数学习 - 反向传播}
 
 
 \end{CJK}
 \end{CJK}