<html><head><meta name="color-scheme" content="light dark"></head><body><pre style="word-wrap: break-word; white-space: pre-wrap;">\def\CTeXPreproc{Created by ctex v0.2.13, don't edit!}
\documentclass[cjk,t,compress,12pt]{beamer}
\usepackage{pstricks}
\usepackage{etex}
\usepackage{eso-pic,graphicx}
\usepackage{fancybox}
\usepackage{amsmath,amssymb}
\usepackage{setspace}
\usepackage{xcolor}
\usepackage{array,multirow}
\usepackage{CJK}
\usepackage{tikz}
\usepackage{tikz-qtree}
\usepackage{hyperref}
\usepackage{changepage}
\usepackage{pgfplots}
\usepackage{subfigure}
\usepackage{tikz-3dplot}
\usepackage{esvect}

\usepackage{tcolorbox}
\tcbuselibrary{skins}

\usetikzlibrary{calc,intersections}
\usetikzlibrary{matrix}
\usetikzlibrary{arrows,decorations.pathreplacing}
\usetikzlibrary{shadows} % LATEX and plain TEX when using Tik Z
\usetikzlibrary{shadows.blur}

\usepgflibrary{arrows} % LATEX and plain TEX and pure pgf
\usetikzlibrary{arrows} % LATEX and plain TEX when using Tik Z
\usetikzlibrary{decorations}
\usetikzlibrary{arrows,shapes}

\usetikzlibrary{positioning,fit,calc}

\usetikzlibrary{mindmap,backgrounds} % mind map

\DeclareMathOperator*{\argmax}{arg\,max}
\DeclareMathOperator*{\argmin}{arg\,min}

\setbeamertemplate{items}[ball]
\usefonttheme[onlymath]{serif}  % fout of math

\definecolor{ugreen}{rgb}{0,0.5,0}
\definecolor{lgreen}{rgb}{0.9,1,0.8}
\definecolor{xtgreen1}{rgb}{0.824,0.898,0.8}
\definecolor{xtgreen}{rgb}{0.914,0.945,0.902}
\definecolor{lightgray}{gray}{0.85}

\setbeamercolor{uppercol}{fg=white,bg=ugreen}
\setbeamercolor{lowercol}{fg=black,bg=xtgreen}

\definecolor{ublue}{rgb}{0.152,0.250,0.545}
\setbeamercolor{uppercolblue}{fg=white,bg=ublue}
\setbeamercolor{lowercolblue}{fg=black,bg=blue!10}


%\usetheme{default}
%\usetheme{Darmstadt}
%\usetheme{Madrid}
%\usetheme{Frankfurt}
%\usetheme{Dresden}
%\usetheme{Boadilla}
%\usecolortheme{dolphin}

\newcounter{mycount1}
\newcounter{mycount2}
\newcounter{mycount3}
\newcounter{mycount4}

\usefonttheme[onlylarge]{structurebold}

\IfFileExists{C:/WINDOWS/win.ini}
{\newcommand{\mycfont}{you}}
{\newcommand{\mycfont}{gbsn}}

\begin{CJK}{GBK}{\mycfont}
\end{CJK}

\setbeamerfont*{frametitle}{size=\large,series=\bfseries}
\setbeamertemplate{navigation symbols}{\begin{CJK}{GBK}{\mycfont} ������ �����������ģ�� \hspace*{2em} Фͩ\&amp;�쾸�� \end{CJK} \hspace*{2em} \today \hspace*{2em} \insertframenumber{}/\inserttotalframenumber}

\setbeamertemplate{itemize items}[circle] % if you want a circle
\setbeamertemplate{itemize subitem}[triangle] % if you wnat a triangle
\setbeamertemplate{itemize subsubitem}[ball] % if you want a ball

\begin{document}

\begin{CJK}{GBK}{\mycfont}

\title{\Large{�����������ģ��}}
\author{\large{\textbf{Фͩ\ \ �쾸��}}}
\institute{
\blue{\url{xiaotong@mail.neu.edu.cn}} \black{} \\
\blue{\url{zhujingbo@mail.neu.edu.cn}} \black{} \\
\vspace{1.0em}
������ѧ ��Ȼ���Դ���ʵ���� \\
\blue{\underline{\url{http://www.nlplab.com}}} \black{} \\
\vspace{0.2cm}
\hspace{0.1cm} \includegraphics[scale=0.1]{../Figures/logo.pdf}
}
\date{}

\maketitle

\setlength{\leftmargini}{1em}
\setlength{\leftmarginii}{1em}

%%%------------------------------------------------------------------------------------------------------------
\section{ΪʲôҪ̸������}

%%%------------------------------------------------------------------------------------------------------------
\subsection{��ʷ}

%%%------------------------------------------------------------------------------------------------------------
%%% ΪʲôҪ̸������
\begin{frame}{ΪʲôҪ̸������}
\begin{itemize}
\item ��Щ��\textbf{���ѧϰ��Deep Learning��}�����˾޴��Ǳ��
    \begin{itemize}
    \item ϯ���˰��������������ڵĺܶ�NLP����
    \item �Ѿ���Ϊ��NLP�з������·�ʽ
    \item ������\textbf{�񾭻�������}����һ����������һ�����ݣ�
    \end{itemize}
\vspace{0.2em}
\begin{center}
\includegraphics[scale=0.45]{./Figures/deeplearning.jpg}
\end{center}
\vspace{0.5em}

\item&lt;2-&gt; \textbf{�˹������磨Artificial Neural Network��}�����ѧϰ��ʵ������

\end{itemize}

\end{frame}

%%%------------------------------------------------------------------------------------------------------------
%%% �򵥵���ʷ
\begin{frame}{����������ѧϰ�ĸ��1940s-1970s��}
\begin{itemize}
\item \textbf{������}��������ڿ������У�Cybernetics��������������������壨Connectionism���б��ἰ
    \begin{itemize}
    \item \textbf{������뷨}��ģ����Ե�����ѧϰ���ƽ��м������ģ
    \item&lt;2-&gt; ����ʹ�����Լ�Ȩ��������������$\textbf{x}$�ͽ��$\textbf{y}$֮�����ϵ
    \vspace{-0.5em}
    \begin{displaymath}
    f(\textbf{x},\textbf{w})=x_1 \cdot w_1 + ... + x_n \cdot w_n
    \end{displaymath}\\
    \vspace{-0.5em}
    ����$\textbf{w}$��Ȩ�ء�����ģ��ҲӰ��������ݶ��½������ڻ���ѧϰ�����ķ�չ��
    \item&lt;3-&gt; ���෽���ľ���Ҳ�����ԣ��޷��������������⣬���������������XOR��ѧϰ����
    \end{itemize}

\end{itemize}

\vspace{-0.5em}
\begin{center}
\includegraphics[scale=0.21]{./Figures/concept-history.jpg}\\
\scriptsize{ͼƬ���ԡ�Deep Learning��}
\end{center}

\end{frame}

%%%------------------------------------------------------------------------------------------------------------
%%% ���ѧϰ�ķ�չ
\begin{frame}{����������ѧϰ�ķ�չ��1980s-1990s��}
\begin{itemize}
\item ���ڣ�����ѧ�����Ѿ������������Ψһ�����Դ�����ѧϰҲ�������µķ�չ�׶Ρ�����˼��Ӱ��޴�
    \begin{itemize}
    \item&lt;2-&gt; \textbf{�������壨Connectionism��}������֪ѧ���У����ڵķ������壨Symbolicism�����ѽ��ʹ������ʹ����Ԫ������������������ĺ���˼���ǣ��������򵥵ļ��㵥Ԫ���ӵ�һ�����ʵ��������Ϊ����\\
        ��Ҳ�ƶ��˷��򴫲���ѵ����������緽����Ӧ�ã�����չ�˰�������ʱ����ģ�����ڵľ��佨ģ������
    \item&lt;3-&gt; \textbf{�ֲ�ʽ��ʾ��Distributed representation��}��һ������ϵͳ���κβ��ֵ����붼Ӧ���Ƕ��������ͬ��ʾ�Ľ�������磬һ�����ʲ���һ�������������ɳɰ���ǧ��������ͬ������������ÿ������������������ʵ�"ĳ��"���档
    \end{itemize}
\item&lt;4-&gt; \alert{�ź�����}��������90������ڣ��ںܶ�Ӧ�������Ƕ������緽���������ߣ����ǽ����û�дﵽԤ�ڡ��ر��ǣ��˷�����ͼģ�͵Ȼ���ѧϰ����ȡ���˺ܺõ�Ч�����������о�������һ�ε͹ȡ�

\end{itemize}

\end{frame}

%%%------------------------------------------------------------------------------------------------------------
%%% ���ѧϰ�ĵ������˳�
\begin{frame}{�������˳���2000s-now��}
\begin{itemize}
\item \textbf{���ѧϰ�ı���}Դ��2006��Hinton���˳ɹ�ѵ����һ������������磨deep belief network����֮�����ѧϰ���˳���ϯ���˻���ѧϰ���˹�����Ӧ���������������ִ����ѧϰ�ĳɹ���������ԭ��
    \begin{enumerate}
    \item \textbf{ģ�ͺ��㷨}��������Ľ�
    \item \textbf{���м�������}������ʹ���ģʵ����Ϊ�˿���
    \item ��Hinton����Ϊ������ѧ�ߵ�\textbf{��������Ͷ��}
    \end{enumerate}
\item&lt;2-&gt; \textbf{��Ӧ�õĽǶ�}���������Ŀ���������ģ������������ҲΪ���ѧϰ�ĳɹ��ṩ������
\end{itemize}

\visible&lt;2-&gt;{
\begin{center}
\begin{tikzpicture}
\scriptsize{
\begin{semilogyaxis}[
    width=.95\textwidth,
    height=.38\textwidth,
    yticklabel style={/pgf/number format/precision=1,/pgf/number format/fixed zerofill},
    xticklabel style={/pgf/number format/1000 sep=},
    xlabel style={yshift=0.5em},
    xlabel={\footnotesize{Year}},ylabel={\footnotesize{\# of sents.}},
    ymin=1,ymax=1000000000000,
    xmin=1999,xmax=2020,xtick={2000,2005,2010,2015,2020},
    legend style={yshift=-5em,xshift=0em,legend cell align=left,legend plot pos=right}
]

\addplot[purple,mark=square,mark=star,very thick] coordinates {(2001,10000) (2005,2000000) (2008,8000000) (2009,9000000) (2011,10000000) (2012,12000000) (2014,20000000) (2016,30000000) (2018,40000000) };
\addlegendentry{\tiny{Bi-text used in MT papers}\ \ \ \ \ \ \ \ \ \ }
\only&lt;3-&gt;{
\addplot[ublue,mark=otimes*,very thick] coordinates {(2005,10000000) (2008,100000000) (2012,3000000000) (2016,5000000000) (2019,10000000000) };
\addlegendentry{\tiny{Bi-text used in practical systems}}
}

\end{semilogyaxis}
}
\end{tikzpicture}
\end{center}
}

\end{frame}

%%%------------------------------------------------------------------------------------------------------------
\subsection{���ѧϰ������}

%%%------------------------------------------------------------------------------------------------------------
%%% �˵���ѧϰ
\begin{frame}{�˵���ѧϰ}
\begin{itemize}
\item ���������������ṩ��һ�ֻ��ƣ�����ֱ�Ӵ�ѧϰ���뵽����Ĺ�ϵ����֮Ϊ\alert{�˵���ѧϰ}
    \begin{itemize}
    \item&lt;2-&gt; \textbf{�����������̵ķ���}����Ҫ�����˹�������������������������������������������
    \item&lt;3-&gt; \textbf{���ڶ˵���ѧϰ�ķ���}��û���˹����������������������ȫ�������罨ģ
    \end{itemize}
\end{itemize}
\vspace{-0.5em}
\begin{center}
\visible&lt;2-&gt;{
\includegraphics[scale=0.31]{./Figures/end2end-learning-1.jpg}\\
}
\visible&lt;3-&gt;{
\Large{\textbf{VS.}}\\
\vspace{0.3em}
\includegraphics[scale=0.31]{./Figures/end2end-learning-2.jpg}\\
}
\end{center}
\end{frame}

%%%------------------------------------------------------------------------------------------------------------
%%% ���ѧϰ�ı���
\begin{frame}{���ѧϰ�ı��� - �����Խ�ģΪ��}
\begin{itemize}
\item \textbf{����}�������Խ�ģ��LM�������ϣ���������������ѧϰ�ķ��������˾޴����ƣ���PTB������PPLֵ�Ѿ��õ����˵��½���PPLԽ��Խ�ã�
	\begin{itemize}
	\item ��ͳ$n$Ԫ�﷨ģ����������ϡ�������
	\item&lt;2-&gt; ������ģ�Ϳ��Ը��õ�����������������
	\end{itemize}
\end{itemize}
\begin{tabular}{l | l | l | r}
ģ�� &amp; ���� &amp; ��� &amp; PPL  \\ \hline
3-gram LM &amp; Brown et al. &amp; 1992 &amp; 178.0 \pause \\ \hline
Feed-forward Neural LM &amp; Bengio et al. &amp; 2003 &amp; 162.2 \\
Recurrent NN-based LM &amp; Mikolov et al. &amp; 2010 &amp; 124.7 \\
Recurrent NN-LDA &amp; Mikolov et al. &amp; 2012 &amp; 92.0 \\
LSTM &amp; Zaremba et al. &amp; 2014 &amp; 78.4 \\
RHN &amp; Zilly et al. &amp; 2016 &amp; 65.4 \\
AWD-LSTM &amp; Merity et al. &amp; 2018 &amp; 58.8 \\
GPT-2 (Transformer) &amp; Radford et al. &amp; 2019 &amp; \alert{35.7}
\end{tabular}
\end{frame}

%%%------------------------------------------------------------------------------------------------------------
\section{���������}

%%%------------------------------------------------------------------------------------------------------------
\subsection{�򵥵�����}

%%%------------------------------------------------------------------------------------------------------------
%%% ��Ԫ
\begin{frame}{������Ļ�����Ԫ - ��Ԫ}
\begin{itemize}
\item ����ѧ�ϣ���Ԫ����ϵͳ�Ļ�����ɵ�Ԫ���ܶ��������������Ӧ����������\\
\begin{center}
\includegraphics[scale=0.25]{./Figures/neuron-real.jpg}\\
\end{center}
\item&lt;2-&gt; ����������˵����\textbf{�˹���Ԫ}��ʵ������������ :)
    \begin{itemize}
    \item ����$\textbf{x}$����$\textbf{w}$�������Ա仯��֮�����ƫ��$\textbf{b}$���ھ��������$f$�����õ�$\textbf{y}$ - ɶ����������
    \end{itemize}
{\Large
\begin{displaymath}
\textbf{y} = f(\textbf{x} \cdot \textbf{w} + \textbf{b})
\end{displaymath}
}
\\
\vspace{-0.5em}
\end{itemize}
\end{frame}

%%%------------------------------------------------------------------------------------------------------------
%%% ��֪��
\begin{frame}{��򵥵��˹���Ԫģ�� - ��֪����Perceptron��}
\begin{itemize}
\item ��֪�����˹���Ԫ��һ��ʵ������������50-60���������󣬶��������о���������ԶӰ�졣
    \begin{itemize}
    \item&lt;2-&gt; \textbf{����}�����ɸ���ֵ������$x_i=0$ or $1$
    \item&lt;3-&gt; ÿһ�����������Ӧһ��\textbf{Ȩ��}$w_i$��ʵ����
    \item&lt;4-&gt; \textbf{���}Ҳ��һ����ֵ�����$y=0$ or $1$�� �жϵ������ǣ�����ͼ�Ȩ���Ƿ���ڣ�����С�ڣ�һ����ֵ$\sigma$��
    \begin{displaymath}
    y = \left\{ \begin{array}{ll}
    0 &amp; \sum_i {w_i \cdot x_i} &lt; \sigma \\
    1 &amp; \sum_i {w_i \cdot x_i} \ge \sigma
    \end{array} \right.
    \end{displaymath}
    \end{itemize}
\end{itemize}

\begin{center}
\begin{tikzpicture}
\begin{scope}
\node [anchor=center,circle,draw,ublue,very thick,minimum size=3.5em,fill=white,drop shadow={shadow xshift=0.1em,shadow yshift=-0.1em}] (neuron) at (0,0) {};
\node [anchor=east] (x1) at ([xshift=-6em]neuron.west) {\Large{$x_1$}};
\node [anchor=center] (x0) at ([yshift=3em]x1.center) {\Large{$x_0$}};
\node [anchor=center] (x2) at ([yshift=-3em]x1.center) {\Large{$x_2$}};
\node [anchor=west] (y) at ([xshift=6em]neuron.east) {\Large{$y$}};

\draw [-&gt;,thick] (x0.east) -- (neuron.150) node [pos=0.5,above] {$w_0$};
\draw [-&gt;,thick] (x1.east) -- (neuron.180) node [pos=0.5,above] {$w_1$};
\draw [-&gt;,thick] (x2.east) -- (neuron.210) node [pos=0.5,above] {$w_2$};
\draw [-&gt;,thick] (neuron.east) -- (y.west);

\visible&lt;2&gt;{
\draw [-&gt;,thick,red] (x0.east) -- (neuron.150) node [pos=0.5,above] {\black{$w_0$}};
\draw [-&gt;,thick,red] (x1.east) -- (neuron.180) node [pos=0.5,above] {\black{$w_1$}};
\draw [-&gt;,thick,red] (x2.east) -- (neuron.210) node [pos=0.5,above] {\black{$w_2$}};
}

\visible&lt;3&gt;{
\draw [-&gt;,thick] (x0.east) -- (neuron.150) node [pos=0.5,above] {\red{$w_0$}};
\draw [-&gt;,thick] (x1.east) -- (neuron.180) node [pos=0.5,above] {\red{$w_1$}};
\draw [-&gt;,thick] (x2.east) -- (neuron.210) node [pos=0.5,above] {\red{$w_2$}};
}

\visible&lt;4-&gt;{
\node [anchor=center] (neuronmath) at (neuron.center) {\red{\small{$\sum \ge \sigma$}}};
}

\visible&lt;5-&gt;{
\node [anchor=south] (prediction) at ([xshift=-2em,yshift=1em]y.north west) {\footnotesize{\red{$x_0 w_0 + x_1 w_1 + x_2 w_2 \ge \sigma$}}};
\draw [-&gt;,thick,red] (neuron.east) -- (y.west);
\node [anchor=west] (yvalue) at ([yshift=0.2em]y.east) {\Large{$=1$}};
}

\end{scope}
\end{tikzpicture}
\end{center}

\end{frame}

%%%------------------------------------------------------------------------------------------------------------
%%% ��֪�� - һ������
\begin{frame}{һ������}
\begin{itemize}
\item һ���ǳ��򵥵����ӡ����磬��һ�����ֻᣬ�����ھ����Ƿ�ȥ�μӣ����������ػ�Ӱ����ľ���
    \begin{itemize}
    \item $x_0$���糡�Ƿ������㹻����
    \item $x_1$��Ʊ���Ƿ����300Ԫ��
    \item $x_2$��Ů�����Ƿ�ϲ�����ֻ᣿
    \end{itemize}
\item&lt;2-&gt; ��ξ��������磬Ů���Ѻ�ϣ������һ�𣬵��Ǿ糡��Զ����Ʊ��500Ԫ�������Щ���ض���ľ��߶���ͬ����Ҫ�ģ���ô����һ���ۺϵ÷֣�
    \begin{displaymath}
    x_0 \cdot w_0 + x_1 \cdot w_1 + x_2 \cdot w_2 = 0 \cdot 1 + 0 \cdot 1 + 1 \cdot 1 = 1
    \end{displaymath}
\item&lt;3-&gt; ����㲻��ʮ�־��ᣬ�ܹ����ܲ����������飬����ܻ���$\sigma=1$������
    \begin{displaymath}
    \sum_i x_i \cdot w_i \ge \sigma
    \end{displaymath}
    \textbf{��ô}�����ȥ�μ����ֻ�
\end{itemize}
\end{frame}

%%%------------------------------------------------------------------------------------------------------------
%%% ��֪�� - һ�����ӣ�Ȩ��
\begin{frame}{һ������ - Ȩ��}
\begin{itemize}
\item ���Կ�����ʵ����������߹��̱����Ͼ���һ����֪��
\item&lt;2-&gt; ���ǣ��˲����������ģ���������Щ����������һЩ���������\textbf{�ز�ū}����˻��Ʊ�ۿ��ĸ���һЩ����ʱ����ò����ȵ�Ȩ�ؼ���ÿ�����ص�Ӱ�죬���磺$w_0=0.5$��$w_1=2$��$w_2=0.5$
\item&lt;3-&gt; Ů�Ѻ�ϣ������һ�𣬵��Ǿ糡��Զ����Ʊ��500Ԫ���ᵼ����\alert{ѡ��ȥ}�����ֻᣨŮ���Ѷ���Ҫ�ˣ�զ����
    \begin{displaymath}
    \sum_i x_i \cdot w_i = 0 \cdot 0.5 + 0 \cdot 2 + 1 \cdot 0.5 = 0.5 &lt; \sigma = 1
    \end{displaymath}
\end{itemize}

\vspace{-1.8em}
\begin{center}
\begin{tikzpicture}
\begin{scope}
\node [anchor=center,circle,draw,ublue,very thick,minimum size=3.5em,fill=white,drop shadow={shadow xshift=0.1em,shadow yshift=-0.1em}] (neuron) at (0,0) {};
\node [anchor=east] (x1) at ([xshift=-6em]neuron.west) {$x_1$:Ʊ�۹��ͣ�};
\node [anchor=center] (x0) at ([yshift=3em]x1.center) {$x_0$:���빻����};
\node [anchor=center] (x2) at ([yshift=-3em]x1.center) {$x_2$:Ů��ϲ����};
\node [anchor=west] (y) at ([xshift=2em]neuron.east) {$y$:ȥ�����ǲ�ȥ��};

\visible&lt;1&gt;{
\draw [-&gt;,thick] (x0.east) -- (neuron.150) node [pos=0.5,above,yshift=0.2em] {\small{$w_0=1$}};
\draw [-&gt;,thick] (x1.east) -- (neuron.180) node [pos=0.5,above,yshift=-0.1em] {\small{$w_1=1$}};
\draw [-&gt;,thick] (x2.east) -- (neuron.210) node [pos=0.5,above,yshift=0.1em] {\small{$w_2=1$}};
}
\draw [-&gt;,thick] (neuron.east) -- (y.west);

\node [anchor=center] (neuronmath) at (neuron.center) {\small{$\sum \ge \sigma$}};

\visible&lt;2-&gt;{
\draw [-&gt;,thin,red] (x0.east) -- (neuron.150) node [pos=0.5,above,yshift=0.2em] {\small{$w_0=.5$}};
\draw [-&gt;,line width=0.8mm,red] (x1.east) -- (neuron.180) node [pos=0.5,above,yshift=-0.1em] {\textbf{\small{$w_1=2$}}};
\draw [-&gt;,thin,red] (x2.east) -- (neuron.210) node [pos=0.5,above,yshift=0.1em] {\small{$w_2=.5$}};
}

\visible&lt;3-&gt;{
\node [anchor=south] (ylabel) at (y.north) {\red{\textbf{��ȥ�ˣ�}}};
}

\end{scope}
\end{tikzpicture}
\end{center}

\end{frame}

%%%------------------------------------------------------------------------------------------------------------
%%% ��֪�� - һ�����ӣ�������ʽ
\begin{frame}{һ������ - ������ʽ}

\begin{itemize}
\item ��������Ů��һ����˺�֮������ʶ�����߲�Ӧ��ֻ���Ƿ�0��1�����أ�Ӧ�ð�``�̶�''���ǽ�����
    \begin{itemize}
    \item $x_0$��10/����
    \item $x_1$��150/Ʊ��
    \item $x_2$��Ů�����Ƿ�ϲ����(�������Ҹ�)
    \end{itemize}
\item&lt;2-&gt; ��ģ���У�$x_0$��$x_1$������������$x_2$��һ����ɢ����
\end{itemize}

\visible&lt;2-&gt;{
\begin{tikzpicture}

\begin{scope}
\draw [-&gt;,thick] (0,0) -- (2.5,0);
\draw [-&gt;,thick] (0,0) -- (0, 1.5);
\draw [-,very thick,ublue,domain=0.6:2,samples=100] plot (\x,{ 1/\x - 0.2});
\node [anchor=east] (ylabel) at (0, 3.2em) {\footnotesize{$x_0$}};
\node [anchor=north] (xlabel) at (5em, 0em) {\scriptsize{����(km)}};
\end{scope}

\begin{scope}[xshift=9em]
\draw [-&gt;,thick] (0,0) -- (2.5,0);
\draw [-&gt;,thick] (0,0) -- (0, 1.5);
\draw [-,very thick,ublue,domain=0.4:2,samples=100] plot (\x,{ 0.5/\x});
\node [anchor=east] (ylabel) at (0, 3.2em) {\footnotesize{$x_1$}};
\node [anchor=north] (xlabel) at (5em, 0em) {\scriptsize{Ʊ��(Ԫ)}};
\end{scope}

\begin{scope}[xshift=18em]
\draw [-&gt;,thick] (0,0) -- (2.5,0);
\draw [-&gt;,thick] (0,0) -- (0, 1.5);
\node [anchor=east] (ylabel) at (0, 3.2em) {\footnotesize{$x_2$}};
\node [anchor=south, fill=ublue, minimum width=1.5em, minimum height=0.1em, inner sep=0] (histogram1) at (1.5em, 0) {};
\node [anchor=south, fill=ublue, minimum width=1.5em, minimum height=3em, inner sep=0] (histogram2) at (4.0em, 0) {};
\node [anchor=north] (hlabel1) at (histogram1.south) {\tiny{Ů�Ѳ�ȥ}};
\node [anchor=north] (hlabel2) at (histogram2.south) {\tiny{Ů��ȥ}};
\end{scope}

\end{tikzpicture}
}

\begin{itemize}
\item&lt;3-&gt; Ů���Ѻ�ϣ������һ�𣬵��Ǿ糡��20kmԶ����Ʊ��500Ԫ��������\ $x_0 = 10/20 = 0.5$��$x_1=150/500 = 0.3$, $x_2=1$���ۺ�����$\sum_i x_i \cdot w_i \ge \sigma$������{\color{red} ȥ�����ֻ�} :)
    \begin{displaymath}
    \sum_i x_i \cdot w_i = 0.5 \cdot 0.5 + 0.3 \cdot 2 + 1 \cdot 0.5 = 1.35 \ge \sigma = 1
    \end{displaymath}
\end{itemize}

\end{frame}

%%%------------------------------------------------------------------------------------------------------------
%%% ��֪�� - һ������ - ѧϰ
\begin{frame}{һ������ - ѧϰ}

\begin{itemize}
\item һ�γɹ������ֻ�֮�����ƺ�������������ֻҪŮ���ѿ��ľͺã�Ϊ�β���������ص�Ȩ�ص�����򵥵ķ�ʽ�ǰ�$w_0$��$w_1$��Ȩ�ض���0��ͬʱ��$w_3 &gt; 0$
\item&lt;3-&gt; �ܿ�����һ�����ֻᣬ����1000���Ʊ�ۣ�����·�ѣ�3000Ԫ����Ȼ��Ů����һֱ��ϲ�����ֻ�ġ������µľ���ģ�ͣ������޷��˵�\alert{����ȥ��}�ⳡ���ֻ�
\item&lt;4-&gt; \textbf{֮��}����Ů�����ָ�����1����˺���ʹ��������
    \begin{itemize}
    \item \alert{����㷢��}��Ů�Ѽ�Ҫ������ͬʱҲ����
    \end{itemize}
\end{itemize}

\begin{center}
\begin{tikzpicture}
\begin{scope}
\node [anchor=center,circle,draw,ublue,very thick,minimum size=3.5em,fill=white,drop shadow={shadow xshift=0.1em,shadow yshift=-0.1em}] (neuron) at (0,0) {};
\node [anchor=east] (x1) at ([xshift=-6em]neuron.west) {$x_1$:���˳̶�\ \ \ \ };
\node [anchor=center] (x0) at ([yshift=3em]x1.center) {$x_0$:Զ���̶�\ \ \ \ };
\node [anchor=center] (x2) at ([yshift=-3em]x1.center) {$x_2$:Ů��ϲ����};
\node [anchor=west] (y) at ([xshift=2em]neuron.east) {$y$:ȥ�����ǲ�ȥ��};

\draw [-&gt;,thick] (neuron.east) -- (y.west);

\node [anchor=center] (neuronmath) at (neuron.center) {\small{$\sum \ge \sigma$}};

\visible&lt;1&gt;{
\draw [-&gt;,thin] (x0.east) -- (neuron.150) node [pos=0.5,above,yshift=0.2em] {\small{$w_0=.5$}};
\draw [-&gt;,line width=0.8mm] (x1.east) -- (neuron.180) node [pos=0.5,above,yshift=-0.1em] {\textbf{\small{$w_1=2$}}};
\draw [-&gt;,thin] (x2.east) -- (neuron.210) node [pos=0.5,above,yshift=0.1em] {\small{$w_2=.5$}};
}

\visible&lt;2-&gt;{
\draw [-&gt;,dotted] (x0.east) -- (neuron.150) node [pos=0.5,above,yshift=0.2em] {\small{$w_0=0$}};
\draw [-&gt;,dotted] (x1.east) -- (neuron.180) node [pos=0.5,above,yshift=-0.1em] {\textbf{\small{$w_1=0$}}};
\draw [-&gt;,line width=1mm] (x2.east) -- (neuron.210) node [pos=0.5,above,yshift=0.1em] {\small{$w_2=10$}};
}


\end{scope}
\end{tikzpicture}
\end{center}

\end{frame}

%%%------------------------------------------------------------------------------------------------------------
%%% ��֪�� - һ������ - ѧϰ(cont)
\begin{frame}{һ������ - Ȩ��ѧϰ}
\begin{itemize}
\item \textbf{ʹ��˼ʹ}���㷢��ÿ�����ص�Ȩ����Ҫ׼ȷ�����ò��ܴﵽ��õľ���Ч��
    \begin{itemize}
    \item ���ȷ����õ�Ȩ�أ�
    \end{itemize}
\item&lt;2-&gt; \textbf{��Ȼ}������һ������ʵ������
    \begin{itemize}
    \item �����ܼ򵥣����ϵس��ԣ����ݽṹ���ϵص���Ȩ��
    \item&lt;10-&gt; �ڽ����˺ܶ��ʵ��󣬷�������Ժõ�һ��Ȩ��
    \end{itemize}
\end{itemize}

\visible&lt;2-&gt;{
\begin{center}
\begin{tikzpicture}

\begin{scope}[scale=0.6]
\visible&lt;3-&gt;{
\draw [-&gt;,thick] (0,0) -- (2.5,0);
\draw [-&gt;,thick] (0,0) -- (0, 1.5);
\draw [-,very thick,ublue,domain=0.6:2,samples=100] plot (\x,{ 1/\x - 0.2});
\node [anchor=east] (ylabel) at (0, 3.2em) {\footnotesize{$x_0$}};
\node [anchor=north] (xlabel) at (5em, 0em) {\tiny{����(km)}};
}

\visible&lt;4-&gt;{
\draw [-,thick] (0.25,-1.5) -- (2.25,-1.5);
\node [anchor=east] (wlabel) at (0.25,-1.5) {\footnotesize{$w_0$}};
}

\visible&lt;5&gt;{\node [anchor=north,fill=ugreen,minimum height=0.5em,minimum width=1.5em] (w0) at (1.25,-1.5) {};}
\visible&lt;6&gt;{\node [anchor=north,fill=ugreen,minimum height=0.3em,minimum width=1.5em] (w0) at (1.25,-1.5) {};}
\visible&lt;7&gt;{\node [anchor=north,fill=ugreen,minimum height=1.8em,minimum width=1.5em] (w0) at (1.25,-1.5) {};}
\visible&lt;8&gt;{\node [anchor=north,fill=ugreen,minimum height=0.3em,minimum width=1.5em] (w0) at (1.25,-1.5) {};}
\visible&lt;9&gt;{\node [anchor=north,fill=ugreen,minimum height=0.3em,minimum width=1.5em] (w0) at (1.25,-1.5) {};}
\visible&lt;10-&gt;{\node [anchor=north,fill=ugreen,minimum height=0.8em,minimum width=1.5em] (w0) at (1.25,-1.5) {};}

\end{scope}

\begin{scope}[scale=0.6,xshift=12em]
\visible&lt;3-&gt;{
\draw [-&gt;,thick] (0,0) -- (2.5,0);
\draw [-&gt;,thick] (0,0) -- (0, 1.5);
\draw [-,very thick,ublue,domain=0.4:2,samples=100] plot (\x,{ 0.5/\x});
\node [anchor=east] (ylabel) at (0, 3.2em) {\footnotesize{$x_1$}};
\node [anchor=north] (xlabel) at (5em, 0em) {\tiny{Ʊ��(Ԫ)}};
}

\visible&lt;4-&gt;{
\draw [-,thick] (0.25,-1.5) -- (2.25,-1.5);
\node [anchor=east] (wlabel) at (0.25,-1.5) {\footnotesize{$w_1$}};
}

\visible&lt;5&gt;{\node [anchor=north,fill=ugreen,minimum height=0.5em,minimum width=1.5em] (w1) at (1.25,-1.5) {};}
\visible&lt;6&gt;{\node [anchor=north,fill=ugreen,minimum height=1.5em,minimum width=1.5em] (w1) at (1.25,-1.5) {};}
\visible&lt;7&gt;{\node [anchor=north,fill=ugreen,minimum height=0.8em,minimum width=1.5em] (w1) at (1.25,-1.5) {};}
\visible&lt;8&gt;{\node [anchor=north,fill=ugreen,minimum height=0.1em,minimum width=1.5em] (w1) at (1.25,-1.5) {};}
\visible&lt;9&gt;{\node [anchor=north,fill=ugreen,minimum height=1.0em,minimum width=1.5em] (w1) at (1.25,-1.5) {};}
\visible&lt;10-&gt;{\node [anchor=north,fill=ugreen,minimum height=0.3em,minimum width=1.5em] (w1) at (1.25,-1.5) {};}

\end{scope}

\begin{scope}[scale=0.6,xshift=24em]
\visible&lt;3-&gt;{
\draw [-&gt;,thick] (0,0) -- (2.5,0);
\draw [-&gt;,thick] (0,0) -- (0, 1.5);
\node [anchor=east] (ylabel) at (0, 3.2em) {\footnotesize{$x_2$}};
\node [anchor=south, fill=ublue, minimum width=0.8em, minimum height=0.1em, inner sep=0] (histogram1) at (1.5em, 0) {};
\node [anchor=south, fill=ublue, minimum width=0.8em, minimum height=2em, inner sep=0] (histogram2) at (4.0em, 0) {};
\node [anchor=north,align=left] (hlabel1) at (histogram1.south) {\tiny{Ů��no}};
\node [anchor=north,align=left] (hlabel2) at ([xshift=0.5em]histogram2.south) {\tiny{Ů��yes}};
}

\visible&lt;4-&gt;{
\draw [-,thick] (0.25,-1.5) -- (2.25,-1.5);
\node [anchor=east] (wlabel) at (0.25,-1.5) {\footnotesize{$w_2$}};
}

\visible&lt;5&gt;{\node [anchor=north,fill=ugreen,minimum height=0.5em,minimum width=1.5em] (w2) at (1.25,-1.5) {};}
\visible&lt;6&gt;{\node [anchor=north,fill=ugreen,minimum height=1.2em,minimum width=1.5em] (w2) at (1.25,-1.5) {};}
\visible&lt;7&gt;{\node [anchor=north,fill=ugreen,minimum height=0.8em,minimum width=1.5em] (w2) at (1.25,-1.5) {};}
\visible&lt;8&gt;{\node [anchor=north,fill=ugreen,minimum height=1.2em,minimum width=1.5em] (w2) at (1.25,-1.5) {};}
\visible&lt;9&gt;{\node [anchor=north,fill=ugreen,minimum height=1.5em,minimum width=1.5em] (w2) at (1.25,-1.5) {};}
\visible&lt;10-&gt;{\node [anchor=north,fill=ugreen,minimum height=1.3em,minimum width=1.5em] (w2) at (1.25,-1.5) {};}

\end{scope}

\end{tikzpicture}
\end{center}

}

\visible&lt;5-&gt;{
\begin{center}
\begin{tabular}{c&lt;{\onslide&lt;5-&gt;}c&lt;{\onslide&lt;6-&gt;}c&lt;{\onslide&lt;7-&gt;}c&lt;{\onslide&lt;8-&gt;}c&lt;{\onslide&lt;9-&gt;}c&lt;{\onslide&lt;10-&gt;}c&lt;{\onslide}}
ʵ�� &amp; 1 &amp; 2 &amp; 3 &amp; 4 &amp; ... &amp; 10k \\
��� &amp; ʧ�� &amp; �ɹ� &amp; ʧ�� &amp; ʧ�� &amp; ... &amp; �ɹ�
\end{tabular}
\end{center}
}

\end{frame}

%%%------------------------------------------------------------------------------------------------------------
%%% ��֪�� - һ������ - �ܽ�
\begin{frame}{һ������ - �ܽ�}
\begin{itemize}
\item �������һ�������⣬������һ�ֺ���������׼ȷ�Ľ��о��߲����򵥡����������ģ���У�����һЩ\alert{����}��Ҫ�ش�
    \begin{itemize}
    \item&lt;2-&gt; �����⽨ģ��������������$\{x_i\}$����ʽ
    \item&lt;3-&gt; �����Ч�ľ���ģ�ͣ���������$y$
    \item&lt;4-&gt; ����ģ�����漰�Ĳ�������Ȩ��$\{w_i\}$��������ֵ
    \end{itemize}
\end{itemize}

\vspace{-2em}
\begin{center}
\begin{tikzpicture}
\begin{scope}
\node [anchor=center,circle,draw,ublue,very thick,minimum size=3.5em,fill=white,drop shadow={shadow xshift=0.1em,shadow yshift=-0.1em}] (neuron) at (0,0) {};

\visible&lt;2-&gt;{
\node [anchor=east] (x1) at ([xshift=-6em]neuron.west) {$x_1$:���˳̶�\ \ \ \ };
\node [anchor=center] (x0) at ([yshift=3em]x1.center) {$x_0$:Զ���̶�\ \ \ \ };
\node [anchor=center] (x2) at ([yshift=-3em]x1.center) {$x_2$:Ů��ϲ����};
}

\visible&lt;3-&gt;{
\node [anchor=west] (y) at ([xshift=2em]neuron.east) {$y$:ȥ�����ǲ�ȥ��};
\node [anchor=center] (neuronmath) at (neuron.center) {\small{$\sum \ge \sigma$}};
}

\draw [-&gt;,thick] (neuron.east) -- (y.west);

\draw [-&gt;,thick] (x0.east) -- (neuron.150);
\draw [-&gt;,thick] (x1.east) -- (neuron.180);
\draw [-&gt;,thick] (x2.east) -- (neuron.210);

\visible&lt;4-&gt;{
\draw [-&gt;,thick] (x0.east) -- (neuron.150) node [pos=0.5,above,yshift=0.2em] {$w_0$};
\draw [-&gt;,thick] (x1.east) -- (neuron.180) node [pos=0.5,above,yshift=-0.1em] {$w_1$};
\draw [-&gt;,thick] (x2.east) -- (neuron.210) node [pos=0.5,above,yshift=0.1em] {$w_2$};
}

\end{scope}
\end{tikzpicture}
\end{center}

\vspace{-0.5em}

\begin{itemize}
\item&lt;5-&gt; \textbf{��Ȼ}����������ݻ��漰��������⣬���Ҳ�ֹ��Щ :)
\end{itemize}

\end{frame}

%%%------------------------------------------------------------------------------------------------------------
\subsection{�˹���Ԫ}

%%%------------------------------------------------------------------------------------------------------------
%%% outline
\begin{frame}{�����˹�������(���ѧϰ)��������������}

\vspace{1em}

\begin{tcolorbox}[enhanced,size=normal,left=2mm,right=1mm,colback=red!5!white,colframe=red!75!black,drop fuzzy shadow]
{\Large
\textbf{1. �˹�������Ļ�����Ԫ��ʲô,}

\vspace{0.4em}
\textbf{\hspace{0.9em} �����ϳ���ǿ���ģ�ͣ�}
}
\end{tcolorbox}

\vspace{0.5em}

\begin{tcolorbox}[enhanced,size=normal,left=2mm,right=1mm,colback=red!5!white,colframe=red!75!black,drop fuzzy shadow]
{\Large
\textbf{2. �˹����������ѧ������ʲô,}

\vspace{0.4em}
\textbf{\hspace{0.9em} ��α��ʵ��������ѧģ�ͣ�}
}
\end{tcolorbox}

\vspace{0.5em}

\begin{tcolorbox}[enhanced,size=normal,left=2mm,right=1mm,colback=red!5!white,colframe=red!75!black,drop fuzzy shadow]
{\Large
\textbf{3. ��ζ�ģ���еĲ�������ѧϰ,}

\vspace{0.4em}
\textbf{\hspace{0.9em} ֮��ʹ��ѧϰ����ģ�ͽ����ƶϣ�}
}
\end{tcolorbox}

\end{frame}

%%%------------------------------------------------------------------------------------------------------------
%%% outline: problem 1
\begin{frame}{����}

\vspace{6em}

\begin{tcolorbox}[enhanced,size=normal,left=2mm,right=1mm,colback=red!5!white,colframe=red!75!black,drop fuzzy shadow]
{\Large
\textbf{�˹�������Ļ�����Ԫ��ʲô,}

\vspace{0.4em}
\textbf{�����ϳ���ǿ���ģ�ͣ�}
}
\end{tcolorbox}

\vspace{2em}
\begin{center}
\begin{tikzpicture}
\node [fill=blue!10] (label) at (0,0) {\Large{$\textbf{y} = ?(\textbf{x})$ }};
\end{tikzpicture}
\end{center}

\end{frame}

%%%------------------------------------------------------------------------------------------------------------
%%% ���Դ�������
\begin{frame}{Ԥ�� - ���Դ���֪ʶ}
\begin{itemize}
\item \textbf{����}��������$a$��ʾһ������(һ����)���ô���$\textbf{a}$��ʾһ������(������)������$a_{ij}$��ʾ$\textbf{a}$��$i$�С���$j$�е�Ԫ��\\
    \begin{displaymath}
    a = 5 \hspace{3em} \textbf{a} = \begin{pmatrix} a_{11} &amp; a_{12} \\ a_{21} &amp; a_{22} \end{pmatrix} = \begin{pmatrix} 1 &amp; 2 \\ 3 &amp; 4 \end{pmatrix}
    \end{displaymath}
\item \textbf{����}��һ������ľ���ֻ��һ�л���һ�У�����Ĭ��ʹ��������������$\textbf{a} = (a_1,a_2,a_3) = (10, 20, 30)$��$\textbf{a}$��Ӧ����������Ϊ$\textbf{a}^T$
\item&lt;2-&gt; \textbf{��������}��������԰�λ����+��-�ȴ������㣬����$\textbf{a} = \begin{pmatrix} 1 &amp; 2 \\ 3 &amp; 4 \end{pmatrix}$��$\textbf{b} = \begin{pmatrix} 1 &amp; 1 \\ 1 &amp; 1 \end{pmatrix}$����$\textbf{a} + \textbf{b} = \begin{pmatrix} 2 &amp; 3 \\ 4 &amp; 5 \end{pmatrix}$
\item&lt;3-&gt; \textbf{�����΢��}����λ���У����ھ���$\textbf{c}$�ͱ���$x$��
    \begin{displaymath}
    \frac{\partial \textbf{c}}{\partial x} = \begin{pmatrix} \frac{\partial c_{11}}{\partial x} &amp; \frac{\partial c_{12}}{\partial x} \\ \frac{\partial c_{21}}{\partial x} &amp; \frac{\partial c_{22}}{\partial x} \end{pmatrix} \hspace{2em} \frac{\partial x}{\partial \textbf{c}} = \begin{pmatrix} \frac{\partial x}{\partial c_{11}} &amp; \frac{\partial x}{\partial c_{12}} \\ \frac{\partial x}{\partial c_{21}} &amp; \frac{\partial x}{\partial c_{22}} \end{pmatrix}
    \end{displaymath}
\end{itemize}

\end{frame}

%%%------------------------------------------------------------------------------------------------------------
%%% ���Դ�������
\begin{frame}{Ԥ�� - ���Դ���֪ʶ(��)}
\begin{itemize}


\item \textbf{����ĳ˷�}������$\textbf{a} \in \mathbb{R}^{n \times k}$��$\textbf{b} \in \mathbb{R}^{k \times m}$����$\textbf{c} = \textbf{a} \textbf{b} \in \mathbb{R}^{n \times m}$��ʾ\textbf{a}��\textbf{b}�ľ���˷�������
    \begin{displaymath}
    c_{pq} = \sum_{i = 1}^k a_{pi} b_{iq}
    \end{displaymath}
    ���ڷ���$\left\{ \begin{array}{l} 5x_{1} + 2x_{2} = y_{1} \\ 3x_{1} + x_{2} = y_{2}\end{array} \right.$�����Ա�ʾΪ$\textbf{a} \textbf{x}^T = \textbf{y}^T$ ����$\textbf{a}=\begin{pmatrix} 5 &amp; 2 \\ 3 &amp; 1 \end{pmatrix}$��$\textbf{x}^T =\begin{pmatrix} x_1 \\ x_2 \end{pmatrix}$��$\textbf{y}^T =\begin{pmatrix} y_1 \\ y_2 \end{pmatrix}$
\item&lt;2-&gt; \textbf{����}
    \begin{itemize}
    \item \textbf{��λ����}������$\textbf{I}$��$I_{ij} = 1$���ҽ���$i=j$������$I_{ij} = 0$
    \item \textbf{ת��}��$\textbf{a}$��ת�ü�Ϊ$\textbf{a}^T$����$a^T_{ji}=a_{ij}$
    \item \textbf{�����}������$\textbf{a}$��������Ϊ$\textbf{a}^{-1}$����$\textbf{a} \textbf{a}^{-1} = \textbf{a}^{-1} \textbf{a} = \textbf{I}$
    \item \textbf{����(����)�ķ���}��$||\textbf{a}||_p = \big( \sum_i |a_i|^p \big)^{\frac{1}{p}}$
    \end{itemize}

\end{itemize}
\end{frame}

%%%------------------------------------------------------------------------------------------------------------
%%% �˹���Ԫ�ĺ�����ʽ
\begin{frame}{�˹���Ԫ��һ������}

\begin{itemize}
\item ��Ԫ��
\end{itemize}

\vspace{-1em}
\begin{center}
\begin{tikzpicture}

\node [anchor=center] (y) at (0,0) {\LARGE{$\textbf{y}$}};
\node [anchor=west] (eq) at (y.east) {\LARGE{$=$}};
\node [anchor=west] (func) at (eq.east) {\LARGE{$f$}};
\node [anchor=west] (brace01) at (func.east) {\LARGE{$($}};
\node [anchor=west] (x) at (brace01.east) {\LARGE{$\textbf{x}$}};
\node [anchor=west] (dot) at (x.east) {\LARGE{$\cdot$}};
\node [anchor=west] (w) at (dot.east) {\LARGE{$\textbf{w}$}};
\node [anchor=west] (plus) at (w.east) {\LARGE{$+$}};
\node [anchor=west] (b) at (plus.east) {\LARGE{$\textbf{b}$}};
\node [anchor=west] (brace02) at (b.east) {\LARGE{$)$}};

\visible&lt;2-&gt;{
\node [anchor=center,fill=yellow!30] (x2) at (x) {\LARGE{$\textbf{x}$}};
\node [anchor=south] (xlabel) at ([yshift=1.5em]x.north) {����};
\draw [&lt;-] ([yshift=0.2em]x2.north) -- (xlabel.south);
}

\visible&lt;3-&gt;{
\node [anchor=center,fill=green!20] (w2) at (w) {\LARGE{$\textbf{w}$}};
\node [anchor=north] (wlabel) at ([yshift=-1.5em]w.south) {����(Ȩ��)};
\draw [&lt;-] ([yshift=-0.2em]w2.south) -- (wlabel.north);
}

\visible&lt;4-&gt;{
\node [anchor=center,fill=purple!20] (b2) at (b) {\LARGE{$\textbf{b}$}};
\node [anchor=south] (blabel) at ([yshift=1.3em]b.north) {ƫ��};
\draw [&lt;-] ([yshift=0.2em]b2.north) -- (blabel.south);
}

\visible&lt;5-&gt;{
\node [anchor=center,fill=blue!20] (func2) at (func) {\LARGE{$f$}};
\node [anchor=north] (funclabel) at ([yshift=-1.1em]func.south) {�����};
\draw [&lt;-] ([yshift=-0.2em]func2.south) -- (funclabel.north);
}

\visible&lt;6-&gt;{
\node [anchor=center,fill=red!20] (y2) at (y) {\LARGE{$\textbf{y}$}};
\node [anchor=south] (ylabel) at ([yshift=1.3em]y.north) {���};
\draw [&lt;-] ([yshift=0.2em]y2.north) -- (ylabel.south);
}

\end{tikzpicture}
\end{center}

\vspace{-1em}
\begin{itemize}
\item&lt;7-&gt; �Ը�֪��Ϊ��
	\begin{itemize}
	\item ���룺$\textbf{x}=(x_0,...,x_n)$
	\item Ȩ�أ�$\textbf{w}=(w_0,...,w_n)$
	\item ƫ�ƣ�$\textbf{b} = (-\sigma)$
	\item �������$f(z)=1$ ��$z \ge 0$, �������$f(z)=0$
	\item �����$\textbf{y}=f(\textbf{x} \cdot \textbf{w} - \sigma)$
	\end{itemize}
\end{itemize}

\end{frame}

%%%------------------------------------------------------------------------------------------------------------
%%% ��ĸ���
\begin{frame}{``��"�ĸ���}
\begin{itemize}
\item ����һ�����⣨��ͬ���룩�����ܻ��ж���������ʱ���԰�\alert{�����ͬ����Ԫ��������}������һ\alert{``��"}
    \begin{itemize}
    \item ���磬����Ԥ����ҪͬʱԤ��ʪ�Ⱥ��¶�
    \end{itemize}
\end{itemize}

\vspace{-2em}

\begin{center}
\begin{tikzpicture}
\begin{scope}

\tikzstyle{neuronnode} = [minimum size=1.5em,circle,draw,ublue,very thick,fill=white,drop shadow={shadow xshift=0.1em,shadow yshift=-0.1em}]

\node [anchor=center,neuronnode] (neuron00) at (0,0) {};
\visible&lt;2-&gt;{
\node [anchor=center,neuronnode] (neuron01) at ([yshift=-3em]neuron00) {};
}
\visible&lt;3-&gt;{
\node [anchor=center,neuronnode] (neuron02) at ([yshift=-3em]neuron01) {};
}

\node [anchor=east] (x0) at ([xshift=-6em]neuron00.west) {$x_0$};
\node [anchor=east] (x1) at ([xshift=-6em]neuron01.west) {$x_1$};
\node [anchor=east] (x2) at ([xshift=-6em]neuron02.west) {$b$};

\node [anchor=west] (y0) at ([xshift=4em]neuron00.east) {$y_0$};

\draw [-&gt;] (x0.east) -- (neuron00.180) node [pos=0.1,above] {\tiny{$w_{00}$}};
\draw [-&gt;] (x1.east) -- (neuron00.200) node [pos=0.1,above] {\tiny{$w_{10}$}};
\draw [-&gt;] (x2.east) -- (neuron00.220) node [pos=0.05,above,yshift=0.3em] {\tiny{$b_{0}$}};
\draw [-&gt;] (neuron00.east) -- (y0.west);

\visible&lt;2-&gt;{
\node [anchor=west] (y1) at ([xshift=4em]neuron01.east) {$y_1$};
\draw [-&gt;] (x0.east) -- (neuron01.160) node [pos=0.4,above] {\tiny{$w_{01}$}};
\draw [-&gt;] (x1.east) -- (neuron01.180) node [pos=0.35,above,yshift=-0.2em] {\tiny{$w_{11}$}};
\draw [-&gt;] (x2.east) -- (neuron01.200) node [pos=0.3,below,yshift=0.2em] {\tiny{$b_{1}$}};
\draw [-&gt;] (neuron01.east) -- (y1.west);
}

\visible&lt;3-&gt;{
\node [anchor=west] (y2) at ([xshift=4em]neuron02.east) {$y_2$};
\draw [-&gt;] (x0.east) -- (neuron02.140) node [pos=0.1,below,yshift=-0.2em] {\tiny{$w_{02}$}};
\draw [-&gt;] (x1.east) -- (neuron02.160) node [pos=0.1,below] {\tiny{$w_{12}$}};
\draw [-&gt;] (x2.east) -- (neuron02.180) node [pos=0.3,below] {\tiny{$b_{2}$}};
\draw [-&gt;] (neuron02.east) -- (y2.west);
}

\visible&lt;4-&gt;{
\node [anchor=east,align=left] (inputlabel) at ([xshift=-0.1em]x1.west) {��������:\\\small{$\textbf{x}=(x_0,x_1)$}};
}
\visible&lt;5-&gt;{
\node [anchor=west,align=left] (outputlabel) at ([xshift=0.1em]y1.east) {�������:\\\small{$\textbf{y}=(y_0,y_1,y_2)$}};
}

\begin{pgfonlayer}{background}
\visible&lt;6-&gt;{
\node [rectangle,inner sep=0.4em,fill=red!20] [fit = (neuron00) (neuron01) (neuron02)] (layer) {};
\node [anchor=south] (layerlabel) at ([yshift=0.2em]layer.north) {һ����Ԫ};
}

\visible&lt;4-&gt;{
\node [rectangle,inner sep=0.1em,fill=ugreen!20] [fit = (x0) (x1)] (inputshadow) {};
}
\visible&lt;5-&gt;{
\node [rectangle,inner sep=0.1em,fill=blue!20] [fit = (y0) (y1) (y2)] (outputshadow) {};
}
\end{pgfonlayer}

\visible&lt;7-&gt;{
\node [anchor=north west] (wlabel) at ([yshift=-1em,xshift=-7em]x2.south) {����(����):$\textbf{w} = \Big( \begin{array}{lll} w_{00} &amp; w_{01} &amp; w_{02} \\ w_{10} &amp; w_{11} &amp; w_{12} \end{array} \Big)$};
}
\visible&lt;8-&gt;{
\node [anchor=west] (blabel) at (wlabel.east) {����(����):$\textbf{b} = (b_0, b_1, b_2)$};
}

\end{scope}
\end{tikzpicture}
\end{center}

\end{frame}

%%%------------------------------------------------------------------------------------------------------------
%%% �����������
\begin{frame}{�����磺���Ա任 + �����}
\begin{itemize}
\item ��������$\textbf{x} \in \mathbb{R}^m$��һ�����������Ȱ�������\textbf{\alert{���Ա任}}ӳ�䵽$\mathbb{R}^m$��֮�󾭹�\textbf{{\color{blue}�����}}�任��$\textbf{y} \in \mathbb{R}^n$
\end{itemize}

\vspace{1em}

\begin{center}
\begin{tikzpicture}

\node [anchor=center] (y) at (0,0) {\Large{$\textbf{y}$}};
\node [anchor=west] (eq) at (y.east) {\Large{$=$}};
\node [anchor=west] (func) at (eq.east) {\Large{$f$}};
\node [anchor=west] (brace01) at (func.east) {\Large{$($}};
\node [anchor=west] (x) at (brace01.east) {\Large{$\textbf{x}$}};
\node [anchor=west] (dot) at (x.east) {\Large{$\cdot$}};
\node [anchor=west] (w) at (dot.east) {\Large{$\textbf{w}$}};
\node [anchor=west] (plus) at (w.east) {\Large{$+$}};
\node [anchor=west] (b) at (plus.east) {\Large{$\textbf{b}$}};
\node [anchor=west] (brace02) at (b.east) {\Large{$)$}};

\node [anchor=center,fill=blue!20] (func2) at (func) {\LARGE{$f$}};
\node [anchor=north] (funclabel) at ([yshift=-1.1em]func.south) {\blue{�����}};
\draw [&lt;-] ([yshift=-0.2em]func2.south) -- (funclabel.north);

\begin{pgfonlayer}{background}
\node [rectangle,inner sep=0.2em,fill=red!20] [fit = (x) (w) (b)] (linear) {};
\node [anchor=north] (linearlabel) at ([yshift=-1.1em]linear.south) {\alert{���Ա任}};
\draw [&lt;-] ([yshift=-0.2em]linear.south) -- (linearlabel.north);
\end{pgfonlayer}

\end{tikzpicture}

\begin{figure}[htp!]

\includegraphics[scale=0.24]{./Figures/wf.png}
% \begin{tikzpicture}
%     \node [rectangle,inner sep=0.2em,fill=red!20] [fit = (x) (w) (b)] (linear) {};
%     \node [anchor=north] (linearlabel) at ([yshift=-1.1em]linear.south) {\alert{���Ա任}}
\end{figure}
\tikz {\node () at (0,0) {}; \node () at (0,10) {};}
\end{center}
\end{frame}


%%%------------------------------------------------------------------------------------------------------------
%%% ���Ա任
\begin{frame}{���Ա任}
\begin{itemize}
\item �������Կռ�$V$������$\textbf{a}$��$\textbf{b} \in V$�������е�����$\alpha$�����Ա任$T(\cdot)$������
\begin{eqnarray}
T(\textbf{a} + \textbf{b}) &amp; = &amp; T(\textbf{a}) + T(\textbf{b}) \nonumber \\
T(\alpha \textbf{a}) &amp; = &amp; \alpha T(\textbf{a}) \nonumber
\end{eqnarray}
\item&lt;2-&gt; ���Ա任��һ�ּ��ν��ͣ�
\end{itemize}

\vspace{-0.5em}
\visible&lt;2-&gt;{
\begin{center}
\begin{tikzpicture}

\node [anchor=west] (x) at (0,0) {\Large{$\textbf{x}$}};
\node [anchor=west] (dot) at (x.east) {\Large{$\cdot$}};
\node [anchor=west] (w) at (dot.east) {\Large{$\textbf{w}$}};
\node [anchor=west] (plus) at (w.east) {\Large{$+$}};
\node [anchor=west] (b) at (plus.east) {\Large{$\textbf{b}$}};

\tikzstyle{neuron} = [rectangle,draw,thick,fill=red!30,red!35,minimum height=2em,minimum width=2em,font=\small]
\node[neuron,anchor=north] (a1) at ([xshift=-6em,yshift=-4em]x.south) {};
\draw[-&gt;,thick] ([xshift=-2em,yshift=0em]a1.south) to ([xshift=3em,yshift=0em]a1.south);
\draw[-&gt;,thick] ([xshift=0em,yshift=-4em]a1.west) to ([xshift=0em,yshift=2em]a1.west);
\node[below] at ([xshift=0.5em,yshift=-1em]a1.west){0};
\node[below] at ([xshift=2em,yshift=-1em]a1.west){1};
\node[below] at ([xshift=-0.5em,yshift=2em]a1.west){1};
\node [anchor=west] (x) at ([xshift=-0.7em,yshift=1em]a1.south) {\Large{$\textbf{F}$}};

\visible&lt;3-&gt;{
\node [anchor=center,fill=green!20] (w2) at (w) {\Large{$\textbf{w}$}};
\node [anchor=north,inner sep=1pt] (wlabel) at ([yshift=-0.7em]w.south) {\small{��ת(rotation)}};
\draw [&lt;-] ([yshift=-0.2em]w2.south) -- (wlabel.north);

\tikzstyle{neuron} = [rectangle,draw,thick,fill=red!30,red!35,minimum height=2em,minimum width=2em,font=\small]
\node[neuron,anchor=north] (a2) at ([xshift=10em,yshift=0em]a1.south) {};
\draw[-&gt;,thick] ([xshift=-2em,yshift=0em]a2.north) to ([xshift=3em,yshift=0em]a2.north);
\draw[-&gt;,thick] ([xshift=0em,yshift=-2em]a2.west) to ([xshift=0em,yshift=4em]a2.west);
\node[above] at ([xshift=0.5em,yshift=1em]a2.west){0};
\node[above] at ([xshift=2em,yshift=1em]a2.west){1};
\node[below] at ([xshift=-0.5em,yshift=0em]a2.west){-1};
\node [anchor=west] (x) at ([xshift=-3.5cm,yshift=2em]a2.north) {\scriptsize{
    $w=\begin{bmatrix}
    1&amp;0&amp;0\\
    0&amp;-1&amp;0\\
    0&amp;0&amp;1
    \end{bmatrix}$}
    };

\node [anchor=west,rotate = 180] (x) at ([xshift=0.7em,yshift=1em]a2.south) {\Large{$\textbf{F}$}};


\draw[-stealth, line width=2pt,dashed] ([xshift=4em,yshift=0em]a1.south) to ([xshift=-3em,yshift=0em]a2.north);
}

\visible&lt;4-&gt;{
\node [anchor=center,fill=purple!20] (b2) at (b) {\Large{$\textbf{b}$}};
\node [anchor=west] (blabel) at ([xshift=1.5em]b2.east) {ƽ��(shift)};
\draw [&lt;-] ([xshift=0.2em]b2.east) -- (blabel.west);

\tikzstyle{neuron} = [rectangle,draw,thick,fill=red!30,red!35,minimum height=2em,minimum width=2em,font=\small]
\node[neuron,anchor=north] (a3) at ([xshift=11em,yshift=2.05em]a2.south) {};
\draw[-&gt;,thick] ([xshift=-3em,yshift=0em]a3.north) to ([xshift=2em,yshift=0em]a3.north);
\draw[-&gt;,thick] ([xshift=-1em,yshift=-2em]a3.west) to ([xshift=-1em,yshift=4em]a3.west);
\node[above] at ([xshift=-0.5em,yshift=1em]a3.west){0};
\node[above] at ([xshift=1em,yshift=1em]a3.west){1};
\node[left] at ([xshift=-0.75em,yshift=-0.5em]a3.west){-1};
\node [anchor=west,rotate = 180] (x) at ([xshift=0.7em,yshift=1em]a3.south) {\Large{$\textbf{F}$}};


\node [anchor=west] (x) at ([xshift=-4cm,yshift=2em]a3.north) {\scriptsize{
    $b=\begin{bmatrix}
    0.5&amp;0&amp;0\\
    0&amp;0&amp;0\\
    0&amp;0&amp;0
    \end{bmatrix}$}
    };
\draw[-stealth, line width=2pt,dashed] ([xshift=3em,yshift=1em]a2.east) to ([xshift=-3em,yshift=1em]a3.west);
}

\end{tikzpicture}
\end{center}
}


\end{frame}


%%%------------------------------------------------------------------------------------------------------------
%%% ���Ա任�������ӵ�ʵ��
\begin{frame}[fragile]{���Ա任������}
\begin{itemize}
\item ���Ա任Ҳ�����ڸ��Ӹ��ӵ��������Ҳ���������ṩ����ϲ�ͬ���ݷֲ�������
    \begin{itemize}
    \item ���磬���ǿ��԰���άͼ��ͶӰ����άƽ����
    \item �ٱ��磬����Ҳ���԰Ѷ�άƽ���ϵ�ͼ��ӳ�䵽��άƽ��
    \end{itemize}
\end{itemize}
\begin{tiny}
$$
\begin{smallmatrix}  \underbrace{
    \left\{
        \begin{smallmatrix}
            \left[
            \begin{array}{cccc}
             1&amp; 0 &amp;0 \\
             0&amp; 1 &amp;0 \\
             0&amp; 0 &amp;1
            \end{array}
            \right ]
            \cdots
            \left[
            \begin{array}{cccc}
                1&amp; 0 &amp;0 \\
                0&amp; 1 &amp;0 \\
                0&amp; 0 &amp;1
            \end{array}
            \right]
        \end{smallmatrix}
        \right\}
     }\\5
\end{smallmatrix}
\times
\begin{smallmatrix}
\left[
    \begin{array}{cccc}
    1\\
    1\\
    1
    \end{array}
\right ]
\end{smallmatrix}
=
\begin{smallmatrix}  \underbrace{
    \left\{
        \begin{smallmatrix}
            \left[
            \begin{array}{cccc}
             1 \\
             1 \\
             1
            \end{array}
            \right ]
            \cdots
            \left[
            \begin{array}{cccc}
                1 \\
                1 \\
                1
            \end{array}
            \right]
        \end{smallmatrix}
        \right\}
     }\\5
\end{smallmatrix}
$$
\end{tiny}
%\vspace{1em}

\newcommand{\plane}[1]{
(-1.95, #1, 1.35) --
++(3.6, 0.6, 0.0) --
++(0.3, -1.8, -2.7) --
++(-3.6, -0.6, -0.0) --
cycle}
\newcommand{\nullspacepicture}{
% bottom part of the row space line
\draw (0,0,0) -- (0.3,-1.8,1.233);
% five planes
\draw[fill=gray!20]\plane{-0.2};
\draw[fill=gray!20]\plane{0.2};
\draw[fill=blue!70!gray]\plane{0.6};
\draw[fill=gray!20]\plane{1};
\draw[fill=gray!20]\plane{1.4};
% top part of the row space line
\draw (-.094,.562,-.385) -- (-0.3,1.8,-1.233);
}
\newcommand{\rangepicture}[1]{
% axes
\draw[help lines,-&gt;] (-2,0) -- (2,0);
\draw[help lines,-&gt;] (0,-2) -- (0,2);
% the line and circles
\draw (1,-2) -- (-1,2);
\draw[fill=#1] (0,0) circle (2.5pt);
\draw[fill=gray!50] (0.2,-0.4) circle (2.5pt);
\draw[fill=gray!50] (0.4,-0.8) circle (2.5pt);
\draw[fill=gray!50] (-0.2,0.4) circle (2.5pt);
\draw[fill=gray!50] (-0.4,0.8) circle (2.5pt);
}

\begin{tikzpicture}[scale=0.95]
\centering
\nullspacepicture
% the label
\node at (-2,1.8) {$\mathbb{R}^3$};
% arrow between diagrams
\path[-&gt;] (3,0) edge[bend left] node[above] {���Ա任} (4.5,0);
\begin{scope}[xshift=7cm]
\rangepicture{blue!70!gray}
\node at (1.8,1.8) {$\mathbb{R}^2$};
\end{scope}
\end{tikzpicture}
\end{frame}


%%%------------------------------------------------------------------------------------------------------------
%%% �����
\begin{frame}{�����}
\begin{itemize}
\item ������������Ϊ�˽��ʵ�������е�\alert{������}�任
    \begin{itemize}
    \item �����Բ����ṩ��������⺯�����������Ժ���ܣ�
    \end{itemize}
\end{itemize}

\vspace{-1em}
\begin{center}
\begin{tikzpicture}
\begin{scope}
\draw [line width=3pt,ublue,-](0,0) -- (-2.0,1);
\node [anchor=north] (linelabel) at (-1.0,-0.5) {\footnotesize{����һ������}};
\end{scope}

\begin{scope}[xshift=10em]
\draw [line width=3pt,ublue,-,line cap=round](0,0) .. controls (-0.5,-0.25) and (-0.5,1).. (-1.3,0.3) .. controls (-2.3,-0.3) and (-1.1,1.8).. (-2.0,1);
\node [] at (-2,1) {\white{$\cdot$}};
\node [anchor=north] (linelabel) at (-1.0,-0.5) {\footnotesize{����һֻ���}};
\end{scope}
\end{tikzpicture}
\end{center}

\begin{itemize}
\item&lt;2-&gt; �򵥵ķ����Ժ���
\end{itemize}

\vspace{-1em}

\visible&lt;2-&gt;{
\begin{center}
\begin{tikzpicture}
\begin{scope}[]
\draw [-&gt;,thick] (-1.5,0) -- (1.5,0);
\draw [-&gt;,thick] (0,-0.1) -- (0,1.5);
\draw [-,very thick,ublue,domain=-1.2:1.2,samples=100] plot (\x,{0.5 * (\x -0.3)^2 + 0.2});
\node [anchor=west] (ylabel) at (0,1.3) {$y$};
\node [anchor=north] (xlabel) at (1.3,0) {$x$};
\node [anchor=north] (func) at (0,-0.8) {\footnotesize{$y = \frac{1}{2}  (x - 0.3)^2 + 0.2$}};
\node [anchor=south west] (flabel) at (func.north west) {\footnotesize{Quadratic:}};
\end{scope}

\begin{scope}[xshift=9.5em]
\draw [-&gt;,thick] (-1.5,0) -- (1.5,0);
\draw [-&gt;,thick] (0,-0.1) -- (0,1.5);
\draw [-,very thick,ublue,domain=-1.2:1.2,samples=100] plot (\x, {0.5 * exp(\x)});
\node [anchor=west] (ylabel) at (0,1.3) {$y$};
\node [anchor=north] (xlabel) at (1.3,0) {$x$};
\node [anchor=north] (func) at (0,-0.8) {\footnotesize{$y = 0.5 \cdot  \exp(x)$}};
\node [anchor=south west] (flabel) at ([xshift=-1.8em]func.north west) {\footnotesize{Exponential:}};
\end{scope}

\begin{scope}[xshift=19em]
\draw [-&gt;,thick] (-1.5,0) -- (1.5,0);
\draw [-&gt;,thick] (0,-0.1) -- (0,1.5);
\draw [-,very thick,ublue,domain=-1.1:1.2,samples=100] plot (\x,{abs(\x -0.2) + 0.1});
\node [anchor=west] (ylabel) at (0,1.3) {$y$};
\node [anchor=north] (xlabel) at (1.3,0) {$x$};
\node [anchor=north] (func) at (0,-0.8) {\footnotesize{$y = |x - 0.3| + 0.1$}};
\node [anchor=south west] (flabel) at ([xshift=-0.4em]func.north west) {\footnotesize{Absolute:}};
\end{scope}
\end{tikzpicture}
\end{center}
}

\end{frame}

%%%------------------------------------------------------------------------------------------------------------
%%% ���õļ����
\begin{frame}{���õļ����}
    \begin{itemize}
    \item �ö�ö࣬�оٲ�ȫ ...
    \end{itemize}
    \vspace{-1em}
    \begin{figure}
    \subfigure[softplus]{
    \centering
    \begin{minipage}{.2\textwidth}
        \begin{tikzpicture}
        \draw[-&gt;](-1.2,0)--(1.2,0)node[left,below,font=\tiny]{$x$};
        \draw[-&gt;](0,-1.2)--(0,1.2)node[right,font=\tiny]{$y$};
        \foreach \x in {-1.0,-0.5,0.0,0.5,1.0}{\draw(\x,0)--(\x,0.05)node[below,outer sep=2pt,font=\tiny]at(\x,0){\x};}
        \foreach \y in {1.0,0.5}{\draw(0,\y)--(0.05,\y)node[left,outer sep=2pt,font=\tiny]at(0,\y){\y};}
        \draw[color=red ,domain=-1.2:1]plot(\x,{ln(1+(exp(\x))});
        \node[black,anchor=south] at (0,1.2) {\small $y = ln(1+e^x)$};
        \end{tikzpicture}
    \end{minipage}%
    }
    \hfill
    \subfigure[sigmoid]{
    \centering
    \begin{minipage}{.2\textwidth}
        \begin{tikzpicture}
        \draw[-&gt;](-1.2,0)--(1.2,0)node[left,below,font=\tiny]{$x$};
        \draw[-&gt;](0,-1.2)--(0,1.2)node[right,font=\tiny]{$y$};
        \draw[dashed](-1.2,1)--(1.2,1);
        \foreach \x in {-1,-0.5,0,0.5,1}{\draw(\x,0)--(\x,0.05)node[below,outer sep=2pt,font=\tiny]at(\x,0){
            \pgfmathparse{(\x)*5}
            \pgfmathresult};}
        \foreach \y in {0.5,1.0}{\draw(0,\y)--(0.05,\y)node[left,outer sep=2pt,font=\tiny]at(0,\y){\y};}
        \draw[color=red,domain=-1.2:1.2]plot(\x,{1/(1+(exp(-5*\x)))});
        \node[black,anchor=south] at (0,1.2) {\small $y = \frac{1}{1+e^{-x}}$};
        \end{tikzpicture}
    \end{minipage}%
    }
    \hfill
    \subfigure[tanh]{
    \centering
    \begin{minipage}{.2\textwidth}
        \begin{tikzpicture}
        \draw[-&gt;](-1.2,0)--(1.2,0)node[left,below,font=\tiny]{$x$};
        \draw[-&gt;](0,-1.2)--(0,1.2)node[right,font=\tiny]{$y$};
        \draw[dashed](-1.2,1)--(1.2,1);
        \draw[dashed](-1.2,-1)--(1.2,-1);
        \foreach \x in {-1.0,-0.5,0.0,0.5,1.0}{\draw(\x,0)--(\x,0.05)node[below,outer sep=2pt,font=\tiny]at(\x,0){\x};}
        \foreach \y in {0.5,1.0}{\draw(0,\y)--(0.05,\y)node[left,outer sep=2pt,font=\tiny]at(0,\y){\y};}
        \draw[color=red ,domain=-1.2:1.2]plot(\x,{tanh(\x)});
        \node[black,anchor=south] at (0,1.2) {\small $y = \frac{e^{x}-e^{-x}}{e^{x}+e^{-x}}$};
        \end{tikzpicture}
    \end{minipage}
    }
    \end{figure}
    \vspace{-1em}
    \begin{figure}
    \subfigure[relu]{
    \centering
    \begin{minipage}{.2\textwidth}
        \begin{tikzpicture}
        \draw[-&gt;](-1.2,0)--(1.2,0)node[left,below,font=\tiny]{$x$};
        \draw[-&gt;](0,-1.2)--(0,1.2)node[right,font=\tiny]{$y$};
        \draw[dashed](-1.2,1)--(1.2,1);
        \draw[dashed](-1.2,-1)--(1.2,-1);
        \foreach \x in {-1.0,-0.5,0.0,0.5,1.0}{\draw(\x,0)--(\x,0.05)node[below,outer sep=2pt,font=\tiny]at(\x,0){\x};}
        \foreach \y in {0.5,1.0}{\draw(0,\y)--(0.05,\y)node[left,outer sep=2pt,font=\tiny]at(0,\y){\y};}
        \draw[color=red ,domain=-1.2:1.2]plot(\x,{max(\x,0)});
        \node[black,anchor=south] at (0,1.2) {\small $y =\max (0, x)$};
        \end{tikzpicture}
    \end{minipage}%
    }
    \hfill
    \subfigure[gaussian]{
    \centering
    \begin{minipage}{.2\textwidth}
        \begin{tikzpicture}
        \draw[-&gt;](-1.2,0)--(1.2,0)node[left,below,font=\tiny]{$x$};
        \draw[-&gt;](0,-1.2)--(0,1.2)node[right,font=\tiny]{$y$};
        \draw[dashed](-1.2,1)--(1.2,1);
        \foreach \x in {-1.0,-0.5,0.0,0.5,1.0}{\draw(\x,0)--(\x,0.05)node[below,outer sep=2pt,font=\tiny]at(\x,0){\x};}
        \foreach \y in {0.5,1.0}{\draw(0,\y)--(0.05,\y)node[left,outer sep=2pt,font=\tiny]at(0,\y){\y};}
        \draw[color=red ,domain=-1.2:1.2]plot(\x,{exp(-1*((\x)^2))});
        \node[black,anchor=south] at (0,1.2) {\small $y =e^{-x^2}$};
        \end{tikzpicture}
    \end{minipage}%
    }
    \hfill
    \subfigure[identity]{
    \centering
    \begin{minipage}{.2\textwidth}
        \begin{tikzpicture}
        \draw[-&gt;](-1.2,0)--(1.2,0)node[left,below,font=\tiny]{$x$};
        \draw[-&gt;](0,-1.2)--(0,1.2)node[right,font=\tiny]{$y$};
        \foreach \x in {-1.0,-0.5,0.0,0.5,1.0}{\draw(\x,0)--(\x,0.05)node[below,outer sep=2pt,font=\tiny]at(\x,0){\x};}
        \foreach \y in {0.5,1.0}{\draw(0,\y)--(0.05,\y)node[left,outer sep=2pt,font=\tiny]at(0,\y){\y};}
        \draw[color=red ,domain=-1:1]plot(\x,\x);
        \node[black,anchor=south] at (0,1.2) {\small $y =x$};
        \end{tikzpicture}
    \end{minipage}
    }

    \end{figure}

\end{frame}


%%%------------------------------------------------------------------------------------------------------------
\subsection{���������}

%%%------------------------------------------------------------------------------------------------------------
%%% һ�� -&gt; ���
\begin{frame}{����Ĳ�}
\begin{itemize}
\item \textbf{����������}�����Ա任 + ������������ԣ�
\item ���ǿ����ظ�����Ĺ��̣�����\textbf{���������}
\end{itemize}

\vspace{-1.0em}
\begin{center}
\begin{tikzpicture}
\begin{scope}[]

\def\neuronsep{1.6}

\tikzstyle{neuronnode} = [minimum size=1.7em,circle,draw,ublue,very thick,inner sep=1pt, fill=white,align=center,drop shadow={shadow xshift=0.1em,shadow yshift=-0.1em}]

%%% layer 1
\foreach \n in {1,...,5}{
    \node [neuronnode] (neuron0\n) at (\n * \neuronsep,0) {\tiny{$f_1$}\\[-1ex] \tiny{$\sum$}};
    \draw [-,ublue] (neuron0\n.east) -- (neuron0\n.west);
}

\foreach \n in {1,...,5}{
    \foreach \m in {1,...,5}{
        \draw [&lt;-] (neuron0\m.south) -- ([yshift=-2em]neuron0\n.south);
    }
    \node [anchor=north] (x\n) at ([yshift=-2em]neuron0\n.south) {$x_\n$};
    \visible&lt;1&gt;{
    \draw [&lt;-,thick] ([yshift=1.5em]neuron0\n.north) -- (neuron0\n.north);
    \node [anchor=south] (y\n) at ([yshift=1.5em]neuron0\n.north) {$y_\n$};
    }
}

\node [anchor=west] (w1label) at ([xshift=-0.5em,yshift=0.5em]x5.north east) {$\textbf{w}_1$};

\begin{pgfonlayer}{background}
\node [rectangle,inner sep=0.2em,fill=red!20] [fit = (neuron01) (neuron05)] (layer01) {};
\end{pgfonlayer}

\node [anchor=west] (layer00label) at ([xshift=1.25em]x5.east) {\alert{�����}};

\visible&lt;2-&gt;{
\node [anchor=west] (layer01label) at ([xshift=1em]layer01.east) {�ڶ���};
}
\visible&lt;4-&gt;{
\node [anchor=west] (layer01label2) at (layer01label.east) {(\alert{����})};
}

%%% layer 2
\visible&lt;2-&gt;{
\foreach \n in {2,...,4}{
    \node [neuronnode] (neuron1\n) at (\n * \neuronsep,4em) {\tiny{$f_2$}\\[-1ex] \tiny{$\sum$}};
    \draw [-,ublue] (neuron1\n.east) -- (neuron1\n.west);
}

\foreach \n in {2,...,4}{
    \foreach \m in {1,...,5}{
        \draw [&lt;-] (neuron1\n.south) -- (neuron0\m.north);
    }
    \visible&lt;2&gt;{
    \draw [&lt;-,thick] ([yshift=1.5em]neuron1\n.north) -- (neuron1\n.north);
    \node [anchor=south] (y\n) at ([yshift=1.5em]neuron1\n.north) {$y_\n$};
    }
}

\node [anchor=west] (w2label) at ([xshift=-2.5em,yshift=5.0em]x5.north east) {$\textbf{w}_2$};

\begin{pgfonlayer}{background}
\visible&lt;2-&gt;{
\node [rectangle,inner sep=0.2em,fill=ugreen!20] [fit = (neuron12) (neuron14)] (layer02) {};
}
\end{pgfonlayer}

\node [anchor=west] (layer02label) at ([xshift=4.9em]layer02.east) {������};
\visible&lt;4-&gt;{
\node [anchor=west] (layer02label2) at (layer02label.east) {(\alert{����})};
}
}

%%% layer 3
\visible&lt;3-&gt;{
\foreach \n in {1,...,5}{
    \node [neuronnode] (neuron2\n) at (\n * \neuronsep,8em) {\tiny{$f_3$}\\[-1ex] \tiny{$\sum$}};
    \draw [-,ublue] (neuron2\n.east) -- (neuron2\n.west);
}

\foreach \n in {1,...,5}{
    \foreach \m in {2,...,4}{
        \draw [&lt;-] (neuron2\n.south) -- (neuron1\m.north);
    }

    \node [anchor=south] (y\n) at ([yshift=1.5em]neuron2\n.north) {$y_\n$};
    \draw [&lt;-,thick] ([yshift=1.5em]neuron2\n.north) -- (neuron2\n.north);
}

\node [anchor=west] (w3label) at ([xshift=-2.5em,yshift=8.5em]x5.north east) {$\textbf{w}_3$};

\begin{pgfonlayer}{background}
\visible&lt;3-&gt;{
\node [rectangle,inner sep=0.2em,fill=blue!20] [fit = (neuron21) (neuron25)] (layer03) {};
}
\end{pgfonlayer}

\node [anchor=west] (layer03label) at ([xshift=1em]layer03.east) {���Ĳ�};
\visible&lt;4-&gt;{
\node [anchor=west] (layer03label2) at (layer03label.east) {(\alert{�����})};
}
}

\end{scope}
\end{tikzpicture}
\end{center}

\end{frame}

%%%------------------------------------------------------------------------------------------------------------
%%% ������������Աƽ��κκ���
\begin{frame}{�����������Աƽ����⺯��}
\begin{itemize}
\item ��һ���򵥵���������Ϊ�������㼤�����sigmoid��
\end{itemize}

\begin{center}
\begin{tikzpicture}

%% a two-layer neural network
\begin{scope}
\tikzstyle{neuronnode} = [minimum size=1.7em,circle,draw,ublue,very thick,inner sep=1pt, fill=white,align=center,drop shadow={shadow xshift=0.1em,shadow yshift=-0.1em}]

%% input and hidden layers
\node [neuronnode] (n10) at (0,0) {\tiny{$f$}\\[-1ex] \tiny{$\sum$}};
\node [neuronnode] (n11) at (1.5,0) {\tiny{$f$}\\[-1ex] \tiny{$\sum$}};
\draw [-,ublue] (n10.west) -- (n10.east);
\draw [-,ublue] (n11.west) -- (n11.east);
\node [anchor=north] (x1) at ([yshift=-6em]n11.south) {$x_1$};
\node [anchor=north] (b) at ([yshift=-6em]n10.south) {$b$};
\visible&lt;1-10&gt;{
\draw [-&gt;,thick] (b.north) -- ([yshift=-0.1em]n10.south);
\draw [-&gt;,thick] (x1.north) -- ([yshift=-0.1em]n10.290);
}
\visible&lt;1&gt;{
\draw [-&gt;,thick] (b.north) -- ([yshift=-0.1em]n11.250);
\draw [-&gt;,thick] (x1.north) -- ([yshift=-0.1em]n11.south);
}

\visible&lt;11-&gt;{
\draw [-&gt;,thick,red] (b.north) -- ([yshift=-0.1em]n10.south);
\draw [-&gt;,thick,ugreen] (x1.north) -- ([yshift=-0.1em]n10.290);
}

\visible&lt;2-&gt;{
\draw [-&gt;,thick,blue] (b.north) -- ([yshift=-0.1em]n11.250);
\draw [-&gt;,thick,purple] (x1.north) -- ([yshift=-0.1em]n11.south);
}

\visible&lt;15-&gt;{
\node [neuronnode] (n12) at (2.7,0) {\tiny{$f$}\\[-1ex] \tiny{$\sum$}};
\node [neuronnode] (n13) at (3.8,0) {\tiny{$f$}\\[-1ex] \tiny{$\sum$}};
\draw [-,ublue] (n12.west) -- (n12.east);
\draw [-,ublue] (n13.west) -- (n13.east);
\draw [-&gt;,thick] (b.north) -- ([yshift=-0.1em]n12.250);
\draw [-&gt;,thick] (x1.north) -- ([yshift=-0.1em]n12.270);
\draw [-&gt;,thick] (b.north) -- ([yshift=-0.1em]n13.230);
\draw [-&gt;,thick] (x1.north) -- ([yshift=-0.1em]n13.250);
}

\visible&lt;16-&gt;{
\node [anchor=west] (morenodes) at (n13.east) {...};
}

%% output layers
\node [neuronnode] (n20) at (0.75,5em) {\scriptsize{$\sum$}};
\visible&lt;1-10&gt;{\draw [-&gt;,thick] ([yshift=0.1em]n10.north) -- ([yshift=-0.1em]n20.250);}
\visible&lt;1-8&gt;{\draw [-&gt;,thick] ([yshift=0.1em]n11.north) -- ([yshift=-0.1em]n20.290);}

\visible&lt;11-&gt;{\draw [-&gt;,thick,brown] ([yshift=0.1em]n10.north) -- ([yshift=-0.1em]n20.250);}
\visible&lt;9-&gt;{\draw [-&gt;,thick,orange] ([yshift=0.1em]n11.north) -- ([yshift=-0.1em]n20.290);}

\node [] (y) at ([yshift=3em]n20.north) {$y$};
\draw [-&gt;,thick] ([yshift=0.1em]n20.north) -- (y.south);

\visible&lt;15-&gt;{
\draw [-&gt;,thick] ([yshift=0.1em]n12.north) -- ([yshift=-0.1em]n20.310);
\draw [-&gt;,thick] ([yshift=0.1em]n13.north) -- ([yshift=-0.1em]n20.330);
}

%% weight and bias
\visible&lt;11-&gt;{\node [anchor=center,rotate=90,fill=white,inner sep=1pt] (b0) at ([yshift=3em,xshift=-0.5em]b.north) {\tiny{$b=-6$}};}
\visible&lt;11-&gt;{\node [anchor=center,rotate=-59,fill=white,inner sep=1pt] (w2) at ([yshift=1.2em,xshift=-1.2em]x1.north) {\tiny{$w=100$}};}

\visible&lt;2-6&gt;{\node [anchor=center,rotate=59,fill=white,inner sep=1pt] (b1) at ([yshift=4.9em,xshift=2.2em]b.north) {\tiny{$b=0$}};}
\visible&lt;7&gt;{\node [anchor=center,rotate=59,fill=white,inner sep=1pt] (b1) at ([yshift=4.9em,xshift=2.2em]b.north) {\tiny{$b=-2$}};}
\visible&lt;8-&gt;{\node [anchor=center,rotate=59,fill=white,inner sep=1pt] (b1) at ([yshift=4.9em,xshift=2.2em]b.north) {\tiny{$b=-4$}};}
\visible&lt;2-4&gt;{\node [anchor=center,rotate=90,fill=white,inner sep=1pt] (w1) at ([yshift=3em,xshift=0.5em]x1.north) {\tiny{$w=1$}};}
\visible&lt;5&gt;{\node [anchor=center,rotate=90,fill=white,inner sep=1pt] (w1) at ([yshift=3em,xshift=0.5em]x1.north) {\tiny{$w=10$}};}
\visible&lt;6-&gt;{\node [anchor=center,rotate=90,fill=white,inner sep=1pt] (w1) at ([yshift=3em,xshift=0.5em]x1.north) {\tiny{$w=100$}};}

\visible&lt;11&gt;{\node [anchor=center,rotate=62,fill=white,inner sep=1pt] (w21) at ([yshift=2em,xshift=0.5em]n10.north) {\tiny{$w'=0.7$}};}
\visible&lt;12-&gt;{\node [anchor=center,rotate=62,fill=white,inner sep=1pt] (w21) at ([yshift=2em,xshift=0.5em]n10.north) {\tiny{$w'=-0.7$}};}

\visible&lt;2-8&gt;{\node [anchor=center,rotate=-62,fill=white,inner sep=1pt] (w22) at ([yshift=2em,xshift=-0.5em]n11.north) {\tiny{$w'=1$}};}
\visible&lt;9&gt;{\node [anchor=center,rotate=-62,fill=white,inner sep=1pt] (w22) at ([yshift=2em,xshift=-0.5em]n11.north) {\tiny{$w'=0.9$}};}
\visible&lt;10-&gt;{\node [anchor=center,rotate=-62,fill=white,inner sep=1pt] (w22) at ([yshift=2em,xshift=-0.5em]n11.north) {\tiny{$w'=0.7$}};}


%% sigmoid box
\begin{scope}
\visible&lt;3-&gt;{
\node [anchor=west] (flabel) at ([xshift=1.2in]y.east) {\footnotesize{sigmoid:}};
\node [anchor=north east] (slabel) at ([xshift=0]flabel.south east) {\footnotesize{sum:}};

\node [anchor=west,inner sep=2pt] (flabel2) at (flabel.east) {\footnotesize{$f(s)=1/(1+e^{-s})$}};
\node [anchor=west,inner sep=2pt] (flabel3) at (slabel.east) {\footnotesize{$s=x_1 \cdot w + b$}};
\draw [-&gt;,thick,dotted] ([yshift=-0.3em,xshift=-0.1em]n11.60)  .. controls +(east:1) and +(west:2) ..  ([xshift=-0.2em]flabel.west) ;

\begin{pgfonlayer}{background}
\visible&lt;3-&gt;{
\node [rectangle,inner sep=0.2em,fill=blue!20,drop shadow={shadow xshift=0.1em,shadow yshift=-0.1em}] [fit = (flabel) (flabel2) (flabel3)] (funcbox) {};
}
\end{pgfonlayer}
}
\end{scope}

%% output illustration
\begin{scope}[xshift=2.8in,yshift=0.1in]
\visible&lt;4-&gt;{
\draw [-&gt;,thick] (-2.2,0) -- (2.2,0);
\draw [-&gt;,thick] (0,0) -- (0,2);
\draw [-] (-0.05,1) -- (0.05,1);
\node [anchor=east,inner sep=1pt] (label1) at (0,1) {\tiny{1}};
\node [anchor=south east,inner sep=1pt] (label2) at (0,0) {\tiny{0}};
}

\visible&lt;4&gt;{\draw [-,very thick,ublue,domain=-2:2,samples=100] plot (\x,{1/(1+exp(-2*\x))});}
\visible&lt;5&gt;{\draw [-,very thick,ublue,domain=-2:2,samples=100] plot (\x,{1/(1+exp(-4*\x))});}
\visible&lt;6&gt;{\draw [-,very thick,ublue,rounded corners=0.1em] (-2,0) -- (0,0) -- (0,1) -- (2,1);}
\visible&lt;7&gt;{\draw [-,very thick,ublue,rounded corners=0.1em] (-2,0) -- (0.25,0) -- (0.25,1) -- (2,1);}
\visible&lt;8&gt;{\draw [-,very thick,ublue,rounded corners=0.1em] (-2,0) -- (0.5,0) -- (0.5,1) -- (2,1);}
\visible&lt;9&gt;{\draw [-,very thick,ublue,rounded corners=0.1em] (-2,0) -- (0.5,0) -- (0.5,0.9) -- (2,0.9);}
\visible&lt;10&gt;{\draw [-,very thick,ublue,rounded corners=0.1em] (-2,0) -- (0.5,0) -- (0.5,0.7) -- (2,0.7);}
\visible&lt;11&gt;{\draw [-,very thick,ublue,rounded corners=0.1em] (-2,0) -- (0.5,0) -- (0.5,0.7) -- (0.7,0.7) -- (0.7,1.4) -- (2,1.4);}
\visible&lt;12-&gt;{\draw [-,very thick,ublue,rounded corners=0.1em] (-2,0) -- (0.5,0) -- (0.5,0.7) -- (0.7,0.7) -- (0.7,0) -- (2,0);}
\visible&lt;15-&gt;{\draw [-,very thick,ublue,rounded corners=0.1em] (-2,0) -- (0.7,0) -- (0.7,0.6) -- (0.9,0.6) -- (0.9,0) -- (2,0);}

\visible&lt;14&gt;{\draw [-&gt;,dashed] (0.6,-0.05) -- (0.6,-0.96in);}
\visible&lt;15-&gt;{\draw [-&gt;,dashed] (0.8,-0.05) -- (0.8,-0.98in);}

\visible&lt;4&gt;{\node [anchor=north west,align=left] (wblabel) at (-2,2) {\scriptsize{$w_1=1$}\\[-0ex] \scriptsize{\ $b_1=0$}};}
\visible&lt;5&gt;{\node [anchor=north west,align=left] (wblabel) at (-2,2) {\alert{\scriptsize{$w_1=10$}}\\[-0ex] \scriptsize{\ $b_1=0$}};}
\visible&lt;6&gt;{\node [anchor=north west,align=left] (wblabel) at (-2,2) {\alert{\scriptsize{$w_1=100$}}\\[-0ex] \scriptsize{\ $b_1=0$}};}
\visible&lt;7&gt;{\node [anchor=north west,align=left] (wblabel) at (-2,2) {\scriptsize{$w_1=100$}\\[-0ex] \alert{\scriptsize{\ $b_1=-2$}}};}
\visible&lt;8&gt;{\node [anchor=north west,align=left] (wblabel) at (-2,2) {\scriptsize{$w_1=100$}\\[-0ex] \alert{\scriptsize{\ $b_1=-4$}}};}
\visible&lt;9&gt;{\node [anchor=north west,align=left] (wblabel) at (-2,2) {\alert{\scriptsize{$w'_1=0.9$}}};}
\visible&lt;10&gt;{\node [anchor=north west,align=left] (wblabel) at (-2,2) {\alert{\scriptsize{$w'_1=0.7$}}};}
\visible&lt;11&gt;{\node [anchor=north west,align=left] (wblabel) at (-2,2) {\alert{\scriptsize{$w_2=100$}}\\[-0ex] \alert{\scriptsize{\ $b_2=-6$}}\\[-0ex] \alert{\scriptsize{\ $w'_2=0.7$}}};}
\visible&lt;12&gt;{\node [anchor=north west,align=left] (wblabel) at (-2,2) {\scriptsize{$w_2=100$}\\[-0ex] \scriptsize{\ $b_2=-6$}\\[-0ex] \alert{\scriptsize{\ $w'_2=-0.7$}}};}
\visible&lt;13-&gt;{\node [anchor=north west,align=left] (wblabel) at (-2.5,2) {\scriptsize{����һ��}\\[-1ex] \scriptsize{step function}};}
\end{scope}

\begin{scope}[xshift=2.8in,yshift=-1.2in]

\visible&lt;13-&gt;{
\draw [-&gt;,thick] (-2.2,0) -- (2.2,0);
\draw [-&gt;,thick] (0,0) -- (0,2);
\draw [-,very thick,red,domain=-1.98:2,samples=100] plot (\x,{0.2 * (\x +0.4)^3 + 1.2 - 0.3 *(\x + 0.8)^2});
}

\visible&lt;14-&gt;{
\foreach \n in {0.5}{
    \pgfmathsetmacro{\result}{0.2 * (\n + 0.1 + 0.4)^3 + 1.2 - 0.3 *(\n + 0.1 + 0.8)^2};
    \draw [-,ublue,thick] (\n,0) -- (\n, \result) -- (\n + 0.2, \result) -- (\n + 0.2, 0);
}
}

\visible&lt;15-&gt;{
\foreach \n in {0.7}{
    \pgfmathsetmacro{\result}{0.2 * (\n + 0.1 + 0.4)^3 + 1.2 - 0.3 *(\n + 0.1 + 0.8)^2};
    \draw [-,ublue,thick] (\n,0) -- (\n, \result) -- (\n + 0.2, \result) -- (\n + 0.2, 0);
}
}

\visible&lt;16-&gt;{
\foreach \n in {-1.9,-1.7,...,1.9}{
    \pgfmathsetmacro{\result}{0.2 * (\n + 0.1 + 0.4)^3 + 1.2 - 0.3 *(\n + 0.1 + 0.8)^2};
    \draw [-,ublue,thick] (\n,0) -- (\n, \result) -- (\n + 0.2, \result) -- (\n + 0.2, 0);
}
}

\visible&lt;14&gt;{\node [anchor=north west,align=left] (wblabel) at (-2.5,2.5) {\scriptsize{������ÿһ�ζ���}\\[-1ex] \scriptsize{��step function}\\[-1ex] \scriptsize{����}};}
\visible&lt;15&gt;{\node [anchor=north west,align=left] (wblabel) at (-2.5,2.5) {\scriptsize{���������Ԫ}\\[-1ex] \scriptsize{������ϸ����}\\[-1ex] \scriptsize{����}};}
\visible&lt;16&gt;{\node [anchor=north west,align=left] (wblabel) at (-2.5,2.5) {\scriptsize{�������㹻���}\\[-1ex] \scriptsize{������Ԫ����}\\[-1ex] \scriptsize{���\alert{���⺯��}}};}

\end{scope}

\end{scope}

\end{tikzpicture}
\end{center}
\end{frame}

%%%------------------------------------------------------------------------------------------------------------
\subsection{������ļ�ʵ�� - ��������}

%%%------------------------------------------------------------------------------------------------------------
%%% outline: problem 2
\begin{frame}{Ȼ��}

\vspace{6em}
\begin{tcolorbox}[enhanced,size=normal,left=2mm,right=1mm,colback=red!5!white,colframe=red!75!black,drop fuzzy shadow]
{\Large
\textbf{�˹����������ѧ������ʲô,}

\vspace{0.4em}
\textbf{��α��ʵ��������ѧģ�ͣ�}
}
\end{tcolorbox}

\vspace{1em}
\begin{center}
\begin{tikzpicture}
\begin{scope}[yshift=6.5em,xshift=1em]
\setcounter{mycount1}{1}
\draw[step=0.5cm,color=orange,thick] (-1,-1) grid (0.5,0.5);
\foreach \y in {+0.25,-0.25,-0.75}
  \foreach \x in {-0.75,-0.25,0.25}{
    \node [fill=orange!20,inner sep=0pt,minimum height=0.49cm,minimum width=0.49cm] at (\x,\y) {\number\value{mycount1}};
    \addtocounter{mycount1}{1};
  }
\end{scope}

\begin{scope}[yshift=6em,xshift=0.5em]
\setcounter{mycount2}{2}
\draw[step=0.5cm,color=blue,thick] (-1,-1) grid (0.5,0.5);
\foreach \y in {+0.25,-0.25,-0.75}
  \foreach \x in {-0.75,-0.25,0.25}{
    \node [fill=blue!20,inner sep=0pt,minimum height=0.49cm,minimum width=0.49cm] at (\x,\y) {\number\value{mycount2}};
    \addtocounter{mycount2}{1};
  }
\end{scope}

\begin{scope}[yshift=5.5em,xshift=0em]
\setcounter{mycount3}{3}
\draw[step=0.5cm,color=ugreen,thick] (-1,-1) grid (0.5,0.5);
\foreach \y in {+0.25,-0.25,-0.75}
  \foreach \x in {-0.75,-0.25,0.25}{
    \node [fill=green!20,inner sep=0pt,minimum height=0.49cm,minimum width=0.49cm] at (\x,\y) {\number\value{mycount3}};
    \addtocounter{mycount3}{1};
  }
\end{scope}

\begin{scope}[yshift=5em,xshift=-0.5em]
\setcounter{mycount4}{4}
\draw[step=0.5cm,color=red,thick] (-1,-1) grid (0.5,0.5);
\foreach \y in {+0.25,-0.25,-0.75}
  \foreach \x in {-0.75,-0.25,0.25}{
    \node [fill=red!20,inner sep=0pt,minimum height=0.49cm,minimum width=0.49cm] at (\x,\y) {\number\value{mycount4}};
    \addtocounter{mycount4}{1};
  }
\end{scope}
\end{tikzpicture}
\end{center}

\end{frame}

%%%------------------------------------------------------------------------------------------------------------
%%% ����
\begin{frame}{������������� - ��������}
\begin{itemize}
\item ���������磬����$\textbf{x}$�����$\textbf{y}$����ʽ��������������
\end{itemize}

\begin{center}
\begin{tikzpicture}

\node [anchor=center] (y) at (0,0) {\LARGE{$\textbf{y}$}};
\node [anchor=west] (eq) at (y.east) {\LARGE{$=$}};
\node [anchor=west] (func) at (eq.east) {\LARGE{$f$}};
\node [anchor=west] (brace01) at (func.east) {\LARGE{$($}};
\node [anchor=west] (x) at (brace01.east) {\LARGE{$\textbf{x}$}};
\node [anchor=west] (dot) at (x.east) {\LARGE{$\cdot$}};
\node [anchor=west] (w) at (dot.east) {\LARGE{$\textbf{w}$}};
\node [anchor=west] (plus) at (w.east) {\LARGE{$+$}};
\node [anchor=west] (b) at (plus.east) {\LARGE{$\textbf{b}$}};
\node [anchor=west] (brace02) at (b.east) {\LARGE{$)$}};

\visible&lt;2-&gt;{
\node [anchor=center,fill=yellow!30] (x2) at (x) {\LARGE{$\textbf{x}$}};
\node [anchor=south] (xlabel) at ([xshift=-3em,yshift=1.5em]x.north) {\alert{����������...}};
\draw [&lt;-] ([yshift=0.2em,xshift=-0.5em]x2.north) -- ([xshift=1em]xlabel.south);

\node [anchor=center,fill=red!20] (y2) at (y) {\LARGE{$\textbf{y}$}};
\draw [&lt;-] ([yshift=0.2em,xshift=0.5em]y2.north) -- ([xshift=-1em]xlabel.south);

\node [anchor=center,fill=green!20] (w2) at (w) {\LARGE{$\textbf{w}$}};
\node [anchor=north] (wlabel) at ([yshift=-1.0em]w.south) {���� e.g.,};
\draw [&lt;-] ([yshift=-0.2em]w2.south) -- (wlabel.north);
\node [anchor=west] (wsample) at ([xshift=-0.5em]wlabel.east) {\footnotesize{$\left(\begin{array}{c c} 1 &amp; 2 \\ 3 &amp; 4 \end{array}\right)$}};

\node [anchor=center,fill=purple!20] (b2) at (b) {\LARGE{$\textbf{b}$}};
\node [anchor=south] (blabel) at ([yshift=1.3em]b.north) {���� e.g.,};
\draw [&lt;-] ([yshift=0.2em]b2.north) -- (blabel.south);
\node [anchor=west] (bsample) at ([xshift=-0.5em]blabel.east) {\footnotesize{$(1, 3)$}};
}

\end{tikzpicture}
\end{center}

\begin{itemize}
\item&lt;3-&gt; $\textbf{x}$��$\textbf{y}$ʵ������һ����tensor�Ķ�������\textbf{����}�����磬
\end{itemize}

\begin{center}
\begin{tikzpicture}
\begin{scope}
\visible&lt;4-&gt;{\node [anchor=west] (vector) at (0,0) {$\textbf{x} = (1,  3)$};}
\visible&lt;5-&gt;{\node [anchor=west] (matrix) at ([xshift=0.1in]vector.east) {$\textbf{x} = \left(\begin{array}{c c} -1 &amp; 3 \\ 0.2 &amp; 2 \end{array}\right)$};}
\visible&lt;6-&gt;{\node [anchor=west] (tensor3d) at ([xshift=0.1in]matrix.east) {ɶ��$\textbf{x} = \left(\begin{array}{c} \left(\begin{array}{c c} -1 &amp; 3 \\ 0.2 &amp; 2 \end{array}\right) \\ \left(\begin{array}{c c} -1 &amp; 3 \\ 0.2 &amp; 2 \end{array}\right) \end{array}\right)$};}
\end{scope}
\end{tikzpicture}
\end{center}

\end{frame}

%%%------------------------------------------------------------------------------------------------------------
%%% �����ļ򵥶���
\begin{frame}{������ʲô}
\begin{itemize}
\item \textbf{���ѧϰ}�У�������``��"�ض���Ϊ\alert{��ά����}
    \begin{itemize}
    \item �����Ľף�rank����ʾ�ж��ٸ������ķ���ÿ����������ɶ��ά�ȱ�ʾ
    \end{itemize}
\end{itemize}

\begin{center}
\begin{tikzpicture}

\begin{scope}
\visible&lt;2-&gt;{
\node [anchor=north] (label) at (0,0) {����};
\node [anchor=center] (label2) at ([yshift=-0.7em]label.south) {scalar};
\node [anchor=center] (rank) at ([yshift=-1.5em]label2.center) {(rank=0)};
\node [anchor=center] (scalar) at ([yshift=5em]label.north) {\Huge{3}};
}
\end{scope}

\begin{scope}[xshift=1in]
\visible&lt;3-&gt;{
\node [anchor=north] (label) at (0,0) {����};
\node [anchor=center] (label2) at ([yshift=-0.7em]label.south) {vector};
\node [anchor=center] (rank) at ([yshift=-1.5em]label2.center) {(rank=1)};
\node [anchor=center] (scalar) at ([yshift=5em]label.north) {$\begin{pmatrix} 2 \\ .3 \\ -8 \\ .2\end{pmatrix}$};
}
\end{scope}

\begin{scope}[xshift=2in]
\visible&lt;4-&gt;{
\node [anchor=north] (label) at (0,0) {����};
\node [anchor=center] (label2) at ([yshift=-0.7em]label.south) {matrix};
\node [anchor=center] (rank) at ([yshift=-1.5em]label2.center) {(rank=2)};
\node [anchor=center] (scalar) at ([yshift=5em]label.north) {$\begin{pmatrix} 1 &amp; 1 &amp; 9 \\ 1 &amp; 0 &amp; 0 \\ 1 &amp; -4 &amp; 7 \end{pmatrix}$};
}
\end{scope}

\begin{scope}[xshift=3.2in]
\visible&lt;5-&gt;{
\node [anchor=north] (label) at (0,0) {3������};
\node [anchor=center] (label2) at ([yshift=-0.7em]label.south) {tensor};
\node [anchor=center] (rank) at ([yshift=-1.5em]label2.center) {(rank=3)};
}
\begin{scope}[yshift=6.5em,xshift=1em]
\visible&lt;5-&gt;{
\setcounter{mycount1}{1}
\draw[step=0.5cm,color=orange,thick] (-1,-1) grid (1,1);
\foreach \y in {+0.75,+0.25,-0.25,-0.75}
  \foreach \x in {-0.75,-0.25,0.25,0.75}{
    \node [fill=orange!20,inner sep=0pt,minimum height=0.49cm,minimum width=0.49cm] at (\x,\y) {\number\value{mycount1}};
    \addtocounter{mycount1}{1};
  }
}
\end{scope}

\begin{scope}[yshift=6em,xshift=0.5em]
\visible&lt;5-&gt;{
\setcounter{mycount2}{1}
\draw[step=0.5cm,color=blue,thick] (-1,-1) grid (1,1);
\foreach \y in {+0.75,+0.25,-0.25,-0.75}
  \foreach \x in {-0.75,-0.25,0.25,0.75}{
    \node [fill=blue!20,inner sep=0pt,minimum height=0.49cm,minimum width=0.49cm] at (\x,\y) {\number\value{mycount2}};
    \addtocounter{mycount2}{1};
  }
}
\end{scope}

\begin{scope}[yshift=5.5em,xshift=0em]
\visible&lt;5-&gt;{
\setcounter{mycount3}{1}
\draw[step=0.5cm,color=ugreen,thick] (-1,-1) grid (1,1);
\foreach \y in {+0.75,+0.25,-0.25,-0.75}
  \foreach \x in {-0.75,-0.25,0.25,0.75}{
    \node [fill=green!20,inner sep=0pt,minimum height=0.49cm,minimum width=0.49cm] at (\x,\y) {\number\value{mycount3}};
    \addtocounter{mycount3}{1};
  }
}
\end{scope}

\begin{scope}[yshift=5em,xshift=-0.5em]
\visible&lt;5-&gt;{
\setcounter{mycount4}{1}
\draw[step=0.5cm,color=red,thick] (-1,-1) grid (1,1);
\foreach \y in {+0.75,+0.25,-0.25,-0.75}
  \foreach \x in {-0.75,-0.25,0.25,0.75}{
    \node [fill=red!20,inner sep=0pt,minimum height=0.49cm,minimum width=0.49cm] at (\x,\y) {\number\value{mycount4}};
    \addtocounter{mycount4}{1};
  }
}
\end{scope}

\end{scope}
\end{tikzpicture}
\end{center}

\end{frame}

%%%------------------------------------------------------------------------------------------------------------
%%% ������һ����ά���Ժ���
\begin{frame}{��ʵ�ϣ��������Ǽ򵥵Ķ�ά���� - ��ţ��˽��� :)}
\begin{itemize}
\item \textbf{�ǳ������ε�˵}������\alert{����}�����;���ļ���չ������˵����ά����\alert{Ҳ����}����������ı�����ʽ
\item&lt;2-&gt; �ϸ������ϣ������ǣ�
    \begin{enumerate}
    \item&lt;2-&gt; \textbf{�������Ķ���}������������ϵ�ı�ʱ����һ������ת����ϵ�ĳ����������һ���������ϵ������任���仯�ļ����������ζ��壩
    \item&lt;3-&gt; \textbf{���ǿ������Ķ���}������������Э����ͨ�������˷�����������������壩
    \item&lt;4-&gt; \textbf{�����Խ��͵Ķ���}��\alert{�����Ƕ������Ժ���}���Ƕ�����һЩ�����ռ�͵ѿ������ϵĶ�������ӳ��
        \begin{itemize}
        \item ������Ϊ$T(v_0,...,v_r)$������������$r$������$\{v_0,...,v_r\}$
        \item ����������ָ������ÿ�����룬�����������Եģ����磬����һ��$v_i$��������
        \vspace{-0.3em}
        \begin{displaymath}
        T(v_0,...,v_i+c \cdot u,...,v_r) = T(v_0,...,v_i,...,v_r) + c \cdot T(v_0,...,u,...,v_r)
        \end{displaymath}
        ���У�$c$Ϊ��������������ʷǳ���Ҫ���������Ƶ���ǰ����������塣
        \end{itemize}
    \end{enumerate}
\end{itemize}
\end{frame}

%%%------------------------------------------------------------------------------------------------------------
%%% ��һ������һ�������Ķ���
\begin{frame}{������һ��``��''������``����''}
\begin{itemize}
\item ������һ�£�
    \begin{itemize}
    \item ���һ�����������������ĳ��λ����ֻ��һ����ֵ�������Ǳ����������ܶ�
    \item �������ͬһ��λ�á��Ӷ���ķ����Ͽ����в�ͬ��ֵ�����������ǡ���þ���˹۲췽�������������������(rank$&gt;$1)������Ӧ������
    \end{itemize}
\end{itemize}

\vspace{-0.8em}
\begin{center}
\tdplotsetmaincoords{50}{140}
\begin{tikzpicture}[scale=2,tdplot_main_coords]
\visible&lt;3-&gt;{
\draw[thick,-&gt;] (0,0,0) -- (1,0,0) node[anchor=north east]{$a$};
\draw[thick,-&gt;] (0,0,0) -- (0,1,0) node[anchor=north west]{$b$};
\draw[thick,-&gt;] (0,0,0) -- (0,0,1) node[anchor=south]{$c$};
}
\pgfmathsetmacro{\ax}{2}
\pgfmathsetmacro{\ay}{2}
\pgfmathsetmacro{\az}{1}
\tdplotsetrotatedcoords{20}{40}{00}
\visible&lt;4-&gt;{
\draw[thick,color=red,tdplot_rotated_coords,-&gt;] (0,0,0)
        -- (.7,0,0) node[anchor=east]{$a'$};
\draw[thick,color=green!50!black,tdplot_rotated_coords,-&gt;] (0,0,0)
        -- (0,.7,0) node[anchor=west]{$b'$};
\draw[thick,color=blue,tdplot_rotated_coords,-&gt;] (0,0,0)
        -- (0,0,.7) node[anchor=south]{$c'$};
}
\tdplottransformmainrot{\ax}{\ay}{\az}

\visible&lt;3-&gt;{\node [anchor=west,inner sep=2pt] (coord1) at (-0.40in,-0.4in) {\footnotesize{����$v=(a,b,c)$}};}
\visible&lt;4-&gt;{\node [anchor=north west,inner sep=2pt] (coord2) at (coord1.south west) {\footnotesize{����$u=(\red{a'}\black{,}{\color{ugreen} b'}\black{,}\blue{c'}\black{)}$}};}

\begin{scope}[xshift=0.4in,yshift=0.35in]
\visible&lt;2-&gt;{
\node [anchor=west,inner sep = 2pt] (description) at (0,0) {\small{$T(v,u)$��һ����ά�ռ�$(x,y,z)$�ϵ�}};
\node [anchor=north west,inner sep = 2pt] (description2) at (description.south west) {\small{2������������$v$��$u$����������}};
}

\visible&lt;5-&gt;{
\node [anchor=north west,inner sep=2pt] (T) at ([yshift=-2em]description2.south west) {\small{$T(v,u)=$}};
\node [anchor=west,inner sep=1pt] (T2) at (T.east) {\footnotesize{$\begin{pmatrix} v_x \\ v_y \\ v_z \end{pmatrix}^T$}};
\node [anchor=west,inner sep=1pt] (T3) at ([xshift=2pt]T2.east) {\footnotesize{$\begin{pmatrix} T_{xx} &amp; T_{xy} &amp; T_{xz} \\ T_{yx} &amp; T_{yy} &amp; T_{yz} \\ T_{zx} &amp; T_{zy} &amp; T_{zz} \end{pmatrix}$}};
\node [anchor=west,inner sep=1pt] (T4) at ([xshift=2pt]T3.east) {\footnotesize{$\begin{pmatrix} u_x \\ u_y \\ u_z \end{pmatrix}$}};
}
\begin{pgfonlayer}{background}
\visible&lt;7-&gt;{
\node [rectangle,inner sep=0pt,fill=red!20,minimum height=3.5em,minimum width=7em] [fit = (T3) ] (TBox) {};
}
\visible&lt;6-&gt;{
\node [rectangle,inner sep=0pt,fill=green!20,minimum height=3.5em,minimum width=3em] [fit = (T2) ] (VBox) {};
\node [rectangle,inner sep=0pt,fill=blue!20,minimum height=3.5em,minimum width=2.5em] [fit = (T4) ] (UBox) {};
}
\end{pgfonlayer}

\visible&lt;6-&gt;{
\draw [&lt;-] (VBox.north) -- ([yshift=0.3em]VBox.north);
\node [anchor=south,align=left] (Vlabel) at ([yshift=0.3em]VBox.north) {\scriptsize{$v$�ڻ������ϵ�ͶӰ}};
\draw [&lt;-] (UBox.north) -- ([yshift=0.3em]UBox.north);
\node [anchor=south,align=left] (Ulabel) at ([yshift=0.3em,xshift=-1em]UBox.north) {\scriptsize{$u$�ڻ������ϵ�ͶӰ}};
}
\visible&lt;7-&gt;{
\draw [&lt;-] (TBox.south) -- ([yshift=-0.3em]TBox.south);
\node [anchor=north,align=left] (Vlabel) at ([xshift=-0.5em,yshift=-0.3em]TBox.south) {\scriptsize{������$3 \times 3$�������ϵķ�����ǡ����``����''��ʾ��}};
\node [anchor=north west,align=left] (Vlabel2) at ([yshift=0.2em]Vlabel.south west) {\scriptsize{��Ϊ$[T]$������һ������ϵ����ת}};
}
\end{scope}
\end{tikzpicture}
\end{center}

\end{frame}

%%%------------------------------------------------------------------------------------------------------------
%%% ��������ѧϰ�ж���һ������
\begin{frame}{``����''��``����''����չ������������ʹ������}
\begin{itemize}
\item ����ǰ��Ŀ��Ժ��� - �����``\alert{�������Ƕ�ά����}''
    \begin{itemize}
    \item ���������󶼿��Կ�����ѧ�ϵ�``����''����չ
    \end{itemize}

\item&lt;2-&gt; ����$T(1:3)$��ʾһ��������������Ԫ��\\
\vspace{0.5em}
\begin{tikzpicture}
\begin{scope}
\node [anchor=north east, inner sep=1pt] (label) at (0,0) {�����洢��};
\draw[step=0.5cm,thick] (0,-0.5) grid (1.5,0);
\setcounter{mycount1}{1}
\foreach \x in {0.25,0.75,1.25}{
    \node [fill=green!20,inner sep=0pt,minimum height=0.49cm,minimum width=0.49cm] at (\x,-0.25) {$\number\value{mycount1}$};
    \addtocounter{mycount1}{1};
}
\end{scope}
\end{tikzpicture}

\item&lt;3-&gt; ����$T(1:2,1:3)$��ʾһ��$3 \times 2$�ľ���\\
\vspace{0.5em}
\begin{tikzpicture}
\begin{scope}
\node [anchor=north east, inner sep=1pt] (label) at (0,0) {�����洢��};
\draw[step=0.5cm,thick] (0,-0.5) grid (3.0,0);
\setcounter{mycount2}{1}
\foreach \x in {0.25,0.75,1.25}{
    \node [fill=green!20,inner sep=0pt,minimum height=0.49cm,minimum width=0.49cm] at (\x,-0.25) {$\number\value{mycount2}$};
    \addtocounter{mycount2}{1};
}
\foreach \x in {1.75,2.25,2.75}{
    \node [fill=red!20,inner sep=0pt,minimum height=0.49cm,minimum width=0.49cm] at (\x,-0.25) {$\number\value{mycount2}$};
    \addtocounter{mycount2}{1};
}
\end{scope}
\end{tikzpicture}

\item&lt;4-&gt; ����$T(1:2,1:2,1:3)$��ʾһ��������������С��$3 \times 2 \times 2$\\
\vspace{0.5em}
\begin{tikzpicture}
\begin{scope}
\node [anchor=north east, inner sep=1pt] (label) at (0,0) {�����洢��};
\draw[step=0.5cm,thick] (0,-0.5) grid (6.0,0);
\setcounter{mycount3}{1}
\foreach \x in {0.25,0.75,1.25}{
    \node [fill=green!20,inner sep=0pt,minimum height=0.49cm,minimum width=0.49cm] at (\x,-0.25) {$\number\value{mycount3}$};
    \addtocounter{mycount3}{1};
}
\foreach \x in {1.75,2.25,2.75}{
    \node [fill=red!20,inner sep=0pt,minimum height=0.49cm,minimum width=0.49cm] at (\x,-0.25) {$\number\value{mycount3}$};
    \addtocounter{mycount3}{1};
}
\foreach \x in {3.25,3.75,4.25}{
    \node [fill=green!20,inner sep=0pt,minimum height=0.49cm,minimum width=0.49cm] at (\x,-0.25) {$\number\value{mycount3}$};
    \addtocounter{mycount3}{1};
}
\foreach \x in {4.75,5.25,5.75}{
    \node [fill=red!20,inner sep=0pt,minimum height=0.49cm,minimum width=0.49cm] at (\x,-0.25) {$\number\value{mycount3}$};
    \addtocounter{mycount3}{1};
}
\draw[decorate,thick,decoration={brace,mirror,raise=0.2em}] (0,-0.50) -- (2.95,-0.50);
\draw[decorate,thick,decoration={brace,mirror,raise=0.2em}] (3.05,-0.50) -- (6,-0.50);
\node [anchor=north] (subtensor1) at (1.5,-0.6) {\footnotesize{$3 \times 2$ sub-tensor}};
\node [anchor=north] (subtensor1) at (4.5,-0.6) {\footnotesize{$3 \times 2$ sub-tensor}};

\end{scope}
\end{tikzpicture}

\item&lt;5-&gt; �߽����������飡���飡���飡
    \begin{itemize}
    \item ��C++��Python�еĶ�ά����һģһ��
    \end{itemize}
\end{itemize}
\end{frame}

%%%------------------------------------------------------------------------------------------------------------
%%% ��������˷�
\begin{frame}{�����ľ���˷�}
\begin{itemize}
\item ����������$\textbf{y}=f(\textbf{x}\cdot \textbf{w} + \textbf{b})$��$\textbf{x} \cdot \textbf{w}$��$\textbf{x} \times \textbf{w}$�����Ա任������$\textbf{x}$������������$\textbf{w}$��һ������
    \begin{itemize}
    \item $\textbf{x} \cdot \textbf{w}$��ʾ���Ǿ���˷������Ϊ$\times$��
    \item ע�⣬���ﲻ�������˷�����Ϊ�����˷�������������
    \item $\textbf{w}$��$n \times m$�ľ���$\textbf{x}$����״��$... \times n$����$\textbf{x}$�ĵ�һά����Ҫ��$\textbf{w}$��������С���\\
    \vspace{0.5em}
    $\textbf{x}(1:4,1:4,\alert{1:4}) \times \textbf{w}(\alert{1:4},1:2) = \textbf{s}(1:4,1:4,1:2)$
    \end{itemize}
\end{itemize}

\begin{center}
\begin{tikzpicture}

\begin{scope}[yshift=6.5em,xshift=1em]
\visible&lt;2-&gt;{
\setcounter{mycount1}{1}
\draw[step=0.5cm,color=orange,thick] (-1,-1) grid (1,1);
\foreach \y in {+0.75,+0.25,-0.25,-0.75}
  \foreach \x in {-0.75,-0.25,0.25,0.75}{
    \node [fill=orange!20,inner sep=0pt,minimum height=0.49cm,minimum width=0.49cm] at (\x,\y) {$\number\value{mycount1}$};
    \addtocounter{mycount1}{1};
  }
}
\end{scope}

\begin{scope}[yshift=6em,xshift=0.5em]
\visible&lt;2-&gt;{
\setcounter{mycount2}{2}
\draw[step=0.5cm,color=blue,thick] (-1,-1) grid (1,1);
\foreach \y in {+0.75,+0.25,-0.25,-0.75}
  \foreach \x in {-0.75,-0.25,0.25,0.75}{
    \node [fill=blue!20,inner sep=0pt,minimum height=0.49cm,minimum width=0.49cm] at (\x,\y) {$\number\value{mycount2}$};
    \addtocounter{mycount2}{1};
  }
}
\end{scope}

\begin{scope}[yshift=5.5em,xshift=0em]
\visible&lt;2-&gt;{
\setcounter{mycount3}{3}
\draw[step=0.5cm,color=ugreen,thick] (-1,-1) grid (1,1);
\foreach \y in {+0.75,+0.25,-0.25,-0.75}
  \foreach \x in {-0.75,-0.25,0.25,0.75}{
    \node [fill=green!20,inner sep=0pt,minimum height=0.49cm,minimum width=0.49cm] at (\x,\y) {$\number\value{mycount3}$};
    \addtocounter{mycount3}{1};
  }
}
\end{scope}

\begin{scope}[yshift=5em,xshift=-0.5em]
\visible&lt;2-&gt;{
\setcounter{mycount4}{4}
\draw[step=0.5cm,color=red,thick] (-1,-1) grid (1,1);
\foreach \y in {+0.75,+0.25,-0.25,-0.75}
  \foreach \x in {-0.75,-0.25,0.25,0.75}{
    \node [fill=red!20,inner sep=0pt,minimum height=0.49cm,minimum width=0.49cm] at (\x,\y) {$\number\value{mycount4}$};
    \addtocounter{mycount4}{1};
  }
\node [anchor=north] (xlabel) at (0,-1.2) {$\textbf{x}$};
}
\end{scope}

\begin{scope}[yshift=5em,xshift=1.5in]
\visible&lt;2-&gt;{
\draw[step=0.5cm,thick] (-0.5,-1) grid (0.5,1.0);
\node [fill=black!20,inner sep=0pt,minimum height=0.49cm,minimum width=0.49cm] at (-0.25,0.75) {\small{$-1$}};
\node [fill=black!20,inner sep=0pt,minimum height=0.49cm,minimum width=0.49cm] at (-0.25,0.25) {$0$};
\node [fill=black!20,inner sep=0pt,minimum height=0.49cm,minimum width=0.49cm] at (-0.25,-0.25) {$1$};
\node [fill=black!20,inner sep=0pt,minimum height=0.49cm,minimum width=0.49cm] at (-0.25,-0.75) {$0$};
\node [fill=black!20,inner sep=0pt,minimum height=0.49cm,minimum width=0.49cm] at (0.25,0.75) {$0$};
\node [fill=black!20,inner sep=0pt,minimum height=0.49cm,minimum width=0.49cm] at (0.25,0.25) {\small{$-1$}};
\node [fill=black!20,inner sep=0pt,minimum height=0.49cm,minimum width=0.49cm] at (0.25,-0.25) {$1$};
\node [fill=black!20,inner sep=0pt,minimum height=0.49cm,minimum width=0.49cm] at (0.25,-0.75) {$0$};
\node [anchor=north] (xlabel) at (0,-1.2) {$\textbf{w}$};
}

\visible&lt;3&gt;{\draw [-&gt;,thick,dashed] (-1.5in+2em+1.5em,-0.3) .. controls +(east:2) and +(west:1) .. (-0.55,0.8) node [pos=0.5,left] {\scriptsize{\textbf{�����}}};}
\visible&lt;4&gt;{\draw [-&gt;,thick,dashed] (-1.5in+2em+1.0em,-0.5) .. controls +(east:2) and +(west:1) .. (-0.55,0.8) node [pos=0.5,left] {\scriptsize{\textbf{�����}}};}
\visible&lt;5&gt;{\draw [-&gt;,thick,dashed] (-1.5in+2em+0.5em,-0.7) .. controls +(east:2.5) and +(west:1) .. (-0.55,0.8) node [pos=0.5,left] {\scriptsize{\textbf{�����}}};}
\visible&lt;6-&gt;{\draw [-&gt;,thick,dashed] (-1.5in+2em,-0.9) .. controls +(east:3) and +(west:1) .. (-0.55,0.8) node [pos=0.5,left] {\scriptsize{\textbf{�����}}};}
\end{scope}

\begin{scope}[yshift=6.5em,xshift=1em+3in]
\visible&lt;3-&gt;{
\draw[step=0.5cm,color=orange,thick] (-0.5,-1) grid (0.5,1.0);
\foreach \y in {+0.75,+0.25,-0.25,-0.75}{
  \setcounter{mycount1}{2}
  \foreach \x in {-0.25,0.25}{
    \node [fill=orange!20,inner sep=0pt,minimum height=0.49cm,minimum width=0.49cm] at (\x,\y) {$\number\value{mycount1}$};
    \addtocounter{mycount1}{-1};
  }
}
}
\end{scope}

\begin{scope}[yshift=6em,xshift=0.5em+3in]
\visible&lt;4-&gt;{
\draw[step=0.5cm,color=blue,thick] (-0.5,-1) grid (0.5,1.0);
\foreach \y in {+0.75,+0.25,-0.25,-0.75}{
  \setcounter{mycount1}{2}
  \foreach \x in {-0.25,0.25}{
    \node [fill=blue!20,inner sep=0pt,minimum height=0.49cm,minimum width=0.49cm] at (\x,\y) {$\number\value{mycount1}$};
    \addtocounter{mycount1}{-1};
  }
}
}
\end{scope}

\begin{scope}[yshift=5.5em,xshift=0em+3in]
\visible&lt;5-&gt;{
\draw[step=0.5cm,color=ugreen,thick] (-0.5,-1) grid (0.5,1.0);
\foreach \y in {+0.75,+0.25,-0.25,-0.75}{
  \setcounter{mycount1}{2}
  \foreach \x in {-0.25,0.25}{
    \node [fill=green!20,inner sep=0pt,minimum height=0.49cm,minimum width=0.49cm] at (\x,\y) {$\number\value{mycount1}$};
    \addtocounter{mycount1}{-1};
  }
}
}
\end{scope}

\begin{scope}[yshift=5.0em,xshift=-0.5em+3in]
\visible&lt;6-&gt;{
\draw[step=0.5cm,color=red,thick] (-0.5,-1) grid (0.5,1.0);
\foreach \y in {+0.75,+0.25,-0.25,-0.75}{
  \setcounter{mycount1}{2}
  \foreach \x in {-0.25,0.25}{
    \node [fill=red!20,inner sep=0pt,minimum height=0.49cm,minimum width=0.49cm] at (\x,\y) {$\number\value{mycount1}$};
    \addtocounter{mycount1}{-1};
  }
}
}

\visible&lt;3-&gt;{
\node [anchor=north] (xlabel) at (0,-1.2) {$\textbf{x} \cdot \textbf{w}$};
\node [anchor=center] (elabel) at (-0.7in,0) {\Huge{$\textbf{=}$}};
}
\end{scope}

\end{tikzpicture}
\end{center}

\end{frame}

%%%------------------------------------------------------------------------------------------------------------
%%% �����ĵ�Ԫ����
\begin{frame}{�����ĵ�Ԫ����}
\begin{itemize}
\item ������$\textbf{y}=f(\textbf{x}\cdot \textbf{w} + \textbf{b})$Ҳ����һЩ�����ĵ�Ԫ������element-wise operation��
	\begin{itemize}
	\item �ӷ���$\textbf{s}+\textbf{b}$������$\textbf{s} = \textbf{x}\cdot \textbf{w}$
	\item �������$f(\cdot)$
	\end{itemize}
\item&lt;2-&gt; \textbf{��Ԫ��}���Ƕ������е�ÿ��λ�ö����мӷ�
	\begin{itemize}
	\item&lt;3-&gt; ��չ���ӷ���\textbf{�㲥}���ظ�����һ���������мӷ�������Ҫ������������״��ͬ
	\end{itemize}
\end{itemize}

\vspace{-1.5em}
\begin{center}
\begin{tikzpicture}
\visible&lt;3-&gt;{
\begin{scope}
\setcounter{mycount1}{1}
\draw[step=0.5cm,color=orange,thick] (-1,-0.5) grid (1,0.5);
\foreach \y in {+0.25,-0.25}
  \foreach \x in {-0.75,-0.25,0.25,0.75}{
    \node [fill=orange!20,inner sep=0pt,minimum height=0.49cm,minimum width=0.49cm] at (\x,\y) {$\number\value{mycount1}$};
    \addtocounter{mycount1}{1};
  }
\node [anchor=south] (varlabel) at (0,0.6) {$\textbf{s}$};
\end{scope}
\begin{scope}[xshift=1.5in]
\setcounter{mycount1}{1}
\draw[step=0.5cm,color=ugreen,thick] (-1,-0) grid (1,0.5);
\foreach \y in {+0.25}
  \foreach \x in {-0.75,-0.25,0.25,0.75}{
    \node [fill=green!20,inner sep=0pt,minimum height=0.49cm,minimum width=0.49cm] at (\x,\y) {$1$};
    \addtocounter{mycount1}{1};
  }
\node [anchor=center] (plabel) at (-4.5em,0) {\huge{$\textbf{+}$}};
\node [anchor=south] (varlabel) at (0,0.6) {$\textbf{b}$};
\end{scope}
\begin{scope}[xshift=3in]
\setcounter{mycount1}{2}
\draw[step=0.5cm,color=orange,thick] (-1,-0.5) grid (1,0.5);
\foreach \y in {+0.25,-0.25}
  \foreach \x in {-0.75,-0.25,0.25,0.75}{
    \node [fill=orange!20,inner sep=0pt,minimum height=0.49cm,minimum width=0.49cm] at (\x,\y) {$\number\value{mycount1}$};
    \addtocounter{mycount1}{1};
  }
\node [anchor=center] (plabel) at (-4.5em,0) {\huge{$\textbf{=}$}};
\node [anchor=south] (varlabel) at (0,0.6) {$\textbf{s+b}$};
\end{scope}
}

\end{tikzpicture}
\end{center}

\vspace{-0.3em}

\begin{itemize}
\item&lt;4-&gt; ���Ƶģ����ǿ������������˷���Ҳ�������������Ҳ��������������������vectorization��
\end{itemize}

\vspace{-0.5em}
\visible&lt;4-&gt;{
\begin{displaymath}
\textrm{Relu} \Big( \begin{pmatrix} 2 \\ -.3 \end{pmatrix} \Big) = \begin{pmatrix} 2 \\ 0 \end{pmatrix}
\end{displaymath}
}

\end{frame}

%%%------------------------------------------------------------------------------------------------------------
%%% ���ѧϰ���߰�
\begin{frame}{���ʵ�֣�- ��Դ����������}
\begin{itemize}
\item ʵ��������Ŀ�Դϵͳ�ܶ࣬һ���򵥺��õĹ��߰�NumPy \url{https://numpy.org/}
    \begin{itemize}
    \item Python�ӿڣ���ά����Ķ���ʹ�÷���
    \item �ṩ��������ʾ��ʹ�õķ�ʽ
    \end{itemize}
\item&lt;2-&gt; ������ܻ��������ܣ�TensorFlow��PyTorch
    \begin{itemize}
    \item Google��Facebook��Ʒ�������б�֤
    \item ����ǿ�󣬽ӿڷḻ
    \item ���Խ��д��ģ�����Ӧ��
    \item �����ɲο���ʵ��
    \end{itemize}

    \includegraphics[scale=0.13]{./Figures/tensorflowpytorch.jpg}
\item&lt;3-&gt; �����������ڸ��µ������ܣ� CNTK��MXNet��PaddlePaddle��Keras��Chainer�� dl4j��NiuTensor��
\end{itemize}
\end{frame}

%%%------------------------------------------------------------------------------------------------------------
%%% NiuTrans.Tensor���߰�
\begin{frame}{NiuTensor}
\begin{itemize}
\item ����ʹ���������е�NiuTensor���߰����н�ѧ \url{http://www.niutrans.com/opensource/niutensor/index.html}
    \begin{itemize}
    \item ��С�ɣ������޸�
    \item C++���Ա�д������߶��Ż�
    \item ͬʱ֧��CPU��GPU�豸
    \item �ḻ����������ӿ�
    \end{itemize}
\end{itemize}

\includegraphics[scale=0.35]{./Figures/niutensor.jpg}

\end{frame}

%%%------------------------------------------------------------------------------------------------------------
%%% ʹ��NiuTensor
\begin{frame}{ʹ��NiuTensor}
\begin{itemize}
\item NiuTensor��ʹ�úܼ򵥣�������һ��C++����
\end{itemize}

\begin{tcolorbox}[enhanced,frame engine=empty,boxrule=0.1mm,size=title,colback=blue!10!white]
\begin{flushleft}
{\scriptsize
\begin{tabbing}
\texttt{\#include "source/tensor/XTensor.h"} \hspace{4em} \= // ����XTensor�����ͷ�ļ� \\

\texttt{using namespace nts;} \&gt; // ����nts�����ռ� \\
\ \\

\texttt{int main(int argc, const char ** argv)\{} \\
\ \ \ \ \texttt{XTensor tensor;} \&gt; // ��������tensor \\

\ \ \ \ \texttt{InitTensor2D(\&amp;tensor, 2, 2, X\_FLOAT);} \&gt; // ��������Ϊ2*2�ľ��� \\

\ \ \ \ \texttt{tensor.SetDataRand();} \&gt; // [0,1]���ȷֲ���ʼ������ \\

\ \ \ \ \texttt{tensor.Dump(stdout);} \&gt; // ����������� \\

\ \ \ \ \texttt{return 0;}\\
\texttt{\}}

\end{tabbing}
}
\end{flushleft}
\end{tcolorbox}

\begin{itemize}
\item&lt;2-&gt; ��������������ʾ����ÿ��Ԫ�ص�ֵ
\begin{itemize}
\item&lt;2-&gt; ��������(order=2)����״��$2 \times 2$ (dimsize=2,2)�����������ǵ����ȸ���(dtype=X\_FLOAT)����ϡ��(dense=1.00)
\end{itemize}
\end{itemize}

\vspace{-0em}
\visible&lt;2-&gt;{
\begin{tcolorbox}[enhanced,frame engine=empty,boxrule=0.1mm,size=title,colback=black!10!white]
\begin{flushleft}
{\scriptsize
\begin{tabbing}
\texttt{order=2 dimsize=2,2 dtype=X\_FLOAT dense=1.000000} \\
\texttt{3.605762e-001 2.992340e-001 1.393780e-001 7.301248e-001}
\end{tabbing}
}
\end{flushleft}
\end{tcolorbox}
}

\end{frame}

%%%------------------------------------------------------------------------------------------------------------
%%% ����XTensor
\begin{frame}{����XTensor}
\begin{itemize}
\item ��������XTensor��ʾ������InitTensor���壬������
    \begin{itemize}
    \item ָ��XTensor���ͱ�����ָ��
    \item �����Ľ�
    \item ��������ά�ȵĴ�С���봫ͳ��ά����Լ��һ����
    \item �������������͵ȣ���ȱʡֵ��
    \end{itemize}
\end{itemize}

\vspace{-0.3em}
\begin{tcolorbox}[enhanced,frame engine=empty,boxrule=0.1mm,size=title,colback=blue!10!white]
\begin{flushleft}
{\scriptsize
\begin{tabbing}
\texttt{XTensor tensor;} \hspace{12em} \= // ��������tensor \\
\texttt{int sizes[6] = \{2,3,4,2,3,4\};} \&gt; // ��������״Ϊ2*3*4*2*3*4 \\
\texttt{InitTensor(\&amp;tensor, 6, sizes, X\_FLOAT);} \&gt; // ������״Ϊsizes��6������
\end{tabbing}
}
\end{flushleft}
\end{tcolorbox}

\visible&lt;2-&gt;{
\begin{itemize}
\item �����Ķ��巽ʽ
\end{itemize}

\vspace{-0.2em}
\begin{tcolorbox}[enhanced,frame engine=empty,boxrule=0.1mm,size=title,colback=blue!10!white]
\begin{flushleft}
{\scriptsize
\begin{tabbing}
\texttt{XTensor a, b, c;} \hspace{11.5em} \= // ��������tensor \\
\texttt{InitTensor1D(\&amp;a, 10, X\_INT);} \&gt; // 10ά������������\\
\texttt{InitTensor1D(\&amp;b, 10);} \&gt; // 10ά��������ȱʡ����(����)\\
\texttt{InitTensor4D(\&amp;c, 10, 20, 30, 40);} \&gt; // 10*20*30*40��4������(����)
\end{tabbing}
}
\end{flushleft}
\end{tcolorbox}
}

\visible&lt;3-&gt;{
\begin{itemize}
\item ֱ����GPU�϶�������
\end{itemize}

\vspace{-0.2em}
\begin{tcolorbox}[enhanced,frame engine=empty,boxrule=0.1mm,size=title,colback=blue!10!white]
\begin{flushleft}
{\scriptsize
\begin{tabbing}
\texttt{XTensor tensorGPU;} \hspace{10.5em} \= // ��������tensor \\
\texttt{InitTensor2D(\&amp;tensorGPU, 10, 20,} $\backslash$ \&gt; // �ڱ��Ϊ0��GPU�϶������� \\
\hspace{6.7em} \texttt{X\_FLOAT, 0);}
\end{tabbing}
}
\end{flushleft}
\end{tcolorbox}
}

\end{frame}

%%%------------------------------------------------------------------------------------------------------------
%%% XTensor�Ĵ�������
\begin{frame}{��������}
\begin{itemize}
\item ���ֵ�Ԫ���ӣ�1�����㣩+��-��*��$\backslash$��Log��Exp�� Power��Absolute�ȣ�����Sigmoid��Softmax�ȼ����
\end{itemize}

\vspace{-0.2em}
\begin{tcolorbox}[enhanced,frame engine=empty,boxrule=0.1mm,size=title,colback=blue!10!white]
\begin{flushleft}
{\scriptsize
\begin{tabbing}
\texttt{XTensor a, b, c, d, e;} \hspace{7em} \= // ��������tensor \\
\texttt{InitTensor3D(\&amp;a, 2, 3, 4);} \&gt; // aΪ2*3*4��3������ \\
\texttt{InitTensor3D(\&amp;b, 2, 3, 4);} \&gt; // bΪ2*3*4��3������ \\
\texttt{InitTensor3D(\&amp;c, 2, 3, 4);} \&gt; // cΪ2*3*4��3������ \\
\texttt{a.SetDataRand();} \&gt; // �����ʼ��a \\
\texttt{b.SetDataRand();} \&gt; // �����ʼ��b \\
\texttt{c.SetDataRand();} \&gt; // �����ʼ��c \\
\texttt{d = a + b * c;} \&gt; // d����ֵΪ a + b * c \\
\texttt{d = ((a + b) * d - b / c ) * d;} \&gt; // d���Ա�Ƕ��ʹ�� \\
\texttt{e = Sigmoid(d);} \&gt; // d���������Sigmoid��ֵ��e
\end{tabbing}
}
\end{flushleft}
\end{tcolorbox}

\visible&lt;2-&gt;{
\begin{itemize}
\item �߽����㣬��õ��Ǿ���˷�(MMul)
\end{itemize}

\vspace{-0.2em}
\begin{tcolorbox}[enhanced,frame engine=empty,boxrule=0.1mm,size=title,colback=blue!10!white]
\begin{flushleft}
{\scriptsize
\begin{tabbing}
\texttt{XTensor a, b, c;} \hspace{10.0em} \= // ��������tensor \\
\texttt{InitTensor4D(\&amp;a, 2, 2, 3, 4);} \&gt; // aΪ2*2*3*4��4������ \\
\texttt{InitTensor2D(\&amp;b, 4, 5);} \&gt; // bΪ4*5�ľ��� \\
\texttt{a.SetDataRand();} \&gt; // �����ʼ��a \\
\texttt{b.SetDataRand();} \&gt; // �����ʼ��b \\
\texttt{c = MMul(a, b);} \&gt; // ����˵Ľ��Ϊ2*2*3*5��4������
\end{tabbing}
}
\end{flushleft}
\end{tcolorbox}
}

\end{frame}

%%%------------------------------------------------------------------------------------------------------------
%%% XTensor����������
\begin{frame}{�������ú���}
\begin{itemize}
\item �����������в�ȫ�����Բο���վ�ϵ���ϸ˵��
\end{itemize}

\footnotesize{
\begin{center}
\begin{tabular}{l|l}
���� &amp; ���� \\ \hline
\texttt{a.Reshape(o, s)} &amp; ��a�任Ϊ��Ϊo����״Ϊs������\\
\texttt{a.Get(pos)} &amp; ȡ������λ��Ϊpos��Ԫ�� \\
\texttt{a.Set(v, pos)} &amp; ��������λ��Ϊpos��Ԫ�ص�ֵ��Ϊv \\
\texttt{a.Dump(file)} &amp; �������浽file�У�fileΪ�ļ���� \\
\texttt{a.Read(file)} &amp; ��file�ж�ȡ������fileΪ�ļ���� \\ \hline
\texttt{Power(a, p)}  &amp; ����ָ��$\textrm{a}^{\textrm{p}}$ \\
\texttt{Linear(a, s, b)}  &amp; ���� a * s + b��s��b����һ���� \\
\texttt{CopyValues(a)} &amp; ����a��һ������ \\
\texttt{ReduceMax(a, d)} &amp; ��a���ŷ���d���й�Լ���õ����ֵ \\
\texttt{ReduceSum(a, d)} &amp; ��a���ŷ���d���й�Լ���õ��� \\
\texttt{Concatenate(a, b, d)} &amp; ����������a��b��d������\\
\texttt{Merge(a, d)} &amp; ������a��d����ϲ�\\
\texttt{Split(a, d, n)} &amp; ������a��d������ѳ�n��\\ \hline
\texttt{Sigmoid(a)}  &amp; ��a����Sigmoid�任 \\
\texttt{Softmax(a)}  &amp; ��a����Softmax�任�������һ������ \\
\texttt{HardTanH(a)} &amp; ��a����hard tanh�任(˫�����еĽ���)\\
\texttt{Relu(a)}     &amp; ��a����Relu�任\\
\end{tabular}
\end{center}
}

\end{frame}

%%%------------------------------------------------------------------------------------------------------------
%%% ����XTensor����������
\begin{frame}{����������}
\begin{itemize}
\item ���Ժܷ���Ĺ���һ����������
\end{itemize}

\begin{tcolorbox}
[bicolor,sidebyside,righthand width=4cm,size=title,frame engine=empty,
 colback=blue!10!white,colbacklower=black!5!white]
 {\scriptsize
\begin{tabbing}
\texttt{XTensor x, y, w, b;} \\
\texttt{InitTensor3D(\&amp;x, 3, 4, 5);} \\
\texttt{InitTensor2D(\&amp;w, 5, 3);} \\
\texttt{InitTensor1D(\&amp;b, 3);} \\
\texttt{...} \\
\texttt{y = Sigmoid(MMul(x, w) + b);}
\end{tabbing}
}
\tcblower
\begin{center}
\begin{tikzpicture}
\node [draw,circle,inner sep=2pt,fill=red!30!white,blur shadow={shadow xshift=1pt,shadow yshift=-1pt}] (x) at (0,0) {\footnotesize{$\textrm{x}$}};
\node [anchor=south,draw,rounded corners,inner sep=2pt,minimum width=4em,fill=green!30!white,blur shadow={shadow xshift=1pt,shadow yshift=-1pt}] (layer) at ([yshift=0.7em]x.north) {\scriptsize{layer}};
\node [anchor=south,draw,circle,inner sep=2pt,fill=red!30!white,blur shadow={shadow xshift=1pt,shadow yshift=-1pt}] (y) at ([yshift=0.7em]layer.north) {\scriptsize{$\textrm{y}$}};
\draw [thick,-&gt;] (x.north) -- (layer.south);
\draw [thick,-&gt;] (layer.north) -- (y.south);
\node [anchor=west,align=left] (xshape) at (x.east) {\tiny{shape: 3*4*5}};
\node [anchor=west,align=left] (yshape) at (y.east) {\tiny{shape: 3*4*3}};
\end{tikzpicture}
\end{center}
\end{tcolorbox}

\visible&lt;2-&gt;{
\begin{itemize}
\item һ���������
\end{itemize}

\begin{tcolorbox}
[bicolor,sidebyside,righthand width=4cm,size=title,frame engine=empty,
 colback=blue!10!white,colbacklower=black!5!white]
 {\scriptsize
\begin{tabbing}
\texttt{XTensor x, y, h1, h2;} \\
\texttt{XTensor w1, b1, w2, w3;} \\
\texttt{InitTensor3D(\&amp;x, 3, 4, 5);} \\
\texttt{InitTensor2D(\&amp;w1, 5, 3);} \\
\texttt{InitTensor1D(\&amp;b1, 3);} \\
\texttt{InitTensor2D(\&amp;w2, 3, 6);} \\
\texttt{InitTensor2D(\&amp;w3, 6, 4);} \\
\texttt{...} \\
\texttt{h1 = Sigmoid(MMul(x, w1) + b1);} \\
\texttt{h2 = HandTanH(MMul(h1, w2));} \\
\texttt{y = Relu(MMul(h2, w3));}
\end{tabbing}
}
\tcblower
\begin{center}
\begin{tikzpicture}
\node [draw,circle,inner sep=2pt,fill=red!30!white,blur shadow={shadow xshift=1pt,shadow yshift=-1pt}] (x) at (0,0) {\footnotesize{$\textrm{x}$}};
\node [anchor=south,draw,rounded corners,inner sep=2pt,minimum width=4em,fill=green!30!white,blur shadow={shadow xshift=1pt,shadow yshift=-1pt}] (layer1) at ([yshift=0.7em]x.north) {\scriptsize{layer1}};
\node [anchor=south,draw,rounded corners,inner sep=2pt,minimum width=4em,fill=green!30!white,blur shadow={shadow xshift=1pt,shadow yshift=-1pt}] (layer2) at ([yshift=1.0em]layer.north) {\scriptsize{layer2}};
\node [anchor=south,draw,rounded corners,inner sep=2pt,minimum width=4em,fill=green!30!white,blur shadow={shadow xshift=1pt,shadow yshift=-1pt}] (layer3) at ([yshift=1.0em]layer2.north) {\scriptsize{layer3}};
\node [anchor=south,draw,circle,inner sep=2pt,fill=red!30!white,blur shadow={shadow xshift=1pt,shadow yshift=-1pt}] (y) at ([yshift=0.7em]layer3.north) {\scriptsize{$\textrm{y}$}};
\draw [thick,-&gt;] (x.north) -- (layer1.south);
\draw [thick,-&gt;] (layer1.north) -- (layer2.south);
\draw [thick,-&gt;] (layer2.north) -- (layer3.south);
\draw [thick,-&gt;] (layer3.north) -- (y.south);
\node [anchor=west,align=left] (xshape) at (x.east) {\tiny{shape: 3*4*5}};
\node [anchor=west,align=left] (yshape) at (y.east) {\tiny{shape: 3*4*4}};
\node [anchor=south west,align=left,inner sep=2pt] (l1shape) at (layer1.north) {\tiny{shape: 3*4*3}};
\node [anchor=south west,align=left,inner sep=2pt] (l2shape) at (layer2.north) {\tiny{shape: 3*4*6}};
\end{tikzpicture}
\end{center}
\end{tcolorbox}
}

\end{frame}

%%%------------------------------------------------------------------------------------------------------------
%%% ����XTensor���������ӵ�������
\begin{frame}{������һ�������}
\begin{itemize}
\item �κ����綼���Թ���������RNN��Transformer ��
\end{itemize}

\begin{tcolorbox}
[bicolor,sidebyside,righthand width=4cm,size=title,frame engine=empty,
 colback=blue!10!white,colbacklower=black!5!white]
 {\scriptsize
\begin{tabbing}
\texttt{XTensor x[3], y[3], r, wh;} \\
\texttt{XTensor h1, h2, w1, b1, h3, h4;} \\
\texttt{XList splits;} \\
\texttt{...} \\
\texttt{for(unsigned i = 0; i &lt; 3; i++)\{} \\
\texttt{\hspace{2em}r = Concatenate(x[i] + r) * wh;}\\
\texttt{\hspace{2em}splits.Add(\&amp;r);}\\
\texttt{\}}\\
\visible&lt;2-&gt;{
\texttt{} \\
\texttt{h1 = Merge(splits, 0);}\\
\texttt{h2 = Relu(h1 * w1 + b1);}\\
\texttt{h3 = h1 + h2;} \\
\texttt{h4 = Softmax(h3);} \\
}
\visible&lt;3-&gt;{
\texttt{} \\
\texttt{Split(h4, splits, 0);} \\
\texttt{} \\
\texttt{for(unsigned i = 0; i &lt; 3; i++)\{} \\
\texttt{\hspace{2em}y[i] = *(XTensor*)splits.Get(i);}\\
\texttt{\hspace{2em}y[i].Dump(stdout);}\\
\texttt{\}}
}
\end{tabbing}
}
\tcblower
\begin{center}
\begin{tikzpicture}
\node [draw,circle,inner sep=1pt,fill=red!30!white,blur shadow={shadow xshift=1pt,shadow yshift=-1pt}] (x1) at (0,0) {\footnotesize{$\textrm{x}_1$}};
\node [anchor=west,draw,circle,inner sep=1pt,fill=red!30!white,blur shadow={shadow xshift=1pt,shadow yshift=-1pt}] (x2) at ([xshift=2em]x1.east) {\footnotesize{$\textrm{x}_2$}};
\node [anchor=west,draw,circle,inner sep=1pt,fill=red!30!white,blur shadow={shadow xshift=1pt,shadow yshift=-1pt}] (x3) at ([xshift=2em]x2.east) {\footnotesize{$\textrm{x}_3$}};
\node [anchor=south,draw,rounded corners,inner sep=2pt,minimum width=2.5em,fill=green!30!white,blur shadow={shadow xshift=1pt,shadow yshift=-1pt}] (rlayer1) at ([yshift=1em]x1.north) {\tiny{rlayer}};
\node [anchor=south,draw,rounded corners,inner sep=2pt,minimum width=2.5em,fill=green!30!white,blur shadow={shadow xshift=1pt,shadow yshift=-1pt}] (rlayer2) at ([yshift=1em]x2.north) {\tiny{rlayer}};
\node [anchor=south,draw,rounded corners,inner sep=2pt,minimum width=2.5em,fill=green!30!white,blur shadow={shadow xshift=1pt,shadow yshift=-1pt}] (rlayer3) at ([yshift=1em]x3.north) {\tiny{rlayer}};
\draw [-&gt;,thick] (x1.north) -- (rlayer1.south);
\draw [-&gt;,thick] (x2.north) -- (rlayer2.south);
\draw [-&gt;,thick] (x3.north) -- (rlayer3.south);
\draw [-&gt;,thick] (rlayer1.east) -- (rlayer2.west);
\draw [-&gt;,thick] (rlayer2.east) -- (rlayer3.west);
\draw [-&gt;,thick] (rlayer1.north) -- ([yshift=1em]rlayer1.north);
\draw [-&gt;,thick] (rlayer2.north) -- ([yshift=1em]rlayer2.north);
\draw [-&gt;,thick] (rlayer3.north) -- ([yshift=1em]rlayer3.north);

\visible&lt;2-&gt;{
\node [anchor=south,draw,rounded corners,inner sep=2pt,minimum width=9.4em,minimum height=1.0em,fill=green!30!white,blur shadow={shadow xshift=1pt,shadow yshift=-1pt}] (h1) at ([yshift=1em]rlayer2.north) {\tiny{h1 = Merge($\cdot$)}};
\node [anchor=south,draw,rounded corners,inner sep=2pt,minimum width=9.4em,minimum height=1.0em,fill=green!30!white,blur shadow={shadow xshift=1pt,shadow yshift=-1pt}] (h2) at ([yshift=1em]h1.north) {\tiny{h2 = Relu($\cdot$)}};
\node [anchor=south,draw,rounded corners,inner sep=2pt,minimum width=9.4em,minimum height=1.0em,fill=green!30!white,blur shadow={shadow xshift=1pt,shadow yshift=-1pt}] (h3) at ([yshift=1em]h2.north) {\tiny{h3 = Sum($\cdot$)}};
\node [anchor=south,draw,rounded corners,inner sep=2pt,minimum width=9.4em,minimum height=1.0em,fill=green!30!white,blur shadow={shadow xshift=1pt,shadow yshift=-1pt}] (h4) at ([yshift=1em]h3.north) {\tiny{h4 = Softmax($\cdot$)}};
\draw [-&gt;,thick] (h1.north) -- (h2.south);
\draw [-&gt;,thick] (h2.north) -- (h3.south);
\draw [-&gt;,thick] (h3.north) -- (h4.south);
\draw [-&gt;,thick,rounded corners] (h1.east) -- ([xshift=0.5em]h1.east) -- ([xshift=0.5em,yshift=0.5em]h2.north east) -- ([xshift=-2em,yshift=0.5em]h2.north east) -- ([xshift=-2em,yshift=1em]h2.north east);
}

\visible&lt;3-&gt;{
\node [anchor=south,draw,rounded corners,inner sep=2pt,minimum width=9.4em,minimum height=1.0em,fill=green!30!white,blur shadow={shadow xshift=1pt,shadow yshift=-1pt}] (slayer) at ([yshift=1em]h4.north) {\tiny{Split($\cdot$)}};
\node [anchor=south,draw,circle,inner sep=1pt,fill=red!30!white,blur shadow={shadow xshift=1pt,shadow yshift=-1pt}] (y2) at ([yshift=1em]slayer.north) {\footnotesize{$\textrm{y}_2$}};
\node [anchor=east,draw,circle,inner sep=1pt,fill=red!30!white,blur shadow={shadow xshift=1pt,shadow yshift=-1pt}] (y1) at ([xshift=-2em]y2.west) {\footnotesize{$\textrm{y}_1$}};
\node [anchor=west,draw,circle,inner sep=1pt,fill=red!30!white,blur shadow={shadow xshift=1pt,shadow yshift=-1pt}] (y3) at ([xshift=2em]y2.east) {\footnotesize{$\textrm{y}_3$}};
\draw [&lt;-,thick] (y1.south) -- ([yshift=-1em]y1.south);
\draw [&lt;-,thick] (y2.south) -- ([yshift=-1em]y2.south);
\draw [&lt;-,thick] (y3.south) -- ([yshift=-1em]y3.south);
}

\visible&lt;2-&gt;{
\draw [-&gt;,thick] (h4.north) -- (slayer.south);
}

\end{tikzpicture}
\end{center}
\end{tcolorbox}

\end{frame}


%%%------------------------------------------------------------------------------------------------------------
\subsection{����ѧϰ - ���򴫲�}

%%%------------------------------------------------------------------------------------------------------------
%%% outline: problem 3
\begin{frame}{����һ������}

\vspace{6em}
\begin{tcolorbox}[enhanced,size=normal,left=2mm,right=1mm,colback=red!5!white,colframe=red!75!black,drop fuzzy shadow]
{\Large
\textbf{��ζ�ģ���еĲ�������ѧϰ,}

\vspace{0.4em}
\textbf{֮��ʹ��ѧϰ����ģ�ͽ����ƶϣ�}
}
\end{tcolorbox}

\vspace{2em}
\begin{center}
\begin{tikzpicture}
\node [fill=blue!10] (label) at (0,0) {\LARGE{$\frac{\partial L(\textbf{w})}{\partial \textbf{w}} = $ ? }};
\end{tikzpicture}
\end{center}

\end{frame}

%%%------------------------------------------------------------------------------------------------------------
%%% ������ = ����ʽ
\begin{frame}{������ = ��������ʽ}
\begin{itemize}
\item ���е������綼���Կ����ɱ����ͺ�����ɵı���ʽ\\
\end{itemize}

\begin{center}
\begin{tikzpicture}
\node [anchor=north west] (eq1) at (0,0) {$\textbf{y} = \textbf{x} + \textbf{b}$};
\node [anchor=north west] (eq2) at (eq1.south west) {$\textbf{y} = \textrm{Relu}(\textbf{x} \cdot \textbf{w} + \textbf{b})$};
\node [anchor=north west] (eq3) at (eq2.south west) {$\textbf{y} = (\textrm{Relu}(\textbf{x} \cdot \textbf{w}_1 + \textbf{b}) + \textbf{x}) \cdot \textbf{w}_2$};
\node [anchor=north west] (eq4) at (eq3.south west) {$\textbf{y} = \textrm{Sigmoid}(\textrm{Relu}(\textbf{x} \cdot \textbf{w}_1 + \textbf{b}_1) + \textbf{x}) \cdot \textbf{w}_2 + \textbf{b}_2$};

\visible&lt;2-&gt;{
\node [anchor=north west,minimum height=1.2em,minimum width=1.2em,fill=green!30!white] (xlabel) at ([yshift=-0.5em,xshift=0.3em]eq4.south west) {};
\node [anchor=west] (xlabel2) at (xlabel.east) {������� - ���û�ָ��};
}

\begin{pgfonlayer}{background}
\visible&lt;2-&gt;{
\node [anchor=south, minimum height=1.6em,minimum width=0.8em,fill=green!30!white] (x1) at ([xshift=-1.3em]eq4.south) {};
\node [anchor=south, minimum height=1.6em,minimum width=0.8em,fill=green!30!white] (x2) at ([xshift=4.9em]eq4.south) {};
}
\end{pgfonlayer}

\visible&lt;3-&gt;{
\node [anchor=north west,minimum height=1.2em,minimum width=1.2em,fill=red!30!white] (wlabel) at ([yshift=-0.3em]xlabel.south west) {};
\node [anchor=west] (wlabel2) at (wlabel.east) {ģ�Ͳ��� - ��ô���ã�����};
}

\begin{pgfonlayer}{background}
\visible&lt;3-&gt;{
\node [anchor=south, minimum height=1.6em,minimum width=1.2em,fill=red!30!white] (w1) at ([xshift=0.2em]eq4.south) {};
\node [anchor=south, minimum height=1.6em,minimum width=1.2em,fill=red!30!white] (b1) at ([xshift=2.5em]eq4.south) {};
\node [anchor=south, minimum height=1.6em,minimum width=1.2em,fill=red!30!white] (w2) at ([xshift=6.85em]eq4.south) {};
\node [anchor=south, minimum height=1.6em,minimum width=1.2em,fill=red!30!white] (b2) at ([xshift=9.2em]eq4.south) {};
}
\end{pgfonlayer}

\end{tikzpicture}
\end{center}

\visible&lt;4-&gt;{
\begin{tcolorbox}[enhanced,size=normal,left=2mm,right=1mm,colback=blue!5!white,colframe=blue!75!black,drop fuzzy shadow]
{\Large
\textbf{�������ˣ�}

\vspace{0.4em}
\textbf{���ȷ��w��b��ʹx��y��Ӧ�ø��ã�}
}
\end{tcolorbox}
}

\end{frame}

%%%------------------------------------------------------------------------------------------------------------
%%% ѧϰ��Ŀ����ʲô
\begin{frame}{Ŀ�꺯������ʧ����}
\begin{itemize}
\item ����һ�����͵��Ż����⣬��������������\\
    \begin{enumerate}
    \item �Ż���Ŀ����ʲô��
    \item ��ε�������$\textbf{w}$��$\textbf{b}$���Ŀ�ꣿ
    \end{enumerate}
\item&lt;2-&gt; \textbf{����Ŀ��}�����ڸ���$\textbf{x}$��ʲô����$\textbf{y}$�Ǻõ�
    \begin{itemize}
    \item ���裺�����������$\{\textbf{x}_1,...,\textbf{x}_n\}$��ÿ��$\textbf{x}_i$����Ӧ\alert{��ȷ��}$\hat{\textbf{y}}_i$
    \item ����һ��������$\textbf{y}=f(\textbf{x})$��ÿ��$\textbf{x}_i$Ҳ����һ�����$\textbf{y}_i$
    \item ������Զ�����$\hat{\textbf{y}}_i$���������$\textbf{y}_i$֮���ƫ������������������С����ƫ��Ϳ��Եõ����õ�ģ��
    \end{itemize}
\end{itemize}

\visible&lt;3-&gt;{
\vspace{-0.7em}
\begin{center}
\begin{tikzpicture}
\begin{scope}[yscale=0.2,xscale=0.8]
\draw[-,very thick,ublue,domain=-4.2:3.5,samples=100] plot (\x,{ - 1/14 * (\x + 4) * (\x + 1) * (\x - 1) * (\x - 3)});
\visible&lt;4-&gt;{
\draw[-,very thick,ugreen,domain=-3.8:3.0,samples=100] plot (\x,{ - 1/14 * (4*\x*\x*\x + 3*\x*\x - 26*\x - 1)});
}
\draw[-&gt;,thick] (-6,0) -- (5,0);
\draw[-&gt;,thick] (-5,-4) -- (-5,5);

\draw [&lt;-] (-2.5,4) -- (-2,5) node [pos=1,right,inner sep=2pt] {\footnotesize{��$\hat{\textbf{y}}_i$}};
\visible&lt;4-&gt;{
\draw [&lt;-] (-3,-3) -- (-2.5,-2) node [pos=0,left,inner sep=2pt] {\footnotesize{Ԥ��$\textbf{y}_i$}};}

\visible&lt;5-&gt;{
\draw [&lt;-] (2.3,1) -- (3.3,2) node [pos=1,right,inner sep=2pt] {\footnotesize{ƫ��$|\hat{\textbf{y}}_i - \textbf{y}_i|$}};
\foreach \x in {-3.8,-3.7,...,3.0}{
    \pgfmathsetmacro{\p}{- 1/14 * (\x + 4) * (\x + 1) * (\x - 1) * (\x - 3)};
    \pgfmathsetmacro{\q}{- 1/14 * (4*\x*\x*\x + 3*\x*\x - 26*\x - 1)};
    \draw [-] (\x,\p) -- (\x, \q);
}
}

\end{scope}
\end{tikzpicture}
\end{center}
}

\vspace{-0.3em}
\begin{itemize}
\item&lt;6-&gt; ������̾���\alert{�����Ż�/ѵ��}����$\hat{\textbf{y}}_i$��$\textbf{y}_i$֮��ƫ��Ķ�������һ��\alert{��ʧ����}��Ҳ����ѵ����\alert{Ŀ�꺯��}�����Ż���Ŀ�����\textbf{��С����ʧ����}
\end{itemize}

\end{frame}

%%%------------------------------------------------------------------------------------------------------------
%%% ������Ŀ�꺯��
\begin{frame}{��������ʧ����}

\begin{itemize}
\item ��ʧ������Ϊ$Loss(\hat{\textbf{y}}_i,\textbf{y}_i)$�����Ϊ$L$�������ǳ��õĶ���
\end{itemize}

\vspace{0.5em}

\footnotesize{
\renewcommand{\arraystretch}{1.2}
\begin{tabular}{l | l | l | l}
���� &amp; ���� &amp; NiuTensorʵ��(\texttt{yh}��ʾ$\hat{\textbf{y}}_i$) &amp; Ӧ�� \\ \hline
0-1 &amp; $L = \left\{ \begin{array}{ll} 0 &amp; \hat{\textbf{y}}_i = \textbf{y}_i \\ 1 &amp; \hat{\textbf{y}}_i \ne \textbf{y}_i \end{array} \right.$ &amp; \scriptsize{\texttt{L = Sign(Absolute(yh - y))}} &amp; ��֪�� \\
Hinge &amp; $L=\max(0,1-\hat{\textbf{y}}_i \cdot \textbf{y}_i)$ &amp; \scriptsize{\texttt{L = Max(0, 1 - yh * y))}} &amp; SVM \\
����ֵ &amp; $L=|\hat{\textbf{y}}_i - \textbf{y}_i|$ &amp; \scriptsize{\texttt{L = Absolute(yh - y)}} &amp; �ع� \\
Logistic &amp; $L=\log(1 + \hat{\textbf{y}}_i \cdot \textbf{y}_i)$ &amp; \scriptsize{\texttt{L = Log(1 + yh * y)}} &amp; �ع� \\
ƽ�� &amp; $L=(\hat{\textbf{y}}_i - \textbf{y}_i)^2$ &amp; \scriptsize{\texttt{L = Power(yh - y, 2)}} &amp; �ع� \\
ָ�� &amp; $L=\exp(- \hat{\textbf{y}}_i \cdot \textbf{y}_i) $ &amp; \scriptsize{\texttt{L = Exp(Negate(yh * y))}} &amp; \scriptsize{AdaBoost} \\
������ &amp; $L=-\sum_k \textbf{y}_i^{[k]} \log \hat{\textbf{y}}_i^{[k]} $ &amp; \scriptsize{\texttt{L = CrossEntropy(y, yh)}} &amp; ����� \\
       &amp; \scriptsize{$\textbf{y}_i^{[k]}$: $\textbf{y}_i$�ĵ�$k$ά} &amp; &amp; \\
\end{tabular}
\renewcommand{\arraystretch}{1.0}
}

\vspace{-0.5em}
\begin{itemize}
\item ע�⣺
    \begin{itemize}
    \item ��ʧ�������Ը������ⲻͬ����ѡ��û�й̶�Ҫ��
    \item ��Щ��ʧ���������������Լ�������罻����Ҫ��$\hat{\textbf{y}}_i$��$\textbf{y}_i$���Ǹ��ʷֲ�
    \end{itemize}
\end{itemize}

\end{frame}

%%%------------------------------------------------------------------------------------------------------------
%%% �Ż�Ŀ�꺯��
\begin{frame}{�����Ż�}

\begin{itemize}
\item ���ڵ�$i$����������($\textbf{x}_i$,$\hat{\textbf{y}}_i$)���������ʧ������������$\textbf{w}$�ĺ�������$\textbf{b}$Ҳ��Ϊһ��$\textbf{w}$������Ϊ$L(\textbf{x}_i,\hat{\textbf{y}}_i;\textbf{w})$�������ѧϰ���Ա�����Ϊ��\\

\begin{displaymath}
\textbf{w}^* = \argmin_{\textbf{w}} \frac{1}{n} \sum_{i=1}^{n} L(\textbf{x}_i,\hat{\textbf{y}}_i;\textbf{w})
\end{displaymath}

$\textbf{w}^*$��ʾ��ѵ������ʹ����ʧ��ƽ��ֵ�ﵽ��С�Ĳ�����$\frac{1}{n} \sum_{i=1}^{n} L(\textbf{x}_i,\hat{\textbf{y}}_i;\textbf{w})$���������ۺ���(cost function)��������ʧ������ֵ�����Ĺ��ơ�

\vspace{0.5em}

\item&lt;2-&gt; �������⣺\textbf{���$\argmin$�����ҵ����ۺ�����Сֵ��}
    \begin{itemize}
    \item ���Ƿǳ����������⣬����һ�µ����µ�IBMģ�ͣ���ʱʹ�õ���EM�㷨
    \item �������ﲢ����һ������ģ��
    \item ��Ҫһ�ָ���ͨ�õ���ⷽ��
    \end{itemize}

\end{itemize}

\end{frame}

%%%------------------------------------------------------------------------------------------------------------
%%% �ݶ��½�
\begin{frame}{�ݶ��½���Gradient Descent��}

\begin{itemize}
\item �����Ŀ�꺯�������ǲ���$\textbf{w}$�ĺ�������Ϊ$J(\textbf{w})$���Ż�Ŀ���ǣ��ҵ�ʹ$J(\textbf{w})$�ﵽ��С��$\textbf{w}$
\item ע�⣬$\textbf{w}$���ܰ������ڸ�ʵ����������SMT��MERT֮��ĵ��η�����������Կ���һ�ָ����ʺϴ���ʵ���������Ż������������˼����\alert{�ݶ��½�}��
    \begin{itemize}
    \item&lt;2-&gt; ���$J(\textbf{w})$����$\textbf{w}$��΢�֣�$\frac{\partial J(\textbf{w})}{\partial \textbf{w}}$��ʾ$J$��$\textbf{w}$���仯���ķ���
    \item&lt;2-&gt; $\textbf{w}$�����ݶȷ�����£��µ�$\textbf{w}$����ʹ�������ӽ���ֵ
    \end{itemize}
\end{itemize}

\pgfplotsset{%
  colormap={whitered}{color(-1cm)=(orange!75!red);color(1cm)=(white)}
}


\begin{center}
\begin{tikzpicture}[
  declare function = {mu1=1;},
  declare function = {mu2=2;},
  declare function = {sigma1=0.5;},
  declare function = {sigma2=1;},
  declare function = {normal(\m,\s)=1/(2*\s*sqrt(pi))*exp(-(x-\m)^2/(2*\s^2));},
  declare function = {bivar(\ma,\sa,\mb,\sb)=1/(2*pi*\sa*\sb) * exp(-((x-\ma)^2/\sa^2 + (y-\mb)^2/\sb^2))/2;}]
  \footnotesize{
  \visible&lt;2-&gt;{
  \begin{scope}
  \begin{axis}[
    colormap name  = whitered,
    width          = 8cm,
    height         = 5cm,
    view           = {20}{45},
    enlargelimits  = false,
    grid           = major,
    domain         = -1:3,
    y domain       = 0:4,
    samples        = 30,
    xlabel         = $\textbf{w}^{[1]}$,
    ylabel         = $\textbf{w}^{[2]}$,
    xlabel style   = {xshift=0em,yshift=0.8em},
    ylabel style   = {xshift=0.2em,yshift=0.8em},
    zlabel         = {$J(\textbf{w})$},
    ztick          = {-0.1},
    colorbar,
    colorbar style = {
      at     = {(1.2,0.5)},
      anchor = north west,
      ytick  = {0,-0.1},
      height = 0.25*\pgfkeysvalueof{/pgfplots/parent axis height},
      title  = {}
    }
  ]

    \addplot3 [surf] {-bivar(mu1,sigma1,mu2,sigma2)};

    \node [circle,fill=red,minimum size=3pt,inner sep=1.5pt] () at (axis cs:0.5,2,-0.01) {};

    \draw [-&gt;,very thick,ublue] (axis cs:0.5,2,-0.01) -- (axis cs:0.8,1.6,-0.03) node [pos=1,right,inner sep=2pt] {\tiny{-$\frac{\partial J(\textbf{w})}{\partial \textbf{w}}$}};
    \draw [-&gt;,very thick,dotted] (axis cs:0.5,2,-0.01) -- (axis cs:0.2,1.5,-0.03);
    \draw [-&gt;,very thick,dotted] (axis cs:0.5,2,-0.01) -- (axis cs:0.2,3.5,-0.03);
    %\draw [black!50] (axis cs:0,-1,0) -- (axis cs:0,4,0);

  \end{axis}
  \end{scope}
  }
  }
\end{tikzpicture}
\end{center}

\end{frame}

%%%------------------------------------------------------------------------------------------------------------
%%% �ݶ��½��ı���
\begin{frame}{�ݶ��½��Ĳ�ͬʵ�ַ�ʽ}

\begin{itemize}
\item \textbf{�ݶ��½�}�����ǿ��������ݶȷ������$\textbf{w}$һС����֮��õ����õ�$\textbf{w}$��֮�����¼����ݶȣ������ظ���������

\begin{displaymath}
\textbf{w}_{t+1} = \textbf{w}_t - \alpha \cdot \frac{\partial J(\textbf{w}_t)}{\partial \textbf{w}_t}
\end{displaymath}

����$t$��ʾ���µĲ�����$\alpha$��һ����������ʾ���²����Ĵ�С��$\alpha$��������Ҫ����������е�������$J(\textbf{w}_t)$����ʽ�����˾�����㷨�����ʵ�֡�

\item&lt;2-&gt; \textbf{�����ݶ��½�(Batch Gradient Descent)}��

\begin{displaymath}
J(\textbf{w}_t) = \frac{1}{n} \sum_{i=1}^{n} L(\textbf{x}_i,\hat{\textbf{y}}_i;\textbf{w}_t)
\end{displaymath}

���ַ���ѵ���ȶ�����������ÿ�θ�����Ҫ������ѵ���������б�����Ч�ʵͣ�����$n$�ܴ󣩣����ģ�����Ϻ���ʹ��

\end{itemize}

\end{frame}

%%%------------------------------------------------------------------------------------------------------------
%%% �ݶ��½��ı���
\begin{frame}{�ݶ��½��Ĳ�ͬʵ�ַ�ʽ(��)}

\begin{itemize}
\item \textbf{����ݶ��½�(Stochastic Gradient Descent)}��

\begin{displaymath}
J(\textbf{w}_t) = L(\textbf{x}_i,\hat{\textbf{y}}_i;\textbf{w}_t)
\end{displaymath}

����������SGD�����л���ѧϰ�Ŀγ��Ｘ�����н��ܡ�ÿ�����ѡȡһ�����������ݶȼ���Ͳ������£����µļ�����۵ͣ�������������������������������ѧϰ(online learning)����������������


\vspace{0.3em}

\item&lt;2-&gt; \textbf{С�����ݶ��½�(Mini-batch Gradient Descent)}��

\begin{displaymath}
J(\textbf{w}_t) = \frac{1}{m} \sum_{i=j}^{j+m} L(\textbf{x}_i,\hat{\textbf{y}}_i;\textbf{w}_t)
\end{displaymath}

ÿ�����ʹ�������������в�������(���������ر��)������һ�����з�����������õķ���֮һ

\end{itemize}

\end{frame}

%%%------------------------------------------------------------------------------------------------------------
%%% �����ݶ��½��ĸĽ�
\begin{frame}{һЩ�Ľ�}

\begin{itemize}
\item \textbf{���ֺ͸Ľ�}����߻����ݶȵķ����������ٶȡ�ѵ���ȶ��Եȣ�����googleһ��
    \begin{itemize}
    \item Momentum, Adagrad, Adadelta, RMSprop, Adam, AdaMax, Nadam, AMSGrad�ȵ�
    \item \footnotesize{\url{http://ruder.io/optimizing-gradient-descent}}
    \end{itemize}
\item&lt;2-&gt; \textbf{���л�}�����ģ���ݴ�����Ҫ�ֲ�ʽ���㣬�ݶȸ��µĲ�����Ҫ���
    \begin{itemize}
    \item \textbf{ͬ������}�����м���ڵ���ɼ����ͳһ���ܲ����²�����Ч���ȶ������ǲ��жȵ�
    \item \textbf{�첽����}��ÿ���ڵ������ʱ���¡����жȸߣ��������ڽڵ��������ܲ�ͬ����������ʮ���ȶ�
    \end{itemize}
\item&lt;3-&gt; \textbf{����}
    \begin{itemize}
    \item ��������ݶ���ʧ�ͱ�ը�����⣬ʹ���ݶȲü����в����ӵ�
    \item �����������ӣ����Զ��ⲿ֪ʶ��ģ����������������ѵ�����ȶ�
    \end{itemize}
\end{itemize}

\end{frame}

%%%------------------------------------------------------------------------------------------------------------
%%% ��μ����ݶ�
\begin{frame}{��μ����ݶ�? - ��ֵ΢��}

\begin{itemize}
\item \textbf{����һ����������}����μ����ݶ�
\begin{displaymath}
\frac{\partial L(\textbf{w})}{\partial \textbf{w}} = ?
\end{displaymath}

\vspace{0.5em}

\item&lt;2-&gt; \textbf{��ֵ΢��} - �򵥴ֱ��ķ���
\begin{displaymath}
\frac{\partial L(\textbf{w})}{\partial \textbf{w}} = \lim_{\Delta \textbf{w} \to 0} \frac{L(\textbf{w} + \Delta \textbf{w}) - L(\textbf{w} - \Delta \textbf{w}) }{2\Delta \textbf{w}}
\end{displaymath}

�������΢�ֹ�ʽ�����ǿ��Խ�$\textbf{w}$�仯һ�������$\Delta \textbf{w}$��ʾ����֮��$L(\cdot)$�ı仯

    \begin{itemize}
    \item&lt;3-&gt; \textbf{�ŵ������}��������ķǳ��򵥣�����ʵ��
    \item&lt;3-&gt; \textbf{ȱ��Ҳ������}��Ч��̫�ͣ����ڸ������硢��������΢��һЩ��ģ�ͻ������޷�ʹ��
    \end{itemize}

\end{itemize}

\end{frame}

%%%------------------------------------------------------------------------------------------------------------
%%% ��μ����ݶ� - ����΢��
\begin{frame}{��μ����ݶ�? - ����΢��}

\begin{itemize}
\item \textbf{����΢��}����������д��΢�ֱ���ʽ�������������ֵ���õ�΢�ֽ�������磬�������±���ʽ
\begin{displaymath}
L(\textbf{w}) = \textbf{x} \cdot \textbf{w} + 2 \textbf{w}^2
\end{displaymath}

\visible&lt;2-&gt;{
\vspace{0.5em}
�����ֶ��Ƶ���΢�ֱ���ʽ

\begin{displaymath}
\frac{\partial L(\textbf{w})}{\partial \textbf{w}} = \textbf{x} + 4 \textbf{w}
\end{displaymath}
}

\visible&lt;3-&gt;{
\vspace{0.5em}
��󣬴���$\textbf{x} = \begin{pmatrix} 2 \\ -3 \end{pmatrix}$��$\textbf{w} = \begin{pmatrix} -1 \\ 1 \end{pmatrix}$���õ�΢�ֽ��\\

\vspace{1em}

\begin{displaymath}
\frac{\partial L(\textbf{w})}{\partial \textbf{w}} =  \begin{pmatrix} 2 \\ -3 \end{pmatrix} + 4 \begin{pmatrix} -1 \\ 1 \end{pmatrix} = \begin{pmatrix} -2 \\ 1 \end{pmatrix}
\end{displaymath}
}

\end{itemize}

\end{frame}

%%%------------------------------------------------------------------------------------------------------------
%%% ����΢�ֵ���������
\begin{frame}{����΢�ֵ���������}

\begin{itemize}
\item \textbf{Expression Swell}����㺯����΢�ֱ���ʽ��ǳ�����
	\begin{itemize}
	\item ����ʽ�߳����״洢�͹���
	\item ������Ҫ����΢�ֵ�\alert{���ֵ}��������΢�ֱ���ʽ
	\end{itemize}
\end{itemize}

\vspace{0.5em}

{\small
\begin{tabular} {l | l | l}
���� &amp; ΢�ֱ���ʽ &amp; �����΢�ֱ���ʽ \\ \hline
$x$ &amp; $1$ &amp; $1$ \\ \hline
$x(x+1)$ &amp; $(x+1)+x$ &amp; $2x + 1$ \\ \hline
$x(x+1)$ &amp; $(x+1)(x^2+x+1)$ &amp; $4x^3+6x^2$ \\
$(x^2+x+1)$ &amp; $+x(x^2+x+1)$ &amp; $+4x+1$ \\
                     &amp; $+x(x+1)(2x+1)$ &amp; \\ \hline
$(x^2+x)$ &amp; $(2x+1)(x^2+x+1)$ &amp; $8x^7+28x^6$ \\
$(x^2+x+1)$ &amp; $(x^4+2x^3+2x^2+x+1)$ &amp; $+48x^5+50x^4$ \\
$(x^4+2x^3$ &amp; $+(2x+1)(x^2+x)$ &amp; $+36x^3+18x^2$ \\
$+2x^2+x+1)$ &amp; \ \ $(x^4+2x^3+2x^2+x+1)$ &amp; $+6x+1$ \\
 &amp; $+(x^2+x)(x^2+x+1)$ &amp; \\
 &amp; \ \ $(4x^3+6x^2+4x+1)$ &amp; \\


\end{tabular}
}

\end{frame}

%%%------------------------------------------------------------------------------------------------------------
%%% �Զ�΢��
\begin{frame}{��μ����ݶ�? - �Զ�΢��}

\begin{itemize}
\item \textbf{�Զ�΢��}�����ӵ�΢�ֱ�ɼ򵥵Ĳ��裬��Щ������ȫ�Զ������������׽��д洢�����㡣�������һ�ַ���ģʽ����������Ҳ����\alert{���򴫲�}˼�룩����������
	\begin{enumerate}
	\item \textbf{ǰ�����}��������������룬������ÿ����������ֵ����Ҳ��������ı�׼ʹ�÷�ʽ
	\item \textbf{�������}��������������������������ÿ���������루���������Ӧ��΢��
	\end{enumerate}
\end{itemize}

\visible&lt;2-&gt;{
\vspace{-1em}
\begin{center}
\begin{tikzpicture}
\begin{scope}
\tikzstyle{layernode} = [draw,thick,fill=ugreen!30!white,blur shadow={shadow xshift=1pt,shadow yshift=-1pt}];

\node [anchor=center,layernode,minimum height=4em,minimum width=1em] (layer01) at (0,0) {};
\node [anchor=north west,layernode,minimum height=3em,minimum width=1em] (layer02) at ([xshift=3em]layer01.north east) {};
\node [anchor=south west,layernode,minimum height=3em,minimum width=1em] (layer03) at ([xshift=7em]layer01.south east) {};
\node [anchor=south west,layernode,minimum height=4em,minimum width=1em] (layer04) at ([xshift=11em]layer01.south east) {};
\node [anchor=south west,layernode,minimum height=4em,minimum width=1em] (layer05) at ([xshift=3em]layer04.south east) {};

\node [anchor=east] (input) at ([xshift=-1em]layer01.west){\scriptsize{����}};
\node [anchor=west] (output) at ([xshift=1em]layer05.east){\scriptsize{���}};

\draw [-&gt;] ([xshift=-1em]layer01.west) -- ([xshift=-0.1em]layer01.west);
\draw [-&gt;] ([xshift=0.1em,yshift=-0.5em]layer01.north east) -- ([xshift=-0.1em,yshift=-0.5em]layer02.north west);
\draw [-&gt;] ([xshift=0.1em,yshift=0.5em]layer01.south east) -- ([xshift=-0.1em,yshift=0.5em]layer03.south west);
\draw [-&gt;] ([xshift=0.1em,yshift=-0.5em]layer02.north east) -- ([xshift=-0.1em,yshift=-0.5em]layer04.north west);
\draw [-&gt;] ([xshift=0.1em,yshift=0.5em]layer03.south east) -- ([xshift=-0.1em,yshift=0.5em]layer04.south west);
\draw [-&gt;] ([xshift=0.1em]layer04.east) -- ([xshift=-0.1em]layer05.west);
\draw [-&gt;] ([xshift=0.1em]layer05.east) -- ([xshift=1.0em]layer05.east);

\visible&lt;3-&gt;{
\draw [-&gt;,very thick,ublue] ([xshift=-1em]layer01.west) -- ([xshift=-0.1em]layer01.west);
}
\visible&lt;4-&gt;{
\draw [-&gt;,very thick,ublue] ([xshift=0.1em,yshift=-0.5em]layer01.north east) -- ([xshift=-0.1em,yshift=-0.5em]layer02.north west);
}
\visible&lt;5-&gt;{
\draw [-&gt;,very thick,ublue] ([xshift=0.1em,yshift=0.5em]layer01.south east) -- ([xshift=-0.1em,yshift=0.5em]layer03.south west);
}
\visible&lt;6-&gt;{
\draw [-&gt;,very thick,ublue] ([xshift=0.1em,yshift=-0.5em]layer02.north east) -- ([xshift=-0.1em,yshift=-0.5em]layer04.north west);
\draw [-&gt;,very thick,ublue] ([xshift=0.1em,yshift=0.5em]layer03.south east) -- ([xshift=-0.1em,yshift=0.5em]layer04.south west);
\draw [-&gt;,very thick,ublue] ([xshift=0.1em]layer04.east) -- ([xshift=-0.1em]layer05.west);
\draw [-&gt;,very thick,ublue] ([xshift=0.1em]layer05.east) -- ([xshift=1.0em]layer05.east);
}

\visible&lt;8-&gt;{
\draw [&lt;-,very thick,red] ([xshift=-1em,yshift=-0.3em]layer01.west) -- ([xshift=-0.1em,yshift=-0.3em]layer01.west);
\draw [&lt;-,very thick,red] ([xshift=0.1em,yshift=-0.8em]layer01.north east) -- ([xshift=-0.1em,yshift=-0.8em]layer02.north west);
\draw [&lt;-,very thick,red] ([xshift=0.1em,yshift=0.2em]layer01.south east) -- ([xshift=-0.1em,yshift=0.2em]layer03.south west);
\draw [&lt;-,very thick,red] ([xshift=0.1em,yshift=-0.8em]layer02.north east) -- ([xshift=-0.1em,yshift=-0.8em]layer04.north west);
\draw [&lt;-,very thick,red] ([xshift=0.1em,yshift=0.2em]layer03.south east) -- ([xshift=-0.1em,yshift=0.2em]layer04.south west);
\draw [&lt;-,very thick,red] ([xshift=0.1em,yshift=-0.3em]layer04.east) -- ([xshift=-0.1em,yshift=-0.3em]layer05.west);
\draw [&lt;-,very thick,red] ([xshift=0.1em,yshift=-0.3em]layer05.east) -- ([xshift=1.0em,yshift=-0.3em]layer05.east);
}

\visible&lt;7-&gt;{
\draw [&lt;-,thin] ([xshift=0.3em,yshift=0.3em]layer04.east) .. controls +(35:1) and +(215:1) .. ([xshift=-2em,yshift=0.3em]layer05.north west) node [pos=1,above] {\scriptsize{ǰ�򣺲�$i$ �����$h_{i}$}};
}
\visible&lt;9-&gt;{
\draw [&lt;-,thin] ([xshift=0.3em,yshift=-0.7em]layer04.east) .. controls +(-35:1) and +(145:1) .. ([xshift=-2em,yshift=-0.3em]layer05.south west) node [pos=1,below] {\scriptsize{����$h_{i}$ �����ݶ�$\frac{\partial L}{\partial h_i}$}};
}

\end{scope}
\end{tikzpicture}
\end{center}
}

\vspace{-1em}
\begin{itemize}
\item&lt;10-&gt; �Զ�΢�ֿ�����\alert{����ͼ}ʵ��(TensorFlow�� NiuTensor ��)����������ͼ�����˿γ̵ķ�Χ������������ѧϰ
\end{itemize}

\end{frame}

%%%------------------------------------------------------------------------------------------------------------
%%% ���򴫲� - ����˵��
\begin{frame}{����˵��}

\begin{itemize}
\item ��һ��$K$��������Ϊ��������ȷһ�·���
    \begin{itemize}
    \item �������ÿ���������ж�����ƫ�������$\textbf{b}$��
    \end{itemize}
\end{itemize}

\vspace{-1em}
\begin{center}
\begin{tikzpicture}
\begin{scope}

\def\neuronsep{1}
\tikzstyle{neuronnode} = [minimum size=1.2em,circle,draw,ublue,very thick,inner sep=1pt, fill=white,align=center,drop shadow={shadow xshift=0.1em,shadow yshift=-0.1em}];

%%% layer 1
\foreach \n in {1,...,4}{
    \node [neuronnode] (neuron0\n) at (\n * \neuronsep,0) {};
    \draw [-&gt;] ([yshift=-0.8em]neuron0\n.south) -- ([yshift=-0.1em]neuron0\n.south) node [pos=0,below] {\tiny{...}};
}


\begin{pgfonlayer}{background}
\node [rectangle,inner sep=0.2em,fill=red!20] [fit = (neuron01) (neuron04)] (layer01) {};
\node [anchor=east] (layer01label) at (layer01.west) {\scriptsize{��$l-1$}};
\end{pgfonlayer}

%%% layer 2
\foreach \n in {1,...,4}{
    \node [neuronnode] (neuron1\n) at (\n * \neuronsep,3em) {};
}

\foreach \n in {1,...,4}{
    \foreach \m in {1,...,4}{
        \draw [&lt;-] (neuron1\n.south) -- (neuron0\m.north);
    }
}

\begin{pgfonlayer}{background}
\node [rectangle,inner sep=0.2em,fill=ugreen!20] [fit = (neuron11) (neuron14)] (layer02) {};
\node [anchor=east] (layer02label) at (layer02.west) {\scriptsize{��$l$}};
\end{pgfonlayer}

%%% layer 3
\foreach \n in {1,...,4}{
    \node [neuronnode] (neuron2\n) at (\n * \neuronsep,6em) {};
    \draw [&lt;-] ([yshift=0.8em]neuron2\n.north) -- ([yshift=0.0em]neuron2\n.north) node [pos=0,above] {\tiny{...}};
}

\foreach \n in {1,...,4}{
    \foreach \m in {1,...,4}{
        \draw [&lt;-] (neuron2\n.south) -- (neuron1\m.north);
    }
}

\begin{pgfonlayer}{background}
\node [rectangle,inner sep=0.2em,fill=blue!20] [fit = (neuron21) (neuron24)] (layer03) {};
\node [anchor=east] (layer03label) at (layer03.west) {\scriptsize{��$l+1$}};
\end{pgfonlayer}

%%% output layer
\foreach \n in {1,...,4}{
    \node [neuronnode] (neuron3\n) at (\n * \neuronsep,9.4em) {};
    \visible&lt;1-3,5-&gt;{
    \draw [&lt;-] ([yshift=0.6em]neuron3\n.north) -- ([yshift=0.0em]neuron3\n.north) node [pos=0,above] {\tiny{output}};
    }
    \visible&lt;4&gt;{
    \draw [&lt;-,red,very thick] ([yshift=0.6em]neuron3\n.north) -- ([yshift=0.0em]neuron3\n.north) node [pos=0,above] {\tiny{output}};
    }
    \draw [-&gt;] ([yshift=-0.6em]neuron3\n.south) -- ([yshift=0.0em]neuron3\n.south);
}

\begin{pgfonlayer}{background}
\node [rectangle,inner sep=0.2em,fill=ugreen!20] [fit = (neuron31) (neuron34)] (layer04) {};
\node [anchor=east] (layer04label) at (layer04.west) {\scriptsize{��$L$(���)}};
\end{pgfonlayer}

\visible&lt;2-&gt;{
\node [neuronnode,draw=red,fill=red!20!white,inner sep=1pt] (neuron12new) at (2 * \neuronsep,3em) {};
\node [anchor=east] (neuronsamplelabel) at ([yshift=-1em]layer02label.south east) {\alert{\textbf{\tiny{��$l$��, ��$i$����Ԫ}}}};
\draw [-&gt;,dashed,very thick,red] ([xshift=-0.2em,yshift=0.2em]neuronsamplelabel.east) .. controls +(30:1) and +(220:1) .. ([xshift=-0em,yshift=-0em]neuron12new.210);
}

\visible&lt;3&gt;{
\foreach \n in {1,...,4}{
\draw [&lt;-,thick,red] (neuron2\n.south) -- (neuron12.north);
}
}

\visible&lt;5-&gt;{
\draw [&lt;-,thick,red] (neuron14.south) -- (neuron04.north);
\node [anchor=north] (wlabel) at (layer02.south east) {\alert{\scriptsize{$w_{4,4}^{l}$}}};
}

\visible&lt;3-&gt;{
\node [anchor=west,align=left] (line01) at ([xshift=1em,yshift=1em]layer04.east) {\footnotesize{$h_{i}^{k}$����$l$��, ��$i$����Ԫ�����}};
\node [anchor=north west,align=left] (line02) at (line01.south west) {\footnotesize{$\textbf{h}^{k}$����$k$������}};
\node [anchor=north west,align=left] (line03) at (line02.south west) {\footnotesize{$\textbf{s}^{k}$����$k$������Ա任$\textbf{s}^k=\textbf{h}^{k-1}\textbf{w}^k$}};
\node [anchor=north west,align=left] (line04) at (line03.south west) {\footnotesize{$f^{k}$����$k$��ļ����$\textbf{h}^k=f^l(\textbf{s}^k)$}};
}
\visible&lt;4-&gt;{
\node [anchor=north west,align=left] (line05) at (line04.south west) {\footnotesize{$\textbf{h}^{K}$�������������}};
}
\visible&lt;5-&gt;{
\node [anchor=north west,align=left] (line06) at (line05.south west) {\footnotesize{$w_{j,i}^{k}$����$k-1$����Ԫ$j$��}\\\footnotesize{��$k$����Ԫ$i$������Ȩ��}};
\node [anchor=north west,align=left] (line07) at (line06.south west) {\footnotesize{$\textbf{w}^{k}$����$k-1$�����$k1$���}\\\footnotesize{����Ȩ��}};
}

\end{scope}
\end{tikzpicture}
\end{center}

\vspace{-1.5em}

\visible&lt;6-&gt;{
\begin{displaymath} \textrm{���ڵ�}k\textrm{��}: \textbf{h}^k = f^k(\textbf{s}^k) = f^k(\sum_j h_{j}^{k-1}w_{j,i}^k) = f^k(\textbf{h}^{k-1} \textbf{w}^k) \end{displaymath}
}

\end{frame}

%%%------------------------------------------------------------------------------------------------------------
%%% �����ķ��򴫲�
\begin{frame}{���򴫲� - �����}

\begin{itemize}
\item �����(�����׶�)
\end{itemize}

\vspace{-1.0em}
\begin{center}
\begin{tikzpicture}
\begin{scope}
\node [anchor=center,minimum height=1.7em,fill=yellow!20,draw] (h) at (0,0) {$\textbf{h}^{K-1}$};
\node [anchor=west,minimum height=1.7em,fill=blue!20,draw] (s) at ([xshift=5.5em]h.east) {$\textbf{s}^{K}$};
\node [anchor=west,minimum height=1.7em,fill=green!20,draw] (h2) at ([xshift=5.5em]s.east) {$\textbf{h}^{K}$};
\node [anchor=west,minimum height=1.7em,fill=orange!20,draw] (l) at ([xshift=5.5em]h2.east) {$L$};
\draw [-&gt;] (h.east) -- (s.west);
\draw [-&gt;] (s.east) -- (h2.west);
\draw [-&gt;] (h2.east) -- (l.west) node [pos=0.5,above] {\tiny{��ʧ}};

\node [anchor=south west,inner sep=2pt] (step100) at ([xshift=0.5em,yshift=-0.8em]h.north east) {\tiny{$\textbf{s}^K = \textbf{h}^{K-1} \textbf{w}^K$}};
\node [anchor=south west,inner sep=2pt] (step101) at (step100.north west) {\tiny{���Ա任}};

\node [anchor=south west,inner sep=2pt] (step200) at ([xshift=0.5em,yshift=-0.8em]s.north east) {\tiny{$\textbf{h}^K = f^K(\textbf{s}^K)$}};
\node [anchor=south west,inner sep=2pt] (step201) at (step200.north west) {\tiny{�����}};

\node [anchor=south,inner sep=1pt] (outputlabel) at ([yshift=0.0em]h2.north) {\tiny{\textbf{�����}}};

\visible&lt;2-&gt;{
\draw[decorate,thick,decoration={brace,mirror,raise=0.4em,amplitude=2mm}] (h.south west) -- (s.south west) node [pos=0.5,below,yshift=-1em] {\scriptsize{\textbf{��һ�׶Σ����Ա任}}};
}
\visible&lt;3-&gt;{
\draw[decorate,thick,decoration={brace,mirror,raise=0.4em,amplitude=2mm}] ([xshift=0.2em]s.south west) -- (l.south east) node [pos=0.5,below,yshift=-1em] (step2) {\scriptsize{\textbf{�ڶ��׶Σ������+��ʧ����}}};
}

\visible&lt;4-&gt;{
\draw [-&gt;,very thick,red] ([yshift=1em,xshift=-0.1em]l.north) -- ([yshift=1em,xshift=0.1em]s.north) node [pos=0.5,above] {\tiny{�������ݶ�\alert{$\frac{\partial L}{\partial \textbf{s}^K} = ?$}}};
\draw [-,very thick,red] ([yshift=0.5em]l.north) -- ([yshift=1.5em]l.north);
\draw [-,very thick,red] ([yshift=0.5em]s.north) -- ([yshift=1.5em]s.north);
}

\end{scope}

\end{tikzpicture}
\end{center}

\begin{itemize}
\item&lt;4-&gt; ���򴫲�����������봫���ݶȣ���������ȿ��ǽ׶ζ�\visible&lt;5-&gt;{����$\pi^k = \frac{\partial L}{\partial \textbf{s}^k}$��ʾ��ʧ$L$�ڵ�$k$�㼤������봦���ݶȣ�������ʽ����}

\vspace{-1.5em}
\visible&lt;5-&gt;{
\begin{eqnarray}
\pi^K &amp; = &amp; \frac{\partial L}{\partial \textbf{s}^K} \nonumber \\
         &amp; = &amp; \frac{\partial L}{\partial \textbf{h}^K} \cdot \frac{\partial \textbf{h}^K}{\partial \textbf{s}^K}  \nonumber \\
         &amp; = &amp; \frac{\partial L}{\partial \textbf{h}^K} \cdot \frac{\partial f^K(\textbf{s}^K)}{\partial \textbf{s}^K}  \nonumber
%\frac{\partial L}{\partial \textbf{w}^K} &amp; = &amp; \frac{\partial L}{\partial \textbf{h}^K} \cdot \frac{\partial \textbf{h}^K}{\partial \textbf{w}^K} \nonumber \\
%                                                         &amp; \visible&lt;4-&gt;{=} &amp; \visible&lt;4-&gt;{\frac{\partial L}{\partial \textbf{h}^K} \cdot \frac{\partial f^K(\textbf{h}^{K-1} \textbf{w}^K)}{\partial \textbf{w}^K} \ \ \ \ (\textrm{��Ϊ}\textbf{h}^K=f^K(\textbf{h}^{K-1} \textbf{w}^K))} \nonumber \\
%                                                         &amp; \visible&lt;5-&gt;{=} &amp; \visible&lt;5-&gt;{\frac{\partial L}{\partial \textbf{h}^K} \cdot \frac{\partial f^K(\textbf{s}^K)}{\partial \textbf{s}^K} \cdot \frac{\partial \textbf{s}^K}{\partial \textbf{w}^K}  \ \ \ (\textrm{��Ϊ}\textbf{s}^K=\textbf{h}^{K-1} \textbf{w}^K)} \nonumber
\end{eqnarray}
}

\end{itemize}

\end{frame}

%%%------------------------------------------------------------------------------------------------------------
%%% �����ķ��򴫲� - �������ӵ�����
\begin{frame}{���򴫲� - �����($\textbf{s}^K$�����ݶ�)}
\begin{center}
\begin{tikzpicture}

\begin{scope}
\node [anchor=center] (factor00) at (0,0) {${\displaystyle \pi^K \ = }$};
\node [anchor=west] (factor01) at (factor00.east) {${\displaystyle \frac{\partial L}{\partial \textbf{h}^K}}$};
\node [anchor=west,inner sep=1pt] (factor02) at (factor01.east) {${\displaystyle \cdot}$};
\node [anchor=west] (factor03) at (factor02.east) {${\displaystyle \frac{\partial f^K(\textbf{s}^K)}{\partial \textbf{s}^K}}$};

\begin{pgfonlayer}{background}
\visible&lt;2-4&gt;{
\node [rectangle,inner sep=0em,fill=red!20] [fit = (factor01)] (p1) {};
}
\visible&lt;3-4&gt;{
\node [rectangle,inner sep=0em,fill=blue!20] [fit = (factor03)] (p2) {};
}
\visible&lt;5-&gt;{
\node [circle,inner sep=0em,fill=green!20] [fit = (factor02)] (p3) {};
}
\end{pgfonlayer}

\end{scope}
\end{tikzpicture}
\end{center}

\begin{itemize}
\item&lt;2-&gt; \raisebox{-0.7em}{\tikz{\node [anchor=west,fill=red!20] (factor01) at (factor00.east) {$\frac{\partial L}{\partial \textbf{h}^K}$};}} ��ʾ��ʧ$L$�����������ı仯�ʣ����磬����$L = \frac{1}{2} ||\hat{\textbf{y}} - \textbf{h}^K||^2$����$\frac{\partial L}{\partial \textbf{h}^K} = \hat{\textbf{y}} - \textbf{h}^K$
\item&lt;3-&gt; \raisebox{-0.7em}{\tikz{\node [anchor=west,fill=blue!20] (factor01) at (factor00.east) {$\frac{\partial f^K(\textbf{s}^K)}{\partial \textbf{s}^K}$};}} ��ʾ�������������Լ�������ı仯�ʣ����磬����$f(\textbf{s}) = \frac{1}{1+\exp(-\textbf{s})}$����$\frac{\partial f(\textbf{s})}{\partial \textbf{s}} = f(\textbf{s})(1-f(\textbf{s}))$
\item&lt;4-&gt; ����������ֱ������$s^K$�����ݶ��൱������ʧ����΢��($\frac{\partial L}{\partial \textbf{h}^K}$)�ͼ����΢��($\frac{\partial f^K(\textbf{s}^K)}{\partial \textbf{s}^K}$) �ĳ˻�\visible&lt;5-&gt;{��ע���������в������ǵ�Ԫ����������������Ԫ�˷�}

\end{itemize}

\visible&lt;4-&gt;{
\vspace{-0.5em}
\begin{center}
\begin{tikzpicture}
\begin{scope}
\node [anchor=west,minimum height=1.7em,fill=blue!20,draw] (s) at (0,0) {$\textbf{s}^{K}$};
\node [anchor=west,minimum height=1.7em,fill=green!20,draw] (h2) at ([xshift=5.5em]s.east) {$\textbf{h}^{K}$};
\node [anchor=west,minimum height=1.7em,fill=orange!20,draw] (l) at ([xshift=5.5em]h2.east) {$L$};
\draw [-&gt;] (s.east) -- (h2.west);
\draw [-&gt;] (h2.east) -- (l.west);

\draw [-&gt;,very thick,red] ([yshift=1em,xshift=-0.1em]l.north) -- ([yshift=1em,xshift=0.1em]h2.north) node [pos=0.5,above] {\tiny{���ݶ�\alert{$\frac{\partial L}{\partial \textbf{h}^K} = ?$}}};
\draw [-&gt;,very thick,red] ([yshift=1em,xshift=-0.1em]h2.north) -- ([yshift=1em,xshift=0.1em]s.north) node [pos=0.5,above] {\tiny{���ݶ�\alert{$\frac{\partial f^K(\textbf{s}^K)}{\partial \textbf{s}^K} = ?$}}};
\draw [-,very thick,red] ([yshift=0.5em]l.north) -- ([yshift=1.5em]l.north);
\draw [-,very thick,red] ([yshift=0.5em]h2.north) -- ([yshift=1.5em]h2.north);
\draw [-,very thick,red] ([yshift=0.5em]s.north) -- ([yshift=1.5em]s.north);

\end{scope}

\end{tikzpicture}
\end{center}
}

\end{frame}

%%%------------------------------------------------------------------------------------------------------------
%%% �����ķ��򴫲� - �� dL/dw
\begin{frame}{���򴫲� - �����($\textbf{h}^{K-1}$�����ݶ�)}

\begin{itemize}
\item �Ѿ��õ�$\textbf{s}^K$�����ݶ�\visible&lt;2-&gt;{�����������������}
	\begin{enumerate}
	\item&lt;2-&gt; ������ʧ$L$���ڵ�$K$���������$\textbf{w}^K$���ݶȣ�$\frac{\partial L}{\partial \textbf{w}^K}$
	\item&lt;2-&gt; ������ʧ$L$���ڵ�$K$������$\textbf{h}^{K-1}$���ݶȣ�$\frac{\partial L}{\partial \textbf{h}^{K-1}}$
	\end{enumerate}
\end{itemize}

\vspace{-0.8em}
\begin{center}
\begin{tikzpicture}
\begin{scope}
\node [anchor=center,minimum height=1.7em,fill=yellow!20,draw] (h) at (0,0) {$\textbf{h}^{K-1}$};
\node [anchor=west,minimum height=1.7em,fill=blue!20,draw] (s) at ([xshift=5.5em]h.east) {$\textbf{s}^{K}$};
\node [anchor=west,minimum height=1.7em,fill=green!20,draw] (h2) at ([xshift=5.5em]s.east) {$\textbf{h}^{K}$};
\node [anchor=west,minimum height=1.7em,fill=orange!20,draw] (l) at ([xshift=5.5em]h2.east) {$L$};
\draw [-&gt;] (h.east) -- (s.west);
\draw [-&gt;] (s.east) -- (h2.west);
\draw [-&gt;] (h2.east) -- (l.west) node [pos=0.5,above] {\tiny{��ʧ}};

\node [anchor=south west,inner sep=2pt] (step100) at ([xshift=0.5em,yshift=-0.8em]h.north east) {\tiny{$\textbf{s}^K = \textbf{h}^{K-1} \textbf{w}^K$}};

\node [anchor=south west,inner sep=2pt] (step200) at ([xshift=0.5em,yshift=-0.8em]s.north east) {\tiny{$\textbf{h}^K = f^K(\textbf{s}^K)$}};

\node [anchor=south,inner sep=1pt] (outputlabel) at ([yshift=0.0em]h2.north) {\tiny{\textbf{�����}}};
\node [anchor=south west] (slabel) at ([yshift=1em,xshift=0.3em]s.north) {\scriptsize{\textbf{\alert{�Ѿ��õ���$\pi^K = \frac{\partial L}{\partial \textbf{s}^K}$}}}};
\draw [-&gt;,red] ([yshift=0.3em]slabel.south) .. controls +(south:0.5) and +(north:0.5) .. ([xshift=0.5em]s.north);

\visible&lt;2-&gt;{
\draw [-&gt;,very thick,red] ([yshift=1em,xshift=-0.1em]s.north) -- ([yshift=1em,xshift=0.1em]h.north) node [pos=0.5,above] {\tiny{\alert{$\frac{\partial L}{\partial \textbf{w}^K} = ?$, $\frac{\partial L}{\partial \textbf{h}^{K-1}} = ?$}}};
\draw [-,very thick,red] ([yshift=0.5em]h.north) -- ([yshift=1.5em]h.north);
\draw [-,very thick,red] ([yshift=0.5em]s.north) -- ([yshift=1.5em]s.north);
}

\end{scope}

\end{tikzpicture}
\end{center}

\begin{itemize}
\item&lt;3-&gt; ����$\textbf{s}^K = \textbf{h}^{K-1} \textbf{w}^K$������$\pi^K = \frac{\partial L}{\partial \textbf{s}^K}$�Ѿ���⣬���Եõ�(��ҪһЩ��ѧ���������Դ�����֪ʶ���Ƶ�һ��!)��

\vspace{-1.2em}

\begin{eqnarray}
\frac{\partial L}{\partial \textbf{w}^K}      &amp; = &amp; [\textbf{h}^{K-1}]^T \pi^K \nonumber \\
\frac{\partial L}{\partial \textbf{h}^{K-1}} &amp; = &amp; \pi^K  [\textbf{w}^K]^T\nonumber
\end{eqnarray}

���$[\textbf{A}]^T$��ʾ$\textbf{A}$��ת�ã�$\pi^K  [\textbf{w}^K]^T$��ʾ����$\pi^K$\alert{�����}$\textbf{w}^K$��ת��
\end{itemize}

\end{frame}

%%%------------------------------------------------------------------------------------------------------------
%%% ����ķ��򴫲�
\begin{frame}{���򴫲� - ����}
\begin{itemize}
\item ������������$k$��$\textbf{h}^k = f^k(\textbf{s}^k) = f^k(\textbf{h}^{k-1}\textbf{w}^k)$��������������������ݶ�$\pi^k=\frac{\partial L}{\partial \textbf{h}^{k}}$����Ҫ
	\begin{enumerate}
	\item ������ʧ$L$���ڵ�$k$���������$\textbf{w}^k$���ݶȣ�$\frac{\partial L}{\partial \textbf{w}^k}$
	\item ������ʧ$L$���ڵ�$k$������$\textbf{h}^{k-1}$���ݶȣ�$\frac{\partial L}{\partial \textbf{h}^{k-1}}$
	\end{enumerate}
\item&lt;2-&gt; ֱ��������һҳ�ķ��������Խ�$\pi^k=\frac{\partial L}{\partial \textbf{h}^{k}} \frac{\partial f^k(\textbf{s}^k)}{\partial \textbf{s}^{k}}$���򴫲�
\vspace{-0.0em}
\begin{eqnarray}
\frac{\partial L}{\partial \textbf{w}^k}      &amp; = &amp; [\textbf{h}^{k-1}]^T \pi^k \nonumber \\
\frac{\partial L}{\partial \textbf{h}^{k-1}} &amp; = &amp; \pi^k  [\textbf{w}^k]^T\nonumber
\end{eqnarray}

\end{itemize}

\visible&lt;3-&gt;{
\begin{center}
\begin{tikzpicture}
\begin{scope}
\node [anchor=center,draw,fill=red!20,minimum height=1.8em,minimum width=2.5em] (h) at (0,0) {$\textbf{h}^{k-1}$};
\node [anchor=west,draw,fill=blue!20,minimum height=1.8em,minimum width=2.5em] (s) at ([xshift=6em]h.east) {$\textbf{s}^{k}$};
\node [anchor=west,draw,fill=green!20,minimum height=1.8em,minimum width=2.5em] (h2) at ([xshift=6em]s.east) {$\textbf{h}^{k}$};
\node [anchor=east] (prev) at ([xshift=-2em]h.west) {...};
\node [anchor=west] (next) at ([xshift=2em]h2.east) {...};
\draw [-&gt;,thick] ([xshift=0.1em]prev.east) -- ([xshift=-0.1em]h.west);
\draw [-&gt;,thick] ([xshift=0.1em]h.east) -- ([xshift=-0.1em]s.west) node [pos=0.5,below] {\tiny{$\textbf{s}^k = \textbf{h}^{k-1}\textbf{w}^k$}};
\draw [-&gt;,thick] ([xshift=0.1em]s.east) -- ([xshift=-0.1em]h2.west) node [pos=0.5,below] {\tiny{$\textbf{h}^k = f(\textbf{s}^{k})$}};
\draw [-&gt;,thick] ([xshift=0.1em]h2.east) -- ([xshift=-0.1em]next.west);

\visible&lt;4-&gt;{
\draw [&lt;-,thick,red] ([xshift=0.1em,yshift=0.4em]h2.east) -- ([xshift=-0.1em,yshift=0.4em]next.west) node [pos=0.8,above] {\tiny{���򴫲�}};
}

\visible&lt;5-&gt;{
\draw [&lt;-,thick,red] ([xshift=0.1em,yshift=0.4em]s.east) -- ([xshift=-0.1em,yshift=0.4em]h2.west) node [pos=0.5,above] {\tiny{���򴫲�}};
}

\visible&lt;6-&gt;{
\draw [&lt;-,thick,red] ([xshift=0.1em,yshift=0.4em]h.east) -- ([xshift=-0.1em,yshift=0.4em]s.west) node [pos=0.5,above] {\tiny{���򴫲�}};
}

\visible&lt;7-&gt;{
\draw [-&gt;,thick,red,dashed] ([yshift=-0.1em]h.south) -- ([yshift=-1em]h.south) -- ([yshift=-1em]h2.south) -- ([yshift=-0.1em]h2.south);
\node [anchor=north,red] (recur) at ([yshift=-1em]s.south) {\scriptsize{$k=k-1$�ظ���������}};
}

\visible&lt;4-&gt;{
\node [anchor=south] (h2label) at (h2.north) {$\frac{\partial L}{\partial \textbf{h}^{k}}$};
}

\visible&lt;5-&gt;{
\node [anchor=south] (slabel) at (s.north) {$\pi^k = \frac{\partial L}{\partial \textbf{s}^{k}}$};
}

\visible&lt;6-&gt;{
\node [anchor=south] (hlabel) at (h.north) {$\frac{\partial L}{\partial \textbf{h}^{k-1}}$, $\frac{\partial L}{\partial \textbf{w}^{k}}$};
}

\end{scope}
\end{tikzpicture}
\end{center}
}

\end{frame}

%%%------------------------------------------------------------------------------------------------------------
%%% ���򴫲�ʵ��
\begin{frame}{���򴫲���ʵ��}
\begin{itemize}
\item ����һ����������������ʵ�ַ��򴫲�
\end{itemize}

\begin{tcolorbox}
[bicolor,sidebyside,righthand width=3.5cm,size=title,frame engine=empty,
 colback=blue!10!white,colbacklower=black!5!white]
 {\scriptsize
\begin{tabbing}
\texttt{XTensor x, y, gold, h[5], w[5], s[5];} \\
\texttt{XTensor dh[5], dw[5], ds[5];} \\
\texttt{...} // ǰ����� \\
\texttt{h[0] = x;} \\
\texttt{y = h[4];} \\

\visible&lt;2-&gt;{
\texttt{} \\
\texttt{CrossEntropyBackward(dldh[4], y, gold);} \\
\texttt{SoftmaxBackward(y, s[4], dh[4], ds[4]);}\\
\texttt{MMul(h[3], {\tiny X\_TRANS}, ds[4], {\tiny X\_NOTRANS}, dw[4]);}\\
}

\visible&lt;3-&gt;{
\texttt{} \\
\texttt{dh[2] = dh[3];}\\
\texttt{dh[1] = dh[3];}\\
}

\visible&lt;4-&gt;{
\texttt{} \\
\texttt{ReluBackward(h[2], s[2], dh[2], ds[2]);}\\
\texttt{MMul(h[1], {\tiny X\_TRANS}, ds[2], {\tiny X\_NOTRANS}, dw[2]);}\\
\texttt{} \\
\texttt{ReluBackward(h[1], s[1], dh[1], ds[1]);}\\
\texttt{MMul(h[0], {\tiny X\_TRANS}, ds[1], {\tiny X\_NOTRANS}, dw[1]);}\\
}

\visible&lt;5-&gt;{
\texttt{} \\
\texttt{for(unsigned i = 0; i &lt; 5; i++)\{} \\
\texttt{} \ \ \ \ ... // ͨ��\alert{\texttt{dw[i]}}���ʲ������ݶ�\\
\texttt{\}}
}

\end{tabbing}
}
\tcblower
\begin{center}
\begin{tikzpicture}


\node [anchor=south,draw,rounded corners,inner sep=2pt,minimum width=8em,minimum height=1.2em,fill=red!30!white,blur shadow={shadow xshift=1pt,shadow yshift=-1pt}] (h1) at (0,0) {\tiny{x (input)}};
\node [anchor=south,draw,rounded corners,inner sep=2pt,minimum width=8em,minimum height=1.2em,fill=green!30!white,blur shadow={shadow xshift=1pt,shadow yshift=-1pt}] (h2) at ([yshift=1.5em]h1.north) {\tiny{h1 = Relu(x * w1)}};
\node [anchor=south,draw,rounded corners,inner sep=2pt,minimum width=8em,minimum height=1.2em,fill=green!30!white,blur shadow={shadow xshift=1pt,shadow yshift=-1pt}] (h3) at ([yshift=1.5em]h2.north) {\tiny{h2 = Relu(h1 * w2)}};
\node [anchor=south,draw,rounded corners,inner sep=2pt,minimum width=8em,minimum height=1.2em,fill=green!30!white,blur shadow={shadow xshift=1pt,shadow yshift=-1pt}] (h4) at ([yshift=1.5em]h3.north) {\tiny{h3 = h2 + h1}};

\visible&lt;1-3&gt;{\draw [-&gt;,thick] (h1.north) -- (h2.south);}
\visible&lt;1-3&gt;{\draw [-&gt;,thick] (h2.north) -- (h3.south);}
\visible&lt;1-2&gt;{\draw [-&gt;,thick] (h3.north) -- (h4.south);}
\visible&lt;1-2&gt;{\draw [-&gt;,thick,rounded corners] (h2.east) -- ([xshift=0.5em]h2.east) -- ([xshift=0.5em,yshift=0.5em]h3.north east) -- ([xshift=-2em,yshift=0.5em]h3.north east) -- ([xshift=-2em,yshift=1.5em]h3.north east);}

\visible&lt;4-&gt;{\draw [&lt;-,very thick,red] (h1.north) -- (h2.south);}
\visible&lt;4-&gt;{\draw [&lt;-,very thick,red] (h2.north) -- (h3.south);}
\visible&lt;3-&gt;{\draw [&lt;-,very thick,red] (h3.north) -- (h4.south);}
\visible&lt;3-&gt;{\draw [&lt;-,very thick,red,rounded corners] (h2.east) -- ([xshift=0.5em]h2.east) -- ([xshift=0.5em,yshift=0.5em]h3.north east) -- ([xshift=-2em,yshift=0.5em]h3.north east) -- ([xshift=-2em,yshift=1.5em]h3.north east);}

\node [anchor=south,draw,rounded corners,inner sep=2pt,minimum width=8.0em,minimum height=1.2em,fill=red!30!white,blur shadow={shadow xshift=1pt,shadow yshift=-1pt}] (slayer) at ([yshift=1.5em]h4.north) {\tiny{h4 = Softmax(h3 * w4) (output)}};
\node [anchor=south] (losslabel) at (slayer.north) {\scriptsize{\textbf{Cross Entropy Loss}}};

\visible&lt;1&gt;{\draw [-&gt;,thick] (h4.north) -- (slayer.south);}
\visible&lt;2-&gt;{\draw [&lt;-,very thick,red] (h4.north) -- (slayer.south);}

\end{tikzpicture}
\end{center}
\end{tcolorbox}

\end{frame}

%%%------------------------------------------------------------------------------------------------------------
%%% �Զ�΢�ֵ�ʵ��
\begin{frame}{���򵥵�ʵ��}
\begin{itemize}
\item ���˵��ǣ����ڼ������е��������ѧϰ��ܶ�ʵ�����Զ�΢�֣�һ���������Ը㶨
\end{itemize}

\begin{tcolorbox}
[bicolor,sidebyside,righthand width=3.5cm,size=title,frame engine=empty,
 colback=blue!10!white,colbacklower=black!5!white]
 {\scriptsize
\begin{tabbing}
\texttt{XTensor x, loss, gold, h[5], w[5], b[5];} \\
\texttt{...} \\

\texttt{} \\
\texttt{h[1] = Relu(MMul(x, w[1]) + b[1]);} \\
\texttt{h[2] = Relu(MMul(h[1], w[2]) + b[2]);} \\
\texttt{h[3] = HardTanH(h[2]);} \\
\texttt{h[4] = Softmax(MMul(h[3], w[3]));} \\
\texttt{loss = CrossEntropy(h[4], gold);} \\

\texttt{} \\
\texttt{XNet net;}\\
\alert{\texttt{net.Backward(loss);} //һ�д���ʵ���Զ�΢��}\\

\texttt{} \\
\texttt{for(unsigned i = 0; i &lt; 5; i++)\{} \\
\texttt{} \ \ \ \ ... // ͨ��\alert{\texttt{w[i].grad}}���ʲ������ݶ�\\
\texttt{\}}

\end{tabbing}
}
\tcblower
\begin{center}
\begin{tikzpicture}


\node [anchor=south,draw,rounded corners,inner sep=2pt,minimum width=8em,minimum height=1.0em,fill=red!30!white,blur shadow={shadow xshift=1pt,shadow yshift=-1pt}] (h1) at (0,0) {\tiny{x (input)}};
\node [anchor=south,draw,rounded corners,inner sep=2pt,minimum width=8em,minimum height=1.0em,fill=green!30!white,blur shadow={shadow xshift=1pt,shadow yshift=-1pt}] (h2) at ([yshift=1.0em]h1.north) {\tiny{h1 = Relu(x * w1 + b1)}};
\node [anchor=south,draw,rounded corners,inner sep=2pt,minimum width=8em,minimum height=1.0em,fill=green!30!white,blur shadow={shadow xshift=1pt,shadow yshift=-1pt}] (h3) at ([yshift=1.0em]h2.north) {\tiny{h2 = Relu(h1 * w2 + b2)}};
\node [anchor=south,draw,rounded corners,inner sep=2pt,minimum width=8em,minimum height=1.0em,fill=green!30!white,blur shadow={shadow xshift=1pt,shadow yshift=-1pt}] (h4) at ([yshift=1.0em]h3.north) {\tiny{h3 = HardTanh(h2)}};

\draw [-&gt;,thick] (h1.north) -- (h2.south);
\draw [-&gt;,thick] (h2.north) -- (h3.south);
\draw [-&gt;,thick] (h3.north) -- (h4.south);

\node [anchor=south,draw,rounded corners,inner sep=2pt,minimum width=8.0em,minimum height=1.0em,fill=red!30!white,blur shadow={shadow xshift=1pt,shadow yshift=-1pt}] (slayer) at ([yshift=1.0em]h4.north) {\tiny{h4 = Softmax(h3 * w4) (output)}};
\node [anchor=south] (losslabel) at (slayer.north) {\scriptsize{\textbf{Cross Entropy Loss}}};

\draw [-&gt;,thick] (h4.north) -- (slayer.south);

\end{tikzpicture}
\end{center}
\end{tcolorbox}

\begin{itemize}
\item ����������Զ�΢��ʵ��Ҳ���Բο�TensorFlow�� PyTorch�ȹ���
\end{itemize}

\end{frame}

%%%------------------------------------------------------------------------------------------------------------
%%% ǰ�������̼�����ֵ�ù�ע������
\begin{frame}{ǰ����㼰��������}
\begin{itemize}
\item \alert{ǰ�����}ʵ���Ͼ������繹���Ĺ��̣������ֳ��÷�ʽ
    \begin{itemize}
    \item \textbf{��̬ͼ}(��PyTorch��NiuTensor)��д�꺯������ʽ��ǰ����㼴��ɣ����ڵ���
    \item \textbf{��̬ͼ}(��TensorFlow)����������ʽ��ɺ󣬲����ܵõ�ǰ�����������Ҫ���Ե���һ��Forward���������Ǽ���ͼ���Խ�������Ż���ִ��Ч�ʽϸ�
    \end{itemize}
\item&lt;2-&gt; ����һЩ���ѧϰϵͳʵ�ֵ����⣬ֵ�ù�ע��������Щ�������˱��γ̵ķ�Χ
    \begin{itemize}
    \item \textbf{�ֲ�ʽѵ��}�����ڸ���ģ�͵ĺ�������ѵ������Ҫ���ö���豸��������࿨��ͬʱѵ��
    \item \textbf{�;��ȼ���}��Ϊ�����Ч�ʿ��Բ��ð뾫�Ȼ��߶��������м���
    \item \textbf{ģ��ѹ��}���������࣬����ѹ��ģ�ͣ�ʹ��ģ�����ڴ洢ͬʱ���ϵͳ����Ч��
    \item \textbf{ѵ�������ͳ���ѡ��}����ͬ����������Ҫ��ͬ��ѵ�����ԣ������������ã��Ӻܶ࣬��Ҫ���۾���
    \end{itemize}
\end{itemize}
\end{frame}

%%%------------------------------------------------------------------------------------------------------------
\section{������ģ��}

%%%------------------------------------------------------------------------------------------------------------
%%% outline: neural language modeling
\begin{frame}{��������}

\vspace{6em}
\begin{tcolorbox}[enhanced,size=normal,left=2mm,right=1mm,colback=red!5!white,colframe=red!75!black,drop fuzzy shadow]
{\Large
\textbf{��ν���Ԫ����Ӧ�õ�NLP��}

\vspace{0.4em}
\textbf{- ����ģ�͵������罨ģ}
}
\end{tcolorbox}

\vspace{2em}
\begin{center}
\begin{tikzpicture}
\end{tikzpicture}
\end{center}

\end{frame}

%%%------------------------------------------------------------------------------------------------------------
\subsection{ǰ����ѭ������ע����������}

%%%------------------------------------------------------------------------------------------------------------
%%% ��NLP���������ܸ�ʲô��
\begin{frame}{��Ȼ���Դ�������������}
\begin{itemize}
\item �����緽������Ȼ���Դ���(NLP)�������µ�˼·
\end{itemize}

\begin{tabular} {l | l}
\textbf{��ͳ����ͳ�Ƶķ���} &amp; \textbf{���ѧϰ����} \\ \hline
����\alert{��ɢ}�ռ�ı�ʾģ�� &amp; ����\alert{����}�ռ�ı�ʾģ�� \\
NLP�����\alert{�����ṹ}���� &amp; �������ṹ���裬\alert{�˵���}ѧϰ \\
\alert{��������}Ϊ�� &amp; ����������������Ҫ\alert{�������} \\
�����������\alert{�洢����Դ} &amp; ģ�ʹ洢���С����\alert{������}
\end{tabular}

\vspace{0em}

\begin{itemize}
\item&lt;2-&gt; ����ģ������Ҳ����ʹ�����ѧϰ����(Ч���ǳ���)
    \begin{itemize}
    \item ����ģ��Ҫ�ش���������������һ���ʴ��ĺû�
    \item ���Ի���һ�µڶ����ᵽ��$n$Ԫ�﷨ģ��
    \end{itemize}
    \vspace{0.5em}
    \begin{displaymath}
    \textbf{P}(w_0 w_1 ... w_m) = ?
    \end{displaymath}

\end{itemize}

\visible&lt;3-&gt;{
\begin{tcolorbox}[enhanced,size=normal,left=2mm,right=1mm,colback=blue!5!white,colframe=blue!75!black,drop fuzzy shadow]
{\Large
\textbf{��ζԴʴ������ɸ��ʽ��н�ģ��}
}
\end{tcolorbox}
}

\end{frame}

%%%------------------------------------------------------------------------------------------------------------
%%% n-gram����ģ��
\begin{frame}{$n$-gram����ģ��}
\begin{itemize}
\item \textbf{��ʽ����}
\begin{eqnarray}
\textrm{P}(w_1 w_2 ... w_m)  &amp; = &amp; \textrm{P}(w_1) \textrm{P}(w_2|w_1) \textrm{P}(w_3 | w_1 w_2) ... \nonumber \\
                                               &amp;    &amp; \textrm{P}(w_m|w_1...w_{m-1}) \nonumber
\end{eqnarray}
\item&lt;2-&gt; \textbf{��ͳ$n$-gram����ģ��}����ǰ�ʽ�������ǰ��$n-1$����
\begin{eqnarray}
\textrm{P}(w_1 w_2 ... w_m)  &amp; = &amp; \textrm{P}(w_1) \textrm{P}(w_2|w_1) \textrm{P}(w_3 | w_1 w_2) ... \nonumber \\
                                               &amp;    &amp; \textrm{P}(w_m|\underbrace{w_{m-n+1}...w_{m-1}}_{\text{ǰ��$n-1$����}}) \nonumber
\end{eqnarray}
\vspace{-1.0em}
\ \ \ \ \ \ ����
\begin{displaymath}
\textrm{P}(w_m | w_{m-n+1} ... w_{m-1})  = \frac{\textrm{count}(w_{m-n+1}...w_{m})}{\textrm{count}(w_{m-n+1}...w_{m-1})}
\end{displaymath}
\ \ \ \ \ \ $\textrm{count}(\cdot)$��ʾ��ѵ��������ͳ�Ƶ�Ƶ��
\end{itemize}
\end{frame}

%%%------------------------------------------------------------------------------------------------------------
%%% n-gram lm =&gt; neural lm
\begin{frame}{$n$-gram���ɸ��ʵ������罨ģ}
\begin{itemize}
\item ��ͳ��$n$-gram����ģ��ʵ���Ͼ���һ����ѯ������$w_{m-n+1} ... w_{m}$��ѯ$n$-gram����$\textrm{P}(w_m | w_{m-n+1} ... w_{m-1})$
    \begin{itemize}
    \item ���ű���������һ��$w_{m-n+1} ... w_{m}$��\alert{��ɢ��ʾ}
    \item ����$n$������\alert{����ϡ��}�����ǳ����أ���Ϊ�������$n$-gram��û������
    \item ��ΪҪά��$n$-gram���������洢���Ĵ�
    \end{itemize}
\item&lt;2-&gt; ��һ��˼·��ֱ�Ӷ�$\textrm{P}(w_m | w_{m-n+1} ... w_{m-1})$���������ռ佨ģ�������庯��$g$�����������$w_{m-n+1} ... w_{m}$��
    \begin{displaymath}
    g(w_{m-n+1} ... w_{m}) \approx \textrm{P}(w_m | w_{m-n+1} ... w_{m-1})
    \end{displaymath}



\item&lt;3-&gt; ��ߴ����Եķ�����ǰ��������(FNN)����ģ��
    \begin{itemize}
    \item �����еľ��䣬���ִ�������ģ�͵���Ʋ�����ԶӰ��
    \end{itemize}

    \textbf{A Neural Probabilistic Language Model}\\
    \textbf{Bengio et al., 2003, Journal of Machine Learning Research 3: 1137-1155}
\end{itemize}
\end{frame}

%%%------------------------------------------------------------------------------------------------------------
%%% FNNLM architecture
\begin{frame}{ǰ������������ģ��(Bengio et al., 2003)}
\begin{itemize}
\item ��4-gram����ģ��Ϊ��
\end{itemize}

\vspace{-1em}
\begin{center}
\begin{tikzpicture}
\begin{scope}
\node [anchor=west] (w0) at (0,0) {\footnotesize{$w_{i-3}$}};
\node [anchor=west] (w1) at ([xshift=2em]w0.east) {\footnotesize{$w_{i-2}$}};
\node [anchor=west] (w2) at ([xshift=2em]w1.east) {\footnotesize{$w_{i-1}$}};
\node [anchor=north] (index0) at ([yshift=0.5em]w0.south) {\tiny(index)};
\node [anchor=north] (index1) at ([yshift=0.5em]w1.south) {\tiny(index)};
\node [anchor=north] (index2) at ([yshift=0.5em]w2.south) {\tiny(index)};
\node [anchor=south,draw,inner sep=3pt] (e0) at ([yshift=1em]w0.north) {\tiny{$e_0=w_{i-3} \textbf{C}$}};
\node [anchor=south,draw,inner sep=3pt] (e1) at ([yshift=1em]w1.north) {\tiny{$e_1=w_{i-2} \textbf{C}$}};
\node [anchor=south,draw,inner sep=3pt] (e2) at ([yshift=1em]w2.north) {\tiny{$e_2=w_{i-1} \textbf{C}$}};
\node [anchor=south,draw,minimum width=9em,inner sep=3pt] (h0) at ([yshift=1.5em]e1.north) {\tiny{$h_0=\textrm{Tanh}([e_0,e_1,e_2] \textbf{H} + \textbf{d})$}};
\node [anchor=south,draw,minimum width=9em,inner sep=3pt] (h1) at ([yshift=1.5em]h0.north) {\tiny{$y=\textrm{Softmax}(h_0 \textbf{U})$}};
\node [anchor=south] (ylabel) at ([yshift=1em]h1.north) {\footnotesize{$\textrm{P}(w_i|w_{i-3}w_{i-2}w_{i-1})$}};

\draw [-&gt;] ([yshift=0.1em]w0.north) -- ([yshift=-0.1em]e0.south);
\draw [-&gt;] ([yshift=0.1em]w1.north) -- ([yshift=-0.1em]e1.south);
\draw [-&gt;] ([yshift=0.1em]w2.north) -- ([yshift=-0.1em]e2.south);
\draw [-&gt;] ([yshift=0.1em]e0.north) -- ([xshift=-2em,yshift=-0.1em]h0.south);
\draw [-&gt;] ([yshift=0.1em]e1.north) -- ([yshift=-0.1em]h0.south);
\draw [-&gt;] ([yshift=0.1em]e2.north) -- ([xshift=2em,yshift=-0.1em]h0.south);
\draw [-&gt;] ([yshift=0.1em]h0.north) -- ([yshift=-0.1em]h1.south);
\draw [-&gt;] ([yshift=0.1em]h1.north) -- ([yshift=-0.1em]ylabel.south);

\visible&lt;6-&gt;{
\draw [-&gt;,dashed,red,thick] ([xshift=1em,yshift=0.1em]e1.north) -- ([xshift=1em,yshift=-0.1em]h1.south);
\draw [-&gt;,dashed,red,thick] ([xshift=-1em,yshift=0.1em]e0.north) .. controls +(north:2) and +(south:1) .. ([xshift=-3em,yshift=-0.1em]h1.south);
\draw [-&gt;,dashed,red,thick] ([xshift=1em,yshift=0.1em]e2.north) .. controls +(north:2) and +(south:1) .. ([xshift=3em,yshift=-0.1em]h1.south);
}

\begin{pgfonlayer}{background}
\visible&lt;2-&gt;{
\node [rectangle,inner sep=0.1em,fill=ugreen!20!white] [fit = (w0) (index0)] (wordbox0) {};
\node [rectangle,inner sep=0.1em,fill=ugreen!20!white] [fit = (w1) (index1)] (wordbox1) {};
\node [rectangle,inner sep=0.1em,fill=ugreen!20!white] [fit = (w2) (index2)] (wordbox2) {};
}
\end{pgfonlayer}

\visible&lt;3-&gt;{
\node [anchor=south,draw,inner sep=3pt,fill=blue!20!white] (e0) at ([yshift=1em]w0.north) {\tiny{$e_0=w_{i-3} \textbf{C}$}};
\node [anchor=south,draw,inner sep=3pt,fill=blue!20!white] (e1) at ([yshift=1em]w1.north) {\tiny{$e_1=w_{i-2} \textbf{C}$}};
\node [anchor=south,draw,inner sep=3pt,fill=blue!20!white] (e2) at ([yshift=1em]w2.north) {\tiny{$e_2=w_{i-1} \textbf{C}$}};
}
\visible&lt;5-&gt;{
\node [anchor=south,draw,minimum width=9em,inner sep=3pt,fill=orange!20!white] (h0) at ([yshift=1.5em]e1.north) {\tiny{$h_0=\textrm{Tanh}([e_0,e_1,e_2] \textbf{H} + \textbf{d})$}};
\node [anchor=south,draw,minimum width=9em,inner sep=3pt,fill=orange!20!white] (h1) at ([yshift=1.5em]h0.north) {\tiny{$y=\textrm{Softmax}(h_0 \textbf{U})$}};
}

\visible&lt;2-&gt;{
\node [anchor=north west] (indexlabel0) at ([yshift=-0.5em,xshift=-1.2em]index0.south west) {\scriptsize{{\color{ugreen} \textbf{One-hot��ʾ}}}};
\node [anchor=north west] (indexlabel1) at ([yshift=0.3em]indexlabel0.south west) {\scriptsize{ÿ������һ���ʻ����С��0-1������ʾ��}};
\node [anchor=north west] (indexlabel2) at ([yshift=0.3em]indexlabel1.south west) {\scriptsize{��һλΪ1������Ϊ0�����磺}};
\node [anchor=north west] (indexlabel3) at ([yshift=0.0em]indexlabel2.south west) {\scriptsize{$(0,0,{\red 1},0,0,0,0,0,0,0,0,0)$}};
\node [anchor=north west] (indexlabel4) at ([xshift=1em,yshift=0.0em]indexlabel3.south west) {\scriptsize{�ʱ��е�3����}};
\draw [-&gt;] ([xshift=1.2em,yshift=-0.2em]indexlabel4.north west) -- ([xshift=1.2em,yshift=0.3em]indexlabel4.north west);
}

\visible&lt;3-&gt;{
\node [anchor=west] (embedinglabel0) at ([xshift=1em,yshift=-1em]e2.east) {\scriptsize{{\blue \textbf{�ʵķֲ�ʽ��ʾ}}}};
\node [anchor=north west] (embedinglabel1) at ([yshift=0.3em]embedinglabel0.south west) {\scriptsize{�ʵ�0-1��ʾ��һ������$\textbf{C}$���������}};
\node [anchor=north west] (embedinglabel2) at ([yshift=0.3em]embedinglabel1.south west) {\scriptsize{��$\textbf{C}$����һ����ѯ��}};
}

\visible&lt;4-&gt;{
\node [anchor=north west] (wordvector) at ([yshift=-1em]embedinglabel2.south west) {\tiny{$(0,0,{\red 1},...)$}};
\node [anchor=west] (timeslabel) at ([xshift=-0.3em]wordvector.east) {\footnotesize{$\times$}};
\node [anchor=north west,inner sep=2pt] (embeddingmatrix) at ([xshift=1em]wordvector.north east) {\tiny{$\begin{pmatrix} 0 &amp; 1 &amp; 3 \\ .2 &amp; -1 &amp; .3 \\ 1 &amp; 7 &amp; .3 \\ ... \end{pmatrix}$}};
\node [anchor=south,inner sep=1pt] (wordvectorlabel) at (wordvector.north) {\scriptsize{$w_{i-1}$}};
\node [anchor=south,inner sep=1pt] (embeddingmatrixlabel) at (embeddingmatrix.north) {\scriptsize{$\textbf{C}$}};
\node [anchor=north west] (selectedlabel) at ([yshift=-2em]wordvector.south west) {\scriptsize{�ڰ�$\textbf{C}$���������������(i.e., $e_{i-1}$)}};

\begin{pgfonlayer}{background}
\visible&lt;4-&gt;{
\node [anchor=north west,fill=blue!20!white,minimum height=0.6em,minimum width=5.0em] (selected) at ([yshift=-1.3em]embeddingmatrix.north west) {};
}
\end{pgfonlayer}
\draw [-&gt;] ([xshift=0.15em,yshift=0.3em]wordvector.south) .. controls +(south:0.3) and +(west:0.5) .. (selected.west);
}

\visible&lt;5-&gt;{
\node [anchor=south west] (hiddenlabel0) at ([yshift=5em]embedinglabel0.north west) {\scriptsize{{\color{orange} \textbf{���������}}}};
\node [anchor=north west] (hiddenlabel1) at ([yshift=0.3em]hiddenlabel0.south west) {\scriptsize{$[e_0,e_1,e_2]$��ʾ����������������һ��}};
\node [anchor=north west] (hiddenlabel2) at ([yshift=0.3em]hiddenlabel1.south west) {\scriptsize{֮�󾭹��������磬���ͨ��Softmax���}};
\node [anchor=north west] (hiddenlabel3) at ([yshift=0.3em]hiddenlabel2.south west) {\scriptsize{ע�⣬$h_0\textbf{U}$�õ����дʵı�ʾ(����)��}};
\node [anchor=north west] (hiddenlabel4) at ([yshift=0.3em]hiddenlabel3.south west) {\scriptsize{Softmaxȷ������ʻ���ϵ�һ���ֲ�}};
}

\visible&lt;6-&gt;{
\node [anchor=south west] (directlabel0) at ([yshift=1em]hiddenlabel0.north west) {\scriptsize{\alert{\textbf{�ײ����ϲ��ֱ������(��ѡ)}}}};
}

\end{scope}
\end{tikzpicture}
\end{center}

\end{frame}

%%%------------------------------------------------------------------------------------------------------------
%%% FNNLM implementation
\begin{frame}{ǰ������������ģ��(FNN LM)��ʵ��}

\begin{itemize}
\item ʵ�ַǳ��򵥣����д���
    \begin{itemize}
    \item ϸ��1����batchingʱ���԰�$w[i]$������չ�������������
    \item ϸ��2��TanHһ�����HardTanHʵ�֣���ΪTanH�������
    \end{itemize}
\end{itemize}

\begin{tcolorbox}
[bicolor,sidebyside,righthand width=3.8cm,size=title,frame engine=empty,
 colback=blue!10!white,colbacklower=black!5!white]
 {\scriptsize
\begin{tabbing}
\texttt{XTensor w[3], e[3], h0, y;} \\
\texttt{XTensor C, H, d, U;} \\
\texttt{...}\\

\texttt{} \\
\texttt{for(unsigned i = 0; i &lt; 3; i++)} \\
\texttt{\ \ \ \ e[i] = MMul(w[i], C);}\\
\texttt{e01 = Concatenate(e[0], e[1], -1);}\\
\texttt{e = Concatenate(e01, e[2], -1);}\\

\texttt{} \\
\texttt{h0 = TanH(MMul(e, H) + d);}\\
\texttt{y = Softmax(MMul(h0, U));}\\

\texttt{} \\
\texttt{for(unsigned k = 0; k &lt; size; k++)\{} \\
\texttt{} \ \ \ \ ... // \alert{\texttt{y}}�ĵ�$k$Ԫ�ر�ʾ $\textrm{P}(w|...)$\\
\texttt{} \ \ \ \ ... // $w$Ϊ�ʻ�����$k$����\\
\texttt{\}}

\end{tabbing}
}
\tcblower
\begin{center}
\begin{tikzpicture}
\begin{scope}
\node [anchor=west] (w0) at (0,0) {\scriptsize{$w_{i-3}$}};
\node [anchor=west] (w1) at ([xshift=0.5em]w0.east) {\scriptsize{$w_{i-2}$}};
\node [anchor=west] (w2) at ([xshift=0.5em]w1.east) {\scriptsize{$w_{i-1}$}};
\node [anchor=north] (index0) at ([yshift=0.5em]w0.south) {\tiny(index)};
\node [anchor=north] (index1) at ([yshift=0.5em]w1.south) {\tiny(index)};
\node [anchor=north] (index2) at ([yshift=0.5em]w2.south) {\tiny(index)};
\node [anchor=south,draw,inner sep=3pt,align=left] (e0) at ([yshift=1.0em]w0.north) {\tiny{$e_0:$}\\\tiny{$w_{i-3} \textbf{C}$}};
\node [anchor=south,draw,inner sep=3pt,align=left] (e1) at ([yshift=1.0em]w1.north) {\tiny{$e_1:$}\\\tiny{$w_{i-2} \textbf{C}$}};
\node [anchor=south,draw,inner sep=3pt,align=left] (e2) at ([yshift=1.0em]w2.north) {\tiny{$e_2:$}\\\tiny{$w_{i-1} \textbf{C}$}};
\node [anchor=south,draw,minimum width=9em,inner sep=3pt] (h0) at ([yshift=1.5em]e1.north) {\tiny{$h_0=\textrm{Tanh}([e_0,e_1,e_2] \textbf{H} + \textbf{d})$}};
\node [anchor=south,draw,minimum width=9em,inner sep=3pt] (h1) at ([yshift=1.5em]h0.north) {\tiny{$y=\textrm{Softmax}(h_0 \textbf{U})$}};
\node [anchor=south] (ylabel) at ([yshift=1em]h1.north) {\scriptsize{$\textrm{P}(w_i|w_{i-3}w_{i-2}w_{i-1})$}};

\draw [-&gt;] ([yshift=0.1em]w0.north) -- ([yshift=-0.1em]e0.south);
\draw [-&gt;] ([yshift=0.1em]w1.north) -- ([yshift=-0.1em]e1.south);
\draw [-&gt;] ([yshift=0.1em]w2.north) -- ([yshift=-0.1em]e2.south);
\draw [-&gt;] ([yshift=0.1em]e0.north) -- ([xshift=-2em,yshift=-0.1em]h0.south);
\draw [-&gt;] ([yshift=0.1em]e1.north) -- ([yshift=-0.1em]h0.south);
\draw [-&gt;] ([yshift=0.1em]e2.north) -- ([xshift=2em,yshift=-0.1em]h0.south);
\draw [-&gt;] ([yshift=0.1em]h0.north) -- ([yshift=-0.1em]h1.south);
\draw [-&gt;] ([yshift=0.1em]h1.north) -- ([yshift=-0.1em]ylabel.south);
\end{scope}
\end{tikzpicture}
\end{center}
\end{tcolorbox}
\vspace{-0.5em}
\footnotesize{ע: size��ʾ�ʻ����С}

\end{frame}

%%%------------------------------------------------------------------------------------------------------------
%%% ������ģ�͸����Ǵ�����ʲô
\begin{frame}{�����Խ�ģ������}

\begin{itemize}
\item Bengio el al. (2003)���д����۵�����
    \begin{enumerate}
    \item ������ÿһ�㾿��ѧ����ʲô \\
    �ʻ㡢�䷨����������һЩ֪ʶ����ν��ͣ�
    \item ����Ĳ����������� - 10�㡢20�㡢100������� \\
    \# of layers: 10 $\to$ 20 $\to$ 100 $\to$ 1000
    \item ����(�������ز��С)���ѡ�� - ��ͬ�������������\\
    ���ʵķֲ�ʽ��ʾά�ȶ��ã�\\
    ������ã�\\
    ��������ѡ��\\
    ...
    \end{enumerate}
\item&lt;2-&gt; ��FNN LM�õ�������
    \begin{itemize}
    \item ���¶������ʲô - �Ǵʵ����һ�����һ��ʵ������
    \item �����������Ժܺõı�ʾ����֮���(�̾���)����
    \item $n$-gram�����ɸ��ʿ���ʹ�������ռ亯����������������ϡ�����⣬ģ�Ͳ�����Ҫ��¼������$n$-gram
    \end{itemize}
\end{itemize}

\end{frame}

%%%------------------------------------------------------------------------------------------------------------
%%% ѭ��������
\begin{frame}{ѭ��������(Recurrent Neural Networks)}

\begin{itemize}
\item FNN LM��Ȼ��Ч�����Ǻʹ�ͳ��$n$-gram LMһ������Ҫ����\alert{����������}����
\begin{center}
\begin{tikzpicture}
\begin{scope}
\node [anchor=west] (w0) at (0,0) {$w_1$};
\node [anchor=west] (w1) at ([xshift=0.5em]w0.east) {$w_2$};
\node [anchor=west] (w2) at ([xshift=0.5em]w1.east) {$...$};
\node [anchor=west] (w3) at ([xshift=0.5em]w2.east) {$w_{m-n+1}$};
\node [anchor=west] (w4) at ([xshift=0.5em]w3.east) {$...$};
\node [anchor=west,fill=green!20!white] (w5) at ([xshift=0.5em]w4.east) {$w_{m}$};
\draw [-&gt;,thick,ublue] (w5.south).. controls +(210:0.5) and +(-30:0.5) .. (w3.south);
\draw [-&gt;,thick,red] (w5.north).. controls +(150:1) and +(30:1) .. (w1.north);
\draw [-&gt;,very thick,ublue] ([xshift=-5em,yshift=1em]w0.west) -- ([xshift=-6.5em,yshift=1em]w0.west) node [pos=0,right] {\scriptsize{����}};
\draw [-&gt;,very thick,red] ([xshift=-5em,yshift=-0.5em]w0.west) -- ([xshift=-6.5em,yshift=-0.5em]w0.west) node [pos=0,right] {\scriptsize{������}};

\end{scope}
\end{tikzpicture}
\end{center}
\item&lt;2-&gt; �ܷ�ֱ�Ӷ�ԭʼ���⽨ģ�������庯��$g$�����������$w_{1} ... w_{m}$��
    \vspace{-0.5em}
    \begin{displaymath}
    g(w_{1} ... w_{m}) \approx \textrm{P}(w_m | w_{1} ... w_{m-1})
    \end{displaymath}
\item&lt;3-&gt; \textbf{ѭ��������(RNNs)}���ԺܺõĽ���������⣬���Ҳ���ɹ���Ӧ�������Խ�ģ����
	\begin{itemize}
	\item ������ÿ���ʵ����ɶ������Ѿ����ɵ����д�
	\item ���ڲ�ͬλ�õĴʵ����ɸ��ʶ�������ͬһ����������
	\end{itemize}
	
        \textbf{Recurrent Neural Network Based Language Model}\\
        \textbf{Mikolov et al., 2010, In Proc. of Interspeech, 1045-1048}
\end{itemize}

\end{frame}

%%%------------------------------------------------------------------------------------------------------------
%%% ѭ��������Ľṹ
\begin{frame}{ѭ����Ԫ}

\begin{itemize}
\item ����������$(\textbf{x}_0,\textbf{x}_1,...,\textbf{x}_t,...)$������$\textbf{x}_t$��ʾ�����е�$t$��Ԫ�أ�Ҳ������\alert{ʱ��$t$}���롣������Ӧ�����������$(\textbf{y}_0,\textbf{y}_1,...,\textbf{y}_t,...)$�� ��ѭ���������У�ÿ��ʱ�̵������������ͬһ��\alert{ѭ����Ԫ}��������\visible&lt;2-&gt;{��������ģ�ͣ�һ�ּ򵥵Ľṹ��}

\visible&lt;2-&gt;{
{\small
\begin{tcolorbox}
[bicolor,sidebyside,righthand width=4.3cm,size=title,frame engine=empty,
 colback=blue!10!white,colbacklower=black!5!white]

\begin{eqnarray}
\textbf{y}_t &amp; = &amp; \textrm{Softmax}(\textbf{h}_t \textbf{V}) \nonumber \\
\textbf{h}_t &amp; = &amp; \textrm{TanH}(\textbf{x}_t \textbf{U} + \textbf{h}_{t-1} \textbf{W}) \nonumber
\end{eqnarray}
\footnotesize{$\textbf{h}_t$: $t$ʱ�̵�����״̬\\
$\textbf{h}_{t-1}$: $t-1$ʱ�̵�����״̬\\
$\textbf{V}, \textbf{U}, \textbf{W}$: ����
}
\tcblower
\begin{center}
\begin{tikzpicture}
\begin{scope}
\node [anchor=west,inner sep=3pt,minimum width=8em] (h) at (0,0) {\tiny{$\textbf{h}_t  =  \textrm{TanH}(\textbf{x}_t \textbf{U} + \textbf{h}_{t-1} \textbf{W})$}};
\node [anchor=south west,inner sep=3pt] (r) at ([yshift=-0.2em]h.north west) {\tiny{ѭ����Ԫ:}};
\begin{pgfonlayer}{background}
\node [rectangle,draw,inner sep=0em,fill=green!20!white] [fit = (r) (h)] (rbox) {};
\end{pgfonlayer}
\node [anchor=south,draw,minimum width=8em,fill=green!20!white] (y) at ([yshift=1.5em]rbox.north) {\tiny{$\textbf{y}_t = \textrm{Softmax}(\textbf{h}_t \textbf{V})$}};
\node [anchor=south,inner sep=2pt] (output) at ([yshift=1em]y.north) {\scriptsize{$\textbf{y}_t$}};
\node [anchor=north,inner sep=2pt] (input) at ([yshift=-1em]h.south) {\scriptsize{$\textbf{x}_t$}};
\draw [-&gt;,thick] (input.north) -- ([yshift=-0.1em]rbox.south);
\draw [-&gt;,thick] ([yshift=0.1em]rbox.north) -- ([yshift=-0.1em]y.south) node [pos=0.5,left] {\tiny{$\textbf{h}_t$}};
\draw [-&gt;,thick] ([yshift=0.1em]y.north) -- (output.south);
\draw [-&gt;,thick] ([xshift=0.1em]rbox.east) -- ([xshift=1em]rbox.east) node [pos=1,above] {\tiny{$\textbf{h}_t$}};
\draw [-&gt;,thick] ([xshift=-1em]rbox.west) -- ([xshift=-0.1em]rbox.west) node [pos=0,above] {\tiny{$\textbf{h}_{t-1}$}};

\end{scope}
\end{tikzpicture}
\end{center}
\end{tcolorbox}
}
}

\item&lt;3-&gt; \textbf{�������ѭ����}$t$ʱ�̵�״̬��$t-1$ʱ��״̬�ĺ�����������̿��Բ��ϱ�ִ��
\end{itemize}

\end{frame}

%%%------------------------------------------------------------------------------------------------------------
%%% ѭ��������ġ����䡱
\begin{frame}{ѭ���������``����''}
\begin{itemize}
\item ѭ����������Լ������ⳤ�ȵ���ʷ����˿��Էǳ��ʺϴ��������������У�������Ȼ���Ծ���
    \begin{itemize}
    \item ע�⣺$\textbf{h}_{t-1}$���Ա����ݵ�����״̬
    \end{itemize}
\end{itemize}

\vspace{-1em}
\begin{eqnarray}
\textbf{h}_t &amp; = &amp; \textrm{TanH}(\textbf{x}_t \textbf{U} + \alert{\textbf{h}_{t-1}} \textbf{W}) \nonumber \\
\visible&lt;2-&gt;{
\textbf{h}_{t+1} &amp; = &amp; \textrm{TanH}(\textbf{x}_{t+1} \textbf{U} + \textbf{h}_{t} \textbf{W}) \nonumber \\
                 &amp; = &amp; \textrm{TanH}(\textbf{x}_{t+1} \textbf{U} + \textrm{TanH}(\textbf{x}_t \textbf{U} + \alert{\textbf{h}_{t-1}} \textbf{W}) \textbf{W}) \nonumber \\
                 }
\visible&lt;3-&gt;{
\textbf{h}_{t+2} &amp; = &amp; \textrm{TanH}(\textbf{x}_{t+2} \textbf{U} + \textbf{h}_{t+1} \textbf{W}) \nonumber \\
                 &amp; = &amp; \textrm{TanH}(\textbf{x}_{t+2} \textbf{U} + \nonumber \\
                 &amp;   &amp; \textrm{TanH}(\textbf{x}_{t+1} \textbf{U} + \textrm{TanH}(\textbf{x}_t \textbf{U} + \alert{\textbf{h}_{t-1}} \textbf{W}) \textbf{W}) \textbf{W}) \nonumber
                 }
\end{eqnarray}

\vspace{-1em}
\begin{center}
\begin{tikzpicture}
\begin{scope}
\tikzstyle{rnnnode} = [draw,inner sep=5pt,fill=green!30!white,blur shadow={shadow xshift=1pt,shadow yshift=-1pt}]
\node [anchor=west,rnnnode] (node1) at (0,0) {\scriptsize{RNN Cell}};
\visible&lt;2-&gt;{
\node [anchor=west,rnnnode] (node2) at ([xshift=4.5em]node1.east) {\scriptsize{RNN Cell}};
}
\visible&lt;3-&gt;{
\node [anchor=west,rnnnode] (node3) at ([xshift=4.5em]node2.east) {\scriptsize{RNN Cell}};
}
\node [anchor=north] (x1) at ([yshift=-1em]node1.south) {\footnotesize{$\textbf{x}_{t}$}};
\visible&lt;2-&gt;{
\node [anchor=north] (x2) at ([yshift=-1em]node2.south) {\footnotesize{$\textbf{x}_{t+1}$}};
}
\visible&lt;3-&gt;{
\node [anchor=north] (x3) at ([yshift=-1em]node3.south) {\footnotesize{$\textbf{x}_{t+2}$}};
}
\node [anchor=south] (h1) at ([yshift=1em]node1.north) {\footnotesize{$\textbf{h}_{t}$}};
\visible&lt;2-&gt;{
\node [anchor=south] (h2) at ([yshift=1em]node2.north) {\footnotesize{$\textbf{h}_{t+1}$}};
}
\visible&lt;3-&gt;{
\node [anchor=south] (h3) at ([yshift=1em]node3.north) {\footnotesize{$\textbf{h}_{t+2}$}};
}
\draw [-&gt;,thick] ([xshift=-1.0em]node1.west)--([xshift=-0.1em]node1.west) node [pos=0,left] {\scriptsize{$\alert{\textbf{h}_{t-1}}$}};
\visible&lt;3-&gt;{
\draw [-&gt;,thick] ([xshift=0.1em]node3.east)--([xshift=1.0em]node3.east) node [pos=1,right] {\scriptsize{$\textbf{h}_{t+2}$}};
}
\draw [-&gt;,thick] ([xshift=0.1em]node1.east)--([xshift=-0.1em]node2.west) node [pos=0.5,above] {\tiny{$\textbf{h}_{t}(\alert{\textbf{h}_{t-1}})$}};
\visible&lt;2-&gt;{
\draw [-&gt;,thick] ([xshift=0.1em]node2.east)--([xshift=-0.1em]node3.west) node [pos=0.5,above] {\tiny{$\textbf{h}_{t+1}(\textbf{h}_{t}(\alert{\textbf{h}_{t-1}}))$}};
}
\draw [-&gt;,thick] (x1.north)--([yshift=-0.1em]node1.south);
\visible&lt;2-&gt;{
\draw [-&gt;,thick] (x2.north)--([yshift=-0.1em]node2.south);
}
\visible&lt;3-&gt;{
\draw [-&gt;,thick] (x3.north)--([yshift=-0.1em]node3.south);
}
\draw [-&gt;,thick] ([yshift=0.1em]node1.north)--(h1.south);
\visible&lt;2-&gt;{
\draw [-&gt;,thick] ([yshift=0.1em]node2.north)--(h2.south);
}
\visible&lt;3-&gt;{
\draw [-&gt;,thick] ([yshift=0.1em]node3.north)--(h3.south);
}

\end{scope}
\end{tikzpicture}
\end{center}

\end{frame}

%%%------------------------------------------------------------------------------------------------------------
%%% ����ѭ�������������ģ��
\begin{frame}{����ѭ�������������ģ��(RNN LM)}
\begin{itemize}
\item ѭ����������Ա�ֱ����������ģ��
    \begin{itemize}
    \item&lt;2-&gt; ��FNN LM���ƣ����ȰѴʴ�one-hot��ʾת���ɷֲ�ʽ��ʾ
    \item&lt;3-&gt; $t$ʱ��Ԥ��$\textrm{P}(x_{t+1}|x_1...x_{t})$
    \item&lt;4-&gt; ���Ե��Ӹ���Ĳ�
    \end{itemize}
\end{itemize}

\visible&lt;2-&gt;{
\begin{center}
\begin{tikzpicture}
\begin{scope}
\tikzstyle{rnnnode} = [draw,inner sep=5pt,minimum width=4em,minimum height=1.5em,fill=green!30!white,blur shadow={shadow xshift=1pt,shadow yshift=-1pt}]
\visible&lt;3-&gt;{
\node [anchor=west,rnnnode] (node11) at (0,0) {\scriptsize{RNN Cell}};
\node [anchor=west,rnnnode] (node12) at ([xshift=2em]node11.east) {\scriptsize{RNN Cell}};
\node [anchor=west,rnnnode] (node13) at ([xshift=2em]node12.east) {\scriptsize{RNN Cell}};
\node [anchor=west,rnnnode] (node14) at ([xshift=2em]node13.east) {\scriptsize{RNN Cell}};
}
\node [anchor=north,rnnnode,fill=red!30!white] (e1) at ([yshift=-1.2em]node11.south) {\tiny{$e_1=w_1\textbf{C}$}};
\node [anchor=north,rnnnode,fill=red!30!white] (e2) at ([yshift=-1.2em]node12.south) {\tiny{$e_2=w_2\textbf{C}$}};
\node [anchor=north,rnnnode,fill=red!30!white] (e3) at ([yshift=-1.2em]node13.south) {\tiny{$e_3=w_3\textbf{C}$}};
\node [anchor=north,rnnnode,fill=red!30!white] (e4) at ([yshift=-1.2em]node14.south) {\tiny{$e_4=w_4\textbf{C}$}};
\node [anchor=north] (w1) at ([yshift=-1em]e1.south) {\footnotesize{$w_1$}};
\node [anchor=north] (w2) at ([yshift=-1em]e2.south) {\footnotesize{$w_2$}};
\node [anchor=north] (w3) at ([yshift=-1em]e3.south) {\footnotesize{$w_3$}};
\node [anchor=north] (w4) at ([yshift=-1em]e4.south) {\footnotesize{$w_4$}};

\draw [-&gt;,thick] ([yshift=0.1em]w1.north)--([yshift=-0.1em]e1.south);
\draw [-&gt;,thick] ([yshift=0.1em]w2.north)--([yshift=-0.1em]e2.south);
\draw [-&gt;,thick] ([yshift=0.1em]w3.north)--([yshift=-0.1em]e3.south);
\draw [-&gt;,thick] ([yshift=0.1em]w4.north)--([yshift=-0.1em]e4.south);

\draw [-&gt;,thick] ([yshift=0.1em]e1.north)--([yshift=-0.1em]node11.south);
\draw [-&gt;,thick] ([yshift=0.1em]e2.north)--([yshift=-0.1em]node12.south);
\draw [-&gt;,thick] ([yshift=0.1em]e3.north)--([yshift=-0.1em]node13.south);
\draw [-&gt;,thick] ([yshift=0.1em]e4.north)--([yshift=-0.1em]node14.south);

\visible&lt;4-&gt;{
\node [anchor=south,rnnnode] (node21) at ([yshift=1.5em]node11.north) {\scriptsize{RNN Cell}};
\node [anchor=south,rnnnode] (node22) at ([yshift=1.5em]node12.north) {\scriptsize{RNN Cell}};
\node [anchor=south,rnnnode] (node23) at ([yshift=1.5em]node13.north) {\scriptsize{RNN Cell}};
\node [anchor=south,rnnnode] (node24) at ([yshift=1.5em]node14.north) {\scriptsize{RNN Cell}};

\node [anchor=south,rnnnode,fill=blue!30!white] (node31) at ([yshift=1.5em]node21.north) {\scriptsize{Softmax($\cdot$)}};
\node [anchor=south,rnnnode,fill=blue!30!white] (node32) at ([yshift=1.5em]node22.north) {\scriptsize{Softmax($\cdot$)}};
\node [anchor=south,rnnnode,fill=blue!30!white] (node33) at ([yshift=1.5em]node23.north) {\scriptsize{Softmax($\cdot$)}};
\node [anchor=south,rnnnode,fill=blue!30!white] (node34) at ([yshift=1.5em]node24.north) {\scriptsize{Softmax($\cdot$)}};
}

\visible&lt;3&gt;{
\node [anchor=south,rnnnode,fill=blue!30!white] (node21) at ([yshift=1.5em]node11.north) {\scriptsize{Softmax($\cdot$)}};
\node [anchor=south,rnnnode,fill=blue!30!white] (node22) at ([yshift=1.5em]node12.north) {\scriptsize{Softmax($\cdot$)}};
\node [anchor=south,rnnnode,fill=blue!30!white] (node23) at ([yshift=1.5em]node13.north) {\scriptsize{Softmax($\cdot$)}};
\node [anchor=south,rnnnode,fill=blue!30!white] (node24) at ([yshift=1.5em]node14.north) {\scriptsize{Softmax($\cdot$)}};

\draw [-&gt;,thick] ([yshift=0.1em]node21.north)--([yshift=-0.1em]node31.south) node[pos=1,above] {\scriptsize{$\textrm{P}(w_2)$}};
\draw [-&gt;,thick] ([yshift=0.1em]node22.north)--([yshift=-0.1em]node32.south) node[pos=1,above] {\scriptsize{$\textrm{P}(w_3|w_2)$}};
\draw [-&gt;,thick] ([yshift=0.1em]node23.north)--([yshift=-0.1em]node33.south) node[pos=1,above] {\scriptsize{$\textrm{P}(w_4|w_2 w_3)$}};
\draw [-&gt;,thick] ([yshift=0.1em]node24.north)--([yshift=-0.1em]node34.south) node[pos=1,above] {\scriptsize{$\textrm{P}(w_5|w_2 w_3 w_4)$}};
}

\visible&lt;4-&gt;{
\draw [-&gt;,thick] ([yshift=0.1em]node31.north)--([yshift=1em]node31.north) node[pos=1,above] {\scriptsize{$\textrm{P}(w_2)$}};
\draw [-&gt;,thick] ([yshift=0.1em]node32.north)--([yshift=1em]node32.north) node[pos=1,above] {\scriptsize{$\textrm{P}(w_3|w_2)$}};
\draw [-&gt;,thick] ([yshift=0.1em]node33.north)--([yshift=1em]node33.north) node[pos=1,above] {\scriptsize{$\textrm{P}(w_4|w_2 w_3)$}};
\draw [-&gt;,thick] ([yshift=0.1em]node34.north)--([yshift=1em]node34.north) node[pos=1,above] {\scriptsize{$\textrm{P}(w_5|w_2 w_3 w_4)$}};

\draw [-&gt;,thick] ([yshift=0.1em]node21.north)--([yshift=-0.1em]node31.south);
\draw [-&gt;,thick] ([yshift=0.1em]node22.north)--([yshift=-0.1em]node32.south);
\draw [-&gt;,thick] ([yshift=0.1em]node23.north)--([yshift=-0.1em]node33.south);
\draw [-&gt;,thick] ([yshift=0.1em]node24.north)--([yshift=-0.1em]node34.south);

\draw [-&gt;,thick] ([xshift=-1em]node21.west)--([xshift=-0.1em]node21.west);
\draw [-&gt;,thick] ([xshift=0.1em]node21.east)--([xshift=-0.1em]node22.west);
\draw [-&gt;,thick] ([xshift=0.1em]node22.east)--([xshift=-0.1em]node23.west);
\draw [-&gt;,thick] ([xshift=0.1em]node23.east)--([xshift=-0.1em]node24.west);
\draw [-&gt;,thick] ([xshift=0.1em]node24.east)--([xshift=1em]node24.east);
}

\visible&lt;3-&gt;{
\draw [-&gt;,thick] ([yshift=0.1em]node11.north)--([yshift=-0.1em]node21.south);
\draw [-&gt;,thick] ([yshift=0.1em]node12.north)--([yshift=-0.1em]node22.south);
\draw [-&gt;,thick] ([yshift=0.1em]node13.north)--([yshift=-0.1em]node23.south);
\draw [-&gt;,thick] ([yshift=0.1em]node14.north)--([yshift=-0.1em]node24.south);

\draw [-&gt;,thick] ([xshift=-1em]node11.west)--([xshift=-0.1em]node11.west);
\draw [-&gt;,thick] ([xshift=0.1em]node11.east)--([xshift=-0.1em]node12.west);
\draw [-&gt;,thick] ([xshift=0.1em]node12.east)--([xshift=-0.1em]node13.west);
\draw [-&gt;,thick] ([xshift=0.1em]node13.east)--([xshift=-0.1em]node14.west);
\draw [-&gt;,thick] ([xshift=0.1em]node14.east)--([xshift=1em]node14.east);
}

\end{scope}
\end{tikzpicture}
\end{center}
}

\end{frame}

%%%------------------------------------------------------------------------------------------------------------
%%% ѭ����Ԫ����ơ��ݶ���ʧ��ѵ��������
\begin{frame}{��һ��������}
\begin{itemize}
\item \textbf{ѭ����Ԫ���}��ѭ����Ԫ����һ�������������ǰʱ�̵��������һʱ�̵�״̬�����ɵ�ǰʱ�̵�״̬
    \begin{displaymath}
    \textbf{h}_t = g(\textbf{x}_t, \textbf{h}_{t-1}; \theta)
    \end{displaymath}
    �ܶ��ַ�ʽ���$g(\cdot)$����������LSTM��GRU��
\item&lt;2-&gt; \textbf{�ݶ���ʧ/��ը}���������б䳤���ڷ��򴫲�ʱѭ����������������ľֲ��ݶ���˼��㣬��ᵼ��\alert{�ݶ���ʧ/��ը����}
    \begin{displaymath}
    \underbrace{0.2 \times 0.3 \times ... \times 0.2 \times 0.1}_{\text{100��}} \approx 0
    \end{displaymath}
    \vspace{-0.8em}
    \begin{itemize}
    \item ���Կ����ݶȲü��������ݶȵĴ�С
    \item Ҳ��������short-cut connection����в�����
    \end{itemize}
\item&lt;2-&gt; \textbf{ѵ��}�������Զ�΢�֣��ⲻ�Ǹ������� :)
\end{itemize}
\end{frame}

%%%------------------------------------------------------------------------------------------------------------
%%% ��ע��������
\begin{frame}{��ע��������(Self-Attention)}

\begin{itemize}
\item RNN LMЧ���ܺã����ǵ����й���,�ʻ�֮����Ϣ����·�����������׳����ݶ���ʧ���ݶȱ�ը�����⡣
\vspace{0.5em}
\begin{center}
\begin{tikzpicture}
\begin{scope}
\node [anchor=west] (w0) at (0,0) {$w_1$};
\node [anchor=west] (w1) at ([xshift=0.5em]w0.east) {$w_2$};
\node [anchor=west] (w2) at ([xshift=0.5em]w1.east) {$w_3$};
\node [anchor=west] (w3) at ([xshift=0.5em]w2.east) {$...$};
\node [anchor=west] (w4) at ([xshift=0.5em]w3.east) {$w_{m-1}$};
\node [anchor=west,fill=green!20!white] (w5) at ([xshift=0.5em]w4.east) {$w_{m}$};
\draw [-&gt;,thick,red] (w1.north).. controls +(130:0.5) and +(50:0.5) .. (w0.north);
\draw [-&gt;,thick,red] (w2.north).. controls +(130:0.5) and +(50:0.5) .. (w1.north);
\draw [-&gt;,thick,red] ([yshift=0.2em]w3.north).. controls +(130:0.5) and +(50:0.5) .. (w2.north);
\draw [-&gt;,thick,red] (w4.north).. controls +(130:0.5) and +(50:0.5) .. ([yshift=0.2em]w3.north);
\draw [-&gt;,thick,red] (w5.north).. controls +(130:0.5) and +(50:0.5) .. (w4.north);
\draw [-&gt;,very thick,red] ([xshift=-5em]w0.west) -- ([xshift=-6.5em]w0.west) node [pos=0,right] {\scriptsize{��Ϣ����}};


\end{scope}

\end{tikzpicture}
\end{center}
\item&lt;2-&gt; �ܷ񽫲�ͬλ��֮��Ĵʻ����Ϣ���ݵľ�������Ϊ1��


\begin{center}
\begin{tikzpicture}
\begin{scope}
\node [anchor=west] (w0) at (0,-2) {$w_1$};
\node [anchor=west] (w1) at ([xshift=0.5em]w0.east) {$w_2$};
\node [anchor=west] (w2) at ([xshift=0.5em]w1.east) {$w_3$};
\node [anchor=west] (w3) at ([xshift=0.5em]w2.east) {$...$};
\node [anchor=west] (w4) at ([xshift=0.5em]w3.east) {$w_{m-1}$};
\node [anchor=west,fill=green!20!white] (w5) at ([xshift=0.5em]w4.east) {$w_{m}$};
\draw [-&gt;,thick,red] (w5.north).. controls +(100:0.8) and +(50:0.8) .. (w0.north);
\draw [-&gt;,thick,red] (w5.north).. controls +(110:0.7) and +(50:0.7) .. (w1.north);
\draw [-&gt;,thick,red] (w5.north).. controls +(120:0.6) and +(50:0.6) .. ([yshift=0.2em]w3.north);
\draw [-&gt;,thick,red] (w5.north).. controls +(130:0.5) and +(50:0.5) .. (w4.north);
\draw [-&gt;,very thick,red] ([xshift=-5em]w0.west) -- ([xshift=-6.5em]w0.west) node [pos=0,right] {\scriptsize{��Ϣ����}};


\end{scope}

\end{tikzpicture}
\end{center}
\item&lt;3-&gt; \textbf{��ע��������(Self-Attention)}���ԺܺõĽ���������������⣬�ڳ��������Խ�ģ����ȡ���˺ܺõ�Ч��
	\begin{itemize}
	\item ����ֵı�ʾ���в�ͬλ��֮��ĸ��ӹ�ϵ
	\item ����ѵ�������Ч��
	\end{itemize}
	
        \textbf{Attention Is All You Need}\\
        \textbf{Vaswani et al., 2017, In Proc. of Neural Information Processing Systems, 6000-6010}
\end{itemize}

\end{frame}

%%%------------------------------------------------------------------------------------------------------------
%%% Transformer architecture
\begin{frame}{Transformer����ģ��(Vaswani et al., 2017)}
\begin{itemize}
\item һ���򵥵�����
\end{itemize}

\vspace{-2em}
\begin{center}
\begin{tikzpicture}
\begin{scope}
\node [anchor=west] (w0) at (0,0) {\footnotesize{$w_{0}$}};
\node [anchor=west] (w1) at ([xshift=4em]w0.east) {\footnotesize{$w_{1}$}};
\node [anchor=west] (w2) at ([xshift=4em]w1.east) {\footnotesize{$w_{2}$}};
\node [anchor=west] (w3) at ([xshift=4em]w2.east) {\footnotesize{$w_{3}$}};
\node [anchor=north] (index0) at ([yshift=0.5em]w0.south) {\tiny(index)};
\node [anchor=north] (index1) at ([yshift=0.5em]w1.south) {\tiny(index)};
\node [anchor=north] (index2) at ([yshift=0.5em]w2.south) {\tiny(index)};
\node [anchor=north] (index3) at ([yshift=0.5em]w3.south) {\tiny(index)};
\node [anchor=south,draw,inner sep=3pt] (e0) at ([yshift=1em]w0.north) {\tiny{$e_0=w_{0} \textbf{C}$}};
\node [anchor=south,draw,inner sep=3pt] (e1) at ([yshift=1em]w1.north) {\tiny{$e_1=w_{1} \textbf{C}$}};
\node [anchor=south,draw,inner sep=3pt] (e2) at ([yshift=1em]w2.north) {\tiny{$e_2=w_{2} \textbf{C}$}};
\node [anchor=south,draw,inner sep=3pt] (e3) at ([yshift=1em]w3.north) {\tiny{$e_3=w_{3} \textbf{C}$}};

\node [anchor=south,draw,inner sep=3pt] (h0) at ([xshift=-0.5em, yshift=1.5em]e0.north) {\tiny{$h_{0}=\textrm{SelfAtt}(e_0,e_3)$}};
\node [anchor=south,draw,inner sep=3pt] (h1) at ([xshift=0.5em, yshift=1.5em]e1.north) {\tiny{$h_{1}=\textrm{SelfAtt}(e_1,e_3)$}};
\node [anchor=south,draw,inner sep=3pt] (h2) at ([xshift=1.5em, yshift=1.5em]e2.north) {\tiny{$h_{2}=\textrm{SelfAtt}(e_2,e_3)$}};
\node [anchor=south,draw,minimum width=9em,inner sep=3pt] (f1) at ([xshift=0.5em, yshift=1.5em]h2.north) {\tiny{$f_3=\textrm{FNN}([h_0,h_1,h_2,e_3])$}};
\node [anchor=south,draw,minimum width=9em,inner sep=3pt] (o1) at ([yshift=1em]f1.north) {\tiny{$y=\textrm{Softmax}(f_3 \textbf{U})$}};
\node [anchor=south] (ylabel) at ([yshift=1em]o1.north) {\footnotesize{$\textrm{P}(w_4|w_{0}w_{1}w_{2}w_{3})$}};

\draw [-&gt;] ([yshift=0.1em]w0.north) -- ([yshift=-0.1em]e0.south);
\draw [-&gt;] ([yshift=0.1em]w1.north) -- ([yshift=-0.1em]e1.south);
\draw [-&gt;] ([yshift=0.1em]w2.north) -- ([yshift=-0.1em]e2.south);
\draw [-&gt;] ([yshift=0.1em]w3.north) -- ([yshift=-0.1em]e3.south);
\draw [-&gt;] ([yshift=0.1em]e0.north) -- ([xshift=0em,yshift=-0.1em]h0.south);
\draw [-&gt;] ([yshift=0.1em]e1.north) -- ([xshift=-0.5em,yshift=-0.1em]h1.south);
\draw [-&gt;] ([yshift=0.1em]e2.north) -- ([xshift=-1em,yshift=-0.1em]h2.south);
\draw [-&gt;] ([yshift=0.1em]e3.north) -- ([xshift=1em,yshift=-0.1em]h0.south);
\draw [-&gt;] ([yshift=0.1em]e3.north) -- ([xshift=1em,yshift=-0.1em]h1.south);
\draw [-&gt;] ([yshift=0.1em]e3.north) -- ([xshift=1em,yshift=-0.1em]h2.south);

\draw [-&gt;] ([yshift=0.1em]h0.north) -- ([xshift=-2em,yshift=-0.1em]f1.south);
\draw [-&gt;] ([yshift=0.1em]e3.north) -- ([xshift=2em,yshift=-0.1em]f1.south);
\draw [-&gt;] ([yshift=0.1em]h1.north) -- ([xshift=-1em,yshift=-0.1em]f1.south);
\draw [-&gt;] ([yshift=0.1em]h2.north) -- ([xshift=0em,yshift=-0.1em]f1.south);
\draw [-&gt;] ([yshift=0.1em]f1.north) -- ([yshift=-0.1em]o1.south);
\draw [-&gt;] ([yshift=0.1em]o1.north) -- ([yshift=-0.1em]ylabel.south);

\visible&lt;2-&gt;{
\node [anchor=south,draw,inner sep=3pt,fill=blue!20!white] (e0) at ([yshift=1em]w0.north) {\tiny{$e_0=w_{0} \textbf{C}$}};
\node [anchor=south,draw,inner sep=3pt,fill=blue!20!white] (e1) at ([yshift=1em]w1.north) {\tiny{$e_1=w_{1} \textbf{C}$}};
\node [anchor=south,draw,inner sep=3pt,fill=blue!20!white] (e2) at ([yshift=1em]w2.north) {\tiny{$e_2=w_{2} \textbf{C}$}};
\node [anchor=south,draw,inner sep=3pt,fill=blue!20!white] (e3) at ([yshift=1em]w3.north) {\tiny{$e_3=w_{3} \textbf{C}$}};
}

\visible&lt;2-&gt;{
\node [anchor=west] (embedinglabel0) at ([xshift=-5em,yshift=-2em]w0.south) {\scriptsize{{\blue \textbf{�ʵķֲ�ʽ��ʾ}}}};
\node [anchor=north west] (embedinglabel1) at ([yshift=0.3em]embedinglabel0.south west) {\scriptsize{ǰ���Ѿ����ܹ���}};
\node [anchor=north west] (embedinglabel2) at ([yshift=0.3em]embedinglabel1.south west) {\scriptsize{����One-hot��ʾ���}};
\node [anchor=north west] (embedinglabel3) at ([yshift=0.3em]embedinglabel2.south west) {\scriptsize{�¼���λ������}};
}

\visible&lt;3-&gt;{
\node [anchor=south,draw,inner sep=3pt,fill=ugreen!20!white] (h0) at ([xshift=-0.5em, yshift=1.5em]e0.north) {\tiny{$h_{0}=\textrm{SelfAtt}(e_0,e_3)$}};
\node [anchor=south,draw,inner sep=3pt,fill=ugreen!20!white] (h1) at ([xshift=0.5em, yshift=1.5em]e1.north) {\tiny{$h_{1}=\textrm{SelfAtt}(e_1,e_3)$}};
\node [anchor=south,draw,inner sep=3pt,fill=ugreen!20!white] (h2) at ([xshift=1.5em, yshift=1.5em]e2.north) {\tiny{$h_{2}=\textrm{SelfAtt}(e_2,e_3)$}};
}

\visible&lt;3-&gt;{
\node [anchor=west] (selfattlabel0) at ([xshift=3em]embedinglabel0.east) {\scriptsize{{\color{ugreen} \textbf{��ע��������}}}};
\node [anchor=west] (selfattlabel1) at ([yshift=-0.3em]selfattlabel0.south west) {\scriptsize{����ʻ�֮�����ض�}};
\node [anchor=west] (selfattlabel2) at ([yshift=-0.3em]selfattlabel1.south west) {\scriptsize{��ͷ��ע��������}};
\node [anchor=west] (directlabel0) at ([yshift=-0.3em]selfattlabel2.south west) {\scriptsize{\alert{\textbf{���潫�����}}}};
}

\visible&lt;4-&gt;{
\node [anchor=south,draw,minimum width=9em,inner sep=3pt,fill=orange!20!white] (f1) at ([xshift=0.5em, yshift=1.5em]h2.north) {\tiny{$f_3=\textrm{FNN}([h_0,h_1,h_2,e_3])$}};
\node [anchor=south,draw,minimum width=9em,inner sep=3pt,fill=orange!20!white] (o1) at ([yshift=1em]f1.north) {\tiny{$y=\textrm{Softmax}(f_3 \textbf{U})$}};
}

\visible&lt;4-&gt;{
\node [anchor=west] (ffnlabel0) at ([xshift=3em]selfattlabel0.east) {\scriptsize{{\color{orange} \textbf{ǰ��������������}}}};
\node [anchor=west] (ffnlabel1) at ([yshift=-0.3em]ffnlabel0.south west) {\scriptsize{˫��ȫ��������}};
\node [anchor=west] (ffnlabel2) at ([yshift=-0.3em]ffnlabel1.south west) {\scriptsize{�����ΪRelu}};
\node [anchor=west] (ffnlabel3) at ([yshift=-0.3em]ffnlabel2.south west) {\scriptsize{���ͨ��Softmax���}};
}


\end{scope}
\end{tikzpicture}
\end{center}

\end{frame}

%%%------------------------------------------------------------------------------------------------------------
%%% Transformer architecture
\begin{frame}{Transformer����ģ��(Vaswani et al., 2017)}
\begin{itemize}
\item ��ͷע��������
\end{itemize}

\vspace{-1.5em}
\begin{center}
\begin{tikzpicture}
\begin{scope}

\node [anchor=west,draw=black!30,inner sep=4pt,fill=ugreen!20!white] (Linear0) at (0,0) {\tiny{Linear}};
\node [anchor=south west,draw=black!50,fill=ugreen!20!white,draw,inner sep=4pt] (Linear01) at ([shift={(-0.2em,-0.2em)}]Linear0.south west) {\tiny{Linear}};
\node [anchor=south west,fill=ugreen!20!white,draw,inner sep=4pt] (Linear02) at ([shift={(-0.2em,-0.2em)}]Linear01.south west) {\tiny{Linear}};
\node [anchor=north] (Q) at ([xshift=0em,yshift=-1em]Linear02.south) {\footnotesize{$Q$}};

\node [anchor=west,draw=black!30,inner sep=4pt,fill=ugreen!20!white] (Linear1) at ([xshift=1.5em]Linear0.east) {\tiny{Linear}};
\node [anchor=south west,draw=black!50,fill=ugreen!20!white,draw,inner sep=4pt] (Linear11) at ([shift={(-0.2em,-0.2em)}]Linear1.south west) {\tiny{Linear}};
\node [anchor=south west,fill=ugreen!20!white,draw,inner sep=4pt] (Linear12) at ([shift={(-0.2em,-0.2em)}]Linear11.south west) {\tiny{Linear}};
\node [anchor=north] (K) at ([xshift=0em,yshift=-1em]Linear12.south) {\footnotesize{$K$}};

\node [anchor=west,draw=black!30,inner sep=4pt,fill=ugreen!20!white] (Linear2) at ([xshift=1.5em]Linear1.east) {\tiny{Linear}};
\node [anchor=south west,draw=black!50,fill=ugreen!20!white,draw,inner sep=4pt] (Linear21) at ([shift={(-0.2em,-0.2em)}]Linear2.south west) {\tiny{Linear}};
\node [anchor=south west,fill=ugreen!20!white,draw,inner sep=4pt] (Linear22) at ([shift={(-0.2em,-0.2em)}]Linear21.south west) {\tiny{Linear}};
\node [anchor=north] (V) at ([xshift=0em,yshift=-1em]Linear22.south) {\footnotesize{$V$}};

\node [anchor=south,draw=black!30,minimum width=9em,inner sep=4pt,fill=blue!20!white] (Scale) at ([yshift=1em]Linear1.north) {\tiny{Scaled Dot-Product Attention}};
\node [anchor=south west,draw=black!50,minimum width=9em,fill=blue!20!white,draw,inner sep=4pt] (Scale1) at ([shift={(-0.2em,-0.2em)}]Scale.south west) {\tiny{Scaled Dot-Product Attention}};
\node [anchor=south west,fill=blue!20!white,draw,minimum width=9em,inner sep=4pt] (Scale2) at ([shift={(-0.2em,-0.2em)}]Scale1.south west) {\tiny{Scaled Dot-Product Attention}};

\node [anchor=south,draw,minimum width=4em,inner sep=4pt,fill=yellow!30] (Concat) at ([yshift=1em]Scale2.north) {\tiny{Concat}};

\node [anchor=south,draw,minimum width=4em,inner sep=4pt,fill=ugreen!20!white] (Linear) at ([yshift=1em]Concat.north) {\tiny{Linear}};


\draw [-&gt;] ([yshift=0.1em]Q.north) -- ([yshift=-0.1em]Linear02.south);
\draw [-,draw=black!50] ([yshift=0.1em]Q.north) -- ([xshift=0.2em,yshift=-0.1em]Linear02.south);
\draw [-,draw=black!30] ([yshift=0.1em]Q.north) -- ([xshift=0.4em,yshift=-0.1em]Linear02.south);

\draw [-&gt;] ([yshift=0.1em]K.north) -- ([yshift=-0.1em]Linear12.south);
\draw [-,draw=black!50] ([yshift=0.1em]K.north) -- ([xshift=0.2em,yshift=-0.1em]Linear12.south);
\draw [-,draw=black!30] ([yshift=0.1em]K.north) -- ([xshift=0.4em,yshift=-0.1em]Linear12.south);

\draw [-&gt;] ([yshift=0.1em]V.north) -- ([yshift=-0.1em]Linear22.south);
\draw [-,draw=black!50] ([yshift=0.1em]V.north) -- ([xshift=0.2em,yshift=-0.1em]Linear22.south);
\draw [-,draw=black!30] ([yshift=0.1em]V.north) -- ([xshift=0.4em,yshift=-0.1em]Linear22.south);

\draw [-&gt;] ([yshift=0em]Linear02.north) -- ([yshift=1em]Linear02.north);
\draw [-,draw=black!50] ([yshift=0em]Linear01.north) -- ([yshift=0.8em]Linear01.north);
\draw [-,draw=black!30] ([yshift=0em]Linear0.north) -- ([yshift=0.6em]Linear0.north);

\draw [-&gt;] ([yshift=0em]Linear12.north) -- ([yshift=1em]Linear12.north);
\draw [-,draw=black!50] ([yshift=0em]Linear11.north) -- ([yshift=0.8em]Linear11.north);
\draw [-,draw=black!30] ([yshift=0em]Linear1.north) -- ([yshift=0.6em]Linear1.north);

\draw [-&gt;] ([yshift=0em]Linear22.north) -- ([yshift=1em]Linear22.north);
\draw [-,draw=black!50] ([yshift=0em]Linear21.north) -- ([yshift=0.8em]Linear21.north);
\draw [-,draw=black!30] ([yshift=0em]Linear2.north) -- ([yshift=0.6em]Linear2.north);

\draw [-&gt;] ([yshift=0em]Scale2.north) -- ([yshift=0em]Concat.south);
\draw [-,draw=black!50] ([yshift=0em]Scale1.north) -- ([yshift=0.8em]Scale1.north);
\draw [-,draw=black!30] ([yshift=0em]Scale.north) -- ([yshift=0.6em]Scale.north);

\draw [-&gt;] ([yshift=0em]Concat.north) -- ([yshift=0em]Linear.south);
\draw [-&gt;] ([yshift=0em]Linear.north) -- ([yshift=1em]Linear.north);

\node [anchor=west] (Multiheadlabel0) at ([xshift=-5em,yshift=-1.2em]Q.south) {\scriptsize{{\blue \textbf{��ͷע����}}}};
\node [anchor=north west] (Multiheadlabel1) at ([yshift=0em]Multiheadlabel0.south west) {\scriptsize{$MultiHead(Q,K,V)=Concat(head_1,...head_n)W^0$}};
\node [anchor=north west] (Multiheadlabel2) at ([yshift=0.2em]Multiheadlabel1.south west) {\scriptsize{������ѹ���ɶ��ά�Ƚ�С��������ֱ�����ע����}};
\node [anchor=north west] (Multiheadlabel3) at ([yshift=0.2em]Multiheadlabel2.south west) {\scriptsize{�ٰѽ���������������Ա任�õ��������}};


\visible&lt;2-&gt;{
\node [anchor=south west,fill=white,draw,inner sep=4pt,minimum width=3.5em,fill=blue!20!white] (MatMul) at ([xshift=8em]Linear22.south west) {\tiny{MatMul}};
\node [anchor=north] (Q1) at ([xshift=-1em,yshift=-1em]MatMul.south) {\footnotesize{$Q$}};
\node [anchor=north] (K1) at ([xshift=1em,yshift=-1em]MatMul.south) {\footnotesize{$K$}};
\node [anchor=south,draw,inner sep=4pt,fill=yellow!30] (Scale3) at ([yshift=1em]MatMul.north) {\tiny{Scale}};
\node [anchor=south,draw,inner sep=4pt,fill=purple!20,minimum width=3.5em] (Mask) at ([yshift=0.8em]Scale3.north) {\tiny{Mask(opt.)}};
\node [anchor=south,draw,inner sep=4pt,fill=ugreen!20!white] (SoftMax) at ([yshift=1em]Mask.north) {\tiny{SoftMax}};
\node [anchor=south,draw,minimum width=3.5em,inner sep=4pt,fill=blue!20!white] (MatMul1) at ([xshift=1.5em,yshift=1em]SoftMax.north) {\tiny{MatMul}};
\node [anchor=north] (V1) at ([xshift=2em]K1.north) {\footnotesize{$V$}};
\node [anchor=north] (null) at ([yshift=0.8em]MatMul1.north) {};

\node [rectangle,draw, densely dashed,inner sep=0.4em] [fit = (MatMul) (MatMul1) (Q1) (K1) (V1) (null)] (inputshadow) {};

\draw [-&gt;] ([yshift=0.1em]Q1.north) -- ([xshift=-1em,yshift=-0.1em]MatMul.south);
\draw [-&gt;] ([yshift=0.1em]K1.north) -- ([xshift=1em,yshift=-0.1em]MatMul.south);
\draw [-&gt;] ([yshift=0.1em]MatMul.north) -- ([yshift=-0.1em]Scale3.south);
\draw [-&gt;] ([yshift=0.1em]Scale3.north) -- ([yshift=-0.1em]Mask.south);
\draw [-&gt;] ([yshift=0.1em]Mask.north) -- ([yshift=-0.1em]SoftMax.south);
\draw [-&gt;] ([yshift=0.1em]SoftMax.north) -- ([yshift=0.9em]SoftMax.north);
\draw [-&gt;] ([yshift=0.1em]V1.north) -- ([yshift=9.1em]V1.north);
\draw [-&gt;] ([yshift=0.1em]MatMul1.north) -- ([yshift=0.8em]MatMul1.north);

\draw [-&gt;,dashed,red,thick] ([xshift=0.1em]Scale.east) .. controls +(east:1) and +(west:1) .. ([xshift=-0.1em,yshift=1em]inputshadow.west);

\node [anchor=west] (Attentionlabel0) at ([xshift=-2em,yshift=-1.2em]Q1.south) {\scriptsize{{\color{ugreen} \textbf{���ڵ�˵���ע����}}}};
\node [anchor=north west] (Attentionlabel1) at ([yshift=0.3em]Attentionlabel0.south west) {\scriptsize{$head_i=softmax(\frac{QK^{T}}{\sqrt{d_k}})V$}};
\node [anchor=north west] (Attentionlabel2) at ([yshift=0.6em]Attentionlabel1.south west) {\scriptsize{����õ�λ�������ļ�Ȩ��}};
\node [anchor=north west] (Attentionlabel3) at ([yshift=0.2em]Attentionlabel2.south west) {\scriptsize{Q,K,V������ͬ��}};
}

\end{scope}
\end{tikzpicture}
\end{center}

\end{frame}

%%%------------------------------------------------------------------------------------------------------------
%%% evaluation
\begin{frame}{����ģ������}
\begin{itemize}
\item ����ģ�͵�����ָ�� - �����(Perplexity, PPL)
\begin{itemize}
\item ����ģ��Ԥ��һ����������������
\item �����Խ�ͣ���ģ��Ч��Խ��
\end{itemize}
\vspace{0.5em}
\begin{displaymath}
\textrm{PPL}(w_1 ... w_m)=\textrm{P}(w_1 ... w_m)^{-1/m}
\end{displaymath}
\vspace{-0.5em}
\item&lt;2-&gt; Penn Treebank(PTB)�ϵ����۽��
\end{itemize}
\vspace{0.0em}
\visible&lt;2-&gt;{
\begin{tabular}{l | l | l | r}
ģ�� &amp; ���� &amp; ��� &amp; PPL \\ \hline
FNN LM &amp; Bengio et al. &amp; 2003 &amp; 162.2 \\
RNN LM &amp; Mikolov et al. &amp; 2010 &amp; 124.7 \\
RNN-LDA LM &amp; Mikolov et al. &amp; 2012 &amp; 92.0 \\
RNN(LSTM) LM &amp; Zaremba et al. &amp; 2014 &amp; 78.4 \\
RHN &amp; Zilly et al. &amp; 2016 &amp; 65.4 \\
RNN(AWD-LSTM) LM &amp; Merity et al. &amp; 2018 &amp; 58.8 \\
GPT-2 (Transformer) &amp; Radford et al. &amp; 2019 &amp; 35.7
\end{tabular}
}

\end{frame}

%%%------------------------------------------------------------------------------------------------------------
\subsection{��Ƕ��}

%%%------------------------------------------------------------------------------------------------------------
%%% �ʵ�one-hot��distributed��ʾ
\begin{frame}{���ʵı�ʾ}
\begin{itemize}
\item ��α�ʾһ�����ʣ�
    \begin{itemize}
    \item \textbf{One-hot}: ������һ���ʵ�$V$���������10k�����ʣ������б�š�ÿ�����ʶ����Ա�ʾΪ10kά��one-hot���������ڱ���Ǹ�ά��Ϊ1������Ϊ0
    \item&lt;2-&gt; \textbf{Distributed}: ������������ģ�ͣ�ÿ�����ʿ��Ա���ʾΪһ��ʵ��������ÿһά����Ӧһ��``����'' - \alert{��Ƕ��}
    \end{itemize}
\end{itemize}

\begin{center}
\begin{tikzpicture}
\begin{scope}
\node [anchor=north west] (o1) at (0,0) {\footnotesize{$\begin{bmatrix} 0 \\ 1 \\ 0 \\ 0 \\ 0 \\ ... \\ 0 \end{bmatrix}$}};
\node [anchor=north west] (o2) at ([xshift=1em]o1.north east) {\footnotesize{$\begin{bmatrix} 0 \\ 0 \\ 0 \\ 1 \\ 0 \\ ... \\ 0 \end{bmatrix}$}};
\node [anchor=north east] (v) at ([xshift=-0em]o1.north west) {\footnotesize{$\begin{matrix} \textrm{\ \ \ \ \ ��}_1 \\ \textrm{\ \ ����}_2 \\ \textrm{\ \ \ \ \ ��}_3 \\ \textrm{\ \ ����}_4 \\ \textrm{\ \ ����}_5 \\ ... \\ \textrm{���}_{10k} \end{matrix}$}};
\node [anchor=south] (w1) at (o1.north) {\footnotesize{����}};
\node [anchor=south] (w2) at (o2.north) {\footnotesize{����}};
\node [anchor=north] (label) at (o1.south) {\footnotesize{���ʵ�one-hot��ʾ}};
\visible&lt;3-&gt;{
\node [anchor=south,fill=red!20!white] (cosine) at (w1.north) {\footnotesize{$cosine(\textrm{`����'},\textrm{`����'})=0$}};
}
\end{scope}

\visible&lt;2-&gt;{
\begin{scope}[xshift=2in]
\node [anchor=north west] (o1) at (0,0) {\footnotesize{$\begin{bmatrix} .1 \\ -1 \\ 2 \\ ... \\ 0 \end{bmatrix}$}};
\node [anchor=north west] (o2) at ([xshift=1em]o1.north east) {\footnotesize{$\begin{bmatrix} 1 \\ 2 \\ .2 \\ ... \\ -1 \end{bmatrix}$}};
\node [anchor=north east] (v) at ([xshift=-0em]o1.north west) {\footnotesize{$\begin{matrix} \textrm{\ \ \ ����}_1 \\ \textrm{\ \ \ ����}_2 \\ \textrm{\ \ \ ����}_3 \\ ... \\ \textrm{����}_{512} \end{matrix}$}};
\node [anchor=south] (w1) at (o1.north) {\footnotesize{����}};
\node [anchor=south] (w2) at (o2.north) {\footnotesize{����}};
\node [anchor=north] (label) at ([yshift=-2em]o1.south) {\footnotesize{���ʵķֲ�ʽ��ʾ(��Ƕ��)}};
\visible&lt;3-&gt;{
\node [anchor=south,fill=red!20!white] (cosine) at (w1.north) {\footnotesize{$cosine(\textrm{`����'},\textrm{`����'})=0.5$}};
}
\end{scope}
}
\end{tikzpicture}
\end{center}

\end{frame}

%%%------------------------------------------------------------------------------------------------------------
%%% �ֲ�ʽ��ʾ���ŵ�
\begin{frame}{Ϊʲô��Ҫ�ֲ�ʽ��ʾ��}
\begin{itemize}
\item \textbf{һ����Ȼ������}���ֲ�ʽ��ʾ��ÿһά����ʲô��˼
    \begin{itemize}
    \item ���԰�ÿһά������Ϊһ�����ԣ����磺�Ա����ߵ�
    \item ���ǣ�ģ�͸�����ǰ�һ��ά�ȿ����������һ��``�̻�''����һ��ͳ�������ϵ�``����''�������˹����ɵ�����
    \end{itemize}
\item&lt;2-&gt; �����ַ�����ʲô�ô���
    \begin{itemize}
    \item �����׿̻�����֮���\alert{������}
    \item �����ռ��ʾģ�Ϳ��Ը�׼ȷ�Ŀ̻��͹���������Ƿ��㼴һ���ж�
    \end{itemize}
\item&lt;2-&gt; Ԥ����һ��������
    \begin{itemize}
    \item �ֲ�ʽ��ʾ������ָ��``����''��``����''�����Ƶ�
    \item ��ʹ``����''û����������г��ֹ���ϵͳ��Ȼ����ͨ������``����''�������Խ���Ԥ��
    \end{itemize}
    \begin{tabular}{l | l}
    ���� Ҫ �ڷ� һ�� \_\_\_\_\_ &amp; Ԥ���¸��� \\ \hline
    ���� Ҫ �ڷ� һ�� \alert{����} &amp; ���� \\
    ���� Ҫ �ڷ� һ�� \blue{����} &amp; û������������Ȼ�Ǻ���Ԥ��
    \end{tabular}
\end{itemize}
\end{frame}

%%%------------------------------------------------------------------------------------------------------------
%%% ��ʵ������ʵķֲ�ʽ��ʾ
\begin{frame}{�ֲ�ʽ��ʾ�Ŀ��ӻ�}
\begin{itemize}
\item \textbf{һ������������}������ $\to$ ����\\
    \begin{displaymath}
    \vv{\textrm{����}} - \vv{\textrm{����}} + \vv{\textrm{Ů��}} = \vv{\textrm{����}}
    \end{displaymath}
    ���$\vv{\textrm{word}}$��ʾ���ʵķֲ�ʽ������ʾ
\item ����ĴʵĿ��ӻ������ƵĴʾ���һ��
\end{itemize}
\begin{center}
\includegraphics[scale=0.4]{./Figures/word-graph.png}
\end{center}
\end{frame}

%%%------------------------------------------------------------------------------------------------------------
%%% ������ģ���еĴ�Ƕ��
\begin{frame}{������ģ���еĴ�Ƕ��}
\begin{itemize}
\item ��������ģ���У���Ҫ�Ѵʱ�ʾ�����ķֲ�ʽ��ʾ
    \begin{itemize}
    \item&lt;2-&gt; ����$\textbf{C}$�Ǵ�Ƕ�����ÿһ�ж�Ӧһ���ʵķֲ�ʽ��ʾ
    \item&lt;3-&gt; $\textbf{C}$����������ģ��ѵ����Ҳ������������ģ��ѵ�����̶���Ƕ�룬������ģ��רע��Ƭ�ε�ѧϰ
    \end{itemize}
\end{itemize}

\vspace{-0.5em}
\begin{center}
\begin{tikzpicture}
\begin{scope}
\node [anchor=center,inner sep=2pt] (e) at (0,0) {\small{$e=w$}};
\node [anchor=west,inner sep=2pt] (c) at (e.east) {\small{$\textbf{C}$}};

\begin{pgfonlayer}{background}
\node [rectangle,inner sep=0.4em,draw,fill=blue!20!white] [fit = (e) (c)] (box) {};
\end{pgfonlayer}

\draw [-&gt;,thick] ([yshift=-1em]box.south)--([yshift=-0.1em]box.south) node [pos=0,below] (bottom1) {\small{����$w$}};
\draw [-&gt;,thick] ([yshift=0.1em]box.north)--([yshift=1em]box.north) node [pos=1,above] (top1) {\scriptsize{$e$=(8,.2,-1,.9,...,1)}};
\node [anchor=north] (bottom2) at ([yshift=0.3em]bottom1.south) {\scriptsize{$w$=(0,0,1,0,...,0)}};
\node [anchor=south] (top2) at ([yshift=-0.3em]top1.north) {\small{$w$�ķֲ�ʽ��ʾ}};

\visible&lt;2-&gt;{
\node [anchor=north west,fill=red!20!white] (cmatrix) at ([xshift=3em,yshift=1.0em]c.north east) {\scriptsize{$\begin{pmatrix} 1 &amp; .2 &amp; -.2 &amp; 8 &amp; ... &amp; 0 \\ .6 &amp; .8 &amp; -2 &amp; 1 &amp; ... &amp; -.2 \\ 8 &amp; .2 &amp; -1 &amp; .9 &amp; ... &amp; 2.3 \\ 1 &amp; 1.2 &amp; -.9 &amp; 3 &amp; ... &amp; .2 \\ ... &amp; ... &amp; ... &amp; ... &amp; ... &amp; ... \\ 1 &amp; .3 &amp; 3 &amp; .9 &amp; ... &amp; 5.1 \end{pmatrix}$}};
\node [anchor=west,inner sep=2pt,fill=red!30!white] (c) at (e.east) {\small{$\textbf{C}$}};
\draw [&lt;-,thick] (c.east) -- ([xshift=3em]c.east);
}

\visible&lt;3-&gt;{
\node [anchor=south,draw,fill=green!20!white] (e2) at ([yshift=1.5em]cmatrix.north) {\scriptsize{�ⲿ��Ƕ��ϵͳ�õ���$\textbf{C}$}};
\draw [-&gt;,very thick,dashed] (e2.south) -- (cmatrix.north);
}

\end{scope}
\end{tikzpicture}
\end{center}

\vspace{-1.0em}

\begin{itemize}
\item&lt;4-&gt; ��Ƕ�����ѧϰ�õ���
    \begin{itemize}
    \item ���Ժ�����ģ�͵���������һ��ѵ���������ٶȽ���
    \item Ҳ���Կ���ʹ��Ч�ʸ��ߵ��ⲿģ�ͣ���word2vec�� Glove�ȣ���������ʹ�ø����ģ������
    \end{itemize}
\end{itemize}

\end{frame}

%%%------------------------------------------------------------------------------------------------------------
\subsection{���ӱ�ʾģ�ͼ�Ԥѵ��}

%%%------------------------------------------------------------------------------------------------------------
%%% ��Ƕ�������
\begin{frame}{������``��''}
\begin{itemize}
\item ��Ƕ���Ѿ���Ϊ���NLPϵͳ�ı��䣬��ȻҲ���������ֻ�ʽ�淨��������``embed everything''�Ŀںţ����Ǵ�Ƕ��Ҳ������
    \begin{itemize}
    \item ÿ���ʶ���ӦΨһ��������ʾ�����Ƕ���һ�ʶ������󣬴�����Ҫͨ�������Ľ������֡�һ�����������ӣ�
    \end{itemize}
    \vspace{0.3em}
    \hspace{6em} Jobs was the CEO of \alert{\underline{apple}}.\\
    \hspace{6em} He finally ate the \alert{\underline{apple}}.
\item&lt;2-&gt; ������������Ϣ
    \begin{itemize}
    \item ���������������µ�˼�������ܼ򵥵ؿ��Ǵʵı�ʾ��Ӧͬʱ��������������Ϣ
    \item ���ھ����е�һ����(����λ��)��ͬʱ��ʾ�ʺ�������
    \end{itemize}
\end{itemize}

\visible&lt;2-&gt;{
\begin{center}
\begin{tikzpicture}
\begin{scope}
\node [anchor=west] (node1) at (0,0) {\footnotesize{Jobs was the CEO of}};
\node [anchor=west] (node2) at ([xshift=-0.2em,yshift=-0.05em]node1.east) {\footnotesize{\alert{\underline{apple}}}};
\node [anchor=west] (node3) at ([xshift=-0.2em,yshift=-0.1em]node2.east) {\footnotesize{.}};
\node [anchor=south,inner sep=2pt,minimum width=2.4em,fill=red!20!white] (node4) at ([yshift=1.5em]node2.north) {\scriptsize{��}};
\node [anchor=north] (label) at ([xshift=1em]node1.south) {\scriptsize{\textbf{�ʱ�ʾģ��}}};
\draw [-&gt;,thick] (node2.north) -- (node4.south);
\end{scope}
\begin{scope}[xshift=2in]
\node [anchor=west] (node1) at (0,0) {\footnotesize{Jobs was the CEO of}};
\node [anchor=west] (node2) at ([xshift=-0.2em,yshift=-0.05em]node1.east) {\footnotesize{\alert{\underline{apple}}}};
\node [anchor=west] (node3) at ([xshift=-0.2em,yshift=-0.1em]node2.east) {\footnotesize{.}};
\node [anchor=south,inner sep=2pt,minimum width=2.4em,fill=red!20!white] (node4) at ([yshift=1.5em]node2.north) {\scriptsize{��}};
\node [anchor=south,inner sep=2pt,minimum width=2.4em,fill=blue!20!white] (node5) at (node4.north) {\scriptsize{������}};
\node [anchor=north] (label) at ([xshift=1em]node1.south) {\scriptsize{\textbf{��+�����ı�ʾģ��}}};
\draw [-&gt;,thick] (node2.north) -- (node4.south);
\draw [-&gt;] ([xshift=1em]node1.north west) .. controls +(north:1) and +(west:2) .. ([yshift=0.2em]node5.west);
\draw [-&gt;] ([xshift=3em]node1.north west) .. controls +(north:0.8) and +(west:1.5) .. ([yshift=-0.2em]node5.west);
\node [anchor=east] (morelines) at ([xshift=-1.5em]node4.west) {...};
\end{scope}
\end{tikzpicture}
\end{center}
}
\end{frame}

%%%------------------------------------------------------------------------------------------------------------
%%% �����ı�ʾģ��
\begin{frame}{��ʾ������Ƭ�� - �����ı�ʾģ��}
\begin{itemize}
\item ������ģ�����Ѿ�������ÿ��λ�õ������ı�ʾ��Ϣ
    \begin{itemize}
    \item ��RNN LMΪ����λ��$i$�������������һ��$w_1...w_i$�ı�ʾ
    \end{itemize}
\end{itemize}

\vspace{-0.5em}
\begin{center}
\begin{tikzpicture}
\begin{scope}
\tikzstyle{rnnnode} = [draw,inner sep=5pt,minimum width=4em,minimum height=1.5em,fill=green!30!white,blur shadow={shadow xshift=1pt,shadow yshift=-1pt}]
\node [anchor=west,rnnnode] (node11) at (0,0) {\scriptsize{RNN Cell}};
\node [anchor=west,rnnnode] (node12) at ([xshift=2em]node11.east) {\scriptsize{RNN Cell}};
\node [anchor=west,rnnnode] (node13) at ([xshift=2em]node12.east) {\scriptsize{RNN Cell}};
\node [anchor=west,rnnnode] (node14) at ([xshift=2em]node13.east) {\scriptsize{RNN Cell}};

\node [anchor=north,rnnnode,fill=red!30!white] (e1) at ([yshift=-1.2em]node11.south) {\scriptsize{embedding}};
\node [anchor=north,rnnnode,fill=red!30!white] (e2) at ([yshift=-1.2em]node12.south) {\scriptsize{embedding}};
\node [anchor=north,rnnnode,fill=red!30!white] (e3) at ([yshift=-1.2em]node13.south) {\scriptsize{embedding}};
\node [anchor=north,rnnnode,fill=red!30!white] (e4) at ([yshift=-1.2em]node14.south) {\scriptsize{embedding}};
\node [anchor=north] (w1) at ([yshift=-1em]e1.south) {\footnotesize{�ǲ�˹}};
\node [anchor=north] (w2) at ([yshift=-1em]e2.south) {\footnotesize{��ְ}};
\node [anchor=north] (w3) at ([yshift=-1em]e3.south) {\footnotesize{��}};
\node [anchor=north] (w4) at ([yshift=-1em]e4.south) {\footnotesize{ƻ��}};

\draw [-&gt;,thick] ([yshift=0.1em]w1.north)--([yshift=-0.1em]e1.south);
\draw [-&gt;,thick] ([yshift=0.1em]w2.north)--([yshift=-0.1em]e2.south);
\draw [-&gt;,thick] ([yshift=0.1em]w3.north)--([yshift=-0.1em]e3.south);
\draw [-&gt;,thick] ([yshift=0.1em]w4.north)--([yshift=-0.1em]e4.south);

\draw [-&gt;,thick] ([yshift=0.1em]e1.north)--([yshift=-0.1em]node11.south);
\draw [-&gt;,thick] ([yshift=0.1em]e2.north)--([yshift=-0.1em]node12.south);
\draw [-&gt;,thick] ([yshift=0.1em]e3.north)--([yshift=-0.1em]node13.south);
\draw [-&gt;,thick] ([yshift=0.1em]e4.north)--([yshift=-0.1em]node14.south);

\node [anchor=south,rnnnode] (node21) at ([yshift=1.5em]node11.north) {\scriptsize{RNN Cell}};
\node [anchor=south,rnnnode] (node22) at ([yshift=1.5em]node12.north) {\scriptsize{RNN Cell}};
\node [anchor=south,rnnnode] (node23) at ([yshift=1.5em]node13.north) {\scriptsize{RNN Cell}};
\node [anchor=south,rnnnode] (node24) at ([yshift=1.5em]node14.north) {\scriptsize{RNN Cell}};

\node [anchor=south] (node31) at ([yshift=1.0em]node21.north) {\scriptsize{�ı�ʾ}};
\node [anchor=south west] (node31new) at ([yshift=-0.3em]node31.north west) {\scriptsize{``�ǲ�˹''}};
\node [anchor=south] (node32) at ([yshift=1.0em]node22.north) {\scriptsize{�ı�ʾ\ \ \ }};
\node [anchor=south west] (node32new) at ([yshift=-0.3em]node32.north west) {\scriptsize{``�ǲ�˹ ��ְ''}};
\node [anchor=south] (node33) at ([yshift=1.0em]node23.north) {\scriptsize{�ı�ʾ\ \ \ \ \ \ \ \ }};
\node [anchor=south west] (node33new) at ([yshift=-0.3em]node33.north west) {\scriptsize{``�ǲ�˹ ��ְ ��''}};
\node [anchor=south] (node34) at ([yshift=1.0em]node24.north) {\scriptsize{�ı�ʾ\ \ \ \ \ \ \ \ }};
\node [anchor=south west] (node34new) at ([yshift=-0.3em]node34.north west) {\scriptsize{``�ǲ�˹ ��ְ �� ƻ��''}};

\draw [-&gt;,thick] ([yshift=0.1em]node21.north)--([yshift=-0.1em]node31.south);
\draw [-&gt;,thick] ([yshift=0.1em]node22.north)--([yshift=-0.1em]node32.south);
\draw [-&gt;,thick] ([yshift=0.1em]node23.north)--([yshift=-0.1em]node33.south);
\draw [-&gt;,thick] ([yshift=0.1em]node24.north)--([yshift=-0.1em]node34.south);

\draw [-&gt;,thick] ([xshift=-1em]node21.west)--([xshift=-0.1em]node21.west);
\draw [-&gt;,thick] ([xshift=0.1em]node21.east)--([xshift=-0.1em]node22.west);
\draw [-&gt;,thick] ([xshift=0.1em]node22.east)--([xshift=-0.1em]node23.west);
\draw [-&gt;,thick] ([xshift=0.1em]node23.east)--([xshift=-0.1em]node24.west);
\draw [-&gt;,thick] ([xshift=0.1em]node24.east)--([xshift=1em]node24.east);

\draw [-&gt;,thick] ([yshift=0.1em]node11.north)--([yshift=-0.1em]node21.south);
\draw [-&gt;,thick] ([yshift=0.1em]node12.north)--([yshift=-0.1em]node22.south);
\draw [-&gt;,thick] ([yshift=0.1em]node13.north)--([yshift=-0.1em]node23.south);
\draw [-&gt;,thick] ([yshift=0.1em]node14.north)--([yshift=-0.1em]node24.south);

\draw [-&gt;,thick] ([xshift=-1em]node11.west)--([xshift=-0.1em]node11.west);
\draw [-&gt;,thick] ([xshift=0.1em]node11.east)--([xshift=-0.1em]node12.west);
\draw [-&gt;,thick] ([xshift=0.1em]node12.east)--([xshift=-0.1em]node13.west);
\draw [-&gt;,thick] ([xshift=0.1em]node13.east)--([xshift=-0.1em]node14.west);
\draw [-&gt;,thick] ([xshift=0.1em]node14.east)--([xshift=1em]node14.east);

\visible&lt;2-&gt;{
\node [anchor=south] (toplabel1) at ([yshift=2em,xshift=-2em]node32new.north) {\footnotesize{``ƻ��''�ı�ʾ��}};
\node [anchor=west,fill=blue!20!white,minimum width=3em] (toplabel2) at (toplabel1.east) {\footnotesize{������}};
}
\visible&lt;3-&gt;{
\node [anchor=west,fill=red!20!white,minimum width=3em] (toplabel3) at (toplabel2.east) {\footnotesize{��}};
}

\begin{pgfonlayer}{background}
\visible&lt;3-&gt;{
\node [rectangle,inner sep=2pt,draw,thick,dashed,red] [fit = (e4)] (r2) {};
\draw [-&gt;,thick,red] (r2.west) .. controls +(west:0.8) and +(south:2) .. ([xshift=1.3em]toplabel3.south);
}
\visible&lt;2-&gt;{
\node [rectangle,inner sep=2pt,draw,thick,dashed,ublue,fill=white] [fit = (node33) (node33new)] (r1) {};
\draw [-&gt;,thick,ublue] ([xshift=-2em]r1.north) .. controls +(north:0.7) and +(south:0.7) .. ([xshift=-0.5em]toplabel2.south);
}
\end{pgfonlayer}

\end{scope}
\end{tikzpicture}
\end{center}

\end{frame}

%%%------------------------------------------------------------------------------------------------------------
%%% ��ǿ��ı�ʾģ�� - ELMO
\begin{frame}{��ǿ�ı�ʾģ�� - ELMO}
\begin{itemize}
\item \textbf{ELMO}(Embedding from Language Models)����˵�������˻�������ģ�͵�Ԥѵ�����ȳ�
    \begin{itemize}
    \item ��Ȼʹ��RNN�ṹ������ѭ����Ԫ������LSTM
    \item ͬʱ�����������Һ���������Ľ�ģ��ʽ��ͬʱ��ʾһ������˺��Ҷ˵�������
    \item �ں����в��������͸�����Ӧ�ã��ṩ�˸��ḻ����Ϣ
    \end{itemize}
\end{itemize}
\vspace{0.5em}
\begin{center}
\begin{tikzpicture}
\begin{scope}[scale=1.2]

\node [anchor=west,draw,inner sep=4pt,fill=blue!20!white] (Lstm0) at (0,0) {\scriptsize{Lstm}};
\node [anchor=west,draw,inner sep=4pt,fill=blue!20!white] (Lstm1) at ([xshift=1em]Lstm0.east) {\scriptsize{Lstm}};
\node [anchor=west,inner sep=4pt] (sep) at ([xshift=1em]Lstm1.east) {\scriptsize{...}};
\node [anchor=west,draw,inner sep=4pt,fill=blue!20!white] (Lstm2) at ([xshift=1em]sep.east) {\scriptsize{Lstm}};

\node [anchor=south,draw,inner sep=4pt,fill=blue!20!white] (Lstm3) at ([yshift=1em]Lstm0.north) {\scriptsize{Lstm}};
\node [anchor=west,draw,inner sep=4pt,fill=blue!20!white] (Lstm4) at ([xshift=1em]Lstm3.east) {\scriptsize{Lstm}};
\node [anchor=west,inner sep=4pt] (sep1) at ([xshift=1em]Lstm4.east) {\scriptsize{...}};
\node [anchor=west,draw,inner sep=4pt,fill=blue!20!white] (Lstm5) at ([xshift=1em]sep1.east) {\scriptsize{Lstm}};

\node [rectangle,rounded corners,draw=black!50,densely dashed,inner sep=0.4em] [fit = (Lstm0) (Lstm2) (Lstm3) (Lstm5)] (inputshadow) {};

\node [anchor=north,draw,inner sep=4pt,fill=ugreen!20!white,minimum width=2em] (e1) at ([xshift=-2em,yshift=-1em]Lstm2.south) {\scriptsize{$E_1$}};
\node [anchor=west,draw,inner sep=4pt,fill=ugreen!20!white,minimum width=2em] (e2) at ([xshift=1em]e1.east) {\scriptsize{$E_2$}};
\node [anchor=west,inner sep=4pt] (sep5) at ([xshift=1em]e2.east) {\scriptsize{...}};
\node [anchor=west,draw,inner sep=4pt,fill=ugreen!20!white,minimum width=2em] (e3) at ([xshift=1em]sep5.east) {\scriptsize{$E_3$}};

\node [anchor=south,draw,inner sep=4pt,fill=yellow!30,minimum width=2em] (t1) at ([xshift=-2em,yshift=1em]Lstm5.north) {\scriptsize{$T_1$}};
\node [anchor=west,draw,inner sep=4pt,fill=yellow!30,minimum width=2em] (t2) at ([xshift=1em]t1.east) {\scriptsize{$T_2$}};
\node [anchor=west,inner sep=4pt] (sep6) at ([xshift=1em]t2.east) {\scriptsize{...}};
\node [anchor=west,draw,inner sep=4pt,fill=yellow!30,minimum width=2em] (t3) at ([xshift=1em]sep6.east) {\scriptsize{$T_3$}};

\node [anchor=west,draw,inner sep=4pt,fill=blue!20!white] (Lstm6) at ([xshift=1.5em]Lstm2.east) {\scriptsize{Lstm}};
\node [anchor=west,draw,inner sep=4pt,fill=blue!20!white] (Lstm7) at ([xshift=1em]Lstm6.east) {\scriptsize{Lstm}};
\node [anchor=west,inner sep=4pt] (sep3) at ([xshift=1em]Lstm7.east) {\scriptsize{...}};
\node [anchor=west,draw,inner sep=4pt,fill=blue!20!white] (Lstm8) at ([xshift=1em]sep3.east) {\scriptsize{Lstm}};

\node [anchor=south,draw,inner sep=4pt,fill=blue!20!white] (Lstm9) at ([yshift=1em]Lstm6.north) {\scriptsize{Lstm}};
\node [anchor=west,draw,inner sep=4pt,fill=blue!20!white] (Lstm10) at ([xshift=1em]Lstm9.east) {\scriptsize{Lstm}};
\node [anchor=west,inner sep=4pt] (sep4) at ([xshift=1em]Lstm10.east) {\scriptsize{...}};
\node [anchor=west,draw,inner sep=4pt,fill=blue!20!white] (Lstm11) at ([xshift=1em]sep4.east) {\scriptsize{Lstm}};

\node [rectangle,rounded corners,draw=black!50,densely dashed,inner sep=0.4em] [fit = (Lstm6) (Lstm8) (Lstm9) (Lstm11)] (inputshadow) {};

\draw [-&gt;] ([xshift=0.1em]Lstm0.east) -- ([xshift=-0.1em]Lstm1.west);
\draw [-&gt;] ([xshift=0.1em]Lstm1.east) -- ([xshift=-0.1em]sep.west);
\draw [-&gt;] ([xshift=0.1em]sep.east) -- ([xshift=-0.1em]Lstm2.west);

\draw [-&gt;] ([xshift=0.1em]Lstm3.east) -- ([xshift=-0.1em]Lstm4.west);
\draw [-&gt;] ([xshift=0.1em]Lstm4.east) -- ([xshift=-0.1em]sep1.west);
\draw [-&gt;] ([xshift=0.1em]sep1.east) -- ([xshift=-0.1em]Lstm5.west);

\draw [-&gt;] ([yshift=0.1em]Lstm0.north) -- ([yshift=-0.1em]Lstm3.south);
\draw [-&gt;] ([yshift=0.1em]Lstm1.north) -- ([yshift=-0.1em]Lstm4.south);
\draw [-&gt;] ([yshift=0.1em]Lstm2.north) -- ([yshift=-0.1em]Lstm5.south);

\draw [-&gt;] ([xshift=0.1em]Lstm6.east) -- ([xshift=-0.1em]Lstm7.west);
\draw [-&gt;] ([xshift=0.1em]Lstm7.east) -- ([xshift=-0.1em]sep3.west);
\draw [-&gt;] ([xshift=0.1em]sep3.east) -- ([xshift=-0.1em]Lstm8.west);

\draw [-&gt;] ([xshift=0.1em]Lstm9.east) -- ([xshift=-0.1em]Lstm10.west);
\draw [-&gt;] ([xshift=0.1em]Lstm10.east) -- ([xshift=-0.1em]sep4.west);
\draw [-&gt;] ([xshift=0.1em]sep4.east) -- ([xshift=-0.1em]Lstm11.west);

\draw [-&gt;] ([yshift=0.1em]Lstm6.north) -- ([yshift=-0.1em]Lstm9.south);
\draw [-&gt;] ([yshift=0.1em]Lstm7.north) -- ([yshift=-0.1em]Lstm10.south);
\draw [-&gt;] ([yshift=0.1em]Lstm8.north) -- ([yshift=-0.1em]Lstm11.south);

\draw [-&gt;] ([yshift=0.1em]e1.north) -- ([yshift=-0.1em]Lstm0.south);
\draw [-&gt;] ([yshift=0.1em]e1.north) -- ([yshift=-0.1em]Lstm6.south);
\draw [-&gt;] ([yshift=0.1em]e2.north) -- ([yshift=-0.1em]Lstm1.south);
\draw [-&gt;] ([yshift=0.1em]e2.north) -- ([yshift=-0.1em]Lstm7.south);
\draw [-&gt;] ([yshift=0.1em]e3.north) -- ([yshift=-0.1em]Lstm2.south);
\draw [-&gt;] ([yshift=0.1em]e3.north) -- ([yshift=-0.1em]Lstm8.south);

\draw [-&gt;] ([yshift=0.1em]Lstm3.north) -- ([xshift=-0.05em,yshift=-0.1em]t1.south);
\draw [-&gt;] ([yshift=0.1em]Lstm9.north) -- ([yshift=-0.1em]t1.south);
\draw [-&gt;] ([yshift=0.1em]Lstm4.north) -- ([xshift=-0.05em,yshift=-0.1em]t2.south);
\draw [-&gt;] ([yshift=0.1em]Lstm10.north) -- ([yshift=-0.1em]t2.south);
\draw [-&gt;] ([yshift=0.1em]Lstm5.north) -- ([xshift=-0.05em,yshift=-0.1em]t3.south);
\draw [-&gt;] ([yshift=0.1em]Lstm11.north) -- ([yshift=-0.1em]t3.south);

\end{scope}
\end{tikzpicture}
\end{center}

\end{frame}

%%%------------------------------------------------------------------------------------------------------------
%%% ��ǿ��ı�ʾģ�� - GTP
\begin{frame}{��ǿ�ı�ʾģ�� - GPT}
\begin{itemize}
\item \textbf{GPT}(Generative Pre-Training)Ҳ��һ�ֻ�������ģ�͵ı�ʾģ��
    \begin{itemize}
    \item �ܹ�������Transformer��������ȡ������ǿ
    \item ����Pre-training + Fine-tuning�Ŀ�ܣ�Ԥѵ����Ϊ����ϵͳ�����Ĳ�����ʼֵ����˿��Ը��õ���ӦĿ������
    \end{itemize}
\end{itemize}
\vspace{0.5em}
\begin{center}
\begin{tikzpicture}
\begin{scope}[scale=1.2]

\node [anchor=west,draw,inner sep=4pt,fill=blue!20!white,minimum width=3em] (Trm0) at (0,0) {\scriptsize{Trm}};
\node [anchor=west,draw,inner sep=4pt,fill=blue!20!white,minimum width=3em] (Trm1) at ([xshift=1em]Trm0.east) {\scriptsize{Trm}};
\node [anchor=west,draw,inner sep=4pt,fill=blue!20!white,minimum width=3em] (Trm2) at ([xshift=1em]Trm1.east) {\scriptsize{Trm}};
\node [anchor=west,draw,inner sep=4pt,fill=blue!20!white,minimum width=3em] (Trm3) at ([xshift=1em]Trm2.east) {\scriptsize{Trm}};
\node [anchor=west,inner sep=4pt] (sep) at ([xshift=1em]Trm3.east) {\scriptsize{...}};
\node [anchor=west,draw,inner sep=4pt,fill=blue!20!white,minimum width=3em] (Trm4) at ([xshift=1em]sep.east) {\scriptsize{Trm}};

\node [anchor=south,draw,inner sep=4pt,fill=blue!20!white,minimum width=3em] (Trm5) at ([yshift=1em]Trm0.north) {\scriptsize{Trm}};
\node [anchor=west,draw,inner sep=4pt,fill=blue!20!white,minimum width=3em] (Trm6) at ([xshift=1em]Trm5.east) {\scriptsize{Trm}};
\node [anchor=west,draw,inner sep=4pt,fill=blue!20!white,minimum width=3em] (Trm7) at ([xshift=1em]Trm6.east) {\scriptsize{Trm}};
\node [anchor=west,draw,inner sep=4pt,fill=blue!20!white,minimum width=3em] (Trm8) at ([xshift=1em]Trm7.east) {\scriptsize{Trm}};
\node [anchor=west,inner sep=4pt] (sep1) at ([xshift=1em]Trm8.east) {\scriptsize{...}};
\node [anchor=west,draw,inner sep=4pt,fill=blue!20!white,minimum width=3em] (Trm9) at ([xshift=1em]sep1.east) {\scriptsize{Trm}};

\node [rectangle,rounded corners,draw=black!50,densely dashed,inner sep=0.4em] [fit = (Trm0) (Trm4) (Trm5) (Trm9)] (inputshadow) {};

\node [anchor=north,draw,inner sep=4pt,fill=ugreen!20!white,minimum width=2em] (e1) at ([yshift=-1em]Trm0.south) {\scriptsize{$E_1$}};
\node [anchor=north,draw,inner sep=4pt,fill=ugreen!20!white,minimum width=2em] (e2) at ([yshift=-1em]Trm1.south) {\scriptsize{$E_2$}};
\node [anchor=north,draw,inner sep=4pt,fill=ugreen!20!white,minimum width=2em] (e3) at ([yshift=-1em]Trm2.south) {\scriptsize{$E_3$}};
\node [anchor=north,draw,inner sep=4pt,fill=ugreen!20!white,minimum width=2em] (e4) at ([yshift=-1em]Trm3.south) {\scriptsize{$E_4$}};
\node [anchor=north,inner sep=4pt] (sep5) at ([yshift=-1em]sep.south) {\scriptsize{...}};
\node [anchor=north,draw,inner sep=4pt,fill=ugreen!20!white,minimum width=2em] (e5) at ([yshift=-1em]Trm4.south) {\scriptsize{$E_5$}};

\node [anchor=south,draw,inner sep=4pt,fill=yellow!30,minimum width=2em] (t1) at ([yshift=1em]Trm5.north) {\scriptsize{$T_1$}};
\node [anchor=south,draw,inner sep=4pt,fill=yellow!30,minimum width=2em] (t2) at ([yshift=1em]Trm6.north) {\scriptsize{$T_2$}};
\node [anchor=south,draw,inner sep=4pt,fill=yellow!30,minimum width=2em] (t3) at ([yshift=1em]Trm7.north) {\scriptsize{$T_3$}};
\node [anchor=south,draw,inner sep=4pt,fill=yellow!30,minimum width=2em] (t4) at ([yshift=1em]Trm8.north) {\scriptsize{$T_4$}};
\node [anchor=south,inner sep=4pt] (sep6) at ([yshift=1em]sep1.north) {\scriptsize{...}};
\node [anchor=south,draw,inner sep=4pt,fill=yellow!30,minimum width=2em] (t5) at ([yshift=1em]Trm9.north) {\scriptsize{$T_5$}};

\draw [-&gt;] ([yshift=0.1em]e1.north) -- ([yshift=-0.1em]Trm0.south);
\draw [-&gt;] ([yshift=0.1em]e1.north) -- ([yshift=-0.1em]Trm1.south);
\draw [-&gt;] ([yshift=0.1em]e1.north) -- ([yshift=-0.1em]Trm2.south);
\draw [-&gt;] ([yshift=0.1em]e1.north) -- ([yshift=-0.1em]Trm3.south);
\draw [-&gt;] ([yshift=0.1em]e1.north) -- ([yshift=-0.1em]Trm4.south);
\draw [-&gt;] ([yshift=0.1em]e2.north) -- ([yshift=-0.1em]Trm1.south);
\draw [-&gt;] ([yshift=0.1em]e3.north) -- ([yshift=-0.1em]Trm2.south);
\draw [-&gt;] ([yshift=0.1em]e3.north) -- ([yshift=-0.1em]Trm3.south);
\draw [-&gt;] ([yshift=0.1em]e3.north) -- ([yshift=-0.1em]Trm4.south);
\draw [-&gt;] ([yshift=0.1em]e4.north) -- ([yshift=-0.1em]Trm3.south);
\draw [-&gt;] ([yshift=0.1em]e5.north) -- ([yshift=-0.1em]Trm4.south);

\draw [-&gt;] ([yshift=0.1em]Trm0.north) -- ([yshift=-0.1em]Trm5.south);
\draw [-&gt;] ([yshift=0.1em]Trm0.north) -- ([yshift=-0.1em]Trm6.south);
\draw [-&gt;] ([yshift=0.1em]Trm0.north) -- ([yshift=-0.1em]Trm7.south);
\draw [-&gt;] ([yshift=0.1em]Trm0.north) -- ([yshift=-0.1em]Trm8.south);
\draw [-&gt;] ([yshift=0.1em]Trm0.north) -- ([yshift=-0.1em]Trm9.south);
\draw [-&gt;] ([yshift=0.1em]Trm1.north) -- ([yshift=-0.1em]Trm6.south);
\draw [-&gt;] ([yshift=0.1em]Trm2.north) -- ([yshift=-0.1em]Trm7.south);
\draw [-&gt;] ([yshift=0.1em]Trm2.north) -- ([yshift=-0.1em]Trm8.south);
\draw [-&gt;] ([yshift=0.1em]Trm2.north) -- ([yshift=-0.1em]Trm9.south);
\draw [-&gt;] ([yshift=0.1em]Trm3.north) -- ([yshift=-0.1em]Trm8.south);
\draw [-&gt;] ([yshift=0.1em]Trm4.north) -- ([yshift=-0.1em]Trm9.south);

\draw [-&gt;] ([yshift=0.1em]Trm5.north) -- ([yshift=-0.1em]t1.south);
\draw [-&gt;] ([yshift=0.1em]Trm6.north) -- ([yshift=-0.1em]t2.south);
\draw [-&gt;] ([yshift=0.1em]Trm7.north) -- ([yshift=-0.1em]t3.south);
\draw [-&gt;] ([yshift=0.1em]Trm8.north) -- ([yshift=-0.1em]t4.south);
\draw [-&gt;] ([yshift=0.1em]Trm9.north) -- ([yshift=-0.1em]t5.south);

\end{scope}
\end{tikzpicture}
\end{center}

\end{frame}

%%%------------------------------------------------------------------------------------------------------------
%%% ��ǿ��ı�ʾģ�� - BERT
\begin{frame}{��ǿ�ı�ʾģ�� - BERT}
\begin{itemize}
\item \textbf{BERT}( Bidirectional Encoder Representations from Transformers)������ǳ��𱬵ı�ʾģ��
    \begin{itemize}
    \item ��Ȼ����Transformer���ǿ������������˵�������(���ԶԱ�GPT)
    \item ʹ����Mask����������ѵ���õ�ģ�͵Ľ�׳�ԣ��������������Ϊ��Ԥѵ����ʾģ�͵��·�ʽ
    \end{itemize}
\end{itemize}
\vspace{0.5em}
\begin{center}
\begin{tikzpicture}
\begin{scope}[scale=1.2]

\node [anchor=west,draw,inner sep=4pt,fill=blue!20!white,minimum width=3em] (Trm0) at (0,0) {\scriptsize{Trm}};
\node [anchor=west,draw,inner sep=4pt,fill=blue!20!white,minimum width=3em] (Trm1) at ([xshift=1em]Trm0.east) {\scriptsize{Trm}};
\node [anchor=west,draw,inner sep=4pt,fill=blue!20!white,minimum width=3em] (Trm2) at ([xshift=1em]Trm1.east) {\scriptsize{Trm}};
\node [anchor=west,draw,inner sep=4pt,fill=blue!20!white,minimum width=3em] (Trm3) at ([xshift=1em]Trm2.east) {\scriptsize{Trm}};
\node [anchor=west,inner sep=4pt] (sep) at ([xshift=1em]Trm3.east) {\scriptsize{...}};
\node [anchor=west,draw,inner sep=4pt,fill=blue!20!white,minimum width=3em] (Trm4) at ([xshift=1em]sep.east) {\scriptsize{Trm}};

\node [anchor=south,draw,inner sep=4pt,fill=blue!20!white,minimum width=3em] (Trm5) at ([yshift=1em]Trm0.north) {\scriptsize{Trm}};
\node [anchor=west,draw,inner sep=4pt,fill=blue!20!white,minimum width=3em] (Trm6) at ([xshift=1em]Trm5.east) {\scriptsize{Trm}};
\node [anchor=west,draw,inner sep=4pt,fill=blue!20!white,minimum width=3em] (Trm7) at ([xshift=1em]Trm6.east) {\scriptsize{Trm}};
\node [anchor=west,draw,inner sep=4pt,fill=blue!20!white,minimum width=3em] (Trm8) at ([xshift=1em]Trm7.east) {\scriptsize{Trm}};
\node [anchor=west,inner sep=4pt] (sep1) at ([xshift=1em]Trm8.east) {\scriptsize{...}};
\node [anchor=west,draw,inner sep=4pt,fill=blue!20!white,minimum width=3em] (Trm9) at ([xshift=1em]sep1.east) {\scriptsize{Trm}};

\node [rectangle,rounded corners,draw=black!50,densely dashed,inner sep=0.4em] [fit = (Trm0) (Trm4) (Trm5) (Trm9)] (inputshadow) {};

\node [anchor=north,draw,inner sep=4pt,fill=ugreen!20!white,minimum width=2em] (e1) at ([yshift=-1em]Trm0.south) {\scriptsize{$E_1$}};
\node [anchor=north,draw,inner sep=4pt,fill=ugreen!20!white,minimum width=2em] (e2) at ([yshift=-1em]Trm1.south) {\scriptsize{$E_2$}};
\node [anchor=north,draw,inner sep=4pt,fill=ugreen!20!white,minimum width=2em] (e3) at ([yshift=-1em]Trm2.south) {\scriptsize{$E_3$}};
\node [anchor=north,draw,inner sep=4pt,fill=ugreen!20!white,minimum width=2em] (e4) at ([yshift=-1em]Trm3.south) {\scriptsize{$E_4$}};
\node [anchor=north,inner sep=4pt] (sep5) at ([yshift=-1em]sep.south) {\scriptsize{...}};
\node [anchor=north,draw,inner sep=4pt,fill=ugreen!20!white,minimum width=2em] (e5) at ([yshift=-1em]Trm4.south) {\scriptsize{$E_5$}};

\node [anchor=south,draw,inner sep=4pt,fill=yellow!30,minimum width=2em] (t1) at ([yshift=1em]Trm5.north) {\scriptsize{$T_1$}};
\node [anchor=south,draw,inner sep=4pt,fill=yellow!30,minimum width=2em] (t2) at ([yshift=1em]Trm6.north) {\scriptsize{$T_2$}};
\node [anchor=south,draw,inner sep=4pt,fill=yellow!30,minimum width=2em] (t3) at ([yshift=1em]Trm7.north) {\scriptsize{$T_3$}};
\node [anchor=south,draw,inner sep=4pt,fill=yellow!30,minimum width=2em] (t4) at ([yshift=1em]Trm8.north) {\scriptsize{$T_4$}};
\node [anchor=south,inner sep=4pt] (sep6) at ([yshift=1em]sep1.north) {\scriptsize{...}};
\node [anchor=south,draw,inner sep=4pt,fill=yellow!30,minimum width=2em] (t5) at ([yshift=1em]Trm9.north) {\scriptsize{$T_5$}};

\draw [-&gt;] ([yshift=0.1em]e1.north) -- ([yshift=-0.1em]Trm0.south);
\draw [-&gt;] ([yshift=0.1em]e1.north) -- ([yshift=-0.1em]Trm1.south);
\draw [-&gt;] ([yshift=0.1em]e1.north) -- ([yshift=-0.1em]Trm2.south);
\draw [-&gt;] ([yshift=0.1em]e1.north) -- ([yshift=-0.1em]Trm3.south);
\draw [-&gt;] ([yshift=0.1em]e1.north) -- ([yshift=-0.1em]Trm4.south);
\draw [-&gt;] ([yshift=0.1em]e2.north) -- ([yshift=-0.1em]Trm1.south);
\draw [-&gt;] ([yshift=0.1em]e3.north) -- ([yshift=-0.1em]Trm0.south);
\draw [-&gt;] ([yshift=0.1em]e3.north) -- ([yshift=-0.1em]Trm1.south);
\draw [-&gt;] ([yshift=0.1em]e3.north) -- ([yshift=-0.1em]Trm2.south);
\draw [-&gt;] ([yshift=0.1em]e3.north) -- ([yshift=-0.1em]Trm3.south);
\draw [-&gt;] ([yshift=0.1em]e3.north) -- ([yshift=-0.1em]Trm4.south);
\draw [-&gt;] ([yshift=0.1em]e4.north) -- ([yshift=-0.1em]Trm3.south);
\draw [-&gt;] ([yshift=0.1em]e5.north) -- ([yshift=-0.1em]Trm4.south);

\draw [-&gt;] ([yshift=0.1em]Trm0.north) -- ([yshift=-0.1em]Trm5.south);
\draw [-&gt;] ([yshift=0.1em]Trm0.north) -- ([yshift=-0.1em]Trm6.south);
\draw [-&gt;] ([yshift=0.1em]Trm0.north) -- ([yshift=-0.1em]Trm7.south);
\draw [-&gt;] ([yshift=0.1em]Trm0.north) -- ([yshift=-0.1em]Trm8.south);
\draw [-&gt;] ([yshift=0.1em]Trm0.north) -- ([yshift=-0.1em]Trm9.south);
\draw [-&gt;] ([yshift=0.1em]Trm1.north) -- ([yshift=-0.1em]Trm6.south);
\draw [-&gt;] ([yshift=0.1em]Trm2.north) -- ([yshift=-0.1em]Trm5.south);
\draw [-&gt;] ([yshift=0.1em]Trm2.north) -- ([yshift=-0.1em]Trm6.south);
\draw [-&gt;] ([yshift=0.1em]Trm2.north) -- ([yshift=-0.1em]Trm7.south);
\draw [-&gt;] ([yshift=0.1em]Trm2.north) -- ([yshift=-0.1em]Trm8.south);
\draw [-&gt;] ([yshift=0.1em]Trm2.north) -- ([yshift=-0.1em]Trm9.south);
\draw [-&gt;] ([yshift=0.1em]Trm3.north) -- ([yshift=-0.1em]Trm8.south);
\draw [-&gt;] ([yshift=0.1em]Trm4.north) -- ([yshift=-0.1em]Trm9.south);

\draw [-&gt;] ([yshift=0.1em]Trm5.north) -- ([yshift=-0.1em]t1.south);
\draw [-&gt;] ([yshift=0.1em]Trm6.north) -- ([yshift=-0.1em]t2.south);
\draw [-&gt;] ([yshift=0.1em]Trm7.north) -- ([yshift=-0.1em]t3.south);
\draw [-&gt;] ([yshift=0.1em]Trm8.north) -- ([yshift=-0.1em]t4.south);
\draw [-&gt;] ([yshift=0.1em]Trm9.north) -- ([yshift=-0.1em]t5.south);

\end{scope}
\end{tikzpicture}
\end{center}

\end{frame}

%%%------------------------------------------------------------------------------------------------------------
%%% Ԥѵ��
\begin{frame}{Ԥѵ��}
\begin{itemize}
\item ����ģ�Ϳ���ʹ�ô����ޱ�ע���ݽ���ѵ�����õ���ģ�Ϳ��Ա�ֱ����������ϵͳ�������е���������Ϊ��

\begin{center}
\begin{tikzpicture}
\node [anchor=south,minimum width=17em,fill=red!20!white] (encoder) at (0,0) {Encoder (����ģ��Ԥ��ѵ��)};
\node [anchor=south,minimum width=17em,fill=blue!20!white] (decoder) at (encoder.north) {Decoder (Ŀ����������ѵ��)};
\end{tikzpicture}
\end{center}

\item&lt;2-&gt; �������˷ǳ��𱬵�\alert{��ʽ}�����ģ����ģ��pre-training + Ŀ������fine-tuning
	\begin{itemize}
	\item ����NLP���񶼿��Ա�����Ϊ���Խ�ģ�����ⲿѵ���õ�������ģ����Ϊģ�����Ŀ��ϵͳ��(������ʼ��)
	\end{itemize}
\end{itemize}

\visible&lt;2-&gt;{
\begin{center}
\begin{tikzpicture}

\begin{scope}

\node [anchor=west,draw,thick,minimum width=4em,minimum height=1.7em,fill=blue!20] (encoder) at (0,0) {ģ��};
\node [anchor=south,minimum width=4em,minimum height=1.7em] (space) at ([yshift=0.3em]encoder.north) {\footnotesize{Ŀ��ϵͳ}};

\begin{pgfonlayer}{background}
\node [rectangle,draw,thick,fill=red!20] [fit = (encoder) (space)] (system) {};
\end{pgfonlayer}

\node [anchor=north] (data) at ([yshift=-1em]system.south) {\scriptsize{\textbf{Ŀ�������б�ע����}}};
\draw [-&gt;,thick] (data.north) -- ([yshift=-0.1em]system.south);
\node [anchor=north] (label) at ([yshift=-0em]data.south) {\scriptsize{(a) standard method}};

\end{scope}

\begin{scope}[xshift=2.8in]

\node [anchor=west,draw,dashed,thick,minimum width=4em,minimum height=1.7em,fill=blue!20] (encoder) at (0,0) {ģ��};
\node [anchor=south,minimum width=4em,minimum height=1.7em] (space) at ([yshift=0.3em]encoder.north) {\footnotesize{Ŀ��ϵͳ}};
\node [anchor=center,draw,thick,minimum width=4em,minimum height=1.7em,fill=green!20] (encoderpre) at ([xshift=-7em]encoder.center) {\footnotesize{����ģ��}};
\draw [-&gt;,thick] (encoderpre.east) -- (encoder.west);

\begin{pgfonlayer}{background}
\node [rectangle,draw,thick,fill=red!20] [fit = (encoder) (space)] (system) {};
\end{pgfonlayer}

\node [anchor=north] (data) at ([yshift=-1em]system.south) {\scriptsize{\textbf{Ŀ�������б�ע����}}};
\draw [-&gt;,thick] (data.north) -- ([yshift=-0.1em]system.south);
\node [anchor=north] (data2) at ([yshift=-1em,xshift=-7em]system.south) {\scriptsize{\textbf{���ģ�ޱ�ע����}}};
\draw [-&gt;,thick] (data2.north) -- ([yshift=-0.1em]encoderpre.south);
\node [anchor=north] (label) at ([yshift=-0em,xshift=-4em]data.south) {\scriptsize{(b) pre-training + fine-tuning}};

\end{scope}

\end{tikzpicture}
\end{center}
}

\end{frame}

%%%------------------------------------------------------------------------------------------------------------
%%% Ԥѵ����������˼·
\begin{frame}{Ԥѵ����������˼·}
\begin{itemize}
\item Ԥѵ��ģ��ˢ����������ͬʱ��������һЩ˼����\\
      Ԥѵ�����������Ǵ�����ʲô��
    \begin{itemize}
    \item �б�ע���������ޣ�Ԥѵ���ṩʹ�ó����ģ���ݵķ���
    \item �Ӵ��ģ�ޱ�ע������ѧϰͨ��֪ʶ��������������
    \item �����縴���Ҳ�����ѵ����Ԥѵ������ʹģ�͹�ע���ʽ�ĸ��ܶ�����
    \end{itemize}
\end{itemize}

\visible&lt;2-&gt;{
\begin{center}
\begin{tikzpicture}
\draw[name path=ellipse,thick] (0,0) circle[x radius = 2, y radius = 1];
\node[rectangle,minimum size=0.1em,inner sep=2pt,fill=red] (p1) at (0.2,0.5) {};
\node[rectangle,minimum size=0.1em,inner sep=2pt,fill=red] (p2) at (0.3,0.6) {};
\node[rectangle,minimum size=0.1em,inner sep=2pt,fill=red] (p3) at (0.1,-0.1) {};
\node[rectangle,minimum size=0.1em,inner sep=2pt,fill=red] (p4) at (0.4,0) {};
\node[rectangle,minimum size=0.1em,inner sep=2pt,fill=red] (p5) at (0.5,0.3) {};
\node[rectangle,minimum size=0.1em,inner sep=2pt,fill=red] (p6) at (0.6,0.1) {};
\node[rectangle,minimum size=0.1em,inner sep=2pt,fill=red] (p7) at (0.7,-0.1) {};
\node[rectangle,minimum size=0.1em,inner sep=2pt,fill=red] (p8) at (-1.2,0.4) {};
\node[rectangle,minimum size=0.1em,inner sep=2pt,fill=red] (p9) at (-1.0,-0.3) {};
\node[rectangle,minimum size=0.1em,inner sep=2pt,fill=red] (p10) at (-0.1,-0.8) {};

\begin{pgfonlayer}{background}
\visible&lt;4-&gt;{
\node [rectangle,inner sep=0.4em,draw,blue] [fit = (p1) (p2) (p3) (p4) (p5) (p6)] (area) {};
}
\end{pgfonlayer}

\draw [-&gt;] (2.5,-0.7) -- (1.8,-0.5) node [pos=0,right] {\scriptsize{ģ�Ͳ�����ռ�}};

\visible&lt;4-&gt;{
\draw [-&gt;] (2.0,0.7) -- (area.20) node [pos=0,right] {\scriptsize{���ʽ���ܶ�����(Ԥѵ��)}};
}
\visible&lt;3-&gt;{
\draw [-&gt;] (-2.0,0.7) -- (p8.west) node [pos=0,left] {\scriptsize{����Ľ�}};
}

\end{tikzpicture}
\end{center}
}

\begin{itemize}
\item&lt;5-&gt; ���������е�Ԥѵ��
    \begin{itemize}
    \item ����������Ԥѵ����û������һ�������ںܶ������������ѵ������������С����һ����Ҳ��Ӧ�������˫�ｨģ��Ԥѵ��Ҳ������µ�Ҫ��
    \end{itemize}
\end{itemize}

\end{frame}

%%%------------------------------------------------------------------------------------------------------------
%%% �ܽ�
\begin{frame}{�ܽ� - ����һ����}
\begin{itemize}
\item ���˺ܶ࣬��ѽ�ۣ�������һ����Ҫ�۵�
    \begin{itemize}
    \item ������û����ô���ӣ����Ų�����
    \item �򵥵�����ṹ������ϳ�ǿ���ģ��
    \item ����ģ�Ϳ�����������ʵ�֣�Ч���ܺã�������ֵ�Ԥѵ���ȷ�ʽ֤����������ģ�͵�Ǳ��
    \end{itemize}
\item&lt;2-&gt; ��Ȼ�кܶ�������Ҫ����
    \begin{itemize}
    \item ������������ṹ(����NLP)\\
          googleһ��LSTM��GRU��CNN
    \item ���ģ�ͺ�ѵ�����������ѧϰ�������``��''?\\
          ���������Դ���ʲô��\\
          �����Ч��ѵ�����ģ�ͣ�
    \item ��ΰ����������ڰ��������������ڵ�����NLP����\\
          ����encoder-decoder���
    \item ���ѧϰ��ʵ������\\
          ``������''�˽��£���Ϊ��ͬ������κ�ģ����ƶ��м���\\
          ...
    \end{itemize}
\end{itemize}
\end{frame}

%%%------------------------------------------------------------------------------------------------------------
%%% last slide
\begin{frame}{�ֽ���һ������~}

\vspace{2em}

\begin{center}
\textbf{���ݺܶ࣬�����˸�ͷ}\\
\textbf{ѧϰ���ѧϰ������Ҫʵ���;���Ļ��ۣ�}

\vspace{2em}

\begin{tikzpicture}

\tikzstyle{rnnnode} = [draw,inner sep=5pt,minimum width=4em,minimum height=1.5em,fill=green!30!white,blur shadow={shadow xshift=1pt,shadow yshift=-1pt}]
\node [anchor=west,rnnnode] (node11) at (0,0) {\tiny{RNN Cell}};
\node [anchor=west,rnnnode] (node12) at ([xshift=2em]node11.east) {\tiny{RNN Cell}};
\node [anchor=west,rnnnode] (node13) at ([xshift=2em]node12.east) {\tiny{RNN Cell}};
\node [anchor=west,rnnnode] (node14) at ([xshift=2em]node13.east) {\tiny{RNN Cell}};

\node [anchor=north,rnnnode,fill=red!30!white] (e1) at ([yshift=-1.2em]node11.south) {\tiny{embedding}};
\node [anchor=north,rnnnode,fill=red!30!white] (e2) at ([yshift=-1.2em]node12.south) {\tiny{embedding}};
\node [anchor=north,rnnnode,fill=red!30!white] (e3) at ([yshift=-1.2em]node13.south) {\tiny{embedding}};
\node [anchor=north,rnnnode,fill=red!30!white] (e4) at ([yshift=-1.2em]node14.south) {\tiny{embedding}};
\node [anchor=north] (w1) at ([yshift=-1em]e1.south) {\footnotesize{$&lt;$s$&gt;$}};
\node [anchor=north] (w2) at ([yshift=-1em]e2.south) {\footnotesize{лл}};
\node [anchor=north] (w3) at ([yshift=-1em]e3.south) {\footnotesize{���}};
\node [anchor=north] (w4) at ([yshift=-1em]e4.south) {\footnotesize{����}};

\draw [-&gt;,thick] ([yshift=0.1em]w1.north)--([yshift=-0.1em]e1.south);
\draw [-&gt;,thick] ([yshift=0.1em]w2.north)--([yshift=-0.1em]e2.south);
\draw [-&gt;,thick] ([yshift=0.1em]w3.north)--([yshift=-0.1em]e3.south);
\draw [-&gt;,thick] ([yshift=0.1em]w4.north)--([yshift=-0.1em]e4.south);

\draw [-&gt;,thick] ([yshift=0.1em]e1.north)--([yshift=-0.1em]node11.south);
\draw [-&gt;,thick] ([yshift=0.1em]e2.north)--([yshift=-0.1em]node12.south);
\draw [-&gt;,thick] ([yshift=0.1em]e3.north)--([yshift=-0.1em]node13.south);
\draw [-&gt;,thick] ([yshift=0.1em]e4.north)--([yshift=-0.1em]node14.south);

\node [anchor=south,rnnnode,fill=red!30!white] (node21) at ([yshift=1.0em]node11.north) {\tiny{Softmax($\cdot$)}};
\node [anchor=south,rnnnode,fill=red!30!white] (node22) at ([yshift=1.0em]node12.north) {\tiny{Softmax($\cdot$)}};
\node [anchor=south,rnnnode,fill=red!30!white] (node23) at ([yshift=1.0em]node13.north) {\tiny{Softmax($\cdot$)}};
\node [anchor=south,rnnnode,fill=red!30!white] (node24) at ([yshift=1.0em]node14.north) {\tiny{Softmax($\cdot$)}};

\node [anchor=south] (output1) at ([yshift=1em]node21.north) {\Large{\textbf{лл}}};
\node [anchor=south] (output2) at ([yshift=1em]node22.north) {\Large{\textbf{���}}};
\node [anchor=south] (output3) at ([yshift=1em]node23.north) {\Large{\textbf{����}}};
\node [anchor=south] (output4) at ([yshift=1em]node24.north) {\Large{\textbf{$&lt;$/s$&gt;$}}};

\draw [-&gt;,thick] ([yshift=0.1em]node21.north)--([yshift=-0.1em]output1.south);
\draw [-&gt;,thick] ([yshift=0.1em]node22.north)--([yshift=-0.1em]output2.south);
\draw [-&gt;,thick] ([yshift=0.1em]node23.north)--([yshift=-0.1em]output3.south);
\draw [-&gt;,thick] ([yshift=0.1em]node24.north)--([yshift=-0.1em]output4.south);

\draw [-&gt;,thick] ([yshift=0.1em]node11.north)--([yshift=-0.1em]node21.south);
\draw [-&gt;,thick] ([yshift=0.1em]node12.north)--([yshift=-0.1em]node22.south);
\draw [-&gt;,thick] ([yshift=0.1em]node13.north)--([yshift=-0.1em]node23.south);
\draw [-&gt;,thick] ([yshift=0.1em]node14.north)--([yshift=-0.1em]node24.south);

\draw [-&gt;,thick] ([xshift=-1em]node11.west)--([xshift=-0.1em]node11.west);
\draw [-&gt;,thick] ([xshift=0.1em]node11.east)--([xshift=-0.1em]node12.west);
\draw [-&gt;,thick] ([xshift=0.1em]node12.east)--([xshift=-0.1em]node13.west);
\draw [-&gt;,thick] ([xshift=0.1em]node13.east)--([xshift=-0.1em]node14.west);
\draw [-&gt;,thick] ([xshift=0.1em]node14.east)--([xshift=1em]node14.east);

\end{tikzpicture}

\end{center}

\end{frame}

\end{CJK}
\end{document}
</pre></body></html>