new update

Section05-Neural-Networks-and-Language-Modeling/section05-test.tex

@@ -120,43 +120,23 @@
 \subsection{词嵌入}
 \subsection{词嵌入}
 
 
 %%%------------------------------------------------------------------------------------------------------------
 %%%------------------------------------------------------------------------------------------------------------
-%%% Transformer architecture
-\begin{frame}{语言模型的评价指标}
-\begin{itemize}
-\item 困惑度(ppl)
+%%% 预训练
+\begin{frame}{预训练}
 \begin{itemize}
 \begin{itemize}
-\item 语言模型预测一个语言样本的能力
-\item 困惑度越低，建模的效果越好
-\end{itemize}
-\end{itemize}
+\item 语言模型可以使用大量无标注数据进行训练，得到的模型可以被直接用于下游系统，以序列到序列任务为例
 
 
 \begin{center}
 \begin{center}
 \begin{tikzpicture}
 \begin{tikzpicture}
-\begin{scope}
-\node [anchor=west] (eq) at (0,0) {$perplexity(s)=p(w_1,w_2,w_3,...,w_m)^{-1/m}$};
-
-\end{scope}
+\node [anchor=south,minimum width=17em,fill=red!20!white] (encoder) at (0,0) {Encoder (语言模型预训练)};
+\node [anchor=south,minimum width=17em,fill=blue!20!white] (decoder) at (encoder.north) {Decoder (带目标任务标注正常训练)};
 \end{tikzpicture}
 \end{tikzpicture}
 \end{center}
 \end{center}
-\vspace{0.5em}
-\begin{tabular}{l | l | l | r}
-模型 & 作者 & 年份 & PPL \\ \hline
-Feed-forward Neural LM & Bengio et al. & 2003 & 162.2 \\
-Recurrent NN-based LM & Mikolov et al. & 2010 & 124.7 \\
-Recurrent NN-LDA & Mikolov et al. & 2012 & 92.0 \\
-LSTM & Zaremba et al. & 2014 & 78.4 \\
-RHN & Zilly et al. & 2016 & 65.4 \\
-AWD-LSTM & Merity et al. & 2018 & 58.8 \\
-GPT-2 (Transformer) & Radford et al. & 2019 & \alert{35.7}
-\end{tabular}
 
 
-\end{frame}
+\item 衍生出了非常火爆的\alert{新范式}，大规模语言模型pre-training + 目标任务fine-tuning
+	\begin{itemize}
+	\item 许多NLP任务都可以被描述为语言建模，在外部训练得到的语言模型作为模块放入目标系统中(参数初始化)
+	\end{itemize}
 
 
-%%%------------------------------------------------------------------------------------------------------------
-%%% 预训练
-\begin{frame}{预训练}
-\begin{itemize}
-\item 语言模型可以使用大量无标注数据进行训练
 \end{itemize}
 \end{itemize}
 \end{frame}
 \end{frame}
 
 

Section05-Neural-Networks-and-Language-Modeling/section05.tex

@@ -4635,6 +4635,38 @@ $\textbf{V}, \textbf{U}, \textbf{W}$: 参数
 \end{frame}
 \end{frame}
 
 
 %%%------------------------------------------------------------------------------------------------------------
 %%%------------------------------------------------------------------------------------------------------------
+%%% evaluation
+\begin{frame}{语言模型评价}
+\begin{itemize}
+\item 语言模型的评价指标 - 困惑度(Perplexity, PPL)
+\begin{itemize}
+\item 语言模型预测一个语言样本的能力
+\item 困惑度越低，建模的效果越好
+\end{itemize}
+\vspace{0.5em}
+\begin{displaymath}
+\textrm{PPL}(w_1 ... w_m)=\textrm{P}(w_1 ... w_m)^{-1/m}
+\end{displaymath}
+\vspace{-0.5em}
+\item<2-> Penn Treebank(PTB)上的评价结果
+\end{itemize}
+\vspace{0.0em}
+\visible<2->{
+\begin{tabular}{l | l | l | r}
+模型 & 作者 & 年份 & PPL \\ \hline
+FNN LM & Bengio et al. & 2003 & 162.2 \\
+RNN LM & Mikolov et al. & 2010 & 124.7 \\
+RNN-LDA LM & Mikolov et al. & 2012 & 92.0 \\
+RNN(LSTM) LM & Zaremba et al. & 2014 & 78.4 \\
+RHN & Zilly et al. & 2016 & 65.4 \\
+RNN(AWD-LSTM) LM & Merity et al. & 2018 & 58.8 \\
+GPT-2 (Transformer) & Radford et al. & 2019 & 35.7
+\end{tabular}
+}
+
+\end{frame}
+
+%%%------------------------------------------------------------------------------------------------------------
 \subsection{词嵌入}
 \subsection{词嵌入}
 
 
 %%%------------------------------------------------------------------------------------------------------------
 %%%------------------------------------------------------------------------------------------------------------