fig-transformer-LM.tex

  \begin{tikzpicture}
\begin{scope}
\node [anchor=west] (w0) at (0,0) {\footnotesize{$w_{0}$}};
\node [anchor=west] (w1) at ([xshift=7em]w0.east) {\footnotesize{$w_{1}$}};
\node [anchor=west] (w2) at ([xshift=7em]w1.east) {\footnotesize{$w_{2}$}};
\node [anchor=west] (w3) at ([xshift=7em]w2.east) {\footnotesize{$w_{3}$}};
\node [anchor=north] (index0) at ([yshift=0.5em]w0.south) {\footnotesize(index)};
\node [anchor=north] (index1) at ([yshift=0.5em]w1.south) {\footnotesize(index)};
\node [anchor=north] (index2) at ([yshift=0.5em]w2.south) {\footnotesize(index)};
\node [anchor=north] (index3) at ([yshift=0.5em]w3.south) {\footnotesize(index)};

{
\node [anchor=south,line width=0.8pt,draw,inner sep=3pt,fill=blue!30!white,minimum width=7.5em] (e0) at ([yshift=1em]w0.north) {\footnotesize{$\textbf{e}_0=w_{0} \textbf{C} + \textrm{PE}(0)$}};
\node [anchor=south,line width=0.8pt,draw,inner sep=3pt,fill=blue!30!white,minimum width=7.5em] (e1) at ([yshift=1em]w1.north) {\footnotesize{$\textbf{e}_1=w_{1} \textbf{C} + \textrm{PE}(1)$}};
\node [anchor=south,line width=0.8pt,draw,inner sep=3pt,fill=blue!30!white,minimum width=7.5em] (e2) at ([yshift=1em]w2.north) {\footnotesize{$\textbf{e}_2=w_{2} \textbf{C} + \textrm{PE}(2)$}};
\node [anchor=south,line width=0.8pt,draw,inner sep=3pt,fill=blue!30!white,minimum width=7.5em] (e3) at ([yshift=1em]w3.north) {\footnotesize{$\textbf{e}_3=w_{3} \textbf{C} + \textrm{PE}(3)$}};
}

{
\node [anchor=south,line width=0.8pt,draw,inner sep=3pt,fill=green!30!white] (h0) at ([ yshift=2.0em]e0.north) {\footnotesize{$\textbf{h}_{0}=\textrm{SelfAtt}(\textbf{e}_0,\textbf{e}_3)$}};
\node [anchor=south,line width=0.8pt,draw,inner sep=3pt,fill=green!30!white] (h1) at ([ yshift=2.0em]e1.north) {\footnotesize{$\textbf{h}_{1}=\textrm{SelfAtt}(\textbf{e}_1,\textbf{e}_3)$}};
\node [anchor=south,line width=0.8pt,draw,inner sep=3pt,fill=green!30!white] (h2) at ([ yshift=2.0em]e2.north) {\footnotesize{$\textbf{h}_{2}=\textrm{SelfAtt}(\textbf{e}_2,\textbf{e}_3)$}};
}


{
\node [anchor=south,line width=0.8pt,draw,minimum width=9em,inner sep=3pt,fill=orange!30!white] (f1) at ([yshift=2.0em]h2.north) {\footnotesize{$\textbf{f}_3=\textrm{FNN}([\textbf{h}_0,\textbf{h}_1,\textbf{h}_2,\textbf{e}_3])$}};
\node [anchor=south,line width=0.8pt,draw,minimum width=9.5em,inner sep=3pt,fill=orange!30!white] (o1) at ([yshift=1.2em]f1.north) {\footnotesize{$\textbf{y}=\textrm{Softmax}(f_3 \textbf{U})$}};
}
\node [anchor=south] (ylabel) at ([yshift=1.2em]o1.north) {\footnotesize{$\textrm{P}(w_4|w_{0}w_{1}w_{2}w_{3})$}};

\draw [->,line width=0.6pt] ([yshift=0.1em]w0.north) -- ([yshift=-0.1em]e0.south);
\draw [->,line width=0.6pt] ([yshift=0.1em]w1.north) -- ([yshift=-0.1em]e1.south);
\draw [->,line width=0.6pt] ([yshift=0.1em]w2.north) -- ([yshift=-0.1em]e2.south);
\draw [->,line width=0.6pt] ([yshift=0.1em]w3.north) -- ([yshift=-0.1em]e3.south);
\draw [->,line width=0.6pt] ([yshift=0.1em]e0.north) -- ([xshift=0em,yshift=-0.1em]h0.south);
\draw [->,line width=0.6pt] ([yshift=0.1em]e1.north) -- ([xshift=0em,yshift=-0.1em]h1.south);
\draw [->,line width=0.6pt] ([yshift=0.1em]e2.north) -- ([xshift=0em,yshift=-0.1em]h2.south);
\draw [->,line width=0.6pt] ([yshift=0.1em]e3.north) -- ([xshift=0.4em,yshift=-0.1em]h0.south);
\draw [->,line width=0.6pt] ([yshift=0.1em]e3.north) -- ([xshift=0.4em,yshift=-0.1em]h1.south);
\draw [->,line width=0.6pt] ([yshift=0.1em]e3.north) -- ([xshift=0.4em,yshift=-0.1em]h2.south);

\draw [->,line width=0.6pt] ([yshift=0.1em]h0.north) -- ([xshift=-2em,yshift=-0.1em]f1.south);
\draw [->,line width=0.6pt] ([yshift=0.1em]e3.north) -- ([xshift=2em,yshift=-0.1em]f1.south);
\draw [->,line width=0.6pt] ([yshift=0.1em]h1.north) -- ([xshift=-1em,yshift=-0.1em]f1.south);
\draw [->,line width=0.6pt] ([yshift=0.1em]h2.north) -- ([xshift=0em,yshift=-0.1em]f1.south);
\draw [->,line width=0.6pt] ([yshift=0.1em]f1.north) -- ([yshift=-0.1em]o1.south);
\draw [->,line width=0.6pt] ([yshift=0.1em]o1.north) -- ([yshift=-0.1em]ylabel.south);
\end{scope}
\end{tikzpicture}