figure-output-layer-structur.tex

% not compatible with [scale=?]


%-----------------------------------------------------------------
 \begin{tikzpicture}
            \setlength{\base}{0.9cm}

            \tikzstyle{rnnnode} = [rounded corners=1pt,minimum height=0.5\base,minimum width=1\base,draw,inner sep=0pt,outer sep=0pt]
            \tikzstyle{wordnode} = [font=\tiny]

            % RNN translation model
            \begin{scope}[local bounding box=RNNMT]
                % RNN Encoder
                \coordinate (eemb0) at (0,0);
                \foreach \x [count=\y from 0] in {1,2,...,3}
                    \node[rnnnode,minimum height=0.5\base,fill=green!30!white,anchor=west] (eemb\x) at ([xshift=0.4\base]eemb\y.east) {\tiny{$e_x()$}};
                \foreach \x in {1,2,...,3}
                    \node[rnnnode,fill=blue!30!white,anchor=south] (enc\x) at ([yshift=0.3\base]eemb\x.north) {};
                    \node[] (enclabel1) at (enc1) {\tiny{$\vectorn{\emph{h}}_{m-2}$}};
                    \node[] (enclabel2) at (enc2) {\tiny{$\vectorn{\emph{h}}_{m-1}$}};
                    \node[rnnnode,fill=purple!30!white] (enclabel3) at (enc3) {\tiny{$\vectorn{\emph{h}}_{m}$}};
                \node[wordnode,left=0.4\base of enc1] (init1) {$\cdots$};
                \node[wordnode,left=0.4\base of eemb1] (init2) {$\cdots$};

                \node[wordnode,below=0pt of eemb1] () {走};
                \node[wordnode,below=0pt of eemb2] () {吗};
                \node[wordnode,below=0pt of eemb3] () {$\langle$eos$\rangle$};

                % RNN Decoder
                \foreach \x in {1,2,...,3}
                    \node[rnnnode,minimum height=0.5\base,fill=green!30!white,anchor=south] (demb\x) at ([yshift=\base]enc\x.north) {\tiny{$e_y()$}};
                \foreach \x in {1,2,...,3}
                    \node[rnnnode,fill=blue!30!white,anchor=south] (dec\x) at ([yshift=0.3\base]demb\x.north) {{\tiny{$\vectorn{\emph{s}}_\x$}}};
                \foreach \x in {1,2,...,3}
                    \node[rnnnode,minimum height=0.5\base,fill=red!30!white,anchor=south] (softmax\x) at ([yshift=0.3\base]dec\x.north) {\tiny{Softmax}};
                \node[wordnode,right=0.4\base of demb3] (end1) {$\cdots$};
                \node[wordnode,right=0.4\base of dec3] (end2) {$\cdots$};
                \node[wordnode,right=0.4\base of softmax3] (end3) {$\cdots$};

                % Decoder input words
                \node[wordnode,below=0pt of demb1] (decwordin) {$\langle$sos$\rangle$};
                \ExtractX{$(demb2.south)$}
                \ExtractY{$(decwordin.base)$}
                \node[wordnode,anchor=base] () at (\XCoord,\YCoord) {Do};
                \ExtractX{$(demb3.south)$}
                \ExtractY{$(decwordin.base)$}
                \node[wordnode,anchor=base] () at (\XCoord,\YCoord) {you};

                % Decoder output words
                \node[wordnode,above=0pt of softmax1] (decwordout) {Do};
                \ExtractX{$(softmax2.north)$}
                \ExtractY{$(decwordout.base)$}
                \node[wordnode,anchor=base] () at (\XCoord,\YCoord) {you};
                \ExtractX{$(softmax3.north)$}
                \ExtractY{$(decwordout.base)$}
                \node[wordnode,anchor=base] () at (\XCoord,\YCoord) {know};

                % Connections
                \draw[-latex'] (init1.east) to (enc1.west);
                \draw[-latex'] (dec3.east) to (end2.west);
                \foreach \x in {1,2,...,3}
                    \draw[-latex'] (eemb\x) to (enc\x);
                \foreach \x in {1,2,...,3}
                    \draw[-latex'] (demb\x) to (dec\x);
                \foreach \x in {1,2,...,3}
                    \draw[-latex'] (dec\x.north) to (softmax\x.south);
                \foreach \x [count=\y from 2] in {1,2}
                {
                    \draw[-latex'] (enc\x.east) to (enc\y.west);
                    \draw[-latex'] (dec\x.east) to (dec\y.west);
                }

                \coordinate (bridge) at ([yshift=0.4\base]enc2.north west);
                \draw[-latex'] (enc3.north) .. controls +(north:0.3\base) and +(east:\base) .. (bridge) .. controls +(west:2.7\base) and +(west:0.3\base) .. (dec1.west);
            \end{scope}

            \begin{scope}
                \coordinate (start) at (8.5\base,0.1\base);

                \node [anchor=center,minimum width=5.7em,minimum height=1.3em,draw,rounded corners=0.3em] (hidden) at (start) {};
                \node [anchor=west,minimum width=1em,minimum size=1em,fill=ugreen!20] (cell01) at ([xshift=0.2em]hidden.west) {\scriptsize{.2}};
                \node [anchor=west,minimum width=1em,minimum size=1em,fill=ugreen!10] (cell02) at (cell01.east) {\scriptsize{-1}};
                \node [anchor=west,minimum width=1em,minimum size=1em,fill=white] (cell03) at (cell02.east) {\scriptsize{$\cdots$}};
                \node [anchor=west,minimum width=1em,minimum size=1em,fill=ugreen!50] (cell04) at (cell03.east) {\scriptsize{5}};

                {
                    \node [anchor=south,minimum width=10.9em,minimum height=1.3em,draw,rounded corners=0.3em] (target) at ([yshift=1.5em]hidden.north) {};
                    \node [anchor=west,minimum width=1em,minimum size=1em,fill=ugreen!10] (cell11) at ([xshift=0.2em]target.west) {\scriptsize{-2}};
                    \node [anchor=west,minimum width=1em,minimum size=1em,fill=ugreen!10] (cell12) at (cell11.east) {\scriptsize{-1}};
                    \node [anchor=west,minimum width=1em,minimum size=1em,fill=ugreen!30] (cell13) at (cell12.east) {\scriptsize{.7}};
                    \node [anchor=west,minimum width=1em,minimum size=1em,fill=white] (cell14) at (cell13.east) {\scriptsize{$\cdots$}};
                    \node [anchor=west,minimum width=1em,minimum size=1em,fill=ugreen!70] (cell15) at (cell14.east) {\scriptsize{6}};
                    \node [anchor=west,minimum width=1em,minimum size=1em,fill=ugreen!10] (cell16) at (cell15.east) {\scriptsize{-3}};
                    \node [anchor=west,minimum width=1em,minimum size=1em,fill=ugreen!10] (cell17) at (cell16.east) {\scriptsize{-1}};
                    \node [anchor=west,minimum width=1em,minimum size=1em,fill=ugreen!20] (cell18) at (cell17.east) {\scriptsize{.2}};
                }

               {
                    \node [anchor=south,minimum width=1em,minimum height=0.2em,fill=ublue!80,inner sep=0pt] (label1) at ([yshift=2.5em]cell11.north) {};
                    \node [anchor=west,rotate=90,font=\tiny] (w1) at (label1.north) {$\langle$eos$\rangle$};
                    \node [anchor=south,minimum width=1em,minimum height=0.3em,fill=ublue!80,inner sep=0pt] (label2) at ([yshift=2.5em]cell12.north) {};
                    \node [anchor=west,rotate=90,font=\tiny] (w2) at (label2.north) {$\langle$sos$\rangle$};
                    \node [anchor=south,minimum width=1em,minimum height=0.5em,fill=ublue!80,inner sep=0pt] (label3) at ([yshift=2.5em]cell13.north) {};
                    \node [anchor=west,rotate=90,font=\tiny] (w3) at (label3.north) {Do};
                    \node [anchor=south,font=\scriptsize] (w4) at ([yshift=2.5em]cell14.north) {$\cdots$};
                    \node [anchor=south,minimum width=1em,minimum height=1em,fill=ublue!80,inner sep=0pt] (label5) at ([yshift=2.5em]cell15.north) {};
                    
                    {
                        \node [anchor=west,rotate=90,font=\tiny] (w5) at (label5.north) {{\color{red} know}};
                    }
                   
                    \node [anchor=south,minimum width=1em,minimum height=0.1em,fill=ublue!80,inner sep=0pt] (label6) at ([yshift=2.5em]cell16.north) {};
                    \node [anchor=west,rotate=90,font=\tiny] (w6) at (label6.north) {you};
                    \node [anchor=south,minimum width=1em,minimum height=0.3em,fill=ublue!80,inner sep=0pt] (label7) at ([yshift=2.5em]cell17.north) {};
                    \node [anchor=west,rotate=90,font=\tiny] (w7) at (label7.north) {?};
                    \node [anchor=south,minimum width=1em,minimum height=0.4em,fill=ublue!80,inner sep=0pt] (label8) at ([yshift=2.5em]cell18.north) {};
                    \node [anchor=west,rotate=90,font=\tiny] (w8) at (label8.north) {have};
                }

                {
                    \filldraw [fill=red!20,draw=white] (target.south west) -- (target.south east) -- ([xshift=-0.2em,yshift=0.1em]hidden.north east) -- ([xshift=0.2em,yshift=0.1em]hidden.north west);
                    \draw [->,thick] ([xshift=0.2em,yshift=0.1em]hidden.north west) -- (target.south west);
                    \draw [->,thick] ([xshift=-0.2em,yshift=0.1em]hidden.north east) -- (target.south east);

                    \node [anchor=south] () at ([yshift=0.3em]hidden.north) {\scriptsize{$\hat{\vectorn{\emph{s}}}_j=\vectorn{\emph{s}}_j \vectorn{\emph{W}}_o$}};
                }

                {
                    \node [rounded corners=0.3em] (softmax) at ([yshift=1.25em]target.north) {\scriptsize{$p(\hat{s}_i)=\frac{e^{\hat{s}_i}}{\sum_j e^{\hat{s}_j}}$}};
                    \filldraw [fill=blue!20,draw=white] ([yshift=0.1em]cell11.north west) {[rounded corners=0.3em] -- (softmax.west)} -- (label1.south west) -- (label8.south east) {[rounded corners=0.3em] -- (softmax.east)} -- ([yshift=0.1em]cell18.north east) -- ([yshift=0.1em]cell11.north west);
                    \node [rounded corners=0.3em] (softmax) at ([yshift=1.25em]target.north) {\scriptsize{$p(\hat{s}_{jk})=\frac{e^{\hat{s}_{jk}}}{\sum_n e^{\hat{s}_{jn}}}$}};
                }
                \draw [-latex'] ([yshift=-0.3cm]hidden.south) to (hidden.south);
                {
                    \draw [-latex'] (w5.east) to ([yshift=0.3cm]w5.east);
                }

                \coordinate (tmp) at ([yshift=-3pt]w5.east);
                \node [draw=red,thick,densely dashed,rounded corners=3pt,inner sep=5pt,fit=(cell01) (cell11) (label1) (label8) (target) (hidden) (tmp)] (output) {};
            \end{scope}

            \draw [->,thick,densely dashed,red] ([yshift=-0.2em]softmax3.east) .. controls +(east:2\base) and +(west:\base) .. (output.west);
        \end{tikzpicture}