15章图片

Chapter15/Figures/figure-activation-function-swish-structure-diagram.tex

+
+%%% outline
+%-------------------------------------------------------------------------
+\begin{tikzpicture}
+\tikzstyle{opt}=[draw,minimum height=2em,minimum width=4em,rounded corners=2pt,thick]
+
+\node[circle,minimum size=2em,draw,fill=red!20] (x1) at (0,0) {\small\bfnew{X}};
+\node[circle,minimum size=2em,draw,fill=red!20] (x2) at (0,4em) {\small\bfnew{X}};
+\node[circle,minimum size=2em,draw,fill=red!20] (x3) at (0,-5em) {\small\bfnew{X}};
+
+\node[anchor=west,opt,fill=yellow!20] (unary1) at ([xshift=3em]x1.east){\small\bfnew{Unary}};
+\node[anchor=west,opt,fill=yellow!20] (unary2) at ([xshift=3em]x2.east){\small\bfnew{Unary}};
+
+\node[opt,fill=blue!20] (binary1) at (12em,2em){\small\bfnew{Binary}};
+\node[opt,fill=blue!20] (binary2) at (25em,-1.5em){\small\bfnew{Binary}};
+\node[anchor=west,opt,fill=yellow!20] (unary3) at ([xshift=3em]binary1.east){\small\bfnew{Unary}};
+\node[anchor=west,opt,fill=yellow!20] (unary4) at ([xshift=16em]x3.east){\small\bfnew{Unary}};
+
+
+\draw[-latex,very thick] (x1.0) -- (unary1.180);
+\draw[-latex,very thick] (x2.0) -- (unary2.180);
+\draw[-latex,very thick] (x3.0) -- (unary4.180);
+\draw[-latex,very thick] (unary1.0) -- ([yshift=-0.2em]binary1.180);
+\draw[-latex,very thick] (unary2.0) -- ([yshift=0.2em]binary1.180);
+\draw[-latex,very thick] (binary1.0) -- (unary3.180);
+\draw[-latex,very thick] (unary3.0) -- ([yshift=0.2em]binary2.180);
+\draw[-latex,very thick] (unary4.0) -- ([yshift=-0.2em]binary2.180);
+
+\begin{pgfonlayer}{background}
+\node[draw=lightgray,fill=lightgray!50,rounded corners=2pt,inner sep=8pt][fit=(x2)(x1)(binary1)]{};
+\end{pgfonlayer}
+\node[anchor=south] at ([yshift=1em]binary1.north){\small\bfnew{Core Unit}};
+\end{tikzpicture}
+
+
+
+

Chapter15/Figures/figure-learning-of-local-structure-combination.tex

+
+%%% outline
+%-------------------------------------------------------------------------
+\begin{tikzpicture}
+\tikzstyle{opt}=[minimum height=1em,minimum width=5em,rounded corners=2pt,thick]
+
+\node[opt] (opt1_0) at (0,0) {固定操作};
+\node[draw,anchor=south,opt,fill=green!20] (opt1_1) at ([yshift=0.8em]opt1_0.north) {操作3};
+\node[draw,anchor=south,opt,fill=cyan!20] (opt1_2) at ([yshift=0.8em]opt1_1.north) {操作2};
+\node[anchor=south,opt] (opt1_3) at ([yshift=0.8em]opt1_2.north) {固定操作};
+\node[draw,anchor=south,opt,fill=yellow!20] (opt1_4) at ([yshift=0.8em]opt1_3.north) {操作4};
+\node[anchor=south,opt] (opt1_5) at ([yshift=0.8em]opt1_4.north) {$\cdots$};
+\node[draw,anchor=south,opt,fill=blue!10] (opt1_6) at ([yshift=0.8em]opt1_5.north) {操作1};
+\node[anchor=south,opt] (opt1_7) at ([yshift=0.8em]opt1_6.north) {};
+
+\node[opt] (opt2_0) at (3,0) {固定操作};
+\node[draw,anchor=south,opt,fill=blue!10] (opt2_1) at ([yshift=0.8em]opt2_0.north) {操作1};
+\node[draw,anchor=south,opt,fill=green!20] (opt2_2) at ([yshift=0.8em]opt2_1.north) {操作3};
+\node[anchor=south,opt] (opt2_3) at ([yshift=0.8em]opt2_2.north) {固定操作};
+\node[draw,anchor=south,opt,fill=red!20] (opt2_4) at ([yshift=0.8em]opt2_3.north) {操作5};
+\node[anchor=south,opt] (opt2_5) at ([yshift=0.8em]opt2_4.north) {$\cdots$};
+\node[draw,anchor=south,opt,fill=cyan!20] (opt2_6) at ([yshift=0.8em]opt2_5.north) {操作2};
+\node[anchor=south,opt] (opt2_7) at ([yshift=0.8em]opt2_6.north) {};
+
+\node[opt] (opt3_0) at (6,0) {固定操作};
+\node[draw,anchor=south,opt,fill=yellow!20] (opt3_1) at ([yshift=0.8em]opt3_0.north) {操作4};
+\node[draw,anchor=south,opt,fill=cyan!20] (opt3_2) at ([yshift=0.8em]opt3_1.north) {操作2};
+\node[anchor=south,opt] (opt3_3) at ([yshift=0.8em]opt3_2.north) {固定操作};
+\node[draw,anchor=south,opt,fill=yellow!20] (opt3_4) at ([yshift=0.8em]opt3_3.north) {操作4};
+\node[anchor=south,opt] (opt3_5) at ([yshift=0.8em]opt3_4.north) {$\cdots$};
+\node[draw,anchor=south,opt,fill=red!20] (opt3_6) at ([yshift=0.8em]opt3_5.north) {操作5};
+\node[anchor=south,opt] (opt3_7) at ([yshift=0.8em]opt3_6.north) {};
+
+\begin{pgfonlayer}{background}
+\node[draw,fill=yellow!20,rounded corners=6pt,inner ysep=2.6em,inner xsep=2.6em] [fit=(opt1_0) (opt3_7)](box4){};
+\node[draw,fill=gray!10,rounded corners=2pt,inner sep=8pt] [fit=(opt1_0) (opt1_7)](box1){};
+\node[draw,fill=gray!10,rounded corners=2pt,inner sep=8pt] [fit=(opt2_0) (opt2_7)](box2){};
+\node[draw,fill=gray!10,rounded corners=2pt,inner sep=8pt] [fit=(opt3_0) (opt3_7)](box3){};
+\end{pgfonlayer}
+
+
+\draw[->,thick] (opt1_0) -- (opt1_1);
+\draw[->,thick] (opt1_1) -- (opt1_2);
+\draw[->,thick] (opt1_2) -- (opt1_3);
+\draw[->,thick] (opt1_3) -- (opt1_4);
+\draw[->,thick] (opt1_4) -- (opt1_5);
+\draw[->,thick] (opt1_5) -- (opt1_6);
+
+\draw[->,thick] (opt2_0) -- (opt2_1);
+\draw[->,thick] (opt2_1) -- (opt2_2);
+\draw[->,thick] (opt2_2) -- (opt2_3);
+\draw[->,thick] (opt2_3) -- (opt2_4);
+\draw[->,thick] (opt2_4) -- (opt2_5);
+\draw[->,thick] (opt2_5) -- (opt2_6);
+
+\draw[->,thick] (opt3_0) -- (opt3_1);
+\draw[->,thick] (opt3_1) -- (opt3_2);
+\draw[->,thick] (opt3_2) -- (opt3_3);
+\draw[->,thick] (opt3_3) -- (opt3_4);
+\draw[->,thick] (opt3_4) -- (opt3_5);
+\draw[->,thick] (opt3_5) -- (opt3_6);
+
+\node[] at ([xshift=-1.2em,yshift=0.2em]opt1_7){\small\bfnew{分支1}};
+\node[] at ([xshift=-1.2em,yshift=0.2em]opt2_7){\small\bfnew{分支2}};
+\node[] at ([xshift=-1.2em,yshift=0.2em]opt3_7){\small\bfnew{分支3}};
+
+\node[] (input) at ([yshift=-5em]opt2_0){\small\bfnew{输入}};
+\node[] (output) at ([yshift=5em]opt2_7){\small\bfnew{输出}};
+
+\draw[->,thick,out=140,in=-30] (box4.-90) to (box1.-90);
+\draw[->,thick,out=40,in=-150] (box4.-90) to (box3.-90);
+\draw[->,thick] (box4.-90) -- (box2.-90);
+
+\draw[->,thick,out=50,in=-130] (box1.90) to (box4.90);
+\draw[->,thick,out=130,in=-50] (box3.90) to (box4.90);
+\draw[->,thick] (box2.90) -- (box4.90);
+
+\draw[->,thick] (input.90) -- (box4.-90);
+\draw[->,thick] (box4.90) -- (output.-90);
+
+\node[] at ([xshift=-2.8em,yshift=1.1em]box1.90){\small\bfnew{模型结构}};
+\node[] at ([xshift=-0.8em]box4.0){$\cdots$};
+\end{tikzpicture}
+
+
+
+

Chapter15/Figures/figure-structure-search-based-on-evolutionary-algorithm.tex

+
+%%% outline
+%-------------------------------------------------------------------------
+\begin{tikzpicture}
+\tikzstyle{node}=[minimum height=2em,minimum width=4.6em,rounded corners=2pt,thick,font=\footnotesize]
+
+\node[node,draw,fill=green!20] (n11) at (0,0) {模型结构1};
+\node[anchor=west,draw,node,fill=red!20] (n12) at ([xshift=0.6em]n11.east){模型结构6};
+\node[anchor=north,draw,node,fill=orange!20] (n13) at ([yshift=-1em]n11.south){模型结构2};
+\node[anchor=west,draw,node,fill=blue!15] (n14) at ([xshift=0.6em]n13.east){模型结构5};
+\node[anchor=north,draw,node,fill=cyan!30] (n15) at ([yshift=-1em]n13.south){模型结构1};
+\node[anchor=west,draw,node,fill=yellow!20] (n16) at ([xshift=0.6em]n15.east){模型结构4};
+\node[inner sep=0pt] (kind) at (-0.7,0.8){\small\bfnew{种群}};
+
+\node[node,draw,fill=green!20] (n21) at (5,0){模型结构1};
+\node[anchor=west,draw,node,fill=red!20] (n22) at ([xshift=0.6em]n21.east){模型结构4};
+\node[anchor=north,node] (n23) at ([yshift=-1em]n21.south){};
+\node[anchor=west,node] (n24) at ([xshift=0.6em]n23.east){};
+\node[anchor=north,node] (n25) at ([yshift=-1em]n23.south){};
+\node[anchor=west,node] (n26) at ([xshift=0.6em]n25.east){};
+\node[inner sep=0pt] (choice) at (4.8,0.8){\small\bfnew{选中的亲本}};
+
+\node[node,draw,fill=green!20,dotted] (n31) at (10,0){模型结构1};
+\node[anchor=west,draw,node,fill=red!20,dotted] (n32) at ([xshift=0.6em]n31.east){模型结构4};
+\node[anchor=north,node] (n33) at ([yshift=-1em]n31.south){};
+\node[anchor=west,node] (n34) at ([xshift=0.6em]n33.east){};
+\node[anchor=north,draw,node,fill=green!40] (n35) at ([yshift=-1em]n33.south){模型结构1};
+\node[anchor=west,draw,node,fill=red!40] (n36) at ([xshift=0.6em]n35.east){模型结构1};
+\node[inner sep=0pt] (change) at (9.7,0.8){\small\bfnew{亲本变异}};
+
+\begin{pgfonlayer}{background}
+\node[rounded corners=4pt,draw,thick,fill=yellow!10,inner sep=4pt,drop shadow][fit=(kind)(n16)](box1){};
+\node[rounded corners=4pt,draw,thick,fill=yellow!10,inner sep=4pt,drop shadow][fit=(choice)(n26)](box2){};
+\node[rounded corners=4pt,draw,thick,fill=yellow!10,inner sep=4pt,drop shadow][fit=(change)(n36)](box3){};
+\end{pgfonlayer}
+
+\draw[->,very thick] (box1.0) -- (box2.180);
+\draw[->,very thick] (box2.0) -- (box3.180);
+\draw[->,very thick] (n31.-90) -- (n35.90);
+\draw[->,very thick] (n32.-90) -- (n36.90);
+\draw[->,very thick] (box3.-90) .. controls ([yshift=-2em,xshift=-1em]box3.-90) and ([yshift=-2em,xshift=1em]box1.-90) .. node[font=\scriptsize,below]{对变异后的结构进行性能评估，选择优秀的结构加入原始种群} (box1.-90);
+\end{tikzpicture}
+
+
+
+

Chapter15/Figures/figure-structure-search-based-on-gradient-method.tex

+
+%%% outline
+%-------------------------------------------------------------------------
+\begin{tikzpicture}
+\tikzstyle{node}=[minimum height=6em,inner sep=4pt,align=left,draw,font=\footnotesize,rounded corners=4pt,thick,drop shadow]
+
+\node[node,fill=red!20] (n1) at (0,0){\scriptsize\bfnew{超网络}： \\ [1ex] 模型结构参数 \\[0.4ex] 网络参数};
+\node[anchor=west,node,fill=yellow!20] (n2) at ([xshift=4em]n1.east){\scriptsize\bfnew{优化后的超网络}： \\ [1ex]模型{\color{red}结构参数}（已优化） \\ [0.4ex]网络参数（已优化）};
+\node[anchor=west,node,fill=blue!20] (n3) at ([xshift=6em]n2.east){\scriptsize\bfnew{找到的模型结构}};
+
+\draw[-latex,thick] (n1.0) -- node[above,align=center,font=\scriptsize]{优化后的\\超网络}(n2.180);
+\draw[-latex,thick] (n2.0) -- node[above,align=center,font=\scriptsize]{根据结构参数\\离散化结构}(n3.180);
+\draw[-latex,out=90,in=100,thick] ([xshift=-2em]n1.90) to node[above,font=\scriptsize]{参数优化}([xshift=2em]n1.90);
+\end{tikzpicture}
+
+
+
+

Chapter15/Figures/figure-structure-search-based-on-reinforcement-learning.tex

+
+%%% outline
+%-------------------------------------------------------------------------
+\begin{tikzpicture}
+\tikzstyle{node}=[minimum height=2em,minimum width=5em,draw,rounded corners=2pt,thick,drop shadow]
+
+\node[node,fill=red!20] (n1) at (0,0){\small\bfnew{环境}};
+\node[anchor=south,node,fill=blue!20] (n2) at ([yshift=5em]n1.north){\small\bfnew{主体}};
+\node[anchor=north,font=\footnotesize] at ([yshift=-0.2em]n1.south){（结构所应用于的任务）};
+\node[anchor=south,font=\footnotesize] at ([yshift=0.2em]n2.north){（结构生成器）};
+
+\draw[-latex,thick] ([yshift=.4em]n1.180) .. controls ([xshift=-3.4em,yshift=.4em]n1.180) and  ([xshift=-3.4em,yshift=-.4em]n2.180) .. node[right,font=\scriptsize,align=left]{\scriptsize\bfnew{奖励} \\ （对输出结果的评价）}([yshift=-.4em]n2.180);
+\draw[-latex,thick] ([yshift=-.4em]n1.180) .. controls ([xshift=-4.4em,yshift=-.4em]n1.180) and  ([xshift=-4.4em,yshift=.4em]n2.180) .. node[left,font=\scriptsize,align=right]{\scriptsize\bfnew{状态} \\ （这个结构在任务中应用 \\ 后得到的输出结果）}([yshift=.4em]n2.180);
+\draw[-latex,thick] (n2.0) .. controls ([xshift=4em]n2.0) and  ([xshift=4em]n1.0) .. node[right,font=\scriptsize,align=left]{\scriptsize\bfnew{动作} \\ （生成一个结构）}(n1.0);
+\end{tikzpicture}
+
+
+
+

Chapter15/Figures/figure-swish-function-image.tex

+
+%%% outline
+%-------------------------------------------------------------------------
+\begin{tikzpicture}
+
+\begin{axis}[
+width=10cm,height=6cm,
+legend style={legend pos=north west,font=\footnotesize,align=center},
+xtick={-5,-4,-3,-2,-1,0,1,2,3},
+ytick={-2,-1,0,1,2,3},
+x tick label style={font=\scriptsize},
+y tick label style={font=\scriptsize},
+xmin=-5,xmax=3,
+ymin=-2,ymax=3,
+y tick style={opacity=0},
+x tick style={opacity=0},
+grid=major,
+title={\small\bfnew{Swish}},
+]
+
+\addplot[cyan,smooth,thick]{x*(1+exp(-10*x))^-1};
+\addplot[red,smooth,thick]{x*(1+exp(-1*x))^-1};
+\addplot[brown,smooth,thick]{x*(1+exp(-0.1*x))^-1};
+\addplot[gray!60, very thick] coordinates {(0,-2) (0,3)};
+\addplot[gray!60, very thick] coordinates {(0,0) (3,0)};
+\legend{$\beta=0.1$,$\beta=1.0$,$ \ \ \beta=10.0$}
+\end{axis}
+\end{tikzpicture}
+
+
+
+

Chapter15/Figures/figure-whole-structure-and-internal-structure-in-rnn.tex

+
+%%% outline
+%-------------------------------------------------------------------------
+\begin{tikzpicture}[scale=0.8]
+\tikzstyle{every node}=[scale=0.8]
+\tikzstyle{node}=[draw,minimum height=1.4em,minimum width=2em,rounded corners=3pt,thick]
+
+\node[node] (n1) at (0,0){};
+\node[node] (n2) at (1.5,0){};
+\node[node] (n3) at (3,0){};
+\node[node] (n4) at (4.5,0){};
+\node[node] (n5) at (1.5,-1.3){};
+\node[node] (n6) at (3,-1.3){};
+\node[node] (n7) at (2.25,-2.4){};
+\node[node] (n8) at (3,1.3){};
+
+\draw[->,thick] (n1.0) -- (n2.180);
+\draw[->,thick] (n2.0) -- (n3.180);
+\draw[->,thick] (n3.0) -- (n4.180);
+\draw[->,thick,out=60,in=180] (n1.90) to (n8.180);
+\draw[->,thick,out=-10,in=90] (n8.0) to (n4.90);
+\draw[->,thick,out=90,in=-90] (n5.90) to (n3.-90);
+\draw[->,thick,out=90,in=-90] (n6.90) to (n4.-90);
+\draw[->,thick,out=90,in=-90] (n7.90) to (n5.-90);
+\draw[->,thick,out=90,in=-90] (n7.90) to (n6.-90);
+\node[font=\huge] (ht) at (-0.2,2.2){$\mathbi h_t$};
+
+\node[draw,font=\huge,inner sep=0pt,minimum width=4em,minimum height=4em,very thick,rounded corners=2pt] (ht-1) at (-3,0) {$\mathbi h_{t-1}$};
+\node[draw,font=\huge,inner sep=0pt,minimum width=4em,minimum height=4em,very thick,rounded corners=2pt] (ht+1) at (7.5,0) {$\mathbi h_{t+1}$};
+
+\node[font=\huge] (xt) at (2.25,-4.2){$x_t$};
+\node[font=\Huge]  at (9,0){$\cdots$};
+\node[font=\Huge]  at (-4.5,0){$\cdots$};
+
+\node[minimum width=3em,minimum height=2em,fill=red!20,rounded corners=6pt] (b1) at (6,-3.8){};
+\node[]  (w1) at (7.8,-3.8){\Large 整体框架};
+\node[minimum width=3em,minimum height=2em,fill=yellow!30,rounded corners=6pt] (b2) at (6,-4.8){};
+\node[] (w2) at (7.8,-4.8){\Large 内部结构};
+
+\begin{pgfonlayer}{background}
+\node[draw,rounded corners=6pt,very thick,fill=yellow!30,minimum width=16em,minimum height=15em] (box1) at (2.25,0){};
+\node[draw=ublue,very thick,drop shadow,inner sep=1.2em,fill=white,xshift=-0.1em] [fit=(b1)(w2)]{};
+\draw[fill=red!20,red!20,rounded corners=6pt] ([yshift=2.4em,xshift=1em]ht-1.east) -- ([yshift=2.4em,xshift=-0.1em]box1.west) -- ([xshift=-8em,xshift=-0.1em]box1.south) -- ([xshift=2em]box1.south) -- ([xshift=2em,yshift=-5em]box1.south) -- ([xshift=0em,yshift=-5em]box1.south) .. controls ([xshift=-12em,yshift=-5em]box1.south) and ([yshift=-2em]ht-1.east) ..([yshift=2.4em]ht-1.east) -- ([yshift=2.4em,xshift=1em]ht-1.east) ;
+\end{pgfonlayer}
+
+\draw[->,very thick] (ht-1.0) -- (box1.180);
+\draw[->,very thick] (box1.0) -- (ht+1.180);
+\draw[->,very thick] (ht-1.90) -- ([yshift=2em]ht-1.90);
+\draw[->,very thick] (ht+1.90) -- ([yshift=2em]ht+1.90);
+\draw[->,very thick] (box1.90) -- ([yshift=2em]box1.90);
+\draw[->,very thick] ([yshift=-2em]ht-1.-90) -- (ht-1.-90);
+\draw[->,very thick] ([yshift=-2em]ht+1.-90) -- (ht+1.-90);
+\draw[->,very thick] ([yshift=-2em]box1.-90) -- (box1.-90);
+
+
+\end{tikzpicture}
+
+
+
+