fig-activation.tex

%%%------------------------------------------------------------------------------------------------------------
\begin{figure}
\subfigure[softplus]{
\centering
\begin{minipage}{.2\textwidth}
\begin{tikzpicture}
 \draw[->](-1.2,0)--(1.2,0)node[left,below,font=\tiny]{$x$};
\draw[->](0,-1.2)--(0,1.2)node[right,font=\tiny]{$y$};
\foreach \x in {-1.0,-0.5,0.0,0.5,1.0}{\draw(\x,0)--(\x,0.05)node[below,outer sep=2pt,font=\tiny]at(\x,0){\x};}
\foreach \y in {1.0,0.5}{\draw(0,\y)--(0.05,\y)node[left,outer sep=2pt,font=\tiny]at(0,\y){\y};}
\draw[color=red ,domain=-1.2:1]plot(\x,{ln(1+(exp(\x))});
\node[black,anchor=south] at (0,1.2) {\small $y = ln(1+e^x)$};
\end{tikzpicture}
\end{minipage}%
}
\hfill
\subfigure[sigmoid]{
\centering
\begin{minipage}{.2\textwidth}
\begin{tikzpicture}
\draw[->](-1.2,0)--(1.2,0)node[left,below,font=\tiny]{$x$};
\draw[->](0,-1.2)--(0,1.2)node[right,font=\tiny]{$y$};
\draw[dashed](-1.2,1)--(1.2,1);
\foreach \x in {-1,-0.5,0,0.5,1}{\draw(\x,0)--(\x,0.05)node[below,outer sep=2pt,font=\tiny]at(\x,0){
\pgfmathparse{(\x)*5}
\pgfmathresult};}
\foreach \y in {0.5,1.0}{\draw(0,\y)--(0.05,\y)node[left,outer sep=2pt,font=\tiny]at(0,\y){\y};}
\draw[color=red,domain=-1.2:1.2]plot(\x,{1/(1+(exp(-5*\x)))});
\node[black,anchor=south] at (0,1.2) {\small $y = \frac{1}{1+e^{-x}}$};
\end{tikzpicture}
   \end{minipage}%
}
\hfill
\subfigure[tanh]{
 \centering
\begin{minipage}{.2\textwidth}
\begin{tikzpicture}
\draw[->](-1.2,0)--(1.2,0)node[left,below,font=\tiny]{$x$};
\draw[->](0,-1.2)--(0,1.2)node[right,font=\tiny]{$y$};
\draw[dashed](-1.2,1)--(1.2,1);
\draw[dashed](-1.2,-1)--(1.2,-1);
\foreach \x in {-1.0,-0.5,0.0,0.5,1.0}{\draw(\x,0)--(\x,0.05)node[below,outer sep=2pt,font=\tiny]at(\x,0){\x};}
\foreach \y in {0.5,1.0}{\draw(0,\y)--(0.05,\y)node[left,outer sep=2pt,font=\tiny]at(0,\y){\y};}
\draw[color=red ,domain=-1.2:1.2]plot(\x,{tanh(\x)});
\node[black,anchor=south] at (0,1.2) {\small $y = \frac{e^{x}-e^{-x}}{e^{x}+e^{-x}}$};
\end{tikzpicture}
\end{minipage}
}\\    \vspace{-1em}
\subfigure[relu]{
\centering
\begin{minipage}{.2\textwidth}
\begin{tikzpicture}
        \draw[->](-1.2,0)--(1.2,0)node[left,below,font=\tiny]{$x$};
        \draw[->](0,-1.2)--(0,1.2)node[right,font=\tiny]{$y$};
        \draw[dashed](-1.2,1)--(1.2,1);
        \draw[dashed](-1.2,-1)--(1.2,-1);
        \foreach \x in {-1.0,-0.5,0.0,0.5,1.0}{\draw(\x,0)--(\x,0.05)node[below,outer sep=2pt,font=\tiny]at(\x,0){\x};}
        \foreach \y in {0.5,1.0}{\draw(0,\y)--(0.05,\y)node[left,outer sep=2pt,font=\tiny]at(0,\y){\y};}
        \draw[color=red ,domain=-1.2:1.2]plot(\x,{max(\x,0)});
        \node[black,anchor=south] at (0,1.2) {\small $y =\max (0, x)$};
        \end{tikzpicture}
    \end{minipage}%
    }
    \hfill
    \subfigure[gaussian]{
    \centering
    \begin{minipage}{.2\textwidth}
        \begin{tikzpicture}
        \draw[->](-1.2,0)--(1.2,0)node[left,below,font=\tiny]{$x$};
        \draw[->](0,-1.2)--(0,1.2)node[right,font=\tiny]{$y$};
        \draw[dashed](-1.2,1)--(1.2,1);
        \foreach \x in {-1.0,-0.5,0.0,0.5,1.0}{\draw(\x,0)--(\x,0.05)node[below,outer sep=2pt,font=\tiny]at(\x,0){\x};}
        \foreach \y in {0.5,1.0}{\draw(0,\y)--(0.05,\y)node[left,outer sep=2pt,font=\tiny]at(0,\y){\y};}
        \draw[color=red ,domain=-1.2:1.2]plot(\x,{exp(-1*((\x)^2))});
        \node[black,anchor=south] at (0,1.2) {\small $y =e^{-x^2}$};
        \end{tikzpicture}
    \end{minipage}%
    }
    \hfill
    \subfigure[identity]{
    \centering
    \begin{minipage}{.2\textwidth}
        \begin{tikzpicture}
        \draw[->](-1.2,0)--(1.2,0)node[left,below,font=\tiny]{$x$};
        \draw[->](0,-1.2)--(0,1.2)node[right,font=\tiny]{$y$};
        \foreach \x in {-1.0,-0.5,0.0,0.5,1.0}{\draw(\x,0)--(\x,0.05)node[below,outer sep=2pt,font=\tiny]at(\x,0){\x};}
        \foreach \y in {0.5,1.0}{\draw(0,\y)--(0.05,\y)node[left,outer sep=2pt,font=\tiny]at(0,\y){\y};}
        \draw[color=red ,domain=-1:1]plot(\x,\x);
        \node[black,anchor=south] at (0,1.2) {\small $y =x$};
        \end{tikzpicture}
    \end{minipage}
    }

    \end{figure}
%%%------------------------------------------------------------------------------------------------------------