% Required:
% \usetikzlibrary{positioning,shadows,decorations.pathreplacing}

\begin{tikzpicture}[
box/.style={
    draw,
    fill=white,
    drop shadow={shadow xshift=0.08cm, shadow yshift=-0.08cm},
    text width=2.7cm,
    minimum height=1.25cm,
    font=\small,
    align=center
},
smallbox/.style={
    draw,
    fill=white,
    drop shadow={shadow xshift=0.06cm, shadow yshift=-0.06cm},
    text width=2.15cm,
    minimum height=0.95cm,
    font=\small,
    align=center
},
rewardbox/.style={
    draw,
    fill=white,
    drop shadow={shadow xshift=0.08cm, shadow yshift=-0.08cm},
    text width=2.7cm,
    minimum height=1.1cm,
    font=\small,
    align=center
},
arrow/.style={
    ->,
    thick
},
dashedarrow/.style={
    ->,
    thick,
    dashed
},
brace/.style={
    decorate,
    decoration={brace, amplitude=0.35cm, raise=0.28cm},
    thick
}
]

% trajectory nodes
\node[box] (plan) {
  \colorbox{blue!20}{\textbf{\textit{Plan}}}\\
  \vspace{.5ex}
  $p$
};

\node[box, right=0.65cm of plan] (step1) {
  \colorbox{red!20}{\textbf{\textit{Step 1}}}\\
  \vspace{.5ex}
  $u_1 \rightarrow o_1$
};

\node[box, right=0.65cm of step1] (step2) {
  \colorbox{red!20}{\textbf{\textit{Step 2}}}\\
  \vspace{.5ex}
  $u_2 \rightarrow o_2$
};

\node[right=0.35cm of step2] (dots) {
  $\cdots$
};

\node[box, right=0.35cm of dots] (stepT) {
  \colorbox{red!20}{\textbf{\textit{Step T}}}\\
  \vspace{.5ex}
  $u_T \rightarrow o_T$
};

\node[rewardbox, right=0.75cm of stepT] (finalreward) {
  \colorbox{green!20}{\textbf{\textit{Final Reward}}}\\
  \vspace{.5ex}
  $R(\tau)$
};

% arrows on trajectory
\draw[arrow] (plan) -- (step1);
\draw[arrow] (step1) -- (step2);
\draw[arrow] (step2) -- (dots);
\draw[arrow] (dots) -- (stepT);
\draw[arrow] (stepT) -- (finalreward);

% brace annotation for the interaction trajectory
\draw[brace]
    ([yshift=0.15cm]plan.north west) --
    ([yshift=0.15cm]stepT.north east)
    node[midway, yshift=0.95cm, font=\small\bfseries] {Interaction Trajectory};

% lower credit assignment module
\node[rewardbox, below=1.15cm of finalreward] (assign) {
\textbf{\textit{Credit Assignment}}\\
Distribute trajectory-level feedback
};

\draw[arrow] (finalreward) -- (assign);

% step rewards
\node[smallbox, below=1.25cm of plan] (rp) {
Plan reward\\
$r_p$
};

\node[smallbox, below=1.25cm of step1] (r1) {
Step reward\\
$r_1$
};

\node[smallbox, below=1.25cm of step2] (r2) {
Step reward\\
$r_2$
};

\node[smallbox, below=1.25cm of stepT] (rT) {
Step reward\\
$r_T$
};

% curved credit-assignment arrows
\draw[dashedarrow]
    (assign.south)
    .. controls +(down:1.25cm) and +(down:1.25cm) ..
    (rT.south);

\draw[dashedarrow]
    (assign.south)
    .. controls +(down:1.55cm) and +(down:1.55cm) ..
    (r2.south);

\draw[dashedarrow]
    (assign.south)
    .. controls +(down:1.85cm) and +(down:1.85cm) ..
    (r1.south);

\draw[dashedarrow]
    (assign.south)
    .. controls +(down:2.15cm) and +(down:2.15cm) ..
    (rp.south);

% local arrows back to decisions
\draw[dashedarrow] (rp) -- (plan);
\draw[dashedarrow] (r1) -- (step1);
\draw[dashedarrow] (r2) -- (step2);
\draw[dashedarrow] (rT) -- (stepT);


\end{tikzpicture}
