update.

rl-introduction.bbl

-\begin{thebibliography}{111}
+\begin{thebibliography}{112}
 \providecommand{\natexlab}[1]{#1}
 \providecommand{\url}[1]{\texttt{#1}}
 \expandafter\ifx\csname urlstyle\endcsname\relax
@@ -441,6 +441,11 @@ Chenglong Wang, Yongyu Mu, Hang Zhou, Yifu Huo, Ziming Zhu, Jiali Zeng, Murun Ya
 \newblock Gram-r$^2$: Self-training generative foundation reward models for reward reasoning.
 \newblock In \emph{Proceedings of the AAAI Conference on Artificial Intelligence}, volume~40, pp.\  33395--33403, 2026{\natexlab{b}}.
 
+\bibitem[Wang et~al.(2026{\natexlab{c}})Wang, Li, Cheng, Ouyang, Yu, Liu, and Chen]{wang-etal:steppo}
+Daoyu Wang, Qingchuan Li, Mingyue Cheng, Jie Ouyang, Shuo Yu, Qi~Liu, and Enhong Chen.
+\newblock Steppo: Step-aligned policy optimization for agentic reinforcement learning.
+\newblock \emph{arXiv preprint arXiv:2604.18401}, 2026{\natexlab{c}}.
+
 \bibitem[Wang et~al.(2021)Wang, Yan, Meng, and Zhou]{wang-etal:2021selective}
 Fusheng Wang, Jianhao Yan, Fandong Meng, and Jie Zhou.
 \newblock Selective knowledge distillation for neural machine translation.

rl-introduction.bib

 
 
 
+@article{wang-etal:steppo,
+  title={StepPO: Step-Aligned Policy Optimization for Agentic Reinforcement Learning},
+  author={Wang, Daoyu and Li, Qingchuan and Cheng, Mingyue and Ouyang, Jie and Yu, Shuo and Liu, Qi and Chen, Enhong},
+  journal={arXiv preprint arXiv:2604.18401},
+  year={2026}
+}
 
 @article{djuhera-etal:tsr,
   title={TSR: Trajectory-Search Rollouts for Multi-Turn RL of LLM Agents},

section6/section6.tex

@@ -105,7 +105,7 @@ Applying RL to agent planning introduces a key challenge: credit assignment. In
     \resizebox{\linewidth}{!}{
         \input{section6/Figures/agent-credit-assignment.tex}
     }
-    \vspace{-1.2cm}
+    \vspace{-1.0cm}
     \caption{Illustration of credit assignment in agentic RL.}
     \label{fig:agent-credit-assignment}
 \end{figure*}
@@ -116,7 +116,17 @@ Given a trajectory $\tau = [p,u_1,o_1,\cdots,u_T,o_T]$, a simple RL objective us
 \begin{eqnarray}
 R(\tau) \rightarrow \{r_p,r_1,\cdots,r_T\}
 \end{eqnarray}
-where $r_p$ evaluates the quality of the generated plan, and $r_t$ evaluates the contribution of the $t$-th interaction step $(u_t,o_t)$ to the final task outcome. The mechanism of credit assignment is illustrated in Figure \ref{fig:agent-credit-assignment}. This decomposition enables the agent to distinguish whether failure comes from an unreasonable plan, an invalid tool call, or a poor adaptation to environmental feedback \citep{li-etal:encouraging,xi-etal:agentprm}.
+where $r_p$ evaluates the quality of the generated plan, and $r_t$ evaluates the contribution of the $t$-th interaction step $(u_t,o_t)$ to the final task outcome. 
+
+The mechanism of credit assignment is illustrated in Figure \ref{fig:agent-credit-assignment}. This decomposition enables the agent to distinguish whether failure comes from an unreasonable plan, an invalid tool call, or a poor adaptation to environmental feedback \citep{li-etal:encouraging,xi-etal:agentprm,wang-etal:steppo}. For example, consider the task of computing the babysitting payment and saving the result into a database. Suppose the agent generates a reasonable plan and correctly calls the calculator, but fails to invoke the database API with the required JSON format. If we only use a final trajectory-level reward, the whole trajectory may receive a low reward, e.g., $R(\tau)=0.3$, even though the early planning and calculation steps are correct. 
+
+For illustration, suppose we define rule-based step-level rewards according to predefined verification criteria, such as whether the plan covers the necessary steps, whether the calculation is correct, whether the JSON schema is valid, and whether the database update succeeds. Credit assignment can then decompose the final feedback into step-level rewards:
+\begin{eqnarray}
+R(\tau)=0.3
+\quad \Rightarrow \quad
+\{r_p=0.4,\ r_1=1.0,\ r_2=1.0,\ r_3=0.7,\ r_4=0.0\}
+\end{eqnarray}
+where $r_p$ indicates that the overall plan is reasonable, $r_1$ and $r_2$ indicate that the agent correctly extracts the numerical values and performs the calculation, $r_3$ indicates that the JSON formatting is partially correct, and $r_4$ indicates that the database update fails. In this way, the agent receives more precise feedback: it should preserve the correct planning.