@@ -59,6 +59,11 @@ Lichang Chen, Shiyang Li, Jun Yan, Hai Wang, Kalpa Gunaratna, Vikas Yadav, Zheng
\newblock In \emph{The Twelfth International Conference on Learning Representations, {ICLR} 2024, Vienna, Austria, May 7-11, 2024}. OpenReview.net, 2024.
title = {ReSearch: Learning to Reason with Search for LLMs via Reinforcement Learning},
author = {Chen, Mingyang and Li, Tianpeng and Sun, Haoze and Zhou, Yijie and Zhu, Chenzheng and Yang, Fan and Zhou, Zenan and Chen, Weipeng and Wang, Haofen and Pan, Jeff Z. and Zhang, Wen and Chen, Huajun},
journal = {arXiv preprint arXiv:2503.19470},
year = {2025}
}
@article{jin-etal:searchr1,
title = {Search-R1: Training LLMs to Reason and Leverage Search Engines with Reinforcement Learning},
author = {Jin, Bowen and Zeng, Hansi and Yue, Zhenrui and Wang, Dong and Zamani, Hamed and Han, Jiawei},
journal = {arXiv preprint arXiv:2503.09516},
year = {2025}
}
@article{singh-etal:artist,
title = {Agentic Reasoning and Tool Integration for LLMs via Reinforcement Learning},
author = {Singh, Joykirat and Magazine, Raghav and Pandya, Yash and Nambi, Akshay},
journal = {arXiv preprint arXiv:2505.01441},
year = {2025}
}
@article{feng-etal:retool,
title = {ReTool: Reinforcement Learning for Strategic Tool Use in LLMs},
author = {Feng, Jiazhan and Huang, Shijue and Qu, Xingwei and Zhang, Ge and Qin, Yujia and Zhong, Baoquan and Jiang, Chengquan and Chi, Jinxin and Zhong, Wanjun},
journal = {arXiv preprint arXiv:2504.11536},
year = {2025}
}
@article{qian-etal:toolrl,
title = {ToolRL: Reward is All Tool Learning Needs},
author = {Qian, Cheng and Acikgoz, Emre Can and He, Qi and Wang, Hongru and Chen, Xiusi and Hakkani-Tr, Dilek and Tur, Gokhan and Ji, Heng},
journal = {arXiv preprint arXiv:2504.13958},
year = {2025}
}
@inproceedings{patil-etal:gorilla,
title = {Gorilla: Large Language Model Connected with Massive APIs},
...
...
@@ -76,15 +111,6 @@
year={2024}
}
@article{qian-etal:toolrl,
title={Toolrl: Reward is all tool learning needs},
author={Qian, Cheng and Acikgoz, Emre Can and He, Qi and Wang, Hongru and Chen, Xiusi and Hakkani-Tur, Dilek and Tur, Gokhan and Ji, Heng},
journal={Advances in Neural Information Processing Systems},
volume={38},
pages={105523--105553},
year={2026}
}
@article{yao-etal:react,
title={React: Synergizing reasoning and acting in language models},
author={Yao, Shunyu and Zhao, Jeffrey and Yu, Dian and Du, Nan and Shafran, Izhak and Narasimhan, Karthik and Cao, Yuan},
@@ -205,13 +205,38 @@ Beyond learning to use a small set of tools, later studies further scale tool-us
% 4. RL for Tool Use
\subsubsubsection{Reinforcement Learning}
Although SFT can teach LLMs to imitate tool-use demonstrations, it is still limited by the coverage and quality of offline trajectories. The model mainly learns the tool-use patterns that appear in the supervised data. As a result, it may fail to generalize to unfamiliar tools, complex tool combinations, or new interaction patterns. More importantly, SFT does not directly optimize whether a tool call is necessary, whether the selected tool is appropriate, or whether the returned observation improves the final answer. This limitation becomes more serious in multi-step tool-use scenarios, \textit{where the model needs to decide when to call a tool, how to formulate the tool input, how to interpret the tool output, and when to stop using tools}.
To address this limitation, researchers have explored RL for tool use. Instead of only imitating fixed demonstrations, the model can interact with tools during rollout and receive feedback from task outcomes or tool execution results. Given a user task $q$ and a set of available tools $\mathcal{T}$, the model generates a tool-use trajectory:
\begin{eqnarray}
\tau = [s_1,u_1,o_1,\cdots,s_T,u_T,o_T,a]
\end{eqnarray}
where $s_t$ denotes the reasoning state at the $t$-th step, $u_t$ denotes the model action, $o_t$ denotes the observation returned by the tool or environment, and $a$ denotes the final answer. Here, $u_t$ can be either a natural language reasoning step, a tool invocation with arguments, or a final response. The goal of RL is to optimize the policy so that the generated trajectory receives a higher reward:
In practice, this objective can be optimized with standard RL algorithms such as PPO or GRPO. Recent studies show that such outcome-driven optimization can help LLMs learn more adaptive tool-use behaviors, such as deciding when to search, when to execute code, and how to revise a tool call after receiving an error message \citep{jin-etal:searchr1,chen-etal:research,feng-etal:retool,singh-etal:artist}.
However, RL-based tool use also introduces several unique challenges. First, the action space is more complex than ordinary text generation. A tool-use action may involve tool selection, argument generation, execution timing, and response integration. A small error in any part can lead to tool failure. Second, the reward signal is often sparse and delayed. The environment may only provide a final success signal after the whole trajectory is completed, making it difficult to identify which tool call causes success or failure. Third, tool use may introduce additional costs. Unnecessary tool calls can increase latency and computation cost, while insufficient tool use may lead to incorrect answers. Therefore, the model needs to learn not only how to use tools, but also how to use them efficiently.
A key direction is to design more informative rewards for tool use. Instead of using only a final-answer reward, we can decompose the reward into multiple components:
\begin{eqnarray}
R(\tau) =
R_{\mathrm{ans}}(\tau)
+ \alpha R_{\mathrm{fmt}}(\tau)
+ \beta R_{\mathrm{exec}}(\tau)
+ \gamma R_{\mathrm{tool}}(\tau)
- \lambda C_{\mathrm{tool}}(\tau)
\end{eqnarray}
where $R_{\mathrm{ans}}$ measures final answer correctness, $R_{\mathrm{fmt}}$ evaluates whether the model follows the required tool-use format, $R_{\mathrm{exec}}$ measures whether tool calls can be successfully executed, and $R_{\mathrm{tool}}$ evaluates whether the model selects appropriate tools and passes valid arguments. $C_{\mathrm{tool}}$ denotes the cost of tool use, such as the number of tool calls, latency, or computational expense. The coefficients $\alpha$, $\beta$, $\gamma$, and $\lambda$ control the relative importance of different reward terms.
In application, this reward decomposition provides denser feedback than final-answer rewards. For example, if the model selects the correct tool but passes an invalid argument, the execution reward can penalize the invalid call while preserving credit for the correct tool selection. If the model reaches the correct answer but uses many unnecessary tool calls, the cost term can discourage inefficient behavior. ToolRL systematically studies this reward design problem and shows that fine-grained reward decomposition can make RL training more stable and effective for tool selection and tool application \citep{qian-etal:toolrl}.
% 5. Efficient Tool Use
\subsubsubsection{Efficient Tool Use}
Another direction is to use RL to improve strategic tool use. For example, ReTool first builds a cold-start model with code-augmented reasoning traces and then applies RL with real-time code execution. During rollout, the model interleaves natural language reasoning with code execution, observes execution results, and revises its subsequent reasoning. This allows the model to discover when and how to invoke a code interpreter based on outcome feedback rather than human-written rules \citep{feng-etal:retool}. Search-R1 and ReSearch follow a similar idea in retrieval-augmented reasoning. They train models to generate search queries during reasoning and use retrieved information to update later steps, rather than relying on fixed retrieval pipelines or hand-crafted search prompts \citep{jin-etal:searchr1,chen-etal:research}.
