\scriptsize
\renewcommand{\arraystretch}{1.02}
\begin{tabular}{@{}>{\raggedright\arraybackslash}p{3.3cm}ccc>{\raggedright\arraybackslash}p{1.6cm}c>{\raggedright\arraybackslash}p{1.7cm}>{\raggedright\arraybackslash}p{2.9cm}@{}}
\toprule[1.1pt]
\multirow{2}{*}{Dataset Name} & \multicolumn{3}{c}{Use Case} & \multirow{2}{*}{Scale} & \multirow{2}{*}{Modality} & \multicolumn{2}{c}{Feedback / Reward Signal} \\ \cmidrule(r){2-4}\cmidrule(l){7-8}
& \begin{tabular}[c]{@{}c@{}}\faIcon{star}\\RM\end{tabular}
& \begin{tabular}[c]{@{}c@{}}\faIcon{calculator}\\Rsn.\end{tabular}
& \begin{tabular}[c]{@{}c@{}}\faIcon{robot}\\Agent\end{tabular}
& & & Source & Category \\ \midrule
\href{https://huggingface.co/datasets/Anthropic/hh-rlhf}{Anthropic/hh-rlhf}
& \CheckmarkBold & \XSolidBrush & \XSolidBrush & 169K & \faIcon{file-alt} & Human & Pairwise preference \\
\href{https://huggingface.co/datasets/stanfordnlp/SHP-2}{stanfordnlp/SHP-2}
& \CheckmarkBold & \XSolidBrush & \XSolidBrush & 4.8M & \faIcon{file-alt} & Human / crowd & Pairwise preference \\
\href{https://huggingface.co/datasets/HuggingFaceH4/stack-exchange-preferences}{H4/stack-exchange-preferences}
& \CheckmarkBold & \XSolidBrush & \XSolidBrush & 10.7M & \faIcon{file-alt} & Crowd score & Score / ranking \\
\href{https://huggingface.co/datasets/openai/summarize_from_feedback}{Summarize from Feedback}
& \CheckmarkBold & \XSolidBrush & \XSolidBrush & 64.8K & \faIcon{file-alt} & Human & Pairwise preference \\
\href{https://huggingface.co/datasets/openbmb/UltraFeedback}{UltraFeedback}
& \CheckmarkBold & \XSolidBrush & \XSolidBrush & 64K / 340K & \faIcon{file-alt} & GPT-4 & Scores, critiques, pairs \\
\href{https://huggingface.co/datasets/berkeley-nest/Nectar}{berkeley-nest/Nectar}
& \CheckmarkBold & \XSolidBrush & \XSolidBrush & 183K / 3.8M & \faIcon{file-alt} & GPT-4 & 7-way ranking \\
\href{https://huggingface.co/datasets/Skywork/Skywork-Reward-Preference-80K-v0.2}{Skywork-Reward-Preference-80K}
& \CheckmarkBold & \XSolidBrush & \XSolidBrush & 80K & \faIcon{file-alt} & Mixed / curated & Pairwise preference \\
\href{https://huggingface.co/datasets/nvidia/HelpSteer}{nvidia/HelpSteer}
& \CheckmarkBold & \XSolidBrush & \XSolidBrush & 37K & \faIcon{file-alt} & Human & Attribute scores \\
\href{https://huggingface.co/datasets/nvidia/HelpSteer2}{nvidia/HelpSteer2}
& \CheckmarkBold & \XSolidBrush & \XSolidBrush & 10K pairs & \faIcon{file-alt} & Human & Attribute scores / preference \\
\href{https://huggingface.co/datasets/nvidia/HelpSteer3}{nvidia/HelpSteer3}
& \CheckmarkBold & \XSolidBrush & \XSolidBrush & 40K pref. & \faIcon{file-alt} & Human & Preference, feedback, edits \\
\href{https://huggingface.co/datasets/lmarena-ai/arena-human-preference-55k}{Arena Human Preference 55K}
& \CheckmarkBold & \XSolidBrush & \XSolidBrush & 55K & \faIcon{file-alt} & Human users & Arena battle preference \\
\href{https://huggingface.co/datasets/MMInstruction/VLFeedback}{VLFeedback}
& \CheckmarkBold & \XSolidBrush & \XSolidBrush & 80K / 380K & \faIcon{file-alt} \faIcon{images} & GPT-4V & Multimodal preference \\
\href{https://github.com/RLHF-V/RLHF-V}{RLHF-V-Dataset}
& \CheckmarkBold & \XSolidBrush & \XSolidBrush & 5.7K & \faIcon{file-alt} \faIcon{images} & Human & Fine-grained correction \\
\href{https://huggingface.co/datasets/openbmb/RLAIF-V-Dataset}{openbmb/RLAIF-V-Dataset}
& \CheckmarkBold & \XSolidBrush & \XSolidBrush & 83K & \faIcon{file-alt} \faIcon{images} & AI feedback & Pairwise preference \\
\href{https://huggingface.co/datasets/OpenGVLab/MMPR-v1.2}{OpenGVLab/MMPR-v1.2}
& \CheckmarkBold & \CheckmarkBold & \XSolidBrush & 3M & \faIcon{file-alt} \faIcon{images} & AI / verifier & Multimodal reasoning preference \\
\href{https://huggingface.co/datasets/open-r1/OpenR1-Math-220k}{open-r1/OpenR1-Math-220k}
& \XSolidBrush & \CheckmarkBold & \XSolidBrush & 220K & \faIcon{file-alt} & Verifier / judge & Math reasoning traces \\
\href{https://huggingface.co/collections/AI-MO/numinamath}{AI-MO/NuminaMath}
& \XSolidBrush & \CheckmarkBold & \XSolidBrush & 900K & \faIcon{file-alt} & Rule / GPT-4 & Math CoT / TIR \\
\href{https://huggingface.co/datasets/PRIME-RL/Eurus-2-RL-Data}{PRIME-RL/Eurus-2-RL-Data}
& \XSolidBrush & \CheckmarkBold & \XSolidBrush & 482K & \faIcon{file-alt} & Verifier & Math/code outcome reward \\
\href{https://huggingface.co/datasets/AgentGym/AgentGym-RL-Data-ID}{AgentGym-RL-Data}
& \XSolidBrush & \CheckmarkBold & \CheckmarkBold & 184K & \faIcon{file-alt} & Env. & Multi-turn agent reward \\
\href{https://github.com/alfworld/alfworld}{ALFWorld}
& \XSolidBrush & \XSolidBrush & \CheckmarkBold & 3.5K train & \faIcon{file-alt} & Env. & Text-world task reward \\
\href{https://github.com/princeton-nlp/WebShop}{WebShop}
& \XSolidBrush & \XSolidBrush & \CheckmarkBold & 12K tasks & \faIcon{file-alt} & Env. & Web shopping reward \\
\href{https://github.com/PeterGriffinJin/Search-R1}{Search-R1 Data}
& \XSolidBrush & \CheckmarkBold & \CheckmarkBold & 170K train & \faIcon{file-alt} & Retriever / rule & Search-tool QA reward \\
\href{https://github.com/langfengQ/verl-agent}{Sokoban}
& \XSolidBrush & \CheckmarkBold & \CheckmarkBold & Env. gen. & \faIcon{file-alt} \faIcon{images} & Env. & Puzzle-solving reward \\
\href{https://github.com/langfengQ/verl-agent}{Gym Cards}
& \XSolidBrush & \CheckmarkBold & \CheckmarkBold & Env. gen. & \faIcon{file-alt} \faIcon{images} & Env. & Logic-game reward \\
\href{https://github.com/OpenBMB/ToolBench}{ToolBench}
& \XSolidBrush & \CheckmarkBold & \CheckmarkBold & 126K & \faIcon{file-alt} & API execution & Tool-use trajectories \\
\bottomrule[1.1pt]
\end{tabular}
