Update section2 notes and figures

6a4cb225 · PolarisZZM · b46553db · 6a4cb225 · 6a4cb225 · 6a4cb225
Commit 6a4cb225 authored Sep 09, 2026 by PolarisZZM
--- a/section2/aml_notes.tex
+++ b/section2/aml_notes.tex
--- a/section2/figures/actor_critic_framework.tex
+++ b/section2/figures/actor_critic_framework.tex
-\begin{tikzpicture}[
+\begin{tikzpicture}[font=\small, >=Latex]
-    font=\small,
+\tikzset{
  box/.style={
-        draw=black,
+    draw=black!75,
+    line width=0.8pt,
    rounded corners=4pt,
-        minimum width=2.8cm,
+    align=center
-        minimum height=1.15cm,
-        align=center,
-        line width=0.8pt
  },
  statebox/.style={
    box,
    fill=yellow!12,
-        minimum width=2.6cm
+    minimum width=2.7cm,
+    minimum height=1.15cm
  },
  actorbox/.style={
    box,
-        fill=blue!10,
+    fill=blue!9,
-        minimum width=3.5cm,
+    minimum width=3.4cm,
    minimum height=1.55cm
  },
  criticbox/.style={
    box,
-        fill=green!12,
+    fill=green!10,
-        minimum width=3.5cm,
+    minimum width=3.4cm,
    minimum height=1.55cm
  },
  envbox/.style={
    box,
-        fill=orange!12,
+    fill=orange!10,
-        minimum width=3.0cm,
+    minimum width=3.1cm,
-        minimum height=1.2cm
+    minimum height=1.15cm
  },
  tdbox/.style={
    box,
-        fill=gray!10,
+    fill=gray!8,
-        minimum width=6.8cm,
+    minimum width=6.2cm,
    minimum height=1.35cm
  },
-    note/.style={
+  rolebox/.style={
-        draw=black,
+    box,
-        rounded corners=4pt,
+    draw=black!55,
-        fill=gray!5,
+    fill=gray!3,
-        minimum width=5.0cm,
+    minimum width=4.6cm,
-        minimum height=0.95cm,
+    minimum height=0.85cm
-        align=center,
-        line width=0.8pt
  },
  flow/.style={
-        draw=black,
+    -{Latex[length=2.2mm,width=1.5mm]},
-        -Latex,
+    draw=black!80,
-        line width=0.8pt
+    line width=0.9pt
  },
-    dashedflow/.style={
+  update/.style={
-        draw=black!75,
+    -{Latex[length=2.2mm,width=1.5mm]},
-        -Latex,
+    draw=black!60,
-        dashed,
+    line width=0.8pt,
-        line width=0.75pt
+    dashed
+  },
+  edgelabel/.style={
+    fill=white,
+    inner sep=1.5pt
  }
-]
+}
 % =========================
 % Nodes
 % =========================
-\node[statebox] (state) at (0,1.8) {当前棋盘状态\\$s_t$};
-\node[actorbox] (actor) at (4.2,3.1) {Actor\\策略网络\\$\pi_{\theta}(a_t\mid s_t)$};
+\node[statebox] (state) at (-6.0,1.25)
+{
-\node[criticbox] (critic) at (10.8,3.1) {Critic\\价值网络\\$V_{\phi}(s)$};
+  当前棋盘状态\\[-1mm]
+  $s_t$
-\node[envbox] (env) at (7.5,0.9) {五子棋环境};
+};
-\node[tdbox] (td) at (7.5,-2.0) {时序差分误差\\[0.08cm]
+\node[actorbox] (actor) at (-2.0,1.75)
-$\delta_t=r_t+\gamma V_{\phi}(s_{t+1})-V_{\phi}(s_t)$};
+{
+  \textbf{Actor}\\
-\node[note] (note1) at (4.2,5.1) {Actor 负责根据状态选择动作};
+  策略网络\\[-1mm]
-\node[note] (note2) at (10.8,5.1) {Critic 负责估计状态价值并评价动作};
+  $\pi_{\theta}(a_t\mid s_t)$
+};
+\node[criticbox] (critic) at (3.5,1.75)
+{
+  \textbf{Critic}\\
+  价值网络\\[-1mm]
+  $V_{\phi}(s)$
+};
+\node[envbox] (env) at (0.75,-0.45)
+{
+  五子棋环境
+};
+\node[tdbox] (td) at (0.75,-3.0)
+{
+  \textbf{时序差分误差}\\[1mm]
+  $\displaystyle
+  \delta_t
+  =
+  r_t+\gamma V_{\phi}(s_{t+1})-V_{\phi}(s_t)
+  $
+};
+\node[rolebox] (actorrole) at (-2.0,4.15)
+{
+  Actor：根据当前状态选择动作
+};
+\node[rolebox] (criticrole) at (3.5,4.15)
+{
+  Critic：估计状态价值并评价当前策略
+};
 % =========================
-% Arrows: state input
+% Forward flow
 % =========================
-\draw[flow] (state.east) -- (actor.west);
-\node at (2.0,2.15) {$s_t$};
 \draw[flow]
  (state.east)
-    .. controls (2.2,1.2) and (7.9,1.3)
+  --
-    .. (critic.west);
+  (actor.west)
-\node at (5.4,1.25) {$s_t$};
+  node[midway, above, edgelabel] {$s_t$};
-% =========================
+\draw[flow]
-% Action and environment
+  (state.north)
-% =========================
+  -- ++(0,1.55)
-\draw[flow] (actor.south east) -- (env.north west);
+  -| (critic.north)
-\node at (5.55,2.05) {动作 $a_t$};
+  node[pos=0.78, above, edgelabel] {$s_t$};
-\draw[flow] (env.north east) -- (critic.south west);
+\draw[flow]
-\node[align=center] at (9.45,2.0) {奖励 $r_t$\\下一状态 $s_{t+1}$};
+  (actor.south east)
+  --
+  (env.north west)
+  node[midway, above left, edgelabel]
+  {动作 $a_t$};
-% =========================
+\draw[flow]
-% TD error construction
+  (env.north east)
-% =========================
+  --
-\draw[flow] (env.south) -- (td.north);
+  (critic.south west)
-\node at (7.0,-0.45) {$r_t$};
+  node[midway, below right, edgelabel]
+  {下一状态 $s_{t+1}$};
-\draw[flow] (critic.south) -- (td.north east);
+\draw[flow]
-\node[align=center] at (10.9,-0.2) {$V_{\phi}(s_t)$\\$V_{\phi}(s_{t+1})$};
+  (env.south)
+  --
+  (td.north)
+  node[midway, right, edgelabel]
+  {奖励 $r_t$};
-% =========================
+\draw[flow]
-% Updates from TD error
+  (critic.south east)
-% =========================
+  to[out=-65,in=20]
-\draw[dashedflow]
+  (td.east)
-    (td.north west)
+  node[pos=0.52, right, edgelabel, align=left]
-    .. controls (5.4,-0.8) and (4.0,0.9)
+  {$V_{\phi}(s_t)$\\
-    .. (actor.south);
+   $V_{\phi}(s_{t+1})$};
-\node[align=center] at (4.3,-0.6) {策略更新\\Actor};
-\draw[dashedflow]
-    (td.north east)
-    .. controls (9.8,-0.8) and (11.0,0.9)
-    .. (critic.south);
-\node[align=center] at (10.8,-0.7) {价值更新\\Critic};
 % =========================
-% Optional environment loop
+% Parameter update
 % =========================
-\draw[dashedflow]
-    (env.west)
+\draw[update]
-    .. controls (5.2,0.1) and (1.8,0.1)
+  (td.west)
-    .. (state.south);
+  to[out=175,in=-75]
-\node[align=center] at (3.2,0.35) {\footnotesize 下一状态用于下一轮决策};
+  (actor.south)
+  node[pos=0.52, left, edgelabel, align=center]
+  {策略更新\\Actor};
+\draw[update]
+  (td.east)
+  to[out=0,in=-80]
+  (critic.south east)
+  node[pos=0.50, right, edgelabel, align=center]
+  {价值更新\\Critic};
 \end{tikzpicture}
\ No newline at end of file
--- a/section2/figures/gobang_policy.tex
+++ b/section2/figures/gobang_policy.tex
@@ -116,17 +116,17 @@
 \node[cand] at (4,9) {D};
 % legend
-\node[blackstone] at (1.2,5.1) {};
+% \node[blackstone] at (1.2,4.6) {};
-\node[anchor=west] at (1.45,5.1) {\footnotesize 黑棋};
+% \node[anchor=west] at (1.45,4.6) {\footnotesize 黑棋};
-\node[whitestone] at (2.4,5.1) {};
+% \node[whitestone] at (2.4,4.6) {};
-\node[anchor=west] at (2.65,5.1) {\footnotesize 白棋};
+% \node[anchor=west] at (2.65,4.6) {\footnotesize 白棋};
-\node[cand, minimum size=5.8mm] at (3.7,5.1) {};
+% \node[cand, minimum size=5.8mm] at (3.7,4.6) {};
-\node[anchor=west] at (3.95,5.1) {\footnotesize 候选位置};
+% \node[anchor=west] at (3.95,4.6) {\footnotesize 候选位置};
-\node[bestcand, minimum size=6.2mm] at (5.05,5.1) {};
+% \node[bestcand, minimum size=6.2mm] at (5.05,4.6) {};
-\node[anchor=west] at (5.3,5.1) {\footnotesize 采样动作};
+% \node[anchor=west] at (5.3,4.6) {\footnotesize 采样动作};
 % =========================
 % Middle panel: policy model
@@ -149,17 +149,17 @@
 \node[titlebox, minimum width=5.0cm] at (15.4,11.55) {落子概率分布};
 \draw[rowbox, fill=white] (12.8,10.1) rectangle (18.0,10.7);
-\node[anchor=west] at (13.05,10.4) {$A:(7,8)\qquad \pi_{\theta}(A\mid s_t)=0.15$};
+\node[anchor=west] at (12.8,10.4) {$A:(7,8)\qquad \pi_{\theta}(A\mid s_t)=0.15$};
 \draw[rowbox, fill=green!15] (12.8,9.15) rectangle (18.0,9.75);
-\node[anchor=west] at (13.05,9.45) {$B:(8,8)\qquad \pi_{\theta}(B\mid s_t)=0.40$};
+\node[anchor=west] at (12.8,9.45) {$B:(8,8)\qquad \pi_{\theta}(B\mid s_t)=0.40$};
-\node[anchor=west, text=green!40!black] at (16.95,9.45) {\footnotesize 概率最高};
+% \node[anchor=west, text=green!40!black] at (16.95,9.45) {\footnotesize 概率最高};
 \draw[rowbox, fill=white] (12.8,8.2) rectangle (18.0,8.8);
-\node[anchor=west] at (13.05,8.5) {$C:(8,9)\qquad \pi_{\theta}(C\mid s_t)=0.25$};
+\node[anchor=west] at (12.8,8.5) {$C:(8,9)\qquad \pi_{\theta}(C\mid s_t)=0.25$};
 \draw[rowbox, fill=white] (12.8,7.25) rectangle (18.0,7.85);
-\node[anchor=west] at (13.05,7.55) {$D:(9,9)\qquad \pi_{\theta}(D\mid s_t)=0.20$};
+\node[anchor=west] at (12.8,7.55) {$D:(9,9)\qquad \pi_{\theta}(D\mid s_t)=0.20$};
 % bottom formula / note
 \draw[infobox] (12.8,5.85) rectangle (18.0,6.85);
@@ -167,11 +167,11 @@
 \node[align=center, text=green!40!black] at (15.4,6.1) {\footnotesize 按概率分布采样动作};
 % sampled action arrow from probability row B back to board B
-\draw[dashedflow]
+% \draw[dashedflow]
-    (12.8,9.45)
+%     (12.8,9.45)
-    .. controls (10.8,10.9) and (6.7,10.8)
+%     .. controls (10.8,10.9) and (6.7,10.8)
-    .. (3,8.35);
+%     .. (3,8.35);
-\node[text=green!40!black] at (8.0,10.45) {\footnotesize 本次采样选择位置 $B=(8,8)$};
+% \node[text=green!40!black] at (8.0,10.45) {\footnotesize 本次采样选择位置 $B=(8,8)$};
 \end{tikzpicture}
\ No newline at end of file
--- a/section2/figures/policy_advantage_update.tex
+++ b/section2/figures/policy_advantage_update.tex
@@ -89,7 +89,7 @@
 % 左侧：当前状态、策略与动作采样
 % ==================================================
-\node[inputbox] (input) at (2.5,4.75) {
+\node[inputbox] (input) at (2.2,4.75) {
    {\bfseries 当前策略与动作}\\[6pt]
    {\bfseries 当前棋盘状态 $s_t$}\\[4pt]
@@ -133,7 +133,7 @@
 % 中间：负优势分支
 % ==================================================
-\node[negativebox] (negative) at (9.0,3.10) {
+\node[negativebox] (negative) at (9.0,2.10) {
    {\bfseries 负优势（$-$）：回报低于基线}\\[6pt]
    实际回报：$G_t=3$\\[3pt]
@@ -173,7 +173,7 @@
 % 右侧：负优势对应的策略更新结果
 % ==================================================
-\node[negativeresult] (negative-result) at (15.6,3.10) {
+\node[negativeresult] (negative-result) at (15.6,2.10) {
    {\bfseries 动作概率降低（$\downarrow$）}\\[7pt]
    更新前：\\[-1pt]
@@ -250,12 +250,12 @@
 % 底部：统一的基线与策略更新原则
 % ==================================================
-\node[principlebox] (principle) at (9.0,0.45) {
+% \node[principlebox] (principle) at (9.0,-0.7) {
-    {\bfseries 基线与策略更新原则}\\[5pt]
+%     {\bfseries 基线与策略更新原则}\\[5pt]
-    常用状态基线为
+%     常用状态基线为
-    $\displaystyle b(s_t)=V^{\pi}(s_t)$。\\[3pt]
+%     $\displaystyle b(s_t)=V^{\pi}(s_t)$。\\[3pt]
-    策略更新关注的不是回报的绝对大小，
+%     策略更新关注的不是回报的绝对大小，
-    而是动作结果相对于当前平均水平的好坏。
+%     而是动作结果相对于当前平均水平的好坏。
-};
+% };
 \end{tikzpicture}
\ No newline at end of file
--- a/section2/figures/ppo_clipping_mechanism.tex
+++ b/section2/figures/ppo_clipping_mechanism.tex
-\begin{tikzpicture}[
+\begin{tikzpicture}[font=\small, >=Latex]
-    font=\small,
+\tikzset{
  flow/.style={
-        draw=black,
+    -{Latex[length=2.2mm,width=1.5mm]},
-        -Latex,
-        line width=0.8pt
-    },
-    dashedflow/.style={
    draw=black!80,
-        -Latex,
+    line width=0.85pt
-        dashed,
-        line width=0.75pt
  },
  panel/.style={
-        draw=black,
+    draw=black!75,
    rounded corners=4pt,
    line width=0.8pt
  },
  titlebox/.style={
-        draw=black,
+    draw=black!70,
-        rounded corners=4pt,
+    rounded corners=3pt,
    fill=blue!5,
-        minimum height=0.72cm,
+    minimum height=0.78cm,
+    inner xsep=8pt,
    align=center
  },
  infobox/.style={
-        draw=black,
+    draw=black!70,
-        rounded corners=4pt,
+    rounded corners=3pt,
-        fill=gray!6,
+    fill=gray!8,
-        minimum width=4.2cm,
+    text width=3.45cm,
    minimum height=1.05cm,
+    inner sep=5pt,
    align=center,
-        line width=0.8pt
+    line width=0.75pt
  },
-    rowbox/.style={
+  casebox/.style={
-        draw=black,
+    draw=black!70,
-        rounded corners=4pt,
+    rounded corners=3pt,
-        minimum width=5.2cm,
+    fill=white,
-        minimum height=1.95cm,
+    text width=4.65cm,
+    minimum height=2.35cm,
+    inner sep=6pt,
    align=left,
-        line width=0.8pt
+    line width=0.75pt
  },
-    pospanel/.style={
+  goodcase/.style={
-        draw=black,
+    casebox,
-        rounded corners=4pt,
+    fill=green!10
-        fill=green!6,
+  },
-        line width=0.8pt
+  badcase/.style={
+    casebox,
+    fill=red!8
  },
-    negpanel/.style={
+  note/.style={
-        draw=black,
+    draw=black!70,
    rounded corners=4pt,
-        fill=red!5,
+    fill=gray!8,
+    text width=10.9cm,
+    minimum height=1.0cm,
+    inner sep=6pt,
+    align=center,
    line width=0.8pt
  }
-]
+}
-% =========================
+% ==========================================================
 % Left panel
-% =========================
+% ==========================================================
-\draw[panel, fill=orange!8] (0.2,3.8) rectangle (5.5,10.4);
-\node[titlebox, minimum width=4.2cm] at (2.85,10.9) {旧策略与裁剪区间};
+\draw[panel, fill=orange!10]
+  (0,3.25) rectangle (4.45,10.55);
-\node[infobox] at (2.85,9.55) {
-    旧策略\\[0.06cm]
+% \node[
+%   titlebox,
+%   minimum width=3.3cm
+% ] at (2.225,10.95)
+% {
+%   旧策略与裁剪区间
+% };
+\node[infobox] at (2.225,9.45)
+{
+  旧策略\\[0.5mm]
  $\pi_{\theta_{\mathrm{old}}}(B\mid s_t)=0.40$
 };
-\node[infobox] at (2.85,7.8) {
+\node[infobox] at (2.225,8.02)
-    裁剪阈值\\[0.06cm]
+{
+  裁剪阈值\\[0.5mm]
  $\epsilon=0.2$
 };
-\node[infobox] at (2.85,6.05) {
+\node[infobox] at (2.225,6.55)
-    概率比值允许范围\\[0.06cm]
+{
+  概率比值允许范围\\[0.5mm]
  $[1-\epsilon,1+\epsilon]=[0.8,1.2]$
 };
-\node[infobox, minimum width=4.4cm, minimum height=1.35cm] at (2.85,4.5) {
+\node[
-    裁剪限制的是\\[0.05cm]
+  infobox,
-    过大概率变化带来的\\[0.05cm]
+  text width=3.65cm,
-    目标函数收益
+  minimum height=1.6cm
+] at (2.225,4.78)
+{
+  裁剪限制的是\\[0.5mm]
+  超出该范围后的\\[0.5mm]
+  目标函数额外收益
 };
-% arrows to two cases
+% ==========================================================
-\draw[flow] (5.65,8.6) -- (6.8,8.9);
+% Arrows from left panel
-\draw[flow] (5.65,5.6) -- (6.8,5.3);
+% ==========================================================
-% =========================
+\draw[flow]
-% Top right panel: positive advantage
+  (4.45,8.55)
-% =========================
+  --
-\draw[pospanel] (6.9,7.25) rectangle (18.1,10.4);
+  (5.55,8.55);
-\node[titlebox, minimum width=4.8cm] at (12.5,10.9) {正优势动作 $\widehat{A}_t>0$};
+\draw[flow]
-\draw[rowbox, fill=white] (7.3,8.05) rectangle (12.2,9.8);
+  (4.45,5.25)
-\node[anchor=west, align=left] at (7.55,9.45) {
+  --
-新策略概率：$0.48$\\
+  (5.55,5.25);
-概率比值：$r_t(\theta)=0.48/0.40=1.20$\\
-位于上界\\
+% ==========================================================
-允许提高动作概率
+% Positive advantage panel
+% ==========================================================
+\draw[panel, fill=green!8]
+  (5.7,7.15) rectangle (17.0,10.55);
+% \node[
+%   titlebox,
+%   minimum width=3.8cm
+% ] at (11.35,10.95)
+% {
+%   正优势动作 $\widehat{A}_t>0$
+% };
+\node[casebox] (posleft) at (8.75,8.88)
+{
+  \textbf{新策略概率：}
+  $0.48$\\[0.8mm]
+  \textbf{概率比值：}\\
+  $r_t(\theta)=0.48/0.40=1.20$\\[0.8mm]
+  \textbf{结论：}\\
+  位于上界，允许提高动作概率
 };
-\draw[rowbox, fill=green!12] (12.8,8.05) rectangle (17.7,9.8);
+\node[goodcase] (posright) at (14.05,8.88)
-\node[anchor=west, align=left] at (13.05,9.45) {
+{
-新策略概率：$0.60$\\
+  \textbf{新策略概率：}
-概率比值：$r_t(\theta)=0.60/0.40=1.50$\\
+  $0.60$\\[0.8mm]
-超出上界 $1.20$\\
-裁剪后按 $1.20$ 计算收益
+  \textbf{概率比值：}\\
-};
+  $r_t(\theta)=0.60/0.40=1.50$\\[0.8mm]
-\node[text=green!40!black] at (9.75,7.55) {\footnotesize 适度提高概率};
+  \textbf{结论：}\\
-\node[text=green!40!black] at (15.25,7.55) {\footnotesize 过大变化不再获得额外鼓励};
+  超出上界 $1.20$，裁剪后按 $1.20$ 计算收益
-% =========================
-% Bottom right panel: negative advantage
-% =========================
-\draw[negpanel] (6.9,3.8) rectangle (18.1,6.95);
-\node[titlebox, minimum width=4.8cm] at (12.5,7.45) {负优势动作 $\widehat{A}_t<0$};
-\draw[rowbox, fill=white] (7.3,4.6) rectangle (12.2,6.35);
-\node[anchor=west, align=left] at (7.55,6.0) {
-新策略概率：$0.32$\\
-概率比值：$r_t(\theta)=0.32/0.40=0.80$\\
-位于下界\\
-允许降低动作概率
 };
-\draw[rowbox, fill=red!10] (12.8,4.6) rectangle (17.7,6.35);
+% \node[
-\node[anchor=west, align=left] at (13.05,6.0) {
+%   text=green!45!black,
-新策略概率：$0.20$\\
+%   font=\footnotesize
-概率比值：$r_t(\theta)=0.20/0.40=0.50$\\
+% ] at (8.75,7.52)
-低于下界 $0.80$\\
+% {
-裁剪后按 $0.80$ 计算收益
+%   适度提高概率
+% };
+% \node[
+%   text=green!45!black,
+%   font=\footnotesize
+% ] at (14.05,7.52)
+% {
+%   过大变化不再获得额外鼓励
+% };
+% ==========================================================
+% Negative advantage panel
+% ==========================================================
+\draw[panel, fill=red!6]
+  (5.7,3.25) rectangle (17.0,6.65);
+% \node[
+%   titlebox,
+%   minimum width=3.8cm
+% ] at (11.35,7.05)
+% {
+%   负优势动作 $\widehat{A}_t<0$
+% };
+\node[casebox] (negleft) at (8.75,4.98)
+{
+  \textbf{新策略概率：}
+  $0.32$\\[0.8mm]
+  \textbf{概率比值：}\\
+  $r_t(\theta)=0.32/0.40=0.80$\\[0.8mm]
+  \textbf{结论：}\\
+  位于下界，允许降低动作概率
 };
-\node[text=red!60!black] at (9.75,4.1) {\footnotesize 适度降低概率};
+\node[badcase] (negright) at (14.05,4.98)
-\node[text=red!60!black] at (15.25,4.1) {\footnotesize 过大变化不再获得额外鼓励};
+{
+  \textbf{新策略概率：}
+  $0.20$\\[0.8mm]
-% =========================
+  \textbf{概率比值：}\\
-% Bottom note
+  $r_t(\theta)=0.20/0.40=0.50$\\[0.8mm]
-% =========================
-\draw[panel, fill=gray!6] (6.9,2.2) rectangle (18.1,3.3);
+  \textbf{结论：}\\
-\node[align=center] at (12.5,2.75) {
+  低于下界 $0.80$，裁剪后按 $0.80$ 计算收益
-PPO 并不直接禁止新策略继续变化\\
-而是限制超出裁剪范围后的额外优化收益
 };
+% \node[
+%   text=red!65!black,
+%   font=\footnotesize
+% ] at (8.75,3.62)
+% {
+%   适度降低概率
+% };
+% \node[
+%   text=red!65!black,
+%   font=\footnotesize
+% ] at (14.05,3.62)
+% {
+%   过大变化不再获得额外鼓励
+% };
+% ==========================================================
+% Bottom note
+% ==========================================================
+% \node[note] at (11.35,1.75)
+% {
+%   PPO 并不直接禁止新策略继续变化，\\
+%   而是限制超出裁剪区间之后所能获得的额外优化收益
+% };
 \end{tikzpicture}
\ No newline at end of file
--- a/section2/figures/ppo_training_process.tex
+++ b/section2/figures/ppo_training_process.tex
@@ -141,11 +141,11 @@
 \draw[flow] (s1.east) -- (s2.west);
 \draw[flow] (s2.east) -- (s3.west);
-\draw[flow] (s3.south west) .. controls (14.0,8.5) and (4.8,8.2) .. (s4.north east);
+\draw[flow] (s3.south west) -- (s4.north east);
 \draw[flow] (s4.east) -- (s5.west);
 \draw[flow] (s5.east) -- (s6.west);
-\draw[flow] (s6.south west) .. controls (14.0,5.6) and (11.7,5.0) .. (s7.north east);
+\draw[flow] (s6.south west) -- (s7.north east);
 \draw[flow] (s7.south) -- (s8.north);
 % =========================
@@ -153,25 +153,24 @@
 % =========================
 \draw[dashedflow]
    (s7.west)
-    .. controls (4.8,4.4) and (4.8,6.0)
+    -- node[midway, above,xshift=5mm] {\footnotesize 重复 $K$ 轮} (s4.south);
-    .. node[midway, above] {\footnotesize 重复 $K$ 轮} (s4.south);
 % =========================
 % Comparison note
 % =========================
-\node[comparebox] at (3.5,2.1) {
+% \node[comparebox] at (3.5,2.1) {
-\textbf{与 A2C 的区别：}\\
+% \textbf{与 A2C 的区别：}\\
-A2C：采集一批数据后通常只更新一次\\
+% A2C：采集一批数据后通常只更新一次\\
-PPO：采集一批数据后可划分小批次并重复更新多轮
+% PPO：采集一批数据后可划分小批次并重复更新多轮
-};
+% };
 % =========================
 % Small note
 % =========================
-\node[comparebox, minimum width=5.0cm, minimum height=1.6cm] at (15.1,2.1) {
+% \node[comparebox, minimum width=5.0cm, minimum height=1.6cm] at (15.1,2.1) {
-\textbf{核心原因：}\\
+% \textbf{核心原因：}\\
-裁剪目标限制单次策略更新幅度\\
+% 裁剪目标限制单次策略更新幅度\\
-因此同一批样本可以在一定范围内重复利用
+% 因此同一批样本可以在一定范围内重复利用
-};
+% };
 \end{tikzpicture}
\ No newline at end of file