modify the shell scripts (add the lower and wmt pipeline)

egs/librispeech/asr/conf/train.yaml

@@ -33,10 +33,4 @@ encoder-embed-dim: 256
 encoder-ffn-embed-dim: 2048
 encoder-layers: 12
 decoder-layers: 6
-encoder-attention-heads: 4
-
-#decoder-embed-dim: 256
-#decoder-ffn-embed-dim: 2048
-#decoder-attention-heads: 4
-#attention-dropout: 0.1
-#activation-dropout: 0.1
+encoder-attention-heads: 4
\ No newline at end of file

egs/librispeech/asr/conf/train_conformer_l.yaml

+train-subset: train-clean-100,train-clean-360,train-other-500
+valid-subset: dev-clean
+
+max-epoch: 100
+max-update: 300000
+
+num-workers: 8
+patience: 10
+no-progress-bar: True
+log-interval: 100
+seed: 1
+report-accuracy: True
+
+arch: s2t_conformer_l
+share-decoder-input-output-embed: True
+optimizer: adam
+clip-norm: 10.0
+lr-scheduler: inverse_sqrt
+warmup-init-lr: 1e-7
+warmup-updates: 10000
+lr: 2e-3
+#adam_betas: (0.9,0.98)
+
+criterion: label_smoothed_cross_entropy
+label_smoothing: 0.1
+
+conv-kernel-sizes: 5,5
+conv-channels: 1024
+
+macaron-style: True
+use-cnn-module: True
+cnn-module-kernel: 31
+
+#dropout: 0.1
+#activation-fn: relu
+#encoder-embed-dim: 256
+#encoder-ffn-embed-dim: 2048
+#encoder-layers: 12
+#decoder-layers: 6
+#encoder-attention-heads: 4
+
+#decoder-embed-dim: 256
+#decoder-ffn-embed-dim: 2048
+#decoder-attention-heads: 4
+#attention-dropout: 0.1
+#activation-dropout: 0.1

egs/librispeech/asr/conf/train_conformer_m.yaml

+train-subset: train-clean-100,train-clean-360,train-other-500
+valid-subset: dev-clean
+
+max-epoch: 100
+max-update: 300000
+
+num-workers: 8
+patience: 10
+no-progress-bar: True
+log-interval: 100
+seed: 1
+report-accuracy: True
+
+arch: s2t_conformer_m
+share-decoder-input-output-embed: True
+optimizer: adam
+clip-norm: 10.0
+lr-scheduler: inverse_sqrt
+warmup-init-lr: 1e-7
+warmup-updates: 10000
+lr: 1e-3
+#adam_betas: (0.9,0.98)
+
+criterion: label_smoothed_cross_entropy
+label_smoothing: 0.1
+
+conv-kernel-sizes: 5,5
+conv-channels: 1024
+
+macaron-style: True
+use-cnn-module: True
+cnn-module-kernel: 31
+
+#dropout: 0.1
+#activation-fn: relu
+#encoder-embed-dim: 256
+#encoder-ffn-embed-dim: 2048
+#encoder-layers: 12
+#decoder-layers: 6
+#encoder-attention-heads: 4
+
+#decoder-embed-dim: 256
+#decoder-ffn-embed-dim: 2048
+#decoder-attention-heads: 4
+#attention-dropout: 0.1
+#activation-dropout: 0.1

egs/librispeech/asr/conf/train_ctc_conformer.yaml

@@ -27,18 +27,18 @@ label_smoothing: 0.1
 
 conv-kernel-sizes: 5,5
 conv-channels: 1024
-dropout: 0.1
-activation-fn: relu
-encoder-embed-dim: 256
-encoder-ffn-embed-dim: 2048
-encoder-layers: 12
-decoder-layers: 6
-encoder-attention-heads: 4
-
 macaron-style: True
 use-cnn-module: True
 cnn-module-kernel: 31
 
+#dropout: 0.1
+#activation-fn: relu
+#encoder-embed-dim: 256
+#encoder-ffn-embed-dim: 2048
+#encoder-layers: 12
+#decoder-layers: 6
+#encoder-attention-heads: 4
+
 #decoder-embed-dim: 256
 #decoder-ffn-embed-dim: 2048
 #decoder-attention-heads: 4

egs/librispeech/asr/conf/train_ctc_conformer_m.yaml

+train-subset: train-clean-100,train-clean-360,train-other-500
+valid-subset: dev-clean
+
+max-epoch: 100
+max-update: 300000
+
+num-workers: 8
+patience: 10
+no-progress-bar: True
+log-interval: 100
+seed: 1
+report-accuracy: True
+
+arch: s2t_conformer_m
+share-decoder-input-output-embed: True
+optimizer: adam
+clip-norm: 10.0
+lr-scheduler: inverse_sqrt
+warmup-init-lr: 1e-7
+warmup-updates: 10000
+lr: 1e-3
+#adam_betas: (0.9,0.98)
+
+ctc-weight: 0.3
+criterion: label_smoothed_cross_entropy_with_ctc
+label_smoothing: 0.1
+
+conv-kernel-sizes: 5,5
+conv-channels: 1024
+macaron-style: True
+use-cnn-module: True
+cnn-module-kernel: 31
+
+#dropout: 0.1
+#activation-fn: relu
+#encoder-embed-dim: 256
+#encoder-ffn-embed-dim: 2048
+#encoder-layers: 12
+#decoder-layers: 6
+#encoder-attention-heads: 4
+
+#decoder-embed-dim: 256
+#decoder-ffn-embed-dim: 2048
+#decoder-attention-heads: 4
+#attention-dropout: 0.1
+#activation-dropout: 0.1
\ No newline at end of file

egs/librispeech/asr/decode.sh

@@ -2,7 +2,8 @@
 
 gpu_num=1
 
-test_subset=(test-clean test-other)
+data_dir=
+test_subset=test-cleam,test-other
 
 exp_name=
 if [ "$#" -eq 1 ]; then
@@ -18,11 +19,17 @@ cmd="./run.sh
     --stop_stage 2
     --gpu_num ${gpu_num}
     --exp_name ${exp_name}
-    --test_subset ${test_subset}
     --n_average ${n_average}
     --beam_size ${beam_size}
     --max_tokens ${max_tokens}
     "
 
+if [[ -n ${data_dir} ]]; then
+    cmd="$cmd --data_dir ${data_dir}"
+fi
+if [[ -n ${test_subset} ]]; then
+    cmd="$cmd --test_subset ${test_subset}"
+fi
+
 echo $cmd
 eval $cmd

egs/librispeech/asr/run.sh

@@ -39,7 +39,7 @@ speed_perturb=0
 
 org_data_dir=/media/data/${dataset}
 data_dir=~/st/data/${dataset}
-test_subset=(dev-clean dev-other test-clean test-other)
+test_subset=dev-clean,dev-other,test-clean,test-other
 
 # exp
 extra_tag=
@@ -60,19 +60,12 @@ step_valid=0
 n_average=10
 beam_size=5
 
-. ./local/parse_options.sh || exit 1;
-
-if [[ $step_valid -eq 1 ]]; then
-    validate_interval=10000
-    save_interval=10000
-    no_epoch_checkpoints=1
-    save_interval_updates=5000
-    keep_interval_updates=3
-else
-    validate_interval=1
-    keep_last_epochs=10
+if [[ ${speed_perturb} -eq 1 ]]; then
+    data_dir=${data_dir}_sp
 fi
 
+. ./local/parse_options.sh || exit 1;
+
 # full path
 train_config=$pwd_dir/conf/${train_config}
 if [[ -z ${exp_name} ]]; then
@@ -84,10 +77,6 @@ if [[ -z ${exp_name} ]]; then
         exp_name=sp_${exp_name}
     fi
 fi
-
-if [[ ${speed_perturb} -eq 1 ]]; then
-    data_dir=${data_dir}_sp
-fi
 model_dir=$root_dir/../checkpoints/$dataset/asr/${exp_name}
 
 if [ ${stage} -le -1 ] && [ ${stop_stage} -ge -1 ]; then
@@ -167,6 +156,16 @@ if [ ${stage} -le 1 ] && [ ${stop_stage} -ge 1 ]; then
         cmd="${cmd}
         --fp16"
     fi
+    if [[ $step_valid -eq 1 ]]; then
+        validate_interval=10000
+        save_interval=10000
+        no_epoch_checkpoints=1
+        save_interval_updates=5000
+        keep_interval_updates=3
+    else
+        validate_interval=1
+        keep_last_epochs=10
+    fi
     if [[ -n $no_epoch_checkpoints && $no_epoch_checkpoints -eq 1 ]]; then
         cmd="$cmd
         --no-epoch-checkpoints"
@@ -241,6 +240,7 @@ if [ ${stage} -le 2 ] && [ ${stop_stage} -ge 2 ]; then
 	result_file=${model_dir}/decode_result
 	[[ -f ${result_file} ]] && rm ${result_file}
 
+    test_subset=(${test_subset//,/ })
 	for subset in ${test_subset[@]}; do
         subset=${subset}
   		cmd="python ${root_dir}/fairseq_cli/generate.py

egs/librispeech/asr/train.sh

@@ -12,20 +12,21 @@ extra_parameter=
 #extra_tag="${extra_tag}"
 #extra_parameter="${extra_parameter} "
 
-exp_tag=test
+exp_tag=
 train_config=train_ctc.yaml
 
-
 cmd="./run.sh
     --stage 1
     --stop_stage 1
     --gpu_num ${gpu_num}
     --update_freq ${update_freq}
-    --exp_tag ${exp_tag}
     --train_config ${train_config}
     --max_tokens ${max_tokens}
     "
 
+if [[ -n ${exp_tag} ]]; then
+    cmd="$cmd --exp_tag ${exp_tag}"
+fi
 if [[ -n ${extra_tag} ]]; then
     cmd="$cmd --extra_tag ${extra_tag}"
 fi

egs/lower/asr/conf/train.yaml

+train-subset: train-clean-100,train-clean-360,train-other-500
+#train-subset: train-clean-100
+valid-subset: dev-clean
+
+max-epoch: 100
+max-update: 300000
+
+num-workers: 8
+patience: 10
+no-progress-bar: True
+log-interval: 100
+seed: 1
+report-accuracy: True
+
+arch: s2t_transformer_s
+share-decoder-input-output-embed: True
+optimizer: adam
+clip-norm: 10.0
+lr-scheduler: inverse_sqrt
+warmup-init-lr: 1e-7
+warmup-updates: 10000
+lr: 2e-3
+#adam_betas: (0.9,0.98)
+
+criterion: label_smoothed_cross_entropy
+label_smoothing: 0.1
+
+conv-kernel-sizes: 5,5
+conv-channels: 1024
+dropout: 0.1
+activation-fn: relu
+encoder-embed-dim: 256
+encoder-ffn-embed-dim: 2048
+encoder-layers: 12
+decoder-layers: 6
+encoder-attention-heads: 4
\ No newline at end of file

egs/lower/asr/conf/train_conformer_l.yaml

+train-subset: train-clean-100,train-clean-360,train-other-500
+valid-subset: dev-clean
+
+max-epoch: 100
+max-update: 300000
+
+num-workers: 8
+patience: 10
+no-progress-bar: True
+log-interval: 100
+seed: 1
+report-accuracy: True
+
+arch: s2t_conformer_l
+share-decoder-input-output-embed: True
+optimizer: adam
+clip-norm: 10.0
+lr-scheduler: inverse_sqrt
+warmup-init-lr: 1e-7
+warmup-updates: 10000
+lr: 2e-3
+#adam_betas: (0.9,0.98)
+
+criterion: label_smoothed_cross_entropy
+label_smoothing: 0.1
+
+conv-kernel-sizes: 5,5
+conv-channels: 1024
+
+macaron-style: True
+use-cnn-module: True
+cnn-module-kernel: 31
+
+#dropout: 0.1
+#activation-fn: relu
+#encoder-embed-dim: 256
+#encoder-ffn-embed-dim: 2048
+#encoder-layers: 12
+#decoder-layers: 6
+#encoder-attention-heads: 4
+
+#decoder-embed-dim: 256
+#decoder-ffn-embed-dim: 2048
+#decoder-attention-heads: 4
+#attention-dropout: 0.1
+#activation-dropout: 0.1

egs/lower/asr/conf/train_conformer_m.yaml

+train-subset: train-clean-100,train-clean-360,train-other-500
+valid-subset: dev-clean
+
+max-epoch: 100
+max-update: 300000
+
+num-workers: 8
+patience: 10
+no-progress-bar: True
+log-interval: 100
+seed: 1
+report-accuracy: True
+
+arch: s2t_conformer_m
+share-decoder-input-output-embed: True
+optimizer: adam
+clip-norm: 10.0
+lr-scheduler: inverse_sqrt
+warmup-init-lr: 1e-7
+warmup-updates: 10000
+lr: 1e-3
+#adam_betas: (0.9,0.98)
+
+criterion: label_smoothed_cross_entropy
+label_smoothing: 0.1
+
+conv-kernel-sizes: 5,5
+conv-channels: 1024
+
+macaron-style: True
+use-cnn-module: True
+cnn-module-kernel: 31
+
+#dropout: 0.1
+#activation-fn: relu
+#encoder-embed-dim: 256
+#encoder-ffn-embed-dim: 2048
+#encoder-layers: 12
+#decoder-layers: 6
+#encoder-attention-heads: 4
+
+#decoder-embed-dim: 256
+#decoder-ffn-embed-dim: 2048
+#decoder-attention-heads: 4
+#attention-dropout: 0.1
+#activation-dropout: 0.1

egs/lower/asr/conf/train_ctc.yaml

+train-subset: train-clean-100,train-clean-360,train-other-500
+#train-subset: train-clean-100
+valid-subset: dev-clean
+
+max-epoch: 100
+max-update: 300000
+
+num-workers: 8
+patience: 10
+no-progress-bar: True
+log-interval: 100
+seed: 1
+report-accuracy: True
+
+arch: s2t_transformer_s
+share-decoder-input-output-embed: True
+optimizer: adam
+clip-norm: 10.0
+lr-scheduler: inverse_sqrt
+warmup-init-lr: 1e-7
+warmup-updates: 10000
+lr: 2e-3
+#adam_betas: (0.9,0.98)
+
+ctc-weight: 0.3
+criterion: label_smoothed_cross_entropy_with_ctc
+label_smoothing: 0.1
+
+conv-kernel-sizes: 5,5
+conv-channels: 1024
+dropout: 0.1
+activation-fn: relu
+encoder-embed-dim: 256
+encoder-ffn-embed-dim: 2048
+encoder-layers: 12
+decoder-layers: 6
+encoder-attention-heads: 4
+
+#decoder-embed-dim: 256
+#decoder-ffn-embed-dim: 2048
+#decoder-attention-heads: 4
+#attention-dropout: 0.1
+#activation-dropout: 0.1

egs/lower/asr/conf/train_ctc_conformer.yaml

+train-subset: train-clean-100,train-clean-360,train-other-500
+valid-subset: dev-clean
+
+max-epoch: 100
+max-update: 300000
+
+num-workers: 8
+patience: 10
+no-progress-bar: True
+log-interval: 100
+seed: 1
+report-accuracy: True
+
+arch: s2t_conformer_s
+share-decoder-input-output-embed: True
+optimizer: adam
+clip-norm: 10.0
+lr-scheduler: inverse_sqrt
+warmup-init-lr: 1e-7
+warmup-updates: 10000
+lr: 2e-3
+#adam_betas: (0.9,0.98)
+
+ctc-weight: 0.3
+criterion: label_smoothed_cross_entropy_with_ctc
+label_smoothing: 0.1
+
+conv-kernel-sizes: 5,5
+conv-channels: 1024
+macaron-style: True
+use-cnn-module: True
+cnn-module-kernel: 31
+
+#dropout: 0.1
+#activation-fn: relu
+#encoder-embed-dim: 256
+#encoder-ffn-embed-dim: 2048
+#encoder-layers: 12
+#decoder-layers: 6
+#encoder-attention-heads: 4
+
+#decoder-embed-dim: 256
+#decoder-ffn-embed-dim: 2048
+#decoder-attention-heads: 4
+#attention-dropout: 0.1
+#activation-dropout: 0.1

egs/lower/asr/conf/train_ctc_conformer_m.yaml

+train-subset: train-clean-100,train-clean-360,train-other-500
+valid-subset: dev-clean
+
+max-epoch: 100
+max-update: 300000
+
+num-workers: 8
+patience: 10
+no-progress-bar: True
+log-interval: 100
+seed: 1
+report-accuracy: True
+
+arch: s2t_conformer_m
+share-decoder-input-output-embed: True
+optimizer: adam
+clip-norm: 10.0
+lr-scheduler: inverse_sqrt
+warmup-init-lr: 1e-7
+warmup-updates: 10000
+lr: 1e-3
+#adam_betas: (0.9,0.98)
+
+ctc-weight: 0.3
+criterion: label_smoothed_cross_entropy_with_ctc
+label_smoothing: 0.1
+
+conv-kernel-sizes: 5,5
+conv-channels: 1024
+macaron-style: True
+use-cnn-module: True
+cnn-module-kernel: 31
+
+#dropout: 0.1
+#activation-fn: relu
+#encoder-embed-dim: 256
+#encoder-ffn-embed-dim: 2048
+#encoder-layers: 12
+#decoder-layers: 6
+#encoder-attention-heads: 4
+
+#decoder-embed-dim: 256
+#decoder-ffn-embed-dim: 2048
+#decoder-attention-heads: 4
+#attention-dropout: 0.1
+#activation-dropout: 0.1
\ No newline at end of file

egs/lower/asr/conf/train_ctc_debug.yaml

+#train-subset: train-clean-100,train-clean-360,train-other-500
+train-subset: train-clean-100
+valid-subset: dev-clean
+
+max-epoch: 100
+max-update: 300000
+
+num-workers: 0
+patience: 10
+no-progress-bar: True
+log-interval: 100
+seed: 1
+report-accuracy: True
+
+arch: s2t_transformer_s
+share-decoder-input-output-embed: True
+optimizer: adam
+clip-norm: 10.0
+lr-scheduler: inverse_sqrt
+warmup-init-lr: 1e-7
+warmup-updates: 10000
+lr: 2e-3
+#adam_betas: (0.9,0.98)
+
+ctc-weight: 0.3
+criterion: label_smoothed_cross_entropy_with_ctc
+label_smoothing: 0.1
+
+conv-kernel-sizes: 5,5
+conv-channels: 1024
+dropout: 0.1
+activation-fn: relu
+encoder-embed-dim: 256
+encoder-ffn-embed-dim: 2048
+encoder-layers: 3
+decoder-layers: 3
+encoder-attention-heads: 4
+
+macaron-style: True
+use-cnn-module: True
+cnn-module-kernel: 31
+
+#decoder-embed-dim: 256
+#decoder-ffn-embed-dim: 2048
+#decoder-attention-heads: 4
+#attention-dropout: 0.1
+#activation-dropout: 0.1

egs/lower/asr/decode.sh

+#! /bin/bash
+
+gpu_num=1
+
+data_dir=
+test_subset=test-cleam,test-other
+
+exp_name=
+if [ "$#" -eq 1 ]; then
+    exp_name=$1
+fi
+
+n_average=10
+beam_size=5
+max_tokens=40000
+
+cmd="./run.sh
+    --stage 2
+    --stop_stage 2
+    --gpu_num ${gpu_num}
+    --exp_name ${exp_name}
+    --n_average ${n_average}
+    --beam_size ${beam_size}
+    --max_tokens ${max_tokens}
+    "
+
+if [[ -n ${data_dir} ]]; then
+    cmd="$cmd --data_dir ${data_dir}"
+fi
+if [[ -n ${test_subset} ]]; then
+    cmd="$cmd --test_subset ${test_subset}"
+fi
+
+echo $cmd
+eval $cmd

egs/lower/asr/local/monitor.sh

+gpu_num=1
+
+while :
+do
+    all_devices=$(seq 0 `gpustat | sed '1,2d' | wc -l`);
+    count=0
+    for dev in ${all_devices[@]}
+    do
+        line=`expr $dev + 2`
+        use=`gpustat -p | head -n $line | tail -1 | cut -d '|' -f4 | wc -w`
+        if [[ $use -eq 0 ]]; then
+            device[$count]=$dev
+            count=`expr $count + 1`
+            if [[ $count -eq $gpu_num ]]; then
+                break
+            fi
+        fi
+    done
+    if [[ ${#device[@]} -lt $gpu_num ]]; then
+        sleep 60s
+    else
+        echo "Run $cmd"
+        eval $cmd
+        sleep 10s
+        exit
+    fi
+done

egs/lower/asr/local/parse_options.sh

+#!/usr/bin/env bash
+
+# Copyright 2012  Johns Hopkins University (Author: Daniel Povey);
+#                 Arnab Ghoshal, Karel Vesely
+
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#  http://www.apache.org/licenses/LICENSE-2.0
+#
+# THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED
+# WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE,
+# MERCHANTABLITY OR NON-INFRINGEMENT.
+# See the Apache 2 License for the specific language governing permissions and
+# limitations under the License.
+
+
+# Parse command-line options.
+# To be sourced by another script (as in ". parse_options.sh").
+# Option format is: --option-name arg
+# and shell variable "option_name" gets set to value "arg."
+# The exception is --help, which takes no arguments, but prints the
+# $help_message variable (if defined).
+
+
+###
+### The --config file options have lower priority to command line
+### options, so we need to import them first...
+###
+
+# Now import all the configs specified by command-line, in left-to-right order
+for ((argpos=1; argpos<$#; argpos++)); do
+  if [ "${!argpos}" == "--config" ]; then
+    argpos_plus1=$((argpos+1))
+    config=${!argpos_plus1}
+    [ ! -r $config ] && echo "$0: missing config '$config'" && exit 1
+    . $config  # source the config file.
+  fi
+done
+
+
+###
+### Now we process the command line options
+###
+while true; do
+  [ -z "${1:-}" ] && break;  # break if there are no arguments
+  case "$1" in
+    # If the enclosing script is called with --help option, print the help
+    # message and exit.  Scripts should put help messages in $help_message
+    --help|-h) if [ -z "$help_message" ]; then echo "No help found." 1>&2;
+      else printf "$help_message\n" 1>&2 ; fi;
+      exit 0 ;;
+    --*=*) echo "$0: options to scripts must be of the form --name value, got '$1'"
+      exit 1 ;;
+    # If the first command-line argument begins with "--" (e.g. --foo-bar),
+    # then work out the variable name as $name, which will equal "foo_bar".
+    --*) name=`echo "$1" | sed s/^--// | sed s/-/_/g`;
+      # Next we test whether the variable in question is undefned-- if so it's
+      # an invalid option and we die.  Note: $0 evaluates to the name of the
+      # enclosing script.
+      # The test [ -z ${foo_bar+xxx} ] will return true if the variable foo_bar
+      # is undefined.  We then have to wrap this test inside "eval" because
+      # foo_bar is itself inside a variable ($name).
+      eval '[ -z "${'$name'+xxx}" ]' && echo "$0: invalid option $1" 1>&2 && exit 1;
+
+      oldval="`eval echo \\$$name`";
+      # Work out whether we seem to be expecting a Boolean argument.
+      if [ "$oldval" == "true" ] || [ "$oldval" == "false" ]; then
+        was_bool=true;
+      else
+        was_bool=false;
+      fi
+
+      # Set the variable to the right value-- the escaped quotes make it work if
+      # the option had spaces, like --cmd "queue.pl -sync y"
+      eval $name=\"$2\";
+
+      # Check that Boolean-valued arguments are really Boolean.
+      if $was_bool && [[ "$2" != "true" && "$2" != "false" ]]; then
+        echo "$0: expected \"true\" or \"false\": $1 $2" 1>&2
+        exit 1;
+      fi
+      shift 2;
+      ;;
+  *) break;
+  esac
+done
+
+
+# Check for an empty argument to the --cmd option, which can easily occur as a
+# result of scripting errors.
+[ ! -z "${cmd+xxx}" ] && [ -z "$cmd" ] && echo "$0: empty argument to --cmd option" 1>&2 && exit 1;
+
+
+true; # so this script returns exit code 0.

egs/lower/asr/local/path.sh

+MAIN_ROOT=$PWD/../../..
+KALDI_ROOT=$MAIN_ROOT/tools/kaldi
+
+export PATH=$PWD/utils/:$KALDI_ROOT/tools/openfst/bin:$PATH
+[ ! -f $KALDI_ROOT/tools/config/common_path.sh ] && echo >&2 "The standard file $KALDI_ROOT/tools/config/common_path.sh is not present -> Exit!" && exit 1
+. $KALDI_ROOT/tools/config/common_path.sh
+export LC_ALL=C
+
+export LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:$MAIN_ROOT/src/lib
+export LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:$MAIN_ROOT/tools/chainer_ctc/ext/warp-ctc/build
+. "${MAIN_ROOT}"/tools/activate_python.sh && . "${MAIN_ROOT}"/tools/extra_path.sh
+export PATH=$MAIN_ROOT/utils:$MAIN_ROOT/espnet/bin:$PATH
+
+export OMP_NUM_THREADS=1
+
+# check extra module installation
+if ! which tokenizer.perl > /dev/null; then
+    echo "Error: it seems that moses is not installed." >&2
+    echo "Error: please install moses as follows." >&2
+    echo "Error: cd ${MAIN_ROOT}/tools && make moses.done" >&2
+    return 1
+fi
+
+# NOTE(kan-bayashi): Use UTF-8 in Python to avoid UnicodeDecodeError when LC_ALL=C
+export PYTHONIOENCODING=UTF-8

egs/lower/asr/local/utils.sh

+
+get_devices(){
+    gpu_num=$1
+    use_cpu=$2
+    device=()
+    while :
+    do
+        record=`mktemp -t temp.record.XXXXXX`
+        gpustat > $record
+        all_devices=$(seq 0 `cat $record | sed '1,2d' | wc -l`);
+        count=0
+        for dev in ${all_devices[@]}
+        do
+            line=`expr $dev + 2`
+            use=`cat $record | head -n $line | tail -1 | cut -d '|' -f3 | cut -d '/' -f1`
+            if [[ $use -lt 100 ]]; then
+                device[$count]=$dev
+                count=`expr $count + 1`
+                if [[ $count -eq $gpu_num ]]; then
+                    break
+                fi
+            fi
+        done
+        if [[ ${#device[@]} -lt $gpu_num ]]; then
+            if [[ $use_cpu -eq 1 ]]; then
+                device=(-1)
+            else
+                sleep 60s
+            fi
+        else
+            break
+        fi
+    done
+
+    echo ${device[*]} | sed 's/ /,/g'
+    return $?
+}
+
+

egs/lower/asr/run.sh

+#! /bin/bash
+
+# Processing LibriSpeech Datasets
+
+# Copyright 2021 Natural Language Processing Laboratory 
+# Xu Chen (xuchenneu@163.com)
+
+# Set bash to 'debug' mode, it will exit on :
+# -e 'error', -u 'undefined variable', -o ... 'error in pipeline', -x 'print commands',
+set -e
+#set -u
+set -o pipefail
+export PYTHONIOENCODING=UTF-8
+
+eval=1
+time=$(date "+%m%d_%H%M")
+
+stage=0
+stop_stage=0
+
+######## hardware ########
+# devices
+device=()
+gpu_num=8
+update_freq=1
+
+root_dir=~/st/Fairseq-S2T
+pwd_dir=$PWD
+
+# dataset
+src_lang=swa
+lang=${src_lang}
+
+dataset=lower
+task=speech_to_text
+vocab_type=unigram
+vocab_size=10000
+speed_perturb=0
+
+org_data_dir=/media/data/${dataset}
+data_dir=~/st/data/${dataset}
+test_subset=test
+
+# exp
+extra_tag=
+extra_parameter=
+exp_tag=baseline
+exp_name=
+
+# config
+train_config=train_ctc.yaml
+data_config=config.yaml
+
+# training setting
+fp16=1
+max_tokens=40000
+step_valid=0
+
+# decoding setting
+n_average=10
+beam_size=5
+
+if [[ ${speed_perturb} -eq 1 ]]; then
+    data_dir=${data_dir}_sp
+fi
+
+. ./local/parse_options.sh || exit 1;
+
+# full path
+train_config=$pwd_dir/conf/${train_config}
+if [[ -z ${exp_name} ]]; then
+    exp_name=$(basename ${train_config%.*})_${exp_tag}
+    if [[ -n ${extra_tag} ]]; then
+        exp_name=${exp_name}_${extra_tag}
+    fi
+    if [[ ${speed_perturb} -eq 1 ]]; then
+        exp_name=sp_${exp_name}
+    fi
+fi
+model_dir=$root_dir/../checkpoints/$dataset/asr/${exp_name}
+
+if [ ${stage} -le -1 ] && [ ${stop_stage} -ge -1 ]; then
+    echo "stage -1: Data Download"
+    # pass
+fi
+
+if [ ${stage} -le 0 ] && [ ${stop_stage} -ge 0 ]; then
+    ### Task dependent. You have to make data the following preparation part by yourself.
+    ### But you can utilize Kaldi recipes in most cases
+    echo "stage 0: Data Preparation"
+
+    if [[ ! -e ${data_dir} ]]; then
+        mkdir -p ${data_dir}
+    fi
+    source ~/tools/audio/bin/activate
+
+    cmd="python ${root_dir}/examples/speech_to_text/prep_librispeech_data.py
+        --data-root ${org_data_dir}
+        --output-root ${data_dir}
+        --vocab-type ${vocab_type}
+        --vocab-size ${vocab_size}"
+    if [[ ${speed_perturb} -eq 1 ]]; then
+        cmd="$cmd
+        --speed-perturb"
+    fi
+    echo -e "\033[34mRun command: \n${cmd} \033[0m"
+    [[ $eval -eq 1 ]] && eval $cmd
+fi
+
+if [ ${stage} -le 1 ] && [ ${stop_stage} -ge 1 ]; then
+    echo "stage 1: ASR Network Training"
+    [[ ! -d ${data_dir} ]] && echo "The data dir ${data_dir} is not existing!" && exit 1;
+
+    if [[ -z ${device} || ${#device[@]} -eq 0 ]]; then
+		if [[ ${gpu_num} -eq 0 ]]; then
+			device=()
+		else
+        	source ./local/utils.sh
+        	device=$(get_devices $gpu_num 0)
+		fi
+    fi
+
+    echo -e "dev=${device} data=${data_dir} model=${model_dir}"
+
+    if [[ ! -d ${model_dir} ]]; then
+        mkdir -p ${model_dir}
+    else
+        echo "${model_dir} exists."
+    fi
+
+    cp ${BASH_SOURCE[0]} ${model_dir}
+    cp ${PWD}/train.sh ${model_dir}
+    cp ${train_config} ${model_dir}
+
+    cmd="python3 -u ${root_dir}/fairseq_cli/train.py
+        ${data_dir}
+        --config-yaml ${data_config}
+        --train-config ${train_config}
+        --task ${task}
+        --max-tokens ${max_tokens}
+        --update-freq ${update_freq}
+        --log-interval 100
+        --save-dir ${model_dir}
+        --tensorboard-logdir ${model_dir}"
+
+    if [[ -n ${extra_parameter} ]]; then
+        cmd="${cmd}
+        ${extra_parameter}"
+    fi
+	if [[ ${gpu_num} -gt 0 ]]; then
+		cmd="${cmd}
+        --distributed-world-size $gpu_num
+        --ddp-backend no_c10d"
+	fi
+    if [[ $fp16 -eq 1 ]]; then
+        cmd="${cmd}
+        --fp16"
+    fi
+    if [[ $step_valid -eq 1 ]]; then
+        validate_interval=10000
+        save_interval=10000
+        no_epoch_checkpoints=1
+        save_interval_updates=5000
+        keep_interval_updates=3
+    else
+        validate_interval=1
+        keep_last_epochs=10
+    fi
+    if [[ -n $no_epoch_checkpoints && $no_epoch_checkpoints -eq 1 ]]; then
+        cmd="$cmd
+        --no-epoch-checkpoints"
+    fi
+    if [[ -n $validate_interval ]]; then
+        cmd="${cmd}
+        --validate-interval $validate_interval "
+    fi
+    if [[ -n $save_interval ]]; then
+        cmd="${cmd}
+        --save-interval $save_interval "
+    fi
+    if [[ -n $keep_last_epochs ]]; then
+        cmd="${cmd}
+        --keep-last-epochs $keep_last_epochs "
+    fi
+    if [[ -n $save_interval_updates ]]; then
+        cmd="${cmd}
+        --save-interval-updates $save_interval_updates"
+        if [[ -n $keep_interval_updates ]]; then
+        cmd="${cmd}
+        --keep-interval-updates $keep_interval_updates"
+        fi
+    fi
+
+    echo -e "\033[34mRun command: \n${cmd} \033[0m"
+
+    # save info
+    log=./history.log
+    echo "${time} | ${device} | ${data_dir} | ${model_dir} " >> $log
+    cat $log | tail -n 50 > tmp.log
+    mv tmp.log $log
+    export CUDA_VISIBLE_DEVICES=${device}
+
+    cmd="nohup ${cmd} >> ${model_dir}/train.log 2>&1 &"
+    if [[ $eval -eq 1 ]]; then
+		eval $cmd
+		sleep 2s
+		tail -n `wc -l ${model_dir}/train.log | awk '{print $1+1}'` -f ${model_dir}/train.log
+	fi
+fi
+wait
+
+if [ ${stage} -le 2 ] && [ ${stop_stage} -ge 2 ]; then
+    echo "stage 2: ASR Decoding"
+    if [[ ${n_average} -ne 1 ]]; then
+        # Average models
+		dec_model=avg_${n_average}_checkpoint.pt
+
+		cmd="python ${root_dir}/scripts/average_checkpoints.py
+        --inputs ${model_dir}
+        --num-epoch-checkpoints ${n_average}
+        --output ${model_dir}/${dec_model}"
+    	echo -e "\033[34mRun command: \n${cmd} \033[0m"
+    	[[ $eval -eq 1 ]] && eval $cmd
+	else
+		dec_model=checkpoint_best.pt
+	fi
+
+    if [[ -z ${device} || ${#device[@]} -eq 0 ]]; then
+		if [[ ${gpu_num} -eq 0 ]]; then
+			device=()
+		else
+        	source ./local/utils.sh
+        	device=$(get_devices $gpu_num 0)
+		fi
+    fi
+    export CUDA_VISIBLE_DEVICES=${device}
+
+	#tmp_file=$(mktemp ${model_dir}/tmp-XXXXX)
+	#trap 'rm -rf ${tmp_file}' EXIT
+	result_file=${model_dir}/decode_result
+	[[ -f ${result_file} ]] && rm ${result_file}
+
+    test_subset=(${test_subset//,/ })
+	for subset in ${test_subset[@]}; do
+        subset=${subset}
+  		cmd="python ${root_dir}/fairseq_cli/generate.py
+        ${data_dir}
+        --config-yaml ${data_config}
+        --gen-subset ${subset}
+        --task speech_to_text
+        --path ${model_dir}/${dec_model}
+        --results-path ${model_dir}
+        --max-tokens ${max_tokens}
+        --beam ${beam_size}
+        --scoring wer"
+    	echo -e "\033[34mRun command: \n${cmd} \033[0m"
+
+        if [[ $eval -eq 1 ]]; then
+    	    eval $cmd
+    	    tail -n 1 ${model_dir}/generate-${subset}.txt >> ${result_file}
+        fi
+	done
+    cat ${result_file}
+fi

egs/lower/asr/train.sh

+#! /bin/bash
+
+# training the model
+
+gpu_num=8
+update_freq=2
+max_tokens=20000
+
+extra_tag=
+extra_parameter=
+
+#extra_tag="${extra_tag}"
+#extra_parameter="${extra_parameter} "
+
+exp_tag=
+train_config=train_ctc.yaml
+
+cmd="./run.sh
+    --stage 1
+    --stop_stage 1
+    --gpu_num ${gpu_num}
+    --update_freq ${update_freq}
+    --train_config ${train_config}
+    --max_tokens ${max_tokens}
+    "
+
+if [[ -n ${exp_tag} ]]; then
+    cmd="$cmd --exp_tag ${exp_tag}"
+fi
+if [[ -n ${extra_tag} ]]; then
+    cmd="$cmd --extra_tag ${extra_tag}"
+fi
+if [[ -n ${extra_parameter} ]]; then
+    cmd="$cmd --extra_parameter \"${extra_parameter}\""
+fi
+
+echo $cmd
+eval $cmd

egs/lower/mt/binary.sh

+set -e
+
+eval=1
+
+root_dir=~/st/Fairseq-S2T
+data_dir=/home/xuchen/st/data/wmt/test
+vocab_dir=/home/xuchen/st/data/wmt/mt/en-de/unigram32000_share
+src_vocab_prefix=spm_unigram32000_share
+tgt_vocab_prefix=spm_unigram32000_share
+
+src_lang=en
+tgt_lang=de
+tokenize=1
+splits=(newstest2014 newstest2016)
+
+for split in ${splits[@]}; do
+    src_file=${data_dir}/${split}.${src_lang}
+    tgt_file=${data_dir}/${split}.${tgt_lang}
+
+    if [[ ${tokenize} -eq 1 ]]; then
+        cmd="tokenizer.perl -l ${src_lang} --threads 8 -no-escape < ${src_file} > ${src_file}.tok"
+        echo -e "\033[34mRun command: \n${cmd} \033[0m"
+        [[ $eval -eq 1 ]] && eval ${cmd}
+
+        cmd="tokenizer.perl -l ${tgt_lang} --threads 8 -no-escape < ${tgt_file} > ${tgt_file}.tok"
+        echo -e "\033[34mRun command: \n${cmd} \033[0m"
+        [[ $eval -eq 1 ]] && eval ${cmd}
+        src_file=${src_file}.tok
+        tgt_file=${tgt_file}.tok
+    fi
+
+    cmd="spm_encode
+    --model ${vocab_dir}/${src_vocab_prefix}.model
+    --output_format=piece
+    < ${src_file}
+    > ${src_file}.spm"
+    echo -e "\033[34mRun command: \n${cmd} \033[0m"
+    [[ $eval -eq 1 ]] && eval ${cmd}
+
+    cmd="spm_encode
+    --model ${vocab_dir}/${tgt_vocab_prefix}.model
+    --output_format=piece
+    < ${tgt_file}
+    > ${tgt_file}.spm"
+    echo -e "\033[34mRun command: \n${cmd} \033[0m"
+    [[ $eval -eq 1 ]] && eval ${cmd}
+
+    src_file=${src_file}.spm
+    tgt_file=${tgt_file}.spm
+
+    mkdir -p ${data_dir}/final
+    cmd="cp ${src_file} ${data_dir}/final/${split}.${src_lang}"
+    echo -e "\033[34mRun command: \n${cmd} \033[0m"
+    [[ $eval -eq 1 ]] && eval ${cmd}
+
+    cmd="cp ${tgt_file} ${data_dir}/final/${split}.${tgt_lang}"
+    echo -e "\033[34mRun command: \n${cmd} \033[0m"
+    [[ $eval -eq 1 ]] && eval ${cmd}
+done
+
+n_set=${#splits[*]}
+for ((i=0;i<$n_set;i++)); do
+    dataset[$i]=${data_dir}/final/${splits[$i]}
+done
+pref=`echo ${dataset[*]} | sed 's/ /,/g'`
+
+cmd="python ${root_dir}/fairseq_cli/preprocess.py
+    --source-lang ${src_lang}
+    --target-lang ${tgt_lang}
+    --testpref ${pref}
+    --destdir ${data_dir}/data-bin
+    --srcdict ${vocab_dir}/${src_vocab_prefix}.txt
+    --tgtdict ${vocab_dir}/${tgt_vocab_prefix}.txt
+    --workers 64"
+
+echo -e "\033[34mRun command: \n${cmd} \033[0m"
+[[ $eval -eq 1 ]] && eval ${cmd}
\ No newline at end of file

egs/lower/mt/conf/train.yaml

 train-subset: train
 valid-subset: valid
 
-max-epoch: 20
+max-epoch: 50
 max-update: 100000
 
-skip-invalid-size-inputs-valid-test: True
 num-workers: 8
 patience: 10
 no-progress-bar: True
 log-interval: 100
 seed: 1
 report-accuracy: True
+skip-invalid-size-inputs-valid-test: True
 
-#load-params: 
-#load-pretrained-encoder-from: 
+#load-pretrained-encoder-from:
 
 arch: transformer
 share-decoder-input-output-embed: True
@@ -30,14 +29,14 @@ label_smoothing: 0.1
 
 dropout: 0.1
 activation-fn: relu
-encoder-embed-dim: 512
+encoder-normalize-before: True
+decoder-normalize-before: True
+encoder-embed-dim: 256
 encoder-ffn-embed-dim: 2048
 encoder-layers: 6
 decoder-layers: 6
-encoder-attention-heads: 8
+encoder-attention-heads: 4
 
-#decoder-embed-dim: 256
-#decoder-ffn-embed-dim: 2048
-#decoder-attention-heads: 4
-#attention-dropout: 0.1
-#activation-dropout: 0.1
+decoder-embed-dim: 256
+decoder-ffn-embed-dim: 2048
+decoder-attention-heads: 4

egs/lower/mt/conf/train_m.yaml

+train-subset: train
+valid-subset: valid
+
+max-epoch: 50
+max-update: 100000
+
+num-workers: 8
+patience: 10
+no-progress-bar: True
+log-interval: 100
+seed: 1
+report-accuracy: True
+skip-invalid-size-inputs-valid-test: True
+
+#load-pretrained-encoder-from:
+
+arch: transformer
+share-decoder-input-output-embed: True
+optimizer: adam
+clip-norm: 10.0
+lr-scheduler: inverse_sqrt
+warmup-init-lr: 1e-7
+warmup-updates: 8000
+lr: 5e-4
+adam_betas: (0.9,0.98)
+
+criterion: label_smoothed_cross_entropy
+label_smoothing: 0.1
+
+dropout: 0.1
+activation-fn: relu
+encoder-normalize-before: True
+decoder-normalize-before: True
+encoder-embed-dim: 512
+encoder-ffn-embed-dim: 2048
+encoder-layers: 6
+decoder-layers: 6
+encoder-attention-heads: 8
+
+decoder-embed-dim: 512
+decoder-ffn-embed-dim: 2048
+decoder-attention-heads: 8

egs/lower/mt/decode.sh

@@ -2,7 +2,8 @@
 
 gpu_num=1
 
-test_subset=(test)
+data_dir=
+test_subset=test
 
 exp_name=
 if [ "$#" -eq 1 ]; then
@@ -11,18 +12,24 @@ fi
 
 n_average=5
 beam_size=5
-max_tokens=4096
+max_tokens=20000
 
 cmd="./run.sh
     --stage 2
     --stop_stage 2
     --gpu_num ${gpu_num}
     --exp_name ${exp_name}
-    --test_subset ${test_subset}
     --n_average ${n_average}
     --beam_size ${beam_size}
     --max_tokens ${max_tokens}
     "
 
+if [[ -n ${data_dir} ]]; then
+    cmd="$cmd --data_dir ${data_dir}"
+fi
+if [[ -n ${test_subset} ]]; then
+    cmd="$cmd --test_subset ${test_subset}"
+fi
+
 echo $cmd
 eval $cmd

egs/lower/mt/run.sh

@@ -20,7 +20,7 @@ stop_stage=0
 
 ######## hardware ########
 # devices
-device=()
+#device=()
 gpu_num=8
 update_freq=1
 
@@ -35,15 +35,20 @@ lang=${src_lang}-${tgt_lang}
 dataset=lower
 task=translation
 vocab_type=unigram
-vocab_size=32000
+vocab_size=10000
 share_dict=1
 
-org_data_dir=/media/data/${dataset}/mt
+use_specific_dict=1
+specific_prefix=st_share10k
+specific_dir=/home/xuchen/st/data/mustc/st/en-de
+src_vocab_prefix=spm_unigram10000_st_share
+tgt_vocab_prefix=spm_unigram10000_st_share
+
+org_data_dir=/media/data/${dataset}
 data_dir=~/st/data/${dataset}/mt/${lang}
-train_prefix=train
-valid_prefix=dev
-test_prefix=test
-test_subset=(test)
+train_subset=train
+valid_subset=dev
+test_subset=test
 
 # exp
 extra_tag=
@@ -64,19 +69,23 @@ bleu_valid=0
 n_average=10
 beam_size=5
 
-. ./local/parse_options.sh || exit 1;
-
-if [[ $step_valid -eq 1 ]]; then
-    validate_interval=10000
-    save_interval=10000
-    no_epoch_checkpoints=1
-    save_interval_updates=5000
-    keep_interval_updates=3
+if [[ ${use_specific_dict} -eq 1 ]]; then
+    exp_tag=${specific_prefix}_${exp_tag}
+    data_dir=${data_dir}/${specific_prefix}
+    mkdir -p ${data_dir}
 else
-    validate_interval=1
-    keep_last_epochs=10
+    data_dir=${data_dir}/${vocab_type}${vocab_size}
+    src_vocab_prefix=spm_${vocab_type}${vocab_size}_${src_lang}
+    tgt_vocab_prefix=spm_${vocab_type}${vocab_size}_${tgt_lang}
+    if [[ $share_dict -eq 1 ]]; then
+        data_dir=${data_dir}_share
+        src_vocab_prefix=spm_${vocab_type}${vocab_size}_share
+        tgt_vocab_prefix=spm_${vocab_type}${vocab_size}_share
+    fi
 fi
 
+. ./local/parse_options.sh || exit 1;
+
 # full path
 train_config=$pwd_dir/conf/${train_config}
 if [[ -z ${exp_name} ]]; then
@@ -85,7 +94,6 @@ if [[ -z ${exp_name} ]]; then
         exp_name=${exp_name}_${extra_tag}
     fi
 fi
-
 model_dir=$root_dir/../checkpoints/$dataset/mt/${exp_name}
 
 if [ ${stage} -le -1 ] && [ ${stop_stage} -ge -1 ]; then
@@ -93,11 +101,6 @@ if [ ${stage} -le -1 ] && [ ${stop_stage} -ge -1 ]; then
     # pass
 fi
 
-data_dir=${data_dir}/${vocab_type}${vocab_size}
-if [[ $share_dict -eq 1 ]]; then
-    data_dir=${data_dir}_share
-fi
-
 if [ ${stage} -le 0 ] && [ ${stop_stage} -ge 0 ]; then
     ### Task dependent. You have to make data the following preparation part by yourself.
     echo "stage 0: MT Data Preparation"
@@ -105,52 +108,54 @@ if [ ${stage} -le 0 ] && [ ${stop_stage} -ge 0 ]; then
         mkdir -p ${data_dir}
     fi
 
-    src_vocab_prefix=spm_${vocab_type}${vocab_size}_${src_lang}
-    tgt_vocab_prefix=spm_${vocab_type}${vocab_size}_${tgt_lang}
-    cmd="python ${root_dir}/examples/speech_to_text/prep_mt_data.py
-        --data-root ${org_data_dir}
-        --output-root ${data_dir}
-        --splits ${train_prefix},${valid_prefix},${test_prefix}
-        --src-lang ${src_lang}
-        --tgt-lang ${tgt_lang}
-        --vocab-type ${vocab_type}
-        --vocab-size ${vocab_size}"
-    if [[ $share_dict -eq 1 ]]; then
-        cmd="$cmd
-        --share"
-        src_vocab_prefix=spm_unigram${vocab_size}_share
-        tgt_vocab_prefix=spm_unigram${vocab_size}_share
+    if [[ ! -f ${data_dir}/${src_vocab_prefix}.txt || ! -f ${data_dir}/${tgt_vocab_prefix}.txt ]]; then
+        if [[ ${use_specific_dict} -eq 0 ]]; then
+            cmd="python ${root_dir}/examples/speech_to_text/prep_mt_data.py
+                --data-root ${org_data_dir}
+                --output-root ${data_dir}
+                --splits ${train_subset},${valid_subset},${test_subset}
+                --src-lang ${src_lang}
+                --tgt-lang ${tgt_lang}
+                --vocab-type ${vocab_type}
+                --vocab-size ${vocab_size}"
+            if [[ $share_dict -eq 1 ]]; then
+                cmd="$cmd
+                --share"
+            fi
+            echo -e "\033[34mRun command: \n${cmd} \033[0m"
+            [[ $eval -eq 1 ]] && eval ${cmd}
+        else
+            cp -r ${specific_dir}/${src_vocab_prefix}.* ${data_dir}
+            cp ${specific_dir}/${tgt_vocab_prefix}.* ${data_dir}
+        fi
     fi
 
-    echo -e "\033[34mRun command: \n${cmd} \033[0m"
-    [[ $eval -eq 1 ]] && eval ${cmd}
-
     mkdir -p ${data_dir}/data
-    for split in ${train_prefix} ${valid_prefix} ${test_prefix}; do
+    for split in ${train_subset} ${valid_subset} ${test_subset}; do
         cmd="spm_encode
         --model ${data_dir}/${src_vocab_prefix}.model
         --output_format=piece
-        < ${org_data_dir}/${lang}/data/${split}/${split}.${src_lang}
+        < ${org_data_dir}/${lang}/data/${split}.${src_lang}
         > ${data_dir}/data/${split}.${src_lang}"
 
         echo -e "\033[34mRun command: \n${cmd} \033[0m"
         [[ $eval -eq 1 ]] && eval ${cmd}
-    
+
         cmd="spm_encode
         --model ${data_dir}/${tgt_vocab_prefix}.model
         --output_format=piece
-        < ${org_data_dir}/${lang}/data/${split}/${split}.${tgt_lang}
+        < ${org_data_dir}/${lang}/data/${split}.${tgt_lang}
         > ${data_dir}/data/${split}.${tgt_lang}"
 
         echo -e "\033[34mRun command: \n${cmd} \033[0m"
         [[ $eval -eq 1 ]] && eval ${cmd}
     done
 
-    cmd="python ${root_dir}/fairseq_cli/preprocess.py 
+    cmd="python ${root_dir}/fairseq_cli/preprocess.py
         --source-lang ${src_lang} --target-lang ${tgt_lang}
-        --trainpref ${data_dir}/data/${train_prefix}
-        --validpref ${data_dir}/data/${valid_prefix}
-        --testpref ${data_dir}/data/${test_prefix}
+        --trainpref ${data_dir}/data/${train_subset}
+        --validpref ${data_dir}/data/${valid_subset}
+        --testpref ${data_dir}/data/${test_subset}
         --destdir ${data_dir}/data-bin
         --srcdict ${data_dir}/${src_vocab_prefix}.txt
         --tgtdict ${data_dir}/${tgt_vocab_prefix}.txt
@@ -212,6 +217,16 @@ if [ ${stage} -le 1 ] && [ ${stop_stage} -ge 1 ]; then
         cmd="${cmd}
         --fp16"
     fi
+    if [[ $step_valid -eq 1 ]]; then
+        validate_interval=10000
+        save_interval=10000
+        no_epoch_checkpoints=1
+        save_interval_updates=5000
+        keep_interval_updates=3
+    else
+        validate_interval=1
+        keep_last_epochs=10
+    fi
     if [[ $bleu_valid -eq 1 ]]; then
         cmd="$cmd
         --eval-bleu
@@ -295,6 +310,7 @@ if [ ${stage} -le 2 ] && [ ${stop_stage} -ge 2 ]; then
 	result_file=${model_dir}/decode_result
 	[[ -f ${result_file} ]] && rm ${result_file}
 
+    test_subset=(${test_subset//,/ })
 	for subset in ${test_subset[@]}; do
   		cmd="python ${root_dir}/fairseq_cli/generate.py
         ${data_dir}

egs/lower/mt/train.sh

@@ -2,7 +2,7 @@
 
 # training the model
 
-gpu_num=8
+gpu_num=1
 update_freq=1
 max_tokens=4096
 

egs/lower/st/conf/train.yaml

+train-subset: train_st
+valid-subset: dev_st
+
+max-epoch: 50
+max-update: 100000
+
+num-workers: 8
+patience: 10
+no-progress-bar: True
+log-interval: 100
+seed: 1
+report-accuracy: True
+
+#load-pretrained-encoder-from:
+#load-pretrained-decoder-from:
+
+arch: s2t_transformer_s
+share-decoder-input-output-embed: True
+optimizer: adam
+clip-norm: 10.0
+lr-scheduler: inverse_sqrt
+warmup-init-lr: 1e-7
+warmup-updates: 10000
+lr: 2e-3
+#adam_betas: (0.9,0.98)
+
+criterion: label_smoothed_cross_entropy
+label_smoothing: 0.1
+
+conv-kernel-sizes: 5,5
+conv-channels: 1024
+dropout: 0.1
+activation-fn: relu
+encoder-embed-dim: 256
+encoder-ffn-embed-dim: 2048
+encoder-layers: 12
+decoder-layers: 6
+encoder-attention-heads: 4
+
+#decoder-embed-dim: 256
+#decoder-ffn-embed-dim: 2048
+#decoder-attention-heads: 4
+#attention-dropout: 0.1
+#activation-dropout: 0.1

egs/lower/st/conf/train_ctc.yaml

+train-subset: train_st
+valid-subset: dev_st
+
+max-epoch: 50
+max-update: 100000
+
+num-workers: 8
+patience: 10
+no-progress-bar: True
+log-interval: 100
+seed: 1
+report-accuracy: True
+
+#load-pretrained-encoder-from:
+#load-pretrained-decoder-from:
+
+arch: s2t_transformer_s
+share-decoder-input-output-embed: True
+optimizer: adam
+clip-norm: 10.0
+lr-scheduler: inverse_sqrt
+warmup-init-lr: 1e-7
+warmup-updates: 10000
+lr: 2e-3
+#adam_betas: (0.9,0.98)
+
+ctc-weight: 0.3
+criterion: label_smoothed_cross_entropy_with_ctc
+label_smoothing: 0.1
+
+conv-kernel-sizes: 5,5
+conv-channels: 1024
+dropout: 0.1
+activation-fn: relu
+encoder-embed-dim: 256
+encoder-ffn-embed-dim: 2048
+encoder-layers: 12
+decoder-layers: 6
+encoder-attention-heads: 4
+
+#decoder-embed-dim: 256
+#decoder-ffn-embed-dim: 2048
+#decoder-attention-heads: 4
+#attention-dropout: 0.1
+#activation-dropout: 0.1

egs/lower/st/conf/train_ctc_conformer.yaml

+train-subset: train_st
+valid-subset: dev_st
+
+max-epoch: 50
+max-update: 100000
+
+num-workers: 8
+patience: 10
+no-progress-bar: True
+log-interval: 100
+seed: 1
+report-accuracy: True
+
+#load-params: 
+#load-pretrained-encoder-from: 
+
+arch: s2t_conformer_s
+share-decoder-input-output-embed: True
+optimizer: adam
+clip-norm: 10.0
+lr-scheduler: inverse_sqrt
+warmup-init-lr: 1e-7
+warmup-updates: 10000
+lr: 2e-3
+#adam_betas: (0.9,0.98)
+
+ctc-weight: 0.3
+criterion: label_smoothed_cross_entropy_with_ctc
+label_smoothing: 0.1
+
+conv-kernel-sizes: 5,5
+conv-channels: 1024
+dropout: 0.1
+activation-fn: relu
+encoder-embed-dim: 256
+encoder-ffn-embed-dim: 2048
+encoder-layers: 12
+decoder-layers: 6
+encoder-attention-heads: 4
+
+macaron-style: True
+use-cnn-module: True
+cnn-module-kernel: 31
+
+#decoder-embed-dim: 256
+#decoder-ffn-embed-dim: 2048
+#decoder-attention-heads: 4
+#attention-dropout: 0.1
+#activation-dropout: 0.1

egs/lower/st/conf/train_ctc_conformer_m.yaml

+train-subset: train_st
+valid-subset: dev_st
+
+max-epoch: 50
+max-update: 100000
+
+num-workers: 8
+patience: 10
+no-progress-bar: True
+log-interval: 100
+seed: 1
+report-accuracy: True
+
+#load-pretrained-encoder-from:
+#load-pretrained-decoder-from:
+
+arch: s2t_conformer_m
+share-decoder-input-output-embed: True
+optimizer: adam
+clip-norm: 10.0
+lr-scheduler: inverse_sqrt
+warmup-init-lr: 1e-7
+warmup-updates: 10000
+lr: 1e-3
+#adam_betas: (0.9,0.98)
+
+ctc-weight: 0.3
+criterion: label_smoothed_cross_entropy_with_ctc
+label_smoothing: 0.1
+
+conv-kernel-sizes: 5,5
+conv-channels: 1024
+#dropout: 0.1
+#activation-fn: relu
+#encoder-embed-dim: 256
+#encoder-ffn-embed-dim: 2048
+#encoder-layers: 12
+#decoder-layers: 6
+#encoder-attention-heads: 4
+
+macaron-style: True
+use-cnn-module: True
+cnn-module-kernel: 31
+
+#decoder-embed-dim: 256
+#decoder-ffn-embed-dim: 2048
+#decoder-attention-heads: 4
+#attention-dropout: 0.1
+#activation-dropout: 0.1

egs/lower/st/conf/train_ctc_debug.yaml

+train-subset: train_st,train_v2
+valid-subset: dev_st
+
+max-epoch: 50
+max-update: 100000
+
+num-workers: 8
+patience: 10
+no-progress-bar: True
+log-interval: 100
+seed: 1
+report-accuracy: True
+
+#load-pretrained-encoder-from: /home/xuchen/st/checkpoints/mustc/asr/train_ctc_baseline/avg_10_checkpoint.pt
+#load-pretrained-decoder-from: /home/xuchen/st/checkpoints/mustc/mt/train_baseline/avg_10_checkpoint.pt
+
+arch: s2t_transformer_s
+share-decoder-input-output-embed: True
+optimizer: adam
+clip-norm: 10.0
+lr-scheduler: inverse_sqrt
+warmup-init-lr: 1e-7
+warmup-updates: 10000
+lr: 2e-3
+#adam_betas: (0.9,0.98)
+
+ctc-weight: 0.3
+criterion: label_smoothed_cross_entropy_with_ctc
+label_smoothing: 0.1
+
+conv-kernel-sizes: 5,5
+conv-channels: 1024
+dropout: 0.1
+activation-fn: relu
+encoder-embed-dim: 256
+encoder-ffn-embed-dim: 2048
+encoder-layers: 12
+decoder-layers: 6
+encoder-attention-heads: 4
+
+#decoder-embed-dim: 256
+#decoder-ffn-embed-dim: 2048
+#decoder-attention-heads: 4
+#attention-dropout: 0.1
+#activation-dropout: 0.1

egs/lower/st/conf/train_ctc_enc_rpr.yaml

+train-subset: train_st
+valid-subset: dev_st
+
+max-epoch: 100
+max-update: 100000
+
+num-workers: 8
+patience: 10
+no-progress-bar: True
+log-interval: 100
+seed: 1
+report-accuracy: True
+
+#load-pretrained-encoder-from:
+#load-pretrained-decoder-from:
+
+arch: s2t_transformer_s
+share-decoder-input-output-embed: True
+optimizer: adam
+clip-norm: 10.0
+lr-scheduler: inverse_sqrt
+warmup-init-lr: 1e-7
+warmup-updates: 10000
+lr: 2e-3
+#adam_betas: (0.9,0.98)
+
+ctc-weight: 0.3
+criterion: label_smoothed_cross_entropy_with_ctc
+label_smoothing: 0.1
+
+conv-kernel-sizes: 5,5
+conv-channels: 1024
+dropout: 0.1
+activation-fn: relu
+encoder-embed-dim: 256
+encoder-ffn-embed-dim: 2048
+encoder-layers: 12
+decoder-layers: 6
+encoder-attention-heads: 4
+encoder-attention-type: rel_selfattn
+
+#decoder-embed-dim: 256
+#decoder-ffn-embed-dim: 2048
+#decoder-attention-heads: 4
+#attention-dropout: 0.1
+#activation-dropout: 0.1

egs/lower/st/conf/train_ctc_sate.yaml

+train-subset: train_st
+valid-subset: dev_st
+
+max-epoch: 50
+max-update: 100000
+
+num-workers: 8
+patience: 10
+no-progress-bar: True
+log-interval: 100
+seed: 1
+report-accuracy: True
+
+#load-pretrained-acoustic-encoder-from: /home/xuchen/st/checkpoints/mustc/asr/train_ctc_baseline/avg_10_checkpoint.pt
+#load-pretrained-text-encoder-from: /home/xuchen/st/checkpoints/mustc/mt/st_share10k_train_baseline/avg_10_checkpoint.pt
+#load-pretrained-decoder-from: /home/xuchen/st/checkpoints/mustc/mt/st_share10k_train_baseline/avg_10_checkpoint.pt
+
+arch: s2t_sate
+share-decoder-input-output-embed: True
+optimizer: adam
+clip-norm: 10.0
+lr-scheduler: inverse_sqrt
+warmup-init-lr: 1e-7
+warmup-updates: 10000
+lr: 2e-3
+#adam_betas: (0.9,0.98)
+
+ctc-weight: 0.3
+criterion: label_smoothed_cross_entropy_with_ctc
+label_smoothing: 0.1
+
+encoder-normalize-before: True
+decoder-normalize-before: True
+conv-kernel-sizes: 5,5
+conv-channels: 1024
+dropout: 0.1
+activation-fn: relu
+encoder-embed-dim: 256
+encoder-ffn-embed-dim: 2048
+encoder-layers: 12
+text-encoder-layers: 6
+decoder-layers: 6
+encoder-attention-heads: 4
+
+macaron-style: True
+use-cnn-module: True
+cnn-module-kernel: 31
+
+acoustic-encoder: transformer
+adapter: league
+
+#decoder-embed-dim: 256
+#decoder-ffn-embed-dim: 2048
+#decoder-attention-heads: 4
+#attention-dropout: 0.1
+#activation-dropout: 0.1

egs/lower/st/conf/train_ctc_sate_conformer.yaml

+train-subset: train_st
+valid-subset: dev_st
+
+max-epoch: 50
+max-update: 100000
+
+num-workers: 8
+patience: 10
+no-progress-bar: True
+log-interval: 100
+seed: 1
+report-accuracy: True
+
+#load-pretrained-acoustic-encoder-from: /home/xuchen/st/checkpoints/mustc/asr/train_ctc_baseline/avg_10_checkpoint.pt
+#load-pretrained-text-encoder-from: /home/xuchen/st/checkpoints/mustc/mt/st_share10k_train_baseline/avg_10_checkpoint.pt
+#load-pretrained-decoder-from: /home/xuchen/st/checkpoints/mustc/mt/st_share10k_train_baseline/avg_10_checkpoint.pt
+
+arch: s2t_sate
+share-decoder-input-output-embed: True
+optimizer: adam
+clip-norm: 10.0
+lr-scheduler: inverse_sqrt
+warmup-init-lr: 1e-7
+warmup-updates: 10000
+lr: 2e-3
+#adam_betas: (0.9,0.98)
+
+ctc-weight: 0.3
+criterion: label_smoothed_cross_entropy_with_ctc
+label_smoothing: 0.1
+
+encoder-normalize-before: True
+decoder-normalize-before: True
+conv-kernel-sizes: 5,5
+conv-channels: 1024
+dropout: 0.1
+activation-fn: relu
+encoder-embed-dim: 256
+encoder-ffn-embed-dim: 2048
+encoder-layers: 12
+text-encoder-layers: 6
+decoder-layers: 6
+encoder-attention-heads: 4
+
+macaron-style: True
+use-cnn-module: True
+cnn-module-kernel: 31
+
+acoustic-encoder: conformer
+adapter: league
+
+#decoder-embed-dim: 256
+#decoder-ffn-embed-dim: 2048
+#decoder-attention-heads: 4
+#attention-dropout: 0.1
+#activation-dropout: 0.1

egs/lower/st/decode.sh

+#! /bin/bash
+
+gpu_num=1
+
+data_dir=
+test_subset=tst-COMMON
+
+exp_name=
+if [ "$#" -eq 1 ]; then
+    exp_name=$1
+fi
+
+n_average=10
+beam_size=5
+max_tokens=40000
+
+cmd="./run.sh
+    --stage 2
+    --stop_stage 2
+    --gpu_num ${gpu_num}
+    --exp_name ${exp_name}
+    --n_average ${n_average}
+    --beam_size ${beam_size}
+    --max_tokens ${max_tokens}
+    "
+
+if [[ -n ${data_dir} ]]; then
+    cmd="$cmd --data_dir ${data_dir}"
+fi
+if [[ -n ${test_subset} ]]; then
+    cmd="$cmd --test_subset ${test_subset}"
+fi
+
+echo $cmd
+eval $cmd

egs/lower/st/local/monitor.sh

+gpu_num=1
+
+while :
+do
+    all_devices=$(seq 0 `gpustat | sed '1,2d' | wc -l`);
+    count=0
+    for dev in ${all_devices[@]}
+    do
+        line=`expr $dev + 2`
+        use=`gpustat -p | head -n $line | tail -1 | cut -d '|' -f4 | wc -w`
+        if [[ $use -eq 0 ]]; then
+            device[$count]=$dev
+            count=`expr $count + 1`
+            if [[ $count -eq $gpu_num ]]; then
+                break
+            fi
+        fi
+    done
+    if [[ ${#device[@]} -lt $gpu_num ]]; then
+        sleep 60s
+    else
+        echo "Run $cmd"
+        eval $cmd
+        sleep 10s
+        exit
+    fi
+done

egs/lower/st/local/parse_options.sh

+#!/usr/bin/env bash
+
+# Copyright 2012  Johns Hopkins University (Author: Daniel Povey);
+#                 Arnab Ghoshal, Karel Vesely
+
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#  http://www.apache.org/licenses/LICENSE-2.0
+#
+# THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED
+# WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE,
+# MERCHANTABLITY OR NON-INFRINGEMENT.
+# See the Apache 2 License for the specific language governing permissions and
+# limitations under the License.
+
+
+# Parse command-line options.
+# To be sourced by another script (as in ". parse_options.sh").
+# Option format is: --option-name arg
+# and shell variable "option_name" gets set to value "arg."
+# The exception is --help, which takes no arguments, but prints the
+# $help_message variable (if defined).
+
+
+###
+### The --config file options have lower priority to command line
+### options, so we need to import them first...
+###
+
+# Now import all the configs specified by command-line, in left-to-right order
+for ((argpos=1; argpos<$#; argpos++)); do
+  if [ "${!argpos}" == "--config" ]; then
+    argpos_plus1=$((argpos+1))
+    config=${!argpos_plus1}
+    [ ! -r $config ] && echo "$0: missing config '$config'" && exit 1
+    . $config  # source the config file.
+  fi
+done
+
+
+###
+### Now we process the command line options
+###
+while true; do
+  [ -z "${1:-}" ] && break;  # break if there are no arguments
+  case "$1" in
+    # If the enclosing script is called with --help option, print the help
+    # message and exit.  Scripts should put help messages in $help_message
+    --help|-h) if [ -z "$help_message" ]; then echo "No help found." 1>&2;
+      else printf "$help_message\n" 1>&2 ; fi;
+      exit 0 ;;
+    --*=*) echo "$0: options to scripts must be of the form --name value, got '$1'"
+      exit 1 ;;
+    # If the first command-line argument begins with "--" (e.g. --foo-bar),
+    # then work out the variable name as $name, which will equal "foo_bar".
+    --*) name=`echo "$1" | sed s/^--// | sed s/-/_/g`;
+      # Next we test whether the variable in question is undefned-- if so it's
+      # an invalid option and we die.  Note: $0 evaluates to the name of the
+      # enclosing script.
+      # The test [ -z ${foo_bar+xxx} ] will return true if the variable foo_bar
+      # is undefined.  We then have to wrap this test inside "eval" because
+      # foo_bar is itself inside a variable ($name).
+      eval '[ -z "${'$name'+xxx}" ]' && echo "$0: invalid option $1" 1>&2 && exit 1;
+
+      oldval="`eval echo \\$$name`";
+      # Work out whether we seem to be expecting a Boolean argument.
+      if [ "$oldval" == "true" ] || [ "$oldval" == "false" ]; then
+        was_bool=true;
+      else
+        was_bool=false;
+      fi
+
+      # Set the variable to the right value-- the escaped quotes make it work if
+      # the option had spaces, like --cmd "queue.pl -sync y"
+      eval $name=\"$2\";
+
+      # Check that Boolean-valued arguments are really Boolean.
+      if $was_bool && [[ "$2" != "true" && "$2" != "false" ]]; then
+        echo "$0: expected \"true\" or \"false\": $1 $2" 1>&2
+        exit 1;
+      fi
+      shift 2;
+      ;;
+  *) break;
+  esac
+done
+
+
+# Check for an empty argument to the --cmd option, which can easily occur as a
+# result of scripting errors.
+[ ! -z "${cmd+xxx}" ] && [ -z "$cmd" ] && echo "$0: empty argument to --cmd option" 1>&2 && exit 1;
+
+
+true; # so this script returns exit code 0.

egs/lower/st/local/path.sh

+MAIN_ROOT=$PWD/../../..
+KALDI_ROOT=$MAIN_ROOT/tools/kaldi
+
+export PATH=$PWD/utils/:$KALDI_ROOT/tools/openfst/bin:$PATH
+[ ! -f $KALDI_ROOT/tools/config/common_path.sh ] && echo >&2 "The standard file $KALDI_ROOT/tools/config/common_path.sh is not present -> Exit!" && exit 1
+. $KALDI_ROOT/tools/config/common_path.sh
+export LC_ALL=C
+
+export LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:$MAIN_ROOT/src/lib
+export LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:$MAIN_ROOT/tools/chainer_ctc/ext/warp-ctc/build
+. "${MAIN_ROOT}"/tools/activate_python.sh && . "${MAIN_ROOT}"/tools/extra_path.sh
+export PATH=$MAIN_ROOT/utils:$MAIN_ROOT/espnet/bin:$PATH
+
+export OMP_NUM_THREADS=1
+
+# check extra module installation
+if ! which tokenizer.perl > /dev/null; then
+    echo "Error: it seems that moses is not installed." >&2
+    echo "Error: please install moses as follows." >&2
+    echo "Error: cd ${MAIN_ROOT}/tools && make moses.done" >&2
+    return 1
+fi
+
+# NOTE(kan-bayashi): Use UTF-8 in Python to avoid UnicodeDecodeError when LC_ALL=C
+export PYTHONIOENCODING=UTF-8

egs/lower/st/local/utils.sh

+
+get_devices(){
+    gpu_num=$1
+    use_cpu=$2
+    device=()
+    while :
+    do
+        record=`mktemp -t temp.record.XXXXXX`
+        gpustat > $record
+        all_devices=$(seq 0 `cat $record | sed '1,2d' | wc -l`);
+        count=0
+        for dev in ${all_devices[@]}
+        do
+            line=`expr $dev + 2`
+            use=`cat $record | head -n $line | tail -1 | cut -d '|' -f3 | cut -d '/' -f1`
+            if [[ $use -lt 100 ]]; then
+                device[$count]=$dev
+                count=`expr $count + 1`
+                if [[ $count -eq $gpu_num ]]; then
+                    break
+                fi
+            fi
+        done
+        if [[ ${#device[@]} -lt $gpu_num ]]; then
+            if [[ $use_cpu -eq 1 ]]; then
+                device=(-1)
+            else
+                sleep 60s
+            fi
+        else
+            break
+        fi
+    done
+
+    echo ${device[*]} | sed 's/ /,/g'
+    return $?
+}
+
+

egs/lower/st/run.sh

+#! /bin/bash
+
+# Processing MuST-C Datasets
+
+# Copyright 2021 Natural Language Processing Laboratory 
+# Xu Chen (xuchenneu@163.com)
+
+# Set bash to 'debug' mode, it will exit on :
+# -e 'error', -u 'undefined variable', -o ... 'error in pipeline', -x 'print commands',
+set -e
+#set -u
+set -o pipefail
+export PYTHONIOENCODING=UTF-8
+
+eval=1
+time=$(date "+%m%d_%H%M")
+
+stage=0
+stop_stage=0
+
+######## hardware ########
+# devices
+device=()
+gpu_num=8
+update_freq=1
+
+root_dir=~/st/Fairseq-S2T
+pwd_dir=$PWD
+
+# dataset
+src_lang=swa
+tgt_lang=en
+lang=${src_lang}-${tgt_lang}
+
+dataset=lower
+task=speech_to_text
+vocab_type=unigram
+asr_vocab_size=5000
+vocab_size=10000
+share_dict=1
+speed_perturb=1
+
+org_data_dir=/media/data/${dataset}
+data_dir=~/st/data/${dataset}/st
+test_subset=tst-COMMON
+
+# exp
+extra_tag=
+extra_parameter=
+exp_tag=baseline
+exp_name=
+
+# config
+train_config=train_ctc.yaml
+
+# training setting
+fp16=1
+max_tokens=40000
+step_valid=0
+bleu_valid=0
+
+# decoding setting
+n_average=10
+beam_size=5
+
+if [[ ${share_dict} -eq 1 ]]; then
+	data_config=config_st_share.yaml
+else
+	data_config=config_st.yaml
+fi
+
+if [[ ${speed_perturb} -eq 1 ]]; then
+    data_dir=${data_dir}_sp
+fi
+
+. ./local/parse_options.sh || exit 1;
+
+# full path
+train_config=$pwd_dir/conf/${train_config}
+if [[ -z ${exp_name} ]]; then
+    exp_name=$(basename ${train_config%.*})_${exp_tag}
+    if [[ -n ${extra_tag} ]]; then
+        exp_name=${exp_name}_${extra_tag}
+    fi
+    if [[ ${speed_perturb} -eq 1 ]]; then
+        exp_name=sp_${exp_name}
+    fi
+fi
+model_dir=$root_dir/../checkpoints/$dataset/st/${exp_name}
+
+if [ ${stage} -le -1 ] && [ ${stop_stage} -ge -1 ]; then
+    echo "stage -1: Data Download"
+    # pass
+fi
+
+if [ ${stage} -le 0 ] && [ ${stop_stage} -ge 0 ]; then
+    ### Task dependent. You have to make data the following preparation part by yourself.
+    ### But you can utilize Kaldi recipes in most cases
+    echo "stage 0: ASR Data Preparation"
+    if [[ ! -e ${data_dir}/${lang} ]]; then
+        mkdir -p ${data_dir}/${lang}
+    fi
+    source ~/tools/audio/bin/activate
+
+    cmd="python ${root_dir}/examples/speech_to_text/prep_mustc_data.py
+        --data-root ${org_data_dir}
+        --output-root ${data_dir}
+        --task asr
+        --vocab-type ${vocab_type}
+        --vocab-size ${asr_vocab_size}"
+    if [[ ${speed_perturb} -eq 1 ]]; then
+        cmd="$cmd
+        --speed-perturb"
+    fi
+    echo -e "\033[34mRun command: \n${cmd} \033[0m"
+    [[ $eval -eq 1 && ${share_dict} -ne 1 ]] && eval $cmd
+
+    echo "stage 0: ST Data Preparation"
+    cmd="python ${root_dir}/examples/speech_to_text/prep_mustc_data.py
+        --data-root ${org_data_dir}
+        --output-root ${data_dir}
+        --task st
+        --add-src
+        --cmvn-type utterance
+        --vocab-type ${vocab_type}
+        --vocab-size ${vocab_size}"
+    if [[ $share_dict -eq 1 ]]; then
+        cmd="$cmd
+        --share"
+	else
+        cmd="$cmd
+        --asr-prefix spm_${vocab_type}${asr_vocab_size}_asr"
+    fi
+    if [[ ${speed_perturb} -eq 1 ]]; then
+        cmd="$cmd
+        --speed-perturb"
+    fi
+
+    echo -e "\033[34mRun command: \n${cmd} \033[0m"
+    [[ $eval -eq 1 ]] && eval ${cmd}
+    deactivate
+fi
+
+data_dir=${data_dir}/${lang}
+
+if [ ${stage} -le 1 ] && [ ${stop_stage} -ge 1 ]; then
+    echo "stage 1: ST Network Training"
+    [[ ! -d ${data_dir} ]] && echo "The data dir ${data_dir} is not existing!" && exit 1;
+
+    if [[ -z ${device} || ${#device[@]} -eq 0 ]]; then
+		if [[ ${gpu_num} -eq 0 ]]; then
+			device=()
+		else
+        	source ./local/utils.sh
+        	device=$(get_devices $gpu_num 0)
+		fi
+    fi
+
+    echo -e "dev=${device} data=${data_dir} model=${model_dir}"
+
+    if [[ ! -d ${model_dir} ]]; then
+        mkdir -p ${model_dir}
+    else
+        echo "${model_dir} exists."
+    fi
+
+    cp ${BASH_SOURCE[0]} ${model_dir}
+    cp ${PWD}/train.sh ${model_dir}
+    cp ${train_config} ${model_dir}
+
+    cmd="python3 -u ${root_dir}/fairseq_cli/train.py
+        ${data_dir}
+        --config-yaml ${data_config}
+        --train-config ${train_config}
+        --task ${task}
+        --max-tokens ${max_tokens}
+        --update-freq ${update_freq}
+        --log-interval 100
+        --save-dir ${model_dir}
+        --tensorboard-logdir ${model_dir}"
+
+    if [[ -n ${extra_parameter} ]]; then
+        cmd="${cmd}
+        ${extra_parameter}"
+    fi
+	if [[ ${gpu_num} -gt 0 ]]; then
+		cmd="${cmd}
+        --distributed-world-size $gpu_num
+        --ddp-backend no_c10d"
+	fi
+    if [[ $fp16 -eq 1 ]]; then
+        cmd="${cmd}
+        --fp16"
+    fi
+    if [[ $step_valid -eq 1 ]]; then
+        validate_interval=10000
+        save_interval=10000
+        no_epoch_checkpoints=1
+        save_interval_updates=5000
+        keep_interval_updates=3
+    else
+        validate_interval=1
+        keep_last_epochs=10
+    fi
+    if [[ $bleu_valid -eq 1 ]]; then
+        cmd="$cmd
+        --eval-bleu
+        --eval-bleu-args '{\"beam\": 1}'
+        --eval-tokenized-bleu
+        --eval-bleu-remove-bpe
+        --best-checkpoint-metric bleu
+        --maximize-best-checkpoint-metric"
+    fi
+    if [[ -n $no_epoch_checkpoints && $no_epoch_checkpoints -eq 1 ]]; then
+        cmd="$cmd
+        --no-epoch-checkpoints"
+    fi
+    if [[ -n $validate_interval ]]; then
+        cmd="${cmd}
+        --validate-interval $validate_interval "
+    fi
+    if [[ -n $save_interval ]]; then
+        cmd="${cmd}
+        --save-interval $save_interval "
+    fi
+    if [[ -n $keep_last_epochs ]]; then
+        cmd="${cmd}
+        --keep-last-epochs $keep_last_epochs "
+    fi
+    if [[ -n $save_interval_updates ]]; then
+        cmd="${cmd}
+        --save-interval-updates $save_interval_updates"
+        if [[ -n $keep_interval_updates ]]; then
+        cmd="${cmd}
+        --keep-interval-updates $keep_interval_updates"
+        fi
+    fi
+
+    echo -e "\033[34mRun command: \n${cmd} \033[0m"
+
+    # save info
+    log=./history.log
+    echo "${time} | ${device} | ${data_dir} | ${model_dir} " >> $log
+    cat $log | tail -n 50 > tmp.log
+    mv tmp.log $log
+    export CUDA_VISIBLE_DEVICES=${device}
+
+    cmd="nohup ${cmd} >> ${model_dir}/train.log 2>&1 &"
+    if [[ $eval -eq 1 ]]; then
+		eval $cmd
+		sleep 2s
+		tail -n `wc -l ${model_dir}/train.log | awk '{print $1+1}'` -f ${model_dir}/train.log
+	fi
+fi
+wait
+
+if [ ${stage} -le 2 ] && [ ${stop_stage} -ge 2 ]; then
+    echo "stage 2: ST Decoding"
+    if [[ ${n_average} -ne 1 ]]; then
+        # Average models
+		dec_model=avg_${n_average}_checkpoint.pt
+
+		cmd="python ${root_dir}/scripts/average_checkpoints.py
+        --inputs ${model_dir}
+        --num-epoch-checkpoints ${n_average}
+        --output ${model_dir}/${dec_model}"
+    	echo -e "\033[34mRun command: \n${cmd} \033[0m"
+    	[[ $eval -eq 1 ]] && eval $cmd
+	else
+		dec_model=checkpoint_best.pt
+	fi
+
+    if [[ -z ${device} || ${#device[@]} -eq 0 ]]; then
+		if [[ ${gpu_num} -eq 0 ]]; then
+			device=()
+		else
+        	source ./local/utils.sh
+        	device=$(get_devices $gpu_num 0)
+		fi
+    fi
+    export CUDA_VISIBLE_DEVICES=${device}
+
+	#tmp_file=$(mktemp ${model_dir}/tmp-XXXXX)
+	#trap 'rm -rf ${tmp_file}' EXIT
+	result_file=${model_dir}/decode_result
+	[[ -f ${result_file} ]] && rm ${result_file}
+
+    test_subset=(${test_subset//,/ })
+	for subset in ${test_subset[@]}; do
+        subset=${subset}_st
+  		cmd="python ${root_dir}/fairseq_cli/generate.py
+        ${data_dir}
+        --config-yaml ${data_config}
+        --gen-subset ${subset}
+        --task speech_to_text
+        --path ${model_dir}/${dec_model}
+        --results-path ${model_dir}
+        --max-tokens ${max_tokens}
+        --beam ${beam_size}
+        --scoring sacrebleu"
+    	echo -e "\033[34mRun command: \n${cmd} \033[0m"
+
+        if [[ $eval -eq 1 ]]; then
+    	    eval $cmd
+    	    tail -n 1 ${model_dir}/generate-${subset}.txt >> ${result_file}
+        fi
+	done
+    cat ${result_file}
+fi

egs/lower/st/train.sh

+#! /bin/bash
+
+# training the model
+
+gpu_num=8
+update_freq=2
+max_tokens=20000
+
+extra_tag=
+extra_parameter=
+
+#extra_tag="${extra_tag}"
+#extra_parameter="${extra_parameter} "
+
+exp_tag=
+train_config=train_ctc.yaml
+
+cmd="./run.sh
+    --stage 1
+    --stop_stage 1
+    --gpu_num ${gpu_num}
+    --update_freq ${update_freq}
+    --train_config ${train_config}
+    --max_tokens ${max_tokens}
+    "
+
+if [[ -n ${exp_tag} ]]; then
+    cmd="$cmd --exp_tag ${exp_tag}"
+fi
+if [[ -n ${extra_tag} ]]; then
+    cmd="$cmd --extra_tag ${extra_tag}"
+fi
+if [[ -n ${extra_parameter} ]]; then
+    cmd="$cmd --extra_parameter \"${extra_parameter}\""
+fi
+
+echo $cmd
+eval $cmd

egs/mustc/asr/conf/train.yaml

@@ -11,8 +11,8 @@ log-interval: 100
 seed: 1
 report-accuracy: True
 
-#load-params: 
-#load-pretrained-encoder-from: 
+#load-pretrained-encoder-from:
+#load-pretrained-decoder-from:
 
 arch: s2t_transformer_s
 share-decoder-input-output-embed: True
@@ -21,7 +21,7 @@ clip-norm: 10.0
 lr-scheduler: inverse_sqrt
 warmup-init-lr: 1e-7
 warmup-updates: 10000
-lr: 1e-3
+lr: 2e-3
 #adam_betas: (0.9,0.98)
 
 criterion: label_smoothed_cross_entropy
@@ -36,9 +36,3 @@ encoder-ffn-embed-dim: 2048
 encoder-layers: 12
 decoder-layers: 6
 encoder-attention-heads: 4
-
-#decoder-embed-dim: 256
-#decoder-ffn-embed-dim: 2048
-#decoder-attention-heads: 4
-#attention-dropout: 0.1
-#activation-dropout: 0.1

egs/mustc/asr/conf/train_ctc.yaml

@@ -11,8 +11,8 @@ log-interval: 100
 seed: 1
 report-accuracy: True
 
-#load-params: 
-#load-pretrained-encoder-from: 
+#load-pretrained-encoder-from:
+#load-pretrained-decoder-from:
 
 arch: s2t_transformer_s
 share-decoder-input-output-embed: True
@@ -21,7 +21,7 @@ clip-norm: 10.0
 lr-scheduler: inverse_sqrt
 warmup-init-lr: 1e-7
 warmup-updates: 10000
-lr: 1e-3
+lr: 2e-3
 #adam_betas: (0.9,0.98)
 
 ctc-weight: 0.3
@@ -36,10 +36,4 @@ encoder-embed-dim: 256
 encoder-ffn-embed-dim: 2048
 encoder-layers: 12
 decoder-layers: 6
-encoder-attention-heads: 4
-
-#decoder-embed-dim: 256
-#decoder-ffn-embed-dim: 2048
-#decoder-attention-heads: 4
-#attention-dropout: 0.1
-#activation-dropout: 0.1
+encoder-attention-heads: 4
\ No newline at end of file

egs/mustc/asr/decode.sh

@@ -2,7 +2,8 @@
 
 gpu_num=1
 
-test_subset=(tst-COMMON)
+data_dir=
+test_subset=tst-COMMON
 
 exp_name=
 if [ "$#" -eq 1 ]; then
@@ -18,11 +19,17 @@ cmd="./run.sh
     --stop_stage 2
     --gpu_num ${gpu_num}
     --exp_name ${exp_name}
-    --test_subset ${test_subset}
     --n_average ${n_average}
     --beam_size ${beam_size}
     --max_tokens ${max_tokens}
     "
 
+if [[ -n ${data_dir} ]]; then
+    cmd="$cmd --data_dir ${data_dir}"
+fi
+if [[ -n ${test_subset} ]]; then
+    cmd="$cmd --test_subset ${test_subset}"
+fi
+
 echo $cmd
 eval $cmd

egs/mustc/asr/run.sh

@@ -40,7 +40,7 @@ speed_perturb=1
 
 org_data_dir=/media/data/${dataset}
 data_dir=~/st/data/${dataset}/asr
-test_subset=(tst-COMMON)
+test_subset=tst-COMMON
 
 # exp
 extra_tag=
@@ -61,19 +61,12 @@ step_valid=0
 n_average=10
 beam_size=5
 
-. ./local/parse_options.sh || exit 1;
-
-if [[ $step_valid -eq 1 ]]; then
-    validate_interval=10000
-    save_interval=10000
-    no_epoch_checkpoints=1
-    save_interval_updates=5000
-    keep_interval_updates=3
-else
-    validate_interval=1
-    keep_last_epochs=10
+if [[ ${speed_perturb} -eq 1 ]]; then
+    data_dir=${data_dir}_sp
 fi
 
+. ./local/parse_options.sh || exit 1;
+
 # full path
 train_config=$pwd_dir/conf/${train_config}
 if [[ -z ${exp_name} ]]; then
@@ -85,10 +78,6 @@ if [[ -z ${exp_name} ]]; then
         exp_name=sp_${exp_name}
     fi
 fi
-
-if [[ ${speed_perturb} -eq 1 ]]; then
-    data_dir=${data_dir}_sp
-fi
 model_dir=$root_dir/../checkpoints/$dataset/asr/${exp_name}
 
 if [ ${stage} -le -1 ] && [ ${stop_stage} -ge -1 ]; then
@@ -170,6 +159,16 @@ if [ ${stage} -le 1 ] && [ ${stop_stage} -ge 1 ]; then
         cmd="${cmd}
         --fp16"
     fi
+    if [[ $step_valid -eq 1 ]]; then
+        validate_interval=10000
+        save_interval=10000
+        no_epoch_checkpoints=1
+        save_interval_updates=5000
+        keep_interval_updates=3
+    else
+        validate_interval=1
+        keep_last_epochs=10
+    fi
     if [[ -n $no_epoch_checkpoints && $no_epoch_checkpoints -eq 1 ]]; then
         cmd="$cmd
         --no-epoch-checkpoints"
@@ -244,6 +243,7 @@ if [ ${stage} -le 2 ] && [ ${stop_stage} -ge 2 ]; then
 	result_file=${model_dir}/decode_result
 	[[ -f ${result_file} ]] && rm ${result_file}
 
+    test_subset=(${test_subset//,/ })
 	for subset in ${test_subset[@]}; do
         subset=${subset}_asr
   		cmd="python ${root_dir}/fairseq_cli/generate.py

egs/mustc/mt/binary.sh

+set -e
+
+eval=1
+
+root_dir=~/st/Fairseq-S2T
+data_dir=/home/xuchen/st/data/wmt/test
+vocab_dir=/home/xuchen/st/data/wmt/mt/en-de/unigram32000_share
+src_vocab_prefix=spm_unigram32000_share
+tgt_vocab_prefix=spm_unigram32000_share
+
+src_lang=en
+tgt_lang=de
+tokenize=1
+splits=(newstest2014 newstest2016)
+
+for split in ${splits[@]}; do
+    src_file=${data_dir}/${split}.${src_lang}
+    tgt_file=${data_dir}/${split}.${tgt_lang}
+
+    if [[ ${tokenize} -eq 1 ]]; then
+        cmd="tokenizer.perl -l ${src_lang} --threads 8 -no-escape < ${src_file} > ${src_file}.tok"
+        echo -e "\033[34mRun command: \n${cmd} \033[0m"
+        [[ $eval -eq 1 ]] && eval ${cmd}
+
+        cmd="tokenizer.perl -l ${tgt_lang} --threads 8 -no-escape < ${tgt_file} > ${tgt_file}.tok"
+        echo -e "\033[34mRun command: \n${cmd} \033[0m"
+        [[ $eval -eq 1 ]] && eval ${cmd}
+        src_file=${src_file}.tok
+        tgt_file=${tgt_file}.tok
+    fi
+
+    cmd="spm_encode
+    --model ${vocab_dir}/${src_vocab_prefix}.model
+    --output_format=piece
+    < ${src_file}
+    > ${src_file}.spm"
+    echo -e "\033[34mRun command: \n${cmd} \033[0m"
+    [[ $eval -eq 1 ]] && eval ${cmd}
+
+    cmd="spm_encode
+    --model ${vocab_dir}/${tgt_vocab_prefix}.model
+    --output_format=piece
+    < ${tgt_file}
+    > ${tgt_file}.spm"
+    echo -e "\033[34mRun command: \n${cmd} \033[0m"
+    [[ $eval -eq 1 ]] && eval ${cmd}
+
+    src_file=${src_file}.spm
+    tgt_file=${tgt_file}.spm
+
+    mkdir -p ${data_dir}/final
+    cmd="cp ${src_file} ${data_dir}/final/${split}.${src_lang}"
+    echo -e "\033[34mRun command: \n${cmd} \033[0m"
+    [[ $eval -eq 1 ]] && eval ${cmd}
+
+    cmd="cp ${tgt_file} ${data_dir}/final/${split}.${tgt_lang}"
+    echo -e "\033[34mRun command: \n${cmd} \033[0m"
+    [[ $eval -eq 1 ]] && eval ${cmd}
+done
+
+n_set=${#splits[*]}
+for ((i=0;i<$n_set;i++)); do
+    dataset[$i]=${data_dir}/final/${splits[$i]}
+done
+pref=`echo ${dataset[*]} | sed 's/ /,/g'`
+
+cmd="python ${root_dir}/fairseq_cli/preprocess.py
+    --source-lang ${src_lang}
+    --target-lang ${tgt_lang}
+    --testpref ${pref}
+    --destdir ${data_dir}/data-bin
+    --srcdict ${vocab_dir}/${src_vocab_prefix}.txt
+    --tgtdict ${vocab_dir}/${tgt_vocab_prefix}.txt
+    --workers 64"
+
+echo -e "\033[34mRun command: \n${cmd} \033[0m"
+[[ $eval -eq 1 ]] && eval ${cmd}
\ No newline at end of file

egs/mustc/mt/conf/train.yaml

@@ -10,9 +10,9 @@ no-progress-bar: True
 log-interval: 100
 seed: 1
 report-accuracy: True
+skip-invalid-size-inputs-valid-test: True
 
-#load-params: 
-#load-pretrained-encoder-from: 
+#load-pretrained-encoder-from:
 
 arch: transformer
 share-decoder-input-output-embed: True
@@ -29,14 +29,14 @@ label_smoothing: 0.1
 
 dropout: 0.1
 activation-fn: relu
+encoder-normalize-before: True
+decoder-normalize-before: True
 encoder-embed-dim: 256
 encoder-ffn-embed-dim: 2048
 encoder-layers: 6
 decoder-layers: 6
 encoder-attention-heads: 4
 
-#decoder-embed-dim: 256
-#decoder-ffn-embed-dim: 2048
-#decoder-attention-heads: 4
-#attention-dropout: 0.1
-#activation-dropout: 0.1
+decoder-embed-dim: 256
+decoder-ffn-embed-dim: 2048
+decoder-attention-heads: 4

egs/mustc/mt/conf/train_m.yaml

+train-subset: train
+valid-subset: valid
+
+max-epoch: 50
+max-update: 100000
+
+num-workers: 8
+patience: 10
+no-progress-bar: True
+log-interval: 100
+seed: 1
+report-accuracy: True
+skip-invalid-size-inputs-valid-test: True
+
+#load-pretrained-encoder-from:
+
+arch: transformer
+share-decoder-input-output-embed: True
+optimizer: adam
+clip-norm: 10.0
+lr-scheduler: inverse_sqrt
+warmup-init-lr: 1e-7
+warmup-updates: 8000
+lr: 5e-4
+adam_betas: (0.9,0.98)
+
+criterion: label_smoothed_cross_entropy
+label_smoothing: 0.1
+
+dropout: 0.1
+activation-fn: relu
+encoder-normalize-before: True
+decoder-normalize-before: True
+encoder-embed-dim: 512
+encoder-ffn-embed-dim: 2048
+encoder-layers: 6
+decoder-layers: 6
+encoder-attention-heads: 8
+
+decoder-embed-dim: 512
+decoder-ffn-embed-dim: 2048
+decoder-attention-heads: 8

egs/mustc/mt/decode.sh

@@ -2,27 +2,34 @@
 
 gpu_num=1
 
-test_subset=(test)
+data_dir=
+test_subset=test
 
 exp_name=
 if [ "$#" -eq 1 ]; then
     exp_name=$1
 fi
 
-n_average=10
+n_average=5
 beam_size=5
-max_tokens=40000
+max_tokens=20000
 
 cmd="./run.sh
     --stage 2
     --stop_stage 2
     --gpu_num ${gpu_num}
     --exp_name ${exp_name}
-    --test_subset ${test_subset}
     --n_average ${n_average}
     --beam_size ${beam_size}
     --max_tokens ${max_tokens}
     "
 
+if [[ -n ${data_dir} ]]; then
+    cmd="$cmd --data_dir ${data_dir}"
+fi
+if [[ -n ${test_subset} ]]; then
+    cmd="$cmd --test_subset ${test_subset}"
+fi
+
 echo $cmd
 eval $cmd

egs/mustc/mt/run.sh

@@ -20,7 +20,7 @@ stop_stage=0
 
 ######## hardware ########
 # devices
-device=()
+#device=()
 gpu_num=8
 update_freq=1
 
@@ -38,12 +38,17 @@ vocab_type=unigram
 vocab_size=10000
 share_dict=1
 
+use_specific_dict=1
+specific_prefix=st_share10k
+specific_dir=/home/xuchen/st/data/mustc/st/en-de
+src_vocab_prefix=spm_unigram10000_st_share
+tgt_vocab_prefix=spm_unigram10000_st_share
+
 org_data_dir=/media/data/${dataset}
 data_dir=~/st/data/${dataset}/mt/${lang}
-train_prefix=train
-valid_prefix=dev
-test_prefix=tst-COMMON
-test_subset=(test)
+train_subset=train
+valid_subset=dev
+test_subset=test
 
 # exp
 extra_tag=
@@ -64,19 +69,23 @@ bleu_valid=0
 n_average=10
 beam_size=5
 
-. ./local/parse_options.sh || exit 1;
-
-if [[ $step_valid -eq 1 ]]; then
-    validate_interval=10000
-    save_interval=10000
-    no_epoch_checkpoints=1
-    save_interval_updates=5000
-    keep_interval_updates=3
+if [[ ${use_specific_dict} -eq 1 ]]; then
+    exp_tag=${specific_prefix}_${exp_tag}
+    data_dir=${data_dir}/${specific_prefix}
+    mkdir -p ${data_dir}
 else
-    validate_interval=1
-    keep_last_epochs=10
+    data_dir=${data_dir}/${vocab_type}${vocab_size}
+    src_vocab_prefix=spm_${vocab_type}${vocab_size}_${src_lang}
+    tgt_vocab_prefix=spm_${vocab_type}${vocab_size}_${tgt_lang}
+    if [[ $share_dict -eq 1 ]]; then
+        data_dir=${data_dir}_share
+        src_vocab_prefix=spm_${vocab_type}${vocab_size}_share
+        tgt_vocab_prefix=spm_${vocab_type}${vocab_size}_share
+    fi
 fi
 
+. ./local/parse_options.sh || exit 1;
+
 # full path
 train_config=$pwd_dir/conf/${train_config}
 if [[ -z ${exp_name} ]]; then
@@ -85,7 +94,6 @@ if [[ -z ${exp_name} ]]; then
         exp_name=${exp_name}_${extra_tag}
     fi
 fi
-
 model_dir=$root_dir/../checkpoints/$dataset/mt/${exp_name}
 
 if [ ${stage} -le -1 ] && [ ${stop_stage} -ge -1 ]; then
@@ -93,11 +101,6 @@ if [ ${stage} -le -1 ] && [ ${stop_stage} -ge -1 ]; then
     # pass
 fi
 
-data_dir=${data_dir}/${vocab_type}${vocab_size}
-if [[ $share_dict -eq 1 ]]; then
-    data_dir=${data_dir}_share
-fi
-
 if [ ${stage} -le 0 ] && [ ${stop_stage} -ge 0 ]; then
     ### Task dependent. You have to make data the following preparation part by yourself.
     echo "stage 0: MT Data Preparation"
@@ -105,32 +108,34 @@ if [ ${stage} -le 0 ] && [ ${stop_stage} -ge 0 ]; then
         mkdir -p ${data_dir}
     fi
 
-    src_vocab_prefix=spm_${vocab_type}${vocab_size}_${src_lang}
-    tgt_vocab_prefix=spm_${vocab_type}${vocab_size}_${tgt_lang}
-    cmd="python ${root_dir}/examples/speech_to_text/prep_mt_data.py
-        --data-root ${org_data_dir}
-        --output-root ${data_dir}
-        --splits ${train_prefix},${valid_prefix},${test_prefix}
-        --src-lang ${src_lang}
-        --tgt-lang ${tgt_lang}
-        --vocab-type ${vocab_type}
-        --vocab-size ${vocab_size}"
-    if [[ $share_dict -eq 1 ]]; then
-        cmd="$cmd
-        --share"
-        src_vocab_prefix=spm_unigram${vocab_size}_share
-        tgt_vocab_prefix=spm_unigram${vocab_size}_share
+    if [[ ! -f ${data_dir}/${src_vocab_prefix}.txt || ! -f ${data_dir}/${tgt_vocab_prefix}.txt ]]; then
+        if [[ ${use_specific_dict} -eq 0 ]]; then
+            cmd="python ${root_dir}/examples/speech_to_text/prep_mt_data.py
+                --data-root ${org_data_dir}
+                --output-root ${data_dir}
+                --splits ${train_subset},${valid_subset},${test_subset}
+                --src-lang ${src_lang}
+                --tgt-lang ${tgt_lang}
+                --vocab-type ${vocab_type}
+                --vocab-size ${vocab_size}"
+            if [[ $share_dict -eq 1 ]]; then
+                cmd="$cmd
+                --share"
+            fi
+            echo -e "\033[34mRun command: \n${cmd} \033[0m"
+            [[ $eval -eq 1 ]] && eval ${cmd}
+        else
+            cp -r ${specific_dir}/${src_vocab_prefix}.* ${data_dir}
+            cp ${specific_dir}/${tgt_vocab_prefix}.* ${data_dir}
+        fi
     fi
 
-    echo -e "\033[34mRun command: \n${cmd} \033[0m"
-    [[ $eval -eq 1 ]] && eval ${cmd}
-
     mkdir -p ${data_dir}/data
-    for split in ${train_prefix} ${valid_prefix} ${test_prefix}; do
+    for split in ${train_subset} ${valid_subset} ${test_subset}; do
         cmd="spm_encode
         --model ${data_dir}/${src_vocab_prefix}.model
         --output_format=piece
-        < ${org_data_dir}/${lang}/data/${split}/${split}.${src_lang}
+        < ${org_data_dir}/${lang}/data/${split}.${src_lang}
         > ${data_dir}/data/${split}.${src_lang}"
 
         echo -e "\033[34mRun command: \n${cmd} \033[0m"
@@ -139,7 +144,7 @@ if [ ${stage} -le 0 ] && [ ${stop_stage} -ge 0 ]; then
         cmd="spm_encode
         --model ${data_dir}/${tgt_vocab_prefix}.model
         --output_format=piece
-        < ${org_data_dir}/${lang}/data/${split}/${split}.${tgt_lang}
+        < ${org_data_dir}/${lang}/data/${split}.${tgt_lang}
         > ${data_dir}/data/${split}.${tgt_lang}"
 
         echo -e "\033[34mRun command: \n${cmd} \033[0m"
@@ -148,9 +153,9 @@ if [ ${stage} -le 0 ] && [ ${stop_stage} -ge 0 ]; then
 
     cmd="python ${root_dir}/fairseq_cli/preprocess.py
         --source-lang ${src_lang} --target-lang ${tgt_lang}
-        --trainpref ${data_dir}/data/${train_prefix}
-        --validpref ${data_dir}/data/${valid_prefix}
-        --testpref ${data_dir}/data/${test_prefix}
+        --trainpref ${data_dir}/data/${train_subset}
+        --validpref ${data_dir}/data/${valid_subset}
+        --testpref ${data_dir}/data/${test_subset}
         --destdir ${data_dir}/data-bin
         --srcdict ${data_dir}/${src_vocab_prefix}.txt
         --tgtdict ${data_dir}/${tgt_vocab_prefix}.txt
@@ -212,6 +217,16 @@ if [ ${stage} -le 1 ] && [ ${stop_stage} -ge 1 ]; then
         cmd="${cmd}
         --fp16"
     fi
+    if [[ $step_valid -eq 1 ]]; then
+        validate_interval=10000
+        save_interval=10000
+        no_epoch_checkpoints=1
+        save_interval_updates=5000
+        keep_interval_updates=3
+    else
+        validate_interval=1
+        keep_last_epochs=10
+    fi
     if [[ $bleu_valid -eq 1 ]]; then
         cmd="$cmd
         --eval-bleu
@@ -295,6 +310,7 @@ if [ ${stage} -le 2 ] && [ ${stop_stage} -ge 2 ]; then
 	result_file=${model_dir}/decode_result
 	[[ -f ${result_file} ]] && rm ${result_file}
 
+    test_subset=(${test_subset//,/ })
 	for subset in ${test_subset[@]}; do
   		cmd="python ${root_dir}/fairseq_cli/generate.py
         ${data_dir}

egs/mustc/st/conf/train.yaml

@@ -11,8 +11,8 @@ log-interval: 100
 seed: 1
 report-accuracy: True
 
-#load-params: 
-#load-pretrained-encoder-from: 
+#load-pretrained-encoder-from:
+#load-pretrained-decoder-from:
 
 arch: s2t_transformer_s
 share-decoder-input-output-embed: True

egs/mustc/st/conf/train_ctc.yaml

@@ -11,9 +11,8 @@ log-interval: 100
 seed: 1
 report-accuracy: True
 
-# load-params:
-load-pretrained-encoder-from:
-load-pretrained-decoder-from:
+#load-pretrained-encoder-from:
+#load-pretrained-decoder-from:
 
 arch: s2t_transformer_s
 share-decoder-input-output-embed: True

egs/mustc/st/conf/train_ctc_conformer_m.yaml

+train-subset: train_st
+valid-subset: dev_st
+
+max-epoch: 50
+max-update: 100000
+
+num-workers: 8
+patience: 10
+no-progress-bar: True
+log-interval: 100
+seed: 1
+report-accuracy: True
+
+#load-pretrained-encoder-from:
+#load-pretrained-decoder-from:
+
+arch: s2t_conformer_m
+share-decoder-input-output-embed: True
+optimizer: adam
+clip-norm: 10.0
+lr-scheduler: inverse_sqrt
+warmup-init-lr: 1e-7
+warmup-updates: 10000
+lr: 1e-3
+#adam_betas: (0.9,0.98)
+
+ctc-weight: 0.3
+criterion: label_smoothed_cross_entropy_with_ctc
+label_smoothing: 0.1
+
+conv-kernel-sizes: 5,5
+conv-channels: 1024
+#dropout: 0.1
+#activation-fn: relu
+#encoder-embed-dim: 256
+#encoder-ffn-embed-dim: 2048
+#encoder-layers: 12
+#decoder-layers: 6
+#encoder-attention-heads: 4
+
+macaron-style: True
+use-cnn-module: True
+cnn-module-kernel: 31
+
+#decoder-embed-dim: 256
+#decoder-ffn-embed-dim: 2048
+#decoder-attention-heads: 4
+#attention-dropout: 0.1
+#activation-dropout: 0.1

egs/mustc/st/conf/train_ctc_debug.yaml

+train-subset: train_st,train_v2
+valid-subset: dev_st
+
+max-epoch: 50
+max-update: 100000
+
+num-workers: 8
+patience: 10
+no-progress-bar: True
+log-interval: 100
+seed: 1
+report-accuracy: True
+
+#load-pretrained-encoder-from: /home/xuchen/st/checkpoints/mustc/asr/train_ctc_baseline/avg_10_checkpoint.pt
+#load-pretrained-decoder-from: /home/xuchen/st/checkpoints/mustc/mt/train_baseline/avg_10_checkpoint.pt
+
+arch: s2t_transformer_s
+share-decoder-input-output-embed: True
+optimizer: adam
+clip-norm: 10.0
+lr-scheduler: inverse_sqrt
+warmup-init-lr: 1e-7
+warmup-updates: 10000
+lr: 2e-3
+#adam_betas: (0.9,0.98)
+
+ctc-weight: 0.3
+criterion: label_smoothed_cross_entropy_with_ctc
+label_smoothing: 0.1
+
+conv-kernel-sizes: 5,5
+conv-channels: 1024
+dropout: 0.1
+activation-fn: relu
+encoder-embed-dim: 256
+encoder-ffn-embed-dim: 2048
+encoder-layers: 12
+decoder-layers: 6
+encoder-attention-heads: 4
+
+#decoder-embed-dim: 256
+#decoder-ffn-embed-dim: 2048
+#decoder-attention-heads: 4
+#attention-dropout: 0.1
+#activation-dropout: 0.1

egs/mustc/st/conf/train_ctc_enc_rpr.yaml

@@ -11,8 +11,8 @@ log-interval: 100
 seed: 1
 report-accuracy: True
 
-#load-params: 
-#load-pretrained-encoder-from: 
+#load-pretrained-encoder-from:
+#load-pretrained-decoder-from:
 
 arch: s2t_transformer_s
 share-decoder-input-output-embed: True

egs/mustc/st/conf/train_ctc_sate.yaml

+train-subset: train_st
+valid-subset: dev_st
+
+max-epoch: 50
+max-update: 100000
+
+num-workers: 8
+patience: 10
+no-progress-bar: True
+log-interval: 100
+seed: 1
+report-accuracy: True
+
+#load-pretrained-acoustic-encoder-from: /home/xuchen/st/checkpoints/mustc/asr/train_ctc_baseline/avg_10_checkpoint.pt
+#load-pretrained-text-encoder-from: /home/xuchen/st/checkpoints/mustc/mt/st_share10k_train_baseline/avg_10_checkpoint.pt
+#load-pretrained-decoder-from: /home/xuchen/st/checkpoints/mustc/mt/st_share10k_train_baseline/avg_10_checkpoint.pt
+
+arch: s2t_sate
+share-decoder-input-output-embed: True
+optimizer: adam
+clip-norm: 10.0
+lr-scheduler: inverse_sqrt
+warmup-init-lr: 1e-7
+warmup-updates: 10000
+lr: 2e-3
+#adam_betas: (0.9,0.98)
+
+ctc-weight: 0.3
+criterion: label_smoothed_cross_entropy_with_ctc
+label_smoothing: 0.1
+
+encoder-normalize-before: True
+decoder-normalize-before: True
+conv-kernel-sizes: 5,5
+conv-channels: 1024
+dropout: 0.1
+activation-fn: relu
+encoder-embed-dim: 256
+encoder-ffn-embed-dim: 2048
+encoder-layers: 12
+text-encoder-layers: 6
+decoder-layers: 6
+encoder-attention-heads: 4
+
+macaron-style: True
+use-cnn-module: True
+cnn-module-kernel: 31
+
+acoustic-encoder: transformer
+adapter: league
+
+#decoder-embed-dim: 256
+#decoder-ffn-embed-dim: 2048
+#decoder-attention-heads: 4
+#attention-dropout: 0.1
+#activation-dropout: 0.1

egs/mustc/st/conf/train_ctc_sate_conformer.yaml

+train-subset: train_st
+valid-subset: dev_st
+
+max-epoch: 50
+max-update: 100000
+
+num-workers: 8
+patience: 10
+no-progress-bar: True
+log-interval: 100
+seed: 1
+report-accuracy: True
+
+#load-pretrained-acoustic-encoder-from: /home/xuchen/st/checkpoints/mustc/asr/train_ctc_baseline/avg_10_checkpoint.pt
+#load-pretrained-text-encoder-from: /home/xuchen/st/checkpoints/mustc/mt/st_share10k_train_baseline/avg_10_checkpoint.pt
+#load-pretrained-decoder-from: /home/xuchen/st/checkpoints/mustc/mt/st_share10k_train_baseline/avg_10_checkpoint.pt
+
+arch: s2t_sate
+share-decoder-input-output-embed: True
+optimizer: adam
+clip-norm: 10.0
+lr-scheduler: inverse_sqrt
+warmup-init-lr: 1e-7
+warmup-updates: 10000
+lr: 2e-3
+#adam_betas: (0.9,0.98)
+
+ctc-weight: 0.3
+criterion: label_smoothed_cross_entropy_with_ctc
+label_smoothing: 0.1
+
+encoder-normalize-before: True
+decoder-normalize-before: True
+conv-kernel-sizes: 5,5
+conv-channels: 1024
+dropout: 0.1
+activation-fn: relu
+encoder-embed-dim: 256
+encoder-ffn-embed-dim: 2048
+encoder-layers: 12
+text-encoder-layers: 6
+decoder-layers: 6
+encoder-attention-heads: 4
+
+macaron-style: True
+use-cnn-module: True
+cnn-module-kernel: 31
+
+acoustic-encoder: conformer
+adapter: league
+
+#decoder-embed-dim: 256
+#decoder-ffn-embed-dim: 2048
+#decoder-attention-heads: 4
+#attention-dropout: 0.1
+#activation-dropout: 0.1

egs/mustc/st/decode.sh

@@ -2,7 +2,8 @@
 
 gpu_num=1
 
-test_subset=(tst-COMMON)
+data_dir=
+test_subset=tst-COMMON
 
 exp_name=
 if [ "$#" -eq 1 ]; then
@@ -18,11 +19,17 @@ cmd="./run.sh
     --stop_stage 2
     --gpu_num ${gpu_num}
     --exp_name ${exp_name}
-    --test_subset ${test_subset}
     --n_average ${n_average}
     --beam_size ${beam_size}
     --max_tokens ${max_tokens}
     "
 
+if [[ -n ${data_dir} ]]; then
+    cmd="$cmd --data_dir ${data_dir}"
+fi
+if [[ -n ${test_subset} ]]; then
+    cmd="$cmd --test_subset ${test_subset}"
+fi
+
 echo $cmd
 eval $cmd

egs/mustc/st/run.sh

@@ -42,7 +42,7 @@ speed_perturb=1
 
 org_data_dir=/media/data/${dataset}
 data_dir=~/st/data/${dataset}/st
-test_subset=(tst-COMMON)
+test_subset=tst-COMMON
 
 # exp
 extra_tag=
@@ -63,25 +63,18 @@ bleu_valid=0
 n_average=10
 beam_size=5
 
-. ./local/parse_options.sh || exit 1;
-
-if [[ $step_valid -eq 1 ]]; then
-    validate_interval=10000
-    save_interval=10000
-    no_epoch_checkpoints=1
-    save_interval_updates=5000
-    keep_interval_updates=3
-else
-    validate_interval=1
-    keep_last_epochs=10
-fi
-
 if [[ ${share_dict} -eq 1 ]]; then
 	data_config=config_st_share.yaml
 else
 	data_config=config_st.yaml
 fi
 
+if [[ ${speed_perturb} -eq 1 ]]; then
+    data_dir=${data_dir}_sp
+fi
+
+. ./local/parse_options.sh || exit 1;
+
 # full path
 train_config=$pwd_dir/conf/${train_config}
 if [[ -z ${exp_name} ]]; then
@@ -93,10 +86,6 @@ if [[ -z ${exp_name} ]]; then
         exp_name=sp_${exp_name}
     fi
 fi
-
-if [[ ${speed_perturb} -eq 1 ]]; then
-    data_dir=${data_dir}_sp
-fi
 model_dir=$root_dir/../checkpoints/$dataset/st/${exp_name}
 
 if [ ${stage} -le -1 ] && [ ${stop_stage} -ge -1 ]; then
@@ -203,6 +192,16 @@ if [ ${stage} -le 1 ] && [ ${stop_stage} -ge 1 ]; then
         cmd="${cmd}
         --fp16"
     fi
+    if [[ $step_valid -eq 1 ]]; then
+        validate_interval=10000
+        save_interval=10000
+        no_epoch_checkpoints=1
+        save_interval_updates=5000
+        keep_interval_updates=3
+    else
+        validate_interval=1
+        keep_last_epochs=10
+    fi
     if [[ $bleu_valid -eq 1 ]]; then
         cmd="$cmd
         --eval-bleu
@@ -286,6 +285,7 @@ if [ ${stage} -le 2 ] && [ ${stop_stage} -ge 2 ]; then
 	result_file=${model_dir}/decode_result
 	[[ -f ${result_file} ]] && rm ${result_file}
 
+    test_subset=(${test_subset//,/ })
 	for subset in ${test_subset[@]}; do
         subset=${subset}_st
   		cmd="python ${root_dir}/fairseq_cli/generate.py

egs/wmt/mt/binary.sh

+set -e
+
+eval=1
+
+root_dir=~/st/Fairseq-S2T
+data_dir=/home/xuchen/st/data/wmt/test
+vocab_dir=/home/xuchen/st/data/wmt/mt/en-de/unigram32000_share
+src_vocab_prefix=spm_unigram32000_share
+tgt_vocab_prefix=spm_unigram32000_share
+
+src_lang=en
+tgt_lang=de
+tokenize=1
+splits=(newstest2014 newstest2016)
+
+for split in ${splits[@]}; do
+    src_file=${data_dir}/${split}.${src_lang}
+    tgt_file=${data_dir}/${split}.${tgt_lang}
+
+    if [[ ${tokenize} -eq 1 ]]; then
+        cmd="tokenizer.perl -l ${src_lang} --threads 8 -no-escape < ${src_file} > ${src_file}.tok"
+        echo -e "\033[34mRun command: \n${cmd} \033[0m"
+        [[ $eval -eq 1 ]] && eval ${cmd}
+
+        cmd="tokenizer.perl -l ${tgt_lang} --threads 8 -no-escape < ${tgt_file} > ${tgt_file}.tok"
+        echo -e "\033[34mRun command: \n${cmd} \033[0m"
+        [[ $eval -eq 1 ]] && eval ${cmd}
+        src_file=${src_file}.tok
+        tgt_file=${tgt_file}.tok
+    fi
+
+    cmd="spm_encode
+    --model ${vocab_dir}/${src_vocab_prefix}.model
+    --output_format=piece
+    < ${src_file}
+    > ${src_file}.spm"
+    echo -e "\033[34mRun command: \n${cmd} \033[0m"
+    [[ $eval -eq 1 ]] && eval ${cmd}
+
+    cmd="spm_encode
+    --model ${vocab_dir}/${tgt_vocab_prefix}.model
+    --output_format=piece
+    < ${tgt_file}
+    > ${tgt_file}.spm"
+    echo -e "\033[34mRun command: \n${cmd} \033[0m"
+    [[ $eval -eq 1 ]] && eval ${cmd}
+
+    src_file=${src_file}.spm
+    tgt_file=${tgt_file}.spm
+
+    mkdir -p ${data_dir}/final
+    cmd="cp ${src_file} ${data_dir}/final/${split}.${src_lang}"
+    echo -e "\033[34mRun command: \n${cmd} \033[0m"
+    [[ $eval -eq 1 ]] && eval ${cmd}
+
+    cmd="cp ${tgt_file} ${data_dir}/final/${split}.${tgt_lang}"
+    echo -e "\033[34mRun command: \n${cmd} \033[0m"
+    [[ $eval -eq 1 ]] && eval ${cmd}
+done
+
+n_set=${#splits[*]}
+for ((i=0;i<$n_set;i++)); do
+    dataset[$i]=${data_dir}/final/${splits[$i]}
+done
+pref=`echo ${dataset[*]} | sed 's/ /,/g'`
+
+cmd="python ${root_dir}/fairseq_cli/preprocess.py
+    --source-lang ${src_lang}
+    --target-lang ${tgt_lang}
+    --testpref ${pref}
+    --destdir ${data_dir}/data-bin
+    --srcdict ${vocab_dir}/${src_vocab_prefix}.txt
+    --tgtdict ${vocab_dir}/${tgt_vocab_prefix}.txt
+    --workers 64"
+
+echo -e "\033[34mRun command: \n${cmd} \033[0m"
+[[ $eval -eq 1 ]] && eval ${cmd}
\ No newline at end of file

egs/wmt/mt/conf/train.yaml

+train-subset: train
+valid-subset: valid
+
+max-epoch: 20
+max-update: 1000000
+
+num-workers: 8
+patience: 10
+no-progress-bar: True
+log-interval: 100
+seed: 1
+report-accuracy: True
+skip-invalid-size-inputs-valid-test: True
+
+#load-pretrained-encoder-from:
+
+arch: transformer
+share-decoder-input-output-embed: True
+optimizer: adam
+clip-norm: 10.0
+lr-scheduler: inverse_sqrt
+warmup-init-lr: 1e-7
+warmup-updates: 8000
+lr: 5e-4
+adam_betas: (0.9,0.98)
+
+criterion: label_smoothed_cross_entropy
+label_smoothing: 0.1
+
+dropout: 0.1
+activation-fn: relu
+encoder-normalize-before: True
+decoder-normalize-before: True
+encoder-embed-dim: 256
+encoder-ffn-embed-dim: 2048
+encoder-layers: 6
+decoder-layers: 6
+encoder-attention-heads: 4
+
+decoder-embed-dim: 256
+decoder-ffn-embed-dim: 2048
+decoder-attention-heads: 4

egs/wmt/mt/conf/train_m.yaml

+train-subset: train
+valid-subset: valid
+
+max-epoch: 20
+max-update: 1000000
+
+num-workers: 8
+patience: 10
+no-progress-bar: True
+log-interval: 100
+seed: 1
+report-accuracy: True
+skip-invalid-size-inputs-valid-test: True
+
+#load-pretrained-encoder-from:
+
+arch: transformer
+share-decoder-input-output-embed: True
+optimizer: adam
+clip-norm: 10.0
+lr-scheduler: inverse_sqrt
+warmup-init-lr: 1e-7
+warmup-updates: 8000
+lr: 5e-4
+adam_betas: (0.9,0.98)
+
+criterion: label_smoothed_cross_entropy
+label_smoothing: 0.1
+
+dropout: 0.1
+activation-fn: relu
+encoder-normalize-before: True
+decoder-normalize-before: True
+encoder-embed-dim: 512
+encoder-ffn-embed-dim: 2048
+encoder-layers: 6
+decoder-layers: 6
+encoder-attention-heads: 8
+
+decoder-embed-dim: 256
+decoder-ffn-embed-dim: 2048
+decoder-attention-heads: 4

egs/wmt/mt/decode.sh

+#! /bin/bash
+
+gpu_num=1
+
+data_dir=
+test_subset=test,test1
+
+exp_name=
+if [ "$#" -eq 1 ]; then
+    exp_name=$1
+fi
+
+n_average=5
+beam_size=5
+max_tokens=20000
+
+cmd="./run.sh
+    --stage 2
+    --stop_stage 2
+    --gpu_num ${gpu_num}
+    --exp_name ${exp_name}
+    --n_average ${n_average}
+    --beam_size ${beam_size}
+    --max_tokens ${max_tokens}
+    "
+
+if [[ -n ${data_dir} ]]; then
+    cmd="$cmd --data_dir ${data_dir}"
+fi
+if [[ -n ${test_subset} ]]; then
+    cmd="$cmd --test_subset ${test_subset}"
+fi
+
+echo $cmd
+eval $cmd

egs/wmt/mt/local/monitor.sh

+gpu_num=1
+
+while :
+do
+    all_devices=$(seq 0 `gpustat | sed '1,2d' | wc -l`);
+    count=0
+    for dev in ${all_devices[@]}
+    do
+        line=`expr $dev + 2`
+        use=`gpustat -p | head -n $line | tail -1 | cut -d '|' -f4 | wc -w`
+        if [[ $use -eq 0 ]]; then
+            device[$count]=$dev
+            count=`expr $count + 1`
+            if [[ $count -eq $gpu_num ]]; then
+                break
+            fi
+        fi
+    done
+    if [[ ${#device[@]} -lt $gpu_num ]]; then
+        sleep 60s
+    else
+        echo "Run $cmd"
+        eval $cmd
+        sleep 10s
+        exit
+    fi
+done

egs/wmt/mt/local/parse_options.sh

+#!/usr/bin/env bash
+
+# Copyright 2012  Johns Hopkins University (Author: Daniel Povey);
+#                 Arnab Ghoshal, Karel Vesely
+
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#  http://www.apache.org/licenses/LICENSE-2.0
+#
+# THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED
+# WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE,
+# MERCHANTABLITY OR NON-INFRINGEMENT.
+# See the Apache 2 License for the specific language governing permissions and
+# limitations under the License.
+
+
+# Parse command-line options.
+# To be sourced by another script (as in ". parse_options.sh").
+# Option format is: --option-name arg
+# and shell variable "option_name" gets set to value "arg."
+# The exception is --help, which takes no arguments, but prints the
+# $help_message variable (if defined).
+
+
+###
+### The --config file options have lower priority to command line
+### options, so we need to import them first...
+###
+
+# Now import all the configs specified by command-line, in left-to-right order
+for ((argpos=1; argpos<$#; argpos++)); do
+  if [ "${!argpos}" == "--config" ]; then
+    argpos_plus1=$((argpos+1))
+    config=${!argpos_plus1}
+    [ ! -r $config ] && echo "$0: missing config '$config'" && exit 1
+    . $config  # source the config file.
+  fi
+done
+
+
+###
+### Now we process the command line options
+###
+while true; do
+  [ -z "${1:-}" ] && break;  # break if there are no arguments
+  case "$1" in
+    # If the enclosing script is called with --help option, print the help
+    # message and exit.  Scripts should put help messages in $help_message
+    --help|-h) if [ -z "$help_message" ]; then echo "No help found." 1>&2;
+      else printf "$help_message\n" 1>&2 ; fi;
+      exit 0 ;;
+    --*=*) echo "$0: options to scripts must be of the form --name value, got '$1'"
+      exit 1 ;;
+    # If the first command-line argument begins with "--" (e.g. --foo-bar),
+    # then work out the variable name as $name, which will equal "foo_bar".
+    --*) name=`echo "$1" | sed s/^--// | sed s/-/_/g`;
+      # Next we test whether the variable in question is undefned-- if so it's
+      # an invalid option and we die.  Note: $0 evaluates to the name of the
+      # enclosing script.
+      # The test [ -z ${foo_bar+xxx} ] will return true if the variable foo_bar
+      # is undefined.  We then have to wrap this test inside "eval" because
+      # foo_bar is itself inside a variable ($name).
+      eval '[ -z "${'$name'+xxx}" ]' && echo "$0: invalid option $1" 1>&2 && exit 1;
+
+      oldval="`eval echo \\$$name`";
+      # Work out whether we seem to be expecting a Boolean argument.
+      if [ "$oldval" == "true" ] || [ "$oldval" == "false" ]; then
+        was_bool=true;
+      else
+        was_bool=false;
+      fi
+
+      # Set the variable to the right value-- the escaped quotes make it work if
+      # the option had spaces, like --cmd "queue.pl -sync y"
+      eval $name=\"$2\";
+
+      # Check that Boolean-valued arguments are really Boolean.
+      if $was_bool && [[ "$2" != "true" && "$2" != "false" ]]; then
+        echo "$0: expected \"true\" or \"false\": $1 $2" 1>&2
+        exit 1;
+      fi
+      shift 2;
+      ;;
+  *) break;
+  esac
+done
+
+
+# Check for an empty argument to the --cmd option, which can easily occur as a
+# result of scripting errors.
+[ ! -z "${cmd+xxx}" ] && [ -z "$cmd" ] && echo "$0: empty argument to --cmd option" 1>&2 && exit 1;
+
+
+true; # so this script returns exit code 0.

egs/wmt/mt/local/path.sh

+MAIN_ROOT=$PWD/../../..
+KALDI_ROOT=$MAIN_ROOT/tools/kaldi
+
+export PATH=$PWD/utils/:$KALDI_ROOT/tools/openfst/bin:$PATH
+[ ! -f $KALDI_ROOT/tools/config/common_path.sh ] && echo >&2 "The standard file $KALDI_ROOT/tools/config/common_path.sh is not present -> Exit!" && exit 1
+. $KALDI_ROOT/tools/config/common_path.sh
+export LC_ALL=C
+
+export LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:$MAIN_ROOT/src/lib
+export LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:$MAIN_ROOT/tools/chainer_ctc/ext/warp-ctc/build
+. "${MAIN_ROOT}"/tools/activate_python.sh && . "${MAIN_ROOT}"/tools/extra_path.sh
+export PATH=$MAIN_ROOT/utils:$MAIN_ROOT/espnet/bin:$PATH
+
+export OMP_NUM_THREADS=1
+
+# check extra module installation
+if ! which tokenizer.perl > /dev/null; then
+    echo "Error: it seems that moses is not installed." >&2
+    echo "Error: please install moses as follows." >&2
+    echo "Error: cd ${MAIN_ROOT}/tools && make moses.done" >&2
+    return 1
+fi
+
+# NOTE(kan-bayashi): Use UTF-8 in Python to avoid UnicodeDecodeError when LC_ALL=C
+export PYTHONIOENCODING=UTF-8

egs/wmt/mt/local/utils.sh

+
+get_devices(){
+    gpu_num=$1
+    use_cpu=$2
+    device=()
+    while :
+    do
+        record=`mktemp -t temp.record.XXXXXX`
+        gpustat > $record
+        all_devices=$(seq 0 `cat $record | sed '1,2d' | wc -l`);
+        count=0
+        for dev in ${all_devices[@]}
+        do
+            line=`expr $dev + 2`
+            use=`cat $record | head -n $line | tail -1 | cut -d '|' -f3 | cut -d '/' -f1`
+            if [[ $use -lt 100 ]]; then
+                device[$count]=$dev
+                count=`expr $count + 1`
+                if [[ $count -eq $gpu_num ]]; then
+                    break
+                fi
+            fi
+        done
+        if [[ ${#device[@]} -lt $gpu_num ]]; then
+            if [[ $use_cpu -eq 1 ]]; then
+                device=(-1)
+            else
+                sleep 60s
+            fi
+        else
+            break
+        fi
+    done
+
+    echo ${device[*]} | sed 's/ /,/g'
+    return $?
+}
+
+

egs/wmt/mt/run.sh

+#! /bin/bash
+
+# Processing MuST-C Datasets
+
+# Copyright 2021 Natural Language Processing Laboratory 
+# Xu Chen (xuchenneu@163.com)
+
+# Set bash to 'debug' mode, it will exit on :
+# -e 'error', -u 'undefined variable', -o ... 'error in pipeline', -x 'print commands',
+set -e
+#set -u
+set -o pipefail
+export PYTHONIOENCODING=UTF-8
+
+eval=1
+time=$(date "+%m%d_%H%M")
+
+stage=0
+stop_stage=0
+
+######## hardware ########
+# devices
+#device=()
+gpu_num=8
+update_freq=1
+
+root_dir=~/st/Fairseq-S2T
+pwd_dir=$PWD
+
+# dataset
+src_lang=en
+tgt_lang=de
+lang=${src_lang}-${tgt_lang}
+
+dataset=wmt
+task=translation
+vocab_type=unigram
+vocab_size=32000
+share_dict=1
+
+use_specific_dict=1
+specific_prefix=st_share10k
+specific_dir=/home/xuchen/st/data/mustc/st/en-de
+src_vocab_prefix=spm_unigram10000_st_share
+tgt_vocab_prefix=spm_unigram10000_st_share
+
+org_data_dir=~/st/data/${dataset}
+data_dir=~/st/data/${dataset}/mt/${lang}
+train_subset=train
+valid_subset=dev
+test_subset=test
+
+# exp
+extra_tag=
+extra_parameter=
+exp_tag=baseline
+exp_name=
+
+# config
+train_config=train.yaml
+
+# training setting
+fp16=1
+max_tokens=4096
+step_valid=0
+bleu_valid=0
+
+# decoding setting
+n_average=10
+beam_size=5
+
+if [[ ${use_specific_dict} -eq 1 ]]; then
+    exp_tag=${specific_prefix}_${exp_tag}
+    data_dir=${data_dir}/${specific_prefix}
+    mkdir -p ${data_dir}
+else
+    data_dir=${data_dir}/${vocab_type}${vocab_size}
+    src_vocab_prefix=spm_${vocab_type}${vocab_size}_${src_lang}
+    tgt_vocab_prefix=spm_${vocab_type}${vocab_size}_${tgt_lang}
+    if [[ $share_dict -eq 1 ]]; then
+        data_dir=${data_dir}_share
+        src_vocab_prefix=spm_${vocab_type}${vocab_size}_share
+        tgt_vocab_prefix=spm_${vocab_type}${vocab_size}_share
+    fi
+fi
+
+. ./local/parse_options.sh || exit 1;
+
+# full path
+train_config=$pwd_dir/conf/${train_config}
+if [[ -z ${exp_name} ]]; then
+    exp_name=$(basename ${train_config%.*})_${exp_tag}
+    if [[ -n ${extra_tag} ]]; then
+        exp_name=${exp_name}_${extra_tag}
+    fi
+fi
+model_dir=$root_dir/../checkpoints/$dataset/mt/${exp_name}
+
+if [ ${stage} -le -1 ] && [ ${stop_stage} -ge -1 ]; then
+    echo "stage -1: Data Download"
+    # pass
+fi
+
+if [ ${stage} -le 0 ] && [ ${stop_stage} -ge 0 ]; then
+    ### Task dependent. You have to make data the following preparation part by yourself.
+    echo "stage 0: MT Data Preparation"
+    if [[ ! -e ${data_dir} ]]; then
+        mkdir -p ${data_dir}
+    fi
+
+    if [[ ! -f ${data_dir}/${src_vocab_prefix}.txt || ! -f ${data_dir}/${tgt_vocab_prefix}.txt ]]; then
+        if [[ ${use_specific_dict} -eq 0 ]]; then
+            cmd="python ${root_dir}/examples/speech_to_text/prep_mt_data.py
+                --data-root ${org_data_dir}
+                --output-root ${data_dir}
+                --splits ${train_subset},${valid_subset},${test_subset}
+                --src-lang ${src_lang}
+                --tgt-lang ${tgt_lang}
+                --vocab-type ${vocab_type}
+                --vocab-size ${vocab_size}"
+            if [[ $share_dict -eq 1 ]]; then
+                cmd="$cmd
+                --share"
+            fi
+            echo -e "\033[34mRun command: \n${cmd} \033[0m"
+            [[ $eval -eq 1 ]] && eval ${cmd}
+        else
+            cp -r ${specific_dir}/${src_vocab_prefix}.* ${data_dir}
+            cp ${specific_dir}/${tgt_vocab_prefix}.* ${data_dir}
+        fi
+    fi
+
+    mkdir -p ${data_dir}/data
+    for split in ${train_subset} ${valid_subset} ${test_subset}; do
+        cmd="spm_encode
+        --model ${data_dir}/${src_vocab_prefix}.model
+        --output_format=piece
+        < ${org_data_dir}/${lang}/data/${split}.${src_lang}
+        > ${data_dir}/data/${split}.${src_lang}"
+
+        echo -e "\033[34mRun command: \n${cmd} \033[0m"
+        [[ $eval -eq 1 ]] && eval ${cmd}
+
+        cmd="spm_encode
+        --model ${data_dir}/${tgt_vocab_prefix}.model
+        --output_format=piece
+        < ${org_data_dir}/${lang}/data/${split}.${tgt_lang}
+        > ${data_dir}/data/${split}.${tgt_lang}"
+
+        echo -e "\033[34mRun command: \n${cmd} \033[0m"
+        [[ $eval -eq 1 ]] && eval ${cmd}
+    done
+
+    cmd="python ${root_dir}/fairseq_cli/preprocess.py
+        --source-lang ${src_lang} --target-lang ${tgt_lang}
+        --trainpref ${data_dir}/data/${train_subset}
+        --validpref ${data_dir}/data/${valid_subset}
+        --testpref ${data_dir}/data/${test_subset}
+        --destdir ${data_dir}/data-bin
+        --srcdict ${data_dir}/${src_vocab_prefix}.txt
+        --tgtdict ${data_dir}/${tgt_vocab_prefix}.txt
+        --workers 64"
+
+    echo -e "\033[34mRun command: \n${cmd} \033[0m"
+    [[ $eval -eq 1 ]] && eval ${cmd}
+fi
+
+data_dir=${data_dir}/data-bin
+
+if [ ${stage} -le 1 ] && [ ${stop_stage} -ge 1 ]; then
+    echo "stage 1: MT Network Training"
+    [[ ! -d ${data_dir} ]] && echo "The data dir ${data_dir} is not existing!" && exit 1;
+
+    if [[ -z ${device} || ${#device[@]} -eq 0 ]]; then
+		if [[ ${gpu_num} -eq 0 ]]; then
+			device=()
+		else
+        	source ./local/utils.sh
+        	device=$(get_devices $gpu_num 0)
+		fi
+    fi
+
+    echo -e "dev=${device} data=${data_dir} model=${model_dir}"
+
+    if [[ ! -d ${model_dir} ]]; then
+        mkdir -p ${model_dir}
+    else
+        echo "${model_dir} exists."
+    fi
+
+    cp ${BASH_SOURCE[0]} ${model_dir}
+    cp ${PWD}/train.sh ${model_dir}
+    cp ${train_config} ${model_dir}
+
+    cmd="python3 -u ${root_dir}/fairseq_cli/train.py
+        ${data_dir}
+        --source-lang ${src_lang}
+        --target-lang ${tgt_lang}
+        --train-config ${train_config}
+        --task ${task}
+        --max-tokens ${max_tokens}
+        --update-freq ${update_freq}
+        --log-interval 100
+        --save-dir ${model_dir}
+        --tensorboard-logdir ${model_dir}"
+
+    if [[ -n ${extra_parameter} ]]; then
+        cmd="${cmd}
+        ${extra_parameter}"
+    fi
+	if [[ ${gpu_num} -gt 0 ]]; then
+		cmd="${cmd}
+        --distributed-world-size $gpu_num
+        --ddp-backend no_c10d"
+	fi
+    if [[ $fp16 -eq 1 ]]; then
+        cmd="${cmd}
+        --fp16"
+    fi
+    if [[ $step_valid -eq 1 ]]; then
+        validate_interval=10000
+        save_interval=10000
+        no_epoch_checkpoints=1
+        save_interval_updates=5000
+        keep_interval_updates=3
+    else
+        validate_interval=1
+        keep_last_epochs=10
+    fi
+    if [[ $bleu_valid -eq 1 ]]; then
+        cmd="$cmd
+        --eval-bleu
+        --eval-bleu-args '{\"beam\": 1}'
+        --eval-tokenized-bleu
+        --eval-bleu-remove-bpe
+        --best-checkpoint-metric bleu
+        --maximize-best-checkpoint-metric"
+    fi
+    if [[ -n $no_epoch_checkpoints && $no_epoch_checkpoints -eq 1 ]]; then
+        cmd="$cmd
+        --no-epoch-checkpoints"
+    fi
+    if [[ -n $validate_interval ]]; then
+        cmd="${cmd}
+        --validate-interval $validate_interval "
+    fi
+    if [[ -n $save_interval ]]; then
+        cmd="${cmd}
+        --save-interval $save_interval "
+    fi
+    if [[ -n $keep_last_epochs ]]; then
+        cmd="${cmd}
+        --keep-last-epochs $keep_last_epochs "
+    fi
+    if [[ -n $save_interval_updates ]]; then
+        cmd="${cmd}
+        --save-interval-updates $save_interval_updates"
+        if [[ -n $keep_interval_updates ]]; then
+        cmd="${cmd}
+        --keep-interval-updates $keep_interval_updates"
+        fi
+    fi
+
+    echo -e "\033[34mRun command: \n${cmd} \033[0m"
+
+    # save info
+    log=./history.log
+    echo "${time} | ${device} | ${data_dir} | ${model_dir} " >> $log
+    cat $log | tail -n 50 > tmp.log
+    mv tmp.log $log
+    export CUDA_VISIBLE_DEVICES=${device}
+
+    cmd="nohup ${cmd} >> ${model_dir}/train.log 2>&1 &"
+    if [[ $eval -eq 1 ]]; then
+		eval $cmd
+		sleep 2s
+		tail -n `wc -l ${model_dir}/train.log | awk '{print $1+1}'` -f ${model_dir}/train.log
+	fi
+fi
+wait
+
+if [ ${stage} -le 2 ] && [ ${stop_stage} -ge 2 ]; then
+    echo "stage 2: MT Decoding"
+    if [[ ${n_average} -ne 1 ]]; then
+        # Average models
+		dec_model=avg_${n_average}_checkpoint.pt
+
+		cmd="python ${root_dir}/scripts/average_checkpoints.py
+        --inputs ${model_dir}
+        --num-epoch-checkpoints ${n_average}
+        --output ${model_dir}/${dec_model}"
+    	echo -e "\033[34mRun command: \n${cmd} \033[0m"
+    	[[ $eval -eq 1 ]] && eval $cmd
+	else
+		dec_model=checkpoint_best.pt
+	fi
+
+    if [[ -z ${device} || ${#device[@]} -eq 0 ]]; then
+		if [[ ${gpu_num} -eq 0 ]]; then
+			device=()
+		else
+        	source ./local/utils.sh
+        	device=$(get_devices $gpu_num 0)
+		fi
+    fi
+    export CUDA_VISIBLE_DEVICES=${device}
+
+	#tmp_file=$(mktemp ${model_dir}/tmp-XXXXX)
+	#trap 'rm -rf ${tmp_file}' EXIT
+	result_file=${model_dir}/decode_result
+	[[ -f ${result_file} ]] && rm ${result_file}
+
+    test_subset=(${test_subset//,/ })
+	for subset in ${test_subset[@]}; do
+  		cmd="python ${root_dir}/fairseq_cli/generate.py
+        ${data_dir}
+        --source-lang ${src_lang}
+        --target-lang ${tgt_lang}
+        --gen-subset ${subset}
+        --task ${task}
+        --path ${model_dir}/${dec_model}
+        --results-path ${model_dir}
+        --max-tokens ${max_tokens}
+        --beam ${beam_size}
+        --scoring sacrebleu"
+    	echo -e "\033[34mRun command: \n${cmd} \033[0m"
+
+        if [[ $eval -eq 1 ]]; then
+    	    eval $cmd
+    	    tail -n 1 ${model_dir}/generate-${subset}.txt >> ${result_file}
+        fi
+	done
+    cat ${result_file}
+fi

egs/wmt/mt/train.sh

+#! /bin/bash
+
+# training the model
+
+gpu_num=7
+update_freq=1
+max_tokens=4096
+
+extra_tag=
+extra_parameter=
+
+#extra_tag="${extra_tag}"
+#extra_parameter="${extra_parameter} "
+
+exp_tag=baseline
+train_config=train.yaml
+
+cmd="./run.sh
+    --stage 1
+    --stop_stage 1
+    --gpu_num ${gpu_num}
+    --update_freq ${update_freq}
+    --train_config ${train_config}
+    --max_tokens ${max_tokens}
+    "
+
+if [[ -n ${exp_tag} ]]; then
+    cmd="$cmd --exp_tag ${exp_tag}"
+fi
+if [[ -n ${extra_tag} ]]; then
+    cmd="$cmd --extra_tag ${extra_tag}"
+fi
+if [[ -n ${extra_parameter} ]]; then
+    cmd="$cmd --extra_parameter \"${extra_parameter}\""
+fi
+
+echo $cmd
+eval $cmd

examples/speech_to_text/data_utils.py

@@ -34,7 +34,7 @@ def gen_vocab(
         f"--model_prefix={output_path_prefix.as_posix()}",
         f"--model_type={model_type}",
         f"--vocab_size={vocab_size}",
-        "--character_coverage=1.0",
+        "--character_coverage=0.9995",
         f"--num_threads={cpu_count()}",
         f"--unk_id={UNK_TOKEN_ID}",
         f"--bos_id={BOS_TOKEN_ID}",