Support the raw audio and online speed perturbation!

I have wanted to do this for a long time!

egs/libri_trans/asr/binary.sh

+set -e
+
+eval=1
+
+lcrm=0
+tokenizer=0
+
+root_dir=~/st/Fairseq-S2T
+data_dir=~/st/data/test
+vocab_dir=~/st/data/mustc/st/en-de
+asr_vocab_prefix=spm_unigram10000_st_share
+
+src_lang=en
+tgt_lang=de
+subsets=(2019)
+
+cp -r ${vocab_dir}/${asr_vocab_prefix}.* ${data_dir}/${src_lang}-${tgt_lang}
+rm -rf ${data_dir}/${src_lang}-${tgt_lang}/fbank80.zip
+
+splits=$(echo ${subsets[*]} | sed 's/ /,/g')
+cmd="python ${root_dir}/examples/speech_to_text/prep_st_data.py
+    --data-root ${data_dir}
+    --output-root ${data_dir}
+    --splits ${splits}
+    --task asr
+    --src-lang ${src_lang}
+    --tgt-lang ${tgt_lang}
+    --add-src
+    --share
+    --asr-prefix ${asr_vocab_prefix}
+    --cmvn-type utterance"
+
+    if [[ ${lcrm} -eq 1 ]]; then
+        cmd="$cmd
+    --lowercase-src
+    --rm-punc-src"
+    fi
+    if [[ ${tokenizer} -eq 1 ]]; then
+        cmd="$cmd
+    --tokenizer"
+    fi
+
+echo -e "\033[34mRun command: \n${cmd} \033[0m"
+[[ $eval -eq 1 ]] && eval ${cmd}

egs/libri_trans/asr/conf/base.yaml

+train-subset: train
+valid-subset: dev
+
+max-epoch: 100
+max-update: 100000
+
+num-workers: 0
+patience: 10
+no-progress-bar: True
+log-interval: 100
+seed: 1
+report-accuracy: True
+
+#load-pretrained-encoder-from:
+#load-pretrained-decoder-from:
+
+arch: s2t_transformer_s
+share-decoder-input-output-embed: True
+optimizer: adam
+clip-norm: 10.0
+lr-scheduler: inverse_sqrt
+warmup-init-lr: 1e-7
+warmup-updates: 10000
+lr: 2e-3
+#adam_betas: (0.9,0.98)
+
+criterion: label_smoothed_cross_entropy_with_ctc
+label_smoothing: 0.1
+
+conv-kernel-sizes: 5,5
+conv-channels: 1024
+dropout: 0.1
+activation-fn: relu
+encoder-embed-dim: 256
+encoder-ffn-embed-dim: 2048
+encoder-layers: 12
+decoder-layers: 6
+encoder-attention-heads: 4
+
+decoder-embed-dim: 256
+decoder-ffn-embed-dim: 2048
+decoder-attention-heads: 4
+attention-dropout: 0.1
+activation-dropout: 0.1

egs/libri_trans/asr/conf/conformer.yaml

+macaron-style: True
+use-cnn-module: True
+cnn-module-kernel: 31

egs/libri_trans/asr/conf/ctc.yaml

+ctc-weight: 0.3

egs/libri_trans/asr/conf/dlcl.yaml

+use-enc-dlcl: True
+use-dec-dlcl: True

egs/libri_trans/asr/conf/local_attn.yaml

+encoder-attention-type: local
+hard-mask-window: 0
+gauss-mask-sigma: 3
+init-mask-weight: 0
\ No newline at end of file

egs/libri_trans/asr/conf/pds_base.yaml

+arch: pdss2t_transformer_s_8
+
+train-subset: train_asr
+valid-subset: dev_asr
+
+max-epoch: 100
+max-update: 100000
+
+num-workers: 8
+patience: 10
+no-progress-bar: True
+log-interval: 100
+seed: 1
+report-accuracy: True
+
+
+#load-pretrained-encoder-from:
+#load-pretrained-decoder-from:
+
+share-decoder-input-output-embed: True
+optimizer: adam
+clip-norm: 10.0
+lr-scheduler: inverse_sqrt
+warmup-init-lr: 1e-7
+warmup-updates: 10000
+lr: 2e-3
+#adam_betas: (0.9,0.98)
+
+criterion: label_smoothed_cross_entropy_with_ctc
+label_smoothing: 0.1
+
+dropout: 0.1
+activation-fn: relu
+encoder-ffn-embed-dim: 2048
+encoder-layers: 12
+decoder-layers: 6
+encoder-attention-heads: 4
+
+decoder-embed-dim: 256
+decoder-ffn-embed-dim: 2048
+decoder-attention-heads: 4

egs/libri_trans/asr/conf/pds_base_16.yaml

+arch: pdss2t_transformer_s_16
+
+encoder-embed-dim: 256
+pyramid-stages: 4
+#pyramid-dropout: 0
+pyramid-layers: 2_2_6_2
+pyramid-ratios: 2_2_2_2
+pyramid-fusion: True
+pyramid-fusion-method: all_conv
+pyramid-embed-dims: 256_256_256_256
+pyramid-ds-method: conv
+pyramid-embed-norm: True
+pyramid-position-embed: 1_1_1_1
+pyramid-kernel-sizes: 5_5_5_5
+pyramid-ffn-ratios: 8_8_8_8
+pyramid-attn-heads: 4_4_4_4
+
+train-subset: train_asr
+valid-subset: dev_asr
+
+max-epoch: 100
+max-update: 100000
+
+num-workers: 8
+patience: 10
+no-progress-bar: True
+log-interval: 100
+seed: 1
+report-accuracy: True
+
+
+#load-pretrained-encoder-from:
+#load-pretrained-decoder-from:
+
+share-decoder-input-output-embed: True
+optimizer: adam
+clip-norm: 10.0
+lr-scheduler: inverse_sqrt
+warmup-init-lr: 1e-7
+warmup-updates: 10000
+lr: 2e-3
+#adam_betas: (0.9,0.98)
+
+criterion: label_smoothed_cross_entropy_with_ctc
+label_smoothing: 0.1
+
+dropout: 0.1
+activation-fn: relu
+encoder-ffn-embed-dim: 2048
+encoder-layers: 12
+decoder-layers: 6
+encoder-attention-heads: 4
+
+decoder-embed-dim: 256
+decoder-ffn-embed-dim: 2048
+decoder-attention-heads: 4

egs/libri_trans/asr/conf/pds_base_32.yaml

+arch: pdss2t_transformer_s_32
+
+encoder-embed-dim: 256
+pyramid-stages: 5
+#pyramid-dropout: 0
+pyramid-layers: 2_2_3_3_2
+pyramid-ratios: 2_2_2_2_2
+pyramid-fusion: True
+pyramid-fusion-method: all_conv
+pyramid-embed-dims: 256_256_256_256_256
+pyramid-ds-method: conv
+pyramid-embed-norm: True
+pyramid-position-embed: 1_1_1_1_1
+pyramid-kernel-sizes: 5_5_5_5_5
+pyramid-ffn-ratios: 8_8_8_8_8
+pyramid-attn-heads: 4_4_4_4_4
+
+train-subset: train_asr
+valid-subset: dev_asr
+
+max-epoch: 100
+max-update: 100000
+
+num-workers: 8
+patience: 10
+no-progress-bar: True
+log-interval: 100
+seed: 1
+report-accuracy: True
+
+
+#load-pretrained-encoder-from:
+#load-pretrained-decoder-from:
+
+share-decoder-input-output-embed: True
+optimizer: adam
+clip-norm: 10.0
+lr-scheduler: inverse_sqrt
+warmup-init-lr: 1e-7
+warmup-updates: 10000
+lr: 2e-3
+#adam_betas: (0.9,0.98)
+
+criterion: label_smoothed_cross_entropy_with_ctc
+label_smoothing: 0.1
+
+dropout: 0.1
+activation-fn: relu
+encoder-ffn-embed-dim: 2048
+encoder-layers: 12
+decoder-layers: 6
+encoder-attention-heads: 4
+
+decoder-embed-dim: 256
+decoder-ffn-embed-dim: 2048
+decoder-attention-heads: 4

egs/libri_trans/asr/conf/pds_base_8.yaml

+arch: pdss2t_transformer_s_8
+
+encoder-embed-dim: 256
+pyramid-stages: 4
+#pyramid-dropout: 0
+pyramid-layers: 3_3_3_3
+pyramid-ratios: 2_2_1_2
+pyramid-fusion: True
+pyramid-fusion-method: all_conv
+pyramid-embed-dims: 256_256_256_256
+pyramid-ds-method: conv
+pyramid-embed-norm: True
+pyramid-position-embed: 1_1_1_1
+pyramid-kernel-sizes: 5_5_5_5
+pyramid-ffn-ratios: 8_8_8_8
+pyramid-attn-heads: 4_4_4_4
+
+train-subset: train_asr
+valid-subset: dev_asr
+
+max-epoch: 100
+max-update: 100000
+
+num-workers: 8
+patience: 10
+no-progress-bar: True
+log-interval: 100
+seed: 1
+report-accuracy: True
+
+
+#load-pretrained-encoder-from:
+#load-pretrained-decoder-from:
+
+share-decoder-input-output-embed: True
+optimizer: adam
+clip-norm: 10.0
+lr-scheduler: inverse_sqrt
+warmup-init-lr: 1e-7
+warmup-updates: 10000
+lr: 2e-3
+#adam_betas: (0.9,0.98)
+
+criterion: label_smoothed_cross_entropy_with_ctc
+label_smoothing: 0.1
+
+dropout: 0.1
+activation-fn: relu
+encoder-ffn-embed-dim: 2048
+encoder-layers: 12
+decoder-layers: 6
+encoder-attention-heads: 4
+
+decoder-embed-dim: 256
+decoder-ffn-embed-dim: 2048
+decoder-attention-heads: 4

egs/libri_trans/asr/conf/rpr.yaml

+encoder-attention-type: rel_selfattn
+#encoder-attention-type: relative
+#max-encoder-relative-length: 100

egs/libri_trans/asr/decode.sh

+#! /bin/bash
+
+gpu_num=1
+
+data_dir=
+test_subset=(test)
+
+exp_name=
+if [ "$#" -eq 1 ]; then
+    exp_name=$1
+fi
+
+n_average=10
+beam_size=5
+len_penalty=1.0
+max_tokens=80000
+dec_model=checkpoint_best.pt
+
+cmd="./run.sh
+    --stage 2
+    --stop_stage 2
+    --gpu_num ${gpu_num}
+    --exp_name ${exp_name}
+    --n_average ${n_average}
+    --beam_size ${beam_size}
+    --len_penalty ${len_penalty}
+    --max_tokens ${max_tokens}
+    --dec_model ${dec_model}
+    "
+
+if [[ -n ${data_dir} ]]; then
+    cmd="$cmd --data_dir ${data_dir}"
+fi
+if [[ ${#test_subset[@]} -ne 0 ]]; then
+    subsets=$(echo ${test_subset[*]} | sed 's/ /,/g')
+    cmd="$cmd --test_subset ${subsets}"
+fi
+
+echo $cmd
+eval $cmd

egs/libri_trans/asr/local/monitor.sh

+gpu_num=4
+cmd="sh train.sh"
+
+while :
+do
+    record=$(mktemp -t temp.record.XXXXXX)
+    gpustat > $record
+    all_devices=$(seq 0 "$(sed '1,2d' ${record} | wc -l)");
+
+    count=0
+    for dev in ${all_devices[@]}
+    do
+        line=$((dev + 2))
+        use=$(head -n $line ${record} | tail -1 | cut -d '|' -f3 | cut -d '/' -f1)
+
+        if [[ $use -lt 100 ]]; then
+            device[$count]=$dev
+            count=$((count + 1))
+            if [[ $count -eq $gpu_num ]]; then
+                break
+            fi
+        fi
+    done
+    if [[ ${#device[@]} -lt $gpu_num ]]; then
+        sleep 60s
+    else
+        echo "Run $cmd"
+        eval $cmd
+        sleep 10s
+        exit
+    fi
+done

egs/libri_trans/asr/local/parse_options.sh

+#!/usr/bin/env bash
+
+# Copyright 2012  Johns Hopkins University (Author: Daniel Povey);
+#                 Arnab Ghoshal, Karel Vesely
+
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#  http://www.apache.org/licenses/LICENSE-2.0
+#
+# THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED
+# WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE,
+# MERCHANTABLITY OR NON-INFRINGEMENT.
+# See the Apache 2 License for the specific language governing permissions and
+# limitations under the License.
+
+
+# Parse command-line options.
+# To be sourced by another script (as in ". parse_options.sh").
+# Option format is: --option-name arg
+# and shell variable "option_name" gets set to value "arg."
+# The exception is --help, which takes no arguments, but prints the
+# $help_message variable (if defined).
+
+
+###
+### The --config file options have lower priority to command line
+### options, so we need to import them first...
+###
+
+# Now import all the configs specified by command-line, in left-to-right order
+for ((argpos=1; argpos<$#; argpos++)); do
+  if [ "${!argpos}" == "--config" ]; then
+    argpos_plus1=$((argpos+1))
+    config=${!argpos_plus1}
+    [ ! -r $config ] && echo "$0: missing config '$config'" && exit 1
+    . $config  # source the config file.
+  fi
+done
+
+
+###
+### Now we process the command line options
+###
+while true; do
+  [ -z "${1:-}" ] && break;  # break if there are no arguments
+  case "$1" in
+    # If the enclosing script is called with --help option, print the help
+    # message and exit.  Scripts should put help messages in $help_message
+    --help|-h) if [ -z "$help_message" ]; then echo "No help found." 1>&2;
+      else printf "$help_message\n" 1>&2 ; fi;
+      exit 0 ;;
+    --*=*) echo "$0: options to scripts must be of the form --name value, got '$1'"
+      exit 1 ;;
+    # If the first command-line argument begins with "--" (e.g. --foo-bar),
+    # then work out the variable name as $name, which will equal "foo_bar".
+    --*) name=`echo "$1" | sed s/^--// | sed s/-/_/g`;
+      # Next we test whether the variable in question is undefned-- if so it's
+      # an invalid option and we die.  Note: $0 evaluates to the name of the
+      # enclosing script.
+      # The test [ -z ${foo_bar+xxx} ] will return true if the variable foo_bar
+      # is undefined.  We then have to wrap this test inside "eval" because
+      # foo_bar is itself inside a variable ($name).
+      eval '[ -z "${'$name'+xxx}" ]' && echo "$0: invalid option $1" 1>&2 && exit 1;
+
+      oldval="`eval echo \\$$name`";
+      # Work out whether we seem to be expecting a Boolean argument.
+      if [ "$oldval" == "true" ] || [ "$oldval" == "false" ]; then
+        was_bool=true;
+      else
+        was_bool=false;
+      fi
+
+      # Set the variable to the right value-- the escaped quotes make it work if
+      # the option had spaces, like --cmd "queue.pl -sync y"
+      eval $name=\"$2\";
+
+      # Check that Boolean-valued arguments are really Boolean.
+      if $was_bool && [[ "$2" != "true" && "$2" != "false" ]]; then
+        echo "$0: expected \"true\" or \"false\": $1 $2" 1>&2
+        exit 1;
+      fi
+      shift 2;
+      ;;
+  *) break;
+  esac
+done
+
+
+# Check for an empty argument to the --cmd option, which can easily occur as a
+# result of scripting errors.
+[ ! -z "${cmd+xxx}" ] && [ -z "$cmd" ] && echo "$0: empty argument to --cmd option" 1>&2 && exit 1;
+
+
+true; # so this script returns exit code 0.

egs/libri_trans/asr/local/utils.sh

+
+get_devices(){
+    gpu_num=$1
+    use_cpu=$2
+    device=()
+    while :
+    do
+        record=$(mktemp -t temp.record.XXXXXX)
+        gpustat > $record
+        all_devices=$(seq 0 "$(sed '1,2d' ${record} | wc -l)");
+
+        count=0
+        for dev in ${all_devices[@]}
+        do
+            line=$((dev + 2))
+            use=$(head -n $line ${record} | tail -1 | cut -d '|' -f3 | cut -d '/' -f1)
+            if [[ $use -lt 100 ]]; then
+                device[$count]=$dev
+                count=$((count + 1))
+                if [[ $count -eq $gpu_num ]]; then
+                    break
+                fi
+            fi
+        done
+        if [[ ${#device[@]} -lt $gpu_num ]]; then
+            if [[ $use_cpu -eq 1 ]]; then
+                device=(-1)
+            else
+                sleep 60s
+            fi
+        else
+            break
+        fi
+    done
+
+    echo ${device[*]} | sed 's/ /,/g'
+    return $?
+}
+
+

egs/libri_trans/asr/run.sh

+#! /bin/bash
+
+# Processing ASR Datasets
+
+# Copyright 2021 Natural Language Processing Laboratory 
+# Xu Chen (xuchenneu@163.com)
+
+# Set bash to 'debug' mode, it will exit on :
+# -e 'error', -u 'undefined variable', -o ... 'error in pipeline', -x 'print commands',
+set -e
+#set -u
+set -o pipefail
+export PYTHONIOENCODING=UTF-8
+
+eval=1
+time=$(date "+%m%d_%H%M")
+
+stage=0
+stop_stage=0
+
+######## hardware ########
+# devices
+#device=()
+gpu_num=0
+update_freq=1
+
+s2t_dir=~/Code/st
+root_dir=${s2t_dir}/Fairseq-S2T
+pwd_dir=$PWD
+
+# dataset
+src_lang=en
+tgt_lang=fr
+lang=${src_lang}-${tgt_lang}
+
+dataset=libri_trans
+task=speech_to_text
+vocab_type=unigram
+vocab_size=1000
+speed_perturb=0
+lcrm=1
+tokenizer=0
+use_raw_audio=1
+
+use_specific_dict=0
+specific_prefix=st
+specific_dir=${s2t_dir}/data/mustc/st/en-de
+asr_vocab_prefix=spm_unigram10000_st_share
+
+org_data_dir=${s2t_dir}/data/${dataset}
+data_dir=${s2t_dir}/data/${dataset}/asr
+train_split=train
+valid_split=dev
+test_split=test
+test_subset=test
+
+# exp
+exp_prefix=$(date "+%m%d")
+extra_tag=
+extra_parameter=
+exp_tag=baseline
+exp_name=
+
+# config
+train_config=base
+data_config=config.yaml
+
+# training setting
+fp16=1
+max_tokens=40000
+step_valid=0
+
+# decoding setting
+dec_model=checkpoint_best.pt
+n_average=10
+beam_size=5
+len_penalty=1.0
+
+if [[ ${speed_perturb} -eq 1 ]]; then
+    data_dir=${data_dir}_sp
+    exp_prefix=${exp_prefix}_sp
+fi
+if [[ ${lcrm} -eq 1 ]]; then
+    data_dir=${data_dir}_lcrm
+    exp_prefix=${exp_prefix}_lcrm
+fi
+if [[ ${use_specific_dict} -eq 1 ]]; then
+    data_dir=${data_dir}_${specific_prefix}
+    exp_prefix=${exp_prefix}_${specific_prefix}
+fi
+if [[ ${tokenizer} -eq 1 ]]; then
+    data_dir=${data_dir}_tok
+    exp_prefix=${exp_prefix}_tok
+fi
+if [[ ${use_raw_audio} -eq 1 ]]; then
+    data_dir=${data_dir}_raw
+    exp_prefix=${exp_prefix}_raw
+fi
+
+. ./local/parse_options.sh || exit 1;
+
+if [[ -z ${exp_name} ]]; then
+    config_string=${train_config//,/_}
+    exp_name=${exp_prefix}_${config_string}_${exp_tag}
+    if [[ -n ${extra_tag} ]]; then
+        exp_name=${exp_name}_${extra_tag}
+    fi
+fi
+model_dir=$root_dir/../checkpoints/$dataset/asr/${exp_name}
+
+if [ ${stage} -le -1 ] && [ ${stop_stage} -ge -1 ]; then
+    echo "stage -1: Data Download"
+    # pass
+fi
+
+if [ ${stage} -le 0 ] && [ ${stop_stage} -ge 0 ]; then
+    ### Task dependent. You have to make data the following preparation part by yourself.
+    ### But you can utilize Kaldi recipes in most cases
+    echo "stage 0: ASR Data Preparation"
+    if [[ ! -e ${data_dir} ]]; then
+        mkdir -p ${data_dir}
+    fi
+
+    cmd="python ${root_dir}/examples/speech_to_text/prep_audio_data.py
+        --data-root ${org_data_dir}
+        --output-root ${data_dir}
+        --task asr
+        --splits ${train_split},${valid_split},${test_split}
+        --src-lang ${src_lang}
+        --vocab-type ${vocab_type}
+        --vocab-size ${vocab_size}"
+
+    if [[ ${use_raw_audio} -eq 1 ]]; then
+        cmd="$cmd
+        --raw"
+    fi
+    if [[ ${use_specific_dict} -eq 1 ]]; then
+        cp -r ${specific_dir}/${asr_vocab_prefix}.* ${data_dir}/${lang}
+        cmd="$cmd
+        --asr-prefix ${asr_vocab_prefix}"
+    fi
+    if [[ ${speed_perturb} -eq 1 ]]; then
+        cmd="$cmd
+        --speed-perturb"
+    fi
+    if [[ ${lcrm} -eq 1 ]]; then
+        cmd="$cmd
+        --lowercase-src
+        --rm-punc-src"
+    fi
+    if [[ ${tokenizer} -eq 1 ]]; then
+        cmd="$cmd
+        --tokenizer"
+    fi
+
+    echo -e "\033[34mRun command: \n${cmd} \033[0m"
+    [[ $eval -eq 1 ]] && eval ${cmd}
+fi
+
+if [ ${stage} -le 1 ] && [ ${stop_stage} -ge 1 ]; then
+    echo "stage 1: ASR Network Training"
+    [[ ! -d ${data_dir} ]] && echo "The data dir ${data_dir} is not existing!" && exit 1;
+
+    if [[ -z ${device} || ${#device[@]} -eq 0 ]]; then
+		if [[ ${gpu_num} -eq 0 ]]; then
+			device=""
+		else
+        	source ./local/utils.sh
+        	device=$(get_devices $gpu_num 0)
+		fi
+    fi
+
+    echo -e "dev=${device} data=${data_dir} model=${model_dir}"
+
+    if [[ ! -d ${model_dir} ]]; then
+        mkdir -p ${model_dir}
+    else
+        echo "${model_dir} exists."
+    fi
+
+    cp ${BASH_SOURCE[0]} ${model_dir}
+    cp ${PWD}/train.sh ${model_dir}
+
+    config_list="${train_config//,/ }"
+    idx=0
+    for config in ${config_list[@]}
+    do
+        config_path=$pwd_dir/conf/${config}.yaml
+        if [[ ! -f ${config_path} ]]; then
+            echo "No config file ${config_path}"
+            exit
+        fi
+        cp ${config_path} ${model_dir}
+
+        if [[ idx -eq 0 ]]; then
+            extra_parameter="${extra_parameter}
+        --train-config ${config_path}"
+        else
+            extra_parameter="${extra_parameter}
+        --train-config${idx} ${config_path}"
+        fi
+        idx=$((idx + 1))
+    done
+
+    cmd="python3 -u ${root_dir}/fairseq_cli/train.py
+        ${data_dir}
+        --config-yaml ${data_config}
+        --task ${task}
+        --max-tokens ${max_tokens}
+        --skip-invalid-size-inputs-valid-test
+        --update-freq ${update_freq}
+        --log-interval 100
+        --save-dir ${model_dir}
+        --tensorboard-logdir ${model_dir}"
+
+	if [[ -n ${extra_parameter} ]]; then
+        cmd="${cmd}
+        ${extra_parameter}"
+    fi
+	if [[ ${gpu_num} -gt 0 ]]; then
+		cmd="${cmd}
+        --distributed-world-size $gpu_num
+        --ddp-backend no_c10d"
+	fi
+    if [[ $fp16 -eq 1 ]]; then
+        cmd="${cmd}
+        --fp16"
+    fi
+    if [[ $step_valid -eq 1 ]]; then
+        validate_interval=1
+        save_interval=1
+        keep_last_epochs=10
+        no_epoch_checkpoints=0
+        save_interval_updates=500
+        keep_interval_updates=10
+    else
+        validate_interval=1
+        keep_last_epochs=10
+    fi
+    if [[ -n $no_epoch_checkpoints && $no_epoch_checkpoints -eq 1 ]]; then
+        cmd="$cmd
+        --no-epoch-checkpoints"
+    fi
+    if [[ -n $validate_interval ]]; then
+        cmd="${cmd}
+        --validate-interval $validate_interval "
+    fi
+    if [[ -n $save_interval ]]; then
+        cmd="${cmd}
+        --save-interval $save_interval "
+    fi
+    if [[ -n $keep_last_epochs ]]; then
+        cmd="${cmd}
+        --keep-last-epochs $keep_last_epochs "
+    fi
+    if [[ -n $save_interval_updates ]]; then
+        cmd="${cmd}
+        --save-interval-updates $save_interval_updates"
+        if [[ -n $keep_interval_updates ]]; then
+        cmd="${cmd}
+        --keep-interval-updates $keep_interval_updates"
+        fi
+    fi
+
+    echo -e "\033[34mRun command: \n${cmd} \033[0m"
+
+    # save info
+    log=./history.log
+    echo "${time} | ${device} | ${data_dir} | ${exp_name} | ${model_dir} " >> $log
+    tail -n 50 ${log} > tmp.log
+    mv tmp.log $log
+    export CUDA_VISIBLE_DEVICES=${device}
+
+    cmd="nohup ${cmd} >> ${model_dir}/train.log 2>&1 &"
+    if [[ $eval -eq 1 ]]; then
+		eval $cmd
+		sleep 2s
+		tail -n "$(wc -l ${model_dir}/train.log | awk '{print $1+1}')" -f ${model_dir}/train.log
+	fi
+fi
+wait
+
+if [ ${stage} -le 2 ] && [ ${stop_stage} -ge 2 ]; then
+    echo "stage 2: ASR Decoding"
+    if [[ ${n_average} -ne 1 ]]; then
+        # Average models
+		dec_model=avg_${n_average}_checkpoint.pt
+
+		cmd="python ${root_dir}/scripts/average_checkpoints.py
+        --inputs ${model_dir}
+        --num-epoch-checkpoints ${n_average}
+        --output ${model_dir}/${dec_model}"
+    	echo -e "\033[34mRun command: \n${cmd} \033[0m"
+    	[[ $eval -eq 1 ]] && eval $cmd
+	else
+		dec_model=${dec_model}
+	fi
+
+    if [[ -z ${device} || ${#device[@]} -eq 0 ]]; then
+		if [[ ${gpu_num} -eq 0 ]]; then
+			device=""
+		else
+        	source ./local/utils.sh
+        	device=$(get_devices $gpu_num 0)
+		fi
+    fi
+    export CUDA_VISIBLE_DEVICES=${device}
+
+	result_file=${model_dir}/decode_result
+	[[ -f ${result_file} ]] && rm ${result_file}
+
+    test_subset=${test_subset//,/ }
+	for subset in ${test_subset[@]}; do
+        subset=${subset}_asr
+  		cmd="python ${root_dir}/fairseq_cli/generate.py
+        ${data_dir}
+        --config-yaml ${data_config}
+        --gen-subset ${subset}
+        --task speech_to_text
+        --path ${model_dir}/${dec_model}
+        --results-path ${model_dir}
+        --max-tokens ${max_tokens}
+        --beam ${beam_size}
+        --lenpen ${len_penalty}
+        --scoring wer
+        --wer-tokenizer 13a
+        --wer-lowercase
+        --wer-remove-punct
+        "
+    	echo -e "\033[34mRun command: \n${cmd} \033[0m"
+
+        if [[ $eval -eq 1 ]]; then
+    	    eval $cmd
+    	    tail -n 1 ${model_dir}/generate-${subset}.txt >> ${result_file}
+        fi
+	done
+    cat ${result_file}
+fi

egs/libri_trans/asr/train.sh

+#! /bin/bash
+
+# training the model
+
+gpu_num=8
+update_freq=1
+max_tokens=40000
+
+extra_tag=
+extra_parameter=
+#extra_tag="${extra_tag}"
+#extra_parameter="${extra_parameter} "
+
+exp_tag=
+
+#config_list=(base)
+#config_list=(ctc)
+#config_list=(base conformer)
+
+#config_list=(pds_base_16)
+config_list=(pds_base_16 conformer rpr)
+
+# exp full name
+exp_name=
+
+train_config=$(echo ${config_list[*]} | sed 's/ /,/g')
+
+cmd="./run.sh
+    --stage 1
+    --stop_stage 1
+    --gpu_num ${gpu_num}
+    --update_freq ${update_freq}
+    --train_config ${train_config}
+    --max_tokens ${max_tokens}
+    "
+
+if [[ -n ${exp_name} ]]; then
+    cmd="$cmd --exp_name ${exp_name}"
+fi
+if [[ -n ${exp_tag} ]]; then
+    cmd="$cmd --exp_tag ${exp_tag}"
+fi
+if [[ -n ${extra_tag} ]]; then
+    cmd="$cmd --extra_tag ${extra_tag}"
+fi
+if [[ -n ${extra_parameter} ]]; then
+    cmd="$cmd --extra_parameter \"${extra_parameter}\""
+fi
+
+echo ${cmd}
+eval ${cmd}

egs/libri_trans/mt/binary.sh

+set -e
+
+eval=1
+
+root_dir=~/st/Fairseq-S2T
+data_dir=/home/xuchen/st/data/wmt/test
+vocab_dir=/home/xuchen/st/data/wmt/mt/en-de/unigram32000_share
+src_vocab_prefix=spm_unigram32000_share
+tgt_vocab_prefix=spm_unigram32000_share
+
+src_lang=en
+tgt_lang=de
+tokenize=1
+splits=(newstest2014 newstest2016)
+
+for split in ${splits[@]}; do
+    src_file=${data_dir}/${split}.${src_lang}
+    tgt_file=${data_dir}/${split}.${tgt_lang}
+
+    if [[ ${tokenize} -eq 1 ]]; then
+        cmd="tokenizer.perl -l ${src_lang} --threads 8 -no-escape < ${src_file} > ${src_file}.tok"
+        echo -e "\033[34mRun command: \n${cmd} \033[0m"
+        [[ $eval -eq 1 ]] && eval ${cmd}
+
+        cmd="tokenizer.perl -l ${tgt_lang} --threads 8 -no-escape < ${tgt_file} > ${tgt_file}.tok"
+        echo -e "\033[34mRun command: \n${cmd} \033[0m"
+        [[ $eval -eq 1 ]] && eval ${cmd}
+        src_file=${src_file}.tok
+        tgt_file=${tgt_file}.tok
+    fi
+
+    cmd="cat ${src_file}"
+    if [[ ${lcrm} -eq 1 ]]; then
+        cmd="python local/lower_rm.py ${src_file}"
+    fi
+    cmd="${cmd}
+    | spm_encode --model ${vocab_dir}/${src_vocab_prefix}.model
+    --output_format=piece
+    > ${src_file}.spm"
+
+    echo -e "\033[34mRun command: \n${cmd} \033[0m"
+    [[ $eval -eq 1 ]] && eval ${cmd}
+
+    cmd="spm_encode
+    --model ${vocab_dir}/${tgt_vocab_prefix}.model
+    --output_format=piece
+    < ${tgt_file}
+    > ${tgt_file}.spm"
+    echo -e "\033[34mRun command: \n${cmd} \033[0m"
+    [[ $eval -eq 1 ]] && eval ${cmd}
+
+    src_file=${src_file}.spm
+    tgt_file=${tgt_file}.spm
+
+    mkdir -p ${data_dir}/final
+    cmd="cp ${src_file} ${data_dir}/final/${split}.${src_lang}"
+    echo -e "\033[34mRun command: \n${cmd} \033[0m"
+    [[ $eval -eq 1 ]] && eval ${cmd}
+
+    cmd="cp ${tgt_file} ${data_dir}/final/${split}.${tgt_lang}"
+    echo -e "\033[34mRun command: \n${cmd} \033[0m"
+    [[ $eval -eq 1 ]] && eval ${cmd}
+done
+
+n_set=${#splits[*]}
+for ((i=0;i<$n_set;i++)); do
+    dataset[$i]=${data_dir}/final/${splits[$i]}
+done
+pref=`echo ${dataset[*]} | sed 's/ /,/g'`
+
+cmd="python ${root_dir}/fairseq_cli/preprocess.py
+    --source-lang ${src_lang}
+    --target-lang ${tgt_lang}
+    --testpref ${pref}
+    --destdir ${data_dir}/data-bin
+    --srcdict ${vocab_dir}/${src_vocab_prefix}.txt
+    --tgtdict ${vocab_dir}/${tgt_vocab_prefix}.txt
+    --workers 64"
+
+echo -e "\033[34mRun command: \n${cmd} \033[0m"
+[[ $eval -eq 1 ]] && eval ${cmd}
\ No newline at end of file

egs/libri_trans/mt/conf/base.yaml

+train-subset: train
+valid-subset: valid
+
+max-epoch: 50
+max-update: 100000
+
+num-workers: 8
+patience: 10
+no-progress-bar: True
+log-interval: 100
+seed: 1
+report-accuracy: True
+skip-invalid-size-inputs-valid-test: True
+
+#load-pretrained-encoder-from:
+#load-pretrained-decoder-from:
+
+arch: transformer
+share-decoder-input-output-embed: True
+optimizer: adam
+clip-norm: 10.0
+lr-scheduler: inverse_sqrt
+warmup-init-lr: 1e-7
+warmup-updates: 8000
+lr: 1e-3
+adam_betas: (0.9,0.997)
+
+criterion: label_smoothed_cross_entropy
+label_smoothing: 0.1
+
+dropout: 0.1
+attention-dropout: 0.1
+activation-dropout: 0.1
+
+activation-fn: relu
+encoder-normalize-before: True
+decoder-normalize-before: True
+encoder-embed-dim: 512
+encoder-ffn-embed-dim: 2048
+encoder-layers: 6
+decoder-layers: 6
+encoder-attention-heads: 8
+
+decoder-embed-dim: 512
+decoder-ffn-embed-dim: 2048
+decoder-attention-heads: 8

egs/libri_trans/mt/conf/base_s.yaml

+train-subset: train
+valid-subset: valid
+
+max-epoch: 50
+max-update: 100000
+
+num-workers: 8
+patience: 10
+no-progress-bar: True
+log-interval: 100
+seed: 1
+report-accuracy: True
+skip-invalid-size-inputs-valid-test: True
+
+#load-pretrained-encoder-from:
+#load-pretrained-decoder-from:
+
+arch: transformer
+share-decoder-input-output-embed: True
+optimizer: adam
+clip-norm: 10.0
+lr-scheduler: inverse_sqrt
+warmup-init-lr: 1e-7
+warmup-updates: 8000
+lr: 1e-3
+adam_betas: (0.9,0.997)
+
+criterion: label_smoothed_cross_entropy
+label_smoothing: 0.1
+
+dropout: 0.1
+attention-dropout: 0.1
+activation-dropout: 0.1
+
+activation-fn: relu
+encoder-normalize-before: True
+decoder-normalize-before: True
+encoder-embed-dim: 256
+encoder-ffn-embed-dim: 2048
+encoder-layers: 6
+decoder-layers: 6
+encoder-attention-heads: 4
+
+decoder-embed-dim: 256
+decoder-ffn-embed-dim: 2048
+decoder-attention-heads: 4

egs/libri_trans/mt/conf/dlcl.yaml

+use-enc-dlcl: True
+use-dec-dlcl: True

egs/libri_trans/mt/conf/rpr.yaml

+#encoder-attention-type: rel_selfattn
+encoder-attention-type: relative
+decoder-attention-type: relative
+max-encoder-relative-length: 20
+max-decoder-relative-length: 20
\ No newline at end of file

egs/libri_trans/mt/decode.sh

+#! /bin/bash
+
+gpu_num=1
+
+data_dir=
+test_subset=(test)
+
+exp_name=
+if [ "$#" -eq 1 ]; then
+    exp_name=$1
+fi
+
+n_average=10
+beam_size=5
+len_penalty=1.0
+max_tokens=80000
+dec_model=checkpoint_best.pt
+
+cmd="./run.sh
+    --stage 2
+    --stop_stage 2
+    --gpu_num ${gpu_num}
+    --exp_name ${exp_name}
+    --n_average ${n_average}
+    --beam_size ${beam_size}
+    --len_penalty ${len_penalty}
+    --max_tokens ${max_tokens}
+    --dec_model ${dec_model}
+    "
+
+if [[ -n ${data_dir} ]]; then
+    cmd="$cmd --data_dir ${data_dir}"
+fi
+if [[ -n ${test_subset} ]]; then
+    test_subset=`echo ${test_subset[*]} | sed 's/ /,/g'`
+    cmd="$cmd --test_subset ${test_subset}"
+fi
+
+echo $cmd
+eval $cmd

egs/libri_trans/mt/local/lower_rm.py

+import sys
+import string
+
+
+in_file = sys.argv[1]
+
+with open(in_file, "r", encoding="utf-8") as f:
+    for line in f.readlines():
+        line = line.strip().lower()
+        for w in string.punctuation:
+            line = line.replace(w, "")
+        line = line.replace("  ", "")
+        print(line)
+

egs/libri_trans/mt/local/monitor.sh

+gpu_num=4
+cmd="sh train.sh"
+
+while :
+do
+    record=$(mktemp -t temp.record.XXXXXX)
+    gpustat > $record
+    all_devices=$(seq 0 "$(sed '1,2d' ${record} | wc -l)");
+
+    count=0
+    for dev in ${all_devices[@]}
+    do
+        line=$((dev + 2))
+        use=$(head -n $line ${record} | tail -1 | cut -d '|' -f3 | cut -d '/' -f1)
+
+        if [[ $use -lt 100 ]]; then
+            device[$count]=$dev
+            count=$((count + 1))
+            if [[ $count -eq $gpu_num ]]; then
+                break
+            fi
+        fi
+    done
+    if [[ ${#device[@]} -lt $gpu_num ]]; then
+        sleep 60s
+    else
+        echo "Run $cmd"
+        eval $cmd
+        sleep 10s
+        exit
+    fi
+done

egs/libri_trans/mt/local/parse_options.sh

+#!/usr/bin/env bash
+
+# Copyright 2012  Johns Hopkins University (Author: Daniel Povey);
+#                 Arnab Ghoshal, Karel Vesely
+
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#  http://www.apache.org/licenses/LICENSE-2.0
+#
+# THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED
+# WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE,
+# MERCHANTABLITY OR NON-INFRINGEMENT.
+# See the Apache 2 License for the specific language governing permissions and
+# limitations under the License.
+
+
+# Parse command-line options.
+# To be sourced by another script (as in ". parse_options.sh").
+# Option format is: --option-name arg
+# and shell variable "option_name" gets set to value "arg."
+# The exception is --help, which takes no arguments, but prints the
+# $help_message variable (if defined).
+
+
+###
+### The --config file options have lower priority to command line
+### options, so we need to import them first...
+###
+
+# Now import all the configs specified by command-line, in left-to-right order
+for ((argpos=1; argpos<$#; argpos++)); do
+  if [ "${!argpos}" == "--config" ]; then
+    argpos_plus1=$((argpos+1))
+    config=${!argpos_plus1}
+    [ ! -r $config ] && echo "$0: missing config '$config'" && exit 1
+    . $config  # source the config file.
+  fi
+done
+
+
+###
+### Now we process the command line options
+###
+while true; do
+  [ -z "${1:-}" ] && break;  # break if there are no arguments
+  case "$1" in
+    # If the enclosing script is called with --help option, print the help
+    # message and exit.  Scripts should put help messages in $help_message
+    --help|-h) if [ -z "$help_message" ]; then echo "No help found." 1>&2;
+      else printf "$help_message\n" 1>&2 ; fi;
+      exit 0 ;;
+    --*=*) echo "$0: options to scripts must be of the form --name value, got '$1'"
+      exit 1 ;;
+    # If the first command-line argument begins with "--" (e.g. --foo-bar),
+    # then work out the variable name as $name, which will equal "foo_bar".
+    --*) name=`echo "$1" | sed s/^--// | sed s/-/_/g`;
+      # Next we test whether the variable in question is undefned-- if so it's
+      # an invalid option and we die.  Note: $0 evaluates to the name of the
+      # enclosing script.
+      # The test [ -z ${foo_bar+xxx} ] will return true if the variable foo_bar
+      # is undefined.  We then have to wrap this test inside "eval" because
+      # foo_bar is itself inside a variable ($name).
+      eval '[ -z "${'$name'+xxx}" ]' && echo "$0: invalid option $1" 1>&2 && exit 1;
+
+      oldval="`eval echo \\$$name`";
+      # Work out whether we seem to be expecting a Boolean argument.
+      if [ "$oldval" == "true" ] || [ "$oldval" == "false" ]; then
+        was_bool=true;
+      else
+        was_bool=false;
+      fi
+
+      # Set the variable to the right value-- the escaped quotes make it work if
+      # the option had spaces, like --cmd "queue.pl -sync y"
+      eval $name=\"$2\";
+
+      # Check that Boolean-valued arguments are really Boolean.
+      if $was_bool && [[ "$2" != "true" && "$2" != "false" ]]; then
+        echo "$0: expected \"true\" or \"false\": $1 $2" 1>&2
+        exit 1;
+      fi
+      shift 2;
+      ;;
+  *) break;
+  esac
+done
+
+
+# Check for an empty argument to the --cmd option, which can easily occur as a
+# result of scripting errors.
+[ ! -z "${cmd+xxx}" ] && [ -z "$cmd" ] && echo "$0: empty argument to --cmd option" 1>&2 && exit 1;
+
+
+true; # so this script returns exit code 0.

egs/libri_trans/mt/local/utils.sh

+
+get_devices(){
+    gpu_num=$1
+    use_cpu=$2
+    device=()
+    while :
+    do
+        record=$(mktemp -t temp.record.XXXXXX)
+        gpustat > $record
+        all_devices=$(seq 0 "$(sed '1,2d' ${record} | wc -l)");
+
+        count=0
+        for dev in ${all_devices[@]}
+        do
+            line=$((dev + 2))
+            use=$(head -n $line ${record} | tail -1 | cut -d '|' -f3 | cut -d '/' -f1)
+            if [[ $use -lt 100 ]]; then
+                device[$count]=$dev
+                count=$((count + 1))
+                if [[ $count -eq $gpu_num ]]; then
+                    break
+                fi
+            fi
+        done
+        if [[ ${#device[@]} -lt $gpu_num ]]; then
+            if [[ $use_cpu -eq 1 ]]; then
+                device=(-1)
+            else
+                sleep 60s
+            fi
+        else
+            break
+        fi
+    done
+
+    echo ${device[*]} | sed 's/ /,/g'
+    return $?
+}
+
+

egs/libri_trans/mt/run.sh

+#! /bin/bash
+
+# Processing MuST-C Datasets
+
+# Copyright 2021 Natural Language Processing Laboratory 
+# Xu Chen (xuchenneu@163.com)
+
+# Set bash to 'debug' mode, it will exit on :
+# -e 'error', -u 'undefined variable', -o ... 'error in pipeline', -x 'print commands',
+set -e
+#set -u
+set -o pipefail
+export PYTHONIOENCODING=UTF-8
+
+eval=1
+time=$(date "+%m%d_%H%M")
+
+stage=0
+stop_stage=0
+
+######## hardware ########
+# devices
+device=()
+gpu_num=8
+update_freq=1
+
+root_dir=~/st/Fairseq-S2T
+pwd_dir=$PWD
+
+# dataset
+src_lang=en
+tgt_lang=de
+lang=${src_lang}-${tgt_lang}
+
+dataset=mt
+task=translation
+vocab_type=unigram
+vocab_size=10000
+share_dict=1
+lcrm=0
+tokenizer=0
+
+use_specific_dict=0
+specific_prefix=st
+specific_dir=/home/xuchen/st/data/mustc/st/en-de/
+src_vocab_prefix=spm_unigram10000_st_share
+tgt_vocab_prefix=spm_unigram10000_st_share
+
+org_data_dir=~/st/data/${dataset}
+data_dir=~/st/data/${dataset}/mt/${lang}
+train_subset=train
+valid_subset=dev
+trans_subset=tst-COMMON
+test_subset=test
+
+# exp
+exp_prefix=${time}
+extra_tag=
+extra_parameter=
+exp_tag=baseline
+exp_name=
+
+# config
+train_config=base_s
+
+# training setting
+fp16=1
+max_tokens=4096
+step_valid=0
+bleu_valid=0
+
+# decoding setting
+dec_model=checkpoint_best.pt
+n_average=10
+beam_size=5
+len_penalty=1.0
+
+if [[ ${use_specific_dict} -eq 1 ]]; then
+    exp_prefix=${specific_prefix}_${exp_prefix}
+    data_dir=${data_dir}/${specific_prefix}
+    mkdir -p ${data_dir}
+else
+    data_dir=${data_dir}/${vocab_type}${vocab_size}
+    src_vocab_prefix=spm_${vocab_type}${vocab_size}_${src_lang}
+    tgt_vocab_prefix=spm_${vocab_type}${vocab_size}_${tgt_lang}
+    if [[ $share_dict -eq 1 ]]; then
+        data_dir=${data_dir}_share
+        src_vocab_prefix=spm_${vocab_type}${vocab_size}_share
+        tgt_vocab_prefix=spm_${vocab_type}${vocab_size}_share
+    fi
+fi
+if [[ ${lcrm} -eq 1 ]]; then
+    data_dir=${data_dir}_lcrm
+    exp_prefix=${exp_prefix}_lcrm
+fi
+if [[ ${tokenizer} -eq 1 ]]; then
+    train_subset=${train_subset}.tok
+    valid_subset=${valid_subset}.tok
+    trans_subset=${trans_subset}.tok
+    data_dir=${data_dir}_tok
+    exp_prefix=${exp_prefix}_tok
+fi
+
+. ./local/parse_options.sh || exit 1;
+
+# full path
+if [[ -z ${exp_name} ]]; then
+    config_string=${train_config//,/_}
+    exp_name=${exp_prefix}_${config_string}_${exp_tag}
+    if [[ -n ${extra_tag} ]]; then
+        exp_name=${exp_name}_${extra_tag}
+    fi
+fi
+model_dir=$root_dir/../checkpoints/$dataset/mt/${exp_name}
+
+if [ ${stage} -le -1 ] && [ ${stop_stage} -ge -1 ]; then
+    echo "stage -1: Data Download"
+    # pass
+fi
+
+if [ ${stage} -le 0 ] && [ ${stop_stage} -ge 0 ]; then
+    ### Task dependent. You have to make data the following preparation part by yourself.
+    echo "stage 0: MT Data Preparation"
+    if [[ ! -e ${data_dir} ]]; then
+        mkdir -p ${data_dir}
+    fi
+
+    if [[ ! -f ${data_dir}/${src_vocab_prefix}.txt || ! -f ${data_dir}/${tgt_vocab_prefix}.txt ]]; then
+        if [[ ${use_specific_dict} -eq 0 ]]; then
+            cmd="python ${root_dir}/examples/speech_to_text/prep_mt_data.py
+                --data-root ${org_data_dir}
+                --output-root ${data_dir}
+                --splits ${train_subset},${valid_subset},${trans_subset}
+                --src-lang ${src_lang}
+                --tgt-lang ${tgt_lang}
+                --vocab-type ${vocab_type}
+                --vocab-size ${vocab_size}"
+            if [[ $share_dict -eq 1 ]]; then
+                cmd="$cmd
+                --share"
+            fi
+            echo -e "\033[34mRun command: \n${cmd} \033[0m"
+            [[ $eval -eq 1 ]] && eval ${cmd}
+        else
+            cp -r ${specific_dir}/${src_vocab_prefix}.* ${data_dir}
+            cp ${specific_dir}/${tgt_vocab_prefix}.* ${data_dir}
+        fi
+    fi
+
+    mkdir -p ${data_dir}/data
+    for split in ${train_subset} ${valid_subset} ${trans_subset}; do
+    {
+        cmd="cat ${org_data_dir}/${lang}/data/${split}/txt/${split}.${src_lang}"
+        if [[ ${lcrm} -eq 1 ]]; then
+            cmd="python local/lower_rm.py ${org_data_dir}/${lang}/data/${split}.${src_lang}"
+        fi
+        cmd="${cmd}
+        | spm_encode --model ${data_dir}/${src_vocab_prefix}.model
+        --output_format=piece
+        > ${data_dir}/data/${split}.${src_lang}"
+
+        echo -e "\033[34mRun command: \n${cmd} \033[0m"
+        [[ $eval -eq 1 ]] && eval ${cmd}
+
+        cmd="spm_encode
+        --model ${data_dir}/${tgt_vocab_prefix}.model
+        --output_format=piece
+        < ${org_data_dir}/${lang}/data/${split}.${tgt_lang}
+        > ${data_dir}/data/${split}.${tgt_lang}"
+
+        echo -e "\033[34mRun command: \n${cmd} \033[0m"
+        [[ $eval -eq 1 ]] && eval ${cmd}
+    }&
+    done
+    wait
+
+    cmd="python ${root_dir}/fairseq_cli/preprocess.py
+        --source-lang ${src_lang} --target-lang ${tgt_lang}
+        --trainpref ${data_dir}/data/${train_subset}
+        --validpref ${data_dir}/data/${valid_subset}
+        --testpref ${data_dir}/data/${trans_subset}
+        --destdir ${data_dir}/data-bin
+        --srcdict ${data_dir}/${src_vocab_prefix}.txt
+        --tgtdict ${data_dir}/${tgt_vocab_prefix}.txt
+        --workers 64"
+
+    echo -e "\033[34mRun command: \n${cmd} \033[0m"
+    [[ $eval -eq 1 ]] && eval ${cmd}
+fi
+
+data_dir=${data_dir}/data-bin
+
+if [ ${stage} -le 1 ] && [ ${stop_stage} -ge 1 ]; then
+    echo "stage 1: MT Network Training"
+    [[ ! -d ${data_dir} ]] && echo "The data dir ${data_dir} is not existing!" && exit 1;
+
+    if [[ -z ${device} || ${#device[@]} -eq 0 ]]; then
+		if [[ ${gpu_num} -eq 0 ]]; then
+			device=""
+		else
+        	source ./local/utils.sh
+        	device=$(get_devices $gpu_num 0)
+		fi
+    fi
+
+    echo -e "dev=${device} data=${data_dir} model=${model_dir}"
+
+    if [[ ! -d ${model_dir} ]]; then
+        mkdir -p ${model_dir}
+    else
+        echo "${model_dir} exists."
+    fi
+
+    cp ${BASH_SOURCE[0]} ${model_dir}
+    cp ${PWD}/train.sh ${model_dir}
+
+    config_list="${train_config//,/ }"
+    idx=0
+    for config in ${config_list[@]}
+    do
+        config_path=$pwd_dir/conf/${config}.yaml
+        if [[ ! -f ${config_path} ]]; then
+            echo "No config file ${config_path}"
+            exit
+        fi
+        cp ${config_path} ${model_dir}
+
+        if [[ idx -eq 0 ]]; then
+            extra_parameter="${extra_parameter}
+        --train-config ${config_path}"
+        else
+            extra_parameter="${extra_parameter}
+        --train-config${idx} ${config_path}"
+        fi
+        idx=$((idx + 1))
+    done
+
+    cmd="python3 -u ${root_dir}/fairseq_cli/train.py
+        ${data_dir}
+        --source-lang ${src_lang}
+        --target-lang ${tgt_lang}
+        --task ${task}
+        --max-tokens ${max_tokens}
+        --skip-invalid-size-inputs-valid-test
+        --update-freq ${update_freq}
+        --log-interval 100
+        --save-dir ${model_dir}
+        --tensorboard-logdir ${model_dir}"
+
+	if [[ -n ${extra_parameter} ]]; then
+        cmd="${cmd}
+        ${extra_parameter}"
+    fi
+	if [[ ${gpu_num} -gt 0 ]]; then
+		cmd="${cmd}
+        --distributed-world-size $gpu_num
+        --ddp-backend no_c10d"
+	fi
+    if [[ $fp16 -eq 1 ]]; then
+        cmd="${cmd}
+        --fp16"
+    fi
+    if [[ $step_valid -eq 1 ]]; then
+        validate_interval=1
+        save_interval=1
+        keep_last_epochs=10
+        no_epoch_checkpoints=0
+        save_interval_updates=500
+        keep_interval_updates=10
+    else
+        validate_interval=1
+        keep_last_epochs=10
+    fi
+    if [[ $bleu_valid -eq 1 ]]; then
+        cmd="$cmd
+        --eval-bleu
+        --eval-bleu-args '{\"beam\": 1}'
+        --eval-tokenized-bleu
+        --eval-bleu-remove-bpe
+        --best-checkpoint-metric bleu
+        --maximize-best-checkpoint-metric"
+    fi
+    if [[ -n $no_epoch_checkpoints && $no_epoch_checkpoints -eq 1 ]]; then
+        cmd="$cmd
+        --no-epoch-checkpoints"
+    fi
+    if [[ -n $validate_interval ]]; then
+        cmd="${cmd}
+        --validate-interval $validate_interval "
+    fi
+    if [[ -n $save_interval ]]; then
+        cmd="${cmd}
+        --save-interval $save_interval "
+    fi
+    if [[ -n $keep_last_epochs ]]; then
+        cmd="${cmd}
+        --keep-last-epochs $keep_last_epochs "
+    fi
+    if [[ -n $save_interval_updates ]]; then
+        cmd="${cmd}
+        --save-interval-updates $save_interval_updates"
+        if [[ -n $keep_interval_updates ]]; then
+        cmd="${cmd}
+        --keep-interval-updates $keep_interval_updates"
+        fi
+    fi
+
+    echo -e "\033[34mRun command: \n${cmd} \033[0m"
+
+    # save info
+    log=./history.log
+    echo "${time} | ${device} | ${data_dir} | ${exp_name} | ${model_dir} " >> $log
+    tail -n 50 ${log} > tmp.log
+    mv tmp.log $log
+    export CUDA_VISIBLE_DEVICES=${device}
+
+    cmd="nohup ${cmd} >> ${model_dir}/train.log 2>&1 &"
+    if [[ $eval -eq 1 ]]; then
+		eval $cmd
+		sleep 2s
+		tail -n "$(wc -l ${model_dir}/train.log | awk '{print $1+1}')" -f ${model_dir}/train.log
+	fi
+fi
+wait
+
+if [ ${stage} -le 2 ] && [ ${stop_stage} -ge 2 ]; then
+    echo "stage 2: MT Decoding"
+    if [[ ${n_average} -ne 1 ]]; then
+        # Average models
+		dec_model=avg_${n_average}_checkpoint.pt
+
+		cmd="python ${root_dir}/scripts/average_checkpoints.py
+        --inputs ${model_dir}
+        --num-epoch-checkpoints ${n_average}
+        --output ${model_dir}/${dec_model}"
+    	echo -e "\033[34mRun command: \n${cmd} \033[0m"
+    	[[ $eval -eq 1 ]] && eval $cmd
+	else
+		dec_model=${dec_model}
+	fi
+
+    if [[ -z ${device} || ${#device[@]} -eq 0 ]]; then
+		if [[ ${gpu_num} -eq 0 ]]; then
+			device=""
+		else
+        	source ./local/utils.sh
+        	device=$(get_devices $gpu_num 0)
+		fi
+    fi
+    export CUDA_VISIBLE_DEVICES=${device}
+
+	result_file=${model_dir}/decode_result
+	[[ -f ${result_file} ]] && rm ${result_file}
+
+    test_subset=(${test_subset//,/ })
+	for subset in ${test_subset[@]}; do
+  		cmd="python ${root_dir}/fairseq_cli/generate.py
+        ${data_dir}
+        --source-lang ${src_lang}
+        --target-lang ${tgt_lang}
+        --gen-subset ${subset}
+        --task ${task}
+        --path ${model_dir}/${dec_model}
+        --results-path ${model_dir}
+        --max-tokens ${max_tokens}
+        --beam ${beam_size}
+        --lenpen ${len_penalty}
+        --post-process sentencepiece
+        --scoring sacrebleu"
+
+        if [[ ${tokenizer} -eq 1 ]]; then
+            cmd="${cmd}
+        --tokenizer moses
+        --moses-source-lang ${src_lang}
+        --moses-target-lang ${tgt_lang}"
+        fi
+
+    	echo -e "\033[34mRun command: \n${cmd} \033[0m"
+
+        if [[ $eval -eq 1 ]]; then
+    	    eval $cmd
+    	    tail -n 1 ${model_dir}/generate-${subset}.txt >> ${result_file}
+        fi
+	done
+    cat ${result_file}
+fi

egs/libri_trans/mt/train.sh

+#! /bin/bash
+
+# training the model
+
+gpu_num=1
+update_freq=1
+max_tokens=8192
+
+exp_tag=baseline
+config_list=(base)
+
+# exp full name
+exp_name=
+
+extra_tag=
+extra_parameter=
+#extra_tag="${extra_tag}"
+#extra_parameter="${extra_parameter} "
+
+train_config=$(echo ${config_list[*]} | sed 's/ /,/g')
+
+cmd="./run.sh
+    --stage 1
+    --stop_stage 1
+    --gpu_num ${gpu_num}
+    --update_freq ${update_freq}
+    --train_config ${train_config}
+    --max_tokens ${max_tokens}
+    "
+
+if [[ -n ${exp_name} ]]; then
+    cmd="$cmd --exp_name ${exp_name}"
+fi
+if [[ -n ${exp_tag} ]]; then
+    cmd="$cmd --exp_tag ${exp_tag}"
+fi
+if [[ -n ${extra_tag} ]]; then
+    cmd="$cmd --extra_tag ${extra_tag}"
+fi
+if [[ -n ${extra_parameter} ]]; then
+    cmd="$cmd --extra_parameter \"${extra_parameter}\""
+fi
+
+echo ${cmd}
+eval ${cmd}

egs/libri_trans/st/binary.sh

+set -e
+
+eval=1
+
+lcrm=1
+tokenizer=0
+
+root_dir=~/st/Fairseq-S2T
+data_dir=/home/xuchen/st/data/test
+vocab_dir=/home/xuchen/st/data/mustc/st_lcrm/en-de
+asr_vocab_prefix=spm_unigram10000_st_share
+st_vocab_prefix=spm_unigram10000_st_share
+
+src_lang=en
+tgt_lang=de
+splits=(2019)
+
+splits=$(echo ${splits[*]} | sed 's/ /_/g')
+
+cp -r ${vocab_dir}/${asr_vocab_prefix}.* ${data_dir}/${src_lang}-${tgt_lang}
+cp -r ${vocab_dir}/${st_vocab_prefix}.* ${data_dir}/${src_lang}-${tgt_lang}
+rm -rf ${data_dir}/${src_lang}-${tgt_lang}/fbank80.zip
+
+cmd="python ${root_dir}/examples/speech_to_text/prep_st_data.py
+    --data-root ${data_dir}
+    --output-root ${data_dir}
+    --splits ${splits}
+    --task st
+    --src-lang ${src_lang}
+    --tgt-lang ${tgt_lang}
+    --add-src
+    --share
+    --asr-prefix ${asr_vocab_prefix}
+    --st-spm-prefix ${st_vocab_prefix}
+    --cmvn-type utterance"
+
+    if [[ ${lcrm} -eq 1 ]]; then
+        cmd="$cmd
+    --lowercase-src
+    --rm-punc-src"
+    fi
+    if [[ ${tokenizer} -eq 1 ]]; then
+        cmd="$cmd
+    --tokenizer"
+    fi
+
+echo -e "\033[34mRun command: \n${cmd} \033[0m"
+[[ $eval -eq 1 ]] && eval ${cmd}

egs/libri_trans/st/conf/base.yaml

+train-subset: train_st
+valid-subset: dev_st
+
+max-epoch: 100
+max-update: 100000
+
+num-workers: 8
+patience: 10
+no-progress-bar: True
+log-interval: 100
+seed: 1
+report-accuracy: True
+
+#load-pretrained-encoder-from:
+#load-pretrained-decoder-from:
+
+arch: s2t_transformer_s
+share-decoder-input-output-embed: True
+optimizer: adam
+clip-norm: 10.0
+lr-scheduler: inverse_sqrt
+warmup-init-lr: 1e-7
+warmup-updates: 10000
+lr: 2e-3
+#adam_betas: (0.9,0.98)
+
+criterion: label_smoothed_cross_entropy_with_ctc
+label_smoothing: 0.1
+
+conv-kernel-sizes: 5,5
+conv-channels: 1024
+dropout: 0.1
+activation-fn: relu
+encoder-embed-dim: 256
+encoder-ffn-embed-dim: 2048
+encoder-layers: 12
+decoder-layers: 6
+encoder-attention-heads: 4
+
+decoder-embed-dim: 256
+decoder-ffn-embed-dim: 2048
+decoder-attention-heads: 4
+attention-dropout: 0.1
+activation-dropout: 0.1

egs/libri_trans/st/conf/conformer.yaml

+macaron-style: True
+use-cnn-module: True
+cnn-module-kernel: 31

egs/libri_trans/st/conf/ctc.yaml

+ctc-weight: 0.3
\ No newline at end of file

egs/libri_trans/st/conf/dlcl.yaml

+use-enc-dlcl: True
+use-dec-dlcl: True

egs/libri_trans/st/conf/local_attn.yaml

+encoder-attention-type: local
+hard-mask-window: 0
+gauss-mask-sigma: 3
+init-mask-weight: 0
\ No newline at end of file

egs/libri_trans/st/conf/pds_base.yaml

+arch: pdss2t_transformer_s_8
+
+train-subset: train_st
+valid-subset: dev_st
+
+max-epoch: 100
+max-update: 100000
+
+num-workers: 8
+patience: 10
+no-progress-bar: True
+log-interval: 100
+seed: 1
+report-accuracy: True
+
+#load-pretrained-encoder-from:
+#load-pretrained-decoder-from:
+
+share-decoder-input-output-embed: True
+optimizer: adam
+clip-norm: 10.0
+lr-scheduler: inverse_sqrt
+warmup-init-lr: 1e-7
+warmup-updates: 10000
+lr: 2e-3
+#adam_betas: (0.9,0.98)
+
+criterion: label_smoothed_cross_entropy_with_ctc
+label_smoothing: 0.1
+
+dropout: 0.1
+activation-fn: relu
+encoder-ffn-embed-dim: 2048
+encoder-layers: 12
+decoder-layers: 6
+encoder-attention-heads: 4
+
+decoder-embed-dim: 256
+decoder-ffn-embed-dim: 2048
+decoder-attention-heads: 4

egs/libri_trans/st/conf/pds_base_16.yaml

+arch: pdss2t_transformer_s_16
+
+encoder-embed-dim: 256
+pyramid-stages: 4
+#pyramid-dropout: 0
+pyramid-layers: 2_2_6_2
+pyramid-ratios: 2_2_2_2
+pyramid-fusion: True
+pyramid-fusion-method: all_conv
+pyramid-embed-dims: 256_256_256_256
+pyramid-ds-method: conv
+pyramid-embed-norm: True
+pyramid-position-embed: 1_1_1_1
+pyramid-kernel-sizes: 5_5_5_5
+pyramid-ffn-ratios: 8_8_8_8
+pyramid-attn-heads: 4_4_4_4
+
+train-subset: train_st
+valid-subset: dev_st
+
+max-epoch: 100
+max-update: 100000
+
+num-workers: 8
+patience: 10
+no-progress-bar: True
+log-interval: 100
+seed: 1
+report-accuracy: True
+
+#load-pretrained-encoder-from:
+#load-pretrained-decoder-from:
+
+share-decoder-input-output-embed: True
+optimizer: adam
+clip-norm: 10.0
+lr-scheduler: inverse_sqrt
+warmup-init-lr: 1e-7
+warmup-updates: 10000
+lr: 2e-3
+#adam_betas: (0.9,0.98)
+
+criterion: label_smoothed_cross_entropy_with_ctc
+label_smoothing: 0.1
+
+dropout: 0.1
+activation-fn: relu
+encoder-ffn-embed-dim: 2048
+encoder-layers: 12
+decoder-layers: 6
+encoder-attention-heads: 4
+
+decoder-embed-dim: 256
+decoder-ffn-embed-dim: 2048
+decoder-attention-heads: 4

egs/libri_trans/st/conf/pds_base_32.yaml

+arch: pdss2t_transformer_s_32
+
+encoder-embed-dim: 256
+pyramid-stages: 5
+#pyramid-dropout: 0
+pyramid-layers: 2_2_3_3_2
+pyramid-ratios: 2_2_2_2_2
+pyramid-fusion: True
+pyramid-fusion-method: all_conv
+pyramid-embed-dims: 256_256_256_256_256
+pyramid-ds-method: conv
+pyramid-embed-norm: True
+pyramid-position-embed: 1_1_1_1_1
+pyramid-kernel-sizes: 5_5_5_5_5
+pyramid-ffn-ratios: 8_8_8_8_8
+pyramid-attn-heads: 4_4_4_4_4
+
+train-subset: train_st
+valid-subset: dev_st
+
+max-epoch: 100
+max-update: 100000
+
+num-workers: 8
+patience: 10
+no-progress-bar: True
+log-interval: 100
+seed: 1
+report-accuracy: True
+
+#load-pretrained-encoder-from:
+#load-pretrained-decoder-from:
+
+share-decoder-input-output-embed: True
+optimizer: adam
+clip-norm: 10.0
+lr-scheduler: inverse_sqrt
+warmup-init-lr: 1e-7
+warmup-updates: 10000
+lr: 2e-3
+#adam_betas: (0.9,0.98)
+
+criterion: label_smoothed_cross_entropy_with_ctc
+label_smoothing: 0.1
+
+dropout: 0.1
+activation-fn: relu
+encoder-ffn-embed-dim: 2048
+encoder-layers: 12
+decoder-layers: 6
+encoder-attention-heads: 4
+
+decoder-embed-dim: 256
+decoder-ffn-embed-dim: 2048
+decoder-attention-heads: 4

egs/libri_trans/st/conf/pds_base_8.yaml

+arch: pdss2t_transformer_s_8
+
+encoder-embed-dim: 256
+pyramid-stages: 4
+#pyramid-dropout: 0
+pyramid-layers: 3_3_3_3
+pyramid-ratios: 2_2_1_2
+pyramid-fusion: True
+pyramid-fusion-method: all_conv
+pyramid-embed-dims: 256_256_256_256
+pyramid-ds-method: conv
+pyramid-embed-norm: True
+pyramid-position-embed: 1_1_1_1
+pyramid-kernel-sizes: 5_5_5_5
+pyramid-ffn-ratios: 8_8_8_8
+pyramid-attn-heads: 4_4_4_4
+
+train-subset: train_st
+valid-subset: dev_st
+
+max-epoch: 100
+max-update: 100000
+
+num-workers: 8
+patience: 10
+no-progress-bar: True
+log-interval: 100
+seed: 1
+report-accuracy: True
+
+#load-pretrained-encoder-from:
+#load-pretrained-decoder-from:
+
+share-decoder-input-output-embed: True
+optimizer: adam
+clip-norm: 10.0
+lr-scheduler: inverse_sqrt
+warmup-init-lr: 1e-7
+warmup-updates: 10000
+lr: 2e-3
+#adam_betas: (0.9,0.98)
+
+criterion: label_smoothed_cross_entropy_with_ctc
+label_smoothing: 0.1
+
+dropout: 0.1
+activation-fn: relu
+encoder-ffn-embed-dim: 2048
+encoder-layers: 12
+decoder-layers: 6
+encoder-attention-heads: 4
+
+decoder-embed-dim: 256
+decoder-ffn-embed-dim: 2048
+decoder-attention-heads: 4

egs/libri_trans/st/conf/rpr.yaml

+encoder-attention-type: rel_selfattn
+#encoder-attention-type: relative
+#decoder-attention-type: relative
+#max-encoder-relative-length: 100
+#max-decoder-relative-length: 20

egs/libri_trans/st/conf/sate_ctc.yaml

+train-subset: train_st
+valid-subset: dev_st
+
+max-epoch: 100
+max-update: 100000
+
+num-workers: 8
+patience: 10
+no-progress-bar: True
+log-interval: 100
+seed: 1
+report-accuracy: True
+
+#load-pretrained-encoder-from:
+#load-pretrained-acoustic-encoder-from:
+#load-pretrained-text-encoder-from:
+#load-pretrained-decoder-from:
+
+arch: s2t_sate
+share-decoder-input-output-embed: True
+optimizer: adam
+clip-norm: 10.0
+lr-scheduler: inverse_sqrt
+warmup-init-lr: 1e-7
+warmup-updates: 10000
+lr: 2e-3
+#adam_betas: (0.9,0.98)
+
+ctc-weight: 0.3
+criterion: label_smoothed_cross_entropy_with_ctc
+label_smoothing: 0.1
+
+encoder-normalize-before: True
+decoder-normalize-before: True
+conv-kernel-sizes: 5,5
+conv-channels: 1024
+dropout: 0.1
+activation-fn: relu
+encoder-embed-dim: 256
+encoder-ffn-embed-dim: 2048
+encoder-layers: 12
+text-encoder-layers: 6
+decoder-layers: 6
+encoder-attention-heads: 4
+
+#macaron-style: True
+#use-cnn-module: True
+#cnn-module-kernel: 31
+
+#acoustic-encoder: pds
+acoustic-encoder: transformer
+adapter: league
+
+encoder-embed-dim: 256
+pyramid-stages: 4
+#pyramid-dropout: 0
+pyramid-layers: 3_3_3_3
+pyramid-ratios: 2_2_1_2
+pyramid-fusion: True
+pyramid-fusion-method: all_conv
+pyramid-embed-dims: 256_256_256_256
+pyramid-ds-method: conv
+pyramid-embed-norm: True
+pyramid-position-embed: 1_1_1_1
+pyramid-kernel-sizes: 5_5_5_5
+pyramid-ffn-ratios: 8_8_8_8
+pyramid-attn-heads: 4_4_4_4
+
+decoder-embed-dim: 256
+decoder-ffn-embed-dim: 2048
+decoder-attention-heads: 4

egs/libri_trans/st/decode.sh

+#! /bin/bash
+
+gpu_num=1
+
+data_dir=
+test_subset=(tst-COMMON)
+
+exp_name=
+if [ "$#" -eq 1 ]; then
+    exp_name=$1
+fi
+
+n_average=10
+beam_size=5
+len_penalty=1.0
+max_tokens=80000
+dec_model=checkpoint_best.pt
+
+cmd="./run.sh
+    --stage 2
+    --stop_stage 2
+    --gpu_num ${gpu_num}
+    --exp_name ${exp_name}
+    --n_average ${n_average}
+    --beam_size ${beam_size}
+    --len_penalty ${len_penalty}
+    --max_tokens ${max_tokens}
+    --dec_model ${dec_model}
+    "
+
+if [[ -n ${data_dir} ]]; then
+    cmd="$cmd --data_dir ${data_dir}"
+fi
+if [[ ${#test_subset[@]} -eq 0 ]]; then
+    subsets=$(echo ${test_subset[*]} | sed 's/ /,/g')
+    cmd="$cmd --test_subset ${subsets}"
+fi
+
+echo $cmd
+eval $cmd

egs/libri_trans/st/ensemble.sh

+set -e
+
+gpu_num=1
+root_dir=/home/xuchen/st/Fairseq-S2T
+ckpt=/home/xuchen/st/checkpoints/mustc-v2/st
+
+model_txt=$1
+set=$2
+test_subset=$3
+
+#data_dir=/home/xuchen/st/data/mustc-v2/st_lcrm/en-de
+#test_subset=(tst-COMMON)
+
+data_dir=/media/data/tst/$set/en-de
+#test_subset=(office)
+#test_subset=(webrtc1)
+#test_subset=(adap2)
+
+data_config=config_st_share.yaml
+result_file=./result
+
+beam_size=5
+lenpen=0.6
+max_tokens=10000
+
+models=()
+i=0
+for line in `cat $model_txt`; do
+    i=`expr $i + 1`
+    
+    model_dir=$ckpt/$line
+    [[ ! -d $model_dir ]] && echo $model_dir && exit 1;
+
+    if [[ -f $model_dir/avg_10_checkpoint.pt ]]; then
+        model=$model_dir/avg_10_checkpoint.pt
+    else
+        model=$model_dir/checkpoint_best.pt
+    fi
+    [[ ! -f $model ]] && echo $model && exit 1;
+
+    models[$i]=$model
+done
+
+models=`echo ${models[*]} | sed 's/ /:/g'`
+
+res_dir=$ckpt/ensemble/$set
+i=0
+while : 
+do
+    if [[ -d $res_dir/$i ]]; then
+        i=`expr $i + 1`
+    else
+        res_dir=$res_dir/$i
+        break
+    fi 
+done
+
+mkdir -p $res_dir
+cp $model_txt $res_dir
+
+
+if [[ -z ${device} || ${#device[@]} -eq 0 ]]; then
+    if [[ ${gpu_num} -eq 0 ]]; then
+        device=()
+    else
+        source ./local/utils.sh
+        device=$(get_devices $gpu_num 0)
+    fi
+fi
+export CUDA_VISIBLE_DEVICES=${device}
+
+for subset in ${test_subset[@]}; do
+    subset=${subset}_st
+    cmd="python ${root_dir}/fairseq_cli/generate.py
+    ${data_dir}
+    --config-yaml ${data_config}
+    --gen-subset ${subset}
+    --task speech_to_text
+    --path ${models}
+    --results-path ${res_dir}
+    --skip-invalid-size-inputs-valid-test
+    --max-tokens ${max_tokens}
+    --beam ${beam_size}
+    --lenpen ${lenpen}
+    --scoring sacrebleu"
+    echo -e "\033[34mRun command: \n${cmd} \033[0m"
+
+    eval $cmd
+    tail -n 1 ${res_dir}/generate-${subset}.txt
+
+    cd $res_dir
+    evaluate.sh translation-${subset}.txt $set
+    cd -
+done
+

egs/libri_trans/st/local/monitor.sh

+gpu_num=4
+cmd="sh train.sh"
+
+while :
+do
+    record=$(mktemp -t temp.record.XXXXXX)
+    gpustat > $record
+    all_devices=$(seq 0 "$(sed '1,2d' ${record} | wc -l)");
+
+    count=0
+    for dev in ${all_devices[@]}
+    do
+        line=$((dev + 2))
+        use=$(head -n $line ${record} | tail -1 | cut -d '|' -f3 | cut -d '/' -f1)
+
+        if [[ $use -lt 100 ]]; then
+            device[$count]=$dev
+            count=$((count + 1))
+            if [[ $count -eq $gpu_num ]]; then
+                break
+            fi
+        fi
+    done
+    if [[ ${#device[@]} -lt $gpu_num ]]; then
+        sleep 60s
+    else
+        echo "Run $cmd"
+        eval $cmd
+        sleep 10s
+        exit
+    fi
+done

egs/libri_trans/st/local/parse_options.sh

+#!/usr/bin/env bash
+
+# Copyright 2012  Johns Hopkins University (Author: Daniel Povey);
+#                 Arnab Ghoshal, Karel Vesely
+
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#  http://www.apache.org/licenses/LICENSE-2.0
+#
+# THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED
+# WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE,
+# MERCHANTABLITY OR NON-INFRINGEMENT.
+# See the Apache 2 License for the specific language governing permissions and
+# limitations under the License.
+
+
+# Parse command-line options.
+# To be sourced by another script (as in ". parse_options.sh").
+# Option format is: --option-name arg
+# and shell variable "option_name" gets set to value "arg."
+# The exception is --help, which takes no arguments, but prints the
+# $help_message variable (if defined).
+
+
+###
+### The --config file options have lower priority to command line
+### options, so we need to import them first...
+###
+
+# Now import all the configs specified by command-line, in left-to-right order
+for ((argpos=1; argpos<$#; argpos++)); do
+  if [ "${!argpos}" == "--config" ]; then
+    argpos_plus1=$((argpos+1))
+    config=${!argpos_plus1}
+    [ ! -r $config ] && echo "$0: missing config '$config'" && exit 1
+    . $config  # source the config file.
+  fi
+done
+
+
+###
+### Now we process the command line options
+###
+while true; do
+  [ -z "${1:-}" ] && break;  # break if there are no arguments
+  case "$1" in
+    # If the enclosing script is called with --help option, print the help
+    # message and exit.  Scripts should put help messages in $help_message
+    --help|-h) if [ -z "$help_message" ]; then echo "No help found." 1>&2;
+      else printf "$help_message\n" 1>&2 ; fi;
+      exit 0 ;;
+    --*=*) echo "$0: options to scripts must be of the form --name value, got '$1'"
+      exit 1 ;;
+    # If the first command-line argument begins with "--" (e.g. --foo-bar),
+    # then work out the variable name as $name, which will equal "foo_bar".
+    --*) name=`echo "$1" | sed s/^--// | sed s/-/_/g`;
+      # Next we test whether the variable in question is undefned-- if so it's
+      # an invalid option and we die.  Note: $0 evaluates to the name of the
+      # enclosing script.
+      # The test [ -z ${foo_bar+xxx} ] will return true if the variable foo_bar
+      # is undefined.  We then have to wrap this test inside "eval" because
+      # foo_bar is itself inside a variable ($name).
+      eval '[ -z "${'$name'+xxx}" ]' && echo "$0: invalid option $1" 1>&2 && exit 1;
+
+      oldval="`eval echo \\$$name`";
+      # Work out whether we seem to be expecting a Boolean argument.
+      if [ "$oldval" == "true" ] || [ "$oldval" == "false" ]; then
+        was_bool=true;
+      else
+        was_bool=false;
+      fi
+
+      # Set the variable to the right value-- the escaped quotes make it work if
+      # the option had spaces, like --cmd "queue.pl -sync y"
+      eval $name=\"$2\";
+
+      # Check that Boolean-valued arguments are really Boolean.
+      if $was_bool && [[ "$2" != "true" && "$2" != "false" ]]; then
+        echo "$0: expected \"true\" or \"false\": $1 $2" 1>&2
+        exit 1;
+      fi
+      shift 2;
+      ;;
+  *) break;
+  esac
+done
+
+
+# Check for an empty argument to the --cmd option, which can easily occur as a
+# result of scripting errors.
+[ ! -z "${cmd+xxx}" ] && [ -z "$cmd" ] && echo "$0: empty argument to --cmd option" 1>&2 && exit 1;
+
+
+true; # so this script returns exit code 0.

egs/libri_trans/st/local/utils.sh

+
+get_devices(){
+    gpu_num=$1
+    use_cpu=$2
+    device=()
+    while :
+    do
+        record=$(mktemp -t temp.record.XXXXXX)
+        gpustat > $record
+        all_devices=$(seq 0 "$(sed '1,2d' ${record} | wc -l)");
+
+        count=0
+        for dev in ${all_devices[@]}
+        do
+            line=$((dev + 2))
+            use=$(head -n $line ${record} | tail -1 | cut -d '|' -f3 | cut -d '/' -f1)
+            if [[ $use -lt 100 ]]; then
+                device[$count]=$dev
+                count=$((count + 1))
+                if [[ $count -eq $gpu_num ]]; then
+                    break
+                fi
+            fi
+        done
+        if [[ ${#device[@]} -lt $gpu_num ]]; then
+            if [[ $use_cpu -eq 1 ]]; then
+                device=(-1)
+            else
+                sleep 60s
+            fi
+        else
+            break
+        fi
+    done
+
+    echo ${device[*]} | sed 's/ /,/g'
+    return $?
+}
+
+

egs/libri_trans/st/run.sh

+#! /bin/bash
+
+# Processing MuST-C Datasets
+
+# Copyright 2021 Natural Language Processing Laboratory 
+# Xu Chen (xuchenneu@163.com)
+
+# Set bash to 'debug' mode, it will exit on :
+# -e 'error', -u 'undefined variable', -o ... 'error in pipeline', -x 'print commands',
+set -e
+#set -u
+set -o pipefail
+export PYTHONIOENCODING=UTF-8
+
+eval=1
+time=$(date "+%m%d_%H%M")
+
+stage=0
+stop_stage=0
+
+######## hardware ########
+# devices
+#device=()
+gpu_num=8
+update_freq=1
+
+root_dir=~/st/Fairseq-S2T
+pwd_dir=$PWD
+
+# dataset
+src_lang=en
+tgt_lang=de
+lang=${src_lang}-${tgt_lang}
+
+dataset=st
+task=speech_to_text
+vocab_type=unigram
+asr_vocab_size=5000
+vocab_size=10000
+share_dict=1
+speed_perturb=0
+lcrm=0
+tokenizer=0
+
+use_specific_dict=0
+specific_prefix=valid
+specific_dir=/home/xuchen/st/data/mustc/st_lcrm/en-de
+asr_vocab_prefix=spm_unigram10000_st_share
+st_vocab_prefix=spm_unigram10000_st_share
+
+org_data_dir=~/st/data/${dataset}
+data_dir=~/st/data/${dataset}/st
+test_subset=tst-COMMON
+
+# exp
+exp_prefix=$(date "+%m%d")
+extra_tag=
+extra_parameter=
+exp_tag=baseline
+exp_name=
+
+# config
+train_config=ctc
+
+# training setting
+fp16=1
+max_tokens=40000
+step_valid=0
+bleu_valid=0
+
+# decoding setting
+dec_model=checkpoint_best.pt
+n_average=10
+beam_size=5
+len_penalty=1.0
+
+if [[ ${share_dict} -eq 1 ]]; then
+	data_config=config_st_share.yaml
+else
+	data_config=config_st.yaml
+fi
+if [[ ${speed_perturb} -eq 1 ]]; then
+    data_dir=${data_dir}_sp
+    exp_prefix=${exp_prefix}_sp
+fi
+if [[ ${lcrm} -eq 1 ]]; then
+    data_dir=${data_dir}_lcrm
+    exp_prefix=${exp_prefix}_lcrm
+fi
+if [[ ${use_specific_dict} -eq 1 ]]; then
+    data_dir=${data_dir}_${specific_prefix}
+    exp_prefix=${exp_prefix}_${specific_prefix}
+fi
+if [[ ${tokenizer} -eq 1 ]]; then
+    data_dir=${data_dir}_tok
+    exp_prefix=${exp_prefix}_tok
+fi
+
+. ./local/parse_options.sh || exit 1;
+
+if [[ -z ${exp_name} ]]; then
+    config_string=${train_config//,/_}
+    exp_name=${exp_prefix}_${config_string}_${exp_tag}
+    if [[ -n ${extra_tag} ]]; then
+        exp_name=${exp_name}_${extra_tag}
+    fi
+fi
+model_dir=$root_dir/../checkpoints/$dataset/st/${exp_name}
+
+if [ ${stage} -le -1 ] && [ ${stop_stage} -ge -1 ]; then
+    echo "stage -1: Data Download"
+    # pass
+fi
+
+if [ ${stage} -le 0 ] && [ ${stop_stage} -ge 0 ]; then
+    ### Task dependent. You have to make data the following preparation part by yourself.
+    ### But you can utilize Kaldi recipes in most cases
+    echo "stage 0: ASR Data Preparation"
+    if [[ ! -e ${data_dir}/${lang} ]]; then
+        mkdir -p ${data_dir}/${lang}
+    fi
+
+    cmd="python ${root_dir}/examples/speech_to_text/prep_asr_data.py
+        --data-root ${org_data_dir}
+        --output-root ${data_dir}
+        --task asr
+        --vocab-type ${vocab_type}
+        --vocab-size ${asr_vocab_size}"
+    if [[ ${speed_perturb} -eq 1 ]]; then
+        cmd="$cmd
+        --speed-perturb"
+    fi
+    echo -e "\033[34mRun command: \n${cmd} \033[0m"
+    [[ $eval -eq 1 && ${share_dict} -ne 1 && ${use_specific_dict} -ne 1 ]] && eval $cmd
+    asr_prefix=spm_${vocab_type}${asr_vocab_size}_asr
+
+    echo "stage 0: ST Data Preparation"
+    cmd="python ${root_dir}/examples/speech_to_text/prep_st_data.py
+        --data-root ${org_data_dir}
+        --output-root ${data_dir}
+        --task st
+        --add-src
+        --cmvn-type utterance
+        --vocab-type ${vocab_type}
+        --vocab-size ${vocab_size}"
+
+    if [[ ${use_specific_dict} -eq 1 ]]; then
+        cp -r ${specific_dir}/${asr_vocab_prefix}.* ${data_dir}/${lang}
+        cp -r ${specific_dir}/${st_vocab_prefix}.* ${data_dir}/${lang}
+        if [[ $share_dict -eq 1 ]]; then
+            cmd="$cmd
+        --share
+        --st-spm-prefix ${st_vocab_prefix}"
+        else
+            cmd="$cmd
+        --st-spm-prefix ${st_vocab_prefix}
+        --asr-prefix ${asr_vocab_prefix}"
+        fi
+    else
+        if [[ $share_dict -eq 1 ]]; then
+            cmd="$cmd
+        --share"
+        else
+            cmd="$cmd
+        --asr-prefix ${asr_prefix}"
+        fi
+    fi
+    if [[ ${speed_perturb} -eq 1 ]]; then
+        cmd="$cmd
+        --speed-perturb"
+    fi
+    if [[ ${lcrm} -eq 1 ]]; then
+        cmd="$cmd
+        --lowercase-src
+        --rm-punc-src"
+    fi
+    if [[ ${tokenizer} -eq 1 ]]; then
+        cmd="$cmd
+        --tokenizer"
+    fi
+
+    echo -e "\033[34mRun command: \n${cmd} \033[0m"
+    [[ $eval -eq 1 ]] && eval ${cmd}
+fi
+
+data_dir=${data_dir}/${lang}
+
+if [ ${stage} -le 1 ] && [ ${stop_stage} -ge 1 ]; then
+    echo "stage 1: ST Network Training"
+    [[ ! -d ${data_dir} ]] && echo "The data dir ${data_dir} is not existing!" && exit 1;
+
+    if [[ -z ${device} || ${#device[@]} -eq 0 ]]; then
+		if [[ ${gpu_num} -eq 0 ]]; then
+			device=""
+		else
+        	source ./local/utils.sh
+        	device=$(get_devices $gpu_num 0)
+		fi
+    fi
+
+    echo -e "dev=${device} data=${data_dir} model=${model_dir}"
+
+    if [[ ! -d ${model_dir} ]]; then
+        mkdir -p ${model_dir}
+    else
+        echo "${model_dir} exists."
+    fi
+
+    cp ${BASH_SOURCE[0]} ${model_dir}
+    cp ${PWD}/train.sh ${model_dir}
+
+    config_list="${train_config//,/ }"
+    idx=0
+    for config in ${config_list[@]}
+    do
+        config_path=$pwd_dir/conf/${config}.yaml
+        if [[ ! -f ${config_path} ]]; then
+            echo "No config file ${config_path}"
+            exit
+        fi
+        cp ${config_path} ${model_dir}
+
+        if [[ idx -eq 0 ]]; then
+            extra_parameter="${extra_parameter}
+        --train-config ${config_path}"
+        else
+            extra_parameter="${extra_parameter}
+        --train-config${idx} ${config_path}"
+        fi
+        idx=$((idx + 1))
+    done
+
+    cmd="python3 -u ${root_dir}/fairseq_cli/train.py
+        ${data_dir}
+        --config-yaml ${data_config}
+        --task ${task}
+        --max-tokens ${max_tokens}
+        --skip-invalid-size-inputs-valid-test
+        --update-freq ${update_freq}
+        --log-interval 100
+        --save-dir ${model_dir}
+        --tensorboard-logdir ${model_dir}"
+
+	if [[ -n ${extra_parameter} ]]; then
+        cmd="${cmd}
+        ${extra_parameter}"
+    fi
+	if [[ ${gpu_num} -gt 0 ]]; then
+		cmd="${cmd}
+        --distributed-world-size $gpu_num
+        --ddp-backend no_c10d"
+	fi
+    if [[ $fp16 -eq 1 ]]; then
+        cmd="${cmd}
+        --fp16"
+    fi
+    if [[ $step_valid -eq 1 ]]; then
+        validate_interval=1
+        save_interval=1
+        keep_last_epochs=10
+        no_epoch_checkpoints=0
+        save_interval_updates=500
+        keep_interval_updates=10
+    else
+        validate_interval=1
+        keep_last_epochs=10
+    fi
+    if [[ $bleu_valid -eq 1 ]]; then
+        cmd="$cmd
+        --eval-bleu
+        --eval-bleu-args '{\"beam\": 1}'
+        --eval-tokenized-bleu
+        --eval-bleu-remove-bpe
+        --best-checkpoint-metric bleu
+        --maximize-best-checkpoint-metric"
+    fi
+    if [[ -n $no_epoch_checkpoints && $no_epoch_checkpoints -eq 1 ]]; then
+        cmd="$cmd
+        --no-epoch-checkpoints"
+    fi
+    if [[ -n $validate_interval ]]; then
+        cmd="${cmd}
+        --validate-interval $validate_interval "
+    fi
+    if [[ -n $save_interval ]]; then
+        cmd="${cmd}
+        --save-interval $save_interval "
+    fi
+    if [[ -n $keep_last_epochs ]]; then
+        cmd="${cmd}
+        --keep-last-epochs $keep_last_epochs "
+    fi
+    if [[ -n $save_interval_updates ]]; then
+        cmd="${cmd}
+        --save-interval-updates $save_interval_updates"
+        if [[ -n $keep_interval_updates ]]; then
+        cmd="${cmd}
+        --keep-interval-updates $keep_interval_updates"
+        fi
+    fi
+
+    echo -e "\033[34mRun command: \n${cmd} \033[0m"
+
+    # save info
+    log=./history.log
+    echo "${time} | ${device} | ${data_dir} | ${exp_name} | ${model_dir} " >> $log
+    tail -n 50 ${log} > tmp.log
+    mv tmp.log $log
+    export CUDA_VISIBLE_DEVICES=${device}
+
+    cmd="nohup ${cmd} >> ${model_dir}/train.log 2>&1 &"
+    if [[ $eval -eq 1 ]]; then
+		eval $cmd
+		sleep 2s
+		tail -n "$(wc -l ${model_dir}/train.log | awk '{print $1+1}')" -f ${model_dir}/train.log
+	fi
+fi
+wait
+
+if [ ${stage} -le 2 ] && [ ${stop_stage} -ge 2 ]; then
+    echo "stage 2: ST Decoding"
+    if [[ ${n_average} -ne 1 ]]; then
+        # Average models
+		dec_model=avg_${n_average}_checkpoint.pt
+
+		cmd="python ${root_dir}/scripts/average_checkpoints.py
+        --inputs ${model_dir}
+        --num-epoch-checkpoints ${n_average}
+        --output ${model_dir}/${dec_model}"
+    	echo -e "\033[34mRun command: \n${cmd} \033[0m"
+    	[[ $eval -eq 1 ]] && eval $cmd
+	else
+		dec_model=${dec_model}
+	fi
+
+    if [[ -z ${device} || ${#device[@]} -eq 0 ]]; then
+		if [[ ${gpu_num} -eq 0 ]]; then
+			device=""
+		else
+        	source ./local/utils.sh
+        	device=$(get_devices $gpu_num 0)
+		fi
+    fi
+    export CUDA_VISIBLE_DEVICES=${device}
+
+	result_file=${model_dir}/decode_result
+	[[ -f ${result_file} ]] && rm ${result_file}
+
+    test_subset=${test_subset//,/ }
+	for subset in "${test_subset[@]}"; do
+        subset=${subset}_st
+  		cmd="python ${root_dir}/fairseq_cli/generate.py
+        ${data_dir}
+        --config-yaml ${data_config}
+        --gen-subset ${subset}
+        --task speech_to_text
+        --path ${model_dir}/${dec_model}
+        --results-path ${model_dir}
+        --max-tokens ${max_tokens}
+        --beam ${beam_size}
+        --lenpen ${len_penalty}
+        --scoring sacrebleu"
+    	echo -e "\033[34mRun command: \n${cmd} \033[0m"
+
+        if [[ $eval -eq 1 ]]; then
+    	    eval $cmd
+    	    tail -n 1 ${model_dir}/generate-${subset}.txt >> ${result_file}
+        fi
+	done
+    cat ${result_file}
+fi

egs/libri_trans/st/train.sh

+#! /bin/bash
+
+# training the model
+
+gpu_num=8
+update_freq=1
+max_tokens=40000
+
+extra_tag=
+extra_parameter=
+#extra_tag="${extra_tag}"
+#extra_parameter="${extra_parameter} "
+
+exp_tag=
+
+#config_list=(base)
+config_list=(ctc)
+#config_list=(sate_ctc)
+#config_list=(ctc conformer rpr)
+#config_list=(base sate)
+
+#config_list=(pds_base)
+#config_list=(pds_base conformer)
+
+# exp full name
+exp_name=
+
+train_config=$(echo ${config_list[*]} | sed 's/ /,/g')
+
+cmd="./run.sh
+    --stage 1
+    --stop_stage 1
+    --gpu_num ${gpu_num}
+    --update_freq ${update_freq}
+    --train_config ${train_config}
+    --max_tokens ${max_tokens}
+    "
+
+if [[ -n ${exp_name} ]]; then
+    cmd="$cmd --exp_name ${exp_name}"
+fi
+if [[ -n ${exp_tag} ]]; then
+    cmd="$cmd --exp_tag ${exp_tag}"
+fi
+if [[ -n ${extra_tag} ]]; then
+    cmd="$cmd --extra_tag ${extra_tag}"
+fi
+if [[ -n ${extra_parameter} ]]; then
+    cmd="$cmd --extra_parameter \"${extra_parameter}\""
+fi
+
+echo ${cmd}
+eval ${cmd}

egs/mustc/asr/run.sh

@@ -39,6 +39,7 @@ vocab_size=5000
 speed_perturb=0
 lcrm=0
 tokenizer=0
+use_raw_audio=0
 
 use_specific_dict=0
 specific_prefix=st
@@ -87,6 +88,10 @@ if [[ ${tokenizer} -eq 1 ]]; then
     data_dir=${data_dir}_tok
     exp_prefix=${exp_prefix}_tok
 fi
+if [[ ${use_raw_audio} -eq 1 ]]; then
+    data_dir=${data_dir}_raw
+    exp_prefix=${exp_prefix}_raw
+fi
 
 . ./local/parse_options.sh || exit 1;
 
@@ -109,9 +114,6 @@ if [ ${stage} -le 0 ] && [ ${stop_stage} -ge 0 ]; then
     ### Task dependent. You have to make data the following preparation part by yourself.
     ### But you can utilize Kaldi recipes in most cases
     echo "stage 0: ASR Data Preparation"
-    if [[ ! -e ${data_dir}/${lang} ]]; then
-        mkdir -p ${data_dir}/${lang}
-    fi
 
     cmd="python ${root_dir}/examples/speech_to_text/prep_mustc_data.py
         --data-root ${org_data_dir}
@@ -120,6 +122,10 @@ if [ ${stage} -le 0 ] && [ ${stop_stage} -ge 0 ]; then
         --vocab-type ${vocab_type}
         --vocab-size ${vocab_size}"
 
+    if [[ ${use_raw_audio} -eq 1 ]]; then
+        cmd="$cmd
+        --raw"
+    fi
     if [[ ${use_specific_dict} -eq 1 ]]; then
         cp -r ${specific_dir}/${asr_vocab_prefix}.* ${data_dir}/${lang}
         cmd="$cmd
@@ -143,8 +149,6 @@ if [ ${stage} -le 0 ] && [ ${stop_stage} -ge 0 ]; then
     [[ $eval -eq 1 ]] && eval ${cmd}
 fi
 
-data_dir=${data_dir}/${lang}
-
 if [ ${stage} -le 1 ] && [ ${stop_stage} -ge 1 ]; then
     echo "stage 1: ASR Network Training"
     [[ ! -d ${data_dir} ]] && echo "The data dir ${data_dir} is not existing!" && exit 1;

egs/template/asr/run.sh

@@ -36,7 +36,7 @@ task=speech_to_text
 vocab_type=unigram
 vocab_size=5000
 speed_perturb=0
-lcrm=0
+lcrm=1
 tokenizer=0
 
 use_specific_dict=0
@@ -47,7 +47,7 @@ asr_vocab_prefix=spm_unigram10000_st_share
 org_data_dir=~/st/data/${dataset}
 data_dir=~/st/data/${dataset}/asr
 train_split=train
-valid_split=valid
+valid_split=dev
 test_split=test
 test_subset=test
 
@@ -117,9 +117,9 @@ if [ ${stage} -le 0 ] && [ ${stop_stage} -ge 0 ]; then
     cmd="python ${root_dir}/examples/speech_to_text/prep_asr_data.py
         --data-root ${org_data_dir}
         --output-root ${data_dir}
-		--task asr
-		--splits ${train_split},${valid_split},${test_split}
-		--lang ${lang}
+        --task asr
+        --splits ${train_split},${valid_split},${test_split}
+        --lang ${lang}
         --vocab-type ${vocab_type}
         --vocab-size ${vocab_size}"
 

examples/speech_to_text/prep_asr_data.py

-#!/usr/bin/env python3
-# Copyright (c) Facebook, Inc. and its affiliates.
-#
-# This source code is licensed under the MIT license found in the
-# LICENSE file in the root directory of this source tree.
-
-import argparse
-import logging
-import os
-from pathlib import Path
-import shutil
-from itertools import groupby
-from tempfile import NamedTemporaryFile
-import string
-import csv
-
-import numpy as np
-import pandas as pd
-import torchaudio
-from examples.speech_to_text.data_utils import (
-    create_zip,
-    extract_fbank_features,
-    filter_manifest_df,
-    gen_config_yaml,
-    gen_vocab,
-    get_zip_manifest,
-    load_df_from_tsv,
-    save_df_to_tsv,
-    cal_gcmvn_stats,
-)
-from torch.utils.data import Dataset
-from tqdm import tqdm
-
-
-log = logging.getLogger(__name__)
-
-
-MANIFEST_COLUMNS = ["id", "audio", "n_frames", "tgt_text", "speaker"]
-
-
-class ASRDataset(Dataset):
-    """
-    Create a Dataset for MuST-C. Each item is a tuple of the form:
-    waveform, sample_rate, source utterance, target utterance, speaker_id,
-    utterance_id
-    """
-
-    def __init__(self, root: str, lang, split: str, speed_perturb: bool = False, tokenizer: bool = False) -> None:
-        _root = Path(root) / f"{lang}" / split
-        wav_root, txt_root = _root / "wav", _root / "txt"
-        if tokenizer:
-            txt_root = _root / "txt.tok"
-        assert _root.is_dir() and wav_root.is_dir() and txt_root.is_dir(), (_root, wav_root, txt_root)
-        # Load audio segments
-        try:
-            import yaml
-        except ImportError:
-            print("Please install PyYAML to load the MuST-C YAML files")
-        with open(txt_root / f"{split}.yaml") as f:
-            segments = yaml.load(f, Loader=yaml.BaseLoader)
-
-        self.speed_perturb = [0.9, 1.0, 1.1] if speed_perturb and split.startswith("train") else None
-        # Load source and target utterances
-        with open(txt_root / f"{split}.{lang}") as f:
-            utterances = [r.strip() for r in f]
-        assert len(segments) == len(utterances)
-        for i, u in enumerate(utterances):
-            segments[i][lang] = u
-        # Gather info
-        self.data = []
-        for wav_filename, _seg_group in groupby(segments, lambda x: x["wav"]):
-            wav_path = wav_root / wav_filename
-            sample_rate = torchaudio.info(wav_path.as_posix()).sample_rate
-            seg_group = sorted(_seg_group, key=lambda x: float(x["offset"]))
-            for i, segment in enumerate(seg_group):
-                offset = int(float(segment["offset"]) * sample_rate)
-                n_frames = int(float(segment["duration"]) * sample_rate)
-                _id = f"{split}_{wav_path.stem}_{i}"
-                self.data.append(
-                    (
-                        wav_path.as_posix(),
-                        offset,
-                        n_frames,
-                        sample_rate,
-                        segment[lang],
-                        segment["speaker_id"] if "speaker_id" in segment else "spk1",
-                        _id,
-                    )
-                )
-
-    def __getitem__(self, n: int):
-        wav_path, offset, n_frames, sr, utt, spk_id, utt_id = self.data[n]
-
-        items = []
-        if self.speed_perturb is None:
-            waveform, _ = torchaudio.load(wav_path, frame_offset=offset, num_frames=n_frames)
-            items.append([waveform, sr, n_frames, utt, spk_id, utt_id])
-        else:
-            for speed in self.speed_perturb:
-                sp_utt_id = f"sp{speed}_" + utt_id
-                sp_n_frames = n_frames / speed
-                if speed == 1.0:
-                    waveform, _ = torchaudio.load(wav_path, frame_offset=offset, num_frames=n_frames)
-                else:
-                    waveform, _ = torchaudio.load(wav_path, frame_offset=offset, num_frames=n_frames)
-                    effects = [
-                        ["speed", f"{speed}"],
-                        ["rate", f"{sr}"]
-                    ]
-                    waveform, _ = torchaudio.sox_effects.apply_effects_tensor(waveform, sr, effects)
-
-                items.append([waveform, sr, sp_n_frames, utt, spk_id, sp_utt_id])
-        return items
-
-    def get_wav(self, n: int, speed_perturb=1.0):
-        wav_path, offset, n_frames, sr, utt, spk_id, utt_id = self.data[n]
-
-        if self.speed_perturb is None or speed_perturb == 1.0:
-            waveform, _ = torchaudio.load(wav_path, frame_offset=offset, num_frames=n_frames)
-        else:
-            waveform, _ = torchaudio.load(wav_path, frame_offset=offset, num_frames=n_frames)
-            effects = [
-                ["speed", f"{speed_perturb}"],
-                ["rate", f"{sr}"]
-            ]
-            waveform, _ = torchaudio.sox_effects.apply_effects_tensor(waveform, sr, effects)
-        return waveform
-
-    def get_fast(self, n: int):
-        wav_path, offset, n_frames, sr, utt, spk_id, utt_id = self.data[n]
-
-        items = []
-        if self.speed_perturb is None:
-            items.append([wav_path, sr, n_frames, utt, spk_id, utt_id])
-        else:
-            for speed in self.speed_perturb:
-                sp_utt_id = f"sp{speed}_" + utt_id
-                sp_n_frames = n_frames / speed
-                items.append([wav_path, sr, sp_n_frames, utt, spk_id, sp_utt_id])
-        return items
-
-    def get_text(self):
-        src_text = []
-        for item in self.data:
-            src_text.append(item[4])
-        return src_text
-
-    def __len__(self) -> int:
-        return len(self.data)
-
-
-def process(args):
-    root = Path(args.data_root).absolute()
-    splits = args.splits.split(",")
-    lang = args.lang
-    cur_root = root / f"{lang}"
-    if not cur_root.is_dir():
-        print(f"{cur_root.as_posix()} does not exist. Skipped.")
-        return
-    if args.output_root is None:
-        output_root = cur_root
-    else:
-        output_root = Path(args.output_root).absolute() / f"{lang}"
-
-    # Extract features
-    if args.speed_perturb:
-        zip_path = output_root / "fbank80_sp.zip"
-    else:
-        zip_path = output_root / "fbank80.zip"
-    index = 0
-
-    gen_feature_flag = False
-    if not Path.exists(zip_path):
-        gen_feature_flag = True
-
-    if args.overwrite or gen_feature_flag:
-        if args.speed_perturb:
-            feature_root = output_root / "fbank80_sp"
-        else:
-            feature_root = output_root / "fbank80"
-        feature_root.mkdir(exist_ok=True)
-
-        for split in splits:
-            print(f"Fetching split {split}...")
-            dataset = ASRDataset(root.as_posix(), lang, split, args.speed_perturb, args.tokenizer)
-            is_train_split = split.startswith("train")
-            print("Extracting log mel filter bank features...")
-            if is_train_split and args.cmvn_type == "global":
-                print("And estimating cepstral mean and variance stats...")
-                gcmvn_feature_list = []
-
-            for idx in tqdm(range(len(dataset))):
-                items = dataset.get_fast(idx)
-                for item in items:
-                    index += 1
-                    wav_path, sr, _, _, _, utt_id = item
-
-                    features_path = (feature_root / f"{utt_id}.npy").as_posix()
-                    if not os.path.exists(features_path):
-                        sp = 1.0
-                        if dataset.speed_perturb is not None:
-                            sp = float(utt_id.split("_")[0].replace("sp", ""))
-                        waveform = dataset.get_wav(idx, sp)
-                        if waveform.shape[1] == 0:
-                            continue
-                        features = extract_fbank_features(waveform, sr, Path(features_path))
-
-                        if split == 'train' and args.cmvn_type == "global" and not utt_id.startswith("sp"):
-                            if len(gcmvn_feature_list) < args.gcmvn_max_num:
-                                gcmvn_feature_list.append(features)
-
-                if is_train_split and args.size != -1 and index > args.size:
-                    break
-
-            if is_train_split and args.cmvn_type == "global":
-                # Estimate and save cmv
-                stats = cal_gcmvn_stats(gcmvn_feature_list)
-                with open(output_root / "gcmvn.npz", "wb") as f:
-                    np.savez(f, mean=stats["mean"], std=stats["std"])
-
-        # Pack features into ZIP
-        print("ZIPing features...")
-        create_zip(feature_root, zip_path)
-
-        # Clean up
-        shutil.rmtree(feature_root)
-
-    gen_manifest_flag = False
-    for split in splits:
-        if not Path.exists(output_root / f"{split}_{args.task}.tsv"):
-            gen_manifest_flag = True
-            break
-
-    train_text = []
-    if args.overwrite or gen_manifest_flag:
-        print("Fetching ZIP manifest...")
-        zip_manifest = get_zip_manifest(zip_path)
-        # Generate TSV manifest
-        print("Generating manifest...")
-        for split in splits:
-            is_train_split = split.startswith("train")
-            manifest = {c: [] for c in MANIFEST_COLUMNS}
-            if args.task == "st" and args.add_src:
-                manifest["src_text"] = []
-
-            dataset = ASRDataset(args.data_root, lang, split, args.speed_perturb, args.tokenizer)
-            for idx in range(len(dataset)):
-                items = dataset.get_fast(idx)
-                for item in items:
-                    _, sr, n_frames, utt, speaker_id, utt_id = item
-                    manifest["id"].append(utt_id)
-                    manifest["audio"].append(zip_manifest[utt_id])
-                    duration_ms = int(n_frames / sr * 1000)
-                    manifest["n_frames"].append(int(1 + (duration_ms - 25) / 10))
-                    if args.lowercase_src:
-                        utt = utt.lower()
-                    if args.rm_punc_src:
-                        for w in string.punctuation:
-                            utt = utt.replace(w, "")
-                    manifest["tgt_text"].append(utt)
-                    manifest["speaker"].append(speaker_id)
-
-                if is_train_split and args.size != -1 and len(manifest["id"]) > args.size:
-                    break
-            if is_train_split:
-                train_text.extend(manifest["tgt_text"])
-
-            df = pd.DataFrame.from_dict(manifest)
-            df = filter_manifest_df(df, is_train_split=is_train_split)
-            save_df_to_tsv(df, output_root / f"{split}_{args.task}.tsv")
-
-    # Generate vocab
-    v_size_str = "" if args.vocab_type == "char" else str(args.vocab_size)
-    spm_filename_prefix = f"spm_{args.vocab_type}{v_size_str}_{args.task}"
-    gen_vocab_flag = True
-
-    if args.asr_prefix is not None:
-        gen_vocab_flag = False
-        spm_filename_prefix = args.asr_prefix
-
-    if gen_vocab_flag:
-        if len(train_text) == 0:
-            print("Loading the training text to build dictionary...")
-
-            for split in args.SPLITS:
-                if split.startswith("train"):
-                    csv_path = output_root / f"{split}_{args.task}.tsv"
-                    with open(csv_path) as f:
-                        reader = csv.DictReader(
-                            f,
-                            delimiter="\t",
-                            quotechar=None,
-                            doublequote=False,
-                            lineterminator="\n",
-                            quoting=csv.QUOTE_NONE,
-                        )
-
-                    tgt_text = [dict(e)["tgt_text"] for e in reader]
-                    train_text.extend(tgt_text)
-
-        with NamedTemporaryFile(mode="w") as f:
-            for t in train_text:
-                f.write(t + "\n")
-            gen_vocab(
-                Path(f.name),
-                output_root / spm_filename_prefix,
-                args.vocab_type,
-                args.vocab_size,
-            )
-
-    # Generate config YAML
-    yaml_filename = f"config_{args.task}.yaml"
-
-    gen_config_yaml(
-        output_root,
-        spm_filename_prefix + ".model",
-        yaml_filename=yaml_filename,
-        specaugment_policy="lb",
-        cmvn_type=args.cmvn_type,
-        gcmvn_path=(
-            output_root / "gcmvn.npz" if args.cmvn_type == "global"
-            else None
-        ),
-        share_src_and_tgt=True if args.task == "asr" else False
-    )
-
-
-def main():
-    parser = argparse.ArgumentParser()
-    parser.add_argument("--data-root", "-d", required=True, type=str)
-    parser.add_argument("--output-root", "-o", default=None, type=str)
-    parser.add_argument(
-        "--vocab-type",
-        default="unigram",
-        required=True,
-        type=str,
-        choices=["bpe", "unigram", "char"],
-    ),
-    parser.add_argument("--vocab-size", default=8000, type=int)
-    parser.add_argument("--task", type=str, default="asr", choices=["asr", "st"])
-    parser.add_argument("--lang", type=str, required=True, help="language")
-    parser.add_argument("--splits", type=str, default="train,dev,test", help="dataset splits")
-    parser.add_argument("--speed-perturb", action="store_true", default=False,
-                        help="apply speed perturbation on wave file")
-    parser.add_argument("--share", action="store_true",
-                        help="share the tokenizer and dictionary of the transcription and translation")
-    parser.add_argument("--asr-prefix", type=str, default=None, help="prefix of the asr dict")
-    parser.add_argument("--lowercase-src", action="store_true", help="lowercase the source text")
-    parser.add_argument("--rm-punc-src", action="store_true", help="remove the punctuation of the source text")
-    parser.add_argument("--tokenizer", action="store_true", help="use tokenizer txt")
-    parser.add_argument("--cmvn-type", default="utterance",
-                        choices=["global", "utterance"],
-                        help="The type of cepstral mean and variance normalization")
-    parser.add_argument("--overwrite", action="store_true", help="overwrite the existing files")
-    parser.add_argument("--gcmvn-max-num", default=150000, type=int,
-                        help=(
-                            "Maximum number of sentences to use to estimate"
-                            "global mean and variance"
-                            ))
-    args = parser.parse_args()
-
-    process(args)
-
-
-if __name__ == "__main__":
-    main()

examples/speech_to_text/prep_st_data.py → examples/speech_to_text/prep_audio_data.py

@@ -32,13 +32,13 @@ from torch.utils.data import Dataset
 from tqdm import tqdm
 
 
-log = logging.getLogger(__name__)
+logger = logging.getLogger(__name__)
 
 
-MANIFEST_COLUMNS = ["id", "audio", "n_frames", "tgt_text", "speaker"]
+MANIFEST_COLUMNS = ["id", "audio", "n_frames", "tgt_text"]
 
 
-class STDataset(Dataset):
+class AudioDataset(Dataset):
     """
     Create a Dataset for MuST-C. Each item is a tuple of the form:
     waveform, sample_rate, source utterance, target utterance, speaker_id,
@@ -46,113 +46,141 @@ class STDataset(Dataset):
     """
 
     def __init__(self, root: str, src_lang, tgt_lang: str, split: str,
-                 speed_perturb: bool = False, tokenizer: bool = False) -> None:
-        _root = Path(root) / f"{src_lang}-{tgt_lang}" / split
+                 speed_perturb: bool = False, size: int = -1, use_raw: bool = False,
+                 tokenizer: bool = False) -> None:
+        _root = Path(root) / "data" / split
         wav_root, txt_root = _root / "wav", _root / "txt"
         if tokenizer:
             txt_root = _root / "txt.tok"
-        assert _root.is_dir() and wav_root.is_dir() and txt_root.is_dir(), (_root, wav_root, txt_root)
-        # Load audio segments
-        try:
-            import yaml
-        except ImportError:
-            print("Please install PyYAML to load the MuST-C YAML files")
-        with open(txt_root / f"{split}.yaml") as f:
-            segments = yaml.load(f, Loader=yaml.BaseLoader)
+        assert wav_root.is_dir() and txt_root.is_dir(), (_root, wav_root, txt_root)
 
+        self.use_raw = use_raw
         self.speed_perturb = [0.9, 1.0, 1.1] if speed_perturb and split.startswith("train") else None
+        self.size = size if split.startswith("train") else -1
+
+        # Load audio segments
+        yaml_file = txt_root / f"{split}.yaml"
+        if yaml_file.is_file():
+            self.mode = "yaml"
+            try:
+                import yaml
+            except ImportError:
+                print("Please install PyYAML to load the MuST-C YAML files")
+
+            with open(yaml_file) as f:
+                segments = yaml.load(f, Loader=yaml.BaseLoader)
+        else:
+            self.mode = "easy"
+            audio_file = txt_root / f"{split}.audio"
+            assert audio_file.is_file(), audio_file
+            with open(audio_file) as f:
+                audios = [line.strip() for line in f.readlines()]
+
+            segments = dict()
+            for idx, audio in enumerate(audios):
+                segments[idx] = {"audio": audio}
+                if 0 < self.size < idx:
+                    break
+
         # Load source and target utterances
+        self.have_src_utt = False
+        self.have_tgt_utt = False
         for _lang in [src_lang, tgt_lang]:
-            with open(txt_root / f"{split}.{_lang}") as f:
-                utterances = [r.strip() for r in f]
-            assert len(segments) == len(utterances)
-            for i, u in enumerate(utterances):
-                segments[i][_lang] = u
+            if _lang is None:
+                continue
+            if Path.exists(txt_root / f"{split}.{_lang}"):
+                if _lang == src_lang:
+                    self.have_src_utt = True
+                else:
+                    self.have_tgt_utt = True
+                with open(txt_root / f"{split}.{_lang}") as f:
+                    utterances = [r.strip() for r in f]
+                assert len(audios) == len(utterances)
+                for idx, u in enumerate(utterances):
+                    segments[idx][_lang] = u
+                    if 0 < self.size < idx:
+                        break
+
         # Gather info
-        self.data = []
-        for wav_filename, _seg_group in groupby(segments, lambda x: x["wav"]):
-            wav_path = wav_root / wav_filename
-            sample_rate = torchaudio.info(wav_path.as_posix()).sample_rate
-            seg_group = sorted(_seg_group, key=lambda x: float(x["offset"]))
-            for i, segment in enumerate(seg_group):
-                offset = int(float(segment["offset"]) * sample_rate)
-                n_frames = int(float(segment["duration"]) * sample_rate)
-                _id = f"{split}_{wav_path.stem}_{i}"
-                self.data.append(
-                    (
-                        wav_path.as_posix(),
-                        offset,
-                        n_frames,
-                        sample_rate,
-                        segment[src_lang],
-                        segment[tgt_lang],
-                        segment["speaker_id"] if "speaker_id" in segment else "spk1",
-                        _id,
-                    )
-                )
+        self.data = dict()
+        if self.mode == "easy":
+            for idx, v in segments.items():
+                if self.speed_perturb is not None:
+                    for perturb in self.speed_perturb:
+                        v["sp"] = perturb
+                        v["id"] = f"{v['audio']}_sp{perturb}"
+                else:
+                    v["id"] = v['audio']
+                v["audio"] = (wav_root / v["audio"].strip()).as_posix() + ".wav"
+                self.data[idx] = v
+
+        elif self.mode == "yaml":
+            idx = 0
+            for wav_filename, _seg_group in groupby(segments, lambda x: x["wav"]):
+                wav_path = wav_root / wav_filename
+                sample_rate = torchaudio.info(wav_path.as_posix()).sample_rate
+                seg_group = sorted(_seg_group, key=lambda x: float(x["offset"]))
+                for i, segment in enumerate(seg_group):
+                    offset = int(float(segment["offset"]) * sample_rate)
+                    n_frames = int(float(segment["duration"]) * sample_rate)
+                    _id = f"{wav_path.stem}_{i}"
+
+                    item = dict()
+                    item["audio"] = wav_path.as_posix()
+                    item["offset"] = offset
+                    item["n_frames"] = n_frames
+                    item["sample_rate"] = sample_rate,
+                    item[src_lang] = segment[src_lang]
+                    if tgt_lang is not None:
+                        item[tgt_lang] = segment[tgt_lang]
+
+                    if self.speed_perturb is not None:
+                        for perturb in self.speed_perturb:
+                            sp_item = item
+                            sp_item["id"] = f"{_id}_sp{perturb}"
+                            sp_item["perturb"] = perturb
+                            self.data[idx] = sp_item
+                            idx += 1
+                    else:
+                        item["id"] = _id
+                        self.data[idx] = item
+                        idx += 1
+                    if 0 < self.size < idx:
+                        break
 
     def __getitem__(self, n: int):
-        wav_path, offset, n_frames, sr, src_utt, tgt_utt, spk_id, utt_id = self.data[n]
+        return self.data[n]
 
-        items = []
-        if self.speed_perturb is None:
-            waveform, _ = torchaudio.load(wav_path, frame_offset=offset, num_frames=n_frames)
-            items.append([waveform, sr, n_frames, src_utt, tgt_utt, spk_id, utt_id])
-        else:
-            for speed in self.speed_perturb:
-                sp_utt_id = f"sp{speed}_" + utt_id
-                sp_n_frames = n_frames / speed
-                if speed == 1.0:
-                    waveform, _ = torchaudio.load(wav_path, frame_offset=offset, num_frames=n_frames)
-                else:
-                    waveform, _ = torchaudio.load(wav_path, frame_offset=offset, num_frames=n_frames)
-                    effects = [
-                        ["speed", f"{speed}"],
-                        ["rate", f"{sr}"]
-                    ]
-                    waveform, _ = torchaudio.sox_effects.apply_effects_tensor(waveform, sr, effects)
+    def get(self, n: int, need_waveform: bool = False):
+        item = self.data[n]
+        audio = item["audio"]
 
-                items.append([waveform, sr, sp_n_frames, src_utt, tgt_utt, spk_id, sp_utt_id])
-        return items
+        if getattr(item, "n_frames", False) and getattr(item, "sample_rate", False):
+            n_frames = item["n_frames"]
+            sample_rate = item["sample_rate"]
+        else:
+            info = torchaudio.info(audio)
+            sample_rate = info.sample_rate
+            n_frames = info.num_frames
+
+        waveform = None
+        if need_waveform:
+            if getattr(item, "offset", False):
+                waveform, sample_rate = torchaudio.load(audio,
+                                                        frame_offset=item["sample_rate"],
+                                                        num_frames=item["n_frames"])
+            else:
+                waveform, sample_rate = torchaudio.load(audio)
 
-    def get_wav(self, n: int, speed_perturb=1.0):
-        wav_path, offset, n_frames, sr, src_utt, tgt_utt, spk_id, utt_id = self.data[n]
+            if getattr(item, "perturb", False):
+                n_frames = n_frames / item['perturb']
+                effects = [
+                    ["speed", f"{item['perturb']}"],
+                    ["rate", f"{sample_rate}"]
+                ]
+                waveform, _ = torchaudio.sox_effects.apply_effects_tensor(waveform, sample_rate, effects)
 
-        if self.speed_perturb is None or speed_perturb == 1.0:
-            waveform, _ = torchaudio.load(wav_path, frame_offset=offset, num_frames=n_frames)
-        else:
-            waveform, _ = torchaudio.load(wav_path, frame_offset=offset, num_frames=n_frames)
-            effects = [
-                ["speed", f"{speed_perturb}"],
-                ["rate", f"{sr}"]
-            ]
-            waveform, _ = torchaudio.sox_effects.apply_effects_tensor(waveform, sr, effects)
-        return waveform
-
-    def get_fast(self, n: int):
-        wav_path, offset, n_frames, sr, src_utt, tgt_utt, spk_id, utt_id = self.data[n]
-
-        items = []
-        if self.speed_perturb is None:
-            items.append([wav_path, sr, n_frames, src_utt, tgt_utt, spk_id, utt_id])
-        else:
-            for speed in self.speed_perturb:
-                sp_utt_id = f"sp{speed}_" + utt_id
-                sp_n_frames = n_frames / speed
-                items.append([wav_path, sr, sp_n_frames, src_utt, tgt_utt, spk_id, sp_utt_id])
-        return items
-
-    def get_src_text(self):
-        src_text = []
-        for item in self.data:
-            src_text.append(item[4])
-        return src_text
-
-    def get_tgt_text(self):
-        tgt_text = []
-        for item in self.data:
-            tgt_text.append(item[5])
-        return tgt_text
+        return waveform, sample_rate, n_frames
 
     def __len__(self) -> int:
         return len(self.data)
@@ -160,67 +188,73 @@ class STDataset(Dataset):
 
 def process(args):
     root = Path(args.data_root).absolute()
+    task = args.task
     splits = args.splits.split(",")
     src_lang = args.src_lang
     tgt_lang = args.tgt_lang
-    cur_root = root / f"{src_lang}-{tgt_lang}"
+    assert (task == "st" and tgt_lang is not None) or (task == "asr" and src_lang is not None)
+
+    cur_root = root
     if not cur_root.is_dir():
-        print(f"{cur_root.as_posix()} does not exist. Skipped.")
-        return
+        logger.error(f"{cur_root.as_posix()} does not exist. Skipped.")
+
     if args.output_root is None:
         output_root = cur_root
     else:
-        output_root = Path(args.output_root).absolute() / f"{src_lang}-{tgt_lang}"
+        output_root = Path(args.output_root).absolute()
 
     # Extract features
+    use_raw = args.raw
+    size = args.size
     if args.speed_perturb:
         zip_path = output_root / "fbank80_sp.zip"
     else:
         zip_path = output_root / "fbank80.zip"
-    index = 0
 
-    gen_feature_flag = False
-    if not Path.exists(zip_path):
-        gen_feature_flag = True
+    if use_raw:
+        gen_feature_flag = False
+    else:
+        gen_feature_flag = False
+        if not Path.exists(zip_path) or args.overwrite:
+            gen_feature_flag = True
 
-    if args.overwrite or gen_feature_flag:
+    if gen_feature_flag:
         if args.speed_perturb:
             feature_root = output_root / "fbank80_sp"
         else:
             feature_root = output_root / "fbank80"
         feature_root.mkdir(exist_ok=True)
 
+        print("Extracting log mel filter bank features...")
         for split in splits:
             print(f"Fetching split {split}...")
-            dataset = STDataset(root.as_posix(), src_lang, tgt_lang, split, args.speed_perturb, args.tokenizer)
             is_train_split = split.startswith("train")
-            print("Extracting log mel filter bank features...")
+            dataset = AudioDataset(root.as_posix(),
+                                   src_lang, tgt_lang, split,
+                                   args.speed_perturb, size, use_raw,
+                                   args.tokenizer)
+
             if is_train_split and args.cmvn_type == "global":
                 print("And estimating cepstral mean and variance stats...")
                 gcmvn_feature_list = []
 
             for idx in tqdm(range(len(dataset))):
-                items = dataset.get_fast(idx)
-                for item in items:
-                    index += 1
-                    wav_path, sr, _, _, _, _, utt_id = item
-
-                    features_path = (feature_root / f"{utt_id}.npy").as_posix()
-                    if not os.path.exists(features_path):
-                        sp = 1.0
-                        if dataset.speed_perturb is not None:
-                            sp = float(utt_id.split("_")[0].replace("sp", ""))
-                        waveform = dataset.get_wav(idx, sp)
-                        if waveform.shape[1] == 0:
-                            continue
-                        features = extract_fbank_features(waveform, sr, Path(features_path))
-
-                        if split == 'train' and args.cmvn_type == "global" and not utt_id.startswith("sp"):
-                            if len(gcmvn_feature_list) < args.gcmvn_max_num:
-                                gcmvn_feature_list.append(features)
-
-                if is_train_split and args.size != -1 and index > args.size:
-                    break
+                item = dataset[idx]
+
+                utt_id = item["id"]
+                features_path = (feature_root / f"{utt_id}.npy").as_posix()
+
+                if os.path.exists(features_path):
+                    continue
+
+                waveform, sample_rate, _ = dataset.get(idx, need_waveform=True)
+                if waveform.shape[1] == 0:
+                    continue
+                features = extract_fbank_features(waveform, sample_rate, Path(features_path))
+
+                if split == 'train' and args.cmvn_type == "global" and not utt_id.startswith("sp"):
+                    if len(gcmvn_feature_list) < args.gcmvn_max_num:
+                        gcmvn_feature_list.append(features)
 
             if is_train_split and args.cmvn_type == "global":
                 # Estimate and save cmv
@@ -237,43 +271,68 @@ def process(args):
 
     gen_manifest_flag = False
     for split in splits:
-        if not Path.exists(output_root / f"{split}_{args.task}.tsv"):
+        if not Path.exists(output_root / f"{split}.tsv"):
             gen_manifest_flag = True
             break
 
     train_text = []
     if args.overwrite or gen_manifest_flag:
-        print("Fetching ZIP manifest...")
-        zip_manifest = get_zip_manifest(zip_path)
+        if not use_raw:
+            print("Fetching ZIP manifest...")
+            zip_manifest = get_zip_manifest(zip_path)
+
         # Generate TSV manifest
         print("Generating manifest...")
         for split in splits:
             is_train_split = split.startswith("train")
             manifest = {c: [] for c in MANIFEST_COLUMNS}
-            if args.task == "st" and args.add_src:
-                manifest["src_text"] = []
 
-            dataset = STDataset(args.data_root, src_lang, tgt_lang, split, args.speed_perturb, args.tokenizer)
-            for idx in range(len(dataset)):
-                items = dataset.get_fast(idx)
-                for item in items:
-                    _, sr, n_frames, src_utt, tgt_utt, speaker_id, utt_id = item
-                    manifest["id"].append(utt_id)
+            dataset = AudioDataset(root.as_posix(),
+                                   src_lang, tgt_lang, split,
+                                   args.speed_perturb, size, use_raw,
+                                   args.tokenizer)
+            if args.task == "st" and args.add_src and dataset.have_src_utt:
+                manifest["src_text"] = []
+            for idx in tqdm(range(len(dataset))):
+                item = dataset[idx]
+                _, sample_rate, n_frames = dataset.get(idx, need_waveform=False)
+                utt_id = item["id"]
+
+                manifest["id"].append(utt_id)
+                if use_raw:
+                    audio_path = item["audio"]
+
+                    # add offset and frames info
+                    if getattr(item, "offset", False):
+                        audio_path = f"{audio_path}:{item['offset']}:{n_frames}"
+                    manifest["audio"].append(audio_path)
+                else:
                     manifest["audio"].append(zip_manifest[utt_id])
-                    duration_ms = int(n_frames / sr * 1000)
-                    manifest["n_frames"].append(int(1 + (duration_ms - 25) / 10))
+                duration_ms = int(n_frames / sample_rate * 1000)
+                manifest["n_frames"].append(int(1 + (duration_ms - 25) / 10))
+
+                if dataset.have_src_utt:
+                    src_utt = item[src_lang]
                     if args.lowercase_src:
                         src_utt = src_utt.lower()
                     if args.rm_punc_src:
                         for w in string.punctuation:
                             src_utt = src_utt.replace(w, "")
-                    manifest["tgt_text"].append(src_utt if args.task == "asr" else tgt_utt)
-                    if args.task == "st" and args.add_src:
+                        src_utt = " ".join(src_utt.split(" "))
+                else:
+                    src_utt = None
+
+                if dataset.have_tgt_utt:
+                    tgt_utt = item[tgt_lang]
+                else:
+                    tgt_utt = None
+                if task == "asr":
+                    manifest["tgt_text"].append(src_utt)
+                elif task == "st":
+                    if args.add_src and src_utt is not None:
                         manifest["src_text"].append(src_utt)
-                    manifest["speaker"].append(speaker_id)
+                    manifest["tgt_text"].append(tgt_utt)
 
-                if is_train_split and args.size != -1 and len(manifest["id"]) > args.size:
-                    break
             if is_train_split:
                 if args.task == "st" and args.add_src and args.share:
                     train_text.extend(manifest["src_text"])
@@ -281,21 +340,21 @@ def process(args):
 
             df = pd.DataFrame.from_dict(manifest)
             df = filter_manifest_df(df, is_train_split=is_train_split)
-            save_df_to_tsv(df, output_root / f"{split}_{args.task}.tsv")
+            save_df_to_tsv(df, output_root / f"{split}.tsv")
 
     # Generate vocab
     v_size_str = "" if args.vocab_type == "char" else str(args.vocab_size)
-    spm_filename_prefix = f"spm_{args.vocab_type}{v_size_str}_{args.task}"
+    spm_filename_prefix = f"spm_{args.vocab_type}{v_size_str}_{task}"
     asr_spm_filename = None
     gen_vocab_flag = True
 
-    if args.task == "st" and args.add_src:
+    if task == "st" and args.add_src:
         if args.share:
             if args.st_spm_prefix is not None:
                 gen_vocab_flag = False
                 spm_filename_prefix = args.st_spm_prefix
             else:
-                spm_filename_prefix = f"spm_{args.vocab_type}{v_size_str}_{args.task}_share"
+                spm_filename_prefix = f"spm_{args.vocab_type}{v_size_str}_{task}_share"
             asr_spm_filename = spm_filename_prefix + ".model"
         else:
             if args.st_spm_prefix is not None:
@@ -303,7 +362,7 @@ def process(args):
                 spm_filename_prefix = args.st_spm_prefix
             assert args.asr_prefix is not None
             asr_spm_filename = args.asr_prefix + ".model"
-    elif args.task == "asr":
+    elif task == "asr":
         if args.asr_prefix is not None:
             gen_vocab_flag = False
             spm_filename_prefix = args.asr_prefix
@@ -314,7 +373,7 @@ def process(args):
 
             for split in args.SPLITS:
                 if split.startswith("train"):
-                    csv_path = output_root / f"{split}_{args.task}.tsv"
+                    csv_path = output_root / f"{split}.tsv"
                     with open(csv_path) as f:
                         reader = csv.DictReader(
                             f,
@@ -325,7 +384,7 @@ def process(args):
                             quoting=csv.QUOTE_NONE,
                         )
 
-                    if args.task == "st" and args.add_src and args.share:
+                    if task == "st" and args.add_src and args.share:
                         for e in reader:
                             src_utt = dict(e)["src_text"]
                             if args.lowercase_src:
@@ -333,7 +392,7 @@ def process(args):
                             if args.rm_punc_src:
                                 for w in string.punctuation:
                                     src_utt = src_utt.replace(w, "")
-                                src_utt = src_utt.replace("  ", "")
+                                src_utt = " ".join(src_utt.split(" "))
                             train_text.append(src_utt)
                     tgt_text = [dict(e)["tgt_text"] for e in reader]
                     train_text.extend(tgt_text)
@@ -349,9 +408,9 @@ def process(args):
             )
 
     # Generate config YAML
-    yaml_filename = f"config_{args.task}.yaml"
-    if args.task == "st" and args.add_src and args.share:
-        yaml_filename = f"config_{args.task}_share.yaml"
+    yaml_filename = f"config.yaml"
+    if task == "st" and args.add_src and args.share:
+        yaml_filename = f"config_share.yaml"
 
     gen_config_yaml(
         output_root,
@@ -364,14 +423,34 @@ def process(args):
             else None
         ),
         asr_spm_filename=asr_spm_filename,
-        share_src_and_tgt=True if args.task == "asr" else False
+        share_src_and_tgt=True if task == "asr" else False
     )
 
 
 def main():
     parser = argparse.ArgumentParser()
+    # general setting
     parser.add_argument("--data-root", "-d", required=True, type=str)
     parser.add_argument("--output-root", "-o", default=None, type=str)
+    parser.add_argument("--task", type=str, default="st", choices=["asr", "st"])
+    parser.add_argument("--src-lang", type=str, required=True, help="source language")
+    parser.add_argument("--tgt-lang", type=str, help="target language")
+    parser.add_argument("--splits", type=str, default="train,dev,test", help="dataset splits")
+    parser.add_argument("--size", default=-1, type=int, help="use part of the data")
+    parser.add_argument("--overwrite", action="store_true", help="overwrite the existing files")
+    # audio setting
+    parser.add_argument("--raw", default=False, action="store_true", help="use the raw audio")
+    parser.add_argument("--speed-perturb", action="store_true", default=False,
+                        help="apply speed perturbation on wave file")
+    parser.add_argument("--cmvn-type", default="utterance",
+                        choices=["global", "utterance"],
+                        help="The type of cepstral mean and variance normalization")
+    parser.add_argument("--gcmvn-max-num", default=150000, type=int,
+                        help=(
+                            "Maximum number of sentences to use to estimate"
+                            "global mean and variance"
+                            ))
+    # text and dictionary settings
     parser.add_argument(
         "--vocab-type",
         default="unigram",
@@ -380,12 +459,6 @@ def main():
         choices=["bpe", "unigram", "char"],
     ),
     parser.add_argument("--vocab-size", default=8000, type=int)
-    parser.add_argument("--task", type=str, default="st", choices=["asr", "st"])
-    parser.add_argument("--src-lang", type=str, required=True, help="source language")
-    parser.add_argument("--tgt-lang", type=str, required=True, help="target language")
-    parser.add_argument("--splits", type=str, default="train,dev,test", help="dataset splits")
-    parser.add_argument("--speed-perturb", action="store_true", default=False,
-                        help="apply speed perturbation on wave file")
     parser.add_argument("--share", action="store_true",
                         help="share the tokenizer and dictionary of the transcription and translation")
     parser.add_argument("--add-src", action="store_true", help="add the src text for st task")
@@ -394,15 +467,7 @@ def main():
     parser.add_argument("--lowercase-src", action="store_true", help="lowercase the source text")
     parser.add_argument("--rm-punc-src", action="store_true", help="remove the punctuation of the source text")
     parser.add_argument("--tokenizer", action="store_true", help="use tokenizer txt")
-    parser.add_argument("--cmvn-type", default="utterance",
-                        choices=["global", "utterance"],
-                        help="The type of cepstral mean and variance normalization")
-    parser.add_argument("--overwrite", action="store_true", help="overwrite the existing files")
-    parser.add_argument("--gcmvn-max-num", default=150000, type=int,
-                        help=(
-                            "Maximum number of sentences to use to estimate"
-                            "global mean and variance"
-                            ))
+
     args = parser.parse_args()
 
     process(args)

examples/speech_to_text/prep_mt_data.py

@@ -33,7 +33,7 @@ class MTDataset(Dataset):
     """
 
     def __init__(self, root: str, src_lang, tgt_lang: str, split: str) -> None:
-        _root = Path(root) / f"{src_lang}-{tgt_lang}" / "data"
+        _root = Path(root) / "data"
         txt_root = _root
         assert _root.is_dir() and txt_root.is_dir(), (_root, txt_root)
         # Load source and target text

fairseq/data/audio/audio_utils.py

@@ -2,7 +2,8 @@ import os.path as op
 from typing import BinaryIO, Optional, Tuple, Union
 
 import numpy as np
-
+import torch
+import torchaudio
 
 def get_waveform(
     path_or_fp: Union[str, BinaryIO], normalization=True
@@ -18,14 +19,12 @@ def get_waveform(
         if ext not in {".flac", ".wav"}:
             raise ValueError(f"Unsupported audio format: {ext}")
 
-    try:
-        import soundfile as sf
-    except ImportError:
-        raise ImportError("Please install soundfile to load WAV/FLAC file")
+    waveform, sample_rate = torchaudio.load(path_or_fp)
+    waveform = waveform.squeeze().numpy()
 
-    waveform, sample_rate = sf.read(path_or_fp, dtype="float32")
     if not normalization:
         waveform *= 2 ** 15  # denormalized to 16-bit signed integers
+
     return waveform, sample_rate
 
 
@@ -54,11 +53,11 @@ def _get_kaldi_fbank(waveform, sample_rate, n_bins=80) -> Optional[np.ndarray]:
 def _get_torchaudio_fbank(waveform, sample_rate, n_bins=80) -> Optional[np.ndarray]:
     """Get mel-filter bank features via TorchAudio."""
     try:
-        import torch
         import torchaudio.compliance.kaldi as ta_kaldi
         import torchaudio.sox_effects as ta_sox
 
-        waveform = torch.from_numpy(waveform)
+        if not isinstance(waveform, torch.Tensor):
+            waveform = torch.from_numpy(waveform)
         if len(waveform.shape) == 1:
             # Mono channel: D -> 1 x D
             waveform = waveform.unsqueeze(0)
@@ -91,3 +90,21 @@ def get_fbank(path_or_fp: Union[str, BinaryIO], n_bins=80) -> np.ndarray:
         )
 
     return features
+
+
+def get_fbank_with_perturb(waveform, sample_rate=16000, n_bins=80):
+    import random
+
+    speed = random.choice([0.9, 1.0, 1.1])
+    if speed != 1.0:
+        waveform = torch.from_numpy(waveform).float().unsqueeze(0)
+        waveform, _ = torchaudio.sox_effects.apply_effects_tensor(
+            waveform, sample_rate,
+            [['speed', str(speed)], ['rate', str(sample_rate)]])
+        waveform = waveform.squeeze()
+
+    features = _get_kaldi_fbank(waveform, sample_rate)
+    if features is None:
+        features = _get_torchaudio_fbank(waveform, sample_rate, n_bins)
+
+    return features

fairseq/data/audio/speech_to_text_dataset.py

@@ -19,7 +19,7 @@ from fairseq.data import (
     ResamplingDataset,
     data_utils as fairseq_data_utils,
 )
-from fairseq.data.audio.audio_utils import get_fbank, get_waveform
+from fairseq.data.audio.audio_utils import get_fbank, get_waveform, get_fbank_with_perturb
 from fairseq.data.audio.feature_transforms import CompositeAudioFeatureTransform
 
 
@@ -117,6 +117,12 @@ class S2TDataConfig(object):
         return self.config.get("use_audio_input", False)
 
     @property
+    def speed_perturb(self):
+        """Needed by the dataset loader to see if the model requires
+        speed perturbation."""
+        return self.config.get("speed_perturb", False)
+
+    @property
     def audio_root(self):
         """Audio paths in the manifest TSV can be relative and this provides
         the root path. Set this to empty string when using absolute paths."""
@@ -253,6 +259,7 @@ class SpeechToTextDataset(FairseqDataset):
     ):
         self.split, self.is_train_split = split, is_train_split
         self.data_cfg = data_cfg
+        self.speed_perturb = data_cfg.speed_perturb
         self.audio_paths, self.n_frames = audio_paths, n_frames
         self.n_samples = len(audio_paths)
         if data_cfg.share_src_and_tgt:
@@ -317,8 +324,12 @@ class SpeechToTextDataset(FairseqDataset):
         self, index: int
     ) -> Tuple[int, torch.Tensor, Optional[torch.Tensor], Optional[torch.Tensor]]:
         source = get_features_or_waveform(
-            self.audio_paths[index], need_waveform=self.data_cfg.use_audio_input
+            self.audio_paths[index],
+            need_waveform=self.data_cfg.use_audio_input or (self.is_train_split and self.speed_perturb)
         )
+        if self.speed_perturb:
+            source = get_fbank_with_perturb(source[0], source[1])
+
         if self.feature_transforms is not None:
             assert not self.data_cfg.use_audio_input
             source = self.feature_transforms(source)

fairseq/tasks/speech_to_text.py

@@ -32,6 +32,12 @@ class SpeechToTextTask(LegacyFairseqTask):
             help="Configuration YAML filename (under manifest root)",
         )
         parser.add_argument(
+            "--speed-perturb",
+            default=False,
+            action="store_true",
+            help="use online speed perturbation",
+        )
+        parser.add_argument(
             "--max-source-positions",
             default=6000,
             type=int,
@@ -50,7 +56,9 @@ class SpeechToTextTask(LegacyFairseqTask):
         super().__init__(args)
         self.src_dict = src_dict
         self.tgt_dict = tgt_dict
+        self.speed_perturb = args.speed_perturb
         self.data_cfg = S2TDataConfig(op.join(args.data, args.config_yaml))
+        self.data_cfg.config["speed_perturb"] = self.speed_perturb
 
     @classmethod
     def setup_task(cls, args, **kwargs):