#! /usr/bin/bash
set -e

#tag=residual_wmt_en2de_baseline
tag=residual_big
#tag=wmt_en2de_bigmodel
#tag=no_share_emb
#tag=wmt_en2de_bigmodel_smalllr

save_dir=checkpoints/$tag

if [ ! -d $save_dir ]; then
	mkdir -p $save_dir
fi
cp ${BASH_SOURCE[0]} $save_dir/train.sh


share_emb=--share-all-embeddings
#share_emb=

adam_betas="'(0.9, 0.98)'"

cmd="python3 -u train.py data-bin/wmt14_en_de_joined_dict 
  --distributed-world-size 8
  --arch dense_stack_transformer_base_wmt_en_de
  $share_emb  
  --optimizer adam --clip-norm 0.0 
  --lr-scheduler inverse_sqrt --warmup-init-lr 1e-07 --warmup-updates 30000 
  --lr 0.002 --min-lr 1e-09 
  --weight-decay 0.0 --criterion label_smoothed_cross_entropy --label-smoothing 0.1  
  --max-tokens 2048 
  --update-freq 2 
  --no-progress-bar  
  --log-interval 100 
  --max-update 300000
  --save-dir $save_dir"
cmd=${cmd}" --adam-betas "${adam_betas}
#echo $cmd
#eval $cmd
#cmd=$(eval $cmd)
#nohup $cmd exec 1> $save_dir/train.log exec 2>&1 & 
#tail -f $save_dir/train.log

cmd="nohup "${cmd}" > $save_dir/train.log 2>&1 &"
eval $cmd
echo $cmd
tail -f $save_dir/train.log

