examples/vm_sound_classification/run.sh

#!/usr/bin/env bash

: <<'END'

sh run.sh --stage 0 --stop_stage 1 --system_version windows --file_folder_name file_dir --final_model_name sound-4-ch32 \
--filename_patterns "E:/Users/tianx/HuggingDatasets/vm_sound_classification/data/wav_finished/wav_finished/en-US/wav_finished/*/*.wav \
E:/Users/tianx/HuggingDatasets/vm_sound_classification/data/wav_finished/id-ID/wav_finished/*/*.wav" \
--label_plan 4

sh run.sh --stage 2 --stop_stage 2 --system_version windows --file_folder_name file_dir --final_model_name sound-2-ch32 \
--filename_patterns "E:/Users/tianx/HuggingDatasets/vm_sound_classification/data/wav_finished/wav_finished/en-US/wav_finished/*/*.wav \
E:/Users/tianx/HuggingDatasets/vm_sound_classification/data/wav_finished/id-ID/wav_finished/*/*.wav" \
--label_plan 4

sh run.sh --stage 0 --stop_stage 6 --system_version centos --file_folder_name file_dir --final_model_name sound-3-ch32 \
--filename_patterns "/data/tianxing/PycharmProjects/datasets/voicemail/*/wav_finished/*/*.wav" \
--label_plan 3 \
--config_file "yaml/conv2d-classifier-3-ch4.yaml"

sh run.sh --stage 0 --stop_stage 6 --system_version centos --file_folder_name file_dir --final_model_name voicemail-ms-my-2-ch32 \
--filename_patterns "/data/tianxing/PycharmProjects/datasets/voicemail/ms-MY/wav_finished/*/*.wav" \
--label_plan 2-voicemail \
--config_file "yaml/conv2d-classifier-2-ch32.yaml"

END


# params
system_version="windows";
verbose=true;
stage=0 # start from 0 if you need to start from data preparation
stop_stage=9

work_dir="$(pwd)"
file_folder_name=file_folder_name
final_model_name=final_model_name
filename_patterns="/data/tianxing/PycharmProjects/datasets/voicemail/*/wav_finished/*/*.wav"
label_plan=4
config_file="yaml/conv2d-classifier-2-ch4.yaml"
pretrained_model=null
nohup_name=nohup.out

country=en-US

# model params
batch_size=64
max_epochs=200
save_top_k=10
patience=5


# parse options
while true; do
  [ -z "${1:-}" ] && break;  # break if there are no arguments
  case "$1" in
    --*) name=$(echo "$1" | sed s/^--// | sed s/-/_/g);
      eval '[ -z "${'"$name"'+xxx}" ]' && echo "$0: invalid option $1" 1>&2 && exit 1;
      old_value="(eval echo \\$$name)";
      if [ "${old_value}" == "true" ] || [ "${old_value}" == "false" ]; then
        was_bool=true;
      else
        was_bool=false;
      fi

      # Set the variable to the right value-- the escaped quotes make it work if
      # the option had spaces, like --cmd "queue.pl -sync y"
      eval "${name}=\"$2\"";

      # Check that Boolean-valued arguments are really Boolean.
      if $was_bool && [[ "$2" != "true" && "$2" != "false" ]]; then
        echo "$0: expected \"true\" or \"false\": $1 $2" 1>&2
        exit 1;
      fi
      shift 2;
      ;;

    *) break;
  esac
done

file_dir="${work_dir}/${file_folder_name}"
final_model_dir="${work_dir}/../../trained_models/${final_model_name}";

dataset="${file_dir}/dataset.xlsx"
train_dataset="${file_dir}/train.xlsx"
valid_dataset="${file_dir}/valid.xlsx"
evaluation_file="${file_dir}/evaluation.xlsx"
vocabulary_dir="${file_dir}/vocabulary"

$verbose && echo "system_version: ${system_version}"
$verbose && echo "file_folder_name: ${file_folder_name}"

if [ $system_version == "windows" ]; then
  alias python3='D:/Users/tianx/PycharmProjects/virtualenv/vm_sound_classification/Scripts/python.exe'
elif [ $system_version == "centos" ] || [ $system_version == "ubuntu" ]; then
  #source /data/local/bin/vm_sound_classification/bin/activate
  alias python3='/data/local/bin/vm_sound_classification/bin/python3'
fi


if [ ${stage} -le 0 ] && [ ${stop_stage} -ge 0 ]; then
  $verbose && echo "stage 0: prepare data"
  cd "${work_dir}" || exit 1
  python3 step_1_prepare_data.py \
  --file_dir "${file_dir}" \
  --filename_patterns "${filename_patterns}" \
  --train_dataset "${train_dataset}" \
  --valid_dataset "${valid_dataset}" \
  --label_plan "${label_plan}" \

fi


if [ ${stage} -le 1 ] && [ ${stop_stage} -ge 1 ]; then
  $verbose && echo "stage 1: make vocabulary"
  cd "${work_dir}" || exit 1
  python3 step_2_make_vocabulary.py \
  --vocabulary_dir "${vocabulary_dir}" \
  --train_dataset "${train_dataset}" \
  --valid_dataset "${valid_dataset}" \

fi


if [ ${stage} -le 2 ] && [ ${stop_stage} -ge 2 ]; then
  $verbose && echo "stage 2: train model"
  cd "${work_dir}" || exit 1
  python3 step_3_train_model.py \
  --vocabulary_dir "${vocabulary_dir}" \
  --train_dataset "${train_dataset}" \
  --valid_dataset "${valid_dataset}" \
  --serialization_dir "${file_dir}" \
  --config_file "${config_file}" \
  --pretrained_model "${pretrained_model}" \

fi


if [ ${stage} -le 3 ] && [ ${stop_stage} -ge 3 ]; then
  $verbose && echo "stage 3: test model"
  cd "${work_dir}" || exit 1
  python3 step_4_evaluation_model.py \
  --dataset "${dataset}" \
  --vocabulary_dir "${vocabulary_dir}" \
  --model_dir "${file_dir}/best" \
  --output_file "${evaluation_file}" \

fi


if [ ${stage} -le 4 ] && [ ${stop_stage} -ge 4 ]; then
  $verbose && echo "stage 4: export model"
  cd "${work_dir}" || exit 1
  python3 step_5_export_models.py \
  --vocabulary_dir "${vocabulary_dir}" \
  --model_dir "${file_dir}/best" \
  --serialization_dir "${file_dir}" \

fi


if [ ${stage} -le 5 ] && [ ${stop_stage} -ge 5 ]; then
  $verbose && echo "stage 5: collect files"
  cd "${work_dir}" || exit 1

  mkdir -p ${final_model_dir}

  cp "${file_dir}/best"/* "${final_model_dir}"
  cp -r "${file_dir}/vocabulary" "${final_model_dir}"

  cp "${file_dir}/evaluation.xlsx" "${final_model_dir}/evaluation.xlsx"

  cp "${file_dir}/trace_model.zip" "${final_model_dir}/trace_model.zip"
  cp "${file_dir}/trace_quant_model.zip" "${final_model_dir}/trace_quant_model.zip"
  cp "${file_dir}/script_model.zip" "${final_model_dir}/script_model.zip"
  cp "${file_dir}/script_quant_model.zip" "${final_model_dir}/script_quant_model.zip"

  cd "${final_model_dir}/.." || exit 1;

  if [ -e "${final_model_name}.zip" ]; then
    rm -rf "${final_model_name}_backup.zip"
    mv "${final_model_name}.zip" "${final_model_name}_backup.zip"
  fi

  zip -r "${final_model_name}.zip" "${final_model_name}"
  rm -rf "${final_model_name}"

fi


if [ ${stage} -le 6 ] && [ ${stop_stage} -ge 6 ]; then
  $verbose && echo "stage 6: clear file_dir"
  cd "${work_dir}" || exit 1

  rm -rf "${file_dir}";

fi