#!/usr/bin/env bash : <<'END' sh run.sh --stage 0 --stop_stage 1 --system_version windows --file_folder_name file_dir --final_model_name sound-4-ch32 \ --filename_patterns "E:/Users/tianx/HuggingDatasets/vm_sound_classification/data/wav_finished/wav_finished/en-US/wav_finished/*/*.wav \ E:/Users/tianx/HuggingDatasets/vm_sound_classification/data/wav_finished/id-ID/wav_finished/*/*.wav" \ --label_plan 4 sh run.sh --stage 2 --stop_stage 2 --system_version windows --file_folder_name file_dir --final_model_name sound-2-ch32 \ --filename_patterns "E:/Users/tianx/HuggingDatasets/vm_sound_classification/data/wav_finished/wav_finished/en-US/wav_finished/*/*.wav \ E:/Users/tianx/HuggingDatasets/vm_sound_classification/data/wav_finished/id-ID/wav_finished/*/*.wav" \ --label_plan 4 sh run.sh --stage 0 --stop_stage 6 --system_version centos --file_folder_name file_dir --final_model_name sound-3-ch32 \ --filename_patterns "/data/tianxing/PycharmProjects/datasets/voicemail/*/wav_finished/*/*.wav" \ --label_plan 3 \ --config_file "yaml/conv2d-classifier-3-ch4.yaml" sh run.sh --stage 0 --stop_stage 6 --system_version centos --file_folder_name file_dir --final_model_name voicemail-ms-my-2-ch32 \ --filename_patterns "/data/tianxing/PycharmProjects/datasets/voicemail/ms-MY/wav_finished/*/*.wav" \ --label_plan 2-voicemail \ --config_file "yaml/conv2d-classifier-2-ch32.yaml" END # params system_version="windows"; verbose=true; stage=0 # start from 0 if you need to start from data preparation stop_stage=9 work_dir="$(pwd)" file_folder_name=file_folder_name final_model_name=final_model_name filename_patterns="/data/tianxing/PycharmProjects/datasets/voicemail/*/wav_finished/*/*.wav" label_plan=4 config_file="yaml/conv2d-classifier-2-ch4.yaml" pretrained_model=null nohup_name=nohup.out country=en-US # model params batch_size=64 max_epochs=200 save_top_k=10 patience=5 # parse options while true; do [ -z "${1:-}" ] && break; # break if there are no arguments case "$1" in --*) name=$(echo "$1" | sed s/^--// | sed s/-/_/g); eval '[ -z "${'"$name"'+xxx}" ]' && echo "$0: invalid option $1" 1>&2 && exit 1; old_value="(eval echo \\$$name)"; if [ "${old_value}" == "true" ] || [ "${old_value}" == "false" ]; then was_bool=true; else was_bool=false; fi # Set the variable to the right value-- the escaped quotes make it work if # the option had spaces, like --cmd "queue.pl -sync y" eval "${name}=\"$2\""; # Check that Boolean-valued arguments are really Boolean. if $was_bool && [[ "$2" != "true" && "$2" != "false" ]]; then echo "$0: expected \"true\" or \"false\": $1 $2" 1>&2 exit 1; fi shift 2; ;; *) break; esac done file_dir="${work_dir}/${file_folder_name}" final_model_dir="${work_dir}/../../trained_models/${final_model_name}"; dataset="${file_dir}/dataset.xlsx" train_dataset="${file_dir}/train.xlsx" valid_dataset="${file_dir}/valid.xlsx" evaluation_file="${file_dir}/evaluation.xlsx" vocabulary_dir="${file_dir}/vocabulary" $verbose && echo "system_version: ${system_version}" $verbose && echo "file_folder_name: ${file_folder_name}" if [ $system_version == "windows" ]; then alias python3='D:/Users/tianx/PycharmProjects/virtualenv/vm_sound_classification/Scripts/python.exe' elif [ $system_version == "centos" ] || [ $system_version == "ubuntu" ]; then #source /data/local/bin/vm_sound_classification/bin/activate alias python3='/data/local/bin/vm_sound_classification/bin/python3' fi if [ ${stage} -le 0 ] && [ ${stop_stage} -ge 0 ]; then $verbose && echo "stage 0: prepare data" cd "${work_dir}" || exit 1 python3 step_1_prepare_data.py \ --file_dir "${file_dir}" \ --filename_patterns "${filename_patterns}" \ --train_dataset "${train_dataset}" \ --valid_dataset "${valid_dataset}" \ --label_plan "${label_plan}" \ fi if [ ${stage} -le 1 ] && [ ${stop_stage} -ge 1 ]; then $verbose && echo "stage 1: make vocabulary" cd "${work_dir}" || exit 1 python3 step_2_make_vocabulary.py \ --vocabulary_dir "${vocabulary_dir}" \ --train_dataset "${train_dataset}" \ --valid_dataset "${valid_dataset}" \ fi if [ ${stage} -le 2 ] && [ ${stop_stage} -ge 2 ]; then $verbose && echo "stage 2: train model" cd "${work_dir}" || exit 1 python3 step_3_train_model.py \ --vocabulary_dir "${vocabulary_dir}" \ --train_dataset "${train_dataset}" \ --valid_dataset "${valid_dataset}" \ --serialization_dir "${file_dir}" \ --config_file "${config_file}" \ --pretrained_model "${pretrained_model}" \ fi if [ ${stage} -le 3 ] && [ ${stop_stage} -ge 3 ]; then $verbose && echo "stage 3: test model" cd "${work_dir}" || exit 1 python3 step_4_evaluation_model.py \ --dataset "${dataset}" \ --vocabulary_dir "${vocabulary_dir}" \ --model_dir "${file_dir}/best" \ --output_file "${evaluation_file}" \ fi if [ ${stage} -le 4 ] && [ ${stop_stage} -ge 4 ]; then $verbose && echo "stage 4: export model" cd "${work_dir}" || exit 1 python3 step_5_export_models.py \ --vocabulary_dir "${vocabulary_dir}" \ --model_dir "${file_dir}/best" \ --serialization_dir "${file_dir}" \ fi if [ ${stage} -le 5 ] && [ ${stop_stage} -ge 5 ]; then $verbose && echo "stage 5: collect files" cd "${work_dir}" || exit 1 mkdir -p ${final_model_dir} cp "${file_dir}/best"/* "${final_model_dir}" cp -r "${file_dir}/vocabulary" "${final_model_dir}" cp "${file_dir}/evaluation.xlsx" "${final_model_dir}/evaluation.xlsx" cp "${file_dir}/trace_model.zip" "${final_model_dir}/trace_model.zip" cp "${file_dir}/trace_quant_model.zip" "${final_model_dir}/trace_quant_model.zip" cp "${file_dir}/script_model.zip" "${final_model_dir}/script_model.zip" cp "${file_dir}/script_quant_model.zip" "${final_model_dir}/script_quant_model.zip" cd "${final_model_dir}/.." || exit 1; if [ -e "${final_model_name}.zip" ]; then rm -rf "${final_model_name}_backup.zip" mv "${final_model_name}.zip" "${final_model_name}_backup.zip" fi zip -r "${final_model_name}.zip" "${final_model_name}" rm -rf "${final_model_name}" fi if [ ${stage} -le 6 ] && [ ${stop_stage} -ge 6 ]; then $verbose && echo "stage 6: clear file_dir" cd "${work_dir}" || exit 1 rm -rf "${file_dir}"; fi