<NME> setup_local_env.sh
<BEF> #!/bin/bash 
# This script sets up development and data environments for 
# a local machine, copy under your home directory and run.
# Note that, Theano is NOT installed by this script.
# To use Byte Pair Encoding, simply pass -b argument.

BPE=false

while getopts ':b' flag; do
  case "${flag}" in
    b) BPE=true 
       echo "Using Byte Pair Encoding" ;;
    *) error 
       echo ""
       echo "Usage: $0 [-b]"
       echo ""
       exit 1 ;;
  esac
done

if [ "$#" -ne 4 ]; then
    echo ""
    echo "Usage: $0 src trg path_to_data path_to_subword"
    echo ""
    exit 1
fi

if [ -z $PYTHON ]; then
    if [ -n `which python3` ]; then
        export PYTHON=python3
    else
        if [ -n `which python2`]; then
            export PYTHON=python2
        else
            if [ -n `which python`]; then
                export PYTHON=python
            fi
        fi
    fi 
fi

if [ -z $PYTHON ]; then
    echo "Please set PYTHON to a Python interpreter"
    exit 1 
fi

echo "Using $PYTHON"

# code directory for cloned repositories
SCRIPT_DIR=$( dirname "${BASH_SOURCE[0]}" )
CODE_DIR=$( realpath "${SCRIPT_DIR}/.." )

# code repository 
CODE_CENTRAL=https://github.com/nyu-dl/dl4mt-tutorial

# our input files will reside here
DATA_DIR=${CODE_DIR}/data

# our trained models will be saved here
MODELS_DIR=${HOME}/models


# clone the repository from github into code directory
if [ ! -d "${CODE_DIR}" ]; then
    echo "Cloning central ..."
    mkdir -p ${CODE_DIR}
    git clone ${CODE_CENTRAL} ${CODE_DIR}
fi

# download the europarl v7 and validation sets and extract
if [ ! -f ${DATA_DIR}/train_data.tgz ]; then 
    curl -o ${DATA_DIR}/train_data.tgz http://www.statmt.org/europarl/v7/fr-en.tgz
else
    echo "${DATA_DIR}/train_data.tgz exists"
fi
if [ ! -f ${DATA_DIR}/valid_data.tgz ]; then
    curl -o ${DATA_DIR}/valid_data.tgz http://matrix.statmt.org/test_sets/newstest2011.tgz
else
    echo "${DATA_DIR}/valid_data.tgz exists"
fi
$PYTHON ${CODE_DIR}/data/extract_files.py \
    -s='fr' -t='en' \
    --source-dev=newstest2011.fr \
    --target-dev=newstest2011.en \
    --outdir=${DATA_DIR}

if [ "$BPE" = true ] ; then

    BPE_DIR=${HOME}/codes/subword-nmt
    BPE_CENTRAL=https://github.com/rsennrich/subword-nmt

    # clone subword-nmt repository
    if [ ! -d "${BPE_DIR}" ]; then
        echo "Cloning BPE central ..."
        mkdir -p ${BPE_DIR}
        git clone ${BPE_CENTRAL} ${BPE_DIR}
    fi

    # follow the preprocessing pipeline for BPE
    ./preprocess.sh 'fr' 'en' ${DATA_DIR} ${BPE_DIR}

else

    # tokenize corresponding files
    perl ${CODE_DIR}/data/tokenizer.perl -l 'fr' < ${DATA_DIR}/test2011/newstest2011.fr > ${DATA_DIR}/newstest2011.fr.tok
    perl ${CODE_DIR}/data/tokenizer.perl -l 'en' < ${DATA_DIR}/test2011/newstest2011.en > ${DATA_DIR}/newstest2011.en.tok
    perl ${CODE_DIR}/data/tokenizer.perl -l 'fr' < ${DATA_DIR}/europarl-v7.fr-en.fr > ${DATA_DIR}/europarl-v7.fr-en.fr.tok
    perl ${CODE_DIR}/data/tokenizer.perl -l 'en' < ${DATA_DIR}/europarl-v7.fr-en.en > ${DATA_DIR}/europarl-v7.fr-en.en.tok

    # extract dictionaries
    $PYTHON ${CODE_DIR}/data/build_dictionary.py ${DATA_DIR}/europarl-v7.fr-en.fr.tok
    $PYTHON ${CODE_DIR}/data/build_dictionary.py ${DATA_DIR}/europarl-v7.fr-en.en.tok

    # shuffle traning data
    $PYTHON ${CODE_DIR}/data/shuffle.py ${DATA_DIR}/europarl-v7.fr-en.en.tok ${DATA_DIR}/europarl-v7.fr-en.fr.tok 
fi

# create model output directory if it does not exist 
if [ ! -d "${MODELS_DIR}" ]; then
    mkdir -p ${MODELS_DIR}
fi

# check if theano is working
$PYTHON -c "from __future__ import print_function; import theano; print('theano available!')"

<MSG> small fixes

<DFF> @@ -18,13 +18,6 @@ while getopts ':b' flag; do
   esac
 done
 
-if [ "$#" -ne 4 ]; then
-    echo ""
-    echo "Usage: $0 src trg path_to_data path_to_subword"
-    echo ""
-    exit 1
-fi
-
 if [ -z $PYTHON ]; then
     if [ -n `which python3` ]; then
         export PYTHON=python3
@@ -48,7 +41,7 @@ echo "Using $PYTHON"
 
 # code directory for cloned repositories
 SCRIPT_DIR=$( dirname "${BASH_SOURCE[0]}" )
-CODE_DIR=$( realpath "${SCRIPT_DIR}/.." )
+CODE_DIR=${SCRIPT_DIR}/..
 
 # code repository 
 CODE_CENTRAL=https://github.com/nyu-dl/dl4mt-tutorial

{'commit': 'diff --git a/data/setup_local_env.sh b/data/setup_local_env.sh\nindex 73d172d..dc1d6a5 100755\n--- a/data/setup_local_env.sh\n+++ b/data/setup_local_env.sh\n@@ -18,13 +18,6 @@ while getopts \':b\' flag; do\n   esac\n done\n \n-if [ "$#" -ne 4 ]; then\n-    echo ""\n-    echo "Usage: $0 src trg path_to_data path_to_subword"\n-    echo ""\n-    exit 1\n-fi\n-\n if [ -z $PYTHON ]; then\n     if [ -n `which python3` ]; then\n         export PYTHON=python3\n@@ -48,7 +41,7 @@ echo "Using $PYTHON"\n \n # code directory for cloned repositories\n SCRIPT_DIR=$( dirname "${BASH_SOURCE[0]}" )\n-CODE_DIR=$( realpath "${SCRIPT_DIR}/.." )\n+CODE_DIR=${SCRIPT_DIR}/..\n \n # code repository \n CODE_CENTRAL=https://github.com/nyu-dl/dl4mt-tutorial\n', 'commit_subject': 'small fixes', 'repo_name': 'nyu-dl/dl4mt-tutorial', 'committer': {'name': 'Jencir Lee', 'email': 'faf0b76a11d9f10ee506a6a64e821507d492ae14@icloud.com', 'time_sec': '1487960676', 'tz_offset': '0', 'date': {'seconds': '1487960676'}}, 'commit_tree': '1fb6be007b6ab44620cadaa90b239c06e2ce6b82', 'commit_message': 'small fixes\n', 'language_name': 'Shell', 'license': 'bsd-3-clause', 'hunks': '@@ -18,13 +18,6 @@ while getopts \':b\' flag; do\n   esac\n done\n \n-if [ "$#" -ne 4 ]; then\n-    echo ""\n-    echo "Usage: $0 src trg path_to_data path_to_subword"\n-    echo ""\n-    exit 1\n-fi\n-\n if [ -z $PYTHON ]; then\n     if [ -n `which python3` ]; then\n         export PYTHON=python3\n@@ -48,7 +41,7 @@ echo "Using $PYTHON"\n \n # code directory for cloned repositories\n SCRIPT_DIR=$( dirname "${BASH_SOURCE[0]}" )\n-CODE_DIR=$( realpath "${SCRIPT_DIR}/.." )\n+CODE_DIR=${SCRIPT_DIR}/..\n \n # code repository \n CODE_CENTRAL=https://github.com/nyu-dl/dl4mt-tutorial\n', 'addition_count': 1, 'deletion_count': 8, 'src_file': 'a/data/setup_local_env.sh', 'tgt_file': 'b/data/setup_local_env.sh', 'file_extension': '.sh', 'before_file': '#!/bin/bash \n# This script sets up development and data environments for \n# a local machine, copy under your home directory and run.\n# Note that, Theano is NOT installed by this script.\n# To use Byte Pair Encoding, simply pass -b argument.\n\nBPE=false\n\nwhile getopts \':b\' flag; do\n  case "${flag}" in\n    b) BPE=true \n       echo "Using Byte Pair Encoding" ;;\n    *) error \n       echo ""\n       echo "Usage: $0 [-b]"\n       echo ""\n       exit 1 ;;\n  esac\ndone\n\nif [ "$#" -ne 4 ]; then\n    echo ""\n    echo "Usage: $0 src trg path_to_data path_to_subword"\n    echo ""\n    exit 1\nfi\n\nif [ -z $PYTHON ]; then\n    if [ -n `which python3` ]; then\n        export PYTHON=python3\n    else\n        if [ -n `which python2`]; then\n            export PYTHON=python2\n        else\n            if [ -n `which python`]; then\n                export PYTHON=python\n            fi\n        fi\n    fi \nfi\n\nif [ -z $PYTHON ]; then\n    echo "Please set PYTHON to a Python interpreter"\n    exit 1 \nfi\n\necho "Using $PYTHON"\n\n# code directory for cloned repositories\nSCRIPT_DIR=$( dirname "${BASH_SOURCE[0]}" )\nCODE_DIR=$( realpath "${SCRIPT_DIR}/.." )\n\n# code repository \nCODE_CENTRAL=https://github.com/nyu-dl/dl4mt-tutorial\n\n# our input files will reside here\nDATA_DIR=${CODE_DIR}/data\n\n# our trained models will be saved here\nMODELS_DIR=${HOME}/models\n\n\n# clone the repository from github into code directory\nif [ ! -d "${CODE_DIR}" ]; then\n    echo "Cloning central ..."\n    mkdir -p ${CODE_DIR}\n    git clone ${CODE_CENTRAL} ${CODE_DIR}\nfi\n\n# download the europarl v7 and validation sets and extract\nif [ ! -f ${DATA_DIR}/train_data.tgz ]; then \n    curl -o ${DATA_DIR}/train_data.tgz http://www.statmt.org/europarl/v7/fr-en.tgz\nelse\n    echo "${DATA_DIR}/train_data.tgz exists"\nfi\nif [ ! -f ${DATA_DIR}/valid_data.tgz ]; then\n    curl -o ${DATA_DIR}/valid_data.tgz http://matrix.statmt.org/test_sets/newstest2011.tgz\nelse\n    echo "${DATA_DIR}/valid_data.tgz exists"\nfi\n$PYTHON ${CODE_DIR}/data/extract_files.py \\\n    -s=\'fr\' -t=\'en\' \\\n    --source-dev=newstest2011.fr \\\n    --target-dev=newstest2011.en \\\n    --outdir=${DATA_DIR}\n\nif [ "$BPE" = true ] ; then\n\n    BPE_DIR=${HOME}/codes/subword-nmt\n    BPE_CENTRAL=https://github.com/rsennrich/subword-nmt\n\n    # clone subword-nmt repository\n    if [ ! -d "${BPE_DIR}" ]; then\n        echo "Cloning BPE central ..."\n        mkdir -p ${BPE_DIR}\n        git clone ${BPE_CENTRAL} ${BPE_DIR}\n    fi\n\n    # follow the preprocessing pipeline for BPE\n    ./preprocess.sh \'fr\' \'en\' ${DATA_DIR} ${BPE_DIR}\n\nelse\n\n    # tokenize corresponding files\n    perl ${CODE_DIR}/data/tokenizer.perl -l \'fr\' < ${DATA_DIR}/test2011/newstest2011.fr > ${DATA_DIR}/newstest2011.fr.tok\n    perl ${CODE_DIR}/data/tokenizer.perl -l \'en\' < ${DATA_DIR}/test2011/newstest2011.en > ${DATA_DIR}/newstest2011.en.tok\n    perl ${CODE_DIR}/data/tokenizer.perl -l \'fr\' < ${DATA_DIR}/europarl-v7.fr-en.fr > ${DATA_DIR}/europarl-v7.fr-en.fr.tok\n    perl ${CODE_DIR}/data/tokenizer.perl -l \'en\' < ${DATA_DIR}/europarl-v7.fr-en.en > ${DATA_DIR}/europarl-v7.fr-en.en.tok\n\n    # extract dictionaries\n    $PYTHON ${CODE_DIR}/data/build_dictionary.py ${DATA_DIR}/europarl-v7.fr-en.fr.tok\n    $PYTHON ${CODE_DIR}/data/build_dictionary.py ${DATA_DIR}/europarl-v7.fr-en.en.tok\n\n    # shuffle traning data\n    $PYTHON ${CODE_DIR}/data/shuffle.py ${DATA_DIR}/europarl-v7.fr-en.en.tok ${DATA_DIR}/europarl-v7.fr-en.fr.tok \nfi\n\n# create model output directory if it does not exist \nif [ ! -d "${MODELS_DIR}" ]; then\n    mkdir -p ${MODELS_DIR}\nfi\n\n# check if theano is working\n$PYTHON -c "from __future__ import print_function; import theano; print(\'theano available!\')"\n', 'message': 'small fixes\n', 'author_name': 'Jencir Lee', 'lang': 'sh'}