%pip install --upgrade --quiet cloudml-hypertune

"""
Explanation: Developing, Training, and Deploying a TensorFlow model on Google Cloud Platform (using CloudShell and Cloud AI Platform)
In this notebook, we will develop a Keras model to predict flight delays using TensorFlow 2.0 as the backend. Unlike flights_model_tf2.ipynb, we will use bash commands so that these can be run from CloudShell. We will also deploy to Cloud AI Platform so that the model can be executed in a serverless way.
Install necessary Python package
End of explanation
"""


# change these to try this notebook out
BUCKET = 'cloud-training-demos-ml'
PROJECT = 'cloud-training-demos'
REGION = 'us-central1'

import os
os.environ['BUCKET'] = BUCKET
os.environ['PROJECT'] = PROJECT
os.environ['REGION'] = REGION

%%bash
gcloud config set project $PROJECT
gcloud config set compute/region $REGION

"""
Explanation: Restart the kernel after doing the pip install
Set environment variables
I'm doing this for the notebook. In CloudShell, you'd do:
<pre>
export PROJECT=cloud-training-demos
</pre>
etc.
End of explanation
"""


%%bash
export PYTHONPATH="$PWD/flights"
python3 -m trainer.task --bucket $BUCKET --train_batch_size=3 --func=read_lines

"""
Explanation: Try out different functions in the model
Reading lines
End of explanation
"""


%%bash
export PYTHONPATH="$PWD/flights"
python3 -m trainer.task --bucket $BUCKET --num_examples=100 --func=find_average_label

"""
Explanation: Finding average label
End of explanation
"""


%%bash
export PYTHONPATH="$PWD/flights"
gsutil -m rm -rf gs://$BUCKET/flights/trained_model
python3 -m trainer.task --bucket $BUCKET --num_examples=1000 --func=linear

gsutil ls gs://$BUCKET/flights/trained_model

"""
Explanation: Linear model
End of explanation
"""


%%bash
export PYTHONPATH="$PWD/flights"
gsutil -m rm -rf gs://$BUCKET/flights/trained_model
python3 -m trainer.task --bucket $BUCKET --num_examples=1000 --func=wide_deep

gsutil ls gs://$BUCKET/flights/trained_model

%%bash
model_dir=$(gsutil ls gs://$BUCKET/flights/trained_model/export | tail -1)
echo $model_dir
saved_model_cli show --tag_set serve --signature_def serving_default --dir $model_dir

"""
Explanation: Wide-and-deep model
End of explanation
"""


%%bash
JOBID=flights_$(date +%Y%m%d_%H%M%S)
gsutil -m rm -rf gs://$BUCKET/flights/trained_model

gcloud beta ai-platform jobs submit training $JOBID \
   --staging-bucket=gs://$BUCKET  --region=$REGION \

   --master-machine-type=n1-standard-4 --scale-tier=CUSTOM \
   -- \
   --bucket=$BUCKET --num_examples=1000000 --func=wide_deep

"""
Explanation: Run on full dataset
Submit the Python package to Cloud AI Platform Training and have it process full dataset.
This is serverless, so you can do the equivalent by running ./train_model.sh from CloudShell.
You don't need a Notebook environment to do this.
End of explanation
"""


%%bash
model_dir=$(gsutil ls gs://$BUCKET/flights/trained_model/export | tail -1)
echo $model_dir
saved_model_cli show --tag_set serve --signature_def serving_default --dir $model_dir

"""
Explanation: The final validation RMSE was 0.214
End of explanation
"""


%%bash
./train_model.sh $BUCKET wide_deep 500000

"""
Explanation: Option 1: Single model training on Cloud AI Platform
This will take 15-20 minutes
End of explanation
"""


%%bash

Comment this line. Make sure to wait for hyperparam job to finish, find best model and change BEST_MODEL in following cell!

JOBID=flights_$(date +%Y%m%d_%H%M%S)
gsutil -m rm -rf gs://$BUCKET/flights/trained_model

gcloud ai-platform jobs submit training $JOBID \
   --staging-bucket=gs://$BUCKET  --region=$REGION \
   --module-name=trainer.task \
   --python-version=3.7 --runtime-version=2.1 \
   --package-path=${PWD}/flights/trainer \
   --config=hyperparam.yaml \
   -- \
   --bucket=$BUCKET --num_examples=100000 --func=wide_deep

"""
Explanation: Option 2: Hyperparameter tuning on Cloud AI Platform
This will take 2-3 hours
End of explanation
"""


%%bash
MODEL_NAME=flights
VERSION_NAME=tf2
BEST_MODEL=""      # Option 1: No hyperparameter tuning
#BEST_MODEL="15/"   # Option 2: CHANGE AS NEEDED 
EXPORT_PATH=$(gsutil ls gs://$BUCKET/flights/trained_model/${BEST_MODEL}export | tail -1)
echo $EXPORT_PATH

if [[ $(gcloud ai-platform models list --format='value(name)' | grep $MODEL_NAME) ]]; then
    echo "$MODEL_NAME already exists"
else
    # create model
    echo "Creating $MODEL_NAME"
    gcloud ai-platform models create --regions=$REGION $MODEL_NAME
fi

if [[ $(gcloud ai-platform versions list --model $MODEL_NAME --format='value(name)' | grep $VERSION_NAME) ]]; then
    echo "Deleting already existing $MODEL_NAME:$VERSION_NAME ... "
    gcloud ai-platform versions delete --quiet --model=$MODEL_NAME $VERSION_NAME
    echo "Please run this cell again if you don't see a Creating message ... "
    sleep 10
fi

# create model
echo "Creating $MODEL_NAME:$VERSION_NAME"
gcloud ai-platform versions create --model=$MODEL_NAME $VERSION_NAME --async \
       --framework=tensorflow --python-version=3.7 --runtime-version=2.1 \
       --origin=$EXPORT_PATH --staging-bucket=gs://$BUCKET

%%writefile example_input.json
{"dep_delay": 14.0, "taxiout": 13.0, "distance": 319.0, "avg_dep_delay": 25.863039, "avg_arr_delay": 27.0, "carrier": "WN", "dep_lat": 32.84722, "dep_lon": -96.85167, "arr_lat": 31.9425, "arr_lon": -102.20194, "origin": "DAL", "dest": "MAF"}
{"dep_delay": -9.0, "taxiout": 21.0, "distance": 301.0, "avg_dep_delay": 41.050808, "avg_arr_delay": -7.0, "carrier": "EV", "dep_lat": 29.984444, "dep_lon": -95.34139, "arr_lat": 27.544167, "arr_lon": -99.46167, "origin": "IAH", "dest": "LRD"}

!gcloud ai-platform predict --model=flights --version=tf2 --json-instances=example_input.json

%%bash
./call_predict.py --project=$PROJECT
echo
echo "With reasoning"
./call_predict_reason.py --project=$PROJECT

"""
Explanation: Deploy model
End of explanation
"""


