!sudo chown -R jupyter:jupyter /home/jupyter/training-data-analyst

# Ensure the right version of Tensorflow is installed.
!pip freeze | grep tensorflow==2.5

from google.cloud import bigquery
import tensorflow as tf
import numpy as np
import shutil
print(tf.__version__)

"""
Explanation: <h1> 2c. Loading large datasets progressively with the tf.data.Dataset </h1>

In this notebook, we continue reading the same small dataset, but refactor our ML pipeline in two small, but significant, ways:

Refactor the input to read data from disk progressively.
Refactor the feature creation so that it is not one-to-one with inputs.

The Pandas function in the previous notebook first read the whole data into memory -- on a large dataset, this won't be an option.
End of explanation
"""


CSV_COLUMNS = ['fare_amount', 'pickuplon','pickuplat','dropofflon','dropofflat','passengers', 'key']
DEFAULTS = [[0.0], [-74.0], [40.0], [-74.0], [40.7], [1.0], ['nokey']]

# TODO: Create an appropriate input function read_dataset
def read_dataset(filename, mode):
    #TODO Add CSV decoder function and dataset creation and methods
    return dataset
  
def get_train_input_fn():
  return read_dataset('./taxi-train.csv', mode = tf.estimator.ModeKeys.TRAIN)

def get_valid_input_fn():
  return read_dataset('./taxi-valid.csv', mode = tf.estimator.ModeKeys.EVAL)

"""
Explanation: <h2> 1. Refactor the input </h2>

Read data created in Lab1a, but this time make it more general, so that we can later handle large datasets. We use the Dataset API for this. It ensures that, as data gets delivered to the model in mini-batches, it is loaded from disk only when needed.
End of explanation
"""


INPUT_COLUMNS = [
    tf.feature_column.numeric_column('pickuplon'),
    tf.feature_column.numeric_column('pickuplat'),
    tf.feature_column.numeric_column('dropofflat'),
    tf.feature_column.numeric_column('dropofflon'),
    tf.feature_column.numeric_column('passengers'),
]

def add_more_features(feats):
  # Nothing to add (yet!)
  return feats

feature_cols = add_more_features(INPUT_COLUMNS)

"""
Explanation: <h2> 2. Refactor the way features are created. </h2>

For now, pass these through (same as previous lab).  However, refactoring this way will enable us to break the one-to-one relationship between inputs and features.
End of explanation
"""


tf.compat.v1.logging.set_verbosity(tf.compat.v1.logging.INFO)
OUTDIR = 'taxi_trained'
shutil.rmtree(OUTDIR, ignore_errors = True) # start fresh each time
model = tf.compat.v1.estimator.LinearRegressor(
      feature_columns = feature_cols, model_dir = OUTDIR)
model.train(input_fn = get_train_input_fn, steps = 200)

"""
Explanation: <h2> Create and train the model </h2>

Note that we train for num_steps * batch_size examples.
End of explanation
"""


metrics = model.evaluate(input_fn = get_valid_input_fn, steps = None)
print('RMSE on dataset = {}'.format(np.sqrt(metrics['average_loss'])))

"""
Explanation: <h3> Evaluate model </h3>

As before, evaluate on the validation data.  We'll do the third refactoring (to move the evaluation into the training loop) in the next lab.
End of explanation
"""


