# We will use some np and pandas for dealing with input data.
import numpy as np
import pandas as pd
# And of course, we need tensorflow.
import tensorflow as tf

from distutils.version import StrictVersion

tf.__version__

"""
Explanation: Introduction
In this notebook, we will 
- Learn how to use BoostedTrees Classifier for training and evaluating
- Explore how training can be speeded up for small datasets
- Will develop intuition for how some of the hyperparameters affect the performance of boosted trees.
End of explanation
"""


tf.logging.set_verbosity(tf.logging.INFO)
tf.set_random_seed(123)

# Load dataset.
dftrain = pd.read_csv('https://storage.googleapis.com/tf-datasets/titanic/train.csv')
dfeval = pd.read_csv('https://storage.googleapis.com/tf-datasets/titanic/eval.csv')
y_train = dftrain.pop('survived')
y_eval = dfeval.pop('survived')

fcol = tf.feature_column
CATEGORICAL_COLUMNS = ['sex', 'n_siblings_spouses', 'parch', 'class', 'deck',
                       'embark_town', 'alone']
NUMERIC_COLUMNS = ['age', 'fare']

def one_hot_cat_column(feature_name, vocab):
  return fcol.indicator_column(
      fcol.categorical_column_with_vocabulary_list(feature_name,
                                                 vocab))
fc = []
for feature_name in CATEGORICAL_COLUMNS:
  # Need to one-hot encode categorical features.
  vocabulary = dftrain[feature_name].unique()
  fc.append(one_hot_cat_column(feature_name, vocabulary))

for feature_name in NUMERIC_COLUMNS:
  fc.append(fcol.numeric_column(feature_name,
                                dtype=tf.float32))

# Prepare the input fn. Use the entire dataset for a batch since this is such a small dataset.

def make_input_fn(X, y, n_epochs=None, do_batching=True):
  def input_fn():
    BATCH_SIZE = len(y)
    dataset = tf.data.Dataset.from_tensor_slices((X.to_dict(orient='list'), y))
    # For training, cycle thru dataset as many times as need (n_epochs=None).    
    dataset = dataset.repeat(n_epochs)  
    if do_batching:
      dataset = dataset.batch(BATCH_SIZE)
    return dataset
  return input_fn

"""
Explanation: Load dataset
We will be using the titanic dataset, where the goal is to predict passenger survival given characteristiscs such as gender, age, class, etc.
End of explanation
"""


TRAIN_SIZE = len(dftrain)
params = {
  'n_trees':10,
  'center_bias':False,
  'l2_regularization':1./TRAIN_SIZE # regularization is per instance, so if you are familiar with XGBoost, you need to divide these values by the num of examples per layer
}


"""
Explanation: Training and Evaluating Classifiers
End of explanation
"""


# Training and evaluation input functions.
n_batches_per_layer = 1  # Use one batch, consisting of the entire dataset to build each layer in the tree.
DO_BATCHING = True

train_input_fn = make_input_fn(dftrain, y_train, n_epochs=None, do_batching=DO_BATCHING)
eval_input_fn = make_input_fn(dfeval, y_eval, n_epochs=1, do_batching=DO_BATCHING)
est = tf.estimator.BoostedTreesClassifier(fc,
                                          n_batches_per_layer=n_batches_per_layer,
                                          **params)

est.train(train_input_fn)

# Eval.
pd.Series(est.evaluate(eval_input_fn))

"""
Explanation: Train and evaluate the model. We will look at accuracy first.
End of explanation
"""


# Training and evaluation input functions.
est = tf.estimator.DNNClassifier([10], fc)
est.train(train_input_fn, max_steps=1000)
pd.Series(est.evaluate(eval_input_fn))

"""
Explanation: DNN classifier.
End of explanation
"""


pred_dicts = list(est.predict(eval_input_fn))
probs = pd.Series([pred['probabilities'][1] for pred in pred_dicts])

probs.plot(kind='hist', bins=20, title='predicted probabilities');

"""
Explanation: Improving the performance
??? Can you get better performance out of the classifier?  What parameters the boosted trees are most sensitive to?
??? Can you see if Boosted trees can overfit? How can you demonstrate it?
Results
Let's understand how our model is performing.
End of explanation
"""


from sklearn.metrics import roc_curve
from matplotlib import pyplot as plt

fpr, tpr, _ = roc_curve(y_eval, probs)
plt.plot(fpr, tpr)
plt.title('ROC curve')
plt.xlabel('false positive rate')
plt.ylabel('true positive rate')
plt.xlim(0,)
plt.ylim(0,);

"""
Explanation: ??? Why are the probabilities right skewed?
Let's plot an ROC curve to understand model performance for various predicition probabilities.
End of explanation
"""


