# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

# ![texto alternativo](https://drive.google.com/uc?id=1CXVh-Dy6T792nWexE0kH45AOqqFj2_wG)
#
#
# <center> <h1>Maestría en Analytics</h1> </center>
# <center> <h1>Modelos Avanzados para el Análisis de Datos 1</h1> </center>
# <center> <h1>Exercise 8 - Car Price Prediction </h1> </center>
#
# **Grupo 3**
# - <NAME>
# - <NAME> 
# - <NAME>
# - <NAME>
#
# 29 June 2020

# # Exercise 8
#
# ## Car Price Prediction
#
# Predict if the price of a car is low or high

# +
#Leyendo datos, convirtiendo dummies
# %matplotlib inline
import pandas as pd
import numpy as np

data = pd.read_csv('../datasets/dataTrain_carListings.zip')
data = data.loc[data['Model'].str.contains('Camry')].drop(['Make', 'State'], axis=1)
data = data.join(pd.get_dummies(data['Model'], prefix='M'))
data

# +
#Definiendo niveles para variables de respuesta Y. Es 1 cuando supera la media de los valores.
data['HighPrice'] = (data['Price'] > data['Price'].mean()).astype(int)
data = data.drop(['Model', 'Price'], axis=1)

data.head()
# -

data.shape

y = data['HighPrice']
X = data.drop(['HighPrice'], axis=1)

y.mean()

# En general, entramos las categorias de la variable de respuesta balanceadas.

# Partir para base de entrenamiento y test en X y y.
from sklearn.model_selection import train_test_split
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.33, random_state=42)


# # Exercise 8.1
#
# Estimate a Decision Tree Classifier Manually using the code created in the Notebook #13
#
# Evaluate the accuracy on the testing set

#Definiendo función para índice de Gini
def gini(y):
    if y.shape[0] == 0:
        return 0
    else:
        return 1 - (y.mean()**2 + (1 - y.mean())**2)


#Definiendo función para Impurity
def gini_impurity(X_col, y, split):
    "Calculate the gain of an split k on feature j"
    
    filter_l = X_col < split
    y_l = y.loc[filter_l]
    y_r = y.loc[~filter_l]
    
    n_l = y_l.shape[0]
    n_r = y_r.shape[0]
    
    gini_y = gini(y)
    gini_l = gini(y_l)
    gini_r = gini(y_r)
    
    gini_impurity_ = gini_y - (n_l / (n_l + n_r) * gini_l + n_r / (n_l + n_r) * gini_r)
    
    return gini_impurity_


# Definiendo función para particiones por percentiles vs predictores.
def best_split(X, y, num_pct=10):
    
    features = range(X.shape[1])
    
    best_split = [0, 0, 0]  # j, split, gain
    
    # For all features
    for j in features:
        
        splits = np.percentile(X.iloc[:, j], np.arange(0, 100, 100.0 / (num_pct+1)).tolist())
        splits = np.unique(splits)[1:]
        
        # For all splits
        for split in splits:
            gain = gini_impurity(X.iloc[:, j], y, split)
                        
            if gain > best_split[2]:
                best_split = [j, split, gain]
    
    return best_split


# Definiendo función para crecer manualmente árbol de clasificación.
def tree_grow(X, y, level=0, min_gain=0.001, max_depth=None, num_pct=10):
    
    # If only one observation
    if X.shape[0] == 1:
        tree = dict(y_pred=y.iloc[:1].values[0], y_prob=0.5, level=level, split=-1, n_samples=1, gain=0)
        return tree
    
    # Calculate the best split
    j, split, gain = best_split(X, y, num_pct)
    
    # save tree and estimate prediction
    y_pred = int(y.mean() >= 0.5) 
    y_prob = (y.sum() + 1.0) / (y.shape[0] + 2.0)  # Laplace correction
    
    tree = dict(y_pred=y_pred, y_prob=y_prob, level=level, split=-1, n_samples=X.shape[0], gain=gain)
    
    # Check stooping criteria
    if gain < min_gain:
        return tree
    if max_depth is not None:
        if level >= max_depth:
            return tree   
    
    # No stooping criteria was meet, then continue to create the partition
    filter_l = X.iloc[:, j] < split
    X_l, y_l = X.loc[filter_l], y.loc[filter_l]
    X_r, y_r = X.loc[~filter_l], y.loc[~filter_l]
    tree['split'] = [j, split]

    # Next iteration to each split
    
    tree['sl'] = tree_grow(X_l, y_l, level + 1, min_gain=min_gain, max_depth=max_depth, num_pct=num_pct)
    tree['sr'] = tree_grow(X_r, y_r, level + 1, min_gain=min_gain, max_depth=max_depth, num_pct=num_pct)
    
    return tree


#Definir función para predicción
def tree_predict(X, tree, proba=False):
    
    predicted = np.ones(X.shape[0])

    # Check if final node
    if tree['split'] == -1:
        if not proba:
            predicted = predicted * tree['y_pred']
        else:
            predicted = predicted * tree['y_prob']
            
    else:
        
        j, split = tree['split']
        filter_l = (X.iloc[:, j] < split)
        X_l = X.loc[filter_l]
        X_r = X.loc[~filter_l]

        if X_l.shape[0] == 0:  # If left node is empty only continue with right
            predicted[~filter_l] = tree_predict(X_r, tree['sr'], proba)
        elif X_r.shape[0] == 0:  # If right node is empty only continue with left
            predicted[filter_l] = tree_predict(X_l, tree['sl'], proba)
        else:
            predicted[filter_l] = tree_predict(X_l, tree['sl'], proba)
            predicted[~filter_l] = tree_predict(X_r, tree['sr'], proba)

    return predicted


tree8_1 = tree_grow(X_train, y_train, level=0, min_gain=0.001, max_depth=None, num_pct=10)

tree8_1

from sklearn.metrics import mean_squared_error, accuracy_score

res = tree_predict(X_test, tree8_1)
Accuracy8_1 = accuracy_score(y_test, res)
Accuracy8_1

# Realizar el Árbol de Clasificación manualmente, nos arroja una exactitud de 84.33% al probarlo con y_test a un split del 0.33

# # Exercise 8.2
#
# Estimate a Bagging of 10 Decision Tree Classifiers Manually using the code created in the Notebook #7
#
# Evaluate the accuracy on the testing set

# +
n_estimators = 10
# set a seed for reproducibility
np.random.seed(123)

n_samples = X_train.shape[0]

# create bootstrap samples (will be used to select rows from the DataFrame)
samples = [np.random.choice(a=n_samples, size=n_samples, replace=True) for _ in range(n_estimators)]
samples
# -

#Entrenar arboles
trees = {}
for i in range(n_estimators):
    trees[i] = tree_grow(X_train.iloc[samples[i]], y_train.iloc[samples[i]], level=0, min_gain=0.001, max_depth=None, num_pct=10)


#Dataframe para el predict
y_pred_df = pd.DataFrame(index=X_test.index, columns=list(range(n_estimators)))


# +
# Predict 
for i in range(n_estimators):
    y_pred_df.iloc[:, i] = tree_predict(X_test, trees[i])

y_pred_df
# -

# Predict using majority voting
y_pred_df.sum(axis=1)[:10]

# +
y_pred = (y_pred_df.sum(axis=1) >= (n_estimators / 2)).astype(np.int)

from sklearn import metrics
metrics.accuracy_score(y_pred, y_test)


# -

# Para este Bagging con 10 árboles creados manualmente, obtenemos un Accuracy de 86.26%, siendo más alto que el obtenido para el arbol individual.

# # Exercise 8.3
#
# Implement the variable max_features on the Decision Tree Classifier created in 11.1.
#
# Compare the impact in the results by varing the parameter max_features
#
# Evaluate the accuracy on the testing set

def tree_grow_mf(X, y, level=0, min_gain=0.001, max_depth=None, num_pct=10, max_features=len(X.columns)):
    # If only one observation
    if X.shape[0] == 1:
        tree = dict(y_pred=y.iloc[:1].values[0], y_prob=0.5, level=level, split=-1, n_samples=1, gain=0)
        return tree
    
    # Calculate the best split
    if max_features>len(X.columns):
        max_features=len(X.columns)
        print("El maximo de columnas soportable es %f", len(X.columns))
    
    import random

    j, split, gain = best_split(X.iloc[:, random.sample(range(0, len(X.columns)), max_features)], y, num_pct)
    
    # save tree and estimate prediction
    y_pred = int(y.mean() >= 0.5) 
    y_prob = (y.sum() + 1.0) / (y.shape[0] + 2.0)  # Laplace correction
    
    tree = dict(y_pred=y_pred, y_prob=y_prob, level=level, split=-1, n_samples=X.shape[0], gain=gain)
    
    # Check stooping criteria
    if gain < min_gain:
        return tree
    if max_depth is not None:
        if level >= max_depth:
            return tree   
    
    # No stooping criteria was meet, then continue to create the partition
    filter_l = X.iloc[:, j] < split
    X_l, y_l = X.loc[filter_l], y.loc[filter_l]
    X_r, y_r = X.loc[~filter_l], y.loc[~filter_l]
    tree['split'] = [j, split]

    # Next iteration to each split
    
    tree['sl'] = tree_grow(X_l, y_l, level + 1, min_gain=min_gain, max_depth=max_depth, num_pct=num_pct)
    tree['sr'] = tree_grow(X_r, y_r, level + 1, min_gain=min_gain, max_depth=max_depth, num_pct=num_pct)
    
    return tree


tree8_3 = tree_grow_mf(X_train, y_train, level=0, min_gain=0.001, max_depth=None, num_pct=10, max_features=5)

res8_3 = tree_predict(X_test, tree8_3)
accuracy8_3 = accuracy_score(y_test, res8_3)
accuracy8_3

# Cuando se incorpora al modelo de clasificación manual, la variable max_features (usando en este caso 5), obtenemos Accuracy del 84.33%

tree8_3_2 = tree_grow_mf(X_train, y_train, level=0, min_gain=0.001, max_depth=None, num_pct=10, max_features=2)
res8_3_2 = tree_predict(X_test, tree8_3_2)
accuracy8_3_2 = accuracy_score(y_test, res8_3_2)
accuracy8_3_2

tree8_3_4 = tree_grow_mf(X_train, y_train, level=0, min_gain=0.001, max_depth=None, num_pct=10, max_features=4)
res8_3_4 = tree_predict(X_test, tree8_3_4)
accuracy8_3_4 = accuracy_score(y_test, res8_3_4)
accuracy8_3_4

tree8_3_3 = tree_grow_mf(X_train, y_train, level=0, min_gain=0.001, max_depth=None, num_pct=10, max_features=3)
res8_3_3 = tree_predict(X_test, tree8_3_3)
accuracy8_3_3 = accuracy_score(y_test, res8_3_3)
accuracy8_3_3


# Al Probar los cambios en el parámetro max_features obtenemos los siguientes resultados:
# * max_features=5, Acurracy = 84.33%
# * max_features=2, Acurracy = 58.24%
# * max_features=4, Acurracy = 84.33%
# * max_features=3, Acurracy = 84.23%

# # Exercise 8.4
#
# Estimate a Bagging of 10 Decision Tree Classifiers with `max_features = log(n_features)`
#
# Evaluate the accuracy on the testing set

# +
###Enfoque Manual para max_features = log(n_features)

def tree_grow_mf2(X, y, level=0, min_gain=0.001, max_depth=None, num_pct=10):
    # If only one observation
    if X.shape[0] == 1:
        tree = dict(y_pred=y.iloc[:1].values[0], y_prob=0.5, level=level, split=-1, n_samples=1, gain=0)
        return tree
    
    # Calculate the best split
    max_features = np.int(np.round(np.log(len(X.columns)), decimals=0))
    
    import random

    j, split, gain = best_split(X.iloc[:, random.sample(range(0, len(X.columns)), max_features)], y, num_pct)
    
    # save tree and estimate prediction
    y_pred = int(y.mean() >= 0.5) 
    y_prob = (y.sum() + 1.0) / (y.shape[0] + 2.0)  # Laplace correction
    
    tree = dict(y_pred=y_pred, y_prob=y_prob, level=level, split=-1, n_samples=X.shape[0], gain=gain)
    
    # Check stooping criteria
    if gain < min_gain:
        return tree
    if max_depth is not None:
        if level >= max_depth:
            return tree   
    
    # No stooping criteria was meet, then continue to create the partition
    filter_l = X.iloc[:, j] < split
    X_l, y_l = X.loc[filter_l], y.loc[filter_l]
    X_r, y_r = X.loc[~filter_l], y.loc[~filter_l]
    tree['split'] = [j, split]

    # Next iteration to each split
    
    tree['sl'] = tree_grow(X_l, y_l, level + 1, min_gain=min_gain, max_depth=max_depth, num_pct=num_pct)
    tree['sr'] = tree_grow(X_r, y_r, level + 1, min_gain=min_gain, max_depth=max_depth, num_pct=num_pct)
    
    return tree


# -

tree8_4 = tree_grow_mf2(X_train, y_train, level=0, min_gain=0.001, max_depth=None, num_pct=10)
res8_4 = tree_predict(X_test, tree8_4)
accuracy8_4 = accuracy_score(y_test, res8_4)
accuracy8_4

# Al hacer el Bagging de estos 10 arboles, usando el criterio Log y de forma manual, encontramos un Accuracy del 84.47%

# # Exercise 8.5
#
# Using sklearn, train a RandomForestClassifier
#
# Evaluate the accuracy on the testing set

# +
import matplotlib.pyplot as plt
from sklearn.ensemble import RandomForestClassifier
from sklearn.model_selection import cross_val_score

clf = RandomForestClassifier(n_estimators=200)
clf

# +
#Train the model using the training sets y_pred=clf.predict(X_test)
clf.fit(X_train,y_train)

y_pred=clf.predict(X_test)

# +
#check the accuracy using actual and predicted values.
#Import scikit-learn metrics module for accuracy calculation
from sklearn import metrics


# Model Accuracy, how often is the classifier correct?
print("Accuracy:",metrics.accuracy_score(y_test, y_pred))
# -

pd.Series(cross_val_score(clf, X, y, cv=10)).describe()

# **Random Forest**: Con este modelo, los resultados del Accuracy son 83,77%  cuando tomamos el split de las bases (33% para Test). Al hacerlo también con Crossvalidation Kfolds=10, la media del Accuracy obtenida es de 83.61%.

# # Exercise 8.6
#
# Find the best parameters of the RandomForestClassifier (max_depth, max_features, n_estimators)
#
# Evaluate the accuracy on the testing set

feature_cols = X.columns
feature_cols

# **Tuning n_estimators**

# +
# Tuning con Crossvalidation
# list of values to try for n_estimators
estimator_range = range(1, 200, 2)

# list to store the average Accuracy for each value of n_estimators
accuracy_scores = []

# use 5-fold cross-validation with each value of n_estimators (WARNING: SLOW!)
for estimator in estimator_range:
    clf = RandomForestClassifier(n_estimators=estimator, random_state=1, n_jobs=-1)
    accuracy_scores.append(cross_val_score(clf, X, y, cv=5, scoring='accuracy').mean())
# -

plt.plot(estimator_range, accuracy_scores)
plt.xlabel('n_estimators')
plt.ylabel('accuracy');

# show the best accuracy and the corresponding n_estimador
sorted(zip(accuracy_scores, estimator_range))[::-1][0]

# +
# Tuning con el split de test al 0.33
# list of values to try for n_estimators
estimator_range = range(1, 200, 2)

# list to store the average Accuracy for each value of n_estimators
accuracy_scores = []

for estimator in estimator_range:
    clf = RandomForestClassifier(n_estimators=estimator, random_state=1, n_jobs=-1)
    clf.fit(X_train,y_train)
    accuracy_scores.append(metrics.accuracy_score(y_test,clf.predict(X_test)))
    
    
plt.plot(estimator_range, accuracy_scores)
plt.xlabel('n_estimators')
plt.ylabel('Accuracy');
# -

# show the best accuracy and the corresponding n_estimador
sorted(zip(accuracy_scores, estimator_range))[::-1][0]

# **n_estimators:** Con CV el mejor parámetro se encuentra en 84.29% con 3 árboles. Por otra parte, usando el split inicial de test al 0.33, el mejor desempeño respecto a la exactitud, lo conseguimos en 84.35% con 29 árboles.

# **Tuning max_features**

# +
#Tuning max_features con CV
# list of values to try for max_features
feature_range = range(1, len(feature_cols)+1)

# list to store the average Accuracy for each value of max_features
accuracy_scores = []

# use 10-fold cross-validation with each value of max_features (WARNING: SLOW!)
for feature in feature_range:
    clf = RandomForestClassifier(n_estimators=200, max_features=feature, random_state=1, n_jobs=-1)
    accuracy_scores.append(cross_val_score(clf, X, y, cv=10, scoring='accuracy').mean())

plt.plot(feature_range, accuracy_scores)
plt.xlabel('max_features')
plt.ylabel('Accuracy');

# -

# show the best accuracy and the corresponding max_features
sorted(zip(accuracy_scores, feature_range))[::-1][0]

# +
#Tuning max_features con split inicial 0.33
# list of values to try for max_features
feature_range = range(1, len(feature_cols)+1)

# list to store the average Accuracy for each value of max_features
accuracy_scores = []


for feature in feature_range:
    clf = RandomForestClassifier(n_estimators=200, max_features=feature, random_state=1, n_jobs=-1)
    clf.fit(X_train,y_train)
    accuracy_scores.append(metrics.accuracy_score(y_test,clf.predict(X_test)).mean())

plt.plot(feature_range, accuracy_scores)
plt.xlabel('max_features')
plt.ylabel('Accuracy');
# -

# show the best accuracy and the corresponding max_features
sorted(zip(accuracy_scores, feature_range))[::-1][0]


# **max_features**: al emplear ambos enfoques, el número de predictores coindice en que debería ser 9. Las exactitudes obtenidas son muy similares. Con CV = 84.67%, con el split de test al 0.33 = 84.83%

# **Tuning max_depth**

# +
# Tuning con Crossvalidation
# list of values to try for n_estimators
max_depth_range = range(1, 20, 1)

# list to store the average Accuracy for each value of n_estimators
accuracy_scores = []

# use 5-fold cross-validation with each value of n_estimators (WARNING: SLOW!)
for max_d in max_depth_range:
    clf = RandomForestClassifier(n_estimators=200, max_depth=max_d, random_state=1, n_jobs=-1)
    accuracy_scores.append(cross_val_score(clf, X, y, cv=5, scoring='accuracy').mean())
    
plt.plot(max_depth_range, accuracy_scores)
plt.xlabel('max_depth')
plt.ylabel('accuracy');
# -

# show the best accuracy and the corresponding max_depth
sorted(zip(accuracy_scores, max_depth_range))[::-1][0]

# +
# Tunning con el split de test al 0.33
# list of values to try for max_depth
max_depth_range = range(1, 20, 1)

# list to store the average Accuracy for each value of n_estimators
accuracy_scores = []

# use 5-fold cross-validation with each value of n_estimators (WARNING: SLOW!)
for max_d in max_depth_range:
    clf = RandomForestClassifier(n_estimators=200, max_depth=max_d, random_state=1, n_jobs=-1)
    clf.fit(X_train,y_train)
    accuracy_scores.append(metrics.accuracy_score(y_test,clf.predict(X_test)))
    
plt.plot(max_depth_range, accuracy_scores)
plt.xlabel('max_depth')
plt.ylabel('accuracy');
# -

# show the best accuracy and the corresponding max_depth
sorted(zip(accuracy_scores, max_depth_range))[::-1][0]

# **Max_Depth**: La profundidad máxima encontrada para el modelo es de 6 layers (Accuracy de 88.28% con CV) o de 7 layers (Accuracy del 88.01% para el modelo entrenado y probado con split al 0.33)

# **Construyendo Modelo con los mejores parámetros**

# +
#Fitting a Random Forest with the best parameters      

clf = RandomForestClassifier(n_estimators=29, max_features=9, max_depth=6, random_state=1, n_jobs=-1)
clf.fit(X, y)
# -

pd.Series(cross_val_score(clf, X, y, cv=3)).describe()

# **Usando la combinación de los mejores parámetros encontramos una media de 88.4% en Accuracy, el mejor conseguido hasta este punto con RandomForest**

# Determinando la importancia de las variables
pd.DataFrame({'feature':feature_cols, 'importance':clf.feature_importances_}).sort_values('importance')

# La variable que más aporta al modelo es "Mileage"

# # Exercise 8.7 
#
# Using xgboost train a XGBClassifier 
#
# Evaluate the accuracy on the testing set

from xgboost import XGBClassifier
from sklearn.metrics import f1_score, accuracy_score

clf = XGBClassifier()
clf

clf.fit(X_train, y_train)
y_pred = clf.predict(X_test)
f1_score(y_pred, y_test.values), accuracy_score(y_pred, y_test.values)

# +
import matplotlib.pyplot as plt
from sklearn import metrics
y_preds = clf.predict_proba(X_test)

preds = y_preds[:,1]

(y_test, preds)

auc_score = metrics.auc(fpr, tpr)

plt.clf()

plt.title('Curva ROC')
plt.plot(fpr, tpr, label='AUC = {:.2f}'.format(auc_score))


plt.plot([0,1],[0,1],'r--')

plt.xlim([-0.1,1.1])
plt.ylim([-0.1,1.1])
plt.ylabel('True Positive Rate')
plt.xlabel('False Positive Rate')

plt.legend(loc='lower right')
plt.show()
# -

# **XGBoost**: para este modelo se obtiene un muy buen resultado de Accuracy 87.90%. El F1_Score también indica un modelo de muy buen desempeño: 89.89%. Obtenemos el resultado de la métrica AUC en 0.94

# # Exercise 8.8
#
# Using xgboost train a XGBClassifier 
#
# Modify the parameters learning rate, gamma, colsample_bytree. Explain what each parameter means.
#
# Evaluate the accuracy on the testing set

np.arange(0,100,10)

# ### Xtreme Gradient Boosting
#

# +

accuracy = list()
param = list()

for lr in np.arange(0.75,0.8,0.005):
    for gamm in np.arange(55,65,1):
        for col_s in np.arange(0.8,1,0.02):
            clf = XGBClassifier(learning_rate=lr, gamma=gamm, colsample_bytree=col_s)
            clf.fit(X_train, y_train)
            y_pred = clf.predict(X_test)
            param.append([lr, gamm, col_s])
            accuracy.append(accuracy_score(y_pred, y_test.values))
sorted(zip(accuracy, param))[::-1][0]


# +
import matplotlib.pyplot as plt
from sklearn import metrics
y_preds = clf.predict_proba(X_test)

preds = y_preds[:,1]

(y_test, preds)

auc_score = metrics.auc(fpr, tpr)

plt.clf()

plt.title('Curva ROC')
plt.plot(fpr, tpr, label='AUC = {:.2f}'.format(auc_score))


plt.plot([0,1],[0,1],'r--')

plt.xlim([-0.1,1.1])
plt.ylim([-0.1,1.1])
plt.ylabel('True Positive Rate')
plt.xlabel('False Positive Rate')

plt.legend(loc='lower right')
plt.show()
# -

# ### Xtreme Gradient Boosting
#

# Al modificar los parámetros de Xtreme Gradient Boosting encontramos uno de los mejores Accuracies conseguidos en todo el taller, es decir 88.41% (igual a RandomForest cuando combina los mejores parámetros). Los parámetros modificados, conseguidos y explicados, se muestran a continuación:
#
# * Learning_rate=0.75, corresponde a la tasa de aprendizaje del modelo y valor que permite al modelo ser más robusto al reducir los pesos en cada iteración.
# * Gamma=60, corresponde a la reducción de pérdida mínima requerida para hacer una partición adicional en un nodo del árbol. actua como un parámetro de regularización.
# * Colsample_bytree=0.98, corresponde a la proporción de submuestra de columnas al construir cada árbol. 
#
#

StarcoderdataJupyterScriptsDedupFiltered