# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     name: python3
# ---

# + [markdown] id="view-in-github" colab_type="text"
# <a href="https://colab.research.google.com/github/Jaydenzk/DS-Unit-2-Applied-Modeling/blob/master/assignment_applied_modeling_completed(module1-4).ipynb" target="_parent"><img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"/></a>

# + [markdown] id="nCc3XZEyG3XV" colab_type="text"
# Lambda School Data Science, Unit 2: Predictive Modeling
#
# # Applied Modeling, Module 1
#
# You will use your portfolio project dataset for all assignments this sprint.
#
# ## Assignment
#
# Complete these tasks for your project, and document your decisions.
#
# - [ ] Choose your target. Which column in your tabular dataset will you predict?
# - [ ] Choose which observations you will use to train, validate, and test your model. And which observations, if any, to exclude.
# - [ ] Determine whether your problem is regression or classification.
# - [ ] Choose your evaluation metric.
# - [ ] Begin with baselines: majority class baseline for classification, or mean baseline for regression, with your metric of choice.
# - [ ] Begin to clean and explore your data.
# - [ ] Choose which features, if any, to exclude. Would some features "leak" information from the future?
#
# ## Reading
# - [Attacking discrimination with smarter machine learning](https://research.google.com/bigpicture/attacking-discrimination-in-ml/), by Google Research, with  interactive visualizations. _"A threshold classifier essentially makes a yes/no decision, putting things in one category or another. We look at how these classifiers work, ways they can potentially be unfair, and how you might turn an unfair classifier into a fairer one. As an illustrative example, we focus on loan granting scenarios where a bank may grant or deny a loan based on a single, automatically computed number such as a credit score."_
# - [How Shopify Capital Uses Quantile Regression To Help Merchants Succeed](https://engineering.shopify.com/blogs/engineering/how-shopify-uses-machine-learning-to-help-our-merchants-grow-their-business)
# - [Maximizing Scarce Maintenance Resources with Data: Applying predictive modeling, precision at k, and clustering to optimize impact](https://towardsdatascience.com/maximizing-scarce-maintenance-resources-with-data-8f3491133050), **by Lambda DS3 student** <NAME>. His blog post extends the Tanzania Waterpumps scenario, far beyond what's in the lecture notebook.
# - [Notebook about how to calculate expected value from a confusion matrix by treating it as a cost-benefit matrix](https://github.com/podopie/DAT18NYC/blob/master/classes/13-expected_value_cost_benefit_analysis.ipynb)
# - [Simple guide to confusion matrix terminology](https://www.dataschool.io/simple-guide-to-confusion-matrix-terminology/) by <NAME>, with video
# - [Visualizing Machine Learning Thresholds to Make Better Business Decisions](https://blog.insightdatascience.com/visualizing-machine-learning-thresholds-to-make-better-business-decisions-4ab07f823415)

# + id="IxdRE5iqYPkV" colab_type="code" outputId="442984b2-c703-468d-b51d-fed977c9020a" colab={"resources": {"http://localhost:8080/nbextensions/google.colab/files.js": {"data": "Ly8gQ29weXJpZ2h0IDIwMTcgR29vZ2xlIExMQwovLwovLyBM<KEY> "ok": true, "headers": [["content-type", "application/javascript"]], "status": 200, "status_text": ""}}, "base_uri": "https://localhost:8080/", "height": 78}
from google.colab import files
uploaded = files.upload()

# + id="DB55aJbefJak" colab_type="code" outputId="746c82b6-9433-425e-fe8b-6db077d39f2f" colab={"base_uri": "https://localhost:8080/", "height": 215}
import os, sys
in_colab = 'google.colab' in sys.modules

if in_colab:
    # Install required python package:
    # category_encoders, version >= 2.0
    # !pip install --upgrade category_encoders

# + id="fOigGm7jMlF8" colab_type="code" outputId="79c49584-7625-49d4-b166-f53ec7b4ce24" colab={"base_uri": "https://localhost:8080/", "height": 233}
# !pip install eli5

# + id="Q82PYL8i_9XM" colab_type="code" colab={}
import pandas as pd
import numpy as np
import seaborn as sns
from matplotlib import pyplot as plt
# %matplotlib inline

df = pd.read_csv('Video_Games_Sales_as_at_22_Dec_2016.csv')

# + id="wNf-hEp8AYIp" colab_type="code" outputId="fe0ba236-29a6-49c9-82b4-667f0f00f211" colab={"base_uri": "https://localhost:8080/", "height": 35}
df.shape

# + id="YlndRP3KAasF" colab_type="code" outputId="bc2b14fa-f41b-4f09-c5a6-434c87181789" colab={"base_uri": "https://localhost:8080/", "height": 323}
df.isnull().sum()

# + id="qxVDxuzgE5d_" colab_type="code" outputId="7e2ddf64-fe58-4506-c310-20a2402d42c0" colab={"base_uri": "https://localhost:8080/", "height": 89}
df.Platform.unique()

# + id="7a1rMHOrHsu2" colab_type="code" outputId="b12ab107-e0fa-4b57-ca3a-9f5f2067b7c2" colab={"base_uri": "https://localhost:8080/", "height": 204}
df.head()

# + id="u2jqDs4SRFh_" colab_type="code" colab={}
## Keep columns's name simple form

df = df.rename(columns={"Year_of_Release": "Year", "NA_Sales": "NA", "EU_Sales": "EU", "JP_Sales": "JP", "Other_Sales": "Other", "Global_Sales": "Global"})
df = df[df["Year"].notnull()]
df = df[df["Genre"].notnull()]
df["Year"] = df["Year"].apply(int)

# + id="G4wO7BPn-abz" colab_type="code" outputId="a9cb6ffe-381a-4ea2-876e-9262316dd352" colab={"base_uri": "https://localhost:8080/", "height": 89}
df = df[(df['Platform'] == 'X360') | (df['Platform'] == 'PC')]

#Let's double check the value counts to be sure
print(pd.value_counts(df["Platform"]))

#Let's see the shape of the data again
print(df.shape)

# + id="8bf8GSNNfYZo" colab_type="code" colab={}
## platforms = {"Playstation" : ["PS", "PS2", "PS3", "PS4"],
##             "Xbox" : ["XB", "X360", "XOne"], 
##             "PC" : ["PC"],
##             "Nintendo" : ["Wii", "WiiU"],
##             "Portable" : ["GB", "GBA", "GC", "DS", "3DS", "PSP", "PSV"]}

# + id="ouMjkdX2FQV8" colab_type="code" outputId="f8bb1735-1457-4258-886d-bfa3eab4c61e" colab={"base_uri": "https://localhost:8080/", "height": 35}
from sklearn.model_selection import train_test_split

# Split into train & test sets first

X_train, X_test = train_test_split(df, train_size=0.8, test_size=0.2, 
                                  stratify=df['Platform'], random_state=42)

# Split X_train into train and val sets

X_train, X_val = train_test_split(X_train, train_size=0.8, test_size=0.2, 
                                 stratify=X_train['Platform'], random_state=42)

# Target is Platform on each features

target = 'Platform'
y_train = X_train[target]
y_val = X_val[target]
y_test = X_test[target]

X_train = X_train.drop(columns=target)
X_val = X_val.drop(columns=target)
X_test = X_test.drop(columns=target)


X_train.shape, y_train.shape, X_val.shape, y_val.shape, X_test.shape, y_test.shape

# + id="e20HMVtxFQPw" colab_type="code" outputId="f3e09cb5-f31b-4381-e6c8-e18cd94c3211" colab={"base_uri": "https://localhost:8080/", "height": 233}
print('Train:\n', y_train.value_counts(normalize=True))
print('Validation:\n', y_val.value_counts(normalize=True))
print('Test:\n', y_test.value_counts(normalize=True))

# + id="Pu-jpSo7FQF3" colab_type="code" outputId="560e9627-e070-455a-85fa-7002b8a01d10" colab={"base_uri": "https://localhost:8080/", "height": 71}
# Normalize y_train

y_train.value_counts(normalize=True)

# + id="ztbr2T2XJb8A" colab_type="code" outputId="0bf457f5-7501-4fa6-bb3a-43e70c2ccb02" colab={"base_uri": "https://localhost:8080/", "height": 35}
# Accuracy Score

from sklearn.metrics import accuracy_score

majority = y_train.mode()[0]
maj_pred = [majority] * len(y_train)
accuracy_score(y_train, maj_pred)

# + id="d1WwwQL4Ju9G" colab_type="code" outputId="3ca477ea-ff30-4e7e-faf2-a7e9a0ace91a" colab={"base_uri": "https://localhost:8080/", "height": 323}
Global = 'Global'
numeric = X_train.select_dtypes('number').columns.drop(Global).tolist()

for col in numeric:
    X_train[col] = X_train[col].astype('object')

print(X_train.shape)
X_train.dtypes

# + id="2DpRxN7MLHQe" colab_type="code" outputId="31cef551-f549-4562-b0c1-72c20a4f2c6f" colab={"base_uri": "https://localhost:8080/", "height": 35}
for col in numeric:
    X_val[col] = X_val[col].astype('object')

print(X_val.shape)

# + id="DkwAbOOwLHOa" colab_type="code" outputId="ad3dfd3f-e638-486f-fe32-72b317a38716" colab={"base_uri": "https://localhost:8080/", "height": 35}
for col in numeric:
    X_test[col] = X_test[col].astype('object')

print(X_test.shape)

# + id="bBFA7BQqLHMe" colab_type="code" outputId="cbced028-ca59-40c8-ef72-af253bf1c1a1" colab={"base_uri": "https://localhost:8080/", "height": 35}
#Random Forest

import category_encoders as ce
from sklearn.impute import SimpleImputer
from sklearn.ensemble import RandomForestClassifier
from sklearn.pipeline import make_pipeline

# Create ML pipeline
pipeline = make_pipeline(
    ce.OrdinalEncoder(),
    SimpleImputer(), 
    RandomForestClassifier(n_estimators=100, n_jobs=-1, random_state=42)
)

# Fit to train
pipeline.fit(X_train, y_train)

# Score on val
print('Validation Accuracy', pipeline.score(X_val, y_val))

# + id="LJ6_PWnLZtc2" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 35} outputId="1a4c3738-f619-46be-f1fa-40cca9cd5978"
# XGboost

from xgboost import XGBClassifier

pipeline = make_pipeline(
    ce.OrdinalEncoder(), 
    XGBClassifier(n_estimators=100, random_state=42, n_jobs=-1)
)

# fit on train, score on val
pipeline.fit(X_train, y_train)

print('Validation Accuracy: ', pipeline.score(X_val, y_val))

# + id="VqC8K6v3LHKm" colab_type="code" outputId="be54d4d8-a24c-443d-930c-b1eb4db3c185" colab={"base_uri": "https://localhost:8080/", "height": 143}
# Let's do a permutation importance to see the most important features
# Rerun the random forest classifier outside the pipeline

transformers = make_pipeline(
    ce.OrdinalEncoder(), 
    SimpleImputer()
)

X_train_transformed = transformers.fit_transform(X_train)
X_val_transformed = transformers.transform(X_val)

model = RandomForestClassifier(n_estimators=100, random_state=42, n_jobs=-1)
model.fit(X_train_transformed, y_train)

# + id="5LK-Rjv2LHIm" colab_type="code" outputId="71e1d983-0d31-4c53-cf50-efbaacac1408" colab={"base_uri": "https://localhost:8080/", "height": 307}
# Let's do eli5 to see which columns to use

import eli5
from eli5.sklearn import PermutationImportance

permuter = PermutationImportance(
    model, scoring='accuracy', n_iter=2, random_state=42
)

permuter.fit(X_val_transformed, y_val)
feature_names = X_val.columns.tolist()
eli5.show_weights(
    permuter, 
    top=None,  # show the permutation importances for all features
    feature_names=feature_names
)

# + id="3ZKcl_stLHE3" colab_type="code" outputId="fdf76bd0-b016-4066-ee9a-f6b124c61e97" colab={"base_uri": "https://localhost:8080/", "height": 179}
from sklearn.metrics import classification_report

# Calculate y_pred
y_pred = pipeline.predict(X_val)
print(classification_report(y_val, y_pred))

# + id="l1woA48PLHC4" colab_type="code" outputId="a1f0290a-4ee5-4e1a-be33-92a031912b9f" colab={"base_uri": "https://localhost:8080/", "height": 269}
# Let's see the heatmap

from sklearn.metrics import confusion_matrix
from sklearn.utils.multiclass import unique_labels
import matplotlib.pyplot as plt
import seaborn as sns
# %matplotlib inline

def plot_confusion_matrix(y_true, y_pred):
    labels = unique_labels(y_true)
    columns = [f'Pred {label}' for label in labels]
    index = [f'Actual {label}' for label in labels]
    table = pd.DataFrame(confusion_matrix(y_true, y_pred), 
                         columns=columns, index=index)
    return sns.heatmap(table, annot=True, fmt='.0f', cmap='viridis')

plot_confusion_matrix(y_val, y_pred);

# + id="sQIY-vHMauhy" colab_type="code" outputId="c1ea4db2-f3f3-4f04-8e02-ff1720973c45" colab={"base_uri": "https://localhost:8080/", "height": 1000}
y_pred_proba = pipeline.predict_proba(X_val)[:,1]
print(y_pred_proba)

threshold = 0.5
ax = sns.distplot(y_pred_proba)
ax.axvline(threshold, color='red');

# + id="a9LDIv_Xds0M" colab_type="code" outputId="2b4c56c4-0694-426d-becb-7123847f1993" colab={"base_uri": "https://localhost:8080/", "height": 35}
# confusion matrix for majority baseline

majority_class = y_train.mode()[0]
y_pred = np.full_like(y_val, fill_value=majority_class)
accuracy_score(y_val, y_pred)

# + id="4-TA8Jy5dyso" colab_type="code" outputId="ffd80db8-6ae2-43f2-b92f-3d108232a32c" colab={"base_uri": "https://localhost:8080/", "height": 269}
plot_confusion_matrix(y_val, y_pred);

# + id="xvCOYT9JcFMy" colab_type="code" colab={}


# + id="4pXVSEzGd3hn" colab_type="code" outputId="47302577-e44b-4963-86b8-c3be2a29ff76" colab={"base_uri": "https://localhost:8080/", "height": 35}
#ROC AUC

from sklearn.metrics import roc_auc_score

y_pred_proba = pipeline.predict_proba(X_val)[:, 1]
roc_auc_score(y_val, y_pred_proba)

# + id="O6ZfPUbgbTfF" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 331} outputId="4622a2e9-4171-48a2-dc4b-cfdf7910ff51"
from sklearn.metrics import roc_curve

fpr, tpr, thresholds = roc_curve(y_val=='M', y_pred_proba)
plt.plot(fpr, tpr)
plt.title('ROC curve')
plt.xlabel('False Positive Rate')
plt.ylabel('True Positive Rate');

# + id="aRSwB3bmcZdV" colab_type="code" colab={}
from yellowbrick.classifier import ClassPredictionError

# Classes assign
classes = ['X360', 'PC']

# Encode X_train & X_val
encoder = ce.OrdinalEncoder()
X_train_encoded = encoder.fit_transform(X_train)
X_val_encoded = encoder.transform(X_val)

# Impute
imputer = SimpleImputer()
X_train_imputed = imputer.fit_transform(X_train_encoded)
X_val_imputed = imputer.transform(X_val_encoded)

visualizer = ClassPredictionError(
    RandomForestClassifier(n_estimators=100, random_state=42, n_jobs=-1), 
    classes=classes
)

# + id="jiwGa1DGgF2j" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 143} outputId="0c2a4a97-718e-4c2f-94ae-b9ea9b43044c"
# Partial Dependence
model = RandomForestClassifier(n_estimators=100, random_state=42, n_jobs=-1)

# Fit model
model.fit(X_train_imputed, y_train)

# + id="y7Xc8yvpgief" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 287} outputId="9e683177-d8f2-413d-8702-ac0a12a31348"
# !pip install pdpbox

# + id="TgX4SEXUgLem" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 582} outputId="3ffe8072-e4d4-499b-f483-38736d6e9302"
# Pdpbox feature for Global

# %matplotlib inline
import matplotlib.pyplot as plt
from pdpbox import pdp

feature = 'Global'
features = X_train.columns.tolist()
pdp_dist = pdp.pdp_isolate(model=model, dataset=X_train_encoded, model_features=features, feature=feature)
pdp.pdp_plot(pdp_dist, feature);

# + id="H0MHds6ngzC0" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 582} outputId="6f823a67-c623-40cf-8037-af90e25f3a38"
# Feature for NA

feature = 'NA'
features = X_train.columns.tolist()
pdp_dist = pdp.pdp_isolate(model=model, dataset=X_train_encoded, model_features=features, feature=feature)
pdp.pdp_plot(pdp_dist, feature);

# + id="EaK5ZdkDhAyR" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 582} outputId="d85f07fb-b8f1-42e6-c1fa-06a23058c1e8"
# Feature for JP

feature = 'JP'
features = X_train.columns.tolist()
pdp_dist = pdp.pdp_isolate(model=model, dataset=X_train_encoded, model_features=features, feature=feature)
pdp.pdp_plot(pdp_dist, feature);

# + id="HF3HvIHRhOek" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 582} outputId="c98734ed-b142-4244-8c85-70a2456f4ff7"
# Feature for EU

feature = 'EU'
features = X_train.columns.tolist()
pdp_dist = pdp.pdp_isolate(model=model, dataset=X_train_encoded, model_features=features, feature=feature)
pdp.pdp_plot(pdp_dist, feature);

# + id="brxIFoNjkQMl" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 582} outputId="331ede75-a630-4837-e7c4-5834b7252142"
# Feature for Other

feature = 'Other'
features = X_train.columns.tolist()
pdp_dist = pdp.pdp_isolate(model=model, dataset=X_train_encoded, model_features=features, feature=feature)
pdp.pdp_plot(pdp_dist, feature);

# + id="zuvYqdz81AP3" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 593} outputId="c77ff82b-c703-480b-b943-66bf29f083d4"
# !pip install shap

# + id="PnwJ4vw-1AKh" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 204} outputId="bcc85ba1-1af6-48cb-d2e4-edc5c49637e3"
# Look at predictions vs actuals
pred_act = pd.DataFrame({
    'y_pred_proba': y_pred_proba, 
    'y_val': y_val
})

pred_act.head()

# + id="ITr6SJU81AFl" colab_type="code" colab={}
PC = pred_act['y_val'] == 'PC'
X360 = ~PC
right = (PC) == (pred_act['y_pred_proba'] > 0.5)
wrong = ~right

# + id="8wXaIril1ADW" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 359} outputId="555eda43-a50e-45b7-9744-5b642ca74df2"
# Sample PC & right
pred_act[PC & right].sample(n=10, random_state=42).sort_values(by='y_pred_proba')

# + id="M-dPZE0x0__W" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 1000} outputId="7da4f09a-211e-4c00-9724-3e99a36235b2"
import category_encoders as ce
from sklearn.impute import SimpleImputer
from sklearn.pipeline import make_pipeline
from xgboost import XGBClassifier

processor = make_pipeline(
    ce.OrdinalEncoder(), 
    SimpleImputer(strategy='median')
)

X_train_processed = processor.fit_transform(X_train)
X_val_processed = processor.transform(X_val)

eval_set = [(X_train_processed, y_train), 
            (X_val_processed, y_val)]

model = XGBClassifier(n_estimators=1000, n_jobs=-1)
model.fit(X_train_processed, y_train, eval_set=eval_set, eval_metric='auc', 
          early_stopping_rounds=10)

# + id="rdXRPSoa0_tW" colab_type="code" colab={}
row = X_val.loc[[3803]]

# + id="0wh7Lrzl12i3" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 192} outputId="081ed3dd-38fc-457e-9cab-b20592af4418"
import shap

explainer = shap.TreeExplainer(model)
row_processed = processor.transform(row)
shap_values = explainer.shap_values(row_processed)

shap.initjs()
shap.force_plot(
    base_value=explainer.expected_value, 
    shap_values=shap_values, 
    features=row
)

# + id="nSrYSq1i12g3" colab_type="code" colab={}


# + id="nUpI8jsg12e7" colab_type="code" colab={}


StarcoderdataJupyterScriptsDedupFiltered