# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     name: python3
# ---

# + [markdown] id="view-in-github" colab_type="text"
# <a href="https://colab.research.google.com/github/Jason-Durr/Projects-in-Machine-Learning/blob/main/HW/Homework2.ipynb" target="_parent"><img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"/></a>

# + [markdown] id="RndMS2nVbbhD"
# <NAME>

# + colab={"resources": {"http://localhost:8080/nbextensions/google.colab/files.js": {"data": "<KEY>", "ok": true, "headers": [["content-type", "application/javascript"]], "status": 200, "status_text": ""}}, "base_uri": "https://localhost:8080/", "height": 89} id="GGV4diTUDwkF" outputId="87d7a1d7-99b2-4d9d-c35d-39e6a842533c"
from google.colab import files
files.upload()


# + colab={"base_uri": "https://localhost:8080/"} id="gaXsSmcv_Cm9" outputId="5f0e5171-4233-4bbe-cde2-d4eebb6f4775"
# !mkdir ~/.kaggle
# !cp kaggle.json ~/.kaggle/
# ! chmod 600 ~/.kaggle/kaggle.json
# !kaggle datasets download -d tobycrabtree/nfl-scores-and-betting-data
# ! unzip /content/nfl-scores-and-betting-data.zip



# + id="k41YJH8Tw0sV"
import pandas as pd
import numpy as np
from sklearn.metrics import accuracy_score




# + id="HNLb0dJjxFDr"
data = pd.read_csv('spreadspoke_scores.csv')
teams = pd.read_csv('nfl_teams.csv')
df = data.dropna(subset=['team_favorite_id','weather_temperature'])
df = df[df['stadium_neutral'] == False]
df = df[df['schedule_week'] != 17]
df.weather_detail = df.weather_detail.fillna('Clear')
df.over_under_line = df.over_under_line.replace(' ',np.NaN)
df.weather_detail.unique()
X = pd.DataFrame()
X["Home_Favorite"] = np.where(df['team_home'].map(teams.set_index('team_name')['team_id']) == df['team_favorite_id'], 1, 0)
X["Temperature"] = (df.weather_temperature.to_numpy() - data.weather_temperature.median())/df.weather_temperature.apply(lambda x: float(x)).std()
X["Humidity"] = (df.weather_humidity.apply(lambda x: float(x)).to_numpy() - df.weather_humidity.apply(lambda x: float(x)).median())/df.weather_humidity.apply(lambda x: float(x)).std()
X["Wind"] = (df.weather_wind_mph.to_numpy() - data.weather_wind_mph.median())/df.weather_wind_mph.apply(lambda x: float(x)).std()
X["Over_Under"] = (df.over_under_line.apply(lambda x: float(x)).to_numpy() - df.over_under_line.apply(lambda x: float(x)).median())/df.over_under_line.apply(lambda x: float(x)).std()
X['Rain'] = df.weather_detail.apply(lambda x: int('Rain' in x)).to_numpy()
X['Snow'] = df.weather_detail.apply(lambda x: int('Snow' in x)).to_numpy()
X['Fog'] = df.weather_detail.apply(lambda x: int('Fog' in x)).to_numpy()
X['Playoff'] = df.schedule_playoff.apply(lambda x: int(x)).to_numpy()
X["Home_Win"] = np.where(df.score_home > df.score_away,1,0)
X = X.dropna()

# + id="SRXA8KND4sfZ"
Y = np.matrix(X['Home_Win']).T
x = X.drop('Home_Win',axis=1).to_numpy()

# + colab={"base_uri": "https://localhost:8080/"} id="8Nkx3JbDeU7l" outputId="a331964b-40a9-4a86-d190-a9ab5c4401d1"
x

# + id="s4VO4aoNWq5C"

sample = np.random.choice(x.shape[0],size=x.shape[0]//5)
x_train = x[~sample,:]
x_test = x[sample,:]
y_train = Y[~sample]
y_test = Y[sample]

# + [markdown] id="TfjNLdmT3r5x"
# # Task 1 Implement Decision Tree Classifier
#

# + colab={"base_uri": "https://localhost:8080/"} id="pd-w31D-bykK" outputId="60a5d712-1338-404e-b016-fe1df5779aaf"
from sklearn import tree
from sklearn.model_selection import RepeatedStratifiedKFold
from sklearn.model_selection import cross_val_score
from numpy import mean
from numpy import std
clf1 = tree.DecisionTreeClassifier(max_depth=3)
clf1 = clf1.fit(x_train, y_train)
accuracy_score(y_test,clf1.predict(x_test))
cv = RepeatedStratifiedKFold(n_splits=10, n_repeats=3, random_state=1)
n_scores = cross_val_score(clf1, x_train, y_train, scoring='accuracy', cv=cv, n_jobs=-1, error_score='raise')
print('Depth 3 Accuracy: %.3f (%.3f)' % (mean(n_scores), std(n_scores)))

# + colab={"base_uri": "https://localhost:8080/"} id="vaT102JmjKDV" outputId="e3b08da1-b589-4512-b8c6-43334b58962d"
clf2 = tree.DecisionTreeClassifier(max_depth=9)
clf2 = clf.fit(x_train, y_train)
accuracy_score(y_test,clf2.predict(x_test))
cv = RepeatedStratifiedKFold(n_splits=10, n_repeats=3, random_state=1)
n_scores = cross_val_score(clf2, x_train, y_train, scoring='accuracy', cv=cv, n_jobs=-1, error_score='raise')
print('Depth 9 Accuracy: %.3f (%.3f)' % (mean(n_scores), std(n_scores)))

# + colab={"base_uri": "https://localhost:8080/"} id="KX_95zzbh7bW" outputId="8e93f213-86b8-434b-d1a4-92a95a291dfa"
clf3 = tree.DecisionTreeClassifier(max_depth=100)
clf3 = clf.fit(x_train, y_train)
accuracy_score(y_test,clf3.predict(x_test))
cv = RepeatedStratifiedKFold(n_splits=10, n_repeats=3, random_state=1)
n_scores = cross_val_score(clf3, x_train, y_train, scoring='accuracy', cv=cv, n_jobs=-1, error_score='raise')
print('Depth 100 Accuracy: %.3f (%.3f)' % (mean(n_scores), std(n_scores)))

# + colab={"base_uri": "https://localhost:8080/"} id="wFLCoRp5f2NS" outputId="bc13157a-6387-4dc6-b4c9-9b1912d3de2c"
clf4 = tree.DecisionTreeClassifier(max_depth=1)
clf4 = clf.fit(x_train, y_train)
accuracy_score(y_test,clf4.predict(x_test))
cv = RepeatedStratifiedKFold(n_splits=10, n_repeats=3, random_state=1)
n_scores = cross_val_score(clf4, x_train, y_train, scoring='accuracy', cv=cv, n_jobs=-1, error_score='raise')
print('Depth 1 Accuracy: %.3f (%.3f)' % (mean(n_scores), std(n_scores)))

# + [markdown] id="xGX8i93IgXtE"
# ### For Comparison The Accuracy of just picking the favorite
#

# + colab={"base_uri": "https://localhost:8080/"} id="NNwfam66gSKc" outputId="f156a07c-2e38-4a6f-943c-a3664720f881"
print("Always Pick the Favorite Accuracy:",accuracy_score(y_train,x_train[:,0]))

# + [markdown] id="x0M0uhhejeYV"
# From modifying the max_depth parameter it forces the tree to prune itself which increases the bias but reduces the overfitting.  This seen when comparing a max depth of 9 and 100 with that of 3 and we see that they preform worse out of sample.  We also see that by going too far the other way it increases the bias to a point where it is the same as our nieve attempt of just picking the favorite everytime and ignoring the rest of the data.  It also shows that the standard deviation of accuracy increases when we increase the allowed size of the tree because there is mor variance

# + [markdown] id="1apuQR1FoN1m"
# # Task 2 Implement Bagging and Boosting and Use k-fold Cross Validation

# + [markdown] id="FyrHwovjobYZ"
# ## Bagging

# + colab={"base_uri": "https://localhost:8080/"} id="t8JSH_-4oZvO" outputId="a00ef4e2-8466-45e2-851f-eb61918bafdd"
from sklearn.ensemble import BaggingClassifier
model = BaggingClassifier(n_estimators=500)
cv = RepeatedStratifiedKFold(n_splits=10, n_repeats=3, random_state=1)
n_scores = cross_val_score(model, x_train, y_train, scoring='accuracy', cv=cv, n_jobs=-1, error_score='raise')
# report performance
print('Bagging Accuracy: %.3f (%.3f)' % (mean(n_scores), std(n_scores)))

# + [markdown] id="ZkkG7itipC9g"
# ## Boosting

# + colab={"base_uri": "https://localhost:8080/"} id="J02KuJoXpFCo" outputId="c6eac158-cd38-4db5-8396-b59f27989cb6"
from sklearn.ensemble import AdaBoostClassifier
model = AdaBoostClassifier(n_estimators=4, random_state=0, algorithm='SAMME')
cv = RepeatedStratifiedKFold(n_splits=10, n_repeats=3, random_state=1)
n_scores = cross_val_score(model, x_train, y_train, scoring='accuracy', cv=cv, n_jobs=-1, error_score='raise')
# report performance
print('Boosting Accuracy: %.3f (%.3f)' % (mean(n_scores), std(n_scores)))

# + [markdown] id="Cy6Fj7wdsgXC"
# This expirament shows that bagging and boosting both helped to achieve better accuracies the almost all of the regular decision trees and this is because it has reduced the variance of the model.  Because boosting focuses on the points that are misclassified it is able to achieve a model with a higher accuracy than bagging but ultimately what this shows is that this data does not contain enough information to beat the naive model of just choosing the favorite to win.

# + [markdown] id="SerUEvZsqVci"
# # Task 3 Compare the effectiveness of the three models implemented above
#
#
# The main metric I have been using to compare the models is the accuracy of prediction compared to that of just selecting the favorite.  I chose this metric because when predicting the outcome of games the goal is do better than the odds setters as predicting the game so just matching their performance means there is no new knowledge being learned.  The reason I did not use a metric like false positives/ false negative or precision/sensitivity is because in this scenario all cases are weighted equally. This is because predicting the home team loses when they win or predicting them to win when they loose has the same consiquences and so there is no need to weight these scenarios differently.  If instead we tried to minimize false negaitves for example and lean toward only predicting they will loose if we are more sure that they will, this might lower the overall accuracy of the model because we are increasing the bias in order to reduce only a certain number of mistakes.  We see this by comparing the confusion matrices of the 4 different trees with different stopping parameters and note that model 1 has a worse accuracy but a better false negative score than the rest of the models which all just pick the team that is favorited.

# + colab={"base_uri": "https://localhost:8080/"} id="_xwPjxtreKzx" outputId="53e52bc3-3f8e-46ac-badc-c11d497426e0"
from sklearn.metrics import confusion_matrix

print(confusion_matrix(y_test,clf1.predict(x_train)))
print(confusion_matrix(y_test,clf2.predict(x_train)))
print(confusion_matrix(y_test,clf3.predict(x_train)))
print(confusion_matrix(y_test,clf4.predict(x_train)))

StarcoderdataJupyterScriptsDedupFiltered