import sklearn
import mglearn

import numpy as np
import pandas as pd

import matplotlib.pyplot as plt
%matplotlib inline

from sklearn.linear_model import LogisticRegression
from sklearn.svm import LinearSVC

X, y = mglearn.datasets.make_forge()

fig, axes = plt.subplots(1, 2, figsize=(10,3))

for model, ax in zip([LinearSVC(), LogisticRegression()], axes):
    clf = model.fit(X, y)
    mglearn.plots.plot_2d_separator(clf, X, fill=False, eps=0.5,
                                   ax=ax, alpha=0.7)
    mglearn.discrete_scatter(X[:,0], X[:,1], y, ax=ax)
    ax.set_title("{}".format(clf.__class__.__name__))
    ax.set_xlabel("Feature 0")
    ax.set_ylabel("Feature 1")
axes[0].legend(loc=4)

"""
Explanation: Introduction to Machine Learning
Andreas Muller and Sarah Guido (2017) O'Reilly
Ch. 2 Supervised Learning
Linear Models for Classification

Decision boundary is a linear function of the input
Binary linear classifier separates two classes using a line, plane, or hyperplane

Algorithms for Learning Linear Models differ in following two ways:

Way they measure how well a particular combination of coefficients and intercepts fits training data
If and what kind of regularization is used

Logistic Regression

Implemented in linear_model.logisticRegression

Linear Support Vector Machines (linear SVMs)

Implemented in svm.LinearSVC

Import packages

Load forge dataset and assign variables
End of explanation
"""


mglearn.plots.plot_linear_svc_regularization()

"""
Explanation: Figure 1. Decision boundaries of linear SVM and logistic regresison on forge data with default parameters

Any new data point above decision boundary classified as 1, any points below classified as 2
Both models apply L2 regularization by default, same way as Ridge regression
Foth both models C is tradeoff parameter that determines the strength of regularization

Regularization Parameter C
High values of C:

Correspond to less regularization, models will fit training set as best as possible
Stresses importance of each individual data point to be classified correctly

Low values of C:

Models put more emphasis on finding coefficient vector (w) close to zero
Cause models to try to adjust to the 'majority' of data points
End of explanation
"""


from sklearn.datasets import load_breast_cancer
cancer = load_breast_cancer()

print(cancer.keys())

print(cancer['target_names'])

print(cancer['feature_names'])

type(cancer)

cancer.data.shape

cancer_df = pd.DataFrame(X_train, columns=cancer.feature_names)
cancer_df.head()


"""
Explanation: Figure 2.

Left panel: Small values of C gives lots of regularization, two misclassified points
Middle panel: Moderate C, model focuses on misclassified points, tilts decision boundary 

(Right panel) for High C:
* Model tries to correctly classify all points correctly with a straight line
* But may not capture overall layout of classes well. Model is likely OVERFITTING! 
* Decision bounday is titled a lot; all points in class 0 are correctly classified.

Linear Models of Classification
Low Dimensional spaces

Linear model for classification may seems restrictive in low-dimensional space
Only allowing for decision boundaries that are straight lines or planes

High Dimensional spaces

Linear models for classification become very powerful
Guarding against OVERFITTING becomes important when considering many features

Example: WI Breast Cancer Study

Load WI Breast Cancer study data
Show features
Visualize dataset
End of explanation
"""


from sklearn.model_selection import train_test_split

X_train, X_test, y_train, y_test = train_test_split(
    cancer.data, cancer.target, stratify=cancer.target, random_state=42)

logreg = LogisticRegression().fit(X_train, y_train)

print("Training set score: {:.3f}".format(logreg.score(X_train, y_train)))
print("Test set score: {:.3f}".format(logreg.score(X_test, y_test)))

"""
Explanation: Logistic Regression: Cancer Data

Split data into TRAIN and TEST sets
Fit the model on Training data
Evaluate model on Text data
End of explanation
"""


logreg100 = LogisticRegression(C=100).fit(X_train, y_train)
print("Training set score: {:.3f}".format(logreg100.score(X_train, y_train)))
print("Test set score: {:.3f}".format(logreg100.score(X_test, y_test)))

"""
Explanation: Regularization Parameter settings

Default setting C=1 provides good performance for train and test sets
Very likely UNDERFITTING 

Use higher value of C to fit more 'flexible' model

C=100 gives higher training set accuracy and slightly higher Test set accuracy
More complex model (flexible) performs better
End of explanation
"""


logreg001 = LogisticRegression(C=0.01).fit(X_train, y_train)
print("Training set score: {:.3f}".format(logreg001.score(X_train, y_train)))
print("Test set score: {:.3f}".format(logreg001.score(X_test, y_test)))

"""
Explanation: Use lower value of C to fit more 'regularized' model

Setting C=0.01 leads model to try to adjust to 'majority' of data points
Decreased model accuracy for both training and Test sets
End of explanation
"""


plt.plot(logreg.coef_.T, 'o', label="C=1")
plt.plot(logreg100.coef_.T, '^', label="C=100")
plt.plot(logreg001.coef_.T, 'v', label="C=0.01")

plt.xticks(range(cancer.data.shape[1]), cancer.feature_names, rotation=90)
plt.hlines(0,0, cancer.data.shape[1])
plt.ylim(-5, 5)

plt.xlabel("Coefficient Index")
plt.xlabel("Coefficient Magnitude")
plt.legend()

"""
Explanation: Plot Coefficients of Logistic Regression for different values of C

By default LogisticRegression applies L2 regularization similar to Ridge regression
Stronger regularization pushes coefficients closer to zero
Parameter values can influence values of Coefficients
End of explanation
"""


for C, marker in zip([0.01, 1, 100], ['v', 'o', '^']):
    lr_l1 = LogisticRegression(C=C, penalty="l1").fit(X_train, y_train)
    print("Training accuracy of L1 logreg with C={:.3f}: {:.2f}".format(
        C, lr_l1.score(X_train, y_train)))
    print("Test accuracy of L1 logreg with C={:.3f}: {:.2f}".format(
        C, lr_l1.score(X_test, y_test)))
    plt.plot(lr_l1.coef_.T, marker, label="C={:.3f}".format(C))
    
plt.xticks(range(cancer.data.shape[1]), cancer.feature_names, rotation=90)
plt.hlines(0,0, cancer.data.shape[1])
plt.xlabel("Coefficient Index")
plt.xlabel("Coefficient Magnitude")

plt.ylim(-5, 5)
plt.legend()

"""
Explanation: Creating More Interpretable Model: L1 Regularization (Lasso)

L1 regularization (Lasso) limits values of most coefficients to zero
Model is limited to using only a few features
End of explanation
"""


