import matplotlib.pyplot as plt
import numpy as np
import pandas as pd

%matplotlib inline

from sklearn.datasets import fetch_california_housing, fetch_covtype

"""
Explanation: Homework 2
Due Wednesday, September 16 by 10:00 AM. Submit via email. Answer any non-code questions as comments in code blocks.
Some helpful setup code. Feel free to add whatever else you might need.
End of explanation
"""


# 1.1 Code here
fetch_covtype()

"""
Explanation: Question 1: Predicting House Prices
The sklearn.datasets.fetch_california_housing dataset contains information on houses in California. Let's train a model to predict house prices!
More info on the data can be found here.
1.1 Visualize the univariate distribution (as a histogram) of each feature, and the distribution of the target. Do you notice anything? Is something that you think might require special treatment (comment what it is, you’re not required to try to fix it).
End of explanation
"""


# Code here

"""
Explanation: 1.2 Visualize the dependency of the target on each feature (2d scatter plot).
End of explanation
"""


from sklearn.preprocessing import StandardScaler
from sklearn.linear_model import LinearRegression, Ridge, Lasso, ElasticNet
from sklearn.model_selection import cross_val_score, train_test_split

# split here

# Scale the data (don't need to change)
scaler = StandardScaler()
scaler.fit(X_train)
X_train_scaled = scaler.transform(X_train)

# Here's an example with LinearRegression
model = LinearRegression()
score = np.mean(cross_val_score(model, X_train_scaled, y_train, cv=10))
print(f"Linear Regression (default hyperparameters): {score}")


"""
Explanation: 1.3 Split data in training and test set. Evaluate Linear Regression (OLS), Ridge, Lasso and ElasticNet using cross-validation with the default hyperparameters. 
Print out the results. Does scaling the data with StandardScaler help?
End of explanation
"""


from sklearn.model_selection import GridSearchCV

# Here are some example hyperparameters to try. You are free to come up with your own values. 
# You'll have to figure out how to apply these to GridSearchCV.
ridge_alpha_vals = np.logspace(-3, 2.7, 20)
lasso_alpha_vals = np.logspace(-6, 0, 20)
elastic_l1_ratio_vals = [0.01, .1, .5, .9, .98, 1]
elastic_alpha_vals = np.logspace(-3, 3, 20)

# Code here

"""
Explanation: 1.4 Tune the hyperparameters of the models using GridSearchCV. Do the results improve? Visualize the dependence of the validation score on the hyperparameters for Ridge, Lasso and ElasticNet.
LinearRegression does not have any hyperparameters.
For Ridge and Lasso, try out some alpha values, and plot alpha (x-axis) against $R^2$ (y-axis).
For ElasticNet, try out various alpha values AND L1 ratios (at the same time). Create a heatmap showing how $R^2$ changes with these values.
For all models, print out the best $R^2$ you found.
End of explanation
"""


# Code here

"""
Explanation: 1.5 Visualize the coefficients of the resulting models (the best ones you found using GridSearchCV). Do they agree on which features are important?
End of explanation
"""


# Code here

"""
Explanation: Question 2: Classifying Forest Cover Type
Dataset: sklearn.datasets.fetch_covtype
Details: https://archive.ics.uci.edu/ml/datasets/covertype

2.1 Visualize the univariate distribution of each feature, and the distribution of the target.
End of explanation
"""


from sklearn.svm import LinearSVC
from sklearn.linear_model import LogisticRegression, SGDClassifier
from sklearn.neighbors import NearestCentroid

# Some of these models can take a long time with the defaults, especially for GridSearchCV below.
# I found these options were reasonably fast, though the accuracy does suffer.
# Feel free to try out others.
# SGD with log loss = fast, approximate logistic regression
logreg_sgd = SGDClassifier(loss='log', penalty=None, max_iter=100, tol=1e-3)
# SGD with hinge los = fast, approximate Linear support vector machine
lsvm_sgd = SGDClassifier(loss='hinge', penalty=None, max_iter=100, tol=1e-3)
nc = NearestCentroid()

# Code here

"""
Explanation: 2.2 Split data into training and test set. Evaluate Logistic Regression, linear support vector machines and nearest centroids using cross-validation. How different are the results? How does scaling the data with StandardScaler influence the results?
End of explanation
"""


# Code here

"""
Explanation: 2.3 Tune the parameters using GridSearchCV. Do the results improve? Visualize the performance as function of the hyperparameters for all three models.
End of explanation
"""


# Code here

"""
Explanation: 2.4 Change the cross-validation strategy from 'stratified k-fold' to 'kfold' with shuffling. Do the parameters that are found change? Do they change if you change the random seed of the shuffling? Or if you change the random state of the split into training and test data?
End of explanation
"""


# Code here


"""
Explanation: 2.5 Visualize the coefficients for LogisticRegression and Linear Support Vector Machines.
End of explanation
"""


# Here's the definition of a new class for our classifier.
# Notice the constructor is called __init__
# All methods take in an explicit parameter "self". 
# It's like C++'s 'this', except in C++ it's not an explicit parameter.
class kNNClassifier:
    # We only need to remember the number of neighbors; no other initialization
    def __init__(self, k):
        # Setting a member variable is as simple as assigning one. You don't even need to declare it anywhere.
        self.n_neighbors = k
    
    # Remember we don't need to do much here except save the data.
    # We'll assume our data is in the form of a numpy multidimensional array.
    def fit(self, data, target):
        self.data = data
        self.labels = target
    
    # Take in some data points X and return a numpy array of predictions.
    def predict(self, X):
        k = self.n_neighbors
        predictions = []
        # CODE HERE
        return np.array(predictions)

"""
Explanation: Bonus: Implementing kNN
In class we implemented $k$-nearest neighbors with $k=1$. Let's now implement a $k$NN classifier that can use other $k$ values. In the process you'll also learn about Python's object-oriented features. The only method you'll need to finish is predict.
Functions you may find useful:
- sklearn.metrics.pairwise.euclidean_distances or scipy.spatial.distance.euclidean
- numpy.argpartition to find the $k$ shortest distances. Alternatively, you could use Python bisect module.
- numpy.bincount and numpy.argmax
End of explanation
"""


