import matplotlib.pyplot as plt
import numpy as np

%matplotlib inline

from sklearn import datasets

diabetes = datasets.load_diabetes()

# Description at http://www4.stat.ncsu.edu/~boos/var.select/diabetes.html

X = diabetes.data
y = diabetes.target

print(X.shape, y.shape)

from sklearn import linear_model

clf = linear_model.LinearRegression()
clf.fit(X, y)

plt.plot(y, clf.predict(X), 'k.')
plt.show()

"""
Explanation: Week 11 - Regression and Classification
In previous weeks we have looked at the steps needed in preparing different types of data for use by machine learning algorithms.
End of explanation
"""


diabetes = datasets.load_diabetes()
X = diabetes.data
y = diabetes.target

clf = linear_model.LinearRegression()
clf.fit(X, y)

plt.plot(y, clf.predict(X), 'k.')
plt.show()

from sklearn import metrics

metrics.mean_squared_error(y, clf.predict(X))

"""
Explanation: All the different models in scikit-learn follow a consistent structure. 

The class is passed any parameters needed at initialization. In this case none are needed.
The fit method takes the features and the target as the parameters X and y.
The predict method takes an array of features and returns the predicted values

These are the basic components with additional methods added when needed. For example, classifiers also have 

A predict_proba method that gives the probability that a sample belongs to each of the classes.
A predict_log_proba method that gives the log of the probability that a sample belongs to each of the classes.

Evaluating models
Before we consider whether we have a good model, or which model to choose, we must first decide on how we will evaluate our models.
Metrics
As part of our evaluation having a single number with which to compare models can be very useful. Choosing a metric that is as close a representation of our goal as possible enables many models to be automatically compared. This can be important when choosing model parameters or comparing different types of algorithm. 
Even if we have a metric we feel is reasonable it can be worthwhile considering in detail the predictions made by any model. Some questions to ask:

Is the model sufficiently sensitive for our use case?
Is the model sufficiently specific for our use case?
Is there any systemic bias?
Does the model perform equally well over the distribution of features?
How does the model perform outside the range of the training data?
Is the model overly dependent on one or two samples in the training dataset?

The metric we decide to use will depend on the type of problem we have (regression or classification) and what aspects of the prediction are most important to us. For example, a decision we might have to make is between:

A model with intermediate errors for all samples
A model with low errors for the majority of samples but with a small number of samples that have large errors.

For these two situations in a regression task we might choose mean_squared_error and mean_absolute_error.
There are lists for regression metrics and classification metrics.
We can apply the mean_squared_error metric to the linear regression model on the diabetes dataset:
End of explanation
"""


diabetes = datasets.load_diabetes()
X = diabetes.data
y = diabetes.target

clf = linear_model.LinearRegression()
clf.fit(X, y)

error = metrics.mean_squared_error(y, clf.predict(X))

rounds = 1000
np.random.seed(0)
errors = []

for i in range(rounds):
    y_shuffle = y.copy()
    np.random.shuffle(y_shuffle)
    clf_shuffle = linear_model.LinearRegression()
    clf_shuffle.fit(X, y_shuffle)
    errors.append(metrics.mean_squared_error(y_shuffle, clf_shuffle.predict(X)))

better_models_by_chance = len([i for i in errors if i <= error])

if better_models_by_chance > 0:
    print('Probability of observing a mean_squared_error of {0} by chance is {1}'.format(error, 
                                                                                    better_models_by_chance / rounds))
else:
    print('Probability of observing a mean_squared_error of {0} by chance is <{1}'.format(error, 
                                                                                    1 / rounds))

"""
Explanation: Although this single number might seem unimpressive, metrics are a key component for model evaluation. As a simple example, we can perform a permutation test to determine whether we might see this performance by chance.
End of explanation
"""


from sklearn import tree

diabetes = datasets.load_diabetes()
X = diabetes.data
y = diabetes.target

clf = tree.DecisionTreeRegressor()
clf.fit(X, y)

plt.plot(y, clf.predict(X), 'k.')
plt.show()


metrics.mean_squared_error(y, clf.predict(X))

from sklearn import neighbors

diabetes = datasets.load_diabetes()
X = diabetes.data
y = diabetes.target

clf = neighbors.KNeighborsRegressor(n_neighbors=1)
clf.fit(X, y)

plt.plot(y, clf.predict(X), 'k.')
plt.show()


metrics.mean_squared_error(y, clf.predict(X))

"""
Explanation: Training, validation, and test datasets
When evaluating different models the approach taken above is not going to work. Particularly for models with high variance, that overfit the training data, we will get very good performance on the training data but perform no better than chance on new data.
End of explanation
"""


from sklearn import neighbors

diabetes = datasets.load_diabetes()
X = diabetes.data
y = diabetes.target

np.random.seed(0)

split = np.random.random(y.shape) > 0.3

X_train = X[split]
y_train = y[split]
X_test = X[np.logical_not(split)]
y_test = y[np.logical_not(split)]

print(X_train.shape, X_test.shape)

clf = neighbors.KNeighborsRegressor(1)
clf.fit(X_train, y_train)

plt.plot(y_test, clf.predict(X_test), 'k.')
plt.show()


metrics.mean_squared_error(y_test, clf.predict(X_test))

diabetes = datasets.load_diabetes()
X = diabetes.data
y = diabetes.target

np.random.seed(0)

split = np.random.random(y.shape) > 0.3

X_train = X[split]
y_train = y[split]
X_test = X[np.logical_not(split)]
y_test = y[np.logical_not(split)]

print(X_train.shape, X_test.shape)

clf = linear_model.LinearRegression()
clf.fit(X_train, y_train)

plt.plot(y_test, clf.predict(X_test), 'k.')
plt.show()


metrics.mean_squared_error(y_test, clf.predict(X_test))

"""
Explanation: Both these models appear to give perfect solutions but all they do is map our test samples back to the training samples and return the associated value.
To understand how our model truly performs we need to evaluate the performance on previously unseen samples. The general approach is to divide a dataset into training, validation and test datasets. Each model is trained on the training dataset. Multiple models can then be compared by evaluating the model against the validation dataset. There is still the potential of choosing a model that performs well on the validation dataset by chance so a final check is made against a test dataset.
This unfortunately means that part of our, often expensively gathered, data can't be used to train our model. Although it is important to leave out a test dataset an alternative approach can be used for the validation dataset. Rather than just building one model we can build multiple models, each time leaving out a different validation dataset. Our validation score is then the average across each of the models. This is known as cross-validation.
Scikit-learn provides classes to support cross-validation but a simple solution can also be implemented directly. Below we will separate out a test dataset to evaluate the nearest neighbor model.
End of explanation
"""


from sklearn import datasets

diabetes = datasets.load_diabetes()

# Description at http://www4.stat.ncsu.edu/~boos/var.select/diabetes.html

X = diabetes.data
y = diabetes.target

print(X.shape, y.shape)

from sklearn import linear_model

clf = linear_model.LassoCV(cv=20)
clf.fit(X, y)

print('Alpha chosen was ', clf.alpha_)

plt.plot(y, clf.predict(X), 'k.')

"""
Explanation: Model types
Scikit-learn includes a variety of different models. The most commonly used algorithms probably include the following:

Regression
Support Vector Machines
Nearest neighbors
Decision trees
Ensembles & boosting

Regression
We have already seen several examples of regression. The basic form is: 
$$f(X) =  \beta_{0}  +  \sum_{j=1}^p X_j\beta_j$$
Each feature is multipled by a coefficient and then the sum returned. This value is then transformed for classification to limit the value to the range 0 to 1.
Support Vector Machines
Support vector machines attempt to project samples into a higher dimensional space such that they can be divided by a hyperplane. A good explanation can be found in this article.
Nearest neighbors
Nearest neighbor methods identify a number of samples from the training set that are close to the new sample and then return the average or most common value depending on the task. 
Decision trees
Decision trees attempt to predict the value of a new sample by learning simple rules from the training samples.
Ensembles & boosting
Ensembles are combinations of other models. Combining different models together can improve performance by boosting generalizability. An average or most common value from the models is returned.
Boosting builds one model and then attempts to reduce the errors with the next model. At each stage the bias in the model is reduced. In this way many weak predictors can be combined into one much more powerful predictor.
I often begin with an ensemble or boosting approach as they typically give very good performance without needing to be carefully optimized. Many of the other algorithms are sensitive to their parameters.
Parameter selection
Many of the models require several different parameters to be specified. Their performance is typically heavily influenced by these parameters and choosing the best values is vital in developing the best model.
Some models have alternative implementations that handle parameter selection in an efficient way.
End of explanation
"""


from sklearn import grid_search

from sklearn import neighbors

diabetes = datasets.load_diabetes()
X = diabetes.data
y = diabetes.target

np.random.seed(0)

split = np.random.random(y.shape) > 0.3

X_train = X[split]
y_train = y[split]
X_test = X[np.logical_not(split)]
y_test = y[np.logical_not(split)]

print(X_train.shape, X_test.shape)

knn = neighbors.KNeighborsRegressor()

parameters = {'n_neighbors':[1,2,3,4,5,6,7,8,9,10]}
clf = grid_search.GridSearchCV(knn, parameters)

clf.fit(X_train, y_train)

plt.plot(y_test, clf.predict(X_test), 'k.')
plt.show()


print(metrics.mean_squared_error(y_test, clf.predict(X_test)))

clf.get_params()

"""
Explanation: There is an expanded example in the documentation.
There are also general classes to handle parameter selection for situations when dedicated classes are not available. As we will often have parameters in preprocessing steps these general classes will be used much more often.
End of explanation
"""


