# import libraries

import matplotlib
import IPython
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import matplotlib as mpl
import pylab
import seaborn as sns
import sklearn as sk

%matplotlib inline

# Load your data here
x_values = pd.read_csv('ex5Linx.dat', sep='\s+', header=None, skiprows=1)
y_values = pd.read_csv('ex5Liny.dat', sep='\s+', header=None, skiprows=1)


"""
Explanation: Regularized Linear Regression

Hopefully, by now you have understood the intuition behind Regularized Linear Regression and the purpose of the lambda value. 
The goal of this exercise is to help you qualitatively understand how changing lambda-values impact the model accuracy. By going through this exercise, you will be able to get a visual idea of how different lambda values affect the model. Make sure you go through the exercise fully and understand what is going on. Feel free to change the lambda values to get a sense of how the model's generalizability is affected.
Data source: http://openclassroom.stanford.edu/MainFolder/DocumentPage.php?course=MachineLearning&doc=exercises/ex5/ex5.html
End of explanation
"""


from sklearn.linear_model import Ridge
from sklearn.preprocessing import PolynomialFeatures
from sklearn.pipeline import make_pipeline

# lambda value of 0
ridge_reg_0 = make_pipeline(PolynomialFeatures(6), Ridge(alpha=0))

ridge_reg_0.fit(x_values, y_values)
y_predicted = ridge_reg_0.predict(x_values)

plt.scatter(x_values,y_values,c='r')
plt.plot(x_values, y_predicted, c='b')

"""
Explanation: To perform Ridge Regression, we will use another module called Ridge that has similar methods to Linear Regression. The first argument that Ridge takes is the lambda-value. Below, there are three different Ridge Regression models that have lambda values of 0, 1, and respectively. You are free to choose different values. First, we will plot our model's prediction when the lambda-value is 0. This is identical to Non-regularized Linear Regression.
End of explanation
"""


# lambda value of 0.8
ridge_reg_1 = make_pipeline(PolynomialFeatures(6), Ridge(alpha=0.8))
ridge_reg_1.fit(x_values, y_values)
y_predicted = ridge_reg_1.predict(x_values)

plt.scatter(x_values,y_values,c='r')
plt.plot(x_values, y_predicted, c='b')


"""
Explanation: It is fairly obvious that this model is overfitting on the data. This is where regularization, more specifically the lambda-value comes into the picture. By increasing the lambda-value, we start penalizing overly complex modules and move towards a more generalizable model.
End of explanation
"""


# lambda value of 8
ridge_reg_10 = make_pipeline(PolynomialFeatures(6), Ridge(alpha=20))
ridge_reg_10.fit(x_values, y_values)
y_predicted = ridge_reg_10.predict(x_values)

plt.scatter(x_values,y_values,c='r')
plt.plot(x_values, y_predicted, c='b')

"""
Explanation: With a lambda-value of 1, notice the drastic change in the model. All the conditions are the same as before except for the lambda-value which is now 1 instead of 0. However, that change has reduced the complexity of the model and it does not seem to be overfitting like the previous model. By seeing this diagram, it should help build the intuition as to how the lambda-value affects a model. Now the question is, does increasing the lambda-value always guarantee a better model? Think about this question before you move onto the last part of the exercise.
End of explanation
"""


