"""
# Gradient Descent 

   Gradient descent is one of the most basic and easy to implement algorithm for finding minima of the cost function. For a mean square cost function there exists only one minima which it's global minima. This is because the mean square cost function is a Quadratic Function.
            
   For a complex cost function with many local minimas, the algorithm that I have given below might not work effectively.
            
   Always check the convergence curve when you're training the model for the first time. If it doesn't converge try decreasing the learning rate(a) and increase the iterations. This is one of the downsides of gradient descent, but in more complex solvers like BGFS, L-BGFS, etc we need not specify the learning rate.
"""
%matplotlib inline
import numpy as np
import matplotlib.pyplot as plt
"""
## Creating synthetic datasets for testing
"""
from sklearn.datasets import make_regression

X, y = make_regression(n_samples = 100, n_features=5,
                            n_informative=1, bias = 150.0,
                            noise = 30, random_state=0)
X[:5], y[:5]
"""
## Linear Regression
"""
"""
### Mean Square Cost Function for linear regression
"""
"""
### Specify loc='in' while using cost function inside Gradient Descent(gd function) 
"""
def cost(x, y, th, loc=None):
    m = len(x)
    if loc != 'in':
        x = np.hstack((np.array([1]*m).reshape(-1,1), x))
    y = y.reshape(-1,1)
    return (np.sum((x.dot(th) - y)**2)) / (2*m)
"""
### Intercept and Coefficient calculating Function for linear regression using Gradient Descent

x     -> the training dataset, a 2D array.

y     -> target values, 1D array.

a     -> learning rate.

itr   -> maximum number of iterations to be performed.
"""
def gd(x, y, a=0.1, itr=100, graph=0):
    m, n = x.shape 
    # m -> no of datapoints
    # n -> no of features
    
    th = np.array([0]*(n+1)).reshape(-1,1)
    # th -> set intercept and coefficient values to 0 initially
    
    x = np.hstack((np.ones((m,1)), x))
    # add a row of 1s to the dataset to mulitply with the intercept term in th
    
    y = y.reshape(-1,1)
    cst = [cost(x, y, th, loc='in')]
    # list to store the cost to check for convergence
    
    for i in range(itr):
        der = (x.T).dot(x.dot(th) - y) / m
        # gradient of the cost function
        
        th = th - (a * der) #updated th
        cst.append(cost(x, y, th, loc='in')) #cost for updated th
    if graph == 1:
        return th, cst, a, itr
    return th
"""
### A diverging model for high learning rate(a)
"""
out = gd(X, y, a=1.6, itr=100, graph=1)
plt.figure()
plt.plot(out[1]) #plot the cost wrt iterations
plt.title('Convergence curve for alpha = {} and max_iteration = {}'.format(out[2], out[3]))
plt.ylabel('Mean square error (Cost function)')
plt.xlabel('Iteration')
plt.show()
print('Intercept: ', out[0][0,0])
print('Coefficients: ', out[0][1:].ravel()) # Gives bad estimates
"""
### A perfectly converging model for appropriate learning rate(a)
"""
out = gd(X, y, a=0.1, itr=500, graph=1)
plt.figure()
plt.plot(out[1]) #plot the cost wrt iterations
plt.title('Convergence curve for alpha = {} and max_iteration = {}'.format(out[2], out[3]))
plt.ylabel('Mean square error (Cost function)')
plt.xlabel('Iteration')
plt.show()
print('Intercept: ', out[0][0,0])
print('Coefficients: ', out[0][1:].ravel()) # Gives proper estimates
"""
### Verifying Results with Scikit-learn
"""
from sklearn.linear_model import LinearRegression
lr = LinearRegression().fit(X, y)
print('Intercept: ', lr.intercept_)
print('Coefficients: ', lr.coef_)