## 3. Bias-variance tradeoff ##

from sklearn.linear_model import LinearRegression
from sklearn.metrics import mean_squared_error
import numpy as np
import matplotlib.pyplot as plt

def train_and_test(cols):
    lr = LinearRegression()
    lr.fit(filtered_cars[cols], filtered_cars['mpg'])
    predictions = lr.predict(filtered_cars[cols])
    # Compute MSE and Variance
    mse = mean_squared_error(filtered_cars['mpg'], predictions)
    
    variance = np.var(predictions)
    return(mse, variance)

cyl_mse, cyl_var = train_and_test(['cylinders'])
print(cyl_mse, cyl_var)

weight_mse, weight_var = train_and_test(['weight'])
print(weight_mse, weight_var)                                         
                                         

## 4. Multivariate models ##

# Our implementation for train_and_test, takes in a list of strings.
def train_and_test(cols):
    # Split into features & target.
    features = filtered_cars[cols]
    target = filtered_cars["mpg"]
    # Fit model.
    lr = LinearRegression()
    lr.fit(features, target)
    # Make predictions on training set.
    predictions = lr.predict(features)
    # Compute MSE and Variance.
    mse = mean_squared_error(filtered_cars["mpg"], predictions)
    variance = np.var(predictions)
    return(mse, variance)

one_mse, one_var = train_and_test(["cylinders"])

two_mse, two_var = train_and_test(['cylinders','displacement'])

three_mse, three_var = train_and_test(['cylinders','displacement','horsepower'])

four_mse, four_var = train_and_test(['cylinders','displacement','horsepower','weight'])

five_mse, five_var = train_and_test(['cylinders','displacement','horsepower','weight','acceleration'])

six_mse, six_var = train_and_test(['cylinders','displacement','horsepower','weight','acceleration','model year'])

seven_mse, seven_var = train_and_test(['cylinders','displacement','horsepower','weight','acceleration','model year','origin'])

print('The MSE and Variance of Two features are: '+ str(two_mse)+' and '+str(two_var) )
print('The MSE and Variance of Three features are: '+ str(three_mse)+' and '+str(three_var) )
print('The MSE and Variance of four features are: '+ str(four_mse)+' and '+str(four_var) )
print('The MSE and Variance of five features are: '+ str(five_mse)+' and '+str(five_var) )
print('The MSE and Variance of six features are: '+ str(six_mse)+' and '+str(six_var) )
print('The MSE and Variance of seven features are: '+ str(seven_mse)+' and '+str(seven_var) )

## 5. Cross validation ##

from sklearn.model_selection import KFold
from sklearn.metrics import mean_squared_error
import numpy as np

def train_and_cross_val(cols):
    features = filtered_cars[cols]
    target = filtered_cars['mpg']
    
    variance_values = []
    mse_values = []
    
    #KFold instance
    kf = KFold(n_splits = 10, shuffle = True, random_state = 3)
     # Iterate through over each fold.
    for train_index, test_index in kf.split(features):
        # Training and test sets.
        X_train, X_test = features.iloc[train_index], features.iloc[test_index]
        y_train, y_test = target.iloc[train_index], target.iloc[test_index]
        
        # Fit the model and make predictions.
        lr = LinearRegression()
        lr.fit(X_train, y_train)
        predictions = lr.predict(X_test)
        
        # Calculate mse and variance values for this fold.
        mse = mean_squared_error(y_test, predictions)
        var = np.var(predictions)

        # Append to arrays to do calculate overall average mse and variance values.
        variance_values.append(var)
        mse_values.append(mse)
   
    # Compute average mse and variance values.
    avg_mse = np.mean(mse_values)
    avg_var = np.mean(variance_values)
    return(avg_mse, avg_var)
        
two_mse, two_var = train_and_cross_val(["cylinders", "displacement"])
three_mse, three_var = train_and_cross_val(["cylinders", "displacement", "horsepower"])
four_mse, four_var = train_and_cross_val(["cylinders", "displacement", "horsepower", "weight"])
five_mse, five_var = train_and_cross_val(["cylinders", "displacement", "horsepower", "weight", "acceleration"])
six_mse, six_var = train_and_cross_val(["cylinders", "displacement", "horsepower", "weight", "acceleration", "model year"])
seven_mse, seven_var = train_and_cross_val(["cylinders", "displacement", "horsepower", "weight", "acceleration","model year", "origin"])
    
print('The MSE and Variance of Two features are: '+ str(two_mse)+' and '+str(two_var) )
print('The MSE and Variance of Three features are: '+ str(three_mse)+' and '+str(three_var) )
print('The MSE and Variance of four features are: '+ str(four_mse)+' and '+str(four_var) )
print('The MSE and Variance of five features are: '+ str(five_mse)+' and '+str(five_var) )
print('The MSE and Variance of six features are: '+ str(six_mse)+' and '+str(six_var) )
print('The MSE and Variance of seven features are: '+ str(seven_mse)+' and '+str(seven_var) )

## 6. Plotting cross-validation error vs. cross-validation variance ##

# We've hidden the `train_and_cross_val` function to save space but you can still call the function!
import matplotlib.pyplot as plt
        
two_mse, two_var = train_and_cross_val(["cylinders", "displacement"])
three_mse, three_var = train_and_cross_val(["cylinders", "displacement", "horsepower"])
four_mse, four_var = train_and_cross_val(["cylinders", "displacement", "horsepower", "weight"])
five_mse, five_var = train_and_cross_val(["cylinders", "displacement", "horsepower", "weight", "acceleration"])
six_mse, six_var = train_and_cross_val(["cylinders", "displacement", "horsepower", "weight", "acceleration", "model year"])
seven_mse, seven_var = train_and_cross_val(["cylinders", "displacement", "horsepower", "weight", "acceleration","model year", "origin"])

plt.scatter([2,3,4,5,6,7], [two_mse, three_mse, four_mse, five_mse, six_mse, seven_mse], c='red')
plt.scatter([2,3,4,5,6,7], [two_var, three_var, four_var, five_var, six_var, seven_var], c='blue')
plt.show()