# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

# # Predicting Sale Price of Homes in Boston

# Here I am running 15 of the top algorithms for predicting prices. I am using the famous Boston Housing Dataset from UCI
#
# In this notebook, I upload the data, prepare it with some feature transforms, and then test the accuracy of 15 different variations of regression algorithms, choosing the best one to tune for the final deployment

# +
#Loading in all libraries
import numpy as np
from numpy import arange
from matplotlib import pyplot as plt
from pandas import read_csv
from pandas import set_option
from pandas.plotting import scatter_matrix
from sklearn.model_selection import train_test_split
from sklearn.model_selection import KFold
from sklearn.model_selection import cross_val_score
from sklearn.preprocessing import StandardScaler
from sklearn.pipeline import Pipeline
from sklearn.model_selection import GridSearchCV


from sklearn.linear_model import LinearRegression
from sklearn.linear_model import Lasso
from sklearn.linear_model import ElasticNet
from sklearn.tree import DecisionTreeRegressor
from sklearn.neighbors import KNeighborsRegressor
from sklearn.svm import SVR
from sklearn.ensemble import RandomForestRegressor
from sklearn.ensemble import GradientBoostingRegressor
from sklearn.ensemble import ExtraTreesRegressor
from sklearn.ensemble import AdaBoostRegressor
from sklearn.metrics import mean_squared_error
# -

# Load dataset
colnames = ['CRIM', 'ZN', 'INDUS', 'CHAS', 'NOX', 'RM', 'AGE', 'DIS', 'RAD', 'TAX', 'PTRATIO', 'B', 'LSTAT', 'MEDV']
dataset = read_csv('housing.csv', delim_whitespace=True, names=colnames)

# ##  Now I am going to look at the Statistics of the dataset

#Descriptive statistics options
set_option('display.width', 150)
set_option('precision', 1)

#Descriptive Statistics
print(dataset.shape)
print()
print(dataset.head(20))
print()
print(dataset.dtypes)
print()
print(dataset.describe())

# #  Here I am visualizing the relationships between the datasets features

#Matplotlib Settings
plt.rcParams['figure.figsize'] = [10, 10]

# histogram
dataset.hist(sharex=False, sharey=False, xlabelsize=1, ylabelsize=1)
plt.suptitle("Histograms", y=1.05, size =16, fontname="Sans", weight='bold')
plt.tight_layout(pad=1)
plt.show()

# density
dataset.plot(kind='density', subplots=True, layout=(4,4), sharex=False, legend=False, fontsize=1)
plt.suptitle("Density", y=1.05, size =16, fontname="Sans", weight='bold')
plt.tight_layout(pad=1)
plt.show()

# box and whisker plots
dataset.plot(kind='box', subplots=True, layout=(4,4), sharex=False, sharey=False)
plt.suptitle("Box Plots", y=1.05, size =16, fontname="Sans", weight='bold')
plt.tight_layout(pad=1)
plt.show()

# scatter plot matrix
scatter_matrix(dataset)
plt.suptitle("Scatter Matrix", size =16, fontname="Sans", weight='bold', y=.92)
plt.show()

# correlation matrix
fig = plt.figure()
ax = fig.add_subplot(111)
cax = ax.matshow(dataset.corr(), vmin=-1, vmax=1, interpolation='none')
fig.colorbar(cax)
ticks = arange(0,14,1)
ax.set_xticks(ticks)
ax.set_yticks(ticks)
ax.set_xticklabels(colnames)
ax.set_yticklabels(colnames)
plt.show()


# ## After visually checking the relationships between the features of the dataset, I am going to split the data set into Train and Validation Sets   

#Separating target variables and predictor variables
X = dataset.values[:,0:13]
Y = dataset.values[:,13]

#Creating Test Dataset
X_train, X_validation, Y_train, Y_validation = train_test_split(X, Y, test_size=.2, random_state=7)

# ## Here I am creating the List of Regression Algorithms for testing

# +
models = [('LR', LinearRegression()),
          ('LASSO', Lasso()),
          ('EN', ElasticNet()),
          ('KNN', KNeighborsRegressor()),
          ('CART', DecisionTreeRegressor()),
          ('S-LR', Pipeline([('Scaler', StandardScaler()), ('LR', LinearRegression())])),
          ('S-LASSO', Pipeline([('Scaler', StandardScaler()), ('LASSO', Lasso())])),
          ('S-EN', Pipeline([('Scaler', StandardScaler()), ('EN', ElasticNet())])),
          ('S-KNN', Pipeline([('Scaler', StandardScaler()), ('KNN', KNeighborsRegressor())])),
          ('S-CART', Pipeline([('Scaler', StandardScaler()), ('CART', DecisionTreeRegressor())])), 
          ('S-SVR', Pipeline([('Scaler', StandardScaler()), ('SVR', SVR(gamma='auto'))])), 
          ('S-AB', Pipeline([('Scaler', StandardScaler()), ('AB', AdaBoostRegressor())])), 
          ('S-GBM', Pipeline([('Scaler', StandardScaler()), ('GBM', GradientBoostingRegressor())])), 
          ('S-RF', Pipeline([('Scaler', StandardScaler()), ('RF', RandomForestRegressor(n_estimators=10))])), 
          ('S-ET', Pipeline([('Scaler', StandardScaler()), ('ET', ExtraTreesRegressor(n_estimators=10))]))]

# ('SVR', SVR(gamma='auto'))
# -

# Test options and evaluation metric
num_folds = 10
seed = 7
scoring = 'neg_mean_squared_error'
kfold = KFold(n_splits=num_folds, random_state=seed, shuffle=True)

# ## Here I am creating a loop to test all models  

results = []
names = []
for name, model in models:
    cv_results = cross_val_score(model, X_train, Y_train, cv=kfold, scoring=scoring)
    results.append(cv_results)
    names.append(name)
    print("%s: %f (%f)" % (name, cv_results.mean(), cv_results.std()))

# # Visual Comparison of Algorithm Performance  

fig = plt.figure(figsize=(10,4), dpi= 100, facecolor='b', edgecolor='w')
fig.suptitle('Algorithm Performance Comparison')
ax = fig.add_subplot(111)
plt.boxplot(results)
ax.set_xticklabels(names)
plt.show()

# ## With these results, I am going to tune a scaled KNN model, a scaled ExtraTrees model, and a scaled GBM model to see which has the lowest error (MSE)

# # Tuning Scaled KNN

# +
scaler = StandardScaler().fit(X_train)
rescaledX = scaler.transform(X_train)
k_values = np.array([1,3,5,7,9,11,13,15,17,19,21])
param_grid = dict(n_neighbors=k_values)
model = KNeighborsRegressor()

#GridSearch
grid = GridSearchCV(estimator=model, param_grid=param_grid, scoring=scoring, cv=kfold)
grid_result = grid.fit(rescaledX, Y_train)

#Print Results
print("Best: %f using %s" % (grid_result.best_score_, grid_result.best_params_))
means = grid_result.cv_results_['mean_test_score']
stdev = grid_result.cv_results_['std_test_score']
params = grid_result.cv_results_['params']
for mean, stdev, param in zip(means, stdev, params):
    print("%f (%f) with: %r" % (mean, stdev, param))
# -

# # Tuning Scaled GBM 

# +
scaler = StandardScaler().fit(X_train)
rescaledX = scaler.transform(X_train)
param_grid = dict(n_estimators=np.array([50,100,150,200,250,300,350,400,450,500]))
model = GradientBoostingRegressor(random_state=seed)

#GridSearch
grid = GridSearchCV(estimator=model, param_grid=param_grid, scoring=scoring, cv=kfold)
grid_result = grid.fit(rescaledX, Y_train)

#Print Results
print("Best: %f using %s" % (grid_result.best_score_, grid_result.best_params_))
means = grid_result.cv_results_['mean_test_score']
stds = grid_result.cv_results_['std_test_score']
params = grid_result.cv_results_['params']
for mean, stdev, param in zip(means, stds, params):
    print("%f (%f) with: %r" % (mean, stdev, param))
# -

# # Tuning Scaled Extra Trees 

# +
scaler = StandardScaler().fit(X_train)
rescaledX = scaler.transform(X_train)
param_grid = dict(n_estimators=np.array([50,100,150,200,250,300,350,400,450,500]))
model = ExtraTreesRegressor(random_state=seed)

grid = GridSearchCV(estimator=model, param_grid=param_grid, scoring=scoring, cv=kfold)
grid_result = grid.fit(rescaledX, Y_train)

print("Best: %f using %s" % (grid_result.best_score_, grid_result.best_params_))
means = grid_result.cv_results_['mean_test_score']
stds = grid_result.cv_results_['std_test_score']
params = grid_result.cv_results_['params']
for mean, stdev, param in zip(means, stds, params):
    print("%f (%f) with: %r" % (mean, stdev, param))
# -

# # Testing Scaled and Tuned ET 

# +
scaler = StandardScaler().fit(X_train)
rescaledX = scaler.transform(X_train)
model = ExtraTreesRegressor(random_state=seed, n_estimators=150)
model.fit(rescaledX, Y_train)

# transform the validation dataset
rescaledValidationX = scaler.transform(X_validation)

#Make Prediction & Print Results
predictions = model.predict(rescaledValidationX)
print(mean_squared_error(Y_validation, predictions))
# -

# # Testing Scaled and Tuned GBM 

# +
scaler = StandardScaler().fit(X_train)
rescaledX = scaler.transform(X_train)
model = GradientBoostingRegressor(random_state=seed, n_estimators=450)
model.fit(rescaledX, Y_train)

# transform the validation dataset
rescaledValidationX = scaler.transform(X_validation)

#Make Prediction & Print Results
test_predictions = model.predict(rescaledValidationX)
print(mean_squared_error(Y_validation, test_predictions))
# -

# ## At this point I have found the "Gradient Boosting Regressor" with Scaled data to be the highest performing Algorithm, with a final Mean Squared Error of 11.97

# ## For the final part, I have added the Prediction Prices into the last column of the orginal dataset, so the accuracy of these predictions can be individually seen on each house 

scaler = StandardScaler().fit(X)
rescaled_original_X = scaler.transform(X)
all_predictions = model.predict(rescaled_original_X)
dataset['predictions'] = all_predictions
dataset.head(20)

StarcoderdataJupyterScriptsDedupFiltered