import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
from sklearn.datasets import load_boston
from sklearn.metrics import r2_score
"""
# Load Dataset
"""
boston = load_boston()
data = pd.DataFrame(data = boston.data, columns = boston.feature_names)
data
"""
![boston.png](attachment:2ad91c6a-8278-48dc-a187-84a6f4525021.png)
"""
data['Price'] = boston.target
data
Missing_value = data.isna().sum()
Missing_value
data.shape[0] - data.count()

data.info()
data.describe()
plt.hist(data['Price'], bins = 50)
var_ind = np.arange(0, 14)
plot = plt.figure(figsize = (20, 10))
plot.subplots_adjust(hspace = 0.5, wspace = 0.5)
for i in range(1, 15):
    a = plot.add_subplot(4, 4, i)
    a.hist(data.iloc[:, var_ind[i - 1]], alpha = 0.7, bins = 30)
    a.title.set_text(data.columns[i - 1])
cor_table = data.iloc[:, var_ind].corr(method = 'pearson')
cor_table
"""
# Split data
"""
from sklearn.model_selection import train_test_split
train, test = train_test_split(data, test_size = 0.25, random_state = 123456)
print(f'The shape of train is: {train.shape}')
print(f'The shape of test is: {test.shape}')
test
X_train = train.iloc[:, :-1]
y_train = train.iloc[:, -1]
X_test = test.iloc[:, :-1]
y_test = test.iloc[:, -1]


print(f'The shape of X_train is: {X_train.shape}')
print(f'The shape of y_train is: {y_train.shape}')
print(f'The shape of X_test is: {X_test.shape}')
print(f'The shape of y_test is: {y_test.shape}')
"""
# Model1: Classic Regression
"""
import statsmodels.formula.api as smf
model1 = smf.ols(formula = 'np.log(Price) ~ CRIM + ZN + INDUS + CHAS + NOX + RM + AGE + DIS + RAD + TAX + PTRATIO + B + LSTAT', data = train)
model1 = model1.fit()
model1.summary()
y_pred_model1 = model1.predict(X_test)
y_pred_model1 = np.exp(y_pred_model1)
abs_err_model1 = np.abs(y_pred_model1 - y_test)
np.mean(abs_err_model1)
#Absolute error mean, median, sd, IQR, max, min
from scipy.stats import iqr
model_comp = pd.DataFrame({'Mean of AbsErrors':    abs_err_model1.mean(),
                           'Median of AbsErrors' : abs_err_model1.median(),
                           'SD of AbsErrors' :     abs_err_model1.std(),
                           'IQR of AbsErrors':     iqr(abs_err_model1),
                           'Min of AbsErrors':     abs_err_model1.min(),
                           'Max of AbsErrors':     abs_err_model1.max()}, index = ['Model1: Classical Regression'])
model_comp
y_pred = y_pred_model1
y_test = y_test
model = 'model1_ClassicalRegression'

plt.figure(figsize=(15, 15), dpi = 150);
plt.subplots_adjust(hspace = 0.25, wspace = 0.25);

plt.subplots(figsize=(16,6));
plt.scatter(x = y_test, y = y_pred, color = 'blue');
#Add 45 degree line
xp = np.linspace(y_test.min(), y_test.max(), 100)
plt.plot(xp, xp, 'k', alpha = 0.9, linewidth = 2, color = 'red')
plt.xlabel('Actual Value');
plt.ylabel('Predicted Value');

plt.title(f'{model}');
plt.subplots(figsize=(17,6));
print(f'The r2-score is: {r2_score(y_pred, y_test)}')
x_points = list(range(len(y_test)));
plt.plot(x_points, y_test, label='y_real', color = 'blue');
plt.plot(x_points, y_pred, label='y_predict', color = 'red');
plt.legend();
plt.title(f'{model}');
plt.show();
abs_err = abs_err_model1
name = 'abs_err_model1 Classical Reg'

plt.figure(figsize=(15, 5), dpi = 200)
plt.subplots_adjust(hspace = 0.25, wspace = 0.25)


plt.subplot(121)
plt.boxplot(abs_err);
plt.title(f'boxplot of {name}');
plt.grid()

plt.subplot(122)
sns.distplot(abs_err, kde = False);
plt.title(f'Histogram of {name}');

logy_train = np.log(y_train)
"""
# Model 2: Ridge Regression
"""
lambda_grid = 10 ** np.linspace(-2, 5, 1000)
lambda_grid
from sklearn.linear_model import Ridge, RidgeCV
ridgecv = RidgeCV(alphas = lambda_grid, cv = 10, normalize = True)
ridgecv.fit(X_train, logy_train)
ridgecv.alpha_


model2 = Ridge(normalize = True, alpha = ridgecv.alpha_)
model2.fit(X_train, logy_train)
y_pred_model2 = model2.predict(X_test)
y_pred_model2 = np.exp(y_pred_model2)
abs_err_model2 = np.abs(y_test - y_pred_model2)
np.mean(abs_err_model2)
model_comp = model_comp.append(pd.DataFrame({'Mean of AbsErrors':    abs_err_model2.mean(),
                                             'Median of AbsErrors' : abs_err_model2.median(),
                                             'SD of AbsErrors' :     abs_err_model2.std(),
                                             'IQR of AbsErrors':     iqr(abs_err_model2),
                                             'Min of AbsErrors':     abs_err_model2.min(),
                                             'Max of AbsErrors':     abs_err_model2.max()}, index = ['Model2: Ridge']), 
                               ignore_index = False)

model_comp
y_pred = y_pred_model2
y_test = y_test
model = 'model2: Ridge Regression'

plt.figure(figsize=(15, 15), dpi = 150);
plt.subplots_adjust(hspace = 0.25, wspace = 0.25);

plt.subplots(figsize=(16,6));
plt.scatter(x = y_test, y = y_pred, color = 'blue');
#Add 45 degree line
xp = np.linspace(y_test.min(), y_test.max(), 100)
plt.plot(xp, xp, 'k', alpha = 0.9, linewidth = 2, color = 'red')
plt.xlabel('Actual Value');
plt.ylabel('Predicted Value');

plt.title(f'{model}');
plt.subplots(figsize=(17,6));
print(f'The r2-score is: {r2_score(y_pred, y_test)}')
x_points = list(range(len(y_test)));
plt.plot(x_points, y_test, label='y_real', color = 'blue');
plt.plot(x_points, y_pred, label='y_predict', color = 'red');
plt.legend();
plt.title(f'{model}');
plt.show();
abs_err = abs_err_model2
name = 'abs_err_model2 Ridge Reg'

plt.figure(figsize=(15, 5), dpi = 200)
plt.subplots_adjust(hspace = 0.25, wspace = 0.25)


plt.subplot(121)
plt.boxplot(abs_err);
plt.title(f'boxplot of {name}');
plt.grid()

plt.subplot(122)
sns.distplot(abs_err, kde = False);
plt.title(f'Histogram of {name}');

"""
# Model 3: Decision Tree
"""
# {'min_samples_leaf': 5, 'max_depth': 6, 'ccp_alpha': 0.00018}
from sklearn.tree import DecisionTreeRegressor
model3 = DecisionTreeRegressor(max_depth = 6, 
                               min_samples_leaf = 5, 
                               ccp_alpha = 0.00018)
model3 = model3.fit(X_train, logy_train)
y_pred_model3 = model3.predict(X_test)
y_pred_model3 = np.exp(y_pred_model3)
abs_err_model3 = np.abs(y_pred_model3 - y_test)
np.mean(abs_err_model3)
# # HyperParameter Tuning
# from sklearn.model_selection import RandomizedSearchCV

# max_depth = [3, 4, 5, 6, 7, 8, 9]
# min_samples_leaf = [5, 7, 6, 8, 9, 4]
# ccp_alpha = [0.0001, 0.001, 0.01, 0.1, 0.00025, 0.00018, 0.0007, 0.0054]
# objective = ['reg:squarederror']

# random_grid = {'max_depth': max_depth, 
#                'min_samples_leaf': min_samples_leaf, 
#                'ccp_alpha': ccp_alpha}

# decision = RandomizedSearchCV(estimator = model3, param_distributions = random_grid, scoring = 'neg_mean_squared_error', n_iter = 10, cv = 8)
# decision.fit(X_train, logy_train)
# decision.best_params_

model_comp = model_comp.append(pd.DataFrame({'Mean of AbsErrors':    abs_err_model3.mean(),
                                             'Median of AbsErrors' : abs_err_model3.median(),
                                             'SD of AbsErrors' :     abs_err_model3.std(),
                                             'IQR of AbsErrors':     iqr(abs_err_model3),
                                             'Min of AbsErrors':     abs_err_model3.min(),
                                             'Max of AbsErrors':     abs_err_model3.max()}, index = ['Model3: Tree']), 
                               ignore_index = False)

model_comp
y_pred = y_pred_model3
y_test = y_test
model = 'model 3 Tree'

plt.figure(figsize=(15, 15), dpi = 150);
plt.subplots_adjust(hspace = 0.25, wspace = 0.25);

plt.subplots(figsize=(16,6));
plt.scatter(x = y_test, y = y_pred, color = 'blue');
#Add 45 degree line
xp = np.linspace(y_test.min(), y_test.max(), 100)
plt.plot(xp, xp, 'k', alpha = 0.9, linewidth = 2, color = 'red')
plt.xlabel('Actual Value');
plt.ylabel('Predicted Value');

plt.title(f'{model}');
plt.subplots(figsize=(17,6));
print(f'The r2-score is: {r2_score(y_pred, y_test)}')
x_points = list(range(len(y_test)));
plt.plot(x_points, y_test, label='y_real', color = 'blue');
plt.plot(x_points, y_pred, label='y_predict', color = 'red');
plt.legend();
plt.grid()
plt.title(f'{model}');
plt.show();
abs_err = abs_err_model3
name = 'abs_err_model3_Decision Tree'

plt.figure(figsize=(15, 5), dpi = 200)
plt.subplots_adjust(hspace = 0.25, wspace = 0.25)


plt.subplot(121)
plt.boxplot(abs_err);
plt.title(f'boxplot of {name}');
plt.grid()

plt.subplot(122)
sns.distplot(abs_err, kde = False);
plt.title(f'Histogram of {name}');

"""
# Model 4: Bagging
"""
from sklearn.ensemble import BaggingRegressor
model4 = BaggingRegressor(n_estimators = 500)
model4.fit(X_train, logy_train)



y_pred_model4 = model4.predict(X_test)
y_pred_model4 = np.exp(y_pred_model4)
abs_err_model4 = np.abs(y_pred_model4 - y_test)
np.mean(abs_err_model4)
model_comp = model_comp.append(pd.DataFrame({'Mean of AbsErrors':    abs_err_model4.mean(),
                                             'Median of AbsErrors' : abs_err_model4.median(),
                                             'SD of AbsErrors' :     abs_err_model4.std(),
                                             'IQR of AbsErrors':     iqr(abs_err_model4),
                                             'Min of AbsErrors':     abs_err_model4.min(),
                                             'Max of AbsErrors':     abs_err_model4.max()}, index = ['Model4: Bagging']), 
                               ignore_index = False)

model_comp
y_pred = y_pred_model4
y_test = y_test
model = 'model 4 Bagging'

plt.figure(figsize=(15, 15), dpi = 150);
plt.subplots_adjust(hspace = 0.25, wspace = 0.25);

plt.subplots(figsize=(16,6));
plt.scatter(x = y_test, y = y_pred, color = 'blue');
#Add 45 degree line
xp = np.linspace(y_test.min(), y_test.max(), 100)
plt.plot(xp, xp, 'k', alpha = 0.9, linewidth = 2, color = 'red')
plt.xlabel('Actual Value');
plt.ylabel('Predicted Value');

plt.title(f'{model}');
plt.subplots(figsize=(17,6));
print(f'The r2-score is: {r2_score(y_pred, y_test)}')
x_points = list(range(len(y_test)));
plt.plot(x_points, y_test, label='y_real', color = 'blue');
plt.plot(x_points, y_pred, label='y_predict', color = 'red');
plt.legend();
plt.grid()
plt.title(f'{model}');
plt.show();
abs_err = abs_err_model4
name = 'abs_err_model4 Bagging'

plt.figure(figsize=(15, 5), dpi = 200)
plt.subplots_adjust(hspace = 0.25, wspace = 0.25)


plt.subplot(121)
plt.boxplot(abs_err);
plt.title(f'boxplot of {name}');
plt.grid()

plt.subplot(122)
sns.distplot(abs_err, kde = False);
plt.title(f'Histogram of {name}');

"""
# Model 5: RandomForest Regression

"""
from sklearn.ensemble import RandomForestRegressor
model5 = RandomForestRegressor(n_estimators = 1220,
                               oob_score = True, 
                               max_features = 'sqrt', 
                               max_depth = 33, 
                               min_samples_split = 2, 
                               min_samples_leaf = 1)

model5.fit(X_train, logy_train)
y_pred_model5 = model5.predict(X_test)
y_pred_model5 = np.exp(y_pred_model5)
abs_err_model5 = np.abs(y_pred_model5 - y_test)
np.mean(abs_err_model5)


# from sklearn.model_selection import RandomizedSearchCV
# n_estimators = [int(x) for x in np.linspace(start = 100, stop = 2000, num = 40)]
# max_features = ['auto', 'sqrt']
# max_depth = [int(x) for x in np.linspace(5, 40, num = 6)]
# min_samples_split = [2, 5, 10, 15, 20,25,30,35,40,100]
# min_samples_leaf = [1, 2, 5, 10]

# random_grid = {'n_estimators': n_estimators,
#                'max_features': max_features,
#                'max_depth': max_depth,
#                'min_samples_split': min_samples_split,
#                'min_samples_leaf': min_samples_leaf}

# rf_random = RandomizedSearchCV(estimator = model5, 
#                                param_distributions = random_grid, 
#                                scoring='neg_mean_squared_error',
#                                n_iter = 10, 
#                                cv = 5, 
#                                verbose=2, 
#                                random_state=42, 
#                                n_jobs = 1)

# rf_random.fit(X_train, logy_train);
# rf_random.best_params_