# This Python 3 environment comes with many helpful analytics libraries installed
# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python
# For example, here's several helpful packages to load

import numpy as np # linear algebra
import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)

# Input data files are available in the read-only "../input/" directory
# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory

import os
for dirname, _, filenames in os.walk('/kaggle/input'):
    for filename in filenames:
        print(os.path.join(dirname, filename))

# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using "Save & Run All" 
# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session
"""
First, we will import necessary libraries
"""
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
"""
Let's import our datasets and check basic information. We also knows from the datasets infor that there are 14 columns, describing details as follow:

- CRIM - per capita crime rate by town
- ZN - proportion of residential land zoned for lots over 25,000 sq.ft.
- INDUS - proportion of non-retail business acres per town.
- CHAS - Charles River dummy variable (1 if tract bounds river; 0 otherwise)
- NOX - nitric oxides concentration (parts per 10 million)
- RM - average number of rooms per dwelling
- AGE - proportion of owner-occupied units built prior to 1940
- DIS - weighted distances to five Boston employment centres
- RAD - index of accessibility to radial highways
- TAX - full-value property-tax rate per \$10,000
- PTRATIO - pupil-teacher ratio by town
- B - 1000(Bk - 0.63)^2 where Bk is the proportion of blacks by town
- LSTAT - percentage of lower status of the population
- MEDV - Median value of owner-occupied homes in $1000's
"""
"""
As the dataset already contain correct data types, there is no need to change it.
"""
columns = ['CRIM', 'ZN','INDUS', 'CHAS', 'NOX', 'RM', 'AGE', 'DIS', 'RAD', 'TAX', 'PTRATIO', 'B', 'LSTAT', 'MEDV']
house_df = pd.read_csv('../input/boston-house-prices/housing.csv', header=None, delimiter='\s+', names=columns)
house_df.head()
# To check whether there is missing number and the data type for each column
house_df.isna().sum()
"""
We observed that there is no missing data
"""
# From the house data, we expect that the datatype for each column should be either integer or float.
house_df.info()
"""
As the dataset already contain correct data types, there is no need to change it.
"""
# We will observe general statistical information from the datasets
house_df.describe()
# Let use see whether MEDV follow normal distribution or not
sns.displot(house_df['MEDV'])
"""
### The distribution is slightly skewed to the right, with additional peal appearing at the right most graph
"""
# In general, bigger room leads to a higher house cost. Let us see the relation between number of room (RM) with house price (MEDV)
sns.lmplot(data=house_df, x='RM', y='MEDV')
# it is interesting that some houses have MEDV of 50.00 regardless of room size. Let's take a look on these houses.
house_df[house_df['MEDV'] == 50.00]
# It seems that for house with MEDV of 50.00 has relatively low LSTAT (percentage of lower status of the population). Let us check whether it is true.
sns.lmplot(data=house_df, x='LSTAT', y='MEDV')
"""
### We see that houses price increases exponentially with increase of LSTAT.It is even more obvious when LSTAT is below 10. It is interesting to see that some people are willing to pay high price to own a house surrounded by owners with higher status, regardless the size of the houses
"""
# Let us compare the correlation between each features, especially compared to the price of the house
plt.figure(figsize=(20,15))
sns.heatmap(house_df.corr(), annot=True)
"""
### We can see that number of room (RM), pupil to teacher ratio(PTRATIO), and percentage of lower status of the population (LSTAT) played the biggest impact to house price (MEDV) compared with other features.
"""
# Let start to predict the house price based on existing features. House Price (MEDV) will be dependent variables (y), and the rest of the features are independent variables. 
X = house_df.iloc[:,:-1].values
y = house_df.iloc[:,-1].values
# The data will also be split into train set and test set.
from sklearn.model_selection import train_test_split
X_train, X_test, y_train, y_test = train_test_split(X,y,test_size=0.2)
"""
### As the features value is not significantly different, I will not do standard scaling here. If results shows low accuracy, I will get back and retest it with standard scaler 
"""
"""
## Linear Regression
"""
# Let's predict house price with linear regression
from sklearn.linear_model import LinearRegression
from sklearn.metrics import r2_score
linreg = LinearRegression()
linreg.fit(X_train,y_train)
y_pred_linreg = linreg.predict(X_test)
r2_score(y_test, y_pred_linreg)
"""
#### the accuracy is decent, but let's see whether we can get a higher score using non-linear regression
"""
"""
## Polynomial Regression
"""
from sklearn.preprocessing import PolynomialFeatures
poly_feat = PolynomialFeatures(degree=2)
X_poly = poly_feat.fit_transform(X_train)
polyreg = LinearRegression()
polyreg.fit(X_poly,y_train)
y_pred_poly = polyreg.predict(poly_feat.transform(X_test))
r2_score(y_test, y_pred_poly)
"""
#### Seems that non-linear regression is more suitable for this dataset. Let use other models (SVR, Decision Tress, and Random Forest) as comparison
"""
"""
## Support Vector Regression (SVR)
"""
from sklearn.svm import SVR
svreg = SVR(kernel='rbf')
svreg.fit(X_train, y_train)
y_pred_svr = svreg.predict(X_test)
r2_score(y_test, y_pred_svr)
"""
Seems SVR is not a suitable model. Even SVR with 'linear' kernel performs better than 'rbf' kernel
"""
"""
## Decision Trees Regression
"""
from sklearn.tree import DecisionTreeRegressor
tree_reg = DecisionTreeRegressor()
tree_reg.fit(X_train, y_train)
y_pred_tree = tree_reg.predict(X_test)
r2_score(y_test, y_pred_tree)
"""
## Random Forest Regression
"""
from sklearn.ensemble import RandomForestRegressor
forest_regressor = RandomForestRegressor(n_estimators=100)
forest_regressor.fit(X_train, y_train)
y_pred_forest = forest_regressor.predict(X_test)
r2_score(y_test, y_pred_forest)
"""
The results between Polynomial regression, Decision Trees, and Random Forest are similar. Let us do k-fold cross validation to select the best model for this dataset to account for different data used for train and test
"""
"""
## K-fold Cross Validation
"""
from sklearn.model_selection import cross_val_score
ml_model = [polyreg, tree_reg, forest_regressor]
for i in ml_model:
    accuracy = cross_val_score(estimator=i, X=X_train, y=y_train, cv=10)
    print('accuracy score of {} is {}'.format(i,accuracy.mean()))
"""
### Between all models, Random Forest Regression has the highest mean accuracy of 86%. Therefore, Random Forest regression is the most suitable model to predict Boston house price based on provided features.
"""