# This Python 3 environment comes with many helpful analytics libraries installed
# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python
# For example, here's several helpful packages to load

import numpy as np # linear algebra
import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)

# Input data files are available in the read-only "../input/" directory
# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory

import os
for dirname, _, filenames in os.walk('/kaggle/input'):
    for filename in filenames:
        print(os.path.join(dirname, filename))

# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using "Save & Run All" 
# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session
"""
<h1 id="tocheading">Table of Contents</h1>
<div id="toc"></div>
"""
%%javascript
$.getScript('https://kmahelona.github.io/ipython_notebook_goodies/ipython_notebook_toc.js')
import numpy as np
import pandas as pd
import seaborn as sns
import scipy.stats as stats
from scipy.stats import norm
import statsmodels.api as sm
import matplotlib.pyplot as plt
from scipy.stats import skew, norm
from sklearn.neighbors import KNeighborsRegressor
%matplotlib inline
#Import the files
df_train = pd.read_csv('../input/house-prices-advanced-regression-techniques/train.csv')
df_test = pd.read_csv('../input/house-prices-advanced-regression-techniques/test.csv')
print('Shape of train data is', df_train.shape)
print ('Shape of tests data is', df_test.shape)
#check the train data
df_train.head()

# so there are around 81 features here in this dataset
#information about the train data set
df_train.info()
df_train.describe()
"""
## Finding missing values
"""
#check for the missing values in the train data
df_train.isnull().sum()
"""
# TRAIN Data
"""
# Check which variables have missing values

Train_columns_with_missing_values = df_train.columns[df_train.isnull().any()]
df_train[Train_columns_with_missing_values].isnull().sum()
df_train.isnull().sum().sum() # total missing values in the entire df
missing_series = df_train.isnull().sum() 
missing_series.where(missing_series>1000)
missing_series.loc[missing_series>1000]/df_train.shape[0] # to locate the missing values that are more than 1000
"""
We can see that the above 4 features have a lot of missing values and they won't brinng any value to the further analysis . To strengthen our analysis the other methods that we could use to see the missing values are:
* Heatmaps
* Histogram plot of all the features and then visual inspection
* missingno library
"""
#Lets look at the histogram of all the features first

df_train.hist(bins=50,figsize=(20,15))
# Now using heatmaps
plt.figure(figsize=(15,8))
sns.heatmap(df_train.isnull(), cbar=False)
# FRom this heatmap too it is evident that the above 4 mentioned vars have more missing values
# Next is to use the missingno library to analyze the missing values
import missingno as msno 

msno.bar(df_train,labels=True,fontsize=10) 
msno.heatmap(df_train,labels=True) 
# Entries marked <1 or >-1 have a correlation that is close to being exactingly negative or positive, but is still not quite perfectly so
# Nullity correlation ranges from -1 (if one variable appears the other definitely does not) to 0 (variables appearing or not appearing have no effect on one another) to 1 (if one variable appears the other definitely also does)
df_train['Alley'].unique()
"""
## Preprocessing
* Drop the missing values
* fill the missing values
    * Fill with meaningful values, after reading data description
    * fill with stat values after looking at the variable (if it is a  continious variable)
"""
# variable - Alley 
# Alley: Type of alley access to property Grvl-Gravel, Pavd-Paved, NA-No alley access
# so nan in alley is not a missing value it means no access -> replace nan with no access
# Py took the a´nan as missing values 
df_train['Alley'].fillna('No Access', inplace=True)
# variable - LotFrontage 
# it is afloat variable , so missing values can be replaced by using median/mean - central tendency mneasures
# which one to use depends on the plot of the data
# https://towardsdatascience.com/understanding-boxplots-5e2df7bcbd51
sns.boxplot(df_train['LotFrontage'])

"""
we can see that there are lot of outliers in this feature and from the distribution plot it can also be seen that the plot is skewed. Outliers data points will have significant impact on the mean and hence, in such cases, it is not recommended to use mean for replacing the missing values. Using mean value for replacing missing values may not create a great model and hence gets ruled out. For symmetric data distribution, one can use mean value for imputing missing values.
"""
"""
![image.png](attachment:image.png)
"""
sns.distplot(df_train['LotFrontage'])

# impute the missing values using median
df_train['LotFrontage'].fillna(df_train['LotFrontage'].median(),inplace =True)
"""
The issues with the following variable is also due to the misintepretation by python , here na means no basement
* BsmtQual          37
* BsmtCond          37
* BsmtExposure      38
* BsmtFinType1      37
* BsmtFinType2      38
"""
df_train['BsmtQual'].fillna('No Basement', inplace=True)
df_train['BsmtCond'].fillna('No Basement', inplace=True)
df_train['BsmtExposure'].fillna('No Basement', inplace=True)
df_train['BsmtFinType1'].fillna('No Basement', inplace=True)
df_train['BsmtFinType2'].fillna('No Basement', inplace=True)
"""
The issues with the following variable is also due to the misintepretation by python , here na means no garage
* GarageType        81
* GarageYrBlt       81
* GarageFinish      81
* GarageQual        81
* GarageCond        81
"""
df_train['GarageType'].fillna('No Garage', inplace=True)
df_train['GarageFinish'].fillna('No Garage', inplace=True)
df_train['GarageQual'].fillna('No Garage', inplace=True)
df_train['GarageCond'].fillna('No Garage', inplace=True)
"""
The issues with the following variable is also due to the misintepretation by python
* FireplaceQu      690 - No Fireplace
* PoolQC          1453 - No Pool
* Fence           1179 - No Fence
* MasVnrType         8 - None
* MasVnrArea         8 - Replace with the most common
* MiscFeature     1406 - None
"""
df_train['PoolQC'].fillna('No Pool', inplace=True)
df_train['Fence'].fillna('No Fence', inplace=True)
df_train['MiscFeature'].fillna('None', inplace=True)
df_train['MasVnrType'].fillna('None', inplace=True)
df_train['FireplaceQu'].fillna('No Fireplace', inplace=True)

df_train['MasVnrArea'].value_counts()
df_train['MasVnrArea'].fillna(df_train['MasVnrArea'].mode()[0], inplace=True)
df_train['Electrical'].fillna(df_train['Electrical'].mode()[0], inplace=True)
#drop the ID and year features
df_train.drop(['GarageYrBlt'],axis=1,inplace=True)
df_train.drop(['Id'],axis=1,inplace=True)
#Finally check the DF after the imputation process
# Now using heatmaps
plt.figure(figsize=(15,8))
sns.heatmap(df_train.isnull(), cbar=False)

# missing values for the train data are succesfully handled
"""
# TEST Data
"""
# Check which variables have missing values in test data

Testcolumns_with_missing_values = df_test.columns[df_test.isnull().any()]
df_test[Testcolumns_with_missing_values].isnull().sum()
#Finally check the DF after the imputation process
# Now using heatmaps
plt.figure(figsize=(15,8))
sns.heatmap(df_test.isnull(), cbar=False)

# missing values for the train data are succesfully handled
df_test['LotFrontage'].fillna(df_test['LotFrontage'].median(),inplace =True)
df_test['Alley'].fillna('No Access', inplace=True)
df_test['BsmtQual'].fillna('No Basement', inplace=True)
df_test['BsmtCond'].fillna('No Basement', inplace=True)
df_test['BsmtExposure'].fillna('No Basement', inplace=True)
df_test['BsmtFinType1'].fillna('No Basement', inplace=True)
df_test['BsmtFinType2'].fillna('No Basement', inplace=True)
df_test['GarageType'].fillna('No Garage', inplace=True)
df_test['GarageFinish'].fillna('No Garage', inplace=True)
df_test['GarageQual'].fillna('No Garage', inplace=True)
df_test['GarageCond'].fillna('No Garage', inplace=True)
df_test['PoolQC'].fillna('No Pool', inplace=True)
df_test['Fence'].fillna('No Fence', inplace=True)
df_test['MiscFeature'].fillna('None', inplace=True)
df_test['MasVnrType'].fillna('None', inplace=True)
df_test['FireplaceQu'].fillna('No Fireplace', inplace=True)
#Categorical features 
df_test['MasVnrArea'].fillna(df_test['MasVnrArea'].mode()[0], inplace=True)
df_test['Electrical'].fillna(df_test['Electrical'].mode()[0], inplace=True)
df_test['MSZoning'].fillna(df_test['MSZoning'].mode()[0], inplace=True)
df_test['Utilities'].fillna(df_test['Utilities'].mode()[0], inplace=True)
df_test['SaleType'].fillna(df_test['SaleType'].mode()[0], inplace=True)
df_test['Functional'].fillna(df_test['Functional'].mode()[0], inplace=True)
df_test['KitchenQual'].fillna(df_test['KitchenQual'].mode()[0], inplace=True)
df_test['BsmtFinSF2'].fillna(df_test['BsmtFinSF2'].mode()[0], inplace=True)
df_test['Exterior1st'].fillna(df_test['Exterior1st'].mode()[0], inplace=True)
df_test['Exterior2nd'].fillna(df_test['Exterior2nd'].mode()[0], inplace=True)
df_test['BsmtFullBath'].fillna(df_test['BsmtFullBath'].mode()[0], inplace=True)
df_test['BsmtHalfBath'].fillna(df_test['BsmtHalfBath'].mode()[0], inplace=True)
#drop the year features
df_test.drop(['GarageYrBlt'],axis=1,inplace=True)

sns.boxplot(df_test['GarageArea'])
df_test['GarageArea'].fillna(df_test['GarageArea'].median(),inplace =True)
sns.boxplot(df_test['GarageCars'])
df_test['GarageCars'].fillna(df_test['GarageCars'].mean(),inplace =True)
sns.boxplot(df_test['BsmtFinSF1'])
df_test['BsmtFinSF1'].fillna(df_test['BsmtFinSF1'].median(),inplace =True)
(df_test['BsmtFullBath'])
sns.distplot(df_test['BsmtUnfSF'])
df_test['BsmtUnfSF'].fillna(df_test['BsmtUnfSF'].median(),inplace =True)
sns.boxplot(df_test['TotalBsmtSF'])
df_test['TotalBsmtSF'].fillna(df_test['TotalBsmtSF'].median(),inplace =True)
df_test['BsmtFullBath'].value_counts()
#Finally check the DF after the imputation process
# Now using heatmaps
plt.figure(figsize=(15,8))
sns.heatmap(df_test.isnull(), cbar=False)

# missing values for the train data are succesfully handled
#Shape after handling the missing data
print('Shape of train data is', df_train.shape)
print ('Shape of tests data is', df_test.shape)
"""
## Correlation Matrix
"""
df_train.corr()
# Correlation Matrix

# Increase the size of the heatmap.
plt.figure(figsize=(25, 15))
# Store heatmap object in a variable to easily access it when you want to include more features (such as title).
# Set the range of values to be displayed on the colormap from -1 to 1, and set the annotation to True to display the correlation values on the heatmap.
heatmap = sns.heatmap(df_train.corr(), vmin=-1, vmax=1, annot=True,cmap='BrBG')
# Give a title to the heatmap. Pad defines the distance of the title from the top of the heatmap.
heatmap.set_title('Correlation Heatmap', fontdict={'fontsize':6}, pad=14);





#check the features that are highly correlated to the output variable
plt.figure(figsize=(8, 12))
heatmap_outputvar = sns.heatmap(df_train.corr()[['SalePrice']].sort_values(by='SalePrice', ascending=False), vmin=-1, vmax=1, annot=True, cmap='BrBG')
heatmap_outputvar.set_title('Features Correlating with Sales Price', fontdict={'fontsize':18}, pad=16);
# most correlated features

corrmat = df_train.corr()
top_corr_features = corrmat.index[abs(corrmat['SalePrice'])>0.5]
plt.figure(figsize=(16,9))
g = sns.heatmap(df_train[top_corr_features].corr(),annot=True,cmap="RdYlGn")
"""
From the above correlation plots it is evident that the features that really matter the most are:
* OverallQual
* GrlivingArea
* GarageCrs
* GarageArea
"""
"""
## Transforming Target & other numerical variable
"""
sns.distplot(df_train['SalePrice']);
#skewness and kurtosis
print("Skewness: %f" % df_train['SalePrice'].skew())
print("Kurtosis: %f" % df_train['SalePrice'].kurt())
# the dist is right skewed, ie it has many outliersspred out to the right
# For parametric models , like regression it is important to have  a belll type distribution of the target variable
# So the idea here is to log transform the output variable
df_train['SalePrice_Log'] = np.log(df_train['SalePrice'])

sns.distplot(df_train['SalePrice_Log']);
# skewness and kurtosis
print("Skewness after Transform: %f" % df_train['SalePrice_Log'].skew())
print("Kurtosis after Transform: %f" % df_train['SalePrice_Log'].kurt())
# dropping old column

df_train.drop('SalePrice', axis= 1, inplace=True)
# Plot skew value for each numerical value
from scipy.stats import skew 
skew_in_vars = df_train [numerical_feats].apply(lambda x: skew(x))
skew_in_vars.sort_values(ascending=False)
skew_in_vars = skew_in_vars[abs(skewness)>0.5]
skew_in_vars.index
skew_features = df_train[skew_in_vars.index]
skew_features.columns
#check correlation again
# Increase the size of the heatmap.
plt.figure(figsize=(25, 15))
# Store heatmap object in a variable to easily access it when you want to include more features (such as title).
# Set the range of values to be displayed on the colormap from -1 to 1, and set the annotation to True to display the correlation values on the heatmap.
heatmap = sns.heatmap(df_train.corr(), vmin=-1, vmax=1, annot=True,cmap='BrBG')
# Give a title to the heatmap. Pad defines the distance of the title from the top of the heatmap.
heatmap.set_title('Correlation Heatmap', fontdict={'fontsize':6}, pad=14);
# most correlated features

corrmat = df_train.corr()
top_corr_features = corrmat.index[abs(corrmat['SalePrice_Log'])>0.5]
plt.figure(figsize=(16,9))
g = sns.heatmap(df_train[top_corr_features].corr(),annot=True,cmap="RdYlGn")
"""
## Categorical features
"""
#First of all we need to get thecolumns where the categorical features are present
# idea is to extract all the columns with type as object
df_train.dtypes.unique()
for types in df_train.dtypes.unique():
    print(types,len(df_train.select_dtypes(types).columns))
    print(df_train.select_dtypes(types).columns)
"""
* The number of categorical data = 43
* Number of numerical features = 36
"""
numerical_feats = df_train.dtypes[df_train.dtypes != "object"].index
print("Number of Numerical features: ", len(numerical_feats))

categorical_feats = df_train.dtypes[df_train.dtypes == "object"].index
print("Number of Categorical features: ", len(categorical_feats))
"""
## Target variable vs Numerical features
"""
nr_rows = 12
nr_cols = 3

fig, axs = plt.subplots(nr_rows, nr_cols, figsize=(nr_cols*3.5,nr_rows*3))

li_plot_num_feats = [c for c in list(numerical_feats) ]

for r in range(0,nr_rows):
    for c in range(0,nr_cols):  
        i = r*nr_cols+c
        if i < len(li_plot_num_feats):
            sns.regplot(df_train[li_plot_num_feats[i]], df_train['SalePrice_Log'], ax = axs[r][c])
            stp = stats.pearsonr(df_train[numerical_feats[i]], df_train['SalePrice_Log'])
          
            str_title = "r = " + "{0:.2f}".format(stp[0]) + "      " "p = " + "{0:.2f}".format(stp[1])
            axs[r][c].set_title(str_title,fontsize=11)
            
plt.tight_layout()    
plt.show()   

nr_rows = 15
nr_cols = 3

fig, axs = plt.subplots(nr_rows, nr_cols, figsize=(nr_cols*4,nr_rows*3))

for r in range(0,nr_rows):
    for c in range(0,nr_cols):  
        i = r*nr_cols+c
        if i < len(categorical_feats):
            sns.boxplot(x=categorical_feats[i], y=df_train['SalePrice_Log'], data=df_train, ax = axs[r][c])
    
plt.tight_layout()    
plt.show() 
"""
* The categorical variables are corelated to the target variable Saleprice if the mean and the distrubition are different / varying for each features of a particular variable

- Varying mean/distribuition → strong correlation → can keep the features
    * SaleType, PoolQC, 'MSZoning', 'Neighborhood','Condition1','HouseStyle','RoofMati', 'Condition2', 'MasVnrType',   'ExterQual','BsmtQual','CentralAir', 'Electrical', 'KitchenQual', 'SaleType' 
"""
"""
Apart from visual inspection, how to check the correaltion of categorical variables to Target variable ?? 
"""
#https://towardsdatascience.com/the-search-for-categorical-correlation-a1cf7f1888c9

# from scipy.stats import chi2_contingency
# def cramers_v(x, y):
#     confusion_matrix = pd.crosstab(x,y)
#     chi2 = chi2_contingency(confusion_matrix)[0]
#     n = confusion_matrix.sum().sum()
#     phi2 = chi2/n
#     r,k = confusion_matrix.shape
#     phi2corr = max(0, phi2-((k-1)*(r-1))/(n-1))
#     rcorr = r-((r-1)**2)/(n-1)
#     kcorr = k-((k-1)**2)/(n-1)
#     return np.sqrt(phi2corr/min((kcorr-1),(rcorr-1)))
# for cat in categorical_feats:
#     print('The relation with {} is'.format(cat) ,cramers_v(df_train[cat],df_train['SalePrice']))
# Drop the columns that are not strongly correlated to the target variable
# numerical features that are less than a threshold corr value
# categorical feats that are manually selected 
numerical_feats
top_corr_features
numerical_feats_below = [ x for x in numerical_feats if x not in  top_corr_features] # list comphrehension to find the diff
categorical_feats_to_keep = ['SaleType', 'PoolQC', 'MSZoning', 'Neighborhood','Condition1','HouseStyle','RoofMatl', 'Condition2', 'MasVnrType', 'ExterQual','BsmtQual','CentralAir', 'Electrical', 'KitchenQual', 'SaleType']
categorical_feats_to_drop = [ x for x in categorical_feats if x not in  categorical_feats_to_keep]
#item for item in first if item not in second
categorical_feats_to_drop
#Features that we are going to drop

print('Numerical features to drop are:\n',numerical_feats_below)
print('---'*50)
print('Categorical features to drop are:\n', categorical_feats_to_drop)
df_test.drop('Id', inplace= True, axis = 1)    

df_test.columns
# Drop the unwanted columns

for df in [df_train, df_test] :
    df.drop(categorical_feats_to_drop, inplace= True, axis = 1)
    df.drop(numerical_feats_below, inplace= True, axis = 1)    
df_train.columns
# check correlation after dropping

plt.figure(figsize=(25, 15))
# Store heatmap object in a variable to easily access it when you want to include more features (such as title).
# Set the range of values to be displayed on the colormap from -1 to 1, and set the annotation to True to display the correlation values on the heatmap.
heatmap = sns.heatmap(df_train.corr(), vmin=-1, vmax=1, annot=True,cmap='BrBG')
# Give a title to the heatmap. Pad defines the distance of the title from the top of the heatmap.
heatmap.set_title('Correlation Heatmap after drops', fontdict={'fontsize':18}, pad=14);
"""
## Categorical features to dummy 
"""
#convert categorical variable into dummy
df_train = pd.get_dummies(df_train)
df_train.head()
df_train.shape
"""
# Modelling
"""
y_train_data  =df_train['SalePrice_Log']
x_train_data = df_train.loc[:, df_train.columns != 'SalePrice_Log']
from sklearn.model_selection import  train_test_split
#split the data to train the model 
X_train,X_test,y_train,y_test = train_test_split(x_train_data,y_train_data,test_size = 0.3,random_state= 0)
X_train.shape,X_test.shape,y_train.shape,y_test.shape
# Classical Models
from sklearn.linear_model import LinearRegression
from sklearn.model_selection import GridSearchCV
from sklearn.metrics import mean_squared_error, make_scorer
from sklearn.model_selection import KFold, cross_val_score

n_folds = 5

scorer = make_scorer(mean_squared_error,greater_is_better = False)
def rmse_CV_train(model):
    kf = KFold(n_folds,shuffle=True,random_state=42).get_n_splits(x_train_data.values)
    rmse = np.sqrt(-cross_val_score(model,X_train,y_train,scoring ="neg_mean_squared_error",cv=kf))
    return (rmse)
def rmse_CV_test(model):
    kf = KFold(n_folds,shuffle=True,random_state=42).get_n_splits(x_train_data.values)
    rmse = np.sqrt(-cross_val_score(model,X_test,y_test,scoring ="neg_mean_squared_error",cv=kf))
    return (rmse)
lr = LinearRegression()
lr.fit(X_train,y_train)
test_pre = lr.predict(X_test)
train_pre = lr.predict(X_train)
print('rmse on train',rmse_CV_train(lr).mean())
print('rmse on test',rmse_CV_test(lr).mean())
#plot between predicted values and residuals
plt.scatter(train_pre, train_pre - y_train, c = "blue",  label = "Training data")
plt.scatter(test_pre,test_pre - y_test, c = "black",  label = "Validation data")
plt.title("Linear regression")
plt.xlabel("Predicted values")
plt.ylabel("Residuals")
plt.legend(loc = "upper left")
plt.hlines(y = 0, xmin = 10.5, xmax = 13.5, color = "red")
plt.show()

# Plot predictions - Real values
plt.scatter(train_pre, y_train, c = "blue",  label = "Training data")
plt.scatter(test_pre, y_test, c = "black",  label = "Validation data")
plt.title("Linear regression")
plt.xlabel("Predicted values")
plt.ylabel("Real values")
plt.legend(loc = "upper left")
plt.plot([10.5, 13.5], [10.5, 13.5], c = "red")
plt.show()
# Boosting models