"""
# Advanced house price prediction
## The goal is to predict the Sales Price for the dataset with the given features.
"""
"""
 # Importing libraries
"""
# This Python 3 environment comes with many helpful analytics libraries installed
# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python
# For example, here's several helpful packages to load

import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
import missingno as msno
from scipy import stats
from scipy.stats import norm

# Input data files are available in the read-only "../input/" directory
# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory

import os
for dirname, _, filenames in os.walk('/kaggle/input'):
    for filename in filenames:
        print(os.path.join(dirname, filename))

# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using "Save & Run All" 
# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session
"""
### Read data
"""
# Reading the train and test data
train_data = pd.read_csv('/kaggle/input/house-prices-advanced-regression-techniques/train.csv')
test_data = pd.read_csv('/kaggle/input/house-prices-advanced-regression-techniques/test.csv')
#checking out the first lines of my DataFrame with the help of the head() function
train_data.head(n=5)
test_data.head(n=5)
"""
# Explore Data
"""
#looking number of rows and columns in the dataset
print('Rows and Columns of train data: ',train_data.shape)
print('Rows and Columns of test data: ',test_data.shape)
# 1460 rows and 81 columns in the data
"""
# Performing data analysis on train data set
"""
# Get the overall concise summary of the DataFrame
# Finding non null values and datatype of each column in train data set
train_data.info()
# Finding number of missing values in each column of train data set
traindata_columns=[]
traindata_columns = train_data.columns.values
for column in traindata_columns:
    if(train_data[column].isnull().sum()>0):
        print(column,train_data[column].isnull().sum())
## Here i am checking the percentage of missing values present in each feature

features_with_na=[features for features in train_data.columns if train_data[features].isnull().sum()>0]
#step print the feature name and the percentage of missing values
for feature in features_with_na:
    print(feature, np.round(train_data[feature].isnull().mean(), 4),  ' % missing values.\n')
# Checking numerical features
numerical_features = [feature for feature in train_data.columns if train_data[feature].dtype != 'O']
print("Number of numerical features: ", len(numerical_features))
# finding the number of temporal features
temporal_features = [feature for feature in numerical_features if 'Year' in feature or 'Yr' in feature]
print("Number of temporal features: ", len(temporal_features))
#Discrete features
discrete_features = [feature for feature in numerical_features if len(train_data[feature].unique()) <=25 
                     and feature not in temporal_features + ['Id']]
print("Length of discrete features: ", len(discrete_features))
#Continous features
continuous_features = [feature for feature in numerical_features if feature not in discrete_features + temporal_features + ['Id']]
print("Length of continuous features: ", len(continuous_features))
# Finding the length of categorical features
categorial_features = [feature for feature in train_data.columns if train_data[feature].dtypes == 'O']
print(categorial_features)
print("Length of categorical features: ", len(categorial_features))
#checking percentage of missing values in categorial data
categorial_with_nan = [feature for feature in categorial_features if train_data[feature].isnull().sum() > 0]
print(categorial_with_nan)
for feature in categorial_with_nan:
    print("Feature {}, has {}% missing values in train dataset", (feature, np.round(train_data[feature].isnull().mean(), 4)))
"""
# Handling the missing values
"""
for feature in categorial_with_nan:
    train_data[feature].fillna('Missing', inplace=True)
print(numerical_features)
# Replace the missing values in train set with median
numerical_with_nan = [feature for feature in numerical_features if train_data[feature].isnull().sum() > 0]
for feature in numerical_with_nan:
      train_data[feature].fillna(train_data[feature].median(), inplace=True)
train_data.shape
# Lets see the relation between Fullbath and SalesPrice.
fig, ax = plt.subplots()
ax.scatter(x = train_data['FullBath'], y = train_data['SalePrice'])
plt.ylabel('SalePrice', fontsize=15)
plt.xlabel('FullBath', fontsize=15)
plt.show()

"""
#### It is observed that Sales price and fullbath is negatively correlated
"""
"""
#### Temporal Variables (Date-time variables)
In the train dataset we have 4 temporal variables
"""
train_data[temporal_features].head()
"""
### Lets see the relation between temporal variables and SalesPrice.
"""
for feature in temporal_features:
    data = train_data.copy()
    
    train_data.groupby(feature)['SalePrice'].median().plot()
    plt.xlabel(feature)
    plt.ylabel('SalePrice')
    plt.title(feature)
    plt.show()
"""
The first 3 plots here look fine as the recent the year house is built/remodling done/garage build, the higher the SalesPrice. But in the 4th plot Sales Price is decreasing as the Year is increasing. Ideally SalesPrice should increase with every passing year.

So lets see the relation between the first 3 year variables and the Year Sold
"""
for feature in temporal_features:
    data = train_data.copy()
    
    if feature != 'YrSold':
        data[feature] = data['YrSold'] - data[feature]
        plt.scatter(data[feature], data['SalePrice'])
        plt.xlabel(feature)
        plt.ylabel('SalePrice')
        plt.title(feature)
        plt.show()
"""
So above scatter plot indicates:

The lesser the difference between house YrSold and house year built/remodling done/garagebuilt, the higher the Sales Price. When Sales Price is less then it means the house is old with no/not recent alterations done. So now we also know that "Houses where faeture values are missing have comparatively low price", because no remodelling or feature enhancements are done recently.
"""
# finding variables which are highly correlated (both positive and negative) with “SalePrice”.
corrmat = train_data.corr()

def getCorrelatedFeature(corrdata, threshold):
    feature = []
    value = []
    for i , index in enumerate(corrdata.index):
        if abs(corrdata[index]) > threshold:
            feature.append(index)
            value.append(corrdata[index])
    df2 = pd.DataFrame(data = value, index=feature, columns=['corr value'] )
    return df2

corr_df = getCorrelatedFeature(corrmat['SalePrice'], 0.5)
corr_df
# Below features are highly correlated with SalePrice
colormap = plt.cm.Blues
correlated_data = train_data[corr_df.index]
fig, ax = plt.subplots(figsize=(11, 11))
sns.heatmap(correlated_data.corr(), annot = True, annot_kws={'size': 12}, square=True,cmap=colormap, linecolor='w', linewidths=0.1)
"""
Now, let’s have a look at the features that are highly correlated with the sale price
"""
# The Living area and Sale Price have roughly a linear relationship
fig, ax = plt.subplots()
ax.scatter(x = train_data['GrLivArea'], y = train_data['SalePrice'])
plt.ylabel('SalePrice', fontsize=15)
plt.xlabel('GrLivArea', fontsize=15)
plt.show()
#Positive correlation between overallqual and salesprice
sns.regplot(x='OverallQual', y='SalePrice', data=train_data, ci=None, scatter = False)
sns.boxplot(y='SalePrice', x = 'GarageArea', data=train_data)
# The below plot clearly shows a linear relationship between ‘SalePrice’ and ‘GarageArea’. The ‘SalePrice’ increases with an increase in ‘GarageArea’.
# plot of ‘YearBuilt’ shows that the distribution is skewed towards the year 2000 and has a long tail which extends till 1900. The linear relationship between the variables is clearer in cases of recently built houses.
sns.jointplot(x='SalePrice', y='YearBuilt', data=train_data, kind='reg', dropna = True)
# TotalBsmtSF is very highly correlated with our target variable SalePrice and
sns.jointplot(x='SalePrice', y='TotalBsmtSF', data=train_data, kind='scatter')
sns.relplot(x='SalePrice', y='YearRemodAdd', data=train_data)
# the YearRemodAdd also has a linear relationship with SalePrice.
# It is highly negatively correlated with our target variable #Relational plot
sns.relplot(x='SalePrice', y='FullBath', data=train_data)
"""
'OverallQual', 'GrLivArea' and 'TotalBsmtSF' have strong correlation with 'SalePrice'
"""
"""
# Analysing Test data set
"""
# Finding number of missing values for test data in each column
test_data_columns=[]
test_data_columns = test_data.columns.values
for column in test_data_columns:
    if(test_data[column].isnull().sum()>0):
        print(column,test_data[column].isnull().sum())
"""
* Filling the missed values in test data set
"""
categorial_with_nan = [feature for feature in categorial_features if test_data[feature].isnull().sum() > 0]
for feature in categorial_with_nan:
    test_data[feature].fillna('Missing', inplace=True)
numerical_with_nan = [feature for feature in numerical_features if feature not in ['SalePrice'] and test_data[feature].isnull().sum() > 0]
for feature in numerical_with_nan:
#     test[feature+'NaN'] = np.where(test[feature].isnull(), 1, 0)
      test_data[feature].fillna(test_data[feature].median(), inplace=True)
test_data.head()
print("Train Dataset", train_data.shape)
print("Test Dataset", test_data.shape)
test_data[temporal_features].head()
"""
* Since the numeric variables are skewed, we will perform log normal distribution.
"""
num_non_zero_skewed_features_train_set = ['LotFrontage', 'LotArea', '1stFlrSF', 'GrLivArea', 'SalePrice']
train_data[num_non_zero_skewed_features_train_set].head()
for feature in num_non_zero_skewed_features_train_set:
    train_data[feature] = np.log(train_data[feature])
"""
We may assume the same numeric features will be skewed in test set as well.
"""
num_non_zero_skewed_features_test_set = ['LotFrontage', 'LotArea', '1stFlrSF', 'GrLivArea']
test_data[num_non_zero_skewed_features_test_set].head()
for feature in num_non_zero_skewed_features_test_set:
    test_data[feature] = np.log(test_data[feature])
train_data[num_non_zero_skewed_features_train_set].head()
test_data[num_non_zero_skewed_features_test_set].head()
"""
### Creating dummy data
"""
train1 = train_data.copy()
test1 = test_data.copy()
data = pd.concat([train1,test1], axis=0)
train_rows = train1.shape[0]
print(data.get('MSZoning'))
for feature in categorial_features:
    dummy = pd.get_dummies(data[feature])
    for col_name in dummy.columns:
        dummy.rename(columns={col_name: feature+"_"+col_name}, inplace=True)
    data = pd.concat([data, dummy], axis = 1)
    data.drop([feature], axis = 1, inplace=True)
train1 = data.iloc[:train_rows, :]
test1 = data.iloc[train_rows:, :] 
train1.head()
test1.head()
print("Train",train1.shape)
print("Test",test1.shape)
"""
# Scaling
* Min-Max scaler is used to normalize the input features/variables
* now i am scaling data by creating an instance of the scaler and scaling it:
"""
from sklearn.preprocessing import MinMaxScaler

scaling_features = [feature for feature in train1.columns if feature not in ['Id', 'SalePrice']]
scaling_features
train1[scaling_features].head()
print(len(scaling_features))
train1[scaling_features].head()
scaler = MinMaxScaler()
scaler.fit(train1[scaling_features])
X_train = scaler.transform(train1[scaling_features])
X_test = scaler.transform(test1[scaling_features])
print("Train", X_train.shape)
print("Test", X_test.shape)
y_train = train1['SalePrice']
X = pd.concat([train1[['Id','SalePrice']].reset_index(drop=True), pd.DataFrame(X_train, columns = scaling_features)], axis =1)
print(X.shape)
X.head()
"""
# Training data
"""
from sklearn.linear_model import ElasticNet, Lasso
from sklearn.ensemble import RandomForestRegressor, GradientBoostingRegressor
from sklearn.kernel_ridge import KernelRidge
from sklearn.pipeline import make_pipeline
from sklearn.preprocessing import RobustScaler
# from sklearn.base import BaseEstimator, TransformerMixin, RegressorMixin, clone
from sklearn.model_selection import KFold, cross_val_score
from sklearn.metrics import mean_squared_error
from xgboost import XGBRegressor
from lightgbm import LGBMRegressor
from sklearn.svm import SVR
# Training models using k-fold cross-validation and seeing the performance(RMSE) of all the models in the given dataset.
# The below function rmse_cv is used to train models of the data created and it returns the RMSE score for the model based on the predictions compared with the actual predictions.

n_folds = 10
def rmsle_cv(model):
    kf = KFold(n_folds, shuffle=True, random_state=42).get_n_splits(X_train)
    rmse= np.sqrt(-cross_val_score(model, X_train, y_train, scoring="neg_mean_squared_error", cv = kf))
    return(rmse)

def rmsle(y_train, y_pred):
    return np.sqrt(mean_squared_error(y_train, y_pred))
"""
* Here we are "regularizing" models ability to structurally prevent overfitting by imposing a penalty on the coefficients.
* Below models with the optimal hyperparameters were evaluated by comparing the predictions of each model with validation data. Each model was evaluated using the root mean square error (RMSE) of model predictions, which is a metric for describing the differences between the predicted values and the observed values for SalePrice. lower RMSE scores are better.
"""
lasso = Lasso(alpha =0.0005, random_state=0)
elasticNet = ElasticNet(alpha=0.0005, l1_ratio=.9, random_state=0)
kernelRidge = KernelRidge(alpha=0.6, kernel='polynomial', degree=2, coef0=2.5)
svr = SVR(C= 20, epsilon= 0.008, gamma=0.0003)
gradientBoosting = GradientBoostingRegressor(n_estimators=3000, learning_rate=0.05,
                                   max_depth=4, max_features='sqrt',
                                   min_samples_leaf=15, min_samples_split=10, 
                                   loss='huber', random_state =0)
xgb = XGBRegressor(colsample_bytree=0.4603, gamma=0.0468, 
                             learning_rate=0.05, max_depth=3, 
                             min_child_weight=1.7817, n_estimators=2200,
                             reg_alpha=0.4640, reg_lambda=0.8571,
                             subsample=0.5213, silent=1,
                             random_state =0, nthread = -1)
lgbm = LGBMRegressor(objective='regression',num_leaves=5,
                              learning_rate=0.05, n_estimators=720,
                              max_bin = 55, bagging_fraction = 0.8,
                              bagging_freq = 5, feature_fraction = 0.2319,
                              feature_fraction_seed=9, bagging_seed=9,
                              min_data_in_leaf =6, min_sum_hessian_in_leaf = 11, random_state=0)
randomForest = RandomForestRegressor(n_estimators=1200,
                          max_depth=15,
                          min_samples_split=5,
                          min_samples_leaf=5,
                          max_features=None,
                          oob_score=True,
                          random_state=0)
scores ={}
# Using Lasso to add the penalty equivalent to the absolute value of the sum of coefficients. This penalty is added to the 
# least square loss function and replaces the squared sum of coefficients from Ridge
score = rmsle_cv(lasso)
print("Lasso:: Mean:",score.mean(), " Std:", score.std())
scores['lasso'] = (score.mean(), score.std())
lasso_model = lasso.fit(X_train, y_train)
y_pred_lasso = lasso_model.predict(X_train)
rmsle(y_train,y_pred_lasso)
# Elastic Net is the combination of both Ridge and Lasso. It adds both the sum of squared coefficients 
# and the absolute sum of the coefficients with the ordinary least square function
score = rmsle_cv(elasticNet)
print("ElasticNet:: Mean:",score.mean(), " Std:", score.std())
scores['elasticNet'] = (score.mean(), score.std())
elasticNet_model = elasticNet.fit(X_train, y_train)
y_pred_elasticNet = elasticNet_model.predict(X_train)
rmsle(y_train,y_pred_elasticNet)
# Kernel ridge regression is a non-parametric form of ridge regression. The aim is to learn a function in the space induced by the respective kernel 𝑘 
# by minimizing a squared loss with a squared norm regularization term.
score = rmsle_cv(kernelRidge)
print("KernelRidge:: Mean:",score.mean(), " Std:", score.std())
scores['kernelRidge'] = (score.mean(), score.std())
kernelRidge_model = kernelRidge.fit(X_train, y_train)
y_pred_kernelRidge = kernelRidge_model.predict(X_train)
rmsle(y_train,y_pred_kernelRidge)
score = rmsle_cv(svr)
print("SVR:: Mean:",score.mean(), " Std:", score.std())
scores['svr'] = (score.mean(), score.std())
svr_model = svr.fit(X_train, y_train)
y_pred_svr = svr_model.predict(X_train)
rmsle(y_train,y_pred_svr)
"""
* Using one of the machine learning technique called Gradient boosting algorithm to minimize the loss of the model
"""
score = rmsle_cv(gradientBoosting)
print("GradientBoostingRegressor:: Mean:",score.mean(), " Std:", score.std())
scores['gradientBoosting'] = (score.mean(), score.std())
gradientBoosting_model = gradientBoosting.fit(X_train, y_train)
y_pred_gradientBoosting = gradientBoosting_model.predict(X_train)
rmsle(y_train,y_pred_gradientBoosting)
"""
* XGBRegressor is used to fine tune and retrieve optimal parameters to reduce rmse
"""
score = rmsle_cv(xgb)
print("XGBRegressor:: Mean:",score.mean(), " Std:", score.std())
scores['xgb'] = (score.mean(), score.std())
xgb_model = xgb.fit(X_train, y_train)
y_pred_xgb = xgb_model.predict(X_train)
rmsle(y_train,y_pred_xgb)
score = rmsle_cv(lgbm)
print("LGBMRegressor:: Mean:",score.mean(), " Std:", score.std())
scores['lgbm'] = (score.mean(), score.std())
lgbm_model = lgbm.fit(X_train, y_train)
y_pred_lgbm = lgbm_model.predict(X_train)
rmsle(y_train,y_pred_lgbm)
"""
* Here Iam using Random Forest regression since the target variable is a continuous real number and also it combines the prediction of multiple decision trees to get more accurate final prediction.
"""
score = rmsle_cv(randomForest)
print("RandomForestRegressor:: Mean:",score.mean(), " Std:", score.std())
scores['randomForest'] = (score.mean(), score.std())
randomForest_model = randomForest.fit(X_train, y_train)
y_pred_randomForest = randomForest_model.predict(X_train)
rmsle(y_train,y_pred_randomForest)
"""
# Stack Models
Ensemble model: Here i am combining multiple models using ensemble model,to boost overall accuracy. 
The combination can be implemented by aggregating the output from each model with two objectives: reducing the model error and maintaining its generalization.
"""
def ensemble_models(X):
    return ((0.1 * lasso_model.predict(X)) +
            (0.1 * elasticNet_model.predict(X)) +
           (0.1 * kernelRidge_model.predict(X)) +
           (0.4 * gradientBoosting_model.predict(X)) + 
           (0.1 * xgb_model.predict(X)) +
           (0.2 * lgbm_model.predict(X)))
averaged_score = rmsle(y_train, ensemble_models(X_train))
scores['averaged'] = (averaged_score, 0)
print('RMSLE score on train data:', averaged_score)
"""
# Visualize model scores
* Plotting the predictions for each model
"""
sns.set_style("white")
fig = plt.figure(figsize=(20, 10))

ax = sns.pointplot(x=list(scores.keys()), y=[score for score, _ in scores.values()], markers=['o'], linestyles=['-'])
for i, score in enumerate(scores.values()):
    ax.text(i, score[0] + 0.002, '{:.6f}'.format(score[0]), horizontalalignment='left', size='large', color='black', weight='semibold')

plt.ylabel('Score (RMSE)', size=20, labelpad=12.5)
plt.xlabel('Model', size=20, labelpad=12.5)
plt.tick_params(axis='x', labelsize=13.5)
plt.tick_params(axis='y', labelsize=12.5)
plt.title('Scores of Models', size=20)
plt.show()

submission = pd.read_csv('/kaggle/input/house-prices-advanced-regression-techniques/sample_submission.csv')
submission.shape
"""
* We have fitted the model and seen its performance. Lets predict the prices for the houses in the actual test data.
"""
test_predict = np.exp(ensemble_models(X_test))
print(test_predict[:5])
"""
# Prediction submission
"""
sub = pd.DataFrame()
sub['Id'] = test_data['Id']
sub['SalePrice'] = test_predict
sub.to_csv('submission.csv',index=False)
sub1 = pd.read_csv('submission.csv')
sub1.head()
"""
Reference: https://www.kaggle.com/charumakhijani/house-price-prediction-top-6-on-leaderboard
"""