"""
<font size="6"><b>House price - Advanced Regression Technique</b></font>

Shout out to these notebooks that helped me a lot in my first notebook on Kaggle:
- <a href="https://www.kaggle.com/pmarcelino/comprehensive-data-exploration-with-python">Comprehensive data exploration with Python - PEDRO MARCELINO</a>
- <a href="https://www.kaggle.com/dgawlik/house-prices-eda">House Prices EDA - DOMINIK GAWLIK</a>
- <a href="https://www.kaggle.com/gcdatkin/top-10-house-price-regression-competition-nb">House Price Regression Competition NB - GABRIEL ATKIN</a>
"""
"""
# 1. Overview

**Data**

The data set is about house price in Ames, Iowa. It contains 79 explainatory variables describing some aspects of residential homes.

**Goal**

The goal is to predict the sale price for each house based on given 79 explainatory variables.

**Metric**

Predictions are evaluated using RMSE between the logarithm of the predicted value and the logarithm of the observed sales price.
"""
# import libraries
# Essentials
import numpy as np
import pandas as pd
import warnings
warnings.filterwarnings('ignore')

# Plot
import matplotlib.pyplot as plt
import seaborn as sns
sns.set_style('darkgrid')

# preprocessing
from sklearn.preprocessing import MinMaxScaler, StandardScaler
from category_encoders import MEstimateEncoder

# models
from sklearn.ensemble import RandomForestRegressor, GradientBoostingRegressor
from xgboost import XGBRegressor
from sklearn.model_selection import KFold, cross_val_score
from sklearn.linear_model import Ridge, RidgeCV
from sklearn.neighbors import KNeighborsRegressor, KNeighborsClassifier

# stats
from scipy.stats import norm
from scipy import stats
# import training data
train_df = pd.read_csv('../input/house-prices-advanced-regression-techniques/train.csv')
# drop id column
train_df = train_df.drop(columns= 'Id')
print(f"Size of training set: {train_df.shape}")
train_df.head()
# import test data
test_df = pd.read_csv('../input/house-prices-advanced-regression-techniques/test.csv')
test_id = test_df['Id']
# drop id column
test_df = test_df.drop(columns='Id')
print(f"Size of test set: {test_df.shape}")
test_df.head()
cols = train_df.columns
quantitatives = train_df.select_dtypes(exclude={'object'}).columns
qualitatives = train_df.select_dtypes(include={'object'}).columns
print(f"Number of quantitative variables: {len(quantitatives)}")
print(quantitatives)
print(f"Number of qualitative variables: {len(qualitatives)}")
print(qualitatives)
train_df.describe()
"""
# 2. Explainatory Data Analysis
"""
"""
## 2.1. Examine Target Variable (Sale Price)
First, we examine the distribution of SalePrice
"""
sns.distplot(train_df['SalePrice'], fit= norm)
plt.title("Sale Price distribution")
plt.xlabel("Sale Price")
plt.ylabel("Frequency")

fig = plt.figure()
res = stats.probplot(train_df['SalePrice'], plot= plt)
# skewness and kurtosis
print(f"Skewness: {train_df['SalePrice'].skew()}")
print(f"Kurtosis: {train_df['SalePrice'].kurt()}")
"""
The SalePrice is positive skewed, and show peakedness.
"""
"""
## 2.2. Numerical Variables

**Heatmap**
"""
corr_matrix = train_df.corr()

fig = plt.figure(figsize=(12, 8))
sns.heatmap(corr_matrix)
plt.title("Correlation heatmap")
top10_corr = corr_matrix.sort_values('SalePrice', ascending= False)[0:10]
top10_corr = top10_corr.loc[:, top10_corr.index]

fig = plt.figure(figsize= (6, 6))
sns.heatmap(top10_corr, annot= True)
plt.title("Zoom in heatmap")
"""
Highly related variables:
- 'TotalBsmtSF' and '1stFlrSF'
- 'GarageCars' and 'GarageArea'
- 'GrLivArea' and 'TotRmsAbvGrd'
- 'YearBuilt' and 'GarageYrBlt'

Variables that correlated with 'SalePrice': 'OverallQual', 'GrLivArea', 'GarageArea', 'TotalBsmtSF', 'YearBuilt'. '1stFlrSF', 'GarageCars', 'TotRmsAbvGrd' are excluded because they are related to one of those 4. 'FullBath' is considered not so important.



**Relation of numerical variables to 'SalePrice'**
"""
n_rows = 12
n_cols = 3

fig, axs = plt.subplots(n_rows, n_cols, figsize= (4*n_cols, 3*n_rows))

for row in range(n_rows):
    for col in range(n_cols):
        i = row * n_cols + col
        if i > len(quantitatives) - 2:
            break
        variable = quantitatives[i]
        sns.scatterplot(x= train_df[variable], y= train_df['SalePrice'], ax= axs[row, col])

plt.tight_layout()
plt.show()
"""
From those charts, highly correlated variables mentioned above have sort of linear relationship with 'SalePrice'. Those charts also imply some exponential relationships, so we can log transform some features to get a better model.

Candidates for log transformation: 'LotFrontage', 'LotArea', 'BsmtFinSF1', 'TotalBsmtSF', '1stFlrSF', '2ndFlrSF', 'GrLivArea', 'GarageArea'.

We can see that some variables are rather categorical variables, and some outliers need to be eliminate.

**Outliers**
"""
# outliers in GrLivArea
train_df = train_df.drop(train_df.loc[(train_df['GrLivArea'] > 4000) & (train_df['SalePrice'] < 200000)].index)
sns.scatterplot(x= train_df['GrLivArea'], y= train_df['SalePrice'])
# outliers in TotalBsmtSF
train_df = train_df.drop(train_df[train_df['TotalBsmtSF'] > 6000].index)
sns.scatterplot(x= train_df['TotalBsmtSF'], y= train_df['SalePrice'])
"""
## 2.3. Categorical Variables

**Relation of categorical variables to 'SalePrice'**
"""
n_rows = 15
n_cols = 3

fig, axs = plt.subplots(n_rows, n_cols, figsize= (4*n_cols, 3*n_rows))

for row in range(n_rows):
    for col in range(n_cols):
        i = row * n_cols + col
        if i >= len(qualitatives):
            break
        variable = qualitatives[i]
        sns.boxplot(x= train_df[variable], y= train_df['SalePrice'], ax= axs[row, col])
        
plt.tight_layout()
plt.show()
"""
From the plots:
- Variables having good disparity with respect to 'SalePrice' are: ‘MSZoning’, ’Neighborhood’, ‘Condition1’, ‘Condition2’, ‘RoofMatl’, ‘MatVnrType’, ’ExterQual’, ‘BsmtQual’, ‘BsmtCond’, ‘CentralAir’, ’KitchenQual’, ‘SaleType’, ‘SaleCondition’.
- House that have excellent Pool Quality tends to have higher sale price.
- Partial sale condition tends to have higher sale price.

# 3. Data Preprocessing
## 3.1. Missing value



"""
train_missing_value = train_df.isnull().sum().sort_values(ascending= False)
train_missing_value = train_missing_value[train_missing_value > 0]
train_missing_value
test_missing_value = test_df.isnull().sum().sort_values(ascending= False)
test_missing_value = test_missing_value[test_missing_value > 0]
test_missing_value
# cut saleprice
sale_price = train_df['SalePrice']
train_df = train_df.drop(columns={'SalePrice'})
# combine test and training data
all_data = pd.concat([train_df, test_df], ignore_index= True)
all_data.shape
def knn_regressor_fill(col, df):
    df = df.copy()
    numeric_non_na_cols = df.select_dtypes(exclude= {'object'}).loc[:, df.isnull().sum() == 0].columns

    train = df.loc[df[col].isna() == False][numeric_non_na_cols]
    labels = df.loc[df[col].isna() == False][col]
    test = df.loc[df[col].isna() == True][numeric_non_na_cols]

    knn = KNeighborsRegressor()
    knn.fit(train, labels)
    preds = knn.predict(test)
    
    df.loc[df[col].isna() == True, col] = preds
    return df[col]
def knn_clf_fill(col, df):
    df = df.copy()
    numeric_non_na_cols = df.select_dtypes(exclude= {'object'}).loc[:, df.isnull().sum() == 0].columns

    train = df.loc[df[col].isna() == False][numeric_non_na_cols]
    labels = df.loc[df[col].isna() == False][col]
    test = df.loc[df[col].isna() == True][numeric_non_na_cols]

    knn = KNeighborsClassifier()
    knn.fit(train, labels)
    preds = knn.predict(test)
    
    df.loc[df[col].isna() == True, col] = preds
    return df[col]
# Missing Categorical variables can be consider as None or NA
fill_na = ['PoolQC', 'MiscFeature', 'Alley', 'Fence', 'FireplaceQu', 'GarageCond', 'GarageType',
          'GarageFinish', 'GarageQual', 'BsmtExposure', 'BsmtFinType2', 'BsmtCond', 'BsmtQual', 'BsmtFinType1']
fill_cat = ['Electrical', 'MSZoning', 'Functional', 'Utilities', 'KitchenQual', 'SaleType', 'Exterior2nd', 'Exterior1st', 'GarageCars']
fill_num = ['LotFrontage', 'BsmtFinSF1', 'BsmtFinSF2', 'GarageArea', 'TotalBsmtSF', 'GarageYrBlt']
fill_0 = ['MasVnrArea', 'BsmtHalfBath', 'BsmtFullBath', 'BsmtUnfSF']

for col in fill_na:
    all_data[col].fillna('NA', inplace= True)

for col in fill_cat:
    #val = train_df[col].mode()[0] 
    #all_data[col].fillna(val, inplace= True)
    # fill with knn maybe ??
    all_data[col] = knn_clf_fill(col, all_data)

for col in fill_num:
    #val =train_df[col].mean()
    #all_data[col].fillna(val, inplace= True)
    # fill with knn maybe ??
    all_data[col] = knn_regressor_fill(col, all_data)
    
for col in fill_0:
    all_data[col].fillna(0, inplace= True)
    
all_data['MasVnrType'].fillna('None', inplace= True)

print(f"Number of missing data: {all_data.isnull().sum().sum()}")
"""
## 3.2. Features Engineering

"""
"""
Add more features
"""
# creating features
all_data['TotBathRms'] = all_data['FullBath'] + all_data['BsmtFullBath'] + 0.5*(all_data['HalfBath'] + all_data['BsmtHalfBath'])
all_data['TotOutsideSF'] = all_data['OpenPorchSF'] + all_data['EnclosedPorch'] + all_data['3SsnPorch'] + \
                            all_data['ScreenPorch'] + all_data['WoodDeckSF']
all_data["LivLotRatio"] = all_data['GrLivArea'] / all_data['LotArea']
all_data["Spaciousness"] = (all_data['1stFlrSF'] + all_data['2ndFlrSF']) / all_data['TotRmsAbvGrd']
all_data['hasPool'] = all_data['PoolArea'].apply(lambda x: 1 if x > 0 else x)
all_data['hasGarage'] = all_data['GarageArea'].apply(lambda x: 1 if x > 0 else x)
all_data['hasBsmt'] = all_data['TotalBsmtSF'].apply(lambda x: 1 if x > 0 else x) 
"""
Target Encoding
"""
target_encoding_cols = train_df.select_dtypes(include= {'object'}).nunique()
target_encoding_cols = target_encoding_cols[target_encoding_cols > 10].index
target_encoding_cols
encoder = MEstimateEncoder(cols= target_encoding_cols.values.tolist(), m= 2)
encoder.fit(all_data[:len(train_df)], sale_price)
all_data = encoder.transform(all_data)
"""
## 3.3. Data Transformation
First, we fix skewness in 'SalePrice'
"""
sale_price = np.log1p(sale_price)

# plot the SalePrice again to check normality
sns.distplot(sale_price, fit= norm)
plt.title("Sale Price distribution")
plt.xlabel("Sale Price")
plt.ylabel("Frequency")

fig = plt.figure()
res = stats.probplot(sale_price, plot= plt)
"""
Now, we log transform skewed numerical variables.
"""
skewness_df = pd.DataFrame(train_df.skew(), columns=['Skewness'], \
                           index= train_df.select_dtypes(exclude=['object']).columns)
skewness_df['Skewed'] = skewness_df['Skewness'].apply(lambda x: True if abs(x) > 0.5 else False)
skewness_df
log_transform_cols = skewness_df[skewness_df['Skewed'] == True].index.to_list()
for col in log_transform_cols:
    all_data.loc[all_data[col] > 0][col+'_log'] = np.log1p(all_data.loc[all_data[col] > 0][col])
"""
## 3.4. Encode Categorical Variables
"""
all_data = pd.get_dummies(all_data)
all_data.shape
"""
# 4. Train Model
## 4.1. Split Data

"""
X_train = all_data[:len(train_df)]
y_train = sale_price
X_test = all_data[len(train_df):]

print(f"Number of training observations: {X_train.shape[0]}")
"""
## 4.2. Set up model and metric
"""
kf = KFold(n_splits= 5, random_state= 7, shuffle= True)
rf_params = {
    'n_estimators': 1000,
    'max_depth' : 10,
    'random_state': 5
}

GBR_params = {
    'learning_rate': 0.0511,
    'n_estimators': 5000,
    'max_depth' : 5,
    'n_iter_no_change' : 5,
    'random_state' : 5
}

xgboost_params = {
    'learning_rate' : 0.115,
    'max_depth' : 6,
    'n_estimators' : 500,
    'random_state' : 5,
    'subsample' : 0.8,
    'gamma' : 0.05,
    'random_state' : 5
}

ridge_params = {
    'alphas' : np.array([1, 1.511, 0.95, 0.93, 0.01, 0.05]),
    'cv' : kf
}
models = {
    "Ridge" : RidgeCV(**ridge_params),
    "RandomForest": RandomForestRegressor(**rf_params),
    "GradientBoosting": GradientBoostingRegressor(**GBR_params),
    "XGBoost": XGBRegressor(**xgboost_params)
}
"""
## 4.3. Train Model and Evaluate
"""
for name, model in models.items():
    model.fit(X_train, y_train)
    print(name + " trained")
for name, model in models.items():
    scores = -cross_val_score(model, X_train, y_train, scoring='neg_root_mean_squared_error', n_jobs= -1, cv=kf)
    print(f"{name}: {scores} -- Average: {scores.mean()}")

"""
## 4.4. Prediction and Submission
"""
final_preds = 0.1 * np.expm1(models['RandomForest'].predict(X_test)) \
            + 0.2 * np.expm1(models['GradientBoosting'].predict(X_test)) \
            + 0.35 * np.expm1(models['XGBoost'].predict(X_test)) \
            + 0.35 * np.expm1(models['Ridge'].predict(X_test))

final_preds
submission = pd.concat([test_id, pd.Series(final_preds, name='SalePrice')], axis=1)
submission
submission.to_csv('./submission.csv', index=False, header=True)