"""
<link rel="preconnect" href="https://fonts.gstatic.com">
<link href="https://fonts.googleapis.com/css2?family=Hachi+Maru+Pop&display=swap" rel="stylesheet">
<link href="https://raw.githubusercontent.com/twilson63/istyle/master/amblin.css" rel="stylesheet">
<div style="width:100%;text-align:center;font-family: 'Hachi Maru Pop', cursive;">
    <h1>House Prices: Advanced Regression Techniques</h1>
</div>
<br><br>

Main objectives are;

1. Frame the problem and look at the big picture.
2. Get the data.
3. Explore the data to gain insights.
4. Prepare the data to better expose the underlying data patterns to Machine Learning algorithms.
5. Explore many different models and shortlist the best ones.
6. Fine-tune your models and combine them into a great solution.
7. Present your solution.

# Frame the Problem

*  **Define the objective.**

The goal is to accurately predict house prices. You will pair your machine learning skills using [Ames Housing dataset](http://www.amstat.org/publications/jse/v19n3/decock.pdf).


* **What are the current solutions/workarounds (if any)?**

The current solution can be seen in [this](https://www.kaggle.com/c/house-prices-advanced-regression-techniques/notebooks).


* **How should you frame this problem (supervised/unsupervised, online/offline, etc.)?**

Supervised, offline (batch), model-based learning.


* **How should performance be measured?**

Submissions are evaluated on [Root-Mean-Squared-Error (RMSE)](https://en.wikipedia.org/wiki/Root-mean-square_deviation).


* **What would be the minimum performance needed to reach the business objective?**

There is no minimum performance to reach the business objective, however, we can compare our results with [leaderboard](https://www.kaggle.com/c/house-prices-advanced-regression-techniques/leaderboard).


* **What are comparable problems? Can you reuse experience or tools?**

Since this is my second competition contribution in Kaggle, I don't have any script to use. However, I can use some techniques from [my first notebook](https://www.kaggle.com/onurserbetci/end-to-end-titanic-project).

# Import Relevant Libraries

Now, we can import necessary libraries.

"""
# Main
import pandas as pd
import numpy as np

# EDA and Data Vizualization
import matplotlib.pyplot as plt
import seaborn as sns
import plotly.express as px
from plotly.subplots import make_subplots
import plotly.graph_objects as go
from plotly.offline import init_notebook_mode, iplot, plot
init_notebook_mode(connected=True)
import plotly.figure_factory as ff

# Feature Engineering
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import OneHotEncoder, StandardScaler, scale, RobustScaler, robust_scale, OrdinalEncoder
from scipy.stats import skewtest, skew, shapiro, boxcox
from sklearn.decomposition import PCA

# Machine Learning
from IPython.display import display, Markdown
from sklearn.svm import LinearSVR, SVR
from sklearn.linear_model import LinearRegression, Ridge, Lasso, ElasticNet, SGDRegressor, RidgeCV, LassoCV, ElasticNetCV
from sklearn.metrics import r2_score, mean_squared_error
from sklearn.tree import DecisionTreeRegressor
from sklearn.ensemble import AdaBoostRegressor, RandomForestRegressor, GradientBoostingRegressor, BaggingRegressor
from sklearn.base import clone
from tqdm import tqdm

print('Imported Successfully!')
"""
# Get the Data

We'll use `pandas` package for reading `.csv` file.
"""
train = pd.read_csv('../input/house-prices-advanced-regression-techniques/train.csv')
submission = pd.read_csv('../input/house-prices-advanced-regression-techniques/test.csv')
df = pd.concat([train,submission]).set_index('Id')
print(f'df shape: {df.shape}\ntrain shape: {train.shape}\ntest shape: {submission.shape}')
"""
We can check how much memory each variables are using in megabytes:
"""
print(f'train.csv: \n{(df.memory_usage(deep=True)/(10**6)).sort_values(ascending=False)}\n\ntest.csv: \n{(submission.memory_usage(deep=True)/(10**6)).sort_values(ascending=False)}')
"""
As we have the data now, we should create test dataset in order to avoid from data snooping.
"""
eda,_=train_test_split(df, test_size=.2, random_state=42)
eda.shape
"""
# Explore the data (EDA)

We should get information about each attribute. Decide whether it is useful or not. There is lot of features, we should try to automate processes.
"""
# Creat price categories
labels=['Lower','Medium','Higher']
eda.loc[:,'price_cat'] = pd.qcut(eda.loc[:,'SalePrice'], 3, labels=labels)

# Aplly log transformation to target for getting normalized results
eda['SalePrice_log'] = np.log(eda['SalePrice'])

# EDA
def _inspect_cat_col(col):
    _print_missing_values(col)
    print(f'Count of each category for {col}:\n')
    print(eda[col].value_counts())
    
def _print_missing_values(col):
    print(f'Number of missing values: {eda[col].isnull().sum()}, Percentage: {eda[col].isnull().sum()/len(eda[col].isnull())}\n')

def _draw_pie_chart(col):
    pd.set_option('mode.chained_assignment', None)
    labels=['Lower','Medium','Higher']
    

    fig = make_subplots(rows=1, cols=3, specs=[[{'type':'domain'}, {'type':'domain'},{'type':'domain'}]], subplot_titles = labels)
    for i,lb in enumerate(labels):
        eda_price_labeled = eda[eda.price_cat==lb]
        eda_price_counted = pd.DataFrame(eda_price_labeled.groupby(col)[col].count()).rename(columns={
            col:'Count'}).reset_index()
        fig.add_trace(go.Pie(values=eda_price_counted.Count, labels=eda_price_counted[col], name=lb),1,i+1)
    fig.update_layout(title_text=f'{col} class distribution for each sale price bins')
    iplot(fig)
    
def _draw_violin_chart(col,price_log=False):
    price_column_name = 'SalePrice_log' if price_log else 'SalePrice'
    fig = go.Figure()

    for cat in eda[col].unique():
        fig.add_trace(go.Violin(x=eda[col][eda[col] == cat],
                                y=eda[price_column_name][eda[col] == cat],
                                name=str(cat),
                                box_visible=True,
                                meanline_visible=True))

    iplot(fig)
    
def _draw_pairplot(col,log_price=False,log_col=False, reciprocal_col=False):
    price_column_name = 'SalePrice_log' if log_price else 'SalePrice'
    if log_col:
        log_column_name = col + '_log'
        eda[log_column_name] = np.log(eda[col])
        col = log_column_name
    
    ax = sns.pairplot(eda.loc[:,[col,price_column_name]], height=4, aspect=1.5, kind='reg')
    ax.fig.suptitle(f'{col} VS {price_column_name}', y=1.05)
    _print_missing_values(col)
    print(f"Correlation: {eda.loc[:,[col,price_column_name]].corr().iloc[0,1]}")
    plt.show()
    
def _create_distplot_log(eda,col):
    fig = ff.create_distplot([eda[col]],[col])
    fig2 = ff.create_distplot([np.log(eda[col])],[col+'_log'])
    iplot(fig)
    print('After log tranformation')
    iplot(fig2)

def automated_eda_report(eda):
    for ind, current_col in enumerate(list(eda)):
        if current_col in ['SalePrice','SalePrice_log','price_cat']:
            continue
        print(f'Current column is {current_col}')
        print(f'Type of current columns is {label_map[current_col]}')
        if label_map[current_col] == 'Categorical':
            _inspect_cat_col(current_col)
            _draw_pie_chart(current_col)
            _draw_violin_chart(current_col)
        elif label_map[current_col] == 'Ordinal Numerical' or label_map[current_col] == 'Ordinal Categorical':
            print(f'Desriptive Stats Results for {current_col}:\n')
            print(eda[current_col].describe())
            _draw_pairplot(current_col,log_price=True)
            
# Feature Engineering

def _print_done(col, trans):
    print(f'PERFECTLY DONE! --> {trans} applied on {col}!')
    
def _log_transformation(df_selected, col):
    df_selected[col+'_trnsfrm'] = np.where(df_selected[col]!=0,
                                           np.log(df_selected[col]), 0)
    return df_selected

def _scale_log_transformation(df_selected, col):
    df_selected[col+'_trnsfrm'] = scale(np.where(df_selected[col]!=0, 
                                                 np.log(df_selected[col]), 0))
    return df_selected

def _scale_transformation(df_selected, col):
    df_selected[col+'_trnsfrm'] = scale(df_selected[col])
    return df_selected
    
def _boxcox_transformation(df_selected, col):
    if np.any(df_selected[col]==0):
        data=df_selected[col]
        posdata = data[data > 0]
        bcdata, lam = boxcox(posdata)
        if lam <= 0:
            x = df_selected[col]
        else:
            x = np.empty_like(data)
            x[data > 0] = bcdata
            x[data == 0] = -1/lam
    else:
        x,_ = boxcox(df_selected[col])
    df_selected[col+'_trnsfrm'] = x
    return df_selected

def _get_transformation_score(df, transformation_fuction, col):
    df_trns=df.copy()
    transformation_fuction(df_trns, col)
    return shapiro(df_trns[col+'_trnsfrm'])[0]
    
def _apply_best_transformation(df,col):
    df_test = df.copy()
    base_score = shapiro(df_test[col])[0]
    
    log_score = _get_transformation_score(df, _log_transformation, col)
    scale_log_score = _get_transformation_score(df, _scale_log_transformation, col)
    scale_score = _get_transformation_score(df, _scale_transformation, col)
    boxcox_score = _get_transformation_score(df, _boxcox_transformation, col)
    
    arr = {'Do nothing':base_score,_log_transformation:log_score,
           _scale_log_transformation:scale_log_score,
           _scale_transformation:scale_score,
           _boxcox_transformation:boxcox_score}
    
    # Select best tranformation with higher score
    best_tranform_func = max(arr, key=arr.get)
    if best_tranform_func == 'Do nothing':
        return df
    else:
        return best_tranform_func(df, col)
    
def _basic_fill(df_selected, col, cat=False):
    df_selected[col] = df_selected.loc[:,col].fillna(df_selected.loc[:,col].value_counts().idxmax()) if cat else df_selected.loc[:,col].fillna(df_selected.loc[:,col].median())

def _custom_encoder(df_selected, col, col_type):
    _col_df = np.array(df_selected.loc[:,col]).reshape(-1,1)
    if col_type == 'Categorical':
        enc = OneHotEncoder(sparse=False)
    elif col_type == 'Ordinal Categorical':
        enc = OrdinalEncoder()
    else:
        raise Exception(f'Cannot encode, check the {col} column!')
    _col_enc = enc.fit_transform(_col_df)
    temp = pd.DataFrame(_col_enc,index=df_selected.index)
    temp.columns= enc.get_feature_names([col]) if col_type == 'Categorical' else [col+'_ord_trnsfrm']
    df_selected=pd.concat([df_selected,temp], axis=1)
    df_selected.drop(col, axis=1, inplace=True)
    return df_selected

def automated_feature_engineering(df_or, label_map):
    df = df_or.copy()
    for col in list(df):
        if col=='SalePrice':
            continue
        type_of_col = label_map[col]
        _cat = True if (type_of_col == 'Categorical') or (type_of_col == 'Ordinal Categorical') else False
        if df[col].isnull().sum() > 0:
            _basic_fill(df, col, cat=_cat)
        if _cat:
            df=_custom_encoder(df, col, type_of_col)
        elif type_of_col == 'Ordinal Numerical':
            _apply_best_transformation(df,col)
        else:
            raise Exception(f'Unrecognized type of column! Please check the {col} column!')
    return df
"""
I realized there are more of columns that not distributed equally or has lot of missing values. Maybe I can handle them in one, because they are taking so much times. Let's say if columns has more than 75% missing values or one feature's category is more than 75%, we are going to drop this attribute. 
"""
threshold_miss = len(eda)*0.75
threshold_distr = len(eda)*0.75
for col in eda:
    if (eda[col].value_counts().iloc[0] >= threshold_distr or eda[col].isnull().sum() >= threshold_miss) and col != 'SalePrice':
        eda.drop(col,axis=1,inplace=True)
print('Preview after dropping')
display(eda.head())
def next_col(last):
    print(f'Next column is {[eda.iloc[:,ind+1].name for ind,col in enumerate(list(eda)) if col==last][0]}')
"""
We should label each feature as categorical or numerical. We'll automate this as much as we can, however it should be applying in under supervision
"""
print('All colum names after drop unnecessary ones:\n')
print(list(eda))
def label_attrs(eda, target_col):
    col_dict={}
    for col in list(eda):
        if col=='price_cat':
            continue
        class_name=type(eda[col].iloc[0]).__name__
        if class_name == 'str':
            col_dict.update({col: 'Categorical'})
        elif class_name == 'float64':
            col_dict.update({col:'Ordinal Numerical'})
        elif eda[col].nunique() < len(eda)*.02:
            if np.abs(eda[col].corr(eda[target_col])) > .15:
                col_dict.update({col:'Ordinal Categorical'})
            else:
                col_dict.update({col: 'Categorical'})
        else:
            col_dict.update({col:'Ordinal Numerical'})
    return col_dict

print('After automated labeling:\n')
label_map=label_attrs(eda, 'SalePrice')
print(label_map)
%%javascript
IPython.OutputArea.auto_scroll_threshold = 9999;
plt.rcParams.update({'figure.max_open_warning': 0})
from IPython.core.interactiveshell import InteractiveShell
InteractiveShell.ast_node_interactivity = "all"
automated_eda_report(eda)
"""
# Feature Engineering
First, we need to drop unuseful features as we did explanatory data anaylsis part. 
"""
"""
### Neighborhood
It provide us the neighborhood of each property.
* Graphs would not be explanatory for the feature such has lot of category like this. We should try different approach if we don't want to lose information about neighborhood of house. We can categorize all as 3 whether they area expensive area or not.
"""
eda_Neighborhood=eda.loc[:,['Neighborhood','price_cat']] # Use eda for getting price_cat column
eda_Neighborhood['count']=1
eda_Neighborhood=eda_Neighborhood.groupby(['Neighborhood','price_cat']).sum()
eda_Neighborhood.fillna(0,inplace=True)
rows=[]
for neighborhood, counts in eda_Neighborhood.unstack().iterrows():
    flg_dict={
        'Neighborhood':neighborhood,
        'Group':counts.idxmax()[1]
    }
    rows.append(flg_dict)
neighborhood_map = pd.DataFrame(rows)
neighborhood_map.set_index('Neighborhood',inplace=True)
display(neighborhood_map.head())
df_Neighborhood=df.copy()
for _,group in neighborhood_map.iterrows():
    for ind,row in df_Neighborhood.iterrows():
        if row['Neighborhood'] == group.name:
            df_Neighborhood.loc[ind,'Neighborhood'] = group.values[0]
# Drop below threshold like we did in EDA
threshold_miss = len(df_Neighborhood)*0.75
threshold_distr = len(df_Neighborhood)*0.75
df_drop = df_Neighborhood.copy()
for col in df_drop:
    if (df_drop[col].value_counts().iloc[0] >= threshold_distr or df_drop[col].isnull().sum() >= threshold_miss) and col != 'SalePrice':
        df_drop.drop(col,axis=1, inplace=True)
print('Completed!\nPreview after dropping')
display(df_drop.head())
"""
### Automated Feature Engineering
"""
import warnings
warnings.filterwarnings('ignore')
df_preprocessed=automated_feature_engineering(df_drop, label_map)
submission_transformed=df_preprocessed[df_preprocessed.SalePrice.isnull()]
df_before_ml = df_preprocessed[~df_preprocessed.SalePrice.isnull()]
y=df_before_ml[['SalePrice']]
X=df_before_ml.drop('SalePrice',axis=1)
X=scale(X)
y=np.log(y)
X_train,X_test, y_train, y_test = train_test_split(X,y,test_size=.2,random_state=42)
%%time


regressors = [
    LinearSVR(random_state=42),DecisionTreeRegressor(random_state=42),RandomForestRegressor(random_state=42),
    LinearRegression(),AdaBoostRegressor(random_state=42),
    AdaBoostRegressor(LinearRegression()),BaggingRegressor(LinearRegression(),n_estimators=50),
    Ridge(),GradientBoostingRegressor(random_state=42),
    Lasso(),ElasticNet(),SGDRegressor()
]
scores={}

tqdm()
for reg in tqdm(regressors):
    reg.fit(X_train, y_train)
    preds = reg.predict(X_test)
    scores.update({
        reg:np.sqrt(mean_squared_error(y_test,preds))
    }) 
for f,sc in scores.items():
    display(Markdown(f'RMSE score of {f}={sc}'))
best_reg = list(scores)[np.argmin(list(scores.values()))]
display(Markdown(f'**Best regressor is {best_reg}**'))
for model_with_cv in [RidgeCV(), LassoCV(random_state=42), ElasticNetCV(random_state=42)]:
    model_with_cv.fit(X,y)
    res=f'$r^2$ of {model_with_cv}={model_with_cv.score(X,y)}'
    display(Markdown(res))
gbr = clone(best_reg)
gbr.fit(X_train, y_train)
gbr_preds = gbr.predict(X_test)
r2_score(y_test, gbr_preds)
from sklearn.inspection import permutation_importance
imp = permutation_importance(gbr,X,y, n_repeats=10, random_state=42)
importance_df=pd.Series(imp.importances_mean).sort_values(ascending=False).to_frame().rename({0:'importance'},axis=1)
importance_df['cumsum']=importance_df.cumsum()
display(importance_df)
good_features=list(importance_df.loc[:importance_df['cumsum'].idxmax(),:].index)
X_feature_extraction = X[:,good_features]
print(X_feature_extraction.shape)
X_train,X_test, y_train, y_test = train_test_split(X_feature_extraction,y,test_size=.2,random_state=42)
gbr.fit(X_train, y_train)
gbr_preds = gbr.predict(X_test)
print(np.sqrt(mean_squared_error(y_test, gbr_preds)))
pca = PCA(n_components='mle')

arr_pca=pca.fit_transform(X_feature_extraction)
X_pca = pd.DataFrame(arr_pca, columns=['PCA'+str(i) for i in range(arr_pca.shape[1])])

graph_var = pca.explained_variance_ratio_

fig = go.Figure(data=[
    go.Bar(name='Explained Variance', x=np.arange(len(graph_var)), y=graph_var*100)
])
fig.add_trace(go.Scatter(x=np.arange(1,len(graph_var)+1), 
                         y=np.cumsum(graph_var)*100,
                         mode='lines',
                         name='Cumulated Explained Variance'))
# Change the bar mode
fig.update_layout(barmode='stack')

print(f'Dataset has total feature of {X_feature_extraction.shape[1]}')
print(f'Total explained variance: {np.sum(pca.explained_variance_ratio_)*100}% with n_components={pca.n_components_}')
fig.show()
gbr = RidgeCV(scoring='neg_root_mean_squared_error').fit(X_pca,y)
rmse=f'**RMSE score of {gbr}={gbr.score(X_pca,y)}**'
display(Markdown(rmse))
X_train,X_test, y_train, y_test = train_test_split(X_pca,np.log(y), test_size=.2, random_state=42)
gbr = Ridge(random_state=42)
gbr.fit(X_train, y_train)
gbr_preds = gbr.predict(X_test)
rmse = np.sqrt(mean_squared_error(y_test, gbr_preds))
display(Markdown(f'**RMSE = {rmse}**'))
r2=f'**$r^2$ of {gbr}={r2_score(y_test,gbr_preds)}**'
display(Markdown(r2))
sns.distplot(np.log(y))
submission_transformed.drop(columns = 'SalePrice', inplace=True)
submission_transformed.index
sub=scale(pca.transform(submission_transformed.iloc[:,good_features]))
res=gbr.predict(sub)
price_predicted=pd.DataFrame(np.exp(np.exp(res)),index=submission_transformed.index).rename({0:'SalePrice'},axis=1)
display(price_predicted)
price_predicted.to_csv('submission.csv')