# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

import numpy as np
import pandas as pd
from scipy import stats
from sklearn.linear_model import Ridge, Lasso, ElasticNet, LinearRegression
import plotly.express as px
import matplotlib.pyplot as plt
import plotly.graph_objects as go
from sklearn.model_selection import train_test_split
from sklearn.model_selection import GridSearchCV
from sklearn.model_selection import KFold, StratifiedKFold
from sklearn.model_selection import cross_val_score
import time

pd.set_option('display.max_rows', 500)

df = pd.read_csv('./../data/ames_housing_price_data_v4.csv')


def seasonal_adjust(df, mo_avg):
    df['SalePrice_adj'] = 0 * len(df)
    for i in range(len(df)):
        price_norm = mo_avg
        df.loc[i, 'SalePrice_adj'] = (df.loc[i, 'SalePrice']) / mo_avg.loc[df.loc[i, 'MoSold'], 'monthly_avg_ratio']
    return df


mo_avg = df.groupby('MoSold').agg(monthly_avg = ('SalePrice', 'mean'), monthly_sd = ('SalePrice', 'std'))

mo_avg['monthly_avg_ratio'] = mo_avg['monthly_avg'] / np.mean(df['SalePrice'])

df = seasonal_adjust(df, mo_avg)

df['SalePrice_adj_log'] = np.log10(df['SalePrice_adj'])

# +
#exclude weird sale types
# -

df['SaleCondition'].value_counts()

df=df[(df['SaleCondition']=='Normal') | (df['SaleCondition']=='Partial')]

price = df['SalePrice']
price_log = df['SalePrice_log']
price_adj = df['SalePrice_adj']
price_adj_log = df['SalePrice_adj_log']

log_cols = []
for col in df.columns:
    if '_log' in col:
        log_cols.append(col)
log_cols

to_dummify = [
    'Street_paved',
    'Alley',
    'LandContour',
    'Utilities',
    'LandSlope',
    'Neighborhood',
    'BldgType',
    #'OverallQual',
    #'OverallCond',
    'RoofStyle',
    'RoofMatl',
    'MasVnrType',
    #'ExterQual',
    #'ExterCond',
    'Foundation',
    'CentralAir',
    #'KitchenQual',
    #'FireplaceQu',
    'GarageFinish',
    #'GarageQual',
    #'GarageCond',
    'PavedDrive',
    #'PoolQC',
    'Fence',
    'MiscFeature',
    #'MoSold',#
    #'HeatingQC_ord',
    'LotShape_com',
    'MSZoning_com',
    'Heating_com',
    'Electrical_com',
    'LotConfig_com',
    #'BsmtCond_ord',
    #'BsmtQual_ord',
    'BsmtExposure_ord',
    'GarageType_com',
    #'number_floors',
    'attic',
    'PUD',
    #'Functional_ord',
    'Garage_age_bin',
    'Remod_age_bin',
    'SaleType',#
    'SaleCondition' #
]

df = pd.get_dummies(df, columns = to_dummify, drop_first = True)

droplist = ['SaleType', 'SaleCondition', 'Garage_age_years', 'Remod_age_years', 'MoSold']
alwaysdrop = ['PID', 'SalePrice', 'SalePrice_log', 'SalePrice_adj', 'SalePrice_adj_log', 'sold_datetime']
df3 = df.drop((alwaysdrop), axis = 1) #+ droplist

df3.head().T

# +
from sklearn import ensemble
from sklearn.model_selection import train_test_split
from sklearn.model_selection import GridSearchCV

#randomForest = ensemble.RandomForestClassifier()
model = ensemble.RandomForestRegressor()

x_train, x_test, y_train, y_test = train_test_split(df3, price, test_size=0.3)

# -

model.set_params(min_samples_leaf=2,min_samples_split=2,max_features=50,n_estimators=3000)
model.fit(x_train,y_train)
print(model.score(x_train, y_train))
print(model.score(x_test, y_test))
features_importance=pd.DataFrame({'Features':x_train.columns,'Score':model.feature_importances_}).sort_values('Score')



# +
features_filtered=features_importance[features_importance['Score']<0.000354]

alwaysdrop = ['PID', 'SalePrice', 'SalePrice_log', 'SalePrice_adj', 'SalePrice_adj_log', 'sold_datetime']

droplist = list(features_filtered['Features']) + alwaysdrop
price = df['SalePrice']

avg_price_df=pd.read_csv('../data/house_surrounding_avg_prices.csv')
avg_price_df2=avg_price_df[['PID','AvgPrice-0.25']].drop_duplicates() #,'AvgPrice-0.5'
df3=df.merge(avg_price_df2,how='left')
df3=df3.merge(feature_df, how='left')

df3 = df3.drop(droplist, axis = 1) #+ droplist
df3=df3.fillna(0)
df3.shape
# -



# +




# +
# grid_para_tree = [{
#     "min_samples_leaf": range(3, 10),
#     "min_samples_split": np.linspace(start=5, stop=20, num=5, dtype=int),
#     "n_estimators": [300],
#     "max_features":[30,35,40,45,50,55,60] #'auto', 'sqrt', 'log2', 5, 10, ,70,100,150
# }]

# grid_search_tree = GridSearchCV(
#     model, grid_para_tree, cv=5, n_jobs=-1)

# # %time grid_search_tree.fit(x_train, y_train)
# print(grid_search_tree.score(x_train, y_train))
# print(grid_search_tree.score(x_test, y_test))
# grid_search_tree.best_params_
# #grid_search_tree.cv_results_['mean_test_score']

# +
model3 = ensemble.RandomForestRegressor()

x_train, x_test, y_train, y_test = train_test_split(df3, price, test_size=0.3)

model3.set_params(min_samples_leaf=3,min_samples_split=3,max_features=50,n_estimators=3000)
model3.fit(x_train,y_train)
print("-"*50)
print(model3.score(x_train, y_train))
print(model3.score(x_test, y_test))
print("-"*50)

# +
features_filtered=features_importance[features_importance['Score']<0.000354]


feature_df=pd.read_csv('../data/house_coordinates_1.0.csv')
feature_df=feature_df.drop(['Address','Coords4','latitude','longitude'],axis=1)

features_importance2=pd.DataFrame({'Features':x_train.columns,'Score':model3.feature_importances_}).sort_values('Score')

features_filtered2=features_importance[features_importance['Score']<0.0001]


alwaysdrop = ['PID', 'SalePrice', 'SalePrice_log', 'SalePrice_adj', 'SalePrice_adj_log', 'sold_datetime']

droplist = list(features_filtered['Features']) + alwaysdrop
price = df['SalePrice']

avg_price_df=pd.read_csv('../data/house_surrounding_avg_prices.csv')
avg_price_df2=avg_price_df[['PID','AvgPrice-0.25']].drop_duplicates() #,'AvgPrice-0.5'
df3=df.merge(avg_price_df2,how='left')
df3=df3.merge(feature_df, how='left')

df3 = df3.drop(droplist, axis = 1) #+ droplist
df3=df3.fillna(0)
df3.shape
# -

grid_search_tree.best_params_












# +
model2 = ensemble.RandomForestRegressor()

x_train, x_test, y_train, y_test = train_test_split(df3, price, test_size=0.3)
feat=int(len(df3.columns)/3)
model2.set_params(min_samples_leaf=2,min_samples_split=2,max_features=feat,n_estimators=3000)
model2.fit(x_train,y_train)
print("-"*50)
print(model2.score(x_train, y_train))
print(model2.score(x_test, y_test))
print("-"*50)

# +
grid_para_tree = [{
    "min_samples_leaf": range(3, 10),
    "min_samples_split": np.linspace(start=5, stop=20, num=5, dtype=int),
    "n_estimators": [300],
    "max_features":[30,35,40,45,50,55,60] #'auto', 'sqrt', 'log2', 5, 10, ,70,100,150
}]

grid_search_tree = GridSearchCV(
    model, grid_para_tree, cv=5, n_jobs=-1)

# %time grid_search_tree.fit(x_train, y_train)
print(grid_search_tree.score(x_train, y_train))
print(grid_search_tree.score(x_test, y_test))
grid_search_tree.best_params_
#grid_search_tree.cv_results_['mean_test_score']
# -

0.9660492504801086
0.9188001671125462

StarcoderdataJupyterScriptsDedupFiltered