# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python [default]
#     language: python
#     name: python3
# ---

# ignore warnings
import warnings
warnings.filterwarnings('ignore')
# get pandas
import pandas as pd
# we'll do some visual checks, get the necessary packages
import matplotlib.pyplot as plt
import seaborn as sns
# %matplotlib inline
import numpy as np
# done, save the model
import pickle

# first load the model
model = pickle.load(open('../models/gbt_combo_reg_v1.pkl', 'rb'))

# then load the dictionary for the transforms
cat_dic = pickle.load(open('../models/gbt_combo_dic_v1.pkl', 'rb'))

# now load the test data
df_raw = pd.read_csv('../data/test.csv')


# wrapper function to perform some simple cleanup
def clean_df(df):
    # drop everything with more than 10% of missing values
    df = df[df.columns[df.isnull().mean() < 0.1]]
    # looks like Garage built goes with Year built, replace NA using that
    df.loc[df['GarageYrBlt'].isnull(),'GarageYrBlt'] = df['YearBuilt']
    return df


def cure_nas(df):
    # get list of categorical variables
    cat_cols = list(df.select_dtypes(include=['object']).columns)
    # get list of non-cat variables
    var_cols = list(df.select_dtypes(include=['int64','float64']).columns)
    # fill na in special case with rule
    # use 1stFlrSF for bsmt when null
    df.loc[df['TotalBsmtSF'].isnull(),'TotalBsmtSF'] = df['1stFlrSF']
    # similar for garage but add factor 1/2
    df.loc[df['GarageArea'].isnull(),'GarageArea'] = df['1stFlrSF']*0.5
    # fill rest of var_cols with 0
    df[var_cols] = df[var_cols].fillna(0)
    # fill na for cats with most frequent
    df[cat_cols] = df[cat_cols].apply(lambda x:x.fillna(x.value_counts().index[0]))    
    return df


df = clean_df(df_raw)
df = cure_nas(df)
df.describe()

# check quickly n/a
total = df.isnull().sum().sort_values(ascending=False)
percent = (df.isnull().sum()/df.isnull().count()).sort_values(ascending=False)
missing_data = pd.concat([total, percent], axis=1, keys=['Total', 'Percent'])
# show columns with most missing entries
missing_data.head(10)


# now prep categorical variables
def cat_handler_df(df,cat_dic):
    # get list of categorical variables
    cat_cols = list(df.select_dtypes(include=['object']).columns)
    # Encoding the variables
    df[cat_cols] = df[cat_cols].apply(lambda x: cat_dic[x.name].transform(x))
    # return the df and the dictionary
    return df


# transform cats
df = cat_handler_df(df,cat_dic)

# prep scikit
input_features = list(df.columns)
input_features.remove('Id')
X = df[input_features].values

y = model.predict(X)

# store output
df['log_SalePrice'] = y

# get the original variable 
df['SalePrice'] = np.exp(df['log_SalePrice']) - 1

# drop the log var
df = df.drop(['log_SalePrice'], axis=1)

df.head()

# save the file for submission
df[['Id','SalePrice']].to_csv('../data/combo_submission_180225.csv',index=False)



StarcoderdataJupyterScriptsDedupFiltered