# This Python 3 environment comes with many helpful analytics libraries installed
# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python
# For example, here's several helpful packages to load

import numpy as np # linear algebra
import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)

# Input data files are available in the read-only "../input/" directory
# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory

import os
for dirname, _, filenames in os.walk('/kaggle/input'):
    for filename in filenames:
        print(os.path.join(dirname, filename))

# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using "Save & Run All" 
# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session
"""
# **Imports**
"""
import pandas as pd
import numpy as np
import seaborn as sns
import matplotlib.pyplot as plt
%matplotlib inline
sns.set_style('whitegrid')
dftrain = pd.read_csv('/kaggle/input/house-prices-advanced-regression-techniques/train.csv')
dftest = pd.read_csv('/kaggle/input/house-prices-advanced-regression-techniques/test.csv')
"""
# **Checkout Data Structures**
"""
dftrain.head()
#shape of train dataset
dftrain.shape
dftrain.describe().transpose()
#average SalePrice is $180,921
dftrain.info()
#mixture of int, obj and some float
#shape of test dataset
dftest.shape
#see distribution of SalesPrice
plt.figure(figsize=(10,6))
sns.histplot(dftrain['SalePrice'])
#most houses are around $100,000 to $200,000
#seem like outlier starts at >450,000
dftrain.corr()['SalePrice'].sort_values(ascending=False)
#corr >=0.5 ['OverallQual', 'GrLivArea', 'GarageCars', 'GarageArea', 'TotalBsmtSF', '1stFlrSF', 'FullBath', 'TotRmsAbvGrd', 'YearBuilt', 'YearRemodAdd']
#corr (-) ['KitchenAbvGr', 'EnclosedPorch', 'MSSubClass', 'OverallCond', 'YrSold', 'LowQualFinSF', 'MiscVal', 'BsmtHalfBath', BsmtFinSF2]
"""
# **Exploratory Data Analysis**
"""
"""
Overall Quality of Materials and finishes
"""
plt.figure(figsize=(10,6))
sns.scatterplot(x='OverallQual', y='SalePrice', data=dftrain)
#house with higher overall material and finish quality has higher sales price
"""
Gound Living Area (sqrft)
"""
plt.figure(figsize=(10,6))
sns.scatterplot(x='GrLivArea', y='SalePrice', data=dftrain)
#strong linear regression shown in the graph
#should cut out houses that has above ground living area over 4,500 sqrft because its outlier will make the model less accurate
dftrain = dftrain.drop(dftrain[dftrain['SalePrice']>600000].index)
dftrain = dftrain.drop(dftrain[dftrain['GrLivArea']>4000].index)
"""
Total Basement Area (sqrft)
"""
plt.figure(figsize=(10,6))
sns.scatterplot(x='TotalBsmtSF', y='SalePrice', data=dftrain)
#some linear relationship but should cut out the outliers too (>3000)
dftrain = dftrain.drop(dftrain[dftrain['TotalBsmtSF']>3000].index)
"""
Garage Area (sqrft)
"""
plt.figure(figsize=(10,6))
sns.scatterplot(x='GarageArea', y='SalePrice', data=dftrain)
#cut out outliers
dftrain = dftrain.drop(dftrain[dftrain['GarageArea']>1200].index)
"""
Garage Car Capacity
"""
plt.figure(figsize=(10,6))
sns.scatterplot(x='GarageCars', y='SalePrice', data=dftrain)
"""
First Floor sqrft
"""
plt.figure(figsize=(10,6))
sns.scatterplot(x='1stFlrSF', y='SalePrice', data=dftrain)
"""
# **Feature Engineering**
"""
"""
Seperate Features
"""
dftrain = dftrain.drop(['Id'], axis=1)

xy_train = dftrain

test_id = dftest['Id']

x_test = dftest.drop(['Id'], axis=1)
#created ttdf to do feature engineer on both train and test df at the same time
ttdf = pd.concat([xy_train, x_test], axis=0)
len(ttdf)
#create table to see null values
null_values = pd.DataFrame(ttdf.isnull().sum().sort_values(ascending=False), columns=['Sum_null'])
null_values = null_values[null_values['Sum_null']>0]
null_values['Percent'] = (null_values['Sum_null']/2906)*100
null_values['Features'] = null_values.index

null_values
"""
**Work with Missing Values**
"""
plt.figure(figsize=(10,10))
sns.heatmap(ttdf.isnull(), yticklabels=False, cbar=False, cmap='PuBu')
ttdf = ttdf.drop((null_values[(null_values['Sum_null']>100)&(null_values['Sum_null']<1420)]).index, axis=1)
ttdf = ttdf.drop(null_values[null_values['Sum_null']>2000].index, axis=1)
"""
Drop duplicated info columns
"""
ttdf = ttdf.drop(['OverallCond'], axis=1)
ttdf = ttdf.drop(['BsmtCond'], axis=1)
ttdf = ttdf.drop(['LandSlope'], axis=1)
ttdf = ttdf.drop(['1stFlrSF'], axis=1)
ttdf = ttdf.drop(['2ndFlrSF'], axis=1)
ttdf = ttdf.drop(['HouseStyle'], axis=1)
ttdf = ttdf.drop(['RoofMatl'], axis=1)
"""
Drop Year built info
"""
ttdf = ttdf.drop(['YearBuilt'], axis=1)
ttdf = ttdf.drop(['YearRemodAdd'], axis=1)
"""
Fill null with info got from description file
"""
ttdf['MasVnrType'] = ttdf['MasVnrType'].fillna('none')
ttdf['BsmtExposure'] = ttdf['BsmtExposure'].fillna('no_bsmt')
ttdf['BsmtFinType1'] = ttdf['BsmtFinType1'].fillna('no_bsmt')
ttdf['BsmtFinType2'] = ttdf['BsmtFinType2'].fillna('no_bsmt')
ttdf['BsmtQual'] = ttdf['BsmtQual'].fillna('no_bsmt')
"""
Fill null with mean values
"""
ttdf['MasVnrArea'] = ttdf['MasVnrArea'].fillna(value=ttdf['MasVnrArea'].mean())
"""
**Combine features**
"""
ttdf['Bathrooms_total'] = (ttdf['FullBath'] + ttdf['BsmtFullBath'] + (0.5* (ttdf['HalfBath']+ttdf['BsmtHalfBath'])))

ttdf.drop(['FullBath'], axis=1, inplace=True)
ttdf.drop(['BsmtFullBath'], axis=1, inplace=True)
ttdf.drop(['HalfBath'], axis=1, inplace=True)
ttdf.drop(['BsmtHalfBath'], axis=1, inplace=True)
ttdf['PorchTotalSF'] = (ttdf['OpenPorchSF'] + ttdf['EnclosedPorch'] + ttdf['3SsnPorch'] + ttdf['ScreenPorch'])

ttdf.drop(['OpenPorchSF'], axis=1, inplace=True)
ttdf.drop(['EnclosedPorch'], axis=1, inplace=True)
ttdf.drop(['3SsnPorch'], axis=1, inplace=True)
ttdf.drop(['ScreenPorch'], axis=1, inplace=True)
"""
**Convert Categorical Feature to Dummy Variables**
"""
ttdf.info()
ttdf.shape
ttdf.select_dtypes(include='object').columns
mszoning = pd.get_dummies(ttdf['MSZoning'], drop_first=True)
street = pd.get_dummies(ttdf['Street'], drop_first=True)
lotshape = pd.get_dummies(ttdf['LotShape'], drop_first=True)
landcontour = pd.get_dummies(ttdf['LandContour'], drop_first=True)
utilities = pd.get_dummies(ttdf['Utilities'], drop_first=True)
lotconfig = pd.get_dummies(ttdf['LotConfig'], drop_first=True)
neighborhood = pd.get_dummies(ttdf['Neighborhood'], drop_first=True)
condition1 = pd.get_dummies(ttdf['Condition1'], drop_first=True)
condition2 = pd.get_dummies(ttdf['Condition2'], drop_first=True)
bldgtype = pd.get_dummies(ttdf['BldgType'], drop_first=True)
roofstyle = pd.get_dummies(ttdf['RoofStyle'], drop_first=True)
exterior1st = pd.get_dummies(ttdf['Exterior1st'], drop_first=True)
exterior2nd = pd.get_dummies(ttdf['Exterior2nd'], drop_first=True)
masvnrtype = pd.get_dummies(ttdf['MasVnrType'], drop_first=True)
exterqual = pd.get_dummies(ttdf['ExterQual'], drop_first=True)
extercond = pd.get_dummies(ttdf['ExterCond'], drop_first=True)
foundation = pd.get_dummies(ttdf['Foundation'], drop_first=True)
bsmtqual = pd.get_dummies(ttdf['BsmtQual'], drop_first=True)
bsmtexposure = pd.get_dummies(ttdf['BsmtExposure'], drop_first=True)
bsmtfintype1 = pd.get_dummies(ttdf['BsmtFinType1'], drop_first=True)
bsmtfintype2 = pd.get_dummies(ttdf['BsmtFinType2'], drop_first=True)
heating = pd.get_dummies(ttdf['Heating'], drop_first=True)
heatingqc = pd.get_dummies(ttdf['HeatingQC'], drop_first=True)
centralair = pd.get_dummies(ttdf['CentralAir'], drop_first=True)
electrical = pd.get_dummies(ttdf['Electrical'], drop_first=True)
kitchenqual = pd.get_dummies(ttdf['KitchenQual'], drop_first=True)
functional = pd.get_dummies(ttdf['Functional'], drop_first=True)
paveddrive = pd.get_dummies(ttdf['PavedDrive'], drop_first=True)
saletype = pd.get_dummies(ttdf['SaleType'], drop_first=True)
salecondition = pd.get_dummies(ttdf['SaleCondition'], drop_first=True)
ttdf = pd.concat([ttdf, mszoning, street, salecondition, saletype, paveddrive, functional, kitchenqual, electrical, centralair, heatingqc, heating, bsmtfintype2, bsmtfintype1, bsmtexposure, bsmtqual, foundation, extercond, exterqual, masvnrtype, exterior2nd, exterior1st, roofstyle, bldgtype, condition2, condition1, neighborhood, lotconfig, utilities, landcontour, lotshape], axis=1)
ttdf.drop(['MSZoning', 'Street', 'LotShape', 'LandContour', 'Utilities', 'LotConfig', 'Neighborhood', 'Condition1', 'Condition2', 'BldgType', 'RoofStyle', 'Exterior1st', 'Exterior2nd', 'MasVnrType', 'ExterQual', 'ExterCond', 'Foundation', 'BsmtQual', 'BsmtExposure', 'BsmtFinType1', 'BsmtFinType2', 'Heating', 'HeatingQC', 'CentralAir', 'Electrical', 'KitchenQual', 'Functional', 'PavedDrive', 'SaleType', 'SaleCondition'], axis=1, inplace=True)
ttdf.info()
#now all columns in numeric form
"""
# **Create Model**
"""
"""
Seperate train and test data
"""
ttdf.shape
ttdf.isnull().sum().sort_values()
trainxy = ttdf[0:1447]
trainxy.isnull().sum().sort_values()
trainxy = trainxy.dropna()
y_train = trainxy['SalePrice'].values
x_train = trainxy.drop("SalePrice",1).values
x_test = ttdf[1447:].drop("SalePrice", 1).values
"""
**Data Preprocessing** 
"""
from sklearn.preprocessing import MinMaxScaler
scaler = MinMaxScaler()
x_train = scaler.fit_transform(x_train)
x_test = scaler.transform(x_test)
x_train.shape
x_test.shape
"""
**Model Creation (ANN)**
"""
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import Dense, Dropout
from sklearn.metrics import mean_squared_error, mean_absolute_error, explained_variance_score
mse = mean_squared_error
model = Sequential()

model.add(Dense(190, activation='relu'))
model.add(Dropout(0.5))
model.add(Dense(90, activation='relu'))
model.add(Dropout(0.5))
model.add(Dense(1))

model.compile(optimizer='adam', loss='mse')

model.fit(x=x_train, y=y_train, batch_size=180, epochs=750)
"""
**See Loss history**
"""
model_loss = pd.DataFrame(model.history.history)
model_loss.plot()
"""
# **Predictions**
"""
predictions = model.predict(x_test)
Predictions = predictions.flatten()
Predictions = pd.Series(Predictions)
Predictions
Test_Id = pd.Series(test_id)
submission = pd.DataFrame({'Id': Test_Id, 'SalePrice': Predictions})
submission.tail(20)
submission = submission.fillna(value=submission['SalePrice'].mean())
"""
**Save Submission File**
"""
submission.to_csv('HPPredictSubmission.csv', index=False)