"""
<a href="https://colab.research.google.com/github/GZabalaG/AIMaster/blob/main/Datasets_Analysis/house_price_prediction.ipynb" target="_parent"><img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"/></a>
"""
"""
# Houses...
"""
import pandas as pd
import numpy as np
from matplotlib import pyplot as plt
import seaborn as sns
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import StandardScaler
from sklearn.preprocessing import PolynomialFeatures
from sklearn.linear_model import LinearRegression
train = pd.read_csv('../input/house-prices-advanced-regression-techniques/train.csv')
test = pd.read_csv('../input/house-prices-advanced-regression-techniques/test.csv')
display(train.head(10))
display(test.head(10))
"""
## Data exploration:

What's going on in this house?
"""
"""
#### Unique values
"""
pd.set_option('display.max_rows', None)

print('Train')
display(train.nunique())

print('\nTest')
display(test.nunique())
"""
#### NaN Values
"""
print('Train')
display(train.isnull().sum())

print('Test')
display(test.isnull().sum())
"""
#### Types
"""
print('Train')
dtypes = train.dtypes
display(dtypes)

print('Test')
dtypes = test.dtypes
display(dtypes)
"""
#### Shape
"""
print(train.shape)
print(test.shape)
"""
#### Describe
"""
train.describe().T
"""
## Data processing

Cleaning the house...
"""
"""
#### Nans... Train:
"""
train['LotFrontage']=train['LotFrontage'].fillna(train['LotFrontage'].mean())
train['BsmtCond']=train['BsmtCond'].fillna(train['BsmtCond'].mode()[0])
train['BsmtQual']=train['BsmtQual'].fillna(train['BsmtQual'].mode()[0])
train['FireplaceQu']=train['FireplaceQu'].fillna(train['FireplaceQu'].mode()[0])
train['GarageType']=train['GarageType'].fillna(train['GarageType'].mode()[0])
train['GarageFinish']=train['GarageFinish'].fillna(train['GarageFinish'].mode()[0])
train['GarageQual']=train['GarageQual'].fillna(train['GarageQual'].mode()[0])
train['GarageCond']=train['GarageCond'].fillna(train['GarageCond'].mode()[0])
train['MasVnrType']=train['MasVnrType'].fillna(train['MasVnrType'].mode()[0])
train['MasVnrArea']=train['MasVnrArea'].fillna(train['MasVnrArea'].mode()[0])
train['BsmtExposure']=train['BsmtExposure'].fillna(train['BsmtExposure'].mode()[0])
train['BsmtFinType2']=train['BsmtFinType2'].fillna(train['BsmtFinType2'].mode()[0])
train.drop(['Id','PoolQC','GarageYrBlt'],axis=1,inplace=True)
train.Alley=train.Alley.map({'NA':0,'Grvl':1,'Pave':2})
train.Fence=train.Fence.map({'MnPrv':1,'MnWw':2,'GdWo':3,'GdPrv':4,'NA':0})
train.MiscFeature=train.MiscFeature.map({'Shed':1,'Gar2':2,'Othr':3,'NA':0})
train.Alley.fillna(train.Alley.mean(),inplace=True)
train.Fence.fillna(train.Fence.mean(),inplace=True)
train.MiscFeature.fillna(train.MiscFeature.mean(),inplace=True)
train.dropna(inplace=True)
"""
#### Nans... Test
"""
test['LotFrontage']=test['LotFrontage'].fillna(test['LotFrontage'].mean())
test['MSZoning']=test['MSZoning'].fillna(test['MSZoning'].mode()[0])
test['BsmtCond']=test['BsmtCond'].fillna(test['BsmtCond'].mode()[0])
test['BsmtQual']=test['BsmtQual'].fillna(test['BsmtQual'].mode()[0])
test['FireplaceQu']=test['FireplaceQu'].fillna(test['FireplaceQu'].mode()[0])
test['GarageType']=test['GarageType'].fillna(test['GarageType'].mode()[0])
test['MasVnrType']=test['MasVnrType'].fillna(test['MasVnrType'].mode()[0])
test['MasVnrArea']=test['MasVnrArea'].fillna(test['MasVnrArea'].mode()[0])
test['GarageFinish']=test['GarageFinish'].fillna(test['GarageFinish'].mode()[0])
test['GarageQual']=test['GarageQual'].fillna(test['GarageQual'].mode()[0])
test['GarageCond']=test['GarageCond'].fillna(test['GarageCond'].mode()[0])
test['BsmtExposure']=test['BsmtExposure'].fillna(test['BsmtExposure'].mode()[0])
test['BsmtFinType2']=test['BsmtFinType2'].fillna(test['BsmtFinType2'].mode()[0])
test['Utilities']=test['Utilities'].fillna(test['Utilities'].mode()[0])
test['Exterior1st']=test['Exterior1st'].fillna(test['Exterior1st'].mode()[0])
test['Exterior2nd']=test['Exterior2nd'].fillna(test['Exterior2nd'].mode()[0])
test['BsmtFinType1']=test['BsmtFinType1'].fillna(test['BsmtFinType1'].mode()[0])
test['BsmtFinSF1']=test['BsmtFinSF1'].fillna(test['BsmtFinSF1'].mode()[0])
test['BsmtFinSF2']=test['BsmtFinSF2'].fillna(test['BsmtFinSF2'].mode()[0])
test['BsmtUnfSF']=test['BsmtUnfSF'].fillna(test['BsmtUnfSF'].mode()[0])
test['TotalBsmtSF']=test['TotalBsmtSF'].fillna(test['TotalBsmtSF'].mode()[0])
test['BsmtFullBath']=test['BsmtFullBath'].fillna(test['BsmtFullBath'].mode()[0])
test['BsmtHalfBath']=test['BsmtHalfBath'].fillna(test['BsmtHalfBath'].mode()[0])
test['KitchenQual']=test['KitchenQual'].fillna(test['KitchenQual'].mode()[0])
test['Functional']=test['Functional'].fillna(test['Functional'].mode()[0])
test['GarageCars']=test['GarageCars'].fillna(test['GarageCars'].mode()[0])
test['GarageArea']=test['GarageArea'].fillna(test['GarageArea'].mode()[0])
test['SaleType']=test['SaleType'].fillna(test['SaleType'].mode()[0])
test['MasVnrArea']=test['MasVnrArea'].fillna(test['MasVnrArea'].mode()[0])
test.drop(['Id','GarageYrBlt','PoolQC'],axis=1,inplace=True)
test.Alley=test.Alley.map({'NA':0,'Grvl':1,'Pave':2})
test.Fence=test.Fence.map({'MnPrv':1,'MnWw':2,'GdWo':3,'GdPrv':4,'NA':0})
test.MiscFeature=test.MiscFeature.map({'Shed':1,'Gar2':2,'Othr':3,'NA':0})
test.Alley.fillna(test.Alley.mean(),inplace=True)
test.Fence.fillna(test.Fence.mean(),inplace=True)
test.MiscFeature.fillna(test.MiscFeature.mean(),inplace=True)
print('Train')
display(train.isnull().sum())

print('\n\nTest')
display(test.isnull().sum())
"""
## Data visualization

Painting houses...
"""
plt.figure(figsize=(12,10))
sns.heatmap(train.corr(),cmap="jet",annot=False,linewidths=1,robust=True)
"""
Top 5 varaiables most correlated to SalePrice: 
1. OverallQual
2. GrLiveArea
3. GarageCars
4. GarageArea
5. 1stFlrSF
"""
"""
#### Drop the not correlated continuous varaiables
"""
train_final = pd.DataFrame()
def correlation(dataset, threshold):
    col_corr = set() # Set of all the names of deleted columns
    corr_matrix = dataset.corr()
    for i in range(len(corr_matrix.columns)):
          if (corr_matrix.iloc[i, -1] >= threshold) and (corr_matrix.columns[-1] not in col_corr):
              colname = corr_matrix.columns[i] # getting the name of column
              col_corr.add(colname)
              if colname in dataset.columns:
                  train_final[colname] = dataset[colname] # deleting the column from the dataset
    display(train_final.head(10))
correlation(train, 0.5)
sns.pairplot(train_final, x_vars= ['SalePrice'], y_vars=['OverallQual', 'YearBuilt', 'YearRemodAdd', 'TotalBsmtSF', '1stFlrSF', 'GrLivArea', 'FullBath', 'TotRmsAbvGrd', 'GarageCars', 'GarageArea'], height=7, aspect=0.7, kind='reg')
"""
## Data prediction

And the price is...
"""
"""
#### Split and standarize
"""
X = train_final.iloc[:, :-1].values
y = train_final.iloc[:, -1].values
sc = StandardScaler()
X = sc.fit_transform(X)
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size = 0.3, random_state = 0)
"""
#### Training...
"""
regr_model = LinearRegression()
regr_model.fit(X_train, y_train)
"""
#### Predictions & Accuracy
"""
y_pred = regr_model.predict(X_test)
np.set_printoptions(precision=2)
print(np.concatenate((y_pred.reshape(len(y_pred), 1), y_test.reshape(len(y_test), 1)), 1))
rmsd = (np.sqrt(np.square(y_pred - y_test))).mean(axis=0)
print('Root mean square deviation:', rmsd)
"""
#### Training... (Poly)
"""
poly_reg = PolynomialFeatures(degree = 2)
X_poly = poly_reg.fit_transform(X_train)

poly_reg_model = LinearRegression()
poly_reg_model.fit(X_poly, y_train)
"""
#### Predictions & Accuracy
"""
X_test_poly = poly_reg.fit_transform(X_test)

y_pred_poly = poly_reg_model.predict(X_test_poly)
print(np.concatenate((y_pred_poly.reshape(len(y_pred_poly), 1), y_test.reshape(len(y_test), 1)), 1))
rmsd = (np.sqrt(np.square(y_pred_poly - y_test))).mean(axis=0)
print('Root mean square deviation:', rmsd)
"""
We got more accuracy with polynomial regressor model.
"""