import numpy as np
import pandas as pd
import seaborn as sns
import matplotlib.pyplot as plt
%matplotlib inline

import os
for dirname, _, filenames in os.walk('/kaggle/input'):
    for filename in filenames:
        print(os.path.join(dirname, filename))
df_train = pd.read_csv("/kaggle/input/house-prices-advanced-regression-techniques/train.csv")

df_test = pd.read_csv("/kaggle/input/house-prices-advanced-regression-techniques/test.csv")
df_train.shape
df_train.columns
df_train.describe()
# taken from: https://www.kaggle.com/pmarcelino/comprehensive-data-exploration-with-python

#missing data
total = df_train.isnull().sum().sort_values(ascending=False)
percent = (df_train.isnull().sum()/df_train.isnull().count()).sort_values(ascending=False)
missing_data = pd.concat([total, percent], axis=1, keys=['Total', 'Percent'])
missing_data.head(20)
"""
# Mapping and Handling Missing Data

Since 'PoolQC', 'MiscFeature', 'Alley', 'Fence', 'FireplaceQu', 'LotFrontage' has more than **15%** percent missing data they are going to be **dropped.**

There are many variables related to the Garage and the Basement. Some are numerical and some are categorical. Therefore, I will map the categorical ones to number (higher is better) to be able to represent them in the correlation heatmap. With the help of the data descriptions and the heatmap, I will look for features that can be dropped.

After mapping, the nan values will also be mapped as a number so the missing values will be **filled**. Besides these, there are 4 features named GarageYrBlt, MasVnrArea, MasVnrType and Electrical that has small amount of missing data. 
* GarageYrBlt will be filled by it's median
* MasVnrArea will be filled by it's mean since it's median is 0
* MasVnrType and Electrical will be filled by it's most common value
"""
df_train.drop(columns = ['PoolQC', 'MiscFeature', 'Alley', 'Fence', 'FireplaceQu', 'LotFrontage'], axis = 1, inplace=True)

year_median = df_train["GarageYrBlt"].median()
area_mean = df_train["MasVnrArea"].mean()


df_train["GarageYrBlt"] = df_train["GarageYrBlt"].fillna(year_median)
df_train["MasVnrArea"] = df_train["MasVnrArea"].fillna(area_mean)
df_train["MasVnrType"] = df_train["MasVnrType"].fillna("BrkFace")
df_train["Electrical"] = df_train["Electrical"].fillna("SBrkr")

df_train.shape
"""
Just to make sure before mappings
"""
df_train["GarageType"] = df_train["GarageType"].astype("str")
df_train["GarageCond"] = df_train["GarageCond"].astype("str")
df_train["GarageFinish"] = df_train["GarageFinish"].astype("str")
df_train["GarageQual"] = df_train["GarageQual"].astype("str")
df_train["BsmtFinType2"] = df_train["BsmtFinType2"].astype("str")
df_train["BsmtExposure"] = df_train["BsmtExposure"].astype("str")
df_train["BsmtCond"] = df_train["BsmtCond"].astype("str")
df_train["BsmtQual"] = df_train["BsmtQual"].astype("str")
df_train["BsmtFinType1"] = df_train["BsmtFinType1"].astype("str")

df_test["GarageType"] = df_test["GarageType"].astype("str")
df_test["GarageCond"] = df_test["GarageCond"].astype("str")
df_test["GarageFinish"] = df_test["GarageFinish"].astype("str")
df_test["GarageQual"] = df_test["GarageQual"].astype("str")
df_test["BsmtFinType2"] = df_test["BsmtFinType2"].astype("str")
df_test["BsmtExposure"] = df_test["BsmtExposure"].astype("str")
df_test["BsmtCond"] = df_test["BsmtCond"].astype("str")
df_test["BsmtQual"] = df_test["BsmtQual"].astype("str")
df_test["BsmtFinType1"] = df_test["BsmtFinType1"].astype("str")
"""
# Ordinal Variable Mapping

Ordinal data is a kind of categorical data with a set **order** to it. In this dataset, some column values are like Excellent, Good, Average, Fair, Poor. This is an example to ordinal variables. 

To be able to work with **categorical data** in our models, we should transform it in a way that they are also represented by numbers. Encoders like LabelEncoder and OneHotEncoder maps the values in alphabetical order. and I couldn't find a way to pass our own mappings.

The value Average would have been 1. This mixed order would not be suitable for the correlation analysis as the increase in the values might not mean anything. Because of that, I have mapped the values as **highest number representing the best value** so that we can see the relationship between the variables in the **correlation heatmap**.

Since GarageType variable is not a ordinal variable, meaning that it's values cannot be ranked amongst each other, it can be label encoded to be represented in the correlation heatmap.
"""
from sklearn.preprocessing import LabelEncoder

label = LabelEncoder()

df_train["GarageType"] = label.fit_transform(df_train["GarageType"])
dict(zip(label.classes_, range(1, len(label.classes_)+1)))
df_test["GarageType"] = label.fit_transform(df_test["GarageType"])
"""
The best value will be mapped as the highest number and the worst is vice versa.

For example:
* Excellent = 6
* Poor = 2
* NA = 1
"""
mapping = {"Ex":6,
           "Gd":5,
           "TA":4,
           "Fa":3,
           "Po":2,
           "nan":1}

df_train["GarageCond"] = df_train["GarageCond"].apply(lambda x : mapping[x])
df_test["GarageCond"] = df_test["GarageCond"].apply(lambda x : mapping[x])
mapping = {"Fin":4,
           "RFn":3,
           "Unf":2,
           "nan":1}
           
df_train["GarageFinish"] = df_train["GarageFinish"].apply(lambda x : mapping[x])
df_test["GarageFinish"] = df_test["GarageFinish"].apply(lambda x : mapping[x])
mapping = {"Ex":6,
           "Gd":5,
           "TA":4,
           "Fa":3,
           "Po":2,
           "nan":1}
           
df_train["GarageQual"] = df_train["GarageQual"].apply(lambda x : mapping[x])
df_test["GarageQual"] = df_test["GarageQual"].apply(lambda x : mapping[x])
mapping = {"GLQ":7,
           "ALQ":6,
           "BLQ":5,
           "Rec":4,
           "LwQ":3,
           "Unf":2,
           "nan":1}
           
df_train["BsmtFinType2"] = df_train["BsmtFinType2"].apply(lambda x : mapping[x])
df_test["BsmtFinType2"] = df_test["BsmtFinType2"].apply(lambda x : mapping[x])
mapping = {"Gd":5,
           "Av":4,
           "Mn":3,
           "No":2,
           "nan":1}
           
df_train["BsmtExposure"] = df_train["BsmtExposure"].apply(lambda x : mapping[x])
df_test["BsmtExposure"] = df_test["BsmtExposure"].apply(lambda x : mapping[x])
mapping = {"GLQ":7,
           "ALQ":6,
           "BLQ":5,
           "Rec":4,
           "LwQ":3,
           "Unf":2,
           "nan":1}
           
df_train["BsmtFinType1"] = df_train["BsmtFinType1"].apply(lambda x : mapping[x])
df_test["BsmtFinType1"] = df_test["BsmtFinType1"].apply(lambda x : mapping[x])
mapping = {"Ex":6,
           "Gd":5,
           "TA":4,
           "Fa":3,
           "Po":2,
           "nan":1}
           
df_train["BsmtCond"] = df_train["BsmtCond"].apply(lambda x : mapping[x])
df_test["BsmtCond"] = df_test["BsmtCond"].apply(lambda x : mapping[x])
mapping = {"Ex":6,
           "Gd":5,
           "TA":4,
           "Fa":3,
           "Po":2,
           "nan":1}
           
df_train["BsmtQual"] = df_train["BsmtQual"].apply(lambda x : mapping[x])
df_test["BsmtQual"] = df_test["BsmtQual"].apply(lambda x : mapping[x])
"""
Creating sub dataframes of variables related to the garage and the basement to analyze them in the correlation heatmap.
"""
df_garage = df_train[["GarageType","GarageCond","GarageYrBlt","GarageFinish","GarageQual","GarageCars","GarageArea"]]

df_basement = df_train[["BsmtFinType2","BsmtExposure","BsmtFinType1","BsmtCond","BsmtQual","BsmtFinSF1","BsmtFinSF2","BsmtUnfSF","TotalBsmtSF"]]
plt.figure(figsize=(16,10))

corr = df_basement.corr()
ax = sns.heatmap(
    corr,
    vmin=-1, vmax=1, center=0,
    cmap=sns.diverging_palette(20, 220, n=200),
    square=True
)
ax.set_xticklabels(
    ax.get_xticklabels(),
    rotation=45,
    horizontalalignment='right'
)

plt.show()
"""
BsmtUnfSF has negative correlation with BsmtFinType1, BsmtFinType2, BsmtFinSF1 and BsmtFinSF2 so it will be dropped.

BsmtFinType1 has strong correlation with BsmtFinSF1 and it is the same case with the 2s.
This shows that a good living quarters have high finished square feet.
I will keep the BsmtFinType1 and 2 variables as they already represent FinSF1 and 2.

Exposure has positive correlations with many of the variables meaning that is represented by them and can be dropped.

Cond has a strong correlation with Qual so I drop Qual.
"""
df_train.drop(columns = ["BsmtUnfSF", "BsmtFinSF1","BsmtFinSF2","BsmtExposure","BsmtQual"], axis = 1, inplace=True)
plt.figure(figsize=(16,10))

corr = df_garage.corr()
ax = sns.heatmap(
    corr,
    vmin=-1, vmax=1, center=0,
    cmap=sns.diverging_palette(20, 220, n=200),
    square=True
)
ax.set_xticklabels(
    ax.get_xticklabels(),
    rotation=45,
    horizontalalignment='right'
)

plt.show()
"""
New garages have more area.

GarageQual and GarageCond are highly correlated, drop one

GarageCars and GarageArea are highly correlated, drop GarageCars

High Quality Garages are Finished, drop GarageFinish
"""
df_train.drop(columns = ["GarageQual", "GarageCars","GarageFinish"], axis = 1, inplace=True)
total = df_train.isnull().sum().sort_values(ascending=False)
percent = (df_train.isnull().sum()/df_train.isnull().count()).sort_values(ascending=False)
missing_data = pd.concat([total, percent], axis=1, keys=['Total', 'Percent'])
missing_data.head(5)
df_train.columns
# taken from: https://www.kaggle.com/pmarcelino/comprehensive-data-exploration-with-python
#saleprice correlation matrix
plt.figure(figsize=(10,10))
k = 10 #number of variables for heatmap
corrmat = df_train.corr()
cols = corrmat.nlargest(k, 'SalePrice')['SalePrice'].index
cm = np.corrcoef(df_train[cols].values.T)
sns.set(font_scale=1.25)
hm = sns.heatmap(cm, cbar=True, annot=True, square=True, fmt='.2f', annot_kws={'size': 10}, yticklabels=cols.values, xticklabels=cols.values)
plt.show()
"""
The features that have the most strong correlation with our target variable SalePrice are OverallQual and GrLivArea

TotalBsmtSF is highly correlated with 1stFlrSF. We can create a new feature to represent them together and drop the existing features.
"""
df_train["TotalSF"] = df_train["TotalBsmtSF"] + df_train["1stFlrSF"] + df_train["2ndFlrSF"]

df_test["TotalSF"] = df_test["TotalBsmtSF"] + df_test["1stFlrSF"] + df_test["2ndFlrSF"]
df_train.drop(columns = ["1stFlrSF", "2ndFlrSF", "TotalBsmtSF"], axis = 1, inplace=True)
"""
Our target variable is right skewed.
"""
data = pd.concat([df_train['SalePrice'], df_train['GrLivArea']], axis=1)
data.plot.scatter(x='GrLivArea', y='SalePrice');
"""
We have 2 points in the bottom right that does not follow the trend and therefore could negatively effect the model. They can be dropped.
"""
df_train.sort_values(by = 'GrLivArea', ascending = False)[:2] #id's are 1299 and 524
df_train = df_train.drop(df_train[df_train['Id'] == 1299].index)
df_train = df_train.drop(df_train[df_train['Id'] == 524].index)
data = pd.concat([df_train['SalePrice'], df_train['TotalSF']], axis=1)
data.plot.scatter(x='TotalSF', y='SalePrice');
# https://www.kaggle.com/pmarcelino/comprehensive-data-exploration-with-python
#box plot overallqual/saleprice
var = 'OverallQual'
data = pd.concat([df_train['SalePrice'], df_train[var]], axis=1)
f, ax = plt.subplots(figsize=(8, 6))
fig = sns.boxplot(x=var, y="SalePrice", data=data)
fig.axis(ymin=0, ymax=800000);
sns.displot(df_train['SalePrice'],kde=True)
# After log transformation
df_train['SalePrice'] = np.log(df_train['SalePrice'])
sns.displot(df_train['SalePrice'],kde=True)
sns.displot(df_train['GrLivArea'],kde=True)
# After log transformation
df_train['GrLivArea'] = np.log(df_train['GrLivArea'])
sns.displot(df_train['GrLivArea'],kde=True)
sns.displot(df_train['TotalSF'],kde=True)
result = np.where( df_train['TotalSF'] > 0.0000000001, df_train['TotalSF'], -10)

# After log transformation
df_train['TotalSF'] = np.log(result)
sns.displot(df_train['TotalSF'],kde=True)

df_train["MSSubClass"].value_counts()
"""
   MSSubClass has integer values but it is actually a categorical variable.  
     
        20	1-STORY 1946 & NEWER ALL STYLES
        30	1-STORY 1945 & OLDER
        40	1-STORY W/FINISHED ATTIC ALL AGES
        45	1-1/2 STORY - UNFINISHED ALL AGES
        50	1-1/2 STORY FINISHED ALL AGES
        60	2-STORY 1946 & NEWER
        70	2-STORY 1945 & OLDER
        75	2-1/2 STORY ALL AGES
        80	SPLIT OR MULTI-LEVEL
        85	SPLIT FOYER
        90	DUPLEX - ALL STYLES AND AGES
       120	1-STORY PUD (Planned Unit Development) - 1946 & NEWER
       150	1-1/2 STORY PUD - ALL AGES
       160	2-STORY PUD - 1946 & NEWER
       180	PUD - MULTILEVEL - INCL SPLIT LEV/FOYER
       190	2 FAMILY CONVERSION - ALL STYLES AND AGES
"""
df_train["MSSubClass"] = df_train["MSSubClass"].astype("str")
# taken from: https://www.kaggle.com/serigne/stacked-regressions-top-4-on-leaderboard

from scipy import stats
from scipy.stats import norm, skew 

numeric_feats = df_train.dtypes[df_train.dtypes != "object"].index

# Check the skew of all numerical features
skewed_feats = df_train[numeric_feats].apply(lambda x: skew(x.dropna())).sort_values(ascending=False)
print("\nSkew in numerical features: \n")
skewness = pd.DataFrame({'Skew' :skewed_feats})
skewness.head(10)
skewness = skewness[abs(skewness) > 0.75]
print("There are {} skewed numerical features to Box Cox transform".format(skewness.shape[0]))
skewed_features = skewness.index

count = 0
for feature in skewed_features:
    df_train[feature] = np.log(df_train[feature])
    count += 1

print("Transfromed ", count, " features.")
df_train.columns
categoric_feats = df_train.dtypes[df_train.dtypes == "object"].index
categoric_feats
"""
Decide map or label encode

* MSSubClass - LE
* MsZoning - LE
* Street - LE
* **LotShape - map**
* Land Contour - LE
* **Utilities - map**
* Lot Config - LE
* **Land Slope - map**
* Neighborhood - LE
* Condition1 - LE
* Condition2 - LE
* BldgType - LE
* HouseStyle - LE
* RoofStyle - LE
* RoofMatl - LE
* Exterior1st - LE
* Exterior2nd - LE
* MasVnrType - LE
* **ExterQual - map**
* **ExterCond - map**
* Foundation - LE
* Heating - LE
* CentralAir - LE
* Electrical - LE
* **KitchenQual - map**
* **Functional - map**
* **PavedDrive - map**
* SaleType - LE
* SaleCondition - LE
"""
mapping = {"Reg":4,"IR1":3,"IR2":2,"IR3":1}
          
df_train["LotShape"] = df_train["LotShape"].apply(lambda x : mapping[x])
df_test["LotShape"] = df_test["LotShape"].apply(lambda x : mapping[x])
df_test["Utilities"].fillna("AllPub")
mapping = {"AllPub":4,"NoSewr":3,"NoSeWa":2,"ELO":1,np.nan:0}
          
df_train["Utilities"] = df_train["Utilities"].apply(lambda x : mapping[x])
df_test["Utilities"] = df_test["Utilities"].apply(lambda x : mapping[x])
mapping = {"Gtl":3,"Mod":2,"Sev":1}
          
df_train["LandSlope"] = df_train["LandSlope"].apply(lambda x : mapping[x])
df_test["LandSlope"] = df_test["LandSlope"].apply(lambda x : mapping[x])
mapping = {"Ex":5,"Gd":4,"TA":3,"Fa":2,"Po":1}
          
df_train["ExterQual"] = df_train["ExterQual"].apply(lambda x : mapping[x])
df_test["ExterQual"] = df_test["ExterQual"].apply(lambda x : mapping[x])
mapping = {"Ex":5,"Gd":4,"TA":3,"Fa":2,"Po":1}

df_train["ExterCond"] = df_train["ExterCond"].apply(lambda x : mapping[x])
df_test["ExterCond"] = df_test["ExterCond"].apply(lambda x : mapping[x])
df_test["KitchenQual"].fillna("TA", inplace=True)
df_test["KitchenQual"].isnull().sum()
mapping = {"Ex":5,"Gd":4,"TA":3,"Fa":2,"Po":1}

df_train["KitchenQual"] = df_train["KitchenQual"].apply(lambda x : mapping[x])
df_test["KitchenQual"] = df_test["KitchenQual"].apply(lambda x : mapping[x])
df_test["Functional"].fillna("Typ", inplace=True)
mapping = {"Typ":8,"Min1":7,"Min2":6,"Mod":5,"Maj1":4,"Maj2":3,"Sev":2,"Sal":1}
          
df_train["Functional"] = df_train["Functional"].apply(lambda x : mapping[x])
df_test["Functional"] = df_test["Functional"].apply(lambda x : mapping[x])
mapping = {"Y":3,"P":2,"N":1}
          
df_train["PavedDrive"] = df_train["PavedDrive"].apply(lambda x : mapping[x])
df_test["PavedDrive"] = df_test["PavedDrive"].apply(lambda x : mapping[x])
df_train["MSSubClass"] = label.fit_transform(df_train["MSSubClass"])
df_train["MSZoning"] = label.fit_transform(df_train["MSZoning"])
df_train["Street"] = label.fit_transform(df_train["Street"])
df_train["LandContour"] = label.fit_transform(df_train["LandContour"])
df_train["LotConfig"] = label.fit_transform(df_train["LotConfig"])
df_train["Neighborhood"] = label.fit_transform(df_train["Neighborhood"])
df_train["Condition1"] = label.fit_transform(df_train["Condition1"])
df_train["Condition2"] = label.fit_transform(df_train["Condition2"])
df_train["BldgType"] = label.fit_transform(df_train["BldgType"])
df_train["HouseStyle"] = label.fit_transform(df_train["HouseStyle"])
df_train["RoofStyle"] = label.fit_transform(df_train["RoofStyle"])
df_train["RoofMatl"] = label.fit_transform(df_train["RoofMatl"])
df_train["Exterior1st"] = label.fit_transform(df_train["Exterior1st"])
df_train["Exterior2nd"] = label.fit_transform(df_train["Exterior2nd"])
df_train["MasVnrType"] = label.fit_transform(df_train["MasVnrType"])
df_train["HeatingQC"] = label.fit_transform(df_train["HeatingQC"])
df_train["Foundation"] = label.fit_transform(df_train["Foundation"])
df_train["Heating"] = label.fit_transform(df_train["Heating"])
df_train["CentralAir"] = label.fit_transform(df_train["CentralAir"])
df_train["Electrical"] = label.fit_transform(df_train["Electrical"])
df_train["SaleType"] = label.fit_transform(df_train["SaleType"])
df_train["SaleCondition"] = label.fit_transform(df_train["SaleCondition"])
df_test["MSZoning"] = df_test["MSZoning"].astype("str")
df_test["Exterior1st"] = df_test["Exterior1st"].astype("str")
df_test["Exterior2nd"] = df_test["Exterior2nd"].astype("str")
df_test["MasVnrType"] = df_test["MasVnrType"].astype("str")
df_test["SaleType"] = df_test["SaleType"].astype("str")
df_test["MSSubClass"] = label.fit_transform(df_test["MSSubClass"])
df_test["MSZoning"] = label.fit_transform(df_test["MSZoning"])
df_test["Street"] = label.fit_transform(df_test["Street"])
df_test["LandContour"] = label.fit_transform(df_test["LandContour"])
df_test["LotConfig"] = label.fit_transform(df_test["LotConfig"])
df_test["Neighborhood"] = label.fit_transform(df_test["Neighborhood"])
df_test["Condition1"] = label.fit_transform(df_test["Condition1"])
df_test["Condition2"] = label.fit_transform(df_test["Condition2"])
df_test["BldgType"] = label.fit_transform(df_test["BldgType"])
df_test["HouseStyle"] = label.fit_transform(df_test["HouseStyle"])
df_test["RoofStyle"] = label.fit_transform(df_test["RoofStyle"])
df_test["RoofMatl"] = label.fit_transform(df_test["RoofMatl"])
df_test["Exterior1st"] = label.fit_transform(df_test["Exterior1st"])
df_test["Exterior2nd"] = label.fit_transform(df_test["Exterior2nd"])
df_test["MasVnrType"] = label.fit_transform(df_test["MasVnrType"])
df_test["HeatingQC"] = label.fit_transform(df_test["HeatingQC"])
df_test["Foundation"] = label.fit_transform(df_test["Foundation"])
df_test["Heating"] = label.fit_transform(df_test["Heating"])
df_test["CentralAir"] = label.fit_transform(df_test["CentralAir"])
df_test["Electrical"] = label.fit_transform(df_test["Electrical"])
df_test["SaleType"] = label.fit_transform(df_test["SaleType"])
df_test["SaleCondition"] = label.fit_transform(df_test["SaleCondition"])
df_train.info()
"""
No object values =)
"""
df_train = df_train.reset_index()
df_train.shape
df_train.columns
df_train.describe()
df_train.columns.to_series()[np.isinf(df_train).any()]
df_train.replace([np.inf, -np.inf], np.nan, inplace=True)
df_test.replace([np.inf, -np.inf], np.nan, inplace=True)
df_test.fillna(0)
df_train["MasVnrArea"] = df_train["MasVnrArea"].fillna(df_train["MasVnrArea"].median())
df_train["LowQualFinSF"] = df_train["LowQualFinSF"].fillna(df_train["LowQualFinSF"].median())
df_train["BsmtFullBath"] = df_train["BsmtFullBath"].fillna(df_train["BsmtFullBath"].mean())
df_train["BsmtHalfBath"] = df_train["BsmtHalfBath"].fillna(df_train["BsmtHalfBath"].mean())
df_train["FullBath"] = df_train["FullBath"].fillna(df_train["FullBath"].mean())
df_train["HalfBath"] = df_train["HalfBath"].fillna(df_train["HalfBath"].mean())
df_train["BedroomAbvGr"] = df_train["BedroomAbvGr"].fillna(df_train["BedroomAbvGr"].median())
df_train["KitchenAbvGr"] = df_train["KitchenAbvGr"].fillna(df_train["KitchenAbvGr"].mean())
df_train["Fireplaces"] = df_train["Fireplaces"].fillna(df_train["Fireplaces"].mean())
df_train["GarageType"] = df_train["GarageType"].fillna(df_train["GarageType"].mean())
df_train["GarageArea"] = df_train["GarageArea"].fillna(df_train["GarageArea"].median())
df_train["WoodDeckSF"] = df_train["WoodDeckSF"].fillna(df_train["WoodDeckSF"].median())
df_train["OpenPorchSF"] = df_train["OpenPorchSF"].fillna(df_train["OpenPorchSF"].median())
df_train["EnclosedPorch"] = df_train["EnclosedPorch"].fillna(df_train["EnclosedPorch"].median())
df_train["3SsnPorch"] = df_train["3SsnPorch"].fillna(df_train["3SsnPorch"].median())
df_train["ScreenPorch"] = df_train["ScreenPorch"].fillna(df_train["ScreenPorch"].median())
df_train["PoolArea"] = df_train["PoolArea"].fillna(df_train["PoolArea"].median())
df_train["MiscVal"] = df_train["MiscVal"].fillna(df_train["MiscVal"].median())
df_train.columns.to_series()[np.isinf(df_train).any()]
def clean_dataset(df_train):
    assert isinstance(df_train, pd.DataFrame), "df needs to be a pd.DataFrame"
    df_train.dropna(inplace=True)
    indices_to_keep =~df_train.isin([np.nan, np.inf, -np.inf]).any(1)
    return df_train[indices_to_keep].astype(np.float64)
clean_dataset(df_train)
sf_median = df_test["TotalSF"].median()
df_test["TotalSF"] = df_test["TotalSF"].fillna(sf_median)
df_test["MasVnrArea"] = df_test["MasVnrArea"].fillna(0)
df_test["BsmtFullBath"] = df_test["BsmtFullBath"].fillna(0)
df_test["BsmtHalfBath"] = df_test["BsmtHalfBath"].fillna(0)
gyrmed = df_test["GarageYrBlt"].median()
df_test["GarageYrBlt"] = df_test["GarageYrBlt"].fillna(gyrmed)
gareamed = df_test["GarageArea"].median()
df_test["GarageArea"] = df_test["GarageArea"].fillna(gareamed)
df_train = df_train.drop(["index"],axis=1)

y  = df_train["SalePrice"]
X = df_train.drop(["SalePrice"],axis=1)
from sklearn.model_selection import train_test_split

X_train, X_test, y_train, y_test = train_test_split(X, y, shuffle=True,random_state=42)
 test_X = df_test[['Id', 'MSSubClass', 'MSZoning', 'LotArea', 'Street', 'LotShape',
       'LandContour', 'Utilities', 'LotConfig', 'LandSlope', 'Neighborhood',
       'Condition1', 'Condition2', 'BldgType', 'HouseStyle', 'OverallQual',
       'OverallCond', 'YearBuilt', 'YearRemodAdd', 'RoofStyle', 'RoofMatl',
       'Exterior1st', 'Exterior2nd', 'MasVnrType', 'MasVnrArea', 'ExterQual',
       'ExterCond', 'Foundation', 'BsmtCond', 'BsmtFinType1', 'BsmtFinType2',
       'Heating', 'HeatingQC', 'CentralAir', 'Electrical', 'LowQualFinSF',
       'GrLivArea', 'BsmtFullBath', 'BsmtHalfBath', 'FullBath', 'HalfBath',
       'BedroomAbvGr', 'KitchenAbvGr', 'KitchenQual', 'TotRmsAbvGrd',
       'Functional', 'Fireplaces', 'GarageType', 'GarageYrBlt', 'GarageArea',
       'GarageCond', 'PavedDrive', 'WoodDeckSF', 'OpenPorchSF',
       'EnclosedPorch', '3SsnPorch', 'ScreenPorch', 'PoolArea', 'MiscVal',
       'MoSold', 'YrSold', 'SaleType', 'SaleCondition','TotalSF']]
from sklearn.linear_model import LinearRegression
linr = LinearRegression()
  
linr.fit(X_train,y_train)
y_preds = linr.predict(X_test)
sns.regplot(x=y_test, y=y_preds, ci=None, color="r")
from sklearn.metrics import mean_squared_error

rmse = mean_squared_error(y_test, y_preds, squared=False)
rmse
from sklearn.metrics import r2_score
print(r2_score(y_test, y_preds))
from sklearn.ensemble import RandomForestRegressor

rf = RandomForestRegressor(random_state=42)

rf.fit(X_train, y_train)
rf_preds = rf.predict(X_test)
rmse = mean_squared_error(y_test, rf_preds, squared=False)
rmse
print(r2_score(y_test, rf_preds))
sns.regplot(x=y_test, y=rf_preds, ci=None, color="r")
from sklearn.linear_model import Ridge

rr = Ridge(alpha=0.005)

rr.fit(X_train, y_train)

rr_preds = rr.predict(X_test)

rmse = mean_squared_error(y_test, rr_preds, squared=False)
rmse
print(r2_score(y_test, rr_preds))
from sklearn.linear_model import Lasso

ls = Lasso(alpha=0.000001)

ls.fit(X_train, y_train)

ls_preds = ls.predict(X_test)

rmse = mean_squared_error(y_test, ls_preds, squared=False)
rmse
print(r2_score(y_test, ls_preds))
preds = linr.predict(test_X)
coefficients = pd.concat([pd.DataFrame(X.columns),pd.DataFrame(np.transpose(linr.coef_))], axis = 1)


coefficients = coefficients.rename(columns={coefficients.columns[0]:"Features",
                                            coefficients.columns[1]:"Weight"})


coefficients[:20]
coefficients[20:40]
coefficients[40:64]
submission = pd.DataFrame({
    'Id': np.asarray(df_test.Id), 
    'SalePrice': preds.astype(int)
})
submission.to_csv('my_submission.csv', index=False)