from IPython.display import Image
Image("../input/image-house-prices-advanced-regression/image.jpeg")

"""
#  Predict sales prices and practice feature engineering, RFs, and gradient boosting

## 1. Problem defition


> Predict the value of the SalePrice variable. 

## 2. Data


https://www.kaggle.com/c/house-prices-advanced-regression-techniques/data



    **Data Details:**
    
* Here's a brief version of what you'll find in the data description file.
* SalePrice - the property's sale price in dollars. This is the target variable that you're trying to predict.
* MSSubClass: The building class
* MSZoning: The general zoning classification
* LotFrontage: Linear feet of street connected to property
* LotArea: Lot size in square feet
* Street: Type of road access
* Alley: Type of alley access
* LotShape: General shape of property
* LandContour: Flatness of the property
* Utilities: Type of utilities available
* LotConfig: Lot configuration
* LandSlope: Slope of property
* Neighborhood: Physical locations within Ames city limits
* Condition1: Proximity to main road or railroad
* Condition2: Proximity to main road or railroad (if a second is present)
* BldgType: Type of dwelling
* HouseStyle: Style of dwelling
* OverallQual: Overall material and finish quality
* OverallCond: Overall condition rating
* YearBuilt: Original construction date
* YearRemodAdd: Remodel date
* RoofStyle: Type of roof
* RoofMatl: Roof material
* Exterior1st: Exterior covering on house
* Exterior2nd: Exterior covering on house (if more than one material)
* MasVnrType: Masonry veneer type
* MasVnrArea: Masonry veneer area in square feet
* ExterQual: Exterior material quality
* ExterCond: Present condition of the material on the exterior
* Foundation: Type of foundation
* BsmtQual: Height of the basement
* BsmtCond: General condition of the basement
* BsmtExposure: Walkout or garden level basement walls
* BsmtFinType1: Quality of basement finished area
* BsmtFinSF1: Type 1 finished square feet
* BsmtFinType2: Quality of second finished area (if present)
* BsmtFinSF2: Type 2 finished square feet
* BsmtUnfSF: Unfinished square feet of basement area
* Total square feet of basement area
* Heating: Type of heating
* HeatingQC: Heating quality and condition
* CentralAir: Central air conditioning
* Electrical: Electrical system
* 1stFlrSF: First Floor square feet
* 2ndFlrSF: Second floor square feet
* LowQualFinSF: Low quality finished square feet (all floors)
* GrLivArea: Above grade (ground) living area square feet
* BsmtFullBath: Basement full bathrooms
* BsmtHalfBath: Basement half bathrooms
* FullBath: Full bathrooms above grade
* HalfBath: Half baths above grade
* Bedroom: Number of bedrooms above basement level
* Kitchen: Number of kitchens
* KitchenQual: Kitchen quality
* TotRmsAbvGrd: Total rooms above grade (does not include bathrooms)
* Functional: Home functionality rating
* Fireplaces: Number of fireplaces
* FireplaceQu: Fireplace quality
* GarageType: Garage location
* GarageYrBlt: Year garage was built
* GarageFinish: Interior finish of the garage
* GarageCars: Size of garage in car capacity
* GarageArea: Size of garage in square feet
* GarageQual: Garage quality
* GarageCond: Garage condition
* PavedDrive: Paved driveway
* WoodDeckSF: Wood deck area in square feet
* OpenPorchSF: Open porch area in square feet
* EnclosedPorch: Enclosed porch area in square feet
* 3SsnPorch: Three season porch area in square feet
* ScreenPorch: Screen porch area in square feet
* PoolArea: Pool area in square feet
* PoolQC: Pool quality
* Fence: Fence quality
* MiscFeature: Miscellaneous feature not covered in other categories
* MiscVal: $Value of miscellaneous feature
* MoSold: Month Sold
* YrSold: Year Sold
* SaleType: Type of sale
* SaleCondition: Condition of sale




"""
"""
##  Evaluation

 The evaluation metric for this competition is the RMSLE (root mean squared log error) between the actual and predicted auction prices
 
 https://www.kaggle.com/c/house-prices-advanced-regression-techniques/overview/evaluation


"""
"""
# 1. Data
"""
"""
## 1.1 Import Data
"""
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import sklearn
import xgboost
df_train = pd.read_csv('../input/house-prices-advanced-regression-techniques/train.csv')
df_train.head()
df_train.shape
df_train.dtypes
df_train.describe()
df_train.isna().sum()
df_train.corr()
# Skewness
df_train.skew()
class_counts = df_train['SalePrice'].value_counts()
class_counts
"""
## 1.2 Data Visualisation
"""
# Histogram
df_train.hist(figsize=(20, 10))
# Scatter plot
fig, ax = plt.subplots()
ax.scatter(df_train['Id'], df_train['SalePrice'])
"""
## 1.3 Data preparation
"""
def preprocess_data(df):
    """
    Performs transformations on df and returns transformed df.
    """
    # Fill the numeric rows with median
    for label, content in df.items():
        if pd.api.types.is_numeric_dtype(content):
            if pd.isnull(content).sum():
                # Add a binary column which tells us if the data was missing or not
                df[label+"_is_missing"] = pd.isnull(content)
                # Fill missing numeric values with median
                df[label] = content.fillna(content.median())
    
        # Filled categorical missing data and turn categories into numbers
        if not pd.api.types.is_numeric_dtype(content):
            df[label+"_is_missing"] = pd.isnull(content)
            # We add +1 to the category code because pandas encodes missing categories as -1
            df[label] = pd.Categorical(content).codes+1
    
    return df
# Process the test data 
df_train = preprocess_data(df_train)
df_train.head()
# Process the test data 
df_train = preprocess_data(df_train)
df_train.head()
df_train.info()
"""
### Modelling
"""
"""
 ## 2.1 Building an evaluation function
"""
from sklearn.linear_model import LogisticRegression
from sklearn.ensemble import RandomForestRegressor
from sklearn import linear_model
from sklearn.linear_model import LinearRegression, Lasso
from xgboost import XGBRegressor
from sklearn.model_selection import train_test_split
from sklearn.metrics import mean_squared_log_error, mean_absolute_error, r2_score

X = df_train.drop('SalePrice', axis=1)
y = df_train['SalePrice']

X_train, X_test, y_train, y_test = train_test_split(X, y, test_size= 0.2)

# Put models in a dictionary
models = {'Logistic Regression': LogisticRegression(),
          'linear_regression' : LinearRegression(),
          'Random Forest': RandomForestRegressor(),
          'linear_model' : linear_model.Lasso(alpha=0.1),
          'XGBoost' : XGBRegressor()
           
          }

# Create a function to fit and score models
def fit_and_score(models, X_train, X_test, y_train, y_test):
    """
    Fits and evaluates given machine learning models.
    models : a dict of differetn Scikit-Learn machine learning models
    X_train : training data (no labels)
    X_test : validation data (no labels)
    y_train : training labels
    y_test : validation labels
    """
    # Set random seed
    np.random.seed(42)
    # Make a dictionary to keep model scores
    model_scores = {}
    # Loop through models
    for name, model in models.items():
        # Fit the model to the data
        model.fit(X_train, y_train)
        # Evaluate the model and append its score to model_scores
        model_scores[name] = model.score(X_test, y_test)
    return model_scores
    
model_scores = fit_and_score(models=models,
                             X_train=X_train,
                             X_test= X_test,
                             y_train=y_train,
                             y_test=y_test)

model_scores


"""
## 2.2 Evaluate Models
"""
models = {'Logistic Regression': LogisticRegression(),
          'linear_regression' : LinearRegression(),
          'Random Forest': RandomForestRegressor(),
          'linear_model' : linear_model.Lasso(alpha=0.1),
          'XGBoost' : XGBRegressor()
           
          }

# Logistic Regression
model1 = LogisticRegression()
model1.fit(X_train, y_train)
y_preds = model1.predict(X_test)
print('Logistic Regression')
print('MAE', mean_absolute_error(y_test, y_preds))
print('RMSLE', mean_squared_log_error(y_test, y_preds))
print('r-squared', r2_score (y_test, y_preds))

# linear_regression
model2 = LinearRegression()
model2 = LogisticRegression()
model2.fit(X_train, y_train)
y_preds = model2.predict(X_test)
print('linear_regression')
print('MAE', mean_absolute_error(y_test, y_preds))
print('RMSLE', mean_squared_log_error(y_test, y_preds))
print('r-squared', r2_score (y_test, y_preds))

# Random Forest
model3 = RandomForestRegressor()
model3.fit(X_train, y_train)
y_preds = model3.predict(X_test)
print('Random Forest')
print('MAE', mean_absolute_error(y_test, y_preds))
print('RMSLE', mean_squared_log_error(y_test, y_preds))
print('r-squared', r2_score (y_test, y_preds))

# linear_model
model4 = linear_model.Lasso(alpha=0.1)
model4.fit(X_train, y_train)
y_preds = model4.predict(X_test)
print('linear_model')
print('MAE', mean_absolute_error(y_test, y_preds))
print('RMSLE', mean_squared_log_error(y_test, y_preds))
print('r-squared', r2_score (y_test, y_preds))

# XGBoost
model5 = XGBRegressor()
model5.fit(X_train, y_train)
y_preds = model5.predict(X_test)
print('XGBoost')
print('MAE', mean_absolute_error(y_test, y_preds))
print('RMSLE', mean_squared_log_error(y_test, y_preds))
print('r-squared', r2_score (y_test, y_preds))





"""
### Choising the model XGBOOST
score: 0.828759856683738
RMSLE 0.021842740178599024   
"""
"""
## Testing our model on a subset (to tune the hyperparameters)
"""
%%time

# Instantiate model
model = XGBRegressor()

# Fit the model
model.fit(X_train, y_train)
y_preds = model.predict(X_test)


"""
### Hyerparameter tuning with GridSearchCV

"""
from sklearn.model_selection import GridSearchCV
grid = {'n_estimators' : [10, 1000],
        'learning_rate': [0.01, 0.1],
        'gamma' : [0.1, 5],
        'max_depth' : [3, 5, 8], 
        'subsample' : [0.8 ,0.9, 1],
        'colsample_bytree' : [0.3, 0,8],
        'gamma' : [0.1, 5]
        
    
}


xgb_model = GridSearchCV(XGBRegressor(),

                              param_grid=grid,
                              
                              cv=5,
                              verbose=20)

xgb_model.fit(X_train, y_train)


# Find the best model hyperparameters
xgb_model.best_params_
best_model = XGBRegressor(colsample_bytree=0.3,
                          gamma=0.1,
                          learning_rate=0.01,
                          max_depth=5,
                          n_estimators=1000,
                          subsample=1)
# Fit the model
best_model.fit(X_train, y_train)
y_preds = best_model.predict(X_test)

best_model.score(X_test, y_test)

print('MAE', mean_absolute_error(y_test, y_preds))
print('MSLE', mean_squared_log_error(y_test, y_preds))
print('r-squared', r2_score (y_test, y_preds))

"""
### Make predictions on test data
"""
# Import the test data
df_test = pd.read_csv('../input/house-prices-advanced-regression-techniques/test.csv')
df_test.head()
# Process the test data 
df_test = preprocess_data(df_test)
df_test.head()
df_train.head()
set(df_test.columns) - set(X_train.columns)
# Manually adjust df_train to missing columns
df_test = df_test.drop('BsmtFinSF1_is_missing', axis=1)                            
df_test = df_test.drop('BsmtFinSF2_is_missing', axis=1)
df_test = df_test.drop('BsmtFullBath_is_missing', axis=1) 
df_test = df_test.drop('BsmtHalfBath_is_missing', axis=1)
df_test = df_test.drop('BsmtUnfSF_is_missing', axis=1)
df_test = df_test.drop('GarageArea_is_missing', axis=1)
df_test = df_test.drop('GarageCars_is_missing', axis=1)
df_test = df_test.drop('TotalBsmtSF_is_missing', axis=1)
df_test.head()
df_test.head()
# Make predictions on updated test data

model.fit(X_train, y_train)

test_preds = model.predict(df_test)
test_preds
"""
## Format predictions asked by Kaggle
"""
df_preds_test = pd.DataFrame()
df_preds_test['Id'] = df_test['Id']
df_preds_test["SalePrice"] = test_preds
df_preds_test
# Export prediction data
df_preds_test.to_csv("./HousePricesAdvanced.csv", index=False)
"""
## Feature importance
"""
# Feature Importance with Extra Trees Classifier
from sklearn.ensemble import ExtraTreesClassifier
# feature extraction
model = ExtraTreesClassifier()
model.fit(X, y)
print(model.feature_importances_)
import seaborn as sns

# Helper function for plotting feature importance
def plot_features(columns, importances, n=20):
    data = (pd.DataFrame({"features": columns,
                        "feature_importance": importances})
          .sort_values("feature_importance", ascending=False)
          .reset_index(drop=True))
    
    sns.barplot(x="feature_importance",
                y="features",
                data=data[:n],
                orient="h")
plt.figure(figsize=(20, 6))
plot_features(X_train.columns, best_model.feature_importances_)