# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

# # 🚜Predicting the Sale Price of Bulldozers using Machine Learning
#
# in this notebook, we're going to go through an example machine learning project with the goal of predicting the sales price of bulldozer.
#
# ## 1. Problem definition
#
# > How well can we predict the future sale price of a bulldozer, given its characteristics and previous examples of how much similar bulldozer have been sold for?
#
# ## 2. Data
#
# The data is downloaded from the Kaggle Bluebook for Bulldozers competitions: https://www.kaggle.com/c/bluebook-for-bulldozers/data
#
# There are three main datasets:
#
# * Train.csv is the training set, which contains data through the end of 2011.
# * Valid.csv is the validation set, which contains data from January 1, 2012 - April 30, 2012 You make predictions on this set throughout the majority of the competition. Your score on this set is used to create the public leaderboard.
# * Test.csv is the test set, which won't be released until the last week of the competition. It contains data from May 1, 2012 - November 2012. Your score on the test set determines your final rank for the competition.
#
# ## 3. Evaluation
#
# The evaluation metric for this competition is the RMSLE (root mean squared log error) between the actual and predicted auction prices.
#
# For more on the evaluation of this project check: https://www.kaggle.com/c/bluebook-for-bulldozers/overview/evaluation
#
# **Note:** The goal for most regression evaluation metrics is to minimize the error. For example, our goal for this project will be to build a machine learning model which minimizes RMSLE.
#
# ## 4. Features
#
# kaggle provides a data dictioary detailing all of the features of the dataset. You can view this data dictionary on Google Sheets: https://docs.google.com/spreadsheets/d/1jUumFSKpVst0k_peYIUIwIMVwLfJ_8MbGW-ATQvyBQI/edit?usp=sharing

import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import sklearn

# Import training and validation sets
df = pd.read_csv("data/bluebook-for-bulldozers/TrainAndValid.csv", low_memory=False)

df.info()

df.isna().sum()

df.columns

fig, ax = plt.subplots()
ax.scatter(df['saledate'][:1000], df["SalePrice"][:1000])

df.SalePrice.plot.hist()

# ### Parsing dates
#
# When we work with time series data, we want to enrich the time & date component as much as possible.
#
# We can do that by telling pandas which of our columns has dates in it using the `parse_dates` parameter.

# import data again but this time parse dates
df = pd.read_csv("data/bluebook-for-bulldozers/TrainAndValid.csv", low_memory=False, parse_dates=["saledate"])

df.saledate.dtype

fig, ax = plt.subplots()
ax.scatter(df['saledate'][:1000], df["SalePrice"][:1000])

df.head()

df.head().T

# ### Sort DataFrame by saledate
#
# When working with time series data, it's a good idea to sort it by date.

# Sort DataFrame in date order
df.sort_values(by=["saledate"], inplace=True, ascending=True)
df.saledate.head(20)

# ### MAke a copy of the original DataFrame
#
# We make a copy of the riginal DataFraame so when we manipulate the copy, we've still got our original data.

# make a copy
df_tmp = df.copy()

df_tmp.saledate.head(20)

# ### Add datetime parameters for `saledate` column

df_tmp[:1].saledate.dt.year

df_tmp[:1].saledate.dt.day

df_tmp[:1].saledate

df_tmp["saleYear"] = df_tmp.saledate.dt.year
df_tmp["saleMonth"] = df_tmp.saledate.dt.month
df_tmp["saleDay"] = df_tmp.saledate.dt.day
df_tmp["saleDayOfWeek"] = df_tmp.saledate.dt.dayofweek
df_tmp["saleDayOfYear"] = df_tmp.saledate.dt.dayofyear

df_tmp.head()

# Now we've enriched our DAtaFrame with date time features, wecan remove `saledate`
df_tmp.drop("saledate", axis=1, inplace=True)

# Check the values of different clumns
df_tmp.state.value_counts()

len(df_tmp)

# ## 5.Modelling
#
# we've done enough EDA(we could always do more) but let's start to do some model-drivenEDA.

# +
# Let's build a machine learning model
from sklearn.ensemble import RandomForestRegressor

model = RandomForestRegressor(n_jobs=-1,
                             random_state=42)
model.fit(df_tmp.drop("SalePrice", axis=1), df_tmp["SalePrice"])
# -

# ### Covert string to categories
#
# One way we can turn all our data into numbers is by converting them into pandas categories.
#
# We can check the different datatypes compatible with pandas here: https://pandas.pydata.org/pandas-docs/stable/reference/general_utility_functions.html

pd.api.types.is_string_dtype(df_tmp["UsageBand"])

# Find the columns which contains string
for label, content in df_tmp.items():
    if pd.api.types.is_string_dtype(content):
        print(label)

# if you're wondering what df.items() does, here's an eg.
rand_dict = {"k1": "Hell0",
            "k2": "Aysha!"}
for key, value in rand_dict.items():
    print(f"key: {key}, value: {value}")

# This will turn all of the string into category values 
for label, content in df_tmp.items():
    if pd.api.types.is_string_dtype(content):
        df_tmp[label] = content.astype("category").cat.as_ordered()

df_tmp.info()

df_tmp.state.cat.categories

df_tmp.state.cat.codes

# Thanks to pandas Categories we now have a way to access all of our data in the form of numbers.
#
# But we still have a bunch of missing datas...

# Check missing datas
df_tmp.isnull().sum()/len(df_tmp)

# ### Save preprocessed data

# Export current tmp dataframe
df_tmp.to_csv("data/bluebook-for-bulldozers/train_tmp.csv", 
             index=False)

# Import preprocessed data
df_tmp = pd.read_csv("data/bluebook-for-bulldozers/train_tmp.csv",
                    low_memory=False)
df_tmp.head().T

df_tmp.isna().sum()

# ### Fill missing values

# #### Fill numerical missing values first

for label, content in df_tmp.items():
    if pd.api.types.is_numeric_dtype(content):
        print(label)

df_tmp.ModelID

# Check for which numeric columns have null values
for label, content in df_tmp.items():
    if pd.api.types.is_numeric_dtype(content):
        if pd.isnull(content).sum():
            print(label)

# Fill the numeric rows with the median
for label, content in df_tmp.items():
    if pd.api.types.is_numeric_dtype(content):
        if pd.isnull(content).sum():
            # Add a binary column which tells us if the data was missing or not
            df_tmp[label+"_is_missing"] = pd.isnull(content)
            # Fill missing numeric values with median
            df_tmp[label] = content.fillna(content.median())

# Check for which numeric columns have null values
for label, content in df_tmp.items():
    if pd.api.types.is_numeric_dtype(content):
        if pd.isnull(content).sum():
            print(label)

# Check to see how many examples were missing 
df_tmp.auctioneerID_is_missing.value_counts()

df_tmp.isna().sum()

# ### Filling and turning categorical variables into numbers

# Check for columns which aren't numeric
for label, content in df_tmp.items():
    if not pd.api.types.is_numeric_dtype(content):
        print(label)

# Turn categorical variables into numbers and fill missing
for label, content in df_tmp.items():
    if not pd.api.types.is_numeric_dtype(content):
        # Add binary coulmn to indicate whether sample had missing values
        df_tmp[label+"_is_missing"] = pd.isnull(content)
        # Turn categories into numbers and add + 1 
        df_tmp[label] = pd.Categorical(content).codes+1

df_tmp.info()

df_tmp.head().T

df_tmp.isna().sum()

# Now that allof our data is numeric as well as our datafrmae has no missing values, we should be able to build a machine learning model.

from sklearn.ensemble import RandomForestRegressor

# +
# %%time

# Instantiate model
model = RandomForestRegressor(n_jobs=-1,
                             random_state=42)
# Fit the model
model.fit(df_tmp.drop("SalePrice", axis=1), df_tmp["SalePrice"])
# -

model.score(df_tmp.drop("SalePrice", axis=1), df_tmp["SalePrice"])

# **Question:**Why does'nt the above metric hold water? (why isn't the metric reliable)

# ### Splitting data into train/validation sets

df_tmp.saleYear.value_counts()

# +
# Split data into training and validation set
df_val = df_tmp[df_tmp.saleYear == 2012]
df_train = df_tmp[df_tmp.saleYear != 2012]

len(df_train), len(df_val)

# +
# Split data into x and y
x_train, y_train = df_train.drop("SalePrice", axis=1), df_train.SalePrice 
x_valid, y_valid = df_val.drop("SalePrice", axis=1), df_val.SalePrice

x_train.shape, y_train.shape, x_valid.shape, y_valid.shape
# -

y_train

# ### Building an evaluation function

# +
# Create evaluation function (the competition uses RMSLE)
from sklearn.metrics import mean_squared_log_error, mean_absolute_error, r2_score

def rmsle(y_test, y_preds):
    '''
    Calculate root mean squared log error between predictions and true labels.
    '''
    return np.sqrt(mean_squared_log_error(y_test, y_preds))

# Create Function to evalaute model on a few different levels
def show_scores(model):
    train_preds = model.predict(x_train)
    val_preds = model.predict(x_valid)
    scores = {"Training MAE": mean_absolute_error(y_train, train_preds),
             "Valid MAE": mean_absolute_error(y_valid, val_preds), 
             "Training RMSLE": rmsle(y_train, train_preds),
             "Valid RMSLE": rmsle(y_valid, val_preds),
             "Training R^2": r2_score(y_train, train_preds),
             "Valid R^2": r2_score(y_valid, val_preds)}
    return scores



# -

# ## Testing our modle on a subset (to tune the hyperparameters)

# + active=""
# # This takes far too long... for experimenting
#
# %%time
# model = RandomForestRegressor(n_jobs=-1,
#                              random_state=42)
# model.fit(x_train, y_train)
# -

len(x_train)

# Change max_samples value
model = RandomForestRegressor(n_jobs=-1,
                             random_state=42,
                             max_samples=10000)

# %%time
# Cutting down on the max number of samples each estimator can see improves training time
model.fit(x_train, y_train)

show_scores(model)

# ### Hyperparameter tuning with RandomizedSearchCV

# +
# %%time
from sklearn.model_selection import RandomizedSearchCV

# Different RandomForestRegressor hyperparameters
rf_grid = {"n_estimators": np.arange(10, 100, 10),
          "max_depth": [None, 3, 5, 10],
          "min_samples_split": np.arange(2, 20, 2),
          "min_samples_leaf": np.arange(1, 20, 2),
          "max_features": [0.5, 1, "sqrt", "auto"],
          "max_samples": [10000]}
# Instantiate RandomizedSearchCV model
rs_model= RandomizedSearchCV(RandomForestRegressor(n_jobs=-1,
                                                  random_state=42),
                            param_distributions=rf_grid,
                            n_iter=2,
                            cv=5,
                            verbose=True)
# Fit the RandomizedSearchCV model
rs_model.fit(x_train, y_train)

# + active=""
# # Find the best model hyperparameters
# rs_model.best_params_
# # O/P
# # {'n_estimators': 50,
# #  'min_samples_split': 10,
# #  'min_samples_leaf': 9,
# #  'max_samples': 10000,
# #  'max_features': 'auto',
# #  'max_depth': 3}

# + active=""
# # Evaluate the RandomizedSearch model
# show_scores(rs_model)
# #O/P
# # {'Training MAE': 11600.450217388972,
# #  'Valid MAE': 13169.983584964879,
# #  'Training RMSLE': 0.49468301761355865,
# #  'Valid RMSLE': 0.49954812812940574,
# #  'Training R^2': 0.49581349333422875,
# #  'Valid R^2': 0.48895962228392753}
# -

# ### Train a model with the best hyperparameters
# **Note:** These were found after 100 iterations of `RandomizesSearchCV`.
#

# +
# %%time

# Most ideal model
ideal_model = RandomForestRegressor(n_estimators=50,
                                   min_samples_leaf=9,
                                   min_samples_split=10,
                                   max_features='auto',
                                   n_jobs=-1,
                                   max_samples=None,
                                   random_state=42)# random state so our results are reproducible
# Fit the ideal model
ideal_model.fit(x_train, y_train)

# + active=""
# # Scores for ideal model (trained on all the data)
# show_scores(ideal_model)
# # O/P
# # {'Training MAE': 3620.6459259911267,
# #  'Valid MAE': 6120.019427314366,
# #  'Training RMSLE': 0.17576470168353692,
# #  'Valid RMSLE': 0.251602610379688,
# #  'Training R^2': 0.9352797759068263,
# #  'Valid R^2': 0.8698400358096663}

# + active=""
# # Scores on rs_model (only trained on 10000 examples)
# show_scores(rs_model)
# # O/P
# # {'Training MAE': 11600.450217388974,
# #  'Valid MAE': 13169.983584964879,
# #  'Training RMSLE': 0.49468301761355865,
# #  'Valid RMSLE': 0.49954812812940574,
# #  'Training R^2': 0.49581349333422886,
# #  'Valid R^2': 0.48895962228392753}
# -

# ### Make predictions on our test data

# importing the test data
df_test = pd.read_csv("Data/bluebook-for-bulldozers/Test.csv",
                     low_memory=False,
                     parse_dates=["saledate"])
df_test.head().T

# Make predictions on the test dataset
test_preds = ideal_model.predict(df_test)


# ### Preprocessing the data (getting the test dataset in the ssame format as our training dataset)
#

def preprocess_data(df):
    """
    Performs transformations on df and returns transformed df.
    """
    df["saleYear"] = df.saledate.dt.year
    df["saleMonth"] = df.saledate.dt.month
    df["saleDay"] = df.saledate.dt.day
    df["saleDayOfWeek"] = df.saledate.dt.dayofweek
    df["saleDayOfYear"] = df.saledate.dt.dayofyear
    
    df.drop("saledate", axis=1, inplace=True)
    
    # Fill the numeric rows with median
    
    for label, content in df.items():
        if pd.api.types.is_numeric_dtype(content):
            if pd.isnull(content).sum():
                # Add a binary column which tells us if the data was missing or not
                df[label+"_is_missing"] = pd.isnull(content)
                # Fill missing numeric values with median
                df[label] = content.fillna(content.median())
    
        # Fill the categorical missing data into numbers
        if not pd.api.types.is_numeric_dtype(content):
            df[label+"_is_missing"] = pd.isnull(content)
            # We add +1 to the category code beacuse pandas encodes missing categories as -1
            df[label] = pd.Categorical(content).codes + 1
    return df


# Process the test data
df_test = preprocess_data(df_test)
df_test.head()

x_train.head()

# MAke predictions on updated test data
test_preds = ideal_model.predict(df_test)

# We can find how the columns differ using sets
set(x_train.columns) - set(df_test.columns)

# Manually adjust df_test to have 'auctioneerID_is_missing' column
df_test['auctioneerID_is_missing'] = False
df_test.head()

# Finally now our test dataframe has the same features as our training dataframe, we can make predictions!

# Make predictions on the test data
test_preds = ideal_model.predict(df_test)

test_preds

# We've made some prediction but they're not in the same format Kaggle is asking for:
# https://www.kaggle.com/c/bluebook-for-bulldozers/overview/evaluation

# Format predictions into the same format Kaggle is after
df_preds = pd.DataFrame()
df_preds["SalesID"] = df_test["SalesID"]
df_preds["SalesPrice"] = test_preds
df_preds

# Export predictions data
df_preds.to_csv("Data/bluebook-for-bulldozers/test_predictions.csv", index=False)

# ### Feature Importance
#
# Feature importance seeks to figure out which different attributes of the data were most importance when it comes to predicting the **target variable** (SalesPrice)

# Find feature importance of our best model
ideal_model.feature_importances_


# Helper function for plotting feature importance
def plot_features(columns, importances, n=20):
    df = (pd.DataFrame({"features": columns,
                       "feature_importances": importances})
         .sort_values("feature_importances", ascending=False)
         .reset_index(drop=True))
    #plot the dataframe
    fig, ax = plt.subplots()
    ax.barh(df["features"][:n], df["feature_importances"][:20])
    ax.set_ylabel("features")
    ax.set_xlabel("feature_importance")
    ax.invert_yaxis()


plot_features(x_train.columns, ideal_model.feature_importances_)

df["Enclosure"].value_counts()



StarcoderdataJupyterScriptsDedupFiltered