# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

# This notebook contains the basemodel prepared on the dataset to explore the data and come with a base model for understanding. I have
# performed below tasks:
#     
#     1) Data Exploration
#     2) Data Pre-processing (handling "-1" and null values)
#     3) Normalizing the feature values using StandardScalar functions in scikit-learn
#     4) Performing feature selection using random forest classifier feature importance
#     5) Training the dataset over basic state of the art classification models
#     6) Interpreting baseline model results and using them for the final model for the better results
#     
#  Intution: I am starting this problem as a binary classification just to understand the data, but, I sense that our job is to just identify "toilet" I may have to handle it as an outlier detection problem (like using OneClassSVM() that uses concept of cost sensetive learning). The more research will be done while making the final model

# Importing basic Dependencies
import pandas as pd
import numpy as np 

# Data Exploration
#    
#     1) Loading the dataset using pandas csv command
#     2) Checking out the features and samples of the data (shape)
#     3) Checking out the first few records
#     4) As I am dealing with supervised classification problem, the label columns which is at the first column of the dataset has to be verified. We are supposed to classify the labels as "toilet" and "not-toilet"

# Loading the data
data = pd.read_csv('ml_problem_set.csv')

# Understanding the shape of the data
data.shape

#checking out the first few samples
data.head(10)

# As it can be seen from the above samples, only the "house_id" and "label" are categorial and rest are continous
#
# As this is basemodel, dropping the "house_id" column it seemes irrelevant for the "label" classification
#
# There are missing values loaded as "-1". As of now replacing them with "NaN" values and dropping them
#
# Next, I will be encoding the categorial "label" into numerical (though there are various technqiues, as of now I am encoding them manually)

# At first, we are seeing how many samples are there for each "label" type
data["label"].value_counts()

# #### Data Prepration

# replacing the missing values i.e. "-1" with "NaN" values
data = data.replace(-1,np.nan)

# Dropping rows containing null values - as of now we are dropping, will be analyzed further in the final model
data = data.dropna()
#checking out the new shape- how many rows lefft
data.shape

# As my task is to just classify the data which has "toilet" and "not a toilet", I am handling it as a binary classification problem, Therefore, I am encoding all "toilet" labels as "0" and rest of the labels as "1"

# Encoding the categorial labels
data['label'].replace({'toilet':0,'sink':1,'shower':1,'bathtub':1,'laundrytub':1,'spigot':1},inplace=True)

#checking out again the first few records to make sure that changes are observed
data.head()

#Checking again the number of samples of each class "Class 0 - toilet" and "Class 1 - "Not a Toilet"
data["label"].value_counts()

#Dropping the "house_id" column - seems unimportant as of now
data.drop(['house_id'],axis=1,inplace = True)

# Checking out the shape- we can observe that one column is decrease as we dropped the "house_id" column
data.shape

#Seperating the data as features and labels
original_features=data.iloc[:,1:]
labels = data.iloc[:,0:1]

# #### Feature Engineering - Feature Normalization and Feature Selection

# +
#Importing Scikit learn libraries

from sklearn.preprocessing import MinMaxScaler
scaler = MinMaxScaler(feature_range=(0, 1))

#Normalizing Train Data Features
scaler_features = scaler.fit(original_features)
normalized_features = scaler_features.transform(semi_features)
# -

#Checking out the whether the features are normalzied - Observe Column 6 in the second cell and here to find difference
pd.DataFrame(normalized_features).head()

# There are almost 216 features. All of them are unknown to me in terms of domain knowledge. Hence, I am using all the features for the feature selection using RandomForesTrees.
# RandomForestTress is the state of the art techniques for the feature selection. It ranks all the features in the order of their importances adding to classify the label
# One can select the top "K" important features that are required for the classification model
#
# Reference: http://scikit-learn.org/stable/auto_examples/ensemble/plot_forest_importances.html

# +
# Importing Matplotlib for plotting the feature importance graph

import matplotlib.pylab as plt


def FeatureSelection(myinputX, myinputY):

    labels = np.array(myinputY).astype(int).ravel()
    inputX = np.array(myinputX)
    
    #Random Forest Model
    model = RandomForestClassifier(random_state = 0)
    model.fit(inputX,labels)
    importances = model.feature_importances_
    
    
    #Plotting the Features agains their importance scores
    indices = np.argsort(importances)[::-1]
    std = np.std([tree.feature_importances_ for tree in model.estimators_],
             axis=0)
    plt.figure(figsize = (40,7))
    plt.title("Feature importances (y-axis) vs Features IDs(x-axis)")
    plt.bar(range(inputX.shape[1]), importances[indices],
       color="r", yerr=std[indices], align="center")
    plt.xticks(range(inputX.shape[1]), indices)
    plt.xlim([-1, inputX.shape[1]])
    plt.show()
    
    # Selecting top featueres which have higher importance values = here we can find "80" features
    # I am changing number of features manually as of now. One has to find the tradeoff
    
    
    myinputX = pd.DataFrame(myinputX)
    newX = myinputX.iloc[:,model.feature_importances_.argsort()[::-1][:80]]
    
    # Converting the X dataframe into tensors and returning the reduced feature set
    myX = newX.as_matrix()

    return myX

# Reduced Feature Set
reduced_features  = FeatureSelection(normalized_features,labels)
# -

# The above graphs displays the "plot" of the importance of each feature with respect to the label. The selection criteria can be made using a recurisve algorithm that removes the reduandant and irrelevant features. As of now I am selecting top "80" features. And the below cell is to verify whether the features are reduced to "80"

pd.DataFrame(reduced_features).head()

# #### Baseline Models - Model Training and Testing

# I have performed below steps for model training and testing
#
#   1) Using train_test_split - splitting the dataset into 85:15 ratio
#   
#   2) I made sure there are "toilet" samples in both training and testing dataset
#   
#   3) I have used a python dictionary to take 5 basic classifiers and performed operatio using training and testing set one by one
#   
#   4) The classification metrics report and the confusion matrix are used as of now to interpret the high level results

# +
#Importing the sklearn dependencies
from sklearn.linear_model import LogisticRegression
from sklearn.tree import DecisionTreeClassifier
from sklearn.ensemble import GradientBoostingClassifier
from sklearn.svm import SVC
from sklearn.naive_bayes import GaussianNB
from sklearn.model_selection import train_test_split
from sklearn import metrics

#Train - Test Split
X_train, X_test, y_train, y_test  = train_test_split(
        reduced_features, 
        labels.as_matrix(),
        train_size=0.85, 
        random_state=1234)

#Checking whether both the training and testing set has acceptable number of "toilet" samples
print ("Number of samples of each class label in training dataset is:")
print (np.unique(y_train, return_counts=True))
print ("Number of samples of each class label in testing dataset is:")
print (np.unique(y_test, return_counts=True))

# Making a dictionary kind of pipeline for few basic classifiers- randomstate to reproduce the results
Classifiers =  {"GB": {"f": GradientBoostingClassifier(random_state = 42)},
                "DT": {"f": DecisionTreeClassifier(random_state = 42)},
                "LR": {"f": LogisticRegression(random_state = 42)},
                "NB": {"f": GaussianNB()},
                "SVC": {"f": SVC(random_state = 42)}}
                
#Preparing the metrics dictionary that will be used in the evaluation
measures = {"class_report": metrics.classification_report, "conf_mat": metrics.confusion_matrix}


#Models Training and prediction - one by one
for model in Classifiers.keys():

    # Fit the Classifiers using the training dataset
    Classifiers[model]["f"].fit(X_train, y_train.ravel())
    
    # Predict
    Classifiers[model]["c"] = Classifiers[model]["f"].predict(X_test)
    
    # Evaluate each model in Regressors
    print ("Results of the classifier model {}".format(model))
    print ("Confusion Matrix for the classifier model {}".format(model))
    
    #printing the metrics results
    for measure in measures.keys():
        print (measures[measure](y_test.ravel(), Classifiers[model]["c"]))
# -

# As it can be seen that GradientBoost Classifier is performing better when compared to other classifiers.
#
# It is able to truly predict 9 samples as "toilet" out of "19" samples.
#
# The fact that this dataset is imbalanced, the precision of "Toilet" class is not up to the mark for all the models
#
# Basic Plan from here:
#
#     1) Use ensemeble techqnieus to handle the imbalanced dataset
#     2) Use cost sensitive learning such that we give more weight to the "toilet" class as it is our priority
#     3) Tune the algorithms using GridSearchCV
#     4) Though, neural networks are not required with the size of dataset, we can give a try
#     5) Use XGBoost
#     6) Better feature selection
#     7) Use other classification metrics

StarcoderdataJupyterScriptsDedupFiltered