# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

# ### Import the needed libraries

# +
import warnings
from itertools import chain
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt

from sklearn.impute import SimpleImputer
from sklearn.preprocessing import OneHotEncoder, MinMaxScaler
from sklearn.model_selection import train_test_split, GridSearchCV
from sklearn.naive_bayes import GaussianNB
from sklearn import metrics
from sklearn.neighbors import KNeighborsClassifier
from sklearn.linear_model import LogisticRegression
from sklearn.svm import SVC
from sklearn.neural_network import MLPClassifier
from sklearn.ensemble import VotingClassifier

from sklearn.exceptions import ConvergenceWarning, DataConversionWarning
warnings.filterwarnings(action='ignore', category=ConvergenceWarning)
warnings.filterwarnings(action='ignore', category=DataConversionWarning)


# -

# ### Semi-Structured Text Processor

def process_text_feature(data: pd.DataFrame, unique: list):
    """
    :param data: Pandas DataFrame - text feature (semi-structured)
    :param unique: List - unique code
    :return: Pandas DataFrame - processed codes
    """
    df = pd.DataFrame(columns=unique,
                           index=range(len(data)))

    for i in range(len(data)):
        text = data.code[i].split(',')

        for j in text:
            index_col = unique.index(j)
            df.loc[i][index_col] = 1

    df.fillna(0, inplace=True)

    return df


# ### Load data and Preprocessing

def preprocess_data():
    """
    Load and pre-process data.
    Implemented the following:
        - removed redundant variables
        - removed variables with high missing ratio
        - handle missing values by imputation
        - processed and incorporated the text codes columns
            - it was extended to multiple variables
        - One-hot encoded the categorical variables (ignored ordering)
        - implemented [0, 1] normalization

    :return: X_data, y_data
    """

    X_data = pd.read_csv("data.csv")
    y_data = X_data.class_label

    # removed redundant variables and the class label
    X_data.drop(["id", "date", "class_label"], axis=1, inplace=True)

    # - calculate % of missing values
    perc_missing = pd.DataFrame(
        {"perc_missing_values": X_data.isnull().mean() * 100}
    ).sort_values("perc_missing_values")

    perc_missing.query("perc_missing_values > 0", inplace=True)
    perc_missing

    perc_missing.iloc[:, 0].plot(
        kind="bar", title="% of missing values per variables", figsize=(15, 10)
    )

    # The following variables will be dropped,
    # because over 50% of them are missing X6 and K6
    X_data.drop(["X6", "K6"], axis=1, inplace=True)

    # Filling missing values by imputation via most_frequent
    imputer = SimpleImputer(missing_values=np.nan, strategy="most_frequent")
    imputer.fit(X_data)
    X_data = pd.DataFrame(imputer.transform(X_data), columns=X_data.columns)

    # working on coded variable (semi structured in nature)
    code_data = pd.DataFrame(X_data.codes.values, columns=["code"])
    X_data.drop(["codes"], axis=1, inplace=True)
    code_unique = sorted(list(set(chain(*code_data.code.str.split(",").tolist()))))
    processed_codes = process_text_feature(code_data, code_unique)

    # One-hot encoding for the categorical variables
    cat_var = ["C1", "C2", "C3", "C4", "C5", "C6"]
    data_encode = X_data[cat_var]
    encoder = OneHotEncoder(handle_unknown="ignore")
    encoder.fit(data_encode)

    data_onehot = pd.DataFrame(
        encoder.transform(data_encode).toarray(),
        columns=encoder.get_feature_names()
    )

    # drop encoded variables from X_data and add data_onehot
    # and processed codes X_data to it
    X_data.drop(cat_var, axis=1, inplace=True)
    X_data = pd.concat([X_data, data_onehot, processed_codes], axis=1)

    scaler = MinMaxScaler()
    scaler.fit(X_data)
    X_data = scaler.fit_transform(X_data)

    return X_data, y_data


# #### Environment Variables

MERIC_SCORING = 'roc_auc'
CV = 10

# # Model Evaluation Metric:

# ROC-AUC will be used to evaluate the models in this project. ROC is a curve (False Positive Rate vs Sensitivity) that show the ability of a binary classifier as its discrimination threshold is varied. While AUC is the area under the curve.

# # <u> Working With Dataset

model_eval_df = pd.DataFrame(
    columns = ['model', 'optimal_param', 'testing_ROC_AUC'], 
    index=range(6)
)

X_TRAIN, Y_TRAIN = preprocess_data()

# ### Distribution of the Class Label

plt.hist(Y_TRAIN)
plt.xticks([0, 1])
plt.title('Distribution of Training Class Label')
plt.show()

# ### Divide Train set into Train and Test

x_train, x_test, y_train, y_test = train_test_split(
    X_TRAIN, Y_TRAIN, test_size=0.2, random_state=42
)

# # <u>1. Model: Naive Bayes

# ### (1a) Fitting the model 

nb_fit = GaussianNB().fit(x_train, y_train)

# ### (1b) Testing

nb_y_pred = nb_fit.predict(x_test)
nb_score = round(metrics.roc_auc_score(y_test, nb_y_pred), 4)
model_eval_df.loc[0] = ('Naive Bayes', '-', nb_score)
print("Testing Score = {}".format(nb_score))

# ### (1c) Confusion Matrix - Actual vs. Predicted

pd.DataFrame(metrics.confusion_matrix(y_test, nb_y_pred))

# # <u>2. Model: KNN

# ### (2a) Fitting the model with optimal Hyperparamters via Grid Search with 5-folds CV

# +
k = np.arange(22, 27)
param_tune = {"n_neighbors": k}

knn = GridSearchCV(
    KNeighborsClassifier(), param_tune, cv=CV, scoring=MERIC_SCORING
)

knn.fit(x_train, y_train)
# -

# ### (2b) The optimal hyperparameters are:

model_eval_df.optimal_param[1] = 'k = {}'.format(*knn.best_params_.values())
print("k = {}".format(*knn.best_params_.values()))

# ### (2c) Testing

knn_y_pred = knn.predict(x_test)
knn_score = round(metrics.roc_auc_score(y_test, knn_y_pred), 4)
model_eval_df.iloc[1, [0, 2]] = ('KNN', knn_score)
print("Testing Score = {}".format(knn_score))

# ### (2d) Confusion Matrix - Actual vs. Predicted

pd.DataFrame(metrics.confusion_matrix(y_test, knn_y_pred))

# # <u>3. Model: Logistic Regression - Ridge

# ### (3a) Fitting the model with optimal Hyperparamters via Grid Search with 5-folds CV

# +
lg_cost = np.logspace(-1, 1, 3)
param_tune = {'C': lg_cost}

lg_regr = GridSearchCV(
    LogisticRegression(solver='lbfgs', max_iter=500), 
    param_tune, cv=CV, scoring=MERIC_SCORING
)

lg_regr.fit(x_train, y_train)
# -

# ### (3b) The optimal hyperparameters are:

model_eval_df.optimal_param[2] = 'C = {}'.format(*lg_regr.best_params_.values())
print("C = {}".format(*lg_regr.best_params_.values()))

# ### (3c) Testing

lg_regr_y_pred = lg_regr.predict(x_test)
lg_regr_score = round(metrics.roc_auc_score(y_test, lg_regr_y_pred), 4)
model_eval_df.iloc[2, [0, 2]] = ('Logistic Regression', lg_regr_score)
print("Testing Score = {}".format(lg_regr_score))

# ### (3d) Confusion Matrix - Actual vs. Predicted

pd.DataFrame(metrics.confusion_matrix(y_test, lg_regr_y_pred))

# # <u>4. Model: SVM - RBF Kernel

# ### (4a) Fitting the model with optimal Hyperparamters via Grid Search with 5-folds CV

# +
svm_cost = np.logspace(-4, -3, 2)

param_tune = {'C': svm_cost}

svm = GridSearchCV(
    SVC(gamma='auto'), param_tune, cv=CV, scoring=MERIC_SCORING
)

svm.fit(x_train, y_train)
# -

# ### (4b) The optimal hyperparameters are:

model_eval_df.optimal_param[3] = 'C = {}'.format(*svm.best_params_.values())
print("C = {}".format(*svm.best_params_.values()))

# ### (4c) Testing

svm_y_pred = svm.predict(x_test)
svm_score = round(metrics.roc_auc_score(y_test, svm_y_pred), 4)
model_eval_df.iloc[3, [0, 2]] = ('SVM', svm_score)
print("Testing Score = {}".format(svm_score))

# ### (4d) Confusion Matrix - Actual vs. Predicted

pd.DataFrame(metrics.confusion_matrix(y_test, svm_y_pred))

# # <u>5. Model: Multi-layer Perceptron

# ### (5a) Fitting the model with optimal Hyperparamters via Grid Search with 5-folds CV

# +
mp_alpha = np.logspace(-4, -3, 2) # l2 penalty
param_tune = {'alpha': mp_alpha}

mlp = GridSearchCV(
    MLPClassifier(hidden_layer_sizes=(512, 256), max_iter=300), 
    param_tune, cv=CV, scoring=MERIC_SCORING
)

mlp.fit(x_train, y_train)
# -

# ### (5b) The optimal hyperparameters are:

model_eval_df.optimal_param[4] = 'Alpha = {}'.format(*mlp.best_params_.values())
print("Alpha = {}".format(*mlp.best_params_.values()))

# ### (5c) Testing

mlp_y_pred = mlp.predict(x_test)
mlp_score = round(metrics.roc_auc_score(y_test, mlp_y_pred), 4)
model_eval_df.iloc[4, [0, 2]] = ('MLP', mlp_score)
print("Testing Score = {}".format(mlp_score))

# ### (5d) Confusion Matrix - Actual vs. Predicted

pd.DataFrame(metrics.confusion_matrix(y_test, mlp_y_pred))

# # <u> Ensemble (with all the Models) by Voting

# +
voting = VotingClassifier(
    estimators=[
        ('nb', nb_fit), 
        ('knn', knn), 
        ('lgr', lg_regr), 
        ('svm', svm), 
        ('mlp', mlp)
    ],
    voting='hard'
)

voting.fit(x_train, y_train)
# -

voting_y_pred = voting.predict(x_test)

voting_score = round(metrics.roc_auc_score(y_test, voting_y_pred), 4)
model_eval_df.loc[5] = ('Ensemble Voting', '-', voting_score)
print("Testing Score = {}".format(voting_score))

# # <u> Summary

model_eval_df.sort_values('testing_ROC_AUC', ascending=False)

StarcoderdataJupyterScriptsDedupFiltered