# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3 (ipykernel)
#     language: python
#     name: python3
# ---

# ## Training
#
# Use Auto ML lib `pycaret` to automate model training and model selection.

import os
os.chdir("..")
# !pwd

from pathlib import Path
import numpy as np
import pandas as pd
from collections import Counter
from pycaret.classification import *
from sklearn.model_selection import train_test_split
from sklearn.metrics import classification_report
from imblearn.combine import SMOTETomek, SMOTEENN

# ## Load data

DATA_DIR = Path("./data")
TMP_DIR = Path("./tmp")
DATA_CSV = DATA_DIR / "full/train.csv"

df = pd.read_csv(DATA_CSV)
df

df.info()

target_names = np.unique(df['target'])
target_names

# + [markdown] tags=[]
# ## Split data
# -

random_seed = 17
X = df.drop('target', axis=1).to_numpy()
y = df['target'].to_numpy()
X_train, X_test, y_train, y_test = train_test_split(
    X, y, test_size=0.1, random_state=random_seed, stratify=y)

Counter(y_train)

Counter(y_test)


# ## Simple predictor: Return majority label

def simple_pred(x):
    return np.zeros(x.shape[0])


y_pred = simple_pred(X_test)
print(classification_report(y_test, y_pred))

# **NOTE**: Returning the majority class achieves 98% accuracy!

# ## Setup pycaret env

train_ds = pd.DataFrame(X_train)
train_ds['target'] = y_train
test_ds = pd.DataFrame(X_test)
test_ds['target'] = y_test

train_ds

test_ds

# ### Baseline: No pyret preprocessing

include=['et', 'lightgbm', 'lr', 'dt', 'svm', 'mlp', 'rf', 'ada']

exp_baseline = setup(
    data=train_ds,
    train_size=0.8, # split training and validation
    test_data=test_ds,
    target='target',
    session_id=random_seed,
    use_gpu=True,
    
    data_split_shuffle=True,
    data_split_stratify=True,
    fold_strategy="stratifiedkfold",
    fold=10,
            
    imputation_type='simple', # there's a small number of missing values
    numeric_imputation='mean', # use mean to replace missing values
    
    fix_imbalance=False,
    # fix_imbalance_method=SMOTETomek(), # imblearn's method
    
    remove_outliers=False,
    outliers_threshold=0.05, # percentage of outliers in distribution's tail
            
    normalize=False,
    normalize_method='zscore', # z=(x-u)/s
    transformation=False,
    transformation_method="yeo-johnson",
            
    feature_interaction=False, # create a*b features
    feature_ratio=False, # create a/b features
    interaction_threshold=0.01, # threshold feature importance
            
    polynomial_features=False,
    polynomial_degree=2,
    polynomial_threshold=0.1,
            
    trigonometry_features=False, # create tan(x), sin(x), cos(x)
    
    feature_selection=False, # use tree models to compute feature importance
    feature_selection_threshold=0.8,
    
    remove_multicollinearity=False, # drop highly correlated features
    multicollinearity_threshold=0.9,
            
    pca=False, # reduce feature space dimension
    pca_method="linear",
    pca_components=0.99, # % components to keep
    
    log_experiment="mlflow",
    log_plots=True,
)

best = compare_models(sort='F1', include=include)

print(best)

# ### Exp 01 - Fix imbalance by using SMOTETomek()

exp_01 = setup(
    data=train_ds,
    train_size=0.8, # split training and validation
    test_data=test_ds,
    target='target',
    session_id=random_seed,
    use_gpu=True,
    
    data_split_shuffle=True,
    data_split_stratify=True,
    fold_strategy="stratifiedkfold",
    fold=10,
            
    imputation_type='simple', # there's a small number of missing values
    numeric_imputation='mean', # use mean to replace missing values
    
    fix_imbalance=False,
    # fix_imbalance_method=SMOTETomek(), # imblearn's method
    
    remove_outliers=False,
    outliers_threshold=0.05, # percentage of outliers in distribution's tail
            
    normalize=False,
    normalize_method='zscore', # z=(x-u)/s

    transformation=False,
    transformation_method="yeo-johnson",
            
    feature_interaction=False, # create a*b features
    feature_ratio=False, # create a/b features
    interaction_threshold=0.01, # threshold feature importance
            
    polynomial_features=False,
    polynomial_degree=2,
    polynomial_threshold=0.1,
            
    trigonometry_features=False, # create tan(x), sin(x), cos(x)
    
    feature_selection=False, # use tree models to compute feature importance
    feature_selection_threshold=0.8,
    
    remove_multicollinearity=False, # drop highly correlated features
    multicollinearity_threshold=0.9,
            
    pca=False, # reduce feature space dimension
    pca_method="linear",
    pca_components=0.99, # % components to keep
    
    log_experiment="mlflow",
    log_plots=True,
)

best = compare_models(sort='F1', include=include)

# ### Exp 01 - Fix imbalance by using SMOTETomek()

include=['et', 'lightgbm', 'lr', 'dt', 'rf']

exp_01 = setup(
    data=train_ds,
    train_size=0.8, # split training and validation
    test_data=test_ds,
    target='target',
    session_id=random_seed,
    use_gpu=True,
    
    data_split_shuffle=True,
    data_split_stratify=True,
    fold_strategy="stratifiedkfold",
    fold=10,
            
    imputation_type='simple', # there's a small number of missing values
    numeric_imputation='mean', # use mean to replace missing values
    
    fix_imbalance=True,
    fix_imbalance_method=SMOTETomek(), # imblearn's method
    
    remove_outliers=False,
    outliers_threshold=0.05, # percentage of outliers in distribution's tail
            
    normalize=False,
    normalize_method='zscore', # z=(x-u)/s

    transformation=False,
    transformation_method="yeo-johnson",
            
    feature_interaction=False, # create a*b features
    feature_ratio=False, # create a/b features
    interaction_threshold=0.01, # threshold feature importance
            
    polynomial_features=False,
    polynomial_degree=2,
    polynomial_threshold=0.1,
            
    trigonometry_features=False, # create tan(x), sin(x), cos(x)
    
    feature_selection=False, # use tree models to compute feature importance
    feature_selection_threshold=0.8,
    
    remove_multicollinearity=False, # drop highly correlated features
    multicollinearity_threshold=0.9,
            
    pca=False, # reduce feature space dimension
    pca_method="linear",
    pca_components=0.99, # % components to keep
    
    log_experiment="mlflow",
    log_plots=True,
)

best = compare_models(sort='F1', include=include)

print(best)

plot_model(best, plot = 'confusion_matrix')

plot_model(best, plot = 'error')

predict_model(best)

# **NOTE**: After playing with several models, lightgbm seems to be one of the top model we should use to move on the next task which is model fine tuning.
#
# In fact, we should try more than 1 model in the fine tuning step because data preparation, model training and model fine tuning are in a cycle. Because of the limited time, I will use only one model and I will not iterate back to the data preparation step.



StarcoderdataJupyterScriptsDedupFiltered