"""
# Highlights  

* Different preprocessing methods and their effect on performance and training time.
* Feature selection methods with anonymous features.
* Effect of reduced feature sets on model performance.
* Framework for model evaluation - creating k-folds, pipelines, generating out-of-fold predictions for each model (if needed for stacking).  

References:
1. scikit-learn documentation
2. [Kaggle's 30 days of ML](https://www.youtube.com/playlist?list=PL98nY_tJQXZnP-k3qCDd1hljVSciDV9_N) youtube playlist by [Abhishek Thakur](https://www.kaggle.com/abhishek). A must-watch for every beginner. The 'custom_cross_val_predict' function borrows heavily from the content discussed in the playlist.
"""
"""
# Imports
"""
SEED = 2311

import time
import gc
import warnings
warnings.filterwarnings('ignore')

import numpy as np
import pandas as pd

from sklearn.ensemble import ExtraTreesClassifier #estimator for feature importances
from sklearn.svm import LinearSVC
from sklearn.feature_selection import SelectFromModel
from sklearn.feature_selection import VarianceThreshold

from sklearn.preprocessing import RobustScaler
from sklearn.preprocessing import QuantileTransformer
from sklearn.preprocessing import StandardScaler
from sklearn.preprocessing import MinMaxScaler
from sklearn.preprocessing import Normalizer

from sklearn.linear_model import LogisticRegression
from sklearn.pipeline import Pipeline

from sklearn.model_selection import StratifiedKFold
from sklearn.metrics import roc_auc_score
data_dir = '../input/tabular-playground-series-nov-2021/'
"""
**Function for reducing memory usage:** *compress*
"""
def compress(df):
    numeric = ['int8', 'int16', 'int32', 'int64', 'float16', 'float32', 'float64']
    
    start_mem = df.memory_usage().sum() / (2 ** 20) #memory in MB
    
    for col in df.columns: 
        col_type = df[col].dtypes
        
        if col_type in numeric:
            c_min, c_max = df[col].min(), df[col].max()
            
            if str(col_type)[:3] == 'int':
                if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:
                    df[col] = df[col].astype(np.int8)
                elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:
                    df[col] = df[col].astype(np.int16)
                elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:
                    df[col] = df[col].astype(np.int32)
                elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:
                    df[col] = df[col].astype(np.int64)
            else:
                if c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:
                    df[col] = df[col].astype(np.float16)
                elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:
                    df[col] = df[col].astype(np.float32)
                else:
                    df[col] = df[col].astype(np.float64)    
    
    end_mem = df.memory_usage().sum() / (2 ** 20)
    percent_reduction = 100 * (start_mem - end_mem) / start_mem
    
    print(f'Memory usage decreased from {start_mem:5.2f} Mb to {end_mem:5.2f} Mb ({percent_reduction:.2f}% reduction)')
    
    return df
%%time
train = compress(pd.read_csv(data_dir + 'train.csv'))
%%time
test = compress(pd.read_csv(data_dir + 'test.csv'))
train.head()
train.shape, test.shape
gc.collect()
"""
# Creating Folds  
Will be using the same folds for cross validating all models here and in future notebooks.
"""
train['fold'] = -1
#chose to create 6 folds since we have 600k samples, nice and round 100k per fold
N_SPLITS = 6

skf = StratifiedKFold(n_splits=N_SPLITS, shuffle=True, random_state=SEED)

for fold, (train_idx, val_idx) in enumerate(skf.split(X=train, y=train.target)):
    train.loc[val_idx, 'fold'] = fold
train.head()
train.fold.value_counts()
%%time
train.to_csv('tps-nov21-6folds.csv', index=False)
features = [f for f in train.columns if f not in ('id', 'target', 'fold')]
gc.collect()
"""
# Preprocessing + Linear model pipelines
"""
robust = ('robust', RobustScaler())
quantile = ('quantile', QuantileTransformer())
standard = ('standard', StandardScaler())
minmax = ('minmax', MinMaxScaler())
norm = ('norm', Normalizer())

clf = ('lr', LogisticRegression(solver='liblinear', random_state=SEED))

model_dict = {
    1: Pipeline([robust, clf]),
    2: Pipeline([quantile, clf]),
    3: Pipeline([standard, clf]),
    4: Pipeline([minmax, clf]),
    5: Pipeline([norm, clf])
}
def custom_cross_val_predict(train, test, features, model):
    oof_preds = {}
    test_preds = []
    scores = []
    
    cv_start = time.time()
    
    for fold in range(N_SPLITS):
        xtrain = train[train.fold != fold].reset_index(drop=True)

        xval = train[train.fold == fold].reset_index(drop=True)    
        val_idx = xval.id.values.tolist()
        
        fold_start = time.time()
        
        model.fit(xtrain[features], xtrain.target)        
        val_preds = model.predict_proba(xval[features])[:,1] #out-of-fold predictions      
        oof_preds.update(dict(zip(val_idx, val_preds)))
        auc = roc_auc_score(xval.target, val_preds)
        scores.append(auc)
        
        fold_end = time.time()
        
        print(f'Fold #{fold}: AUC = {auc:.5f}\t[Time: {fold_end - fold_start:.2f} secs]')
        
        test_preds.append(model.predict_proba(test[features])[:,1])
        
    cv_end = time.time()
    print(f'Average AUC = {np.mean(scores):.5f} with std. dev. = {np.std(scores):.5f}')
    print(f'[Total time: {cv_end - cv_start:.2f} secs]')
    
    oof_preds = pd.DataFrame.from_dict(oof_preds, orient='index').reset_index()
    test_preds = np.mean(np.column_stack(test_preds), axis=1)
    
    return oof_preds, test_preds

for model_id, model in model_dict.items():
    print('----- MODEL-' + str(model_id) + ' -----')
    
    oof_preds, test_preds = custom_cross_val_predict(train, test, features, model)
    
    oof_preds.columns = ['id', 'oof' + str(model_id)] #extracting model number from model_dict key
    oof_preds.to_csv('oof' + str(model_id) + '.csv', index=False)
    
    output = pd.DataFrame({'id': test.id, 'target': test_preds})
    output.to_csv('submission' + str(model_id) + '.csv', index=False)
!head submission1.csv
#can be merged based on 'id' into meta-dataset for stacking model
!head oof3.csv
"""
1. Applying QuantileTransformer (Model-2) and Normalizer (Model-5) resulted in a drop in performance.  
2. Applying RobustScaler (Model-1), StandardScaler (Model-3) and MinMaxScaler (Model-4) gave nearly equal AUC but the training time is significantly less with StandardScaler. Our Logistic Regression classifier is able to fit faster when the data is centered (mean = 0) with unit variance.  

Based on these results, **we will stick with StandardScaler for our model pipeline** when we experiment with different feature selection methods.  

Another point to remember is that this variation in performance may not hold for other datasets and classifiers. In case we see significant improvement in scores for a particular feature set after feature selection, we can come back to this step and experiment with preprocessing again.
"""
model = model_dict[3] #StandardScaler() -> LogisticRegression()
gc.collect()
"""
# [Feature Selection](https://en.wikipedia.org/wiki/Feature_selection)  

Motivation - simplifying the models and reducing training times, by reducing the amount of data to be processed while maintaining relevant information  
Since the features are anonymous, we cannot apply informed modifications. Thus, we will experiment with some feature selection methods based on the samples themselves.  
For different methods, we will end up with (hopefully) different subsets of features. We will evaluate them using the model we selected earlier and compare the performance with the results we have for the complete dataset.

Another branch of methods to reduce dimensions (a.k.a features) is [feature projection](https://en.wikipedia.org/wiki/Feature_extraction) (a.k.a feature extraction). They involve deriving new features from the original ones to transform the data into fewer dimensions. We will not be exploring these methods here.
"""
"""
**Using [VarianceThreshold](https://scikit-learn.org/stable/modules/generated/sklearn.feature_selection.VarianceThreshold.html#sklearn-feature-selection-variancethreshold)**  

Low variance features can be discarded since they will not have a significant effect on the target variable.
"""
vt_default_selector = VarianceThreshold().fit(train[features]) #default threshold = 0
feature_mask = vt_default_selector.get_support() #array of booleans 

feature_mask[:10]
#extracting list of selected features using the feature mask obtained from SelectFromModel
def get_selected_features(feature_mask, input_features):
    return [b for a, b in zip(feature_mask, input_features) if a]
vt_default_features = get_selected_features(feature_mask, features)

vt_default_features[:5], vt_default_features[-5:], len(vt_default_features)
"""
There are no zero-variance features in our original dataset. Hence, no features were discarded. Let us add a threshold to remove low-variance features.
"""
vt_custom_selector = VarianceThreshold(threshold=0.1).fit(train[features])
feature_mask = vt_custom_selector.get_support()
vt_custom_features = get_selected_features(feature_mask, features)
len(vt_custom_features)
print(f'Dropped features: {sorted(list(set(features) - set(vt_custom_features)))}')
"""
**Using [SelectFromModel](https://scikit-learn.org/stable/modules/feature_selection.html#feature-selection-using-selectfrommodel)**  

The estimator calculates feature importances and the transformer selects those above a certain threshold.
"""
"""
Tree-based feature selection
"""
%%time
tree_estimator = ExtraTreesClassifier(
    n_estimators=150, 
    random_state=SEED
)

tree_estimator.fit(train[features], train.target)
tree_mean_selector = SelectFromModel(tree_estimator, prefit=True, threshold='mean')
feature_mask = tree_mean_selector.get_support()
tree_mean_features = get_selected_features(feature_mask, features)
len(tree_mean_features)
tree_median_selector = SelectFromModel(tree_estimator, prefit=True, threshold='median')
feature_mask = tree_median_selector.get_support()
tree_median_features = get_selected_features(feature_mask, features)
len(tree_median_features)
"""
L1-based (linear model) feature selection
"""
%%time
linear_estimator = LinearSVC(C=0.5, penalty="l1", dual=False, random_state=SEED)
linear_estimator.fit(train[features], train.target)
linear_mean_selector = SelectFromModel(linear_estimator, prefit=True, threshold='mean')
feature_mask = linear_mean_selector.get_support()
linear_mean_features = get_selected_features(feature_mask, features)
len(linear_mean_features)
linear_median_selector = SelectFromModel(linear_estimator, prefit=True, threshold='median')
feature_mask = linear_median_selector.get_support()
linear_median_features = get_selected_features(feature_mask, features)
len(linear_median_features)
feature_dict = {
    6: vt_custom_features, #feature scaling models were numbered up to 5. 
    7: tree_mean_features,
    8: tree_median_features,
    9: linear_mean_features,
    10: linear_median_features
}
for fid, features in feature_dict.items():
    print('----- MODEL-' + str(fid) + ' -----')
    
    oof_preds, test_preds = custom_cross_val_predict(train, test, features, model)
    
    oof_preds.columns = ['id', 'oof' + str(fid)]
    oof_preds.to_csv('oof' + str(fid) + '.csv', index=False)
    
    output = pd.DataFrame({'id': test.id, 'target': test_preds})
    output.to_csv('submission' + str(fid) + '.csv', index=False)
"""
Well, our performance has suffered. So what did we achieve?  

* Our 'base model' i.e., Model-3 had an average AUC of 0.74852 and it used the complete dataset.
* With half the data and in half the time, we have an average AUC of 0.73674 (Model-8)
* With almost one-fourth the data and in almost one-fourth the time, we have an average AUC of 0.72691 (Model-7)

We gained time and memory for a slight hit to our performance.  
(Or a very large hit, depending on your attachment to the public LB :D )
"""
"""
**This was an experiment with feature preprocessing and selection.**  
* I browsed the scikit-learn documentation and chose preprocessing methods which I thought would modify the data in noticeably different ways. You can do the same and choose from several other methods.  
* My approach was the same for feature selection methods. The chosen methods gave different feature sets depending on their internal mechanism of feature importance. I evaluated the feature selection methods only after pruning the preprocessing experiment. You can choose otherwise.  
* And finally, all the methods were evaluated with a single classifier (logistic regression). The wide variety of classifiers certainly does not need coverage, hence the lack of focus there. You can choose any other classifier to replicate the experiment.
"""
"""
**Time to submit!**  
(The submission associated with the notebook will be for Model-3)

The folds dataset, out-of-fold predictions, and test predictions for all the models will be saved with the notebook.
"""