# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

# + _cell_guid="b1076dfc-b9ad-4769-8c92-a6c4dae69d19" _uuid="8f2839f25d086af736a60e9eeb907d3b93b6e0e5" papermill={"duration": 3.346056, "end_time": "2021-10-28T12:53:30.578876", "exception": false, "start_time": "2021-10-28T12:53:27.232820", "status": "completed"} tags=[]
import pandas as pd
import numpy as np
import lightgbm as lgb
import pickle
import os
import gc
from sklearn.preprocessing import MinMaxScaler
from matplotlib import pyplot as plt
import seaborn as sns
from sklearn.model_selection import StratifiedKFold
import json
from tqdm.notebook import tqdm

from utilities import (
    RANDOM_STATE, TARGET_COL, N_FOLD, FOLD_STRAT_NAME,
)

INPUT_PATH = '../input/tabular-playground-series-oct-2021'
PATH_NOTEBOOK = '../input/preprocess-gpu'
PATH_PARAM = '../input/optuna-lightgbm'

TEST_PATH = [
#     '../input/xgboost-tuned',
    '../input/lightgbm-tuned',
#     '../input/catboost-benchmark',
#     '../input/nn-cat-binned',
]

# + papermill={"duration": 0.029879, "end_time": "2021-10-28T12:53:30.623818", "exception": false, "start_time": "2021-10-28T12:53:30.593939", "status": "completed"} tags=[]
with open(os.path.join(PATH_NOTEBOOK, 'feature_dic.pkl'), 'rb') as file:
    feature_dic = pickle.load(file)

#CONSTANT
FEATURE = feature_dic['feature']
CAT_COL = feature_dic['categorical']
NUMERIC_COL = feature_dic['numerical']

# + papermill={"duration": 11.155268, "end_time": "2021-10-28T12:53:41.792880", "exception": false, "start_time": "2021-10-28T12:53:30.637612", "status": "completed"} tags=[]
train = pd.read_pickle(
    os.path.join(PATH_NOTEBOOK, 'train_unscaled.pkl')
)

# + papermill={"duration": 4.294666, "end_time": "2021-10-28T12:53:46.101613", "exception": false, "start_time": "2021-10-28T12:53:41.806947", "status": "completed"} tags=[]
test = pd.read_pickle(
    os.path.join(PATH_NOTEBOOK, 'test_unscaled.pkl')
)

# + papermill={"duration": 1.659546, "end_time": "2021-10-28T12:53:47.775252", "exception": false, "start_time": "2021-10-28T12:53:46.115706", "status": "completed"} tags=[]
for i, path in tqdm(enumerate(TEST_PATH)):

    sub = pd.read_csv(
        os.path.join(path, 'submission.csv')
    )
    
    sub['target_fold'] = (sub['target']>=.5).astype(int)

    #DEFINE FOLD same as PREPROCESS
    strat_fold = StratifiedKFold(n_splits = N_FOLD, random_state = RANDOM_STATE, shuffle = True)
    sub[FOLD_STRAT_NAME] = -1

    for j, (_, test_index) in enumerate(strat_fold.split(sub, sub['target_fold'])):
        sub.loc[test_index, FOLD_STRAT_NAME] = j

    pseudo_test = pd.concat([test, sub[[FOLD_STRAT_NAME, 'target']]], axis = 1)
    train = pd.concat([train, pseudo_test], axis = 0)

    gc.collect()

# + papermill={"duration": 0.040946, "end_time": "2021-10-28T12:53:47.830775", "exception": false, "start_time": "2021-10-28T12:53:47.789829", "status": "completed"} tags=[]
del test

# + papermill={"duration": 0.021623, "end_time": "2021-10-28T12:53:47.868335", "exception": false, "start_time": "2021-10-28T12:53:47.846712", "status": "completed"} tags=[]
train.reset_index(drop = True, inplace = True)

# + papermill={"duration": 0.141547, "end_time": "2021-10-28T12:53:48.023952", "exception": false, "start_time": "2021-10-28T12:53:47.882405", "status": "completed"} tags=[]
FOLD_LIST = list(range(train[FOLD_STRAT_NAME].nunique()))

gc.collect()

# + [markdown] papermill={"duration": 0.014324, "end_time": "2021-10-28T12:53:48.052907", "exception": false, "start_time": "2021-10-28T12:53:48.038583", "status": "completed"} tags=[]
# # TRAIN

# + papermill={"duration": 0.029837, "end_time": "2021-10-28T12:53:48.096996", "exception": false, "start_time": "2021-10-28T12:53:48.067159", "status": "completed"} tags=[]
params = {
    'objective': 'xentropy',
    'boosting_type': 'gbdt',
    'learning_rate': 0.01,
    'random_state': RANDOM_STATE,
    'verbose': -1,
    'n_jobs': -1,
    'device_type': 'gpu',
}

#import lgb optimized parameter (file_name is wrong :()

with open(os.path.join(PATH_PARAM, 'final_xgb_param.pkl'), 'rb') as file:
    param_lgb_final = pickle.load(file)
    
params.update(param_lgb_final)

#pretty print
print(json.dumps(params, indent=4))

# + papermill={"duration": 27388.205609, "end_time": "2021-10-28T20:30:16.317005", "exception": false, "start_time": "2021-10-28T12:53:48.111396", "status": "completed"} tags=[]
score = 0
model_list = []

for i, fold_ in enumerate(FOLD_LIST):

    mask_train = (train[FOLD_STRAT_NAME] != fold_)
    mask_test = (train[FOLD_STRAT_NAME] == fold_)
    
    
    train_x, train_y = train.loc[mask_train, FEATURE], train.loc[mask_train, TARGET_COL]
    test_x, test_y = train.loc[mask_test, FEATURE], train.loc[mask_test, TARGET_COL]

    model = lgb.train(
        params,
        lgb.Dataset(train_x, label=train_y,categorical_feature=CAT_COL), 30000,
        valid_sets = lgb.Dataset(test_x, label=test_y,categorical_feature=CAT_COL),
        valid_names ='validation', verbose_eval=100, early_stopping_rounds = 100,
    )
        
    #evaluate score and save model for importance/prediction
    score_fold = model.best_score['validation']['cross_entropy'] 
    score += score_fold/N_FOLD
    
    model_list.append(model)
    
    print('\nFold: {}; Cross Entropy: {:.5f}\n'.format(fold_, score_fold))
    print('-'*50)
    print('\n\n\n')
    gc.collect()
    
print('CV-Cross Entropy: {:.5f}\n'.format(score))

# + [markdown] papermill={"duration": 0.395383, "end_time": "2021-10-28T20:30:17.101381", "exception": false, "start_time": "2021-10-28T20:30:16.705998", "status": "completed"} tags=[]
# # Feature importance

# + papermill={"duration": 0.4273, "end_time": "2021-10-28T20:30:17.945432", "exception": false, "start_time": "2021-10-28T20:30:17.518132", "status": "completed"} tags=[]
feature_importances = pd.DataFrame()
feature_importances['feature'] = FEATURE


for fold_, model in enumerate(model_list):
    feature_importances['fold_{}'.format(fold_ + 1)] = model.feature_importance(importance_type='gain')

# + papermill={"duration": 1.124166, "end_time": "2021-10-28T20:30:19.465371", "exception": false, "start_time": "2021-10-28T20:30:18.341205", "status": "completed"} tags=[]
scaler = MinMaxScaler(feature_range=(0, 100))

average_importance = feature_importances.drop('feature', axis = 1).mean(axis=1)

feature_importances['average'] = scaler.fit_transform(X=pd.DataFrame(average_importance))
feature_importances = feature_importances.sort_values(by='average', ascending=False)

feature_importances[['feature', 'average']].to_csv('feature_importances.csv',index=False)

fig = plt.figure(figsize=(12,8))
sns.barplot(data=feature_importances.head(50), x='average', y='feature');
plt.title(f'50 TOP feature importance over {N_FOLD} average')

# + papermill={"duration": 0.527646, "end_time": "2021-10-28T20:30:20.386956", "exception": false, "start_time": "2021-10-28T20:30:19.859310", "status": "completed"} tags=[]
del train
gc.collect()

# + [markdown] papermill={"duration": 0.396117, "end_time": "2021-10-28T20:30:21.177168", "exception": false, "start_time": "2021-10-28T20:30:20.781051", "status": "completed"} tags=[]
# # TEST Blending

# + papermill={"duration": 1.223517, "end_time": "2021-10-28T20:30:22.875903", "exception": false, "start_time": "2021-10-28T20:30:21.652386", "status": "completed"} tags=[]
test = pd.read_pickle(
    os.path.join(PATH_NOTEBOOK, 'test_unscaled.pkl')
)

# + papermill={"duration": 4617.764537, "end_time": "2021-10-28T21:47:21.038793", "exception": false, "start_time": "2021-10-28T20:30:23.274256", "status": "completed"} tags=[]
pred_test = np.zeros(test.shape[0])

for fold_, model in enumerate(model_list):
    pred_test += model.predict(test)/N_FOLD

# + [markdown] papermill={"duration": 0.660835, "end_time": "2021-10-28T21:47:22.092384", "exception": false, "start_time": "2021-10-28T21:47:21.431549", "status": "completed"} tags=[]
# # SAVE RESULT

# + papermill={"duration": 2.211439, "end_time": "2021-10-28T21:47:24.733666", "exception": false, "start_time": "2021-10-28T21:47:22.522227", "status": "completed"} tags=[]
submission = pd.read_csv(os.path.join(INPUT_PATH, 'sample_submission.csv'))
submission['target'] = pred_test

submission.to_csv('submission.csv', index = False)

StarcoderdataJupyterScriptsDedupFiltered