"""
## <center>30 Days of ML competition</center>
### <center>Full top 5% soulution</center>
<center><img src='https://storage.googleapis.com/kaggle-media/Images/30_Days_ML_Hero.png' width='240' height="240" ></center>

This notebook contains full solution from reading and analysis of data to building stacking pipeline.


#### Dataset:
The dataset is used for this competition is synthetic (and generated using a CTGAN), but based on a real dataset. The original dataset deals with predicting the amount of an insurance claim. 
* 'cat0' - 'cat9' categorical features
* 'cont0' - 'cont13' continuous features
* 'target' - continous target
"""
import os
import time
import numpy as np # linear algebra
import pandas as pd # data processing
import matplotlib.pyplot as plt # plotting
import seaborn as sns # plotting
from sklearn.metrics import mean_squared_error # MSE metric
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import OrdinalEncoder # ordinal encoding categorical variables
from sklearn.model_selection import StratifiedKFold
from sklearn.linear_model import Ridge
from sklearn.ensemble import RandomForestRegressor
from xgboost import XGBRegressor
from catboost import CatBoostRegressor
from lightgbm import LGBMRegressor

SEED = 91 # random seed

pd.set_option('display.max_columns', 100)
"""
# 1. Load data and first look
"""
PATH = '/kaggle/input/30-days-of-ml/'

print('Files in directory:')
for dirname, _, filenames in os.walk('/kaggle/input'):
    for filename in filenames:
        print('  '+os.path.join(dirname, filename))
print()

# Load the training data
try:
    df_train = pd.read_csv(PATH+'train.csv', index_col=0)
    df_test = pd.read_csv(PATH+'test.csv', index_col=0)
    submission = pd.read_csv(PATH+'sample_submission.csv', index_col=0)
    print('All of the data has been loaded successfully!')
except Exception as err:
    print(repr(err))
print()
full_lenght_data = len(df_train) + len(df_test)
print(f"train: {len(df_train)}  ({100*len(df_train)/full_lenght_data:.3f} %)")
print(f"test:  {len(df_test)} ({100*len(df_test)/full_lenght_data:.3f} %)")
"""
Check if exist missing values
"""
if not sum(df_train.isna().any()) and not sum(df_test.isna().any()):
    print('No missing values in dataset')
df_train.head(5)
"""
# 2. Exploratory Data Analysis (EDA)
"""
"""
Use only meaningful features  
(some features are deleted as useless because of feature importance of built regression model)  
See here: https://www.kaggle.com/sergeyzemskov/catboost-feature-importance-with-shap  
I haven't saved all notebooks comparison of feature importance
"""
CAT_FEATURES = ['cat0', 'cat1', 'cat2', 'cat3', 'cat4', 'cat5', 'cat8', 'cat9']
NUM_FEATURES = ['cont0', 'cont1', 'cont2', 'cont3', 'cont4', 'cont5', 'cont6', 'cont7', 'cont8',
                'cont9', 'cont10', 'cont11', 'cont12', 'cont13']
ALL_FEATURES = CAT_FEATURES+NUM_FEATURES
"""
### Categorical features
Comparison of distribution of features in train and test data
"""
# join train and test dat
df_full = pd.concat([df_test[CAT_FEATURES], df_train[CAT_FEATURES]]).sort_index()
df_full['test'] = 'train'
df_full.loc[df_test.index,'test'] = 'test'
plt.figure(figsize=(20,8))
plt.subplots_adjust(hspace=0.5, wspace=0.3)
#sns.set_palette("Spectral")
for i, col in enumerate(CAT_FEATURES):
    plt.subplot(2, 4, i+1)
    (df_full[col]
     .groupby(df_full['test'])
     .value_counts(normalize=True)
     .rename('proportion')
     .reset_index()
     .pipe((sns.barplot, "data"), x=col, y='proportion', hue='test'))
    plt.title(col)
plt.show()
"""
Variables of categorical features on test and train data are distributed very similar
"""
"""
### Continous features
"""
df_train[NUM_FEATURES].describe()
"""
### Target
"""
df_train['target'].describe()
f,ax=plt.subplots(figsize=(12,4))
sns.histplot(df_train['target'])
plt.legend()
plt.xticks(np.arange(0,10.8,.5))
plt.show()
"""
# 3. Data preprocessing
"""
"""
X - fatures,  
y - target
"""
X = df_train.drop(['target'], axis=1).copy()
y = df_train['target'].copy()
X_test = df_test.copy()
X.head()
"""
Apply one-hot encoding to categorical features cat1 and cat8. In cat8 I use rare categories as not A, C or E
"""
X['cat1_A'] = X['cat1'].apply(lambda x: 1 if x=='A' else 0)
X['cat1_B'] = X['cat1'].apply(lambda x: 1 if x=='B' else 0)
X['cat8_A'] = X['cat8'].apply(lambda x: 1 if x=='A' else 0)
X['cat8_C'] = X['cat8'].apply(lambda x: 1 if x=='C' else 0)
X['cat8_E'] = X['cat8'].apply(lambda x: 1 if x=='E' else 0)

X_test['cat1_A'] = X_test['cat1'].apply(lambda x: 1 if x=='A' else 0)
X_test['cat1_B'] = X_test['cat1'].apply(lambda x: 1 if x=='B' else 0)
X_test['cat8_A'] = X_test['cat8'].apply(lambda x: 1 if x=='A' else 0)
X_test['cat8_C'] = X_test['cat8'].apply(lambda x: 1 if x=='C' else 0)
X_test['cat8_E'] = X_test['cat8'].apply(lambda x: 1 if x=='E' else 0)
"""
I made small research where I find reasons try to use anomaly peaks of cont1 as additional categorical features:  
https://www.kaggle.com/sergeyzemskov/eda-predict-anomalies-of-target-experiment/edit/run/73746908
"""
peaks_l = [0.24, 0.295, 0.36, 0.421, 0.488, 0.555, 0.622, 0.84, 0.89]
peaks_r = [0.271, 0.321, 0.38, 0.44, 0.508, 0.575, 0.638, 0.848, 0.9]
X['cont1_cat'] = X['cont1'].apply(
    lambda x: 1 if [True for i in range(len(peaks_l)) if (x>peaks_l[i] and x<peaks_r[i])] else 0
)
X['cont1_cat'].value_counts()
X_test['cont1_cat'] = X_test['cont1'].apply(
    lambda x: 1 if [True for i in range(len(peaks_l)) if (x>peaks_l[i] and x<peaks_r[i])] else 0
)
X_test['cont1_cat'].value_counts()
EXTRA_FEATURES = ['cat1_A', 'cat1_B', 'cat8_A', 'cat8_C', 'cat8_E', 'cont1_cat']
ALL_FEATURES += EXTRA_FEATURES
CAT_FEATURES += EXTRA_FEATURES
"""
### Ordinal encode of categorical columns
"""
ordinal_encoder = OrdinalEncoder()
X[CAT_FEATURES] = ordinal_encoder.fit_transform(X[CAT_FEATURES]).astype(int)
X_test[CAT_FEATURES] = ordinal_encoder.transform(X_test[CAT_FEATURES]).astype(int)

X.head()
"""
### Correlation matrix of generated features
"""
corrMatrix = X[['cat1', 'cat1_A', 'cat1_B', 'cat8', 'cat8_A', 'cat8_C', 'cat8_E', 'cont1_cat']].corr(method='pearson', min_periods=1)
plt.figure(figsize=(8,4))
# Generate a mask for the upper triangle
mask = np.triu(np.ones_like(corrMatrix, dtype=bool), k=1)
ax = sns.heatmap(corrMatrix, annot=True, mask=mask, cbar_kws={"shrink": .7}, cmap='coolwarm')
plt.show()
"""
Remove cat1 and cat8 due strong correlation with derived features
"""
CAT_FEATURES.remove('cat1')
ALL_FEATURES.remove('cat1')
CAT_FEATURES.remove('cat8')
ALL_FEATURES.remove('cat8')
X[ALL_FEATURES].head(5)
"""
# 4. Model
"""
"""
For cross-validation use KFold with 7 folds
"""
N_FOLD = 5
#kf = StratifiedKFold(n_splits=N_FOLD, shuffle=True, random_state=SEED)
"""
I tried StratifiedKFold but it doesn't improve my result on leaderbord.  
Thanks [Kaihua Zhang](https://www.kaggle.com/zhangkaihua88/30ml-continuous-target-stratification#Continuous-Target-StratifiedKFold) for research
"""
class ContinuousTargetStratifiedKFold:
    """
    Obtain stratified cross validation splits from the continuous targets
    -------
    Parameters
    -------
    - n_splits(int): Number of folds
    - nbins(int): The criteria to bin by(from pd.cut)
    - shuffle(bool): Whether to shuffle
    - random_state(int):
    -------
    Examples
    -------
    >>> cskf = ContinuousTargetStratifiedKFold(n_splits=n_s)
    >>> for fold_no, (t, v) in enumerate(skf.split(target, target)):
            pass
    -------
    Sources
    -------
    - [Continuous Target Stratification](https://www.kaggle.com/tolgadincer/continuous-target-stratification)
    - [How to generate a custom cross-validation generator in scikit-learn?](https://stackoverflow.com/questions/30040597/how-to-generate-a-custom-cross-validation-generator-in-scikit-learn/39721727)
    """
    def __init__(self, n_splits=N_FOLD, nbins=100, shuffle=False,
                 random_state=None):
        self.skf = StratifiedKFold(n_splits=n_splits,
                                   shuffle=shuffle,
                                   random_state=random_state)
        self.nbins = nbins
        self.n_splits = n_splits

    def split(self, X, y):
        y = pd.qcut(y, self.nbins, labels=False) # implortant to use qcut
        for train_index, test_index in self.skf.split(y, y):
            yield train_index, test_index

    def get_n_splits(self, X=None, y=None):
        return self.n_splits
kf = ContinuousTargetStratifiedKFold(n_splits=N_FOLD, nbins=30, shuffle=True, random_state=SEED)
model_results = {'model': [], 'score': [], 'training_time': []}

def add_model_result(dic, model, score, time=None):
    dic['model'].append(model)
    dic['score'].append(score)
    if time:
        dic['training_time'].append(time)
"""
### 0 level (base)
"""
"""
I use various hyperparameters. Also, every model has his own features set.
"""
models = {
    'XGBoost 1': {
        'model': XGBRegressor(
            booster = 'gbtree',
            eval_metric = "rmse",
            n_estimators = 10000,
            learning_rate = 0.05,
            reg_lambda = 7,
            reg_alpha = 25.8,
            subsample = 0.9,
            colsample_bytree = 0.12,
            max_depth = 3,
            random_state = SEED 
        ),
        'features': ALL_FEATURES,
        'feature_importance': 0
    },

    'XGBoost 2': {
        'model': XGBRegressor(
            booster = 'gbtree',
            eval_metric = "rmse",
            n_estimators = 10000,
            learning_rate = 0.035,
            reg_lambda = 5,
            reg_alpha = 25.6,
            subsample = 0.9,
            colsample_bytree = 0.12,
            max_depth = 3,
            random_state = SEED 
        ),
        'features': ALL_FEATURES,
        'feature_importance': 0
    },

    'XGBoost 3': { 
        'model': XGBRegressor(
            booster = 'gbtree',
            eval_metric = "rmse",
            n_estimators = 10000,
            learning_rate = 0.04,
            reg_lambda = 0.5,
            reg_alpha = 24.5,
            subsample = 0.82,
            colsample_bytree = 0.13,
            max_depth = 3,
            random_state = SEED 
        ),
        'features': [f for f in ALL_FEATURES if f not in ['cat0', 'cat2', 'cat4', 'cat9', 'cont1_cat']],
        'feature_importance': 0
    },

    'XGBoost 4': {
        'model': XGBRegressor(
            booster = 'gbtree',
            eval_metric = "rmse",
            n_estimators = 10000,
            learning_rate = 0.04,
            reg_lambda = 5,
            reg_alpha = 25.5,
            subsample = 0.97,
            colsample_bytree = 0.11,
            max_depth = 3,
            random_state = SEED 
        ),
        'features':  [f for f in ALL_FEATURES if f not in ['cat0', 'cat2', 'cat3', 'cat4', 'cat5', 'cat9', 'cont1_cat']],
        'feature_importance': 0
    },
    
    # https://www.kaggle.com/sudipbishwakarma/30-days-of-ml-stacking-lgbm-xgboost
    'LightGBM': {
        'model': LGBMRegressor(
            max_depth = 10, # reduce max depth from 18
            learning_rate = 0.03,
            metric = 'rmse', 
            n_estimators = 20000,
            reg_alpha = 10.924491968127692,
            reg_lambda = 17.396730654687218,
            colsample_bytree = 0.11807135201147481,
            subsample = 0.7582562557431147,
            num_leaves = 63,
            min_child_samples = 27,
            max_bin = 523,
            n_jobs = -1,
            random_seed = SEED
        ),
        'features': ALL_FEATURES,
        'feature_importance': 0
    },

    'CatBoost': {
        'model': CatBoostRegressor(
            loss_function = 'RMSE',
            eval_metric = 'RMSE',
            depth = 4,
            l2_leaf_reg = 4.2,
            od_type = 'IncToDec',
            bagging_temperature = 290,
            cat_features = [f for f in CAT_FEATURES if f not in ['cat0']],
            iterations = 10000,
            thread_count = 4,
            random_seed = SEED
        ),
        'features': [f for f in ALL_FEATURES if f not in ['cat0']],
        'feature_importance': 0
    },
}
"""
Train and predict using every model that was described above
"""
model_results_level0 = {'model': [], 'score': [], 'training_time': []}

for m in models:
    print(f"{m}:")
    X[m] = np.zeros((X.shape[0],))
    predictions_valid = np.zeros((X.shape[0],))
    predictions = 0
    model_fi = 0
    mean_rmse = 0
    
    start_time = time.time()
    for num, (train_idx, valid_idx) in enumerate(kf.split(y,y)):
        # split the train data into train and validation
        X_train = X.iloc[train_idx][models[m]['features']]
        X_valid = X.iloc[valid_idx][models[m]['features']]
        y_train = y.iloc[train_idx]
        y_valid = y.iloc[valid_idx]
        
        model = models[m]['model']
        model.fit(X_train, y_train,
                  eval_set = [(X_valid, y_valid)],
                  early_stopping_rounds = 70,
                  verbose = False
                 )
        
        # Mean of the predictions
        predictions += model.predict(X_test[models[m]['features']]) / N_FOLD
        
        # Mean of feature importance
        models[m]['feature_importance'] += model.feature_importances_ / N_FOLD 
        
        # Out of Fold predictions
        predictions_valid[valid_idx] = model.predict(X_valid)
        fold_rmse = np.sqrt(mean_squared_error(y_valid, predictions_valid[valid_idx]))
        print(f"Fold {num} | RMSE: {fold_rmse:.3f}")
        
        mean_rmse += fold_rmse / N_FOLD
        
    X[m] = predictions_valid
    X_test[m] = predictions
    add_model_result(model_results_level0, m, mean_rmse, time.time()-start_time)
    print(f"Overall RMSE: {mean_rmse:.6f}")
    
pd.DataFrame(model_results_level0).sort_values('score')
"""
We can see that LightGBM is fastest model in these conditions.
"""
"""
### Feature importance
Let's look at feature importance for every model
"""
df_fi = pd.concat([pd.DataFrame(models[m]['feature_importance'], index=models[m]['features'], columns=[m]) for m in models],
                  axis=1)
df_fi.sort_values('XGBoost 1', ascending=False)
"""
Sum up all feature importance to get overall feature importance.  
100% - sum of importance values of all features for one model
"""
df_fi = df_fi.fillna(0).apply(lambda x: x/sum(x)*100)
df_fi['overall'] = df_fi.apply(lambda x: sum(x), axis=1)
df_fi = df_fi.apply(lambda x: x/sum(x)*100)
df_fi.sort_values('overall', ascending=False)
"""
### Blending and stacking
"""
"""
Use target prediction from previous step as features
"""
"""
### 1 level
"""
useful_features = models.keys()
"""
Reshuffle folds
"""
N_FOLD = 5
#kf = StratifiedKFold(n_splits=N_FOLD, shuffle=True, random_state=SEED)
kf = ContinuousTargetStratifiedKFold(n_splits=N_FOLD, nbins=50, shuffle=True, random_state=SEED)
stacking_models_1 = {
    'ridge': Ridge(
        alpha = 2.5
    ),
    
    'xgb': XGBRegressor(
        booster = 'gbtree',
        n_estimators = 3000,
        max_depth = 1,
        reg_alpha = 27.36157337726597,
        reg_lambda = 33.49289864031397,
        subsample =  0.96,
        colsample_bytree = 0.25360286709413943,
        min_child_weight = 6.867848163290424,
        n_jobs = 4,
        random_state = SEED,
    ),

    'rforrest': RandomForestRegressor(
        n_estimators=400,
        max_depth=4,
        n_jobs= -1, 
        random_state= SEED,
    ),

    'catboost': CatBoostRegressor(
        loss_function = 'RMSE',
        eval_metric = 'RMSE',
        depth = 1,
        iterations = 6000,
        early_stopping_rounds = 50,
        od_type = 'IncToDec',
        l2_leaf_reg = 4.482142193283575,
        bagging_temperature = 45.91463334134044,
        thread_count = 4, 
        verbose = False,
        random_state = SEED,
    ),
    
    'lgbm': LGBMRegressor(
        metric = 'rmse',      
        n_estimators = 50000,    
        reg_alpha = 10.924491968127692,
        reg_lambda = 17.396730654687218,
        learning_rate = 0.09985133666265425,
        max_depth = 5,
        num_leaves = 5,
        min_child_samples = 10,    
        max_bin = 523,
        colsample_bytree = 0.11807135201147481,
        n_jobs = 4,   
        random_state = SEED, 
    ),
}
model_results_level1 = {'model': [], 'score': [], 'training_time': []}

for i in stacking_models_1:
    predictions_valid = np.zeros((X.shape[0],))
    predictions = 0
    mean_rmse = 0

    start_time = time.time()
    for num, (train_idx, valid_idx) in enumerate(kf.split(y,y)):
        # split the train data into train and validation
        X_train = X.iloc[train_idx][useful_features]
        X_valid = X.iloc[valid_idx][useful_features]
        y_train = y.iloc[train_idx]
        y_valid = y.iloc[valid_idx]

        model = stacking_models_1[i]
        model.fit(X_train, y_train)

        # Mean of the predictions
        predictions += model.predict(X_test[useful_features]) / N_FOLD

        # Out of Fold predictions
        predictions_valid[valid_idx] = model.predict(X_valid)
        fold_rmse = np.sqrt(mean_squared_error(y_valid, predictions_valid[valid_idx]))
        print(f"Fold {num} | RMSE: {fold_rmse:.3f}")

        mean_rmse += fold_rmse / N_FOLD

    print(f"Meta model {i} RMSE: {mean_rmse:.6f}\n")
    X[f"pred_level1_{i}"] = predictions_valid
    X_test[f"pred_level1_{i}"] = predictions
    add_model_result(model_results_level1, i, mean_rmse, time.time()-start_time)
    
pd.DataFrame(model_results_level1).sort_values('score')
"""
### 2 level
"""
useful_features = [f"pred_level1_{i}" for i in stacking_models_1.keys()]
N_FOLD = 5
#kf = StratifiedKFold(n_splits=N_FOLD, shuffle=True, random_state=SEED+1)
kf = ContinuousTargetStratifiedKFold(n_splits=N_FOLD, nbins=100, shuffle=True, random_state=SEED+1)
stacking_models_2 = {
    'ridge': Ridge(
        alpha = 3.0
    ),

    'catboost': CatBoostRegressor(
        loss_function = 'RMSE',
        eval_metric = 'RMSE',
        depth = 1,
        iterations = 6000,
        early_stopping_rounds = 50,
        od_type = 'IncToDec',
        l2_leaf_reg = 2.427672821898561,
        bagging_temperature = 87.74424148586161,
        thread_count = 4, 
        verbose = False,
        random_state = SEED+1,
    ),

    'rforrest': RandomForestRegressor(
        n_estimators=500,
        max_depth=4,
        n_jobs= -1, 
        random_state= SEED+1,
    ),
}
model_results_level2 = {'model': [], 'score': [], 'training_time': []}

for i in stacking_models_2:
    predictions_valid = np.zeros((X.shape[0],))
    predictions = 0
    mean_rmse = 0

    start_time = time.time()
    for num, (train_idx, valid_idx) in enumerate(kf.split(y,y)):
        # split the train data into train and validation
        X_train = X.iloc[train_idx][useful_features]
        X_valid = X.iloc[valid_idx][useful_features]
        y_train = y.iloc[train_idx]
        y_valid = y.iloc[valid_idx]

        model = stacking_models_2[i]
        model.fit(X_train, y_train)

        # Mean of the predictions
        predictions += model.predict(X_test[useful_features]) / N_FOLD

        # Out of Fold predictions
        predictions_valid[valid_idx] = model.predict(X_valid)
        fold_rmse = np.sqrt(mean_squared_error(y_valid, predictions_valid[valid_idx]))
        print(f"Fold {num} | RMSE: {fold_rmse:.3f}")

        mean_rmse += fold_rmse / N_FOLD

    print(f"Meta model {i} RMSE: {mean_rmse:.6f}\n")
    X[f"pred_level2_{i}"] = predictions_valid
    X_test[f"pred_level2_{i}"] = predictions
    add_model_result(model_results_level2, i, mean_rmse, time.time()-start_time)
    
pd.DataFrame(model_results_level2).sort_values('score')
"""
### Final meta model
"""
"""
My final scheme of models  
<img src='https://snipboard.io/6SJToD.jpg' width='900' height="500" >
"""
useful_features = [f"pred_level2_{i}" for i in stacking_models_2.keys()]
N_FOLD = 7
#kf = StratifiedKFold(n_splits=N_FOLD, shuffle=True, random_state=SEED+2)
kf = ContinuousTargetStratifiedKFold(n_splits=N_FOLD, nbins=200, shuffle=True, random_state=SEED+2)
"""
In the las step I use Ridge regression model with alpha hyperparameter = 3.0
"""
predictions_valid = np.zeros((X.shape[0],))
final_predictions = 0
mean_rmse = 0

for num, (train_idx, valid_idx) in enumerate(kf.split(y,y)):
    # split the train data into train and validation
    X_train = X.iloc[train_idx][useful_features]
    X_valid = X.iloc[valid_idx][useful_features]
    y_train = y.iloc[train_idx]
    y_valid = y.iloc[valid_idx]
    
    model = Ridge(alpha=3.0)
    model.fit(X_train, y_train)
    
    # Mean of the predictions
    final_predictions += model.predict(X_test[useful_features]) / N_FOLD
    
    # Out of Fold predictions
    predictions_valid[valid_idx] = model.predict(X_valid)
    fold_rmse = np.sqrt(mean_squared_error(y_valid, predictions_valid[valid_idx]))
    print(f"Fold {num} | RMSE: {fold_rmse:.3f}")
    
    mean_rmse += fold_rmse / N_FOLD
    
print(f"\nOverall RMSE: {mean_rmse:.6f}")
final_predictions
"""
# 5. Submit predictions
Save the predictions to a CSV file
"""
output = pd.DataFrame({'Id': df_test.index,
                       'target': final_predictions})
output.to_csv('submission.csv', index=False)
"""
 ## _If you find it useful please upvote_
"""