!pip install -U scikit-learn --q
import sklearn
print(sklearn.__version__)
import pandas as pd
from collections import Counter
from itertools import product
from tqdm.notebook import tqdm
import numpy as np
import optuna
import shap
import copy

import matplotlib.pyplot as plt
import seaborn as sns

from sklearn import preprocessing,model_selection,decomposition,compose,pipeline,metrics,ensemble,impute
from sklearn.utils import class_weight,compute_sample_weight
from sklearn.ensemble import GradientBoostingClassifier,GradientBoostingRegressor
import xgboost as xgb
import lightgbm as lgb


optuna.logging.set_verbosity(optuna.logging.WARNING)

shap.initjs()
def Preprocessor(data,scaler):
    
    cat_cols = data.select_dtypes('object').columns.tolist()
    num_cols = data.select_dtypes(exclude='object').columns.tolist()
    
    numerical_transformer = pipeline.Pipeline(steps=[
        ('imputer',impute.SimpleImputer(
            missing_values=np.nan,
            strategy='most_frequent',
            fill_value=None,
            verbose=0,
            copy=True,
            add_indicator=False
        )),
        ('scaler',getattr(preprocessing,scaler)())
    ])

    categorical_transformer = pipeline.Pipeline(steps=[
        ('imputer',impute.SimpleImputer(
            missing_values=np.nan,
            strategy='constant',
            fill_value='missing',
            verbose=0,
            copy=True,
            add_indicator=False
        )),
        ('encoder',preprocessing.OrdinalEncoder(handle_unknown='use_encoded_value',unknown_value=-999))
    ])

    preprocessor = compose.ColumnTransformer(transformers=[
        ('numerical_transformer',numerical_transformer,num_cols),
        ('categorical_transformer',categorical_transformer,cat_cols)
    ])
    
    preprocessor.fit(data)
    return preprocessor,cat_cols,num_cols
def cross_vall_split(df,n_split):
    df['fold'] = None
    kf = model_selection.KFold(n_splits=n_split,shuffle=True,random_state=1234)
    for fold,(train_idx,valid_idx) in enumerate(kf.split(df)):
        df.loc[valid_idx,'fold'] = fold
    return df
def kde_target(var_name, df):
    
    if df[var_name].dtype=='object':
        print("Not a numeric column")
        return None
    corr = df['loan_status'].corr(df[var_name])
    
    FullyPaid = df.loc[df['loan_status'] == 0, var_name].median()
    Default = df.loc[df['loan_status'] == 1, var_name].median()
    
    plt.figure(figsize = (12, 6))
    
    sns.kdeplot(df.loc[df['loan_status'] == 0, var_name], label = 'Fully Paid',)
    sns.kdeplot(df.loc[df['loan_status'] == 1, var_name], label = 'Default',)

    
    plt.xlabel(var_name); plt.ylabel('Density'); plt.title('%s Distribution' % var_name)
    plt.legend();
    
    print('The correlation between %s and the TARGET is %0.4f' % (var_name, corr))
    
    print('Median value for FullyPaid = %0.4f' % FullyPaid)
    print('Median value for Default = %0.4f' % Default)
df = pd.read_feather('../input/d/ransakaravihara/lending-club-loan-cleaned/Cleaned_loandata.feather')
"""
### Feature engineering
"""
for col in df.columns:
    if df[col].isna().sum()>0:print(col,"Has ",df[col].isna().sum()," null values")
df['emp_title_test'] = df['emp_title'].fillna("missing")
df['emp_length_test'] = df['emp_length'].fillna("missing")
cols_to_drop = ['emp_length_test','emp_title_test']
"""
- Create new features
"""
df['issue_d_year'] = pd.DatetimeIndex(df['issue_d']).year  
df['issue_d_month'] = pd.DatetimeIndex(df['issue_d']).month  
df['last_pymnt_d_year'] = pd.DatetimeIndex(df['last_pymnt_d']).year  
df['last_pymnt_d_month'] = pd.DatetimeIndex(df['last_pymnt_d']).month 
df['last_credit_pull_d_year'] = pd.DatetimeIndex(df['last_credit_pull_d']).year  
df['last_credit_pull_d_month'] = pd.DatetimeIndex(df['last_credit_pull_d']).month 
df['term'] = df.term.apply(lambda x:float(x[:2]))
df['int_rate'] = df.int_rate .apply(lambda x:float(x.replace("%","")))
lstatus_map = {v:k for k,v in dict(enumerate(df['loan_status'].unique())).items()}
#encode target
df['loan_status'] = df['loan_status'].map(lstatus_map)
int_to_lstatus = {val:key for key,val in lstatus_map.items()}
def plot_target_vs_cat(df,x,figsize=(30,20)):
    if df[x].nunique()<=10:
        nrows = round(df[x].nunique()/2)
        ncols = 2
    elif df[x].nunique()<=30:
        nrows = round(df[x].nunique()/3)
        ncols = 3
    else:
        print("Large unique values...skipping")
        return None
    fig,ax = plt.subplots(nrows=nrows,ncols=ncols,figsize=figsize)
    for axi,val in zip(ax.flatten(),df[x].unique()):
        e = df.query(f"{x}=='{val}'").reset_index(drop=True)
        e['loan_status'] = e['loan_status'].map(int_to_lstatus)
        e = pd.DataFrame(e['loan_status'].value_counts()).reset_index()
        colors = sns.color_palette('pastel')[0:len(e)]
        axi.pie(e['loan_status'], labels = e['index'], colors = colors, autopct='%.0f%%')
        axi.set_title(val,fontweight='bold')
    plt.show()
df.select_dtypes('object').columns.tolist()
"""
#### plot whether emp_length feature has eny major impact on target. Also, find is there are any meaning behind missing emp_length
"""
plot_target_vs_cat(df,'emp_length_test',figsize=(30,30))
"""
#### **conclusion**
 - While all other emp_length features behave as same, emp_lenght missing behaving deferently. See the fully_paid/default ratio of emp_title missing populations
"""
# Filling na
df['emp_length'] = df['emp_length'].fillna('missing')
"""
#### emp_title_test vs target column impact
"""
top_20_emp = df\
            .groupby(['emp_title_test'])\
            .agg({"id":'count','loan_status':sum})\
            .reset_index()\
            .sort_values('id',ascending=False)\
            .head(20)\
            .reset_index(drop=True)
top_20_emp['default_ratio'] = top_20_emp['loan_status']/top_20_emp['id']
top_20_emp
df['emp_title'] = df['emp_title'].fillna('missing')
df['emp_title'] = df['emp_title'].apply(lambda s: s.lower())
plot_target_vs_cat(df,'revol_util',figsize=(30,30))
"""
#### Find home owner ship distribution against loan status
"""
plot_target_vs_cat(df,'home_ownership',figsize=(20,20))
plot_target_vs_cat(df,'earliest_cr_line',figsize=(20,20))
"""
### Plot selected numerical features against target
"""
kde_target("loan_amnt",df)
"""
## Conclusion
 - If loan amount ~10,000 or lower there is high chance to fully pay the loan
"""
kde_target("funded_amnt",df)
"""
## Conclusion
 - Simmilar to the loan amount featue
 
 Cheking correlation of funded_amnt and loan_amnt
"""
df['funded_amnt'].corr(df['loan_amnt'])
cols_to_drop.append('funded_amnt')
kde_target("term",df)
"""
## Conclusion
 - Higher the number of payments on the loan there is high risk to loan to default
"""
kde_target("int_rate",df)
"""
## Conclusion
 - Higher the interest rate on the loan there is high risk to loan to default
"""
kde_target("installment",df)
"""
## Conclusion
 - Higher the installment amount there is high risk to loan will be default
"""
kde_target("earliest_cr_line",df)
kde_target("dti",df)
kde_target("tot_hi_cred_lim",df)
df['last_pymnt_d_year'].fillna(df['last_pymnt_d_year'].median(),inplace=True)
df['last_pymnt_d_month'].fillna(df['last_pymnt_d_month'].median(),inplace=True)
df['last_credit_pull_d_year'].fillna(df['last_credit_pull_d_year'].median(),inplace=True)
df['last_credit_pull_d_month'].fillna(df['last_credit_pull_d_month'].median(),inplace=True)
datetime_cols = df.select_dtypes('datetime').columns.tolist()
cols_to_drop.extend(datetime_cols)
print(cols_to_drop)
df.drop(cols_to_drop,axis=1,inplace=True)
scaler = 'None'
y = df['loan_status'].map(int_to_lstatus)
y.value_counts()
y = y.map(lstatus_map)
ids = df['id']
df.drop(['loan_status','id'],axis=1,inplace=True)
from sklearn.model_selection import train_test_split
x_train,x_test,y_train,y_test = train_test_split(df,y,test_size = 0.10 ,random_state = 2, stratify =y)
x_train,x_valid,y_train,y_valid = train_test_split(x_train,y_train,test_size = 0.20 ,random_state = 2, stratify =y_train)

x_train.shape,y_train.shape,x_test.shape,y_test.shape,x_valid.shape,y_valid.shape
cat_cols = df.select_dtypes('object').columns.tolist()
num_cols = df.select_dtypes(exclude='object').columns.tolist()

numerical_transformer = pipeline.Pipeline(steps=[
    ('scaler',getattr(preprocessing,scaler)() if scaler!='None' else None)
])

categorical_transformer = pipeline.Pipeline(steps=[
    ('encoder',preprocessing.OrdinalEncoder(handle_unknown='use_encoded_value',unknown_value=np.nan))
])

preprocessor = compose.ColumnTransformer(transformers=[
    ('numerical_transformer',numerical_transformer,num_cols),
    ('categorical_transformer',categorical_transformer,cat_cols)
])
preprocessor.fit(x_train)

x_train = pd.DataFrame(preprocessor.transform(x_train),columns=x_train.columns)
x_test = pd.DataFrame(preprocessor.transform(x_test),columns=x_train.columns)
x_valid = pd.DataFrame(preprocessor.transform(x_valid),columns=x_train.columns)
"""
### Modeling
##### XGBoost Modeling
"""
def xgb_f1(y, t, threshold=0.5):
    """Helper funcion for XGBoost f1_score"""
    t = t.get_label()
    y_bin = [1. if y_cont > threshold else 0. for y_cont in y]
    return 'f1',metrics.f1_score(t,y_bin)
model = xgb.XGBClassifier(n_estimators=20,use_label_encoder=False,tree_method='gpu_hist',gpu_id=0,random_state=1234)
model.fit(x_train,y_train,eval_set=[(x_valid,y_valid)],eval_metric=xgb_f1,verbose=False)
test_predictions_xgb = model.predict(x_test)
train_preds_xgb = model.predict(x_train)
valid_preds_xgb = model.predict(x_valid)

test_proba_xgb = model.predict_proba(x_test)
train_proba_xgb = model.predict_proba(x_train)
valid_proba_xgb = model.predict_proba(x_valid)
cm_test_xgb = metrics.confusion_matrix(y_test,test_predictions_xgb)
cm_train_xgb = metrics.confusion_matrix(y_train,train_preds_xgb)
"""
##### LighGBM Modeling
"""
lgb_bst = lgb.LGBMClassifier(n_estimators=20,random_state=1234)
lgb_bst.fit(x_train,y_train,eval_set=[(x_valid,y_valid)],early_stopping_rounds=10)
test_predictions_lgb = lgb_bst.predict(x_test)
train_preds_lgb = lgb_bst.predict(x_train)
valid_preds_lgb = lgb_bst.predict(x_valid)

test_proba_lgb = lgb_bst.predict_proba(x_test)
train_proba_lgb = lgb_bst.predict_proba(x_train)
valid_proba_lgb = lgb_bst.predict_proba(x_valid)
cm_test_lgb = metrics.confusion_matrix(y_test,test_predictions_lgb)
cm_train_lgb = metrics.confusion_matrix(y_train,train_preds_lgb)
fig,ax = plt.subplots(2,2,figsize=(15,15))
model_cms = dict(zip(['LGB_TEST','LGB_TRAIN','XGB_TEST','XGB_TRAIN'],[cm_test_lgb,cm_train_lgb,cm_test_xgb,cm_train_xgb]))

for axi,con_mat in zip(ax.flatten(),model_cms):
    sns.heatmap(model_cms[con_mat],annot=True,cmap='viridis',fmt=".0f",cbar=False,ax=axi,xticklabels=False,yticklabels=False)
    axi.text(x=0.85,y=0,s=con_mat,backgroundcolor='y')
"""
### Explain the model
"""
xgb.plot_importance(model,max_num_features=20)
plt.title("Top 20 features")
plt.show()
explainer = shap.TreeExplainer(model)
shap_values = explainer.shap_values(x_train)
print(f"Check example of {int_to_lstatus[y_train.iloc[100]]}")
shap.force_plot(explainer.expected_value, shap_values[100,:], x_train.iloc[100,:])
print(f"Check example of {int_to_lstatus[y_train.iloc[39]]}")
shap.force_plot(explainer.expected_value, shap_values[39,:], x_train.iloc[39,:])
shap.force_plot(explainer.expected_value, shap_values[:1000,:], x_train.iloc[:1000,:])
shap.summary_plot(shap_values, x_train, plot_type="bar")
shap.summary_plot(shap_values, x_train)
"""
- ### Conclusions
 - High installments caused to loan to default
 - Low inquries in past 6 mnths tends to loan to default
 - Higher the loan amount, the default chance is high
 - Higher the dti(borrower's debt repayment capacity) tends to fully paid the loan
"""
"""
 - ##### Current model using morethan 80 features. Having such a large featueres tends to data pipeline more complicated and hard to maintain. Trying to reduce the no. of features
"""
vals= np.abs(shap_values).mean(0)
feature_importance = pd.DataFrame(list(zip(x_train.columns,vals)),columns=['col_name','feature_importance_vals'])
feature_importance = feature_importance.sort_values(by=['feature_importance_vals'],ascending=False).reset_index(drop=True)
features_to_use = feature_importance['col_name'][0:30].tolist()
filtered_cols = copy.deepcopy(features_to_use)
filtered_cols.append('loan_status')
df['loan_status'] = y
fig, ax = plt.subplots(figsize=(20, 15))
sns.heatmap(
        df[features_to_use].corr(), 
        cmap = sns.diverging_palette(220, 10, as_cmap = True),
        square=True, 
        cbar=False,
        ax=ax,
#         annot=True, 
        linewidths=0.1,vmax=1.0, linecolor='white',
        annot_kws={'fontsize':12 })

plt.show()
del df['loan_status']
features_to_use.remove('funded_amnt_inv')
features_to_use
bst_xgb = xgb.XGBClassifier(n_estimators=20,use_label_encoder=False,tree_method='gpu_hist',gpu_id=0,random_state=1234)
bst_xgb.fit(x_train[features_to_use],y_train,eval_set=[(x_valid[features_to_use],y_valid)],eval_metric=xgb_f1,verbose=True)
bst_lgb = lgb.LGBMClassifier(n_estimators=20,random_state=1234)
bst_lgb.fit(x_train[features_to_use],y_train,eval_set=[(x_valid[features_to_use],y_valid)])
test_predictions_lgb = bst_lgb.predict(x_test[features_to_use])
train_preds_lgb = bst_lgb.predict(x_train[features_to_use])
valid_preds_lgb = bst_lgb.predict(x_valid[features_to_use])

test_predictions_xgb = bst_xgb.predict(x_test[features_to_use])
train_preds_xgb = bst_lgb.predict(x_train[features_to_use])
valid_preds_xgb = bst_xgb.predict(x_valid[features_to_use])
cm_test_lgb = metrics.confusion_matrix(y_test,test_predictions_lgb)
cm_train_lgb = metrics.confusion_matrix(y_train,train_preds_lgb)

cm_test_xgb = metrics.confusion_matrix(y_test,test_predictions_xgb)
cm_train_xgb = metrics.confusion_matrix(y_train,train_preds_xgb)
fig,ax = plt.subplots(2,2,figsize=(15,15))
model_cms = dict(zip(['LGB_TEST','LGB_TRAIN','XGB_TEST','XGB_TRAIN'],[cm_test_lgb,cm_train_lgb,cm_test_xgb,cm_train_xgb]))

for axi,con_mat in zip(ax.flatten(),model_cms):
    sns.heatmap(model_cms[con_mat],annot=True,cmap='viridis',fmt=".0f",cbar=False,ax=axi,xticklabels=True,yticklabels=True)
    axi.text(x=0.85,y=0,s=con_mat,backgroundcolor='y')
"""
## As per above model test results,
 - LGB Model
  - 0 predictions for loan defaults as fully paid
  - 788 predictions for loan fully paid as loan defaults (This cause revenue loss for the bank)
 - XGB Model
  - 8 predictions for loan defaults as fully paid
  - 458 predictions for loan fully paid as loan defaults (This cause revenue loss for the bank but loss is lower than LGB)
  
**Hence, I'd Pick final model as XGB Model**
"""
features_to_use = features_to_use
features_to_use.extend(['id','loan_status'])
df['id'] = ids
df['loan_status'] = y
"""
**Even our model preformed pretty well, saving the data for further model fine tuning. Not going to tune model in this notebook.**
"""
features_to_use  = list(set(features_to_use))
df[features_to_use].to_feather('train_loan_data.feather')