# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

# ### INTRODUCTION
#
# - Read to Data
# - Data Visualization
# - Data Preprocessing
# - Building Model Pipelines
# - Data Submission
#

# ##### Note
# After series of test with different models for this task, CatBoost was the best performing model on the test data,so if this notebook must run, the cells where other models were tested should be ignored even though it wont throw up an error as the steps were all wrapped up in a pipeline. In addition, the winner notebook has a roc_auc score of 0.845... whereas this notebook got auc score of 0.843...,which is given by the catboost classifier. 

# ### Read to Data

import pandas as pd
train = pd.read_csv('./Train1.csv')
test = pd.read_csv('./Test1.csv')

train.head()

test.head()

train.info()

train.describe()

test.describe()

# check for missing values
train.isna().sum()

test.isna().sum()

# The above operation so far shows our data has alot of mising values, now we will explore each field to ascertain which column is trival to us

# check the shape of our data
print('Train Data Shape'+ str(train.shape))
print('=========================')
print( 'Test Data Shape'+ str(test.shape))

# Our data shows we have two major categorical variable aside the ApplicantID which is set out for clients profiling, they are the target variable(default status) and form field47.Now lets take a look at the distribution of the target variable by using data visualization

## We explore percentage of the missing values in our train data
total = train.isnull().sum().sort_values(ascending=False)
percent = (train.isnull().sum()/train.isnull().count()).sort_values(ascending=False)
missing_data = pd.concat([total,percent],axis=1,keys =['Total','Percent'])
missing_data[missing_data.Total > 0]

total = test.isnull().sum().sort_values(ascending=False)
percent = (train.isnull().sum()/test.isnull().count()).sort_values(ascending=False)
missing_data = pd.concat([total,percent],axis=1,keys =['Total','Percent'])
missing_data[missing_data.Total > 0]

# ### Data Visualization

# ###### Exploring our categorical data for more insight.

# import visualization libraries
import matplotlib.pyplot as plt
import numpy as np
import matplotlib as pyplot
import seaborn as sns
sns.set_style('whitegrid')
# %matplotlib inline

# Exploring our traget variable
print(train['default_status'].value_counts())

# creating a pictorial visual for the above info
labels = 'Yes', 'No'
sizes = [train.default_status[train['default_status']=='yes'].count(), train.default_status[train['default_status']=='no'].count()]
explode = (0, 0.2)
fig1, ax1 = plt.subplots(figsize=(15,8))
ax1.pie(sizes, explode=explode, labels=labels, autopct='%1.1f%%', shadow = True, startangle=90)
ax1.axis('equal')
plt.title('Proportion of customers who defaulted ??')
plt.show()


# The above diagram suggested 24.5% of customers are loan defaulters whereas those who did not difault are with the percantage 75.5%. Now our major concern is to build a model that will correctly track these small percentage of customers who are most likely to default loan issued by the company.

## exploring the form field 47
train['form_field47'].unique()

sns.countplot(x='form_field47',hue='default_status', data=train)
plt.title('Chat on Distribution of default status based on Product Type')
plt.show()

# ###### observations:
# - The above indicates clearly that between the customers who applied for charge and lending services, that customers who go for charge services type have greater chances of loan default
# - On the other hand, it is also observed that there is a greater number of customers who applied for charge and did not default their loans,therefore indicating large number of customers on both classes. This could be as a result of the fact most of the customers prefer charge services.

# ### Data Preprocessing
#

# ###### Exploring our continous variables for more insight

# Now we explore our numeric variables but we need to take care of the missing values first.
sns.heatmap(train.isnull())
plt.show()

# The missing values in both our train and test data will be filled with their respective median,the reason i chose to fill with median instead of mean is because median is not affected by outliers so it would be wise to fill with it so our original data will be well intact without deformities

train.fillna(train.median(), inplace=True)
#train = train.fillna(-99)
# We also fill for our test data
test.fillna(test.median(), inplace=True)
#test = test.fillna(-99)

# verifying if our missing train data have been filled
sns.heatmap(train.isnull())
plt.show()

# verifying if our missing test data have been filled
test.isna().sum()

#now lets verify whether our train data still retains the original info
train.describe()

# As we can see our data is still retaining its original info after filling the missing values with their respective median,also our continous data is numeric in nature we will look at how each field is distributed for a better analysis. Now let us go ahead to explore them

train.hist(bins=20,figsize=(12,16))
plt.show()

# It appears most of these fields are not properly distributed,but we will look at how some of these fields interact with the target variable.

# +
fig, ax = plt.subplots(5,2, figsize=(20,20))
sns.boxplot(y='form_field1',x='default_status',hue='default_status', data=train, ax=ax[0][0])
sns.boxplot(y='form_field26', x='default_status',hue='default_status',data=train, ax=ax[0][1])
sns.boxplot(y='form_field27', x='default_status',hue='default_status', data=train, ax=ax[1][0])
sns.boxplot(y='form_field29',x='default_status',hue='default_status',data=train, ax=ax[1][1])
sns.boxplot(y='form_field42',x='default_status',hue='default_status',data=train, ax=ax[2][0])
sns.boxplot(y='form_field44',x='default_status',hue='default_status',data=train, ax=ax[2][1])
sns.boxplot(y='form_field25',x='default_status',hue='default_status',data=train, ax=ax[3][0])
sns.boxplot(y='form_field30',x='default_status',hue='default_status',data=train, ax=ax[3][1])
sns.boxplot(y='form_field43',x='default_status',hue='default_status',data=train, ax=ax[4][0])
sns.boxplot(y='form_field33',x='default_status',hue='default_status',data=train, ax=ax[4][1])

plt.show()
# -

# ###### observations:
# - Based on credit worthiness for form_field1, there is a greater percentage of customers who appear to be credit worthy based on their historic data compared to the customers whom are not
# - Unsurprisingly for form_field26(Tenure of active credit card). Customers with lower number of tenure of all active credit cards are loan defaulters.
# - The above immediate assumption is also applicable for form_field27(Tenure of revolving active credit cards)
# - Worringly for the organization a greater percentage of the customers default their loan compared to those who do not with respect to the financial stress index across all customers.This could be worrisome to the organisation as it would incure increasing cost of fund to borrowers as majority of the customers'financial stress index is way below average.
# - There is no much significant difference in the ratio of the maximum amount due on all active credit lines and the sum of the amounts due on all active credit lines with customers who are likely to default their loan as in the case of form_field44. 

# ### Feature Engineering
# Now we prepare our data features for our model building. First we start by taking care of our categorical features, I will be applying the One Hot encoding to transform the categorical features, which will help generating dummy variables that also have a correlation with the target variable.

# +
# apply one hot encoding to our train data
#train = pd.get_dummies(train, columns=['form_field47'])
# apply one hot encoding to our test data
#test = pd.get_dummies(test, columns=['form_field47'])
# -

# Now we apply label encoder to transform our categorical train and test set
from sklearn.preprocessing import LabelEncoder
train['default_status'] = LabelEncoder().fit_transform(train.default_status)
train['form_field47'] = LabelEncoder().fit_transform(train.form_field47)
test['form_field47'] = LabelEncoder().fit_transform(test.form_field47)

# feature selection , here we drop the ApplicantID for profiling
X = train.drop(columns=['Applicant_ID','default_status'], axis=1)
y = train.default_status
#y = pd.factorize(train['default_status'])[0].reshape(-1)

# We are going to be using only our train data for testing, we carve out a portion for training and a portion for evaluating our model in the ratio of 70:30 respectively

from sklearn.model_selection import train_test_split
# splitting our train data into train and validation
x_train,x_eval,y_train,y_eval = train_test_split(X,y, test_size=0.3,
                                                 stratify = y,
                                                 shuffle=True,random_state=1)

# +
#import libraries for oversampling task
#import imblearn
#from pandas import DataFrame
#from imblearn.over_sampling import SMOTE
#smote = SMOTE(random_state =1)

#x_train_bal,y_train_bal = smote.fit_sample(x_train,y_train)

# +
# Our data is numeric in nature so it would be imperative to scale the data for better performance by setting them to a mean values of 0s and std of 1s
from sklearn.preprocessing import StandardScaler
scaler = StandardScaler()

x_train_scaled = scaler.fit_transform(x_train) 
x_eval_scaled = scaler.transform(x_eval)
# -

# Before testing our model on other algorithms lets apply logistic Regression to help us map out features that contribute greatly to the possibilites of a customer defaulting their loans by using the coefficient plots.

# +
from sklearn.linear_model import LogisticRegression

model = LogisticRegression()
result = model.fit(x_train_scaled,y_train)
y_pred = model.predict(x_eval_scaled)
# -

# print model coefficients and store in a list
coefficients = result.coef_.tolist()[0]
#create an empty dataframe to store values
coefficients_df = pd.DataFrame()
coefficients_df['Features'] = x_train.columns
# create a new column called coefficients
coefficients_df['Coefficient'] = coefficients

# plot coefficient values
coefficients_df= coefficients_df.sort_values(by='Coefficient', ascending=False)
fig,ax = plt.subplots(figsize=(15,15))
sns.barplot(x='Coefficient',y='Features', data=coefficients_df)
plt.title('Factors contributring to loan default')
plt.xlabel('Logistic Regression Coefficients')
plt.axvline(x=0, color='black')
sns.despine()

# The above chat shows the summary of how the features correlates with the target variable,on the left side indicates negative correlation meaning that increase these features leads to a decreased chance in the observation of customer loan default. With reverse being the case for the features on the right side with positive coeficients 

# ### Building  Models 

# *I will be testing the following Algorithms for evaluation;*
#
# - Balanced RandomForest Classifier
# - ExtraTreesClassifier
# - XGB Boosting(Extreme Gradient Boosting Classifier)
# - Lightgbm Classifer
# - Decision Tree Classifier

# +
#importing neccessary libraries for building our model
from sklearn.ensemble import ExtraTreesClassifier
from imblearn.ensemble import BalancedRandomForestClassifier
from sklearn.tree import DecisionTreeClassifier
from catboost import CatBoostClassifier
import lightgbm as lgb
import xgboost as xgb

# import neccessary libraries for result evaluation
from sklearn.metrics import recall_score , accuracy_score
from sklearn.metrics import roc_curve
from sklearn.metrics import roc_auc_score
import sklearn.metrics as metrics
from sklearn.metrics import classification_report
from sklearn.metrics import plot_confusion_matrix
import warnings
warnings.filterwarnings('ignore')
# -

# storing our model in a list
models =  [
    
    ['BalancedRandomForestClassifier:', BalancedRandomForestClassifier()],
           ['ExtraTreesClassifier:', ExtraTreesClassifier()],
    ['lightgbm:',lgb.LGBMClassifier()], ['DecisionTreeClassifier:', DecisionTreeClassifier()],
            ['Xgboost:', xgb.XGBClassifier()]
]

model_data = []
for name,curr_model in models:
    curr_model_data = {}
    curr_model.random_state = 42
    curr_model_data['Name'] = name
    curr_model.fit(x_train_scaled,y_train)
   
     # Measuring the model performance
    curr_model_data['Accuracy Score']= metrics.accuracy_score(y_eval,curr_model.predict(x_eval_scaled))
    curr_model_data['Recall Score']= metrics.recall_score(y_eval,curr_model.predict(x_eval_scaled))
    curr_model_data['roc_auc_score']= metrics.roc_auc_score(y_eval,curr_model.predict(x_eval_scaled))
    
    # we try and measure the difference between the train set score and evaluated score to capture any signal of overfitting
    curr_model_data['Train Set Score'] = curr_model.score(x_train_scaled,y_train)
    curr_model_data['Test Set Score'] = curr_model.score(x_eval_scaled,y_eval)
    model_data.append(curr_model_data)

# print our result
model_data

# storing the above result in a dataFrame for better visualization
df = pd.DataFrame(model_data)

df

# As we can see the balanced random forest classifier has a better result than the rest of the model bearing in mind we are to predict the probabilities and not the actual labels of customers who default their loan. Prior to this point the probabilites have been predicted using the brfc as the model but,but it didnt do well on the test data as it gave a [LB_score of 0.8137].Which could mean the model overfitted and didnt capture the true negatives quite well. I will be selecting the best three models in the above result and try to improve it by applying polynomial features of degree 2.

# ###### Metric Result evaluation

# #### CatBoost Testing

# +
# assign categorical features
#cat_features = X_copy.loc[:, ['form_field47','default_status']]
#print(cat_features)
# -

from sklearn.preprocessing import PolynomialFeatures
from sklearn.pipeline import Pipeline

# +
# i will wrap some of the steps in a pipeline for easy application
trans = PolynomialFeatures(degree=2)

model1 = CatBoostClassifier(
iterations =100,
depth=8,
random_seed = 42,
learning_rate =0.1,
eval_metric = 'AUC',
#cat_features = cat_features,
verbose = False
)
# -

cat_pipeline = Pipeline(steps=[('t', trans), ('m', model1)])

cat_pipeline.fit(x_train_scaled,y_train)

cat_pipeline_pred = cat_pipeline.predict_proba(x_eval_scaled)

# +
cat_pipeline_roc_curve = roc_auc_score(y_eval, cat_pipeline.predict(x_eval_scaled))
fpr,tpr,thresholds = roc_curve(y_eval,cat_pipeline.predict_proba(x_eval_scaled)[:,1])

plt.figure()
plt.plot(fpr,tpr,label='Catboost(area=%0.2f)' % cat_pipeline_roc_curve)
plt.plot([0, 1],[0, 1],'k--')
plt.xlim([0.0, 1.0])
plt.ylim([0.0, 1.05])
plt.xlabel('False Positive Rate')
plt.ylabel('True Positve Rate')
plt.title('Reciever Operating Characteristic')
plt.legend(loc='lower right')
plt.show()
# -

# ### Testing for lightgbm

## Applying polynomial features to lightgbm model
model2 = lgb.LGBMClassifier()

lgb_pipeline = Pipeline(steps=[('t', trans), ('m', model2)])

lgb_pipeline.fit(x_train_scaled,y_train)

lgb_pipeline_pred = lgb_pipeline.predict_proba(x_eval_scaled)

# +
lgb_pipeline_roc_curve = roc_auc_score(y_eval, lgb_pipeline.predict(x_eval_scaled))
fpr,tpr,thresholds = roc_curve(y_eval,lgb_pipeline.predict_proba(x_eval_scaled)[:,1])

plt.figure()
plt.plot(fpr,tpr,label='LightgbmClassifier(area=%0.2f)' % lgb_pipeline_roc_curve)
plt.plot([0, 1],[0, 1],'k--')
plt.xlim([0.0, 1.0])
plt.ylim([0.0, 1.05])
plt.xlabel('False Positive Rate')
plt.ylabel('True Positve Rate')
plt.title('Reciever Operating Characteristic')
plt.legend(loc='lower right')
plt.show()
# -

# ### Testing for BalancedRandomforest

# Applying polynomial features to randomforest
model3 =  BalancedRandomForestClassifier(n_estimators=500,bootstrap=False,class_weight='balanced', random_state=0)
brfc_pipeline = Pipeline(steps=[('t', trans), ('m', model3)])

brfc_pipeline.fit(x_train_scaled,y_train)

brfc_pipeline_pred = brfc_pipeline.predict_proba(x_eval_scaled)

# +
brfc_pipeline_roc_curve = roc_auc_score(y_eval, brfc_pipeline.predict(x_eval_scaled))
fpr,tpr,thresholds = roc_curve(y_eval,brfc_pipeline.predict_proba(x_eval_scaled)[:,1])

plt.figure()
plt.plot(fpr,tpr,label='Balanced Rfc(area=%0.2f)' % brfc_pipeline_roc_curve)
plt.plot([0, 1],[0, 1],'k--')
plt.xlim([0.0, 1.0])
plt.ylim([0.0, 1.05])
plt.xlabel('False Positive Rate')
plt.ylabel('True Positve Rate')
plt.title('Reciever Operating Characteristic')
plt.legend(loc='lower right')
plt.show()
# -



# The above confusion matrix is showing we have 9489+3062 correct prediction and 1053 + 3106 incorrect prediction. Now the baseline of our model is not to miss out on any clients who has a high probability of default their loan. So to improve this classification i will be selecting the top 9 features with negative coefficients and the top 9 features with a positive coefficients with the target variable to see how it performs on the test data 

# ### Apply full train to our Data

# ###### Train data

# +
X_train = train.drop(columns=['Applicant_ID','default_status'],axis =1)

y_train = train.default_status
X_train_scaled = scaler.fit_transform(X_train)
# -

# ###### Test  data

# +
X_test = test.drop(columns=['Applicant_ID'], axis =1)

X_test_scaled = scaler.transform(X_test)
# -

cat_pipeline.fit(X_train_scaled,y_train)
cat_pipeline_pred = cat_pipeline.predict_proba(X_test_scaled)

# ###### load the submission file

submission_df = pd.read_csv('./SampleSubmission.csv')
submission_df.head()

np.testing.assert_array_equal(test.index.values, submission_df.index.values)

submission_df['default_status'] = cat_pipeline_pred[:, 1]

submission_df.head()

# saving the submission file to csv
submission_df.to_csv('Loan_default_cat7_pipeline.csv', index=False)





StarcoderdataJupyterScriptsDedupFiltered