"""
## Import Libraries
"""
import numpy as np
import pandas as pd
"""
## Import Data
The dataset contains all available data for more than 800,000 consumer loans issued from 2007 to 2015 by Lending Club: a large US peer-to-peer lending company. There are several different versions of this dataset. We have used a version available on kaggle.com. You can find it here: https://www.kaggle.com/wendykan/lending-club-loan-data/version/1
We divided the data into two periods because we assume that some data are available at the moment when we need to build Expected Loss models, and some data comes from applications after. Later, we investigate whether the applications we have after we built the Probability of Default (PD) model have similar characteristics with the applications we used to build the PD model.
"""
loan_data_backup = pd.read_csv('/kaggle/input/loan-data-2007-2014/loan_data_2007_2014/loan_data_2007_2014.csv')
loan_data = loan_data_backup.copy()
"""
## Explore Data
"""
loan_data.head()
pd.options.display.max_columns = None
pd.options.display.max_rows = None
# Sets the pandas dataframe options to display all columns/ rows.
loan_data.head()
loan_data.columns.values
# Displays all column names.
loan_data.info()
# Displays column names, complete (non-missing) cases per column, and datatype per column.
"""
## Preprocessing - I
"""
"""
## Preprocessing few continuous variables
"""
loan_data['emp_length'].unique()
loan_data['emp_length_int'] = loan_data['emp_length'].str.replace('\+ years', '')
loan_data['emp_length_int'] = loan_data['emp_length_int'].str.replace('< 1 year', str(0))
loan_data['emp_length_int'] = loan_data['emp_length_int'].str.replace('n/a', str(0))
loan_data['emp_length_int'] = loan_data['emp_length_int'].str.replace('years', '')
loan_data['emp_length_int'] = loan_data['emp_length_int'].str.replace('year', '')
type(loan_data['emp_length_int'][0])
loan_data['emp_length_int'] = pd.to_numeric(loan_data['emp_length_int'])
type(loan_data['emp_length_int'][0])
loan_data['term'].unique()
loan_data['term_int'] = loan_data['term'].str.replace('\+ months', '')
loan_data['term_int'] = loan_data['term'].str.replace('months', '')
type(loan_data['term_int'][0])
loan_data['term_int'] = pd.to_numeric(loan_data['term_int'])
loan_data['earliest_cr_line'].head()
loan_data['earliest_cr_line_date'] = pd.to_datetime(loan_data['earliest_cr_line'], format = '%b-%y')
loan_data['earliest_cr_line_date'] = loan_data['earliest_cr_line_date'].mask(loan_data['earliest_cr_line_date'].dt.year > 2017, 
                                         loan_data['earliest_cr_line_date'] - pd.offsets.DateOffset(years=100))
type(loan_data['earliest_cr_line_date'][0])
loan_data['mths_since_earliest_cr_line'] = round((pd.to_datetime('2017-12-01') - loan_data['earliest_cr_line_date'])/np.timedelta64(1, 'M'))
loan_data['mths_since_earliest_cr_line'].describe()
loan_data.loc[:, ['earliest_cr_line', 'earliest_cr_line_date', 'mths_since_earliest_cr_line']][loan_data['mths_since_earliest_cr_line'] < 0]
loan_data['issue_d'].head()
loan_data['issue_date'] = pd.to_datetime(loan_data['issue_d'], format = '%b-%y')
#loan_data['issue_date'] = loan_data['issue_date'].mask(loan_data['issue_date'].dt.year > 2017, loan_data['issue_date'] - pd.offsets.DateOffset(years=100))
loan_data['issue_date'].head()
loan_data['mths_since_issue_d'] = round(pd.to_numeric((pd.to_datetime('2017-12-01') - loan_data['issue_date']) / np.timedelta64(1, 'M')))
loan_data['mths_since_issue_d'].describe()
"""
## Creating Dummy variables
"""
loan_data_dummies = [pd.get_dummies(loan_data['grade'], prefix= 'grade', prefix_sep = ":"),
                     pd.get_dummies(loan_data['sub_grade'], prefix= 'sub_grade', prefix_sep = ":"),
                     pd.get_dummies(loan_data['home_ownership'], prefix= 'home_ownership', prefix_sep = ":"),
                     pd.get_dummies(loan_data['verification_status'], prefix= 'verification_status', prefix_sep = ":"),
                     pd.get_dummies(loan_data['loan_status'], prefix= 'loan_status', prefix_sep = ":"),
                     pd.get_dummies(loan_data['purpose'], prefix= 'purpose', prefix_sep = ":"),
                     pd.get_dummies(loan_data['addr_state'], prefix= 'addr_state', prefix_sep = ":"),
                     pd.get_dummies(loan_data['initial_list_status'], prefix= 'initial_list_status', prefix_sep = ":")]
loan_data_dummies = pd.concat(loan_data_dummies, axis = 1)
type(loan_data_dummies)
loan_data = pd.concat([loan_data, loan_data_dummies], axis = 1)
loan_data.columns
"""
## Checking for missing values
"""
loan_data.isnull().sum()
loan_data['total_rev_hi_lim'].fillna(loan_data['funded_amnt'], inplace = True)
loan_data['annual_inc'].fillna(loan_data['annual_inc'].mean(), inplace=True)
loan_data['mths_since_earliest_cr_line'].fillna(0, inplace = True) 
loan_data['acc_now_delinq'].fillna(0, inplace = True) 
loan_data['total_acc'].fillna(0, inplace = True) 
loan_data['pub_rec'].fillna(0, inplace = True) 
loan_data['open_acc'].fillna(0, inplace = True) 
loan_data['inq_last_6mths'].fillna(0, inplace = True) 
loan_data['delinq_2yrs'].fillna(0, inplace = True) 
loan_data['emp_length_int'].fillna(0, inplace = True) 
"""
## Defining Dependent variable
"""
"""
### Good/bad(default) definition, Default and non-default accounts
"""
loan_data['loan_status'].unique()
loan_data['loan_status'].value_counts()/loan_data['loan_status'].count()
loan_data['good_bad'] = np.where(loan_data['loan_status'].isin(['Charged Off', 'Default',
                                                                   'Does not meet the credit policy. Status:Charged Off',
                                                                   'Late (31-120 days)']), 0, 1)
loan_data['good_bad'].value_counts()
"""
## Train/Test Split
"""
from sklearn.model_selection import train_test_split
loan_data_inputs_train, loan_data_inputs_test, loan_data_targets_train, loan_data_targets_test = train_test_split(loan_data.drop('good_bad', axis = 1), loan_data['good_bad'], test_size = 0.2, random_state = 42)
"""
## Weight of Evidence Calculation
"""
"""
Since data preparation binning will be same for both training and test data, combining both dataframes to a list so that changes can be applied uniformly.
"""
data_cleaner = [loan_data_inputs_train, loan_data_inputs_test]
"""
### Weight of Evidence Calculation for 1 variable
"""
loan_data_inputs_train['grade'].unique()
df1 = pd.concat([loan_data_inputs_train['grade'], loan_data_targets_train], axis = 1)
df1.head()
df1.groupby(df1.columns.values[0], as_index = False)[df1.columns.values[1]].count()
df1.groupby(df1.columns.values[0], as_index = False)[df1.columns.values[1]].mean()
df1 = pd.concat([df1.groupby(df1.columns.values[0], as_index = False)[df1.columns.values[1]].count(),
                 df1.groupby(df1.columns.values[0], as_index = False)[df1.columns.values[1]].mean()], axis = 1)
df1.head()
df1 = df1.iloc[: , [0, 1, 3]]
df1.columns = [df1.columns.values[0], 'n_obs', 'prop_good']
df1['prop_n_obs'] = df1['n_obs'] / df1['n_obs'].sum()
df1['n_good'] = df1['prop_good'] * df1['n_obs']
df1['n_bad'] = (1 - df1['prop_good']) * df1['n_obs']
df1.head()
df1['prop_n_good'] = df1['n_good'] / df1['n_good'].sum()
df1['prop_n_bad'] = df1['n_bad'] / df1['n_bad'].sum()
df1.head()
df1['WoE'] = np.log(df1['prop_n_good'] / df1['prop_n_bad'])
df1.head()
df1 = df1.sort_values(['WoE'])
df1 = df1.reset_index(drop = True)
df1.head()
"""
Calculating the diff in proportion of good/bad as we move across categories
"""
df1['diff_prop_good'] = df1['prop_good'].diff().abs()
df1['diff_WoE'] = df1['WoE'].diff().abs()
df1
df1['IV'] = (df1['prop_n_good'] - df1['prop_n_bad']) * df1['WoE']
df1['IV'] = df1['IV'].sum()
df1
"""
### Automating WOE calculation
"""
def woe_discrete(df, discrete_variable_name, good_bad_variable_df):
    df = pd.concat([df[discrete_variable_name], good_bad_variable_df], axis = 1)
    df = pd.concat([df.groupby(df.columns.values[0], as_index = False)[df.columns.values[1]].count(),
                    df.groupby(df.columns.values[0], as_index = False)[df.columns.values[1]].mean()], axis = 1)
    df = df.iloc[: , [0, 1, 3]]
    df.columns = [df.columns.values[0], 'n_obs', 'prop_good']
    df['prop_n_obs'] = df['n_obs'] / df['n_obs'].sum()
    df['n_good'] = df['prop_good'] * df['n_obs']
    df['n_bad'] = (1 - df['prop_good']) * df['n_obs']
    df['prop_n_good'] = df['n_good'] / df['n_good'].sum()
    df['prop_n_bad'] = df['n_bad'] / df['n_bad'].sum()
    df['WoE'] = np.log(df['prop_n_good'] / df['prop_n_bad'])
    df = df.sort_values(['WoE'])
    df = df.reset_index(drop = True)
    df['diff_prop_good'] = df['prop_good'].diff().abs()
    df['diff_WoE'] = df['WoE'].diff().abs()
    df['IV'] = (df['prop_n_good'] - df['prop_n_bad']) * df['WoE']
    df['IV'] = df['IV'].sum()
    return(df)
df_temp = woe_discrete(loan_data_inputs_train, 'grade', loan_data_targets_train)
df_temp
"""
### Preprocessing discrete variables: Visualising results
"""
import matplotlib.pyplot as plt
import seaborn as sns
sns.set()
def plot_by_woe(df_WOE, rotation_of_xaxis_labels = 90):
    # reading the first column from the WOE dataset, and coercing to string
    x = np.array(df_WOE.iloc[:, 0].apply(str))
    y = df_WOE['WoE']
    plt.figure(figsize = (18, 6))
    plt.plot(x, y, marker = 'o', linestyle = '--', color = 'k')
    plt.xlabel(df_WOE.columns[0])
    plt.ylabel('Weight of Evidence')
    plt.title('Weight of Evidence by' + df_WOE.columns[0])
    plt.xticks(rotation = rotation_of_xaxis_labels)
plot_by_woe(df_temp)
"""
### Preprocessing Discrete Variables: Creating dummy variables, Part 1
"""
df_temp = woe_discrete(loan_data_inputs_train, 'home_ownership', loan_data_targets_train)
df_temp
plot_by_woe(df_temp)
"""
Other, none and any categories represent very few observations. We combine them to not lose the info they bring. Combining underrepresented categories. They are combined w/ the riskiest category w/ enough number of observations. In this case: rent
"""
for dataset in data_cleaner:  
    dataset['home_ownership:RENT_OTHER_ANY_NONE'] = sum([dataset['home_ownership:RENT'], 
                                                            dataset['home_ownership:OTHER'], 
                                                            dataset['home_ownership:ANY'],
                                                            dataset['home_ownership:NONE']])
"""
#### Address State
"""
loan_data_inputs_train['addr_state'].unique()
df_temp = woe_discrete(loan_data_inputs_train, 'addr_state', loan_data_targets_train )
df_temp
plot_by_woe(df_temp)
"""
There are fewer than 50 states in the results. There are no borrowers from 1 of the states. (North Dakota)
"""
for dataset in data_cleaner: 
    if ['addr_state:ND'] in dataset.columns.values:
        pass
    else:
        dataset['addr_state:ND'] = 0
"""
Weight of evidence is not calculated for 2 states. There are no bads. The first two states have very few observations. These can be clubbed w/ the next riskiest category
"""
plot_by_woe(df_temp.iloc[2: -2, :])
"""
Nevada has a lower WOE than the states after. It could be clubbed with the low proportion states we had excluded. The state with no records (North Dakota) can be included here. This would be a conservative approach to take. For states with unknown values, assignin them the worst WOE value. The first dummy variable will include NE, IA, NV, FL, HI and AL. The states at the end will include WV, NH, WY, DC, ME and ID. Plotting the remaining states: 

"""
plot_by_woe(df_temp.iloc[6: -6, :])
"""
NY and CA have the highest number of observations. Most borrowers. These can be in a separate category dummy of their own. 
The other states can be in one category/ grouped together. But leaving out NY and CA because these need to be in a separate category. 

NM and VA could be combined
OK, TN, MO, LA, MD, NC could be combined to one. 
These can't be combined together because they are separated by NY. (10% of observations come from NY)

UT, KY, AZ and NJ can be comibined into one category. They have similar WOE and neither has very high number of values. 

AR, MI, PA, OH and MN have similar WOE. RI, MA, DE, SD and IN belong to one category. 
GA, WA and OR belong together. WI and MT need to be combined.

TX, IL and CT. have similar WOE but Texas has too many values. It can stay a separate category. 
IL and CT can be combined

KS, SC, CO, VT, AK and MS combined into one category
They have very high WOE but too few observations. This makes WoE less reliable. They can be combined together.
"""
for dataset in data_cleaner: 
    dataset['addr_state:NE_IA_NV_FL_HI_AL'] = sum([dataset['addr_state:NE'], dataset['addr_state:IA'],
                                                     dataset['addr_state:NV'], dataset['addr_state:FL'], 
                                                     dataset['addr_state:HI'], dataset['addr_state:AL']])
    
    dataset['addr_state:NM_VA'] = sum([dataset['addr_state:NM'], dataset['addr_state:VA']])
    
    dataset['addr_state:OK_TN_MO_LA_MD_NC'] = sum([dataset['addr_state:OK'], dataset['addr_state:TN'],
                                                     dataset['addr_state:MO'], dataset['addr_state:LA'], 
                                                     dataset['addr_state:MD'], dataset['addr_state:NC']])
    
    dataset['addr_state:UT_KY_AZ_NJ'] = sum([dataset['addr_state:UT'], dataset['addr_state:KY'],
                                                     dataset['addr_state:AZ'], dataset['addr_state:NJ']])
    
    dataset['addr_state:AR_MI_PA_OH_MN']= sum([dataset['addr_state:AR'], dataset['addr_state:MI'],
                                                  dataset['addr_state:PA'], dataset['addr_state:OH'], 
                                                  dataset['addr_state:MN']])
    
    dataset['addr_state:RI_MA_DE_SD_IN']= sum([dataset['addr_state:RI'], dataset['addr_state:MA'],
                                                  dataset['addr_state:DE'], dataset['addr_state:SD'],
                                                  dataset['addr_state:IN']])
    
    dataset['addr_state:GA_WA_OR'] = sum([dataset['addr_state:GA'], dataset['addr_state:WA'], 
                                          dataset['addr_state:OR']])
    
    dataset['addr_state:WI_MT'] = sum([dataset['addr_state:WI'], dataset['addr_state:MT']])
    
    dataset['addr_state:IL_CT'] = sum([dataset['addr_state:IL'], dataset['addr_state:CT']])
    
    dataset['addr_state:KS_SC_CO_VT_AK_MS'] = sum([dataset['addr_state:KS'], dataset['addr_state:SC'],
                                                     dataset['addr_state:CO'], dataset['addr_state:VT'], 
                                                     dataset['addr_state:AK'], dataset['addr_state:MS']])
    
    dataset['addr_state:WV_NH_WY_DC_ME_ID'] = sum([dataset['addr_state:WV'], dataset['addr_state:NH'],
                                                     dataset['addr_state:WY'], dataset['addr_state:DC'], 
                                                     dataset['addr_state:ME'], dataset['addr_state:ID']])
"""
#### verification_status
"""
loan_data_inputs_train['verification_status'].unique()
df_temp = woe_discrete(loan_data_inputs_train, 'verification_status', loan_data_targets_train)
df_temp
plot_by_woe(df_temp)
"""
The WOE of all these values is sufficiently different. All categories have enough observations. This variable can remain as is. The lowest WOE belongs to verified. That can become the reference category
"""
"""
#### purpose
"""
loan_data_inputs_train['purpose'].unique()
df_temp = woe_discrete(loan_data_inputs_train, 'purpose', loan_data_targets_train)
df_temp
plot_by_woe(df_temp)
"""
Debt consolidation and credit card can remain a separate value. It has too many values. 

"""
#dataset['purpose:debt_consolidation']
#dataset['purpose:credit_card']
for dataset in data_cleaner: 
    dataset['purpose:educ__sm_b__wedd__ren_en__mov__house'] = sum([dataset['purpose:educational'], 
                                                                       dataset['purpose:small_business'],
                                                                       dataset['purpose:wedding'], 
                                                                       dataset['purpose:renewable_energy'],
                                                                       dataset['purpose:moving'], 
                                                                       dataset['purpose:house']])
    dataset['purpose:oth__med__vacation'] = sum([dataset['purpose:other'], 
                                                     dataset['purpose:medical'],
                                                     dataset['purpose:vacation']])
    dataset['purpose:major_purch__car__home_impr'] = sum([dataset['purpose:major_purchase'],
                                                              dataset['purpose:car'],
                                                              dataset['purpose:home_improvement']])
"""
#### initial_list_status
"""
loan_data_inputs_train['initial_list_status'].unique()
df_temp = woe_discrete(loan_data_inputs_train, 'initial_list_status', loan_data_targets_train)
df_temp
plot_by_woe(df_temp)
"""
There are only 2 categories and they both bins have enough values- no need to combine these. 
"""
"""
### Preprocessing Continuous Variables: Automating Calculations and Visualizing Results
"""
def woe_ordered_continuous(df, discrete_variabe_name, good_bad_variable_df):
    df = pd.concat([df[discrete_variabe_name], good_bad_variable_df], axis = 1)
    df = pd.concat([df.groupby(df.columns.values[0], as_index = False)[df.columns.values[1]].count(),
                    df.groupby(df.columns.values[0], as_index = False)[df.columns.values[1]].mean()], axis = 1)
    df = df.iloc[:, [0, 1, 3]]
    df.columns = [df.columns.values[0], 'n_obs', 'prop_good']
    df['prop_n_obs'] = df['n_obs'] / df['n_obs'].sum()
    df['n_good'] = df['prop_good'] * df['n_obs']
    df['n_bad'] = (1 - df['prop_good']) * df['n_obs']
    df['prop_n_good'] = df['n_good'] / df['n_good'].sum()
    df['prop_n_bad'] = df['n_bad'] / df['n_bad'].sum()
    df['WoE'] = np.log(df['prop_n_good'] / df['prop_n_bad'])
    #df = df.sort_values(['WoE'])
    #df = df.reset_index(drop = True)
    df['diff_prop_good'] = df['prop_good'].diff().abs()
    df['diff_WoE'] = df['WoE'].diff().abs()
    df['IV'] = (df['prop_n_good'] - df['prop_n_bad']) * df['WoE']
    df['IV'] = df['IV'].sum()
    return df
"""
### Preprocessing Continuous Variables: Creating Dummy Variables
"""
"""
#### Term
"""
loan_data_inputs_train['term_int'].unique()
df_temp = woe_ordered_continuous(loan_data_inputs_train, 'term_int', loan_data_targets_train)
df_temp
plot_by_woe(df_temp)
for dataset in data_cleaner:
    dataset['term:36'] = np.where((dataset['term_int'] == 36), 1, 0)
    dataset['term:60'] = np.where((dataset['term_int'] == 60), 1, 0)
"""
#### Employment Length
"""
loan_data_inputs_train['emp_length_int'].unique()
df_temp = woe_ordered_continuous(loan_data_inputs_train, 'emp_length_int', loan_data_targets_train)
df_temp
plot_by_woe(df_temp)
"""
0 years of emplyment can be separate category. 1 year of employment
2-4
5-6
7-9
10
"""
for dataset in data_cleaner: 
    dataset['emp_length:0'] = np.where(dataset['emp_length_int'].isin([0]), 1, 0)
    dataset['emp_length:1'] = np.where(dataset['emp_length_int'].isin([1]), 1, 0)
    dataset['emp_length:2-4'] = np.where(dataset['emp_length_int'].isin(range(2, 5)), 1, 0)
    dataset['emp_length:5-6'] = np.where(dataset['emp_length_int'].isin(range(5, 7)), 1, 0)
    dataset['emp_length:7-9'] = np.where(dataset['emp_length_int'].isin(range(7, 10)), 1, 0)
    dataset['emp_length:10'] = np.where(dataset['emp_length_int'].isin([10]), 1, 0)
"""
#### Months since Issue
"""
loan_data_inputs_train['mths_since_issue_d'].unique()
"""
This variable has too many values to use as is. It can be converted to categorical using the cut function
"""
loan_data_inputs_train['mths_since_issue_factor'] = pd.cut(loan_data_inputs_train['mths_since_issue_d'], 50)
loan_data_inputs_train['mths_since_issue_factor'].unique()
df_temp = woe_ordered_continuous(loan_data_inputs_train, 'mths_since_issue_factor', loan_data_targets_train)
df_temp
plot_by_woe(df_temp)
"""
The first three bins need to be kept separate from the rest of the categories. They have higher WOEs than the rest of the categories. Plotting the remaining bins to see which ones can be binned together:
"""
plot_by_woe(df_temp.iloc[3:, :])
"""
The first 4 bins have similar weights of evidence (41.4, 48.6]. The next two categories can be grouped together.(48.6, 52.2]. The next 7 variables have similar WOE and can be grouped together (52.2, 64.8]. 

The rest of the values have highly varying values. This is a warning to check their bin sizes. Usually small bin sizes could contribute to such variation. These can be combined into one dummy variable till there is an upward trend, and the rest of them together. (64.8, 84.60] & (84.60, Inf)
"""
for dataset in data_cleaner: 
    dataset['mths_since_issue_d:<38'] = np.where(dataset['mths_since_issue_d'].isin(range(38)), 1, 0)
    dataset['mths_since_issue_d:38-39'] = np.where(dataset['mths_since_issue_d'].isin(range(38, 40)), 1, 0)
    dataset['mths_since_issue_d:40-41'] = np.where(dataset['mths_since_issue_d'].isin(range(40, 42)), 1, 0)
    dataset['mths_since_issue_d:42-48'] = np.where(dataset['mths_since_issue_d'].isin(range(42, 49)), 1, 0)
    dataset['mths_since_issue_d:49-52'] = np.where(dataset['mths_since_issue_d'].isin(range(49, 53)), 1, 0)
    dataset['mths_since_issue_d:53-64'] = np.where(dataset['mths_since_issue_d'].isin(range(53, 65)), 1, 0)
    dataset['mths_since_issue_d:65-84'] = np.where(dataset['mths_since_issue_d'].isin(range(65, 85)), 1, 0)
    dataset['mths_since_issue_d:>84'] = np.where(dataset['mths_since_issue_d'].isin(range(85, int(dataset['mths_since_issue_d'].max()))), 1, 0)
"""
#### interest rate
"""
loan_data_inputs_train['int_rate'].unique()
loan_data_inputs_train['int_rate_factor'] = pd.cut(loan_data_inputs_train['int_rate'], 50)
df_temp = woe_ordered_continuous(loan_data_inputs_train, 'int_rate_factor', loan_data_targets_train)
df_temp
plot_by_woe(df_temp)
"""
Largely the relationship of interest rate w/ WOE is monotonic. The boundaries of the bins will not have an impact on the monotonicity of this relationship. 

Identifying cases where WOE differs significantly from the previous categories (huge jumps or lows).
The first ten bins have very large WOEs have low bin size. They can be combined together. This is till 9.548. 
Then the bins till 12.025 see a reduction in WOE.The other points where a huge change in WOE is seen are 15.74, 20.281, 23.583. 

The bins after that have some variation but have very small bin sizes. 

For a continuous variables, the condition will not look up a list. Instead we will use greater than/less than conditions. 
"""
for dataset in data_cleaner: 
    dataset['int_rate:<9.548'] = np.where((dataset['int_rate'] <= 9.548), 1, 0)
    dataset['int_rate:9.548-12.025'] = np.where((dataset['int_rate'] > 9.548) & (dataset['int_rate'] <= 12.025), 1, 0)
    dataset['int_rate:12.025-15.74'] = np.where((dataset['int_rate'] > 12.025) & (dataset['int_rate'] <= 15.74), 1, 0)
    dataset['int_rate:15.74-20.281'] = np.where((dataset['int_rate'] > 15.74) & (dataset['int_rate'] <= 20.281), 1, 0)
    dataset['int_rate:>20.281'] = np.where((dataset['int_rate'] > 20.281), 1, 0)
"""
#### Funded amount
"""
loan_data_inputs_train['funded_amnt_factor'] = pd.cut(loan_data_inputs_train['funded_amnt'], 50)
df_temp = woe_ordered_continuous(loan_data_inputs_train, 'funded_amnt_factor', loan_data_targets_train)
df_temp
plot_by_woe(df_temp)
"""
Consecutive intervals have very different weights of evidence. All of these variations are around a trend that is almost horizontal. Also, the IV of the variable is low. This is variable with weak predictive power (WoE varies greatly, and there is no association w/ the independent variable. )

This variable can be left out of the model. 
"""
"""
#### months since credit line
"""
loan_data_inputs_train['mths_since_earliest_cr_line'].unique()
loan_data_inputs_train['mths_since_earliest_cr_line_factor'] = pd.cut(loan_data_inputs_train['mths_since_earliest_cr_line'], 50)
df_temp = woe_ordered_continuous(loan_data_inputs_train, 'mths_since_earliest_cr_line_factor', loan_data_targets_train)
df_temp
loan_data_inputs_train['mths_since_earliest_cr_line'].min()
plot_by_woe(df_temp)
plot_by_woe(df_temp.iloc[6:, :])
for dataset in data_cleaner: 
    dataset['mths_since_earliest_cr_line:<140'] = np.where(dataset['mths_since_earliest_cr_line'].isin(range(140)), 1, 0)
    dataset['mths_since_earliest_cr_line:141-164'] = np.where(dataset['mths_since_earliest_cr_line'].isin(range(140, 165)), 1, 0)
    dataset['mths_since_earliest_cr_line:165-247'] = np.where(dataset['mths_since_earliest_cr_line'].isin(range(165, 248)), 1, 0)
    dataset['mths_since_earliest_cr_line:248-270'] = np.where(dataset['mths_since_earliest_cr_line'].isin(range(248, 271)), 1, 0)
    dataset['mths_since_earliest_cr_line:271-352'] = np.where(dataset['mths_since_earliest_cr_line'].isin(range(271, 353)), 1, 0)
    dataset['mths_since_earliest_cr_line:>352'] = np.where(dataset['mths_since_earliest_cr_line'].isin(range(353, int(dataset['mths_since_earliest_cr_line'].max()))), 1, 0)
"""
#### Installment
"""
loan_data_inputs_train['installment'].unique()
loan_data_inputs_train['installment_factor'] = pd.cut(loan_data_inputs_train['installment'], 50)
df_temp = woe_ordered_continuous(loan_data_inputs_train, 'installment_factor', loan_data_targets_train)
df_temp
plot_by_woe(df_temp)
"""
Installement has very low IV, and a highly erratic WoE graph. 
"""
"""
#### Delinquency in the last 2 years
"""
loan_data_inputs_train['delinq_2yrs'].unique()
df_temp = woe_ordered_continuous(loan_data_inputs_train, 'delinq_2yrs', loan_data_targets_train)
df_temp
plot_by_woe(df_temp)
"""
Most accounts have 0 delinquencies. They can remain in a separate category. Then the values from 1-3 delinquencies. The bins after that have very few values, so they can all be clubbed together. 
"""
for dataset in data_cleaner: 
    dataset['delinq_2yrs:0'] = np.where((dataset['delinq_2yrs'] == 0), 1, 0)
    dataset['delinq_2yrs:1-3'] = np.where((dataset['delinq_2yrs'] >= 1) & (dataset['delinq_2yrs'] <= 3), 1, 0)
    dataset['delinq_2yrs:>=4'] = np.where((dataset['delinq_2yrs'] >= 9), 1, 0)
"""
#### Inquiries in the last 6 months
"""
loan_data_inputs_train['inq_last_6mths'].unique()
df_temp = woe_ordered_continuous(loan_data_inputs_train, 'inq_last_6mths', loan_data_targets_train)
df_temp
plot_by_woe(df_temp)
"""
The values equal to 0 need to be a separate category. Bins from 1-3 can be combined. Then the next cutoff can be around 6. The rest of the bins have very few values and can be combined together. 
"""
for dataset in data_cleaner: 
    dataset['inq_last_6mths:0'] = np.where((dataset['inq_last_6mths'] == 0), 1, 0)
    dataset['inq_last_6mths:1-2'] = np.where((dataset['inq_last_6mths'] >= 1) & (dataset['inq_last_6mths'] <= 2), 1, 0)
    dataset['inq_last_6mths:3-6'] = np.where((dataset['inq_last_6mths'] >= 3) & (dataset['inq_last_6mths'] <= 6), 1, 0)
    dataset['inq_last_6mths:>6'] = np.where((dataset['inq_last_6mths'] > 6), 1, 0)
"""
#### Number of trades
"""
loan_data_inputs_train['open_acc'].unique()
df_temp = woe_ordered_continuous(loan_data_inputs_train, 'open_acc', loan_data_targets_train)
df_temp
plot_by_woe(df_temp)
for dataset in data_cleaner: 
    dataset['open_acc:0'] = np.where((dataset['open_acc'] == 0), 1, 0)
    dataset['open_acc:1-3'] = np.where((dataset['open_acc'] >= 1) & (dataset['open_acc'] <= 3), 1, 0)
    dataset['open_acc:4-12'] = np.where((dataset['open_acc'] >= 4) & (dataset['open_acc'] <= 12), 1, 0)
    dataset['open_acc:13-17'] = np.where((dataset['open_acc'] >= 13) & (dataset['open_acc'] <= 17), 1, 0)
    dataset['open_acc:18-22'] = np.where((dataset['open_acc'] >= 18) & (dataset['open_acc'] <= 22), 1, 0)
    dataset['open_acc:23-25'] = np.where((dataset['open_acc'] >= 23) & (dataset['open_acc'] <= 25), 1, 0)
    dataset['open_acc:26-30'] = np.where((dataset['open_acc'] >= 26) & (dataset['open_acc'] <= 30), 1, 0)
    dataset['open_acc:>=31'] = np.where((dataset['open_acc'] >= 31), 1, 0)
"""
#### Number of public records
"""
loan_data_inputs_train['pub_rec'].unique()
df_temp = woe_ordered_continuous(loan_data_inputs_train, 'pub_rec', loan_data_targets_train)
df_temp
plot_by_woe(df_temp)
for dataset in data_cleaner: 
    dataset['pub_rec:0-2'] = np.where((dataset['pub_rec'] >= 0) & (dataset['pub_rec'] <= 2), 1, 0)
    dataset['pub_rec:3-4'] = np.where((dataset['pub_rec'] >= 3) & (dataset['pub_rec'] <= 4), 1, 0)
    dataset['pub_rec:>=5'] = np.where((dataset['pub_rec'] >= 5), 1, 0)
"""
#### Number of accounts
"""
loan_data_inputs_train['total_acc'].unique()
loan_data_inputs_train['total_acc_factor']= pd.cut(loan_data_inputs_train['total_acc'], 50)
df_temp = woe_ordered_continuous(loan_data_inputs_train, 'total_acc_factor', loan_data_targets_train)
df_temp
plot_by_woe(df_temp)
for dataset in data_cleaner: 
    dataset['total_acc:<=27'] = np.where((dataset['total_acc'] <= 27), 1, 0)
    dataset['total_acc:28-51'] = np.where((dataset['total_acc'] >= 28) & (dataset['total_acc'] <= 51), 1, 0)
    dataset['total_acc:>=52'] = np.where((dataset['total_acc'] >= 52), 1, 0)
"""
#### Accounts now delinquent
"""
loan_data_inputs_train['acc_now_delinq'].unique()
df_temp = woe_ordered_continuous(loan_data_inputs_train, 'acc_now_delinq', loan_data_targets_train)
df_temp
plot_by_woe(df_temp)
for dataset in data_cleaner: 
    dataset['acc_now_delinq:0'] = np.where((dataset['acc_now_delinq'] == 0), 1, 0)
    dataset['acc_now_delinq:>=1'] = np.where((dataset['acc_now_delinq'] >= 1), 1, 0)
"""
#### Annual income
"""
"""
This variable can have only positive values and has values running upto millions.
"""
loan_data_inputs_train['annual_inc_factor'] = pd.cut(loan_data_inputs_train['annual_inc'], 50)
# Here we do fine-classing: using the 'cut' method, we split the variable into 50 categories by its values.
df_temp = woe_ordered_continuous(loan_data_inputs_train, 'annual_inc_factor', loan_data_targets_train)
# We calculate weight of evidence.
df_temp
"""
There are a large number of values in the first interval (~94% of values). In this case, it might be worthwhile to split the variable into more classes. Eg. 100
"""
loan_data_inputs_train['annual_inc_factor'] = pd.cut(loan_data_inputs_train['annual_inc'], 100)
# Here we do fine-classing: using the 'cut' method, we split the variable into 100 categories by its values.
df_temp = woe_ordered_continuous(loan_data_inputs_train, 'annual_inc_factor', loan_data_targets_train)
# We calculate weight of evidence.
df_temp
"""
This makes things better, but still the first category ends up with 60% of observations. This is logical, because there are very few people with high incomes. We can increate one dummy variable for rich , while exploring the rest of the categories separately. Let the cutoff for rich start at the 3rd bin (144693.64). 
"""
"""
Initial examination shows that there are too few individuals with large income and too many with small income. Hence, we are going to have one category for more than 150K, and we are going to apply our approach to determine the categories of everyone with 140k or less.
"""
# Initial examination shows that there are too few individuals with large income and too many with small income.
# Hence, we are going to have one category for more than 150K, and we are going to apply our approach to determine
# the categories of everyone with 140k or less.
loan_data_inputs_train_temp = loan_data_inputs_train.loc[loan_data_inputs_train['annual_inc'] <= 140000, : ]
#loan_data_temp = loan_data_temp.reset_index(drop = True)
#loan_data_inputs_train_temp
loan_data_inputs_train_temp["annual_inc_factor"] = pd.cut(loan_data_inputs_train_temp['annual_inc'], 50)
loan_data_inputs_train_temp["annual_inc_factor"].unique()
df_temp = woe_ordered_continuous(loan_data_inputs_train_temp, 'annual_inc_factor', 
                                 loan_data_targets_train[loan_data_inputs_train_temp.index])
df_temp
plot_by_woe(df_temp) 
# WoE is monotonically increasing with income, so we split income in 12 categories
for dataset in data_cleaner: 
    dataset['annual_inc:<20K'] = np.where((dataset['annual_inc'] <= 20000), 1, 0)
    dataset['annual_inc:20K-30K'] = np.where((dataset['annual_inc'] > 20000) & (dataset['annual_inc'] <= 30000), 1, 0)
    dataset['annual_inc:30K-40K'] = np.where((dataset['annual_inc'] > 30000) & (dataset['annual_inc'] <= 40000), 1, 0)
    dataset['annual_inc:40K-50K'] = np.where((dataset['annual_inc'] > 40000) & (dataset['annual_inc'] <= 50000), 1, 0)
    dataset['annual_inc:50K-60K'] = np.where((dataset['annual_inc'] > 50000) & (dataset['annual_inc'] <= 60000), 1, 0)
    dataset['annual_inc:60K-70K'] = np.where((dataset['annual_inc'] > 60000) & (dataset['annual_inc'] <= 70000), 1, 0)
    dataset['annual_inc:70K-80K'] = np.where((dataset['annual_inc'] > 70000) & (dataset['annual_inc'] <= 80000), 1, 0)
    dataset['annual_inc:80K-90K'] = np.where((dataset['annual_inc'] > 80000) & (dataset['annual_inc'] <= 90000), 1, 0)
    dataset['annual_inc:90K-100K'] = np.where((dataset['annual_inc'] > 90000) & (dataset['annual_inc'] <= 100000), 1, 0)
    dataset['annual_inc:100K-120K'] = np.where((dataset['annual_inc'] > 100000) & (dataset['annual_inc'] <= 120000), 1, 0)
    dataset['annual_inc:120K-140K'] = np.where((dataset['annual_inc'] > 120000) & (dataset['annual_inc'] <= 140000), 1, 0)
    dataset['annual_inc:>140K'] = np.where((dataset['annual_inc'] > 140000), 1, 0)

"""
#### Months since last delinquency
"""
loan_data_inputs_train['mths_since_last_delinq'].unique()
"""
This variable has null values. The binning exercise should not consider those. However, instead  of dropping these variables, we can introduce a new dummy variable that takes on a value 1 when the value is nan and 0 when its not. Examining the WoE of the remaining values:
"""
loan_data_inputs_train_temp = loan_data_inputs_train[pd.notnull(loan_data_inputs_train['mths_since_last_delinq'])]
loan_data_inputs_train_temp['mths_since_last_delinq_factor'] = pd.cut(loan_data_inputs_train_temp['mths_since_last_delinq'], 50)

df_temp = woe_ordered_continuous(loan_data_inputs_train_temp, 'mths_since_last_delinq_factor', 
                                 loan_data_targets_train[loan_data_inputs_train_temp.index])
# We calculate weight of evidence.
df_temp
plot_by_woe(df_temp, 90)
"""
Weight of evidence for the first category is considerably lower than the next one. So that can be a separate category (0-3]. The next jumps in WoE happen around 30 an then 56. The values after 56 have very few observations in each bin and hence can be combined together. 
"""
for dataset in data_cleaner: 
    dataset['mths_since_last_delinq:Missing'] = np.where((dataset['mths_since_last_delinq'].isnull()), 1, 0)
    dataset['mths_since_last_delinq:0-3'] = np.where((dataset['mths_since_last_delinq'] >= 0) & (dataset['mths_since_last_delinq'] <= 3), 1, 0)
    dataset['mths_since_last_delinq:4-30'] = np.where((dataset['mths_since_last_delinq'] >= 4) & (dataset['mths_since_last_delinq'] <= 30), 1, 0)
    dataset['mths_since_last_delinq:31-56'] = np.where((dataset['mths_since_last_delinq'] >= 31) & (dataset['mths_since_last_delinq'] <= 56), 1, 0)
    dataset['mths_since_last_delinq:>=57'] = np.where((dataset['mths_since_last_delinq'] >= 57), 1, 0)
"""
#### Debt to income ratio
"""
loan_data_inputs_train['dti'].unique()
loan_data_inputs_train['dti'].max()
# dti
loan_data_inputs_train['dti_factor'] = pd.cut(loan_data_inputs_train['dti'], 50)
# Here we do fine-classing: using the 'cut' method, we split the variable into 100 categories by its values.
df_temp = woe_ordered_continuous(loan_data_inputs_train, 'dti_factor', loan_data_targets_train)
# We calculate weight of evidence.
df_temp
plot_by_woe(df_temp, 90)
"""
The variable has a largely downward trend around WoE. But there are very few values greater than 35.
"""
loan_data_inputs_train_temp = loan_data_inputs_train.loc[loan_data_inputs_train['dti'] <= 35, : ]
loan_data_inputs_train_temp['dti_factor'] = pd.cut(loan_data_inputs_train_temp['dti'], 50)
# Here we do fine-classing: using the 'cut' method, we split the variable into 50 categories by its values.
df_temp = woe_ordered_continuous(loan_data_inputs_train_temp, 'dti_factor',
                                 loan_data_targets_train[loan_data_inputs_train_temp.index])
# We calculate weight of evidence.
df_temp
plot_by_woe(df_temp, 90)
for dataset in data_cleaner: 
    dataset['dti:<=1.4'] = np.where((dataset['dti'] <= 1.4), 1, 0)
    dataset['dti:1.4-3.5'] = np.where((dataset['dti'] > 1.4) & (dataset['dti'] <= 3.5), 1, 0)
    dataset['dti:3.5-7.7'] = np.where((dataset['dti'] > 3.5) & (dataset['dti'] <= 7.7), 1, 0)
    dataset['dti:7.7-10.5'] = np.where((dataset['dti'] > 7.7) & (dataset['dti'] <= 10.5), 1, 0)
    dataset['dti:10.5-16.1'] = np.where((dataset['dti'] > 10.5) & (dataset['dti'] <= 16.1), 1, 0)
    dataset['dti:16.1-20.3'] = np.where((dataset['dti'] > 16.1) & (dataset['dti'] <= 20.3), 1, 0)
    dataset['dti:20.3-21.7'] = np.where((dataset['dti'] > 20.3) & (dataset['dti'] <= 21.7), 1, 0)
    dataset['dti:21.7-22.4'] = np.where((dataset['dti'] > 21.7) & (dataset['dti'] <= 22.4), 1, 0)
    dataset['dti:22.4-35'] = np.where((dataset['dti'] > 22.4) & (dataset['dti'] <= 35), 1, 0)
    dataset['dti:>35'] = np.where((dataset['dti'] > 35), 1, 0)
"""
#### mths_since_last_record
"""
loan_data_inputs_train['mths_since_last_record'].unique()
"""
This is variable with null values. 
"""
loan_data_inputs_train_temp = loan_data_inputs_train[pd.notnull(loan_data_inputs_train['mths_since_last_record'])]
loan_data_inputs_train_temp['mths_since_last_record_factor'] = pd.cut(loan_data_inputs_train_temp['mths_since_last_record'], 50)
df_temp = woe_ordered_continuous(loan_data_inputs_train_temp, 'mths_since_last_record_factor',
                                 loan_data_targets_train[loan_data_inputs_train_temp.index])
df_temp
plot_by_woe(df_temp, 90)
# Categories: 'Missing', '0-2', '3-20', '21-31', '32-80', '81-86', '>86'
for dataset in data_cleaner: 
    dataset['mths_since_last_record:Missing'] = np.where((dataset['mths_since_last_record'].isnull()), 1, 0)
    dataset['mths_since_last_record:0-2'] = np.where((dataset['mths_since_last_record'] >= 0) & (dataset['mths_since_last_record'] <= 2), 1, 0)
    dataset['mths_since_last_record:3-20'] = np.where((dataset['mths_since_last_record'] >= 3) & (dataset['mths_since_last_record'] <= 20), 1, 0)
    dataset['mths_since_last_record:21-31'] = np.where((dataset['mths_since_last_record'] >= 21) & (dataset['mths_since_last_record'] <= 31), 1, 0)
    dataset['mths_since_last_record:32-80'] = np.where((dataset['mths_since_last_record'] >= 32) & (dataset['mths_since_last_record'] <= 80), 1, 0)
    dataset['mths_since_last_record:81-86'] = np.where((dataset['mths_since_last_record'] >= 81) & (dataset['mths_since_last_record'] <= 86), 1, 0)
    dataset['mths_since_last_record:>86'] = np.where((dataset['mths_since_last_record'] > 86), 1, 0)
loan_data_inputs_train['mths_since_last_record:>86'].isnull().sum()

"""
### Total Revolving Limit
"""
loan_data_inputs_train['total_rev_hi_lim'].unique()
loan_data_inputs_train_temp = loan_data_inputs_train[pd.notnull(loan_data_inputs_train['total_rev_hi_lim'])]
loan_data_inputs_train_temp['total_rev_hi_lim_factor'] = pd.cut(loan_data_inputs_train_temp['total_rev_hi_lim'], 50)
df_temp = woe_ordered_continuous(loan_data_inputs_train_temp, 'total_rev_hi_lim_factor',
                                 loan_data_targets_train[loan_data_inputs_train_temp.index])
df_temp
"""
This 99% of observations lie in the first bin, we need to dissect this into further bins to calculate WoE. 
"""
loan_data_inputs_train_temp = loan_data_inputs_train.loc[loan_data_inputs_train['total_rev_hi_lim'] <= 200000, : ]
loan_data_inputs_train_temp['total_rev_hi_lim_factor'] = pd.cut(loan_data_inputs_train_temp['total_rev_hi_lim'], 50)
df_temp = woe_ordered_continuous(loan_data_inputs_train_temp, 'total_rev_hi_lim_factor',
                                 loan_data_targets_train[loan_data_inputs_train_temp.index])
df_temp
plot_by_woe(df_temp, 90)
for dataset in data_cleaner:
    dataset['total_rev_hi_lim:Missing'] = np.where((dataset['total_rev_hi_lim'].isnull()), 1, 0)
    dataset['total_rev_hi_lim:<=5K'] = np.where((dataset['total_rev_hi_lim'] <= 5000), 1, 0)
    dataset['total_rev_hi_lim:5K-10K'] = np.where((dataset['total_rev_hi_lim'] > 5000) & (dataset['total_rev_hi_lim'] <= 10000), 1, 0)
    dataset['total_rev_hi_lim:10K-20K'] = np.where((dataset['total_rev_hi_lim'] > 10000) & (dataset['total_rev_hi_lim'] <= 20000), 1, 0)
    dataset['total_rev_hi_lim:20K-30K'] = np.where((dataset['total_rev_hi_lim'] > 20000) & (dataset['total_rev_hi_lim'] <= 30000), 1, 0)
    dataset['total_rev_hi_lim:30K-40K'] = np.where((dataset['total_rev_hi_lim'] > 30000) & (dataset['total_rev_hi_lim'] <= 40000), 1, 0)
    dataset['total_rev_hi_lim:40K-55K'] = np.where((dataset['total_rev_hi_lim'] > 40000) & (dataset['total_rev_hi_lim'] <= 55000), 1, 0)
    dataset['total_rev_hi_lim:55K-95K'] = np.where((dataset['total_rev_hi_lim'] > 55000) & (dataset['total_rev_hi_lim'] <= 95000), 1, 0)
    dataset['total_rev_hi_lim:>95K'] = np.where((dataset['total_rev_hi_lim'] > 95000), 1, 0)
            
             
"""
## Saving Output
"""
loan_data_inputs_train.to_csv('loan_data_inputs_train.csv')
loan_data_targets_train.to_csv('loan_data_targets_train.csv')
loan_data_inputs_test.to_csv('loan_data_inputs_test.csv')
loan_data_targets_test.to_csv('loan_data_targets_test.csv')