# This Python 3 environment comes with many helpful analytics libraries installed
# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python
# For example, here's several helpful packages to load

import numpy as np # linear algebra
import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)
import plotly.graph_objects as go
import seaborn as sn
# Input data files are available in the read-only "../input/" directory
# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory

import os
for dirname, _, filenames in os.walk('/kaggle/input'):
    for filename in filenames:
        print(os.path.join(dirname, filename))

# You can write up to 5GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using "Save & Run All" 
# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session
import xgboost as xgb
from xgboost import plot_importance
from matplotlib import pyplot as plt
from sklearn.model_selection import train_test_split
data_app_train = pd.read_csv('/kaggle/input/home-credit-default-risk/application_train.csv')
data_app_train['SK_ID_CURR'] = data_app_train['SK_ID_CURR'].astype(str)
import plotly.graph_objects as go
import plotly.express as px
data_app_test = pd.read_csv('/kaggle/input/home-credit-default-risk/application_test.csv')
data_app_test['SK_ID_CURR'] = data_app_test['SK_ID_CURR'].astype(str)
data_app_test['EMERGENCYSTATE_MODE'].unique()
'XNA' in data_app_test['ORGANIZATION_TYPE'].unique().tolist()
pd.set_option('display.max_rows', None)
data_null_value_col = data_app_train.isna().sum()[data_app_train.isna().sum()!=0].reset_index().rename(columns={'index': 'col_name', 0: '#'})
data_null_value_col['%'] = data_null_value_col['#'] / data_app_train.shape[0]
data_null_value_col
pd.set_option('display.max_rows', 60)
def default_rate_cal(df, col, threshold):
    print('default rate high band: ', df[df[col]>threshold]['TARGET'].sum()/df[df[col]>threshold].shape[0])
    print('default rate low band: ', df[df[col]>threshold]['TARGET'].sum()/df[df[col]<=threshold].shape[0])
default_rate_cal(data_app_train, 'AMT_INCOME_TOTAL', 600000)
list_cat_col = [
    'NAME_CONTRACT_TYPE',
    'CODE_GENDER',
    'FLAG_OWN_CAR',
    'FLAG_OWN_REALTY',
    'NAME_TYPE_SUITE',
    'NAME_INCOME_TYPE',
    'NAME_EDUCATION_TYPE',
    'NAME_FAMILY_STATUS',
    'NAME_HOUSING_TYPE',
    'OCCUPATION_TYPE',
    'ORGANIZATION_TYPE',
    'FONDKAPREMONT_MODE',
    'HOUSETYPE_MODE',
    'WALLSMATERIAL_MODE',
    'EMERGENCYSTATE_MODE',
#     'FLAG_MOBIL',
    'FLAG_EMP_PHONE',
#     'FLAG_WORK_PHONE',
#     'FLAG_CONT_MOBILE',
    'FLAG_PHONE',
#     'FLAG_EMAIL',
    'AMT_REQ_CREDIT_BUREAU_HOUR',
    'AMT_REQ_CREDIT_BUREAU_DAY',
    'AMT_REQ_CREDIT_BUREAU_WEEK',
    'AMT_REQ_CREDIT_BUREAU_MON',
    'AMT_REQ_CREDIT_BUREAU_QRT',
    'AMT_REQ_CREDIT_BUREAU_YEAR',
    'REG_REGION_NOT_LIVE_REGION',
    'REG_REGION_NOT_WORK_REGION',
#     'LIVE_REGION_NOT_WORK_REGION',
    'REG_CITY_NOT_LIVE_CITY',
    'REG_CITY_NOT_WORK_CITY',
    'LIVE_CITY_NOT_WORK_CITY'
]

list_num_col = [
    'TARGET',
    'CNT_CHILDREN',
    'AMT_INCOME_TOTAL',
    'AMT_CREDIT',
    'AMT_ANNUITY',
    'AMT_GOODS_PRICE',
    'REGION_POPULATION_RELATIVE',
    'DAYS_EMPLOYED',
    'DAYS_REGISTRATION',
    'DAYS_ID_PUBLISH',
    'OWN_CAR_AGE',
    'CNT_FAM_MEMBERS',
    'REGION_RATING_CLIENT',
    'REGION_RATING_CLIENT_W_CITY',
    'EXT_SOURCE_1',
    'EXT_SOURCE_2',
    'EXT_SOURCE_3',
    'APARTMENTS_AVG',
    'BASEMENTAREA_AVG',
    'YEARS_BEGINEXPLUATATION_AVG',
    'YEARS_BUILD_AVG',
    'COMMONAREA_AVG',
    'ELEVATORS_AVG',
    'ENTRANCES_AVG',
    'FLOORSMAX_AVG',
    'FLOORSMIN_AVG',
    'LANDAREA_AVG',
    'LIVINGAPARTMENTS_AVG',
    'LIVINGAREA_AVG',
    'NONLIVINGAPARTMENTS_AVG',
    'NONLIVINGAREA_AVG',
    'APARTMENTS_MODE',
    'BASEMENTAREA_MODE',
    'YEARS_BEGINEXPLUATATION_MODE',
    'YEARS_BUILD_MODE',
    'COMMONAREA_MODE',
    'ELEVATORS_MODE',
    'ENTRANCES_MODE',
    'FLOORSMAX_MODE',
    'FLOORSMIN_MODE',
    'LANDAREA_MODE',
    'LIVINGAPARTMENTS_MODE',
    'LIVINGAREA_MODE',
    'NONLIVINGAPARTMENTS_MODE',
    'NONLIVINGAREA_MODE',
    'APARTMENTS_MEDI',
    'BASEMENTAREA_MEDI',
    'YEARS_BEGINEXPLUATATION_MEDI',
    'YEARS_BUILD_MEDI',
    'COMMONAREA_MEDI',
    'ELEVATORS_MEDI',
    'ENTRANCES_MEDI',
    'FLOORSMAX_MEDI',
    'FLOORSMIN_MEDI',
    'LANDAREA_MEDI',
    'LIVINGAPARTMENTS_MEDI',
    'LIVINGAREA_MEDI',
    'NONLIVINGAPARTMENTS_MEDI',
    'NONLIVINGAREA_MEDI',
    'FONDKAPREMONT_MODE',
    'HOUSETYPE_MODE',
    'TOTALAREA_MODE',
    'OBS_30_CNT_SOCIAL_CIRCLE',
    'DEF_30_CNT_SOCIAL_CIRCLE',
    'OBS_60_CNT_SOCIAL_CIRCLE',
    'DEF_60_CNT_SOCIAL_CIRCLE',
    'DAYS_LAST_PHONE_CHANGE',
    'FLAG_DOCUMENT_2',
    'FLAG_DOCUMENT_3',
    'FLAG_DOCUMENT_4',
    'FLAG_DOCUMENT_5',
    'FLAG_DOCUMENT_6',
    'FLAG_DOCUMENT_7',
    'FLAG_DOCUMENT_8',
    'FLAG_DOCUMENT_9',
    'FLAG_DOCUMENT_10',
    'FLAG_DOCUMENT_11',
    'FLAG_DOCUMENT_12',
    'FLAG_DOCUMENT_13',
    'FLAG_DOCUMENT_14',
    'FLAG_DOCUMENT_15',
    'FLAG_DOCUMENT_16',
    'FLAG_DOCUMENT_17',
    'FLAG_DOCUMENT_18',
    'FLAG_DOCUMENT_19',
    'FLAG_DOCUMENT_20',
    'FLAG_DOCUMENT_21'
]

list_flag_to_cat = [
#     'FLAG_MOBIL',
    'FLAG_EMP_PHONE',
#     'FLAG_WORK_PHONE',
#     'FLAG_CONT_MOBILE',
    'FLAG_PHONE',
#     'FLAG_EMAIL',
    'REG_REGION_NOT_LIVE_REGION',
    'REG_REGION_NOT_WORK_REGION',
    'REG_CITY_NOT_LIVE_CITY',
    'REG_CITY_NOT_WORK_CITY',
    'LIVE_CITY_NOT_WORK_CITY'
]

list_to_remove = [
    'WEEKDAY_APPR_PROCESS_START',
    'HOUR_APPR_PROCESS_START',
    'FLAG_MOBIL',
    'FLAG_CONT_MOBILE',
    'FLAG_EMAIL',
    'FLAG_WORK_PHONE',
    'LIVE_REGION_NOT_WORK_REGION',
    
]
"""
## Remove useless columns
"""
data_app_train.drop(list_to_remove, axis=1, inplace=True)
"""
## Flag variable to cat
"""
data_app_train[list_flag_to_cat] = data_app_train[list_flag_to_cat].astype(str)
df_non_default_ratio = pd.DataFrame(columns=['col_value', 'TARGET', 'count', 'total', 'col_name', 'ratio'])
for col in list_cat_col:
    x_axis = data_app_train[col].unique().tolist()
    y_axis = data_app_train[[col, 'TARGET', 'SK_ID_CURR']].groupby([col, 'TARGET']).count().reset_index().rename(columns={col: 'col_value', 'SK_ID_CURR': 'count'})
    df_total = data_app_train[[col, 'SK_ID_CURR']].groupby([col]).count().reset_index().rename(columns={col: 'col_value', 'SK_ID_CURR': 'total'})
    for col_value in list(set(x_axis).difference(set(y_axis[y_axis['TARGET']==0]['col_value'].values.tolist()))):
        y_axis.loc[len(y_axis)] = [col_value, 0, 0]
    for col_value in list(set(x_axis).difference(set(y_axis[y_axis['TARGET']==1]['col_value'].values.tolist()))):
        y_axis.loc[len(y_axis)] = [col_value, 1, 0]
    y_axis = y_axis.merge(df_total, on=['col_value'], how='left')
    y_axis['col_name'] = col
    y_axis['ratio'] = y_axis['count'] / (y_axis['total'] - y_axis['count'])
    df_non_default_ratio = df_non_default_ratio.append(y_axis)

pd.set_option('display.max_rows', None)
df_non_default_ratio
pd.set_option('display.max_rows', 60)
"""
### CODE_GENDER
* Exclude XNA, which is not in test data and not material
"""
data_app_train.shape
data_app_train = data_app_train[data_app_train['CODE_GENDER']!='XNA']
data_app_train.shape
"""
### FLAG_OWN_CAR, OWN_CAR_AGE and DAYS_BIRTH
* Flag own car but no own car age, Flag not own car but has own car age
* OWN_CAR_AGE > DAYS_BIRTH
* new feature: age_when_own_car
"""
data_app_train[data_app_train['FLAG_OWN_CAR']=='N']['OWN_CAR_AGE'].unique()
data_app_train[data_app_train['FLAG_OWN_CAR']=='Y']['OWN_CAR_AGE'].unique()
data_app_train['age_when_own_car'] = data_app_train['DAYS_BIRTH'] / (-365) - data_app_train['OWN_CAR_AGE']
data_app_train = data_app_train[(data_app_train['age_when_own_car'] >= 0) | (data_app_train['age_when_own_car'].isna())]
data_app_train.drop(['OWN_CAR_AGE'], axis=1, inplace=True)
data_app_train.shape
"""
### FLAG_OWN_REALTY
* Delete since Non-default ratio very similar
"""
data_app_train.drop(['FLAG_OWN_REALTY'], axis=1, inplace=True)
data_app_train.shape
"""
### NAME_TYPE_SUITE
* drop na since only 1000+ rows
* combine some column values
"""
data_app_train.dropna(subset=['NAME_TYPE_SUITE'], axis=0, inplace=True)
data_app_train.shape
data_app_train.loc[(data_app_train['NAME_TYPE_SUITE']=='Children') |
                   (data_app_train['NAME_TYPE_SUITE']=='Family'), 'NAME_TYPE_SUITE_cat'] = 'high'
data_app_train.loc[(data_app_train['NAME_TYPE_SUITE']=='Spouse, partner') |
                   (data_app_train['NAME_TYPE_SUITE']=='Unaccompanied'), 'NAME_TYPE_SUITE_cat'] = 'mid'
data_app_train.loc[(data_app_train['NAME_TYPE_SUITE']=='Group of people') |
                   (data_app_train['NAME_TYPE_SUITE']=='Other_A'), 'NAME_TYPE_SUITE_cat'] = 'low'
data_app_train.loc[data_app_train['NAME_TYPE_SUITE']=='Other_B', 'NAME_TYPE_SUITE_cat'] = 'btm'
"""
### NAME_INCOME_TYPE
* Combine columns values

"""
data_app_train.loc[(data_app_train['NAME_INCOME_TYPE']=='Businessman') |
                   (data_app_train['NAME_INCOME_TYPE']=='Student'), 'NAME_INCOME_TYPE_cat'] = 'no_default'
data_app_train.loc[(data_app_train['NAME_INCOME_TYPE']=='State servant') |
                   (data_app_train['NAME_INCOME_TYPE']=='Pensioner'), 'NAME_INCOME_TYPE_cat'] = 'high'
data_app_train.loc[data_app_train['NAME_INCOME_TYPE']=='Commercial associate', 'NAME_INCOME_TYPE_cat'] = 'mid'
data_app_train.loc[data_app_train['NAME_INCOME_TYPE']=='Working', 'NAME_INCOME_TYPE_cat'] = 'low'
data_app_train.loc[(data_app_train['NAME_INCOME_TYPE']=='Maternity leave') |
                   (data_app_train['NAME_INCOME_TYPE']=='Unemployed'), 'NAME_INCOME_TYPE_cat'] = 'high_default'
"""
### NAME_EDUCATION_TYPE
* combine Incomplete higher	and Secondary / secondary special
"""
data_app_train['NAME_EDUCATION_TYPE_cat'] = data_app_train['NAME_EDUCATION_TYPE']
data_app_train.loc[(data_app_train['NAME_EDUCATION_TYPE_cat']=='Incomplete higher') |
                   (data_app_train['NAME_EDUCATION_TYPE_cat']=='Secondary / secondary special'), 'NAME_EDUCATION_TYPE_cat'] = 'not_well_edu'
"""
### NAME_FAMILY_STATUS
* drop unknown
* combine values
"""
data_app_train = data_app_train[data_app_train['NAME_FAMILY_STATUS']!='Unknown']
data_app_train.shape
data_app_train.loc[(data_app_train['NAME_FAMILY_STATUS']=='Civil marriage') |
                   (data_app_train['NAME_FAMILY_STATUS']=='Single / not married'), 'NAME_FAMILY_STATUS_cat'] = 'low'
data_app_train.loc[(data_app_train['NAME_FAMILY_STATUS']=='Married') |
                   (data_app_train['NAME_FAMILY_STATUS']=='Separated'), 'NAME_FAMILY_STATUS_cat'] = 'mid'
data_app_train.loc[data_app_train['NAME_FAMILY_STATUS']=='Widow', 'NAME_FAMILY_STATUS_cat'] = 'high'
"""
### NAME_HOUSING_TYPE
* Combine values
"""
data_app_train.loc[(data_app_train['NAME_HOUSING_TYPE']=='Rented apartment') |
                   (data_app_train['NAME_HOUSING_TYPE']=='With parents'), 'NAME_HOUSING_TYPE_cat'] = 'low'
data_app_train.loc[(data_app_train['NAME_HOUSING_TYPE']=='Co-op apartment') |
                   (data_app_train['NAME_HOUSING_TYPE']=='House / apartment') |
                   (data_app_train['NAME_HOUSING_TYPE']=='Municipal apartment'), 'NAME_HOUSING_TYPE_cat'] = 'mid'
data_app_train.loc[data_app_train['NAME_HOUSING_TYPE']=='Office apartment', 'NAME_HOUSING_TYPE_cat'] = 'high'
"""
### OCCUPATION_TYPE
* Combine values
"""
data_app_train.loc[(data_app_train['OCCUPATION_TYPE']=='Accountants')|
                    (data_app_train['OCCUPATION_TYPE']=='Core staff')|
                    (data_app_train['OCCUPATION_TYPE']=='HR staff')|
                    (data_app_train['OCCUPATION_TYPE']=='High skill tech staff')|
                    (data_app_train['OCCUPATION_TYPE']=='IT staff')|
                    (data_app_train['OCCUPATION_TYPE']=='Managers')|
                    (data_app_train['OCCUPATION_TYPE']=='Medicine staff')|
                    (data_app_train['OCCUPATION_TYPE']=='Private service staff')|
                    (data_app_train['OCCUPATION_TYPE']=='Realty agents')|
                    (data_app_train['OCCUPATION_TYPE']=='Secretaries'), 'OCCUPATION_TYPE_cat'] = 'high'
data_app_train.loc[(data_app_train['OCCUPATION_TYPE']=='Cleaning staff')|
                    (data_app_train['OCCUPATION_TYPE']=='Cooking staff')|
                    (data_app_train['OCCUPATION_TYPE']=='Drivers')|
                    (data_app_train['OCCUPATION_TYPE']=='Laborers')|
                    (data_app_train['OCCUPATION_TYPE']=='Low-skill Laborers')|
                    (data_app_train['OCCUPATION_TYPE']=='Sales staff')|
                    (data_app_train['OCCUPATION_TYPE']=='Security staff')|
                    (data_app_train['OCCUPATION_TYPE']=='Waiters/barmen staff'), 'OCCUPATION_TYPE_cat'] = 'mid'
data_app_train.loc[data_app_train['OCCUPATION_TYPE_cat'].isna(), 'OCCUPATION_TYPE_cat'] = 'mid'
list_modeled_cat = [
    'NAME_CONTRACT_TYPE',
    'CODE_GENDER',
    'FLAG_OWN_CAR',
    'age_when_own_car',
    'NAME_TYPE_SUITE_cat',
    'NAME_INCOME_TYPE_cat',
    'NAME_EDUCATION_TYPE_cat',
    'NAME_FAMILY_STATUS',
    'NAME_HOUSING_TYPE_cat',
    'OCCUPATION_TYPE_cat',
    'FLAG_EMP_PHONE',
    'FLAG_PHONE',
    'EMERGENCYSTATE_MODE',
    'REG_CITY_NOT_LIVE_CITY',
    'REG_CITY_NOT_WORK_CITY',
    'LIVE_CITY_NOT_WORK_CITY',
    'REG_REGION_NOT_LIVE_REGION'
]
data_app_train['NAME_HOUSING_TYPE_cat'].unique()

data_app_train_train = data_app_train[list_modeled_cat+['EXT_SOURCE_1', 'EXT_SOURCE_2', 'EXT_SOURCE_3', 'TARGET']]
data_app_train_train['TARGET'] = data_app_train_train['TARGET'].astype(str)

data_app_train_xgboost = pd.concat([data_app_train_train[['EXT_SOURCE_1', 'EXT_SOURCE_2', 'EXT_SOURCE_3', 'TARGET']], pd.get_dummies(data_app_train_train[list_modeled_cat])], axis=1)


X_train, X_test, y_train, y_test = train_test_split(data_app_train_xgboost.drop(['TARGET'], axis=1).values, data_app_train_xgboost['TARGET'].values, test_size=0.2, random_state=1234565)
params = {
    'booster': 'dart',
    'objective': 'binary:logistic',
    'eval_metric': 'auc',
    'verbosity': 1,
    'eta': 0.005,
    'max_depth':7
    
}

plst = params.items()

dtrain = xgb.DMatrix(X_train, y_train)
num_rounds = 50
model = xgb.train(list(params.items()), dtrain, num_rounds)


dtest = xgb.DMatrix(X_test)
ans = model.predict(dtest)

from sklearn.metrics import roc_auc_score


cnt1 = 0
cnt2 = 0
for i in range(len(y_test)):
    if ans[i] == y_test[i]:
        cnt1 += 1
    else:
        cnt2 += 1

print("Accuracy: %.2f %% " % (100 * cnt1 / (cnt1 + cnt2)))
print('roc: ', roc_auc_score(y_test, ans))


plot_importance(model)
plt.show()
"""
# EDA
"""
"""
## Days to Years
"""
data_app_train = pd.read_csv('/kaggle/input/home-credit-default-risk/application_train.csv')
data_app_train['SK_ID_CURR'] = data_app_train['SK_ID_CURR'].astype(str)
data_app_train['DAYS_BIRTH_year'] = data_app_train['DAYS_BIRTH'] / -365
data_app_train['DAYS_EMPLOYED_year'] = data_app_train['DAYS_EMPLOYED'] / -365
data_app_train.loc[data_app_train['DAYS_EMPLOYED']==365243, 'DAYS_EMPLOYED_year'] = 0
data_app_train['DAYS_REGISTRATION_year'] = data_app_train['DAYS_REGISTRATION'] / -365
data_app_train['DAYS_ID_PUBLISH_year'] = data_app_train['DAYS_ID_PUBLISH'] / -365
data_app_train['DAYS_LAST_PHONE_CHANGE_year'] = data_app_train['DAYS_LAST_PHONE_CHANGE'] / -365
plt.figure(figsize=(40, 30))
corrMatrix = data_app_train[list_num_col].corr()
sn.heatmap(corrMatrix)
plt.show()
pd.set_option('display.max_rows', None)
corrMatrix[['TARGET']].sort_values(by='TARGET', ascending=False)
pd.set_option('display.max_rows', 60)
"""
## Bar Chart to show ratios
"""
def colored_and_styled_bar_chart(col, title):
    x_axis = data_app_train[col].unique().tolist()
    y_axis = data_app_train[[col, 'TARGET']].groupby([col, 'TARGET']).count().reset_index()
#     if len(y_axis[y_axis['TARGET']==0][col].values.tolist()) < len(x_axis):
        
    y_axis_default = data_app_train[data_app_train['TARGET']==1][[col, 'TARGET']].groupby([col]).count()['TARGET'].values.tolist()
    y_axis_non_default = data_app_train[data_app_train['TARGET']==0][[col, 'TARGET']].groupby([col]).count()['TARGET'].values.tolist()
    fig = go.Figure()
    fig.add_trace(go.Bar(x=x_axis,
                    y=y_axis_non_default,
                    name='Not Default',
                    marker_color='rgb(55, 83, 109)',
                         texttemplate='%{y:.2s}', textposition='outside'
#                     text=data_app_train[data_app_train['TARGET']==0][[col, 'TARGET']].groupby([col]).count()['TARGET'].values.tolist(),
#                     textposition='outside',
                    ))
    fig.add_trace(go.Bar(x=x_axis,
                    y=y_axis_default,
                    name='Default',
                    marker_color='rgb(26, 118, 255)',
                         texttemplate='%{y:.2s}', textposition='outside'
#                     text=data_app_train[data_app_train['TARGET']==1][[col, 'TARGET']].groupby([col]).count()['TARGET'].values.tolist(),
#                     textposition='outside',
                    ))

    fig.update_layout(
        title=title,
        xaxis_tickfont_size=14,
        yaxis=dict(
            title='Count',
            titlefont_size=16,
            tickfont_size=14,
        ),
        legend=dict(
            x=0,
            y=1.0,
            bgcolor='rgba(255, 255, 255, 0)',
            bordercolor='rgba(255, 255, 255, 0)'
        ),
        barmode='group',
        bargap=0.15, # gap between bars of adjacent location coordinates.
        bargroupgap=0.1 # gap between bars of the same location coordinate.
    )
#     fig.update_layout(uniformtext_minsize=8, uniformtext_mode='hide')
#     fig.update_traces(texttemplate='%{text:.2s}', textposition='outside')
    fig.show()
#     for i in range(len(x_axis)):
#         if y_axis_default[i] == 0:
#             print('Non-default Ratio of ', x_axis[i], ': 1')
#         else:
#             print('Non-default Ratio of ', x_axis[i], ': ', y_axis_non_default[i] / y_axis_default[i])
colored_and_styled_bar_chart('NAME_CONTRACT_TYPE', 'Contract Type')
colored_and_styled_bar_chart('CODE_GENDER', 'Gender')
colored_and_styled_bar_chart('NAME_INCOME_TYPE', 'Income Type')
colored_and_styled_bar_chart('NAME_TYPE_SUITE', 'Suite Type')
"""
* Non-default ratio almost 10, except for Other B about 1.15
"""
colored_and_styled_bar_chart('NAME_EDUCATION_TYPE', 'Education Type Defaults')
"""
Very few # for those 4 types of income
State servant and pensioners have similar default rate while working has highest default rate
"""
colored_and_styled_bar_chart('NAME_FAMILY_STATUS', 'Family Status Defaults')
"""
* Unknown has no defaults
* separated is highly possible to default
* other status has similar non-default ratio 10:1
"""
colored_and_styled_bar_chart('NAME_HOUSING_TYPE', 'Housing Defaults')
"""
* similar non-default ratio
"""
colored_and_styled_bar_chart('OCCUPATION_TYPE', 'Occupation Type')
"""
* non-default ratio \> 10: Laborers, Managers, Drivers, Sales staff, Cleaning staff, Medicine staff, Security staff, High skill tech staff, Waiters/barmen staff, Lowe-skill Laboreres, Realty agents
* non-default ratio \< 10: Core Staff, Accountants, Cooking staff, Private Service staff, secretaries, IT staff
"""
colored_and_styled_bar_chart('FLAG_EMP_PHONE', 'Employer Phone')
"""
* with employer phone, has increased the ND ratio a lot
"""
colored_and_styled_bar_chart('FLAG_WORK_PHONE', 'Work Phone')
colored_and_styled_bar_chart('FLAG_CONT_MOBILE', 'Cont Mobile')
"""
* No obvisous difference
"""

colored_and_styled_bar_chart('FLAG_MOBIL', 'Mobile')
colored_and_styled_bar_chart('FLAG_PHONE', 'Phone')
"""
* ND ratio Minor diff
"""
colored_and_styled_bar_chart('FLAG_EMAIL', 'Email')
"""
* ND ratio super close
"""
colored_and_styled_bar_chart('CNT_FAM_MEMBERS', '# of Family')
"""
* Family with 2 members has the highest non-default ratio
* Family members with 1, 3, 4, 5 has around 10 non-default ratio
* \> 5 does not have enough data points to make decision, set as one category to let the model learn.
"""
colored_and_styled_bar_chart('REGION_RATING_CLIENT', 'Region Rating')
"""
* No idea what the column means
* 11.66, 19.375, 7.9
* rating 1 => normal range, rating 2 => good range, rating 3 => bad range
"""
colored_and_styled_bar_chart('REGION_RATING_CLIENT_W_CITY', 'Region Rating with City')
"""
* 11.66, 19.41, 7.8
* Very similar to region rating
"""
colored_and_styled_bar_chart('WEEKDAY_APPR_PROCESS_START', 'Process Start Weekday')
"""
* Very similar and normal range
"""
colored_and_styled_bar_chart('HOUR_APPR_PROCESS_START', 'Process Start Hour')
colored_and_styled_bar_chart('ORGANIZATION_TYPE', 'Organization Type')
data_app_train['FONDKAPREMONT_MODE'] = data_app_train['FONDKAPREMONT_MODE'].fillna('blank')
colored_and_styled_bar_chart('FONDKAPREMONT_MODE', 'FONDKAPREMONT_MODE')
data_app_train['HOUSETYPE_MODE'] = data_app_train['HOUSETYPE_MODE'].fillna('blank')
colored_and_styled_bar_chart('HOUSETYPE_MODE', 'HOUSETYPE_MODE')
data_app_train['WALLSMATERIAL_MODE'] = data_app_train['WALLSMATERIAL_MODE'].fillna('blank')
colored_and_styled_bar_chart('WALLSMATERIAL_MODE', 'WALLSMATERIAL_MODE')
data_app_train['EMERGENCYSTATE_MODE'] = data_app_train['EMERGENCYSTATE_MODE'].fillna('blank')
colored_and_styled_bar_chart('EMERGENCYSTATE_MODE', 'EMERGENCYSTATE_MODE')
def histagram(col):
    fig = px.histogram(data_app_train, x=col, color='TARGET')
    fig.show()
"""
* Does not have obvious trends
"""
histagram('REGION_POPULATION_RELATIVE')
data_app_train['DAYS_BIRTH_year'] = data_app_train['DAYS_BIRTH'] / -365
data_app_train['DAYS_EMPLOYED_year'] = data_app_train['DAYS_EMPLOYED'] / -365
data_app_train.loc[data_app_train['DAYS_EMPLOYED']==365243, 'DAYS_EMPLOYED_year'] = 0
data_app_train['DAYS_REGISTRATION_year'] = data_app_train['DAYS_REGISTRATION'] / -365
data_app_train['DAYS_ID_PUBLISH_year'] = data_app_train['DAYS_ID_PUBLISH'] / -365
data_app_train['DAYS_LAST_PHONE_CHANGE_year'] = data_app_train['DAYS_LAST_PHONE_CHANGE'] / -365
histagram('DAYS_BIRTH_year')
"""
* \> 27, increase a lot
* from 27 onwards, gradually decrease
* \> 65 sudden decrease
"""
histagram('DAYS_EMPLOYED_year')
"""
* Gradually decrease
* entry = 365243 are either pensioner or unemployed
"""
histagram('DAYS_REGISTRATION_year')
"""
* Gradually decrease when registration year increase
"""
histagram('DAYS_ID_PUBLISH_year')
"""
* similar trned for defaults and non-defaults
* heap in 10.6-12.6
"""
histagram('OWN_CAR_AGE')
histagram('DAYS_LAST_PHONE_CHANGE')