"""
<a href="https://colab.research.google.com/github/thanhnguyen2612/diveintocode-ml/blob/master/ML_Week3_ass.ipynb" target="_parent"><img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"/></a>
"""
"""
# Exploratory data analysis Class assignment Credit information analysis
"""
"""
## [Problem 1] Understanding the content of the competition

*   Home Credit is a non-bank financial company, they provide installment financial loans to unbanked people who have no or little credit history.
*   The competition seeks for Kagglers' support to learn deeper the costumer's data they have and from such predicts their clients' repayment abilities.
*   The ability to predict this knowledge helps the company manage risk, understand more about their clients and ensure the services.


"""
"""
## Preparing the dataset
"""
# This Python 3 environment comes with many helpful analytics libraries installed
# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python
# For example, here's several helpful packages to load

import numpy as np # linear algebra
import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)
import matplotlib.pyplot as plt
import seaborn as sns
%matplotlib inline

# Input data files are available in the read-only "../input/" directory
# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory

import os
for dirname, _, filenames in os.walk('/kaggle/input'):
    for filename in filenames:
        print(os.path.join(dirname, filename))

# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using "Save & Run All" 
# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session
data = pd.read_csv('../input/home-credit-default-risk/application_train.csv')
"""
## [Problem 2] Understanding the overview of data
"""
"""
### Overview
"""
data.head()
data.tail()
print("Size of dataframe:", data.shape)
data.describe()
print("All column attributes:")
print(data.columns.values)
"""
### Check missing data
"""
def missing_overview(data):
    total_missing_values = data.isnull().sum(axis=0).sort_values(ascending=False)
    missing_ratio = (data.isnull().sum() / data.isnull().count() * 100).sort_values(ascending=False)

    return pd.concat([total_missing_values, missing_ratio], axis=1, keys=['total', 'missing_ratio'])

print(missing_overview(data).head(20))
"""
Maximum percent of missing samples is about 70% in total
"""
"""
Percentage of target classes
"""
temp = data['TARGET'].value_counts()
target_classes = pd.DataFrame({'class': temp.index,
                               'amount': temp.values})

plt.figure(figsize=(10, 6))
plt.title("Percentage of classes")
plt.xlabel("class")
plt.ylabel("percent")
sns.barplot(x='class', y='amount', data=target_classes)
plt.show()

print("It's seem that dataset is imbalanced, the number of class 0 dominates that of class 1.")
"""
## [Problem 3] Defining issues
"""
missing_df = missing_overview(data)
num_of_severve_missing_cols = missing_df[missing_df['missing_ratio'] > 15].shape[0]
print("Number of column attributes have over 15% missing values:", num_of_severve_missing_cols)
print(f"It's about {num_of_severve_missing_cols / data.shape[1] * 100}% of columns")
"""
Nearly half the number of attributes contain missing values severely (assume 15% is severe).
The target to be predicted is highly imbalanced, which can lead to inefficient prediction model
"""
"""
Some issues/questions:

1.   TARGETs are imbalanced maybe because the data is collected among the same group of people or attributes.
2.   Some features have fairly similar names or aspects of information. May be there is correlation among them
3.   Does data follow some probability distribution?
4.   Does imbalance TARGET spoil features' distribution?


"""
"""
## [Problem 4] Data exploration
"""
def column_with_target_stats(data, column, label_rotate=False, target=1):
    """
    Use samples with TARGET=1 to show distribution of 'column' feature
    """
    df = data[data['TARGET'] == target]
    
    fig, ax = plt.subplots(1, 3, figsize=(24, 6))
    fig.suptitle(f"Insight of {column} with TARGET={target}")

    temp = data[column].value_counts()
    temp = pd.DataFrame({column: temp.index, 'count': temp.values})
    temp = temp.sort_values(['count'], ascending=False).reset_index(drop=True)
    s = sns.barplot(x=column, y='count', data=temp, ax=ax[0])
    if label_rotate:
        s.set_xticklabels(s.get_xticklabels(),rotation=90)
    ax[0].set_title("Number of samples in total")

    temp = df[column].value_counts()
    temp = pd.DataFrame({column: temp.index, 'count': temp.values})
    temp = temp.sort_values(['count'], ascending=False).reset_index(drop=True)
    s = sns.barplot(x=column, y='count', data=temp, ax=ax[1])
    if label_rotate:
        s.set_xticklabels(s.get_xticklabels(),rotation=90)
    ax[1].set_title("Number of samples with TARGET=1")

    values = data[column].value_counts()
    temp = {value: df[df[column] == value].shape[0] / count for value, count in values.items()}
    temp = pd.DataFrame({column: temp.keys(), 'percent': temp.values()})
    temp = temp.sort_values(['percent'], ascending=False).reset_index(drop=True)
    s = sns.barplot(x=column, y='percent', data=temp, ax=ax[2])
    if label_rotate:
        s.set_xticklabels(s.get_xticklabels(),rotation=90)
    ax[2].set_title("Percent of target with value 1 [%]")

    plt.show()
column_with_target_stats(data, 'NAME_CONTRACT_TYPE', target=1)
"""
Cash loans contracts mostly occur in un-repayment events. However, **Revolving loans** still has a medium level of not repaying chance (4.5%) in comparison with **Cash loans** (8.5%)
"""
column_with_target_stats(data, 'CODE_GENDER')
"""
The number of female clients is greater than male one. But males have a higher chance of not returning loans (10%) than females (7%) -> **CODE_GENDER** may affect **TARGET**
"""
"""
Examine other columns:
"""
low_cardinality_cols = list((filter(lambda col: data[col].nunique() <= 10, data.columns)))
print("Low cardinality columns")
print(low_cardinality_cols)
column_with_target_stats(data, 'FLAG_OWN_CAR')
column_with_target_stats(data, 'FLAG_OWN_REALTY')
column_with_target_stats(data, 'FLAG_MOBIL')
"""
All clients provide phone number...
"""
column_with_target_stats(data, 'HOUSETYPE_MODE')
column_with_target_stats(data, 'EMERGENCYSTATE_MODE')
column_with_target_stats(data, 'NAME_TYPE_SUITE', True)
column_with_target_stats(data, 'NAME_INCOME_TYPE', True)
"""
Mostly contracts made for employed clients, but 10% of them didn't repay their loans. However, the unemployed and maternity-leave population consist of small amount of samples but over 35% and 40% didn't repay, respectively. Seem reasonable...
"""
column_with_target_stats(data, 'NAME_FAMILY_STATUS', True)
"""
The percentage of not repayment among family-group is about 8% in average. **Single/not married** gains the most percentage amount (10%), which is also reasonable somehow... **Civil marriage** is little higher.
"""
# Drop missing value
missing_data = missing_overview(data)
drop_df = data.drop(columns=missing_data[missing_data['total'] > 5].index)
print("Remain:", drop_df.columns.values)
"""
Consider other chance of correlationship
"""
corrmat = drop_df.corr()

plt.figure(figsize=(20, 12))
sns.heatmap(corrmat, square=True)
plt.show()
"""
According to heatmap, some positive correlating group of features:

*   CNT_CHILDREN & CNT_FAM_MEMBERS & REGION_RATING_CLIENT
*   FLAG_MOBIL & DAYS_LAST_PHONE_CHANGE, etc.
"""
# 10 features/cols have the highest correlation coefficient values with target
cols = corrmat.nlargest(10, 'TARGET').index
top10_corrmat = np.corrcoef(drop_df[cols].values.T)
sns.set(font_scale=1.25)
plt.figure(figsize=(12, 9))
sns.heatmap(top10_corrmat, cbar=True, annot=True, square=True, fmt='.2f', annot_kws={'size': 10}, yticklabels=cols.values, xticklabels=cols.values)
plt.show()
"""
Explore distribution of real-value column features
"""
# Plot distribution of one feature
def plot_distribution(data, column, color='r'):
    plt.figure(figsize=(10,6))
    plt.title(f"Distribution of {column}")
    sns.histplot(data[column].dropna(), color=color, kde=True, bins=100)
    plt.show()

# Plot distribution of multiple features, with TARGET = 1/0 on the same graph
def plot_distribution_comp(data, columns, nrow=2):
    t1 = data.loc[data['TARGET'] != 0]
    t0 = data.loc[data['TARGET'] == 0]

    sns.set_style('whitegrid')
    plt.figure()
    fig, ax = plt.subplots(nrow, 2, figsize=(12,6*nrow))

    i = 0
    for col in columns:
        i += 1
        plt.subplot(nrow, 2, i)
        sns.kdeplot(t1[col], bw_adjust=0.1, label="TARGET = 1")
        sns.kdeplot(t0[col], bw_adjust=0.1, label="TARGET = 0")
        plt.xlabel(col, fontsize=12)
        plt.ylabel('Density plot', fontsize=12)
        plt.legend()
        locs, labels = plt.xticks()
        plt.tick_params(axis='both', which='major', labelsize=12)
    plt.show()
plot_distribution(data, 'AMT_INCOME_TOTAL')
plot_distribution(data, 'AMT_CREDIT')
plot_distribution(data, 'AMT_ANNUITY', 'b')
plot_distribution(data, 'AMT_GOODS_PRICE', 'y')
"""
The data not follows normal distribution
"""
cols = ['AMT_ANNUITY','AMT_GOODS_PRICE','DAYS_EMPLOYED', 'DAYS_REGISTRATION','DAYS_BIRTH','DAYS_ID_PUBLISH']
plot_distribution_comp(data, cols, 3)
"""
It's seem that the distributions of interval values from features remain between 2 sets of samples (TARGET=1 and TARGET=0) no matter of imbalancing shown previously.
"""