#Read the train_loan_predection.csv into pandas DataFrame
loan_df = pd.read_csv('./data/loan/train_loan_predection.csv')

"""
Explanation: Data Analysis - Problem Statement
About Company: 
<p>Dream Housing Finance company deals in all home loans. They have presence across all urban, semi urban and rural areas. Customer first apply for home loan after that company validates the customer eligibility for loan.</p>
</br>
<p>Company wants to automate the loan eligibility process (real time) based on customer detail provided while filling online application form. These details are Gender, Marital Status, Education, Number of Dependents, Income, Loan Amount, Credit History and others. To automate this process, they have given a problem to identify the customers segments, those are eligible for loan amount so that they can specifically target these customers. Here they have provided a partial data set.</p>

Variable Description
|Variable|Description    |
|:--------|:---------------|
|Loan_ID |Unique Loan ID |
|Gender|Male/ Female   |
|Married|Applicant married (Y/N)|
|Dependents|Number of dependents|
|Education|Applicant Education (Graduate/ Under Graduate)|
|Self_Employed|Self employed (Y/N)|
|ApplicantIncome|Applicant income|
|CoapplicantIncome|Coapplicant income|
|LoanAmount|Loan amount in thousands|
|Loan_Amount_Term|Term of loan in months|
|Credit_History|credit history meets guidelines|
|Property_Area|Urban/ Semi Urban/ Rural|
|Loan_Status|Loan approved (Y/N)|
End of explanation
"""


loan_df.head()

len(loan_df)

loan_df.info()

"""
Explanation: Let's look at the data
End of explanation
"""


len(loan_df.Loan_ID.unique())

loan_df = loan_df.set_index('Loan_ID')

loan_df.head(2)

loan_df.loc['LP001008']

"""
Explanation: Set the Index, this is not mandatory
End of explanation
"""


loan_df.isnull().sum()

loan_df[loan_df.Gender.isnull()]

loan_df[loan_df.Married.isnull()]

"""
Explanation: Check for Null values
End of explanation
"""


len(loan_df)

loan_df = loan_df.dropna(how='any')

len(loan_df)

loan_df.isnull().sum()

"""
Explanation: Let's drop all of the rows if one of the cell has null value
End of explanation
"""


loan_df.columns

loan_df.dtypes

loan_df.Dependents.value_counts()

loan_df.Dependents.apply(lambda x:str(type(x))).value_counts()

# Convert Dependents column to integers
loan_df['Dependents'] = loan_df.Dependents.apply(lambda x: int(x.replace('+','')))

# Now Dependents colum has descrete values
loan_df.Dependents.value_counts()

loan_df.dtypes

# add Dummy column 'Count', will be used for aggregations
loan_df['Count'] = 1

"""
Explanation: Verify column data types and convert if necessary
End of explanation
"""


loan_df.ApplicantIncome.describe()

columns = ['ApplicantIncome', 'CoapplicantIncome']
loan_df[columns].plot(kind='box',figsize=(20,6))

columns = ['ApplicantIncome', 'CoapplicantIncome','LoanAmount']
loan_df[columns].plot(kind='hist',bins=20,grid=True,alpha=0.3)

"""
Explanation: Analyze numeric columns
End of explanation
"""


column_name = 'Gender'
pd.DataFrame(loan_df[column_name].value_counts()*100/len(loan_df)).plot(kind='bar')

"""
Explanation: Analyze categorical columns
End of explanation
"""


column_name = "Married"
#column_name = "Gender"
#column_name = "Dependents"
#column_name = "Loan_Amount_Term"
#column_name = "Self_Employed"
#column_name = "Education"
#column_name = 'Credit_History'

columns = [column_name,'Loan_Status','Count']

grp_df = loan_df[columns].groupby([column_name,'Loan_Status']).sum()
grp_df = grp_df.unstack()*100/len(loan_df)
grp_df.plot(kind='bar',stacked=True,figsize=(8,4))

"""
Explanation: Let's find out how our target variable Loan_Status is related with other categorical variables
End of explanation
"""


# Loan sanctioned to few of them even though their credit history is not 
loan_df[(loan_df.Credit_History==0) & (loan_df.Loan_Status=='Y')]

"""
Explanation: Loan has been sancationed to very few people who has Credit_History as 0
End of explanation
"""


loan_status_map = {'Y':1,'N':0}
loan_df['Loan_Status_int'] = loan_df.Loan_Status.map(loan_status_map)

loan_df['LoanAmount_pct'] = loan_df.LoanAmount *100 / loan_df.ApplicantIncome

pd.cut(loan_df['LoanAmount_pct'],[0,1,2,3,4,5,6,8,10,20,100]).value_counts().sort_index()

loan_df.Loan_Amount_Term.value_counts()

plt.scatter(loan_df.LoanAmount_pct,loan_df.ApplicantIncome)
plt.show()

loan_df[loan_df.LoanAmount_pct > 10]

loan_df.LoanAmount_pct.plot(kind='hist',bins=20)

loan_df['TotalIncome'] = loan_df.ApplicantIncome + loan_df.CoapplicantIncome

loan_df[loan_df.Loan_Status=='Y'].TotalIncome.plot(kind='hist',bins=20)

loan_df[loan_df.Loan_Status=='N'].TotalIncome.plot(kind='hist',bins=20)

loan_df[(loan_df.TotalIncome > 30000) & (loan_df.Loan_Status=='N')]

temp_df = loan_df[(loan_df.Credit_History==1) & (loan_df.Loan_Status=='N')]
print(len(temp_df))
temp_df.head()

loan_df[loan_df.Loan_Status=='N'].describe(include='all')

loan_df[loan_df.Loan_Status=='Y'].describe(include='all')

"""
Explanation: Addition of more fields
End of explanation
"""


loan_df.columns

columns_retained = ['Gender', 'Married', 'Dependents', 'Education', 'Self_Employed',
       'ApplicantIncome', 'CoapplicantIncome', 'LoanAmount',
       'Loan_Amount_Term', 'Credit_History', 'Property_Area', 'Loan_Status']

columns_retained = ['Gender', 'Married', 'Dependents', 'Education', 'Self_Employed',
       'ApplicantIncome', 'CoapplicantIncome', 'LoanAmount',
       'Loan_Amount_Term', 'Credit_History', 'Property_Area', 'Loan_Status']


columns = ['Gender','Married','Education','Self_Employed','Property_Area','Loan_Status']
df = loan_df[columns_retained]
df = pd.get_dummies(df,columns=columns,drop_first=True)

df.head()

import sklearn

from sklearn.model_selection import train_test_split

X_train,X_test = train_test_split(df,test_size=0.2, random_state=42)

y_train = X_train['Loan_Status_Y']
y_test = X_test['Loan_Status_Y']
X_train = X_train.drop('Loan_Status_Y',axis=1)
X_test = X_test.drop('Loan_Status_Y',axis=1)

from sklearn.ensemble import RandomForestClassifier
from sklearn.metrics import confusion_matrix,accuracy_score

clf = RandomForestClassifier(n_estimators=500,n_jobs=-1,max_features=None)

clf.fit(X_train,y_train)

d = dict(zip(clf.feature_importances_,X_test.columns))

for k in sorted(d.keys(),reverse=True):
    print(d[k])
    

y_pred = clf.predict(X_test)

confusion_matrix(y_pred=y_pred,y_true=y_test)

accuracy_score(y_pred=y_pred,y_true=y_test)

df_test[df_test.LoanAmount.isnull()]

df_test = df_test.dropna(how='any')

len(df_test)

df_test['Dependents'] = df_test.Dependents.apply(lambda x: int(x.replace('+','')))

columns_retained = ['Gender', 'Married', 'Dependents', 'Education', 'Self_Employed',
       'ApplicantIncome', 'CoapplicantIncome', 'LoanAmount',
       'Loan_Amount_Term', 'Credit_History', 'Property_Area']

columns = ['Gender','Married','Education','Self_Employed','Property_Area']
df_test = df_test[columns_retained]
df_test = pd.get_dummies(df_test,columns=columns,drop_first=True)

y_test_pred = clf.predict(df_test)

df_test['Loan_Status_Y'] = y_test_pred

df_test[df_test.Loan_Status_Y==0]

x =loan_df.LoanAmount*100/loan_df.ApplicantIncome
b = loan_df[(loan_df.LoanAmount*100/loan_df.ApplicantIncome > 8) ]

x.head()

b[b.Loan_Status=='N']

"""
Explanation: Let's prepare the data for building the model
End of explanation
"""


