"""
### Importing Libraries
"""
import numpy as np
import pandas as pd
import seaborn as sns
import matplotlib.pyplot as plt
"""
## Reading Files
"""
df_bill_amt = pd.read_csv("../input/sukouz-equnix/bill_amount.csv")
df_demographics = pd.read_csv("../input/sukouz-equnix/demographics.csv")
df_bill_id = pd.read_csv("../input/sukouz-equnix/bill_id.csv")
df_clinical_data = pd.read_csv("../input/sukouz-equnix/clinical_data.csv")
"""
## Quick look at the files
"""
df_bill_amt.head()
df_demographics.head()
df_bill_id.head()
df_clinical_data.head()
"""
## Quick look at the target 
"""
df_bill_amt.amount.hist(color = 'orange', edgecolor = 'black', figsize = (10,8), bins = 25)   # target is skewed
df_bill_id.count()
df_bill_id.patient_id.nunique()
df_clinical_data.head()
df_clinical_data.id.nunique()
df_clinical_data.shape
"""
## Checking if All ids Are present in All files
"""
set(df_clinical_data.id) ==  set(df_bill_id.patient_id) 
set(df_bill_id.bill_id) == set(df_bill_amt.bill_id) 
set(df_demographics.patient_id) == set(df_bill_id.patient_id)  
"""
## Renaming "id"  to "patient_id"
"""
df_clinical_data.rename(columns = {'id':'patient_id'}, inplace = True)
"""
## Preparing flat file by merging the files
"""
df_bill_clinical = pd.merge(df_bill_id,  df_clinical_data, 
                            on = ['patient_id', 'date_of_admission'], how = 'inner')

df_bill_cinical_amt = pd.merge(df_bill_clinical, df_bill_amt, on = 'bill_id', how = 'inner' )

df_final = pd.merge(df_bill_cinical_amt, df_demographics, on = 'patient_id', how = 'inner')
"""
### Converting date time columns
"""
df_final.date_of_admission = pd.to_datetime(df_final.date_of_admission, format = "%Y-%m-%d")
df_final.date_of_discharge = pd.to_datetime(df_final.date_of_discharge, format = "%Y-%m-%d")
df_final.date_of_birth = pd.to_datetime(df_final.date_of_birth, format = "%Y-%m-%d")
df_final.count()   
df_final.isna().sum()[df_final.isna().sum() != 0]
"""
## Columns Having Missing Values
"""
"""
####  Looking at the above counts we can identify the columns which are having misssing values
"""
list_missing_data_columns = ["medical_history_2", "medical_history_5"] # needs to be looked sepeartely
%%javascript
IPython.OutputArea.prototype._should_scroll = function(lines) {
    return false;
}
"""
## Quick EDA
"""
## If you Re Run the this cell after data cleaning (after the last cell) it will plot the clean results. 
##ie. We can re run after df_final complete 

list_sepearte_columns = []   

count = 0

for i in df_final.columns[2:]:
    C = df_final[i].nunique()
    #print(i + "  "+ str(C))
    
    if(C < 20):
        (df_final[i].value_counts()*100/len(df_final[i])).plot(kind = 'bar')
        plt.title(i + " (Count Plot)")
        plt.ylabel("Frequency (%)")
        plt.xlabel("Unique Values")
        plt.show()
        count = count+1
    else:
        if(df_final[i].dtype != 'O'):
            df_final[i].hist(color = 'orange', edgecolor = 'black', figsize = (10,8), bins = 25)
            plt.ylabel("Frequency (count)")
            plt.xlabel(i)
            plt.title("Histogram")
            plt.show()
            count = count+1
        else:
            list_sepearte_columns.append(i)
            
list_sepearte_columns  # Thus we see we plotted all columns
"""
# Data Cleaning
"""
"""
### A look in to columns haveing missing value
"""
"""
#### medical_history_2 and medical_history_5 have missing value
"""
df_final.medical_history_2.count()/len(df_final.medical_history_2)

df_final.medical_history_5.count()/len(df_final.medical_history_2)
df_final.medical_history_5.isna().sum()
df_final.medical_history_2.isna().sum()
"""
### in the abouve columns Around 7-9 % rows are missing 
"""
"""
### Above 2 variable can be imputed or can be given as 3rd catergory missing. instead of inputing i will assigng it as another category. OneHotEncoding can be done to get the feature representation for the model. This we will do during model building 
"""
df_final.medical_history_2 = df_final.medical_history_2.fillna("missing")
df_final.medical_history_5 = df_final.medical_history_5.fillna("missing")
"""
### Treatment for "medical_history_3" column
"""
df_final.medical_history_3.value_counts()
"""
###  looking at the distribution and count "No" has count coparable to 0 and "yes" has count comparable to 1 thus using common sense replacing "yes" by 1 and "No" by 1. although its better to confirm with business
"""
 df_final.medical_history_3 =  df_final.medical_history_3.replace({"Yes":'1', "No":'0'}).astype('int8')
 df_final.medical_history_3.value_counts()
"""
### Correcting gender, Race and resedential_status columns
"""
df_final.gender.value_counts()
 df_final.gender =  df_final.gender.replace({"m":1, "f":0, 'Male':1, 'Female':0}).astype('int8')
df_final.gender.value_counts()  ## corrected
df_final.race.value_counts()
mapper = {'Chinese':'Chinese', 'Malay':'Malay', 'chinese':'Chinese', 
          'Indian':'Indian', 'Others':'Others', 'India':'Indian'}
df_final.race =  df_final.race.map(mapper)
df_final.race.value_counts()  ## corrected
df_final.resident_status.value_counts()
mapper = {'Singaporean':'Singaporean', 'Singapore citizen':'Singaporean', 'PR':'PR',
          'Foreigner':'Foreigner'}
df_final.resident_status =  df_final.resident_status.replace(mapper)
df_final.resident_status.value_counts() ## corrected
"""
# Feature Engineering
"""
df_final.date_of_admission = pd.to_datetime(df_final.date_of_admission, format = "%Y-%m-%d")
df_final.date_of_discharge = pd.to_datetime(df_final.date_of_discharge, format = "%Y-%m-%d")
df_final.date_of_birth = pd.to_datetime(df_final.date_of_birth, format = "%Y-%m-%d")
"""
### Geeting Age of the patients
"""
df_final['Age'] = ((df_final.date_of_admission - df_final.date_of_birth).dt.days)/365
df_final['Age'].hist(color = 'orange', edgecolor = 'black', figsize = (10,8), bins = 25)  # looks okay
"""
### Number of days paitent has stayed
"""
df_final['Stay_days'] = (df_final.date_of_discharge - df_final.date_of_admission).dt.days
df_final['Stay_days'].hist(color = 'orange', edgecolor = 'black', figsize = (10,8), bins = 25)
"""
### BMI
"""
df_final['BMI'] = df_final.weight / (df_final.height/100)**2
df_final['BMI'].hist(color = 'orange', edgecolor = 'black', figsize = (10,8), bins = 25)
"""
### Seasonal Features
"""
df_final['month'] = df_final['date_of_discharge'].dt.month
df_final['weekofyear'] = df_final['date_of_discharge'].dt.weekofyear
df_final['month'].hist(color = 'orange', edgecolor = 'black', figsize = (10,8), bins = 25)
df_final['weekofyear'].hist(color = 'orange', edgecolor = 'black', figsize = (10,8), bins = 25)
df_final.columns
cat_features = [ 'patient_id', 'medical_history_1', 'medical_history_2',
       'medical_history_3', 'medical_history_4', 'medical_history_5',
       'medical_history_6', 'medical_history_7', 'preop_medication_1',
       'preop_medication_2', 'preop_medication_3', 'preop_medication_4',
       'preop_medication_5', 'preop_medication_6', 'symptom_1', 'symptom_2',
       'symptom_3', 'symptom_4', 'symptom_5', 'gender', 'race', 'resident_status',
       'month', 'weekofyear']    # categoriclal features
numerical_features = ['lab_result_1', 'lab_result_2',
       'lab_result_3', 'weight', 'height', 'Stay_days', 'BMI', 'Age', 'amount'
       ]     # Numerical features
len(cat_features)  # categorical features count
len(numerical_features)  # numerical features count
"""
###  Checking statistics of numerical columns
"""
df_final[numerical_features].describe()
"""
### Checking the correlation b/w numerical columns and target
"""
df_final[numerical_features].corr()
"""
### Plotting Heatmap of Correlations
"""
plt.figure(figsize = (12,8))
ax = sns.heatmap(df_final[numerical_features].corr(), annot=True, 
                 cmap=plt.cm.Blues, linecolor ='black',  
                 linewidths=1) 
bottom, top = ax.get_ylim()
ax.set_ylim(bottom + 0.5, top - 0.5)
plt.tight_layout()
plt.show()
"""
#### Looking at the above numbers it appears to be no linear correlation (except Age) between target and the numerical features. thus nonlinear model might do a better job  
"""
"""
## As we saw in the EDA that target distribution is extremly skewed. Thus log tranformation (of box - cox) transformation will be good idea to ensure train test split is less biased. here we are doing log transform
"""
df_final['target_transformed'] = np.log1p(df_final.amount)   # adding 1 plus to take care of target having value zero
df_final['target_transformed'].hist(color = 'orange', edgecolor = 'black', figsize = (10,8), bins = 25)
"""
#### Binning target to ensure startified split the data into train and test
"""
df_final['target_bins'] = pd.cut(df_final['target_transformed'] ,bins = 5).astype('object')  
#this will be used in splitting the train and test
df_final['target_bins'].value_counts()
"""
### Removing non relevent columns for modeling
"""
df_final.drop(columns = ['date_of_admission', 'date_of_discharge', 'date_of_birth'], inplace = True)
"""
### Saving the flat file for model building 
"""
df_final.to_csv("created_flat_file.csv", index = False)
"""
### Remarks : Looking at the number of categorical features and past experience CatBoost will be appropriate for this data. Although we will build linear model baseline
"""
df_final.columns
"""
## Analysing patinet_id
"""
print(f"Unique Percentage of patinet_id : {df_final.patient_id.nunique()*100/len(df_final.patient_id)}")
"""
#### Given the repeating nature of patinet, patient_id can be used for model building.  while production two model can be used one witout patient_id and other whith patient_id. when a new patient is coming accordingly prdiction can be use. In this case i am going to build a model with patient_id as a feature. 
"""
"""
# More EDA on cleaned Data
"""
"""
## Exploring columns having binary value and its effect on target  

We are looking at the distibution of target for different value for binary columns (ie. 0 and 1) In the below plots blue is the distibution of target when feature value is equal to 1 and orange is distibution of target when feature value is 0.  Here we are using Log transformed distibution of target because it is very difficult to distigush the distibution of orginal target value

### If there is significant difference in the distibution means the feature will be imporantant in the model
"""
binary_features  = [  'medical_history_1', 'medical_history_2',
       'medical_history_3', 'medical_history_4', 'medical_history_5',
       'medical_history_6', 'medical_history_7', 'preop_medication_1',
       'preop_medication_2', 'preop_medication_3', 'preop_medication_4',
       'preop_medication_5', 'preop_medication_6', 'symptom_1', 'symptom_2',
       'symptom_3', 'symptom_4', 'symptom_5', 'gender']



for i in range(0, len(binary_features), 2):
    
    
    fig, axarr = plt.subplots(1, 2, figsize=(16, 8))
    try:   
        df_final[df_final[binary_features[i]] == 1]['target_transformed'].hist(density = True, ax = axarr[0])
        df_final[df_final[binary_features[i]] == 0]['target_transformed'].hist(density = True,
                                                                               alpha = .5,
                                                                               ax = axarr[0])
        axarr[0].title.set_text("Histograms of Target for " + '"'+ binary_features[i] +   '"' + " : 0 and 1")
        axarr[0].set_xlabel("Log transformed target Values")
        axarr[0].set_ylabel("Normalized Frequency")
        
        #plt.show()

        df_final[df_final[binary_features[i+1]] == 1]['target_transformed'].hist(density = True, ax = axarr[1])
        df_final[df_final[binary_features[i+1]] == 0]['target_transformed'].hist(density = True,alpha = .5,
                                                                                 ax = axarr[1])
        axarr[1].title.set_text("Histograms of Target for " + '"'+ binary_features[i+1] +   '"' + " : 0 and 1")
        axarr[1].set_xlabel("Log transformed target Values")
        axarr[1].set_ylabel("Normalized Frequency")
        axarr[1].tight_layout()

        #plt.show()
    except:
         plt.show()