"""
# <font color='darkblue'>Diabetes prediction</font>

#### This notebook includes a basic random forest classification model analysis. Each snippet of code shows explanation.Hopefully this will be helpful for someone who is trying kaggle competition for the first time.<br>

### <span style='background :yellow'>Please feel free to upvote and comment if you have any questions<span>
"""
"""
![](https://media1.tenor.com/images/22d8ce3fc622207434a52bfaa7b92593/tenor.gif?itemid=20036149)
"""
"""
![](http://)<h1><left>Lets get started!!</left></h1>

![](https://media1.tenor.com/images/1a68aae1cddacead28b4f4626d098297/tenor.gif?itemid=20034977)
"""
"""
Diabetes is one of the most prevailing chronic conditions in the US. Based on the adverse effects the Coronavirus has on patients with diabetes, it is vital to understand the patient history for a personalized treatment plan for a patient.
"""
"""
### <center> <font color='darkblue'>                                 Lets import all necessary Libraries!<font>
### <center><font color='darkblue'>                            All packages that this notebook needs are below. :)<font>
    
"""
"""
![](https://media1.tenor.com/images/e5d4fff0602734f5e0d23ec814dc31c5/tenor.gif?itemid=19594862)
"""
import pandas as pd
import numpy as np
import seaborn as sns
from sklearn.model_selection import train_test_split
from sklearn.metrics import roc_auc_score
from sklearn.metrics import roc_curve
import matplotlib.pyplot as plt
from sklearn.preprocessing import LabelEncoder
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import StandardScaler
from sklearn.ensemble import RandomForestRegressor
from sklearn.ensemble import RandomForestClassifier
from sklearn import metrics
from sklearn.metrics import classification_report, confusion_matrix, accuracy_score
from tensorflow import keras
from tensorflow.keras.layers import Conv1D, MaxPooling1D, Dense, LeakyReLU
from tensorflow.keras.layers import Flatten, Activation, BatchNormalization
from tensorflow.keras.losses import BinaryCrossentropy
from tensorflow.keras import layers
import tensorflow as tf
import xgboost as xgb
from xgboost import XGBRegressor
from sklearn.metrics import r2_score
from sklearn.model_selection import RandomizedSearchCV
from scipy.stats import uniform
from sklearn.model_selection import StratifiedKFold
from sklearn.model_selection import cross_val_score
from sklearn.model_selection import train_test_split
from sklearn.model_selection import StratifiedKFold
from sklearn.metrics import accuracy_score
from sklearn import metrics
from sklearn.model_selection import RandomizedSearchCV, GridSearchCV
from lightgbm import LGBMClassifier

"""
### <Left> <font color='darkblue'>Import datasets<font>
"""
data_dictionary = pd.read_csv("../input/widsdatathon2021/DataDictionaryWiDS2021.csv")
unlabeled = pd.read_csv("../input/widsdatathon2021/UnlabeledWiDS2021.csv")
training = pd.read_csv("../input/widsdatathon2021/TrainingWiDS2021.csv")
unlabeled = pd.read_csv("../input/widsdatathon2021/UnlabeledWiDS2021.csv")

#Creating a copy before doing any modifications
trainingeda=training.copy()
column_datatype_mapping = dict(zip(data_dictionary['Variable Name'], data_dictionary['Data Type']))
del training['Unnamed: 0']
del unlabeled['Unnamed: 0']
training.diabetes_mellitus.value_counts()
"""
### <Left> <font color='darkblue'>Data Manipulation<font>
"""
print(len(training), len(unlabeled))

all_data = training.drop(['diabetes_mellitus'], axis=1).append(unlabeled,ignore_index=True).drop(['encounter_id', 
                          'hospital_id'], axis=1)

for col in all_data.columns:
    if all_data.dtypes[col] == "object":
        all_data[col] = all_data[col].fillna("NA")
        all_data[col] = LabelEncoder().fit_transform(all_data[col])
        all_data[col]= all_data[col].astype('category')
    elif column_datatype_mapping[col] == "binary":
        all_data[col] = all_data[col].fillna(-1)
    elif column_datatype_mapping[col] == "numeric":
        all_data[col] = all_data[col].fillna(0)
    else:
        all_data[col] = all_data[col].fillna(all_data[col].median())
    
len(all_data)


all_data['weight'] = all_data['bmi']*(all_data['height']/100)*(all_data['height']/100)
"""
### <Left> <font color='darkblue'>EDA<font>
"""
"""
### <Left> <font color='purple'> 1. Age distribution by Males Vs Females<font>

"""
plt.rcParams["figure.figsize"] = (7,5)
trainingeda.groupby('gender').age.plot(kind='kde')
plt.xlabel("Age")

"""
### <span style="background-color:lightblue">It is clear from the above graph that the age distribution is pretty similar between males and females.</span>


"""
"""
### <Left> <font color='purple'> 2. Mean BMI for different age groups<font>

"""
bins = [0,18, 30, 40, 50, 60, 70, 120]
labels = ['0-17','18-29', '30-39', '40-49', '50-59', '60-69', '70+']
trainingeda['agerange'] = pd.cut(trainingeda.age, bins, labels = labels,include_lowest = True)
trainingedaar=pd.DataFrame(trainingeda.groupby('agerange')['bmi'].mean())


plt.rcParams["figure.figsize"] = (5,3)
fig = plt.figure()
ax = fig.add_axes([0,0,1,1])
trainingedaar=trainingedaar.reset_index()
ax.bar(trainingedaar['agerange'],trainingedaar['bmi'],color='slateblue')
plt.xlabel("Age")
plt.ylabel("Mean_BMI")
plt.show()



"""
### <span style="background-color:lightblue">Based on the graph, BMI increases upto 40 - 49 Yrs and then decreases.</span>
"""
(trainingeda.columns).to_list()
"""
### <Left> <font color='purple'> 3. Did Diabetic patients weigh more?<font>
"""



trainingwe=(trainingeda[['diabetes_mellitus','weight']])
print(trainingwe.shape)
trainingwe.dropna(inplace=True)
print(trainingwe.shape)
trainingwei=pd.DataFrame(trainingwe.groupby('diabetes_mellitus')['weight'].mean())

plt.rcParams["figure.figsize"] = (5,3)


fig = plt.figure()
ax = fig.add_axes([0,0,1,1])
trainingwei=trainingwei.reset_index()
trainingwei['diabetes_mellitus'] = trainingwei['diabetes_mellitus'].apply(str)

ax.bar(trainingwei['diabetes_mellitus'],trainingwei['weight'],color='slateblue')
plt.xlabel("0:Non-diabetic   1:Dibetic")
plt.ylabel("Mean weight")
plt.show()

"""
### <span style="background-color:lightblue">Diabetic patients weight 10+ pounds more when compared to non diabetic patients. When a person takes insulin as a therapy for diabetes, their body may absorb too much glucose from food, resulting in weight gain.<br>
<br>
</span><br>
    
    source:https://www.medicalnewstoday.com/articles/325328#:~:text=Eating%20more%20calories%20than%20the,food%2C%20resulting%20in%20weight%20gain.
"""
"""
### <Left> <font color='purple'> 4. Imbalanced training dataset?<font>
"""
imbalan=pd.DataFrame(trainingeda.groupby('diabetes_mellitus')['Unnamed: 0'].count())
imbalan=imbalan.reset_index()
imbalan['perc']=imbalan['Unnamed: 0']/len(trainingeda['Unnamed: 0'])


fig = plt.figure()
ax = fig.add_axes([0,0,1,1])

imbalan['diabetes_mellitus'] = imbalan['diabetes_mellitus'].apply(str)

ax.bar(imbalan['diabetes_mellitus'],imbalan['perc'],color='slateblue')
plt.xlabel("0:Non-diabetic   1:Dibetic")
plt.ylabel("perc")
plt.show()
"""
### <span style="background-color:lightblue">The dataset is obviously imbalanced as we can see from the graph above. So, we need to create a sample with balanced dataset. Balanced here means having approximately same number of records in the sample training dataset.</span><br>
    
"""
"""
### <Left> <font color='purple'> 5. Levels in Diabetic vs. non diabetic patients?<font>
    
    
source :https://www.kaggle.com/iamleonie/wids-datathon-2021-diabetes-detection
"""
#Replace the null values with mean

trainingeda['creatinine_apache'] =trainingeda['creatinine_apache'].fillna(trainingeda['creatinine_apache'].mean())
trainingeda['bun_apache'] =trainingeda['bun_apache'].fillna(trainingeda['bun_apache'].mean())
trainingeda['d1_creatinine_mean']=pd.DataFrame(trainingeda.groupby('encounter_id')['creatinine_apache'].mean())
trainingeda['d1_bun_mean']=pd.DataFrame(trainingeda.groupby('encounter_id')['bun_apache'].mean())

sns.pairplot(data=trainingeda, vars=["creatinine_apache", "weight", "bmi", "bun_apache"], hue='diabetes_mellitus');


no_diabetes = trainingeda[trainingeda.diabetes_mellitus == 0]
diabetes = trainingeda[trainingeda.diabetes_mellitus == 1]

#Visualization code
fig, (ax1, ax2) = plt.subplots(ncols=2)
fig.set_size_inches(12, 5)
#non-Diabetic patient correlation visualization

spike_cols_nd = [col for col in no_diabetes.columns if 'apache' in col]
trainingapache_nd=no_diabetes[spike_cols_nd]
corrs = trainingapache_nd.corr()
# sns.set(rc={'figure.figsize':(15,15)})
mask = np.zeros_like(corrs)
mask[np.triu_indices_from(mask)] = True
sns.heatmap(corrs, cmap='Spectral_r', mask=mask, square=True, vmin=-.4, vmax=.4,ax=ax1).set_title('Non diabetic patient levels')


#Diabetic patient correlation visualization
spike_cols = [col for col in diabetes.columns if 'apache' in col]
trainingapache=diabetes[spike_cols]
# sns.set(rc={'figure.figsize':(15,15)})
corrsd = trainingapache.corr()
mask = np.zeros_like(corrsd)
mask[np.triu_indices_from(mask)] = True
sns.heatmap(corrsd, cmap='Spectral_r', mask=mask, square=True, vmin=-.4, vmax=.4,ax=ax2).set_title('Diabetic patient levels')
fig.tight_layout()
"""
### <Left> <font color='goldenrod'> Top 10 highly correlated variables - Diabetc Vs. Non diabetic<font>
"""
# Create correlation matrix 
corr_mat = trainingapache.corr(method='pearson') 
  
# Retain upper triangular values of correlation matrix and 
# make Lower triangular values Null 
upper_corr_mat = corr_mat.where( 
    np.triu(np.ones(corr_mat.shape), k=1).astype(np.bool)) 
  
# Convert to 1-D series and drop Null values 
unique_corr_pairs = upper_corr_mat.unstack().dropna() 
  
# Sort correlation pairs 
sorted_mat = unique_corr_pairs.sort_values() 



# Create correlation matrix 
corr_mat_nd = trainingapache_nd.corr(method='pearson') 
  
# Retain upper triangular values of correlation matrix and 
# make Lower triangular values Null 
upper_corr_mat_nd = corr_mat_nd.where( 
    np.triu(np.ones(corr_mat_nd.shape), k=1).astype(np.bool)) 
  
# Convert to 1-D series and drop Null values 
unique_corr_pairs_nd = upper_corr_mat_nd.unstack().dropna() 
  
# Sort correlation pairs 
sorted_mat_nd = unique_corr_pairs_nd.sort_values() 


sorted_nd=pd.DataFrame(sorted_mat_nd).head(10)
print('Non diabetic patients')
print(sorted_nd.reset_index())
print(' ')
print(' ')
sorteddiab=pd.DataFrame(sorted_mat).head(10)
print('Diabetic patients')
print(sorteddiab.reset_index())
"""
### <span style="background-color:lightblue">Looking at the top 10 highly correlated variables  among diabetic Vs. Non- diabetic patients,verbal component,opening eyes,motor skill scale is highly correlated with ventilation for Non diabetic patients when compared to non diabetic patients. This shows that the diabetic patients who are on ventilation are less responsive on an average when comapred to non-diabetic  patients.
 </span><br>
    
"""
pd.DataFrame(trainingeda.groupby('diabetes_mellitus')['Unnamed: 0'].count())
"""

### <Left> <font color='purple'> 6. Make the sample data (Balanced dataset)<font>
"""
df1 = trainingeda[trainingeda['diabetes_mellitus']==1].sample(n=25000)
df0 = trainingeda[trainingeda['diabetes_mellitus']==0].sample(n=25000)
sample=pd.DataFrame(pd.concat([df1,df0]))

balaned=pd.DataFrame(sample.groupby('diabetes_mellitus')['Unnamed: 0'].count())
balaned=balaned.reset_index()

balaned
fig = plt.figure()
ax = fig.add_axes([0,0,1,1])

balaned['diabetes_mellitus'] = balaned['diabetes_mellitus'].apply(str)

ax.bar(balaned['diabetes_mellitus'],balaned['Unnamed: 0'],color='slateblue')
plt.xlabel("0:Non-diabetic   1:Dibetic")
plt.ylabel("perc")
plt.show()
"""
### <Left> <font color='purple'> 6. Data Manipulation of balanced dataset<font>
"""
print(len(training), len(unlabeled))

all_data = sample.drop(['diabetes_mellitus','Unnamed: 0','agerange','d1_creatinine_mean','d1_bun_mean','encounter_id','hospital_id'], axis=1)

for col in all_data.columns:
    if all_data.dtypes[col] == "object":
        all_data[col] = all_data[col].fillna("NA")
        all_data[col] = LabelEncoder().fit_transform(all_data[col])
        all_data[col]= all_data[col].astype('category')
    elif column_datatype_mapping[col] == "binary":
        all_data[col] = all_data[col].fillna(-1)
    elif column_datatype_mapping[col] == "numeric":
        all_data[col] = all_data[col].fillna(0)
    else:
        all_data[col] = all_data[col].fillna(all_data[col].median())
    
len(all_data)

"""
### <Left> <font color='purple'> 7. Data Manipulation of unlabeled dataset<font>
"""
print(len(training), len(unlabeled))

unlabeled1 =unlabeled.drop(['encounter_id','hospital_id'], axis=1)

for col in unlabeled1.columns:
    if unlabeled1.dtypes[col] == "object":
        unlabeled1[col] = unlabeled1[col].fillna("NA")
        unlabeled1[col] = LabelEncoder().fit_transform(unlabeled1[col])
        unlabeled1[col]= unlabeled1[col].astype('category')
    elif column_datatype_mapping[col] == "binary":
        unlabeled1[col] = unlabeled1[col].fillna(-1)
    elif column_datatype_mapping[col] == "numeric":
        unlabeled1[col] = unlabeled1[col].fillna(0)
    else:
        unlabeled1[col] = unlabeled1[col].fillna(unlabeled1[col].median())
    
len(unlabeled1)

"""
### <Left> <font color='purple'> 8. Random Forest Regresson model<font>
"""
"""
### <Left> <font color='darkblue'>Test and train data split<font>
"""
df_train = all_data.iloc[:len(training)]#[selected_columns]
df_pred = all_data.iloc[-len(unlabeled):].reset_index(drop=True)#[selected_columns]
Y = training['diabetes_mellitus']
x_train, x_test, y_train, y_test = train_test_split(df_train, sample['diabetes_mellitus'], test_size=0.20, random_state=42,shuffle=True )
clf = RandomForestClassifier(n_estimators=4, random_state=0)
clf.fit(x_train, y_train)
name='RandomForest'

prob_y_valrf = clf.predict_proba(x_test)[:,1]

#Make Predictions
pred_y_0 = clf.predict_proba(unlabeled1)[:,1]
pd.DataFrame(pred_y_0)
subs=unlabeled[['encounter_id']]
subs['diabetes_mellitus']=pd.DataFrame(pred_y_0)
subs.to_csv('./Prediction_with_rf.csv',index=False)

"""
### <Left> <font color='purple'> 9. CNN <font>
"""
df_train = all_data.iloc[:len(training)]#[selected_columns]
df_pred = all_data.iloc[-len(unlabeled):].reset_index(drop=True)#[selected_columns]
Y = training['diabetes_mellitus']
x_train, x_test, y_train, y_test = train_test_split(df_train, sample['diabetes_mellitus'], test_size=0.20, random_state=42,shuffle=True )
x_train = x_train.values.reshape(-1, x_train.shape[1], 1)  # reshaping for convnet
x_test = x_test.values.reshape(-1, x_test.shape[1], 1)  # reshaping for convnet
def build_model():
    model = keras.models.Sequential()
    model.add(Conv1D(384, kernel_size=3, input_shape=x_train.shape[1:]))
    model.add(BatchNormalization())
    model.add(Activation('relu'))
    model.add(MaxPooling1D(pool_size=2))
    model.add(Conv1D(32, kernel_size=3))
    model.add(BatchNormalization())
    model.add(Activation('relu'))
    model.add(Flatten()) 
    model.add(Dense(1))
    model.add(Activation("sigmoid"))
    model.compile(optimizer=keras.optimizers.Adam(lr=0.001),
                  loss=BinaryCrossentropy(label_smoothing=0.0921),
                  metrics=[tf.keras.metrics.AUC(name = 'auc'), "accuracy"])
    return model
    
model = build_model()
model.fit(x=x_train, y=y_train,
          epochs=25, batch_size=2048,
          callbacks=[tf.keras.callbacks.EarlyStopping('val_auc', mode='max',patience=3)],
          validation_data=(x_test, y_test))

model.fit(df_train.values.reshape(-1, df_pred.shape[1], 1), sample['diabetes_mellitus'],
          epochs=3,
          batch_size=1024)


#Make Predictions
pd.DataFrame({
    'encounter_id': unlabeled['encounter_id'].values,
    'diabetes_mellitus': model.predict_proba(
        df_pred.values.reshape(-1, df_pred.shape[1], 1)).flatten()
             }).to_csv('./Prediction_with_CNN1D.csv',index=False)
pred_cnn= model.predict_proba(
        df_pred.values.reshape(-1, df_pred.shape[1], 1)).flatten()
            
"""
### <Left> <font color='purple'> 10. XG Boost<font>
"""
# Remove Dependent Variable
df_train_1 = sample.loc[:,'encounter_id':'solid_tumor_with_metastasis'].reset_index(drop=True)
# Dependent Variable
y = sample['diabetes_mellitus']
df_test_1 = unlabeled.loc[:,'encounter_id':'solid_tumor_with_metastasis'].reset_index(drop=True)
cat_clmns = df_train_1.dtypes[df_train_1.dtypes == "object"].index 
numeric_clmns = df_train_1.dtypes[df_train_1.dtypes != "object"].index

#Categorical Values
df_train_1[cat_clmns] = df_train_1[cat_clmns].fillna(df_train_1[cat_clmns].mode().loc[0])
# Numerical
df_train_1[numeric_clmns] = df_train_1[numeric_clmns].fillna(df_train_1[numeric_clmns].mean())

# Impute Test 
# Categorical Values
df_test_1[cat_clmns] = df_test_1[cat_clmns].fillna(df_train_1[cat_clmns].mode().loc[0])
# Numerical
df_test_1[numeric_clmns] = df_test_1[numeric_clmns].fillna(df_train_1[numeric_clmns].mean())
## XGBoost
from xgboost import XGBClassifier
import xgboost as xgb

# process columns, apply LabelEncoder to categorical features
for c in cat_clmns:
    label_enc = LabelEncoder()
    label_enc.fit(df_train_1[c].values)
    # Training Data
    df_train_1[c] = label_enc.transform(df_train_1[c].values)
    # Testing Data
    df_test_1[c] = label_enc.transform(df_test_1[c].values)
df_train_2 = df_train_1.drop(['encounter_id', 'hospital_id'], axis = 1)
df_test_2 = df_test_1.drop(['encounter_id', 'hospital_id'], axis = 1)

nfold = 5
skf = StratifiedKFold(n_splits=nfold, shuffle=True, random_state=2019)
X_train, X_test, y_train, y_test = train_test_split(df_train_2, y, test_size=0.2, random_state=42)


st_xgb = XGBClassifier()

st_xgb.fit(X_train,y_train)

pred_xgb= st_xgb.predict_proba(X_test)[:,1]
pred_xgb





# frame_xgb = { 'encounter_id': df_test_1['encounter_id'], 'diabetes_mellitus': y_pred_lgb } 
# result = pd.DataFrame(frame_xgb) 
# result.to_csv('xgb_gpu.csv',index=False)

pred_xgb= st_xgb.predict_proba(df_test_2)
pred_xgb
#This gives only 79% so not great

pred=0.5*pred_xgb+0.5*pred_cnn

df_test_1['diabetes_mellitus'] = pred
df_test_1[["encounter_id","diabetes_mellitus"]].to_csv("submission_cnn+xgb.csv",index=False)
from sklearn.ensemble import StackingClassifier

st_clf = StackingClassifier(estimators=[('xgb', train_model), ('cnn', model)], 
                            final_estimator= RandomizedSearchCV(xgbb, 
                                   param_distributions=params, 
                                   scoring='roc_auc', 
                                   cv=skf.split(X_train,y_train), 
                                   verbose=3, 
                                   random_state=1001 ))
st_clf.fit(X_train,y_train)
"""
### <Left> <font color='purple'> 11. LGBM Classifier<font>
"""

model = LGBMClassifier(
                              random_state=33,
                              early_stopping_rounds = 250,
                              n_estimators=1000,
                              boosting_type='gbdt', num_leaves=151, max_depth=- 1, learning_rate=0.02, subsample_for_bin=200, 
                              min_split_gain=0.5, min_child_weight=0.001, min_child_samples=20, subsample=1.0, subsample_freq=0, 
                              colsample_bytree=.75, reg_alpha=1.3, reg_lambda=0.1,  n_jobs=- 1,
                              silent=True, importance_type='split')

model.fit(
    X_train,
    y_train,
    eval_set=[(X_test, y_test)],
    eval_metric='auc',  
    verbose=False,
)
print(f"accuracy score is {accuracy_score(y_test, model.predict(X_test))}") #0.02- 0.8423
print(metrics.classification_report(y_test, model.predict(X_test), labels=[0, 1]))
del df_test_2['diabetes_mellitus']
df_test_2
pred = model.predict_proba(df_test_2)[:,1]
df_test_1['diabetes_mellitus'] = pred
df_test_1[["encounter_id","diabetes_mellitus"]].to_csv("submission_tuned_lgbm_2.csv",index=False)