# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

# <h1 align="center">Diabetes Data Analysis</h1>
#
# All information regarding the features and dataset can be found in this research arcticle:
# Impact of HbA1c Measurement on Hospital Readmission Rates: Analysis of 70,000 Clinical Database Patient Records
#
# In this project we want to know how different features affect diabetes in general.
# For this kernel, we will be using a diabetes readmission dataset to explore the different frameworks for model explainability
#
#
# Machine learning models that can be used in the medical field should be interpretable.
# Humans should know why these models decided on a conclusion.
# The problem is the more complex an ML model gets the less interpretable it gets.
# In this kernel we will examine techniques and frameworks in interpreting ML models.

#import libraries
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
# %matplotlib inline

# read the file and create a pandas dataframe
data = pd.read_csv('dataset/diabetic_data.csv')

# check the dimensions of the data
data.shape

# first 5 rows of data
data.head()

# we get discriptive statistics of numerical variable

#discribtion of numerical data
data[['time_in_hospital','num_lab_procedures','num_procedures','num_medications',
        'number_outpatient','number_emergency','number_inpatient',
        'number_diagnoses']].describe()

#no of unique patient
len(np.unique(data['patient_nbr']))

# + Remove duplicate recod based on patient_nbr column

#remove duplicate patient recods
data = data.drop_duplicates(subset = 'patient_nbr', keep = 'first')

# + Plot some column data

# the response variable 'readmitted' in the original dataset contains three categories.
# 11% of patients were readmitted within 30 days (<30)
# 35% of patients were readmitted after 30 days (>30)
# 54% of patients were never readmitted (NO)
data.groupby('readmitted').size().plot(kind='bar')
plt.ylabel('Count')
plt.title("Distribution of Readmitted Data")
plt.show()

data['readmitted'] = pd.Series([0 if val in ['NO', '>30'] else  val 
                         for val in data['readmitted']], index=data.index)
data['readmitted'] = pd.Series([1 if val in ['<30'] else  val 
                         for val in data['readmitted']], index=data.index)

#values counts of readmited column
data.readmitted.value_counts()

# the response variable 'readmitted' in the original dataset contains three categories.
# 11% of patients were readmitted within 30 days (<30)
# 35% of patients were readmitted after 30 days (>30)
# 54% of patients were never readmitted (NO)
data.groupby('readmitted').size().plot(kind='bar')
plt.ylabel('Count')
plt.title("Distribution of Readmitted Data")
plt.show()

# + Drop unnecessary column from data

# remove irrelevant features
data.drop(['encounter_id','patient_nbr', 'weight', 'payer_code','max_glu_serum','A1Cresult'], axis=1, inplace=True)

# remove rows that have NA in 'race', 'diag_1', 'diag_2', or 'diag_3' and 'gender'
# remove rows that have invalid values in 'gender'
data = data[data['race'] != '?']
data = data[data['diag_1'] != '?']
data = data[data['diag_2'] != '?']
data = data[data['diag_3'] != '?']
data = data[data['gender'] != 'Unknown/Invalid']

# check 'age' feature
data.groupby('age').size().plot(kind='bar')
plt.ylabel('Count')
plt.title("Distribution of Age")
plt.show()

# > from above graph we see that 60 to 100 age group data is large so we make single group for this

# Recategorize 'age' so that the population is more evenly distributed
data['age'] = pd.Series(['[20-60)' if val in ['[20-30)', '[30-40)', '[40-50)', '[50-60)'] else val 
                         for val in data['age']], index=data.index)
data['age'] = pd.Series(['[60-100)' if val in ['[60-70)','[70-80)','[80-90)', '[90-100)'] else val 
                         for val in data['age']], index=data.index)
data.groupby('age').size().plot(kind='bar')
plt.ylabel('Count')
plt.title("Distribution of Age")
plt.show()

data.groupby('number_outpatient').size().plot(kind='bar')
plt.ylabel('Count')
plt.title("number_outpatient v/s Count ")
plt.show()

data.groupby('number_emergency').size().plot(kind='bar')
plt.title("number_emergency v/s Count")
plt.ylabel('Count')
plt.show()

data.groupby('number_inpatient').size().plot(kind='bar')
plt.title("number_inpatient v/s Count")
plt.ylabel('Count')
plt.show()

# remove the other medications
data.drop(['metformin', 'repaglinide', 'nateglinide', 'chlorpropamide', 'glimepiride', 
           'acetohexamide', 'glipizide', 'glyburide', 'tolbutamide', 'pioglitazone', 
           'rosiglitazone', 'acarbose', 'miglitol', 'troglitazone', 'tolazamide', 'examide', 
           'citoglipton', 'glyburide-metformin', 'glipizide-metformin', 'glimepiride-pioglitazone',
           'metformin-rosiglitazone', 'metformin-pioglitazone','insulin'], axis=1, inplace=True)

# +
# Recategorize 'age' so that the population is more evenly distributed
data['discharge_disposition_id'] = pd.Series(['Home' if val in [1] else val 
                         for val in data['discharge_disposition_id']], index=data.index)

data['discharge_disposition_id'] = pd.Series(['Anather' if val in [2,3,4,5,6] else val 
                         for val in data['discharge_disposition_id']], index=data.index)

data['discharge_disposition_id'] = pd.Series(['Expired' if val in [11,19,20,21] else val 
                         for val in data['discharge_disposition_id']], index=data.index)

data['discharge_disposition_id'] = pd.Series(['NaN' if val in [18,25,26] else val
                         for val in data['discharge_disposition_id']], index=data.index)
                        
data['discharge_disposition_id'] = pd.Series(['other' if val in [7,8,9,10,12,13,14,15,16,17,22,23,24,27,28,29,30] else val 
                         for val in data['discharge_disposition_id']], index=data.index)
# -

# original 'admission_source_id' contains 25 levels
# reduce 'admission_source_id' into 3 categories
data['admission_source_id'] = pd.Series(['Emergency Room' if val == 7 else 'Referral' if val in [1,2,3] else 
                                         'NaN' if val in [15,17,20,21] else 'Other source' 
                                              for val in data['admission_source_id']], index=data.index)

# original 'admission_type_id' contains 8 levels
# reduce 'admission_type_id' into 2 categories
data['admission_type_id'] = pd.Series(['Emergency' if val == 1 else 'Other type' 
                                              for val in data['admission_type_id']], index=data.index)

# Extract codes related to heart disease
data = data.loc[data['diag_1'].isin(['410','411','412','413','414','415','420','421','422','423','424','425','426','427','428','429','430']) | data['diag_2'].isin(['410','411','412','413','414','415','420','421','422','423','424','425','426','427','428','429','430']) | data['diag_3'].isin(['410','411','412','413','414','415','420','421','422','423','424','425','426','427','428','429','430'])]


data.shape

import random
#create variable emergency visits
data['emergency_visits'] = [random.randint(0, 5) for _ in range(26703)]

#create variable emergency visits
data['acuity_of_admission'] = [random.randint(1, 5) for _ in range(26703)]

#create variable emergency visits
data['comorbidity'] = [random.randint(1, 15) for _ in range(26703)]

categarical_colmun=["age","race","gender","medical_specialty","change","diabetesMed","discharge_disposition_id","admission_source_id","admission_type_id","diag_1","diag_2","diag_3"]
dtypes = {c: 'category' for c in categarical_colmun}
data=data.astype(dtypes)

# conver categarical variable into categary code
for i in categarical_colmun:
    data[i]=data[i].cat.codes

# apply square root transformation on right skewed count data to reduce the effects of extreme values.
# here log transformation is not appropriate because the data is Poisson distributed and contains many zero values.
data['number_outpatient'] = data['number_outpatient'].apply(lambda x: np.sqrt(x + 0.5))
data['number_emergency'] = data['number_emergency'].apply(lambda x: np.sqrt(x + 0.5))
data['number_inpatient'] = data['number_inpatient'].apply(lambda x: np.sqrt(x + 0.5))

# +
# feature scaling, features are standardized to have zero mean and unit variance
feature_scale_cols = ['time_in_hospital', 'num_lab_procedures', 'num_procedures', 'num_medications', 
                      'number_diagnoses', 'number_inpatient', 'number_emergency', 'number_outpatient']

from sklearn import preprocessing
scaler = preprocessing.StandardScaler().fit(data[feature_scale_cols])
data_scaler = scaler.transform(data[feature_scale_cols])

data_scaler_df = pd.DataFrame(data=data_scaler, columns=feature_scale_cols, index=data.index)
data.drop(feature_scale_cols, axis=1, inplace=True)
data = pd.concat([data, data_scaler_df], axis=1)
# -

# create X (features) and y (response)
X = data.drop(['readmitted'], axis=1)
y = data['readmitted']

y.value_counts()

# #### Find Top Features in data

# split X and y into cross-validation (75%) and testing (25%) data sets
from sklearn.model_selection import train_test_split
X_cv, X_test, y_cv, y_test = train_test_split(X, y, test_size=0.25)

# +
# fit Random Forest model to the cross-validation data
from sklearn.ensemble import RandomForestClassifier
forest = RandomForestClassifier()
forest.fit(X_cv, y_cv)
importances = forest.feature_importances_

# make importance relative to the max importance
feature_importance = 100.0 * (importances / importances.max())
sorted_idx = np.argsort(feature_importance)
feature_names = list(X_cv.columns.values)
feature_names_sort = [feature_names[indice] for indice in sorted_idx]
pos = np.arange(sorted_idx.shape[0]) + .5
print('Top 10 features are: ')
for feature in feature_names_sort[::-1][:10]:
    print(feature)

# plot the result
plt.figure(figsize=(12, 10))
plt.barh(pos, feature_importance[sorted_idx], align='center')
plt.yticks(pos, feature_names_sort)
plt.title('Relative Feature Importance', fontsize=20)
plt.show()
# -

# ### Use over sampling method for handle imbalanace data

from imblearn.over_sampling import SMOTE
from sklearn import metrics
from collections import Counter

oversample = SMOTE()
X, y = oversample.fit_resample(X, y)

X_train, X_test, y_train, y_test = train_test_split(X, y, test_size = .3)

# ### Logistic Regession

from sklearn.linear_model import LogisticRegression
clf=LogisticRegression(solver='saga')
clf.fit(X_train, y_train)
print(clf.score(X_test, y_test))

from sklearn.metrics import cohen_kappa_score
y_pred=clf.predict(X_test)
cohen_kappa_score(y_test, y_pred)

#Classification Score
from sklearn.metrics import classification_report
print(classification_report(y_test, y_pred))

# ### Random Forest 

rf =  RandomForestClassifier()
rf.fit(X_train, y_train)
print(rf.score(X_test, y_test))

# cohen kappa score
from sklearn.metrics import cohen_kappa_score
y_pred=rf.predict(X_test)
cohen_kappa_score(y_test, y_pred)

#Classification Score
from sklearn.metrics import classification_report
print(classification_report(y_test, y_pred))

StarcoderdataJupyterScriptsDedupFiltered