# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 2
#     language: python
#     name: python2
# ---

# <div style="width:900px;background:#F9EECF;border:1px solid black;text-align:left;padding:20px;">
# <span style="color:purple;font-size:13pt"><b>Data Dictionary</span></b>
# </div>
#
# <div style="width:900px;float:left;padding:20px;align:left;">
# Descriptors:
# <br><br>
# <style type="text/css">
# .tg  {border-collapse:collapse;border-spacing:0;}
# .tg td{font-family:Arial, sans-serif;font-size:14px;padding:10px 5px;border-style:solid;border-width:1px;overflow:hidden;word-break:normal;}
# .tg th{font-family:Arial, sans-serif;font-size:14px;font-weight:normal;padding:10px 5px;border-style:solid;border-width:1px;overflow:hidden;word-break:normal;}
# .tg .tg-2lp6{font-weight:bold;background-color:#bbdaff;vertical-align:top}
# .tg .tg-amwm{font-weight:bold;text-align:center;vertical-align:top}
# .tg .tg-36xf{font-weight:bold;background-color:#bbdaff}
# .tg .tg-yw4l{vertical-align:top}
# .tg .tg-yw42{vertical-align:top;color:blue}
# </style>
# <table class="tg">
#
#   <tr>
#     <th class="tg-36xf">Feature Name</th>
#     <th class="tg-2lp6">Description</th>
#     <th class="tg-2lp6">Metrics</th>
#   </tr>
#           
# <tr>
#     <td class="tg-yw4l">RecordID</td>
#     <td class="tg-yw4l">A unique integer for each ICU stay </td>
#     <td class="tg-yw4l">Integer</td>
# </tr>
# <tr>
#     <td class="tg-yw4l">Age</td>
#     <td class="tg-yw4l">Age</td>
#     <td class="tg-yw4l">(years)</td>
# </tr>
#
# <tr>
#     <td class="tg-yw4l">Height</td>
#     <td class="tg-yw4l">Height</td>
#     <td class="tg-yw4l">(cm)</td>
# </tr>
#
# <tr>
#     <td class="tg-yw4l">ICUtype</td>
#     <td class="tg-yw4l">ICU Type</td>
#     <td class="tg-yw4l">(1: Coronary Care Unit, 2: Cardiac Surgery Recovery Unit, <br>3: Medical ICU, or 4: Surgical ICU)</td>
# </tr>
#
# <tr>
#     <td class="tg-yw4l">Gender</td>
#     <td class="tg-yw4l">Gender</td>
#     <td class="tg-yw4l">(0: female, or 1: male)</td>
# </tr>
#
# </table>
# <br><br>
# These 37 variables may be observed once, more than once, or not at all in some cases:
# <br><br>
#
# <table class="tg">
# <tr>
#     <th class="tg-36xf">Feature Name</th>
#     <th class="tg-2lp6">Description</th>
#     <th class="tg-2lp6">Metrics</th>
# </tr>
#           
# <tr>
#     <td class="tg-yw4l">Albumin</td>
#     <td class="tg-yw4l">Albumin </td>
#     <td class="tg-yw4l">(g/dL)</td>
# </tr>
# <tr>
#     <td class="tg-yw4l">ALP</td>
#     <td class="tg-yw4l">Alkaline phosphatase</td>
#     <td class="tg-yw4l">(IU/L)</td>
# </tr>
# <tr>
# <td class="tg-yw4l">ALT</td>
# <td class="tg-yw4l">Alanine transaminase</td> 
# <td class="tg-yw4l">(IU/L)</td>
# </tr>
# <tr>
# <td class="tg-yw4l">AST</td>
# <td class="tg-yw4l">Aspartate transaminase</td>
# <td class="tg-yw4l">(IU/L)</td>
# </tr>
# <tr>
# <td class="tg-yw4l">Bilirubin</td>
# <td class="tg-yw4l">Bilirubin</td>
# <td class="tg-yw4l">(mg/dL)</td>
# </tr>
# <tr>
# <td class="tg-yw4l">BUN</td>
# <td class="tg-yw4l">Blood urea nitrogen</td>
# <td class="tg-yw4l">(mg/dL)</td>
# </tr>
# <tr>
# <td class="tg-yw4l">Cholesterol</td>
# <td class="tg-yw4l">Cholesterol</td>
# <td class="tg-yw4l">(mg/dL)</td>
# </tr>
# <tr>
# <td class="tg-yw4l">Creatinine</td>
# <td class="tg-yw4l">Serum creatinine</td>
# <td class="tg-yw4l">(mg/dL)</td>
# </tr>
#
# <tr>
# <td class="tg-yw4l">DiasABP <td class="tg-yw4l">Invasive diastolic arterial blood pressure <td class="tg-yw4l">(mmHg)</tr>
#
# <tr><td class="tg-yw4l">FiO2 <td class="tg-yw4l">Fractional inspired O2 <td class="tg-yw4l">(0-1)</tr>
# <tr><td class="tg-yw4l">GCS <td class="tg-yw4l">Glasgow Coma Score <td class="tg-yw4l">(3-15)</tr>
# <tr><td class="tg-yw4l">Glucose <td class="tg-yw4l">Serum glucose <td class="tg-yw4l">(mg/dL)</tr>
# <tr><td class="tg-yw4l">HCO3 <td class="tg-yw4l">Serum bicarbonate <td class="tg-yw4l">(mmol/L)</tr>
# <tr><td class="tg-yw4l">HCT <td class="tg-yw4l">Hematocrit <td class="tg-yw4l">(%)</tr>
# <tr><td class="tg-yw4l">HR <td class="tg-yw4l">Heart rate <td class="tg-yw4l">(bpm)</tr>
# <tr><td class="tg-yw4l">K <td class="tg-yw4l">Serum potassium <td class="tg-yw4l">(mEq/L)</tr>
# <tr><td class="tg-yw4l">Lactate <td class="tg-yw4l">Lactate<td class="tg-yw4l">(mmol/L)</tr>
# <tr><td class="tg-yw4l">Mg <td class="tg-yw4l">Serum magnesium <td class="tg-yw4l">(mmol/L)</tr>
# <tr><td class="tg-yw4l">MAP <td class="tg-yw4l">Invasive mean arterial blood pressure <td class="tg-yw4l">(mmHg)</tr>
# <tr><td class="tg-yw4l">MechVent <td class="tg-yw4l">Mechanical ventilation respiration <td class="tg-yw4l">(0:false, or 1:true)</tr>
# <tr><td class="tg-yw4l">Na <td class="tg-yw4l">Serum sodium <td class="tg-yw4l">(mEq/L)</tr>
# <tr><td class="tg-yw4l">NIDiasABP <td class="tg-yw4l">Non-invasive diastolic arterial blood pressure <td class="tg-yw4l">(mmHg)</tr>
# <tr><td class="tg-yw4l">NIMAP <td class="tg-yw4l">Non-invasive mean arterial blood pressure <td class="tg-yw4l">(mmHg)</tr>
# <tr><td class="tg-yw4l">NISysABP <td class="tg-yw4l">Non-invasive systolic arterial blood pressure <td class="tg-yw4l">(mmHg)</tr>
# <tr><td class="tg-yw4l">PaCO2 <td class="tg-yw4l">partial pressure of arterial CO2 <td class="tg-yw4l">(mmHg)</tr>
# <tr><td class="tg-yw4l">PaO2 <td class="tg-yw4l">Partial pressure of arterial O2 <td class="tg-yw4l">(mmHg)</tr>
# <tr><td class="tg-yw4l">pH <td class="tg-yw4l">Arterial pH <td class="tg-yw4l">(0-14)</tr>
# <tr><td class="tg-yw4l">Platelets <td class="tg-yw4l">Platelets<td class="tg-yw4l">(cells/nL)</tr>
# <tr><td class="tg-yw4l">RespRate <td class="tg-yw4l">Respiration rate <td class="tg-yw4l">(bpm)</tr>
# <tr><td class="tg-yw4l">SaO2 <td class="tg-yw4l">O2 saturation in hemoglobin <td class="tg-yw4l">(%)</tr>
# <tr><td class="tg-yw4l">SysABP <td class="tg-yw4l">Invasive systolic arterial blood pressure <td class="tg-yw4l">(mmHg)</tr>
# <tr><td class="tg-yw4l">Temp <td class="tg-yw4l">Temperature <td class="tg-yw4l">(°C)</tr>
# <tr><td class="tg-yw4l">TropI <td class="tg-yw4l">Troponin-I <td class="tg-yw4l">(μg/L)</tr>
# <tr><td class="tg-yw4l">TropT <td class="tg-yw4l">Troponin-T <td class="tg-yw4l">(μg/L)</tr>
# <tr><td class="tg-yw4l">Urine <td class="tg-yw4l">Urine output <td class="tg-yw4l">(mL)</tr>
# <tr><td class="tg-yw4l">WBC <td class="tg-yw4l">White blood cell count <td class="tg-yw4l">(cells/nL)</tr>
# <tr><td class="tg-yw4l">Weight <td class="tg-yw4l">Weight<td class="tg-yw4l">(kg)</tr></table>
#
# <br><br>
# Outcomes-Related Descriptors:
#
# <table class="tg">
# <tr>
#     <th class="tg-36xf">Outcomes</th>
#     <th class="tg-2lp6">Description</th>
#     <th class="tg-2lp6">Metrics</th>
# </tr>
#           
# <tr>
#     <td class="tg-yw4l">SAPS-I score</td>
#     <td class="tg-yw4l">(Le Gall et al., 1984) </td>
#     <td class="tg-yw4l">between 0 to 163</td>
# </tr>
#   
# <tr>
#     <td class="tg-yw4l">SOFA score</td>
#     <td class="tg-yw4l">(Ferreira et al., 2001) </td>
#     <td class="tg-yw4l">between 0 to 4</td>
# </tr>
#   
# <tr>
#     <td class="tg-yw4l">Length of stay</td>
#     <td class="tg-yw4l">Length of stay </td>
#     <td class="tg-yw4l">(days)</td>
# </tr>
#
# <tr>
#     <td class="tg-yw4l">Survival</td>
#     <td class="tg-yw4l">Survival</td>
#     <td class="tg-yw4l">(days)</td>
# </tr>
#
# <tr>
#     <td class="tg-yw42"><b>In-hospital death</b></td>
#     <td class="tg-yw42"><b>Target Variable</b></td>
#     <td class="tg-yw42"><b>(0: survivor, or 1: died in-hospital)</b></td>
# </tr>
#  

# <div style="width:900px;background:#F9EECF;border:1px solid black;text-align:left;padding:20px;">
#
# <span style="color:purple;font-size:13pt"><b>Import Packages</span></b>
#
# </div>

# +
# Import packages

import pandas as pd
import numpy as np


import seaborn as sns

from sklearn.preprocessing import RobustScaler, OneHotEncoder
from sklearn.pipeline import FeatureUnion
from sklearn.base import BaseEstimator, TransformerMixin

from statsmodels.stats.outliers_influence import variance_inflation_factor

pd.set_option('display.max_columns', 200)
pd.set_option('display.max_rows',200)


sns.set_style('whitegrid')
sns.set(rc={"figure.figsize": (15, 8)})

# %config InlineBackend.figure_format = 'retina'
# %matplotlib inline

# +
# Simple cleaning on csv file

mortality = pd.read_csv('mortality_filled_median.csv')
mortality.drop(['Unnamed: 0'],axis=1,inplace=True)
mortality.set_index('recordid',inplace=True)
# -

X = mortality.drop('in-hospital_death',axis=1)   # identify predictors 


# <div style="width:900px;background:#F9EECF;border:1px solid black;text-align:left;padding:20px;">
#
# <p>
#
# <span style="color:purple;font-size:13pt"><b>Feature Selection</span></b>
# <br><br>
# <b>Step 1: Preprocess</b>
# <br>
# Get dummy variables for categorical features
# <br>
# Scale numerical features with Robust Scaler 
# <br>
# (Scaled with median and the interquartile range so that outliers can be taken into account)
# <br><br>
# <b>Step 2: Variance Inflation Factor</b> 
# <br>Drop columns that are multi-colinear with each other
# <br><br>
# <b>Step 3: Decision Trees on Feature Importance</b>
# <br> Rank features based on importance
# <br><br>
# <b>Step 4: Reduce dimensionality using PCA</b>
# </div>

class Categorical_Extractor(BaseEstimator, TransformerMixin):
    
    def __init__(self):
        pass

    def get_dummy_(self,df):
        
        icu_dummy = pd.get_dummies(df['icutype'],drop_first=True)
        df = pd.concat((df['gender'],icu_dummy),axis=1)
        
        return df.values.reshape(-1, 4)
    
    def transform(self, X, *args):
        
        X = self.get_dummy_(X)
        
        return X


    def fit(self, X, *args):
        return self


class Numerical_Extractor(BaseEstimator, TransformerMixin):
    
    def __init__(self):
        pass

    def robust_scale_(self,df,threshold=10):
        
        num_col = []
        for col, num in df.nunique().iteritems():
            if num > threshold:
                num_col.append(col)
                
        df = df[num_col]
        
        rscaler = RobustScaler()
        array = rscaler.fit_transform(df)
        
        return array.reshape(-1,array.shape[1])
    
    def transform(self, X, *args):
        
        X = self.robust_scale_(X)
        
        return X


    def fit(self, X, *args):
        return self

feature_union = FeatureUnion([  
    ('numerical_scaler',Numerical_Extractor()),
    ('categorical_transform',Categorical_Extractor())
])

transformed_X = feature_union.transform(X)

new_columns = list(X.columns)
new_columns.extend(['icutype3','icutype4'])

transformed_X = pd.DataFrame(transformed_X,columns=new_columns,index=X.index)

transformed_X.rename(columns={'icutype':'icutype2'},inplace=True)

transformed_X.head()

transformed_X.shape

# <div style="width:900px;background:#F9EECF;border:1px solid black;text-align:left;padding:20px;">
#
# We have a total of 110 features now.
#
# </div>

# +
# Variance Inflation Factor

vif = [variance_inflation_factor(transformed_X.iloc[:,:-4].values,i) 
       for i in range(transformed_X.iloc[:,:-4].shape[1])]

# +
sns.set_style('whitegrid')
sns.set(rc={"figure.figsize": (15, 15)})

vif_df = pd.DataFrame(transformed_X.iloc[:,:-4].columns,columns=['Features'])
vif_df['VIF'] = vif

sns.barplot(x='VIF',y='Features',data=vif_df)
# -

remove_col = list(vif_df[vif_df['VIF']>5]['Features'])

selected_X = transformed_X.drop(remove_col,axis=1)
selected_X.shape



StarcoderdataJupyterScriptsDedupFiltered