# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     name: python3
# ---

# + [markdown] id="V45LHACzJxGi" colab_type="text"
# ## Mount Drive and Download Dataset

# + id="0P7x_4ULHbr_" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 121} outputId="09f62261-7e3e-4660-ba5c-475eaac90a5f"
from google.colab import drive
drive.mount('/content/gdrive')

# + id="Q_2Ll8meIBKw" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 34} outputId="ce5abcc1-8dd1-4a03-b947-98ebb8de22e8"
# !unzip -uq '/content/gdrive/My Drive/chexpertdataset.zip'

# + [markdown] id="Ps-jyezTKZkC" colab_type="text"
# ## Import Basic Packages

# + id="xtfiz4KMIF81" colab_type="code" colab={}
import cv2
import numpy as np
import pandas as pd
# %matplotlib inline
from matplotlib import pyplot as plt

# + [markdown] id="NYo5P1rgKe5N" colab_type="text"
# ## Read and clean the  DataFrame

# + id="du1b0of2IL6m" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 387} outputId="43ec1ce5-6edc-4939-e9b1-e092f3054af3"
df = pd.read_csv('CheXpert-v1.0-small/train.csv') # Earlier, df_train
df_val = pd.read_csv('CheXpert-v1.0-small/valid.csv')


# + id="hLx4C5qXJ0_R" colab_type="code" colab={}
def clean_df(df):
  # Focusing only on 5 classes:
  df = df[[
    'Path', 
    'Atelectasis',
    'Cardiomegaly',
    'Consolidation',
    'Edema',
    'Pleural Effusion'
  ]]

  # Handling the NaN values
  df = df.fillna(0)

  # Handling the uncertain values
  ## Different policy for each feature:
  u_ones = ['Atelectasis', 'Edema']
  u_zeros = ['Cardiomegaly', 'Consolidation', 'Pleural Effusion']
  df[u_ones]  = df[u_ones].replace(-1, 1)
  df[u_zeros] = df[u_zeros].replace(-1, 0)

  return df


# + id="jd4QXLd8IO9U" colab_type="code" colab={}
df = df[[
  'Path', 
  'Atelectasis',
  'Cardiomegaly',
  'Consolidation',
  'Edema',
  'Pleural Effusion'
]]

# Handling the NaN values
df = df.fillna(0)

# Handling the uncertain values
## Different policy for each feature:
u_ones = ['Atelectasis', 'Edema']
u_zeros = ['Cardiomegaly', 'Consolidation', 'Pleural Effusion']
df[u_ones]  = df[u_ones].replace(-1, 1)
df[u_zeros] = df[u_zeros].replace(-1, 0)

# + id="71vrB_-lJ7JX" colab_type="code" colab={}
df = clean_df(df)
df_val = clean_df(df_val)

# + [markdown] id="psjtHtTSKkUg" colab_type="text"
# ## Set a few constants and to-be-used Class names

# + id="fihb_ZPwIRnM" colab_type="code" colab={}
BATCH_SIZE = 32
IMAGE_SIZE = 224
CLASSES = [ 
  'Atelectasis',
  'Cardiomegaly',
  'Consolidation',
  'Edema',
  'Pleural Effusion'
]

# + id="sKOTlwhsIfFm" colab_type="code" colab={}
FRAC = 0.003 # Fraction of total data to be taken as sample
SHAPE = (320, 390, 3) # Common shape for featurewise centering & normalization

sample_paths = df['Path'].sample(frac=FRAC).to_numpy()
X_temp = np.array([np.array(cv2.imread(path, 1), dtype=float) for path in sample_paths])
X_sample = np.array([x for x in X_temp if x.shape == SHAPE])


# + [markdown] id="8sZkamIiKtAk" colab_type="text"
# ## A custon preprocessing function to experiment with
#
# We tried histogram equalization to mimic deeper constrasts in the image, but unfortunately, it didn't return results as we had expected, so we dropped this in the end.

# + id="6F-v478nfKxm" colab_type="code" colab={}
def image_preprocess(img):
  img = cv2.convertScaleAbs(img)
  img = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
  dst = cv2.equalizeHist(img)
  dst = cv2.cvtColor(dst, cv2.COLOR_GRAY2RGB)
  dst = dst.astype(np.float64)
  return dst


# + [markdown] id="oV9--WjVNHoT" colab_type="text"
# ## Set up Image Data Generator for Data Augmentation

# + id="isOE73_YIpI-" colab_type="code" colab={}
from keras.preprocessing.image import ImageDataGenerator as IDG

datagen = IDG(
    rescale=1./255, 
    featurewise_center=True,
    featurewise_std_normalization=True,
    rotation_range=0.1,
    zoom_range = 0.1,
    width_shift_range=0.2,
    height_shift_range=0.2,
    horizontal_flip=True,
    validation_split = 0.1,
    fill_mode = 'nearest',
)

datagen.fit(X_sample)

test_datagen = IDG(rescale=1./255)


# + id="C53TZWrAIsDe" colab_type="code" colab={}
def get_gen():
  train_gen = datagen.flow_from_dataframe(
      dataframe = df,
      #directory = '/content/CheXpert-v1.0-small/train',
      x_col = 'Path',
      y_col = CLASSES, #'classes',
      class_mode='raw',
      #validate_filenames = False,
      seed=42,
      shuffle=True,
      target_size=(IMAGE_SIZE, IMAGE_SIZE), 
      batch_size=BATCH_SIZE, 
      subset = 'training'
  )

  val_gen = datagen.flow_from_dataframe(
      dataframe = df,
      #directory = '/content/CheXpert-v1.0-small/train',
      x_col = 'Path',
      y_col = CLASSES, #'classes',
      class_mode='raw',
      #validate_filenames = False,
      seed=42,
      shuffle=True,
      target_size=(IMAGE_SIZE, IMAGE_SIZE), 
      batch_size=BATCH_SIZE, 
      #classes = columns,
      subset = 'validation'
  )

  return train_gen, val_gen


# + [markdown] id="DtdurrZ8NPSi" colab_type="text"
# ## Build the Model

# + id="AYyWvcJjI2pj" colab_type="code" colab={}
# Building on top of the base:
from keras.applications import DenseNet121
from keras.models import Sequential
from keras.layers import BatchNormalization, Conv2D, GlobalAveragePooling2D
from keras.layers.core import Flatten, Dense, Dropout

def build_model():
  # The convolutional base:
  model_base = DenseNet121(
      weights='imagenet', include_top=False, input_shape=(IMAGE_SIZE, IMAGE_SIZE, 3)
      )
  #model_base.trainable = False
  # Unfreezing all the layers:
  for layer in model_base.layers:
      layer.trainable = True

  model = Sequential()
  model.add(model_base) # Adding the base as a layer
  model.add(GlobalAveragePooling2D())
  model.add(Dense(1024, activation='relu'))
  model.add(BatchNormalization())
  model.add(Dropout(0.3))
  #model.add(Flatten())
  #model.add(Dense(1024, activation='relu'))
  #model.add(Dropout(0.25))
  model.add(Dense(5, activation='sigmoid'))
  
  return model


# + id="Y1ETpsqjJWLB" colab_type="code" colab={}
from keras.metrics import AUC, categorical_accuracy as catacc
from keras.callbacks import EarlyStopping, ModelCheckpoint
from keras.optimizers import Adam

auc = AUC()
adam = Adam(learning_rate=0.0001) # 0.05 of default

es = EarlyStopping(monitor='val_categorical_accuracy', mode='max', verbose=1, patience=2)
mc = ModelCheckpoint(
    filepath='densenet121-keras-2.h5', verbose=1 #, save_best_only=True
)

cb_list = [es, mc] # Will add clr later, as we'll have to tune it's hyperparameters

model = build_model()
model.compile(
    loss='binary_crossentropy',
    optimizer=adam,
    metrics=[auc, catacc] # Earlier, 'acc' 
)

# + id="rDe4VJEiNU1J" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 353} outputId="b7482725-045d-470b-aee4-9e190afc2a2a"
model.summary()

# + id="LriJldfIKlRf" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 50} outputId="10385280-deef-4632-abdb-a6788e13b1dd"
train_gen, val_gen = get_gen()

# + [markdown] id="l65tytBsQnvx" colab_type="text"
# Set train_steps, validation_steps and number of epochs and let the training begin !

# + id="4yzklBXCJYgo" colab_type="code" colab={}
# Training constants:
TRAIN_STEPS = train_gen.n//BATCH_SIZE
VAL_STEPS   = val_gen.n//BATCH_SIZE
N_EPOCHS = 5

# + id="fivmIamUNtL9" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 34} outputId="9830c8d3-89e8-4b5f-8ab3-b12628675824"
TRAIN_STEPS

# + id="tlh96xbRJgJ3" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 272} outputId="286d03ef-9a6e-4df6-fd7a-6fd8528e2a9a"
history = model.fit_generator(
    train_gen,
    steps_per_epoch=TRAIN_STEPS,
    epochs=N_EPOCHS,
    validation_data=val_gen,
    validation_steps=VAL_STEPS,
    callbacks = cb_list
)

# + [markdown] id="Yyun8p45SVsC" colab_type="text"
# Google Colab Runtime limitations :(
#
#   ## Create test generator and plot ROCAUCs for our model !

# + id="pMnqRcNMLFap" colab_type="code" colab={}
test_gen = test_datagen.flow_from_dataframe(
    dataframe = df_val,
    #directory = '/content/CheXpert-v1.0-small/valid',
    x_col = 'Path',
    y_col = CLASSES, #'classes',
    class_mode='raw',
    #validate_filenames = False,
    target_size=(IMAGE_SIZE, IMAGE_SIZE), 
    batch_size=1, 
    shuffle = False,
    #classes = columns,
)

# + id="gvr6i_PxMxSK" colab_type="code" colab={}
y_labels = df_val[CLASSES].to_numpy()
y_pred = model.predict_generator(test_gen, steps=test_gen.n) 

# + id="9OTCkP1hM2_b" colab_type="code" colab={}
from sklearn.metrics import roc_curve
from sklearn.metrics import auc

plt.figure(1)
plt.plot([0, 1], [0, 1], 'k--')

for i in range(len(CLASSES)):
   fpr, tpr, thresholds = roc_curve(y_labels[:, i], y_pred[:, i])
   individual_auc = auc(fpr, tpr)
   plt.plot(fpr, tpr, label= (CLASSES[i] + '(area = {})'.format(individual_auc)))

    
plt.xlabel('False positive rate')
plt.ylabel('True positive rate')
plt.title('ROC curve')
plt.legend(loc='best')
plt.show()

# + [markdown] id="kEwbA0tHV1U2" colab_type="text"
# ![alt text](https://i.ibb.co/jbXfhPs/densenet121-rescale.png)

StarcoderdataJupyterScriptsDedupFiltered