# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

# # Project: Identification of Pneumonia CXR 
#
# ## Analysis of Intensity Profiles
# ### Author: <NAME>, December 2020, last update: 05/01/2021

# !pip install -U seaborn >> /dev/null

# +
import pandas as pd
import numpy as np
import random

import matplotlib.pyplot as plt
# %matplotlib inline
import os
import seaborn as sns
from itertools import chain
import glob
from sklearn.model_selection import StratifiedKFold
from scipy import stats

sns.set()
# -

assert sns.__version__ >= '0.11.0'

# ## Full Dataset (NIH CXR-14) analysis

# +
# the file with the complete list of images and their metadata
FULL_LIST = "./metadata/Data_Entry_2017.csv"

def prepare_df_with_all_diseases(f_name):
    # add the columns for eac disease (0,1)
    full_df = pd.read_csv(FULL_LIST)
    # remove useless last column
    full_df.drop(['Unnamed: 11', 'OriginalImage[Width', 'Height]', 
              'OriginalImagePixelSpacing[x', 'y]'], axis = 1, inplace=True)
    print('Number of samples in dataset is:', full_df.shape[0])
    
    # add one column per label
    all_labels = np.unique(list(chain(*full_df['Finding Labels'].map(lambda x: x.split('|')).tolist())))
    all_labels = [x for x in all_labels if len(x)>0]
    
    for c_label in all_labels:
        if len(c_label)>1: # leave out empty labels
            full_df[c_label] = full_df['Finding Labels'].map(lambda finding: 1.0 if c_label in finding else 0)
    
    return full_df


# -

full_df = prepare_df_with_all_diseases(FULL_LIST)

# +
# build the full path list
all_image_list = glob.glob(os.path.join('/data','images*', '*', '*.png'))

all_image_paths = {os.path.basename(x): x for x in all_image_list}

# this contains pneumonia and all disease co-morb. with pneumonia
diseases_list = ['Pneumonia','Atelectasis', 'Cardiomegaly', 'Consolidation', 'Edema', 'Effusion', 'Emphysema', 
                      'Fibrosis', 'Hernia', 'Infiltration', 'Mass', 
                      'Nodule', 'Pleural_Thickening', 'Pneumothorax']


# +
#
# take a subsample and compute the intensity avg and std
# 
def compute_intensity_profile_all(label, df):
    
    N_SAMPLES = 100
    
    image_list = list(df['Image Index'].values)
    n_images = len(image_list)
    
    print('* Compute intensity Profile Stats for:', label)
    
    # sample N_SAMPLES from the list
    image_list = random.sample(image_list, N_SAMPLES)
    
    # threshold to remove background and high intensity pixels
    THRESH1 = 20
    THRESH2 = 250
    
    pixel_list = []
    
    for path in image_list:
        img = plt.imread(all_image_paths[path])
        img = img * 255.
        img_mask = (img > THRESH1) & (img < THRESH2)
        
        img = img[img_mask]
        
        pixel_list.extend(img.tolist())
    
    # compute mean and std for the images
    o_avg_int = round(np.mean(pixel_list), 1)
    o_std_int = round(np.std(pixel_list), 1)
    
    # num_images, avg_intensity, std_intensity
    return n_images, o_avg_int, o_std_int

def select_disease(label, df):
    condition = df[label] == 1.0
    
    return df.loc[condition]

def select_any_disease(df):
    condition = df['No Finding'] == 0.0
    
    return df.loc[condition]


# -

# ### Compute Stats for profiles for the different diseases

# +
# compute stats
    
all_list = diseases_list + ['No Finding']

stats_for_df = []

for disease in all_list:
    # select the partial df with only rows for that disease
    disease_df = select_disease(disease, full_df)
    
    n_imgs, avg_int, avg_std = compute_intensity_profile_all(disease, disease_df)
    
    stats_for_df.append([n_imgs, avg_int, avg_std])

# +
# aggregate stats for pixel intensity in a Dataframe
# and displays

df_stats = pd.DataFrame(stats_for_df, columns=['# images', 'avg_intensity', 'std'])
df_stats['category'] = pd.Series(all_list)

# to reorder columns
# new_column_list = ['category'] + all_list
df_stats = df_stats[['category', '# images', 'avg_intensity', 'std']]

# finally displays
df_stats.head(15)
# -

# ### Comments:
# The difference in average values is less than the standard deviation for each population (Pneumonia, NO Finding, etc).
# Therefore, I would conclude that **these stats are NOT a good predictor for Pneumonia**.

# ### Plot Intensity Profile for Pneumonia and all other comorbitities

# +
BINRANGE = (0, 255)
BINS = 255

def plot_intensity_profile_pneumonia_vs_nofinding(df):
    
    plt.figure(figsize=(25,25))
    
    # thresholds to remve background and very high intensity pixels
    THRESH1 = 20
    THRESH2 = 250
    # less sample to reduce computation time
    N_SAMPLES = 100
    
    label_list = diseases_list + ['No Finding']
    
    for i, label in enumerate(label_list):
        print('#', end =" "),
        condition = (df[label] == 1.0)
        image_list = list(df.loc[condition, 'Image Index'].values)
        
        # sample N_SAMPLES from the list
        image_list = random.sample(image_list, N_SAMPLES)
        
        pixel_intensities = []
        
        for num, path in enumerate(image_list):
            img = plt.imread(all_image_paths[path])
            img = img * 255.
            img_mask = (img > THRESH1) & (img < THRESH2)
            
            pixel_intensities.extend(img[img_mask].tolist())
        
        plt.subplot(5,3,i+1)
        plt.xlabel('pixel intensity')
        plt.ylabel('fraction')
        plt.ylim(0, 0.008)
        plt.title(label)
        plt.hist(pixel_intensities, bins = BINS, density = True);


# -

# ### Intensity profile compared: Pneumonia vs other comorbidities and No Finding

# now I can get the Intensity profile of ONLY the image with Pneumonia = 1
plot_intensity_profile_pneumonia_vs_nofinding(full_df)

# ##  Conclusions
# If we compare the profiles we see that: 
# * for some diseases (Emphysema, Infiltration) is similar to Pneumonia.
# * It is not very different for No Finding.
# * For some diseases (Fibrosys, Nodule, Pleural Tickening, Hernia) it is very different.
#
#
# 1. The profile therefore is not a very good predictor to distinguish between a Pneumonia CXR and NO Finding
# 2. The fact that we have some diseases where the profile is similar to Pneumonia profile, with high chance, makes more difficult, for the DNN model, to distinguish between Pneumonia and those diseases.



StarcoderdataJupyterScriptsDedupFiltered