"""
## Introduction:
Pneumonia is an inflammatory condition of the lung primarily affecting the small air sacs known as alveoli. Symptoms typically include some combination of productive or dry cough, chest pain, fever and difficulty breathing. The severity of the condition is variable. Pneumonia is usually caused by infection with viruses or bacteria, and less commonly by other microorganisms. Identifying the responsible pathogen can be difficult. 
Pneumonia is typically diagnosed based on a combination of physical signs and a chest X-ray.

![image.png](attachment:image.png)

In this notebook, It is tried to classify pneumonia using x-ray photographs in the data set with Convolutional Neural Network (CNN).
"""
"""
## Content:

- <a href='#1.'> 1. Importing Libraries</a>
- <a href='#2.'> 2. Loading and Checking Data</a>
- <a href='#3.'> 3. Preprocessing</a>
- <a href='#4.'> 4. Implementing CNN</a>
- <a href='#5.'> 5. Evaluation</a>
- <a href='#6.'> 6. Conclusion</a>
- <a href='#7.'> 7. References</a>
"""
"""
## <a id='1.'> 1. Importing Libraries</a> 
"""
import numpy as np                                 
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
import os                                         
import cv2 
from random import shuffle 

import tensorflow as tf                      
from tensorflow.keras.layers import BatchNormalization
from tensorflow.keras.callbacks import ReduceLROnPlateau
from tensorflow.keras.preprocessing import image
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import Conv2D, MaxPooling2D, Flatten, Dense, Dropout, Activation
from tensorflow.keras.preprocessing.image import ImageDataGenerator
from skimage.util import img_as_float

from zipfile import ZipFile 
from sklearn.metrics import classification_report 
import warnings
warnings.filterwarnings('ignore')
"""
## <a id='2.'> 2. Loading and Checking Dataset</a> 
"""
# Data directories and categories
train_data_dir = "../input/chest-xray-pneumonia/chest_xray/train"
test_data_dir = "../input/chest-xray-pneumonia/chest_xray/test"
CATEGORIES = ["NORMAL","PNEUMONIA"]
# Visualizing an image from data
for category in CATEGORIES:
    path = os.path.join(train_data_dir, category)
    for img in os.listdir(path):
        img_array = cv2.imread(os.path.join(path, img), cv2.IMREAD_GRAYSCALE)
        plt.imshow(img_array, cmap="gray")
        plt.show()
        break
    break
print(img_array.shape)
IMG_SIZE = 150
# Visualizing an image with its lowered resolution
new_array = cv2.resize(img_array, (IMG_SIZE, IMG_SIZE ))
plt.imshow(new_array, cmap = "gray")
plt.show()
"""
## <a id='3.'> 3. Preprocessing</a> 
"""
# Creating training data
training_data = []

def create_training_data():
    for category in CATEGORIES:
        path = os.path.join(train_data_dir, category)
        class_num = CATEGORIES.index(category)
        for img in os.listdir(path):
            try:
                img_array = cv2.imread(os.path.join(path,img), cv2.IMREAD_GRAYSCALE)
                new_array = cv2.resize(img_array, (IMG_SIZE, IMG_SIZE))
                training_data.append([new_array, class_num])
            except Exception as e:
                pass
create_training_data()
print(len(training_data))
# Creating test data

test_data = []

def create_test_data():
    for category1 in CATEGORIES:
        path1 = os.path.join(test_data_dir, category1)
        class_num = CATEGORIES.index(category1)
        for img in os.listdir(path1):
            try:               
                img_array1 = cv2.imread(os.path.join(path1,img), cv2.IMREAD_GRAYSCALE)
                new_array1 = cv2.resize(img_array1, (IMG_SIZE, IMG_SIZE))
                test_data.append([new_array1, class_num])
            except Exception as e:
                pass
create_test_data()
print(len(test_data))
# Datasets are sorted by categories. We need to shuffle them. 
import random 

random.shuffle(training_data)
random.shuffle(test_data)
# Checking data is imbalance.

list = []
for i in training_data:
    if(i[1] == 0):
        list.append("Pneumonia")
    else:
        list.append("Normal")
sns.countplot(list)   
X_train = []
y_train = []
X_test = []
y_test = []
for features, label in training_data:
    X_train.append(features)
    y_train.append(label)
for features, label in test_data:
    X_test.append(features)
    y_test.append(label)
# Normalize the data
X_train = np.array(X_train) / 255
X_test = np.array(X_test) / 255
# Reshape
X_train = np.array(X_train).reshape(-1, IMG_SIZE, IMG_SIZE, 1)
y_train = np.array(y_train)
X_test = np.array(X_test).reshape(-1, IMG_SIZE, IMG_SIZE, 1)
y_test = np.array(y_test)
# Handling the imbalance

datagen = ImageDataGenerator(
        featurewise_center=False,  # set input mean to 0 over the dataset
        samplewise_center=False,  # set each sample mean to 0
        featurewise_std_normalization=False,  # divide inputs by std of the dataset
        samplewise_std_normalization=False,  # divide each input by its std
        zca_whitening=False,  # apply ZCA whitening
        rotation_range = 30,  # randomly rotate images in the range (degrees, 0 to 180)
        zoom_range = 0.2, # Randomly zoom image 
        width_shift_range=0.1,  # randomly shift images horizontally (fraction of total width)
        height_shift_range=0.1,  # randomly shift images vertically (fraction of total height)
        horizontal_flip = True,  # randomly flip images
        vertical_flip=False)  # randomly flip images


datagen.fit(X_train)
"""
## <a id='4.'> 4. Implementing CNN</a>
"""
# Creating the model
model = Sequential()
model.add(Conv2D(32 , (3,3) , strides = 1 , padding = 'same' , activation = 'relu' , input_shape = (IMG_SIZE,IMG_SIZE,1)))
model.add(BatchNormalization())
model.add(MaxPooling2D((2,2) , strides = 2 , padding = 'same'))
model.add(Conv2D(16 , (3,3) , strides = 1 , padding = 'same' , activation = 'relu'))
model.add(Dropout(0.1))
model.add(BatchNormalization())
model.add(MaxPooling2D((2,2) , strides = 2 , padding = 'same'))
model.add(Conv2D(16 , (3,3) , strides = 1 , padding = 'same' , activation = 'relu'))
model.add(Dropout(0.2))
model.add(BatchNormalization())
model.add(MaxPooling2D((2,2) , strides = 2 , padding = 'same'))
model.add(Flatten())
model.add(Dense(units = 64 , activation = 'relu'))
model.add(Dropout(0.2))
model.add(Dense(units = 1 , activation = 'sigmoid'))
model.compile(optimizer = "rmsprop" , loss = 'binary_crossentropy' , metrics = ['accuracy'])
model.summary()
learning_rate_reduction = ReduceLROnPlateau(monitor='val_accuracy', patience = 2, verbose=1,factor=0.3, min_lr=0.000001)
history = model.fit(datagen.flow(X_train,y_train, batch_size = 32) ,epochs = 12 , validation_data = datagen.flow(X_test, y_test) ,callbacks = [learning_rate_reduction])
"""
## <a id='5.'> 5. Evaluation</a>
"""
predictions = model.predict_classes(X_test)
predictions = predictions.reshape(1,-1)[0]
print(classification_report(y_test, predictions, target_names = ['Pneumonia','Normal']))
from sklearn.metrics import confusion_matrix
cm = confusion_matrix(y_test,predictions)
cm
f,ax = plt.subplots(figsize=(8, 8))
sns.heatmap(cm, annot=True, linewidths=0.01,cmap="Blues",linecolor="gray", fmt= '.1f',ax=ax)
plt.xlabel("Predicted Label")
plt.ylabel("True Label")
plt.title("Confusion Matrix")
plt.show()
# Losses
plt.subplots(figsize=(8,6))
plt.plot(history.history["loss"], label = "train loss")
plt.plot(history.history["val_loss"], label = "test loss")
plt.xlabel("Epochs")
plt.ylabel("Loss")
plt.title("Losses of fit w LR reduction")
plt.legend()
plt.savefig("Losses of fit w LR reduction")
plt.show()
# Losses
plt.subplots(figsize=(8,6))
plt.plot(history.history["accuracy"], label = "train accuracy")
plt.plot(history.history["val_accuracy"], label = "test accuracy")
plt.xlabel("Epochs")
plt.ylabel("Accuracy")
plt.title("Accuracies of fit w LR reduction")
plt.legend()
plt.savefig("Accuracies of fit w LR reduction")
plt.show()
"""
## <a id='6.'> 6. Conclusion</a>
"""
"""
It is implemented CNN to predict the pneumonia from chest X-ray images. Our model predicted the results with accuracy of 0.88. We also have overfitting problem. I'll try to deal with overfitting in the next updates of notebook. 
Note: If you feedback me, I'll be greatful.
"""
"""
## <a id='7.'> 7. References</a> 
"""
"""
1. https://en.wikipedia.org/wiki/Pneumonia
2. https://www.kaggle.com/senolcomert/convolutional-neural-network-cnn-tutorial/edit
3. https://pythonprogramming.net/introduction-deep-learning-python-tensorflow-keras/
4. https://www.kaggle.com/madz2000/pneumonia-detection-using-cnn-92-6-accuracy
"""