"""
在 [Pneumonia Detection by VGG16](https://www.kaggle.com/whitelee/pneumonia-detection-by-vgg16) 中我們透過 Transfer Learning 的作法 fine-tune VGG16 model 取得了不錯的預測結果 (F1 Score 約 0.93 ~ 0.94)，後續作了許多嘗試 (對影像作Blur, Shapern 或 Morphology 處理、換 model 等) 不過都無法取得突破。

在嘗試過程中接觸到 `CAM (Class Activation Mapping)` 這個實作，CAM 可以用來 `幫助我們理解圖片/影像中那個部份是讓 CNN 作出最終決策的依據`。
CAM 最早由 Bolei Zhou 等人於2016年在 [Learning Deep Features for Discriminative Localization](https://arxiv.org/abs/1512.04150) 論文中提出，2017年Selvaraju et al.等人則提出改良版本 [Grad-CAM](https://arxiv.org/abs/1610.02391) 以克服 CAM 在 CNN 結構上的限制。

Grad-CAM 的概念是透過反向傳播 (Back Propagation) 推算各類別對圖片中各部位的梯度值(Gradient) 作為權重，以繪製類別的heatmap。
想對 Grad-CAM 有多一點了解的可參考這篇 [文章](https://medium.com/%E6%89%8B%E5%AF%AB%E7%AD%86%E8%A8%98/grad-cam-introduction-d0e48eb64adb)。

我想 CAM 可以提供兩個很有價值的用途:
1. 藉 CAM 作為 debug 依據，找出對模型有幫助之圖片處理方式 (因為看完heatmap結果後依然沒什麼方向，目前沒有嘗試；若看完本文有什麼idea歡迎分享)。
2. 在實際應用上，亦提供 heatmap 告知使用者 model 決策的部位，以協助使用者決定是否接受 model 的判斷。

以下展示 Fine-tune ResNet 模型，並套用 CAM 產生 TP / FP / TN / FN 之 heatmap 觀察模型的決策依據。
"""
!rm ./*.hdf5
import numpy as np # linear algebra
import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)
import seaborn as sns
import matplotlib.pyplot as plt
import matplotlib.image as mimg
import seaborn as sns
%matplotlib inline
from sklearn.metrics import confusion_matrix

import cv2
import os
import glob
"""
### 資料前處理
"""
# Input data files are available in the "../input/" directory.
INPUT_PATH = "../input/pneumonia-detection/chest_xray"

# List the files in the input directory.
print(os.listdir(INPUT_PATH))
base_dir = INPUT_PATH
train_dir = os.path.join(base_dir, 'train')
val_dir = os.path.join(base_dir, 'val')
test_dir = os.path.join(base_dir, 'test')

train_0_dir = os.path.join(train_dir, 'Normal'.upper())
train_1_dir = os.path.join(train_dir, 'Pneumonia'.upper())

val_0_dir = os.path.join(val_dir, 'Normal'.upper())
val_1_dir = os.path.join(val_dir, 'Pneumonia'.upper())

test_0_dir = os.path.join(test_dir, 'Normal'.upper())
test_1_dir = os.path.join(test_dir, 'Pneumonia'.upper())

def get_data_list():
    train_0_list = [os.path.join(train_0_dir, fn) for fn in os.listdir(train_0_dir)]
    train_1_list = [os.path.join(train_1_dir, fn) for fn in os.listdir(train_1_dir)]
    val_0_list = [os.path.join(val_0_dir, fn) for fn in os.listdir(val_0_dir)]
    val_1_list = [os.path.join(val_1_dir, fn) for fn in os.listdir(val_1_dir)]
    test_0_list = [os.path.join(test_0_dir, fn) for fn in os.listdir(test_0_dir)]
    test_1_list = [os.path.join(test_1_dir, fn) for fn in os.listdir(test_1_dir)]

    # list dir numbers
    print('total picture numbers in train_0_dir: ', len(train_0_list))
    print('total picture numbers in train_1_dir: ', len(train_1_list))
    print('total picture numbers in val_0_dir: ', len(val_0_list))
    print('total picture numbers in val_1_dir: ', len(val_1_list))
    print('total picture numbers in test_0_dir: ', len(test_0_list))
    print('total picture numbers in test_1_dir: ', len(test_1_list))

    return (train_0_list, train_1_list, val_0_list, val_1_list, test_0_list, test_1_list)
(train_0_list, train_1_list, val_0_list, val_1_list, test_0_list, test_1_list) = get_data_list()
"""
###### Move images from train to val list to increase validation set

我們觀察到 `調整 validation set 中 0 (mv_cnt_0) 與 1 (mv_cnt_1) 的比例對模型的預測能力會產生影響` ，大致上來說 (mv_cnt_0, mv_cnt_1) 為 (300, 300) 時 F1 score 表現較佳，變更為 (200, 400) 時會有更好的 Recall；這也說明了 Validation set 及 Validation 策略設計的重要性。
"""
import random 
(mv_cnt_0, mv_cnt_1) = (300, 300)

if len(val_0_list) < mv_cnt_0:
    mv_list_0 = random.sample(train_0_list, mv_cnt_0)
    mv_list_1 = random.sample(train_1_list, mv_cnt_1)
    train_0_list = [fn for fn in train_0_list if not fn in mv_list_0]
    train_1_list = [fn for fn in train_1_list if not fn in mv_list_1]
    val_0_list += mv_list_0
    val_1_list += mv_list_1
    
    print('total picture numbers in train_0_dir: ', len(train_0_list))
    print('total picture numbers in train_1_dir: ', len(train_1_list))
    print('total picture numbers in val_0_dir: ', len(val_0_list))
    print('total picture numbers in val_1_dir: ', len(val_1_list))
    print('total picture numbers in test_0_dir: ', len(test_0_list))
    print('total picture numbers in test_1_dir: ', len(test_1_list))
"""
Preprocess images with the following operations and create dataset
* resize to 224*224
* Only capture (y1, y2, x1, x2) = (top,top+200, left,left+200) (left, top) = (15, 40) to focus on lung part only
"""
(left, top) = (15, 40)
(y1, y2, x1, x2) = (top,top+200, left,left+200)
def image_resize(img_path):
    # print(dataset.shape)
    
    im = cv2.imread(img_path)
    im = cv2.resize(im, (224,224))
    if im.shape[2] == 1:
        # np.dstack(): Stack arrays in sequence depth-wise (along third axis).
        # https://docs.scipy.org/doc/numpy/reference/generated/numpy.dstack.html
        im = np.dstack([im, im, im])
        
        # ----------------------------------------------------------------------------------------
        # cv2.cvtColor(): The function converts an input image from one color space to another. 
        # [Ref.1]: "cvtColor - OpenCV Documentation"
        #     - https://docs.opencv.org/2.4/modules/imgproc/doc/miscellaneous_transformations.html
        # [Ref.2]: "Python计算机视觉编程- 第十章 OpenCV" 
        #     - https://yongyuan.name/pcvwithpython/chapter10.html
        # ----------------------------------------------------------------------------------------
    x_image = cv2.cvtColor(im, cv2.COLOR_BGR2RGB)
    x_image = x_image[y1:y2, x1:x2]
    x_image = cv2.resize(x_image, (150,150))
    # Normalization
    # x_image = x_image.astype(np.float32)/255.
    return x_image
"""
======== Show pictures after being resized=====
"""
import matplotlib.pyplot as plt
import matplotlib.image as mimg
%matplotlib inline
import cv2
import numpy as np
fn_list_0 = train_0_list[:4]
fn_list_1 = train_1_list[:4]

fig, ax = plt.subplots(2, 4, figsize=(20,10))
for i, axi in enumerate(ax.flat):
    img_path = None
    if i < 4:
        img_path = fn_list_0[i]
    else:
        img_path = fn_list_1[i-4]
    img = image_resize(img_path)#.astype(np.uint8)
    axi.imshow(img, cmap='bone')
    axi.set_title(img_path.split('/')[-1])
    axi.set(xticks=[], yticks=[])
"""
Make datasets for train, val and test 
"""
def create_dataset(img_path_list_0, img_path_list_1, return_fn = False):
    # list of the paths of all the image files
    normal = img_path_list_0
    pneumonia = img_path_list_1

    # --------------------------------------------------------------
    # Data-paths' format in (img_path, label) 
    # labels : for [ Normal cases = 0 ] & [ Pneumonia cases = 1 ]
    # --------------------------------------------------------------
    normal_data = [(image, 0) for image in normal]
    pneumonia_data = [(image, 1) for image in pneumonia]

    image_data = normal_data + pneumonia_data

    # Get a pandas dataframe for the data paths 
    image_data = pd.DataFrame(image_data, columns=['image', 'label'])
#     print(image_data.head(5))
    # Shuffle the data 
    image_data = image_data.sample(frac=1., random_state=100).reset_index(drop=True)
    
    # Importing both image & label datasets...
    (x_images, y_labels) = ([image_resize(image_data.iloc[i][0]) for i in range(len(image_data))], 
                         [image_data.iloc[i][1] for i in range(len(image_data))])

    # Convert the list into numpy arrays
    x_images = np.array(x_images)
    y_labels = np.array(y_labels)
    
    print("Total number of images: ", x_images.shape)
    print("Total number of labels: ", y_labels.shape)
    
    if not return_fn:
        return (x_images, y_labels)
    else:
        return (x_images, y_labels, image_data.image.values)
# Import train dataset...
(x_train, y_train) = create_dataset(train_0_list, train_1_list)

print(x_train.shape)
print(y_train.shape)
# Import val dataset...
(x_val, y_val) = create_dataset(val_0_list, val_1_list)
"""
Apply ResNet50 module of keras for training & prediction

在此我們 `使用 ResNet50 在 Keras 的實作 (使用 TensorFlow 1.x 版本為 backend) 來訓練`；事實上 ResNet50 在訓練過程中的數值並不是很正常，但跑來結果不比 VGG16 差，且本篇重點在 CAM，看倌若有興趣可以 folk 此 notebook 過去改成VGG16試試看。
"""
from keras.applications.resnet50 import ResNet50
from keras.applications.vgg16 import VGG16
from keras.applications.inception_resnet_v2 import InceptionResNetV2
# base_model = ResNet50(weights='../input/keras-pretrained-models/resnet50_weights_tf_dim_ordering_tf_kernels_notop.h5', input_shape=(150, 150, 3), include_top = False, pooling = 'avg')
# base_model = VGG16(weights='../input/keras-pretrained-models/vgg16_weights_tf_dim_ordering_tf_kernels_notop.h5', input_shape=(150, 150, 3), include_top = False)
base_model = InceptionResNetV2(weights='../input/keras-pretrained-models/inception_resnet_v2_weights_tf_dim_ordering_tf_kernels_notop.h5', 
                   input_shape=(150, 150, 3), include_top = False)
base_model.summary()
# create data generator (without data augment)
from keras.preprocessing.image import ImageDataGenerator
import numpy as np
import keras.backend as K

# rescale all image by 1/255 
data_batch_size = 20

def get_f1(y_true, y_pred): #taken from old keras source code
    true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))
    possible_positives = K.sum(K.round(K.clip(y_true, 0, 1)))
    predicted_positives = K.sum(K.round(K.clip(y_pred, 0, 1)))
    precision = true_positives / (predicted_positives + K.epsilon())
    recall = true_positives / (possible_positives + K.epsilon())
    f1_val = 2*(precision*recall)/(precision+recall+K.epsilon())
    return f1_val
from sklearn import metrics
from sklearn.metrics import confusion_matrix
import seaborn as sns

def get_pred_score(y_true, y_pred):
    mat = confusion_matrix(y_true, y_pred)
    print(mat)

    plt.figure(figsize=(8,6))
    sns.heatmap(mat, square=False, annot=True, fmt ='d', cbar=True, annot_kws={"size": 16})
    plt.title('0 : Normal   1 : Pneumonia', fontsize = 20)
    plt.xticks(fontsize = 16)
    plt.yticks(fontsize = 16)
    plt.xlabel('predicted value', fontsize = 20)
    plt.ylabel('true value', fontsize = 20)
    plt.show()

    tn, fp, fn, tp = mat.ravel()
    print('\ntn = {}, fp = {}, fn = {}, tp = {} '.format(tn, fp, fn, tp))

    precision = tp/(tp+fp)
    recall = tp/(tp+fn)
    accuracy = (tp+tn)/(tp+tn+fp+fn)
    f1_score = 2. * precision * recall / (precision + recall)
    f2_score = 5. * precision * recall / (4. * precision + recall)

    print("Test Recall of the model \t = {:.4f}".format(recall))
    print("Test Precision of the model \t = {:.4f}".format(precision))
    print("Test Accuracy of the model \t = {:.4f}".format(accuracy))
    print("Test F1 score of the model \t = {:.4f}".format(f1_score))
    print("Test F2 score of the model \t = {:.4f}".format(f2_score))
from keras import layers, models, Model
from keras import optimizers
from keras.callbacks import ModelCheckpoint, EarlyStopping
# Import train dataset...
(x_test, y_test, test_fns) = create_dataset(test_0_list, test_1_list, return_fn=True)
"""
### Add dense layer on top of conv_base 

與 [Pneumonia Detection by VGG16](https://www.kaggle.com/whitelee/pneumonia-detection-by-vgg16) 的實作不同，在此我們使用 `函數式的寫法來建立 Model` ，而非將 base_model 鑲在 Sequential 中；如此一來後續在 CAM 實作中較容易找到 last_conv_layer 。
"""
x = base_model.output
# x = layers.GlobalAveragePooling2D()(x)
x = layers.Flatten()(x)
x = layers.Dense(512, activation = 'relu')(x)
x = layers.Dropout(0.5)(x)
x = layers.Dense(1, activation='sigmoid')(x)
model = Model(base_model.input,x)
"""
若設定 trainable 為 False， 則 Fine-tune 參數為 0 個，這是比較奇怪的地方，因此我們不更改trainable設定，維持 492 個參數；但 492 個參數使得每個 epoch 的訓練時間長達 42 秒 (相較 VGG16 每個 epoch 為19秒)；若使用 TF 2.0 為 backend 則在設定 trainable 後 Fine-tune 參數變成 4 個，訓練時間縮減，但預測能力下降許多。
"""
# # Freezing a layer or set of layers means preventing their weights from being updated during training.
# freezing_layer = None
# model.trainable = False
print('This is the number of trainable weights: ', len(model.trainable_weights))
# for layer in reversed(model.layers):
#     # check to see if the layer has a 4D output
#     if len(layer.output_shape) == 4:
#         freezing_layer = layer.name
#         break
# print('Freezing layer = ', freezing_layer)
# set_trainable = False
# for layer in model.layers:
#     if layer.name == freezing_layer:
#         set_trainable = True
#       # set trainable = True for layers after block5_conv1
#     if set_trainable:
#         layer.trainable = True
#         print(layer.name)
#     else:
#         layer.trainable = False
# print('This is the number of trainable weights after freezing the conv base:', len(model.trainable_weights))
model.summary()
# show trainable weights
# [x.name for x in model.trainable_weights]
# use ImageGenerator to generate more training data
train_datagen = ImageDataGenerator(
    rescale=1./255,  # Rescales all images by 1/255
    rotation_range = 10,
    width_shift_range = 0.2, height_shift_range = 0.2,
    fill_mode = 'nearest', shear_range = 0.2,
    zoom_range = 0.2, horizontal_flip=False, 
)
train_datagen.fit(x_train)
val_datagen = ImageDataGenerator(rescale=1./255) #validation set no need to augment
val_datagen.fit(x_val)

train_generator = train_datagen.flow(x_train, y_train, batch_size=32) #increase batch size to 32
val_generator = val_datagen.flow(x_val, y_val, batch_size=32) #increase batch size to 32
default_lr = 1e-4 
adp_optimizer = optimizers.RMSprop(lr=default_lr, rho=0.9, epsilon=1e-08, decay=0.0)
model.compile(optimizer=adp_optimizer, loss="binary_crossentropy", metrics=["accuracy", get_f1])
"""
設定 call back 保留最佳 val_loss 之權重值
"""
# Define a checkpoint callback for method2:
checkpoint_name = 'Weights-m2-{epoch:03d}--{val_loss:.5f}.hdf5'
checkpoint2 = ModelCheckpoint(checkpoint_name, monitor='val_loss', verbose = 1, save_best_only = True, mode ='auto')
es = EarlyStopping(monitor='val_loss', patience=5)
callbacks_list2 = [checkpoint2]
"""
訓練期間可看到最佳的 val_loss 降為 0，或許是有overfit的現象發生... 不過發生在 validation set 上是比較少見；讀者有興趣可以調大 validation set 的總量跑跑看。
"""
history = model.fit_generator(train_generator, steps_per_epoch=100, epochs=30, validation_data=val_generator, validation_steps=20, callbacks=callbacks_list2)
import matplotlib.pyplot as plt


acc = history.history['accuracy']
val_acc = history.history['val_accuracy']
epochs = range(1, len(acc)+1)
loss = history.history['loss']
val_loss = history.history['val_loss']
f1 = history.history['get_f1']
val_f1 = history.history['val_get_f1']

plt.plot(epochs, acc, 'bo', label='Train Acc')
plt.plot(epochs, val_acc, 'b', label='Validation Acc')
plt.title('Accuracy')
plt.legend()
plt.figure()
plt.plot(epochs, f1, 'bo', label='Train F1')
plt.plot(epochs, val_f1, 'b', label='Validation F1')
plt.title('F1 score')
plt.legend()
plt.figure()
plt.plot(epochs, loss, 'bo', label='Train Loss')
plt.plot(epochs, val_loss, 'b', label='Validation Loss')
plt.title('Loss')
plt.legend()
plt.figure()

plt.show()
"""
==== Prepare test data for prediction ===
"""
test_data = []
test_labels = []
for (test_img, label) in zip(x_test, y_test):
    test_data.append(test_img.astype(np.float32)/255)
    test_labels.append(label)

test_data = np.array(test_data)
test_labels = np.array(test_labels)

print("Total number of test examples: ", test_data.shape)
print("Total number of labels:", test_labels.shape)
def predict(model, test_data):
    pred_prob = model.predict(test_data, batch_size=data_batch_size)
    pred_res = np.asarray([1 if x > 0.5 else 0 for x in [x[0] for x in pred_prob]]) 
    return (pred_res, [x[0] for x in pred_prob])
# Load best weight of model
from pathlib import Path
w_fnl = [str(fn) for fn in Path('./').glob('Weights-m2-*.hdf5')]
w_fnl.sort(reverse=True)
wights_file = w_fnl[0] # choose the best checkpoint 
model.load_weights(wights_file) # load it
model.compile(optimizer=adp_optimizer, loss="binary_crossentropy", metrics=["accuracy", get_f1])
(y_pred, y_pred_prob) = predict(model, test_data)
get_pred_score(test_labels, y_pred)
"""
### Extract false case for advanced analysis

訓練完 Model 以後，取出 TN, TP, FN, FP 的 case 來套用 CAM ，觀察其 heatmap 狀況。
"""
pred_result = pd.DataFrame({'imgPath': test_fns, 'label': test_labels, 'pred': y_pred, 'pred_prob': y_pred_prob})
pred_result['fn'] = pred_result.imgPath.apply(lambda ip: ip.split('/')[-1])
pred_result.head()
false_result = pred_result[pred_result.pred != pred_result.label]
true_0_fns = pred_result[(pred_result.pred == pred_result.label) & (pred_result.label == 0)].fn.values
true_1_fns = pred_result[(pred_result.pred == pred_result.label) & (pred_result.label == 1)].fn.values
false_0_fns = false_result[false_result.label == 0].fn.values
false_1_fns = false_result[false_result.label == 1].fn.values
print(true_0_fns.shape, true_1_fns.shape, false_0_fns.shape, false_1_fns.shape)
"""
### Apply CAM to one FN case
"""
# image_path = os.path.join(test_0_dir, false_0_fns[0])
fn = false_0_fns[1]
folder = test_0_dir
image_path = os.path.join(folder, fn)
print(image_path)
# img = image.load_img(image_path, target_size=(224, 224))
img = image_resize(image_path)
img = img.astype(np.float32)/255
img_x = np.expand_dims(img, axis=0)
img_x.shape
"""
### Grad-CAM

此 Grad-CAM 的實作是參考 [這篇文章](https://www.pyimagesearch.com/2020/03/09/grad-cam-visualize-class-activation-maps-with-keras-tensorflow-and-deep-learning/) 的；主要差別在於該文以 TF 2.0 為 backend，而我們使用 TF 1.x。

看倌們若是 Google `Deeplearning CAM` 會看到有許多不同的 CAM 實作方式，但原理大同小異，`都是計算某個類別(通常是機率最大的類別)的最終 output 關於 model 中最後一個 conv2d 層之梯度值作為圖像各部位的權重` 來繪制heatmap；差別在於如何對heatmap作normalize使其更能突顯出重要的決策部位。
"""
from keras.models import Model
import tensorflow as tf
import keras.backend as K

class GradCAM:
    def __init__(self, model, classIdx=0, layerName=None):
        # store the model, the class index used to measure the class
        # activation map, and the layer to be used when visualizing
        # the class activation map
        self.model = model
        self.classIdx = classIdx
        self.layerName = layerName
        self.sess = tf.compat.v1.Session()
        # if the layer name is None, attempt to automatically find
        # the target output layer
        if self.layerName is None:
            self.layerName = self.find_target_layer()

    def find_target_layer(self):
        # 尋找最後一層Conv layer
        # attempt to find the final convolutional layer in the network
        # by looping over the layers of the network in reverse order
        for layer in reversed(self.model.layers):
            # check to see if the layer has a 4D output
            if len(layer.output_shape) == 4:
                # model中出現的最後一個輸出維度為4的層即尋找目標
                return layer.name
        # otherwise, we could not find a 4D layer so the GradCAM
        # algorithm cannot be applied
        raise ValueError("Could not find 4D layer. Cannot apply GradCAM.")

    def compute_heatmap(self, image, eps=1e-8):
        # construct our gradient model by supplying (1) the inputs
        # to our pre-trained model, (2) the output of the (presumably)
        # final 4D layer in the network, and (3) the output of the
        # softmax activations from the model
#         gradModel = Model(
#             inputs=[self.model.inputs],
#             outputs=[self.model.get_layer(self.layerName).output,
#                 self.model.output])
            # record operations for automatic differentiation
        
        pred = self.model.predict(image)
        predictions = self.model.output
        # model 對輸入作完預測後取出要繪制heatmap的類別，因我們的model為二元分類，故classIdx固定為0
        loss = predictions[:, self.classIdx]
        convOutputs = self.model.get_layer(self.layerName).output

        # 用 gradients 函式計算梯度值作為後面畫heatmap的權重
        # use automatic differentiation to compute the gradients
        grads = K.gradients(loss, convOutputs)[0]

        pooled_grads = K.mean(grads, axis=(0, 1, 2))
        iterate = K.function([self.model.input],[pooled_grads, convOutputs[0]])
        (pooled_grads_value, conv_layer_output_value) = iterate([image])
        for i in range(512):
            # 對conv layer的輸出乘上權重以作為繪制heatmap的raw data
            conv_layer_output_value[:, :, i] *= pooled_grads_value[i]
        
        heatmap = np.mean(conv_layer_output_value, axis=-1)
        # grab the spatial dimensions of the input image and resize
        # the output class activation map to match the input image
        # dimensions
        (w, h) = (image.shape[2], image.shape[1])
        # heatmap = cv2.resize(cam.numpy(), (w, h))
        heatmap = cv2.resize(heatmap, (w, h))
        # 正規化 heatmap的raw data使值落在0~255之間(image data的合理範圍)
        # normalize the heatmap such that all values lie in the range
        # [0, 1], scale the resulting values to the range [0, 255],
        # and then convert to an unsigned 8-bit integer
        numer = heatmap - np.min(heatmap)
        denom = (heatmap.max() - heatmap.min()) + eps
        heatmap = numer / denom
        heatmap = (heatmap * 255).astype("uint8")
        # return the resulting heatmap to the calling function
        return (heatmap, pred)

    def overlay_heatmap(self, heatmap, image, alpha=0.5,
        colormap=cv2.COLORMAP_VIRIDIS):
        # apply the supplied color map to the heatmap and then
        # overlay the heatmap on the input image
        heatmap = cv2.applyColorMap(heatmap, colormap)
        output = cv2.addWeighted(image, alpha, heatmap, 1 - alpha, 0)
        # return a 2-tuple of the color mapped heatmap and the output,
        # overlaid image
        return (heatmap, output)
"""
接下來先嘗試對單張輸入繪制heatmap，觀察繪制及與原圖之疊合效果
"""
cam = GradCAM(model)
(heatmap, _) = cam.compute_heatmap(img_x)
img = image_resize(image_path)
print(heatmap.shape, type(heatmap))
print(img[:, :, 0].shape, type(img[:, :, 0]))
"""
將 heatmap 與原始圖型疊合，alpha 值使用0.2 (越高的 alpha 值會讓原圖越清晰，heatmap越模糊)
"""
(heatmap, output) = cam.overlay_heatmap(np.dstack([heatmap, heatmap, heatmap]), img, alpha=0.2)
plt.imshow(heatmap) # show heatmap
"""
我們使用 VIRIDIS 色階來繪制 Heatmap，越亮處(金黃色)表示權重值越高

![CV2中的常用色階](https://www.pyimagesearch.com/wp-content/uploads/2020/03/keras_gradcam_colormap.png)
圖片出處：https://www.pyimagesearch.com/wp-content/uploads/2020/03/keras_gradcam_colormap.png
"""
plt.imshow(output) # 疊合圖
"""
由疊合圖可以看出金黃色並未很好的涵蓋肺部區域，很有可能因沒有涵蓋到病灶而誤判
"""
"""
### List heatmaps of TP / TN / FP / FN cases 

各 case 取出 15 張原始圖來繪制 heatmap，觀察判斷錯誤之 case 是否是因 model 用了錯誤的部份作決策。
"""
# Create global CAM class
cam = GradCAM(model)
def get_heatmap_with_pic(base_dir, fn, cam_model=cam):
#     print(fn)
    image_path = os.path.join(base_dir, fn)
    title1 = fn
    img = image_resize(image_path)
    img_nz = img.astype(np.float32)/255
    img_x = np.expand_dims(img_nz, axis=0)
    (heatmap, pred_prob) = cam_model.compute_heatmap(img_x)
    (heatmap, output) = cam_model.overlay_heatmap(np.dstack([heatmap, heatmap, heatmap]), img, alpha=0.2)
    title2 = 'pred_prob: {}'.format(pred_prob)
    title3 = 'combined'
    return (img, title1, heatmap, title2, output, title3)
# 使用 subplots 將原始圖、heatmap及疊合圖並排呈現
def draw_heatmap_on_plt(base_dir, fn_list, cat='tp', cam_model=cam):
    # Draw heatmap for true cases
    pics_per_row = 2
    cols = 3 * pics_per_row
    rows = int(len(fn_list) / pics_per_row)
    fig, ax = plt.subplots(rows, cols, figsize=(cols*5,rows*5))
    for i, axi in enumerate(ax.flat):
        title = ''
        show_img = None
        idx = i % 3
        if idx == 0:
            (img, title1, heatmap, title2, superimposed_img, title3) = get_heatmap_with_pic(base_dir, fn_list[int(i / 3)])
            show_img = img
            title = '{} ({})'.format(title1, cat)
        elif idx == 1:
            show_img = heatmap
            title = title2
        else:
            show_img = superimposed_img
            title = title3
        axi.imshow(show_img, cmap='bone')
        axi.set_title(title)
        axi.set(xticks=[], yticks=[])
import datetime
disp_num = 15 #每個類別取15張原圖
pic_sec = 20 # 畫一張 heatmap 約要20秒(以kaggle的notebook規格)
disp_cnt = min(disp_num, len(true_0_fns))
print('Will take around {} secs from {}'.format(disp_cnt*pic_sec, str(datetime.datetime.now())))
draw_heatmap_on_plt(test_0_dir, true_0_fns[:disp_cnt], 'tn')
disp_cnt = min(disp_num, len(true_1_fns))
print('Will take around {} secs from {}'.format(disp_cnt*pic_sec, str(datetime.datetime.now())))
draw_heatmap_on_plt(test_1_dir, true_1_fns[:disp_cnt], 'tp')
"""
1. 由TN及TP的圖形可以觀察到理想的 heatmap 亮處應由圖片中間呈圓形展開，或是在中間形成柱狀，形成中間亮兩邊暗圖形。
1. 事實上模型對部份 TP case 的決策區域也不是很理想(雖亮處由中間展開但亦有涵蓋不足情況)，但因Test set 中 positive比例也較高，也不排除有歪打正著的現象。
"""
disp_cnt = min(disp_num, len(false_0_fns))
print('Will take around {} secs from {}'.format(disp_cnt*pic_sec, str(datetime.datetime.now())))
draw_heatmap_on_plt(test_0_dir, false_0_fns[:disp_cnt], 'fn')
disp_cnt = min(disp_num, len(false_1_fns))
print('Will take around {} secs from {}'.format(disp_cnt*pic_sec, str(datetime.datetime.now())))
draw_heatmap_on_plt(test_1_dir, false_1_fns[:disp_cnt], 'fp')
"""
1. 由 FN 及 FP 圖形的 heatmap 可以看出多半是亮處偏在角落或是明顯遺漏肺部某區域，當使用者看到這樣的heatmap或許應該考慮是否接受模型判斷結果
1. 有些 heatmap 涵蓋良好但仍然誤判的 case 或許應再檢視原圖檔看是否有需要調整label(判定結果)
"""
"""
### 結語

由以上 CASE 的 heatmap 觀察，大多數的CASE的亮區都還合理(自認為)，當然涵蓋度是否足夠可能要加入Domain專家的判斷才比較準確。但我們相信在整個資料分析過程的各個部份都可由 CAM 獲得幫助；Heatmap 提供了可視化的決策資訊讓開發人員可以對模型的效能有另一種面向的觀察(例如模型是真的判斷出來的還是猜對的)，使用者也可由亮區知道模型判斷依據是否合理，甚至可以作為回頭檢視資料(input)品質的參考。

以上是小弟這陣子的學習分享，各位看倌若有任何想法或建議也歡迎提出討論～
"""