first commit

app.py

@@ -0,0 +1,368 @@
+import gradio as gr
+import matplotlib.pyplot as plt
+import numpy as np
+from PIL import Image
+
+import cv2
+import matplotlib
+
+import clip
+
+from utils import *
+
+matplotlib.get_cachedir()
+plt.rc('font', family="Times New Roman")
+from sklearn import metrics
+
+import torch
+from torchvision import transforms
+
+from models.submodular_vit_efficient_plus import MultiModalSubModularExplanationEfficientPlus
+
+data_transform = transforms.Compose(
+    [
+        transforms.Resize(
+            (224,224), interpolation=transforms.InterpolationMode.BICUBIC
+        ),
+        # transforms.CenterCrop(224),
+        transforms.ToTensor(),
+        transforms.Normalize(
+            mean=(0.48145466, 0.4578275, 0.40821073),
+            std=(0.26862954, 0.26130258, 0.27577711),
+        ),
+    ]
+)
+
+class CLIPModel_Super(torch.nn.Module):
+    def __init__(self, 
+                 type="ViT-L/14", 
+                 download_root=None,
+                 device = "cuda"):
+        super().__init__()
+        self.device = device
+        self.model, _ = clip.load(type, device=self.device, download_root=download_root)
+        
+        self.model = self.model.float()
+        
+    def forward(self, vision_inputs):
+        """
+        Input:
+            vision_inputs: torch.size([B,C,W,H])
+        Output:
+            embeddings: a d-dimensional vector torch.size([B,d])
+        """
+        vision_inputs = vision_inputs.type(torch.float32)
+        
+        with torch.no_grad():
+            image_features = self.model.encode_image(vision_inputs)
+            image_features /= image_features.norm(dim=-1, keepdim=True)
+        
+        return image_features
+    
+def transform_vision_data(image):
+    """
+    Input:
+        image: An image read by opencv [w,h,c]
+    Output:
+        image: After preproccessing, is a tensor [c,w,h]
+    """
+    image = Image.fromarray(image)
+    image = data_transform(image)
+    return image
+
+device = "cuda" if torch.cuda.is_available() else "cpu"
+# Instantiate model
+vis_model = CLIPModel_Super("ViT-L/14", device=device, download_root="./ckpt")
+vis_model.eval()
+vis_model.to(device)
+print("load clip model")
+
+semantic_path = "./clip_vitl_imagenet_zeroweights.pt"
+if os.path.exists(semantic_path):
+    semantic_feature = torch.load(semantic_path, map_location="cpu")
+    semantic_feature = semantic_feature.to(device)
+    semantic_feature = semantic_feature.type(torch.float32)
+
+explainer = MultiModalSubModularExplanationEfficientPlus(
+        vis_model, semantic_feature, transform_vision_data, device=device, 
+        lambda1=0.01, 
+        lambda2=0.05, 
+        lambda3=20., 
+        lambda4=5.)
+
+def add_value_decrease(smdl_mask, json_file):
+    single_mask = np.zeros_like(smdl_mask[0].mean(-1))
+    
+    value_list_1 = np.array(json_file["consistency_score"]) + np.array(json_file["collaboration_score"])
+    
+    value_list_2 = np.array([json_file["baseline_score"]] + json_file["consistency_score"][:-1]) + np.array([1 - json_file["org_score"]] + json_file["collaboration_score"][:-1])
+    
+    value_list = value_list_1 - value_list_2
+    
+    values = []
+    value = 0
+    for smdl_single_mask, smdl_value in zip(smdl_mask, value_list):
+        value = value - abs(smdl_value)
+        single_mask[smdl_single_mask.sum(-1)>0] = value
+        values.append(value)
+    
+    attribution_map = single_mask - single_mask.min()
+    attribution_map /= attribution_map.max()
+    
+    return attribution_map, np.array(values)
+
+def visualization(image, submodular_image_set, saved_json_file, vis_image, index=None, compute_params=True):
+    
+    insertion_ours_images = []
+    # deletion_ours_images = []
+
+    insertion_image = submodular_image_set[0] - submodular_image_set[0]
+    insertion_ours_images.append(insertion_image)
+    # deletion_ours_images.append(image - insertion_image)
+    for smdl_sub_mask in submodular_image_set[:]:
+        insertion_image = insertion_image.copy() + smdl_sub_mask
+        insertion_ours_images.append(insertion_image)
+        # deletion_ours_images.append(image - insertion_image)
+
+    insertion_ours_images_input_results = np.array([1-saved_json_file["collaboration_score"][-1]] + saved_json_file["consistency_score"])
+
+    if index == None:
+        ours_best_index = np.argmax(insertion_ours_images_input_results)
+    else:
+        ours_best_index = index
+    x = [(insertion_ours_image.sum(-1)!=0).sum() / (image.shape[0] * image.shape[1]) for insertion_ours_image in insertion_ours_images]
+    i = len(x)
+
+    fig, [ax1, ax2, ax3] = plt.subplots(1,3, gridspec_kw = {'width_ratios':[1, 1, 1.5]}, figsize=(30,8))
+    ax1.spines["left"].set_visible(False)
+    ax1.spines["right"].set_visible(False)
+    ax1.spines["top"].set_visible(False)
+    ax1.spines["bottom"].set_visible(False)
+    ax1.xaxis.set_visible(False)
+    ax1.yaxis.set_visible(False)
+    ax1.set_title('Attribution Map', fontsize=54)
+    ax1.set_facecolor('white')
+    ax1.imshow(vis_image.astype(np.uint8))
+    
+    ax2.spines["left"].set_visible(False)
+    ax2.spines["right"].set_visible(False)
+    ax2.spines["top"].set_visible(False)
+    ax2.spines["bottom"].set_visible(False)
+    ax2.xaxis.set_visible(True)
+    ax2.yaxis.set_visible(False)
+    ax2.set_title('Searched Region', fontsize=54)
+    ax2.set_facecolor('white')
+    ax2.set_xlabel("Highest conf. {:.4f}".format(insertion_ours_images_input_results.max()), fontsize=44)
+    ax2.tick_params(axis='x', which='both', bottom=False, top=False, labelbottom=False)
+
+    ax3.set_xlim((0, 1))
+    ax3.set_ylim((0, 1))
+    
+    ax3.set_ylabel('Recognition Score', fontsize=44)
+    ax3.set_xlabel('Percentage of image revealed', fontsize=44)
+    ax3.tick_params(axis='both', which='major', labelsize=36)
+
+    x_ = x[:i]
+    ours_y = insertion_ours_images_input_results[:i]
+    ax3.plot(x_, ours_y, color='dodgerblue', linewidth=3.5)  # draw curve
+    ax3.set_facecolor('white')
+    ax3.spines['bottom'].set_color('black')
+    ax3.spines['bottom'].set_linewidth(2.0)
+    ax3.spines['top'].set_color('none')
+    ax3.spines['left'].set_color('black')
+    ax3.spines['left'].set_linewidth(2.0)
+    ax3.spines['right'].set_color('none')
+
+    # plt.legend(["Ours"], fontsize=40, loc="upper left")
+    ax3.scatter(x_[-1], ours_y[-1], color='dodgerblue', s=54)  # Plot latest point
+    # 在曲线下方填充淡蓝色
+    ax3.fill_between(x_, ours_y, color='dodgerblue', alpha=0.1)
+
+    kernel = np.ones((3, 3), dtype=np.uint8)
+    # ax3.plot([x_[ours_best_index], x_[ours_best_index]], [0, 1], color='red', linewidth=3.5)  # 绘制红色曲线
+    ax3.axvline(x=x_[int(ours_best_index)], color='red', linewidth=3.5)  # 绘制红色垂直线
+
+    # Ours
+    mask = (image - insertion_ours_images[int(ours_best_index)]).mean(-1)
+    mask[mask>0] = 1
+
+    if int(ours_best_index) != 0:
+        dilate = cv2.dilate(mask, kernel, 3)
+        # erosion = cv2.erode(dilate, kernel, iterations=3)
+        # dilate = cv2.dilate(erosion, kernel, 2)
+        edge = dilate - mask
+        # erosion = cv2.erode(dilate, kernel, iterations=1)
+
+    image_debug = image.copy()
+
+    image_debug[mask>0] = image_debug[mask>0] * 0.5
+    if int(ours_best_index) != 0:
+        image_debug[edge>0] = np.array([255,0,0])
+    ax2.imshow(image_debug)
+    
+    if compute_params:
+        auc = metrics.auc(x, insertion_ours_images_input_results)
+    
+    ax3.set_title('Insertion Curve', fontsize=54)
+    
+    fig.canvas.draw()
+    img_curve = np.frombuffer(fig.canvas.tostring_rgb(), dtype=np.uint8)
+    img_curve = img_curve.reshape(fig.canvas.get_width_height()[::-1] + (3,))
+    
+    plt.close(fig)  # 关闭图形以释放资源
+    
+    if compute_params:
+        return img_curve, insertion_ours_images_input_results.max(), auc, ours_best_index
+    else:
+        return img_curve
+    
+def gen_cam(image, mask):
+    """
+    Generate heatmap
+        :param image: [H,W,C]
+        :param mask: [H,W],range 0-1
+        :return: tuple(cam,heatmap)
+    """
+    # Read image
+    # image = cv2.resize(cv2.imread(image_path), (224,224))
+    # mask->heatmap
+    heatmap = cv2.applyColorMap(np.uint8(mask), cv2.COLORMAP_COOL)
+    heatmap = np.float32(heatmap)
+
+    # merge heatmap to original image
+    cam = 0.5*heatmap + 0.5*np.float32(image)
+    return cam, (heatmap).astype(np.uint8)
+
+def norm_image(image):
+    """
+    Normalization image
+    :param image: [H,W,C]
+    :return:
+    """
+    image = image.copy()
+    image -= np.max(np.min(image), 0)
+    image /= np.max(image)
+    image *= 255.
+    return np.uint8(image)
+
+def read_image(file_path):
+    image = Image.open(file_path)
+    image = image.convert("RGB")
+    return np.array(image)
+
+# 使用同一个示例图像 "shark.png"
+default_images = {
+    # "Default Image": read_image("images/shark.png"),
+    "Example: Tiger Shark": read_image("images/shark.png"),
+    "Example: Quail": read_image("images/bird.png")  # 所有选项都使用相同的图片
+}
+
+def interpret_image(uploaded_image, slider, text_input):
+    # 使用上传的图像（如果有），否则使用生成的图像
+    if uploaded_image is not None:
+        image = np.array(uploaded_image)
+    else:
+        return None, 0, 0
+
+    image = cv2.resize(image, (224, 224))
+    element_sets_V = SubRegionDivision(image, mode="slico", region_size=30) 
+    
+    explainer.k = len(element_sets_V)
+    
+    global submodular_image_set
+    global saved_json_file
+    global im
+    submodular_image, submodular_image_set, saved_json_file = explainer(element_sets_V, id=None)
+    
+    attribution_map, value_list = add_value_decrease(submodular_image_set, saved_json_file)
+    im, heatmap = gen_cam(image, norm_image(attribution_map))
+    
+    image_curve, highest_confidence, insertion_auc_score, ours_best_index = visualization(image, submodular_image_set, saved_json_file, im, index=None)
+
+    text_output_class = "The method explains why the CLIP (ViT-B/16) model identifies an image as {}.".format(imagenet_classes[explainer.target_label])
+
+    return image_curve, highest_confidence, insertion_auc_score, text_output_class
+
+def visualization_slider(uploaded_image, slider):
+    # 使用上传的图像（如果有），否则使用生成的图像
+    if uploaded_image is not None:
+        image = np.array(uploaded_image)
+    else:
+        return None, 0, 0
+
+    image = cv2.resize(image, (224, 224))
+    
+    image_curve = visualization(image, submodular_image_set, saved_json_file, im, index=slider, compute_params=False)
+    
+    return image_curve
+
+def update_image(thumbnail_name):
+    # 返回对应缩略图的图像数据
+    return default_images[thumbnail_name]
+
+# 创建 Gradio 界面
+with gr.Blocks() as demo:
+    with gr.Row():
+        with gr.Column():
+            # 第一排：上传图像输入框和一个缩略图
+            with gr.Row():
+                # 上传图像输入框
+                image_input = gr.Image(label="Upload Image", type="numpy")
+                
+                # 第一个缩略图和按钮
+                with gr.Column():
+                    # gr.Image(value=default_images["Default Image"], type="numpy")
+                    # button_default = gr.Button(value="Default Image")
+                    # button_default.click(
+                    #     fn=lambda k="Default Image": update_image(k),
+                    #     inputs=[],
+                    #     outputs=image_input
+                    # )
+                    gr.Textbox("Thank you for using our interpretable attribution method, which originates from the ICLR 2024 Oral paper titled \"Less is More: Fewer Interpretable Regions via Submodular Subset Selection.\" We have now implemented this method on the multimodal ViT model and achieved promising results in explaining model predictions. A key feature of our approach is its ability to clarify the reasons behind the model's prediction errors. We invite you to try out this demo and explore its capabilities. The source code is available at https://github.com/RuoyuChen10/SMDL-Attribution.\nYou can upload an image yourself or select one from the following, then click the button Interpreting Model to get the result. The demo currently does not support selecting categories or descriptions by yourself. If you are interested, you can try it from the source code.", label="Instructions for use", interactive=False)
+            
+            # 第二排：两个缩略图
+            with gr.Row():
+                for key in default_images.keys():
+                    with gr.Column():
+                        gr.Image(value=default_images[key], type="numpy")
+                        button = gr.Button(value=key)
+                        button.click(
+                            fn=lambda k=key: update_image(k),
+                            inputs=[],
+                            outputs=image_input
+                        )
+
+            # 文本输入框和滑块
+            text_input = gr.Textbox(label="Text Input", placeholder="Enter some text here... (optional)")
+            
+            
+        with gr.Column():
+            # 输出图像和控件
+            image_output = gr.Image(label="Output Image")
+            
+            slider = gr.Slider(minimum=0, maximum=50, step=1, label="Confidence Slider")
+            
+            text_output_class = gr.Textbox(label="Explaining Category")
+            with gr.Row():
+                # 最高置信度和插入 AUC Score 并排显示
+                text_output_confidence = gr.Textbox(label="Highest Confidence")
+                text_output_auc = gr.Textbox(label="Insertion AUC Score")
+            
+            interpret_button = gr.Button("Interpreting Model")
+
+    # 定义解释模型按钮点击事件
+    interpret_button.click(
+        fn=interpret_image,
+        inputs=[image_input, slider, text_input],
+        outputs=[image_output, text_output_confidence, text_output_auc, text_output_class]
+    )
+    
+    # 实时更新的滑块
+    slider.change(
+        fn=visualization_slider,
+        inputs=[image_input, slider],
+        outputs=[image_output]
+    )
+
+# 启动 Gradio 应用
+demo.launch()

clip_vitl_imagenet_zeroweights.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a28c797d2d204c666968d76ad9fb6d4bfc9de4bc39afc591ade407623fed3a85
+size 1536865

models/submodular_vit_efficient_plus.py

@@ -0,0 +1,247 @@
+import math
+import random
+import numpy as np
+
+from tqdm import tqdm
+import cv2
+from PIL import Image
+
+import torch
+import torch.nn.functional as F
+
+from .submodular_vit_torch import MultiModalSubModularExplanation
+
+class MultiModalSubModularExplanationEfficientPlus(MultiModalSubModularExplanation):
+    def __init__(self, 
+                 model,
+                 semantic_feature,
+                 preproccessing_function,
+                 k = 40,
+                 lambda1 = 1.0,
+                 lambda2 = 1.0,
+                 lambda3 = 1.0,
+                 lambda4 = 1.0,
+                 device = "cuda",
+                 pending_samples = 8):
+        super(MultiModalSubModularExplanationEfficientPlus, self).__init__(
+            k = k,
+            model = model,
+            semantic_feature = semantic_feature,
+            preproccessing_function = preproccessing_function,
+            
+            lambda1 = lambda1,
+            lambda2 = lambda2,
+            lambda3 = lambda3,
+            lambda4 = lambda4,
+            
+            device = device)
+        
+        # Parameters of the submodular
+        self.pending_samples = pending_samples
+    
+    def evaluation_maximun_sample(self, 
+                                  main_set, 
+                                  decrease_set,
+                                  candidate_set, 
+                                  partition_image_set):
+        """
+        Given a subset, return a best sample index
+        """
+        sub_index_sets = []
+        for candidate_ in candidate_set:
+            sub_index_sets.append(
+                np.concatenate((main_set, np.array([candidate_]))).astype(int))
+            
+
+        sub_index_sets_decrease = []
+        for candidate_ in candidate_set:
+            sub_index_sets_decrease.append(
+                np.concatenate((decrease_set, np.array([candidate_]))).astype(int))
+
+        # merge images / 组合图像
+        sub_images = torch.stack([
+            self.preproccessing_function(
+                self.merge_image(sub_index_set, partition_image_set)
+            ) for sub_index_set in sub_index_sets])
+        
+        batch_input_images = sub_images.to(self.device)
+        
+        with torch.no_grad():
+            # 2. Effectiveness Score
+            score_effectiveness = self.proccess_compute_effectiveness_score(sub_index_sets)
+            score_effectiveness_decrease = self.proccess_compute_effectiveness_score(sub_index_sets_decrease)
+            
+            # 3. Consistency Score
+            score_consistency = self.proccess_compute_consistency_score(batch_input_images)
+            
+            # 1. Confidence Score
+            score_confidence = self.proccess_compute_confidence_score()
+            
+            # 4. Collaboration Score
+            sub_images_reverse = torch.stack([
+                self.preproccessing_function(
+                    self.org_img - self.merge_image(sub_index_set, partition_image_set)
+                ) for sub_index_set in sub_index_sets])
+        
+            batch_input_images_reverse = sub_images_reverse.to(self.device)
+            
+            score_collaboration = 1 - self.proccess_compute_consistency_score(batch_input_images_reverse)
+            
+            # submodular score
+            # smdl_score = self.lambda1 * score_confidence + self.lambda2 * score_effectiveness +  self.lambda3 * score_consistency + self.lambda4 * score_collaborations
+            smdl_score = self.lambda1 * score_confidence + self.lambda2 * score_effectiveness + self.lambda3 * score_consistency + self.lambda4 * score_collaboration
+            arg_max_index = smdl_score.argmax().cpu().item()
+            
+            # if self.lambda1 != 0:
+            self.saved_json_file["confidence_score_increase"].append(score_confidence[arg_max_index].cpu().item())
+            self.saved_json_file["effectiveness_score_increase"].append(score_effectiveness[arg_max_index].cpu().item())
+            self.saved_json_file["consistency_score_increase"].append(score_consistency[arg_max_index].cpu().item())
+            self.saved_json_file["collaboration_score_increase"].append(score_collaboration[arg_max_index].cpu().item())
+            self.saved_json_file["smdl_score"].append(smdl_score[arg_max_index].cpu().item())
+
+            if len(candidate_set) > self.pending_samples:
+                smdl_score_decrease = self.lambda1 * score_confidence + self.lambda2 * score_effectiveness_decrease + self.lambda3 * score_consistency + self.lambda4 * score_collaboration
+                
+                # Select the sample with the worst score as the negative sample estimate
+                negtive_sampels_indexes = smdl_score_decrease.topk(self.pending_samples, largest = False).indices.cpu().numpy()
+                
+                if arg_max_index in negtive_sampels_indexes:
+                    negtive_sampels_indexes = negtive_sampels_indexes.tolist()
+                    negtive_sampels_indexes.remove(arg_max_index)
+                    negtive_sampels_indexes = np.array(negtive_sampels_indexes)
+                
+                sub_index_negtive_sets = np.array(sub_index_sets_decrease)[negtive_sampels_indexes]
+                
+                # merge images / 组合图像
+                sub_images_decrease = torch.stack([
+                    self.preproccessing_function(
+                        self.merge_image(sub_index_set, partition_image_set)
+                    ) for sub_index_set in sub_index_negtive_sets])
+                
+                sub_images_decrease_reverse = torch.stack([
+                    self.preproccessing_function(
+                        self.org_img - self.merge_image(sub_index_set, partition_image_set)
+                    ) for sub_index_set in sub_index_negtive_sets])
+                
+                # 2. Effectiveness Score
+                score_effectiveness_decrease_ = score_effectiveness_decrease[negtive_sampels_indexes]
+                
+                # 3. Consistency Score
+                score_consistency_decrease = self.proccess_compute_consistency_score(sub_images_decrease.to(self.device))
+                
+                # 1. Confidence Score
+                score_confidence_decrease = self.proccess_compute_confidence_score()
+                
+                # 4. Collaboration Score
+                score_collaboration_decrease = 1 - self.proccess_compute_consistency_score(sub_images_decrease_reverse.to(self.device))
+                
+                smdl_score_decrease = self.lambda1 * score_confidence_decrease + self.lambda2 * score_effectiveness_decrease_ + self.lambda3 * score_consistency_decrease + self.lambda4 * score_collaboration_decrease
+                arg_min_index = smdl_score_decrease.argmin().cpu().item()
+                
+                decrease_set = sub_index_negtive_sets[arg_min_index]
+
+                self.saved_json_file["confidence_score_decrease"].append(score_confidence_decrease[arg_min_index].cpu().item())
+                self.saved_json_file["effectiveness_score_decrease"].append(score_effectiveness_decrease_[arg_min_index].cpu().item())
+                self.saved_json_file["consistency_score_decrease"].append(1-score_collaboration_decrease[arg_min_index].cpu().item())
+                self.saved_json_file["collaboration_score_decrease"].append(1-score_consistency_decrease[arg_min_index].cpu().item())
+
+            return sub_index_sets[arg_max_index], decrease_set
+    
+    def save_file_init(self):
+        self.saved_json_file = {}
+        self.saved_json_file["sub-k"] = self.k
+        
+        self.saved_json_file["confidence_score"] = []
+        self.saved_json_file["effectiveness_score"] = []
+        self.saved_json_file["consistency_score"] = []
+        self.saved_json_file["collaboration_score"] = []
+        
+        self.saved_json_file["confidence_score_increase"] = []
+        self.saved_json_file["effectiveness_score_increase"] = []
+        self.saved_json_file["consistency_score_increase"] = []
+        self.saved_json_file["collaboration_score_increase"] = []
+        
+        self.saved_json_file["confidence_score_decrease"] = []
+        self.saved_json_file["effectiveness_score_decrease"] = []
+        self.saved_json_file["consistency_score_decrease"] = []
+        self.saved_json_file["collaboration_score_decrease"] = []
+        
+        self.saved_json_file["smdl_score"] = []
+        self.saved_json_file["lambda1"] = self.lambda1
+        self.saved_json_file["lambda2"] = self.lambda2
+        self.saved_json_file["lambda3"] = self.lambda3
+        self.saved_json_file["lambda4"] = self.lambda4
+        
+    def get_merge_set(self, partition):
+        """
+        """
+        Subset = np.array([])
+        Subset_decrease = np.array([])
+        
+        indexes = np.arange(len(partition))
+        
+        # First calculate the similarity of each element to facilitate calculation of effectiveness score.
+        self.calculate_distance_of_each_element(partition)
+        
+        self.smdl_score_best = 0
+        
+        loop_times = int((self.k-self.pending_samples)/2) + self.pending_samples
+        for j in tqdm(range(loop_times)):
+            diff = np.setdiff1d(indexes, np.concatenate((Subset, Subset_decrease)))  # in indexes but not in Subset
+            
+            sub_candidate_indexes = diff
+            if len(diff) == 1:
+                Subset = np.concatenate((Subset, np.array(diff)))
+                break
+            
+            Subset, Subset_decrease = self.evaluation_maximun_sample(Subset, Subset_decrease, sub_candidate_indexes, partition)
+        
+        sub_images = torch.stack([
+            self.preproccessing_function(
+                self.org_img
+            ),
+            self.preproccessing_function(
+                self.org_img - self.org_img
+            ),
+        ])
+        scores = self.proccess_compute_consistency_score(sub_images.to(self.device))
+        
+        self.saved_json_file["org_score"] = scores[0].cpu().item()
+        self.saved_json_file["baseline_score"] = scores[1].cpu().item()
+        
+        self.saved_json_file["consistency_score"] = self.saved_json_file["consistency_score_increase"] + self.saved_json_file["consistency_score_decrease"][::-1] + [scores[0].cpu().item()]
+        self.saved_json_file["collaboration_score"] = self.saved_json_file["collaboration_score_increase"] + self.saved_json_file["collaboration_score_decrease"][::-1] + [1-scores[1].cpu().item()]
+        
+        Subset = np.concatenate((Subset, Subset_decrease[::-1]))
+        
+        return Subset.astype(int)
+    
+    def __call__(self, image_set, id = None):
+        """
+        Compute Source Face Submodular Score
+            @image_set: [mask_image 1, ..., mask_image m] (cv2 format)
+        """
+        # V_partition = self.partition_collection(image_set)  # [ [image1, image2, ...], [image1, image2, ...], ...  ]
+    
+        self.save_file_init()
+        
+        self.org_img = np.array(image_set).sum(0).astype(np.uint8)      
+        source_image = self.preproccessing_function(self.org_img)
+
+        self.source_feature = self.model(source_image.unsqueeze(0).to(self.device))
+        if id == None:
+            self.target_label = (self.source_feature @ self.semantic_feature.T).argmax().cpu().item()
+        else:
+            self.target_label = id
+        
+        Subset_merge = np.array(image_set)
+        Submodular_Subset = self.get_merge_set(Subset_merge)  # array([17, 42, 49, ...])
+            
+        submodular_image_set = Subset_merge[Submodular_Subset]  # sub_k x (112, 112, 3)
+        
+        
+        submodular_image = submodular_image_set.sum(0).astype(np.uint8)
+        self.saved_json_file["smdl_score_max"] = max(self.saved_json_file["smdl_score"])
+        self.saved_json_file["smdl_score_max_index"] = self.saved_json_file["smdl_score"].index(self.saved_json_file["smdl_score_max"])
+
+        return submodular_image, submodular_image_set, self.saved_json_file

models/submodular_vit_torch.py

@@ -0,0 +1,283 @@
+import math
+import random
+import numpy as np
+
+from tqdm import tqdm
+import cv2
+from PIL import Image
+
+import torch
+import torch.nn.functional as F
+# import torchvision.transforms as transforms
+
+from itertools import combinations
+from collections import OrderedDict
+
+class MultiModalSubModularExplanation(object):
+    def __init__(self, 
+                 model,
+                 semantic_feature,
+                 preproccessing_function,
+                 k = 40,
+                 lambda1 = 1.0,
+                 lambda2 = 1.0,
+                 lambda3 = 1.0,
+                 lambda4 = 1.0,
+                 device = "cuda"):
+        super(MultiModalSubModularExplanation, self).__init__()
+        
+        # Parameters of the submodular
+        self.k = k
+        
+        self.model = model
+        self.semantic_feature = semantic_feature
+        self.preproccessing_function = preproccessing_function
+        
+        self.lambda1 = lambda1
+        self.lambda2 = lambda2
+        self.lambda3 = lambda3
+        self.lambda4 = lambda4
+        
+        self.device = device
+        
+    def partition_collection(self, image_set):
+        """
+        Divide m image elements into n sets
+        """
+        image_set_size = len(image_set)
+        sample_size_per_partition = image_set_size
+        
+        image_set_clone = list(image_set)
+        random.shuffle(image_set_clone)
+        
+        V_partition = [image_set_clone[i: i + sample_size_per_partition] for i in range(0, image_set_size, sample_size_per_partition)]
+        
+        assert len(V_partition[0]) == sample_size_per_partition
+        
+        self.s_size = sample_size_per_partition
+        # assert image_set_size > sample_size_per_partition * self.k  # 其实就是 self.n > self.k ?
+        return V_partition
+    
+    def merge_image(self, sub_index_set, partition_image_set):
+        """
+        merge image
+        """
+        sub_image_set_ = np.array(partition_image_set)[sub_index_set]
+        image = sub_image_set_.sum(0)
+
+        return image.astype(np.uint8)
+    
+    # def compute_effectiveness_score(self, features):
+    #     """
+    #     Computes Eeffectiveness Score: The point should be distant from all the other elements in the subset.
+    #     features: torch.Size(batch, d)
+    #     """
+    #     norm_feature = F.normalize(features, p=2, dim=1)
+    #     # Consine Similarity
+    #     cosine_similarity = torch.mm(norm_feature, norm_feature.t())
+    #     cosine_similarity = torch.clamp(cosine_similarity, min=-1, max=1)
+    #     # Normlize 0-1
+    #     cosine_dist = torch.arccos(cosine_similarity) / math.pi
+        
+    #     if cosine_dist.shape[0] == 1:
+    #         eye = 1 - torch.eye(norm_feature.shape[0], device=self.device)
+    #         masked_dist = cosine_dist * eye
+    #         e_score = torch.sum(torch.min(masked_dist, dim=1).values)
+    #     else:
+    #         eye = torch.eye(norm_feature.shape[0], device=self.device)
+    #         adjusted_cosine_dist = cosine_dist + eye
+    #         e_score = torch.sum(torch.min(adjusted_cosine_dist, dim=1).values)
+        
+    #     return e_score # tensor(0.0343, device='cuda:0')
+        
+    # def proccess_compute_effectiveness_score_v1(self, components_image_feature, combination_list):
+    #     """
+    #     Compute each S's effectiveness score
+    #     """
+    #     e_scores = []
+    #     for sub_index in combination_list:
+    #         sub_feature_set = components_image_feature[sub_index]
+    #         e_score = self.compute_effectiveness_score(sub_feature_set)
+    #         e_scores.append(e_score)
+        
+    #     return torch.stack(e_scores)
+    
+    def proccess_compute_confidence_score(self):
+        """
+        Compute confidence score
+        """
+        # visual_features = self.model(batch_input_images)
+        # predicted_scores = torch.softmax(visual_features @ self.semantic_feature.T, dim=-1)
+        entropy = - torch.sum(self.predicted_scores * torch.log(self.predicted_scores + 1e-7), dim=1)
+        max_entropy = torch.log(torch.tensor(self.predicted_scores.shape[1])).to(self.device)
+        confidence = 1 - (entropy / max_entropy)
+        return confidence 
+    
+    def proccess_compute_effectiveness_score(self, sub_index_sets):
+        """
+        Compute each S's effectiveness score
+        """
+        e_scores = []
+        
+        for sub_index in sub_index_sets:
+            cosine_dist = self.effectiveness_dist[:, np.array(sub_index)]    # [len(element) , len(main_set)]
+            cosine_dist = cosine_dist[np.array(sub_index), :]
+            
+            eye = torch.eye(cosine_dist.shape[0], device=self.device)
+            adjusted_cosine_dist = cosine_dist + eye
+            e_score = torch.sum(torch.min(adjusted_cosine_dist, dim=1).values)
+            e_scores.append(e_score)
+        
+        effectiveness_score = torch.stack(e_scores)
+        if len(sub_index_sets[0]) == 1:
+            effectiveness_score = effectiveness_score * 0
+        return effectiveness_score
+    
+    def proccess_compute_consistency_score(self, batch_input_images):
+        """
+        Compute each consistency score
+        """
+        with torch.no_grad():
+            visual_features = self.model(batch_input_images)
+            self.predicted_scores = torch.softmax(visual_features @ self.semantic_feature.T, dim=-1)
+            consistency_scores = self.predicted_scores[:, self.target_label]
+
+        return consistency_scores
+    
+    def evaluation_maximun_sample(self, 
+                                  main_set, 
+                                  candidate_set, 
+                                  partition_image_set):
+        """
+        Given a subset, return a best sample index
+        """
+        sub_index_sets = []
+        for candidate_ in candidate_set:
+            sub_index_sets.append(
+                np.concatenate((main_set, np.array([candidate_]))).astype(int))
+       
+        # merge images / 组合图像
+        sub_images = torch.stack([
+            self.preproccessing_function(
+                self.merge_image(sub_index_set, partition_image_set)
+            ) for sub_index_set in sub_index_sets])
+        
+        batch_input_images = sub_images.to(self.device)
+        
+        with torch.no_grad():
+            
+            # 2. Effectiveness Score
+            score_effectiveness = self.proccess_compute_effectiveness_score(sub_index_sets)
+        
+            # 3. Consistency Score
+            score_consistency = self.proccess_compute_consistency_score(batch_input_images)
+            
+            # 1. Confidence Score
+            score_confidence = self.proccess_compute_confidence_score()
+            
+            # 4. Collaboration Score
+            sub_images_reverse = torch.stack([
+                self.preproccessing_function(
+                    self.org_img - self.merge_image(sub_index_set, partition_image_set)
+                ) for sub_index_set in sub_index_sets])
+        
+            batch_input_images_reverse = sub_images_reverse.to(self.device)
+            
+            score_collaboration = 1 - self.proccess_compute_consistency_score(batch_input_images_reverse)
+            
+            # 1. Confidence Score
+            # score_confidence = self.proccess_compute_confidence_score()
+            
+            # submodular score
+            smdl_score = self.lambda1 * score_confidence + self.lambda2 * score_effectiveness +  self.lambda3 * score_consistency + self.lambda4 * score_collaboration
+            # smdl_score = self.lambda2 * score_effectiveness +  self.lambda3 * score_consistency + self.lambda4 * score_collaboration
+            arg_max_index = smdl_score.argmax().cpu().item()
+            
+            # if self.lambda1 != 0:
+            self.saved_json_file["confidence_score"].append(score_confidence[arg_max_index].cpu().item())
+            self.saved_json_file["effectiveness_score"].append(score_effectiveness[arg_max_index].cpu().item())
+            self.saved_json_file["consistency_score"].append(score_consistency[arg_max_index].cpu().item())
+            self.saved_json_file["collaboration_score"].append(score_collaboration[arg_max_index].cpu().item())
+            self.saved_json_file["smdl_score"].append(smdl_score[arg_max_index].cpu().item())
+        
+        return sub_index_sets[arg_max_index]
+    
+    def save_file_init(self):
+        self.saved_json_file = {}
+        self.saved_json_file["sub-k"] = self.k
+        self.saved_json_file["confidence_score"] = []
+        self.saved_json_file["effectiveness_score"] = []
+        self.saved_json_file["consistency_score"] = []
+        self.saved_json_file["collaboration_score"] = []
+        self.saved_json_file["smdl_score"] = []
+        self.saved_json_file["lambda1"] = self.lambda1
+        self.saved_json_file["lambda2"] = self.lambda2
+        self.saved_json_file["lambda3"] = self.lambda3
+        self.saved_json_file["lambda4"] = self.lambda4
+    
+    def calculate_distance_of_each_element(self, partition_image_set):
+        """
+        Calculate the similarity of each element, obtain a similarity matrix
+        """
+        with torch.no_grad():
+            partition_images = torch.stack([
+                self.preproccessing_function(
+                    partition_image
+                ) for partition_image in partition_image_set]).to(self.device)
+            partition_image_features = self.model(partition_images)
+            
+            norm_feature = F.normalize(partition_image_features, p=2, dim=1)
+            # Consine Similarity
+            cosine_similarity = torch.mm(norm_feature, norm_feature.t())
+            cosine_similarity = torch.clamp(cosine_similarity, min=-1, max=1)
+            
+            # Normlize 0-1
+            self.effectiveness_dist = torch.arccos(cosine_similarity) / math.pi
+    
+    def get_merge_set(self, partition):
+        """
+        """
+        Subset = np.array([])
+        
+        indexes = np.arange(len(partition))
+        
+        # First calculate the similarity of each element to facilitate calculation of effectiveness score.
+        self.calculate_distance_of_each_element(partition)
+        
+        self.smdl_score_best = 0
+        
+        for j in tqdm(range(self.k)):
+            diff = np.setdiff1d(indexes, np.array(Subset))  # in indexes but not in Subset
+            
+            sub_candidate_indexes = diff
+            
+            Subset = self.evaluation_maximun_sample(Subset, sub_candidate_indexes, partition)
+        
+        return Subset
+    
+    def __call__(self, image_set, id = None):
+        """
+        Compute Source Face Submodular Score
+            @image_set: [mask_image 1, ..., mask_image m] (cv2 format)
+        """
+        # V_partition = self.partition_collection(image_set)  # [ [image1, image2, ...], [image1, image2, ...], ...  ]
+    
+        self.save_file_init()
+        
+        self.org_img = np.array(image_set).sum(0).astype(np.uint8)      
+        source_image = self.preproccessing_function(self.org_img)
+
+        self.source_feature = self.model(source_image.unsqueeze(0).to(self.device))
+        self.target_label = id
+        
+        Subset_merge = np.array(image_set)
+        Submodular_Subset = self.get_merge_set(Subset_merge)  # array([17, 42, 49, ...])
+            
+        submodular_image_set = Subset_merge[Submodular_Subset]  # sub_k x (112, 112, 3)
+        
+        
+        submodular_image = submodular_image_set.sum(0).astype(np.uint8)
+        self.saved_json_file["smdl_score_max"] = max(self.saved_json_file["smdl_score"])
+        self.saved_json_file["smdl_score_max_index"] = self.saved_json_file["smdl_score"].index(self.saved_json_file["smdl_score_max"])
+        
+        return submodular_image, submodular_image_set, self.saved_json_file

utils.py

@@ -0,0 +1,246 @@
+import scipy
+
+import os
+import numpy as np
+import cv2
+
+from scipy.ndimage import gaussian_filter
+from matplotlib import pyplot as plt
+# plt.style.use('seaborn')
+from matplotlib.colors import ListedColormap
+
+imagenet_classes = ["tench", "goldfish", "great white shark", "tiger shark", "hammerhead shark", "electric ray", "stingray", "rooster", "hen", "ostrich", "brambling", "goldfinch", "house finch", "junco", "indigo bunting", "American robin", "bulbul", "jay", "magpie", "chickadee", "American dipper", "kite (bird of prey)", "bald eagle", "vulture", "great grey owl", "fire salamander", "smooth newt", "newt", "spotted salamander", "axolotl", "American bullfrog", "tree frog", "tailed frog", "loggerhead sea turtle", "leatherback sea turtle", "mud turtle", "terrapin", "box turtle", "banded gecko", "green iguana", "Carolina anole", "desert grassland whiptail lizard", "agama", "frilled-necked lizard", "alligator lizard", "Gila monster", "European green lizard", "chameleon", "Komodo dragon", "Nile crocodile", "American alligator", "triceratops", "worm snake", "ring-necked snake", "eastern hog-nosed snake", "smooth green snake", "kingsnake", "garter snake", "water snake", "vine snake", "night snake", "boa constrictor", "African rock python", "Indian cobra", "green mamba", "sea snake", "Saharan horned viper", "eastern diamondback rattlesnake", "sidewinder rattlesnake", "trilobite", "harvestman", "scorpion", "yellow garden spider", "barn spider", "European garden spider", "southern black widow", "tarantula", "wolf spider", "tick", "centipede", "black grouse", "ptarmigan", "ruffed grouse", "prairie grouse", "peafowl", "quail", "partridge", "african grey parrot", "macaw", "sulphur-crested cockatoo", "lorikeet", "coucal", "bee eater", "hornbill", "hummingbird", "jacamar", "toucan", "duck", "red-breasted merganser", "goose", "black swan", "tusker", "echidna", "platypus", "wallaby", "koala", "wombat", "jellyfish", "sea anemone", "brain coral", "flatworm", "nematode", "conch", "snail", "slug", "sea slug", "chiton", "chambered nautilus", "Dungeness crab", "rock crab", "fiddler crab", "red king crab", "American lobster", "spiny lobster", "crayfish", "hermit crab", "isopod", "white stork", "black stork", "spoonbill", "flamingo", "little blue heron", "great egret", "bittern bird", "crane bird", "limpkin", "common gallinule", "American coot", "bustard", "ruddy turnstone", "dunlin", "common redshank", "dowitcher", "oystercatcher", "pelican", "king penguin", "albatross", "grey whale", "killer whale", "dugong", "sea lion", "Chihuahua", "Japanese Chin", "Maltese", "Pekingese", "Shih Tzu", "King Charles Spaniel", "Papillon", "toy terrier", "Rhodesian Ridgeback", "Afghan Hound", "Basset Hound", "Beagle", "Bloodhound", "Bluetick Coonhound", "Black and Tan Coonhound", "Treeing Walker Coonhound", "English foxhound", "Redbone Coonhound", "borzoi", "Irish Wolfhound", "Italian Greyhound", "Whippet", "Ibizan Hound", "Norwegian Elkhound", "Otterhound", "Saluki", "Scottish Deerhound", "Weimaraner", "Staffordshire Bull Terrier", "American Staffordshire Terrier", "Bedlington Terrier", "Border Terrier", "Kerry Blue Terrier", "Irish Terrier", "Norfolk Terrier", "Norwich Terrier", "Yorkshire Terrier", "Wire Fox Terrier", "Lakeland Terrier", "Sealyham Terrier", "Airedale Terrier", "Cairn Terrier", "Australian Terrier", "Dandie Dinmont Terrier", "Boston Terrier", "Miniature Schnauzer", "Giant Schnauzer", "Standard Schnauzer", "Scottish Terrier", "Tibetan Terrier", "Australian Silky Terrier", "Soft-coated Wheaten Terrier", "West Highland White Terrier", "Lhasa Apso", "Flat-Coated Retriever", "Curly-coated Retriever", "Golden Retriever", "Labrador Retriever", "Chesapeake Bay Retriever", "German Shorthaired Pointer", "Vizsla", "English Setter", "Irish Setter", "Gordon Setter", "Brittany dog", "Clumber Spaniel", "English Springer Spaniel", "Welsh Springer Spaniel", "Cocker Spaniel", "Sussex Spaniel", "Irish Water Spaniel", "Kuvasz", "Schipperke", "Groenendael dog", "Malinois", "Briard", "Australian Kelpie", "Komondor", "Old English Sheepdog", "Shetland Sheepdog", "collie", "Border Collie", "Bouvier des Flandres dog", "Rottweiler", "German Shepherd Dog", "Dobermann", "Miniature Pinscher", "Greater Swiss Mountain Dog", "Bernese Mountain Dog", "Appenzeller Sennenhund", "Entlebucher Sennenhund", "Boxer", "Bullmastiff", "Tibetan Mastiff", "French Bulldog", "Great Dane", "St. Bernard", "husky", "Alaskan Malamute", "Siberian Husky", "Dalmatian", "Affenpinscher", "Basenji", "pug", "Leonberger", "Newfoundland dog", "Great Pyrenees dog", "Samoyed", "Pomeranian", "Chow Chow", "Keeshond", "brussels griffon", "Pembroke Welsh Corgi", "Cardigan Welsh Corgi", "Toy Poodle", "Miniature Poodle", "Standard Poodle", "Mexican hairless dog (xoloitzcuintli)", "grey wolf", "Alaskan tundra wolf", "red wolf or maned wolf", "coyote", "dingo", "dhole", "African wild dog", "hyena", "red fox", "kit fox", "Arctic fox", "grey fox", "tabby cat", "tiger cat", "Persian cat", "Siamese cat", "Egyptian Mau", "cougar", "lynx", "leopard", "snow leopard", "jaguar", "lion", "tiger", "cheetah", "brown bear", "American black bear", "polar bear", "sloth bear", "mongoose", "meerkat", "tiger beetle", "ladybug", "ground beetle", "longhorn beetle", "leaf beetle", "dung beetle", "rhinoceros beetle", "weevil", "fly", "bee", "ant", "grasshopper", "cricket insect", "stick insect", "cockroach", "praying mantis", "cicada", "leafhopper", "lacewing", "dragonfly", "damselfly", "red admiral butterfly", "ringlet butterfly", "monarch butterfly", "small white butterfly", "sulphur butterfly", "gossamer-winged butterfly", "starfish", "sea urchin", "sea cucumber", "cottontail rabbit", "hare", "Angora rabbit", "hamster", "porcupine", "fox squirrel", "marmot", "beaver", "guinea pig", "common sorrel horse", "zebra", "pig", "wild boar", "warthog", "hippopotamus", "ox", "water buffalo", "bison", "ram (adult male sheep)", "bighorn sheep", "Alpine ibex", "hartebeest", "impala (antelope)", "gazelle", "arabian camel", "llama", "weasel", "mink", "European polecat", "black-footed ferret", "otter", "skunk", "badger", "armadillo", "three-toed sloth", "orangutan", "gorilla", "chimpanzee", "gibbon", "siamang", "guenon", "patas monkey", "baboon", "macaque", "langur", "black-and-white colobus", "proboscis monkey", "marmoset", "white-headed capuchin", "howler monkey", "titi monkey", "Geoffroy's spider monkey", "common squirrel monkey", "ring-tailed lemur", "indri", "Asian elephant", "African bush elephant", "red panda", "giant panda", "snoek fish", "eel", "silver salmon", "rock beauty fish", "clownfish", "sturgeon", "gar fish", "lionfish", "pufferfish", "abacus", "abaya", "academic gown", "accordion", "acoustic guitar", "aircraft carrier", "airliner", "airship", "altar", "ambulance", "amphibious vehicle", "analog clock", "apiary", "apron", "trash can", "assault rifle", "backpack", "bakery", "balance beam", "balloon", "ballpoint pen", "Band-Aid", "banjo", "baluster / handrail", "barbell", "barber chair", "barbershop", "barn", "barometer", "barrel", "wheelbarrow", "baseball", "basketball", "bassinet", "bassoon", "swimming cap", "bath towel", "bathtub", "station wagon", "lighthouse", "beaker", "military hat (bearskin or shako)", "beer bottle", "beer glass", "bell tower", "baby bib", "tandem bicycle", "bikini", "ring binder", "binoculars", "birdhouse", "boathouse", "bobsleigh", "bolo tie", "poke bonnet", "bookcase", "bookstore", "bottle cap", "hunting bow", "bow tie", "brass memorial plaque", "bra", "breakwater", "breastplate", "broom", "bucket", "buckle", "bulletproof vest", "high-speed train", "butcher shop", "taxicab", "cauldron", "candle", "cannon", "canoe", "can opener", "cardigan", "car mirror", "carousel", "tool kit", "cardboard box / carton", "car wheel", "automated teller machine", "cassette", "cassette player", "castle", "catamaran", "CD player", "cello", "mobile phone", "chain", "chain-link fence", "chain mail", "chainsaw", "storage chest", "chiffonier", "bell or wind chime", "china cabinet", "Christmas stocking", "church", "movie theater", "cleaver", "cliff dwelling", "cloak", "clogs", "cocktail shaker", "coffee mug", "coffeemaker", "spiral or coil", "combination lock", "computer keyboard", "candy store", "container ship", "convertible", "corkscrew", "cornet", "cowboy boot", "cowboy hat", "cradle", "construction crane", "crash helmet", "crate", "infant bed", "Crock Pot", "croquet ball", "crutch", "cuirass", "dam", "desk", "desktop computer", "rotary dial telephone", "diaper", "digital clock", "digital watch", "dining table", "dishcloth", "dishwasher", "disc brake", "dock", "dog sled", "dome", "doormat", "drilling rig", "drum", "drumstick", "dumbbell", "Dutch oven", "electric fan", "electric guitar", "electric locomotive", "entertainment center", "envelope", "espresso machine", "face powder", "feather boa", "filing cabinet", "fireboat", "fire truck", "fire screen", "flagpole", "flute", "folding chair", "football helmet", "forklift", "fountain", "fountain pen", "four-poster bed", "freight car", "French horn", "frying pan", "fur coat", "garbage truck", "gas mask or respirator", "gas pump", "goblet", "go-kart", "golf ball", "golf cart", "gondola", "gong", "gown", "grand piano", "greenhouse", "radiator grille", "grocery store", "guillotine", "hair clip", "hair spray", "half-track", "hammer", "hamper", "hair dryer", "hand-held computer", "handkerchief", "hard disk drive", "harmonica", "harp", "combine harvester", "hatchet", "holster", "home theater", "honeycomb", "hook", "hoop skirt", "gymnastic horizontal bar", "horse-drawn vehicle", "hourglass", "iPod", "clothes iron", "carved pumpkin", "jeans", "jeep", "T-shirt", "jigsaw puzzle", "rickshaw", "joystick", "kimono", "knee pad", "knot", "lab coat", "ladle", "lampshade", "laptop computer", "lawn mower", "lens cap", "letter opener", "library", "lifeboat", "lighter", "limousine", "ocean liner", "lipstick", "slip-on shoe", "lotion", "music speaker", "loupe magnifying glass", "sawmill", "magnetic compass", "messenger bag", "mailbox", "tights", "one-piece bathing suit", "manhole cover", "maraca", "marimba", "mask", "matchstick", "maypole", "maze", "measuring cup", "medicine cabinet", "megalith", "microphone", "microwave oven", "military uniform", "milk can", "minibus", "miniskirt", "minivan", "missile", "mitten", "mixing bowl", "mobile home", "ford model t", "modem", "monastery", "monitor", "moped", "mortar and pestle", "graduation cap", "mosque", "mosquito net", "vespa", "mountain bike", "tent", "computer mouse", "mousetrap", "moving van", "muzzle", "metal nail", "neck brace", "necklace", "baby pacifier", "notebook computer", "obelisk", "oboe", "ocarina", "odometer", "oil filter", "pipe organ", "oscilloscope", "overskirt", "bullock cart", "oxygen mask", "product packet / packaging", "paddle", "paddle wheel", "padlock", "paintbrush", "pajamas", "palace", "pan flute", "paper towel", "parachute", "parallel bars", "park bench", "parking meter", "railroad car", "patio", "payphone", "pedestal", "pencil case", "pencil sharpener", "perfume", "Petri dish", "photocopier", "plectrum", "Pickelhaube", "picket fence", "pickup truck", "pier", "piggy bank", "pill bottle", "pillow", "ping-pong ball", "pinwheel", "pirate ship", "drink pitcher", "block plane", "planetarium", "plastic bag", "plate rack", "farm plow", "plunger", "Polaroid camera", "pole", "police van", "poncho", "pool table", "soda bottle", "plant pot", "potter's wheel", "power drill", "prayer rug", "printer", "prison", "missile", "projector", "hockey puck", "punching bag", "purse", "quill", "quilt", "race car", "racket", "radiator", "radio", "radio telescope", "rain barrel", "recreational vehicle", "fishing casting reel", "reflex camera", "refrigerator", "remote control", "restaurant", "revolver", "rifle", "rocking chair", "rotisserie", "eraser", "rugby ball", "ruler measuring stick", "sneaker", "safe", "safety pin", "salt shaker", "sandal", "sarong", "saxophone", "scabbard", "weighing scale", "school bus", "schooner", "scoreboard", "CRT monitor", "screw", "screwdriver", "seat belt", "sewing machine", "shield", "shoe store", "shoji screen / room divider", "shopping basket", "shopping cart", "shovel", "shower cap", "shower curtain", "ski", "balaclava ski mask", "sleeping bag", "slide rule", "sliding door", "slot machine", "snorkel", "snowmobile", "snowplow", "soap dispenser", "soccer ball", "sock", "solar thermal collector", "sombrero", "soup bowl", "keyboard space bar", "space heater", "space shuttle", "spatula", "motorboat", "spider web", "spindle", "sports car", "spotlight", "stage", "steam locomotive", "through arch bridge", "steel drum", "stethoscope", "scarf", "stone wall", "stopwatch", "stove", "strainer", "tram", "stretcher", "couch", "stupa", "submarine", "suit", "sundial", "sunglasses", "sunglasses", "sunscreen", "suspension bridge", "mop", "sweatshirt", "swim trunks / shorts", "swing", "electrical switch", "syringe", "table lamp", "tank", "tape player", "teapot", "teddy bear", "television", "tennis ball", "thatched roof", "front curtain", "thimble", "threshing machine", "throne", "tile roof", "toaster", "tobacco shop", "toilet seat", "torch", "totem pole", "tow truck", "toy store", "tractor", "semi-trailer truck", "tray", "trench coat", "tricycle", "trimaran", "tripod", "triumphal arch", "trolleybus", "trombone", "hot tub", "turnstile", "typewriter keyboard", "umbrella", "unicycle", "upright piano", "vacuum cleaner", "vase", "vaulted or arched ceiling", "velvet fabric", "vending machine", "vestment", "viaduct", "violin", "volleyball", "waffle iron", "wall clock", "wallet", "wardrobe", "military aircraft", "sink", "washing machine", "water bottle", "water jug", "water tower", "whiskey jug", "whistle", "hair wig", "window screen", "window shade", "Windsor tie", "wine bottle", "airplane wing", "wok", "wooden spoon", "wool", "split-rail fence", "shipwreck", "sailboat", "yurt", "website", "comic book", "crossword", "traffic or street sign", "traffic light", "dust jacket", "menu", "plate", "guacamole", "consomme", "hot pot", "trifle", "ice cream", "popsicle", "baguette", "bagel", "pretzel", "cheeseburger", "hot dog", "mashed potatoes", "cabbage", "broccoli", "cauliflower", "zucchini", "spaghetti squash", "acorn squash", "butternut squash", "cucumber", "artichoke", "bell pepper", "cardoon", "mushroom", "Granny Smith apple", "strawberry", "orange", "lemon", "fig", "pineapple", "banana", "jackfruit", "cherimoya (custard apple)", "pomegranate", "hay", "carbonara", "chocolate syrup", "dough", "meatloaf", "pizza", "pot pie", "burrito", "red wine", "espresso", "tea cup", "eggnog", "mountain", "bubble", "cliff", "coral reef", "geyser", "lakeshore", "promontory", "sandbar", "beach", "valley", "volcano", "baseball player", "bridegroom", "scuba diver", "rapeseed", "daisy", "yellow lady's slipper", "corn", "acorn", "rose hip", "horse chestnut seed", "coral fungus", "agaric", "gyromitra", "stinkhorn mushroom", "earth star fungus", "hen of the woods mushroom", "bolete", "corn cob", "toilet paper"]
+
+imagenet_templates = [
+    'a bad photo of a {}.',
+    'a photo of many {}.',
+    'a sculpture of a {}.',
+    'a photo of the hard to see {}.',
+    'a low resolution photo of the {}.',
+    'a rendering of a {}.',
+    'graffiti of a {}.',
+    'a bad photo of the {}.',
+    'a cropped photo of the {}.',
+    'a tattoo of a {}.',
+    'the embroidered {}.',
+    'a photo of a hard to see {}.',
+    'a bright photo of a {}.',
+    'a photo of a clean {}.',
+    'a photo of a dirty {}.',
+    'a dark photo of the {}.',
+    'a drawing of a {}.',
+    'a photo of my {}.',
+    'the plastic {}.',
+    'a photo of the cool {}.',
+    'a close-up photo of a {}.',
+    'a black and white photo of the {}.',
+    'a painting of the {}.',
+    'a painting of a {}.',
+    'a pixelated photo of the {}.',
+    'a sculpture of the {}.',
+    'a bright photo of the {}.',
+    'a cropped photo of a {}.',
+    'a plastic {}.',
+    'a photo of the dirty {}.',
+    'a jpeg corrupted photo of a {}.',
+    'a blurry photo of the {}.',
+    'a photo of the {}.',
+    'a good photo of the {}.',
+    'a rendering of the {}.',
+    'a {} in a video game.',
+    'a photo of one {}.',
+    'a doodle of a {}.',
+    'a close-up photo of the {}.',
+    'a photo of a {}.',
+    'the origami {}.',
+    'the {} in a video game.',
+    'a sketch of a {}.',
+    'a doodle of the {}.',
+    'a origami {}.',
+    'a low resolution photo of a {}.',
+    'the toy {}.',
+    'a rendition of the {}.',
+    'a photo of the clean {}.',
+    'a photo of a large {}.',
+    'a rendition of a {}.',
+    'a photo of a nice {}.',
+    'a photo of a weird {}.',
+    'a blurry photo of a {}.',
+    'a cartoon {}.',
+    'art of a {}.',
+    'a sketch of the {}.',
+    'a embroidered {}.',
+    'a pixelated photo of a {}.',
+    'itap of the {}.',
+    'a jpeg corrupted photo of the {}.',
+    'a good photo of a {}.',
+    'a plushie {}.',
+    'a photo of the nice {}.',
+    'a photo of the small {}.',
+    'a photo of the weird {}.',
+    'the cartoon {}.',
+    'art of the {}.',
+    'a drawing of the {}.',
+    'a photo of the large {}.',
+    'a black and white photo of a {}.',
+    'the plushie {}.',
+    'a dark photo of a {}.',
+    'itap of a {}.',
+    'graffiti of the {}.',
+    'a toy {}.',
+    'itap of my {}.',
+    'a photo of a cool {}.',
+    'a photo of a small {}.',
+    'a tattoo of the {}.',
+]
+
+cub_classes = [
+    'Black footed Albatross', 'Laysan Albatross', 'Sooty Albatross', 'Groove billed Ani', 'Crested Auklet', 'Least Auklet', 'Parakeet Auklet', 'Rhinoceros Auklet', 'Brewer Blackbird', 'Red winged Blackbird', 'Rusty Blackbird', 'Yellow headed Blackbird', 'Bobolink', 'Indigo Bunting', 'Lazuli Bunting', 'Painted Bunting', 'Cardinal', 'Spotted Catbird', 'Gray Catbird', 'Yellow breasted Chat', 'Eastern Towhee', 'Chuck will Widow', 'Brandt Cormorant', 'Red faced Cormorant', 'Pelagic Cormorant', 'Bronzed Cowbird', 'Shiny Cowbird', 'Brown Creeper', 'American Crow', 'Fish Crow', 'Black billed Cuckoo', 'Mangrove Cuckoo', 'Yellow billed Cuckoo', 'Gray crowned Rosy Finch', 'Purple Finch', 'Northern Flicker', 'Acadian Flycatcher', 'Great Crested Flycatcher', 'Least Flycatcher', 'Olive sided Flycatcher', 'Scissor tailed Flycatcher', 'Vermilion Flycatcher', 'Yellow bellied Flycatcher', 'Frigatebird', 'Northern Fulmar', 'Gadwall', 'American Goldfinch', 'European Goldfinch', 'Boat tailed Grackle', 'Eared Grebe', 'Horned Grebe', 'Pied billed Grebe', 'Western Grebe', 'Blue Grosbeak', 'Evening Grosbeak', 'Pine Grosbeak', 'Rose breasted Grosbeak', 'Pigeon Guillemot', 'California Gull', 'Glaucous winged Gull', 'Heermann Gull', 'Herring Gull', 'Ivory Gull', 'Ring billed Gull', 'Slaty backed Gull', 'Western Gull', 'Anna Hummingbird', 'Ruby throated Hummingbird', 'Rufous Hummingbird', 'Green Violetear', 'Long tailed Jaeger', 'Pomarine Jaeger', 'Blue Jay', 'Florida Jay', 'Green Jay', 'Dark eyed Junco', 'Tropical Kingbird', 'Gray Kingbird', 'Belted Kingfisher', 'Green Kingfisher', 'Pied Kingfisher', 'Ringed Kingfisher', 'White breasted Kingfisher', 'Red legged Kittiwake', 'Horned Lark', 'Pacific Loon', 'Mallard', 'Western Meadowlark', 'Hooded Merganser', 'Red breasted Merganser', 'Mockingbird', 'Nighthawk', 'Clark Nutcracker', 'White breasted Nuthatch', 'Baltimore Oriole', 'Hooded Oriole', 'Orchard Oriole', 'Scott Oriole', 'Ovenbird', 'Brown Pelican', 'White Pelican', 'Western Wood Pewee', 'Sayornis', 'American Pipit', 'Whip poor Will', 'Horned Puffin', 'Common Raven', 'White necked Raven', 'American Redstart', 'Geococcyx', 'Loggerhead Shrike', 'Great Grey Shrike', 'Baird Sparrow', 'Black throated Sparrow', 'Brewer Sparrow', 'Chipping Sparrow', 'Clay colored Sparrow', 'House Sparrow', 'Field Sparrow', 'Fox Sparrow', 'Grasshopper Sparrow', 'Harris Sparrow', 'Henslow Sparrow', 'Le Conte Sparrow', 'Lincoln Sparrow', 'Nelson Sharp tailed Sparrow', 'Savannah Sparrow', 'Seaside Sparrow', 'Song Sparrow', 'Tree Sparrow', 'Vesper Sparrow', 'White crowned Sparrow', 'White throated Sparrow', 'Cape Glossy Starling', 'Bank Swallow', 'Barn Swallow', 'Cliff Swallow', 'Tree Swallow', 'Scarlet Tanager', 'Summer Tanager', 'Arctic Tern', 'Black Tern', 'Caspian Tern', 'Common Tern', 'Elegant Tern', 'Forsters Tern', 'Least Tern', 'Green tailed Towhee', 'Brown Thrasher', 'Sage Thrasher', 'Black capped Vireo', 'Blue headed Vireo', 'Philadelphia Vireo', 'Red eyed Vireo', 'Warbling Vireo', 'White eyed Vireo', 'Yellow throated Vireo', 'Bay breasted Warbler', 'Black and white Warbler', 'Black throated Blue Warbler', 'Blue winged Warbler', 'Canada Warbler', 'Cape May Warbler', 'Cerulean Warbler', 'Chestnut sided Warbler', 'Golden winged Warbler', 'Hooded Warbler', 'Kentucky Warbler', 'Magnolia Warbler', 'Mourning Warbler', 'Myrtle Warbler', 'Nashville Warbler', 'Orange crowned Warbler', 'Palm Warbler', 'Pine Warbler', 'Prairie Warbler', 'Prothonotary Warbler', 'Swainson Warbler', 'Tennessee Warbler', 'Wilson Warbler', 'Worm eating Warbler', 'Yellow Warbler', 'Northern Waterthrush', 'Louisiana Waterthrush', 'Bohemian Waxwing', 'Cedar Waxwing', 'American Three toed Woodpecker', 'Pileated Woodpecker', 'Red bellied Woodpecker', 'Red cockaded Woodpecker', 'Red headed Woodpecker', 'Downy Woodpecker', 'Bewick Wren', 'Cactus Wren', 'Carolina Wren', 'House Wren', 'Marsh Wren', 'Rock Wren', 'Winter Wren', 'Common Yellowthroat'
+]
+
+cub_templates = [
+    'a photo of a {}, a type of bird.',
+]
+
+lc_lung_classes = [
+    "lung adenocarcinoma",
+    "benign lung",
+    "lung squamous cell carcinoma"
+]
+
+lc_lung_template = 'a histopathology slide showing '
+
+def load_image(path, size=224):
+    img = cv2.resize(cv2.imread(path)[...,::-1], (size, size))
+    return img.astype(np.float32)
+
+def set_size(w,h):
+    """Set matplot figure size"""
+    plt.rcParams["figure.figsize"] = [w,h]
+
+def show(img, p=False, smooth=False, minn=None, maxx = None, **kwargs):
+    """ Display numpy/tf tensor """ 
+    img = np.array(img, dtype=np.float32)
+
+    # check if channel first
+    if img.shape[0] == 1:
+        img = img[0]
+    elif img.shape[0] == 3:
+        img = np.moveaxis(img, 0, -1)
+
+    # check if cmap
+    if img.shape[-1] == 1:
+        img = img[:,:,0]
+
+    # normalize
+    if minn is None:
+        if img.max() > 1 or img.min() < 0:
+            img -= img.min(); img/=img.max()
+
+    # check if clip percentile
+    if p is not False:
+        img = np.clip(img, np.percentile(img, p), np.percentile(img, 100-p))
+    
+    if smooth and len(img.shape) == 2:
+        img = gaussian_filter(img, smooth)
+
+    if minn is not None:
+        plt.imshow(img, vmin = minn, vmax =maxx, **kwargs)
+    else:
+        plt.imshow(img,  **kwargs)
+    plt.axis('off')
+    plt.grid(None)
+    
+def get_alpha_cmap(cmap):
+    if isinstance(cmap, str):
+        cmap = plt.get_cmap(cmap)
+
+    alpha_cmap = cmap(np.arange(cmap.N))
+    alpha_cmap[:,-1] = np.linspace(0, 1, cmap.N)
+    alpha_cmap = ListedColormap(alpha_cmap)
+
+    return alpha_cmap
+
+def mkdir(name):
+    '''
+    Create folder
+    '''
+    isExists=os.path.exists(name)
+    if not isExists:
+        try:
+            os.makedirs(name)
+        except:
+            print("already exist")
+    return 0
+
+def norm(image):
+    """
+    :param image: [H,W]
+    :return:
+    """
+    image = image.copy()
+    image -= np.max(np.min(image), 0)
+    image /= np.max(image)
+    
+    return image
+
+def norm_image(image):
+    """
+    :param image: [H,W,C]
+    :return:
+    """
+    image = image.copy()
+    image -= np.max(np.min(image), 0)
+    image /= np.max(image)
+    image *= 255.
+    return np.uint8(image)
+
+def gen_cam(image, mask):
+    """
+    生成CAM图
+    :param image: [H,W,C],原始图像
+    :param mask: [H,W],范围0~1
+    :return: tuple(cam,heatmap)
+    """
+    # mask to heatmap
+    heatmap = cv2.applyColorMap(np.uint8(255 * mask), cv2.COLORMAP_JET)
+    # heatmap = np.float32(heatmap) / 255
+    # heatmap = heatmap[..., ::-1]  # gbr to rgb
+
+    # merge heatmap to original image
+    cam = 0.5 * heatmap + 0.5 * image
+    return norm_image(cam), heatmap
+
+def norm(image):
+    """
+    :param image: [H,W,C]
+    :return:
+    """
+    image = image.copy()
+    image -= np.max(np.min(image), 0)
+    image /= np.max(image)
+    return image
+
+def SubRegionDivision(image, mode="slico", region_size=30):
+    element_sets_V = []
+    if mode == "slico":
+        slic = cv2.ximgproc.createSuperpixelSLIC(image, region_size=region_size, ruler = 20.0) 
+        slic.iterate(20)     # The number of iterations, the larger the better the effect
+        label_slic = slic.getLabels()        # Get superpixel label
+        number_slic = slic.getNumberOfSuperpixels()  # Get the number of superpixels
+
+        for i in range(number_slic):
+            img_copp = image.copy()
+            img_copp = img_copp * (label_slic == i)[:,:, np.newaxis]
+            element_sets_V.append(img_copp)
+    elif mode == "seeds":
+        seeds = cv2.ximgproc.createSuperpixelSEEDS(image.shape[1], image.shape[0], image.shape[2], num_superpixels=50, num_levels=3)
+        seeds.iterate(image,10)  # The input image size must be the same as the initialization shape and the number of iterations is 10
+        label_seeds = seeds.getLabels()
+        number_seeds = seeds.getNumberOfSuperpixels()
+
+        for i in range(number_seeds):
+            img_copp = image.copy()
+            img_copp = img_copp * (label_seeds == i)[:,:, np.newaxis]
+            element_sets_V.append(img_copp)
+    return element_sets_V