add model debugging

app.py

@@ -113,6 +113,7 @@ explainer = MultiModalSubModularExplanationEfficientPlus(
         lambda2=0.05, 
         lambda3=20., 
         lambda4=5.)
+explainer.org_semantic_feature = semantic_feature
 
 def add_value_decrease(smdl_mask, json_file):
     single_mask = np.zeros_like(smdl_mask[0].mean(-1))
@@ -276,13 +277,16 @@ def norm_image(image):
 def read_image(file_path):
     image = Image.open(file_path)
     image = image.convert("RGB")
+    image = image.resize((512,512))
     return np.array(image)
 
 # 使用同一个示例图像 "shark.png"
 default_images = {
     # "Default Image": read_image("images/shark.png"),
-    "Example: Tiger Shark": read_image("images/shark.png"),
-    "Example: Quail": read_image("images/bird.png")  # 所有选项都使用相同的图片
+    "Example: tiger shark": read_image("images/shark.png"),
+    "Example: quail": read_image("images/bird.png"),  # 所有选项都使用相同的图片
+    "Example: tabby cat or lion": read_image("images/cat_lion.jpeg"),
+    "Example: rabbit or duck": read_image("images/rabbit-duck.jpg"),
 }
 
 def interpret_image(uploaded_image, slider, text_input):
@@ -296,21 +300,66 @@ def interpret_image(uploaded_image, slider, text_input):
     element_sets_V = SubRegionDivision(image, mode="slico", region_size=40) 
     
     explainer.k = len(element_sets_V)
-    print(len(element_sets_V))
     
     global submodular_image_set
     global saved_json_file
+    
+    image_input = explainer.preproccessing_function(image).unsqueeze(0)
+    predicted_class = (explainer.model(image_input.to(explainer.device)) @ explainer.semantic_feature.T).argmax().cpu().item()
+    
+    # input
+    if text_input == "":
+        target_id = predicted_class
+    else:
+        if text_input in imagenet_classes:
+            target_id = imagenet_classes.index(text_input)
+        else:
+            target_id = -1
+            texts = [text_input]
+            texts = clip.tokenize(texts).to(device) #tokenize
+            
+            with torch.no_grad():
+                class_embeddings = vis_model.model.encode_text(texts)
+                class_embeddings /= class_embeddings.norm(dim=-1, keepdim=True)
+                class_embeddings = class_embeddings.to(device) * 100
+            
+            explainer.semantic_feature = torch.cat((explainer.org_semantic_feature, class_embeddings), dim=0)
+    
     # global im
-    submodular_image, submodular_image_set, saved_json_file = explainer(element_sets_V, id=None)
+    submodular_image, submodular_image_set, saved_json_file = explainer(element_sets_V, id=target_id)
     
     # attribution_map, value_list = add_value_decrease(submodular_image_set, saved_json_file)
     # im, heatmap = gen_cam(image, norm_image(attribution_map))
     
     image_curve, highest_confidence, insertion_auc_score, ours_best_index = visualization(image, submodular_image_set, saved_json_file, index=None)
 
-    text_output_class = "The method explains why the CLIP (ViT-B/16) model identifies an image as {}.".format(imagenet_classes[explainer.target_label])
+    if target_id == -1:
+        text_output_class = "This method explains that CLIP is interested in describing \"{}\".".format(text_input)
+    else:
+        text_output_class = "The method explains why the CLIP (ViT-B/16) model identifies an image as {}.".format(imagenet_classes[explainer.target_label])
+
+    text_output_predict = "The image is predicted as {}".format(imagenet_classes[predicted_class])
+    
+    explainer.semantic_feature = explainer.org_semantic_feature
+
+    return image_curve, highest_confidence, insertion_auc_score, text_output_class, text_output_predict, None
+
+def predict_image(uploaded_image):
+    # 使用上传的图像（如果有），否则使用生成的图像
+    if uploaded_image is not None:
+        image = np.array(uploaded_image)
+    else:
+        return None, 0, 0
 
-    return image_curve, highest_confidence, insertion_auc_score, text_output_class
+    image = cv2.resize(image, (224, 224))
+    
+    image_input = explainer.preproccessing_function(image).unsqueeze(0)
+    predicted_class = (explainer.model(image_input.to(explainer.device)) @ explainer.semantic_feature.T).argmax().cpu().item()
+    
+    text_output_predict = "The image is predicted as {}".format(imagenet_classes[predicted_class])
+    
+    return text_output_predict
+    
 
 def visualization_slider(uploaded_image, slider):
     # 使用上传的图像（如果有），否则使用生成的图像
@@ -331,7 +380,7 @@ def update_image(thumbnail_name):
 
 # 创建 Gradio 界面
 with gr.Blocks() as demo:
-    gr.Markdown("# Semantic Region Attribution via Submodular Subset Selection")  # 使用Markdown添加标题
+    gr.Markdown("# Semantic Region Attribution and Mistake Discovery via Submodular Subset Selection")  # 使用Markdown添加标题
     
     gr.Markdown("Since huggingface only has ordinary CPUs available, our sub-region division is relatively coarse-grained, which may affect the attribution performance. The inference time is about 5 minutes (GPU is about 4s). If you are interested, you can try our source code. We have written many scripts to facilitate visualization.")
     with gr.Row():
@@ -352,6 +401,9 @@ with gr.Blocks() as demo:
                     # )
                     gr.Textbox("Thank you for using our interpretable attribution method, which originates from the ICLR 2024 Oral paper titled \"Less is More: Fewer Interpretable Regions via Submodular Subset Selection.\" We have now implemented this method on the multimodal ViT model and achieved promising results in explaining model predictions. A key feature of our approach is its ability to clarify the reasons behind the model's prediction errors. We invite you to try out this demo and explore its capabilities. The source code is available at https://github.com/RuoyuChen10/SMDL-Attribution.\nYou can upload an image yourself or select one from the following, then click the button Interpreting Model to get the result. The demo currently does not support selecting categories or descriptions by yourself. If you are interested, you can try it from the source code.", label="Instructions for use", interactive=False)
             
+            # 文本输入框和滑块
+            text_input = gr.Textbox(label="Text Input", placeholder="You can choose what you want to explain. You can enter a word (e.g., 'Rabbit') or a description (e.g., 'A photo of a rabbit'). If the input is empty, the model will explain the predicted category.")
+            
             # 第二排：两个缩略图
             with gr.Row():
                 for key in default_images.keys():
@@ -363,10 +415,6 @@ with gr.Blocks() as demo:
                             inputs=[],
                             outputs=image_input
                         )
-
-            # 文本输入框和滑块
-            text_input = gr.Textbox(label="Text Input", placeholder="Enter some text here... (optional)")
-            
             
         with gr.Column():
             # 输出图像和控件
@@ -374,19 +422,29 @@ with gr.Blocks() as demo:
             
             slider = gr.Slider(minimum=0, maximum=34, step=1, label="Number of Sub-regions")
             
+            text_output_predict = gr.Textbox(label="Predicted Category")
             text_output_class = gr.Textbox(label="Explaining Category")
+            
             with gr.Row():
                 # 最高置信度和插入 AUC Score 并排显示
                 text_output_confidence = gr.Textbox(label="Highest Confidence")
                 text_output_auc = gr.Textbox(label="Insertion AUC Score")
             
-            interpret_button = gr.Button("Interpreting Model")
+            with gr.Row():
+                predict_button = gr.Button("Model Inference")
+                interpret_button = gr.Button("Interpreting Model")
 
     # 定义解释模型按钮点击事件
     interpret_button.click(
         fn=interpret_image,
         inputs=[image_input, slider, text_input],
-        outputs=[image_output, text_output_confidence, text_output_auc, text_output_class]
+        outputs=[image_output, text_output_confidence, text_output_auc, text_output_class, text_output_predict, text_input]
+    )
+    
+    predict_button.click(
+        fn=predict_image,
+        inputs=[image_input],
+        outputs=[text_output_predict]
     )
     
     # 实时更新的滑块