biomedparse+biomedllama_3b_multimodal

app.py

@@ -380,6 +380,14 @@ def process_image(image_path, user_prompt, modality=None):
         analysis_results = []
         colors = [(255,0,0), (0,255,0), (0,0,255), (255,255,0), (255,0,255)]
         
+        # Add color mapping to analysis with more natural language
+        color_descriptions = []
+        for idx, prompt in enumerate(relevant_prompts):
+            color = colors[idx % len(colors)]
+            color_name = {(255,0,0): "red", (0,255,0): "green", (0,0,255): "blue", 
+                         (255,255,0): "yellow", (255,0,255): "magenta"}[color]
+            color_descriptions.append(f"The {prompt} is highlighted in {color_name} color")
+        
         for idx, prompt in enumerate(relevant_prompts):
             try:
                 mask_list = interactive_infer_image(model, pil_image, [prompt])
@@ -407,13 +415,14 @@ def process_image(image_path, user_prompt, modality=None):
             results = [image]  # Return original image if no overlays were created
             
         detailed_analysis = ""
-        # try:
-        analysis_prompt = f"Give the modality, organ, analysis, abnormalities (if any), treatment (if abnormalities are present) for this image. Focus more on the user question. which is: {user_prompt}"
-        msgs = [{'role': 'user', 'content': [pil_image, analysis_prompt]}]
-        
-        # llm_response = ""
         if llm_model and llm_tokenizer:
             try:
+                # Add color legend with more natural language
+                detailed_analysis += "\n\n As shown in the images outputs details:\n \n" + "\n".join(color_descriptions)
+                
+                analysis_prompt = f"Give the modality, organ, analysis, abnormalities (if any), treatment (if abnormalities are present) for this image. Focus more on the user question. which is: {user_prompt}"
+                msgs = [{'role': 'user', 'content': [pil_image, analysis_prompt]}]
+                
                 for new_text in llm_model.chat(
                     image=pil_image,
                     msgs=msgs,