biomedparse+biomedllama_3b_multimodal

app.py

@@ -208,64 +208,6 @@ def extract_modality_and_prompts(llm_output):
     
     return detected_modality, relevant_prompts
 
-def extract_modality_from_llm(llm_output):
-    """Extract modality from LLM output and map it to BIOMEDPARSE_MODES"""
-    llm_output = llm_output.lower()
-    
-    modality_keywords = {
-        'ct': {
-            'abdomen': 'CT-Abdomen',
-            'chest': 'CT-Chest',
-            'liver': 'CT-Liver'
-        },
-        'mri': {
-            'abdomen': 'MRI-Abdomen',
-            'cardiac': 'MRI-Cardiac',
-            'heart': 'MRI-Cardiac',
-            'flair': 'MRI-FLAIR-Brain',
-            't1': 'MRI-T1-Gd-Brain',
-            'contrast': 'MRI-T1-Gd-Brain',
-            'brain': 'MRI-FLAIR-Brain'
-        },
-        'x-ray': {'chest': 'X-Ray-Chest'},
-        'ultrasound': {'cardiac': 'Ultrasound-Cardiac', 'heart': 'Ultrasound-Cardiac'},
-        'endoscopy': {'': 'Endoscopy'},
-        'fundus': {'': 'Fundus'},
-        'dermoscopy': {'': 'Dermoscopy'},
-        'oct': {'': 'OCT'},
-        'pathology': {'': 'Pathology'}
-    }
-    
-    for modality, subtypes in modality_keywords.items():
-        if modality in llm_output:
-            for keyword, specific_modality in subtypes.items():
-                if not keyword or keyword in llm_output:
-                    return specific_modality
-            return next(iter(subtypes.values()))
-    
-    return None
-
-def extract_clinical_findings(llm_output, modality):
-    """Extract relevant clinical findings that match available anatomical sites in BIOMEDPARSE_MODES"""
-    available_sites = BIOMEDPARSE_MODES.get(modality, [])
-    findings = []
-    
-    # Convert sites to lowercase for case-insensitive matching
-    sites_lower = {site.lower(): site for site in available_sites}
-    
-    # Look for each available site in the LLM output
-    for site_lower, original_site in sites_lower.items():
-        if site_lower in llm_output.lower():
-            findings.append(original_site)
-    
-    # Add additional findings from MODALITY_PROMPTS if available
-    if modality in MODALITY_PROMPTS:
-        for prompt in MODALITY_PROMPTS[modality]:
-            if prompt.lower() in llm_output.lower() and prompt not in findings:
-                findings.append(prompt)
-    
-    return findings
-
 def on_mode_dropdown_change(selected_mode):
     if selected_mode in IMAGE_INFERENCE_MODES:
         return [
@@ -376,49 +318,49 @@ def process_image(image_path, user_prompt, modality=None):
             detected_modality = "X-Ray-Chest"  # Fallback modality
             relevant_prompts = MODALITY_PROMPTS["X-Ray-Chest"]
         
-        results = []
-        analysis_results = []
-        colors = [(255,0,0), (0,255,0), (0,0,255), (255,255,0), (255,0,255)]
+        # results = []
+        # analysis_results = []
+        # colors = [(255,0,0), (0,255,0), (0,0,255), (255,255,0), (255,0,255)]
         
-        # Add color mapping to analysis with more natural language
-        color_descriptions = []
-        for idx, prompt in enumerate(relevant_prompts):
-            color = colors[idx % len(colors)]
-            color_name = {(255,0,0): "red", (0,255,0): "green", (0,0,255): "blue", 
-                         (255,255,0): "yellow", (255,0,255): "magenta"}[color]
-            color_descriptions.append(f"The {prompt} is highlighted in {color_name} color")
+        # # Add color mapping to analysis with more natural language
+        # color_descriptions = []
+        # for idx, prompt in enumerate(relevant_prompts):
+        #     color = colors[idx % len(colors)]
+        #     color_name = {(255,0,0): "red", (0,255,0): "green", (0,0,255): "blue", 
+        #                  (255,255,0): "yellow", (255,0,255): "magenta"}[color]
+        #     color_descriptions.append(f"The {prompt} is highlighted in {color_name} color")
         
-        for idx, prompt in enumerate(relevant_prompts):
-            try:
-                mask_list = interactive_infer_image(model, pil_image, [prompt])
-                if mask_list is None or len(mask_list) == 0:
-                    analysis_results.append(f"No mask generated for '{prompt}'")
-                    continue
+        # for idx, prompt in enumerate(relevant_prompts):
+        #     try:
+        #         mask_list = interactive_infer_image(model, pil_image, [prompt])
+        #         if mask_list is None or len(mask_list) == 0:
+        #             analysis_results.append(f"No mask generated for '{prompt}'")
+        #             continue
                 
-                pred_mask = mask_list[0]
-                # Check if mask is valid using numpy's any() function
-                if pred_mask is None or not np.any(pred_mask):
-                    analysis_results.append(f"Empty mask generated for '{prompt}'")
-                    continue
+        #         pred_mask = mask_list[0]
+        #         # Check if mask is valid using numpy's any() function
+        #         if pred_mask is None or not np.any(pred_mask):
+        #             analysis_results.append(f"Empty mask generated for '{prompt}'")
+        #             continue
                 
-                overlay_image = image.copy()
-                color = colors[idx % len(colors)]
-                mask_indices = pred_mask > 0.5
-                if np.any(mask_indices):  # Use np.any() for boolean array check
-                    overlay_image[mask_indices] = color
-                    results.append(overlay_image)
-            except Exception as e:
-                print(f"Error processing finding {prompt}: {str(e)}")
-                analysis_results.append(f"Failed to process '{prompt}': {str(e)}")
+        #         overlay_image = image.copy()
+        #         color = colors[idx % len(colors)]
+        #         mask_indices = pred_mask > 0.5
+        #         if np.any(mask_indices):  # Use np.any() for boolean array check
+        #             overlay_image[mask_indices] = color
+        #             results.append(overlay_image)
+        #     except Exception as e:
+        #         print(f"Error processing finding {prompt}: {str(e)}")
+        #         analysis_results.append(f"Failed to process '{prompt}': {str(e)}")
         
-        if not results:
-            results = [image]  # Return original image if no overlays were created
+        # if not results:
+        #     results = [image]  # Return original image if no overlays were created
             
         detailed_analysis = ""
         if llm_model and llm_tokenizer:
             try:
                 # Add color legend with more natural language
-                detailed_analysis += "\n\n As shown in the images outputs details:\n \n" + "\n".join(color_descriptions)
+                # detailed_analysis += "\n\n As shown in the images outputs details:\n \n" + "\n".join(color_descriptions)
                 
                 analysis_prompt = f"Give the modality, organ, analysis, abnormalities (if any), treatment (if abnormalities are present) for this image. Focus more on the user question. which is: {user_prompt}"
                 msgs = [{'role': 'user', 'content': [pil_image, analysis_prompt]}]
@@ -434,6 +376,44 @@ def process_image(image_path, user_prompt, modality=None):
                     detailed_analysis += new_text
                 
                 # Add segmentation details in a more natural way
+                results = []
+                analysis_results = []
+                colors = [(255,0,0), (0,255,0), (0,0,255), (255,255,0), (255,0,255)]
+                
+                # Add color mapping to analysis with more natural language
+                color_descriptions = []
+                for idx, prompt in enumerate(relevant_prompts):
+                    if prompt in detailed_analysis.lower():
+                        color = colors[idx % len(colors)]
+                        color_name = {(255,0,0): "red", (0,255,0): "green", (0,0,255): "blue", 
+                                    (255,255,0): "yellow", (255,0,255): "magenta"}[color]
+                        color_descriptions.append(f"The {prompt} is highlighted in {color_name} color")
+                
+                for idx, prompt in enumerate(relevant_prompts):
+                    try:
+                        mask_list = interactive_infer_image(model, pil_image, [prompt])
+                        if mask_list is None or len(mask_list) == 0:
+                            analysis_results.append(f"No mask generated for '{prompt}'")
+                            continue
+                        
+                        pred_mask = mask_list[0]
+                        # Check if mask is valid using numpy's any() function
+                        if pred_mask is None or not np.any(pred_mask):
+                            analysis_results.append(f"Empty mask generated for '{prompt}'")
+                            continue
+                        
+                        overlay_image = image.copy()
+                        color = colors[idx % len(colors)]
+                        mask_indices = pred_mask > 0.5
+                        if np.any(mask_indices):  # Use np.any() for boolean array check
+                            overlay_image[mask_indices] = color
+                            results.append(overlay_image)
+                    except Exception as e:
+                        print(f"Error processing finding {prompt}: {str(e)}")
+                        analysis_results.append(f"Failed to process '{prompt}': {str(e)}")
+                
+                if not results:
+                    results = [image]  # Return original image if no overlays were created       
                 detailed_analysis += "\n\nIn the segmentation analysis:"
                 if color_descriptions:
                     detailed_analysis += " " + " ".join(color_descriptions) + "."