app.py

from transformers import AutoModelForCausalLM, AutoProcessor
from PIL import Image
import torch
import gradio as gr
import requests
import tempfile

device = torch.device("cpu")
model = AutoModelForCausalLM.from_pretrained("microsoft/maira-2", trust_remote_code=True)
processor = AutoProcessor.from_pretrained("microsoft/maira-2", trust_remote_code=True)
model = model.eval().to(device)

def get_sample_data():
    """Download sample medical images and data"""
    frontal_url = "https://openi.nlm.nih.gov/imgs/512/145/145/CXR145_IM-0290-1001.png"
    lateral_url = "https://openi.nlm.nih.gov/imgs/512/145/145/CXR145_IM-0290-2001.png"
    
    def download_image(url):
        response = requests.get(url, headers={"User-Agent": "MAIRA-2"}, stream=True)
        return Image.open(response.raw)
    
    return {
        "frontal": download_image(frontal_url),
        "lateral": download_image(lateral_url),
        "indication": "Dyspnea.",
        "technique": "PA and lateral views of the chest.",
        "comparison": "None.",
        "phrase": "Pleural effusion."
    }

def save_temp_image(img):
    """Save PIL image to temporary file"""
    temp_file = tempfile.NamedTemporaryFile(suffix=".png", delete=False)
    img.save(temp_file.name)
    return temp_file.name

def load_sample_findings():
    """Load sample data for findings generation"""
    sample = get_sample_data()
    return [
        save_temp_image(sample["frontal"]),
        save_temp_image(sample["lateral"]),
        sample["indication"],
        sample["technique"],
        sample["comparison"],
        None, None, None, False
    ]

def load_sample_phrase():
    """Load sample data for phrase grounding"""
    sample = get_sample_data()
    return [save_temp_image(sample["frontal"]), sample["phrase"]]

def generate_report(frontal_path, lateral_path, indication, technique, comparison, 
                   prior_frontal_path, prior_lateral_path, prior_report, grounding):
    """Generate radiology report with optional grounding"""
    try:
        # Load images
        current_frontal = Image.open(frontal_path)
        current_lateral = Image.open(lateral_path)
        prior_frontal = Image.open(prior_frontal_path) if prior_frontal_path else None
        prior_lateral = Image.open(prior_lateral_path) if prior_lateral_path else None

        # Process inputs
        processed = processor.format_and_preprocess_reporting_input(
            current_frontal=current_frontal,
            current_lateral=current_lateral,
            prior_frontal=prior_frontal,
            prior_lateral=prior_lateral,
            indication=indication,
            technique=technique,
            comparison=comparison,
            prior_report=prior_report or None,
            return_tensors="pt",
            get_grounding=grounding
        ).to(device)

        # Generate report
        outputs = model.generate(**processed, 
                               max_new_tokens=450 if grounding else 300,
                               use_cache=True)
        
        # Decode and format
        prompt_length = processed["input_ids"].shape[-1]
        decoded = processor.decode(outputs[0][prompt_length:], skip_special_tokens=True)
        return processor.convert_output_to_plaintext_or_grounded_sequence(decoded.lstrip())
    
    except Exception as e:
        return f"Error: {str(e)}"

def ground_phrase(frontal_path, phrase):
    """Perform phrase grounding on image"""
    try:
        frontal = Image.open(frontal_path)
        processed = processor.format_and_preprocess_phrase_grounding_input(
            frontal_image=frontal,
            phrase=phrase,
            return_tensors="pt"
        ).to(device)
        
        outputs = model.generate(**processed, max_new_tokens=150, use_cache=True)
        
        prompt_length = processed["input_ids"].shape[-1]
        decoded = processor.decode(outputs[0][prompt_length:], skip_special_tokens=True)
        return processor.convert_output_to_plaintext_or_grounded_sequence(decoded)
    
    except Exception as e:
        return f"Error: {str(e)}"

# Gradio UI
with gr.Blocks(title="MAIRA-2 Medical Imaging Assistant") as demo:
    gr.Markdown("# MAIRA-2 Medical Imaging Assistant\nAI-powered radiology report generation and phrase grounding")
    
    with gr.Tab("Report Generation"):
        with gr.Row():
            with gr.Column():
                gr.Markdown("## Current Study")
                frontal = gr.Image(label="Frontal View", type="filepath")
                lateral = gr.Image(label="Lateral View", type="filepath")
                indication = gr.Textbox(label="Clinical Indication")
                technique = gr.Textbox(label="Imaging Technique")
                comparison = gr.Textbox(label="Comparison")
                
                gr.Markdown("## Prior Study (Optional)")
                prior_frontal = gr.Image(label="Prior Frontal View", type="filepath")
                prior_lateral = gr.Image(label="Prior Lateral View", type="filepath")
                prior_report = gr.Textbox(label="Prior Report")
                
                grounding = gr.Checkbox(label="Include Grounding")
                sample_btn = gr.Button("Load Sample Data")
                
            with gr.Column():
                report_output = gr.Textbox(label="Generated Report", lines=10)
                generate_btn = gr.Button("Generate Report")
        
        sample_btn.click(load_sample_findings,
                        outputs=[frontal, lateral, indication, technique, comparison,
                               prior_frontal, prior_lateral, prior_report, grounding])
        generate_btn.click(generate_report,
                         inputs=[frontal, lateral, indication, technique, comparison,
                               prior_frontal, prior_lateral, prior_report, grounding],
                         outputs=report_output)
    
    with gr.Tab("Phrase Grounding"):
        with gr.Row():
            with gr.Column():
                pg_frontal = gr.Image(label="Frontal View", type="filepath")
                phrase = gr.Textbox(label="Phrase to Ground")
                pg_sample_btn = gr.Button("Load Sample Data")
            with gr.Column():
                pg_output = gr.Textbox(label="Grounding Result", lines=3)
                pg_btn = gr.Button("Find Phrase")
        
        pg_sample_btn.click(load_sample_phrase,
                           outputs=[pg_frontal, phrase])
        pg_btn.click(ground_phrase,
                    inputs=[pg_frontal, phrase],
                    outputs=pg_output)

demo.launch()