import time import gradio as gr import torch import numpy as np from PIL import Image import time import io import subprocess import sys # Install required packages def install_packages(): packages = [ "transformers", "accelerate", "timm", "easyocr" ] for package in packages: try: subprocess.check_call([sys.executable, "-m", "pip", "install", package]) except: print(f"Warning: Could not install {package}") # Install packages at startup install_packages() from transformers import AutoProcessor, AutoModelForImageTextToText, AutoConfig # Global variables for model processor = None model = None config = None ocr_reader = None def load_model(): """Load the Gemma 3n model""" global processor, model, config, ocr_reader try: print("🚀 Loading Gemma 3n model...") GEMMA_PATH = "google/gemma-3n-e2b-it" # Load configuration config = AutoConfig.from_pretrained(GEMMA_PATH, trust_remote_code=True) print("✅ Config loaded") # Load processor processor = AutoProcessor.from_pretrained(GEMMA_PATH, trust_remote_code=True) print("✅ Processor loaded") # Load model model = AutoModelForImageTextToText.from_pretrained( GEMMA_PATH, config=config, torch_dtype="auto", device_map="auto", trust_remote_code=True ) print("✅ Model loaded successfully!") # Set up compilation fix import torch._dynamo torch._dynamo.config.suppress_errors = True # Initialize OCR try: import easyocr ocr_reader = easyocr.Reader(['en'], gpu=False, verbose=False) print("✅ EasyOCR initialized") except Exception as e: print(f"⚠️ EasyOCR not available: {e}") ocr_reader = None return True except Exception as e: print(f"❌ Model loading failed: {e}") return False def generate_soap_note(text): """Generate SOAP note using Gemma 3n""" if model is None or processor is None: return "❌ Model not loaded. Please wait for initialization." soap_prompt = f"""You are a medical AI assistant. Convert the following medical notes into a properly formatted SOAP note. Medical notes: {text} Please format as: S - SUBJECTIVE: (chief complaint, history of present illness, past medical history, medications, allergies) O - OBJECTIVE: (vital signs, physical examination findings) A - ASSESSMENT: (diagnosis/clinical impression) P - PLAN: (treatment plan, follow-up instructions) Generate a complete, professional SOAP note:""" messages = [{ "role": "system", "content": [{"type": "text", "text": "You are an expert medical AI assistant specialized in creating SOAP notes from medical documentation."}] }, { "role": "user", "content": [{"type": "text", "text": soap_prompt}] }] try: inputs = processor.apply_chat_template( messages, add_generation_prompt=True, tokenize=True, return_dict=True, return_tensors="pt" ).to(model.device) input_len = inputs["input_ids"].shape[-1] with torch.no_grad(): outputs = model.generate( **inputs, max_new_tokens=400, do_sample=True, temperature=0.1, top_p=0.95, pad_token_id=processor.tokenizer.eos_token_id, disable_compile=True ) response = processor.batch_decode( outputs[:, input_len:], skip_special_tokens=True )[0].strip() return response except Exception as e: return f"❌ SOAP generation failed: {str(e)}" def extract_text_from_image(image): """Extract text using EasyOCR - fast processing""" if ocr_reader is None: return "❌ OCR not available" try: if hasattr(image, 'convert'): image = image.convert('RGB') img_array = np.array(image) results = ocr_reader.readtext(img_array, detail=0, paragraph=True) if results: return ' '.join(results).strip() else: return "❌ No text detected in image" except Exception as e: return f"❌ OCR failed: {str(e)}" def process_medical_input(image, text): """Main processing function for the Gradio interface""" if image is not None and text.strip(): return "⚠️ Please provide either an image OR text, not both.", "" if image is not None: # Process image print("🔍 Extracting text from image...") extracted_text = extract_text_from_image(image) if extracted_text.startswith('❌'): return extracted_text, "" print("🤖 Generating SOAP note...") soap_note = generate_soap_note(extracted_text) return extracted_text, soap_note elif text.strip(): # Process text directly print("🤖 Generating SOAP note from text...") soap_note = generate_soap_note(text.strip()) return text.strip(), soap_note else: return "❌ Please provide either an image or text input.", "" def create_demo(): """Create the Gradio demo interface""" # Sample text for demonstration sample_text = """Patient: John Smith, 45yo male CC: Chest pain Vitals: BP 140/90, HR 88, RR 16, O2 98%, Temp 98.6F HPI: Patient reports crushing chest pain x 2 hours, radiating to left arm PMH: HTN, DM Type 2 Current Meds: Lisinopril 10mg daily, Metformin 500mg BID PE: Diaphoretic, anxious appearance EKG: ST elevation in leads II, III, aVF""" with gr.Blocks(title="Medical OCR SOAP Generator", theme=gr.themes.Soft()) as demo: gr.HTML("""

🏥 Medical OCR SOAP Generator - LIVE DEMO

🎯 For Competition Judges - Quick 2-Minute Demo:

📋 SAMPLE IMAGE PROVIDED:

👆 Download "docs-note-to-upload.jpg" from the Files tab above, then upload it below

OR click "Try Sample Medical Text" button for instant text demo

Demo Steps:

  1. Upload the sample image (docs-note-to-upload.jpg from Files tab) OR click sample text button
  2. Click "Generate SOAP Note"
  3. Wait ~2 minutes for AI processing (first time only)
  4. See professional SOAP note generated by Gemma 3n

✅ What This Demo Shows:

⚠️ Note: First generation takes ~2 minutes as model loads. Subsequent ones are faster.


""") with gr.Row(): with gr.Column(): image_input = gr.Image( type="pil", label="📷 Upload Medical Image", height=300 ) text_input = gr.Textbox( label="📝 Or Enter Medical Text", placeholder=sample_text, lines=8, max_lines=15 ) submit_btn = gr.Button( "Generate SOAP Note", variant="primary", size="lg" ) with gr.Column(): extracted_output = gr.Textbox( label="📋 Extracted/Input Text", lines=6, max_lines=10 ) soap_output = gr.Textbox( label="🏥 Generated SOAP Note", lines=12, max_lines=20 ) # Example section gr.Markdown("### 📋 Quick Test Example") example_btn = gr.Button("Try Sample Medical Text", variant="secondary") def load_example(): return sample_text, None example_btn.click( load_example, outputs=[text_input, image_input] ) # Process function submit_btn.click( process_medical_input, inputs=[image_input, text_input], outputs=[extracted_output, soap_output] ) gr.Markdown(""" --- **About:** This application uses Google's Gemma 3n model for medical text understanding and EasyOCR for handwriting recognition. All processing is done locally for HIPAA compliance. **Competition Entry:** Medical AI Innovation Challenge 2024 """) return demo # Initialize the application if __name__ == "__main__": print("🚀 Starting Medical OCR SOAP Generator...") # Load model model_loaded = load_model() if model_loaded: print("✅ All systems ready!") demo = create_demo() demo.launch( share=True, server_name="0.0.0.0", server_port=7860 ) else: print("❌ Failed to load model. Creating fallback demo...") def fallback_demo(): return "❌ Model loading failed. Please check the logs.", "❌ Model not available." demo = gr.Interface( fn=fallback_demo, inputs=[ gr.Image(type="pil", label="Upload Medical Image"), gr.Textbox(label="Enter Medical Text", lines=5) ], outputs=[ gr.Textbox(label="Status"), gr.Textbox(label="Error Message") ], title="❌ Medical OCR - Model Loading Failed" ) demo.launch(share=True) import io import subprocess import sys import cv2 # Install required packages def install_packages(): packages = [ "transformers", "accelerate", "timm", "easyocr", "opencv-python" ] for package in packages: try: subprocess.check_call([sys.executable, "-m", "pip", "install", package]) except: print(f"Warning: Could not install {package}") # Install packages at startup install_packages() from transformers import AutoProcessor, AutoModelForImageTextToText, AutoConfig # Global variables for model processor = None model = None config = None ocr_reader = None def load_model(): """Load the Gemma 3n model""" global processor, model, config, ocr_reader try: print("🚀 Loading Gemma 3n model...") GEMMA_PATH = "google/gemma-3n-e2b-it" # Load configuration config = AutoConfig.from_pretrained(GEMMA_PATH, trust_remote_code=True) print("✅ Config loaded") # Load processor processor = AutoProcessor.from_pretrained(GEMMA_PATH, trust_remote_code=True) print("✅ Processor loaded") # Load model model = AutoModelForImageTextToText.from_pretrained( GEMMA_PATH, config=config, torch_dtype="auto", device_map="auto", trust_remote_code=True ) print("✅ Model loaded successfully!") # Set up compilation fix import torch._dynamo torch._dynamo.config.suppress_errors = True # Initialize OCR try: import easyocr ocr_reader = easyocr.Reader(['en'], gpu=False, verbose=False) print("✅ EasyOCR initialized") except Exception as e: print(f"⚠️ EasyOCR not available: {e}") ocr_reader = None return True except Exception as e: print(f"❌ Model loading failed: {e}") return False def generate_soap_note(text): """Generate SOAP note using Gemma 3n""" if model is None or processor is None: return "❌ Model not loaded. Please wait for initialization." soap_prompt = f"""You are a medical AI assistant. Convert the following medical notes into a properly formatted SOAP note. Medical notes: {text} Please format as: S - SUBJECTIVE: (chief complaint, history of present illness, past medical history, medications, allergies) O - OBJECTIVE: (vital signs, physical examination findings) A - ASSESSMENT: (diagnosis/clinical impression) P - PLAN: (treatment plan, follow-up instructions) Generate a complete, professional SOAP note:""" messages = [{ "role": "system", "content": [{"type": "text", "text": "You are an expert medical AI assistant specialized in creating SOAP notes from medical documentation."}] }, { "role": "user", "content": [{"type": "text", "text": soap_prompt}] }] try: inputs = processor.apply_chat_template( messages, add_generation_prompt=True, tokenize=True, return_dict=True, return_tensors="pt" ).to(model.device) input_len = inputs["input_ids"].shape[-1] with torch.no_grad(): outputs = model.generate( **inputs, max_new_tokens=400, do_sample=True, temperature=0.1, top_p=0.95, pad_token_id=processor.tokenizer.eos_token_id, disable_compile=True ) response = processor.batch_decode( outputs[:, input_len:], skip_special_tokens=True )[0].strip() return response except Exception as e: return f"❌ SOAP generation failed: {str(e)}" def preprocess_image_for_ocr(image): """Preprocess image for better OCR results using CLAHE""" try: if hasattr(image, 'convert'): image = image.convert('RGB') img_array = np.array(image) # Convert to grayscale if len(img_array.shape) == 3: gray = cv2.cvtColor(img_array, cv2.COLOR_RGB2GRAY) else: gray = img_array # Resize if too small height, width = gray.shape if height < 300 or width < 300: scale = max(300/height, 300/width) new_height = int(height * scale) new_width = int(width * scale) gray = cv2.resize(gray, (new_width, new_height), interpolation=cv2.INTER_CUBIC) # Enhance image with CLAHE gray = cv2.medianBlur(gray, 3) clahe = cv2.createCLAHE(clipLimit=2.0, tileGridSize=(8,8)) gray = clahe.apply(gray) _, gray = cv2.threshold(gray, 0, 255, cv2.THRESH_BINARY + cv2.THRESH_OTSU) return gray except Exception as e: print(f"⚠️ Image preprocessing failed: {e}") # Fallback to original image if preprocessing fails return np.array(image) def extract_text_from_image(image): """Extract text using EasyOCR with CLAHE preprocessing""" if ocr_reader is None: return "❌ OCR not available" try: # Apply CLAHE preprocessing for better OCR processed_img = preprocess_image_for_ocr(image) results = ocr_reader.readtext(processed_img, detail=0, paragraph=True) if results: return ' '.join(results).strip() else: return "❌ No text detected in image" except Exception as e: return f"❌ OCR failed: {str(e)}" def process_medical_input(image, text): """Main processing function for the Gradio interface""" if image is not None and text.strip(): return "⚠️ Please provide either an image OR text, not both.", "" if image is not None: # Process image print("🔍 Extracting text from image...") extracted_text = extract_text_from_image(image) if extracted_text.startswith('❌'): return extracted_text, "" print("🤖 Generating SOAP note...") soap_note = generate_soap_note(extracted_text) return extracted_text, soap_note elif text.strip(): # Process text directly print("🤖 Generating SOAP note from text...") soap_note = generate_soap_note(text.strip()) return text.strip(), soap_note else: return "❌ Please provide either an image or text input.", "" def create_demo(): """Create the Gradio demo interface""" # Sample text for demonstration sample_text = """Patient: John Smith, 45yo male CC: Chest pain Vitals: BP 140/90, HR 88, RR 16, O2 98%, Temp 98.6F HPI: Patient reports crushing chest pain x 2 hours, radiating to left arm PMH: HTN, DM Type 2 Current Meds: Lisinopril 10mg daily, Metformin 500mg BID PE: Diaphoretic, anxious appearance EKG: ST elevation in leads II, III, aVF""" with gr.Blocks(title="Medical OCR SOAP Generator", theme=gr.themes.Soft()) as demo: gr.HTML("""

🏥 Medical OCR SOAP Generator - LIVE DEMO

🎯 For Competition Judges - Quick 2-Minute Demo:

📋 SAMPLE IMAGE PROVIDED:

👆 Download "docs-note-to-upload.jpg" from the Files tab above, then upload it below

OR click "Try Sample Medical Text" button for instant text demo

Demo Steps:

  1. Upload the sample image (docs-note-to-upload.jpg from Files tab) OR click sample text button
  2. Click "Generate SOAP Note"
  3. Wait ~60-90 seconds for AI processing (first time only)
  4. See professional SOAP note generated by Gemma 3n

✅ What This Demo Shows:

⚠️ Note: First generation takes ~60-90 seconds as model loads. Subsequent ones are faster.


""") with gr.Row(): with gr.Column(): image_input = gr.Image( type="pil", label="📷 Upload Medical Image", height=300 ) text_input = gr.Textbox( label="📝 Or Enter Medical Text", placeholder=sample_text, lines=8, max_lines=15 ) submit_btn = gr.Button( "Generate SOAP Note", variant="primary", size="lg" ) with gr.Column(): extracted_output = gr.Textbox( label="📋 Extracted/Input Text", lines=6, max_lines=10 ) soap_output = gr.Textbox( label="🏥 Generated SOAP Note", lines=12, max_lines=20 ) # Example section gr.Markdown("### 📋 Quick Test Example") example_btn = gr.Button("Try Sample Medical Text", variant="secondary") def load_example(): return sample_text, None example_btn.click( load_example, outputs=[text_input, image_input] ) # Process function submit_btn.click( process_medical_input, inputs=[image_input, text_input], outputs=[extracted_output, soap_output] ) gr.Markdown(""" --- **About:** This application uses Google's Gemma 3n model for medical text understanding and EasyOCR for handwriting recognition. All processing is done locally for HIPAA compliance. **Competition Entry:** Medical AI Innovation Challenge 2024 """) return demo # Initialize the application if __name__ == "__main__": print("🚀 Starting Medical OCR SOAP Generator...") # Load model model_loaded = load_model() if model_loaded: print("✅ All systems ready!") demo = create_demo() demo.launch( share=True, server_name="0.0.0.0", server_port=7860 ) else: print("❌ Failed to load model. Creating fallback demo...") def fallback_demo(): return "❌ Model loading failed. Please check the logs.", "❌ Model not available." demo = gr.Interface( fn=fallback_demo, inputs=[ gr.Image(type="pil", label="Upload Medical Image"), gr.Textbox(label="Enter Medical Text", lines=5) ], outputs=[ gr.Textbox(label="Status"), gr.Textbox(label="Error Message") ], title="❌ Medical OCR - Model Loading Failed" ) demo.launch(share=True)