Update app.py

app.py

@@ -57,6 +57,7 @@ def extract_medical_data(df: pd.DataFrame) -> Dict[str, Any]:
     
     for _, row in df.iterrows():
         record = {
+            'booking': row.get('Booking Number', ''),
             'form_name': row.get('Form Name', ''),
             'form_item': row.get('Form Item', ''),
             'response': row.get('Item Response', ''),
@@ -69,117 +70,134 @@ def extract_medical_data(df: pd.DataFrame) -> Dict[str, Any]:
     return medical_data
 
 def identify_red_flags(records: List[Dict[str, Any]]) -> Dict[str, Any]:
-    """Identify potential red flags in medical records"""
+    """Identify potential red flags across all medical records"""
     red_flags = {
         'symptoms': defaultdict(list),
         'medications': defaultdict(list),
         'diagnoses': defaultdict(list),
         'vitals': defaultdict(list),
-        'labs': defaultdict(list)
+        'labs': defaultdict(list),
+        'patients': defaultdict(list)
     }
     
-    for record in records:
-        form_name = record['form_name'].lower()
-        item = record['form_item'].lower()
-        response = record['response'].lower()
-        
-        # Symptom patterns
-        if 'pain' in item or 'symptom' in form_name:
-            if 'severe' in response or 'chronic' in response:
-                red_flags['symptoms'][item].append(response)
-        
-        # Medication checks
-        elif 'medication' in form_name or 'drug' in form_name:
-            if 'interaction' in response or 'allergy' in response:
-                red_flags['medications'][item].append(response)
-        
-        # Diagnosis inconsistencies
-        elif 'diagnosis' in form_name:
-            if 'rule out' in response or 'possible' in response:
-                red_flags['diagnoses'][item].append(response)
-        
-        # Abnormal vitals
-        elif 'vital' in form_name:
-            try:
-                value = float(re.search(r'\d+\.?\d*', response).group())
-                if ('blood pressure' in item and value > 140) or \
-                   ('heart rate' in item and (value < 50 or value > 100)) or \
-                   ('temperature' in item and value > 38):
-                    red_flags['vitals'][item].append(response)
-            except:
-                pass
-        
-        # Abnormal labs
-        elif 'lab' in form_name or 'test' in form_name:
-            if 'abnormal' in response or 'high' in response or 'low' in response:
-                red_flags['labs'][item].append(response)
+    for booking, patient_records in records.items():
+        for record in patient_records:
+            form_name = record['form_name'].lower()
+            item = record['form_item'].lower()
+            response = record['response'].lower()
+            
+            # Symptom patterns
+            if 'pain' in item or 'symptom' in form_name:
+                if 'severe' in response or 'chronic' in response:
+                    red_flags['symptoms'][item].append((booking, response))
+            
+            # Medication checks
+            elif 'medication' in form_name or 'drug' in form_name:
+                if 'interaction' in response or 'allergy' in response:
+                    red_flags['medications'][item].append((booking, response))
+            
+            # Diagnosis inconsistencies
+            elif 'diagnosis' in form_name:
+                if 'rule out' in response or 'possible' in response:
+                    red_flags['diagnoses'][item].append((booking, response))
+            
+            # Abnormal vitals
+            elif 'vital' in form_name:
+                try:
+                    value = float(re.search(r'\d+\.?\d*', response).group())
+                    if ('blood pressure' in item and value > 140) or \
+                       ('heart rate' in item and (value < 50 or value > 100)) or \
+                       ('temperature' in item and value > 38):
+                        red_flags['vitals'][item].append((booking, response))
+                except:
+                    pass
+            
+            # Abnormal labs
+            elif 'lab' in form_name or 'test' in form_name:
+                if 'abnormal' in response or 'high' in response or 'low' in response:
+                    red_flags['labs'][item].append((booking, response))
     
     return red_flags
 
-def generate_analysis_prompt(booking: str, records: List[Dict[str, Any]], red_flags: Dict[str, Any]) -> str:
-    """Generate structured prompt for analysis"""
-    records_text = "\n".join(
-        f"- {r['form_name']}: {r['form_item']} = {r['response']} ({r['date']} by {r['interviewer']})\n  {r['description']}"
-        for r in records
-    )
+def generate_combined_prompt(all_records: Dict[str, Any], red_flags: Dict[str, Any]]) -> str:
+    """Generate a single comprehensive prompt for all patient data"""
+    # Create summary of all records
+    records_summary = []
+    for booking, records in all_records.items():
+        records_summary.append(f"\n## Patient {booking}")
+        for r in records:
+            records_summary.append(
+                f"- {r['form_name']}: {r['form_item']} = {r['response']} "
+                f"({r['date']} by {r['interviewer']})\n  {r['description']}"
+            )
     
-    red_flags_text = "\n".join(
-        f"### {category.capitalize()} Red Flags\n" + "\n".join(
-            f"- {item}: {', '.join(responses)}"
-            for item, responses in items.items()
-        )
-        for category, items in red_flags.items() if items
-    )
+    # Format red flags with patient references
+    red_flags_text = []
+    for category, items in red_flags.items():
+        if items:
+            red_flags_text.append(f"\n### {category.capitalize()} Red Flags")
+            for item, entries in items.items():
+                patient_entries = defaultdict(list)
+                for booking, response in entries:
+                    patient_entries[booking].append(response)
+                
+                for booking, responses in patient_entries.items():
+                    red_flags_text.append(
+                        f"- {item} (Patient {booking}): {', '.join(responses)}"
+                    )
     
     prompt = f"""
-**Patient Booking Number**: {booking}
+**COMPREHENSIVE PATIENT ANALYSIS**
 
 **Medical Records Summary**:
-{records_text}
+{"".join(records_summary)}
 
-**Identified Red Flags**:
-{red_flags_text if red_flags_text else "No obvious red flags detected"}
+**Identified Red Flags Across All Patients**:
+{"".join(red_flags_text) if red_flags_text else "No obvious red flags detected"}
 
-**Comprehensive Analysis Instructions**:
-1. Review all medical data and red flags above
-2. Identify any potential missed diagnoses based on symptoms, labs, and clinical findings
-3. Check for medication conflicts or inappropriate prescriptions
-4. Note any incomplete assessments or missing diagnostic workups
-5. Flag any urgent follow-up needs or critical findings
-6. Provide recommendations in clear, actionable terms
+**Analysis Instructions**:
+1. Review ALL patient data holistically
+2. Identify patterns that might indicate systemic issues
+3. Check for recurring medication problems across patients
+4. Note any common missed diagnoses
+5. Flag any urgent cases needing immediate attention
+6. Provide overall clinical recommendations
 
 **Required Output Format**:
-### Missed Diagnoses
-- [List any conditions that may have been overlooked based on the data]
+### Summary of Findings
+[Overview of most significant findings across all patients]
+
+### Common Missed Diagnoses
+- [Conditions frequently overlooked across multiple patients]
+- [Specific patients affected: Booking numbers]
 
-### Medication Issues
-- [List any medication conflicts, inappropriate prescriptions, or missing medications]
+### Recurring Medication Issues
+- [Common drug interactions or inappropriate prescriptions]
+- [Patients affected]
 
-### Assessment Gaps
-- [List any incomplete assessments or missing diagnostic tests]
+### Systemic Assessment Gaps
+- [Patterns of incomplete assessments across patients]
+- [Recommended additional tests]
 
-### Urgent Follow-up
-- [List any findings requiring immediate attention]
+### Critical Cases Needing Follow-up
+- [Patients requiring urgent attention]
+- [Specific reasons]
 
-### Clinical Recommendations
-- [Provide specific recommendations for next steps]
+### Overall Recommendations
+- [General recommendations for clinical practice]
+- [Specific actions for different patient groups]
 """
     return prompt
 
-def parse_excel_to_prompts(file_path: str) -> List[Tuple[str, str]]:
-    """Parse Excel file into analysis prompts with red flag detection"""
+def parse_excel_to_combined_prompt(file_path: str) -> str:
+    """Parse Excel file into a single comprehensive analysis prompt"""
     try:
         xl = pd.ExcelFile(file_path)
         df = xl.parse(xl.sheet_names[0], header=0).fillna("")
         medical_data = extract_medical_data(df)
-        
-        prompts = []
-        for booking, records in medical_data.items():
-            red_flags = identify_red_flags(records)
-            prompt = generate_analysis_prompt(booking, records, red_flags)
-            prompts.append((booking, prompt))
-        
-        return prompts
+        red_flags = identify_red_flags(medical_data)
+        prompt = generate_combined_prompt(medical_data, red_flags)
+        return prompt
     except Exception as e:
         raise ValueError(f"Error parsing Excel file: {str(e)}")
 
@@ -204,14 +222,10 @@ def init_agent():
     agent.init_model()
     return agent
 
-def format_markdown(text: str) -> str:
-    """Convert markdown text to HTML for better display"""
-    return markdown.markdown(text, extensions=['fenced_code', 'tables'])
-
 def create_ui(agent):
     """Create Gradio UI interface"""
     with gr.Blocks(theme=gr.themes.Soft(), title="Clinical Oversight Assistant") as demo:
-        gr.Markdown("# 🏥 Clinical Oversight Assistant (Missed Diagnosis Detection)")
+        gr.Markdown("# 🏥 Comprehensive Clinical Analysis")
         
         with gr.Tabs():
             with gr.TabItem("Analysis"):
@@ -231,12 +245,12 @@ def create_ui(agent):
                         )
                         with gr.Row():
                             clear_btn = gr.Button("Clear", variant="secondary")
-                            send_btn = gr.Button("Analyze", variant="primary")
+                            send_btn = gr.Button("Analyze All Patients", variant="primary")
                     
                     # Right column - Outputs
                     with gr.Column(scale=2):
                         chatbot = gr.Chatbot(
-                            label="Analysis Results",
+                            label="Comprehensive Analysis Results",
                             height=600,
                             bubble_full_width=False,
                             show_copy_button=True,
@@ -253,94 +267,62 @@ def create_ui(agent):
                 
                 1. **Upload Excel File**: Select your patient records Excel file
                 2. **Add Instructions** (Optional): Provide any specific analysis requests
-                3. **Click Analyze**: The system will process each patient record
-                4. **Review Results**: Analysis appears in the chat window
-                5. **Download Report**: Get a full text report of all findings
+                3. **Click Analyze**: The system will process ALL patient records together
+                4. **Review Results**: Comprehensive analysis appears in the chat window
+                5. **Download Report**: Get a complete text report of all findings
                 
-                ### Excel File Requirements
-                Your Excel file must contain these columns:
-                - Booking Number (patient identifier)
-                - Form Name (type of medical form)
-                - Form Item (specific field name)
-                - Item Response (patient response or value)
-                - Interview Date (date of recording)
-                - Interviewer (who recorded the data)
-                - Description (additional notes)
-                
-                ### Analysis Includes
-                - **Missed diagnoses**: Potential conditions not identified
-                - **Medication issues**: Conflicts, side effects, inappropriate prescriptions
-                - **Assessment gaps**: Missing tests or incomplete evaluations
-                - **Urgent follow-up**: Critical findings needing immediate attention
-                - **Clinical recommendations**: Actionable next steps
+                ### Key Features
+                - **Holistic analysis** of all patient records
+                - **Pattern detection** across multiple patients
+                - **Systemic issues** identification
+                - **Prioritized recommendations** based on severity
                 """)
         
-        def format_message(role: str, content: str) -> Tuple[str, str]:
-            """Format messages for the chatbot in (user, bot) format"""
-            if role == "user":
-                return (content, None)
-            else:
-                return (None, content)
-        
         def analyze(message: str, chat_history: List[Tuple[str, str]], file) -> Tuple[List[Tuple[str, str]], str]:
-            """Main analysis function"""
+            """Main analysis function for all patients"""
             if not file:
                 raise gr.Error("Please upload an Excel file first")
             
             try:
-                # Initialize chat history with user message
-                new_history = chat_history + [format_message("user", message)]
-                new_history.append(format_message("assistant", "⏳ Processing Excel data..."))
+                # Initialize chat history
+                new_history = chat_history + [(message, None)]
+                new_history.append((None, "⏳ Processing all patient data..."))
                 yield new_history, None
                 
-                prompts = parse_excel_to_prompts(file.name)
-                full_output = ""
+                # Generate combined prompt
+                prompt = parse_excel_to_combined_prompt(file.name)
                 
-                for idx, (booking, prompt) in enumerate(prompts, 1):
-                    chunk_output = ""
-                    try:
-                        for result in agent.run_gradio_chat(
-                            message=prompt,
-                            history=[],
-                            temperature=0.2,
-                            max_new_tokens=1024,
-                            max_token=4096,
-                            call_agent=False,
-                            conversation=[],
-                        ):
-                            if isinstance(result, list):
-                                for r in result:
-                                    if hasattr(r, 'content') and r.content:
-                                        cleaned = clean_response(r.content)
-                                        chunk_output += cleaned + "\n"
-                            elif isinstance(result, str):
-                                cleaned = clean_response(result)
-                                chunk_output += cleaned + "\n"
-                            
-                            if chunk_output:
-                                output = f"## Patient Booking: {booking}\n{chunk_output.strip()}\n"
-                                new_history[-1] = format_message("assistant", output)
-                                yield new_history, None
-                                
-                    except Exception as e:
-                        error_msg = f"⚠️ Error processing booking {booking}: {str(e)}"
-                        new_history.append(format_message("assistant", error_msg))
-                        yield new_history, None
-                        continue
-                        
-                    if chunk_output:
-                        output = f"## Patient Booking: {booking}\n{chunk_output.strip()}\n"
-                        new_history.append(format_message("assistant", output))
-                        full_output += output + "\n"
+                # Run analysis
+                full_output = ""
+                for result in agent.run_gradio_chat(
+                    message=prompt,
+                    history=[],
+                    temperature=0.2,
+                    max_new_tokens=2048,  # Increased for comprehensive analysis
+                    max_token=4096,
+                    call_agent=False,
+                    conversation=[],
+                ):
+                    if isinstance(result, list):
+                        for r in result:
+                            if hasattr(r, 'content') and r.content:
+                                cleaned = clean_response(r.content)
+                                full_output += cleaned + "\n"
+                    elif isinstance(result, str):
+                        cleaned = clean_response(result)
+                        full_output += cleaned + "\n"
+                    
+                    if full_output:
+                        new_history[-1] = (None, full_output.strip())
                         yield new_history, None
                 
                 # Save report
                 file_hash_value = file_hash(file.name)
                 timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
-                report_path = os.path.join(report_dir, f"{file_hash_value}_{timestamp}_report.md")
+                report_path = os.path.join(report_dir, f"comprehensive_{file_hash_value}_{timestamp}_report.md")
                 
                 with open(report_path, "w", encoding="utf-8") as f:
-                    f.write("# Clinical Oversight Analysis Report\n\n")
+                    f.write("# Comprehensive Clinical Analysis Report\n\n")
                     f.write(f"**Generated on**: {timestamp}\n\n")
                     f.write(f"**Source file**: {file.name}\n\n")
                     f.write(full_output)
@@ -348,7 +330,7 @@ def create_ui(agent):
                 yield new_history, report_path if os.path.exists(report_path) else None
                 
             except Exception as e:
-                new_history.append(format_message("assistant", f"❌ Error: {str(e)}"))
+                new_history.append((None, f"❌ Error: {str(e)}"))
                 yield new_history, None
                 raise gr.Error(f"Analysis failed: {str(e)}")