Update app.py

app.py

@@ -107,8 +107,15 @@ def log_system_usage(tag=""):
 
 def clean_response(text: str) -> str:
     text = sanitize_utf8(text)
+    # Remove tool calls, JSON data, and repetitive phrases
     text = re.sub(r"\[TOOL_CALLS\].*", "", text, flags=re.DOTALL)
+    text = re.sub(r"\['get_[^\]]+\']\n?", "", text)  # Remove tool names
+    text = re.sub(r"\{'meta':\s*\{.*?\}\s*,\s*'results':\s*\[.*?\]\}\n?", "", text, flags=re.DOTALL)  # Remove JSON
+    text = re.sub(r"To analyze the medical records for clinical oversights.*?begin by reviewing.*?\n", "", text, flags=re.DOTALL)
     text = re.sub(r"\n{3,}", "\n\n", text).strip()
+    # Only keep text under analysis headings or relevant content
+    if not re.search(r"(Missed Diagnoses|Medication Conflicts|Incomplete Assessments|Urgent Follow-up)", text):
+        return ""
     return text
 
 def init_agent():
@@ -125,7 +132,7 @@ def init_agent():
         tool_files_dict={"new_tool": target_tool_path},
         force_finish=True,
         enable_checker=True,
-        step_rag_num=4,
+        step_rag_num=2,
         seed=100,
         additional_default_tools=[],
     )
@@ -157,8 +164,8 @@ def create_ui(agent):
                     extracted = "\n".join(results)
                     file_hash_value = file_hash(files[0].name) if files else ""
 
-            # Split extracted text into chunks of ~6,000 characters
-            chunk_size = 6000
+            # Split extracted text into chunks of ~4,000 characters
+            chunk_size = 4000
             chunks = [extracted[i:i + chunk_size] for i in range(0, len(extracted), chunk_size)]
             combined_response = ""
 
@@ -190,7 +197,7 @@ Begin analysis:
                 if history and history[-1]["content"].startswith("⏳"):
                     history.pop()
 
-                # Process each chunk and stream results in real-time
+                # Process each chunk and stream cleaned results
                 for chunk_idx, chunk in enumerate(chunks, 1):
                     # Update UI with progress
                     history.append({"role": "assistant", "content": f"🔄 Processing Chunk {chunk_idx} of {len(chunks)}..."})
@@ -215,7 +222,7 @@ Begin analysis:
                                     cleaned = clean_response(m.content)
                                     if cleaned:
                                         chunk_response += cleaned + "\n"
-                                        # Update UI with partial response
+                                        # Stream partial response to UI
                                         if history[-1]["content"].startswith("🔄"):
                                             history[-1] = {"role": "assistant", "content": f"--- Analysis for Chunk {chunk_idx} ---\n{chunk_response.strip()}"}
                                         else:
@@ -225,7 +232,7 @@ Begin analysis:
                             cleaned = clean_response(chunk_output)
                             if cleaned:
                                 chunk_response += cleaned + "\n"
-                                # Update UI with partial response
+                                # Stream partial response to UI
                                 if history[-1]["content"].startswith("🔄"):
                                     history[-1] = {"role": "assistant", "content": f"--- Analysis for Chunk {chunk_idx} ---\n{chunk_response.strip()}"}
                                 else:
@@ -233,7 +240,8 @@ Begin analysis:
                                 yield history, None
 
                     # Append completed chunk response to combined response
-                    combined_response += f"--- Analysis for Chunk {chunk_idx} ---\n{chunk_response}\n"
+                    if chunk_response:
+                        combined_response += f"--- Analysis for Chunk {chunk_idx} ---\n{chunk_response}\n"
 
                 # Finalize UI with complete response
                 if combined_response:
@@ -241,7 +249,7 @@ Begin analysis:
                 else:
                     history.append({"role": "assistant", "content": "No oversights identified."})
 
-                # Generate report file
+                # Generate report file with cleaned response
                 report_path = os.path.join(report_dir, f"{file_hash_value}_report.txt") if file_hash_value else None
                 if report_path:
                     with open(report_path, "w", encoding="utf-8") as f: