Default output to 'output' instead of 'telemetry_output'. Save eval output to DF (#32)

.gitignore

@@ -167,4 +167,4 @@ cython_debug/
 .idea/
 .vscode/
 
-telemetry_output
+output

src/surf_spot_finder/evaluation/evaluate.py

@@ -1,9 +1,11 @@
 import json
+import os
 import sys
 from textwrap import dedent
 from typing import Any, Dict, List, Optional
 from loguru import logger
 from fire import Fire
+import pandas as pd
 from surf_spot_finder.cli import find_surf_spot
 from surf_spot_finder.config import (
     Config,
@@ -53,9 +55,6 @@ def evaluate_telemetry(test_case: TestCase, telemetry_path: str) -> bool:
     # Extract the final answer from the telemetry
     processor = TelemetryProcessor.create(agent_type)
     hypothesis_answer = processor.extract_hypothesis_answer(trace=telemetry)
-    logger.info(
-        f"""<yellow>Hypothesis Final answer extracted: {hypothesis_answer}</yellow>"""
-    )
     # Verify agent behavior against checkpoints using llm-as-a-judge
     llm_judge = "openai/gpt-4o"
     checkpoint_results = verify_checkpoints(
@@ -72,6 +71,10 @@ def evaluate_telemetry(test_case: TestCase, telemetry_path: str) -> bool:
         model=llm_judge,
     )
     # Summarize results
+    output_message = ""
+    output_message += (
+        f"""<yellow>Hypothesis Final answer extracted: {hypothesis_answer}</yellow>\n"""
+    )
 
     verification_results = checkpoint_results + hypothesis_answer_results
     failed_checks = [r for r in verification_results if not r.passed]
@@ -86,7 +89,7 @@ def evaluate_telemetry(test_case: TestCase, telemetry_path: str) -> bool:
                 - {check.criteria}
                 - {check.reason}</green>"""
             )
-            logger.info(message)
+            output_message += message + "\n"
     if failed_checks:
         for check in failed_checks:
             message = dedent(
@@ -95,14 +98,43 @@ def evaluate_telemetry(test_case: TestCase, telemetry_path: str) -> bool:
                 - {check.criteria}
                 - {check.reason}</red>"""
             )
-            logger.error(message)
+            output_message += message + "\n"
     else:
-        logger.info("<green>All checkpoints passed!</green>")
-    logger.info(f"<green>Passed checkpoints: {len(passed_checks)}</green>")
-    logger.info(f"<red>Failed checkpoints: {len(failed_checks)}</red>")
-    logger.info("<green>=====================================</green>")
-    logger.info(f"<green>Score: {won_points}/{won_points + missed_points}</green>")
-    logger.info("<green>=====================================</green>")
+        output_message += "<green>All checkpoints passed!</green>\n"
+    output_message += f"<green>Passed checkpoints: {len(passed_checks)}</green>\n"
+    output_message += f"<red>Failed checkpoints: {len(failed_checks)}</red>\n"
+    output_message += "<green>=====================================</green>\n"
+    output_message += (
+        f"<green>Score: {won_points}/{won_points + missed_points}</green>\n"
+    )
+    output_message += "<green>=====================================</green>\n"
+    logger.info(output_message)
+    # See if the test_case.output_path file exists.
+    if os.path.exists(test_case.output_path):
+        df = pd.read_json(test_case.output_path, orient="records", lines=True)
+    else:
+        df = pd.DataFrame()
+    df = pd.concat(
+        [
+            df,
+            pd.DataFrame(
+                [
+                    {
+                        "test_case_path": test_case.test_case_path,
+                        "output_message": output_message,
+                        "telemetry_path": telemetry_path,
+                        "hypothesis_answer": hypothesis_answer,
+                        "passed_checks": len(passed_checks),
+                        "failed_checks": len(failed_checks),
+                        "score": round(
+                            won_points / (won_points + missed_points) * 100, 2
+                        ),
+                    }
+                ]
+            ),
+        ]
+    )
+    df.to_json(test_case.output_path, orient="records", lines=True)
 
 
 def evaluate(

src/surf_spot_finder/evaluation/telemetry/langchain_telemetry.py

@@ -20,8 +20,15 @@ class LangchainTelemetryProcessor(TelemetryProcessor):
                 message = json.loads(content)["messages"][0]
                 message = self.parse_generic_key_value_string(message)
                 base_message = BaseMessage(content=message["content"], type="AGENT")
-                print(base_message.text())
-                return base_message.text()
+                # Use the interpreted string for printing
+                final_text = base_message.text()
+                # Either decode escape sequences if they're present
+                try:
+                    final_text = final_text.encode().decode("unicode_escape")
+                except UnicodeDecodeError:
+                    # If that fails, the escape sequences might already be interpreted
+                    pass
+                return final_text
 
         raise ValueError("No agent final answer found in trace")
 

src/surf_spot_finder/evaluation/telemetry/openai_telemetry.py

@@ -70,7 +70,8 @@ class OpenAITelemetryProcessor(TelemetryProcessor):
                     "tool_name": tool_name,
                     "input": attributes.get("input.value", ""),
                     "output": tool_output,
-                    "status": span.get("status", {}).get("status_code"),
+                    # Can't add status yet because it isn't being set by openinference
+                    # "status": span.get("status", {}).get("status_code"),
                 }
                 span_info["input"] = json.loads(span_info["input"])
 

src/surf_spot_finder/evaluation/test_case.py

@@ -27,6 +27,8 @@ class TestCase(BaseModel):
     ground_truth: List[Dict[str, Any]] = Field(default_factory=list)
     checkpoints: List[CheckpointCriteria] = Field(default_factory=list)
     final_answer_criteria: List[CheckpointCriteria] = Field(default_factory=list)
+    test_case_path: str
+    output_path: str = "output/results.json"
 
     @classmethod
     def from_yaml(cls, test_case_path: str) -> "TestCase":
@@ -56,4 +58,6 @@ class TestCase(BaseModel):
             item for item in test_case_dict["ground_truth"] if isinstance(item, dict)
         ]
 
+        test_case_dict["test_case_path"] = test_case_path
+
         return cls.model_validate(test_case_dict)

src/surf_spot_finder/tracing.py

@@ -46,7 +46,7 @@ def get_tracer_provider(
     project_name: str,
     json_tracer: bool,
     agent_type: AgentType,
-    output_dir: str = "telemetry_output",
+    output_dir: str = "output",
 ) -> tuple[TracerProvider, str | None]:
     """
     Create a tracer_provider based on the selected mode.
@@ -57,7 +57,7 @@ def get_tracer_provider(
         agent_type: The type of agent being used.
         output_dir: The directory where the telemetry output will be stored.
             Only used if `json_tracer=True`.
-            Defaults to "telemetry_output".
+            Defaults to "output".
 
     Returns:
         tracer_provider: The configured tracer provider