update sampled

app.py

@@ -21,13 +21,13 @@ Note that not every model has a response for every puzzle.
 import gradio as gr
 import pandas as pd
 import numpy as np
-from metrics import load_results, accuracy_by_model_and_time
+from metrics import load_results_sample_one_only, accuracy_by_model_and_time
 import metrics
 from pathlib import Path
 
 def get_model_response(prompt_id, model_name):
     query = f"""
-        SELECT completion FROM results.completions 
+        SELECT completion FROM sampled
         WHERE prompt_id = {prompt_id} AND parent_dir = '{model_name}'
     """
     response = conn.sql(query).fetchone()
@@ -56,10 +56,10 @@ def display_model_response(puzzle_id, model_name, show_thoughts):
         return response.strip()
 
 
-conn = load_results()
+conn = load_results_sample_one_only()
 
 # Get all unique model names
-model_names = [item[0] for item in conn.sql("SELECT DISTINCT parent_dir FROM results.completions").fetchall()]
+model_names = [item[0] for item in conn.sql("SELECT DISTINCT parent_dir FROM sampled").fetchall()]
 model_names.sort()
 # Just for display.
 cleaned_model_names = [name.replace("completions-", "") for name in model_names]
@@ -84,7 +84,7 @@ def build_table():
     query += """
         clip_text(c.challenge, 40) as challenge_clipped,
         FROM challenges c
-        LEFT JOIN results.completions r
+        LEFT JOIN sampled r
         ON c.ID = r.prompt_id
         GROUP BY c.ID, c.challenge, c.answer
     """

metrics.py

@@ -3,6 +3,18 @@ import duckdb
 import textwrap
 from typing import List, Tuple
 import argparse
+import unicodedata
+
+import unicodedata
+import re
+
+def normalize_text(text: str) -> str:
+    """Normalize text to remove accents, convert to lowercase, and strip spaces."""
+    text = unicodedata.normalize("NFKD", text)  # Decomposes letters with accents (e.g., é → e + ́)
+    text = "".join([c for c in text if not unicodedata.combining(c)])  # Remove diacritics
+    text = text.lower().strip()  # Convert to lowercase and strip spaces
+    return text
+
 
 def _parse_answer(text: str) -> List[List[str]]:
     """
@@ -17,7 +29,7 @@ def _parse_answer(text: str) -> List[List[str]]:
     distinct phrases that may be present in any order. All other characters
     are dropped.
     """
-    text = text.lower()
+    text = normalize_text(text)
     alternatives = re.split(r';', text)
     result = [ ]
     for alternative in alternatives:
@@ -43,6 +55,7 @@ def _check_answer(completion: str, answer: str) -> bool:
     completion = _answer_without_thoughts(completion).lower()
     completion  = re.sub(r'[^\w\s]', ' ', completion) # this replaces punctuations with space, aligning with the _parse_answer function's ' '.join
     completion = re.sub(r'\s+', ' ', completion) # normalize consecutive (Unicode) spaces to finish aligning with _parse_answer
+    completion = normalize_text(completion)
     alternative_answers = _parse_answer(answer)
     for answer_phrases in alternative_answers:
         # if all(phrase in completion for phrase in answer_phrases):
@@ -71,6 +84,37 @@ def load_results():
     conn.create_function("wrap_text", _wrap_text)
     return conn
 
+def load_results_sample_one_only():
+    conn = duckdb.connect(":memory:")
+    conn.execute("ATTACH DATABASE 'results.duckdb' AS results (READ_ONLY)")
+    
+    query = """
+    CREATE TABLE sampled AS
+    WITH numbered AS (
+        SELECT *,
+            ROW_NUMBER() OVER (PARTITION BY parent_dir, prompt ORDER BY prompt_id) AS rn
+        FROM results.completions
+    )
+    SELECT prompt_id, parent_dir, prompt, completion
+    FROM numbered
+    WHERE rn = 1;
+    """
+    conn.execute(query).fetchall()
+    # #print how how many rows are in the table
+    # print(conn.execute("SELECT COUNT(*) FROM sampled").fetchall())
+    # #describe the sampled table
+    # print(conn.execute("DESCRIBE sampled").fetchall())
+
+    conn.execute("""
+        CREATE TABLE challenges AS 
+        SELECT * FROM 'puzzles_cleaned.csv'
+        WHERE Warnings IS NULL OR Warnings NOT LIKE '%(E)%'
+    """)
+    conn.create_function("check_answer", _check_answer)
+    conn.create_function("clip_text", _clip_text)
+    conn.create_function("wrap_text", _wrap_text)
+    return conn
+
 def r1_accuracy_by_completion_length(conn,model_name):
     """
     For the responses from the completions-r1 model:
@@ -154,8 +198,8 @@ def accuracy_by_model(conn):
         WITH AnswerCheck AS (
             SELECT 
                 results.parent_dir AS model,
-                COUNT(*) AS total,
-                SUM(CAST(check_answer(results.completion, challenges.answer) AS INTEGER)) AS correct
+                SUM(results.count) AS total,
+                SUM(results.count * CAST(check_answer(results.completion, challenges.answer) AS INTEGER)) AS correct
             FROM 
                 results.completions results
             JOIN 
@@ -174,11 +218,52 @@ def accuracy_by_model(conn):
             AnswerCheck
     """)
 
+def accuracy_by_model_only_one(conn):
+    query = """
+    WITH FirstResponses AS (
+        SELECT 
+            parent_dir AS model,
+            prompt_id,
+            completion,
+            count,
+            ROW_NUMBER() OVER (PARTITION BY parent_dir, prompt_id) AS rn
+        FROM results.completions
+        WHERE parent_dir = 'completions-r1_cursor_hosted'  -- Only consider rows where parent_dir is 'r1_cursor_hosted'
+    ),
+    AnswerCheck AS (
+        SELECT 
+            fr.model,
+            SUM(fr.count) AS total,
+            SUM(fr.count * CAST(check_answer(fr.completion, c.answer) AS INTEGER)) AS correct
+        FROM FirstResponses fr
+        JOIN challenges c ON fr.prompt_id = c.ID
+        WHERE fr.rn = 1  -- Select only the first response per model per prompt
+        GROUP BY fr.model
+    )
+    SELECT 
+        model,
+        total,
+        correct,
+        ROUND(correct / total, 2) AS accuracy
+    FROM AnswerCheck;
+    """
+    return conn.sql(query)
+
+
 def main():
     parser = argparse.ArgumentParser()
     parser.add_argument("--by-model-and-time", action="store_true")
     args = parser.parse_args()
-    conn = load_results()
+    conn = load_results_sample_one_only()
+    query = """
+    SELECT parent_dir, prompt_id, COUNT(DISTINCT completion) AS completion_count
+    FROM sampled
+    GROUP BY parent_dir, prompt_id
+    HAVING COUNT(DISTINCT completion) == 1;
+    """
+    wrongones = conn.execute(query).fetchall()
+    assert not wrongones, f"Found {len(wrongones)} prompts with not just one completion"
+    
     if args.by_model_and_time:
         print(accuracy_by_model_and_time(conn))
     else: