skip excluded tasks

app.py

@@ -189,7 +189,10 @@ def all_challenges_view():
 
     def select_table_item(evt: gr.SelectData):
         model_index = evt.index[1]
-        challenge_id = evt.index[0]
+        # challenge_id = evt.index[0]
+        row_index = evt.index[0]  # The row index of the selected row
+        # Map the row index to the challenge_id (which is the 'ID' in your DataFrame)
+        challenge_id = relabelled_df.iloc[row_index]['ID']
         model_name = model_columns[model_index] if model_index in valid_model_indices else default_model
         return (model_name, challenge_id)
 

lighteval_to_duckdb.py

@@ -0,0 +1,72 @@
+import duckdb
+from pathlib import Path
+import argparse
+import re
+import ast
+from datasets import load_dataset
+
+def export_to_duckdb(conn, results_dir: Path):
+    # SQL schema (matching results.completions)
+    create_string = """CREATE TABLE IF NOT EXISTS completions  (
+        parent_dir TEXT NOT NULL,
+        prompt_id INT NOT NULL,
+        prompt TEXT NOT NULL,
+        completion TEXT NOT NULL,
+        count INT NOT NULL,
+        temperature FLOAT NOT NULL,
+        top_p FLOAT NOT NULL, 
+        max_tokens INT NOT NULL,
+        PRIMARY KEY (parent_dir, prompt_id, prompt, completion)
+    )"""
+    conn.execute(create_string)
+    created = 0  # Counter for inserted rows
+    parquet_files = list(results_dir.rglob("*.parquet"))
+    for parquet_file in parquet_files:
+        parquet_file = str(parquet_file)
+        # Load the dataset from the Parquet file
+        details = load_dataset("parquet", data_files=parquet_file, split="train")
+
+        for completion_item in details:
+            # Extract values safely
+            parent_dir = parquet_file.split("/results/details/")[1].split("/")[0]
+            specifics = ast.literal_eval(completion_item['specifics'])  # Convert string to dictionary
+            prompt_id = specifics.get('ID', -1)
+            prompt = completion_item.get("full_prompt", "")
+            completion = ast.literal_eval(completion_item.get("predictions", ""))[0]
+            while type(completion) != str:
+                completion = completion[0]
+            count = 0  # Placeholder value
+            temperature = 0.0  # Placeholder value
+            top_p = 0.0  # Placeholder value
+            max_tokens = 2048  # Placeholder value
+
+            # Insert into DuckDB
+            conn.execute(
+                "INSERT INTO completions VALUES (?, ?, ?, ?, ?, ?, ?, ?)",
+                (parent_dir, prompt_id, prompt, completion, count, temperature, top_p, max_tokens),
+            )
+            created += 1
+
+    conn.commit()
+    print(f"Created {created} rows.")
+
+
+def to_duckdb(db_path: str, results_dir: Path):
+    """Create a DuckDB connection and export data."""
+    conn = duckdb.connect(db_path)
+    export_to_duckdb(conn, results_dir)
+    conn.close()
+
+def main():
+    """Command-line interface for exporting Parquet data to DuckDB."""
+    parser = argparse.ArgumentParser()
+    parser.add_argument("db_path", type=str, help="Path to DuckDB database file")
+    parser.add_argument("results_dir", type=Path, help="Path to results dir")
+    args = parser.parse_args()
+    to_duckdb(args.db_path, args.results_dir)
+
+if __name__ == "__main__":
+    main()
+
+#results_dir = "/mnt/ssd/aryawu/lighteval/results/details"
+#python npr_to_duckdb.py "/mnt/ssd/aryawu/lighteval/results.duckdb" "/mnt/ssd/aryawu/lighteval/results/details"

metrics.py

@@ -60,7 +60,12 @@ def _wrap_text(text: str, width: int) -> str:
 def load_results():
     conn = duckdb.connect(":memory:")
     conn.execute("ATTACH DATABASE 'results.duckdb' AS results (READ_ONLY)")
-    conn.execute("CREATE TABLE challenges as SELECT * FROM 'puzzles_cleaned.csv'")
+    # conn.execute("CREATE TABLE challenges as SELECT * FROM 'puzzles_cleaned.csv'")
+    conn.execute("""
+        CREATE TABLE challenges AS 
+        SELECT * FROM 'puzzles_cleaned.csv'
+        WHERE Warnings IS NULL OR Warnings NOT LIKE '%(E)%'
+    """)
     conn.create_function("check_answer", _check_answer)
     conn.create_function("clip_text", _clip_text)
     conn.create_function("wrap_text", _wrap_text)