x

lighteval_to_duckdb.py

@@ -5,6 +5,10 @@ import re
 import ast
 from datasets import load_dataset
 
+"""
+This script exports the results from the Parquet files in the results directory to a DuckDB database.
+"""
+
 def export_to_duckdb(conn, results_dir: Path):
     # SQL schema (matching results.completions)
     create_string = """CREATE TABLE IF NOT EXISTS completions  (
@@ -68,5 +72,4 @@ def main():
 if __name__ == "__main__":
     main()
 
-#results_dir = "/mnt/ssd/aryawu/lighteval/results/details"
-#python npr_to_duckdb.py "/mnt/ssd/aryawu/lighteval/results.duckdb" "/mnt/ssd/aryawu/lighteval/results/details"
+#python npr_to_duckdb.py "pathto/lighteval/results.duckdb" "pathto/lighteval/results/details"