open-r1-eval-leaderboard
/
eval_results
/AI-MO
/deepseek-math-7b-sft
/aimo_v00.01
/math
/results_2024-04-24T19-54-27.416740.json
| { | |
| "config_general": { | |
| "lighteval_sha": "?", | |
| "num_fewshot_seeds": 1, | |
| "override_batch_size": 2, | |
| "max_samples": null, | |
| "job_id": "", | |
| "start_time": 10978769.444749627, | |
| "end_time": 10990915.711477632, | |
| "total_evaluation_time_secondes": "12146.26672800444", | |
| "model_name": "AI-MO/deepseek-math-7b-sft", | |
| "model_sha": "4084c5bb5d592fab5e5455ef0a215d6490ff7de0", | |
| "model_dtype": "torch.bfloat16", | |
| "model_size": "12.93 GB", | |
| "config": null | |
| }, | |
| "results": { | |
| "lighteval|math:algebra|0": { | |
| "qem": 0.5181128896377422, | |
| "qem_stderr": 0.014509167981143354 | |
| }, | |
| "lighteval|math:counting_and_probability|0": { | |
| "qem": 0.25738396624472576, | |
| "qem_stderr": 0.02010214193276712 | |
| }, | |
| "lighteval|math:geometry|0": { | |
| "qem": 0.2860125260960334, | |
| "qem_stderr": 0.020669209864607734 | |
| }, | |
| "lighteval|math:intermediate_algebra|0": { | |
| "qem": 0.19269102990033224, | |
| "qem_stderr": 0.013132498782610072 | |
| }, | |
| "lighteval|math:number_theory|0": { | |
| "qem": 0.25, | |
| "qem_stderr": 0.018651177423824772 | |
| }, | |
| "lighteval|math:prealgebra|0": { | |
| "qem": 0.529276693455798, | |
| "qem_stderr": 0.016922503388767985 | |
| }, | |
| "lighteval|math:precalculus|0": { | |
| "qem": 0.2326007326007326, | |
| "qem_stderr": 0.01809747469809874 | |
| }, | |
| "lighteval|math:_average|0": { | |
| "qem": 0.3237254054193377, | |
| "qem_stderr": 0.01744059629597425 | |
| }, | |
| "all": { | |
| "qem": 0.3237254054193377, | |
| "qem_stderr": 0.01744059629597425 | |
| } | |
| }, | |
| "versions": { | |
| "lighteval|math:algebra|0": 0, | |
| "lighteval|math:counting_and_probability|0": 0, | |
| "lighteval|math:geometry|0": 0, | |
| "lighteval|math:intermediate_algebra|0": 0, | |
| "lighteval|math:number_theory|0": 0, | |
| "lighteval|math:prealgebra|0": 0, | |
| "lighteval|math:precalculus|0": 0 | |
| }, | |
| "config_tasks": { | |
| "lighteval|math:algebra": { | |
| "name": "math:algebra", | |
| "prompt_function": "math", | |
| "hf_repo": "lighteval/MATH", | |
| "hf_subset": "algebra", | |
| "metric": [ | |
| "quasi_exact_match_math" | |
| ], | |
| "hf_avail_splits": [ | |
| "train", | |
| "test", | |
| "validation" | |
| ], | |
| "evaluation_splits": [ | |
| "test" | |
| ], | |
| "few_shots_split": null, | |
| "few_shots_select": null, | |
| "generation_size": 2048, | |
| "stop_sequence": [ | |
| "\n" | |
| ], | |
| "output_regex": null, | |
| "frozen": false, | |
| "suite": [ | |
| "lighteval", | |
| "math" | |
| ], | |
| "original_num_docs": 1187, | |
| "effective_num_docs": 1187, | |
| "trust_dataset": true, | |
| "must_remove_duplicate_docs": null | |
| }, | |
| "lighteval|math:counting_and_probability": { | |
| "name": "math:counting_and_probability", | |
| "prompt_function": "math", | |
| "hf_repo": "lighteval/MATH", | |
| "hf_subset": "counting_and_probability", | |
| "metric": [ | |
| "quasi_exact_match_math" | |
| ], | |
| "hf_avail_splits": [ | |
| "train", | |
| "test", | |
| "validation" | |
| ], | |
| "evaluation_splits": [ | |
| "test" | |
| ], | |
| "few_shots_split": null, | |
| "few_shots_select": null, | |
| "generation_size": 2048, | |
| "stop_sequence": [ | |
| "\n" | |
| ], | |
| "output_regex": null, | |
| "frozen": false, | |
| "suite": [ | |
| "lighteval", | |
| "math" | |
| ], | |
| "original_num_docs": 474, | |
| "effective_num_docs": 474, | |
| "trust_dataset": true, | |
| "must_remove_duplicate_docs": null | |
| }, | |
| "lighteval|math:geometry": { | |
| "name": "math:geometry", | |
| "prompt_function": "math", | |
| "hf_repo": "lighteval/MATH", | |
| "hf_subset": "geometry", | |
| "metric": [ | |
| "quasi_exact_match_math" | |
| ], | |
| "hf_avail_splits": [ | |
| "train", | |
| "test", | |
| "validation" | |
| ], | |
| "evaluation_splits": [ | |
| "test" | |
| ], | |
| "few_shots_split": null, | |
| "few_shots_select": null, | |
| "generation_size": 2048, | |
| "stop_sequence": [ | |
| "\n" | |
| ], | |
| "output_regex": null, | |
| "frozen": false, | |
| "suite": [ | |
| "lighteval", | |
| "math" | |
| ], | |
| "original_num_docs": 479, | |
| "effective_num_docs": 479, | |
| "trust_dataset": true, | |
| "must_remove_duplicate_docs": null | |
| }, | |
| "lighteval|math:intermediate_algebra": { | |
| "name": "math:intermediate_algebra", | |
| "prompt_function": "math", | |
| "hf_repo": "lighteval/MATH", | |
| "hf_subset": "intermediate_algebra", | |
| "metric": [ | |
| "quasi_exact_match_math" | |
| ], | |
| "hf_avail_splits": [ | |
| "train", | |
| "test", | |
| "validation" | |
| ], | |
| "evaluation_splits": [ | |
| "test" | |
| ], | |
| "few_shots_split": null, | |
| "few_shots_select": null, | |
| "generation_size": 2048, | |
| "stop_sequence": [ | |
| "\n" | |
| ], | |
| "output_regex": null, | |
| "frozen": false, | |
| "suite": [ | |
| "lighteval", | |
| "math" | |
| ], | |
| "original_num_docs": 903, | |
| "effective_num_docs": 903, | |
| "trust_dataset": true, | |
| "must_remove_duplicate_docs": null | |
| }, | |
| "lighteval|math:number_theory": { | |
| "name": "math:number_theory", | |
| "prompt_function": "math", | |
| "hf_repo": "lighteval/MATH", | |
| "hf_subset": "number_theory", | |
| "metric": [ | |
| "quasi_exact_match_math" | |
| ], | |
| "hf_avail_splits": [ | |
| "train", | |
| "test", | |
| "validation" | |
| ], | |
| "evaluation_splits": [ | |
| "test" | |
| ], | |
| "few_shots_split": null, | |
| "few_shots_select": null, | |
| "generation_size": 2048, | |
| "stop_sequence": [ | |
| "\n" | |
| ], | |
| "output_regex": null, | |
| "frozen": false, | |
| "suite": [ | |
| "lighteval", | |
| "math" | |
| ], | |
| "original_num_docs": 540, | |
| "effective_num_docs": 540, | |
| "trust_dataset": true, | |
| "must_remove_duplicate_docs": null | |
| }, | |
| "lighteval|math:prealgebra": { | |
| "name": "math:prealgebra", | |
| "prompt_function": "math", | |
| "hf_repo": "lighteval/MATH", | |
| "hf_subset": "prealgebra", | |
| "metric": [ | |
| "quasi_exact_match_math" | |
| ], | |
| "hf_avail_splits": [ | |
| "train", | |
| "test", | |
| "validation" | |
| ], | |
| "evaluation_splits": [ | |
| "test" | |
| ], | |
| "few_shots_split": null, | |
| "few_shots_select": null, | |
| "generation_size": 2048, | |
| "stop_sequence": [ | |
| "\n" | |
| ], | |
| "output_regex": null, | |
| "frozen": false, | |
| "suite": [ | |
| "lighteval", | |
| "math" | |
| ], | |
| "original_num_docs": 871, | |
| "effective_num_docs": 871, | |
| "trust_dataset": true, | |
| "must_remove_duplicate_docs": null | |
| }, | |
| "lighteval|math:precalculus": { | |
| "name": "math:precalculus", | |
| "prompt_function": "math", | |
| "hf_repo": "lighteval/MATH", | |
| "hf_subset": "precalculus", | |
| "metric": [ | |
| "quasi_exact_match_math" | |
| ], | |
| "hf_avail_splits": [ | |
| "train", | |
| "test", | |
| "validation" | |
| ], | |
| "evaluation_splits": [ | |
| "test" | |
| ], | |
| "few_shots_split": null, | |
| "few_shots_select": null, | |
| "generation_size": 2048, | |
| "stop_sequence": [ | |
| "\n" | |
| ], | |
| "output_regex": null, | |
| "frozen": false, | |
| "suite": [ | |
| "lighteval", | |
| "math" | |
| ], | |
| "original_num_docs": 546, | |
| "effective_num_docs": 546, | |
| "trust_dataset": true, | |
| "must_remove_duplicate_docs": null | |
| } | |
| }, | |
| "summary_tasks": { | |
| "lighteval|math:algebra|0": { | |
| "hashes": { | |
| "hash_examples": "37a2fd2f076d2e49", | |
| "hash_full_prompts": "461f2ab197cdfd01", | |
| "hash_input_tokens": "bd3e731d75bc03e6", | |
| "hash_cont_tokens": "184c18d9da42610c" | |
| }, | |
| "truncated": 1187, | |
| "non_truncated": 0, | |
| "padded": 63, | |
| "non_padded": 1124, | |
| "effective_few_shots": 0.0, | |
| "num_truncated_few_shots": 0 | |
| }, | |
| "lighteval|math:counting_and_probability|0": { | |
| "hashes": { | |
| "hash_examples": "97b4892e28bc078b", | |
| "hash_full_prompts": "e2cc9eb28ca159b1", | |
| "hash_input_tokens": "55728dc6558ad660", | |
| "hash_cont_tokens": "6a16d12ef76d7988" | |
| }, | |
| "truncated": 474, | |
| "non_truncated": 0, | |
| "padded": 17, | |
| "non_padded": 457, | |
| "effective_few_shots": 0.0, | |
| "num_truncated_few_shots": 0 | |
| }, | |
| "lighteval|math:geometry|0": { | |
| "hashes": { | |
| "hash_examples": "9e9f0228b8b3d093", | |
| "hash_full_prompts": "e581f42fefeed8c9", | |
| "hash_input_tokens": "6615f77877bf8cef", | |
| "hash_cont_tokens": "6e4434a00f09b30a" | |
| }, | |
| "truncated": 479, | |
| "non_truncated": 0, | |
| "padded": 60, | |
| "non_padded": 419, | |
| "effective_few_shots": 0.0, | |
| "num_truncated_few_shots": 0 | |
| }, | |
| "lighteval|math:intermediate_algebra|0": { | |
| "hashes": { | |
| "hash_examples": "cfe73a8e28ae94de", | |
| "hash_full_prompts": "7557d30ba2cc15ef", | |
| "hash_input_tokens": "608f04e5c215ffad", | |
| "hash_cont_tokens": "3cd6a35e7f9a8bf8" | |
| }, | |
| "truncated": 903, | |
| "non_truncated": 0, | |
| "padded": 15, | |
| "non_padded": 888, | |
| "effective_few_shots": 0.0, | |
| "num_truncated_few_shots": 0 | |
| }, | |
| "lighteval|math:number_theory|0": { | |
| "hashes": { | |
| "hash_examples": "4ee5237cf144afac", | |
| "hash_full_prompts": "7b983a6fe1e3d0a8", | |
| "hash_input_tokens": "0a9b44cdfde7644d", | |
| "hash_cont_tokens": "1f8da8a6c0aaec67" | |
| }, | |
| "truncated": 539, | |
| "non_truncated": 1, | |
| "padded": 1, | |
| "non_padded": 539, | |
| "effective_few_shots": 0.0, | |
| "num_truncated_few_shots": 0 | |
| }, | |
| "lighteval|math:prealgebra|0": { | |
| "hashes": { | |
| "hash_examples": "3fb3afeb885f73d8", | |
| "hash_full_prompts": "736a03165dbf8f49", | |
| "hash_input_tokens": "434f9915c1b3fd21", | |
| "hash_cont_tokens": "602746299047a2fe" | |
| }, | |
| "truncated": 871, | |
| "non_truncated": 0, | |
| "padded": 18, | |
| "non_padded": 853, | |
| "effective_few_shots": 0.0, | |
| "num_truncated_few_shots": 0 | |
| }, | |
| "lighteval|math:precalculus|0": { | |
| "hashes": { | |
| "hash_examples": "753e25ab9ec4b46c", | |
| "hash_full_prompts": "632ca5aa79f53b59", | |
| "hash_input_tokens": "7808ce9bbfbc6f20", | |
| "hash_cont_tokens": "e0d69ceb6d3123bb" | |
| }, | |
| "truncated": 546, | |
| "non_truncated": 0, | |
| "padded": 4, | |
| "non_padded": 542, | |
| "effective_few_shots": 0.0, | |
| "num_truncated_few_shots": 0 | |
| } | |
| }, | |
| "summary_general": { | |
| "hashes": { | |
| "hash_examples": "b76099aa9092a203", | |
| "hash_full_prompts": "2d2e105e1a0eeab9", | |
| "hash_input_tokens": "ec60d48c4ec91205", | |
| "hash_cont_tokens": "d0987184d3421c0b" | |
| }, | |
| "truncated": 4999, | |
| "non_truncated": 1, | |
| "padded": 178, | |
| "non_padded": 4822, | |
| "num_truncated_few_shots": 0 | |
| } | |
| } |