open-r1-eval-leaderboard
/
eval_results
/AI-MO
/deepseek-math-7b-sft
/aimo_v03.01
/math
/results_2024-04-26T04-51-02.678286.json
| { | |
| "config_general": { | |
| "lighteval_sha": "?", | |
| "num_fewshot_seeds": 1, | |
| "override_batch_size": 2, | |
| "max_samples": null, | |
| "job_id": "", | |
| "start_time": 4213395.073455049, | |
| "end_time": 4222017.114041344, | |
| "total_evaluation_time_secondes": "8622.040586295538", | |
| "model_name": "AI-MO/deepseek-math-7b-sft", | |
| "model_sha": "dd01bf2509b1dede9c1f2afef533c04f5657c530", | |
| "model_dtype": "torch.bfloat16", | |
| "model_size": "12.93 GB", | |
| "config": null | |
| }, | |
| "results": { | |
| "lighteval|math:algebra|0": { | |
| "qem": 0.6773378264532435, | |
| "qem_stderr": 0.013574828385302717 | |
| }, | |
| "lighteval|math:counting_and_probability|0": { | |
| "qem": 0.36075949367088606, | |
| "qem_stderr": 0.022080579003443235 | |
| }, | |
| "lighteval|math:geometry|0": { | |
| "qem": 0.348643006263048, | |
| "qem_stderr": 0.02179646532161935 | |
| }, | |
| "lighteval|math:intermediate_algebra|0": { | |
| "qem": 0.2469545957918051, | |
| "qem_stderr": 0.014358724698059553 | |
| }, | |
| "lighteval|math:number_theory|0": { | |
| "qem": 0.37962962962962965, | |
| "qem_stderr": 0.020903123748455175 | |
| }, | |
| "lighteval|math:prealgebra|0": { | |
| "qem": 0.6463834672789897, | |
| "qem_stderr": 0.016208833993652156 | |
| }, | |
| "lighteval|math:precalculus|0": { | |
| "qem": 0.29853479853479853, | |
| "qem_stderr": 0.0196020862304134 | |
| }, | |
| "lighteval|math:_average|0": { | |
| "qem": 0.42260611680320004, | |
| "qem_stderr": 0.0183606630544208 | |
| }, | |
| "all": { | |
| "qem": 0.42260611680320004, | |
| "qem_stderr": 0.0183606630544208 | |
| } | |
| }, | |
| "versions": { | |
| "lighteval|math:algebra|0": 0, | |
| "lighteval|math:counting_and_probability|0": 0, | |
| "lighteval|math:geometry|0": 0, | |
| "lighteval|math:intermediate_algebra|0": 0, | |
| "lighteval|math:number_theory|0": 0, | |
| "lighteval|math:prealgebra|0": 0, | |
| "lighteval|math:precalculus|0": 0 | |
| }, | |
| "config_tasks": { | |
| "lighteval|math:algebra": { | |
| "name": "math:algebra", | |
| "prompt_function": "math", | |
| "hf_repo": "lighteval/MATH", | |
| "hf_subset": "algebra", | |
| "metric": [ | |
| "quasi_exact_match_math" | |
| ], | |
| "hf_avail_splits": [ | |
| "train", | |
| "test", | |
| "validation" | |
| ], | |
| "evaluation_splits": [ | |
| "test" | |
| ], | |
| "few_shots_split": null, | |
| "few_shots_select": null, | |
| "generation_size": 2048, | |
| "stop_sequence": [ | |
| "\n" | |
| ], | |
| "output_regex": null, | |
| "frozen": false, | |
| "suite": [ | |
| "lighteval", | |
| "math" | |
| ], | |
| "original_num_docs": 1187, | |
| "effective_num_docs": 1187, | |
| "trust_dataset": true, | |
| "must_remove_duplicate_docs": null | |
| }, | |
| "lighteval|math:counting_and_probability": { | |
| "name": "math:counting_and_probability", | |
| "prompt_function": "math", | |
| "hf_repo": "lighteval/MATH", | |
| "hf_subset": "counting_and_probability", | |
| "metric": [ | |
| "quasi_exact_match_math" | |
| ], | |
| "hf_avail_splits": [ | |
| "train", | |
| "test", | |
| "validation" | |
| ], | |
| "evaluation_splits": [ | |
| "test" | |
| ], | |
| "few_shots_split": null, | |
| "few_shots_select": null, | |
| "generation_size": 2048, | |
| "stop_sequence": [ | |
| "\n" | |
| ], | |
| "output_regex": null, | |
| "frozen": false, | |
| "suite": [ | |
| "lighteval", | |
| "math" | |
| ], | |
| "original_num_docs": 474, | |
| "effective_num_docs": 474, | |
| "trust_dataset": true, | |
| "must_remove_duplicate_docs": null | |
| }, | |
| "lighteval|math:geometry": { | |
| "name": "math:geometry", | |
| "prompt_function": "math", | |
| "hf_repo": "lighteval/MATH", | |
| "hf_subset": "geometry", | |
| "metric": [ | |
| "quasi_exact_match_math" | |
| ], | |
| "hf_avail_splits": [ | |
| "train", | |
| "test", | |
| "validation" | |
| ], | |
| "evaluation_splits": [ | |
| "test" | |
| ], | |
| "few_shots_split": null, | |
| "few_shots_select": null, | |
| "generation_size": 2048, | |
| "stop_sequence": [ | |
| "\n" | |
| ], | |
| "output_regex": null, | |
| "frozen": false, | |
| "suite": [ | |
| "lighteval", | |
| "math" | |
| ], | |
| "original_num_docs": 479, | |
| "effective_num_docs": 479, | |
| "trust_dataset": true, | |
| "must_remove_duplicate_docs": null | |
| }, | |
| "lighteval|math:intermediate_algebra": { | |
| "name": "math:intermediate_algebra", | |
| "prompt_function": "math", | |
| "hf_repo": "lighteval/MATH", | |
| "hf_subset": "intermediate_algebra", | |
| "metric": [ | |
| "quasi_exact_match_math" | |
| ], | |
| "hf_avail_splits": [ | |
| "train", | |
| "test", | |
| "validation" | |
| ], | |
| "evaluation_splits": [ | |
| "test" | |
| ], | |
| "few_shots_split": null, | |
| "few_shots_select": null, | |
| "generation_size": 2048, | |
| "stop_sequence": [ | |
| "\n" | |
| ], | |
| "output_regex": null, | |
| "frozen": false, | |
| "suite": [ | |
| "lighteval", | |
| "math" | |
| ], | |
| "original_num_docs": 903, | |
| "effective_num_docs": 903, | |
| "trust_dataset": true, | |
| "must_remove_duplicate_docs": null | |
| }, | |
| "lighteval|math:number_theory": { | |
| "name": "math:number_theory", | |
| "prompt_function": "math", | |
| "hf_repo": "lighteval/MATH", | |
| "hf_subset": "number_theory", | |
| "metric": [ | |
| "quasi_exact_match_math" | |
| ], | |
| "hf_avail_splits": [ | |
| "train", | |
| "test", | |
| "validation" | |
| ], | |
| "evaluation_splits": [ | |
| "test" | |
| ], | |
| "few_shots_split": null, | |
| "few_shots_select": null, | |
| "generation_size": 2048, | |
| "stop_sequence": [ | |
| "\n" | |
| ], | |
| "output_regex": null, | |
| "frozen": false, | |
| "suite": [ | |
| "lighteval", | |
| "math" | |
| ], | |
| "original_num_docs": 540, | |
| "effective_num_docs": 540, | |
| "trust_dataset": true, | |
| "must_remove_duplicate_docs": null | |
| }, | |
| "lighteval|math:prealgebra": { | |
| "name": "math:prealgebra", | |
| "prompt_function": "math", | |
| "hf_repo": "lighteval/MATH", | |
| "hf_subset": "prealgebra", | |
| "metric": [ | |
| "quasi_exact_match_math" | |
| ], | |
| "hf_avail_splits": [ | |
| "train", | |
| "test", | |
| "validation" | |
| ], | |
| "evaluation_splits": [ | |
| "test" | |
| ], | |
| "few_shots_split": null, | |
| "few_shots_select": null, | |
| "generation_size": 2048, | |
| "stop_sequence": [ | |
| "\n" | |
| ], | |
| "output_regex": null, | |
| "frozen": false, | |
| "suite": [ | |
| "lighteval", | |
| "math" | |
| ], | |
| "original_num_docs": 871, | |
| "effective_num_docs": 871, | |
| "trust_dataset": true, | |
| "must_remove_duplicate_docs": null | |
| }, | |
| "lighteval|math:precalculus": { | |
| "name": "math:precalculus", | |
| "prompt_function": "math", | |
| "hf_repo": "lighteval/MATH", | |
| "hf_subset": "precalculus", | |
| "metric": [ | |
| "quasi_exact_match_math" | |
| ], | |
| "hf_avail_splits": [ | |
| "train", | |
| "test", | |
| "validation" | |
| ], | |
| "evaluation_splits": [ | |
| "test" | |
| ], | |
| "few_shots_split": null, | |
| "few_shots_select": null, | |
| "generation_size": 2048, | |
| "stop_sequence": [ | |
| "\n" | |
| ], | |
| "output_regex": null, | |
| "frozen": false, | |
| "suite": [ | |
| "lighteval", | |
| "math" | |
| ], | |
| "original_num_docs": 546, | |
| "effective_num_docs": 546, | |
| "trust_dataset": true, | |
| "must_remove_duplicate_docs": null | |
| } | |
| }, | |
| "summary_tasks": { | |
| "lighteval|math:algebra|0": { | |
| "hashes": { | |
| "hash_examples": "37a2fd2f076d2e49", | |
| "hash_full_prompts": "461f2ab197cdfd01", | |
| "hash_input_tokens": "bd3e731d75bc03e6", | |
| "hash_cont_tokens": "8b12597582e2af8a" | |
| }, | |
| "truncated": 1187, | |
| "non_truncated": 0, | |
| "padded": 63, | |
| "non_padded": 1124, | |
| "effective_few_shots": 0.0, | |
| "num_truncated_few_shots": 0 | |
| }, | |
| "lighteval|math:counting_and_probability|0": { | |
| "hashes": { | |
| "hash_examples": "97b4892e28bc078b", | |
| "hash_full_prompts": "e2cc9eb28ca159b1", | |
| "hash_input_tokens": "55728dc6558ad660", | |
| "hash_cont_tokens": "ee728214399bbce1" | |
| }, | |
| "truncated": 474, | |
| "non_truncated": 0, | |
| "padded": 17, | |
| "non_padded": 457, | |
| "effective_few_shots": 0.0, | |
| "num_truncated_few_shots": 0 | |
| }, | |
| "lighteval|math:geometry|0": { | |
| "hashes": { | |
| "hash_examples": "9e9f0228b8b3d093", | |
| "hash_full_prompts": "e581f42fefeed8c9", | |
| "hash_input_tokens": "6615f77877bf8cef", | |
| "hash_cont_tokens": "3632ce692fc1b61c" | |
| }, | |
| "truncated": 479, | |
| "non_truncated": 0, | |
| "padded": 60, | |
| "non_padded": 419, | |
| "effective_few_shots": 0.0, | |
| "num_truncated_few_shots": 0 | |
| }, | |
| "lighteval|math:intermediate_algebra|0": { | |
| "hashes": { | |
| "hash_examples": "cfe73a8e28ae94de", | |
| "hash_full_prompts": "7557d30ba2cc15ef", | |
| "hash_input_tokens": "608f04e5c215ffad", | |
| "hash_cont_tokens": "a83dd8d0765c48b1" | |
| }, | |
| "truncated": 903, | |
| "non_truncated": 0, | |
| "padded": 15, | |
| "non_padded": 888, | |
| "effective_few_shots": 0.0, | |
| "num_truncated_few_shots": 0 | |
| }, | |
| "lighteval|math:number_theory|0": { | |
| "hashes": { | |
| "hash_examples": "4ee5237cf144afac", | |
| "hash_full_prompts": "7b983a6fe1e3d0a8", | |
| "hash_input_tokens": "0a9b44cdfde7644d", | |
| "hash_cont_tokens": "c1a5a1b43fc8ffb6" | |
| }, | |
| "truncated": 539, | |
| "non_truncated": 1, | |
| "padded": 1, | |
| "non_padded": 539, | |
| "effective_few_shots": 0.0, | |
| "num_truncated_few_shots": 0 | |
| }, | |
| "lighteval|math:prealgebra|0": { | |
| "hashes": { | |
| "hash_examples": "3fb3afeb885f73d8", | |
| "hash_full_prompts": "736a03165dbf8f49", | |
| "hash_input_tokens": "434f9915c1b3fd21", | |
| "hash_cont_tokens": "6cd2735b70a01c28" | |
| }, | |
| "truncated": 871, | |
| "non_truncated": 0, | |
| "padded": 18, | |
| "non_padded": 853, | |
| "effective_few_shots": 0.0, | |
| "num_truncated_few_shots": 0 | |
| }, | |
| "lighteval|math:precalculus|0": { | |
| "hashes": { | |
| "hash_examples": "753e25ab9ec4b46c", | |
| "hash_full_prompts": "632ca5aa79f53b59", | |
| "hash_input_tokens": "7808ce9bbfbc6f20", | |
| "hash_cont_tokens": "29aee73acae487bc" | |
| }, | |
| "truncated": 546, | |
| "non_truncated": 0, | |
| "padded": 4, | |
| "non_padded": 542, | |
| "effective_few_shots": 0.0, | |
| "num_truncated_few_shots": 0 | |
| } | |
| }, | |
| "summary_general": { | |
| "hashes": { | |
| "hash_examples": "b76099aa9092a203", | |
| "hash_full_prompts": "2d2e105e1a0eeab9", | |
| "hash_input_tokens": "ec60d48c4ec91205", | |
| "hash_cont_tokens": "2362c0024ff536ef" | |
| }, | |
| "truncated": 4999, | |
| "non_truncated": 1, | |
| "padded": 178, | |
| "non_padded": 4822, | |
| "num_truncated_few_shots": 0 | |
| } | |
| } |