open-r1-eval-leaderboard
/
eval_results
/AI-MO
/deepseek-math-7b-dpo
/ultrainteract_v00.00
/math
/results_2024-04-25T19-59-52.618027.json
| { | |
| "config_general": { | |
| "lighteval_sha": "?", | |
| "num_fewshot_seeds": 1, | |
| "override_batch_size": 2, | |
| "max_samples": null, | |
| "job_id": "", | |
| "start_time": 3233009.06714829, | |
| "end_time": 3245728.816519074, | |
| "total_evaluation_time_secondes": "12719.749370784033", | |
| "model_name": "AI-MO/deepseek-math-7b-dpo", | |
| "model_sha": "de8cb5901344e6f0d2229c51ec0a289c4f62843f", | |
| "model_dtype": "torch.bfloat16", | |
| "model_size": "12.93 GB", | |
| "config": null | |
| }, | |
| "results": { | |
| "lighteval|math:algebra|0": { | |
| "qem": 0.5754001684919966, | |
| "qem_stderr": 0.014352665275085122 | |
| }, | |
| "lighteval|math:counting_and_probability|0": { | |
| "qem": 0.2468354430379747, | |
| "qem_stderr": 0.01982522509053896 | |
| }, | |
| "lighteval|math:geometry|0": { | |
| "qem": 0.3298538622129436, | |
| "qem_stderr": 0.02150461134726607 | |
| }, | |
| "lighteval|math:intermediate_algebra|0": { | |
| "qem": 0.1982281284606866, | |
| "qem_stderr": 0.013274090660712124 | |
| }, | |
| "lighteval|math:number_theory|0": { | |
| "qem": 0.2796296296296296, | |
| "qem_stderr": 0.019331926349147863 | |
| }, | |
| "lighteval|math:prealgebra|0": { | |
| "qem": 0.5797933409873708, | |
| "qem_stderr": 0.01673433467950255 | |
| }, | |
| "lighteval|math:precalculus|0": { | |
| "qem": 0.2600732600732601, | |
| "qem_stderr": 0.018790743352015988 | |
| }, | |
| "lighteval|math:_average|0": { | |
| "qem": 0.35283054755626597, | |
| "qem_stderr": 0.01768765667918124 | |
| }, | |
| "all": { | |
| "qem": 0.35283054755626597, | |
| "qem_stderr": 0.01768765667918124 | |
| } | |
| }, | |
| "versions": { | |
| "lighteval|math:algebra|0": 0, | |
| "lighteval|math:counting_and_probability|0": 0, | |
| "lighteval|math:geometry|0": 0, | |
| "lighteval|math:intermediate_algebra|0": 0, | |
| "lighteval|math:number_theory|0": 0, | |
| "lighteval|math:prealgebra|0": 0, | |
| "lighteval|math:precalculus|0": 0 | |
| }, | |
| "config_tasks": { | |
| "lighteval|math:algebra": { | |
| "name": "math:algebra", | |
| "prompt_function": "math", | |
| "hf_repo": "lighteval/MATH", | |
| "hf_subset": "algebra", | |
| "metric": [ | |
| "quasi_exact_match_math" | |
| ], | |
| "hf_avail_splits": [ | |
| "train", | |
| "test", | |
| "validation" | |
| ], | |
| "evaluation_splits": [ | |
| "test" | |
| ], | |
| "few_shots_split": null, | |
| "few_shots_select": null, | |
| "generation_size": 2048, | |
| "stop_sequence": [ | |
| "\n" | |
| ], | |
| "output_regex": null, | |
| "frozen": false, | |
| "suite": [ | |
| "lighteval", | |
| "math" | |
| ], | |
| "original_num_docs": 1187, | |
| "effective_num_docs": 1187, | |
| "trust_dataset": true, | |
| "must_remove_duplicate_docs": null | |
| }, | |
| "lighteval|math:counting_and_probability": { | |
| "name": "math:counting_and_probability", | |
| "prompt_function": "math", | |
| "hf_repo": "lighteval/MATH", | |
| "hf_subset": "counting_and_probability", | |
| "metric": [ | |
| "quasi_exact_match_math" | |
| ], | |
| "hf_avail_splits": [ | |
| "train", | |
| "test", | |
| "validation" | |
| ], | |
| "evaluation_splits": [ | |
| "test" | |
| ], | |
| "few_shots_split": null, | |
| "few_shots_select": null, | |
| "generation_size": 2048, | |
| "stop_sequence": [ | |
| "\n" | |
| ], | |
| "output_regex": null, | |
| "frozen": false, | |
| "suite": [ | |
| "lighteval", | |
| "math" | |
| ], | |
| "original_num_docs": 474, | |
| "effective_num_docs": 474, | |
| "trust_dataset": true, | |
| "must_remove_duplicate_docs": null | |
| }, | |
| "lighteval|math:geometry": { | |
| "name": "math:geometry", | |
| "prompt_function": "math", | |
| "hf_repo": "lighteval/MATH", | |
| "hf_subset": "geometry", | |
| "metric": [ | |
| "quasi_exact_match_math" | |
| ], | |
| "hf_avail_splits": [ | |
| "train", | |
| "test", | |
| "validation" | |
| ], | |
| "evaluation_splits": [ | |
| "test" | |
| ], | |
| "few_shots_split": null, | |
| "few_shots_select": null, | |
| "generation_size": 2048, | |
| "stop_sequence": [ | |
| "\n" | |
| ], | |
| "output_regex": null, | |
| "frozen": false, | |
| "suite": [ | |
| "lighteval", | |
| "math" | |
| ], | |
| "original_num_docs": 479, | |
| "effective_num_docs": 479, | |
| "trust_dataset": true, | |
| "must_remove_duplicate_docs": null | |
| }, | |
| "lighteval|math:intermediate_algebra": { | |
| "name": "math:intermediate_algebra", | |
| "prompt_function": "math", | |
| "hf_repo": "lighteval/MATH", | |
| "hf_subset": "intermediate_algebra", | |
| "metric": [ | |
| "quasi_exact_match_math" | |
| ], | |
| "hf_avail_splits": [ | |
| "train", | |
| "test", | |
| "validation" | |
| ], | |
| "evaluation_splits": [ | |
| "test" | |
| ], | |
| "few_shots_split": null, | |
| "few_shots_select": null, | |
| "generation_size": 2048, | |
| "stop_sequence": [ | |
| "\n" | |
| ], | |
| "output_regex": null, | |
| "frozen": false, | |
| "suite": [ | |
| "lighteval", | |
| "math" | |
| ], | |
| "original_num_docs": 903, | |
| "effective_num_docs": 903, | |
| "trust_dataset": true, | |
| "must_remove_duplicate_docs": null | |
| }, | |
| "lighteval|math:number_theory": { | |
| "name": "math:number_theory", | |
| "prompt_function": "math", | |
| "hf_repo": "lighteval/MATH", | |
| "hf_subset": "number_theory", | |
| "metric": [ | |
| "quasi_exact_match_math" | |
| ], | |
| "hf_avail_splits": [ | |
| "train", | |
| "test", | |
| "validation" | |
| ], | |
| "evaluation_splits": [ | |
| "test" | |
| ], | |
| "few_shots_split": null, | |
| "few_shots_select": null, | |
| "generation_size": 2048, | |
| "stop_sequence": [ | |
| "\n" | |
| ], | |
| "output_regex": null, | |
| "frozen": false, | |
| "suite": [ | |
| "lighteval", | |
| "math" | |
| ], | |
| "original_num_docs": 540, | |
| "effective_num_docs": 540, | |
| "trust_dataset": true, | |
| "must_remove_duplicate_docs": null | |
| }, | |
| "lighteval|math:prealgebra": { | |
| "name": "math:prealgebra", | |
| "prompt_function": "math", | |
| "hf_repo": "lighteval/MATH", | |
| "hf_subset": "prealgebra", | |
| "metric": [ | |
| "quasi_exact_match_math" | |
| ], | |
| "hf_avail_splits": [ | |
| "train", | |
| "test", | |
| "validation" | |
| ], | |
| "evaluation_splits": [ | |
| "test" | |
| ], | |
| "few_shots_split": null, | |
| "few_shots_select": null, | |
| "generation_size": 2048, | |
| "stop_sequence": [ | |
| "\n" | |
| ], | |
| "output_regex": null, | |
| "frozen": false, | |
| "suite": [ | |
| "lighteval", | |
| "math" | |
| ], | |
| "original_num_docs": 871, | |
| "effective_num_docs": 871, | |
| "trust_dataset": true, | |
| "must_remove_duplicate_docs": null | |
| }, | |
| "lighteval|math:precalculus": { | |
| "name": "math:precalculus", | |
| "prompt_function": "math", | |
| "hf_repo": "lighteval/MATH", | |
| "hf_subset": "precalculus", | |
| "metric": [ | |
| "quasi_exact_match_math" | |
| ], | |
| "hf_avail_splits": [ | |
| "train", | |
| "test", | |
| "validation" | |
| ], | |
| "evaluation_splits": [ | |
| "test" | |
| ], | |
| "few_shots_split": null, | |
| "few_shots_select": null, | |
| "generation_size": 2048, | |
| "stop_sequence": [ | |
| "\n" | |
| ], | |
| "output_regex": null, | |
| "frozen": false, | |
| "suite": [ | |
| "lighteval", | |
| "math" | |
| ], | |
| "original_num_docs": 546, | |
| "effective_num_docs": 546, | |
| "trust_dataset": true, | |
| "must_remove_duplicate_docs": null | |
| } | |
| }, | |
| "summary_tasks": { | |
| "lighteval|math:algebra|0": { | |
| "hashes": { | |
| "hash_examples": "37a2fd2f076d2e49", | |
| "hash_full_prompts": "461f2ab197cdfd01", | |
| "hash_input_tokens": "bd3e731d75bc03e6", | |
| "hash_cont_tokens": "b9092248b20eaaa5" | |
| }, | |
| "truncated": 1187, | |
| "non_truncated": 0, | |
| "padded": 63, | |
| "non_padded": 1124, | |
| "effective_few_shots": 0.0, | |
| "num_truncated_few_shots": 0 | |
| }, | |
| "lighteval|math:counting_and_probability|0": { | |
| "hashes": { | |
| "hash_examples": "97b4892e28bc078b", | |
| "hash_full_prompts": "e2cc9eb28ca159b1", | |
| "hash_input_tokens": "55728dc6558ad660", | |
| "hash_cont_tokens": "7f1eba1a5996cc6a" | |
| }, | |
| "truncated": 474, | |
| "non_truncated": 0, | |
| "padded": 17, | |
| "non_padded": 457, | |
| "effective_few_shots": 0.0, | |
| "num_truncated_few_shots": 0 | |
| }, | |
| "lighteval|math:geometry|0": { | |
| "hashes": { | |
| "hash_examples": "9e9f0228b8b3d093", | |
| "hash_full_prompts": "e581f42fefeed8c9", | |
| "hash_input_tokens": "6615f77877bf8cef", | |
| "hash_cont_tokens": "e19391771925c55c" | |
| }, | |
| "truncated": 479, | |
| "non_truncated": 0, | |
| "padded": 60, | |
| "non_padded": 419, | |
| "effective_few_shots": 0.0, | |
| "num_truncated_few_shots": 0 | |
| }, | |
| "lighteval|math:intermediate_algebra|0": { | |
| "hashes": { | |
| "hash_examples": "cfe73a8e28ae94de", | |
| "hash_full_prompts": "7557d30ba2cc15ef", | |
| "hash_input_tokens": "608f04e5c215ffad", | |
| "hash_cont_tokens": "7ba006ff6433a433" | |
| }, | |
| "truncated": 903, | |
| "non_truncated": 0, | |
| "padded": 15, | |
| "non_padded": 888, | |
| "effective_few_shots": 0.0, | |
| "num_truncated_few_shots": 0 | |
| }, | |
| "lighteval|math:number_theory|0": { | |
| "hashes": { | |
| "hash_examples": "4ee5237cf144afac", | |
| "hash_full_prompts": "7b983a6fe1e3d0a8", | |
| "hash_input_tokens": "0a9b44cdfde7644d", | |
| "hash_cont_tokens": "42911230efc70e45" | |
| }, | |
| "truncated": 539, | |
| "non_truncated": 1, | |
| "padded": 1, | |
| "non_padded": 539, | |
| "effective_few_shots": 0.0, | |
| "num_truncated_few_shots": 0 | |
| }, | |
| "lighteval|math:prealgebra|0": { | |
| "hashes": { | |
| "hash_examples": "3fb3afeb885f73d8", | |
| "hash_full_prompts": "736a03165dbf8f49", | |
| "hash_input_tokens": "434f9915c1b3fd21", | |
| "hash_cont_tokens": "136754225cd821ba" | |
| }, | |
| "truncated": 871, | |
| "non_truncated": 0, | |
| "padded": 18, | |
| "non_padded": 853, | |
| "effective_few_shots": 0.0, | |
| "num_truncated_few_shots": 0 | |
| }, | |
| "lighteval|math:precalculus|0": { | |
| "hashes": { | |
| "hash_examples": "753e25ab9ec4b46c", | |
| "hash_full_prompts": "632ca5aa79f53b59", | |
| "hash_input_tokens": "7808ce9bbfbc6f20", | |
| "hash_cont_tokens": "d1af8ccbf0122848" | |
| }, | |
| "truncated": 546, | |
| "non_truncated": 0, | |
| "padded": 4, | |
| "non_padded": 542, | |
| "effective_few_shots": 0.0, | |
| "num_truncated_few_shots": 0 | |
| } | |
| }, | |
| "summary_general": { | |
| "hashes": { | |
| "hash_examples": "b76099aa9092a203", | |
| "hash_full_prompts": "2d2e105e1a0eeab9", | |
| "hash_input_tokens": "ec60d48c4ec91205", | |
| "hash_cont_tokens": "ad3c5068b326cc30" | |
| }, | |
| "truncated": 4999, | |
| "non_truncated": 1, | |
| "padded": 178, | |
| "non_padded": 4822, | |
| "num_truncated_few_shots": 0 | |
| } | |
| } |