open-r1-eval-leaderboard
/
eval_results
/AI-MO
/mistral-7b-sft
/aimo_v03.01
/math
/results_2024-04-24T01-00-19.177728.json
edbeeching
HF staff
Upload eval_results/AI-MO/mistral-7b-sft/aimo_v03.01/math/results_2024-04-24T01-00-19.177728.json with huggingface_hub
79a90ca
verified
| { | |
| "config_general": { | |
| "lighteval_sha": "?", | |
| "num_fewshot_seeds": 1, | |
| "override_batch_size": 4, | |
| "max_samples": null, | |
| "job_id": "", | |
| "start_time": 1064705.542373272, | |
| "end_time": 1074734.880624278, | |
| "total_evaluation_time_secondes": "10029.338251006091", | |
| "model_name": "AI-MO/mistral-7b-sft", | |
| "model_sha": "147a5797ae74bc37a561429745b8781fde7e9eff", | |
| "model_dtype": "torch.bfloat16", | |
| "model_size": "13.99 GB", | |
| "config": null | |
| }, | |
| "results": { | |
| "lighteval|math:algebra|0": { | |
| "qem": 0.32771693344566133, | |
| "qem_stderr": 0.013629602274488393 | |
| }, | |
| "lighteval|math:counting_and_probability|0": { | |
| "qem": 0.0970464135021097, | |
| "qem_stderr": 0.013611058115372826 | |
| }, | |
| "lighteval|math:geometry|0": { | |
| "qem": 0.13778705636743216, | |
| "qem_stderr": 0.015765129113075753 | |
| }, | |
| "lighteval|math:intermediate_algebra|0": { | |
| "qem": 0.10963455149501661, | |
| "qem_stderr": 0.01040290428349701 | |
| }, | |
| "lighteval|math:number_theory|0": { | |
| "qem": 0.09444444444444444, | |
| "qem_stderr": 0.012596543048318216 | |
| }, | |
| "lighteval|math:prealgebra|0": { | |
| "qem": 0.29506314580941445, | |
| "qem_stderr": 0.015462259793596663 | |
| }, | |
| "lighteval|math:precalculus|0": { | |
| "qem": 0.14835164835164835, | |
| "qem_stderr": 0.015225728875338516 | |
| }, | |
| "lighteval|math:_average|0": { | |
| "qem": 0.1728634562022467, | |
| "qem_stderr": 0.013813317929098196 | |
| }, | |
| "all": { | |
| "qem": 0.1728634562022467, | |
| "qem_stderr": 0.013813317929098196 | |
| } | |
| }, | |
| "versions": { | |
| "lighteval|math:algebra|0": 0, | |
| "lighteval|math:counting_and_probability|0": 0, | |
| "lighteval|math:geometry|0": 0, | |
| "lighteval|math:intermediate_algebra|0": 0, | |
| "lighteval|math:number_theory|0": 0, | |
| "lighteval|math:prealgebra|0": 0, | |
| "lighteval|math:precalculus|0": 0 | |
| }, | |
| "config_tasks": { | |
| "lighteval|math:algebra": { | |
| "name": "math:algebra", | |
| "prompt_function": "math", | |
| "hf_repo": "lighteval/MATH", | |
| "hf_subset": "algebra", | |
| "metric": [ | |
| "quasi_exact_match_math" | |
| ], | |
| "hf_avail_splits": [ | |
| "train", | |
| "test", | |
| "validation" | |
| ], | |
| "evaluation_splits": [ | |
| "test" | |
| ], | |
| "few_shots_split": null, | |
| "few_shots_select": null, | |
| "generation_size": 2048, | |
| "stop_sequence": [ | |
| "\n" | |
| ], | |
| "output_regex": null, | |
| "frozen": false, | |
| "suite": [ | |
| "lighteval", | |
| "math" | |
| ], | |
| "original_num_docs": 1187, | |
| "effective_num_docs": 1187, | |
| "trust_dataset": true, | |
| "must_remove_duplicate_docs": null | |
| }, | |
| "lighteval|math:counting_and_probability": { | |
| "name": "math:counting_and_probability", | |
| "prompt_function": "math", | |
| "hf_repo": "lighteval/MATH", | |
| "hf_subset": "counting_and_probability", | |
| "metric": [ | |
| "quasi_exact_match_math" | |
| ], | |
| "hf_avail_splits": [ | |
| "train", | |
| "test", | |
| "validation" | |
| ], | |
| "evaluation_splits": [ | |
| "test" | |
| ], | |
| "few_shots_split": null, | |
| "few_shots_select": null, | |
| "generation_size": 2048, | |
| "stop_sequence": [ | |
| "\n" | |
| ], | |
| "output_regex": null, | |
| "frozen": false, | |
| "suite": [ | |
| "lighteval", | |
| "math" | |
| ], | |
| "original_num_docs": 474, | |
| "effective_num_docs": 474, | |
| "trust_dataset": true, | |
| "must_remove_duplicate_docs": null | |
| }, | |
| "lighteval|math:geometry": { | |
| "name": "math:geometry", | |
| "prompt_function": "math", | |
| "hf_repo": "lighteval/MATH", | |
| "hf_subset": "geometry", | |
| "metric": [ | |
| "quasi_exact_match_math" | |
| ], | |
| "hf_avail_splits": [ | |
| "train", | |
| "test", | |
| "validation" | |
| ], | |
| "evaluation_splits": [ | |
| "test" | |
| ], | |
| "few_shots_split": null, | |
| "few_shots_select": null, | |
| "generation_size": 2048, | |
| "stop_sequence": [ | |
| "\n" | |
| ], | |
| "output_regex": null, | |
| "frozen": false, | |
| "suite": [ | |
| "lighteval", | |
| "math" | |
| ], | |
| "original_num_docs": 479, | |
| "effective_num_docs": 479, | |
| "trust_dataset": true, | |
| "must_remove_duplicate_docs": null | |
| }, | |
| "lighteval|math:intermediate_algebra": { | |
| "name": "math:intermediate_algebra", | |
| "prompt_function": "math", | |
| "hf_repo": "lighteval/MATH", | |
| "hf_subset": "intermediate_algebra", | |
| "metric": [ | |
| "quasi_exact_match_math" | |
| ], | |
| "hf_avail_splits": [ | |
| "train", | |
| "test", | |
| "validation" | |
| ], | |
| "evaluation_splits": [ | |
| "test" | |
| ], | |
| "few_shots_split": null, | |
| "few_shots_select": null, | |
| "generation_size": 2048, | |
| "stop_sequence": [ | |
| "\n" | |
| ], | |
| "output_regex": null, | |
| "frozen": false, | |
| "suite": [ | |
| "lighteval", | |
| "math" | |
| ], | |
| "original_num_docs": 903, | |
| "effective_num_docs": 903, | |
| "trust_dataset": true, | |
| "must_remove_duplicate_docs": null | |
| }, | |
| "lighteval|math:number_theory": { | |
| "name": "math:number_theory", | |
| "prompt_function": "math", | |
| "hf_repo": "lighteval/MATH", | |
| "hf_subset": "number_theory", | |
| "metric": [ | |
| "quasi_exact_match_math" | |
| ], | |
| "hf_avail_splits": [ | |
| "train", | |
| "test", | |
| "validation" | |
| ], | |
| "evaluation_splits": [ | |
| "test" | |
| ], | |
| "few_shots_split": null, | |
| "few_shots_select": null, | |
| "generation_size": 2048, | |
| "stop_sequence": [ | |
| "\n" | |
| ], | |
| "output_regex": null, | |
| "frozen": false, | |
| "suite": [ | |
| "lighteval", | |
| "math" | |
| ], | |
| "original_num_docs": 540, | |
| "effective_num_docs": 540, | |
| "trust_dataset": true, | |
| "must_remove_duplicate_docs": null | |
| }, | |
| "lighteval|math:prealgebra": { | |
| "name": "math:prealgebra", | |
| "prompt_function": "math", | |
| "hf_repo": "lighteval/MATH", | |
| "hf_subset": "prealgebra", | |
| "metric": [ | |
| "quasi_exact_match_math" | |
| ], | |
| "hf_avail_splits": [ | |
| "train", | |
| "test", | |
| "validation" | |
| ], | |
| "evaluation_splits": [ | |
| "test" | |
| ], | |
| "few_shots_split": null, | |
| "few_shots_select": null, | |
| "generation_size": 2048, | |
| "stop_sequence": [ | |
| "\n" | |
| ], | |
| "output_regex": null, | |
| "frozen": false, | |
| "suite": [ | |
| "lighteval", | |
| "math" | |
| ], | |
| "original_num_docs": 871, | |
| "effective_num_docs": 871, | |
| "trust_dataset": true, | |
| "must_remove_duplicate_docs": null | |
| }, | |
| "lighteval|math:precalculus": { | |
| "name": "math:precalculus", | |
| "prompt_function": "math", | |
| "hf_repo": "lighteval/MATH", | |
| "hf_subset": "precalculus", | |
| "metric": [ | |
| "quasi_exact_match_math" | |
| ], | |
| "hf_avail_splits": [ | |
| "train", | |
| "test", | |
| "validation" | |
| ], | |
| "evaluation_splits": [ | |
| "test" | |
| ], | |
| "few_shots_split": null, | |
| "few_shots_select": null, | |
| "generation_size": 2048, | |
| "stop_sequence": [ | |
| "\n" | |
| ], | |
| "output_regex": null, | |
| "frozen": false, | |
| "suite": [ | |
| "lighteval", | |
| "math" | |
| ], | |
| "original_num_docs": 546, | |
| "effective_num_docs": 546, | |
| "trust_dataset": true, | |
| "must_remove_duplicate_docs": null | |
| } | |
| }, | |
| "summary_tasks": { | |
| "lighteval|math:algebra|0": { | |
| "hashes": { | |
| "hash_examples": "37a2fd2f076d2e49", | |
| "hash_full_prompts": "461f2ab197cdfd01", | |
| "hash_input_tokens": "57c833b893656794", | |
| "hash_cont_tokens": "a0bc063195b7e0b7" | |
| }, | |
| "truncated": 1187, | |
| "non_truncated": 0, | |
| "padded": 186, | |
| "non_padded": 1001, | |
| "effective_few_shots": 0.0, | |
| "num_truncated_few_shots": 0 | |
| }, | |
| "lighteval|math:counting_and_probability|0": { | |
| "hashes": { | |
| "hash_examples": "97b4892e28bc078b", | |
| "hash_full_prompts": "e2cc9eb28ca159b1", | |
| "hash_input_tokens": "d88ca301ee288391", | |
| "hash_cont_tokens": "89a7163dc423b0e4" | |
| }, | |
| "truncated": 474, | |
| "non_truncated": 0, | |
| "padded": 42, | |
| "non_padded": 432, | |
| "effective_few_shots": 0.0, | |
| "num_truncated_few_shots": 0 | |
| }, | |
| "lighteval|math:geometry|0": { | |
| "hashes": { | |
| "hash_examples": "9e9f0228b8b3d093", | |
| "hash_full_prompts": "e581f42fefeed8c9", | |
| "hash_input_tokens": "84e7aa576c6b632c", | |
| "hash_cont_tokens": "0e23eaa84b1b7a37" | |
| }, | |
| "truncated": 479, | |
| "non_truncated": 0, | |
| "padded": 129, | |
| "non_padded": 350, | |
| "effective_few_shots": 0.0, | |
| "num_truncated_few_shots": 0 | |
| }, | |
| "lighteval|math:intermediate_algebra|0": { | |
| "hashes": { | |
| "hash_examples": "cfe73a8e28ae94de", | |
| "hash_full_prompts": "7557d30ba2cc15ef", | |
| "hash_input_tokens": "51b363a350a9b8e8", | |
| "hash_cont_tokens": "1e5be24d4a2e7a76" | |
| }, | |
| "truncated": 900, | |
| "non_truncated": 3, | |
| "padded": 54, | |
| "non_padded": 849, | |
| "effective_few_shots": 0.0, | |
| "num_truncated_few_shots": 0 | |
| }, | |
| "lighteval|math:number_theory|0": { | |
| "hashes": { | |
| "hash_examples": "4ee5237cf144afac", | |
| "hash_full_prompts": "7b983a6fe1e3d0a8", | |
| "hash_input_tokens": "a981b91cee6fdb46", | |
| "hash_cont_tokens": "3f7120f8571185e8" | |
| }, | |
| "truncated": 540, | |
| "non_truncated": 0, | |
| "padded": 7, | |
| "non_padded": 533, | |
| "effective_few_shots": 0.0, | |
| "num_truncated_few_shots": 0 | |
| }, | |
| "lighteval|math:prealgebra|0": { | |
| "hashes": { | |
| "hash_examples": "3fb3afeb885f73d8", | |
| "hash_full_prompts": "736a03165dbf8f49", | |
| "hash_input_tokens": "9768d108e479ce04", | |
| "hash_cont_tokens": "81fc828924d24cb6" | |
| }, | |
| "truncated": 871, | |
| "non_truncated": 0, | |
| "padded": 55, | |
| "non_padded": 816, | |
| "effective_few_shots": 0.0, | |
| "num_truncated_few_shots": 0 | |
| }, | |
| "lighteval|math:precalculus|0": { | |
| "hashes": { | |
| "hash_examples": "753e25ab9ec4b46c", | |
| "hash_full_prompts": "632ca5aa79f53b59", | |
| "hash_input_tokens": "046d89bc5abb40cf", | |
| "hash_cont_tokens": "53065f4de66dbad2" | |
| }, | |
| "truncated": 546, | |
| "non_truncated": 0, | |
| "padded": 24, | |
| "non_padded": 522, | |
| "effective_few_shots": 0.0, | |
| "num_truncated_few_shots": 0 | |
| } | |
| }, | |
| "summary_general": { | |
| "hashes": { | |
| "hash_examples": "b76099aa9092a203", | |
| "hash_full_prompts": "2d2e105e1a0eeab9", | |
| "hash_input_tokens": "cda754064cfd1d21", | |
| "hash_cont_tokens": "ddc601b2d9e0f6d3" | |
| }, | |
| "truncated": 4997, | |
| "non_truncated": 3, | |
| "padded": 497, | |
| "non_padded": 4503, | |
| "num_truncated_few_shots": 0 | |
| } | |
| } |