open-r1-eval-leaderboard
/
eval_results
/Qwen
/Qwen1.5-0.5B-Chat
/main
/math
/results_2024-04-21T21-08-03.974672.json
| { | |
| "config_general": { | |
| "lighteval_sha": "?", | |
| "num_fewshot_seeds": 1, | |
| "override_batch_size": 4, | |
| "max_samples": null, | |
| "job_id": "", | |
| "start_time": 867482.247466206, | |
| "end_time": 873665.762882805, | |
| "total_evaluation_time_secondes": "6183.515416598995", | |
| "model_name": "Qwen/Qwen1.5-0.5B-Chat", | |
| "model_sha": "f82bd3692de0283f4a4b31e06d164dd8467fb52e", | |
| "model_dtype": "torch.bfloat16", | |
| "model_size": "1.05 GB", | |
| "config": null | |
| }, | |
| "results": { | |
| "lighteval|math:algebra|0": { | |
| "qem": 0.0, | |
| "qem_stderr": 0.0 | |
| }, | |
| "lighteval|math:counting_and_probability|0": { | |
| "qem": 0.0, | |
| "qem_stderr": 0.0 | |
| }, | |
| "lighteval|math:geometry|0": { | |
| "qem": 0.0, | |
| "qem_stderr": 0.0 | |
| }, | |
| "lighteval|math:intermediate_algebra|0": { | |
| "qem": 0.0, | |
| "qem_stderr": 0.0 | |
| }, | |
| "lighteval|math:number_theory|0": { | |
| "qem": 0.0, | |
| "qem_stderr": 0.0 | |
| }, | |
| "lighteval|math:prealgebra|0": { | |
| "qem": 0.001148105625717566, | |
| "qem_stderr": 0.0011481056257175708 | |
| }, | |
| "lighteval|math:precalculus|0": { | |
| "qem": 0.0, | |
| "qem_stderr": 0.0 | |
| }, | |
| "lighteval|math:_average|0": { | |
| "qem": 0.00016401508938822373, | |
| "qem_stderr": 0.0001640150893882244 | |
| }, | |
| "all": { | |
| "qem": 0.00016401508938822373, | |
| "qem_stderr": 0.0001640150893882244 | |
| } | |
| }, | |
| "versions": { | |
| "lighteval|math:algebra|0": 0, | |
| "lighteval|math:counting_and_probability|0": 0, | |
| "lighteval|math:geometry|0": 0, | |
| "lighteval|math:intermediate_algebra|0": 0, | |
| "lighteval|math:number_theory|0": 0, | |
| "lighteval|math:prealgebra|0": 0, | |
| "lighteval|math:precalculus|0": 0 | |
| }, | |
| "config_tasks": { | |
| "lighteval|math:algebra": { | |
| "name": "math:algebra", | |
| "prompt_function": "math", | |
| "hf_repo": "lighteval/MATH", | |
| "hf_subset": "algebra", | |
| "metric": [ | |
| "quasi_exact_match_math" | |
| ], | |
| "hf_avail_splits": [ | |
| "train", | |
| "test", | |
| "validation" | |
| ], | |
| "evaluation_splits": [ | |
| "test" | |
| ], | |
| "few_shots_split": null, | |
| "few_shots_select": null, | |
| "generation_size": 2048, | |
| "stop_sequence": [ | |
| "\n" | |
| ], | |
| "output_regex": null, | |
| "frozen": false, | |
| "suite": [ | |
| "lighteval", | |
| "math" | |
| ], | |
| "original_num_docs": 1187, | |
| "effective_num_docs": 1187, | |
| "trust_dataset": true, | |
| "must_remove_duplicate_docs": null | |
| }, | |
| "lighteval|math:counting_and_probability": { | |
| "name": "math:counting_and_probability", | |
| "prompt_function": "math", | |
| "hf_repo": "lighteval/MATH", | |
| "hf_subset": "counting_and_probability", | |
| "metric": [ | |
| "quasi_exact_match_math" | |
| ], | |
| "hf_avail_splits": [ | |
| "train", | |
| "test", | |
| "validation" | |
| ], | |
| "evaluation_splits": [ | |
| "test" | |
| ], | |
| "few_shots_split": null, | |
| "few_shots_select": null, | |
| "generation_size": 2048, | |
| "stop_sequence": [ | |
| "\n" | |
| ], | |
| "output_regex": null, | |
| "frozen": false, | |
| "suite": [ | |
| "lighteval", | |
| "math" | |
| ], | |
| "original_num_docs": 474, | |
| "effective_num_docs": 474, | |
| "trust_dataset": true, | |
| "must_remove_duplicate_docs": null | |
| }, | |
| "lighteval|math:geometry": { | |
| "name": "math:geometry", | |
| "prompt_function": "math", | |
| "hf_repo": "lighteval/MATH", | |
| "hf_subset": "geometry", | |
| "metric": [ | |
| "quasi_exact_match_math" | |
| ], | |
| "hf_avail_splits": [ | |
| "train", | |
| "test", | |
| "validation" | |
| ], | |
| "evaluation_splits": [ | |
| "test" | |
| ], | |
| "few_shots_split": null, | |
| "few_shots_select": null, | |
| "generation_size": 2048, | |
| "stop_sequence": [ | |
| "\n" | |
| ], | |
| "output_regex": null, | |
| "frozen": false, | |
| "suite": [ | |
| "lighteval", | |
| "math" | |
| ], | |
| "original_num_docs": 479, | |
| "effective_num_docs": 479, | |
| "trust_dataset": true, | |
| "must_remove_duplicate_docs": null | |
| }, | |
| "lighteval|math:intermediate_algebra": { | |
| "name": "math:intermediate_algebra", | |
| "prompt_function": "math", | |
| "hf_repo": "lighteval/MATH", | |
| "hf_subset": "intermediate_algebra", | |
| "metric": [ | |
| "quasi_exact_match_math" | |
| ], | |
| "hf_avail_splits": [ | |
| "train", | |
| "test", | |
| "validation" | |
| ], | |
| "evaluation_splits": [ | |
| "test" | |
| ], | |
| "few_shots_split": null, | |
| "few_shots_select": null, | |
| "generation_size": 2048, | |
| "stop_sequence": [ | |
| "\n" | |
| ], | |
| "output_regex": null, | |
| "frozen": false, | |
| "suite": [ | |
| "lighteval", | |
| "math" | |
| ], | |
| "original_num_docs": 903, | |
| "effective_num_docs": 903, | |
| "trust_dataset": true, | |
| "must_remove_duplicate_docs": null | |
| }, | |
| "lighteval|math:number_theory": { | |
| "name": "math:number_theory", | |
| "prompt_function": "math", | |
| "hf_repo": "lighteval/MATH", | |
| "hf_subset": "number_theory", | |
| "metric": [ | |
| "quasi_exact_match_math" | |
| ], | |
| "hf_avail_splits": [ | |
| "train", | |
| "test", | |
| "validation" | |
| ], | |
| "evaluation_splits": [ | |
| "test" | |
| ], | |
| "few_shots_split": null, | |
| "few_shots_select": null, | |
| "generation_size": 2048, | |
| "stop_sequence": [ | |
| "\n" | |
| ], | |
| "output_regex": null, | |
| "frozen": false, | |
| "suite": [ | |
| "lighteval", | |
| "math" | |
| ], | |
| "original_num_docs": 540, | |
| "effective_num_docs": 540, | |
| "trust_dataset": true, | |
| "must_remove_duplicate_docs": null | |
| }, | |
| "lighteval|math:prealgebra": { | |
| "name": "math:prealgebra", | |
| "prompt_function": "math", | |
| "hf_repo": "lighteval/MATH", | |
| "hf_subset": "prealgebra", | |
| "metric": [ | |
| "quasi_exact_match_math" | |
| ], | |
| "hf_avail_splits": [ | |
| "train", | |
| "test", | |
| "validation" | |
| ], | |
| "evaluation_splits": [ | |
| "test" | |
| ], | |
| "few_shots_split": null, | |
| "few_shots_select": null, | |
| "generation_size": 2048, | |
| "stop_sequence": [ | |
| "\n" | |
| ], | |
| "output_regex": null, | |
| "frozen": false, | |
| "suite": [ | |
| "lighteval", | |
| "math" | |
| ], | |
| "original_num_docs": 871, | |
| "effective_num_docs": 871, | |
| "trust_dataset": true, | |
| "must_remove_duplicate_docs": null | |
| }, | |
| "lighteval|math:precalculus": { | |
| "name": "math:precalculus", | |
| "prompt_function": "math", | |
| "hf_repo": "lighteval/MATH", | |
| "hf_subset": "precalculus", | |
| "metric": [ | |
| "quasi_exact_match_math" | |
| ], | |
| "hf_avail_splits": [ | |
| "train", | |
| "test", | |
| "validation" | |
| ], | |
| "evaluation_splits": [ | |
| "test" | |
| ], | |
| "few_shots_split": null, | |
| "few_shots_select": null, | |
| "generation_size": 2048, | |
| "stop_sequence": [ | |
| "\n" | |
| ], | |
| "output_regex": null, | |
| "frozen": false, | |
| "suite": [ | |
| "lighteval", | |
| "math" | |
| ], | |
| "original_num_docs": 546, | |
| "effective_num_docs": 546, | |
| "trust_dataset": true, | |
| "must_remove_duplicate_docs": null | |
| } | |
| }, | |
| "summary_tasks": { | |
| "lighteval|math:algebra|0": { | |
| "hashes": { | |
| "hash_examples": "37a2fd2f076d2e49", | |
| "hash_full_prompts": "3cd2a9fbe38ca7d1", | |
| "hash_input_tokens": "af20703b79058645", | |
| "hash_cont_tokens": "fe52ffbbb871419b" | |
| }, | |
| "truncated": 1187, | |
| "non_truncated": 0, | |
| "padded": 162, | |
| "non_padded": 1025, | |
| "effective_few_shots": 0.0, | |
| "num_truncated_few_shots": 0 | |
| }, | |
| "lighteval|math:counting_and_probability|0": { | |
| "hashes": { | |
| "hash_examples": "97b4892e28bc078b", | |
| "hash_full_prompts": "23c0b10a3f2357ea", | |
| "hash_input_tokens": "b4c45e9b83b3e1c5", | |
| "hash_cont_tokens": "42d9b4d881832736" | |
| }, | |
| "truncated": 474, | |
| "non_truncated": 0, | |
| "padded": 37, | |
| "non_padded": 437, | |
| "effective_few_shots": 0.0, | |
| "num_truncated_few_shots": 0 | |
| }, | |
| "lighteval|math:geometry|0": { | |
| "hashes": { | |
| "hash_examples": "9e9f0228b8b3d093", | |
| "hash_full_prompts": "25ae739a5bd7cf61", | |
| "hash_input_tokens": "5841f4204130c263", | |
| "hash_cont_tokens": "00ac23b316de2151" | |
| }, | |
| "truncated": 478, | |
| "non_truncated": 1, | |
| "padded": 116, | |
| "non_padded": 363, | |
| "effective_few_shots": 0.0, | |
| "num_truncated_few_shots": 0 | |
| }, | |
| "lighteval|math:intermediate_algebra|0": { | |
| "hashes": { | |
| "hash_examples": "cfe73a8e28ae94de", | |
| "hash_full_prompts": "c8813c4dcc73d3b8", | |
| "hash_input_tokens": "b7735888ef889dec", | |
| "hash_cont_tokens": "39fac2a1c17a2e94" | |
| }, | |
| "truncated": 901, | |
| "non_truncated": 2, | |
| "padded": 54, | |
| "non_padded": 849, | |
| "effective_few_shots": 0.0, | |
| "num_truncated_few_shots": 0 | |
| }, | |
| "lighteval|math:number_theory|0": { | |
| "hashes": { | |
| "hash_examples": "4ee5237cf144afac", | |
| "hash_full_prompts": "603a451eb11974d5", | |
| "hash_input_tokens": "93e5d92f4261e2c8", | |
| "hash_cont_tokens": "af9ceab4afe70f1c" | |
| }, | |
| "truncated": 540, | |
| "non_truncated": 0, | |
| "padded": 4, | |
| "non_padded": 536, | |
| "effective_few_shots": 0.0, | |
| "num_truncated_few_shots": 0 | |
| }, | |
| "lighteval|math:prealgebra|0": { | |
| "hashes": { | |
| "hash_examples": "3fb3afeb885f73d8", | |
| "hash_full_prompts": "e5e12c93d4a8be8a", | |
| "hash_input_tokens": "35fa5bc74f072f40", | |
| "hash_cont_tokens": "c997620dc5abc01e" | |
| }, | |
| "truncated": 871, | |
| "non_truncated": 0, | |
| "padded": 46, | |
| "non_padded": 825, | |
| "effective_few_shots": 0.0, | |
| "num_truncated_few_shots": 0 | |
| }, | |
| "lighteval|math:precalculus|0": { | |
| "hashes": { | |
| "hash_examples": "753e25ab9ec4b46c", | |
| "hash_full_prompts": "eabf79561a9582e5", | |
| "hash_input_tokens": "51ed0786cadde815", | |
| "hash_cont_tokens": "ede52acd76fdf266" | |
| }, | |
| "truncated": 546, | |
| "non_truncated": 0, | |
| "padded": 23, | |
| "non_padded": 523, | |
| "effective_few_shots": 0.0, | |
| "num_truncated_few_shots": 0 | |
| } | |
| }, | |
| "summary_general": { | |
| "hashes": { | |
| "hash_examples": "b76099aa9092a203", | |
| "hash_full_prompts": "5a707e66f34d9ab5", | |
| "hash_input_tokens": "59f68ba3e73a437f", | |
| "hash_cont_tokens": "ab3aa0922abddcb3" | |
| }, | |
| "truncated": 4997, | |
| "non_truncated": 3, | |
| "padded": 442, | |
| "non_padded": 4558, | |
| "num_truncated_few_shots": 0 | |
| } | |
| } |