open-r1-eval-leaderboard
/
eval_results
/Qwen
/Qwen1.5-0.5B-Chat
/main
/mini_math
/results_2024-04-23T11-35-30.275417.json
edbeeching
HF staff
Upload eval_results/Qwen/Qwen1.5-0.5B-Chat/main/mini_math/results_2024-04-23T11-35-30.275417.json with huggingface_hub
7d6194a
verified
| { | |
| "config_general": { | |
| "lighteval_sha": "?", | |
| "num_fewshot_seeds": 1, | |
| "override_batch_size": 4, | |
| "max_samples": null, | |
| "job_id": "", | |
| "start_time": 113890.410646821, | |
| "end_time": 114685.798350712, | |
| "total_evaluation_time_secondes": "795.3877038909995", | |
| "model_name": "Qwen/Qwen1.5-0.5B-Chat", | |
| "model_sha": "f82bd3692de0283f4a4b31e06d164dd8467fb52e", | |
| "model_dtype": "torch.bfloat16", | |
| "model_size": "1.05 GB", | |
| "config": null | |
| }, | |
| "results": { | |
| "custom|mini_math:level_1|0": { | |
| "qem": 0.0, | |
| "qem_stderr": 0.0 | |
| }, | |
| "custom|mini_math:level_2|0": { | |
| "qem": 0.0, | |
| "qem_stderr": 0.0 | |
| }, | |
| "custom|mini_math:level_3|0": { | |
| "qem": 0.0, | |
| "qem_stderr": 0.0 | |
| }, | |
| "custom|mini_math:level_4|0": { | |
| "qem": 0.0, | |
| "qem_stderr": 0.0 | |
| }, | |
| "custom|mini_math:level_5|0": { | |
| "qem": 0.0, | |
| "qem_stderr": 0.0 | |
| }, | |
| "custom|mini_math:_average|0": { | |
| "qem": 0.0, | |
| "qem_stderr": 0.0 | |
| }, | |
| "all": { | |
| "qem": 0.0, | |
| "qem_stderr": 0.0 | |
| } | |
| }, | |
| "versions": { | |
| "custom|mini_math:level_1|0": 0, | |
| "custom|mini_math:level_2|0": 0, | |
| "custom|mini_math:level_3|0": 0, | |
| "custom|mini_math:level_4|0": 0, | |
| "custom|mini_math:level_5|0": 0 | |
| }, | |
| "config_tasks": { | |
| "custom|mini_math:level_1": { | |
| "name": "mini_math:level_1", | |
| "prompt_function": "math", | |
| "hf_repo": "HuggingFaceH4/lighteval-mini-math", | |
| "hf_subset": "Level 1", | |
| "metric": [ | |
| "quasi_exact_match_math" | |
| ], | |
| "hf_avail_splits": [ | |
| "train", | |
| "test" | |
| ], | |
| "evaluation_splits": [ | |
| "test" | |
| ], | |
| "few_shots_split": null, | |
| "few_shots_select": null, | |
| "generation_size": 2048, | |
| "stop_sequence": null, | |
| "output_regex": null, | |
| "frozen": false, | |
| "suite": [ | |
| "custom", | |
| "mini_math" | |
| ], | |
| "original_num_docs": 38, | |
| "effective_num_docs": 38, | |
| "trust_dataset": null, | |
| "must_remove_duplicate_docs": null | |
| }, | |
| "custom|mini_math:level_2": { | |
| "name": "mini_math:level_2", | |
| "prompt_function": "math", | |
| "hf_repo": "HuggingFaceH4/lighteval-mini-math", | |
| "hf_subset": "Level 2", | |
| "metric": [ | |
| "quasi_exact_match_math" | |
| ], | |
| "hf_avail_splits": [ | |
| "train", | |
| "test" | |
| ], | |
| "evaluation_splits": [ | |
| "test" | |
| ], | |
| "few_shots_split": null, | |
| "few_shots_select": null, | |
| "generation_size": 2048, | |
| "stop_sequence": null, | |
| "output_regex": null, | |
| "frozen": false, | |
| "suite": [ | |
| "custom", | |
| "mini_math" | |
| ], | |
| "original_num_docs": 71, | |
| "effective_num_docs": 71, | |
| "trust_dataset": null, | |
| "must_remove_duplicate_docs": null | |
| }, | |
| "custom|mini_math:level_3": { | |
| "name": "mini_math:level_3", | |
| "prompt_function": "math", | |
| "hf_repo": "HuggingFaceH4/lighteval-mini-math", | |
| "hf_subset": "Level 3", | |
| "metric": [ | |
| "quasi_exact_match_math" | |
| ], | |
| "hf_avail_splits": [ | |
| "train", | |
| "test" | |
| ], | |
| "evaluation_splits": [ | |
| "test" | |
| ], | |
| "few_shots_split": null, | |
| "few_shots_select": null, | |
| "generation_size": 2048, | |
| "stop_sequence": null, | |
| "output_regex": null, | |
| "frozen": false, | |
| "suite": [ | |
| "custom", | |
| "mini_math" | |
| ], | |
| "original_num_docs": 78, | |
| "effective_num_docs": 78, | |
| "trust_dataset": null, | |
| "must_remove_duplicate_docs": null | |
| }, | |
| "custom|mini_math:level_4": { | |
| "name": "mini_math:level_4", | |
| "prompt_function": "math", | |
| "hf_repo": "HuggingFaceH4/lighteval-mini-math", | |
| "hf_subset": "Level 4", | |
| "metric": [ | |
| "quasi_exact_match_math" | |
| ], | |
| "hf_avail_splits": [ | |
| "train", | |
| "test" | |
| ], | |
| "evaluation_splits": [ | |
| "test" | |
| ], | |
| "few_shots_split": null, | |
| "few_shots_select": null, | |
| "generation_size": 2048, | |
| "stop_sequence": null, | |
| "output_regex": null, | |
| "frozen": false, | |
| "suite": [ | |
| "custom", | |
| "mini_math" | |
| ], | |
| "original_num_docs": 109, | |
| "effective_num_docs": 109, | |
| "trust_dataset": null, | |
| "must_remove_duplicate_docs": null | |
| }, | |
| "custom|mini_math:level_5": { | |
| "name": "mini_math:level_5", | |
| "prompt_function": "math", | |
| "hf_repo": "HuggingFaceH4/lighteval-mini-math", | |
| "hf_subset": "Level 5", | |
| "metric": [ | |
| "quasi_exact_match_math" | |
| ], | |
| "hf_avail_splits": [ | |
| "train", | |
| "test" | |
| ], | |
| "evaluation_splits": [ | |
| "test" | |
| ], | |
| "few_shots_split": null, | |
| "few_shots_select": null, | |
| "generation_size": 2048, | |
| "stop_sequence": null, | |
| "output_regex": null, | |
| "frozen": false, | |
| "suite": [ | |
| "custom", | |
| "mini_math" | |
| ], | |
| "original_num_docs": 107, | |
| "effective_num_docs": 107, | |
| "trust_dataset": null, | |
| "must_remove_duplicate_docs": null | |
| } | |
| }, | |
| "summary_tasks": { | |
| "custom|mini_math:level_1|0": { | |
| "hashes": { | |
| "hash_examples": "781a25d9a2f29e87", | |
| "hash_full_prompts": "09ef7f6df57f47ef", | |
| "hash_input_tokens": "a52e14904d82f4f0", | |
| "hash_cont_tokens": "1378bece65ca5077" | |
| }, | |
| "truncated": 38, | |
| "non_truncated": 0, | |
| "padded": 27, | |
| "non_padded": 11, | |
| "effective_few_shots": 0.0, | |
| "num_truncated_few_shots": 0 | |
| }, | |
| "custom|mini_math:level_2|0": { | |
| "hashes": { | |
| "hash_examples": "471a4a8568c70994", | |
| "hash_full_prompts": "71779330375b1791", | |
| "hash_input_tokens": "3882edc6f5609345", | |
| "hash_cont_tokens": "ff9806e2946dc746" | |
| }, | |
| "truncated": 71, | |
| "non_truncated": 0, | |
| "padded": 24, | |
| "non_padded": 47, | |
| "effective_few_shots": 0.0, | |
| "num_truncated_few_shots": 0 | |
| }, | |
| "custom|mini_math:level_3|0": { | |
| "hashes": { | |
| "hash_examples": "aa6979a90c9a7776", | |
| "hash_full_prompts": "a9999d3932558466", | |
| "hash_input_tokens": "01ceca3580b9eb5e", | |
| "hash_cont_tokens": "fbf0bffb8a2612ef" | |
| }, | |
| "truncated": 78, | |
| "non_truncated": 0, | |
| "padded": 36, | |
| "non_padded": 42, | |
| "effective_few_shots": 0.0, | |
| "num_truncated_few_shots": 0 | |
| }, | |
| "custom|mini_math:level_4|0": { | |
| "hashes": { | |
| "hash_examples": "2e9e3be067de6ec6", | |
| "hash_full_prompts": "30f669c1c4439175", | |
| "hash_input_tokens": "4a8097cf379ba41c", | |
| "hash_cont_tokens": "88e54955e7213376" | |
| }, | |
| "truncated": 109, | |
| "non_truncated": 0, | |
| "padded": 38, | |
| "non_padded": 71, | |
| "effective_few_shots": 0.0, | |
| "num_truncated_few_shots": 0 | |
| }, | |
| "custom|mini_math:level_5|0": { | |
| "hashes": { | |
| "hash_examples": "18b402b510f9a746", | |
| "hash_full_prompts": "0c8f48ff502c1190", | |
| "hash_input_tokens": "b6ecf04a8c2e48b4", | |
| "hash_cont_tokens": "c603094defd72220" | |
| }, | |
| "truncated": 106, | |
| "non_truncated": 1, | |
| "padded": 38, | |
| "non_padded": 69, | |
| "effective_few_shots": 0.0, | |
| "num_truncated_few_shots": 0 | |
| } | |
| }, | |
| "summary_general": { | |
| "hashes": { | |
| "hash_examples": "470ce01cab0383c9", | |
| "hash_full_prompts": "36a8a8499eb2fd53", | |
| "hash_input_tokens": "a651497058968421", | |
| "hash_cont_tokens": "16318eec6b11890e" | |
| }, | |
| "truncated": 402, | |
| "non_truncated": 1, | |
| "padded": 163, | |
| "non_padded": 240, | |
| "num_truncated_few_shots": 0 | |
| } | |
| } |