{ "config_general": { "lighteval_sha": "?", "num_fewshot_seeds": 1, "override_batch_size": 1, "max_samples": null, "job_id": "", "start_time": 3562036.976720208, "end_time": 3580157.72115306, "total_evaluation_time_secondes": "18120.744432852138", "model_name": "AI-MO/deepseek-math-7b-sft", "model_sha": "3110e41537403c0ce4a0318f61ac3bead085cdf6", "model_dtype": "torch.bfloat16", "model_size": "12.93 GB", "config": null }, "results": { "lighteval|math:algebra|0": { "qem": 0.497893850042123, "qem_stderr": 0.014518568796862752 }, "lighteval|math:counting_and_probability|0": { "qem": 0.2510548523206751, "qem_stderr": 0.019937869321473834 }, "lighteval|math:geometry|0": { "qem": 0.29018789144050106, "qem_stderr": 0.020758568281959495 }, "lighteval|math:intermediate_algebra|0": { "qem": 0.14839424141749724, "qem_stderr": 0.01183653400175703 }, "lighteval|math:number_theory|0": { "qem": 0.2518518518518518, "qem_stderr": 0.018697002853812705 }, "lighteval|math:prealgebra|0": { "qem": 0.5579793340987371, "qem_stderr": 0.016837232852669793 }, "lighteval|math:precalculus|0": { "qem": 0.21794871794871795, "qem_stderr": 0.017684652252975615 }, "lighteval|math:_average|0": { "qem": 0.31647296273144326, "qem_stderr": 0.017181489765930175 }, "all": { "qem": 0.31647296273144326, "qem_stderr": 0.017181489765930175 } }, "versions": { "lighteval|math:algebra|0": 0, "lighteval|math:counting_and_probability|0": 0, "lighteval|math:geometry|0": 0, "lighteval|math:intermediate_algebra|0": 0, "lighteval|math:number_theory|0": 0, "lighteval|math:prealgebra|0": 0, "lighteval|math:precalculus|0": 0 }, "config_tasks": { "lighteval|math:algebra": { "name": "math:algebra", "prompt_function": "math", "hf_repo": "lighteval/MATH", "hf_subset": "algebra", "metric": [ "quasi_exact_match_math" ], "hf_avail_splits": [ "train", "test", "validation" ], "evaluation_splits": [ "test" ], "few_shots_split": null, "few_shots_select": null, "generation_size": 2048, "stop_sequence": [ "\n" ], "output_regex": null, "frozen": false, "suite": [ "lighteval", "math" ], "original_num_docs": 1187, "effective_num_docs": 1187, "trust_dataset": true, "must_remove_duplicate_docs": null }, "lighteval|math:counting_and_probability": { "name": "math:counting_and_probability", "prompt_function": "math", "hf_repo": "lighteval/MATH", "hf_subset": "counting_and_probability", "metric": [ "quasi_exact_match_math" ], "hf_avail_splits": [ "train", "test", "validation" ], "evaluation_splits": [ "test" ], "few_shots_split": null, "few_shots_select": null, "generation_size": 2048, "stop_sequence": [ "\n" ], "output_regex": null, "frozen": false, "suite": [ "lighteval", "math" ], "original_num_docs": 474, "effective_num_docs": 474, "trust_dataset": true, "must_remove_duplicate_docs": null }, "lighteval|math:geometry": { "name": "math:geometry", "prompt_function": "math", "hf_repo": "lighteval/MATH", "hf_subset": "geometry", "metric": [ "quasi_exact_match_math" ], "hf_avail_splits": [ "train", "test", "validation" ], "evaluation_splits": [ "test" ], "few_shots_split": null, "few_shots_select": null, "generation_size": 2048, "stop_sequence": [ "\n" ], "output_regex": null, "frozen": false, "suite": [ "lighteval", "math" ], "original_num_docs": 479, "effective_num_docs": 479, "trust_dataset": true, "must_remove_duplicate_docs": null }, "lighteval|math:intermediate_algebra": { "name": "math:intermediate_algebra", "prompt_function": "math", "hf_repo": "lighteval/MATH", "hf_subset": "intermediate_algebra", "metric": [ "quasi_exact_match_math" ], "hf_avail_splits": [ "train", "test", "validation" ], "evaluation_splits": [ "test" ], "few_shots_split": null, "few_shots_select": null, "generation_size": 2048, "stop_sequence": [ "\n" ], "output_regex": null, "frozen": false, "suite": [ "lighteval", "math" ], "original_num_docs": 903, "effective_num_docs": 903, "trust_dataset": true, "must_remove_duplicate_docs": null }, "lighteval|math:number_theory": { "name": "math:number_theory", "prompt_function": "math", "hf_repo": "lighteval/MATH", "hf_subset": "number_theory", "metric": [ "quasi_exact_match_math" ], "hf_avail_splits": [ "train", "test", "validation" ], "evaluation_splits": [ "test" ], "few_shots_split": null, "few_shots_select": null, "generation_size": 2048, "stop_sequence": [ "\n" ], "output_regex": null, "frozen": false, "suite": [ "lighteval", "math" ], "original_num_docs": 540, "effective_num_docs": 540, "trust_dataset": true, "must_remove_duplicate_docs": null }, "lighteval|math:prealgebra": { "name": "math:prealgebra", "prompt_function": "math", "hf_repo": "lighteval/MATH", "hf_subset": "prealgebra", "metric": [ "quasi_exact_match_math" ], "hf_avail_splits": [ "train", "test", "validation" ], "evaluation_splits": [ "test" ], "few_shots_split": null, "few_shots_select": null, "generation_size": 2048, "stop_sequence": [ "\n" ], "output_regex": null, "frozen": false, "suite": [ "lighteval", "math" ], "original_num_docs": 871, "effective_num_docs": 871, "trust_dataset": true, "must_remove_duplicate_docs": null }, "lighteval|math:precalculus": { "name": "math:precalculus", "prompt_function": "math", "hf_repo": "lighteval/MATH", "hf_subset": "precalculus", "metric": [ "quasi_exact_match_math" ], "hf_avail_splits": [ "train", "test", "validation" ], "evaluation_splits": [ "test" ], "few_shots_split": null, "few_shots_select": null, "generation_size": 2048, "stop_sequence": [ "\n" ], "output_regex": null, "frozen": false, "suite": [ "lighteval", "math" ], "original_num_docs": 546, "effective_num_docs": 546, "trust_dataset": true, "must_remove_duplicate_docs": null } }, "summary_tasks": { "lighteval|math:algebra|0": { "hashes": { "hash_examples": "37a2fd2f076d2e49", "hash_full_prompts": "461f2ab197cdfd01", "hash_input_tokens": "8209eebabb8132e4", "hash_cont_tokens": "dce3c8b3a21ece07" }, "truncated": 1187, "non_truncated": 0, "padded": 0, "non_padded": 1187, "effective_few_shots": 0.0, "num_truncated_few_shots": 0 }, "lighteval|math:counting_and_probability|0": { "hashes": { "hash_examples": "97b4892e28bc078b", "hash_full_prompts": "e2cc9eb28ca159b1", "hash_input_tokens": "3c116046c93accc7", "hash_cont_tokens": "c87f701476bbcce3" }, "truncated": 474, "non_truncated": 0, "padded": 0, "non_padded": 474, "effective_few_shots": 0.0, "num_truncated_few_shots": 0 }, "lighteval|math:geometry|0": { "hashes": { "hash_examples": "9e9f0228b8b3d093", "hash_full_prompts": "e581f42fefeed8c9", "hash_input_tokens": "91420fc689743b29", "hash_cont_tokens": "794654d76991bc84" }, "truncated": 479, "non_truncated": 0, "padded": 0, "non_padded": 479, "effective_few_shots": 0.0, "num_truncated_few_shots": 0 }, "lighteval|math:intermediate_algebra|0": { "hashes": { "hash_examples": "cfe73a8e28ae94de", "hash_full_prompts": "7557d30ba2cc15ef", "hash_input_tokens": "e47e75b9d3b1247a", "hash_cont_tokens": "fb07fe0a5c595311" }, "truncated": 903, "non_truncated": 0, "padded": 0, "non_padded": 903, "effective_few_shots": 0.0, "num_truncated_few_shots": 0 }, "lighteval|math:number_theory|0": { "hashes": { "hash_examples": "4ee5237cf144afac", "hash_full_prompts": "7b983a6fe1e3d0a8", "hash_input_tokens": "8f8d85639345093d", "hash_cont_tokens": "8fccdd28ac529677" }, "truncated": 540, "non_truncated": 0, "padded": 0, "non_padded": 540, "effective_few_shots": 0.0, "num_truncated_few_shots": 0 }, "lighteval|math:prealgebra|0": { "hashes": { "hash_examples": "3fb3afeb885f73d8", "hash_full_prompts": "736a03165dbf8f49", "hash_input_tokens": "e1c72cbbee4a9c7f", "hash_cont_tokens": "88d730aab7d93c17" }, "truncated": 871, "non_truncated": 0, "padded": 0, "non_padded": 871, "effective_few_shots": 0.0, "num_truncated_few_shots": 0 }, "lighteval|math:precalculus|0": { "hashes": { "hash_examples": "753e25ab9ec4b46c", "hash_full_prompts": "632ca5aa79f53b59", "hash_input_tokens": "8218370d326a70ad", "hash_cont_tokens": "865718a7cfb00f9a" }, "truncated": 546, "non_truncated": 0, "padded": 0, "non_padded": 546, "effective_few_shots": 0.0, "num_truncated_few_shots": 0 } }, "summary_general": { "hashes": { "hash_examples": "b76099aa9092a203", "hash_full_prompts": "2d2e105e1a0eeab9", "hash_input_tokens": "59674354dfc8622f", "hash_cont_tokens": "44c603e4bf490796" }, "truncated": 5000, "non_truncated": 0, "padded": 0, "non_padded": 5000, "num_truncated_few_shots": 0 } }