open-r1-eval-leaderboard
/
eval_results
/meta-llama
/Meta-Llama-3-8B-Instruct
/main
/agieval
/results_2024-04-22T11-51-37.908232.json
| { | |
| "config_general": { | |
| "lighteval_sha": "?", | |
| "num_fewshot_seeds": 1, | |
| "override_batch_size": 4, | |
| "max_samples": null, | |
| "job_id": "", | |
| "start_time": 1988758.699140144, | |
| "end_time": 1989306.575083681, | |
| "total_evaluation_time_secondes": "547.8759435368702", | |
| "model_name": "meta-llama/Meta-Llama-3-8B-Instruct", | |
| "model_sha": "1448453bdb895762499deb4176c1dd83b145fac1", | |
| "model_dtype": "torch.bfloat16", | |
| "model_size": "15.08 GB", | |
| "config": null | |
| }, | |
| "results": { | |
| "lighteval|agieval:aqua-rat|0": { | |
| "acc": 0.25196850393700787, | |
| "acc_stderr": 0.027294353392553605, | |
| "acc_norm": 0.24803149606299213, | |
| "acc_norm_stderr": 0.02715144709118171 | |
| }, | |
| "lighteval|agieval:gaokao-biology|0": { | |
| "acc": 0.46190476190476193, | |
| "acc_stderr": 0.034485192220162664, | |
| "acc_norm": 0.38095238095238093, | |
| "acc_norm_stderr": 0.033591100467499906 | |
| }, | |
| "lighteval|agieval:gaokao-chemistry|0": { | |
| "acc": 0.32367149758454106, | |
| "acc_stderr": 0.03259848850179343, | |
| "acc_norm": 0.32367149758454106, | |
| "acc_norm_stderr": 0.032598488501793435 | |
| }, | |
| "lighteval|agieval:gaokao-chinese|0": { | |
| "acc": 0.3130081300813008, | |
| "acc_stderr": 0.02962583267300022, | |
| "acc_norm": 0.2967479674796748, | |
| "acc_norm_stderr": 0.029185445861037912 | |
| }, | |
| "lighteval|agieval:gaokao-english|0": { | |
| "acc": 0.696078431372549, | |
| "acc_stderr": 0.026336613469046637, | |
| "acc_norm": 0.5620915032679739, | |
| "acc_norm_stderr": 0.028408302020332687 | |
| }, | |
| "lighteval|agieval:gaokao-geography|0": { | |
| "acc": 0.5175879396984925, | |
| "acc_stderr": 0.035511462395976016, | |
| "acc_norm": 0.49246231155778897, | |
| "acc_norm_stderr": 0.035529414706104066 | |
| }, | |
| "lighteval|agieval:gaokao-history|0": { | |
| "acc": 0.5234042553191489, | |
| "acc_stderr": 0.03265019475033582, | |
| "acc_norm": 0.39574468085106385, | |
| "acc_norm_stderr": 0.031967586978353627 | |
| }, | |
| "lighteval|agieval:gaokao-mathqa|0": { | |
| "acc": 0.2792022792022792, | |
| "acc_stderr": 0.023979060299146256, | |
| "acc_norm": 0.27635327635327633, | |
| "acc_norm_stderr": 0.023903505003127216 | |
| }, | |
| "lighteval|agieval:gaokao-physics|0": { | |
| "acc": 0.395, | |
| "acc_stderr": 0.03465370682892271, | |
| "acc_norm": 0.38, | |
| "acc_norm_stderr": 0.0344081327303582 | |
| }, | |
| "lighteval|agieval:logiqa-en|0": { | |
| "acc": 0.41321044546851, | |
| "acc_stderr": 0.01931390783165284, | |
| "acc_norm": 0.34408602150537637, | |
| "acc_norm_stderr": 0.01863375065717621 | |
| }, | |
| "lighteval|agieval:logiqa-zh|0": { | |
| "acc": 0.31490015360983103, | |
| "acc_stderr": 0.018218251493671664, | |
| "acc_norm": 0.33640552995391704, | |
| "acc_norm_stderr": 0.01853216950935858 | |
| }, | |
| "lighteval|agieval:lsat-ar|0": { | |
| "acc": 0.2217391304347826, | |
| "acc_stderr": 0.027451496604058916, | |
| "acc_norm": 0.1782608695652174, | |
| "acc_norm_stderr": 0.025291655246273914 | |
| }, | |
| "lighteval|agieval:lsat-lr|0": { | |
| "acc": 0.5333333333333333, | |
| "acc_stderr": 0.022112806381564227, | |
| "acc_norm": 0.32941176470588235, | |
| "acc_norm_stderr": 0.020832367421292207 | |
| }, | |
| "lighteval|agieval:lsat-rc|0": { | |
| "acc": 0.6394052044609665, | |
| "acc_stderr": 0.029331239329958934, | |
| "acc_norm": 0.35687732342007433, | |
| "acc_norm_stderr": 0.029264357329058688 | |
| }, | |
| "lighteval|agieval:sat-en|0": { | |
| "acc": 0.7572815533980582, | |
| "acc_stderr": 0.029943540553570538, | |
| "acc_norm": 0.48058252427184467, | |
| "acc_norm_stderr": 0.034895171350660135 | |
| }, | |
| "lighteval|agieval:sat-en-without-passage|0": { | |
| "acc": 0.3932038834951456, | |
| "acc_stderr": 0.034115627597025605, | |
| "acc_norm": 0.2669902912621359, | |
| "acc_norm_stderr": 0.030897665523458157 | |
| }, | |
| "lighteval|agieval:sat-math|0": { | |
| "acc": 0.38181818181818183, | |
| "acc_stderr": 0.032829506847783727, | |
| "acc_norm": 0.2681818181818182, | |
| "acc_norm_stderr": 0.02993603001489284 | |
| }, | |
| "lighteval|agieval:_average|0": { | |
| "acc": 0.43627751088934646, | |
| "acc_stderr": 0.028850075362954343, | |
| "acc_norm": 0.34805007393976223, | |
| "acc_norm_stderr": 0.02853097590658585 | |
| }, | |
| "all": { | |
| "acc": 0.43627751088934646, | |
| "acc_stderr": 0.028850075362954343, | |
| "acc_norm": 0.34805007393976223, | |
| "acc_norm_stderr": 0.02853097590658585 | |
| } | |
| }, | |
| "versions": { | |
| "lighteval|agieval:aqua-rat|0": 0, | |
| "lighteval|agieval:gaokao-biology|0": 0, | |
| "lighteval|agieval:gaokao-chemistry|0": 0, | |
| "lighteval|agieval:gaokao-chinese|0": 0, | |
| "lighteval|agieval:gaokao-english|0": 0, | |
| "lighteval|agieval:gaokao-geography|0": 0, | |
| "lighteval|agieval:gaokao-history|0": 0, | |
| "lighteval|agieval:gaokao-mathqa|0": 0, | |
| "lighteval|agieval:gaokao-physics|0": 0, | |
| "lighteval|agieval:logiqa-en|0": 0, | |
| "lighteval|agieval:logiqa-zh|0": 0, | |
| "lighteval|agieval:lsat-ar|0": 0, | |
| "lighteval|agieval:lsat-lr|0": 0, | |
| "lighteval|agieval:lsat-rc|0": 0, | |
| "lighteval|agieval:sat-en|0": 0, | |
| "lighteval|agieval:sat-en-without-passage|0": 0, | |
| "lighteval|agieval:sat-math|0": 0 | |
| }, | |
| "config_tasks": { | |
| "lighteval|agieval:aqua-rat": { | |
| "name": "agieval:aqua-rat", | |
| "prompt_function": "agieval", | |
| "hf_repo": "dmayhem93/agieval-aqua-rat", | |
| "hf_subset": "default", | |
| "metric": [ | |
| "loglikelihood_acc", | |
| "loglikelihood_acc_norm_nospace" | |
| ], | |
| "hf_avail_splits": [ | |
| "test" | |
| ], | |
| "evaluation_splits": [ | |
| "test" | |
| ], | |
| "few_shots_split": null, | |
| "few_shots_select": "random_sampling", | |
| "generation_size": 1, | |
| "stop_sequence": null, | |
| "output_regex": null, | |
| "frozen": false, | |
| "suite": [ | |
| "lighteval" | |
| ], | |
| "original_num_docs": 254, | |
| "effective_num_docs": 254, | |
| "trust_dataset": true, | |
| "must_remove_duplicate_docs": null | |
| }, | |
| "lighteval|agieval:gaokao-biology": { | |
| "name": "agieval:gaokao-biology", | |
| "prompt_function": "agieval", | |
| "hf_repo": "dmayhem93/agieval-gaokao-biology", | |
| "hf_subset": "default", | |
| "metric": [ | |
| "loglikelihood_acc", | |
| "loglikelihood_acc_norm_nospace" | |
| ], | |
| "hf_avail_splits": [ | |
| "test" | |
| ], | |
| "evaluation_splits": [ | |
| "test" | |
| ], | |
| "few_shots_split": null, | |
| "few_shots_select": "random_sampling", | |
| "generation_size": 1, | |
| "stop_sequence": null, | |
| "output_regex": null, | |
| "frozen": false, | |
| "suite": [ | |
| "lighteval" | |
| ], | |
| "original_num_docs": 210, | |
| "effective_num_docs": 210, | |
| "trust_dataset": true, | |
| "must_remove_duplicate_docs": null | |
| }, | |
| "lighteval|agieval:gaokao-chemistry": { | |
| "name": "agieval:gaokao-chemistry", | |
| "prompt_function": "agieval", | |
| "hf_repo": "dmayhem93/agieval-gaokao-chemistry", | |
| "hf_subset": "default", | |
| "metric": [ | |
| "loglikelihood_acc", | |
| "loglikelihood_acc_norm_nospace" | |
| ], | |
| "hf_avail_splits": [ | |
| "test" | |
| ], | |
| "evaluation_splits": [ | |
| "test" | |
| ], | |
| "few_shots_split": null, | |
| "few_shots_select": "random_sampling", | |
| "generation_size": 1, | |
| "stop_sequence": null, | |
| "output_regex": null, | |
| "frozen": false, | |
| "suite": [ | |
| "lighteval" | |
| ], | |
| "original_num_docs": 207, | |
| "effective_num_docs": 207, | |
| "trust_dataset": true, | |
| "must_remove_duplicate_docs": null | |
| }, | |
| "lighteval|agieval:gaokao-chinese": { | |
| "name": "agieval:gaokao-chinese", | |
| "prompt_function": "agieval", | |
| "hf_repo": "dmayhem93/agieval-gaokao-chinese", | |
| "hf_subset": "default", | |
| "metric": [ | |
| "loglikelihood_acc", | |
| "loglikelihood_acc_norm_nospace" | |
| ], | |
| "hf_avail_splits": [ | |
| "test" | |
| ], | |
| "evaluation_splits": [ | |
| "test" | |
| ], | |
| "few_shots_split": null, | |
| "few_shots_select": "random_sampling", | |
| "generation_size": 1, | |
| "stop_sequence": null, | |
| "output_regex": null, | |
| "frozen": false, | |
| "suite": [ | |
| "lighteval" | |
| ], | |
| "original_num_docs": 246, | |
| "effective_num_docs": 246, | |
| "trust_dataset": true, | |
| "must_remove_duplicate_docs": null | |
| }, | |
| "lighteval|agieval:gaokao-english": { | |
| "name": "agieval:gaokao-english", | |
| "prompt_function": "agieval", | |
| "hf_repo": "dmayhem93/agieval-gaokao-english", | |
| "hf_subset": "default", | |
| "metric": [ | |
| "loglikelihood_acc", | |
| "loglikelihood_acc_norm_nospace" | |
| ], | |
| "hf_avail_splits": [ | |
| "test" | |
| ], | |
| "evaluation_splits": [ | |
| "test" | |
| ], | |
| "few_shots_split": null, | |
| "few_shots_select": "random_sampling", | |
| "generation_size": 1, | |
| "stop_sequence": null, | |
| "output_regex": null, | |
| "frozen": false, | |
| "suite": [ | |
| "lighteval" | |
| ], | |
| "original_num_docs": 306, | |
| "effective_num_docs": 306, | |
| "trust_dataset": true, | |
| "must_remove_duplicate_docs": null | |
| }, | |
| "lighteval|agieval:gaokao-geography": { | |
| "name": "agieval:gaokao-geography", | |
| "prompt_function": "agieval", | |
| "hf_repo": "dmayhem93/agieval-gaokao-geography", | |
| "hf_subset": "default", | |
| "metric": [ | |
| "loglikelihood_acc", | |
| "loglikelihood_acc_norm_nospace" | |
| ], | |
| "hf_avail_splits": [ | |
| "test" | |
| ], | |
| "evaluation_splits": [ | |
| "test" | |
| ], | |
| "few_shots_split": null, | |
| "few_shots_select": "random_sampling", | |
| "generation_size": 1, | |
| "stop_sequence": null, | |
| "output_regex": null, | |
| "frozen": false, | |
| "suite": [ | |
| "lighteval" | |
| ], | |
| "original_num_docs": 199, | |
| "effective_num_docs": 199, | |
| "trust_dataset": true, | |
| "must_remove_duplicate_docs": null | |
| }, | |
| "lighteval|agieval:gaokao-history": { | |
| "name": "agieval:gaokao-history", | |
| "prompt_function": "agieval", | |
| "hf_repo": "dmayhem93/agieval-gaokao-history", | |
| "hf_subset": "default", | |
| "metric": [ | |
| "loglikelihood_acc", | |
| "loglikelihood_acc_norm_nospace" | |
| ], | |
| "hf_avail_splits": [ | |
| "test" | |
| ], | |
| "evaluation_splits": [ | |
| "test" | |
| ], | |
| "few_shots_split": null, | |
| "few_shots_select": "random_sampling", | |
| "generation_size": 1, | |
| "stop_sequence": null, | |
| "output_regex": null, | |
| "frozen": false, | |
| "suite": [ | |
| "lighteval" | |
| ], | |
| "original_num_docs": 235, | |
| "effective_num_docs": 235, | |
| "trust_dataset": true, | |
| "must_remove_duplicate_docs": null | |
| }, | |
| "lighteval|agieval:gaokao-mathqa": { | |
| "name": "agieval:gaokao-mathqa", | |
| "prompt_function": "agieval", | |
| "hf_repo": "dmayhem93/agieval-gaokao-mathqa", | |
| "hf_subset": "default", | |
| "metric": [ | |
| "loglikelihood_acc", | |
| "loglikelihood_acc_norm_nospace" | |
| ], | |
| "hf_avail_splits": [ | |
| "test" | |
| ], | |
| "evaluation_splits": [ | |
| "test" | |
| ], | |
| "few_shots_split": null, | |
| "few_shots_select": "random_sampling", | |
| "generation_size": 1, | |
| "stop_sequence": null, | |
| "output_regex": null, | |
| "frozen": false, | |
| "suite": [ | |
| "lighteval" | |
| ], | |
| "original_num_docs": 351, | |
| "effective_num_docs": 351, | |
| "trust_dataset": true, | |
| "must_remove_duplicate_docs": null | |
| }, | |
| "lighteval|agieval:gaokao-physics": { | |
| "name": "agieval:gaokao-physics", | |
| "prompt_function": "agieval", | |
| "hf_repo": "dmayhem93/agieval-gaokao-physics", | |
| "hf_subset": "default", | |
| "metric": [ | |
| "loglikelihood_acc", | |
| "loglikelihood_acc_norm_nospace" | |
| ], | |
| "hf_avail_splits": [ | |
| "test" | |
| ], | |
| "evaluation_splits": [ | |
| "test" | |
| ], | |
| "few_shots_split": null, | |
| "few_shots_select": "random_sampling", | |
| "generation_size": 1, | |
| "stop_sequence": null, | |
| "output_regex": null, | |
| "frozen": false, | |
| "suite": [ | |
| "lighteval" | |
| ], | |
| "original_num_docs": 200, | |
| "effective_num_docs": 200, | |
| "trust_dataset": true, | |
| "must_remove_duplicate_docs": null | |
| }, | |
| "lighteval|agieval:logiqa-en": { | |
| "name": "agieval:logiqa-en", | |
| "prompt_function": "agieval", | |
| "hf_repo": "dmayhem93/agieval-logiqa-en", | |
| "hf_subset": "default", | |
| "metric": [ | |
| "loglikelihood_acc", | |
| "loglikelihood_acc_norm_nospace" | |
| ], | |
| "hf_avail_splits": [ | |
| "test" | |
| ], | |
| "evaluation_splits": [ | |
| "test" | |
| ], | |
| "few_shots_split": null, | |
| "few_shots_select": "random_sampling", | |
| "generation_size": 1, | |
| "stop_sequence": null, | |
| "output_regex": null, | |
| "frozen": false, | |
| "suite": [ | |
| "lighteval" | |
| ], | |
| "original_num_docs": 651, | |
| "effective_num_docs": 651, | |
| "trust_dataset": true, | |
| "must_remove_duplicate_docs": null | |
| }, | |
| "lighteval|agieval:logiqa-zh": { | |
| "name": "agieval:logiqa-zh", | |
| "prompt_function": "agieval", | |
| "hf_repo": "dmayhem93/agieval-logiqa-zh", | |
| "hf_subset": "default", | |
| "metric": [ | |
| "loglikelihood_acc", | |
| "loglikelihood_acc_norm_nospace" | |
| ], | |
| "hf_avail_splits": [ | |
| "test" | |
| ], | |
| "evaluation_splits": [ | |
| "test" | |
| ], | |
| "few_shots_split": null, | |
| "few_shots_select": "random_sampling", | |
| "generation_size": 1, | |
| "stop_sequence": null, | |
| "output_regex": null, | |
| "frozen": false, | |
| "suite": [ | |
| "lighteval" | |
| ], | |
| "original_num_docs": 651, | |
| "effective_num_docs": 651, | |
| "trust_dataset": true, | |
| "must_remove_duplicate_docs": null | |
| }, | |
| "lighteval|agieval:lsat-ar": { | |
| "name": "agieval:lsat-ar", | |
| "prompt_function": "agieval", | |
| "hf_repo": "dmayhem93/agieval-lsat-ar", | |
| "hf_subset": "default", | |
| "metric": [ | |
| "loglikelihood_acc", | |
| "loglikelihood_acc_norm_nospace" | |
| ], | |
| "hf_avail_splits": [ | |
| "test" | |
| ], | |
| "evaluation_splits": [ | |
| "test" | |
| ], | |
| "few_shots_split": null, | |
| "few_shots_select": "random_sampling", | |
| "generation_size": 1, | |
| "stop_sequence": null, | |
| "output_regex": null, | |
| "frozen": false, | |
| "suite": [ | |
| "lighteval" | |
| ], | |
| "original_num_docs": 230, | |
| "effective_num_docs": 230, | |
| "trust_dataset": true, | |
| "must_remove_duplicate_docs": null | |
| }, | |
| "lighteval|agieval:lsat-lr": { | |
| "name": "agieval:lsat-lr", | |
| "prompt_function": "agieval", | |
| "hf_repo": "dmayhem93/agieval-lsat-lr", | |
| "hf_subset": "default", | |
| "metric": [ | |
| "loglikelihood_acc", | |
| "loglikelihood_acc_norm_nospace" | |
| ], | |
| "hf_avail_splits": [ | |
| "test" | |
| ], | |
| "evaluation_splits": [ | |
| "test" | |
| ], | |
| "few_shots_split": null, | |
| "few_shots_select": "random_sampling", | |
| "generation_size": 1, | |
| "stop_sequence": null, | |
| "output_regex": null, | |
| "frozen": false, | |
| "suite": [ | |
| "lighteval" | |
| ], | |
| "original_num_docs": 510, | |
| "effective_num_docs": 510, | |
| "trust_dataset": true, | |
| "must_remove_duplicate_docs": null | |
| }, | |
| "lighteval|agieval:lsat-rc": { | |
| "name": "agieval:lsat-rc", | |
| "prompt_function": "agieval", | |
| "hf_repo": "dmayhem93/agieval-lsat-rc", | |
| "hf_subset": "default", | |
| "metric": [ | |
| "loglikelihood_acc", | |
| "loglikelihood_acc_norm_nospace" | |
| ], | |
| "hf_avail_splits": [ | |
| "test" | |
| ], | |
| "evaluation_splits": [ | |
| "test" | |
| ], | |
| "few_shots_split": null, | |
| "few_shots_select": "random_sampling", | |
| "generation_size": 1, | |
| "stop_sequence": null, | |
| "output_regex": null, | |
| "frozen": false, | |
| "suite": [ | |
| "lighteval" | |
| ], | |
| "original_num_docs": 269, | |
| "effective_num_docs": 269, | |
| "trust_dataset": true, | |
| "must_remove_duplicate_docs": null | |
| }, | |
| "lighteval|agieval:sat-en": { | |
| "name": "agieval:sat-en", | |
| "prompt_function": "agieval", | |
| "hf_repo": "dmayhem93/agieval-sat-en", | |
| "hf_subset": "default", | |
| "metric": [ | |
| "loglikelihood_acc", | |
| "loglikelihood_acc_norm_nospace" | |
| ], | |
| "hf_avail_splits": [ | |
| "test" | |
| ], | |
| "evaluation_splits": [ | |
| "test" | |
| ], | |
| "few_shots_split": null, | |
| "few_shots_select": "random_sampling", | |
| "generation_size": 1, | |
| "stop_sequence": null, | |
| "output_regex": null, | |
| "frozen": false, | |
| "suite": [ | |
| "lighteval" | |
| ], | |
| "original_num_docs": 206, | |
| "effective_num_docs": 206, | |
| "trust_dataset": true, | |
| "must_remove_duplicate_docs": null | |
| }, | |
| "lighteval|agieval:sat-en-without-passage": { | |
| "name": "agieval:sat-en-without-passage", | |
| "prompt_function": "agieval", | |
| "hf_repo": "dmayhem93/agieval-sat-en-without-passage", | |
| "hf_subset": "default", | |
| "metric": [ | |
| "loglikelihood_acc", | |
| "loglikelihood_acc_norm_nospace" | |
| ], | |
| "hf_avail_splits": [ | |
| "test" | |
| ], | |
| "evaluation_splits": [ | |
| "test" | |
| ], | |
| "few_shots_split": null, | |
| "few_shots_select": "random_sampling", | |
| "generation_size": 1, | |
| "stop_sequence": null, | |
| "output_regex": null, | |
| "frozen": false, | |
| "suite": [ | |
| "lighteval" | |
| ], | |
| "original_num_docs": 206, | |
| "effective_num_docs": 206, | |
| "trust_dataset": true, | |
| "must_remove_duplicate_docs": null | |
| }, | |
| "lighteval|agieval:sat-math": { | |
| "name": "agieval:sat-math", | |
| "prompt_function": "agieval", | |
| "hf_repo": "dmayhem93/agieval-sat-math", | |
| "hf_subset": "default", | |
| "metric": [ | |
| "loglikelihood_acc", | |
| "loglikelihood_acc_norm_nospace" | |
| ], | |
| "hf_avail_splits": [ | |
| "test" | |
| ], | |
| "evaluation_splits": [ | |
| "test" | |
| ], | |
| "few_shots_split": null, | |
| "few_shots_select": "random_sampling", | |
| "generation_size": 1, | |
| "stop_sequence": null, | |
| "output_regex": null, | |
| "frozen": false, | |
| "suite": [ | |
| "lighteval" | |
| ], | |
| "original_num_docs": 220, | |
| "effective_num_docs": 220, | |
| "trust_dataset": true, | |
| "must_remove_duplicate_docs": null | |
| } | |
| }, | |
| "summary_tasks": { | |
| "lighteval|agieval:aqua-rat|0": { | |
| "hashes": { | |
| "hash_examples": "f09607f69e5b7525", | |
| "hash_full_prompts": "286855998c6e5e88", | |
| "hash_input_tokens": "c8a11a37fad94991", | |
| "hash_cont_tokens": "ef0b3383d6f03cf7" | |
| }, | |
| "truncated": 0, | |
| "non_truncated": 254, | |
| "padded": 1270, | |
| "non_padded": 0, | |
| "effective_few_shots": 0.0, | |
| "num_truncated_few_shots": 0 | |
| }, | |
| "lighteval|agieval:gaokao-biology|0": { | |
| "hashes": { | |
| "hash_examples": "f262eaf4a72db963", | |
| "hash_full_prompts": "079710887794e92b", | |
| "hash_input_tokens": "760164ddee928404", | |
| "hash_cont_tokens": "6dbb7db97194d8c0" | |
| }, | |
| "truncated": 0, | |
| "non_truncated": 210, | |
| "padded": 840, | |
| "non_padded": 0, | |
| "effective_few_shots": 0.0, | |
| "num_truncated_few_shots": 0 | |
| }, | |
| "lighteval|agieval:gaokao-chemistry|0": { | |
| "hashes": { | |
| "hash_examples": "47f2e649f58d9da5", | |
| "hash_full_prompts": "34855f051b6dd679", | |
| "hash_input_tokens": "eb3e767b26311b8d", | |
| "hash_cont_tokens": "b01f9f3c4d008784" | |
| }, | |
| "truncated": 0, | |
| "non_truncated": 207, | |
| "padded": 825, | |
| "non_padded": 6, | |
| "effective_few_shots": 0.0, | |
| "num_truncated_few_shots": 0 | |
| }, | |
| "lighteval|agieval:gaokao-chinese|0": { | |
| "hashes": { | |
| "hash_examples": "1010b21fde4726ab", | |
| "hash_full_prompts": "49905c6e90bb4ffb", | |
| "hash_input_tokens": "ebfb3b7cbf519cb9", | |
| "hash_cont_tokens": "c14e536a2664f693" | |
| }, | |
| "truncated": 0, | |
| "non_truncated": 246, | |
| "padded": 981, | |
| "non_padded": 3, | |
| "effective_few_shots": 0.0, | |
| "num_truncated_few_shots": 0 | |
| }, | |
| "lighteval|agieval:gaokao-english|0": { | |
| "hashes": { | |
| "hash_examples": "4864e492a350ae93", | |
| "hash_full_prompts": "0d368ac714d82cbd", | |
| "hash_input_tokens": "97b152ce2f9dd276", | |
| "hash_cont_tokens": "b0295fd391ea9065" | |
| }, | |
| "truncated": 0, | |
| "non_truncated": 306, | |
| "padded": 1224, | |
| "non_padded": 0, | |
| "effective_few_shots": 0.0, | |
| "num_truncated_few_shots": 0 | |
| }, | |
| "lighteval|agieval:gaokao-geography|0": { | |
| "hashes": { | |
| "hash_examples": "ec3a021e37650e7d", | |
| "hash_full_prompts": "ae39d50701103ed2", | |
| "hash_input_tokens": "03fe8ffbd234ac1f", | |
| "hash_cont_tokens": "5696e6aef46a4012" | |
| }, | |
| "truncated": 0, | |
| "non_truncated": 199, | |
| "padded": 794, | |
| "non_padded": 2, | |
| "effective_few_shots": 0.0, | |
| "num_truncated_few_shots": 0 | |
| }, | |
| "lighteval|agieval:gaokao-history|0": { | |
| "hashes": { | |
| "hash_examples": "b3fad1596f1ae1f9", | |
| "hash_full_prompts": "d846ac9f414ea77f", | |
| "hash_input_tokens": "792a5a317dd88386", | |
| "hash_cont_tokens": "e2947e88a3f08a49" | |
| }, | |
| "truncated": 0, | |
| "non_truncated": 235, | |
| "padded": 938, | |
| "non_padded": 2, | |
| "effective_few_shots": 0.0, | |
| "num_truncated_few_shots": 0 | |
| }, | |
| "lighteval|agieval:gaokao-mathqa|0": { | |
| "hashes": { | |
| "hash_examples": "1d1088556861b0b0", | |
| "hash_full_prompts": "2d6d02d7c7878410", | |
| "hash_input_tokens": "23b49a4c37eb5643", | |
| "hash_cont_tokens": "7bce9c6eac88e31d" | |
| }, | |
| "truncated": 0, | |
| "non_truncated": 351, | |
| "padded": 1403, | |
| "non_padded": 1, | |
| "effective_few_shots": 0.0, | |
| "num_truncated_few_shots": 0 | |
| }, | |
| "lighteval|agieval:gaokao-physics|0": { | |
| "hashes": { | |
| "hash_examples": "eb05f035c7bfca2f", | |
| "hash_full_prompts": "c32439b20ae8ddc3", | |
| "hash_input_tokens": "83afe03525a018b6", | |
| "hash_cont_tokens": "77f9a76c75c408ee" | |
| }, | |
| "truncated": 0, | |
| "non_truncated": 200, | |
| "padded": 792, | |
| "non_padded": 8, | |
| "effective_few_shots": 0.0, | |
| "num_truncated_few_shots": 0 | |
| }, | |
| "lighteval|agieval:logiqa-en|0": { | |
| "hashes": { | |
| "hash_examples": "0a688a45f69c21e0", | |
| "hash_full_prompts": "b758941be76e6c93", | |
| "hash_input_tokens": "3b4e102d5a77c2ee", | |
| "hash_cont_tokens": "bdb78fbd84121ffa" | |
| }, | |
| "truncated": 0, | |
| "non_truncated": 651, | |
| "padded": 2591, | |
| "non_padded": 13, | |
| "effective_few_shots": 0.0, | |
| "num_truncated_few_shots": 0 | |
| }, | |
| "lighteval|agieval:logiqa-zh|0": { | |
| "hashes": { | |
| "hash_examples": "620d6888b6012ea5", | |
| "hash_full_prompts": "00581fe3b81ab4bf", | |
| "hash_input_tokens": "47c668eacbda6e4c", | |
| "hash_cont_tokens": "a470f1c94ce464df" | |
| }, | |
| "truncated": 0, | |
| "non_truncated": 651, | |
| "padded": 2577, | |
| "non_padded": 27, | |
| "effective_few_shots": 0.0, | |
| "num_truncated_few_shots": 0 | |
| }, | |
| "lighteval|agieval:lsat-ar|0": { | |
| "hashes": { | |
| "hash_examples": "627c8f5ccd5da209", | |
| "hash_full_prompts": "c35e6779d96ec186", | |
| "hash_input_tokens": "21aca9a9e3907790", | |
| "hash_cont_tokens": "5a2d4bf9575205d0" | |
| }, | |
| "truncated": 0, | |
| "non_truncated": 230, | |
| "padded": 1134, | |
| "non_padded": 16, | |
| "effective_few_shots": 0.0, | |
| "num_truncated_few_shots": 0 | |
| }, | |
| "lighteval|agieval:lsat-lr|0": { | |
| "hashes": { | |
| "hash_examples": "794641c86de172f5", | |
| "hash_full_prompts": "28132acff0b72fac", | |
| "hash_input_tokens": "21b8cf84591f6416", | |
| "hash_cont_tokens": "b7efbd859fa461cb" | |
| }, | |
| "truncated": 0, | |
| "non_truncated": 510, | |
| "padded": 2528, | |
| "non_padded": 22, | |
| "effective_few_shots": 0.0, | |
| "num_truncated_few_shots": 0 | |
| }, | |
| "lighteval|agieval:lsat-rc|0": { | |
| "hashes": { | |
| "hash_examples": "35981ed917ea01cf", | |
| "hash_full_prompts": "38671d826aec9b04", | |
| "hash_input_tokens": "e6836cf65003774d", | |
| "hash_cont_tokens": "25cd21c6f88e868d" | |
| }, | |
| "truncated": 0, | |
| "non_truncated": 269, | |
| "padded": 1345, | |
| "non_padded": 0, | |
| "effective_few_shots": 0.0, | |
| "num_truncated_few_shots": 0 | |
| }, | |
| "lighteval|agieval:sat-en|0": { | |
| "hashes": { | |
| "hash_examples": "041c39c646536a1e", | |
| "hash_full_prompts": "7cfcf777a80aab88", | |
| "hash_input_tokens": "338b5d3aa5e5d50c", | |
| "hash_cont_tokens": "a32456da5b0cb4d6" | |
| }, | |
| "truncated": 0, | |
| "non_truncated": 206, | |
| "padded": 821, | |
| "non_padded": 0, | |
| "effective_few_shots": 0.0, | |
| "num_truncated_few_shots": 0 | |
| }, | |
| "lighteval|agieval:sat-en-without-passage|0": { | |
| "hashes": { | |
| "hash_examples": "e4d9284367dff68f", | |
| "hash_full_prompts": "858c585678a575e3", | |
| "hash_input_tokens": "0fa7ff75d35b4ef7", | |
| "hash_cont_tokens": "b5be143c7f4c4669" | |
| }, | |
| "truncated": 0, | |
| "non_truncated": 206, | |
| "padded": 814, | |
| "non_padded": 7, | |
| "effective_few_shots": 0.0, | |
| "num_truncated_few_shots": 0 | |
| }, | |
| "lighteval|agieval:sat-math|0": { | |
| "hashes": { | |
| "hash_examples": "01db7291603fc1a0", | |
| "hash_full_prompts": "22304b8d6215acee", | |
| "hash_input_tokens": "b41cfbbe32e7050d", | |
| "hash_cont_tokens": "05e08502d89d7a73" | |
| }, | |
| "truncated": 0, | |
| "non_truncated": 220, | |
| "padded": 873, | |
| "non_padded": 7, | |
| "effective_few_shots": 0.0, | |
| "num_truncated_few_shots": 0 | |
| } | |
| }, | |
| "summary_general": { | |
| "hashes": { | |
| "hash_examples": "da3af66181f18ddf", | |
| "hash_full_prompts": "38bd3cda94f79eaf", | |
| "hash_input_tokens": "4f8569cc545f17c5", | |
| "hash_cont_tokens": "0e86a0b3e19b1c9c" | |
| }, | |
| "truncated": 0, | |
| "non_truncated": 5151, | |
| "padded": 21750, | |
| "non_padded": 114, | |
| "num_truncated_few_shots": 0 | |
| } | |
| } |