{ "config_general": { "lighteval_sha": "?", "num_fewshot_seeds": 1, "override_batch_size": 4, "max_samples": null, "job_id": "", "start_time": 1988758.699140144, "end_time": 1989306.575083681, "total_evaluation_time_secondes": "547.8759435368702", "model_name": "meta-llama/Meta-Llama-3-8B-Instruct", "model_sha": "1448453bdb895762499deb4176c1dd83b145fac1", "model_dtype": "torch.bfloat16", "model_size": "15.08 GB", "config": null }, "results": { "lighteval|agieval:aqua-rat|0": { "acc": 0.25196850393700787, "acc_stderr": 0.027294353392553605, "acc_norm": 0.24803149606299213, "acc_norm_stderr": 0.02715144709118171 }, "lighteval|agieval:gaokao-biology|0": { "acc": 0.46190476190476193, "acc_stderr": 0.034485192220162664, "acc_norm": 0.38095238095238093, "acc_norm_stderr": 0.033591100467499906 }, "lighteval|agieval:gaokao-chemistry|0": { "acc": 0.32367149758454106, "acc_stderr": 0.03259848850179343, "acc_norm": 0.32367149758454106, "acc_norm_stderr": 0.032598488501793435 }, "lighteval|agieval:gaokao-chinese|0": { "acc": 0.3130081300813008, "acc_stderr": 0.02962583267300022, "acc_norm": 0.2967479674796748, "acc_norm_stderr": 0.029185445861037912 }, "lighteval|agieval:gaokao-english|0": { "acc": 0.696078431372549, "acc_stderr": 0.026336613469046637, "acc_norm": 0.5620915032679739, "acc_norm_stderr": 0.028408302020332687 }, "lighteval|agieval:gaokao-geography|0": { "acc": 0.5175879396984925, "acc_stderr": 0.035511462395976016, "acc_norm": 0.49246231155778897, "acc_norm_stderr": 0.035529414706104066 }, "lighteval|agieval:gaokao-history|0": { "acc": 0.5234042553191489, "acc_stderr": 0.03265019475033582, "acc_norm": 0.39574468085106385, "acc_norm_stderr": 0.031967586978353627 }, "lighteval|agieval:gaokao-mathqa|0": { "acc": 0.2792022792022792, "acc_stderr": 0.023979060299146256, "acc_norm": 0.27635327635327633, "acc_norm_stderr": 0.023903505003127216 }, "lighteval|agieval:gaokao-physics|0": { "acc": 0.395, "acc_stderr": 0.03465370682892271, "acc_norm": 0.38, "acc_norm_stderr": 0.0344081327303582 }, "lighteval|agieval:logiqa-en|0": { "acc": 0.41321044546851, "acc_stderr": 0.01931390783165284, "acc_norm": 0.34408602150537637, "acc_norm_stderr": 0.01863375065717621 }, "lighteval|agieval:logiqa-zh|0": { "acc": 0.31490015360983103, "acc_stderr": 0.018218251493671664, "acc_norm": 0.33640552995391704, "acc_norm_stderr": 0.01853216950935858 }, "lighteval|agieval:lsat-ar|0": { "acc": 0.2217391304347826, "acc_stderr": 0.027451496604058916, "acc_norm": 0.1782608695652174, "acc_norm_stderr": 0.025291655246273914 }, "lighteval|agieval:lsat-lr|0": { "acc": 0.5333333333333333, "acc_stderr": 0.022112806381564227, "acc_norm": 0.32941176470588235, "acc_norm_stderr": 0.020832367421292207 }, "lighteval|agieval:lsat-rc|0": { "acc": 0.6394052044609665, "acc_stderr": 0.029331239329958934, "acc_norm": 0.35687732342007433, "acc_norm_stderr": 0.029264357329058688 }, "lighteval|agieval:sat-en|0": { "acc": 0.7572815533980582, "acc_stderr": 0.029943540553570538, "acc_norm": 0.48058252427184467, "acc_norm_stderr": 0.034895171350660135 }, "lighteval|agieval:sat-en-without-passage|0": { "acc": 0.3932038834951456, "acc_stderr": 0.034115627597025605, "acc_norm": 0.2669902912621359, "acc_norm_stderr": 0.030897665523458157 }, "lighteval|agieval:sat-math|0": { "acc": 0.38181818181818183, "acc_stderr": 0.032829506847783727, "acc_norm": 0.2681818181818182, "acc_norm_stderr": 0.02993603001489284 }, "lighteval|agieval:_average|0": { "acc": 0.43627751088934646, "acc_stderr": 0.028850075362954343, "acc_norm": 0.34805007393976223, "acc_norm_stderr": 0.02853097590658585 }, "all": { "acc": 0.43627751088934646, "acc_stderr": 0.028850075362954343, "acc_norm": 0.34805007393976223, "acc_norm_stderr": 0.02853097590658585 } }, "versions": { "lighteval|agieval:aqua-rat|0": 0, "lighteval|agieval:gaokao-biology|0": 0, "lighteval|agieval:gaokao-chemistry|0": 0, "lighteval|agieval:gaokao-chinese|0": 0, "lighteval|agieval:gaokao-english|0": 0, "lighteval|agieval:gaokao-geography|0": 0, "lighteval|agieval:gaokao-history|0": 0, "lighteval|agieval:gaokao-mathqa|0": 0, "lighteval|agieval:gaokao-physics|0": 0, "lighteval|agieval:logiqa-en|0": 0, "lighteval|agieval:logiqa-zh|0": 0, "lighteval|agieval:lsat-ar|0": 0, "lighteval|agieval:lsat-lr|0": 0, "lighteval|agieval:lsat-rc|0": 0, "lighteval|agieval:sat-en|0": 0, "lighteval|agieval:sat-en-without-passage|0": 0, "lighteval|agieval:sat-math|0": 0 }, "config_tasks": { "lighteval|agieval:aqua-rat": { "name": "agieval:aqua-rat", "prompt_function": "agieval", "hf_repo": "dmayhem93/agieval-aqua-rat", "hf_subset": "default", "metric": [ "loglikelihood_acc", "loglikelihood_acc_norm_nospace" ], "hf_avail_splits": [ "test" ], "evaluation_splits": [ "test" ], "few_shots_split": null, "few_shots_select": "random_sampling", "generation_size": 1, "stop_sequence": null, "output_regex": null, "frozen": false, "suite": [ "lighteval" ], "original_num_docs": 254, "effective_num_docs": 254, "trust_dataset": true, "must_remove_duplicate_docs": null }, "lighteval|agieval:gaokao-biology": { "name": "agieval:gaokao-biology", "prompt_function": "agieval", "hf_repo": "dmayhem93/agieval-gaokao-biology", "hf_subset": "default", "metric": [ "loglikelihood_acc", "loglikelihood_acc_norm_nospace" ], "hf_avail_splits": [ "test" ], "evaluation_splits": [ "test" ], "few_shots_split": null, "few_shots_select": "random_sampling", "generation_size": 1, "stop_sequence": null, "output_regex": null, "frozen": false, "suite": [ "lighteval" ], "original_num_docs": 210, "effective_num_docs": 210, "trust_dataset": true, "must_remove_duplicate_docs": null }, "lighteval|agieval:gaokao-chemistry": { "name": "agieval:gaokao-chemistry", "prompt_function": "agieval", "hf_repo": "dmayhem93/agieval-gaokao-chemistry", "hf_subset": "default", "metric": [ "loglikelihood_acc", "loglikelihood_acc_norm_nospace" ], "hf_avail_splits": [ "test" ], "evaluation_splits": [ "test" ], "few_shots_split": null, "few_shots_select": "random_sampling", "generation_size": 1, "stop_sequence": null, "output_regex": null, "frozen": false, "suite": [ "lighteval" ], "original_num_docs": 207, "effective_num_docs": 207, "trust_dataset": true, "must_remove_duplicate_docs": null }, "lighteval|agieval:gaokao-chinese": { "name": "agieval:gaokao-chinese", "prompt_function": "agieval", "hf_repo": "dmayhem93/agieval-gaokao-chinese", "hf_subset": "default", "metric": [ "loglikelihood_acc", "loglikelihood_acc_norm_nospace" ], "hf_avail_splits": [ "test" ], "evaluation_splits": [ "test" ], "few_shots_split": null, "few_shots_select": "random_sampling", "generation_size": 1, "stop_sequence": null, "output_regex": null, "frozen": false, "suite": [ "lighteval" ], "original_num_docs": 246, "effective_num_docs": 246, "trust_dataset": true, "must_remove_duplicate_docs": null }, "lighteval|agieval:gaokao-english": { "name": "agieval:gaokao-english", "prompt_function": "agieval", "hf_repo": "dmayhem93/agieval-gaokao-english", "hf_subset": "default", "metric": [ "loglikelihood_acc", "loglikelihood_acc_norm_nospace" ], "hf_avail_splits": [ "test" ], "evaluation_splits": [ "test" ], "few_shots_split": null, "few_shots_select": "random_sampling", "generation_size": 1, "stop_sequence": null, "output_regex": null, "frozen": false, "suite": [ "lighteval" ], "original_num_docs": 306, "effective_num_docs": 306, "trust_dataset": true, "must_remove_duplicate_docs": null }, "lighteval|agieval:gaokao-geography": { "name": "agieval:gaokao-geography", "prompt_function": "agieval", "hf_repo": "dmayhem93/agieval-gaokao-geography", "hf_subset": "default", "metric": [ "loglikelihood_acc", "loglikelihood_acc_norm_nospace" ], "hf_avail_splits": [ "test" ], "evaluation_splits": [ "test" ], "few_shots_split": null, "few_shots_select": "random_sampling", "generation_size": 1, "stop_sequence": null, "output_regex": null, "frozen": false, "suite": [ "lighteval" ], "original_num_docs": 199, "effective_num_docs": 199, "trust_dataset": true, "must_remove_duplicate_docs": null }, "lighteval|agieval:gaokao-history": { "name": "agieval:gaokao-history", "prompt_function": "agieval", "hf_repo": "dmayhem93/agieval-gaokao-history", "hf_subset": "default", "metric": [ "loglikelihood_acc", "loglikelihood_acc_norm_nospace" ], "hf_avail_splits": [ "test" ], "evaluation_splits": [ "test" ], "few_shots_split": null, "few_shots_select": "random_sampling", "generation_size": 1, "stop_sequence": null, "output_regex": null, "frozen": false, "suite": [ "lighteval" ], "original_num_docs": 235, "effective_num_docs": 235, "trust_dataset": true, "must_remove_duplicate_docs": null }, "lighteval|agieval:gaokao-mathqa": { "name": "agieval:gaokao-mathqa", "prompt_function": "agieval", "hf_repo": "dmayhem93/agieval-gaokao-mathqa", "hf_subset": "default", "metric": [ "loglikelihood_acc", "loglikelihood_acc_norm_nospace" ], "hf_avail_splits": [ "test" ], "evaluation_splits": [ "test" ], "few_shots_split": null, "few_shots_select": "random_sampling", "generation_size": 1, "stop_sequence": null, "output_regex": null, "frozen": false, "suite": [ "lighteval" ], "original_num_docs": 351, "effective_num_docs": 351, "trust_dataset": true, "must_remove_duplicate_docs": null }, "lighteval|agieval:gaokao-physics": { "name": "agieval:gaokao-physics", "prompt_function": "agieval", "hf_repo": "dmayhem93/agieval-gaokao-physics", "hf_subset": "default", "metric": [ "loglikelihood_acc", "loglikelihood_acc_norm_nospace" ], "hf_avail_splits": [ "test" ], "evaluation_splits": [ "test" ], "few_shots_split": null, "few_shots_select": "random_sampling", "generation_size": 1, "stop_sequence": null, "output_regex": null, "frozen": false, "suite": [ "lighteval" ], "original_num_docs": 200, "effective_num_docs": 200, "trust_dataset": true, "must_remove_duplicate_docs": null }, "lighteval|agieval:logiqa-en": { "name": "agieval:logiqa-en", "prompt_function": "agieval", "hf_repo": "dmayhem93/agieval-logiqa-en", "hf_subset": "default", "metric": [ "loglikelihood_acc", "loglikelihood_acc_norm_nospace" ], "hf_avail_splits": [ "test" ], "evaluation_splits": [ "test" ], "few_shots_split": null, "few_shots_select": "random_sampling", "generation_size": 1, "stop_sequence": null, "output_regex": null, "frozen": false, "suite": [ "lighteval" ], "original_num_docs": 651, "effective_num_docs": 651, "trust_dataset": true, "must_remove_duplicate_docs": null }, "lighteval|agieval:logiqa-zh": { "name": "agieval:logiqa-zh", "prompt_function": "agieval", "hf_repo": "dmayhem93/agieval-logiqa-zh", "hf_subset": "default", "metric": [ "loglikelihood_acc", "loglikelihood_acc_norm_nospace" ], "hf_avail_splits": [ "test" ], "evaluation_splits": [ "test" ], "few_shots_split": null, "few_shots_select": "random_sampling", "generation_size": 1, "stop_sequence": null, "output_regex": null, "frozen": false, "suite": [ "lighteval" ], "original_num_docs": 651, "effective_num_docs": 651, "trust_dataset": true, "must_remove_duplicate_docs": null }, "lighteval|agieval:lsat-ar": { "name": "agieval:lsat-ar", "prompt_function": "agieval", "hf_repo": "dmayhem93/agieval-lsat-ar", "hf_subset": "default", "metric": [ "loglikelihood_acc", "loglikelihood_acc_norm_nospace" ], "hf_avail_splits": [ "test" ], "evaluation_splits": [ "test" ], "few_shots_split": null, "few_shots_select": "random_sampling", "generation_size": 1, "stop_sequence": null, "output_regex": null, "frozen": false, "suite": [ "lighteval" ], "original_num_docs": 230, "effective_num_docs": 230, "trust_dataset": true, "must_remove_duplicate_docs": null }, "lighteval|agieval:lsat-lr": { "name": "agieval:lsat-lr", "prompt_function": "agieval", "hf_repo": "dmayhem93/agieval-lsat-lr", "hf_subset": "default", "metric": [ "loglikelihood_acc", "loglikelihood_acc_norm_nospace" ], "hf_avail_splits": [ "test" ], "evaluation_splits": [ "test" ], "few_shots_split": null, "few_shots_select": "random_sampling", "generation_size": 1, "stop_sequence": null, "output_regex": null, "frozen": false, "suite": [ "lighteval" ], "original_num_docs": 510, "effective_num_docs": 510, "trust_dataset": true, "must_remove_duplicate_docs": null }, "lighteval|agieval:lsat-rc": { "name": "agieval:lsat-rc", "prompt_function": "agieval", "hf_repo": "dmayhem93/agieval-lsat-rc", "hf_subset": "default", "metric": [ "loglikelihood_acc", "loglikelihood_acc_norm_nospace" ], "hf_avail_splits": [ "test" ], "evaluation_splits": [ "test" ], "few_shots_split": null, "few_shots_select": "random_sampling", "generation_size": 1, "stop_sequence": null, "output_regex": null, "frozen": false, "suite": [ "lighteval" ], "original_num_docs": 269, "effective_num_docs": 269, "trust_dataset": true, "must_remove_duplicate_docs": null }, "lighteval|agieval:sat-en": { "name": "agieval:sat-en", "prompt_function": "agieval", "hf_repo": "dmayhem93/agieval-sat-en", "hf_subset": "default", "metric": [ "loglikelihood_acc", "loglikelihood_acc_norm_nospace" ], "hf_avail_splits": [ "test" ], "evaluation_splits": [ "test" ], "few_shots_split": null, "few_shots_select": "random_sampling", "generation_size": 1, "stop_sequence": null, "output_regex": null, "frozen": false, "suite": [ "lighteval" ], "original_num_docs": 206, "effective_num_docs": 206, "trust_dataset": true, "must_remove_duplicate_docs": null }, "lighteval|agieval:sat-en-without-passage": { "name": "agieval:sat-en-without-passage", "prompt_function": "agieval", "hf_repo": "dmayhem93/agieval-sat-en-without-passage", "hf_subset": "default", "metric": [ "loglikelihood_acc", "loglikelihood_acc_norm_nospace" ], "hf_avail_splits": [ "test" ], "evaluation_splits": [ "test" ], "few_shots_split": null, "few_shots_select": "random_sampling", "generation_size": 1, "stop_sequence": null, "output_regex": null, "frozen": false, "suite": [ "lighteval" ], "original_num_docs": 206, "effective_num_docs": 206, "trust_dataset": true, "must_remove_duplicate_docs": null }, "lighteval|agieval:sat-math": { "name": "agieval:sat-math", "prompt_function": "agieval", "hf_repo": "dmayhem93/agieval-sat-math", "hf_subset": "default", "metric": [ "loglikelihood_acc", "loglikelihood_acc_norm_nospace" ], "hf_avail_splits": [ "test" ], "evaluation_splits": [ "test" ], "few_shots_split": null, "few_shots_select": "random_sampling", "generation_size": 1, "stop_sequence": null, "output_regex": null, "frozen": false, "suite": [ "lighteval" ], "original_num_docs": 220, "effective_num_docs": 220, "trust_dataset": true, "must_remove_duplicate_docs": null } }, "summary_tasks": { "lighteval|agieval:aqua-rat|0": { "hashes": { "hash_examples": "f09607f69e5b7525", "hash_full_prompts": "286855998c6e5e88", "hash_input_tokens": "c8a11a37fad94991", "hash_cont_tokens": "ef0b3383d6f03cf7" }, "truncated": 0, "non_truncated": 254, "padded": 1270, "non_padded": 0, "effective_few_shots": 0.0, "num_truncated_few_shots": 0 }, "lighteval|agieval:gaokao-biology|0": { "hashes": { "hash_examples": "f262eaf4a72db963", "hash_full_prompts": "079710887794e92b", "hash_input_tokens": "760164ddee928404", "hash_cont_tokens": "6dbb7db97194d8c0" }, "truncated": 0, "non_truncated": 210, "padded": 840, "non_padded": 0, "effective_few_shots": 0.0, "num_truncated_few_shots": 0 }, "lighteval|agieval:gaokao-chemistry|0": { "hashes": { "hash_examples": "47f2e649f58d9da5", "hash_full_prompts": "34855f051b6dd679", "hash_input_tokens": "eb3e767b26311b8d", "hash_cont_tokens": "b01f9f3c4d008784" }, "truncated": 0, "non_truncated": 207, "padded": 825, "non_padded": 6, "effective_few_shots": 0.0, "num_truncated_few_shots": 0 }, "lighteval|agieval:gaokao-chinese|0": { "hashes": { "hash_examples": "1010b21fde4726ab", "hash_full_prompts": "49905c6e90bb4ffb", "hash_input_tokens": "ebfb3b7cbf519cb9", "hash_cont_tokens": "c14e536a2664f693" }, "truncated": 0, "non_truncated": 246, "padded": 981, "non_padded": 3, "effective_few_shots": 0.0, "num_truncated_few_shots": 0 }, "lighteval|agieval:gaokao-english|0": { "hashes": { "hash_examples": "4864e492a350ae93", "hash_full_prompts": "0d368ac714d82cbd", "hash_input_tokens": "97b152ce2f9dd276", "hash_cont_tokens": "b0295fd391ea9065" }, "truncated": 0, "non_truncated": 306, "padded": 1224, "non_padded": 0, "effective_few_shots": 0.0, "num_truncated_few_shots": 0 }, "lighteval|agieval:gaokao-geography|0": { "hashes": { "hash_examples": "ec3a021e37650e7d", "hash_full_prompts": "ae39d50701103ed2", "hash_input_tokens": "03fe8ffbd234ac1f", "hash_cont_tokens": "5696e6aef46a4012" }, "truncated": 0, "non_truncated": 199, "padded": 794, "non_padded": 2, "effective_few_shots": 0.0, "num_truncated_few_shots": 0 }, "lighteval|agieval:gaokao-history|0": { "hashes": { "hash_examples": "b3fad1596f1ae1f9", "hash_full_prompts": "d846ac9f414ea77f", "hash_input_tokens": "792a5a317dd88386", "hash_cont_tokens": "e2947e88a3f08a49" }, "truncated": 0, "non_truncated": 235, "padded": 938, "non_padded": 2, "effective_few_shots": 0.0, "num_truncated_few_shots": 0 }, "lighteval|agieval:gaokao-mathqa|0": { "hashes": { "hash_examples": "1d1088556861b0b0", "hash_full_prompts": "2d6d02d7c7878410", "hash_input_tokens": "23b49a4c37eb5643", "hash_cont_tokens": "7bce9c6eac88e31d" }, "truncated": 0, "non_truncated": 351, "padded": 1403, "non_padded": 1, "effective_few_shots": 0.0, "num_truncated_few_shots": 0 }, "lighteval|agieval:gaokao-physics|0": { "hashes": { "hash_examples": "eb05f035c7bfca2f", "hash_full_prompts": "c32439b20ae8ddc3", "hash_input_tokens": "83afe03525a018b6", "hash_cont_tokens": "77f9a76c75c408ee" }, "truncated": 0, "non_truncated": 200, "padded": 792, "non_padded": 8, "effective_few_shots": 0.0, "num_truncated_few_shots": 0 }, "lighteval|agieval:logiqa-en|0": { "hashes": { "hash_examples": "0a688a45f69c21e0", "hash_full_prompts": "b758941be76e6c93", "hash_input_tokens": "3b4e102d5a77c2ee", "hash_cont_tokens": "bdb78fbd84121ffa" }, "truncated": 0, "non_truncated": 651, "padded": 2591, "non_padded": 13, "effective_few_shots": 0.0, "num_truncated_few_shots": 0 }, "lighteval|agieval:logiqa-zh|0": { "hashes": { "hash_examples": "620d6888b6012ea5", "hash_full_prompts": "00581fe3b81ab4bf", "hash_input_tokens": "47c668eacbda6e4c", "hash_cont_tokens": "a470f1c94ce464df" }, "truncated": 0, "non_truncated": 651, "padded": 2577, "non_padded": 27, "effective_few_shots": 0.0, "num_truncated_few_shots": 0 }, "lighteval|agieval:lsat-ar|0": { "hashes": { "hash_examples": "627c8f5ccd5da209", "hash_full_prompts": "c35e6779d96ec186", "hash_input_tokens": "21aca9a9e3907790", "hash_cont_tokens": "5a2d4bf9575205d0" }, "truncated": 0, "non_truncated": 230, "padded": 1134, "non_padded": 16, "effective_few_shots": 0.0, "num_truncated_few_shots": 0 }, "lighteval|agieval:lsat-lr|0": { "hashes": { "hash_examples": "794641c86de172f5", "hash_full_prompts": "28132acff0b72fac", "hash_input_tokens": "21b8cf84591f6416", "hash_cont_tokens": "b7efbd859fa461cb" }, "truncated": 0, "non_truncated": 510, "padded": 2528, "non_padded": 22, "effective_few_shots": 0.0, "num_truncated_few_shots": 0 }, "lighteval|agieval:lsat-rc|0": { "hashes": { "hash_examples": "35981ed917ea01cf", "hash_full_prompts": "38671d826aec9b04", "hash_input_tokens": "e6836cf65003774d", "hash_cont_tokens": "25cd21c6f88e868d" }, "truncated": 0, "non_truncated": 269, "padded": 1345, "non_padded": 0, "effective_few_shots": 0.0, "num_truncated_few_shots": 0 }, "lighteval|agieval:sat-en|0": { "hashes": { "hash_examples": "041c39c646536a1e", "hash_full_prompts": "7cfcf777a80aab88", "hash_input_tokens": "338b5d3aa5e5d50c", "hash_cont_tokens": "a32456da5b0cb4d6" }, "truncated": 0, "non_truncated": 206, "padded": 821, "non_padded": 0, "effective_few_shots": 0.0, "num_truncated_few_shots": 0 }, "lighteval|agieval:sat-en-without-passage|0": { "hashes": { "hash_examples": "e4d9284367dff68f", "hash_full_prompts": "858c585678a575e3", "hash_input_tokens": "0fa7ff75d35b4ef7", "hash_cont_tokens": "b5be143c7f4c4669" }, "truncated": 0, "non_truncated": 206, "padded": 814, "non_padded": 7, "effective_few_shots": 0.0, "num_truncated_few_shots": 0 }, "lighteval|agieval:sat-math|0": { "hashes": { "hash_examples": "01db7291603fc1a0", "hash_full_prompts": "22304b8d6215acee", "hash_input_tokens": "b41cfbbe32e7050d", "hash_cont_tokens": "05e08502d89d7a73" }, "truncated": 0, "non_truncated": 220, "padded": 873, "non_padded": 7, "effective_few_shots": 0.0, "num_truncated_few_shots": 0 } }, "summary_general": { "hashes": { "hash_examples": "da3af66181f18ddf", "hash_full_prompts": "38bd3cda94f79eaf", "hash_input_tokens": "4f8569cc545f17c5", "hash_cont_tokens": "0e86a0b3e19b1c9c" }, "truncated": 0, "non_truncated": 5151, "padded": 21750, "non_padded": 114, "num_truncated_few_shots": 0 } }