{ "config_general": { "lighteval_sha": "?", "num_fewshot_seeds": 1, "override_batch_size": 1, "max_samples": null, "job_id": "", "start_time": 1032334.39355707, "end_time": 1032764.678856595, "total_evaluation_time_secondes": "430.2852995248977", "model_name": "HuggingFaceH4/starcoder2-15b-dpo", "model_sha": "3fea85086795d1c7043849eb550e631fbf8426f4", "model_dtype": "torch.bfloat16", "model_size": "30.04 GB", "config": null }, "results": { "lighteval|agieval:aqua-rat|0": { "acc": 0.3031496062992126, "acc_stderr": 0.02889601929036988, "acc_norm": 0.2952755905511811, "acc_norm_stderr": 0.028678944926860864 }, "lighteval|agieval:gaokao-biology|0": { "acc": 0.3476190476190476, "acc_stderr": 0.032940430891650836, "acc_norm": 0.35714285714285715, "acc_norm_stderr": 0.033144012047664914 }, "lighteval|agieval:gaokao-chemistry|0": { "acc": 0.30917874396135264, "acc_stderr": 0.032199864940004486, "acc_norm": 0.3333333333333333, "acc_norm_stderr": 0.03284430927214311 }, "lighteval|agieval:gaokao-chinese|0": { "acc": 0.4105691056910569, "acc_stderr": 0.03142870877522598, "acc_norm": 0.4024390243902439, "acc_norm_stderr": 0.03132983555771989 }, "lighteval|agieval:gaokao-english|0": { "acc": 0.6797385620915033, "acc_stderr": 0.026716118380156847, "acc_norm": 0.6339869281045751, "acc_norm_stderr": 0.027582811415159607 }, "lighteval|agieval:gaokao-geography|0": { "acc": 0.4221105527638191, "acc_stderr": 0.035099658550329385, "acc_norm": 0.4321608040201005, "acc_norm_stderr": 0.03520487250258454 }, "lighteval|agieval:gaokao-history|0": { "acc": 0.4595744680851064, "acc_stderr": 0.03257901482099834, "acc_norm": 0.43829787234042555, "acc_norm_stderr": 0.032436186361081004 }, "lighteval|agieval:gaokao-mathqa|0": { "acc": 0.28774928774928776, "acc_stderr": 0.024198561654366728, "acc_norm": 0.28774928774928776, "acc_norm_stderr": 0.024198561654366735 }, "lighteval|agieval:gaokao-physics|0": { "acc": 0.335, "acc_stderr": 0.033458517029435815, "acc_norm": 0.355, "acc_norm_stderr": 0.033920910080708536 }, "lighteval|agieval:logiqa-en|0": { "acc": 0.3425499231950845, "acc_stderr": 0.01861386882920801, "acc_norm": 0.36251920122887865, "acc_norm_stderr": 0.01885568797958507 }, "lighteval|agieval:logiqa-zh|0": { "acc": 0.36098310291858676, "acc_stderr": 0.018838352954538687, "acc_norm": 0.37019969278033793, "acc_norm_stderr": 0.01893925025913045 }, "lighteval|agieval:lsat-ar|0": { "acc": 0.18695652173913044, "acc_stderr": 0.025763772398512325, "acc_norm": 0.16521739130434782, "acc_norm_stderr": 0.02454125880854541 }, "lighteval|agieval:lsat-lr|0": { "acc": 0.4117647058823529, "acc_stderr": 0.021814296283441937, "acc_norm": 0.3686274509803922, "acc_norm_stderr": 0.021383450873181317 }, "lighteval|agieval:lsat-rc|0": { "acc": 0.483271375464684, "acc_stderr": 0.030525261933744594, "acc_norm": 0.3940520446096654, "acc_norm_stderr": 0.029848812493479995 }, "lighteval|agieval:sat-en|0": { "acc": 0.6553398058252428, "acc_stderr": 0.0331934128585908, "acc_norm": 0.5631067961165048, "acc_norm_stderr": 0.034642250552412775 }, "lighteval|agieval:sat-en-without-passage|0": { "acc": 0.42718446601941745, "acc_stderr": 0.03454921537431906, "acc_norm": 0.32524271844660196, "acc_norm_stderr": 0.0327190473759639 }, "lighteval|agieval:sat-math|0": { "acc": 0.33636363636363636, "acc_stderr": 0.03192622349349311, "acc_norm": 0.2909090909090909, "acc_norm_stderr": 0.030690753276711096 }, "lighteval|agieval:_average|0": { "acc": 0.3975942889216777, "acc_stderr": 0.02898478226225805, "acc_norm": 0.37501529905928377, "acc_norm_stderr": 0.02888005620219407 }, "all": { "acc": 0.3975942889216777, "acc_stderr": 0.02898478226225805, "acc_norm": 0.37501529905928377, "acc_norm_stderr": 0.02888005620219407 } }, "versions": { "lighteval|agieval:aqua-rat|0": 0, "lighteval|agieval:gaokao-biology|0": 0, "lighteval|agieval:gaokao-chemistry|0": 0, "lighteval|agieval:gaokao-chinese|0": 0, "lighteval|agieval:gaokao-english|0": 0, "lighteval|agieval:gaokao-geography|0": 0, "lighteval|agieval:gaokao-history|0": 0, "lighteval|agieval:gaokao-mathqa|0": 0, "lighteval|agieval:gaokao-physics|0": 0, "lighteval|agieval:logiqa-en|0": 0, "lighteval|agieval:logiqa-zh|0": 0, "lighteval|agieval:lsat-ar|0": 0, "lighteval|agieval:lsat-lr|0": 0, "lighteval|agieval:lsat-rc|0": 0, "lighteval|agieval:sat-en|0": 0, "lighteval|agieval:sat-en-without-passage|0": 0, "lighteval|agieval:sat-math|0": 0 }, "config_tasks": { "lighteval|agieval:aqua-rat": { "name": "agieval:aqua-rat", "prompt_function": "agieval", "hf_repo": "dmayhem93/agieval-aqua-rat", "hf_subset": "default", "metric": [ "loglikelihood_acc", "loglikelihood_acc_norm_nospace" ], "hf_avail_splits": [ "test" ], "evaluation_splits": [ "test" ], "few_shots_split": null, "few_shots_select": "random_sampling", "generation_size": 1, "stop_sequence": null, "output_regex": null, "frozen": false, "suite": [ "lighteval" ], "original_num_docs": 254, "effective_num_docs": 254, "trust_dataset": true, "must_remove_duplicate_docs": null }, "lighteval|agieval:gaokao-biology": { "name": "agieval:gaokao-biology", "prompt_function": "agieval", "hf_repo": "dmayhem93/agieval-gaokao-biology", "hf_subset": "default", "metric": [ "loglikelihood_acc", "loglikelihood_acc_norm_nospace" ], "hf_avail_splits": [ "test" ], "evaluation_splits": [ "test" ], "few_shots_split": null, "few_shots_select": "random_sampling", "generation_size": 1, "stop_sequence": null, "output_regex": null, "frozen": false, "suite": [ "lighteval" ], "original_num_docs": 210, "effective_num_docs": 210, "trust_dataset": true, "must_remove_duplicate_docs": null }, "lighteval|agieval:gaokao-chemistry": { "name": "agieval:gaokao-chemistry", "prompt_function": "agieval", "hf_repo": "dmayhem93/agieval-gaokao-chemistry", "hf_subset": "default", "metric": [ "loglikelihood_acc", "loglikelihood_acc_norm_nospace" ], "hf_avail_splits": [ "test" ], "evaluation_splits": [ "test" ], "few_shots_split": null, "few_shots_select": "random_sampling", "generation_size": 1, "stop_sequence": null, "output_regex": null, "frozen": false, "suite": [ "lighteval" ], "original_num_docs": 207, "effective_num_docs": 207, "trust_dataset": true, "must_remove_duplicate_docs": null }, "lighteval|agieval:gaokao-chinese": { "name": "agieval:gaokao-chinese", "prompt_function": "agieval", "hf_repo": "dmayhem93/agieval-gaokao-chinese", "hf_subset": "default", "metric": [ "loglikelihood_acc", "loglikelihood_acc_norm_nospace" ], "hf_avail_splits": [ "test" ], "evaluation_splits": [ "test" ], "few_shots_split": null, "few_shots_select": "random_sampling", "generation_size": 1, "stop_sequence": null, "output_regex": null, "frozen": false, "suite": [ "lighteval" ], "original_num_docs": 246, "effective_num_docs": 246, "trust_dataset": true, "must_remove_duplicate_docs": null }, "lighteval|agieval:gaokao-english": { "name": "agieval:gaokao-english", "prompt_function": "agieval", "hf_repo": "dmayhem93/agieval-gaokao-english", "hf_subset": "default", "metric": [ "loglikelihood_acc", "loglikelihood_acc_norm_nospace" ], "hf_avail_splits": [ "test" ], "evaluation_splits": [ "test" ], "few_shots_split": null, "few_shots_select": "random_sampling", "generation_size": 1, "stop_sequence": null, "output_regex": null, "frozen": false, "suite": [ "lighteval" ], "original_num_docs": 306, "effective_num_docs": 306, "trust_dataset": true, "must_remove_duplicate_docs": null }, "lighteval|agieval:gaokao-geography": { "name": "agieval:gaokao-geography", "prompt_function": "agieval", "hf_repo": "dmayhem93/agieval-gaokao-geography", "hf_subset": "default", "metric": [ "loglikelihood_acc", "loglikelihood_acc_norm_nospace" ], "hf_avail_splits": [ "test" ], "evaluation_splits": [ "test" ], "few_shots_split": null, "few_shots_select": "random_sampling", "generation_size": 1, "stop_sequence": null, "output_regex": null, "frozen": false, "suite": [ "lighteval" ], "original_num_docs": 199, "effective_num_docs": 199, "trust_dataset": true, "must_remove_duplicate_docs": null }, "lighteval|agieval:gaokao-history": { "name": "agieval:gaokao-history", "prompt_function": "agieval", "hf_repo": "dmayhem93/agieval-gaokao-history", "hf_subset": "default", "metric": [ "loglikelihood_acc", "loglikelihood_acc_norm_nospace" ], "hf_avail_splits": [ "test" ], "evaluation_splits": [ "test" ], "few_shots_split": null, "few_shots_select": "random_sampling", "generation_size": 1, "stop_sequence": null, "output_regex": null, "frozen": false, "suite": [ "lighteval" ], "original_num_docs": 235, "effective_num_docs": 235, "trust_dataset": true, "must_remove_duplicate_docs": null }, "lighteval|agieval:gaokao-mathqa": { "name": "agieval:gaokao-mathqa", "prompt_function": "agieval", "hf_repo": "dmayhem93/agieval-gaokao-mathqa", "hf_subset": "default", "metric": [ "loglikelihood_acc", "loglikelihood_acc_norm_nospace" ], "hf_avail_splits": [ "test" ], "evaluation_splits": [ "test" ], "few_shots_split": null, "few_shots_select": "random_sampling", "generation_size": 1, "stop_sequence": null, "output_regex": null, "frozen": false, "suite": [ "lighteval" ], "original_num_docs": 351, "effective_num_docs": 351, "trust_dataset": true, "must_remove_duplicate_docs": null }, "lighteval|agieval:gaokao-physics": { "name": "agieval:gaokao-physics", "prompt_function": "agieval", "hf_repo": "dmayhem93/agieval-gaokao-physics", "hf_subset": "default", "metric": [ "loglikelihood_acc", "loglikelihood_acc_norm_nospace" ], "hf_avail_splits": [ "test" ], "evaluation_splits": [ "test" ], "few_shots_split": null, "few_shots_select": "random_sampling", "generation_size": 1, "stop_sequence": null, "output_regex": null, "frozen": false, "suite": [ "lighteval" ], "original_num_docs": 200, "effective_num_docs": 200, "trust_dataset": true, "must_remove_duplicate_docs": null }, "lighteval|agieval:logiqa-en": { "name": "agieval:logiqa-en", "prompt_function": "agieval", "hf_repo": "dmayhem93/agieval-logiqa-en", "hf_subset": "default", "metric": [ "loglikelihood_acc", "loglikelihood_acc_norm_nospace" ], "hf_avail_splits": [ "test" ], "evaluation_splits": [ "test" ], "few_shots_split": null, "few_shots_select": "random_sampling", "generation_size": 1, "stop_sequence": null, "output_regex": null, "frozen": false, "suite": [ "lighteval" ], "original_num_docs": 651, "effective_num_docs": 651, "trust_dataset": true, "must_remove_duplicate_docs": null }, "lighteval|agieval:logiqa-zh": { "name": "agieval:logiqa-zh", "prompt_function": "agieval", "hf_repo": "dmayhem93/agieval-logiqa-zh", "hf_subset": "default", "metric": [ "loglikelihood_acc", "loglikelihood_acc_norm_nospace" ], "hf_avail_splits": [ "test" ], "evaluation_splits": [ "test" ], "few_shots_split": null, "few_shots_select": "random_sampling", "generation_size": 1, "stop_sequence": null, "output_regex": null, "frozen": false, "suite": [ "lighteval" ], "original_num_docs": 651, "effective_num_docs": 651, "trust_dataset": true, "must_remove_duplicate_docs": null }, "lighteval|agieval:lsat-ar": { "name": "agieval:lsat-ar", "prompt_function": "agieval", "hf_repo": "dmayhem93/agieval-lsat-ar", "hf_subset": "default", "metric": [ "loglikelihood_acc", "loglikelihood_acc_norm_nospace" ], "hf_avail_splits": [ "test" ], "evaluation_splits": [ "test" ], "few_shots_split": null, "few_shots_select": "random_sampling", "generation_size": 1, "stop_sequence": null, "output_regex": null, "frozen": false, "suite": [ "lighteval" ], "original_num_docs": 230, "effective_num_docs": 230, "trust_dataset": true, "must_remove_duplicate_docs": null }, "lighteval|agieval:lsat-lr": { "name": "agieval:lsat-lr", "prompt_function": "agieval", "hf_repo": "dmayhem93/agieval-lsat-lr", "hf_subset": "default", "metric": [ "loglikelihood_acc", "loglikelihood_acc_norm_nospace" ], "hf_avail_splits": [ "test" ], "evaluation_splits": [ "test" ], "few_shots_split": null, "few_shots_select": "random_sampling", "generation_size": 1, "stop_sequence": null, "output_regex": null, "frozen": false, "suite": [ "lighteval" ], "original_num_docs": 510, "effective_num_docs": 510, "trust_dataset": true, "must_remove_duplicate_docs": null }, "lighteval|agieval:lsat-rc": { "name": "agieval:lsat-rc", "prompt_function": "agieval", "hf_repo": "dmayhem93/agieval-lsat-rc", "hf_subset": "default", "metric": [ "loglikelihood_acc", "loglikelihood_acc_norm_nospace" ], "hf_avail_splits": [ "test" ], "evaluation_splits": [ "test" ], "few_shots_split": null, "few_shots_select": "random_sampling", "generation_size": 1, "stop_sequence": null, "output_regex": null, "frozen": false, "suite": [ "lighteval" ], "original_num_docs": 269, "effective_num_docs": 269, "trust_dataset": true, "must_remove_duplicate_docs": null }, "lighteval|agieval:sat-en": { "name": "agieval:sat-en", "prompt_function": "agieval", "hf_repo": "dmayhem93/agieval-sat-en", "hf_subset": "default", "metric": [ "loglikelihood_acc", "loglikelihood_acc_norm_nospace" ], "hf_avail_splits": [ "test" ], "evaluation_splits": [ "test" ], "few_shots_split": null, "few_shots_select": "random_sampling", "generation_size": 1, "stop_sequence": null, "output_regex": null, "frozen": false, "suite": [ "lighteval" ], "original_num_docs": 206, "effective_num_docs": 206, "trust_dataset": true, "must_remove_duplicate_docs": null }, "lighteval|agieval:sat-en-without-passage": { "name": "agieval:sat-en-without-passage", "prompt_function": "agieval", "hf_repo": "dmayhem93/agieval-sat-en-without-passage", "hf_subset": "default", "metric": [ "loglikelihood_acc", "loglikelihood_acc_norm_nospace" ], "hf_avail_splits": [ "test" ], "evaluation_splits": [ "test" ], "few_shots_split": null, "few_shots_select": "random_sampling", "generation_size": 1, "stop_sequence": null, "output_regex": null, "frozen": false, "suite": [ "lighteval" ], "original_num_docs": 206, "effective_num_docs": 206, "trust_dataset": true, "must_remove_duplicate_docs": null }, "lighteval|agieval:sat-math": { "name": "agieval:sat-math", "prompt_function": "agieval", "hf_repo": "dmayhem93/agieval-sat-math", "hf_subset": "default", "metric": [ "loglikelihood_acc", "loglikelihood_acc_norm_nospace" ], "hf_avail_splits": [ "test" ], "evaluation_splits": [ "test" ], "few_shots_split": null, "few_shots_select": "random_sampling", "generation_size": 1, "stop_sequence": null, "output_regex": null, "frozen": false, "suite": [ "lighteval" ], "original_num_docs": 220, "effective_num_docs": 220, "trust_dataset": true, "must_remove_duplicate_docs": null } }, "summary_tasks": { "lighteval|agieval:aqua-rat|0": { "hashes": { "hash_examples": "f09607f69e5b7525", "hash_full_prompts": "f0af1499da980246", "hash_input_tokens": "8baf74e20ece8648", "hash_cont_tokens": "9b1938f401dd8b11" }, "truncated": 0, "non_truncated": 254, "padded": 1270, "non_padded": 0, "effective_few_shots": 0.0, "num_truncated_few_shots": 0 }, "lighteval|agieval:gaokao-biology|0": { "hashes": { "hash_examples": "f262eaf4a72db963", "hash_full_prompts": "4027de4b1cdd1c67", "hash_input_tokens": "39dc1edf1aee0fbb", "hash_cont_tokens": "688fa7bad594603f" }, "truncated": 0, "non_truncated": 210, "padded": 840, "non_padded": 0, "effective_few_shots": 0.0, "num_truncated_few_shots": 0 }, "lighteval|agieval:gaokao-chemistry|0": { "hashes": { "hash_examples": "47f2e649f58d9da5", "hash_full_prompts": "39de31ab927f9675", "hash_input_tokens": "20e47fcf97b7770e", "hash_cont_tokens": "b1270b8e647495ca" }, "truncated": 0, "non_truncated": 207, "padded": 830, "non_padded": 1, "effective_few_shots": 0.0, "num_truncated_few_shots": 0 }, "lighteval|agieval:gaokao-chinese|0": { "hashes": { "hash_examples": "1010b21fde4726ab", "hash_full_prompts": "3b4313f1bd85fd2e", "hash_input_tokens": "3898b0f0f090f3d0", "hash_cont_tokens": "d3b9d7bfcbb19698" }, "truncated": 0, "non_truncated": 246, "padded": 982, "non_padded": 2, "effective_few_shots": 0.0, "num_truncated_few_shots": 0 }, "lighteval|agieval:gaokao-english|0": { "hashes": { "hash_examples": "4864e492a350ae93", "hash_full_prompts": "547863254a606496", "hash_input_tokens": "239abc88ee71e803", "hash_cont_tokens": "cfc7630481f785d5" }, "truncated": 0, "non_truncated": 306, "padded": 1224, "non_padded": 0, "effective_few_shots": 0.0, "num_truncated_few_shots": 0 }, "lighteval|agieval:gaokao-geography|0": { "hashes": { "hash_examples": "ec3a021e37650e7d", "hash_full_prompts": "0a7cffbf555ab29e", "hash_input_tokens": "b4c7b4b03d284285", "hash_cont_tokens": "720e9ef67de27447" }, "truncated": 0, "non_truncated": 199, "padded": 793, "non_padded": 3, "effective_few_shots": 0.0, "num_truncated_few_shots": 0 }, "lighteval|agieval:gaokao-history|0": { "hashes": { "hash_examples": "b3fad1596f1ae1f9", "hash_full_prompts": "b8aca4146c3435af", "hash_input_tokens": "9789eb63f352f670", "hash_cont_tokens": "e06f636fede16bab" }, "truncated": 0, "non_truncated": 235, "padded": 928, "non_padded": 12, "effective_few_shots": 0.0, "num_truncated_few_shots": 0 }, "lighteval|agieval:gaokao-mathqa|0": { "hashes": { "hash_examples": "1d1088556861b0b0", "hash_full_prompts": "1441e196c635c040", "hash_input_tokens": "4c7d3d8c766b4133", "hash_cont_tokens": "9a043c4851ecba43" }, "truncated": 0, "non_truncated": 351, "padded": 1401, "non_padded": 3, "effective_few_shots": 0.0, "num_truncated_few_shots": 0 }, "lighteval|agieval:gaokao-physics|0": { "hashes": { "hash_examples": "eb05f035c7bfca2f", "hash_full_prompts": "be15722274b1466d", "hash_input_tokens": "cad736440c03b505", "hash_cont_tokens": "1c5e032af2b83252" }, "truncated": 0, "non_truncated": 200, "padded": 795, "non_padded": 5, "effective_few_shots": 0.0, "num_truncated_few_shots": 0 }, "lighteval|agieval:logiqa-en|0": { "hashes": { "hash_examples": "0a688a45f69c21e0", "hash_full_prompts": "ca179e67bdc726a6", "hash_input_tokens": "1fbf31a77a24b094", "hash_cont_tokens": "a5d0dff7311f907e" }, "truncated": 0, "non_truncated": 651, "padded": 2588, "non_padded": 16, "effective_few_shots": 0.0, "num_truncated_few_shots": 0 }, "lighteval|agieval:logiqa-zh|0": { "hashes": { "hash_examples": "620d6888b6012ea5", "hash_full_prompts": "55e305ed89c6e580", "hash_input_tokens": "de67a5e6f4908ede", "hash_cont_tokens": "c3a71ac47a57ec15" }, "truncated": 0, "non_truncated": 651, "padded": 2585, "non_padded": 19, "effective_few_shots": 0.0, "num_truncated_few_shots": 0 }, "lighteval|agieval:lsat-ar|0": { "hashes": { "hash_examples": "627c8f5ccd5da209", "hash_full_prompts": "59e010e22954d5b7", "hash_input_tokens": "859b7791e0b3096c", "hash_cont_tokens": "1c426a8ed2791d40" }, "truncated": 0, "non_truncated": 230, "padded": 1150, "non_padded": 0, "effective_few_shots": 0.0, "num_truncated_few_shots": 0 }, "lighteval|agieval:lsat-lr|0": { "hashes": { "hash_examples": "794641c86de172f5", "hash_full_prompts": "efc3c1a3a1586d3e", "hash_input_tokens": "9b0cfa412091f4be", "hash_cont_tokens": "0dbb27d96287a25f" }, "truncated": 0, "non_truncated": 510, "padded": 2526, "non_padded": 24, "effective_few_shots": 0.0, "num_truncated_few_shots": 0 }, "lighteval|agieval:lsat-rc|0": { "hashes": { "hash_examples": "35981ed917ea01cf", "hash_full_prompts": "b80e2b86e1eb0cea", "hash_input_tokens": "481cbc8831175348", "hash_cont_tokens": "da450aec35e6a43a" }, "truncated": 0, "non_truncated": 269, "padded": 1345, "non_padded": 0, "effective_few_shots": 0.0, "num_truncated_few_shots": 0 }, "lighteval|agieval:sat-en|0": { "hashes": { "hash_examples": "041c39c646536a1e", "hash_full_prompts": "4eb610121b313521", "hash_input_tokens": "500851b7c0e4f612", "hash_cont_tokens": "79a85abe49b36aab" }, "truncated": 0, "non_truncated": 206, "padded": 821, "non_padded": 0, "effective_few_shots": 0.0, "num_truncated_few_shots": 0 }, "lighteval|agieval:sat-en-without-passage|0": { "hashes": { "hash_examples": "e4d9284367dff68f", "hash_full_prompts": "532ea18906ff2f4e", "hash_input_tokens": "1592fd5eff06e0b0", "hash_cont_tokens": "79a85abe49b36aab" }, "truncated": 0, "non_truncated": 206, "padded": 816, "non_padded": 5, "effective_few_shots": 0.0, "num_truncated_few_shots": 0 }, "lighteval|agieval:sat-math|0": { "hashes": { "hash_examples": "01db7291603fc1a0", "hash_full_prompts": "1422fad2e0cca51f", "hash_input_tokens": "1af07c78239d2621", "hash_cont_tokens": "aee1d0a52fed5f88" }, "truncated": 0, "non_truncated": 220, "padded": 873, "non_padded": 7, "effective_few_shots": 0.0, "num_truncated_few_shots": 0 } }, "summary_general": { "hashes": { "hash_examples": "da3af66181f18ddf", "hash_full_prompts": "e89209d4ce68d63a", "hash_input_tokens": "5bfe88e83744723c", "hash_cont_tokens": "2c3ea9683e04c17d" }, "truncated": 0, "non_truncated": 5151, "padded": 21767, "non_padded": 97, "num_truncated_few_shots": 0 } }