{ "config_general": { "lighteval_sha": "?", "num_fewshot_seeds": 1, "override_batch_size": 1, "max_samples": null, "job_id": "", "start_time": 8646774.98843263, "end_time": 8647277.082037227, "total_evaluation_time_secondes": "502.0936045963317", "model_name": "meta-llama/Llama-2-13b-chat-hf", "model_sha": "29655417e51232f4f2b9b5d3e1418e5a9b04e80e", "model_dtype": "torch.bfloat16", "model_size": "24.32 GB", "config": null }, "results": { "lighteval|agieval:aqua-rat|0": { "acc": 0.2047244094488189, "acc_stderr": 0.02536783354473852, "acc_norm": 0.2047244094488189, "acc_norm_stderr": 0.02536783354473853 }, "lighteval|agieval:gaokao-biology|0": { "acc": 0.28095238095238095, "acc_stderr": 0.03109009446934462, "acc_norm": 0.3142857142857143, "acc_norm_stderr": 0.03211151353994384 }, "lighteval|agieval:gaokao-chemistry|0": { "acc": 0.30917874396135264, "acc_stderr": 0.03219986494000449, "acc_norm": 0.2560386473429952, "acc_norm_stderr": 0.03040845392239327 }, "lighteval|agieval:gaokao-chinese|0": { "acc": 0.1991869918699187, "acc_stderr": 0.025516024108089824, "acc_norm": 0.21138211382113822, "acc_norm_stderr": 0.02608461238305184 }, "lighteval|agieval:gaokao-english|0": { "acc": 0.5490196078431373, "acc_stderr": 0.02849199358617156, "acc_norm": 0.4477124183006536, "acc_norm_stderr": 0.028472938478033522 }, "lighteval|agieval:gaokao-geography|0": { "acc": 0.3316582914572864, "acc_stderr": 0.03345893526874066, "acc_norm": 0.3015075376884422, "acc_norm_stderr": 0.03261349587454653 }, "lighteval|agieval:gaokao-history|0": { "acc": 0.33191489361702126, "acc_stderr": 0.03078373675774565, "acc_norm": 0.28936170212765955, "acc_norm_stderr": 0.02964400657700962 }, "lighteval|agieval:gaokao-mathqa|0": { "acc": 0.2706552706552707, "acc_stderr": 0.023748744034266786, "acc_norm": 0.2706552706552707, "acc_norm_stderr": 0.023748744034266786 }, "lighteval|agieval:gaokao-physics|0": { "acc": 0.27, "acc_stderr": 0.03147145152843339, "acc_norm": 0.315, "acc_norm_stderr": 0.032928657464464904 }, "lighteval|agieval:logiqa-en|0": { "acc": 0.3563748079877112, "acc_stderr": 0.01878509246182, "acc_norm": 0.3210445468509985, "acc_norm_stderr": 0.01831245670147612 }, "lighteval|agieval:logiqa-zh|0": { "acc": 0.26574500768049153, "acc_stderr": 0.017326040808935687, "acc_norm": 0.3333333333333333, "acc_norm_stderr": 0.018490006540841045 }, "lighteval|agieval:lsat-ar|0": { "acc": 0.21739130434782608, "acc_stderr": 0.027256850838819964, "acc_norm": 0.22608695652173913, "acc_norm_stderr": 0.027641785707241334 }, "lighteval|agieval:lsat-lr|0": { "acc": 0.3568627450980392, "acc_stderr": 0.02123457379560983, "acc_norm": 0.28823529411764703, "acc_norm_stderr": 0.02007626467805952 }, "lighteval|agieval:lsat-rc|0": { "acc": 0.4200743494423792, "acc_stderr": 0.030149620080212006, "acc_norm": 0.31226765799256506, "acc_norm_stderr": 0.02830778120469435 }, "lighteval|agieval:sat-en|0": { "acc": 0.6019417475728155, "acc_stderr": 0.03418799390613399, "acc_norm": 0.4077669902912621, "acc_norm_stderr": 0.03432222290260263 }, "lighteval|agieval:sat-en-without-passage|0": { "acc": 0.3640776699029126, "acc_stderr": 0.03360641055142781, "acc_norm": 0.25728155339805825, "acc_norm_stderr": 0.030530892446123826 }, "lighteval|agieval:sat-math|0": { "acc": 0.2863636363636364, "acc_stderr": 0.030547478139694613, "acc_norm": 0.2636363636363636, "acc_norm_stderr": 0.02977328576472747 }, "lighteval|agieval:_average|0": { "acc": 0.3303601093059411, "acc_stderr": 0.027954278754128786, "acc_norm": 0.2953129711654506, "acc_norm_stderr": 0.0275785265743656 }, "all": { "acc": 0.3303601093059411, "acc_stderr": 0.027954278754128786, "acc_norm": 0.2953129711654506, "acc_norm_stderr": 0.0275785265743656 } }, "versions": { "lighteval|agieval:aqua-rat|0": 0, "lighteval|agieval:gaokao-biology|0": 0, "lighteval|agieval:gaokao-chemistry|0": 0, "lighteval|agieval:gaokao-chinese|0": 0, "lighteval|agieval:gaokao-english|0": 0, "lighteval|agieval:gaokao-geography|0": 0, "lighteval|agieval:gaokao-history|0": 0, "lighteval|agieval:gaokao-mathqa|0": 0, "lighteval|agieval:gaokao-physics|0": 0, "lighteval|agieval:logiqa-en|0": 0, "lighteval|agieval:logiqa-zh|0": 0, "lighteval|agieval:lsat-ar|0": 0, "lighteval|agieval:lsat-lr|0": 0, "lighteval|agieval:lsat-rc|0": 0, "lighteval|agieval:sat-en|0": 0, "lighteval|agieval:sat-en-without-passage|0": 0, "lighteval|agieval:sat-math|0": 0 }, "config_tasks": { "lighteval|agieval:aqua-rat": { "name": "agieval:aqua-rat", "prompt_function": "agieval", "hf_repo": "dmayhem93/agieval-aqua-rat", "hf_subset": "default", "metric": [ "loglikelihood_acc", "loglikelihood_acc_norm_nospace" ], "hf_avail_splits": [ "test" ], "evaluation_splits": [ "test" ], "few_shots_split": null, "few_shots_select": "random_sampling", "generation_size": 1, "stop_sequence": null, "output_regex": null, "frozen": false, "suite": [ "lighteval" ], "original_num_docs": 254, "effective_num_docs": 254, "trust_dataset": true, "must_remove_duplicate_docs": null }, "lighteval|agieval:gaokao-biology": { "name": "agieval:gaokao-biology", "prompt_function": "agieval", "hf_repo": "dmayhem93/agieval-gaokao-biology", "hf_subset": "default", "metric": [ "loglikelihood_acc", "loglikelihood_acc_norm_nospace" ], "hf_avail_splits": [ "test" ], "evaluation_splits": [ "test" ], "few_shots_split": null, "few_shots_select": "random_sampling", "generation_size": 1, "stop_sequence": null, "output_regex": null, "frozen": false, "suite": [ "lighteval" ], "original_num_docs": 210, "effective_num_docs": 210, "trust_dataset": true, "must_remove_duplicate_docs": null }, "lighteval|agieval:gaokao-chemistry": { "name": "agieval:gaokao-chemistry", "prompt_function": "agieval", "hf_repo": "dmayhem93/agieval-gaokao-chemistry", "hf_subset": "default", "metric": [ "loglikelihood_acc", "loglikelihood_acc_norm_nospace" ], "hf_avail_splits": [ "test" ], "evaluation_splits": [ "test" ], "few_shots_split": null, "few_shots_select": "random_sampling", "generation_size": 1, "stop_sequence": null, "output_regex": null, "frozen": false, "suite": [ "lighteval" ], "original_num_docs": 207, "effective_num_docs": 207, "trust_dataset": true, "must_remove_duplicate_docs": null }, "lighteval|agieval:gaokao-chinese": { "name": "agieval:gaokao-chinese", "prompt_function": "agieval", "hf_repo": "dmayhem93/agieval-gaokao-chinese", "hf_subset": "default", "metric": [ "loglikelihood_acc", "loglikelihood_acc_norm_nospace" ], "hf_avail_splits": [ "test" ], "evaluation_splits": [ "test" ], "few_shots_split": null, "few_shots_select": "random_sampling", "generation_size": 1, "stop_sequence": null, "output_regex": null, "frozen": false, "suite": [ "lighteval" ], "original_num_docs": 246, "effective_num_docs": 246, "trust_dataset": true, "must_remove_duplicate_docs": null }, "lighteval|agieval:gaokao-english": { "name": "agieval:gaokao-english", "prompt_function": "agieval", "hf_repo": "dmayhem93/agieval-gaokao-english", "hf_subset": "default", "metric": [ "loglikelihood_acc", "loglikelihood_acc_norm_nospace" ], "hf_avail_splits": [ "test" ], "evaluation_splits": [ "test" ], "few_shots_split": null, "few_shots_select": "random_sampling", "generation_size": 1, "stop_sequence": null, "output_regex": null, "frozen": false, "suite": [ "lighteval" ], "original_num_docs": 306, "effective_num_docs": 306, "trust_dataset": true, "must_remove_duplicate_docs": null }, "lighteval|agieval:gaokao-geography": { "name": "agieval:gaokao-geography", "prompt_function": "agieval", "hf_repo": "dmayhem93/agieval-gaokao-geography", "hf_subset": "default", "metric": [ "loglikelihood_acc", "loglikelihood_acc_norm_nospace" ], "hf_avail_splits": [ "test" ], "evaluation_splits": [ "test" ], "few_shots_split": null, "few_shots_select": "random_sampling", "generation_size": 1, "stop_sequence": null, "output_regex": null, "frozen": false, "suite": [ "lighteval" ], "original_num_docs": 199, "effective_num_docs": 199, "trust_dataset": true, "must_remove_duplicate_docs": null }, "lighteval|agieval:gaokao-history": { "name": "agieval:gaokao-history", "prompt_function": "agieval", "hf_repo": "dmayhem93/agieval-gaokao-history", "hf_subset": "default", "metric": [ "loglikelihood_acc", "loglikelihood_acc_norm_nospace" ], "hf_avail_splits": [ "test" ], "evaluation_splits": [ "test" ], "few_shots_split": null, "few_shots_select": "random_sampling", "generation_size": 1, "stop_sequence": null, "output_regex": null, "frozen": false, "suite": [ "lighteval" ], "original_num_docs": 235, "effective_num_docs": 235, "trust_dataset": true, "must_remove_duplicate_docs": null }, "lighteval|agieval:gaokao-mathqa": { "name": "agieval:gaokao-mathqa", "prompt_function": "agieval", "hf_repo": "dmayhem93/agieval-gaokao-mathqa", "hf_subset": "default", "metric": [ "loglikelihood_acc", "loglikelihood_acc_norm_nospace" ], "hf_avail_splits": [ "test" ], "evaluation_splits": [ "test" ], "few_shots_split": null, "few_shots_select": "random_sampling", "generation_size": 1, "stop_sequence": null, "output_regex": null, "frozen": false, "suite": [ "lighteval" ], "original_num_docs": 351, "effective_num_docs": 351, "trust_dataset": true, "must_remove_duplicate_docs": null }, "lighteval|agieval:gaokao-physics": { "name": "agieval:gaokao-physics", "prompt_function": "agieval", "hf_repo": "dmayhem93/agieval-gaokao-physics", "hf_subset": "default", "metric": [ "loglikelihood_acc", "loglikelihood_acc_norm_nospace" ], "hf_avail_splits": [ "test" ], "evaluation_splits": [ "test" ], "few_shots_split": null, "few_shots_select": "random_sampling", "generation_size": 1, "stop_sequence": null, "output_regex": null, "frozen": false, "suite": [ "lighteval" ], "original_num_docs": 200, "effective_num_docs": 200, "trust_dataset": true, "must_remove_duplicate_docs": null }, "lighteval|agieval:logiqa-en": { "name": "agieval:logiqa-en", "prompt_function": "agieval", "hf_repo": "dmayhem93/agieval-logiqa-en", "hf_subset": "default", "metric": [ "loglikelihood_acc", "loglikelihood_acc_norm_nospace" ], "hf_avail_splits": [ "test" ], "evaluation_splits": [ "test" ], "few_shots_split": null, "few_shots_select": "random_sampling", "generation_size": 1, "stop_sequence": null, "output_regex": null, "frozen": false, "suite": [ "lighteval" ], "original_num_docs": 651, "effective_num_docs": 651, "trust_dataset": true, "must_remove_duplicate_docs": null }, "lighteval|agieval:logiqa-zh": { "name": "agieval:logiqa-zh", "prompt_function": "agieval", "hf_repo": "dmayhem93/agieval-logiqa-zh", "hf_subset": "default", "metric": [ "loglikelihood_acc", "loglikelihood_acc_norm_nospace" ], "hf_avail_splits": [ "test" ], "evaluation_splits": [ "test" ], "few_shots_split": null, "few_shots_select": "random_sampling", "generation_size": 1, "stop_sequence": null, "output_regex": null, "frozen": false, "suite": [ "lighteval" ], "original_num_docs": 651, "effective_num_docs": 651, "trust_dataset": true, "must_remove_duplicate_docs": null }, "lighteval|agieval:lsat-ar": { "name": "agieval:lsat-ar", "prompt_function": "agieval", "hf_repo": "dmayhem93/agieval-lsat-ar", "hf_subset": "default", "metric": [ "loglikelihood_acc", "loglikelihood_acc_norm_nospace" ], "hf_avail_splits": [ "test" ], "evaluation_splits": [ "test" ], "few_shots_split": null, "few_shots_select": "random_sampling", "generation_size": 1, "stop_sequence": null, "output_regex": null, "frozen": false, "suite": [ "lighteval" ], "original_num_docs": 230, "effective_num_docs": 230, "trust_dataset": true, "must_remove_duplicate_docs": null }, "lighteval|agieval:lsat-lr": { "name": "agieval:lsat-lr", "prompt_function": "agieval", "hf_repo": "dmayhem93/agieval-lsat-lr", "hf_subset": "default", "metric": [ "loglikelihood_acc", "loglikelihood_acc_norm_nospace" ], "hf_avail_splits": [ "test" ], "evaluation_splits": [ "test" ], "few_shots_split": null, "few_shots_select": "random_sampling", "generation_size": 1, "stop_sequence": null, "output_regex": null, "frozen": false, "suite": [ "lighteval" ], "original_num_docs": 510, "effective_num_docs": 510, "trust_dataset": true, "must_remove_duplicate_docs": null }, "lighteval|agieval:lsat-rc": { "name": "agieval:lsat-rc", "prompt_function": "agieval", "hf_repo": "dmayhem93/agieval-lsat-rc", "hf_subset": "default", "metric": [ "loglikelihood_acc", "loglikelihood_acc_norm_nospace" ], "hf_avail_splits": [ "test" ], "evaluation_splits": [ "test" ], "few_shots_split": null, "few_shots_select": "random_sampling", "generation_size": 1, "stop_sequence": null, "output_regex": null, "frozen": false, "suite": [ "lighteval" ], "original_num_docs": 269, "effective_num_docs": 269, "trust_dataset": true, "must_remove_duplicate_docs": null }, "lighteval|agieval:sat-en": { "name": "agieval:sat-en", "prompt_function": "agieval", "hf_repo": "dmayhem93/agieval-sat-en", "hf_subset": "default", "metric": [ "loglikelihood_acc", "loglikelihood_acc_norm_nospace" ], "hf_avail_splits": [ "test" ], "evaluation_splits": [ "test" ], "few_shots_split": null, "few_shots_select": "random_sampling", "generation_size": 1, "stop_sequence": null, "output_regex": null, "frozen": false, "suite": [ "lighteval" ], "original_num_docs": 206, "effective_num_docs": 206, "trust_dataset": true, "must_remove_duplicate_docs": null }, "lighteval|agieval:sat-en-without-passage": { "name": "agieval:sat-en-without-passage", "prompt_function": "agieval", "hf_repo": "dmayhem93/agieval-sat-en-without-passage", "hf_subset": "default", "metric": [ "loglikelihood_acc", "loglikelihood_acc_norm_nospace" ], "hf_avail_splits": [ "test" ], "evaluation_splits": [ "test" ], "few_shots_split": null, "few_shots_select": "random_sampling", "generation_size": 1, "stop_sequence": null, "output_regex": null, "frozen": false, "suite": [ "lighteval" ], "original_num_docs": 206, "effective_num_docs": 206, "trust_dataset": true, "must_remove_duplicate_docs": null }, "lighteval|agieval:sat-math": { "name": "agieval:sat-math", "prompt_function": "agieval", "hf_repo": "dmayhem93/agieval-sat-math", "hf_subset": "default", "metric": [ "loglikelihood_acc", "loglikelihood_acc_norm_nospace" ], "hf_avail_splits": [ "test" ], "evaluation_splits": [ "test" ], "few_shots_split": null, "few_shots_select": "random_sampling", "generation_size": 1, "stop_sequence": null, "output_regex": null, "frozen": false, "suite": [ "lighteval" ], "original_num_docs": 220, "effective_num_docs": 220, "trust_dataset": true, "must_remove_duplicate_docs": null } }, "summary_tasks": { "lighteval|agieval:aqua-rat|0": { "hashes": { "hash_examples": "f09607f69e5b7525", "hash_full_prompts": "3861b27efd5a6170", "hash_input_tokens": "b81ddf32d9e36b1f", "hash_cont_tokens": "d68d99aa7aafcf10" }, "truncated": 0, "non_truncated": 254, "padded": 1270, "non_padded": 0, "effective_few_shots": 0.0, "num_truncated_few_shots": 0 }, "lighteval|agieval:gaokao-biology|0": { "hashes": { "hash_examples": "f262eaf4a72db963", "hash_full_prompts": "1ef14dcf5933ff33", "hash_input_tokens": "0e02a46ea2b2e076", "hash_cont_tokens": "fb818e5d1e6cfaf9" }, "truncated": 0, "non_truncated": 210, "padded": 840, "non_padded": 0, "effective_few_shots": 0.0, "num_truncated_few_shots": 0 }, "lighteval|agieval:gaokao-chemistry|0": { "hashes": { "hash_examples": "47f2e649f58d9da5", "hash_full_prompts": "324a4d07d3b7de40", "hash_input_tokens": "ab927b5975095914", "hash_cont_tokens": "cae820a904612b14" }, "truncated": 0, "non_truncated": 207, "padded": 831, "non_padded": 0, "effective_few_shots": 0.0, "num_truncated_few_shots": 0 }, "lighteval|agieval:gaokao-chinese|0": { "hashes": { "hash_examples": "1010b21fde4726ab", "hash_full_prompts": "938611fd6ecdbf4e", "hash_input_tokens": "72d5df618421255b", "hash_cont_tokens": "4f99bcd4d0795887" }, "truncated": 0, "non_truncated": 246, "padded": 979, "non_padded": 5, "effective_few_shots": 0.0, "num_truncated_few_shots": 0 }, "lighteval|agieval:gaokao-english|0": { "hashes": { "hash_examples": "4864e492a350ae93", "hash_full_prompts": "2cff929f74eb0813", "hash_input_tokens": "364e98db90c628be", "hash_cont_tokens": "8b0fdf9d99f09625" }, "truncated": 0, "non_truncated": 306, "padded": 1217, "non_padded": 7, "effective_few_shots": 0.0, "num_truncated_few_shots": 0 }, "lighteval|agieval:gaokao-geography|0": { "hashes": { "hash_examples": "ec3a021e37650e7d", "hash_full_prompts": "79c20e1e52638229", "hash_input_tokens": "fb6db8d803e25b33", "hash_cont_tokens": "0144954f291c77ff" }, "truncated": 0, "non_truncated": 199, "padded": 796, "non_padded": 0, "effective_few_shots": 0.0, "num_truncated_few_shots": 0 }, "lighteval|agieval:gaokao-history|0": { "hashes": { "hash_examples": "b3fad1596f1ae1f9", "hash_full_prompts": "541c270a0bbab3fd", "hash_input_tokens": "4461fd574f259120", "hash_cont_tokens": "f7f0c87190e11336" }, "truncated": 0, "non_truncated": 235, "padded": 940, "non_padded": 0, "effective_few_shots": 0.0, "num_truncated_few_shots": 0 }, "lighteval|agieval:gaokao-mathqa|0": { "hashes": { "hash_examples": "1d1088556861b0b0", "hash_full_prompts": "6002200193304ad6", "hash_input_tokens": "e6fb6d8930aa10fa", "hash_cont_tokens": "1c92bd434c8ca7cf" }, "truncated": 0, "non_truncated": 351, "padded": 1404, "non_padded": 0, "effective_few_shots": 0.0, "num_truncated_few_shots": 0 }, "lighteval|agieval:gaokao-physics|0": { "hashes": { "hash_examples": "eb05f035c7bfca2f", "hash_full_prompts": "7b606fe0e48d1136", "hash_input_tokens": "1c39aecbf26045ac", "hash_cont_tokens": "2c78afa11a936dc4" }, "truncated": 0, "non_truncated": 200, "padded": 800, "non_padded": 0, "effective_few_shots": 0.0, "num_truncated_few_shots": 0 }, "lighteval|agieval:logiqa-en|0": { "hashes": { "hash_examples": "0a688a45f69c21e0", "hash_full_prompts": "70df665a01ea64f7", "hash_input_tokens": "61f22bb8a98d5e05", "hash_cont_tokens": "1c7594dbf0a0a533" }, "truncated": 0, "non_truncated": 651, "padded": 2588, "non_padded": 16, "effective_few_shots": 0.0, "num_truncated_few_shots": 0 }, "lighteval|agieval:logiqa-zh|0": { "hashes": { "hash_examples": "620d6888b6012ea5", "hash_full_prompts": "79066930a44b0f7e", "hash_input_tokens": "2fe19dd623007024", "hash_cont_tokens": "16ad57705a79dec4" }, "truncated": 0, "non_truncated": 651, "padded": 2585, "non_padded": 19, "effective_few_shots": 0.0, "num_truncated_few_shots": 0 }, "lighteval|agieval:lsat-ar|0": { "hashes": { "hash_examples": "627c8f5ccd5da209", "hash_full_prompts": "254286f37d4a76ce", "hash_input_tokens": "dcb768476dabfd30", "hash_cont_tokens": "32e72f9ac5e990db" }, "truncated": 0, "non_truncated": 230, "padded": 1139, "non_padded": 11, "effective_few_shots": 0.0, "num_truncated_few_shots": 0 }, "lighteval|agieval:lsat-lr|0": { "hashes": { "hash_examples": "794641c86de172f5", "hash_full_prompts": "cf1badbb8c42f6a8", "hash_input_tokens": "f46eacac0dc8f43d", "hash_cont_tokens": "f59d4751382d6cd2" }, "truncated": 0, "non_truncated": 510, "padded": 2527, "non_padded": 23, "effective_few_shots": 0.0, "num_truncated_few_shots": 0 }, "lighteval|agieval:lsat-rc|0": { "hashes": { "hash_examples": "35981ed917ea01cf", "hash_full_prompts": "5fbc2eae9474469b", "hash_input_tokens": "29375d042cf27c7e", "hash_cont_tokens": "3e5cd22ff2929a47" }, "truncated": 0, "non_truncated": 269, "padded": 1345, "non_padded": 0, "effective_few_shots": 0.0, "num_truncated_few_shots": 0 }, "lighteval|agieval:sat-en|0": { "hashes": { "hash_examples": "041c39c646536a1e", "hash_full_prompts": "171c266251542e36", "hash_input_tokens": "1ad83cc18fd43db0", "hash_cont_tokens": "3bc244ff75c1a591" }, "truncated": 0, "non_truncated": 206, "padded": 821, "non_padded": 0, "effective_few_shots": 0.0, "num_truncated_few_shots": 0 }, "lighteval|agieval:sat-en-without-passage|0": { "hashes": { "hash_examples": "e4d9284367dff68f", "hash_full_prompts": "66bc76f2704c6ca7", "hash_input_tokens": "afd505d4ceeeaa45", "hash_cont_tokens": "3bc244ff75c1a591" }, "truncated": 0, "non_truncated": 206, "padded": 821, "non_padded": 0, "effective_few_shots": 0.0, "num_truncated_few_shots": 0 }, "lighteval|agieval:sat-math|0": { "hashes": { "hash_examples": "01db7291603fc1a0", "hash_full_prompts": "6016bbecd8642dbe", "hash_input_tokens": "86192e0452c1707e", "hash_cont_tokens": "5175b61accc0c5d1" }, "truncated": 0, "non_truncated": 220, "padded": 879, "non_padded": 1, "effective_few_shots": 0.0, "num_truncated_few_shots": 0 } }, "summary_general": { "hashes": { "hash_examples": "da3af66181f18ddf", "hash_full_prompts": "3554a21b6b69cbf8", "hash_input_tokens": "4abd38d6a674e0cd", "hash_cont_tokens": "0fc139d3b2070461" }, "truncated": 0, "non_truncated": 5151, "padded": 21782, "non_padded": 82, "num_truncated_few_shots": 0 } }