{ "config_general": { "lighteval_sha": "?", "num_fewshot_seeds": 1, "override_batch_size": 1, "max_samples": null, "job_id": "", "start_time": 2079825.461451948, "end_time": 2080892.035621395, "total_evaluation_time_secondes": "1066.5741694471799", "model_name": "meta-llama/Llama-2-70b-chat-hf", "model_sha": "8b17e6f4e86be78cf54afd49ddb517d4e274c13f", "model_dtype": "torch.bfloat16", "model_size": "128.64 GB", "config": null }, "results": { "lighteval|bigbench:causal_judgment|0": { "acc": 0.5421052631578948, "acc_stderr": 0.03624046284425958 }, "lighteval|bigbench:date_understanding|0": { "acc": 0.024390243902439025, "acc_stderr": 0.008041225285314473 }, "lighteval|bigbench:disambiguation_qa|0": { "acc": 0.47674418604651164, "acc_stderr": 0.031155388583850818 }, "lighteval|bigbench:geometric_shapes|0": { "acc": 0.1, "acc_stderr": 0.015833394419077815 }, "lighteval|bigbench:logical_deduction_five_objects|0": { "acc": 0.254, "acc_stderr": 0.01948659680164338 }, "lighteval|bigbench:logical_deduction_seven_objects|0": { "acc": 0.15142857142857144, "acc_stderr": 0.013558433870404417 }, "lighteval|bigbench:logical_deduction_three_objects|0": { "acc": 0.4033333333333333, "acc_stderr": 0.028370197016959944 }, "lighteval|bigbench:movie_recommendation|0": { "acc": 0.358, "acc_stderr": 0.021461434862859122 }, "lighteval|bigbench:navigate|0": { "acc": 0.5, "acc_stderr": 0.015819299929208316 }, "lighteval|bigbench:reasoning_about_colored_objects|0": { "acc": 0.19, "acc_stderr": 0.008774308761784201 }, "lighteval|bigbench:ruin_names|0": { "acc": 0.2700892857142857, "acc_stderr": 0.021000749078822385 }, "lighteval|bigbench:salient_translation_error_detection|0": { "acc": 0.18537074148296592, "acc_stderr": 0.012307023122568653 }, "lighteval|bigbench:snarks|0": { "acc": 0.5359116022099447, "acc_stderr": 0.037171551011396645 }, "lighteval|bigbench:sports_understanding|0": { "acc": 0.68, "acc_stderr": 0.014758652303574886 }, "lighteval|bigbench:temporal_sequences|0": { "acc": 0.049, "acc_stderr": 0.006829761756140931 }, "lighteval|bigbench:tracking_shuffled_objects_five_objects|0": { "acc": 0.2008, "acc_stderr": 0.011335178737965248 }, "lighteval|bigbench:tracking_shuffled_objects_seven_objects|0": { "acc": 0.14114285714285715, "acc_stderr": 0.00832520647820533 }, "lighteval|bigbench:tracking_shuffled_objects_three_objects|0": { "acc": 0.4033333333333333, "acc_stderr": 0.028370197016959944 }, "lighteval|bigbench:_average|0": { "acc": 0.30364718987511874, "acc_stderr": 0.018824392326722 }, "all": { "acc": 0.30364718987511874, "acc_stderr": 0.018824392326722 } }, "versions": { "lighteval|bigbench:causal_judgment|0": 0, "lighteval|bigbench:date_understanding|0": 0, "lighteval|bigbench:disambiguation_qa|0": 0, "lighteval|bigbench:geometric_shapes|0": 0, "lighteval|bigbench:logical_deduction_five_objects|0": 0, "lighteval|bigbench:logical_deduction_seven_objects|0": 0, "lighteval|bigbench:logical_deduction_three_objects|0": 0, "lighteval|bigbench:movie_recommendation|0": 0, "lighteval|bigbench:navigate|0": 0, "lighteval|bigbench:reasoning_about_colored_objects|0": 0, "lighteval|bigbench:ruin_names|0": 0, "lighteval|bigbench:salient_translation_error_detection|0": 0, "lighteval|bigbench:snarks|0": 0, "lighteval|bigbench:sports_understanding|0": 0, "lighteval|bigbench:temporal_sequences|0": 0, "lighteval|bigbench:tracking_shuffled_objects_five_objects|0": 0, "lighteval|bigbench:tracking_shuffled_objects_seven_objects|0": 0, "lighteval|bigbench:tracking_shuffled_objects_three_objects|0": 0 }, "config_tasks": { "lighteval|bigbench:causal_judgment": { "name": "bigbench:causal_judgment", "prompt_function": "bbh_lighteval", "hf_repo": "lighteval/bbh", "hf_subset": "causal_judgement", "metric": [ "loglikelihood_acc_single_token" ], "hf_avail_splits": [ "train" ], "evaluation_splits": [ "train" ], "few_shots_split": null, "few_shots_select": null, "generation_size": -1, "stop_sequence": [ "", "Q:", "\n\n" ], "output_regex": null, "frozen": false, "suite": [ "lighteval" ], "original_num_docs": 190, "effective_num_docs": 190, "trust_dataset": true, "must_remove_duplicate_docs": null }, "lighteval|bigbench:date_understanding": { "name": "bigbench:date_understanding", "prompt_function": "bbh_lighteval", "hf_repo": "lighteval/bbh", "hf_subset": "date_understanding", "metric": [ "loglikelihood_acc_single_token" ], "hf_avail_splits": [ "train" ], "evaluation_splits": [ "train" ], "few_shots_split": null, "few_shots_select": null, "generation_size": -1, "stop_sequence": [ "", "Q:", "\n\n" ], "output_regex": null, "frozen": false, "suite": [ "lighteval" ], "original_num_docs": 369, "effective_num_docs": 369, "trust_dataset": true, "must_remove_duplicate_docs": null }, "lighteval|bigbench:disambiguation_qa": { "name": "bigbench:disambiguation_qa", "prompt_function": "bbh_lighteval", "hf_repo": "lighteval/bbh", "hf_subset": "disambiguation_qa", "metric": [ "loglikelihood_acc_single_token" ], "hf_avail_splits": [ "train" ], "evaluation_splits": [ "train" ], "few_shots_split": null, "few_shots_select": null, "generation_size": -1, "stop_sequence": [ "", "Q:", "\n\n" ], "output_regex": null, "frozen": false, "suite": [ "lighteval" ], "original_num_docs": 258, "effective_num_docs": 258, "trust_dataset": true, "must_remove_duplicate_docs": null }, "lighteval|bigbench:geometric_shapes": { "name": "bigbench:geometric_shapes", "prompt_function": "bbh_lighteval", "hf_repo": "lighteval/bbh", "hf_subset": "geometric_shapes", "metric": [ "loglikelihood_acc_single_token" ], "hf_avail_splits": [ "train" ], "evaluation_splits": [ "train" ], "few_shots_split": null, "few_shots_select": null, "generation_size": -1, "stop_sequence": [ "", "Q:", "\n\n" ], "output_regex": null, "frozen": false, "suite": [ "lighteval" ], "original_num_docs": 360, "effective_num_docs": 360, "trust_dataset": true, "must_remove_duplicate_docs": null }, "lighteval|bigbench:logical_deduction_five_objects": { "name": "bigbench:logical_deduction_five_objects", "prompt_function": "bbh_lighteval", "hf_repo": "lighteval/bbh", "hf_subset": "logical_deduction_five_objects", "metric": [ "loglikelihood_acc_single_token" ], "hf_avail_splits": [ "train" ], "evaluation_splits": [ "train" ], "few_shots_split": null, "few_shots_select": null, "generation_size": -1, "stop_sequence": [ "", "Q:", "\n\n" ], "output_regex": null, "frozen": false, "suite": [ "lighteval" ], "original_num_docs": 500, "effective_num_docs": 500, "trust_dataset": true, "must_remove_duplicate_docs": null }, "lighteval|bigbench:logical_deduction_seven_objects": { "name": "bigbench:logical_deduction_seven_objects", "prompt_function": "bbh_lighteval", "hf_repo": "lighteval/bbh", "hf_subset": "logical_deduction_seven_objects", "metric": [ "loglikelihood_acc_single_token" ], "hf_avail_splits": [ "train" ], "evaluation_splits": [ "train" ], "few_shots_split": null, "few_shots_select": null, "generation_size": -1, "stop_sequence": [ "", "Q:", "\n\n" ], "output_regex": null, "frozen": false, "suite": [ "lighteval" ], "original_num_docs": 700, "effective_num_docs": 700, "trust_dataset": true, "must_remove_duplicate_docs": null }, "lighteval|bigbench:logical_deduction_three_objects": { "name": "bigbench:logical_deduction_three_objects", "prompt_function": "bbh_lighteval", "hf_repo": "lighteval/bbh", "hf_subset": "logical_deduction_three_objects", "metric": [ "loglikelihood_acc_single_token" ], "hf_avail_splits": [ "train" ], "evaluation_splits": [ "train" ], "few_shots_split": null, "few_shots_select": null, "generation_size": -1, "stop_sequence": [ "", "Q:", "\n\n" ], "output_regex": null, "frozen": false, "suite": [ "lighteval" ], "original_num_docs": 300, "effective_num_docs": 300, "trust_dataset": true, "must_remove_duplicate_docs": null }, "lighteval|bigbench:movie_recommendation": { "name": "bigbench:movie_recommendation", "prompt_function": "bbh_lighteval", "hf_repo": "lighteval/bbh", "hf_subset": "movie_recommendation", "metric": [ "loglikelihood_acc_single_token" ], "hf_avail_splits": [ "train" ], "evaluation_splits": [ "train" ], "few_shots_split": null, "few_shots_select": null, "generation_size": -1, "stop_sequence": [ "", "Q:", "\n\n" ], "output_regex": null, "frozen": false, "suite": [ "lighteval" ], "original_num_docs": 500, "effective_num_docs": 500, "trust_dataset": true, "must_remove_duplicate_docs": null }, "lighteval|bigbench:navigate": { "name": "bigbench:navigate", "prompt_function": "bbh_lighteval", "hf_repo": "lighteval/bbh", "hf_subset": "navigate", "metric": [ "loglikelihood_acc_single_token" ], "hf_avail_splits": [ "train" ], "evaluation_splits": [ "train" ], "few_shots_split": null, "few_shots_select": null, "generation_size": -1, "stop_sequence": [ "", "Q:", "\n\n" ], "output_regex": null, "frozen": false, "suite": [ "lighteval" ], "original_num_docs": 1000, "effective_num_docs": 1000, "trust_dataset": true, "must_remove_duplicate_docs": null }, "lighteval|bigbench:reasoning_about_colored_objects": { "name": "bigbench:reasoning_about_colored_objects", "prompt_function": "bbh_lighteval", "hf_repo": "lighteval/bbh", "hf_subset": "reasoning_about_colored_objects", "metric": [ "loglikelihood_acc_single_token" ], "hf_avail_splits": [ "train" ], "evaluation_splits": [ "train" ], "few_shots_split": null, "few_shots_select": null, "generation_size": -1, "stop_sequence": [ "", "Q:", "\n\n" ], "output_regex": null, "frozen": false, "suite": [ "lighteval" ], "original_num_docs": 2000, "effective_num_docs": 2000, "trust_dataset": true, "must_remove_duplicate_docs": null }, "lighteval|bigbench:ruin_names": { "name": "bigbench:ruin_names", "prompt_function": "bbh_lighteval", "hf_repo": "lighteval/bbh", "hf_subset": "ruin_names", "metric": [ "loglikelihood_acc_single_token" ], "hf_avail_splits": [ "train" ], "evaluation_splits": [ "train" ], "few_shots_split": null, "few_shots_select": null, "generation_size": -1, "stop_sequence": [ "", "Q:", "\n\n" ], "output_regex": null, "frozen": false, "suite": [ "lighteval" ], "original_num_docs": 448, "effective_num_docs": 448, "trust_dataset": true, "must_remove_duplicate_docs": null }, "lighteval|bigbench:salient_translation_error_detection": { "name": "bigbench:salient_translation_error_detection", "prompt_function": "bbh_lighteval", "hf_repo": "lighteval/bbh", "hf_subset": "salient_translation_error_detection", "metric": [ "loglikelihood_acc_single_token" ], "hf_avail_splits": [ "train" ], "evaluation_splits": [ "train" ], "few_shots_split": null, "few_shots_select": null, "generation_size": -1, "stop_sequence": [ "", "Q:", "\n\n" ], "output_regex": null, "frozen": false, "suite": [ "lighteval" ], "original_num_docs": 998, "effective_num_docs": 998, "trust_dataset": true, "must_remove_duplicate_docs": null }, "lighteval|bigbench:snarks": { "name": "bigbench:snarks", "prompt_function": "bbh_lighteval", "hf_repo": "lighteval/bbh", "hf_subset": "snarks", "metric": [ "loglikelihood_acc_single_token" ], "hf_avail_splits": [ "train" ], "evaluation_splits": [ "train" ], "few_shots_split": null, "few_shots_select": null, "generation_size": -1, "stop_sequence": [ "", "Q:", "\n\n" ], "output_regex": null, "frozen": false, "suite": [ "lighteval" ], "original_num_docs": 181, "effective_num_docs": 181, "trust_dataset": true, "must_remove_duplicate_docs": null }, "lighteval|bigbench:sports_understanding": { "name": "bigbench:sports_understanding", "prompt_function": "bbh_lighteval", "hf_repo": "lighteval/bbh", "hf_subset": "sports_understanding", "metric": [ "loglikelihood_acc_single_token" ], "hf_avail_splits": [ "train" ], "evaluation_splits": [ "train" ], "few_shots_split": null, "few_shots_select": null, "generation_size": -1, "stop_sequence": [ "", "Q:", "\n\n" ], "output_regex": null, "frozen": false, "suite": [ "lighteval" ], "original_num_docs": 1000, "effective_num_docs": 1000, "trust_dataset": true, "must_remove_duplicate_docs": null }, "lighteval|bigbench:temporal_sequences": { "name": "bigbench:temporal_sequences", "prompt_function": "bbh_lighteval", "hf_repo": "lighteval/bbh", "hf_subset": "temporal_sequences", "metric": [ "loglikelihood_acc_single_token" ], "hf_avail_splits": [ "train" ], "evaluation_splits": [ "train" ], "few_shots_split": null, "few_shots_select": null, "generation_size": -1, "stop_sequence": [ "", "Q:", "\n\n" ], "output_regex": null, "frozen": false, "suite": [ "lighteval" ], "original_num_docs": 1000, "effective_num_docs": 1000, "trust_dataset": true, "must_remove_duplicate_docs": null }, "lighteval|bigbench:tracking_shuffled_objects_five_objects": { "name": "bigbench:tracking_shuffled_objects_five_objects", "prompt_function": "bbh_lighteval", "hf_repo": "lighteval/bbh", "hf_subset": "tracking_shuffled_objects_five_objects", "metric": [ "loglikelihood_acc_single_token" ], "hf_avail_splits": [ "train" ], "evaluation_splits": [ "train" ], "few_shots_split": null, "few_shots_select": null, "generation_size": -1, "stop_sequence": [ "", "Q:", "\n\n" ], "output_regex": null, "frozen": false, "suite": [ "lighteval" ], "original_num_docs": 1250, "effective_num_docs": 1250, "trust_dataset": true, "must_remove_duplicate_docs": null }, "lighteval|bigbench:tracking_shuffled_objects_seven_objects": { "name": "bigbench:tracking_shuffled_objects_seven_objects", "prompt_function": "bbh_lighteval", "hf_repo": "lighteval/bbh", "hf_subset": "tracking_shuffled_objects_seven_objects", "metric": [ "loglikelihood_acc_single_token" ], "hf_avail_splits": [ "train" ], "evaluation_splits": [ "train" ], "few_shots_split": null, "few_shots_select": null, "generation_size": -1, "stop_sequence": [ "", "Q:", "\n\n" ], "output_regex": null, "frozen": false, "suite": [ "lighteval" ], "original_num_docs": 1750, "effective_num_docs": 1750, "trust_dataset": true, "must_remove_duplicate_docs": null }, "lighteval|bigbench:tracking_shuffled_objects_three_objects": { "name": "bigbench:tracking_shuffled_objects_three_objects", "prompt_function": "bbh_lighteval", "hf_repo": "lighteval/bbh", "hf_subset": "tracking_shuffled_objects_three_objects", "metric": [ "loglikelihood_acc_single_token" ], "hf_avail_splits": [ "train" ], "evaluation_splits": [ "train" ], "few_shots_split": null, "few_shots_select": null, "generation_size": -1, "stop_sequence": [ "", "Q:", "\n\n" ], "output_regex": null, "frozen": false, "suite": [ "lighteval" ], "original_num_docs": 300, "effective_num_docs": 300, "trust_dataset": true, "must_remove_duplicate_docs": null } }, "summary_tasks": { "lighteval|bigbench:causal_judgment|0": { "hashes": { "hash_examples": "dfb1ae47218f2850", "hash_full_prompts": "4c7f2f76409f70f4", "hash_input_tokens": "dfde6ce631994bcf", "hash_cont_tokens": "b568f0838688d45f" }, "truncated": 0, "non_truncated": 190, "padded": 189, "non_padded": 1, "effective_few_shots": 0.0, "num_truncated_few_shots": 0 }, "lighteval|bigbench:date_understanding|0": { "hashes": { "hash_examples": "2b823c41500a6ec2", "hash_full_prompts": "e83dd502c036bded", "hash_input_tokens": "9464bd17bbfbf52f", "hash_cont_tokens": "148a2d93f0a3fed7" }, "truncated": 0, "non_truncated": 369, "padded": 369, "non_padded": 0, "effective_few_shots": 0.0, "num_truncated_few_shots": 0 }, "lighteval|bigbench:disambiguation_qa|0": { "hashes": { "hash_examples": "2a4c3d41db198cea", "hash_full_prompts": "9cf3c0f75fc49921", "hash_input_tokens": "bffd456de920d0b5", "hash_cont_tokens": "0bb9ee243095a267" }, "truncated": 0, "non_truncated": 258, "padded": 258, "non_padded": 0, "effective_few_shots": 0.0, "num_truncated_few_shots": 0 }, "lighteval|bigbench:geometric_shapes|0": { "hashes": { "hash_examples": "24aa261103911b72", "hash_full_prompts": "862e0cf9b72a95a4", "hash_input_tokens": "9f7b4f4296011f1e", "hash_cont_tokens": "f687789618cc5b4e" }, "truncated": 0, "non_truncated": 360, "padded": 360, "non_padded": 0, "effective_few_shots": 0.0, "num_truncated_few_shots": 0 }, "lighteval|bigbench:logical_deduction_five_objects|0": { "hashes": { "hash_examples": "cb5bdc92afc41f83", "hash_full_prompts": "238b886ee1e7b36b", "hash_input_tokens": "582759660fad6bb0", "hash_cont_tokens": "cc346802490e368d" }, "truncated": 0, "non_truncated": 500, "padded": 500, "non_padded": 0, "effective_few_shots": 0.0, "num_truncated_few_shots": 0 }, "lighteval|bigbench:logical_deduction_seven_objects|0": { "hashes": { "hash_examples": "b6805ea696739f9f", "hash_full_prompts": "ec2aefb5b9986a54", "hash_input_tokens": "2548836d56fce744", "hash_cont_tokens": "2a94547c7122ac8a" }, "truncated": 0, "non_truncated": 700, "padded": 700, "non_padded": 0, "effective_few_shots": 0.0, "num_truncated_few_shots": 0 }, "lighteval|bigbench:logical_deduction_three_objects|0": { "hashes": { "hash_examples": "0509e5712ab9bcdb", "hash_full_prompts": "f369763a9ab136d8", "hash_input_tokens": "97e768427d717e33", "hash_cont_tokens": "0810414105730f5a" }, "truncated": 0, "non_truncated": 300, "padded": 283, "non_padded": 17, "effective_few_shots": 0.0, "num_truncated_few_shots": 0 }, "lighteval|bigbench:movie_recommendation|0": { "hashes": { "hash_examples": "530cc6f737830f45", "hash_full_prompts": "7aee02c6af822316", "hash_input_tokens": "ff6f5e54dccb748a", "hash_cont_tokens": "39b78764b1f62817" }, "truncated": 0, "non_truncated": 500, "padded": 496, "non_padded": 4, "effective_few_shots": 0.0, "num_truncated_few_shots": 0 }, "lighteval|bigbench:navigate|0": { "hashes": { "hash_examples": "7962ef85d0058b9a", "hash_full_prompts": "b6eff183ee6f8e88", "hash_input_tokens": "6e6625533763e732", "hash_cont_tokens": "929d37605ce0219d" }, "truncated": 0, "non_truncated": 1000, "padded": 994, "non_padded": 6, "effective_few_shots": 0.0, "num_truncated_few_shots": 0 }, "lighteval|bigbench:reasoning_about_colored_objects|0": { "hashes": { "hash_examples": "39be1ab1677a651d", "hash_full_prompts": "897f7f976f40ec00", "hash_input_tokens": "8ccf9a502879e6a4", "hash_cont_tokens": "60f23832d085ccc8" }, "truncated": 0, "non_truncated": 2000, "padded": 2000, "non_padded": 0, "effective_few_shots": 0.0, "num_truncated_few_shots": 0 }, "lighteval|bigbench:ruin_names|0": { "hashes": { "hash_examples": "e9b96b31d2154941", "hash_full_prompts": "cad13f7801d6f123", "hash_input_tokens": "18ee7a859c4683fe", "hash_cont_tokens": "3a14d53f73c7b38b" }, "truncated": 0, "non_truncated": 448, "padded": 448, "non_padded": 0, "effective_few_shots": 0.0, "num_truncated_few_shots": 0 }, "lighteval|bigbench:salient_translation_error_detection|0": { "hashes": { "hash_examples": "951ac59f7ad0427d", "hash_full_prompts": "7c45b95866240537", "hash_input_tokens": "c77c5cccd00f938c", "hash_cont_tokens": "1195a841bd9abbcc" }, "truncated": 0, "non_truncated": 998, "padded": 998, "non_padded": 0, "effective_few_shots": 0.0, "num_truncated_few_shots": 0 }, "lighteval|bigbench:snarks|0": { "hashes": { "hash_examples": "3a53eb9b9d758534", "hash_full_prompts": "713ab9a15534316c", "hash_input_tokens": "dd3e22f508de2312", "hash_cont_tokens": "c1ec66334133c92a" }, "truncated": 0, "non_truncated": 181, "padded": 176, "non_padded": 5, "effective_few_shots": 0.0, "num_truncated_few_shots": 0 }, "lighteval|bigbench:sports_understanding|0": { "hashes": { "hash_examples": "bd65741f00770373", "hash_full_prompts": "241b2cf353df5ecf", "hash_input_tokens": "4191638b06174ffa", "hash_cont_tokens": "96ca07f613d55c35" }, "truncated": 0, "non_truncated": 1000, "padded": 1000, "non_padded": 0, "effective_few_shots": 0.0, "num_truncated_few_shots": 0 }, "lighteval|bigbench:temporal_sequences|0": { "hashes": { "hash_examples": "1d13139f47cb2df7", "hash_full_prompts": "729db25fed2f940e", "hash_input_tokens": "58c2ebacc1a03ca0", "hash_cont_tokens": "fd792f95bc07e152" }, "truncated": 0, "non_truncated": 1000, "padded": 1000, "non_padded": 0, "effective_few_shots": 0.0, "num_truncated_few_shots": 0 }, "lighteval|bigbench:tracking_shuffled_objects_five_objects|0": { "hashes": { "hash_examples": "8770a702a9646648", "hash_full_prompts": "74fbca2d47f8529e", "hash_input_tokens": "c3596088a6d4bb85", "hash_cont_tokens": "823f94ba98681e0e" }, "truncated": 0, "non_truncated": 1250, "padded": 1233, "non_padded": 17, "effective_few_shots": 0.0, "num_truncated_few_shots": 0 }, "lighteval|bigbench:tracking_shuffled_objects_seven_objects|0": { "hashes": { "hash_examples": "b469b7d073824a59", "hash_full_prompts": "bb13b4b0abdc1636", "hash_input_tokens": "b586533e6a04f73d", "hash_cont_tokens": "76f29108032b2ebb" }, "truncated": 0, "non_truncated": 1750, "padded": 1729, "non_padded": 21, "effective_few_shots": 0.0, "num_truncated_few_shots": 0 }, "lighteval|bigbench:tracking_shuffled_objects_three_objects|0": { "hashes": { "hash_examples": "0509e5712ab9bcdb", "hash_full_prompts": "f369763a9ab136d8", "hash_input_tokens": "19f2be77920da3b7", "hash_cont_tokens": "faf55b66443be9d9" }, "truncated": 0, "non_truncated": 300, "padded": 282, "non_padded": 18, "effective_few_shots": 0.0, "num_truncated_few_shots": 0 } }, "summary_general": { "hashes": { "hash_examples": "51a30c4501ba4586", "hash_full_prompts": "a5ca2bdc97de22da", "hash_input_tokens": "52cc4a335d912c7a", "hash_cont_tokens": "b7c4466916e24dad" }, "truncated": 0, "non_truncated": 13104, "padded": 13015, "non_padded": 89, "num_truncated_few_shots": 0 } }