{ "config_general": { "lighteval_sha": "?", "num_fewshot_seeds": 1, "override_batch_size": 4, "max_samples": null, "job_id": "", "start_time": 796129.532502433, "end_time": 797008.24915019, "total_evaluation_time_secondes": "878.7166477569845", "model_name": "meta-llama/Meta-Llama-3-70B-Instruct", "model_sha": "5fcb2901844dde3111159f24205b71c25900ffbd", "model_dtype": "torch.bfloat16", "model_size": "131.73 GB", "config": null }, "results": { "lighteval|bigbench:causal_judgment|0": { "acc": 0.5210526315789473, "acc_stderr": 0.03633739504773335 }, "lighteval|bigbench:date_understanding|0": { "acc": 0.986449864498645, "acc_stderr": 0.006026781111722431 }, "lighteval|bigbench:disambiguation_qa|0": { "acc": 0.31007751937984496, "acc_stderr": 0.028851521910581428 }, "lighteval|bigbench:geometric_shapes|0": { "acc": 0.1, "acc_stderr": 0.01583339441907784 }, "lighteval|bigbench:logical_deduction_five_objects|0": { "acc": 0.2, "acc_stderr": 0.01790645924143384 }, "lighteval|bigbench:logical_deduction_seven_objects|0": { "acc": 0.14285714285714285, "acc_stderr": 0.013235458703202278 }, "lighteval|bigbench:logical_deduction_three_objects|0": { "acc": 0.3333333333333333, "acc_stderr": 0.027262027336984396 }, "lighteval|bigbench:movie_recommendation|0": { "acc": 0.244, "acc_stderr": 0.019226734893614587 }, "lighteval|bigbench:navigate|0": { "acc": 0.5, "acc_stderr": 0.015819299929208316 }, "lighteval|bigbench:reasoning_about_colored_objects|0": { "acc": 0.178, "acc_stderr": 0.0085553841116954 }, "lighteval|bigbench:ruin_names|0": { "acc": 0.27901785714285715, "acc_stderr": 0.021214094157265963 }, "lighteval|bigbench:salient_translation_error_detection|0": { "acc": 0.14829659318637275, "acc_stderr": 0.011255432818843761 }, "lighteval|bigbench:snarks|0": { "acc": 0.46408839779005523, "acc_stderr": 0.03717155101139654 }, "lighteval|bigbench:sports_understanding|0": { "acc": 0.728, "acc_stderr": 0.014078856992462623 }, "lighteval|bigbench:temporal_sequences|0": { "acc": 1.0, "acc_stderr": 0.0 }, "lighteval|bigbench:tracking_shuffled_objects_five_objects|0": { "acc": 0.2, "acc_stderr": 0.011318236699485788 }, "lighteval|bigbench:tracking_shuffled_objects_seven_objects|0": { "acc": 0.14285714285714285, "acc_stderr": 0.008367248752248818 }, "lighteval|bigbench:tracking_shuffled_objects_three_objects|0": { "acc": 0.3333333333333333, "acc_stderr": 0.027262027336984396 }, "lighteval|bigbench:_average|0": { "acc": 0.37840910088653756, "acc_stderr": 0.0177623280263301 }, "all": { "acc": 0.37840910088653756, "acc_stderr": 0.0177623280263301 } }, "versions": { "lighteval|bigbench:causal_judgment|0": 0, "lighteval|bigbench:date_understanding|0": 0, "lighteval|bigbench:disambiguation_qa|0": 0, "lighteval|bigbench:geometric_shapes|0": 0, "lighteval|bigbench:logical_deduction_five_objects|0": 0, "lighteval|bigbench:logical_deduction_seven_objects|0": 0, "lighteval|bigbench:logical_deduction_three_objects|0": 0, "lighteval|bigbench:movie_recommendation|0": 0, "lighteval|bigbench:navigate|0": 0, "lighteval|bigbench:reasoning_about_colored_objects|0": 0, "lighteval|bigbench:ruin_names|0": 0, "lighteval|bigbench:salient_translation_error_detection|0": 0, "lighteval|bigbench:snarks|0": 0, "lighteval|bigbench:sports_understanding|0": 0, "lighteval|bigbench:temporal_sequences|0": 0, "lighteval|bigbench:tracking_shuffled_objects_five_objects|0": 0, "lighteval|bigbench:tracking_shuffled_objects_seven_objects|0": 0, "lighteval|bigbench:tracking_shuffled_objects_three_objects|0": 0 }, "config_tasks": { "lighteval|bigbench:causal_judgment": { "name": "bigbench:causal_judgment", "prompt_function": "bbh_lighteval", "hf_repo": "lighteval/bbh", "hf_subset": "causal_judgement", "metric": [ "loglikelihood_acc_single_token" ], "hf_avail_splits": [ "train" ], "evaluation_splits": [ "train" ], "few_shots_split": null, "few_shots_select": null, "generation_size": -1, "stop_sequence": [ "", "Q:", "\n\n" ], "output_regex": null, "frozen": false, "suite": [ "lighteval" ], "original_num_docs": 190, "effective_num_docs": 190, "trust_dataset": true, "must_remove_duplicate_docs": null }, "lighteval|bigbench:date_understanding": { "name": "bigbench:date_understanding", "prompt_function": "bbh_lighteval", "hf_repo": "lighteval/bbh", "hf_subset": "date_understanding", "metric": [ "loglikelihood_acc_single_token" ], "hf_avail_splits": [ "train" ], "evaluation_splits": [ "train" ], "few_shots_split": null, "few_shots_select": null, "generation_size": -1, "stop_sequence": [ "", "Q:", "\n\n" ], "output_regex": null, "frozen": false, "suite": [ "lighteval" ], "original_num_docs": 369, "effective_num_docs": 369, "trust_dataset": true, "must_remove_duplicate_docs": null }, "lighteval|bigbench:disambiguation_qa": { "name": "bigbench:disambiguation_qa", "prompt_function": "bbh_lighteval", "hf_repo": "lighteval/bbh", "hf_subset": "disambiguation_qa", "metric": [ "loglikelihood_acc_single_token" ], "hf_avail_splits": [ "train" ], "evaluation_splits": [ "train" ], "few_shots_split": null, "few_shots_select": null, "generation_size": -1, "stop_sequence": [ "", "Q:", "\n\n" ], "output_regex": null, "frozen": false, "suite": [ "lighteval" ], "original_num_docs": 258, "effective_num_docs": 258, "trust_dataset": true, "must_remove_duplicate_docs": null }, "lighteval|bigbench:geometric_shapes": { "name": "bigbench:geometric_shapes", "prompt_function": "bbh_lighteval", "hf_repo": "lighteval/bbh", "hf_subset": "geometric_shapes", "metric": [ "loglikelihood_acc_single_token" ], "hf_avail_splits": [ "train" ], "evaluation_splits": [ "train" ], "few_shots_split": null, "few_shots_select": null, "generation_size": -1, "stop_sequence": [ "", "Q:", "\n\n" ], "output_regex": null, "frozen": false, "suite": [ "lighteval" ], "original_num_docs": 360, "effective_num_docs": 360, "trust_dataset": true, "must_remove_duplicate_docs": null }, "lighteval|bigbench:logical_deduction_five_objects": { "name": "bigbench:logical_deduction_five_objects", "prompt_function": "bbh_lighteval", "hf_repo": "lighteval/bbh", "hf_subset": "logical_deduction_five_objects", "metric": [ "loglikelihood_acc_single_token" ], "hf_avail_splits": [ "train" ], "evaluation_splits": [ "train" ], "few_shots_split": null, "few_shots_select": null, "generation_size": -1, "stop_sequence": [ "", "Q:", "\n\n" ], "output_regex": null, "frozen": false, "suite": [ "lighteval" ], "original_num_docs": 500, "effective_num_docs": 500, "trust_dataset": true, "must_remove_duplicate_docs": null }, "lighteval|bigbench:logical_deduction_seven_objects": { "name": "bigbench:logical_deduction_seven_objects", "prompt_function": "bbh_lighteval", "hf_repo": "lighteval/bbh", "hf_subset": "logical_deduction_seven_objects", "metric": [ "loglikelihood_acc_single_token" ], "hf_avail_splits": [ "train" ], "evaluation_splits": [ "train" ], "few_shots_split": null, "few_shots_select": null, "generation_size": -1, "stop_sequence": [ "", "Q:", "\n\n" ], "output_regex": null, "frozen": false, "suite": [ "lighteval" ], "original_num_docs": 700, "effective_num_docs": 700, "trust_dataset": true, "must_remove_duplicate_docs": null }, "lighteval|bigbench:logical_deduction_three_objects": { "name": "bigbench:logical_deduction_three_objects", "prompt_function": "bbh_lighteval", "hf_repo": "lighteval/bbh", "hf_subset": "logical_deduction_three_objects", "metric": [ "loglikelihood_acc_single_token" ], "hf_avail_splits": [ "train" ], "evaluation_splits": [ "train" ], "few_shots_split": null, "few_shots_select": null, "generation_size": -1, "stop_sequence": [ "", "Q:", "\n\n" ], "output_regex": null, "frozen": false, "suite": [ "lighteval" ], "original_num_docs": 300, "effective_num_docs": 300, "trust_dataset": true, "must_remove_duplicate_docs": null }, "lighteval|bigbench:movie_recommendation": { "name": "bigbench:movie_recommendation", "prompt_function": "bbh_lighteval", "hf_repo": "lighteval/bbh", "hf_subset": "movie_recommendation", "metric": [ "loglikelihood_acc_single_token" ], "hf_avail_splits": [ "train" ], "evaluation_splits": [ "train" ], "few_shots_split": null, "few_shots_select": null, "generation_size": -1, "stop_sequence": [ "", "Q:", "\n\n" ], "output_regex": null, "frozen": false, "suite": [ "lighteval" ], "original_num_docs": 500, "effective_num_docs": 500, "trust_dataset": true, "must_remove_duplicate_docs": null }, "lighteval|bigbench:navigate": { "name": "bigbench:navigate", "prompt_function": "bbh_lighteval", "hf_repo": "lighteval/bbh", "hf_subset": "navigate", "metric": [ "loglikelihood_acc_single_token" ], "hf_avail_splits": [ "train" ], "evaluation_splits": [ "train" ], "few_shots_split": null, "few_shots_select": null, "generation_size": -1, "stop_sequence": [ "", "Q:", "\n\n" ], "output_regex": null, "frozen": false, "suite": [ "lighteval" ], "original_num_docs": 1000, "effective_num_docs": 1000, "trust_dataset": true, "must_remove_duplicate_docs": null }, "lighteval|bigbench:reasoning_about_colored_objects": { "name": "bigbench:reasoning_about_colored_objects", "prompt_function": "bbh_lighteval", "hf_repo": "lighteval/bbh", "hf_subset": "reasoning_about_colored_objects", "metric": [ "loglikelihood_acc_single_token" ], "hf_avail_splits": [ "train" ], "evaluation_splits": [ "train" ], "few_shots_split": null, "few_shots_select": null, "generation_size": -1, "stop_sequence": [ "", "Q:", "\n\n" ], "output_regex": null, "frozen": false, "suite": [ "lighteval" ], "original_num_docs": 2000, "effective_num_docs": 2000, "trust_dataset": true, "must_remove_duplicate_docs": null }, "lighteval|bigbench:ruin_names": { "name": "bigbench:ruin_names", "prompt_function": "bbh_lighteval", "hf_repo": "lighteval/bbh", "hf_subset": "ruin_names", "metric": [ "loglikelihood_acc_single_token" ], "hf_avail_splits": [ "train" ], "evaluation_splits": [ "train" ], "few_shots_split": null, "few_shots_select": null, "generation_size": -1, "stop_sequence": [ "", "Q:", "\n\n" ], "output_regex": null, "frozen": false, "suite": [ "lighteval" ], "original_num_docs": 448, "effective_num_docs": 448, "trust_dataset": true, "must_remove_duplicate_docs": null }, "lighteval|bigbench:salient_translation_error_detection": { "name": "bigbench:salient_translation_error_detection", "prompt_function": "bbh_lighteval", "hf_repo": "lighteval/bbh", "hf_subset": "salient_translation_error_detection", "metric": [ "loglikelihood_acc_single_token" ], "hf_avail_splits": [ "train" ], "evaluation_splits": [ "train" ], "few_shots_split": null, "few_shots_select": null, "generation_size": -1, "stop_sequence": [ "", "Q:", "\n\n" ], "output_regex": null, "frozen": false, "suite": [ "lighteval" ], "original_num_docs": 998, "effective_num_docs": 998, "trust_dataset": true, "must_remove_duplicate_docs": null }, "lighteval|bigbench:snarks": { "name": "bigbench:snarks", "prompt_function": "bbh_lighteval", "hf_repo": "lighteval/bbh", "hf_subset": "snarks", "metric": [ "loglikelihood_acc_single_token" ], "hf_avail_splits": [ "train" ], "evaluation_splits": [ "train" ], "few_shots_split": null, "few_shots_select": null, "generation_size": -1, "stop_sequence": [ "", "Q:", "\n\n" ], "output_regex": null, "frozen": false, "suite": [ "lighteval" ], "original_num_docs": 181, "effective_num_docs": 181, "trust_dataset": true, "must_remove_duplicate_docs": null }, "lighteval|bigbench:sports_understanding": { "name": "bigbench:sports_understanding", "prompt_function": "bbh_lighteval", "hf_repo": "lighteval/bbh", "hf_subset": "sports_understanding", "metric": [ "loglikelihood_acc_single_token" ], "hf_avail_splits": [ "train" ], "evaluation_splits": [ "train" ], "few_shots_split": null, "few_shots_select": null, "generation_size": -1, "stop_sequence": [ "", "Q:", "\n\n" ], "output_regex": null, "frozen": false, "suite": [ "lighteval" ], "original_num_docs": 1000, "effective_num_docs": 1000, "trust_dataset": true, "must_remove_duplicate_docs": null }, "lighteval|bigbench:temporal_sequences": { "name": "bigbench:temporal_sequences", "prompt_function": "bbh_lighteval", "hf_repo": "lighteval/bbh", "hf_subset": "temporal_sequences", "metric": [ "loglikelihood_acc_single_token" ], "hf_avail_splits": [ "train" ], "evaluation_splits": [ "train" ], "few_shots_split": null, "few_shots_select": null, "generation_size": -1, "stop_sequence": [ "", "Q:", "\n\n" ], "output_regex": null, "frozen": false, "suite": [ "lighteval" ], "original_num_docs": 1000, "effective_num_docs": 1000, "trust_dataset": true, "must_remove_duplicate_docs": null }, "lighteval|bigbench:tracking_shuffled_objects_five_objects": { "name": "bigbench:tracking_shuffled_objects_five_objects", "prompt_function": "bbh_lighteval", "hf_repo": "lighteval/bbh", "hf_subset": "tracking_shuffled_objects_five_objects", "metric": [ "loglikelihood_acc_single_token" ], "hf_avail_splits": [ "train" ], "evaluation_splits": [ "train" ], "few_shots_split": null, "few_shots_select": null, "generation_size": -1, "stop_sequence": [ "", "Q:", "\n\n" ], "output_regex": null, "frozen": false, "suite": [ "lighteval" ], "original_num_docs": 1250, "effective_num_docs": 1250, "trust_dataset": true, "must_remove_duplicate_docs": null }, "lighteval|bigbench:tracking_shuffled_objects_seven_objects": { "name": "bigbench:tracking_shuffled_objects_seven_objects", "prompt_function": "bbh_lighteval", "hf_repo": "lighteval/bbh", "hf_subset": "tracking_shuffled_objects_seven_objects", "metric": [ "loglikelihood_acc_single_token" ], "hf_avail_splits": [ "train" ], "evaluation_splits": [ "train" ], "few_shots_split": null, "few_shots_select": null, "generation_size": -1, "stop_sequence": [ "", "Q:", "\n\n" ], "output_regex": null, "frozen": false, "suite": [ "lighteval" ], "original_num_docs": 1750, "effective_num_docs": 1750, "trust_dataset": true, "must_remove_duplicate_docs": null }, "lighteval|bigbench:tracking_shuffled_objects_three_objects": { "name": "bigbench:tracking_shuffled_objects_three_objects", "prompt_function": "bbh_lighteval", "hf_repo": "lighteval/bbh", "hf_subset": "tracking_shuffled_objects_three_objects", "metric": [ "loglikelihood_acc_single_token" ], "hf_avail_splits": [ "train" ], "evaluation_splits": [ "train" ], "few_shots_split": null, "few_shots_select": null, "generation_size": -1, "stop_sequence": [ "", "Q:", "\n\n" ], "output_regex": null, "frozen": false, "suite": [ "lighteval" ], "original_num_docs": 300, "effective_num_docs": 300, "trust_dataset": true, "must_remove_duplicate_docs": null } }, "summary_tasks": { "lighteval|bigbench:causal_judgment|0": { "hashes": { "hash_examples": "dfb1ae47218f2850", "hash_full_prompts": "59c75adfe9a31a36", "hash_input_tokens": "d367bb45f59d1c1a", "hash_cont_tokens": "7abd56823b0f881d" }, "truncated": 0, "non_truncated": 190, "padded": 189, "non_padded": 1, "effective_few_shots": 0.0, "num_truncated_few_shots": 0 }, "lighteval|bigbench:date_understanding|0": { "hashes": { "hash_examples": "2b823c41500a6ec2", "hash_full_prompts": "884a091accfcb273", "hash_input_tokens": "a7ab94b84b243a48", "hash_cont_tokens": "2200d35a17fcfadf" }, "truncated": 0, "non_truncated": 369, "padded": 369, "non_padded": 0, "effective_few_shots": 0.0, "num_truncated_few_shots": 0 }, "lighteval|bigbench:disambiguation_qa|0": { "hashes": { "hash_examples": "2a4c3d41db198cea", "hash_full_prompts": "4e7bae13576ed64d", "hash_input_tokens": "595bd87275fcf560", "hash_cont_tokens": "dd2e010c817c594d" }, "truncated": 0, "non_truncated": 258, "padded": 258, "non_padded": 0, "effective_few_shots": 0.0, "num_truncated_few_shots": 0 }, "lighteval|bigbench:geometric_shapes|0": { "hashes": { "hash_examples": "24aa261103911b72", "hash_full_prompts": "581dc240081a5d44", "hash_input_tokens": "429977eeea021e91", "hash_cont_tokens": "bd1e625b921480c9" }, "truncated": 0, "non_truncated": 360, "padded": 360, "non_padded": 0, "effective_few_shots": 0.0, "num_truncated_few_shots": 0 }, "lighteval|bigbench:logical_deduction_five_objects|0": { "hashes": { "hash_examples": "cb5bdc92afc41f83", "hash_full_prompts": "467c64e62f5268c8", "hash_input_tokens": "fd02498d81f88010", "hash_cont_tokens": "b859a88ce2f1987b" }, "truncated": 0, "non_truncated": 500, "padded": 500, "non_padded": 0, "effective_few_shots": 0.0, "num_truncated_few_shots": 0 }, "lighteval|bigbench:logical_deduction_seven_objects|0": { "hashes": { "hash_examples": "b6805ea696739f9f", "hash_full_prompts": "0ecc606738c5ca4a", "hash_input_tokens": "87c52da26faaae85", "hash_cont_tokens": "92b6a4d8c0f0f7c2" }, "truncated": 0, "non_truncated": 700, "padded": 700, "non_padded": 0, "effective_few_shots": 0.0, "num_truncated_few_shots": 0 }, "lighteval|bigbench:logical_deduction_three_objects|0": { "hashes": { "hash_examples": "0509e5712ab9bcdb", "hash_full_prompts": "92218dd1c7cb33a8", "hash_input_tokens": "b229650974dcd7a7", "hash_cont_tokens": "41fe514efb632853" }, "truncated": 0, "non_truncated": 300, "padded": 267, "non_padded": 33, "effective_few_shots": 0.0, "num_truncated_few_shots": 0 }, "lighteval|bigbench:movie_recommendation|0": { "hashes": { "hash_examples": "530cc6f737830f45", "hash_full_prompts": "d5861db561509a2b", "hash_input_tokens": "1f4ceca0a58f8fe0", "hash_cont_tokens": "d0089dce2f90542e" }, "truncated": 0, "non_truncated": 500, "padded": 500, "non_padded": 0, "effective_few_shots": 0.0, "num_truncated_few_shots": 0 }, "lighteval|bigbench:navigate|0": { "hashes": { "hash_examples": "7962ef85d0058b9a", "hash_full_prompts": "ec163d66b9f3d03c", "hash_input_tokens": "08d0397109277cc1", "hash_cont_tokens": "989b10dafac90407" }, "truncated": 0, "non_truncated": 1000, "padded": 990, "non_padded": 10, "effective_few_shots": 0.0, "num_truncated_few_shots": 0 }, "lighteval|bigbench:reasoning_about_colored_objects|0": { "hashes": { "hash_examples": "39be1ab1677a651d", "hash_full_prompts": "8d0f1eb86028a5d6", "hash_input_tokens": "b7965cad3396a6b4", "hash_cont_tokens": "bea580eabc54d6a4" }, "truncated": 0, "non_truncated": 2000, "padded": 2000, "non_padded": 0, "effective_few_shots": 0.0, "num_truncated_few_shots": 0 }, "lighteval|bigbench:ruin_names|0": { "hashes": { "hash_examples": "e9b96b31d2154941", "hash_full_prompts": "901d7ff41fc0ee8e", "hash_input_tokens": "1f76f54b792ebe2f", "hash_cont_tokens": "67949635d7b35df4" }, "truncated": 0, "non_truncated": 448, "padded": 443, "non_padded": 5, "effective_few_shots": 0.0, "num_truncated_few_shots": 0 }, "lighteval|bigbench:salient_translation_error_detection|0": { "hashes": { "hash_examples": "951ac59f7ad0427d", "hash_full_prompts": "bf3ca7d2f21c2a2f", "hash_input_tokens": "1f2fa9aa4d92db9c", "hash_cont_tokens": "7091834cc929babc" }, "truncated": 0, "non_truncated": 998, "padded": 998, "non_padded": 0, "effective_few_shots": 0.0, "num_truncated_few_shots": 0 }, "lighteval|bigbench:snarks|0": { "hashes": { "hash_examples": "3a53eb9b9d758534", "hash_full_prompts": "b6a540690bddada6", "hash_input_tokens": "123c54819600d026", "hash_cont_tokens": "445b91a8f6611775" }, "truncated": 0, "non_truncated": 181, "padded": 180, "non_padded": 1, "effective_few_shots": 0.0, "num_truncated_few_shots": 0 }, "lighteval|bigbench:sports_understanding|0": { "hashes": { "hash_examples": "bd65741f00770373", "hash_full_prompts": "bf934b4518e08adf", "hash_input_tokens": "f8861cd8326e12a4", "hash_cont_tokens": "61b7586b7d2997a9" }, "truncated": 0, "non_truncated": 1000, "padded": 1000, "non_padded": 0, "effective_few_shots": 0.0, "num_truncated_few_shots": 0 }, "lighteval|bigbench:temporal_sequences|0": { "hashes": { "hash_examples": "1d13139f47cb2df7", "hash_full_prompts": "f1b38a6175096665", "hash_input_tokens": "9b2630aff90471b8", "hash_cont_tokens": "58a836d9f1cd718b" }, "truncated": 0, "non_truncated": 1000, "padded": 1000, "non_padded": 0, "effective_few_shots": 0.0, "num_truncated_few_shots": 0 }, "lighteval|bigbench:tracking_shuffled_objects_five_objects|0": { "hashes": { "hash_examples": "8770a702a9646648", "hash_full_prompts": "5d6ffe9b6461b651", "hash_input_tokens": "989a3385867007b8", "hash_cont_tokens": "ea6242ff1afdd5d4" }, "truncated": 0, "non_truncated": 1250, "padded": 1239, "non_padded": 11, "effective_few_shots": 0.0, "num_truncated_few_shots": 0 }, "lighteval|bigbench:tracking_shuffled_objects_seven_objects|0": { "hashes": { "hash_examples": "b469b7d073824a59", "hash_full_prompts": "a61210e25dbf195d", "hash_input_tokens": "416406a726b3b787", "hash_cont_tokens": "9aae23ebb1e0f579" }, "truncated": 0, "non_truncated": 1750, "padded": 1709, "non_padded": 41, "effective_few_shots": 0.0, "num_truncated_few_shots": 0 }, "lighteval|bigbench:tracking_shuffled_objects_three_objects|0": { "hashes": { "hash_examples": "0509e5712ab9bcdb", "hash_full_prompts": "92218dd1c7cb33a8", "hash_input_tokens": "24d85c6c80ecfad4", "hash_cont_tokens": "419f5cf4c2029190" }, "truncated": 0, "non_truncated": 300, "padded": 264, "non_padded": 36, "effective_few_shots": 0.0, "num_truncated_few_shots": 0 } }, "summary_general": { "hashes": { "hash_examples": "51a30c4501ba4586", "hash_full_prompts": "b40f7b2153556dee", "hash_input_tokens": "547b2506c8d2a0b5", "hash_cont_tokens": "dad86a65dc1c0fc2" }, "truncated": 0, "non_truncated": 13104, "padded": 12966, "non_padded": 138, "num_truncated_few_shots": 0 } }