{ "config_general": { "lighteval_sha": "?", "num_fewshot_seeds": 1, "override_batch_size": 1, "max_samples": null, "job_id": "", "start_time": 421804.303392443, "end_time": 422069.48693289, "total_evaluation_time_secondes": "265.183540447033", "model_name": "HuggingFaceH4/zephyr-7b-beta-ift", "model_sha": "0a50b8076f15d8735364b82bcc234a142998519a", "model_dtype": "torch.bfloat16", "model_size": "13.99 GB", "config": null }, "results": { "harness|bbh:causal_judgment|3": { "em": 0.36363636363636365, "em_stderr": 0.03527198153014412, "qem": 0.36363636363636365, "qem_stderr": 0.03527198153014412, "pem": 0.5561497326203209, "pem_stderr": 0.03642987131924727, "pqem": 0.5561497326203209, "pqem_stderr": 0.03642987131924727, "perfect_em": 0.36363636363636365, "perfect_em_stderr": 0.03527198153014412 }, "harness|bbh:date_understanding|3": { "em": 0.48, "em_stderr": 0.031660853408495185, "qem": 0.48, "qem_stderr": 0.031660853408495185, "pem": 0.48, "pem_stderr": 0.031660853408495185, "pqem": 0.572, "pqem_stderr": 0.031355968923772605, "perfect_em": 0.48, "perfect_em_stderr": 0.031660853408495185 }, "harness|bbh:disambiguation_qa|3": { "em": 0.636, "em_stderr": 0.030491555220405555, "qem": 0.636, "qem_stderr": 0.030491555220405555, "pem": 0.636, "pem_stderr": 0.030491555220405555, "pqem": 0.712, "pqem_stderr": 0.02869700458739822, "perfect_em": 0.636, "perfect_em_stderr": 0.030491555220405555 }, "harness|bbh:geometric_shapes|3": { "em": 0.228, "em_stderr": 0.026587432487268452, "qem": 0.228, "qem_stderr": 0.026587432487268452, "pem": 0.228, "pem_stderr": 0.026587432487268452, "pqem": 0.228, "pqem_stderr": 0.026587432487268452, "perfect_em": 0.228, "perfect_em_stderr": 0.026587432487268452 }, "harness|bbh:logical_deduction_five_objects|3": { "em": 0.368, "em_stderr": 0.030562070620993163, "qem": 0.368, "qem_stderr": 0.030562070620993163, "pem": 0.368, "pem_stderr": 0.030562070620993163, "pqem": 0.456, "pqem_stderr": 0.03156328506121339, "perfect_em": 0.368, "perfect_em_stderr": 0.030562070620993163 }, "harness|bbh:logical_deduction_seven_objects|3": { "em": 0.272, "em_stderr": 0.028200088296310006, "qem": 0.272, "qem_stderr": 0.028200088296310006, "pem": 0.272, "pem_stderr": 0.028200088296310006, "pqem": 0.388, "pqem_stderr": 0.03088103874899391, "perfect_em": 0.272, "perfect_em_stderr": 0.028200088296310006 }, "harness|bbh:logical_deduction_three_objects|3": { "em": 0.524, "em_stderr": 0.03164968895968782, "qem": 0.524, "qem_stderr": 0.03164968895968782, "pem": 0.524, "pem_stderr": 0.03164968895968782, "pqem": 0.712, "pqem_stderr": 0.02869700458739821, "perfect_em": 0.524, "perfect_em_stderr": 0.03164968895968782 }, "harness|bbh:movie_recommendation|3": { "em": 0.6465863453815262, "em_stderr": 0.030354919577080145, "qem": 0.6465863453815262, "qem_stderr": 0.030354919577080145, "pem": 0.6506024096385542, "pem_stderr": 0.030275543348462008, "pqem": 0.7028112449799196, "pqem_stderr": 0.029020821148267553, "perfect_em": 0.6465863453815262, "perfect_em_stderr": 0.030354919577080145 }, "harness|bbh:navigate|3": { "em": 0.592, "em_stderr": 0.031145209846548505, "qem": 0.592, "qem_stderr": 0.031145209846548505, "pem": 0.592, "pem_stderr": 0.031145209846548505, "pqem": 0.592, "pqem_stderr": 0.031145209846548505, "perfect_em": 0.592, "perfect_em_stderr": 0.031145209846548505 }, "harness|bbh:reasoning_about_colored_objects|3": { "em": 0.296, "em_stderr": 0.028928939388379617, "qem": 0.296, "qem_stderr": 0.028928939388379617, "pem": 0.364, "pem_stderr": 0.030491555220405562, "pqem": 0.5, "pqem_stderr": 0.031686212526223896, "perfect_em": 0.296, "perfect_em_stderr": 0.028928939388379617 }, "harness|bbh:ruin_names|3": { "em": 0.36693548387096775, "em_stderr": 0.03066693445085009, "qem": 0.36693548387096775, "qem_stderr": 0.03066693445085009, "pem": 0.36693548387096775, "pem_stderr": 0.03066693445085009, "pqem": 0.4717741935483871, "pqem_stderr": 0.03176350513565306, "perfect_em": 0.36693548387096775, "perfect_em_stderr": 0.03066693445085009 }, "harness|bbh:salient_translation_error_detection|3": { "em": 0.308, "em_stderr": 0.029256928606501864, "qem": 0.308, "qem_stderr": 0.029256928606501864, "pem": 0.308, "pem_stderr": 0.029256928606501864, "pqem": 0.444, "pqem_stderr": 0.03148684942554574, "perfect_em": 0.308, "perfect_em_stderr": 0.029256928606501864 }, "harness|bbh:snarks|3": { "em": 0.5056179775280899, "em_stderr": 0.037579929004759816, "qem": 0.5056179775280899, "qem_stderr": 0.037579929004759816, "pem": 0.5056179775280899, "pem_stderr": 0.037579929004759816, "pqem": 0.5280898876404494, "pqem_stderr": 0.03752294651708461, "perfect_em": 0.5056179775280899, "perfect_em_stderr": 0.037579929004759816 }, "harness|bbh:sports_understanding|3": { "em": 0.02, "em_stderr": 0.008872139507342685, "qem": 0.02, "qem_stderr": 0.008872139507342685, "pem": 0.78, "pem_stderr": 0.026251792824605824, "pqem": 0.78, "pqem_stderr": 0.026251792824605824, "perfect_em": 0.02, "perfect_em_stderr": 0.008872139507342685 }, "harness|bbh:temporal_sequences|3": { "em": 0.088, "em_stderr": 0.017953084777052885, "qem": 0.088, "qem_stderr": 0.017953084777052885, "pem": 0.088, "pem_stderr": 0.017953084777052885, "pqem": 0.348, "pqem_stderr": 0.030186568464511683, "perfect_em": 0.088, "perfect_em_stderr": 0.017953084777052885 }, "harness|bbh:tracking_shuffled_objects_five_objects|3": { "em": 0.176, "em_stderr": 0.024133497525457116, "qem": 0.176, "qem_stderr": 0.024133497525457116, "pem": 0.176, "pem_stderr": 0.024133497525457116, "pqem": 0.376, "pqem_stderr": 0.030696336267394583, "perfect_em": 0.176, "perfect_em_stderr": 0.024133497525457116 }, "harness|bbh:tracking_shuffled_objects_seven_objects|3": { "em": 0.112, "em_stderr": 0.01998553693917145, "qem": 0.112, "qem_stderr": 0.01998553693917145, "pem": 0.112, "pem_stderr": 0.01998553693917145, "pqem": 0.244, "pqem_stderr": 0.02721799546455318, "perfect_em": 0.112, "perfect_em_stderr": 0.01998553693917145 }, "harness|bbh:tracking_shuffled_objects_three_objects|3": { "em": 0.376, "em_stderr": 0.030696336267394583, "qem": 0.376, "qem_stderr": 0.030696336267394583, "pem": 0.376, "pem_stderr": 0.030696336267394583, "pqem": 0.688, "pqem_stderr": 0.029361067575219817, "perfect_em": 0.376, "perfect_em_stderr": 0.030696336267394583 }, "harness|bbh:_average|3": { "em": 0.3532653428009415, "em_stderr": 0.02799984035632461, "qem": 0.3532653428009415, "qem_stderr": 0.02799984035632461, "pem": 0.410183644647663, "pem_stderr": 0.02911210606242318, "pqem": 0.5166013921549487, "pqem_stderr": 0.030586161717272256, "perfect_em": 0.3532653428009415, "perfect_em_stderr": 0.02799984035632461 } }, "versions": { "harness|bbh:causal_judgment|3": 0, "harness|bbh:date_understanding|3": 0, "harness|bbh:disambiguation_qa|3": 0, "harness|bbh:geometric_shapes|3": 0, "harness|bbh:logical_deduction_five_objects|3": 0, "harness|bbh:logical_deduction_seven_objects|3": 0, "harness|bbh:logical_deduction_three_objects|3": 0, "harness|bbh:movie_recommendation|3": 0, "harness|bbh:navigate|3": 0, "harness|bbh:reasoning_about_colored_objects|3": 0, "harness|bbh:ruin_names|3": 0, "harness|bbh:salient_translation_error_detection|3": 0, "harness|bbh:snarks|3": 0, "harness|bbh:sports_understanding|3": 0, "harness|bbh:temporal_sequences|3": 0, "harness|bbh:tracking_shuffled_objects_five_objects|3": 0, "harness|bbh:tracking_shuffled_objects_seven_objects|3": 0, "harness|bbh:tracking_shuffled_objects_three_objects|3": 0 }, "config_tasks": { "harness|bbh:causal_judgment": { "name": "bbh:causal_judgment", "prompt_function": "bbh_causal_judgment", "hf_repo": "lukaemon/bbh", "hf_subset": "causal_judgement", "metric": [ "exact_match", "quasi_exact_match", "prefix_exact_match", "prefix_quasi_exact_match", "perfect_exact_match" ], "hf_avail_splits": [ "test" ], "evaluation_splits": [ "test" ], "few_shots_split": null, "few_shots_select": null, "generation_size": 20, "stop_sequence": [ "", "Q:", "\n\n" ], "output_regex": null, "frozen": false, "suite": [ "harness" ], "original_num_docs": 187, "effective_num_docs": 187, "trust_dataset": true, "must_remove_duplicate_docs": null }, "harness|bbh:date_understanding": { "name": "bbh:date_understanding", "prompt_function": "bbh_date_understanding", "hf_repo": "lukaemon/bbh", "hf_subset": "date_understanding", "metric": [ "exact_match", "quasi_exact_match", "prefix_exact_match", "prefix_quasi_exact_match", "perfect_exact_match" ], "hf_avail_splits": [ "test" ], "evaluation_splits": [ "test" ], "few_shots_split": null, "few_shots_select": null, "generation_size": 20, "stop_sequence": [ "", "Q:", "\n\n" ], "output_regex": null, "frozen": false, "suite": [ "harness" ], "original_num_docs": 250, "effective_num_docs": 250, "trust_dataset": true, "must_remove_duplicate_docs": null }, "harness|bbh:disambiguation_qa": { "name": "bbh:disambiguation_qa", "prompt_function": "bbh_disambiguation_qa", "hf_repo": "lukaemon/bbh", "hf_subset": "disambiguation_qa", "metric": [ "exact_match", "quasi_exact_match", "prefix_exact_match", "prefix_quasi_exact_match", "perfect_exact_match" ], "hf_avail_splits": [ "test" ], "evaluation_splits": [ "test" ], "few_shots_split": null, "few_shots_select": null, "generation_size": 20, "stop_sequence": [ "", "Q:", "\n\n" ], "output_regex": null, "frozen": false, "suite": [ "harness" ], "original_num_docs": 250, "effective_num_docs": 250, "trust_dataset": true, "must_remove_duplicate_docs": null }, "harness|bbh:geometric_shapes": { "name": "bbh:geometric_shapes", "prompt_function": "bbh_geometric_shapes", "hf_repo": "lukaemon/bbh", "hf_subset": "geometric_shapes", "metric": [ "exact_match", "quasi_exact_match", "prefix_exact_match", "prefix_quasi_exact_match", "perfect_exact_match" ], "hf_avail_splits": [ "test" ], "evaluation_splits": [ "test" ], "few_shots_split": null, "few_shots_select": null, "generation_size": 20, "stop_sequence": [ "", "Q:", "\n\n" ], "output_regex": null, "frozen": false, "suite": [ "harness" ], "original_num_docs": 250, "effective_num_docs": 250, "trust_dataset": true, "must_remove_duplicate_docs": null }, "harness|bbh:logical_deduction_five_objects": { "name": "bbh:logical_deduction_five_objects", "prompt_function": "bbh_logical_deduction_five_objects", "hf_repo": "lukaemon/bbh", "hf_subset": "logical_deduction_five_objects", "metric": [ "exact_match", "quasi_exact_match", "prefix_exact_match", "prefix_quasi_exact_match", "perfect_exact_match" ], "hf_avail_splits": [ "test" ], "evaluation_splits": [ "test" ], "few_shots_split": null, "few_shots_select": null, "generation_size": 20, "stop_sequence": [ "", "Q:", "\n\n" ], "output_regex": null, "frozen": false, "suite": [ "harness" ], "original_num_docs": 250, "effective_num_docs": 250, "trust_dataset": true, "must_remove_duplicate_docs": null }, "harness|bbh:logical_deduction_seven_objects": { "name": "bbh:logical_deduction_seven_objects", "prompt_function": "bbh_logical_deduction_seven_objects", "hf_repo": "lukaemon/bbh", "hf_subset": "logical_deduction_seven_objects", "metric": [ "exact_match", "quasi_exact_match", "prefix_exact_match", "prefix_quasi_exact_match", "perfect_exact_match" ], "hf_avail_splits": [ "test" ], "evaluation_splits": [ "test" ], "few_shots_split": null, "few_shots_select": null, "generation_size": 20, "stop_sequence": [ "", "Q:", "\n\n" ], "output_regex": null, "frozen": false, "suite": [ "harness" ], "original_num_docs": 250, "effective_num_docs": 250, "trust_dataset": true, "must_remove_duplicate_docs": null }, "harness|bbh:logical_deduction_three_objects": { "name": "bbh:logical_deduction_three_objects", "prompt_function": "bbh_logical_deduction_three_objects", "hf_repo": "lukaemon/bbh", "hf_subset": "logical_deduction_three_objects", "metric": [ "exact_match", "quasi_exact_match", "prefix_exact_match", "prefix_quasi_exact_match", "perfect_exact_match" ], "hf_avail_splits": [ "test" ], "evaluation_splits": [ "test" ], "few_shots_split": null, "few_shots_select": null, "generation_size": 20, "stop_sequence": [ "", "Q:", "\n\n" ], "output_regex": null, "frozen": false, "suite": [ "harness" ], "original_num_docs": 250, "effective_num_docs": 250, "trust_dataset": true, "must_remove_duplicate_docs": null }, "harness|bbh:movie_recommendation": { "name": "bbh:movie_recommendation", "prompt_function": "bbh_movie_recommendation", "hf_repo": "lukaemon/bbh", "hf_subset": "movie_recommendation", "metric": [ "exact_match", "quasi_exact_match", "prefix_exact_match", "prefix_quasi_exact_match", "perfect_exact_match" ], "hf_avail_splits": [ "test" ], "evaluation_splits": [ "test" ], "few_shots_split": null, "few_shots_select": null, "generation_size": 20, "stop_sequence": [ "", "Q:", "\n\n" ], "output_regex": null, "frozen": false, "suite": [ "harness" ], "original_num_docs": 249, "effective_num_docs": 249, "trust_dataset": true, "must_remove_duplicate_docs": null }, "harness|bbh:navigate": { "name": "bbh:navigate", "prompt_function": "bbh_navigate", "hf_repo": "lukaemon/bbh", "hf_subset": "navigate", "metric": [ "exact_match", "quasi_exact_match", "prefix_exact_match", "prefix_quasi_exact_match", "perfect_exact_match" ], "hf_avail_splits": [ "test" ], "evaluation_splits": [ "test" ], "few_shots_split": null, "few_shots_select": null, "generation_size": 20, "stop_sequence": [ "", "Q:", "\n\n" ], "output_regex": null, "frozen": false, "suite": [ "harness" ], "original_num_docs": 250, "effective_num_docs": 250, "trust_dataset": true, "must_remove_duplicate_docs": null }, "harness|bbh:reasoning_about_colored_objects": { "name": "bbh:reasoning_about_colored_objects", "prompt_function": "bbh_reasoning_about_colored_objects", "hf_repo": "lukaemon/bbh", "hf_subset": "reasoning_about_colored_objects", "metric": [ "exact_match", "quasi_exact_match", "prefix_exact_match", "prefix_quasi_exact_match", "perfect_exact_match" ], "hf_avail_splits": [ "test" ], "evaluation_splits": [ "test" ], "few_shots_split": null, "few_shots_select": null, "generation_size": 20, "stop_sequence": [ "", "Q:", "\n\n" ], "output_regex": null, "frozen": false, "suite": [ "harness" ], "original_num_docs": 250, "effective_num_docs": 250, "trust_dataset": true, "must_remove_duplicate_docs": null }, "harness|bbh:ruin_names": { "name": "bbh:ruin_names", "prompt_function": "bbh_ruin_names", "hf_repo": "lukaemon/bbh", "hf_subset": "ruin_names", "metric": [ "exact_match", "quasi_exact_match", "prefix_exact_match", "prefix_quasi_exact_match", "perfect_exact_match" ], "hf_avail_splits": [ "test" ], "evaluation_splits": [ "test" ], "few_shots_split": null, "few_shots_select": null, "generation_size": 20, "stop_sequence": [ "", "Q:", "\n\n" ], "output_regex": null, "frozen": false, "suite": [ "harness" ], "original_num_docs": 248, "effective_num_docs": 248, "trust_dataset": true, "must_remove_duplicate_docs": null }, "harness|bbh:salient_translation_error_detection": { "name": "bbh:salient_translation_error_detection", "prompt_function": "bbh_salient_translation_error_detection", "hf_repo": "lukaemon/bbh", "hf_subset": "salient_translation_error_detection", "metric": [ "exact_match", "quasi_exact_match", "prefix_exact_match", "prefix_quasi_exact_match", "perfect_exact_match" ], "hf_avail_splits": [ "test" ], "evaluation_splits": [ "test" ], "few_shots_split": null, "few_shots_select": null, "generation_size": 20, "stop_sequence": [ "", "Q:", "\n\n" ], "output_regex": null, "frozen": false, "suite": [ "harness" ], "original_num_docs": 250, "effective_num_docs": 250, "trust_dataset": true, "must_remove_duplicate_docs": null }, "harness|bbh:snarks": { "name": "bbh:snarks", "prompt_function": "bbh_snarks", "hf_repo": "lukaemon/bbh", "hf_subset": "snarks", "metric": [ "exact_match", "quasi_exact_match", "prefix_exact_match", "prefix_quasi_exact_match", "perfect_exact_match" ], "hf_avail_splits": [ "test" ], "evaluation_splits": [ "test" ], "few_shots_split": null, "few_shots_select": null, "generation_size": 20, "stop_sequence": [ "", "Q:", "\n\n" ], "output_regex": null, "frozen": false, "suite": [ "harness" ], "original_num_docs": 178, "effective_num_docs": 178, "trust_dataset": true, "must_remove_duplicate_docs": null }, "harness|bbh:sports_understanding": { "name": "bbh:sports_understanding", "prompt_function": "bbh_sports_understanding", "hf_repo": "lukaemon/bbh", "hf_subset": "sports_understanding", "metric": [ "exact_match", "quasi_exact_match", "prefix_exact_match", "prefix_quasi_exact_match", "perfect_exact_match" ], "hf_avail_splits": [ "test" ], "evaluation_splits": [ "test" ], "few_shots_split": null, "few_shots_select": null, "generation_size": 20, "stop_sequence": [ "", "Q:", "\n\n" ], "output_regex": null, "frozen": false, "suite": [ "harness" ], "original_num_docs": 250, "effective_num_docs": 250, "trust_dataset": true, "must_remove_duplicate_docs": null }, "harness|bbh:temporal_sequences": { "name": "bbh:temporal_sequences", "prompt_function": "bbh_temporal_sequences", "hf_repo": "lukaemon/bbh", "hf_subset": "temporal_sequences", "metric": [ "exact_match", "quasi_exact_match", "prefix_exact_match", "prefix_quasi_exact_match", "perfect_exact_match" ], "hf_avail_splits": [ "test" ], "evaluation_splits": [ "test" ], "few_shots_split": null, "few_shots_select": null, "generation_size": 20, "stop_sequence": [ "", "Q:", "\n\n" ], "output_regex": null, "frozen": false, "suite": [ "harness" ], "original_num_docs": 250, "effective_num_docs": 250, "trust_dataset": true, "must_remove_duplicate_docs": null }, "harness|bbh:tracking_shuffled_objects_five_objects": { "name": "bbh:tracking_shuffled_objects_five_objects", "prompt_function": "bbh_tracking_shuffled_objects_five_objects", "hf_repo": "lukaemon/bbh", "hf_subset": "tracking_shuffled_objects_five_objects", "metric": [ "exact_match", "quasi_exact_match", "prefix_exact_match", "prefix_quasi_exact_match", "perfect_exact_match" ], "hf_avail_splits": [ "test" ], "evaluation_splits": [ "test" ], "few_shots_split": null, "few_shots_select": null, "generation_size": 20, "stop_sequence": [ "", "Q:", "\n\n" ], "output_regex": null, "frozen": false, "suite": [ "harness" ], "original_num_docs": 250, "effective_num_docs": 250, "trust_dataset": true, "must_remove_duplicate_docs": null }, "harness|bbh:tracking_shuffled_objects_seven_objects": { "name": "bbh:tracking_shuffled_objects_seven_objects", "prompt_function": "bbh_tracking_shuffled_objects_seven_objects", "hf_repo": "lukaemon/bbh", "hf_subset": "tracking_shuffled_objects_seven_objects", "metric": [ "exact_match", "quasi_exact_match", "prefix_exact_match", "prefix_quasi_exact_match", "perfect_exact_match" ], "hf_avail_splits": [ "test" ], "evaluation_splits": [ "test" ], "few_shots_split": null, "few_shots_select": null, "generation_size": 20, "stop_sequence": [ "", "Q:", "\n\n" ], "output_regex": null, "frozen": false, "suite": [ "harness" ], "original_num_docs": 250, "effective_num_docs": 250, "trust_dataset": true, "must_remove_duplicate_docs": null }, "harness|bbh:tracking_shuffled_objects_three_objects": { "name": "bbh:tracking_shuffled_objects_three_objects", "prompt_function": "bbh_tracking_shuffled_objects_three_objects", "hf_repo": "lukaemon/bbh", "hf_subset": "tracking_shuffled_objects_three_objects", "metric": [ "exact_match", "quasi_exact_match", "prefix_exact_match", "prefix_quasi_exact_match", "perfect_exact_match" ], "hf_avail_splits": [ "test" ], "evaluation_splits": [ "test" ], "few_shots_split": null, "few_shots_select": null, "generation_size": 20, "stop_sequence": [ "", "Q:", "\n\n" ], "output_regex": null, "frozen": false, "suite": [ "harness" ], "original_num_docs": 250, "effective_num_docs": 250, "trust_dataset": true, "must_remove_duplicate_docs": null } }, "summary_tasks": { "harness|bbh:causal_judgment|3": { "hashes": { "hash_examples": "63218f5ae055ab2b", "hash_full_prompts": "7303fa1d0fe0b29a", "hash_input_tokens": "79663e73bb5ce6ac", "hash_cont_tokens": "372cb5cf1df956ef" }, "truncated": 187, "non_truncated": 0, "padded": 0, "non_padded": 187, "effective_few_shots": 3.0, "num_truncated_few_shots": 0 }, "harness|bbh:date_understanding|3": { "hashes": { "hash_examples": "f145c7a06def3c8e", "hash_full_prompts": "69e60d10afa5a6f1", "hash_input_tokens": "e9bd5760c58a1104", "hash_cont_tokens": "8c0865bfd5d6ecf9" }, "truncated": 250, "non_truncated": 0, "padded": 0, "non_padded": 250, "effective_few_shots": 3.0, "num_truncated_few_shots": 0 }, "harness|bbh:disambiguation_qa|3": { "hashes": { "hash_examples": "19677fd1773f7eb9", "hash_full_prompts": "ae0a8fd428f9aee3", "hash_input_tokens": "b3625dcc25d708b2", "hash_cont_tokens": "5de2352d7fcfa99e" }, "truncated": 250, "non_truncated": 0, "padded": 0, "non_padded": 250, "effective_few_shots": 3.0, "num_truncated_few_shots": 0 }, "harness|bbh:geometric_shapes|3": { "hashes": { "hash_examples": "76c7b11a13cc72a9", "hash_full_prompts": "76633257f67207f9", "hash_input_tokens": "c16e8768d8c9056f", "hash_cont_tokens": "09918400c9ea6b83" }, "truncated": 250, "non_truncated": 0, "padded": 0, "non_padded": 250, "effective_few_shots": 3.0, "num_truncated_few_shots": 0 }, "harness|bbh:logical_deduction_five_objects|3": { "hashes": { "hash_examples": "0e958c856332a745", "hash_full_prompts": "3c96645848786efd", "hash_input_tokens": "915443ee37f164dc", "hash_cont_tokens": "1b95ed6253158cb7" }, "truncated": 250, "non_truncated": 0, "padded": 0, "non_padded": 250, "effective_few_shots": 3.0, "num_truncated_few_shots": 0 }, "harness|bbh:logical_deduction_seven_objects|3": { "hashes": { "hash_examples": "ab9de25a5eb40d09", "hash_full_prompts": "185c5851c101ee66", "hash_input_tokens": "66d532c31ef57236", "hash_cont_tokens": "ee899db484d19856" }, "truncated": 250, "non_truncated": 0, "padded": 0, "non_padded": 250, "effective_few_shots": 3.0, "num_truncated_few_shots": 0 }, "harness|bbh:logical_deduction_three_objects|3": { "hashes": { "hash_examples": "3c6bf52517714218", "hash_full_prompts": "8ba2d94357e589d0", "hash_input_tokens": "d51c6ad06efbf88b", "hash_cont_tokens": "c78fc8827c2409a9" }, "truncated": 250, "non_truncated": 0, "padded": 0, "non_padded": 250, "effective_few_shots": 3.0, "num_truncated_few_shots": 0 }, "harness|bbh:movie_recommendation|3": { "hashes": { "hash_examples": "2d9dc4975935d31a", "hash_full_prompts": "a411e216d0f5f626", "hash_input_tokens": "e17a3080d43ae54f", "hash_cont_tokens": "abe57201c291f601" }, "truncated": 249, "non_truncated": 0, "padded": 0, "non_padded": 249, "effective_few_shots": 3.0, "num_truncated_few_shots": 0 }, "harness|bbh:navigate|3": { "hashes": { "hash_examples": "ba91dcdb9a064255", "hash_full_prompts": "ebb3084ecc78a46a", "hash_input_tokens": "90854b0ca565c8f5", "hash_cont_tokens": "106bfc7a0d9f53bd" }, "truncated": 250, "non_truncated": 0, "padded": 0, "non_padded": 250, "effective_few_shots": 3.0, "num_truncated_few_shots": 0 }, "harness|bbh:reasoning_about_colored_objects|3": { "hashes": { "hash_examples": "a6ba328c4c3385d2", "hash_full_prompts": "38328d016a4ebef3", "hash_input_tokens": "b45b5a8a531e8bf5", "hash_cont_tokens": "19f3d4375e80f9c8" }, "truncated": 250, "non_truncated": 0, "padded": 0, "non_padded": 250, "effective_few_shots": 3.0, "num_truncated_few_shots": 0 }, "harness|bbh:ruin_names|3": { "hashes": { "hash_examples": "2ef28d5f2d4fdd25", "hash_full_prompts": "9c7d0493c37182d6", "hash_input_tokens": "627b6058879c9350", "hash_cont_tokens": "b13730f22563b515" }, "truncated": 248, "non_truncated": 0, "padded": 0, "non_padded": 248, "effective_few_shots": 3.0, "num_truncated_few_shots": 0 }, "harness|bbh:salient_translation_error_detection|3": { "hashes": { "hash_examples": "c13f25ec8ffed496", "hash_full_prompts": "edccd4061b168b78", "hash_input_tokens": "7d4d7e481ad8766b", "hash_cont_tokens": "50734314f7cae042" }, "truncated": 250, "non_truncated": 0, "padded": 0, "non_padded": 250, "effective_few_shots": 3.0, "num_truncated_few_shots": 0 }, "harness|bbh:snarks|3": { "hashes": { "hash_examples": "5f6db7bff7f6f22e", "hash_full_prompts": "31cafd95ab850a44", "hash_input_tokens": "616900bacd0ba7ca", "hash_cont_tokens": "b774b909eab451ee" }, "truncated": 178, "non_truncated": 0, "padded": 0, "non_padded": 178, "effective_few_shots": 3.0, "num_truncated_few_shots": 0 }, "harness|bbh:sports_understanding|3": { "hashes": { "hash_examples": "042afbe5d9c1f02d", "hash_full_prompts": "3d46581e9bbec2d0", "hash_input_tokens": "8e9e99c22dd3a8d2", "hash_cont_tokens": "25f31a0722d36082" }, "truncated": 250, "non_truncated": 0, "padded": 0, "non_padded": 250, "effective_few_shots": 3.0, "num_truncated_few_shots": 0 }, "harness|bbh:temporal_sequences|3": { "hashes": { "hash_examples": "803a05f352eb6afc", "hash_full_prompts": "4a54db144a5dd222", "hash_input_tokens": "24789970b2290dd3", "hash_cont_tokens": "799becc48ad28c99" }, "truncated": 250, "non_truncated": 0, "padded": 0, "non_padded": 250, "effective_few_shots": 3.0, "num_truncated_few_shots": 0 }, "harness|bbh:tracking_shuffled_objects_five_objects|3": { "hashes": { "hash_examples": "2bbac6db7ab0d527", "hash_full_prompts": "e3079106787cc311", "hash_input_tokens": "9036045cff895b08", "hash_cont_tokens": "fb54b12a82bb6d2d" }, "truncated": 250, "non_truncated": 0, "padded": 0, "non_padded": 250, "effective_few_shots": 3.0, "num_truncated_few_shots": 0 }, "harness|bbh:tracking_shuffled_objects_seven_objects|3": { "hashes": { "hash_examples": "845caf093ac2b58c", "hash_full_prompts": "6364e5b860590ec8", "hash_input_tokens": "7100c488aa0764ff", "hash_cont_tokens": "4657ed62d033183f" }, "truncated": 250, "non_truncated": 0, "padded": 0, "non_padded": 250, "effective_few_shots": 3.0, "num_truncated_few_shots": 0 }, "harness|bbh:tracking_shuffled_objects_three_objects|3": { "hashes": { "hash_examples": "9004f14d5a32b9a8", "hash_full_prompts": "01aef56c4d1fe9fe", "hash_input_tokens": "b9690a5d32a586fc", "hash_cont_tokens": "8ead035cb04fb1cf" }, "truncated": 250, "non_truncated": 0, "padded": 0, "non_padded": 250, "effective_few_shots": 3.0, "num_truncated_few_shots": 0 } }, "summary_general": { "hashes": { "hash_examples": "4ff1e3dc5703575d", "hash_full_prompts": "1cbeab0a00117cb8", "hash_input_tokens": "3608679dab4ce40e", "hash_cont_tokens": "e566def04c5d82fa" }, "truncated": 4362, "non_truncated": 0, "padded": 0, "non_padded": 4362, "num_truncated_few_shots": 0 } }