{ "config_general": { "lighteval_sha": "?", "num_fewshot_seeds": 1, "override_batch_size": 1, "max_samples": null, "job_id": "", "start_time": 895778.173710304, "end_time": 897081.949710843, "total_evaluation_time_secondes": "1303.7760005390737", "model_name": "Qwen/Qwen1.5-72B-Chat", "model_sha": "1a6ccc1215278f962c794b1848c710c29ef4053d", "model_dtype": "torch.bfloat16", "model_size": "135.9 GB", "config": null }, "results": { "harness|bbh:causal_judgment|3": { "em": 0.6149732620320856, "em_stderr": 0.03567936280544673, "qem": 0.6149732620320856, "qem_stderr": 0.03567936280544673, "pem": 0.6149732620320856, "pem_stderr": 0.03567936280544673, "pqem": 0.6149732620320856, "pqem_stderr": 0.03567936280544673, "perfect_em": 0.6149732620320856, "perfect_em_stderr": 0.03567936280544673 }, "harness|bbh:date_understanding|3": { "em": 0.576, "em_stderr": 0.03131803437491614, "qem": 0.576, "qem_stderr": 0.03131803437491614, "pem": 0.576, "pem_stderr": 0.03131803437491614, "pqem": 0.676, "pqem_stderr": 0.02965829492454558, "perfect_em": 0.576, "perfect_em_stderr": 0.03131803437491614 }, "harness|bbh:disambiguation_qa|3": { "em": 0.664, "em_stderr": 0.02993325909419152, "qem": 0.664, "qem_stderr": 0.02993325909419152, "pem": 0.664, "pem_stderr": 0.02993325909419152, "pqem": 0.78, "pqem_stderr": 0.026251792824605828, "perfect_em": 0.664, "perfect_em_stderr": 0.02993325909419152 }, "harness|bbh:geometric_shapes|3": { "em": 0.388, "em_stderr": 0.03088103874899392, "qem": 0.388, "qem_stderr": 0.03088103874899392, "pem": 0.388, "pem_stderr": 0.03088103874899392, "pqem": 0.388, "pqem_stderr": 0.03088103874899392, "perfect_em": 0.388, "perfect_em_stderr": 0.03088103874899392 }, "harness|bbh:logical_deduction_five_objects|3": { "em": 0.556, "em_stderr": 0.03148684942554574, "qem": 0.556, "qem_stderr": 0.03148684942554574, "pem": 0.556, "pem_stderr": 0.03148684942554574, "pqem": 0.644, "pqem_stderr": 0.030343680657153215, "perfect_em": 0.556, "perfect_em_stderr": 0.03148684942554574 }, "harness|bbh:logical_deduction_seven_objects|3": { "em": 0.488, "em_stderr": 0.03167708558254708, "qem": 0.488, "qem_stderr": 0.03167708558254708, "pem": 0.488, "pem_stderr": 0.03167708558254708, "pqem": 0.568, "pqem_stderr": 0.0313918107654294, "perfect_em": 0.488, "perfect_em_stderr": 0.03167708558254708 }, "harness|bbh:logical_deduction_three_objects|3": { "em": 0.772, "em_stderr": 0.02658743248726847, "qem": 0.772, "qem_stderr": 0.02658743248726847, "pem": 0.772, "pem_stderr": 0.02658743248726847, "pqem": 0.848, "pqem_stderr": 0.02275202449176547, "perfect_em": 0.772, "perfect_em_stderr": 0.02658743248726847 }, "harness|bbh:movie_recommendation|3": { "em": 0.8393574297188755, "em_stderr": 0.023317283401377554, "qem": 0.8393574297188755, "qem_stderr": 0.023317283401377554, "pem": 0.8433734939759037, "pem_stderr": 0.023078988107708014, "pqem": 0.8875502008032129, "pqem_stderr": 0.02006087930418837, "perfect_em": 0.8393574297188755, "perfect_em_stderr": 0.023317283401377554 }, "harness|bbh:navigate|3": { "em": 0.6, "em_stderr": 0.031046021028253247, "qem": 0.6, "qem_stderr": 0.031046021028253247, "pem": 0.6, "pem_stderr": 0.031046021028253247, "pqem": 0.6, "pqem_stderr": 0.031046021028253247, "perfect_em": 0.6, "perfect_em_stderr": 0.031046021028253247 }, "harness|bbh:reasoning_about_colored_objects|3": { "em": 0.664, "em_stderr": 0.029933259094191516, "qem": 0.664, "qem_stderr": 0.029933259094191516, "pem": 0.664, "pem_stderr": 0.029933259094191516, "pqem": 0.716, "pqem_stderr": 0.028576958730437394, "perfect_em": 0.664, "perfect_em_stderr": 0.029933259094191516 }, "harness|bbh:ruin_names|3": { "em": 0.8145161290322581, "em_stderr": 0.024731726954093584, "qem": 0.8145161290322581, "qem_stderr": 0.024731726954093584, "pem": 0.8145161290322581, "pem_stderr": 0.024731726954093584, "pqem": 0.875, "pqem_stderr": 0.021043140573169826, "perfect_em": 0.8145161290322581, "perfect_em_stderr": 0.024731726954093584 }, "harness|bbh:salient_translation_error_detection|3": { "em": 0.592, "em_stderr": 0.0311452098465485, "qem": 0.592, "qem_stderr": 0.0311452098465485, "pem": 0.592, "pem_stderr": 0.0311452098465485, "pqem": 0.688, "pqem_stderr": 0.029361067575219817, "perfect_em": 0.592, "perfect_em_stderr": 0.0311452098465485 }, "harness|bbh:snarks|3": { "em": 0.8595505617977528, "em_stderr": 0.026116145785378953, "qem": 0.8595505617977528, "qem_stderr": 0.026116145785378953, "pem": 0.8595505617977528, "pem_stderr": 0.026116145785378953, "pqem": 0.9044943820224719, "pqem_stderr": 0.022091782723697606, "perfect_em": 0.8595505617977528, "perfect_em_stderr": 0.026116145785378953 }, "harness|bbh:sports_understanding|3": { "em": 0.864, "em_stderr": 0.02172334261705205, "qem": 0.864, "qem_stderr": 0.02172334261705205, "pem": 0.864, "pem_stderr": 0.02172334261705205, "pqem": 0.864, "pqem_stderr": 0.02172334261705205, "perfect_em": 0.864, "perfect_em_stderr": 0.02172334261705205 }, "harness|bbh:temporal_sequences|3": { "em": 0.596, "em_stderr": 0.031096688184825295, "qem": 0.596, "qem_stderr": 0.031096688184825295, "pem": 0.596, "pem_stderr": 0.031096688184825295, "pqem": 0.744, "pqem_stderr": 0.02765710871820491, "perfect_em": 0.596, "perfect_em_stderr": 0.031096688184825295 }, "harness|bbh:tracking_shuffled_objects_five_objects|3": { "em": 0.24, "em_stderr": 0.027065293652239007, "qem": 0.24, "qem_stderr": 0.027065293652239007, "pem": 0.24, "pem_stderr": 0.027065293652239007, "pqem": 0.396, "pqem_stderr": 0.030993197854577853, "perfect_em": 0.24, "perfect_em_stderr": 0.027065293652239007 }, "harness|bbh:tracking_shuffled_objects_seven_objects|3": { "em": 0.216, "em_stderr": 0.02607865766373272, "qem": 0.216, "qem_stderr": 0.02607865766373272, "pem": 0.216, "pem_stderr": 0.02607865766373272, "pqem": 0.336, "pqem_stderr": 0.029933259094191512, "perfect_em": 0.216, "perfect_em_stderr": 0.02607865766373272 }, "harness|bbh:tracking_shuffled_objects_three_objects|3": { "em": 0.264, "em_stderr": 0.027934518957690908, "qem": 0.264, "qem_stderr": 0.027934518957690908, "pem": 0.264, "pem_stderr": 0.027934518957690908, "pqem": 0.556, "pqem_stderr": 0.03148684942554574, "perfect_em": 0.264, "perfect_em_stderr": 0.027934518957690908 }, "harness|bbh:_average|3": { "em": 0.5893554101433873, "em_stderr": 0.02876395609468295, "qem": 0.5893554101433873, "qem_stderr": 0.02876395609468295, "pem": 0.5895785248243333, "pem_stderr": 0.028750717467256864, "pqem": 0.6714454358254317, "pqem_stderr": 0.02782953404791547, "perfect_em": 0.5893554101433873, "perfect_em_stderr": 0.02876395609468295 } }, "versions": { "harness|bbh:causal_judgment|3": 0, "harness|bbh:date_understanding|3": 0, "harness|bbh:disambiguation_qa|3": 0, "harness|bbh:geometric_shapes|3": 0, "harness|bbh:logical_deduction_five_objects|3": 0, "harness|bbh:logical_deduction_seven_objects|3": 0, "harness|bbh:logical_deduction_three_objects|3": 0, "harness|bbh:movie_recommendation|3": 0, "harness|bbh:navigate|3": 0, "harness|bbh:reasoning_about_colored_objects|3": 0, "harness|bbh:ruin_names|3": 0, "harness|bbh:salient_translation_error_detection|3": 0, "harness|bbh:snarks|3": 0, "harness|bbh:sports_understanding|3": 0, "harness|bbh:temporal_sequences|3": 0, "harness|bbh:tracking_shuffled_objects_five_objects|3": 0, "harness|bbh:tracking_shuffled_objects_seven_objects|3": 0, "harness|bbh:tracking_shuffled_objects_three_objects|3": 0 }, "config_tasks": { "harness|bbh:causal_judgment": { "name": "bbh:causal_judgment", "prompt_function": "bbh_causal_judgment", "hf_repo": "lukaemon/bbh", "hf_subset": "causal_judgement", "metric": [ "exact_match", "quasi_exact_match", "prefix_exact_match", "prefix_quasi_exact_match", "perfect_exact_match" ], "hf_avail_splits": [ "test" ], "evaluation_splits": [ "test" ], "few_shots_split": null, "few_shots_select": null, "generation_size": 20, "stop_sequence": [ "", "Q:", "\n\n" ], "output_regex": null, "frozen": false, "suite": [ "harness" ], "original_num_docs": 187, "effective_num_docs": 187, "trust_dataset": true, "must_remove_duplicate_docs": null }, "harness|bbh:date_understanding": { "name": "bbh:date_understanding", "prompt_function": "bbh_date_understanding", "hf_repo": "lukaemon/bbh", "hf_subset": "date_understanding", "metric": [ "exact_match", "quasi_exact_match", "prefix_exact_match", "prefix_quasi_exact_match", "perfect_exact_match" ], "hf_avail_splits": [ "test" ], "evaluation_splits": [ "test" ], "few_shots_split": null, "few_shots_select": null, "generation_size": 20, "stop_sequence": [ "", "Q:", "\n\n" ], "output_regex": null, "frozen": false, "suite": [ "harness" ], "original_num_docs": 250, "effective_num_docs": 250, "trust_dataset": true, "must_remove_duplicate_docs": null }, "harness|bbh:disambiguation_qa": { "name": "bbh:disambiguation_qa", "prompt_function": "bbh_disambiguation_qa", "hf_repo": "lukaemon/bbh", "hf_subset": "disambiguation_qa", "metric": [ "exact_match", "quasi_exact_match", "prefix_exact_match", "prefix_quasi_exact_match", "perfect_exact_match" ], "hf_avail_splits": [ "test" ], "evaluation_splits": [ "test" ], "few_shots_split": null, "few_shots_select": null, "generation_size": 20, "stop_sequence": [ "", "Q:", "\n\n" ], "output_regex": null, "frozen": false, "suite": [ "harness" ], "original_num_docs": 250, "effective_num_docs": 250, "trust_dataset": true, "must_remove_duplicate_docs": null }, "harness|bbh:geometric_shapes": { "name": "bbh:geometric_shapes", "prompt_function": "bbh_geometric_shapes", "hf_repo": "lukaemon/bbh", "hf_subset": "geometric_shapes", "metric": [ "exact_match", "quasi_exact_match", "prefix_exact_match", "prefix_quasi_exact_match", "perfect_exact_match" ], "hf_avail_splits": [ "test" ], "evaluation_splits": [ "test" ], "few_shots_split": null, "few_shots_select": null, "generation_size": 20, "stop_sequence": [ "", "Q:", "\n\n" ], "output_regex": null, "frozen": false, "suite": [ "harness" ], "original_num_docs": 250, "effective_num_docs": 250, "trust_dataset": true, "must_remove_duplicate_docs": null }, "harness|bbh:logical_deduction_five_objects": { "name": "bbh:logical_deduction_five_objects", "prompt_function": "bbh_logical_deduction_five_objects", "hf_repo": "lukaemon/bbh", "hf_subset": "logical_deduction_five_objects", "metric": [ "exact_match", "quasi_exact_match", "prefix_exact_match", "prefix_quasi_exact_match", "perfect_exact_match" ], "hf_avail_splits": [ "test" ], "evaluation_splits": [ "test" ], "few_shots_split": null, "few_shots_select": null, "generation_size": 20, "stop_sequence": [ "", "Q:", "\n\n" ], "output_regex": null, "frozen": false, "suite": [ "harness" ], "original_num_docs": 250, "effective_num_docs": 250, "trust_dataset": true, "must_remove_duplicate_docs": null }, "harness|bbh:logical_deduction_seven_objects": { "name": "bbh:logical_deduction_seven_objects", "prompt_function": "bbh_logical_deduction_seven_objects", "hf_repo": "lukaemon/bbh", "hf_subset": "logical_deduction_seven_objects", "metric": [ "exact_match", "quasi_exact_match", "prefix_exact_match", "prefix_quasi_exact_match", "perfect_exact_match" ], "hf_avail_splits": [ "test" ], "evaluation_splits": [ "test" ], "few_shots_split": null, "few_shots_select": null, "generation_size": 20, "stop_sequence": [ "", "Q:", "\n\n" ], "output_regex": null, "frozen": false, "suite": [ "harness" ], "original_num_docs": 250, "effective_num_docs": 250, "trust_dataset": true, "must_remove_duplicate_docs": null }, "harness|bbh:logical_deduction_three_objects": { "name": "bbh:logical_deduction_three_objects", "prompt_function": "bbh_logical_deduction_three_objects", "hf_repo": "lukaemon/bbh", "hf_subset": "logical_deduction_three_objects", "metric": [ "exact_match", "quasi_exact_match", "prefix_exact_match", "prefix_quasi_exact_match", "perfect_exact_match" ], "hf_avail_splits": [ "test" ], "evaluation_splits": [ "test" ], "few_shots_split": null, "few_shots_select": null, "generation_size": 20, "stop_sequence": [ "", "Q:", "\n\n" ], "output_regex": null, "frozen": false, "suite": [ "harness" ], "original_num_docs": 250, "effective_num_docs": 250, "trust_dataset": true, "must_remove_duplicate_docs": null }, "harness|bbh:movie_recommendation": { "name": "bbh:movie_recommendation", "prompt_function": "bbh_movie_recommendation", "hf_repo": "lukaemon/bbh", "hf_subset": "movie_recommendation", "metric": [ "exact_match", "quasi_exact_match", "prefix_exact_match", "prefix_quasi_exact_match", "perfect_exact_match" ], "hf_avail_splits": [ "test" ], "evaluation_splits": [ "test" ], "few_shots_split": null, "few_shots_select": null, "generation_size": 20, "stop_sequence": [ "", "Q:", "\n\n" ], "output_regex": null, "frozen": false, "suite": [ "harness" ], "original_num_docs": 249, "effective_num_docs": 249, "trust_dataset": true, "must_remove_duplicate_docs": null }, "harness|bbh:navigate": { "name": "bbh:navigate", "prompt_function": "bbh_navigate", "hf_repo": "lukaemon/bbh", "hf_subset": "navigate", "metric": [ "exact_match", "quasi_exact_match", "prefix_exact_match", "prefix_quasi_exact_match", "perfect_exact_match" ], "hf_avail_splits": [ "test" ], "evaluation_splits": [ "test" ], "few_shots_split": null, "few_shots_select": null, "generation_size": 20, "stop_sequence": [ "", "Q:", "\n\n" ], "output_regex": null, "frozen": false, "suite": [ "harness" ], "original_num_docs": 250, "effective_num_docs": 250, "trust_dataset": true, "must_remove_duplicate_docs": null }, "harness|bbh:reasoning_about_colored_objects": { "name": "bbh:reasoning_about_colored_objects", "prompt_function": "bbh_reasoning_about_colored_objects", "hf_repo": "lukaemon/bbh", "hf_subset": "reasoning_about_colored_objects", "metric": [ "exact_match", "quasi_exact_match", "prefix_exact_match", "prefix_quasi_exact_match", "perfect_exact_match" ], "hf_avail_splits": [ "test" ], "evaluation_splits": [ "test" ], "few_shots_split": null, "few_shots_select": null, "generation_size": 20, "stop_sequence": [ "", "Q:", "\n\n" ], "output_regex": null, "frozen": false, "suite": [ "harness" ], "original_num_docs": 250, "effective_num_docs": 250, "trust_dataset": true, "must_remove_duplicate_docs": null }, "harness|bbh:ruin_names": { "name": "bbh:ruin_names", "prompt_function": "bbh_ruin_names", "hf_repo": "lukaemon/bbh", "hf_subset": "ruin_names", "metric": [ "exact_match", "quasi_exact_match", "prefix_exact_match", "prefix_quasi_exact_match", "perfect_exact_match" ], "hf_avail_splits": [ "test" ], "evaluation_splits": [ "test" ], "few_shots_split": null, "few_shots_select": null, "generation_size": 20, "stop_sequence": [ "", "Q:", "\n\n" ], "output_regex": null, "frozen": false, "suite": [ "harness" ], "original_num_docs": 248, "effective_num_docs": 248, "trust_dataset": true, "must_remove_duplicate_docs": null }, "harness|bbh:salient_translation_error_detection": { "name": "bbh:salient_translation_error_detection", "prompt_function": "bbh_salient_translation_error_detection", "hf_repo": "lukaemon/bbh", "hf_subset": "salient_translation_error_detection", "metric": [ "exact_match", "quasi_exact_match", "prefix_exact_match", "prefix_quasi_exact_match", "perfect_exact_match" ], "hf_avail_splits": [ "test" ], "evaluation_splits": [ "test" ], "few_shots_split": null, "few_shots_select": null, "generation_size": 20, "stop_sequence": [ "", "Q:", "\n\n" ], "output_regex": null, "frozen": false, "suite": [ "harness" ], "original_num_docs": 250, "effective_num_docs": 250, "trust_dataset": true, "must_remove_duplicate_docs": null }, "harness|bbh:snarks": { "name": "bbh:snarks", "prompt_function": "bbh_snarks", "hf_repo": "lukaemon/bbh", "hf_subset": "snarks", "metric": [ "exact_match", "quasi_exact_match", "prefix_exact_match", "prefix_quasi_exact_match", "perfect_exact_match" ], "hf_avail_splits": [ "test" ], "evaluation_splits": [ "test" ], "few_shots_split": null, "few_shots_select": null, "generation_size": 20, "stop_sequence": [ "", "Q:", "\n\n" ], "output_regex": null, "frozen": false, "suite": [ "harness" ], "original_num_docs": 178, "effective_num_docs": 178, "trust_dataset": true, "must_remove_duplicate_docs": null }, "harness|bbh:sports_understanding": { "name": "bbh:sports_understanding", "prompt_function": "bbh_sports_understanding", "hf_repo": "lukaemon/bbh", "hf_subset": "sports_understanding", "metric": [ "exact_match", "quasi_exact_match", "prefix_exact_match", "prefix_quasi_exact_match", "perfect_exact_match" ], "hf_avail_splits": [ "test" ], "evaluation_splits": [ "test" ], "few_shots_split": null, "few_shots_select": null, "generation_size": 20, "stop_sequence": [ "", "Q:", "\n\n" ], "output_regex": null, "frozen": false, "suite": [ "harness" ], "original_num_docs": 250, "effective_num_docs": 250, "trust_dataset": true, "must_remove_duplicate_docs": null }, "harness|bbh:temporal_sequences": { "name": "bbh:temporal_sequences", "prompt_function": "bbh_temporal_sequences", "hf_repo": "lukaemon/bbh", "hf_subset": "temporal_sequences", "metric": [ "exact_match", "quasi_exact_match", "prefix_exact_match", "prefix_quasi_exact_match", "perfect_exact_match" ], "hf_avail_splits": [ "test" ], "evaluation_splits": [ "test" ], "few_shots_split": null, "few_shots_select": null, "generation_size": 20, "stop_sequence": [ "", "Q:", "\n\n" ], "output_regex": null, "frozen": false, "suite": [ "harness" ], "original_num_docs": 250, "effective_num_docs": 250, "trust_dataset": true, "must_remove_duplicate_docs": null }, "harness|bbh:tracking_shuffled_objects_five_objects": { "name": "bbh:tracking_shuffled_objects_five_objects", "prompt_function": "bbh_tracking_shuffled_objects_five_objects", "hf_repo": "lukaemon/bbh", "hf_subset": "tracking_shuffled_objects_five_objects", "metric": [ "exact_match", "quasi_exact_match", "prefix_exact_match", "prefix_quasi_exact_match", "perfect_exact_match" ], "hf_avail_splits": [ "test" ], "evaluation_splits": [ "test" ], "few_shots_split": null, "few_shots_select": null, "generation_size": 20, "stop_sequence": [ "", "Q:", "\n\n" ], "output_regex": null, "frozen": false, "suite": [ "harness" ], "original_num_docs": 250, "effective_num_docs": 250, "trust_dataset": true, "must_remove_duplicate_docs": null }, "harness|bbh:tracking_shuffled_objects_seven_objects": { "name": "bbh:tracking_shuffled_objects_seven_objects", "prompt_function": "bbh_tracking_shuffled_objects_seven_objects", "hf_repo": "lukaemon/bbh", "hf_subset": "tracking_shuffled_objects_seven_objects", "metric": [ "exact_match", "quasi_exact_match", "prefix_exact_match", "prefix_quasi_exact_match", "perfect_exact_match" ], "hf_avail_splits": [ "test" ], "evaluation_splits": [ "test" ], "few_shots_split": null, "few_shots_select": null, "generation_size": 20, "stop_sequence": [ "", "Q:", "\n\n" ], "output_regex": null, "frozen": false, "suite": [ "harness" ], "original_num_docs": 250, "effective_num_docs": 250, "trust_dataset": true, "must_remove_duplicate_docs": null }, "harness|bbh:tracking_shuffled_objects_three_objects": { "name": "bbh:tracking_shuffled_objects_three_objects", "prompt_function": "bbh_tracking_shuffled_objects_three_objects", "hf_repo": "lukaemon/bbh", "hf_subset": "tracking_shuffled_objects_three_objects", "metric": [ "exact_match", "quasi_exact_match", "prefix_exact_match", "prefix_quasi_exact_match", "perfect_exact_match" ], "hf_avail_splits": [ "test" ], "evaluation_splits": [ "test" ], "few_shots_split": null, "few_shots_select": null, "generation_size": 20, "stop_sequence": [ "", "Q:", "\n\n" ], "output_regex": null, "frozen": false, "suite": [ "harness" ], "original_num_docs": 250, "effective_num_docs": 250, "trust_dataset": true, "must_remove_duplicate_docs": null } }, "summary_tasks": { "harness|bbh:causal_judgment|3": { "hashes": { "hash_examples": "63218f5ae055ab2b", "hash_full_prompts": "148a4c45a8d2b858", "hash_input_tokens": "41f3903cf7efd5be", "hash_cont_tokens": "e93d8d76d4dddd8d" }, "truncated": 187, "non_truncated": 0, "padded": 0, "non_padded": 187, "effective_few_shots": 3.0, "num_truncated_few_shots": 0 }, "harness|bbh:date_understanding|3": { "hashes": { "hash_examples": "f145c7a06def3c8e", "hash_full_prompts": "e79a3237877b106e", "hash_input_tokens": "ae3f6744fd7add8b", "hash_cont_tokens": "8c6d71b9a9c752bb" }, "truncated": 250, "non_truncated": 0, "padded": 0, "non_padded": 250, "effective_few_shots": 3.0, "num_truncated_few_shots": 0 }, "harness|bbh:disambiguation_qa|3": { "hashes": { "hash_examples": "19677fd1773f7eb9", "hash_full_prompts": "9458fa1926b438bb", "hash_input_tokens": "a9ea8e42e7ca4f0f", "hash_cont_tokens": "7db54489d9d427f8" }, "truncated": 250, "non_truncated": 0, "padded": 0, "non_padded": 250, "effective_few_shots": 3.0, "num_truncated_few_shots": 0 }, "harness|bbh:geometric_shapes|3": { "hashes": { "hash_examples": "76c7b11a13cc72a9", "hash_full_prompts": "78ead1e22de562a8", "hash_input_tokens": "ded49f2dd58c24f6", "hash_cont_tokens": "db89ed479d075689" }, "truncated": 250, "non_truncated": 0, "padded": 0, "non_padded": 250, "effective_few_shots": 3.0, "num_truncated_few_shots": 0 }, "harness|bbh:logical_deduction_five_objects|3": { "hashes": { "hash_examples": "0e958c856332a745", "hash_full_prompts": "0d7cff0e511b49e7", "hash_input_tokens": "d16bae7ef2034333", "hash_cont_tokens": "1f4919dffa3a134c" }, "truncated": 250, "non_truncated": 0, "padded": 0, "non_padded": 250, "effective_few_shots": 3.0, "num_truncated_few_shots": 0 }, "harness|bbh:logical_deduction_seven_objects|3": { "hashes": { "hash_examples": "ab9de25a5eb40d09", "hash_full_prompts": "db7b7b19919ef4a7", "hash_input_tokens": "8b1a10d2204d99c3", "hash_cont_tokens": "ff391bdb10cfe252" }, "truncated": 250, "non_truncated": 0, "padded": 0, "non_padded": 250, "effective_few_shots": 3.0, "num_truncated_few_shots": 0 }, "harness|bbh:logical_deduction_three_objects|3": { "hashes": { "hash_examples": "3c6bf52517714218", "hash_full_prompts": "fd6a5580415c1e21", "hash_input_tokens": "d48e4b55d3c0ab6c", "hash_cont_tokens": "96770fd477911169" }, "truncated": 250, "non_truncated": 0, "padded": 0, "non_padded": 250, "effective_few_shots": 3.0, "num_truncated_few_shots": 0 }, "harness|bbh:movie_recommendation|3": { "hashes": { "hash_examples": "2d9dc4975935d31a", "hash_full_prompts": "b7c9fdf1a2ad8106", "hash_input_tokens": "019f6011d8ebb7b4", "hash_cont_tokens": "c65ea0feb714d001" }, "truncated": 249, "non_truncated": 0, "padded": 0, "non_padded": 249, "effective_few_shots": 3.0, "num_truncated_few_shots": 0 }, "harness|bbh:navigate|3": { "hashes": { "hash_examples": "ba91dcdb9a064255", "hash_full_prompts": "35aa68650803f91c", "hash_input_tokens": "fbf7bbc927857899", "hash_cont_tokens": "397518cdac8ba301" }, "truncated": 250, "non_truncated": 0, "padded": 0, "non_padded": 250, "effective_few_shots": 3.0, "num_truncated_few_shots": 0 }, "harness|bbh:reasoning_about_colored_objects|3": { "hashes": { "hash_examples": "a6ba328c4c3385d2", "hash_full_prompts": "954618143d9d5c6d", "hash_input_tokens": "eab386685d65e6d7", "hash_cont_tokens": "c9511762801670a8" }, "truncated": 250, "non_truncated": 0, "padded": 0, "non_padded": 250, "effective_few_shots": 3.0, "num_truncated_few_shots": 0 }, "harness|bbh:ruin_names|3": { "hashes": { "hash_examples": "2ef28d5f2d4fdd25", "hash_full_prompts": "fd807f4380c14312", "hash_input_tokens": "f17c7a83f10826d5", "hash_cont_tokens": "3b0acadb3bb8f058" }, "truncated": 248, "non_truncated": 0, "padded": 0, "non_padded": 248, "effective_few_shots": 3.0, "num_truncated_few_shots": 0 }, "harness|bbh:salient_translation_error_detection|3": { "hashes": { "hash_examples": "c13f25ec8ffed496", "hash_full_prompts": "49ab3bc1ed62613f", "hash_input_tokens": "4ecdc0361737b984", "hash_cont_tokens": "eb8a1f488e70e53c" }, "truncated": 250, "non_truncated": 0, "padded": 0, "non_padded": 250, "effective_few_shots": 3.0, "num_truncated_few_shots": 0 }, "harness|bbh:snarks|3": { "hashes": { "hash_examples": "5f6db7bff7f6f22e", "hash_full_prompts": "fa5c1ca26f4a8d48", "hash_input_tokens": "e0e9ee6065d02980", "hash_cont_tokens": "9a1668d8c3b79a7a" }, "truncated": 178, "non_truncated": 0, "padded": 0, "non_padded": 178, "effective_few_shots": 3.0, "num_truncated_few_shots": 0 }, "harness|bbh:sports_understanding|3": { "hashes": { "hash_examples": "042afbe5d9c1f02d", "hash_full_prompts": "607b29401b4907ec", "hash_input_tokens": "e2868a0e92e68765", "hash_cont_tokens": "46b51525fc124f66" }, "truncated": 250, "non_truncated": 0, "padded": 0, "non_padded": 250, "effective_few_shots": 3.0, "num_truncated_few_shots": 0 }, "harness|bbh:temporal_sequences|3": { "hashes": { "hash_examples": "803a05f352eb6afc", "hash_full_prompts": "faab4b5e14b9304e", "hash_input_tokens": "26a0ab6e389383c3", "hash_cont_tokens": "1977d5de1ecb2dfc" }, "truncated": 250, "non_truncated": 0, "padded": 0, "non_padded": 250, "effective_few_shots": 3.0, "num_truncated_few_shots": 0 }, "harness|bbh:tracking_shuffled_objects_five_objects|3": { "hashes": { "hash_examples": "2bbac6db7ab0d527", "hash_full_prompts": "7e689cfb3916666f", "hash_input_tokens": "d0e6770bedbf925a", "hash_cont_tokens": "ce9cfa52d8c85406" }, "truncated": 250, "non_truncated": 0, "padded": 0, "non_padded": 250, "effective_few_shots": 3.0, "num_truncated_few_shots": 0 }, "harness|bbh:tracking_shuffled_objects_seven_objects|3": { "hashes": { "hash_examples": "845caf093ac2b58c", "hash_full_prompts": "a80a61e259878fa0", "hash_input_tokens": "9b1288a55911b79d", "hash_cont_tokens": "700a13bfc0ce35fc" }, "truncated": 250, "non_truncated": 0, "padded": 0, "non_padded": 250, "effective_few_shots": 3.0, "num_truncated_few_shots": 0 }, "harness|bbh:tracking_shuffled_objects_three_objects|3": { "hashes": { "hash_examples": "9004f14d5a32b9a8", "hash_full_prompts": "fc66cf32f54cd46f", "hash_input_tokens": "1328545243e4b107", "hash_cont_tokens": "e4bffb2467fc2a79" }, "truncated": 250, "non_truncated": 0, "padded": 0, "non_padded": 250, "effective_few_shots": 3.0, "num_truncated_few_shots": 0 } }, "summary_general": { "hashes": { "hash_examples": "4ff1e3dc5703575d", "hash_full_prompts": "3758756e616fd780", "hash_input_tokens": "3ee0dfd31c556ab0", "hash_cont_tokens": "84a9b5a4627ad8f3" }, "truncated": 4362, "non_truncated": 0, "padded": 0, "non_padded": 4362, "num_truncated_few_shots": 0 } }