{ "config_general": { "lighteval_sha": "?", "num_fewshot_seeds": 1, "override_batch_size": 1, "max_samples": null, "job_id": "", "start_time": 5122577.949751535, "end_time": 5122918.221141459, "total_evaluation_time_secondes": "340.2713899239898", "model_name": "HuggingFaceH4/zephyr-7b-beta", "model_sha": "b70e0c9a2d9e14bd1e812d3c398e5f313e93b473", "model_dtype": "torch.bfloat16", "model_size": "13.99 GB", "config": null }, "results": { "harness|bbh:causal_judgment|3": { "em": 0.15508021390374332, "em_stderr": 0.026541744475095306, "qem": 0.15508021390374332, "qem_stderr": 0.026541744475095306, "pem": 0.5614973262032086, "pem_stderr": 0.03638341809400995, "pqem": 0.5614973262032086, "pqem_stderr": 0.03638341809400995, "perfect_em": 0.15508021390374332, "perfect_em_stderr": 0.026541744475095306 }, "harness|bbh:date_understanding|3": { "em": 0.468, "em_stderr": 0.03162125257572551, "qem": 0.468, "qem_stderr": 0.03162125257572551, "pem": 0.476, "pem_stderr": 0.03164968895968782, "pqem": 0.596, "pqem_stderr": 0.03109668818482529, "perfect_em": 0.468, "perfect_em_stderr": 0.03162125257572551 }, "harness|bbh:disambiguation_qa|3": { "em": 0.62, "em_stderr": 0.030760116042626046, "qem": 0.62, "qem_stderr": 0.030760116042626046, "pem": 0.62, "pem_stderr": 0.030760116042626046, "pqem": 0.716, "pqem_stderr": 0.02857695873043741, "perfect_em": 0.62, "perfect_em_stderr": 0.030760116042626046 }, "harness|bbh:geometric_shapes|3": { "em": 0.14, "em_stderr": 0.021989409645240265, "qem": 0.14, "qem_stderr": 0.021989409645240265, "pem": 0.212, "pem_stderr": 0.025901884690541162, "pqem": 0.212, "pqem_stderr": 0.025901884690541162, "perfect_em": 0.14, "perfect_em_stderr": 0.021989409645240265 }, "harness|bbh:logical_deduction_five_objects|3": { "em": 0.36, "em_stderr": 0.03041876402517499, "qem": 0.36, "qem_stderr": 0.03041876402517499, "pem": 0.364, "pem_stderr": 0.030491555220405562, "pqem": 0.5, "pqem_stderr": 0.031686212526223896, "perfect_em": 0.36, "perfect_em_stderr": 0.03041876402517499 }, "harness|bbh:logical_deduction_seven_objects|3": { "em": 0.28, "em_stderr": 0.02845414827783232, "qem": 0.28, "qem_stderr": 0.02845414827783232, "pem": 0.284, "pem_stderr": 0.028576958730437408, "pqem": 0.42, "pqem_stderr": 0.03127799950463662, "perfect_em": 0.28, "perfect_em_stderr": 0.02845414827783232 }, "harness|bbh:logical_deduction_three_objects|3": { "em": 0.544, "em_stderr": 0.03156328506121339, "qem": 0.544, "qem_stderr": 0.03156328506121339, "pem": 0.548, "pem_stderr": 0.03153986449255662, "pqem": 0.784, "pqem_stderr": 0.026078657663732716, "perfect_em": 0.544, "perfect_em_stderr": 0.03156328506121339 }, "harness|bbh:movie_recommendation|3": { "em": 0.5662650602409639, "em_stderr": 0.03146996403326648, "qem": 0.5662650602409639, "qem_stderr": 0.03146996403326648, "pem": 0.6345381526104418, "pem_stderr": 0.030579054441446426, "pqem": 0.7670682730923695, "pqem_stderr": 0.02684142031987394, "perfect_em": 0.5662650602409639, "perfect_em_stderr": 0.03146996403326648 }, "harness|bbh:navigate|3": { "em": 0.596, "em_stderr": 0.031096688184825298, "qem": 0.596, "qem_stderr": 0.031096688184825298, "pem": 0.6, "pem_stderr": 0.031046021028253257, "pqem": 0.6, "pqem_stderr": 0.031046021028253257, "perfect_em": 0.596, "perfect_em_stderr": 0.031096688184825298 }, "harness|bbh:reasoning_about_colored_objects|3": { "em": 0.252, "em_stderr": 0.027513851933031352, "qem": 0.252, "qem_stderr": 0.027513851933031352, "pem": 0.308, "pem_stderr": 0.029256928606501864, "pqem": 0.452, "pqem_stderr": 0.03153986449255662, "perfect_em": 0.252, "perfect_em_stderr": 0.027513851933031352 }, "harness|bbh:ruin_names|3": { "em": 0.3064516129032258, "em_stderr": 0.02933397125939671, "qem": 0.3064516129032258, "qem_stderr": 0.02933397125939671, "pem": 0.3548387096774194, "pem_stderr": 0.030443961913180523, "pqem": 0.5564516129032258, "pqem_stderr": 0.03161081756372432, "perfect_em": 0.3064516129032258, "perfect_em_stderr": 0.02933397125939671 }, "harness|bbh:salient_translation_error_detection|3": { "em": 0.384, "em_stderr": 0.030821679117375385, "qem": 0.384, "qem_stderr": 0.030821679117375385, "pem": 0.384, "pem_stderr": 0.030821679117375385, "pqem": 0.524, "pqem_stderr": 0.03164968895968782, "perfect_em": 0.384, "perfect_em_stderr": 0.030821679117375385 }, "harness|bbh:snarks|3": { "em": 0.5112359550561798, "em_stderr": 0.03757281091983853, "qem": 0.5112359550561798, "qem_stderr": 0.03757281091983853, "pem": 0.550561797752809, "pem_stderr": 0.037389649660569645, "pqem": 0.6853932584269663, "pqem_stderr": 0.03490336455459108, "perfect_em": 0.5112359550561798, "perfect_em_stderr": 0.03757281091983853 }, "harness|bbh:sports_understanding|3": { "em": 0.36, "em_stderr": 0.030418764025174985, "qem": 0.36, "qem_stderr": 0.030418764025174985, "pem": 0.784, "pem_stderr": 0.02607865766373272, "pqem": 0.784, "pqem_stderr": 0.02607865766373272, "perfect_em": 0.36, "perfect_em_stderr": 0.030418764025174985 }, "harness|bbh:temporal_sequences|3": { "em": 0.152, "em_stderr": 0.022752024491765464, "qem": 0.152, "qem_stderr": 0.022752024491765464, "pem": 0.156, "pem_stderr": 0.022995023034068748, "pqem": 0.428, "pqem_stderr": 0.03135596892377261, "perfect_em": 0.152, "perfect_em_stderr": 0.022752024491765464 }, "harness|bbh:tracking_shuffled_objects_five_objects|3": { "em": 0.148, "em_stderr": 0.022503547243806144, "qem": 0.148, "qem_stderr": 0.022503547243806144, "pem": 0.148, "pem_stderr": 0.022503547243806144, "pqem": 0.348, "pqem_stderr": 0.030186568464511707, "perfect_em": 0.148, "perfect_em_stderr": 0.022503547243806144 }, "harness|bbh:tracking_shuffled_objects_seven_objects|3": { "em": 0.108, "em_stderr": 0.019669559381568755, "qem": 0.108, "qem_stderr": 0.019669559381568755, "pem": 0.116, "pem_stderr": 0.020293429803083844, "pqem": 0.26, "pqem_stderr": 0.027797315752644308, "perfect_em": 0.108, "perfect_em_stderr": 0.019669559381568755 }, "harness|bbh:tracking_shuffled_objects_three_objects|3": { "em": 0.34, "em_stderr": 0.030020073605457904, "qem": 0.34, "qem_stderr": 0.030020073605457904, "pem": 0.34, "pem_stderr": 0.030020073605457904, "pqem": 0.652, "pqem_stderr": 0.030186568464511683, "perfect_em": 0.34, "perfect_em_stderr": 0.030020073605457904 }, "harness|bbh:_average|3": { "em": 0.3495018245613396, "em_stderr": 0.028584536349911933, "qem": 0.3495018245613396, "qem_stderr": 0.028584536349911933, "pem": 0.41341311034688205, "pem_stderr": 0.029262861797096727, "pqem": 0.5470228039236539, "pqem_stderr": 0.030233226421237065, "perfect_em": 0.3495018245613396, "perfect_em_stderr": 0.028584536349911933 } }, "versions": { "harness|bbh:causal_judgment|3": 0, "harness|bbh:date_understanding|3": 0, "harness|bbh:disambiguation_qa|3": 0, "harness|bbh:geometric_shapes|3": 0, "harness|bbh:logical_deduction_five_objects|3": 0, "harness|bbh:logical_deduction_seven_objects|3": 0, "harness|bbh:logical_deduction_three_objects|3": 0, "harness|bbh:movie_recommendation|3": 0, "harness|bbh:navigate|3": 0, "harness|bbh:reasoning_about_colored_objects|3": 0, "harness|bbh:ruin_names|3": 0, "harness|bbh:salient_translation_error_detection|3": 0, "harness|bbh:snarks|3": 0, "harness|bbh:sports_understanding|3": 0, "harness|bbh:temporal_sequences|3": 0, "harness|bbh:tracking_shuffled_objects_five_objects|3": 0, "harness|bbh:tracking_shuffled_objects_seven_objects|3": 0, "harness|bbh:tracking_shuffled_objects_three_objects|3": 0 }, "config_tasks": { "harness|bbh:causal_judgment": { "name": "bbh:causal_judgment", "prompt_function": "bbh_causal_judgment", "hf_repo": "lukaemon/bbh", "hf_subset": "causal_judgement", "metric": [ "exact_match", "quasi_exact_match", "prefix_exact_match", "prefix_quasi_exact_match", "perfect_exact_match" ], "hf_avail_splits": [ "test" ], "evaluation_splits": [ "test" ], "few_shots_split": null, "few_shots_select": null, "generation_size": 20, "stop_sequence": [ "", "Q:", "\n\n" ], "output_regex": null, "frozen": false, "suite": [ "harness" ], "original_num_docs": 187, "effective_num_docs": 187, "trust_dataset": true, "must_remove_duplicate_docs": null }, "harness|bbh:date_understanding": { "name": "bbh:date_understanding", "prompt_function": "bbh_date_understanding", "hf_repo": "lukaemon/bbh", "hf_subset": "date_understanding", "metric": [ "exact_match", "quasi_exact_match", "prefix_exact_match", "prefix_quasi_exact_match", "perfect_exact_match" ], "hf_avail_splits": [ "test" ], "evaluation_splits": [ "test" ], "few_shots_split": null, "few_shots_select": null, "generation_size": 20, "stop_sequence": [ "", "Q:", "\n\n" ], "output_regex": null, "frozen": false, "suite": [ "harness" ], "original_num_docs": 250, "effective_num_docs": 250, "trust_dataset": true, "must_remove_duplicate_docs": null }, "harness|bbh:disambiguation_qa": { "name": "bbh:disambiguation_qa", "prompt_function": "bbh_disambiguation_qa", "hf_repo": "lukaemon/bbh", "hf_subset": "disambiguation_qa", "metric": [ "exact_match", "quasi_exact_match", "prefix_exact_match", "prefix_quasi_exact_match", "perfect_exact_match" ], "hf_avail_splits": [ "test" ], "evaluation_splits": [ "test" ], "few_shots_split": null, "few_shots_select": null, "generation_size": 20, "stop_sequence": [ "", "Q:", "\n\n" ], "output_regex": null, "frozen": false, "suite": [ "harness" ], "original_num_docs": 250, "effective_num_docs": 250, "trust_dataset": true, "must_remove_duplicate_docs": null }, "harness|bbh:geometric_shapes": { "name": "bbh:geometric_shapes", "prompt_function": "bbh_geometric_shapes", "hf_repo": "lukaemon/bbh", "hf_subset": "geometric_shapes", "metric": [ "exact_match", "quasi_exact_match", "prefix_exact_match", "prefix_quasi_exact_match", "perfect_exact_match" ], "hf_avail_splits": [ "test" ], "evaluation_splits": [ "test" ], "few_shots_split": null, "few_shots_select": null, "generation_size": 20, "stop_sequence": [ "", "Q:", "\n\n" ], "output_regex": null, "frozen": false, "suite": [ "harness" ], "original_num_docs": 250, "effective_num_docs": 250, "trust_dataset": true, "must_remove_duplicate_docs": null }, "harness|bbh:logical_deduction_five_objects": { "name": "bbh:logical_deduction_five_objects", "prompt_function": "bbh_logical_deduction_five_objects", "hf_repo": "lukaemon/bbh", "hf_subset": "logical_deduction_five_objects", "metric": [ "exact_match", "quasi_exact_match", "prefix_exact_match", "prefix_quasi_exact_match", "perfect_exact_match" ], "hf_avail_splits": [ "test" ], "evaluation_splits": [ "test" ], "few_shots_split": null, "few_shots_select": null, "generation_size": 20, "stop_sequence": [ "", "Q:", "\n\n" ], "output_regex": null, "frozen": false, "suite": [ "harness" ], "original_num_docs": 250, "effective_num_docs": 250, "trust_dataset": true, "must_remove_duplicate_docs": null }, "harness|bbh:logical_deduction_seven_objects": { "name": "bbh:logical_deduction_seven_objects", "prompt_function": "bbh_logical_deduction_seven_objects", "hf_repo": "lukaemon/bbh", "hf_subset": "logical_deduction_seven_objects", "metric": [ "exact_match", "quasi_exact_match", "prefix_exact_match", "prefix_quasi_exact_match", "perfect_exact_match" ], "hf_avail_splits": [ "test" ], "evaluation_splits": [ "test" ], "few_shots_split": null, "few_shots_select": null, "generation_size": 20, "stop_sequence": [ "", "Q:", "\n\n" ], "output_regex": null, "frozen": false, "suite": [ "harness" ], "original_num_docs": 250, "effective_num_docs": 250, "trust_dataset": true, "must_remove_duplicate_docs": null }, "harness|bbh:logical_deduction_three_objects": { "name": "bbh:logical_deduction_three_objects", "prompt_function": "bbh_logical_deduction_three_objects", "hf_repo": "lukaemon/bbh", "hf_subset": "logical_deduction_three_objects", "metric": [ "exact_match", "quasi_exact_match", "prefix_exact_match", "prefix_quasi_exact_match", "perfect_exact_match" ], "hf_avail_splits": [ "test" ], "evaluation_splits": [ "test" ], "few_shots_split": null, "few_shots_select": null, "generation_size": 20, "stop_sequence": [ "", "Q:", "\n\n" ], "output_regex": null, "frozen": false, "suite": [ "harness" ], "original_num_docs": 250, "effective_num_docs": 250, "trust_dataset": true, "must_remove_duplicate_docs": null }, "harness|bbh:movie_recommendation": { "name": "bbh:movie_recommendation", "prompt_function": "bbh_movie_recommendation", "hf_repo": "lukaemon/bbh", "hf_subset": "movie_recommendation", "metric": [ "exact_match", "quasi_exact_match", "prefix_exact_match", "prefix_quasi_exact_match", "perfect_exact_match" ], "hf_avail_splits": [ "test" ], "evaluation_splits": [ "test" ], "few_shots_split": null, "few_shots_select": null, "generation_size": 20, "stop_sequence": [ "", "Q:", "\n\n" ], "output_regex": null, "frozen": false, "suite": [ "harness" ], "original_num_docs": 249, "effective_num_docs": 249, "trust_dataset": true, "must_remove_duplicate_docs": null }, "harness|bbh:navigate": { "name": "bbh:navigate", "prompt_function": "bbh_navigate", "hf_repo": "lukaemon/bbh", "hf_subset": "navigate", "metric": [ "exact_match", "quasi_exact_match", "prefix_exact_match", "prefix_quasi_exact_match", "perfect_exact_match" ], "hf_avail_splits": [ "test" ], "evaluation_splits": [ "test" ], "few_shots_split": null, "few_shots_select": null, "generation_size": 20, "stop_sequence": [ "", "Q:", "\n\n" ], "output_regex": null, "frozen": false, "suite": [ "harness" ], "original_num_docs": 250, "effective_num_docs": 250, "trust_dataset": true, "must_remove_duplicate_docs": null }, "harness|bbh:reasoning_about_colored_objects": { "name": "bbh:reasoning_about_colored_objects", "prompt_function": "bbh_reasoning_about_colored_objects", "hf_repo": "lukaemon/bbh", "hf_subset": "reasoning_about_colored_objects", "metric": [ "exact_match", "quasi_exact_match", "prefix_exact_match", "prefix_quasi_exact_match", "perfect_exact_match" ], "hf_avail_splits": [ "test" ], "evaluation_splits": [ "test" ], "few_shots_split": null, "few_shots_select": null, "generation_size": 20, "stop_sequence": [ "", "Q:", "\n\n" ], "output_regex": null, "frozen": false, "suite": [ "harness" ], "original_num_docs": 250, "effective_num_docs": 250, "trust_dataset": true, "must_remove_duplicate_docs": null }, "harness|bbh:ruin_names": { "name": "bbh:ruin_names", "prompt_function": "bbh_ruin_names", "hf_repo": "lukaemon/bbh", "hf_subset": "ruin_names", "metric": [ "exact_match", "quasi_exact_match", "prefix_exact_match", "prefix_quasi_exact_match", "perfect_exact_match" ], "hf_avail_splits": [ "test" ], "evaluation_splits": [ "test" ], "few_shots_split": null, "few_shots_select": null, "generation_size": 20, "stop_sequence": [ "", "Q:", "\n\n" ], "output_regex": null, "frozen": false, "suite": [ "harness" ], "original_num_docs": 248, "effective_num_docs": 248, "trust_dataset": true, "must_remove_duplicate_docs": null }, "harness|bbh:salient_translation_error_detection": { "name": "bbh:salient_translation_error_detection", "prompt_function": "bbh_salient_translation_error_detection", "hf_repo": "lukaemon/bbh", "hf_subset": "salient_translation_error_detection", "metric": [ "exact_match", "quasi_exact_match", "prefix_exact_match", "prefix_quasi_exact_match", "perfect_exact_match" ], "hf_avail_splits": [ "test" ], "evaluation_splits": [ "test" ], "few_shots_split": null, "few_shots_select": null, "generation_size": 20, "stop_sequence": [ "", "Q:", "\n\n" ], "output_regex": null, "frozen": false, "suite": [ "harness" ], "original_num_docs": 250, "effective_num_docs": 250, "trust_dataset": true, "must_remove_duplicate_docs": null }, "harness|bbh:snarks": { "name": "bbh:snarks", "prompt_function": "bbh_snarks", "hf_repo": "lukaemon/bbh", "hf_subset": "snarks", "metric": [ "exact_match", "quasi_exact_match", "prefix_exact_match", "prefix_quasi_exact_match", "perfect_exact_match" ], "hf_avail_splits": [ "test" ], "evaluation_splits": [ "test" ], "few_shots_split": null, "few_shots_select": null, "generation_size": 20, "stop_sequence": [ "", "Q:", "\n\n" ], "output_regex": null, "frozen": false, "suite": [ "harness" ], "original_num_docs": 178, "effective_num_docs": 178, "trust_dataset": true, "must_remove_duplicate_docs": null }, "harness|bbh:sports_understanding": { "name": "bbh:sports_understanding", "prompt_function": "bbh_sports_understanding", "hf_repo": "lukaemon/bbh", "hf_subset": "sports_understanding", "metric": [ "exact_match", "quasi_exact_match", "prefix_exact_match", "prefix_quasi_exact_match", "perfect_exact_match" ], "hf_avail_splits": [ "test" ], "evaluation_splits": [ "test" ], "few_shots_split": null, "few_shots_select": null, "generation_size": 20, "stop_sequence": [ "", "Q:", "\n\n" ], "output_regex": null, "frozen": false, "suite": [ "harness" ], "original_num_docs": 250, "effective_num_docs": 250, "trust_dataset": true, "must_remove_duplicate_docs": null }, "harness|bbh:temporal_sequences": { "name": "bbh:temporal_sequences", "prompt_function": "bbh_temporal_sequences", "hf_repo": "lukaemon/bbh", "hf_subset": "temporal_sequences", "metric": [ "exact_match", "quasi_exact_match", "prefix_exact_match", "prefix_quasi_exact_match", "perfect_exact_match" ], "hf_avail_splits": [ "test" ], "evaluation_splits": [ "test" ], "few_shots_split": null, "few_shots_select": null, "generation_size": 20, "stop_sequence": [ "", "Q:", "\n\n" ], "output_regex": null, "frozen": false, "suite": [ "harness" ], "original_num_docs": 250, "effective_num_docs": 250, "trust_dataset": true, "must_remove_duplicate_docs": null }, "harness|bbh:tracking_shuffled_objects_five_objects": { "name": "bbh:tracking_shuffled_objects_five_objects", "prompt_function": "bbh_tracking_shuffled_objects_five_objects", "hf_repo": "lukaemon/bbh", "hf_subset": "tracking_shuffled_objects_five_objects", "metric": [ "exact_match", "quasi_exact_match", "prefix_exact_match", "prefix_quasi_exact_match", "perfect_exact_match" ], "hf_avail_splits": [ "test" ], "evaluation_splits": [ "test" ], "few_shots_split": null, "few_shots_select": null, "generation_size": 20, "stop_sequence": [ "", "Q:", "\n\n" ], "output_regex": null, "frozen": false, "suite": [ "harness" ], "original_num_docs": 250, "effective_num_docs": 250, "trust_dataset": true, "must_remove_duplicate_docs": null }, "harness|bbh:tracking_shuffled_objects_seven_objects": { "name": "bbh:tracking_shuffled_objects_seven_objects", "prompt_function": "bbh_tracking_shuffled_objects_seven_objects", "hf_repo": "lukaemon/bbh", "hf_subset": "tracking_shuffled_objects_seven_objects", "metric": [ "exact_match", "quasi_exact_match", "prefix_exact_match", "prefix_quasi_exact_match", "perfect_exact_match" ], "hf_avail_splits": [ "test" ], "evaluation_splits": [ "test" ], "few_shots_split": null, "few_shots_select": null, "generation_size": 20, "stop_sequence": [ "", "Q:", "\n\n" ], "output_regex": null, "frozen": false, "suite": [ "harness" ], "original_num_docs": 250, "effective_num_docs": 250, "trust_dataset": true, "must_remove_duplicate_docs": null }, "harness|bbh:tracking_shuffled_objects_three_objects": { "name": "bbh:tracking_shuffled_objects_three_objects", "prompt_function": "bbh_tracking_shuffled_objects_three_objects", "hf_repo": "lukaemon/bbh", "hf_subset": "tracking_shuffled_objects_three_objects", "metric": [ "exact_match", "quasi_exact_match", "prefix_exact_match", "prefix_quasi_exact_match", "perfect_exact_match" ], "hf_avail_splits": [ "test" ], "evaluation_splits": [ "test" ], "few_shots_split": null, "few_shots_select": null, "generation_size": 20, "stop_sequence": [ "", "Q:", "\n\n" ], "output_regex": null, "frozen": false, "suite": [ "harness" ], "original_num_docs": 250, "effective_num_docs": 250, "trust_dataset": true, "must_remove_duplicate_docs": null } }, "summary_tasks": { "harness|bbh:causal_judgment|3": { "hashes": { "hash_examples": "63218f5ae055ab2b", "hash_full_prompts": "fa8168f39a475fb0", "hash_input_tokens": "787f75e06fd43c0d", "hash_cont_tokens": "02b5f6876479ee6f" }, "truncated": 187, "non_truncated": 0, "padded": 0, "non_padded": 187, "effective_few_shots": 3.0, "num_truncated_few_shots": 0 }, "harness|bbh:date_understanding|3": { "hashes": { "hash_examples": "f145c7a06def3c8e", "hash_full_prompts": "2cceeea606638d49", "hash_input_tokens": "10c13d6fb8af7c22", "hash_cont_tokens": "da80e7905b3d4564" }, "truncated": 250, "non_truncated": 0, "padded": 0, "non_padded": 250, "effective_few_shots": 3.0, "num_truncated_few_shots": 0 }, "harness|bbh:disambiguation_qa|3": { "hashes": { "hash_examples": "19677fd1773f7eb9", "hash_full_prompts": "d8f1ba70c22ae578", "hash_input_tokens": "c21a88707f480cab", "hash_cont_tokens": "6e64a77989631706" }, "truncated": 250, "non_truncated": 0, "padded": 0, "non_padded": 250, "effective_few_shots": 3.0, "num_truncated_few_shots": 0 }, "harness|bbh:geometric_shapes|3": { "hashes": { "hash_examples": "76c7b11a13cc72a9", "hash_full_prompts": "52a60ed1d0113b8b", "hash_input_tokens": "10e113b2cf3fa584", "hash_cont_tokens": "70da2b36605721bc" }, "truncated": 250, "non_truncated": 0, "padded": 0, "non_padded": 250, "effective_few_shots": 3.0, "num_truncated_few_shots": 0 }, "harness|bbh:logical_deduction_five_objects|3": { "hashes": { "hash_examples": "0e958c856332a745", "hash_full_prompts": "253aa9791c941909", "hash_input_tokens": "0bc166cab0aed76a", "hash_cont_tokens": "e0f40cb91e3668e4" }, "truncated": 250, "non_truncated": 0, "padded": 0, "non_padded": 250, "effective_few_shots": 3.0, "num_truncated_few_shots": 0 }, "harness|bbh:logical_deduction_seven_objects|3": { "hashes": { "hash_examples": "ab9de25a5eb40d09", "hash_full_prompts": "aa6117f601cd268e", "hash_input_tokens": "ab99c78b48e3a0bb", "hash_cont_tokens": "e15546cc120f758f" }, "truncated": 250, "non_truncated": 0, "padded": 0, "non_padded": 250, "effective_few_shots": 3.0, "num_truncated_few_shots": 0 }, "harness|bbh:logical_deduction_three_objects|3": { "hashes": { "hash_examples": "3c6bf52517714218", "hash_full_prompts": "1892b050bc7848a4", "hash_input_tokens": "a720b56aa7c52551", "hash_cont_tokens": "19ceb44a47d6a83e" }, "truncated": 250, "non_truncated": 0, "padded": 0, "non_padded": 250, "effective_few_shots": 3.0, "num_truncated_few_shots": 0 }, "harness|bbh:movie_recommendation|3": { "hashes": { "hash_examples": "2d9dc4975935d31a", "hash_full_prompts": "8e00606ed3407167", "hash_input_tokens": "c825ab1c99245a17", "hash_cont_tokens": "3db5d490f746e528" }, "truncated": 249, "non_truncated": 0, "padded": 0, "non_padded": 249, "effective_few_shots": 3.0, "num_truncated_few_shots": 0 }, "harness|bbh:navigate|3": { "hashes": { "hash_examples": "ba91dcdb9a064255", "hash_full_prompts": "8d50c5baf1df7aef", "hash_input_tokens": "f234e6b28ea1fa49", "hash_cont_tokens": "b75cef2daa7b5aa7" }, "truncated": 250, "non_truncated": 0, "padded": 0, "non_padded": 250, "effective_few_shots": 3.0, "num_truncated_few_shots": 0 }, "harness|bbh:reasoning_about_colored_objects|3": { "hashes": { "hash_examples": "a6ba328c4c3385d2", "hash_full_prompts": "3d2441a21c12a960", "hash_input_tokens": "f3b577892955aa84", "hash_cont_tokens": "3a39b267c7438e86" }, "truncated": 250, "non_truncated": 0, "padded": 0, "non_padded": 250, "effective_few_shots": 3.0, "num_truncated_few_shots": 0 }, "harness|bbh:ruin_names|3": { "hashes": { "hash_examples": "2ef28d5f2d4fdd25", "hash_full_prompts": "ba95caa786f313b1", "hash_input_tokens": "9954b30d4205604a", "hash_cont_tokens": "f0ed57d273446ff6" }, "truncated": 248, "non_truncated": 0, "padded": 0, "non_padded": 248, "effective_few_shots": 3.0, "num_truncated_few_shots": 0 }, "harness|bbh:salient_translation_error_detection|3": { "hashes": { "hash_examples": "c13f25ec8ffed496", "hash_full_prompts": "a8512d174e1cab8f", "hash_input_tokens": "3e738df24b7eddf8", "hash_cont_tokens": "303a352b96e7ada1" }, "truncated": 250, "non_truncated": 0, "padded": 0, "non_padded": 250, "effective_few_shots": 3.0, "num_truncated_few_shots": 0 }, "harness|bbh:snarks|3": { "hashes": { "hash_examples": "5f6db7bff7f6f22e", "hash_full_prompts": "ff91d81466b9041f", "hash_input_tokens": "21388b09e13d0208", "hash_cont_tokens": "f1b8acd779f9be16" }, "truncated": 178, "non_truncated": 0, "padded": 0, "non_padded": 178, "effective_few_shots": 3.0, "num_truncated_few_shots": 0 }, "harness|bbh:sports_understanding|3": { "hashes": { "hash_examples": "042afbe5d9c1f02d", "hash_full_prompts": "a59324d9eb37e0f5", "hash_input_tokens": "0ad41bb8d2290a5b", "hash_cont_tokens": "4349ad0a7c0ef88c" }, "truncated": 250, "non_truncated": 0, "padded": 0, "non_padded": 250, "effective_few_shots": 3.0, "num_truncated_few_shots": 0 }, "harness|bbh:temporal_sequences|3": { "hashes": { "hash_examples": "803a05f352eb6afc", "hash_full_prompts": "1b3971192bf481e7", "hash_input_tokens": "3051b60940ccceab", "hash_cont_tokens": "672a26a849c36b02" }, "truncated": 250, "non_truncated": 0, "padded": 0, "non_padded": 250, "effective_few_shots": 3.0, "num_truncated_few_shots": 0 }, "harness|bbh:tracking_shuffled_objects_five_objects|3": { "hashes": { "hash_examples": "2bbac6db7ab0d527", "hash_full_prompts": "7ef4567d2fcf5094", "hash_input_tokens": "b841310ee5531238", "hash_cont_tokens": "77d91b7062568db8" }, "truncated": 250, "non_truncated": 0, "padded": 0, "non_padded": 250, "effective_few_shots": 3.0, "num_truncated_few_shots": 0 }, "harness|bbh:tracking_shuffled_objects_seven_objects|3": { "hashes": { "hash_examples": "845caf093ac2b58c", "hash_full_prompts": "196a0f8712857624", "hash_input_tokens": "3e738df24b7eddf8", "hash_cont_tokens": "7451006fd3eba0cf" }, "truncated": 250, "non_truncated": 0, "padded": 0, "non_padded": 250, "effective_few_shots": 3.0, "num_truncated_few_shots": 0 }, "harness|bbh:tracking_shuffled_objects_three_objects|3": { "hashes": { "hash_examples": "9004f14d5a32b9a8", "hash_full_prompts": "592a03f0518f17b6", "hash_input_tokens": "19e0ef1dd5ae9d33", "hash_cont_tokens": "3c3725d0040edbd9" }, "truncated": 250, "non_truncated": 0, "padded": 0, "non_padded": 250, "effective_few_shots": 3.0, "num_truncated_few_shots": 0 } }, "summary_general": { "hashes": { "hash_examples": "4ff1e3dc5703575d", "hash_full_prompts": "0d80ce968d89d4ef", "hash_input_tokens": "72bda1e7aeb34786", "hash_cont_tokens": "72154389368339d0" }, "truncated": 4362, "non_truncated": 0, "padded": 0, "non_padded": 4362, "num_truncated_few_shots": 0 } }