{ "config_general": { "lighteval_sha": "?", "num_fewshot_seeds": 1, "override_batch_size": 1, "max_samples": null, "job_id": "", "start_time": 1564442.483351112, "end_time": 1564777.847840123, "total_evaluation_time_secondes": "335.36448901100084", "model_name": "google/gemma-2b-it", "model_sha": "718cb189da9c5b2e55abe86f2eeffee9b4ae0dad", "model_dtype": "torch.bfloat16", "model_size": "4.68 GB", "config": null }, "results": { "harness|bbh:causal_judgment|3": { "em": 0.3689839572192513, "em_stderr": 0.03538078548260322, "qem": 0.3689839572192513, "qem_stderr": 0.03538078548260322, "pem": 0.5240641711229946, "pem_stderr": 0.03661929361528702, "pqem": 0.5240641711229946, "pqem_stderr": 0.03661929361528702, "perfect_em": 0.1443850267379679, "perfect_em_stderr": 0.02577174317334218 }, "harness|bbh:date_understanding|3": { "em": 0.0, "em_stderr": 0.0, "qem": 0.0, "qem_stderr": 0.0, "pem": 0.276, "pem_stderr": 0.028328537274211352, "pqem": 0.348, "pqem_stderr": 0.030186568464511683, "perfect_em": 0.0, "perfect_em_stderr": 0.0 }, "harness|bbh:disambiguation_qa|3": { "em": 0.0, "em_stderr": 0.0, "qem": 0.0, "qem_stderr": 0.0, "pem": 0.396, "pem_stderr": 0.03099319785457785, "pqem": 0.544, "pqem_stderr": 0.03156328506121339, "perfect_em": 0.0, "perfect_em_stderr": 0.0 }, "harness|bbh:geometric_shapes|3": { "em": 0.124, "em_stderr": 0.02088638225867326, "qem": 0.124, "qem_stderr": 0.02088638225867326, "pem": 0.128, "pem_stderr": 0.021172081336336492, "pqem": 0.128, "pqem_stderr": 0.021172081336336492, "perfect_em": 0.124, "perfect_em_stderr": 0.02088638225867326 }, "harness|bbh:logical_deduction_five_objects|3": { "em": 0.004, "em_stderr": 0.004, "qem": 0.004, "qem_stderr": 0.004, "pem": 0.212, "pem_stderr": 0.025901884690541162, "pqem": 0.392, "pqem_stderr": 0.0309382076204012, "perfect_em": 0.004, "perfect_em_stderr": 0.004 }, "harness|bbh:logical_deduction_seven_objects|3": { "em": 0.0, "em_stderr": 0.0, "qem": 0.0, "qem_stderr": 0.0, "pem": 0.164, "pem_stderr": 0.023465261002076753, "pqem": 0.224, "pqem_stderr": 0.026421361687347905, "perfect_em": 0.0, "perfect_em_stderr": 0.0 }, "harness|bbh:logical_deduction_three_objects|3": { "em": 0.008, "em_stderr": 0.005645483676690171, "qem": 0.008, "qem_stderr": 0.005645483676690171, "pem": 0.4, "pem_stderr": 0.031046021028253244, "pqem": 0.64, "pqem_stderr": 0.030418764025174985, "perfect_em": 0.008, "perfect_em_stderr": 0.005645483676690171 }, "harness|bbh:movie_recommendation|3": { "em": 0.0, "em_stderr": 0.0, "qem": 0.0, "qem_stderr": 0.0, "pem": 0.5301204819277109, "pem_stderr": 0.03169236937317327, "pqem": 0.6024096385542169, "pqem_stderr": 0.03107692497419258, "perfect_em": 0.0, "perfect_em_stderr": 0.0 }, "harness|bbh:navigate|3": { "em": 0.116, "em_stderr": 0.02029342980308385, "qem": 0.116, "qem_stderr": 0.02029342980308385, "pem": 0.596, "pem_stderr": 0.031096688184825298, "pqem": 0.596, "pqem_stderr": 0.031096688184825298, "perfect_em": 0.02, "perfect_em_stderr": 0.008872139507342683 }, "harness|bbh:reasoning_about_colored_objects|3": { "em": 0.0, "em_stderr": 0.0, "qem": 0.0, "qem_stderr": 0.0, "pem": 0.176, "pem_stderr": 0.02413349752545711, "pqem": 0.312, "pqem_stderr": 0.029361067575219817, "perfect_em": 0.0, "perfect_em_stderr": 0.0 }, "harness|bbh:ruin_names|3": { "em": 0.0, "em_stderr": 0.0, "qem": 0.0, "qem_stderr": 0.0, "pem": 0.2056451612903226, "pem_stderr": 0.025716866102588432, "pqem": 0.46774193548387094, "pqem_stderr": 0.03174795841398569, "perfect_em": 0.0, "perfect_em_stderr": 0.0 }, "harness|bbh:salient_translation_error_detection|3": { "em": 0.0, "em_stderr": 0.0, "qem": 0.0, "qem_stderr": 0.0, "pem": 0.224, "pem_stderr": 0.026421361687347905, "pqem": 0.364, "pqem_stderr": 0.03049155522040554, "perfect_em": 0.0, "perfect_em_stderr": 0.0 }, "harness|bbh:snarks|3": { "em": 0.033707865168539325, "em_stderr": 0.01356541902000238, "qem": 0.033707865168539325, "qem_stderr": 0.01356541902000238, "pem": 0.449438202247191, "pem_stderr": 0.03738964966056965, "pqem": 0.5617977528089888, "pqem_stderr": 0.03729414592947276, "perfect_em": 0.033707865168539325, "perfect_em_stderr": 0.01356541902000238 }, "harness|bbh:sports_understanding|3": { "em": 0.316, "em_stderr": 0.029462657598578676, "qem": 0.316, "qem_stderr": 0.029462657598578676, "pem": 0.468, "pem_stderr": 0.031621252575725504, "pqem": 0.468, "pqem_stderr": 0.031621252575725504, "perfect_em": 0.296, "perfect_em_stderr": 0.028928939388379635 }, "harness|bbh:temporal_sequences|3": { "em": 0.0, "em_stderr": 0.0, "qem": 0.0, "qem_stderr": 0.0, "pem": 0.108, "pem_stderr": 0.019669559381568752, "pqem": 0.392, "pqem_stderr": 0.0309382076204012, "perfect_em": 0.0, "perfect_em_stderr": 0.0 }, "harness|bbh:tracking_shuffled_objects_five_objects|3": { "em": 0.0, "em_stderr": 0.0, "qem": 0.0, "qem_stderr": 0.0, "pem": 0.204, "pem_stderr": 0.02553712157454814, "pqem": 0.24, "pqem_stderr": 0.027065293652239007, "perfect_em": 0.0, "perfect_em_stderr": 0.0 }, "harness|bbh:tracking_shuffled_objects_seven_objects|3": { "em": 0.0, "em_stderr": 0.0, "qem": 0.0, "qem_stderr": 0.0, "pem": 0.148, "pem_stderr": 0.02250354724380614, "pqem": 0.204, "pqem_stderr": 0.02553712157454815, "perfect_em": 0.0, "perfect_em_stderr": 0.0 }, "harness|bbh:tracking_shuffled_objects_three_objects|3": { "em": 0.0, "em_stderr": 0.0, "qem": 0.0, "qem_stderr": 0.0, "pem": 0.396, "pem_stderr": 0.030993197854577853, "pqem": 0.548, "pqem_stderr": 0.03153986449255662, "perfect_em": 0.0, "perfect_em_stderr": 0.0 }, "harness|bbh:_average|3": { "em": 0.05392732346598836, "em_stderr": 0.007179675435535086, "qem": 0.05392732346598836, "qem_stderr": 0.007179675435535086, "pem": 0.3114037786993455, "pem_stderr": 0.028016743775859554, "pqem": 0.41977852766500395, "pqem_stderr": 0.03028275789021361, "perfect_em": 0.03500516066147262, "perfect_em_stderr": 0.005981672612468351 } }, "versions": { "harness|bbh:causal_judgment|3": 0, "harness|bbh:date_understanding|3": 0, "harness|bbh:disambiguation_qa|3": 0, "harness|bbh:geometric_shapes|3": 0, "harness|bbh:logical_deduction_five_objects|3": 0, "harness|bbh:logical_deduction_seven_objects|3": 0, "harness|bbh:logical_deduction_three_objects|3": 0, "harness|bbh:movie_recommendation|3": 0, "harness|bbh:navigate|3": 0, "harness|bbh:reasoning_about_colored_objects|3": 0, "harness|bbh:ruin_names|3": 0, "harness|bbh:salient_translation_error_detection|3": 0, "harness|bbh:snarks|3": 0, "harness|bbh:sports_understanding|3": 0, "harness|bbh:temporal_sequences|3": 0, "harness|bbh:tracking_shuffled_objects_five_objects|3": 0, "harness|bbh:tracking_shuffled_objects_seven_objects|3": 0, "harness|bbh:tracking_shuffled_objects_three_objects|3": 0 }, "config_tasks": { "harness|bbh:causal_judgment": { "name": "bbh:causal_judgment", "prompt_function": "bbh_causal_judgment", "hf_repo": "lukaemon/bbh", "hf_subset": "causal_judgement", "metric": [ "exact_match", "quasi_exact_match", "prefix_exact_match", "prefix_quasi_exact_match", "perfect_exact_match" ], "hf_avail_splits": [ "test" ], "evaluation_splits": [ "test" ], "few_shots_split": null, "few_shots_select": null, "generation_size": 20, "stop_sequence": [ "", "Q:", "\n\n" ], "output_regex": null, "frozen": false, "suite": [ "harness" ], "original_num_docs": 187, "effective_num_docs": 187, "trust_dataset": true, "must_remove_duplicate_docs": null }, "harness|bbh:date_understanding": { "name": "bbh:date_understanding", "prompt_function": "bbh_date_understanding", "hf_repo": "lukaemon/bbh", "hf_subset": "date_understanding", "metric": [ "exact_match", "quasi_exact_match", "prefix_exact_match", "prefix_quasi_exact_match", "perfect_exact_match" ], "hf_avail_splits": [ "test" ], "evaluation_splits": [ "test" ], "few_shots_split": null, "few_shots_select": null, "generation_size": 20, "stop_sequence": [ "", "Q:", "\n\n" ], "output_regex": null, "frozen": false, "suite": [ "harness" ], "original_num_docs": 250, "effective_num_docs": 250, "trust_dataset": true, "must_remove_duplicate_docs": null }, "harness|bbh:disambiguation_qa": { "name": "bbh:disambiguation_qa", "prompt_function": "bbh_disambiguation_qa", "hf_repo": "lukaemon/bbh", "hf_subset": "disambiguation_qa", "metric": [ "exact_match", "quasi_exact_match", "prefix_exact_match", "prefix_quasi_exact_match", "perfect_exact_match" ], "hf_avail_splits": [ "test" ], "evaluation_splits": [ "test" ], "few_shots_split": null, "few_shots_select": null, "generation_size": 20, "stop_sequence": [ "", "Q:", "\n\n" ], "output_regex": null, "frozen": false, "suite": [ "harness" ], "original_num_docs": 250, "effective_num_docs": 250, "trust_dataset": true, "must_remove_duplicate_docs": null }, "harness|bbh:geometric_shapes": { "name": "bbh:geometric_shapes", "prompt_function": "bbh_geometric_shapes", "hf_repo": "lukaemon/bbh", "hf_subset": "geometric_shapes", "metric": [ "exact_match", "quasi_exact_match", "prefix_exact_match", "prefix_quasi_exact_match", "perfect_exact_match" ], "hf_avail_splits": [ "test" ], "evaluation_splits": [ "test" ], "few_shots_split": null, "few_shots_select": null, "generation_size": 20, "stop_sequence": [ "", "Q:", "\n\n" ], "output_regex": null, "frozen": false, "suite": [ "harness" ], "original_num_docs": 250, "effective_num_docs": 250, "trust_dataset": true, "must_remove_duplicate_docs": null }, "harness|bbh:logical_deduction_five_objects": { "name": "bbh:logical_deduction_five_objects", "prompt_function": "bbh_logical_deduction_five_objects", "hf_repo": "lukaemon/bbh", "hf_subset": "logical_deduction_five_objects", "metric": [ "exact_match", "quasi_exact_match", "prefix_exact_match", "prefix_quasi_exact_match", "perfect_exact_match" ], "hf_avail_splits": [ "test" ], "evaluation_splits": [ "test" ], "few_shots_split": null, "few_shots_select": null, "generation_size": 20, "stop_sequence": [ "", "Q:", "\n\n" ], "output_regex": null, "frozen": false, "suite": [ "harness" ], "original_num_docs": 250, "effective_num_docs": 250, "trust_dataset": true, "must_remove_duplicate_docs": null }, "harness|bbh:logical_deduction_seven_objects": { "name": "bbh:logical_deduction_seven_objects", "prompt_function": "bbh_logical_deduction_seven_objects", "hf_repo": "lukaemon/bbh", "hf_subset": "logical_deduction_seven_objects", "metric": [ "exact_match", "quasi_exact_match", "prefix_exact_match", "prefix_quasi_exact_match", "perfect_exact_match" ], "hf_avail_splits": [ "test" ], "evaluation_splits": [ "test" ], "few_shots_split": null, "few_shots_select": null, "generation_size": 20, "stop_sequence": [ "", "Q:", "\n\n" ], "output_regex": null, "frozen": false, "suite": [ "harness" ], "original_num_docs": 250, "effective_num_docs": 250, "trust_dataset": true, "must_remove_duplicate_docs": null }, "harness|bbh:logical_deduction_three_objects": { "name": "bbh:logical_deduction_three_objects", "prompt_function": "bbh_logical_deduction_three_objects", "hf_repo": "lukaemon/bbh", "hf_subset": "logical_deduction_three_objects", "metric": [ "exact_match", "quasi_exact_match", "prefix_exact_match", "prefix_quasi_exact_match", "perfect_exact_match" ], "hf_avail_splits": [ "test" ], "evaluation_splits": [ "test" ], "few_shots_split": null, "few_shots_select": null, "generation_size": 20, "stop_sequence": [ "", "Q:", "\n\n" ], "output_regex": null, "frozen": false, "suite": [ "harness" ], "original_num_docs": 250, "effective_num_docs": 250, "trust_dataset": true, "must_remove_duplicate_docs": null }, "harness|bbh:movie_recommendation": { "name": "bbh:movie_recommendation", "prompt_function": "bbh_movie_recommendation", "hf_repo": "lukaemon/bbh", "hf_subset": "movie_recommendation", "metric": [ "exact_match", "quasi_exact_match", "prefix_exact_match", "prefix_quasi_exact_match", "perfect_exact_match" ], "hf_avail_splits": [ "test" ], "evaluation_splits": [ "test" ], "few_shots_split": null, "few_shots_select": null, "generation_size": 20, "stop_sequence": [ "", "Q:", "\n\n" ], "output_regex": null, "frozen": false, "suite": [ "harness" ], "original_num_docs": 249, "effective_num_docs": 249, "trust_dataset": true, "must_remove_duplicate_docs": null }, "harness|bbh:navigate": { "name": "bbh:navigate", "prompt_function": "bbh_navigate", "hf_repo": "lukaemon/bbh", "hf_subset": "navigate", "metric": [ "exact_match", "quasi_exact_match", "prefix_exact_match", "prefix_quasi_exact_match", "perfect_exact_match" ], "hf_avail_splits": [ "test" ], "evaluation_splits": [ "test" ], "few_shots_split": null, "few_shots_select": null, "generation_size": 20, "stop_sequence": [ "", "Q:", "\n\n" ], "output_regex": null, "frozen": false, "suite": [ "harness" ], "original_num_docs": 250, "effective_num_docs": 250, "trust_dataset": true, "must_remove_duplicate_docs": null }, "harness|bbh:reasoning_about_colored_objects": { "name": "bbh:reasoning_about_colored_objects", "prompt_function": "bbh_reasoning_about_colored_objects", "hf_repo": "lukaemon/bbh", "hf_subset": "reasoning_about_colored_objects", "metric": [ "exact_match", "quasi_exact_match", "prefix_exact_match", "prefix_quasi_exact_match", "perfect_exact_match" ], "hf_avail_splits": [ "test" ], "evaluation_splits": [ "test" ], "few_shots_split": null, "few_shots_select": null, "generation_size": 20, "stop_sequence": [ "", "Q:", "\n\n" ], "output_regex": null, "frozen": false, "suite": [ "harness" ], "original_num_docs": 250, "effective_num_docs": 250, "trust_dataset": true, "must_remove_duplicate_docs": null }, "harness|bbh:ruin_names": { "name": "bbh:ruin_names", "prompt_function": "bbh_ruin_names", "hf_repo": "lukaemon/bbh", "hf_subset": "ruin_names", "metric": [ "exact_match", "quasi_exact_match", "prefix_exact_match", "prefix_quasi_exact_match", "perfect_exact_match" ], "hf_avail_splits": [ "test" ], "evaluation_splits": [ "test" ], "few_shots_split": null, "few_shots_select": null, "generation_size": 20, "stop_sequence": [ "", "Q:", "\n\n" ], "output_regex": null, "frozen": false, "suite": [ "harness" ], "original_num_docs": 248, "effective_num_docs": 248, "trust_dataset": true, "must_remove_duplicate_docs": null }, "harness|bbh:salient_translation_error_detection": { "name": "bbh:salient_translation_error_detection", "prompt_function": "bbh_salient_translation_error_detection", "hf_repo": "lukaemon/bbh", "hf_subset": "salient_translation_error_detection", "metric": [ "exact_match", "quasi_exact_match", "prefix_exact_match", "prefix_quasi_exact_match", "perfect_exact_match" ], "hf_avail_splits": [ "test" ], "evaluation_splits": [ "test" ], "few_shots_split": null, "few_shots_select": null, "generation_size": 20, "stop_sequence": [ "", "Q:", "\n\n" ], "output_regex": null, "frozen": false, "suite": [ "harness" ], "original_num_docs": 250, "effective_num_docs": 250, "trust_dataset": true, "must_remove_duplicate_docs": null }, "harness|bbh:snarks": { "name": "bbh:snarks", "prompt_function": "bbh_snarks", "hf_repo": "lukaemon/bbh", "hf_subset": "snarks", "metric": [ "exact_match", "quasi_exact_match", "prefix_exact_match", "prefix_quasi_exact_match", "perfect_exact_match" ], "hf_avail_splits": [ "test" ], "evaluation_splits": [ "test" ], "few_shots_split": null, "few_shots_select": null, "generation_size": 20, "stop_sequence": [ "", "Q:", "\n\n" ], "output_regex": null, "frozen": false, "suite": [ "harness" ], "original_num_docs": 178, "effective_num_docs": 178, "trust_dataset": true, "must_remove_duplicate_docs": null }, "harness|bbh:sports_understanding": { "name": "bbh:sports_understanding", "prompt_function": "bbh_sports_understanding", "hf_repo": "lukaemon/bbh", "hf_subset": "sports_understanding", "metric": [ "exact_match", "quasi_exact_match", "prefix_exact_match", "prefix_quasi_exact_match", "perfect_exact_match" ], "hf_avail_splits": [ "test" ], "evaluation_splits": [ "test" ], "few_shots_split": null, "few_shots_select": null, "generation_size": 20, "stop_sequence": [ "", "Q:", "\n\n" ], "output_regex": null, "frozen": false, "suite": [ "harness" ], "original_num_docs": 250, "effective_num_docs": 250, "trust_dataset": true, "must_remove_duplicate_docs": null }, "harness|bbh:temporal_sequences": { "name": "bbh:temporal_sequences", "prompt_function": "bbh_temporal_sequences", "hf_repo": "lukaemon/bbh", "hf_subset": "temporal_sequences", "metric": [ "exact_match", "quasi_exact_match", "prefix_exact_match", "prefix_quasi_exact_match", "perfect_exact_match" ], "hf_avail_splits": [ "test" ], "evaluation_splits": [ "test" ], "few_shots_split": null, "few_shots_select": null, "generation_size": 20, "stop_sequence": [ "", "Q:", "\n\n" ], "output_regex": null, "frozen": false, "suite": [ "harness" ], "original_num_docs": 250, "effective_num_docs": 250, "trust_dataset": true, "must_remove_duplicate_docs": null }, "harness|bbh:tracking_shuffled_objects_five_objects": { "name": "bbh:tracking_shuffled_objects_five_objects", "prompt_function": "bbh_tracking_shuffled_objects_five_objects", "hf_repo": "lukaemon/bbh", "hf_subset": "tracking_shuffled_objects_five_objects", "metric": [ "exact_match", "quasi_exact_match", "prefix_exact_match", "prefix_quasi_exact_match", "perfect_exact_match" ], "hf_avail_splits": [ "test" ], "evaluation_splits": [ "test" ], "few_shots_split": null, "few_shots_select": null, "generation_size": 20, "stop_sequence": [ "", "Q:", "\n\n" ], "output_regex": null, "frozen": false, "suite": [ "harness" ], "original_num_docs": 250, "effective_num_docs": 250, "trust_dataset": true, "must_remove_duplicate_docs": null }, "harness|bbh:tracking_shuffled_objects_seven_objects": { "name": "bbh:tracking_shuffled_objects_seven_objects", "prompt_function": "bbh_tracking_shuffled_objects_seven_objects", "hf_repo": "lukaemon/bbh", "hf_subset": "tracking_shuffled_objects_seven_objects", "metric": [ "exact_match", "quasi_exact_match", "prefix_exact_match", "prefix_quasi_exact_match", "perfect_exact_match" ], "hf_avail_splits": [ "test" ], "evaluation_splits": [ "test" ], "few_shots_split": null, "few_shots_select": null, "generation_size": 20, "stop_sequence": [ "", "Q:", "\n\n" ], "output_regex": null, "frozen": false, "suite": [ "harness" ], "original_num_docs": 250, "effective_num_docs": 250, "trust_dataset": true, "must_remove_duplicate_docs": null }, "harness|bbh:tracking_shuffled_objects_three_objects": { "name": "bbh:tracking_shuffled_objects_three_objects", "prompt_function": "bbh_tracking_shuffled_objects_three_objects", "hf_repo": "lukaemon/bbh", "hf_subset": "tracking_shuffled_objects_three_objects", "metric": [ "exact_match", "quasi_exact_match", "prefix_exact_match", "prefix_quasi_exact_match", "perfect_exact_match" ], "hf_avail_splits": [ "test" ], "evaluation_splits": [ "test" ], "few_shots_split": null, "few_shots_select": null, "generation_size": 20, "stop_sequence": [ "", "Q:", "\n\n" ], "output_regex": null, "frozen": false, "suite": [ "harness" ], "original_num_docs": 250, "effective_num_docs": 250, "trust_dataset": true, "must_remove_duplicate_docs": null } }, "summary_tasks": { "harness|bbh:causal_judgment|3": { "hashes": { "hash_examples": "63218f5ae055ab2b", "hash_full_prompts": "da4486f76ca37d71", "hash_input_tokens": "2f4439cd00aeae0e", "hash_cont_tokens": "1478f84094c52bb4" }, "truncated": 187, "non_truncated": 0, "padded": 0, "non_padded": 187, "effective_few_shots": 3.0, "num_truncated_few_shots": 0 }, "harness|bbh:date_understanding|3": { "hashes": { "hash_examples": "f145c7a06def3c8e", "hash_full_prompts": "ab81948b0abe57b9", "hash_input_tokens": "d36736b69b6676a1", "hash_cont_tokens": "45ecc819fa52c55c" }, "truncated": 250, "non_truncated": 0, "padded": 0, "non_padded": 250, "effective_few_shots": 3.0, "num_truncated_few_shots": 0 }, "harness|bbh:disambiguation_qa|3": { "hashes": { "hash_examples": "19677fd1773f7eb9", "hash_full_prompts": "7a5061f70150c09e", "hash_input_tokens": "1f6e1e8254e2c651", "hash_cont_tokens": "3c8b7521886046ae" }, "truncated": 250, "non_truncated": 0, "padded": 0, "non_padded": 250, "effective_few_shots": 3.0, "num_truncated_few_shots": 0 }, "harness|bbh:geometric_shapes|3": { "hashes": { "hash_examples": "76c7b11a13cc72a9", "hash_full_prompts": "e063deff2e84a2a6", "hash_input_tokens": "f6cae0e55367ae6c", "hash_cont_tokens": "dafd4cc6d39b3660" }, "truncated": 250, "non_truncated": 0, "padded": 0, "non_padded": 250, "effective_few_shots": 3.0, "num_truncated_few_shots": 0 }, "harness|bbh:logical_deduction_five_objects|3": { "hashes": { "hash_examples": "0e958c856332a745", "hash_full_prompts": "7544b780705e9654", "hash_input_tokens": "069f2e5e4f4819e1", "hash_cont_tokens": "becfc64d5179c60b" }, "truncated": 250, "non_truncated": 0, "padded": 0, "non_padded": 250, "effective_few_shots": 3.0, "num_truncated_few_shots": 0 }, "harness|bbh:logical_deduction_seven_objects|3": { "hashes": { "hash_examples": "ab9de25a5eb40d09", "hash_full_prompts": "9ba71ddb2add0b58", "hash_input_tokens": "61154b5b2d23fd61", "hash_cont_tokens": "5ab6772d0e983d20" }, "truncated": 250, "non_truncated": 0, "padded": 0, "non_padded": 250, "effective_few_shots": 3.0, "num_truncated_few_shots": 0 }, "harness|bbh:logical_deduction_three_objects|3": { "hashes": { "hash_examples": "3c6bf52517714218", "hash_full_prompts": "78aebd4f41995a84", "hash_input_tokens": "8611de524f2cb180", "hash_cont_tokens": "839ba0f1b24b0dc2" }, "truncated": 250, "non_truncated": 0, "padded": 0, "non_padded": 250, "effective_few_shots": 3.0, "num_truncated_few_shots": 0 }, "harness|bbh:movie_recommendation|3": { "hashes": { "hash_examples": "2d9dc4975935d31a", "hash_full_prompts": "290a864a2acc3828", "hash_input_tokens": "dd866e9aefb525a2", "hash_cont_tokens": "510f5e07a5c4999c" }, "truncated": 249, "non_truncated": 0, "padded": 0, "non_padded": 249, "effective_few_shots": 3.0, "num_truncated_few_shots": 0 }, "harness|bbh:navigate|3": { "hashes": { "hash_examples": "ba91dcdb9a064255", "hash_full_prompts": "4d8fad912ba90363", "hash_input_tokens": "aea22a5b526e9934", "hash_cont_tokens": "e3a02b03316478f9" }, "truncated": 250, "non_truncated": 0, "padded": 0, "non_padded": 250, "effective_few_shots": 3.0, "num_truncated_few_shots": 0 }, "harness|bbh:reasoning_about_colored_objects|3": { "hashes": { "hash_examples": "a6ba328c4c3385d2", "hash_full_prompts": "96ca31772c57eeeb", "hash_input_tokens": "f31e7a9ef88c8872", "hash_cont_tokens": "f933686f8b1e23a4" }, "truncated": 250, "non_truncated": 0, "padded": 0, "non_padded": 250, "effective_few_shots": 3.0, "num_truncated_few_shots": 0 }, "harness|bbh:ruin_names|3": { "hashes": { "hash_examples": "2ef28d5f2d4fdd25", "hash_full_prompts": "f3ad065935c46de7", "hash_input_tokens": "5cc6ead631865b9a", "hash_cont_tokens": "2eb1209a83b1d09e" }, "truncated": 248, "non_truncated": 0, "padded": 0, "non_padded": 248, "effective_few_shots": 3.0, "num_truncated_few_shots": 0 }, "harness|bbh:salient_translation_error_detection|3": { "hashes": { "hash_examples": "c13f25ec8ffed496", "hash_full_prompts": "80fc33e0cd811a8d", "hash_input_tokens": "6004f29b100793e5", "hash_cont_tokens": "f39b3fc4a19d51f6" }, "truncated": 250, "non_truncated": 0, "padded": 0, "non_padded": 250, "effective_few_shots": 3.0, "num_truncated_few_shots": 0 }, "harness|bbh:snarks|3": { "hashes": { "hash_examples": "5f6db7bff7f6f22e", "hash_full_prompts": "55586142735dea08", "hash_input_tokens": "f78df86ae6e4e291", "hash_cont_tokens": "053f4b3ea75f4de6" }, "truncated": 178, "non_truncated": 0, "padded": 0, "non_padded": 178, "effective_few_shots": 3.0, "num_truncated_few_shots": 0 }, "harness|bbh:sports_understanding|3": { "hashes": { "hash_examples": "042afbe5d9c1f02d", "hash_full_prompts": "1c523efd25e2ebfd", "hash_input_tokens": "176635253aa6d0e9", "hash_cont_tokens": "b66980c534eac2de" }, "truncated": 250, "non_truncated": 0, "padded": 0, "non_padded": 250, "effective_few_shots": 3.0, "num_truncated_few_shots": 0 }, "harness|bbh:temporal_sequences|3": { "hashes": { "hash_examples": "803a05f352eb6afc", "hash_full_prompts": "caa56ea50f0068e1", "hash_input_tokens": "74b504d9b40add38", "hash_cont_tokens": "634c4602e9316257" }, "truncated": 250, "non_truncated": 0, "padded": 0, "non_padded": 250, "effective_few_shots": 3.0, "num_truncated_few_shots": 0 }, "harness|bbh:tracking_shuffled_objects_five_objects|3": { "hashes": { "hash_examples": "2bbac6db7ab0d527", "hash_full_prompts": "4c9e2f3952f99f7d", "hash_input_tokens": "36d0e7377cce953e", "hash_cont_tokens": "f59c1a7a0882977b" }, "truncated": 250, "non_truncated": 0, "padded": 0, "non_padded": 250, "effective_few_shots": 3.0, "num_truncated_few_shots": 0 }, "harness|bbh:tracking_shuffled_objects_seven_objects|3": { "hashes": { "hash_examples": "845caf093ac2b58c", "hash_full_prompts": "d4f4f0656462d45f", "hash_input_tokens": "d255d2f245886c58", "hash_cont_tokens": "6374bde1214f79d7" }, "truncated": 250, "non_truncated": 0, "padded": 0, "non_padded": 250, "effective_few_shots": 3.0, "num_truncated_few_shots": 0 }, "harness|bbh:tracking_shuffled_objects_three_objects|3": { "hashes": { "hash_examples": "9004f14d5a32b9a8", "hash_full_prompts": "be9c55b344ca07ab", "hash_input_tokens": "55c63804a84860f6", "hash_cont_tokens": "583fdb1bded8b71b" }, "truncated": 250, "non_truncated": 0, "padded": 0, "non_padded": 250, "effective_few_shots": 3.0, "num_truncated_few_shots": 0 } }, "summary_general": { "hashes": { "hash_examples": "4ff1e3dc5703575d", "hash_full_prompts": "ec35636f85fd5ee2", "hash_input_tokens": "016924e640ab4527", "hash_cont_tokens": "7fc2f6f782e5890b" }, "truncated": 4362, "non_truncated": 0, "padded": 0, "non_padded": 4362, "num_truncated_few_shots": 0 } }