open-r1-eval-leaderboard
/
eval_results
/google
/gemma-2b-it
/main
/bbh
/results_2024-03-18T20-36-40.384973.json
| { | |
| "config_general": { | |
| "lighteval_sha": "?", | |
| "num_fewshot_seeds": 1, | |
| "override_batch_size": 1, | |
| "max_samples": null, | |
| "job_id": "", | |
| "start_time": 1564442.483351112, | |
| "end_time": 1564777.847840123, | |
| "total_evaluation_time_secondes": "335.36448901100084", | |
| "model_name": "google/gemma-2b-it", | |
| "model_sha": "718cb189da9c5b2e55abe86f2eeffee9b4ae0dad", | |
| "model_dtype": "torch.bfloat16", | |
| "model_size": "4.68 GB", | |
| "config": null | |
| }, | |
| "results": { | |
| "harness|bbh:causal_judgment|3": { | |
| "em": 0.3689839572192513, | |
| "em_stderr": 0.03538078548260322, | |
| "qem": 0.3689839572192513, | |
| "qem_stderr": 0.03538078548260322, | |
| "pem": 0.5240641711229946, | |
| "pem_stderr": 0.03661929361528702, | |
| "pqem": 0.5240641711229946, | |
| "pqem_stderr": 0.03661929361528702, | |
| "perfect_em": 0.1443850267379679, | |
| "perfect_em_stderr": 0.02577174317334218 | |
| }, | |
| "harness|bbh:date_understanding|3": { | |
| "em": 0.0, | |
| "em_stderr": 0.0, | |
| "qem": 0.0, | |
| "qem_stderr": 0.0, | |
| "pem": 0.276, | |
| "pem_stderr": 0.028328537274211352, | |
| "pqem": 0.348, | |
| "pqem_stderr": 0.030186568464511683, | |
| "perfect_em": 0.0, | |
| "perfect_em_stderr": 0.0 | |
| }, | |
| "harness|bbh:disambiguation_qa|3": { | |
| "em": 0.0, | |
| "em_stderr": 0.0, | |
| "qem": 0.0, | |
| "qem_stderr": 0.0, | |
| "pem": 0.396, | |
| "pem_stderr": 0.03099319785457785, | |
| "pqem": 0.544, | |
| "pqem_stderr": 0.03156328506121339, | |
| "perfect_em": 0.0, | |
| "perfect_em_stderr": 0.0 | |
| }, | |
| "harness|bbh:geometric_shapes|3": { | |
| "em": 0.124, | |
| "em_stderr": 0.02088638225867326, | |
| "qem": 0.124, | |
| "qem_stderr": 0.02088638225867326, | |
| "pem": 0.128, | |
| "pem_stderr": 0.021172081336336492, | |
| "pqem": 0.128, | |
| "pqem_stderr": 0.021172081336336492, | |
| "perfect_em": 0.124, | |
| "perfect_em_stderr": 0.02088638225867326 | |
| }, | |
| "harness|bbh:logical_deduction_five_objects|3": { | |
| "em": 0.004, | |
| "em_stderr": 0.004, | |
| "qem": 0.004, | |
| "qem_stderr": 0.004, | |
| "pem": 0.212, | |
| "pem_stderr": 0.025901884690541162, | |
| "pqem": 0.392, | |
| "pqem_stderr": 0.0309382076204012, | |
| "perfect_em": 0.004, | |
| "perfect_em_stderr": 0.004 | |
| }, | |
| "harness|bbh:logical_deduction_seven_objects|3": { | |
| "em": 0.0, | |
| "em_stderr": 0.0, | |
| "qem": 0.0, | |
| "qem_stderr": 0.0, | |
| "pem": 0.164, | |
| "pem_stderr": 0.023465261002076753, | |
| "pqem": 0.224, | |
| "pqem_stderr": 0.026421361687347905, | |
| "perfect_em": 0.0, | |
| "perfect_em_stderr": 0.0 | |
| }, | |
| "harness|bbh:logical_deduction_three_objects|3": { | |
| "em": 0.008, | |
| "em_stderr": 0.005645483676690171, | |
| "qem": 0.008, | |
| "qem_stderr": 0.005645483676690171, | |
| "pem": 0.4, | |
| "pem_stderr": 0.031046021028253244, | |
| "pqem": 0.64, | |
| "pqem_stderr": 0.030418764025174985, | |
| "perfect_em": 0.008, | |
| "perfect_em_stderr": 0.005645483676690171 | |
| }, | |
| "harness|bbh:movie_recommendation|3": { | |
| "em": 0.0, | |
| "em_stderr": 0.0, | |
| "qem": 0.0, | |
| "qem_stderr": 0.0, | |
| "pem": 0.5301204819277109, | |
| "pem_stderr": 0.03169236937317327, | |
| "pqem": 0.6024096385542169, | |
| "pqem_stderr": 0.03107692497419258, | |
| "perfect_em": 0.0, | |
| "perfect_em_stderr": 0.0 | |
| }, | |
| "harness|bbh:navigate|3": { | |
| "em": 0.116, | |
| "em_stderr": 0.02029342980308385, | |
| "qem": 0.116, | |
| "qem_stderr": 0.02029342980308385, | |
| "pem": 0.596, | |
| "pem_stderr": 0.031096688184825298, | |
| "pqem": 0.596, | |
| "pqem_stderr": 0.031096688184825298, | |
| "perfect_em": 0.02, | |
| "perfect_em_stderr": 0.008872139507342683 | |
| }, | |
| "harness|bbh:reasoning_about_colored_objects|3": { | |
| "em": 0.0, | |
| "em_stderr": 0.0, | |
| "qem": 0.0, | |
| "qem_stderr": 0.0, | |
| "pem": 0.176, | |
| "pem_stderr": 0.02413349752545711, | |
| "pqem": 0.312, | |
| "pqem_stderr": 0.029361067575219817, | |
| "perfect_em": 0.0, | |
| "perfect_em_stderr": 0.0 | |
| }, | |
| "harness|bbh:ruin_names|3": { | |
| "em": 0.0, | |
| "em_stderr": 0.0, | |
| "qem": 0.0, | |
| "qem_stderr": 0.0, | |
| "pem": 0.2056451612903226, | |
| "pem_stderr": 0.025716866102588432, | |
| "pqem": 0.46774193548387094, | |
| "pqem_stderr": 0.03174795841398569, | |
| "perfect_em": 0.0, | |
| "perfect_em_stderr": 0.0 | |
| }, | |
| "harness|bbh:salient_translation_error_detection|3": { | |
| "em": 0.0, | |
| "em_stderr": 0.0, | |
| "qem": 0.0, | |
| "qem_stderr": 0.0, | |
| "pem": 0.224, | |
| "pem_stderr": 0.026421361687347905, | |
| "pqem": 0.364, | |
| "pqem_stderr": 0.03049155522040554, | |
| "perfect_em": 0.0, | |
| "perfect_em_stderr": 0.0 | |
| }, | |
| "harness|bbh:snarks|3": { | |
| "em": 0.033707865168539325, | |
| "em_stderr": 0.01356541902000238, | |
| "qem": 0.033707865168539325, | |
| "qem_stderr": 0.01356541902000238, | |
| "pem": 0.449438202247191, | |
| "pem_stderr": 0.03738964966056965, | |
| "pqem": 0.5617977528089888, | |
| "pqem_stderr": 0.03729414592947276, | |
| "perfect_em": 0.033707865168539325, | |
| "perfect_em_stderr": 0.01356541902000238 | |
| }, | |
| "harness|bbh:sports_understanding|3": { | |
| "em": 0.316, | |
| "em_stderr": 0.029462657598578676, | |
| "qem": 0.316, | |
| "qem_stderr": 0.029462657598578676, | |
| "pem": 0.468, | |
| "pem_stderr": 0.031621252575725504, | |
| "pqem": 0.468, | |
| "pqem_stderr": 0.031621252575725504, | |
| "perfect_em": 0.296, | |
| "perfect_em_stderr": 0.028928939388379635 | |
| }, | |
| "harness|bbh:temporal_sequences|3": { | |
| "em": 0.0, | |
| "em_stderr": 0.0, | |
| "qem": 0.0, | |
| "qem_stderr": 0.0, | |
| "pem": 0.108, | |
| "pem_stderr": 0.019669559381568752, | |
| "pqem": 0.392, | |
| "pqem_stderr": 0.0309382076204012, | |
| "perfect_em": 0.0, | |
| "perfect_em_stderr": 0.0 | |
| }, | |
| "harness|bbh:tracking_shuffled_objects_five_objects|3": { | |
| "em": 0.0, | |
| "em_stderr": 0.0, | |
| "qem": 0.0, | |
| "qem_stderr": 0.0, | |
| "pem": 0.204, | |
| "pem_stderr": 0.02553712157454814, | |
| "pqem": 0.24, | |
| "pqem_stderr": 0.027065293652239007, | |
| "perfect_em": 0.0, | |
| "perfect_em_stderr": 0.0 | |
| }, | |
| "harness|bbh:tracking_shuffled_objects_seven_objects|3": { | |
| "em": 0.0, | |
| "em_stderr": 0.0, | |
| "qem": 0.0, | |
| "qem_stderr": 0.0, | |
| "pem": 0.148, | |
| "pem_stderr": 0.02250354724380614, | |
| "pqem": 0.204, | |
| "pqem_stderr": 0.02553712157454815, | |
| "perfect_em": 0.0, | |
| "perfect_em_stderr": 0.0 | |
| }, | |
| "harness|bbh:tracking_shuffled_objects_three_objects|3": { | |
| "em": 0.0, | |
| "em_stderr": 0.0, | |
| "qem": 0.0, | |
| "qem_stderr": 0.0, | |
| "pem": 0.396, | |
| "pem_stderr": 0.030993197854577853, | |
| "pqem": 0.548, | |
| "pqem_stderr": 0.03153986449255662, | |
| "perfect_em": 0.0, | |
| "perfect_em_stderr": 0.0 | |
| }, | |
| "harness|bbh:_average|3": { | |
| "em": 0.05392732346598836, | |
| "em_stderr": 0.007179675435535086, | |
| "qem": 0.05392732346598836, | |
| "qem_stderr": 0.007179675435535086, | |
| "pem": 0.3114037786993455, | |
| "pem_stderr": 0.028016743775859554, | |
| "pqem": 0.41977852766500395, | |
| "pqem_stderr": 0.03028275789021361, | |
| "perfect_em": 0.03500516066147262, | |
| "perfect_em_stderr": 0.005981672612468351 | |
| } | |
| }, | |
| "versions": { | |
| "harness|bbh:causal_judgment|3": 0, | |
| "harness|bbh:date_understanding|3": 0, | |
| "harness|bbh:disambiguation_qa|3": 0, | |
| "harness|bbh:geometric_shapes|3": 0, | |
| "harness|bbh:logical_deduction_five_objects|3": 0, | |
| "harness|bbh:logical_deduction_seven_objects|3": 0, | |
| "harness|bbh:logical_deduction_three_objects|3": 0, | |
| "harness|bbh:movie_recommendation|3": 0, | |
| "harness|bbh:navigate|3": 0, | |
| "harness|bbh:reasoning_about_colored_objects|3": 0, | |
| "harness|bbh:ruin_names|3": 0, | |
| "harness|bbh:salient_translation_error_detection|3": 0, | |
| "harness|bbh:snarks|3": 0, | |
| "harness|bbh:sports_understanding|3": 0, | |
| "harness|bbh:temporal_sequences|3": 0, | |
| "harness|bbh:tracking_shuffled_objects_five_objects|3": 0, | |
| "harness|bbh:tracking_shuffled_objects_seven_objects|3": 0, | |
| "harness|bbh:tracking_shuffled_objects_three_objects|3": 0 | |
| }, | |
| "config_tasks": { | |
| "harness|bbh:causal_judgment": { | |
| "name": "bbh:causal_judgment", | |
| "prompt_function": "bbh_causal_judgment", | |
| "hf_repo": "lukaemon/bbh", | |
| "hf_subset": "causal_judgement", | |
| "metric": [ | |
| "exact_match", | |
| "quasi_exact_match", | |
| "prefix_exact_match", | |
| "prefix_quasi_exact_match", | |
| "perfect_exact_match" | |
| ], | |
| "hf_avail_splits": [ | |
| "test" | |
| ], | |
| "evaluation_splits": [ | |
| "test" | |
| ], | |
| "few_shots_split": null, | |
| "few_shots_select": null, | |
| "generation_size": 20, | |
| "stop_sequence": [ | |
| "</s>", | |
| "Q:", | |
| "\n\n" | |
| ], | |
| "output_regex": null, | |
| "frozen": false, | |
| "suite": [ | |
| "harness" | |
| ], | |
| "original_num_docs": 187, | |
| "effective_num_docs": 187, | |
| "trust_dataset": true, | |
| "must_remove_duplicate_docs": null | |
| }, | |
| "harness|bbh:date_understanding": { | |
| "name": "bbh:date_understanding", | |
| "prompt_function": "bbh_date_understanding", | |
| "hf_repo": "lukaemon/bbh", | |
| "hf_subset": "date_understanding", | |
| "metric": [ | |
| "exact_match", | |
| "quasi_exact_match", | |
| "prefix_exact_match", | |
| "prefix_quasi_exact_match", | |
| "perfect_exact_match" | |
| ], | |
| "hf_avail_splits": [ | |
| "test" | |
| ], | |
| "evaluation_splits": [ | |
| "test" | |
| ], | |
| "few_shots_split": null, | |
| "few_shots_select": null, | |
| "generation_size": 20, | |
| "stop_sequence": [ | |
| "</s>", | |
| "Q:", | |
| "\n\n" | |
| ], | |
| "output_regex": null, | |
| "frozen": false, | |
| "suite": [ | |
| "harness" | |
| ], | |
| "original_num_docs": 250, | |
| "effective_num_docs": 250, | |
| "trust_dataset": true, | |
| "must_remove_duplicate_docs": null | |
| }, | |
| "harness|bbh:disambiguation_qa": { | |
| "name": "bbh:disambiguation_qa", | |
| "prompt_function": "bbh_disambiguation_qa", | |
| "hf_repo": "lukaemon/bbh", | |
| "hf_subset": "disambiguation_qa", | |
| "metric": [ | |
| "exact_match", | |
| "quasi_exact_match", | |
| "prefix_exact_match", | |
| "prefix_quasi_exact_match", | |
| "perfect_exact_match" | |
| ], | |
| "hf_avail_splits": [ | |
| "test" | |
| ], | |
| "evaluation_splits": [ | |
| "test" | |
| ], | |
| "few_shots_split": null, | |
| "few_shots_select": null, | |
| "generation_size": 20, | |
| "stop_sequence": [ | |
| "</s>", | |
| "Q:", | |
| "\n\n" | |
| ], | |
| "output_regex": null, | |
| "frozen": false, | |
| "suite": [ | |
| "harness" | |
| ], | |
| "original_num_docs": 250, | |
| "effective_num_docs": 250, | |
| "trust_dataset": true, | |
| "must_remove_duplicate_docs": null | |
| }, | |
| "harness|bbh:geometric_shapes": { | |
| "name": "bbh:geometric_shapes", | |
| "prompt_function": "bbh_geometric_shapes", | |
| "hf_repo": "lukaemon/bbh", | |
| "hf_subset": "geometric_shapes", | |
| "metric": [ | |
| "exact_match", | |
| "quasi_exact_match", | |
| "prefix_exact_match", | |
| "prefix_quasi_exact_match", | |
| "perfect_exact_match" | |
| ], | |
| "hf_avail_splits": [ | |
| "test" | |
| ], | |
| "evaluation_splits": [ | |
| "test" | |
| ], | |
| "few_shots_split": null, | |
| "few_shots_select": null, | |
| "generation_size": 20, | |
| "stop_sequence": [ | |
| "</s>", | |
| "Q:", | |
| "\n\n" | |
| ], | |
| "output_regex": null, | |
| "frozen": false, | |
| "suite": [ | |
| "harness" | |
| ], | |
| "original_num_docs": 250, | |
| "effective_num_docs": 250, | |
| "trust_dataset": true, | |
| "must_remove_duplicate_docs": null | |
| }, | |
| "harness|bbh:logical_deduction_five_objects": { | |
| "name": "bbh:logical_deduction_five_objects", | |
| "prompt_function": "bbh_logical_deduction_five_objects", | |
| "hf_repo": "lukaemon/bbh", | |
| "hf_subset": "logical_deduction_five_objects", | |
| "metric": [ | |
| "exact_match", | |
| "quasi_exact_match", | |
| "prefix_exact_match", | |
| "prefix_quasi_exact_match", | |
| "perfect_exact_match" | |
| ], | |
| "hf_avail_splits": [ | |
| "test" | |
| ], | |
| "evaluation_splits": [ | |
| "test" | |
| ], | |
| "few_shots_split": null, | |
| "few_shots_select": null, | |
| "generation_size": 20, | |
| "stop_sequence": [ | |
| "</s>", | |
| "Q:", | |
| "\n\n" | |
| ], | |
| "output_regex": null, | |
| "frozen": false, | |
| "suite": [ | |
| "harness" | |
| ], | |
| "original_num_docs": 250, | |
| "effective_num_docs": 250, | |
| "trust_dataset": true, | |
| "must_remove_duplicate_docs": null | |
| }, | |
| "harness|bbh:logical_deduction_seven_objects": { | |
| "name": "bbh:logical_deduction_seven_objects", | |
| "prompt_function": "bbh_logical_deduction_seven_objects", | |
| "hf_repo": "lukaemon/bbh", | |
| "hf_subset": "logical_deduction_seven_objects", | |
| "metric": [ | |
| "exact_match", | |
| "quasi_exact_match", | |
| "prefix_exact_match", | |
| "prefix_quasi_exact_match", | |
| "perfect_exact_match" | |
| ], | |
| "hf_avail_splits": [ | |
| "test" | |
| ], | |
| "evaluation_splits": [ | |
| "test" | |
| ], | |
| "few_shots_split": null, | |
| "few_shots_select": null, | |
| "generation_size": 20, | |
| "stop_sequence": [ | |
| "</s>", | |
| "Q:", | |
| "\n\n" | |
| ], | |
| "output_regex": null, | |
| "frozen": false, | |
| "suite": [ | |
| "harness" | |
| ], | |
| "original_num_docs": 250, | |
| "effective_num_docs": 250, | |
| "trust_dataset": true, | |
| "must_remove_duplicate_docs": null | |
| }, | |
| "harness|bbh:logical_deduction_three_objects": { | |
| "name": "bbh:logical_deduction_three_objects", | |
| "prompt_function": "bbh_logical_deduction_three_objects", | |
| "hf_repo": "lukaemon/bbh", | |
| "hf_subset": "logical_deduction_three_objects", | |
| "metric": [ | |
| "exact_match", | |
| "quasi_exact_match", | |
| "prefix_exact_match", | |
| "prefix_quasi_exact_match", | |
| "perfect_exact_match" | |
| ], | |
| "hf_avail_splits": [ | |
| "test" | |
| ], | |
| "evaluation_splits": [ | |
| "test" | |
| ], | |
| "few_shots_split": null, | |
| "few_shots_select": null, | |
| "generation_size": 20, | |
| "stop_sequence": [ | |
| "</s>", | |
| "Q:", | |
| "\n\n" | |
| ], | |
| "output_regex": null, | |
| "frozen": false, | |
| "suite": [ | |
| "harness" | |
| ], | |
| "original_num_docs": 250, | |
| "effective_num_docs": 250, | |
| "trust_dataset": true, | |
| "must_remove_duplicate_docs": null | |
| }, | |
| "harness|bbh:movie_recommendation": { | |
| "name": "bbh:movie_recommendation", | |
| "prompt_function": "bbh_movie_recommendation", | |
| "hf_repo": "lukaemon/bbh", | |
| "hf_subset": "movie_recommendation", | |
| "metric": [ | |
| "exact_match", | |
| "quasi_exact_match", | |
| "prefix_exact_match", | |
| "prefix_quasi_exact_match", | |
| "perfect_exact_match" | |
| ], | |
| "hf_avail_splits": [ | |
| "test" | |
| ], | |
| "evaluation_splits": [ | |
| "test" | |
| ], | |
| "few_shots_split": null, | |
| "few_shots_select": null, | |
| "generation_size": 20, | |
| "stop_sequence": [ | |
| "</s>", | |
| "Q:", | |
| "\n\n" | |
| ], | |
| "output_regex": null, | |
| "frozen": false, | |
| "suite": [ | |
| "harness" | |
| ], | |
| "original_num_docs": 249, | |
| "effective_num_docs": 249, | |
| "trust_dataset": true, | |
| "must_remove_duplicate_docs": null | |
| }, | |
| "harness|bbh:navigate": { | |
| "name": "bbh:navigate", | |
| "prompt_function": "bbh_navigate", | |
| "hf_repo": "lukaemon/bbh", | |
| "hf_subset": "navigate", | |
| "metric": [ | |
| "exact_match", | |
| "quasi_exact_match", | |
| "prefix_exact_match", | |
| "prefix_quasi_exact_match", | |
| "perfect_exact_match" | |
| ], | |
| "hf_avail_splits": [ | |
| "test" | |
| ], | |
| "evaluation_splits": [ | |
| "test" | |
| ], | |
| "few_shots_split": null, | |
| "few_shots_select": null, | |
| "generation_size": 20, | |
| "stop_sequence": [ | |
| "</s>", | |
| "Q:", | |
| "\n\n" | |
| ], | |
| "output_regex": null, | |
| "frozen": false, | |
| "suite": [ | |
| "harness" | |
| ], | |
| "original_num_docs": 250, | |
| "effective_num_docs": 250, | |
| "trust_dataset": true, | |
| "must_remove_duplicate_docs": null | |
| }, | |
| "harness|bbh:reasoning_about_colored_objects": { | |
| "name": "bbh:reasoning_about_colored_objects", | |
| "prompt_function": "bbh_reasoning_about_colored_objects", | |
| "hf_repo": "lukaemon/bbh", | |
| "hf_subset": "reasoning_about_colored_objects", | |
| "metric": [ | |
| "exact_match", | |
| "quasi_exact_match", | |
| "prefix_exact_match", | |
| "prefix_quasi_exact_match", | |
| "perfect_exact_match" | |
| ], | |
| "hf_avail_splits": [ | |
| "test" | |
| ], | |
| "evaluation_splits": [ | |
| "test" | |
| ], | |
| "few_shots_split": null, | |
| "few_shots_select": null, | |
| "generation_size": 20, | |
| "stop_sequence": [ | |
| "</s>", | |
| "Q:", | |
| "\n\n" | |
| ], | |
| "output_regex": null, | |
| "frozen": false, | |
| "suite": [ | |
| "harness" | |
| ], | |
| "original_num_docs": 250, | |
| "effective_num_docs": 250, | |
| "trust_dataset": true, | |
| "must_remove_duplicate_docs": null | |
| }, | |
| "harness|bbh:ruin_names": { | |
| "name": "bbh:ruin_names", | |
| "prompt_function": "bbh_ruin_names", | |
| "hf_repo": "lukaemon/bbh", | |
| "hf_subset": "ruin_names", | |
| "metric": [ | |
| "exact_match", | |
| "quasi_exact_match", | |
| "prefix_exact_match", | |
| "prefix_quasi_exact_match", | |
| "perfect_exact_match" | |
| ], | |
| "hf_avail_splits": [ | |
| "test" | |
| ], | |
| "evaluation_splits": [ | |
| "test" | |
| ], | |
| "few_shots_split": null, | |
| "few_shots_select": null, | |
| "generation_size": 20, | |
| "stop_sequence": [ | |
| "</s>", | |
| "Q:", | |
| "\n\n" | |
| ], | |
| "output_regex": null, | |
| "frozen": false, | |
| "suite": [ | |
| "harness" | |
| ], | |
| "original_num_docs": 248, | |
| "effective_num_docs": 248, | |
| "trust_dataset": true, | |
| "must_remove_duplicate_docs": null | |
| }, | |
| "harness|bbh:salient_translation_error_detection": { | |
| "name": "bbh:salient_translation_error_detection", | |
| "prompt_function": "bbh_salient_translation_error_detection", | |
| "hf_repo": "lukaemon/bbh", | |
| "hf_subset": "salient_translation_error_detection", | |
| "metric": [ | |
| "exact_match", | |
| "quasi_exact_match", | |
| "prefix_exact_match", | |
| "prefix_quasi_exact_match", | |
| "perfect_exact_match" | |
| ], | |
| "hf_avail_splits": [ | |
| "test" | |
| ], | |
| "evaluation_splits": [ | |
| "test" | |
| ], | |
| "few_shots_split": null, | |
| "few_shots_select": null, | |
| "generation_size": 20, | |
| "stop_sequence": [ | |
| "</s>", | |
| "Q:", | |
| "\n\n" | |
| ], | |
| "output_regex": null, | |
| "frozen": false, | |
| "suite": [ | |
| "harness" | |
| ], | |
| "original_num_docs": 250, | |
| "effective_num_docs": 250, | |
| "trust_dataset": true, | |
| "must_remove_duplicate_docs": null | |
| }, | |
| "harness|bbh:snarks": { | |
| "name": "bbh:snarks", | |
| "prompt_function": "bbh_snarks", | |
| "hf_repo": "lukaemon/bbh", | |
| "hf_subset": "snarks", | |
| "metric": [ | |
| "exact_match", | |
| "quasi_exact_match", | |
| "prefix_exact_match", | |
| "prefix_quasi_exact_match", | |
| "perfect_exact_match" | |
| ], | |
| "hf_avail_splits": [ | |
| "test" | |
| ], | |
| "evaluation_splits": [ | |
| "test" | |
| ], | |
| "few_shots_split": null, | |
| "few_shots_select": null, | |
| "generation_size": 20, | |
| "stop_sequence": [ | |
| "</s>", | |
| "Q:", | |
| "\n\n" | |
| ], | |
| "output_regex": null, | |
| "frozen": false, | |
| "suite": [ | |
| "harness" | |
| ], | |
| "original_num_docs": 178, | |
| "effective_num_docs": 178, | |
| "trust_dataset": true, | |
| "must_remove_duplicate_docs": null | |
| }, | |
| "harness|bbh:sports_understanding": { | |
| "name": "bbh:sports_understanding", | |
| "prompt_function": "bbh_sports_understanding", | |
| "hf_repo": "lukaemon/bbh", | |
| "hf_subset": "sports_understanding", | |
| "metric": [ | |
| "exact_match", | |
| "quasi_exact_match", | |
| "prefix_exact_match", | |
| "prefix_quasi_exact_match", | |
| "perfect_exact_match" | |
| ], | |
| "hf_avail_splits": [ | |
| "test" | |
| ], | |
| "evaluation_splits": [ | |
| "test" | |
| ], | |
| "few_shots_split": null, | |
| "few_shots_select": null, | |
| "generation_size": 20, | |
| "stop_sequence": [ | |
| "</s>", | |
| "Q:", | |
| "\n\n" | |
| ], | |
| "output_regex": null, | |
| "frozen": false, | |
| "suite": [ | |
| "harness" | |
| ], | |
| "original_num_docs": 250, | |
| "effective_num_docs": 250, | |
| "trust_dataset": true, | |
| "must_remove_duplicate_docs": null | |
| }, | |
| "harness|bbh:temporal_sequences": { | |
| "name": "bbh:temporal_sequences", | |
| "prompt_function": "bbh_temporal_sequences", | |
| "hf_repo": "lukaemon/bbh", | |
| "hf_subset": "temporal_sequences", | |
| "metric": [ | |
| "exact_match", | |
| "quasi_exact_match", | |
| "prefix_exact_match", | |
| "prefix_quasi_exact_match", | |
| "perfect_exact_match" | |
| ], | |
| "hf_avail_splits": [ | |
| "test" | |
| ], | |
| "evaluation_splits": [ | |
| "test" | |
| ], | |
| "few_shots_split": null, | |
| "few_shots_select": null, | |
| "generation_size": 20, | |
| "stop_sequence": [ | |
| "</s>", | |
| "Q:", | |
| "\n\n" | |
| ], | |
| "output_regex": null, | |
| "frozen": false, | |
| "suite": [ | |
| "harness" | |
| ], | |
| "original_num_docs": 250, | |
| "effective_num_docs": 250, | |
| "trust_dataset": true, | |
| "must_remove_duplicate_docs": null | |
| }, | |
| "harness|bbh:tracking_shuffled_objects_five_objects": { | |
| "name": "bbh:tracking_shuffled_objects_five_objects", | |
| "prompt_function": "bbh_tracking_shuffled_objects_five_objects", | |
| "hf_repo": "lukaemon/bbh", | |
| "hf_subset": "tracking_shuffled_objects_five_objects", | |
| "metric": [ | |
| "exact_match", | |
| "quasi_exact_match", | |
| "prefix_exact_match", | |
| "prefix_quasi_exact_match", | |
| "perfect_exact_match" | |
| ], | |
| "hf_avail_splits": [ | |
| "test" | |
| ], | |
| "evaluation_splits": [ | |
| "test" | |
| ], | |
| "few_shots_split": null, | |
| "few_shots_select": null, | |
| "generation_size": 20, | |
| "stop_sequence": [ | |
| "</s>", | |
| "Q:", | |
| "\n\n" | |
| ], | |
| "output_regex": null, | |
| "frozen": false, | |
| "suite": [ | |
| "harness" | |
| ], | |
| "original_num_docs": 250, | |
| "effective_num_docs": 250, | |
| "trust_dataset": true, | |
| "must_remove_duplicate_docs": null | |
| }, | |
| "harness|bbh:tracking_shuffled_objects_seven_objects": { | |
| "name": "bbh:tracking_shuffled_objects_seven_objects", | |
| "prompt_function": "bbh_tracking_shuffled_objects_seven_objects", | |
| "hf_repo": "lukaemon/bbh", | |
| "hf_subset": "tracking_shuffled_objects_seven_objects", | |
| "metric": [ | |
| "exact_match", | |
| "quasi_exact_match", | |
| "prefix_exact_match", | |
| "prefix_quasi_exact_match", | |
| "perfect_exact_match" | |
| ], | |
| "hf_avail_splits": [ | |
| "test" | |
| ], | |
| "evaluation_splits": [ | |
| "test" | |
| ], | |
| "few_shots_split": null, | |
| "few_shots_select": null, | |
| "generation_size": 20, | |
| "stop_sequence": [ | |
| "</s>", | |
| "Q:", | |
| "\n\n" | |
| ], | |
| "output_regex": null, | |
| "frozen": false, | |
| "suite": [ | |
| "harness" | |
| ], | |
| "original_num_docs": 250, | |
| "effective_num_docs": 250, | |
| "trust_dataset": true, | |
| "must_remove_duplicate_docs": null | |
| }, | |
| "harness|bbh:tracking_shuffled_objects_three_objects": { | |
| "name": "bbh:tracking_shuffled_objects_three_objects", | |
| "prompt_function": "bbh_tracking_shuffled_objects_three_objects", | |
| "hf_repo": "lukaemon/bbh", | |
| "hf_subset": "tracking_shuffled_objects_three_objects", | |
| "metric": [ | |
| "exact_match", | |
| "quasi_exact_match", | |
| "prefix_exact_match", | |
| "prefix_quasi_exact_match", | |
| "perfect_exact_match" | |
| ], | |
| "hf_avail_splits": [ | |
| "test" | |
| ], | |
| "evaluation_splits": [ | |
| "test" | |
| ], | |
| "few_shots_split": null, | |
| "few_shots_select": null, | |
| "generation_size": 20, | |
| "stop_sequence": [ | |
| "</s>", | |
| "Q:", | |
| "\n\n" | |
| ], | |
| "output_regex": null, | |
| "frozen": false, | |
| "suite": [ | |
| "harness" | |
| ], | |
| "original_num_docs": 250, | |
| "effective_num_docs": 250, | |
| "trust_dataset": true, | |
| "must_remove_duplicate_docs": null | |
| } | |
| }, | |
| "summary_tasks": { | |
| "harness|bbh:causal_judgment|3": { | |
| "hashes": { | |
| "hash_examples": "63218f5ae055ab2b", | |
| "hash_full_prompts": "da4486f76ca37d71", | |
| "hash_input_tokens": "2f4439cd00aeae0e", | |
| "hash_cont_tokens": "1478f84094c52bb4" | |
| }, | |
| "truncated": 187, | |
| "non_truncated": 0, | |
| "padded": 0, | |
| "non_padded": 187, | |
| "effective_few_shots": 3.0, | |
| "num_truncated_few_shots": 0 | |
| }, | |
| "harness|bbh:date_understanding|3": { | |
| "hashes": { | |
| "hash_examples": "f145c7a06def3c8e", | |
| "hash_full_prompts": "ab81948b0abe57b9", | |
| "hash_input_tokens": "d36736b69b6676a1", | |
| "hash_cont_tokens": "45ecc819fa52c55c" | |
| }, | |
| "truncated": 250, | |
| "non_truncated": 0, | |
| "padded": 0, | |
| "non_padded": 250, | |
| "effective_few_shots": 3.0, | |
| "num_truncated_few_shots": 0 | |
| }, | |
| "harness|bbh:disambiguation_qa|3": { | |
| "hashes": { | |
| "hash_examples": "19677fd1773f7eb9", | |
| "hash_full_prompts": "7a5061f70150c09e", | |
| "hash_input_tokens": "1f6e1e8254e2c651", | |
| "hash_cont_tokens": "3c8b7521886046ae" | |
| }, | |
| "truncated": 250, | |
| "non_truncated": 0, | |
| "padded": 0, | |
| "non_padded": 250, | |
| "effective_few_shots": 3.0, | |
| "num_truncated_few_shots": 0 | |
| }, | |
| "harness|bbh:geometric_shapes|3": { | |
| "hashes": { | |
| "hash_examples": "76c7b11a13cc72a9", | |
| "hash_full_prompts": "e063deff2e84a2a6", | |
| "hash_input_tokens": "f6cae0e55367ae6c", | |
| "hash_cont_tokens": "dafd4cc6d39b3660" | |
| }, | |
| "truncated": 250, | |
| "non_truncated": 0, | |
| "padded": 0, | |
| "non_padded": 250, | |
| "effective_few_shots": 3.0, | |
| "num_truncated_few_shots": 0 | |
| }, | |
| "harness|bbh:logical_deduction_five_objects|3": { | |
| "hashes": { | |
| "hash_examples": "0e958c856332a745", | |
| "hash_full_prompts": "7544b780705e9654", | |
| "hash_input_tokens": "069f2e5e4f4819e1", | |
| "hash_cont_tokens": "becfc64d5179c60b" | |
| }, | |
| "truncated": 250, | |
| "non_truncated": 0, | |
| "padded": 0, | |
| "non_padded": 250, | |
| "effective_few_shots": 3.0, | |
| "num_truncated_few_shots": 0 | |
| }, | |
| "harness|bbh:logical_deduction_seven_objects|3": { | |
| "hashes": { | |
| "hash_examples": "ab9de25a5eb40d09", | |
| "hash_full_prompts": "9ba71ddb2add0b58", | |
| "hash_input_tokens": "61154b5b2d23fd61", | |
| "hash_cont_tokens": "5ab6772d0e983d20" | |
| }, | |
| "truncated": 250, | |
| "non_truncated": 0, | |
| "padded": 0, | |
| "non_padded": 250, | |
| "effective_few_shots": 3.0, | |
| "num_truncated_few_shots": 0 | |
| }, | |
| "harness|bbh:logical_deduction_three_objects|3": { | |
| "hashes": { | |
| "hash_examples": "3c6bf52517714218", | |
| "hash_full_prompts": "78aebd4f41995a84", | |
| "hash_input_tokens": "8611de524f2cb180", | |
| "hash_cont_tokens": "839ba0f1b24b0dc2" | |
| }, | |
| "truncated": 250, | |
| "non_truncated": 0, | |
| "padded": 0, | |
| "non_padded": 250, | |
| "effective_few_shots": 3.0, | |
| "num_truncated_few_shots": 0 | |
| }, | |
| "harness|bbh:movie_recommendation|3": { | |
| "hashes": { | |
| "hash_examples": "2d9dc4975935d31a", | |
| "hash_full_prompts": "290a864a2acc3828", | |
| "hash_input_tokens": "dd866e9aefb525a2", | |
| "hash_cont_tokens": "510f5e07a5c4999c" | |
| }, | |
| "truncated": 249, | |
| "non_truncated": 0, | |
| "padded": 0, | |
| "non_padded": 249, | |
| "effective_few_shots": 3.0, | |
| "num_truncated_few_shots": 0 | |
| }, | |
| "harness|bbh:navigate|3": { | |
| "hashes": { | |
| "hash_examples": "ba91dcdb9a064255", | |
| "hash_full_prompts": "4d8fad912ba90363", | |
| "hash_input_tokens": "aea22a5b526e9934", | |
| "hash_cont_tokens": "e3a02b03316478f9" | |
| }, | |
| "truncated": 250, | |
| "non_truncated": 0, | |
| "padded": 0, | |
| "non_padded": 250, | |
| "effective_few_shots": 3.0, | |
| "num_truncated_few_shots": 0 | |
| }, | |
| "harness|bbh:reasoning_about_colored_objects|3": { | |
| "hashes": { | |
| "hash_examples": "a6ba328c4c3385d2", | |
| "hash_full_prompts": "96ca31772c57eeeb", | |
| "hash_input_tokens": "f31e7a9ef88c8872", | |
| "hash_cont_tokens": "f933686f8b1e23a4" | |
| }, | |
| "truncated": 250, | |
| "non_truncated": 0, | |
| "padded": 0, | |
| "non_padded": 250, | |
| "effective_few_shots": 3.0, | |
| "num_truncated_few_shots": 0 | |
| }, | |
| "harness|bbh:ruin_names|3": { | |
| "hashes": { | |
| "hash_examples": "2ef28d5f2d4fdd25", | |
| "hash_full_prompts": "f3ad065935c46de7", | |
| "hash_input_tokens": "5cc6ead631865b9a", | |
| "hash_cont_tokens": "2eb1209a83b1d09e" | |
| }, | |
| "truncated": 248, | |
| "non_truncated": 0, | |
| "padded": 0, | |
| "non_padded": 248, | |
| "effective_few_shots": 3.0, | |
| "num_truncated_few_shots": 0 | |
| }, | |
| "harness|bbh:salient_translation_error_detection|3": { | |
| "hashes": { | |
| "hash_examples": "c13f25ec8ffed496", | |
| "hash_full_prompts": "80fc33e0cd811a8d", | |
| "hash_input_tokens": "6004f29b100793e5", | |
| "hash_cont_tokens": "f39b3fc4a19d51f6" | |
| }, | |
| "truncated": 250, | |
| "non_truncated": 0, | |
| "padded": 0, | |
| "non_padded": 250, | |
| "effective_few_shots": 3.0, | |
| "num_truncated_few_shots": 0 | |
| }, | |
| "harness|bbh:snarks|3": { | |
| "hashes": { | |
| "hash_examples": "5f6db7bff7f6f22e", | |
| "hash_full_prompts": "55586142735dea08", | |
| "hash_input_tokens": "f78df86ae6e4e291", | |
| "hash_cont_tokens": "053f4b3ea75f4de6" | |
| }, | |
| "truncated": 178, | |
| "non_truncated": 0, | |
| "padded": 0, | |
| "non_padded": 178, | |
| "effective_few_shots": 3.0, | |
| "num_truncated_few_shots": 0 | |
| }, | |
| "harness|bbh:sports_understanding|3": { | |
| "hashes": { | |
| "hash_examples": "042afbe5d9c1f02d", | |
| "hash_full_prompts": "1c523efd25e2ebfd", | |
| "hash_input_tokens": "176635253aa6d0e9", | |
| "hash_cont_tokens": "b66980c534eac2de" | |
| }, | |
| "truncated": 250, | |
| "non_truncated": 0, | |
| "padded": 0, | |
| "non_padded": 250, | |
| "effective_few_shots": 3.0, | |
| "num_truncated_few_shots": 0 | |
| }, | |
| "harness|bbh:temporal_sequences|3": { | |
| "hashes": { | |
| "hash_examples": "803a05f352eb6afc", | |
| "hash_full_prompts": "caa56ea50f0068e1", | |
| "hash_input_tokens": "74b504d9b40add38", | |
| "hash_cont_tokens": "634c4602e9316257" | |
| }, | |
| "truncated": 250, | |
| "non_truncated": 0, | |
| "padded": 0, | |
| "non_padded": 250, | |
| "effective_few_shots": 3.0, | |
| "num_truncated_few_shots": 0 | |
| }, | |
| "harness|bbh:tracking_shuffled_objects_five_objects|3": { | |
| "hashes": { | |
| "hash_examples": "2bbac6db7ab0d527", | |
| "hash_full_prompts": "4c9e2f3952f99f7d", | |
| "hash_input_tokens": "36d0e7377cce953e", | |
| "hash_cont_tokens": "f59c1a7a0882977b" | |
| }, | |
| "truncated": 250, | |
| "non_truncated": 0, | |
| "padded": 0, | |
| "non_padded": 250, | |
| "effective_few_shots": 3.0, | |
| "num_truncated_few_shots": 0 | |
| }, | |
| "harness|bbh:tracking_shuffled_objects_seven_objects|3": { | |
| "hashes": { | |
| "hash_examples": "845caf093ac2b58c", | |
| "hash_full_prompts": "d4f4f0656462d45f", | |
| "hash_input_tokens": "d255d2f245886c58", | |
| "hash_cont_tokens": "6374bde1214f79d7" | |
| }, | |
| "truncated": 250, | |
| "non_truncated": 0, | |
| "padded": 0, | |
| "non_padded": 250, | |
| "effective_few_shots": 3.0, | |
| "num_truncated_few_shots": 0 | |
| }, | |
| "harness|bbh:tracking_shuffled_objects_three_objects|3": { | |
| "hashes": { | |
| "hash_examples": "9004f14d5a32b9a8", | |
| "hash_full_prompts": "be9c55b344ca07ab", | |
| "hash_input_tokens": "55c63804a84860f6", | |
| "hash_cont_tokens": "583fdb1bded8b71b" | |
| }, | |
| "truncated": 250, | |
| "non_truncated": 0, | |
| "padded": 0, | |
| "non_padded": 250, | |
| "effective_few_shots": 3.0, | |
| "num_truncated_few_shots": 0 | |
| } | |
| }, | |
| "summary_general": { | |
| "hashes": { | |
| "hash_examples": "4ff1e3dc5703575d", | |
| "hash_full_prompts": "ec35636f85fd5ee2", | |
| "hash_input_tokens": "016924e640ab4527", | |
| "hash_cont_tokens": "7fc2f6f782e5890b" | |
| }, | |
| "truncated": 4362, | |
| "non_truncated": 0, | |
| "padded": 0, | |
| "non_padded": 4362, | |
| "num_truncated_few_shots": 0 | |
| } | |
| } |