open-r1-eval-leaderboard
/
eval_results
/Nexusflow
/Starling-LM-7B-beta
/main
/bbh
/results_2024-03-28T19-46-29.903219.json
| { | |
| "config_general": { | |
| "lighteval_sha": "?", | |
| "num_fewshot_seeds": 1, | |
| "override_batch_size": 1, | |
| "max_samples": null, | |
| "job_id": "", | |
| "start_time": 240766.002289505, | |
| "end_time": 240991.609879191, | |
| "total_evaluation_time_secondes": "225.6075896859984", | |
| "model_name": "Nexusflow/Starling-LM-7B-beta", | |
| "model_sha": "aa21e7f117d41f9463b6d48d2e127bbf2e93256d", | |
| "model_dtype": "torch.bfloat16", | |
| "model_size": "13.61 GB", | |
| "config": null | |
| }, | |
| "results": { | |
| "lighteval|bigbench:causal_judgment|0": { | |
| "acc": 0.5842105263157895, | |
| "acc_stderr": 0.0358501132552001 | |
| }, | |
| "lighteval|bigbench:date_understanding|0": { | |
| "acc": 0.5609756097560976, | |
| "acc_stderr": 0.02586976028731276 | |
| }, | |
| "lighteval|bigbench:disambiguation_qa|0": { | |
| "acc": 0.6627906976744186, | |
| "acc_stderr": 0.029489771935966462 | |
| }, | |
| "lighteval|bigbench:geometric_shapes|0": { | |
| "acc": 0.11388888888888889, | |
| "acc_stderr": 0.01676631204862353 | |
| }, | |
| "lighteval|bigbench:logical_deduction_five_objects|0": { | |
| "acc": 0.274, | |
| "acc_stderr": 0.019966103540279476 | |
| }, | |
| "lighteval|bigbench:logical_deduction_seven_objects|0": { | |
| "acc": 0.17714285714285713, | |
| "acc_stderr": 0.014440607250978076 | |
| }, | |
| "lighteval|bigbench:logical_deduction_three_objects|0": { | |
| "acc": 0.45666666666666667, | |
| "acc_stderr": 0.02880694721939613 | |
| }, | |
| "lighteval|bigbench:movie_recommendation|0": { | |
| "acc": 0.656, | |
| "acc_stderr": 0.02126575803797874 | |
| }, | |
| "lighteval|bigbench:navigate|0": { | |
| "acc": 0.5, | |
| "acc_stderr": 0.015819299929208316 | |
| }, | |
| "lighteval|bigbench:reasoning_about_colored_objects|0": { | |
| "acc": 0.486, | |
| "acc_stderr": 0.011178751372184865 | |
| }, | |
| "lighteval|bigbench:ruin_names|0": { | |
| "acc": 0.5267857142857143, | |
| "acc_stderr": 0.023615228730356076 | |
| }, | |
| "lighteval|bigbench:salient_translation_error_detection|0": { | |
| "acc": 0.37374749498997994, | |
| "acc_stderr": 0.015322029851655386 | |
| }, | |
| "lighteval|bigbench:snarks|0": { | |
| "acc": 0.5082872928176796, | |
| "acc_stderr": 0.03726268022638987 | |
| }, | |
| "lighteval|bigbench:sports_understanding|0": { | |
| "acc": 0.734, | |
| "acc_stderr": 0.013979965645145158 | |
| }, | |
| "lighteval|bigbench:temporal_sequences|0": { | |
| "acc": 0.284, | |
| "acc_stderr": 0.014267009061031307 | |
| }, | |
| "lighteval|bigbench:tracking_shuffled_objects_five_objects|0": { | |
| "acc": 0.1544, | |
| "acc_stderr": 0.010224095936080034 | |
| }, | |
| "lighteval|bigbench:tracking_shuffled_objects_seven_objects|0": { | |
| "acc": 0.10857142857142857, | |
| "acc_stderr": 0.007438855904603836 | |
| }, | |
| "lighteval|bigbench:tracking_shuffled_objects_three_objects|0": { | |
| "acc": 0.45666666666666667, | |
| "acc_stderr": 0.02880694721939613 | |
| }, | |
| "lighteval|bigbench:_average|0": { | |
| "acc": 0.423229657987566, | |
| "acc_stderr": 0.020576124302877013 | |
| }, | |
| "all": { | |
| "acc": 0.423229657987566, | |
| "acc_stderr": 0.020576124302877013 | |
| } | |
| }, | |
| "versions": { | |
| "lighteval|bigbench:causal_judgment|0": 0, | |
| "lighteval|bigbench:date_understanding|0": 0, | |
| "lighteval|bigbench:disambiguation_qa|0": 0, | |
| "lighteval|bigbench:geometric_shapes|0": 0, | |
| "lighteval|bigbench:logical_deduction_five_objects|0": 0, | |
| "lighteval|bigbench:logical_deduction_seven_objects|0": 0, | |
| "lighteval|bigbench:logical_deduction_three_objects|0": 0, | |
| "lighteval|bigbench:movie_recommendation|0": 0, | |
| "lighteval|bigbench:navigate|0": 0, | |
| "lighteval|bigbench:reasoning_about_colored_objects|0": 0, | |
| "lighteval|bigbench:ruin_names|0": 0, | |
| "lighteval|bigbench:salient_translation_error_detection|0": 0, | |
| "lighteval|bigbench:snarks|0": 0, | |
| "lighteval|bigbench:sports_understanding|0": 0, | |
| "lighteval|bigbench:temporal_sequences|0": 0, | |
| "lighteval|bigbench:tracking_shuffled_objects_five_objects|0": 0, | |
| "lighteval|bigbench:tracking_shuffled_objects_seven_objects|0": 0, | |
| "lighteval|bigbench:tracking_shuffled_objects_three_objects|0": 0 | |
| }, | |
| "config_tasks": { | |
| "lighteval|bigbench:causal_judgment": { | |
| "name": "bigbench:causal_judgment", | |
| "prompt_function": "bbh_lighteval", | |
| "hf_repo": "lighteval/bbh", | |
| "hf_subset": "causal_judgement", | |
| "metric": [ | |
| "loglikelihood_acc_single_token" | |
| ], | |
| "hf_avail_splits": [ | |
| "train" | |
| ], | |
| "evaluation_splits": [ | |
| "train" | |
| ], | |
| "few_shots_split": null, | |
| "few_shots_select": null, | |
| "generation_size": -1, | |
| "stop_sequence": [ | |
| "</s>", | |
| "Q:", | |
| "\n\n" | |
| ], | |
| "output_regex": null, | |
| "frozen": false, | |
| "suite": [ | |
| "lighteval" | |
| ], | |
| "original_num_docs": 190, | |
| "effective_num_docs": 190, | |
| "trust_dataset": true, | |
| "must_remove_duplicate_docs": null | |
| }, | |
| "lighteval|bigbench:date_understanding": { | |
| "name": "bigbench:date_understanding", | |
| "prompt_function": "bbh_lighteval", | |
| "hf_repo": "lighteval/bbh", | |
| "hf_subset": "date_understanding", | |
| "metric": [ | |
| "loglikelihood_acc_single_token" | |
| ], | |
| "hf_avail_splits": [ | |
| "train" | |
| ], | |
| "evaluation_splits": [ | |
| "train" | |
| ], | |
| "few_shots_split": null, | |
| "few_shots_select": null, | |
| "generation_size": -1, | |
| "stop_sequence": [ | |
| "</s>", | |
| "Q:", | |
| "\n\n" | |
| ], | |
| "output_regex": null, | |
| "frozen": false, | |
| "suite": [ | |
| "lighteval" | |
| ], | |
| "original_num_docs": 369, | |
| "effective_num_docs": 369, | |
| "trust_dataset": true, | |
| "must_remove_duplicate_docs": null | |
| }, | |
| "lighteval|bigbench:disambiguation_qa": { | |
| "name": "bigbench:disambiguation_qa", | |
| "prompt_function": "bbh_lighteval", | |
| "hf_repo": "lighteval/bbh", | |
| "hf_subset": "disambiguation_qa", | |
| "metric": [ | |
| "loglikelihood_acc_single_token" | |
| ], | |
| "hf_avail_splits": [ | |
| "train" | |
| ], | |
| "evaluation_splits": [ | |
| "train" | |
| ], | |
| "few_shots_split": null, | |
| "few_shots_select": null, | |
| "generation_size": -1, | |
| "stop_sequence": [ | |
| "</s>", | |
| "Q:", | |
| "\n\n" | |
| ], | |
| "output_regex": null, | |
| "frozen": false, | |
| "suite": [ | |
| "lighteval" | |
| ], | |
| "original_num_docs": 258, | |
| "effective_num_docs": 258, | |
| "trust_dataset": true, | |
| "must_remove_duplicate_docs": null | |
| }, | |
| "lighteval|bigbench:geometric_shapes": { | |
| "name": "bigbench:geometric_shapes", | |
| "prompt_function": "bbh_lighteval", | |
| "hf_repo": "lighteval/bbh", | |
| "hf_subset": "geometric_shapes", | |
| "metric": [ | |
| "loglikelihood_acc_single_token" | |
| ], | |
| "hf_avail_splits": [ | |
| "train" | |
| ], | |
| "evaluation_splits": [ | |
| "train" | |
| ], | |
| "few_shots_split": null, | |
| "few_shots_select": null, | |
| "generation_size": -1, | |
| "stop_sequence": [ | |
| "</s>", | |
| "Q:", | |
| "\n\n" | |
| ], | |
| "output_regex": null, | |
| "frozen": false, | |
| "suite": [ | |
| "lighteval" | |
| ], | |
| "original_num_docs": 360, | |
| "effective_num_docs": 360, | |
| "trust_dataset": true, | |
| "must_remove_duplicate_docs": null | |
| }, | |
| "lighteval|bigbench:logical_deduction_five_objects": { | |
| "name": "bigbench:logical_deduction_five_objects", | |
| "prompt_function": "bbh_lighteval", | |
| "hf_repo": "lighteval/bbh", | |
| "hf_subset": "logical_deduction_five_objects", | |
| "metric": [ | |
| "loglikelihood_acc_single_token" | |
| ], | |
| "hf_avail_splits": [ | |
| "train" | |
| ], | |
| "evaluation_splits": [ | |
| "train" | |
| ], | |
| "few_shots_split": null, | |
| "few_shots_select": null, | |
| "generation_size": -1, | |
| "stop_sequence": [ | |
| "</s>", | |
| "Q:", | |
| "\n\n" | |
| ], | |
| "output_regex": null, | |
| "frozen": false, | |
| "suite": [ | |
| "lighteval" | |
| ], | |
| "original_num_docs": 500, | |
| "effective_num_docs": 500, | |
| "trust_dataset": true, | |
| "must_remove_duplicate_docs": null | |
| }, | |
| "lighteval|bigbench:logical_deduction_seven_objects": { | |
| "name": "bigbench:logical_deduction_seven_objects", | |
| "prompt_function": "bbh_lighteval", | |
| "hf_repo": "lighteval/bbh", | |
| "hf_subset": "logical_deduction_seven_objects", | |
| "metric": [ | |
| "loglikelihood_acc_single_token" | |
| ], | |
| "hf_avail_splits": [ | |
| "train" | |
| ], | |
| "evaluation_splits": [ | |
| "train" | |
| ], | |
| "few_shots_split": null, | |
| "few_shots_select": null, | |
| "generation_size": -1, | |
| "stop_sequence": [ | |
| "</s>", | |
| "Q:", | |
| "\n\n" | |
| ], | |
| "output_regex": null, | |
| "frozen": false, | |
| "suite": [ | |
| "lighteval" | |
| ], | |
| "original_num_docs": 700, | |
| "effective_num_docs": 700, | |
| "trust_dataset": true, | |
| "must_remove_duplicate_docs": null | |
| }, | |
| "lighteval|bigbench:logical_deduction_three_objects": { | |
| "name": "bigbench:logical_deduction_three_objects", | |
| "prompt_function": "bbh_lighteval", | |
| "hf_repo": "lighteval/bbh", | |
| "hf_subset": "logical_deduction_three_objects", | |
| "metric": [ | |
| "loglikelihood_acc_single_token" | |
| ], | |
| "hf_avail_splits": [ | |
| "train" | |
| ], | |
| "evaluation_splits": [ | |
| "train" | |
| ], | |
| "few_shots_split": null, | |
| "few_shots_select": null, | |
| "generation_size": -1, | |
| "stop_sequence": [ | |
| "</s>", | |
| "Q:", | |
| "\n\n" | |
| ], | |
| "output_regex": null, | |
| "frozen": false, | |
| "suite": [ | |
| "lighteval" | |
| ], | |
| "original_num_docs": 300, | |
| "effective_num_docs": 300, | |
| "trust_dataset": true, | |
| "must_remove_duplicate_docs": null | |
| }, | |
| "lighteval|bigbench:movie_recommendation": { | |
| "name": "bigbench:movie_recommendation", | |
| "prompt_function": "bbh_lighteval", | |
| "hf_repo": "lighteval/bbh", | |
| "hf_subset": "movie_recommendation", | |
| "metric": [ | |
| "loglikelihood_acc_single_token" | |
| ], | |
| "hf_avail_splits": [ | |
| "train" | |
| ], | |
| "evaluation_splits": [ | |
| "train" | |
| ], | |
| "few_shots_split": null, | |
| "few_shots_select": null, | |
| "generation_size": -1, | |
| "stop_sequence": [ | |
| "</s>", | |
| "Q:", | |
| "\n\n" | |
| ], | |
| "output_regex": null, | |
| "frozen": false, | |
| "suite": [ | |
| "lighteval" | |
| ], | |
| "original_num_docs": 500, | |
| "effective_num_docs": 500, | |
| "trust_dataset": true, | |
| "must_remove_duplicate_docs": null | |
| }, | |
| "lighteval|bigbench:navigate": { | |
| "name": "bigbench:navigate", | |
| "prompt_function": "bbh_lighteval", | |
| "hf_repo": "lighteval/bbh", | |
| "hf_subset": "navigate", | |
| "metric": [ | |
| "loglikelihood_acc_single_token" | |
| ], | |
| "hf_avail_splits": [ | |
| "train" | |
| ], | |
| "evaluation_splits": [ | |
| "train" | |
| ], | |
| "few_shots_split": null, | |
| "few_shots_select": null, | |
| "generation_size": -1, | |
| "stop_sequence": [ | |
| "</s>", | |
| "Q:", | |
| "\n\n" | |
| ], | |
| "output_regex": null, | |
| "frozen": false, | |
| "suite": [ | |
| "lighteval" | |
| ], | |
| "original_num_docs": 1000, | |
| "effective_num_docs": 1000, | |
| "trust_dataset": true, | |
| "must_remove_duplicate_docs": null | |
| }, | |
| "lighteval|bigbench:reasoning_about_colored_objects": { | |
| "name": "bigbench:reasoning_about_colored_objects", | |
| "prompt_function": "bbh_lighteval", | |
| "hf_repo": "lighteval/bbh", | |
| "hf_subset": "reasoning_about_colored_objects", | |
| "metric": [ | |
| "loglikelihood_acc_single_token" | |
| ], | |
| "hf_avail_splits": [ | |
| "train" | |
| ], | |
| "evaluation_splits": [ | |
| "train" | |
| ], | |
| "few_shots_split": null, | |
| "few_shots_select": null, | |
| "generation_size": -1, | |
| "stop_sequence": [ | |
| "</s>", | |
| "Q:", | |
| "\n\n" | |
| ], | |
| "output_regex": null, | |
| "frozen": false, | |
| "suite": [ | |
| "lighteval" | |
| ], | |
| "original_num_docs": 2000, | |
| "effective_num_docs": 2000, | |
| "trust_dataset": true, | |
| "must_remove_duplicate_docs": null | |
| }, | |
| "lighteval|bigbench:ruin_names": { | |
| "name": "bigbench:ruin_names", | |
| "prompt_function": "bbh_lighteval", | |
| "hf_repo": "lighteval/bbh", | |
| "hf_subset": "ruin_names", | |
| "metric": [ | |
| "loglikelihood_acc_single_token" | |
| ], | |
| "hf_avail_splits": [ | |
| "train" | |
| ], | |
| "evaluation_splits": [ | |
| "train" | |
| ], | |
| "few_shots_split": null, | |
| "few_shots_select": null, | |
| "generation_size": -1, | |
| "stop_sequence": [ | |
| "</s>", | |
| "Q:", | |
| "\n\n" | |
| ], | |
| "output_regex": null, | |
| "frozen": false, | |
| "suite": [ | |
| "lighteval" | |
| ], | |
| "original_num_docs": 448, | |
| "effective_num_docs": 448, | |
| "trust_dataset": true, | |
| "must_remove_duplicate_docs": null | |
| }, | |
| "lighteval|bigbench:salient_translation_error_detection": { | |
| "name": "bigbench:salient_translation_error_detection", | |
| "prompt_function": "bbh_lighteval", | |
| "hf_repo": "lighteval/bbh", | |
| "hf_subset": "salient_translation_error_detection", | |
| "metric": [ | |
| "loglikelihood_acc_single_token" | |
| ], | |
| "hf_avail_splits": [ | |
| "train" | |
| ], | |
| "evaluation_splits": [ | |
| "train" | |
| ], | |
| "few_shots_split": null, | |
| "few_shots_select": null, | |
| "generation_size": -1, | |
| "stop_sequence": [ | |
| "</s>", | |
| "Q:", | |
| "\n\n" | |
| ], | |
| "output_regex": null, | |
| "frozen": false, | |
| "suite": [ | |
| "lighteval" | |
| ], | |
| "original_num_docs": 998, | |
| "effective_num_docs": 998, | |
| "trust_dataset": true, | |
| "must_remove_duplicate_docs": null | |
| }, | |
| "lighteval|bigbench:snarks": { | |
| "name": "bigbench:snarks", | |
| "prompt_function": "bbh_lighteval", | |
| "hf_repo": "lighteval/bbh", | |
| "hf_subset": "snarks", | |
| "metric": [ | |
| "loglikelihood_acc_single_token" | |
| ], | |
| "hf_avail_splits": [ | |
| "train" | |
| ], | |
| "evaluation_splits": [ | |
| "train" | |
| ], | |
| "few_shots_split": null, | |
| "few_shots_select": null, | |
| "generation_size": -1, | |
| "stop_sequence": [ | |
| "</s>", | |
| "Q:", | |
| "\n\n" | |
| ], | |
| "output_regex": null, | |
| "frozen": false, | |
| "suite": [ | |
| "lighteval" | |
| ], | |
| "original_num_docs": 181, | |
| "effective_num_docs": 181, | |
| "trust_dataset": true, | |
| "must_remove_duplicate_docs": null | |
| }, | |
| "lighteval|bigbench:sports_understanding": { | |
| "name": "bigbench:sports_understanding", | |
| "prompt_function": "bbh_lighteval", | |
| "hf_repo": "lighteval/bbh", | |
| "hf_subset": "sports_understanding", | |
| "metric": [ | |
| "loglikelihood_acc_single_token" | |
| ], | |
| "hf_avail_splits": [ | |
| "train" | |
| ], | |
| "evaluation_splits": [ | |
| "train" | |
| ], | |
| "few_shots_split": null, | |
| "few_shots_select": null, | |
| "generation_size": -1, | |
| "stop_sequence": [ | |
| "</s>", | |
| "Q:", | |
| "\n\n" | |
| ], | |
| "output_regex": null, | |
| "frozen": false, | |
| "suite": [ | |
| "lighteval" | |
| ], | |
| "original_num_docs": 1000, | |
| "effective_num_docs": 1000, | |
| "trust_dataset": true, | |
| "must_remove_duplicate_docs": null | |
| }, | |
| "lighteval|bigbench:temporal_sequences": { | |
| "name": "bigbench:temporal_sequences", | |
| "prompt_function": "bbh_lighteval", | |
| "hf_repo": "lighteval/bbh", | |
| "hf_subset": "temporal_sequences", | |
| "metric": [ | |
| "loglikelihood_acc_single_token" | |
| ], | |
| "hf_avail_splits": [ | |
| "train" | |
| ], | |
| "evaluation_splits": [ | |
| "train" | |
| ], | |
| "few_shots_split": null, | |
| "few_shots_select": null, | |
| "generation_size": -1, | |
| "stop_sequence": [ | |
| "</s>", | |
| "Q:", | |
| "\n\n" | |
| ], | |
| "output_regex": null, | |
| "frozen": false, | |
| "suite": [ | |
| "lighteval" | |
| ], | |
| "original_num_docs": 1000, | |
| "effective_num_docs": 1000, | |
| "trust_dataset": true, | |
| "must_remove_duplicate_docs": null | |
| }, | |
| "lighteval|bigbench:tracking_shuffled_objects_five_objects": { | |
| "name": "bigbench:tracking_shuffled_objects_five_objects", | |
| "prompt_function": "bbh_lighteval", | |
| "hf_repo": "lighteval/bbh", | |
| "hf_subset": "tracking_shuffled_objects_five_objects", | |
| "metric": [ | |
| "loglikelihood_acc_single_token" | |
| ], | |
| "hf_avail_splits": [ | |
| "train" | |
| ], | |
| "evaluation_splits": [ | |
| "train" | |
| ], | |
| "few_shots_split": null, | |
| "few_shots_select": null, | |
| "generation_size": -1, | |
| "stop_sequence": [ | |
| "</s>", | |
| "Q:", | |
| "\n\n" | |
| ], | |
| "output_regex": null, | |
| "frozen": false, | |
| "suite": [ | |
| "lighteval" | |
| ], | |
| "original_num_docs": 1250, | |
| "effective_num_docs": 1250, | |
| "trust_dataset": true, | |
| "must_remove_duplicate_docs": null | |
| }, | |
| "lighteval|bigbench:tracking_shuffled_objects_seven_objects": { | |
| "name": "bigbench:tracking_shuffled_objects_seven_objects", | |
| "prompt_function": "bbh_lighteval", | |
| "hf_repo": "lighteval/bbh", | |
| "hf_subset": "tracking_shuffled_objects_seven_objects", | |
| "metric": [ | |
| "loglikelihood_acc_single_token" | |
| ], | |
| "hf_avail_splits": [ | |
| "train" | |
| ], | |
| "evaluation_splits": [ | |
| "train" | |
| ], | |
| "few_shots_split": null, | |
| "few_shots_select": null, | |
| "generation_size": -1, | |
| "stop_sequence": [ | |
| "</s>", | |
| "Q:", | |
| "\n\n" | |
| ], | |
| "output_regex": null, | |
| "frozen": false, | |
| "suite": [ | |
| "lighteval" | |
| ], | |
| "original_num_docs": 1750, | |
| "effective_num_docs": 1750, | |
| "trust_dataset": true, | |
| "must_remove_duplicate_docs": null | |
| }, | |
| "lighteval|bigbench:tracking_shuffled_objects_three_objects": { | |
| "name": "bigbench:tracking_shuffled_objects_three_objects", | |
| "prompt_function": "bbh_lighteval", | |
| "hf_repo": "lighteval/bbh", | |
| "hf_subset": "tracking_shuffled_objects_three_objects", | |
| "metric": [ | |
| "loglikelihood_acc_single_token" | |
| ], | |
| "hf_avail_splits": [ | |
| "train" | |
| ], | |
| "evaluation_splits": [ | |
| "train" | |
| ], | |
| "few_shots_split": null, | |
| "few_shots_select": null, | |
| "generation_size": -1, | |
| "stop_sequence": [ | |
| "</s>", | |
| "Q:", | |
| "\n\n" | |
| ], | |
| "output_regex": null, | |
| "frozen": false, | |
| "suite": [ | |
| "lighteval" | |
| ], | |
| "original_num_docs": 300, | |
| "effective_num_docs": 300, | |
| "trust_dataset": true, | |
| "must_remove_duplicate_docs": null | |
| } | |
| }, | |
| "summary_tasks": { | |
| "lighteval|bigbench:causal_judgment|0": { | |
| "hashes": { | |
| "hash_examples": "dfb1ae47218f2850", | |
| "hash_full_prompts": "f2bb991fd51bb99f", | |
| "hash_input_tokens": "0d45021bff6fed40", | |
| "hash_cont_tokens": "9a9bd8b966072e64" | |
| }, | |
| "truncated": 0, | |
| "non_truncated": 190, | |
| "padded": 189, | |
| "non_padded": 1, | |
| "effective_few_shots": 0.0, | |
| "num_truncated_few_shots": 0 | |
| }, | |
| "lighteval|bigbench:date_understanding|0": { | |
| "hashes": { | |
| "hash_examples": "2b823c41500a6ec2", | |
| "hash_full_prompts": "e83cfda3473c2321", | |
| "hash_input_tokens": "49289439543c6c14", | |
| "hash_cont_tokens": "ca1827d9c35df800" | |
| }, | |
| "truncated": 0, | |
| "non_truncated": 369, | |
| "padded": 369, | |
| "non_padded": 0, | |
| "effective_few_shots": 0.0, | |
| "num_truncated_few_shots": 0 | |
| }, | |
| "lighteval|bigbench:disambiguation_qa|0": { | |
| "hashes": { | |
| "hash_examples": "2a4c3d41db198cea", | |
| "hash_full_prompts": "de8edb2c3e81f1a0", | |
| "hash_input_tokens": "9e011df68d144c29", | |
| "hash_cont_tokens": "c3e79f545508f21a" | |
| }, | |
| "truncated": 0, | |
| "non_truncated": 258, | |
| "padded": 258, | |
| "non_padded": 0, | |
| "effective_few_shots": 0.0, | |
| "num_truncated_few_shots": 0 | |
| }, | |
| "lighteval|bigbench:geometric_shapes|0": { | |
| "hashes": { | |
| "hash_examples": "24aa261103911b72", | |
| "hash_full_prompts": "857754458a216e36", | |
| "hash_input_tokens": "99af61c287b4b46b", | |
| "hash_cont_tokens": "7351a9b4012a13a4" | |
| }, | |
| "truncated": 0, | |
| "non_truncated": 360, | |
| "padded": 360, | |
| "non_padded": 0, | |
| "effective_few_shots": 0.0, | |
| "num_truncated_few_shots": 0 | |
| }, | |
| "lighteval|bigbench:logical_deduction_five_objects|0": { | |
| "hashes": { | |
| "hash_examples": "cb5bdc92afc41f83", | |
| "hash_full_prompts": "1d43855dfb3188e4", | |
| "hash_input_tokens": "ddd02a0155536371", | |
| "hash_cont_tokens": "3791ab917f69e6af" | |
| }, | |
| "truncated": 0, | |
| "non_truncated": 500, | |
| "padded": 500, | |
| "non_padded": 0, | |
| "effective_few_shots": 0.0, | |
| "num_truncated_few_shots": 0 | |
| }, | |
| "lighteval|bigbench:logical_deduction_seven_objects|0": { | |
| "hashes": { | |
| "hash_examples": "b6805ea696739f9f", | |
| "hash_full_prompts": "f594b9f9ea03f8ce", | |
| "hash_input_tokens": "ad889abc7a07f1c2", | |
| "hash_cont_tokens": "9c00f5e49f3e5f98" | |
| }, | |
| "truncated": 0, | |
| "non_truncated": 700, | |
| "padded": 687, | |
| "non_padded": 13, | |
| "effective_few_shots": 0.0, | |
| "num_truncated_few_shots": 0 | |
| }, | |
| "lighteval|bigbench:logical_deduction_three_objects|0": { | |
| "hashes": { | |
| "hash_examples": "0509e5712ab9bcdb", | |
| "hash_full_prompts": "a35945161fbe4048", | |
| "hash_input_tokens": "38d25a2a78527053", | |
| "hash_cont_tokens": "02df3d11c26ce1da" | |
| }, | |
| "truncated": 0, | |
| "non_truncated": 300, | |
| "padded": 297, | |
| "non_padded": 3, | |
| "effective_few_shots": 0.0, | |
| "num_truncated_few_shots": 0 | |
| }, | |
| "lighteval|bigbench:movie_recommendation|0": { | |
| "hashes": { | |
| "hash_examples": "530cc6f737830f45", | |
| "hash_full_prompts": "31485051852b9f32", | |
| "hash_input_tokens": "0b7b0096d10192f2", | |
| "hash_cont_tokens": "be520838bf2427bc" | |
| }, | |
| "truncated": 0, | |
| "non_truncated": 500, | |
| "padded": 494, | |
| "non_padded": 6, | |
| "effective_few_shots": 0.0, | |
| "num_truncated_few_shots": 0 | |
| }, | |
| "lighteval|bigbench:navigate|0": { | |
| "hashes": { | |
| "hash_examples": "7962ef85d0058b9a", | |
| "hash_full_prompts": "1fc6da9622d3209e", | |
| "hash_input_tokens": "c8903e4c0fa93e91", | |
| "hash_cont_tokens": "ce2f051624b0c7f2" | |
| }, | |
| "truncated": 0, | |
| "non_truncated": 1000, | |
| "padded": 981, | |
| "non_padded": 19, | |
| "effective_few_shots": 0.0, | |
| "num_truncated_few_shots": 0 | |
| }, | |
| "lighteval|bigbench:reasoning_about_colored_objects|0": { | |
| "hashes": { | |
| "hash_examples": "39be1ab1677a651d", | |
| "hash_full_prompts": "fe78aaa6eba66782", | |
| "hash_input_tokens": "a412a20185f9c939", | |
| "hash_cont_tokens": "87abd4073f64b19a" | |
| }, | |
| "truncated": 0, | |
| "non_truncated": 2000, | |
| "padded": 1991, | |
| "non_padded": 9, | |
| "effective_few_shots": 0.0, | |
| "num_truncated_few_shots": 0 | |
| }, | |
| "lighteval|bigbench:ruin_names|0": { | |
| "hashes": { | |
| "hash_examples": "e9b96b31d2154941", | |
| "hash_full_prompts": "5aa4d00447058e82", | |
| "hash_input_tokens": "448ed4af38c50f9e", | |
| "hash_cont_tokens": "046bbbbddb05b429" | |
| }, | |
| "truncated": 0, | |
| "non_truncated": 448, | |
| "padded": 447, | |
| "non_padded": 1, | |
| "effective_few_shots": 0.0, | |
| "num_truncated_few_shots": 0 | |
| }, | |
| "lighteval|bigbench:salient_translation_error_detection|0": { | |
| "hashes": { | |
| "hash_examples": "951ac59f7ad0427d", | |
| "hash_full_prompts": "ca5ac9a0a62b1030", | |
| "hash_input_tokens": "c16c2779e9122841", | |
| "hash_cont_tokens": "12a73509c698a7dc" | |
| }, | |
| "truncated": 0, | |
| "non_truncated": 998, | |
| "padded": 998, | |
| "non_padded": 0, | |
| "effective_few_shots": 0.0, | |
| "num_truncated_few_shots": 0 | |
| }, | |
| "lighteval|bigbench:snarks|0": { | |
| "hashes": { | |
| "hash_examples": "3a53eb9b9d758534", | |
| "hash_full_prompts": "aa8cf31c793715bf", | |
| "hash_input_tokens": "2f5a31f715bc150b", | |
| "hash_cont_tokens": "11293a1bfa806eba" | |
| }, | |
| "truncated": 0, | |
| "non_truncated": 181, | |
| "padded": 173, | |
| "non_padded": 8, | |
| "effective_few_shots": 0.0, | |
| "num_truncated_few_shots": 0 | |
| }, | |
| "lighteval|bigbench:sports_understanding|0": { | |
| "hashes": { | |
| "hash_examples": "bd65741f00770373", | |
| "hash_full_prompts": "93e300b0c02bfe7e", | |
| "hash_input_tokens": "03b0d58149c92d1b", | |
| "hash_cont_tokens": "a934157fb362ce28" | |
| }, | |
| "truncated": 0, | |
| "non_truncated": 1000, | |
| "padded": 1000, | |
| "non_padded": 0, | |
| "effective_few_shots": 0.0, | |
| "num_truncated_few_shots": 0 | |
| }, | |
| "lighteval|bigbench:temporal_sequences|0": { | |
| "hashes": { | |
| "hash_examples": "1d13139f47cb2df7", | |
| "hash_full_prompts": "a1437cb805dd0aa4", | |
| "hash_input_tokens": "861b3ca56d4b1bb0", | |
| "hash_cont_tokens": "326f6f7b7bdf4692" | |
| }, | |
| "truncated": 0, | |
| "non_truncated": 1000, | |
| "padded": 988, | |
| "non_padded": 12, | |
| "effective_few_shots": 0.0, | |
| "num_truncated_few_shots": 0 | |
| }, | |
| "lighteval|bigbench:tracking_shuffled_objects_five_objects|0": { | |
| "hashes": { | |
| "hash_examples": "8770a702a9646648", | |
| "hash_full_prompts": "bc353ccf0e803508", | |
| "hash_input_tokens": "68d821ac602e405c", | |
| "hash_cont_tokens": "30ca4364426258e1" | |
| }, | |
| "truncated": 0, | |
| "non_truncated": 1250, | |
| "padded": 1240, | |
| "non_padded": 10, | |
| "effective_few_shots": 0.0, | |
| "num_truncated_few_shots": 0 | |
| }, | |
| "lighteval|bigbench:tracking_shuffled_objects_seven_objects|0": { | |
| "hashes": { | |
| "hash_examples": "b469b7d073824a59", | |
| "hash_full_prompts": "8051e0f1969fcef1", | |
| "hash_input_tokens": "4f410bc1c0eca013", | |
| "hash_cont_tokens": "f76ba63a583d749e" | |
| }, | |
| "truncated": 0, | |
| "non_truncated": 1750, | |
| "padded": 1675, | |
| "non_padded": 75, | |
| "effective_few_shots": 0.0, | |
| "num_truncated_few_shots": 0 | |
| }, | |
| "lighteval|bigbench:tracking_shuffled_objects_three_objects|0": { | |
| "hashes": { | |
| "hash_examples": "0509e5712ab9bcdb", | |
| "hash_full_prompts": "a35945161fbe4048", | |
| "hash_input_tokens": "a079f5a0f92cb7d5", | |
| "hash_cont_tokens": "8aea5a43cb736677" | |
| }, | |
| "truncated": 0, | |
| "non_truncated": 300, | |
| "padded": 294, | |
| "non_padded": 6, | |
| "effective_few_shots": 0.0, | |
| "num_truncated_few_shots": 0 | |
| } | |
| }, | |
| "summary_general": { | |
| "hashes": { | |
| "hash_examples": "51a30c4501ba4586", | |
| "hash_full_prompts": "a93a8c11cb614cb6", | |
| "hash_input_tokens": "736308fbff86d9d0", | |
| "hash_cont_tokens": "b4e1747eea39656b" | |
| }, | |
| "truncated": 0, | |
| "non_truncated": 13104, | |
| "padded": 12941, | |
| "non_padded": 163, | |
| "num_truncated_few_shots": 0 | |
| } | |
| } |