open-r1-eval-leaderboard
/
eval_results
/mistralai
/Mixtral-8x22B-Instruct-v0.1
/main
/bbh
/results_2024-04-17T14-56-54.833079.json
| { | |
| "config_general": { | |
| "lighteval_sha": "?", | |
| "num_fewshot_seeds": 1, | |
| "override_batch_size": 1, | |
| "max_samples": null, | |
| "job_id": "", | |
| "start_time": 600068.786639362, | |
| "end_time": 602062.303593423, | |
| "total_evaluation_time_secondes": "1993.5169540609932", | |
| "model_name": "mistralai/Mixtral-8x22B-Instruct-v0.1", | |
| "model_sha": "796bc4393fd5e7e0c0ff1c44de2526419f163003", | |
| "model_dtype": "torch.bfloat16", | |
| "model_size": "263.69 GB", | |
| "config": null | |
| }, | |
| "results": { | |
| "lighteval|bigbench:causal_judgment|0": { | |
| "acc": 0.6736842105263158, | |
| "acc_stderr": 0.0341048643533449 | |
| }, | |
| "lighteval|bigbench:date_understanding|0": { | |
| "acc": 0.6233062330623306, | |
| "acc_stderr": 0.025259284393370166 | |
| }, | |
| "lighteval|bigbench:disambiguation_qa|0": { | |
| "acc": 0.6085271317829457, | |
| "acc_stderr": 0.03044557954579361 | |
| }, | |
| "lighteval|bigbench:geometric_shapes|0": { | |
| "acc": 0.33055555555555555, | |
| "acc_stderr": 0.024827459203325592 | |
| }, | |
| "lighteval|bigbench:logical_deduction_five_objects|0": { | |
| "acc": 0.566, | |
| "acc_stderr": 0.022187215803029004 | |
| }, | |
| "lighteval|bigbench:logical_deduction_seven_objects|0": { | |
| "acc": 0.5142857142857142, | |
| "acc_stderr": 0.01890401620957065 | |
| }, | |
| "lighteval|bigbench:logical_deduction_three_objects|0": { | |
| "acc": 0.79, | |
| "acc_stderr": 0.023555243542102446 | |
| }, | |
| "lighteval|bigbench:movie_recommendation|0": { | |
| "acc": 0.654, | |
| "acc_stderr": 0.021294951277234637 | |
| }, | |
| "lighteval|bigbench:navigate|0": { | |
| "acc": 0.558, | |
| "acc_stderr": 0.015712507211864204 | |
| }, | |
| "lighteval|bigbench:reasoning_about_colored_objects|0": { | |
| "acc": 0.6595, | |
| "acc_stderr": 0.010598869893602352 | |
| }, | |
| "lighteval|bigbench:ruin_names|0": { | |
| "acc": 0.7790178571428571, | |
| "acc_stderr": 0.01962449705224273 | |
| }, | |
| "lighteval|bigbench:salient_translation_error_detection|0": { | |
| "acc": 0.5160320641282565, | |
| "acc_stderr": 0.015827016657875177 | |
| }, | |
| "lighteval|bigbench:snarks|0": { | |
| "acc": 0.8121546961325967, | |
| "acc_stderr": 0.02911275452466063 | |
| }, | |
| "lighteval|bigbench:sports_understanding|0": { | |
| "acc": 0.795, | |
| "acc_stderr": 0.012772554096113114 | |
| }, | |
| "lighteval|bigbench:temporal_sequences|0": { | |
| "acc": 0.9, | |
| "acc_stderr": 0.009491579957525057 | |
| }, | |
| "lighteval|bigbench:tracking_shuffled_objects_five_objects|0": { | |
| "acc": 0.1808, | |
| "acc_stderr": 0.010889626034172734 | |
| }, | |
| "lighteval|bigbench:tracking_shuffled_objects_seven_objects|0": { | |
| "acc": 0.09714285714285714, | |
| "acc_stderr": 0.007081415067962661 | |
| }, | |
| "lighteval|bigbench:tracking_shuffled_objects_three_objects|0": { | |
| "acc": 0.79, | |
| "acc_stderr": 0.023555243542102446 | |
| }, | |
| "lighteval|bigbench:_average|0": { | |
| "acc": 0.6026670177644127, | |
| "acc_stderr": 0.01973581546477178 | |
| }, | |
| "all": { | |
| "acc": 0.6026670177644127, | |
| "acc_stderr": 0.01973581546477178 | |
| } | |
| }, | |
| "versions": { | |
| "lighteval|bigbench:causal_judgment|0": 0, | |
| "lighteval|bigbench:date_understanding|0": 0, | |
| "lighteval|bigbench:disambiguation_qa|0": 0, | |
| "lighteval|bigbench:geometric_shapes|0": 0, | |
| "lighteval|bigbench:logical_deduction_five_objects|0": 0, | |
| "lighteval|bigbench:logical_deduction_seven_objects|0": 0, | |
| "lighteval|bigbench:logical_deduction_three_objects|0": 0, | |
| "lighteval|bigbench:movie_recommendation|0": 0, | |
| "lighteval|bigbench:navigate|0": 0, | |
| "lighteval|bigbench:reasoning_about_colored_objects|0": 0, | |
| "lighteval|bigbench:ruin_names|0": 0, | |
| "lighteval|bigbench:salient_translation_error_detection|0": 0, | |
| "lighteval|bigbench:snarks|0": 0, | |
| "lighteval|bigbench:sports_understanding|0": 0, | |
| "lighteval|bigbench:temporal_sequences|0": 0, | |
| "lighteval|bigbench:tracking_shuffled_objects_five_objects|0": 0, | |
| "lighteval|bigbench:tracking_shuffled_objects_seven_objects|0": 0, | |
| "lighteval|bigbench:tracking_shuffled_objects_three_objects|0": 0 | |
| }, | |
| "config_tasks": { | |
| "lighteval|bigbench:causal_judgment": { | |
| "name": "bigbench:causal_judgment", | |
| "prompt_function": "bbh_lighteval", | |
| "hf_repo": "lighteval/bbh", | |
| "hf_subset": "causal_judgement", | |
| "metric": [ | |
| "loglikelihood_acc_single_token" | |
| ], | |
| "hf_avail_splits": [ | |
| "train" | |
| ], | |
| "evaluation_splits": [ | |
| "train" | |
| ], | |
| "few_shots_split": null, | |
| "few_shots_select": null, | |
| "generation_size": -1, | |
| "stop_sequence": [ | |
| "</s>", | |
| "Q:", | |
| "\n\n" | |
| ], | |
| "output_regex": null, | |
| "frozen": false, | |
| "suite": [ | |
| "lighteval" | |
| ], | |
| "original_num_docs": 190, | |
| "effective_num_docs": 190, | |
| "trust_dataset": true, | |
| "must_remove_duplicate_docs": null | |
| }, | |
| "lighteval|bigbench:date_understanding": { | |
| "name": "bigbench:date_understanding", | |
| "prompt_function": "bbh_lighteval", | |
| "hf_repo": "lighteval/bbh", | |
| "hf_subset": "date_understanding", | |
| "metric": [ | |
| "loglikelihood_acc_single_token" | |
| ], | |
| "hf_avail_splits": [ | |
| "train" | |
| ], | |
| "evaluation_splits": [ | |
| "train" | |
| ], | |
| "few_shots_split": null, | |
| "few_shots_select": null, | |
| "generation_size": -1, | |
| "stop_sequence": [ | |
| "</s>", | |
| "Q:", | |
| "\n\n" | |
| ], | |
| "output_regex": null, | |
| "frozen": false, | |
| "suite": [ | |
| "lighteval" | |
| ], | |
| "original_num_docs": 369, | |
| "effective_num_docs": 369, | |
| "trust_dataset": true, | |
| "must_remove_duplicate_docs": null | |
| }, | |
| "lighteval|bigbench:disambiguation_qa": { | |
| "name": "bigbench:disambiguation_qa", | |
| "prompt_function": "bbh_lighteval", | |
| "hf_repo": "lighteval/bbh", | |
| "hf_subset": "disambiguation_qa", | |
| "metric": [ | |
| "loglikelihood_acc_single_token" | |
| ], | |
| "hf_avail_splits": [ | |
| "train" | |
| ], | |
| "evaluation_splits": [ | |
| "train" | |
| ], | |
| "few_shots_split": null, | |
| "few_shots_select": null, | |
| "generation_size": -1, | |
| "stop_sequence": [ | |
| "</s>", | |
| "Q:", | |
| "\n\n" | |
| ], | |
| "output_regex": null, | |
| "frozen": false, | |
| "suite": [ | |
| "lighteval" | |
| ], | |
| "original_num_docs": 258, | |
| "effective_num_docs": 258, | |
| "trust_dataset": true, | |
| "must_remove_duplicate_docs": null | |
| }, | |
| "lighteval|bigbench:geometric_shapes": { | |
| "name": "bigbench:geometric_shapes", | |
| "prompt_function": "bbh_lighteval", | |
| "hf_repo": "lighteval/bbh", | |
| "hf_subset": "geometric_shapes", | |
| "metric": [ | |
| "loglikelihood_acc_single_token" | |
| ], | |
| "hf_avail_splits": [ | |
| "train" | |
| ], | |
| "evaluation_splits": [ | |
| "train" | |
| ], | |
| "few_shots_split": null, | |
| "few_shots_select": null, | |
| "generation_size": -1, | |
| "stop_sequence": [ | |
| "</s>", | |
| "Q:", | |
| "\n\n" | |
| ], | |
| "output_regex": null, | |
| "frozen": false, | |
| "suite": [ | |
| "lighteval" | |
| ], | |
| "original_num_docs": 360, | |
| "effective_num_docs": 360, | |
| "trust_dataset": true, | |
| "must_remove_duplicate_docs": null | |
| }, | |
| "lighteval|bigbench:logical_deduction_five_objects": { | |
| "name": "bigbench:logical_deduction_five_objects", | |
| "prompt_function": "bbh_lighteval", | |
| "hf_repo": "lighteval/bbh", | |
| "hf_subset": "logical_deduction_five_objects", | |
| "metric": [ | |
| "loglikelihood_acc_single_token" | |
| ], | |
| "hf_avail_splits": [ | |
| "train" | |
| ], | |
| "evaluation_splits": [ | |
| "train" | |
| ], | |
| "few_shots_split": null, | |
| "few_shots_select": null, | |
| "generation_size": -1, | |
| "stop_sequence": [ | |
| "</s>", | |
| "Q:", | |
| "\n\n" | |
| ], | |
| "output_regex": null, | |
| "frozen": false, | |
| "suite": [ | |
| "lighteval" | |
| ], | |
| "original_num_docs": 500, | |
| "effective_num_docs": 500, | |
| "trust_dataset": true, | |
| "must_remove_duplicate_docs": null | |
| }, | |
| "lighteval|bigbench:logical_deduction_seven_objects": { | |
| "name": "bigbench:logical_deduction_seven_objects", | |
| "prompt_function": "bbh_lighteval", | |
| "hf_repo": "lighteval/bbh", | |
| "hf_subset": "logical_deduction_seven_objects", | |
| "metric": [ | |
| "loglikelihood_acc_single_token" | |
| ], | |
| "hf_avail_splits": [ | |
| "train" | |
| ], | |
| "evaluation_splits": [ | |
| "train" | |
| ], | |
| "few_shots_split": null, | |
| "few_shots_select": null, | |
| "generation_size": -1, | |
| "stop_sequence": [ | |
| "</s>", | |
| "Q:", | |
| "\n\n" | |
| ], | |
| "output_regex": null, | |
| "frozen": false, | |
| "suite": [ | |
| "lighteval" | |
| ], | |
| "original_num_docs": 700, | |
| "effective_num_docs": 700, | |
| "trust_dataset": true, | |
| "must_remove_duplicate_docs": null | |
| }, | |
| "lighteval|bigbench:logical_deduction_three_objects": { | |
| "name": "bigbench:logical_deduction_three_objects", | |
| "prompt_function": "bbh_lighteval", | |
| "hf_repo": "lighteval/bbh", | |
| "hf_subset": "logical_deduction_three_objects", | |
| "metric": [ | |
| "loglikelihood_acc_single_token" | |
| ], | |
| "hf_avail_splits": [ | |
| "train" | |
| ], | |
| "evaluation_splits": [ | |
| "train" | |
| ], | |
| "few_shots_split": null, | |
| "few_shots_select": null, | |
| "generation_size": -1, | |
| "stop_sequence": [ | |
| "</s>", | |
| "Q:", | |
| "\n\n" | |
| ], | |
| "output_regex": null, | |
| "frozen": false, | |
| "suite": [ | |
| "lighteval" | |
| ], | |
| "original_num_docs": 300, | |
| "effective_num_docs": 300, | |
| "trust_dataset": true, | |
| "must_remove_duplicate_docs": null | |
| }, | |
| "lighteval|bigbench:movie_recommendation": { | |
| "name": "bigbench:movie_recommendation", | |
| "prompt_function": "bbh_lighteval", | |
| "hf_repo": "lighteval/bbh", | |
| "hf_subset": "movie_recommendation", | |
| "metric": [ | |
| "loglikelihood_acc_single_token" | |
| ], | |
| "hf_avail_splits": [ | |
| "train" | |
| ], | |
| "evaluation_splits": [ | |
| "train" | |
| ], | |
| "few_shots_split": null, | |
| "few_shots_select": null, | |
| "generation_size": -1, | |
| "stop_sequence": [ | |
| "</s>", | |
| "Q:", | |
| "\n\n" | |
| ], | |
| "output_regex": null, | |
| "frozen": false, | |
| "suite": [ | |
| "lighteval" | |
| ], | |
| "original_num_docs": 500, | |
| "effective_num_docs": 500, | |
| "trust_dataset": true, | |
| "must_remove_duplicate_docs": null | |
| }, | |
| "lighteval|bigbench:navigate": { | |
| "name": "bigbench:navigate", | |
| "prompt_function": "bbh_lighteval", | |
| "hf_repo": "lighteval/bbh", | |
| "hf_subset": "navigate", | |
| "metric": [ | |
| "loglikelihood_acc_single_token" | |
| ], | |
| "hf_avail_splits": [ | |
| "train" | |
| ], | |
| "evaluation_splits": [ | |
| "train" | |
| ], | |
| "few_shots_split": null, | |
| "few_shots_select": null, | |
| "generation_size": -1, | |
| "stop_sequence": [ | |
| "</s>", | |
| "Q:", | |
| "\n\n" | |
| ], | |
| "output_regex": null, | |
| "frozen": false, | |
| "suite": [ | |
| "lighteval" | |
| ], | |
| "original_num_docs": 1000, | |
| "effective_num_docs": 1000, | |
| "trust_dataset": true, | |
| "must_remove_duplicate_docs": null | |
| }, | |
| "lighteval|bigbench:reasoning_about_colored_objects": { | |
| "name": "bigbench:reasoning_about_colored_objects", | |
| "prompt_function": "bbh_lighteval", | |
| "hf_repo": "lighteval/bbh", | |
| "hf_subset": "reasoning_about_colored_objects", | |
| "metric": [ | |
| "loglikelihood_acc_single_token" | |
| ], | |
| "hf_avail_splits": [ | |
| "train" | |
| ], | |
| "evaluation_splits": [ | |
| "train" | |
| ], | |
| "few_shots_split": null, | |
| "few_shots_select": null, | |
| "generation_size": -1, | |
| "stop_sequence": [ | |
| "</s>", | |
| "Q:", | |
| "\n\n" | |
| ], | |
| "output_regex": null, | |
| "frozen": false, | |
| "suite": [ | |
| "lighteval" | |
| ], | |
| "original_num_docs": 2000, | |
| "effective_num_docs": 2000, | |
| "trust_dataset": true, | |
| "must_remove_duplicate_docs": null | |
| }, | |
| "lighteval|bigbench:ruin_names": { | |
| "name": "bigbench:ruin_names", | |
| "prompt_function": "bbh_lighteval", | |
| "hf_repo": "lighteval/bbh", | |
| "hf_subset": "ruin_names", | |
| "metric": [ | |
| "loglikelihood_acc_single_token" | |
| ], | |
| "hf_avail_splits": [ | |
| "train" | |
| ], | |
| "evaluation_splits": [ | |
| "train" | |
| ], | |
| "few_shots_split": null, | |
| "few_shots_select": null, | |
| "generation_size": -1, | |
| "stop_sequence": [ | |
| "</s>", | |
| "Q:", | |
| "\n\n" | |
| ], | |
| "output_regex": null, | |
| "frozen": false, | |
| "suite": [ | |
| "lighteval" | |
| ], | |
| "original_num_docs": 448, | |
| "effective_num_docs": 448, | |
| "trust_dataset": true, | |
| "must_remove_duplicate_docs": null | |
| }, | |
| "lighteval|bigbench:salient_translation_error_detection": { | |
| "name": "bigbench:salient_translation_error_detection", | |
| "prompt_function": "bbh_lighteval", | |
| "hf_repo": "lighteval/bbh", | |
| "hf_subset": "salient_translation_error_detection", | |
| "metric": [ | |
| "loglikelihood_acc_single_token" | |
| ], | |
| "hf_avail_splits": [ | |
| "train" | |
| ], | |
| "evaluation_splits": [ | |
| "train" | |
| ], | |
| "few_shots_split": null, | |
| "few_shots_select": null, | |
| "generation_size": -1, | |
| "stop_sequence": [ | |
| "</s>", | |
| "Q:", | |
| "\n\n" | |
| ], | |
| "output_regex": null, | |
| "frozen": false, | |
| "suite": [ | |
| "lighteval" | |
| ], | |
| "original_num_docs": 998, | |
| "effective_num_docs": 998, | |
| "trust_dataset": true, | |
| "must_remove_duplicate_docs": null | |
| }, | |
| "lighteval|bigbench:snarks": { | |
| "name": "bigbench:snarks", | |
| "prompt_function": "bbh_lighteval", | |
| "hf_repo": "lighteval/bbh", | |
| "hf_subset": "snarks", | |
| "metric": [ | |
| "loglikelihood_acc_single_token" | |
| ], | |
| "hf_avail_splits": [ | |
| "train" | |
| ], | |
| "evaluation_splits": [ | |
| "train" | |
| ], | |
| "few_shots_split": null, | |
| "few_shots_select": null, | |
| "generation_size": -1, | |
| "stop_sequence": [ | |
| "</s>", | |
| "Q:", | |
| "\n\n" | |
| ], | |
| "output_regex": null, | |
| "frozen": false, | |
| "suite": [ | |
| "lighteval" | |
| ], | |
| "original_num_docs": 181, | |
| "effective_num_docs": 181, | |
| "trust_dataset": true, | |
| "must_remove_duplicate_docs": null | |
| }, | |
| "lighteval|bigbench:sports_understanding": { | |
| "name": "bigbench:sports_understanding", | |
| "prompt_function": "bbh_lighteval", | |
| "hf_repo": "lighteval/bbh", | |
| "hf_subset": "sports_understanding", | |
| "metric": [ | |
| "loglikelihood_acc_single_token" | |
| ], | |
| "hf_avail_splits": [ | |
| "train" | |
| ], | |
| "evaluation_splits": [ | |
| "train" | |
| ], | |
| "few_shots_split": null, | |
| "few_shots_select": null, | |
| "generation_size": -1, | |
| "stop_sequence": [ | |
| "</s>", | |
| "Q:", | |
| "\n\n" | |
| ], | |
| "output_regex": null, | |
| "frozen": false, | |
| "suite": [ | |
| "lighteval" | |
| ], | |
| "original_num_docs": 1000, | |
| "effective_num_docs": 1000, | |
| "trust_dataset": true, | |
| "must_remove_duplicate_docs": null | |
| }, | |
| "lighteval|bigbench:temporal_sequences": { | |
| "name": "bigbench:temporal_sequences", | |
| "prompt_function": "bbh_lighteval", | |
| "hf_repo": "lighteval/bbh", | |
| "hf_subset": "temporal_sequences", | |
| "metric": [ | |
| "loglikelihood_acc_single_token" | |
| ], | |
| "hf_avail_splits": [ | |
| "train" | |
| ], | |
| "evaluation_splits": [ | |
| "train" | |
| ], | |
| "few_shots_split": null, | |
| "few_shots_select": null, | |
| "generation_size": -1, | |
| "stop_sequence": [ | |
| "</s>", | |
| "Q:", | |
| "\n\n" | |
| ], | |
| "output_regex": null, | |
| "frozen": false, | |
| "suite": [ | |
| "lighteval" | |
| ], | |
| "original_num_docs": 1000, | |
| "effective_num_docs": 1000, | |
| "trust_dataset": true, | |
| "must_remove_duplicate_docs": null | |
| }, | |
| "lighteval|bigbench:tracking_shuffled_objects_five_objects": { | |
| "name": "bigbench:tracking_shuffled_objects_five_objects", | |
| "prompt_function": "bbh_lighteval", | |
| "hf_repo": "lighteval/bbh", | |
| "hf_subset": "tracking_shuffled_objects_five_objects", | |
| "metric": [ | |
| "loglikelihood_acc_single_token" | |
| ], | |
| "hf_avail_splits": [ | |
| "train" | |
| ], | |
| "evaluation_splits": [ | |
| "train" | |
| ], | |
| "few_shots_split": null, | |
| "few_shots_select": null, | |
| "generation_size": -1, | |
| "stop_sequence": [ | |
| "</s>", | |
| "Q:", | |
| "\n\n" | |
| ], | |
| "output_regex": null, | |
| "frozen": false, | |
| "suite": [ | |
| "lighteval" | |
| ], | |
| "original_num_docs": 1250, | |
| "effective_num_docs": 1250, | |
| "trust_dataset": true, | |
| "must_remove_duplicate_docs": null | |
| }, | |
| "lighteval|bigbench:tracking_shuffled_objects_seven_objects": { | |
| "name": "bigbench:tracking_shuffled_objects_seven_objects", | |
| "prompt_function": "bbh_lighteval", | |
| "hf_repo": "lighteval/bbh", | |
| "hf_subset": "tracking_shuffled_objects_seven_objects", | |
| "metric": [ | |
| "loglikelihood_acc_single_token" | |
| ], | |
| "hf_avail_splits": [ | |
| "train" | |
| ], | |
| "evaluation_splits": [ | |
| "train" | |
| ], | |
| "few_shots_split": null, | |
| "few_shots_select": null, | |
| "generation_size": -1, | |
| "stop_sequence": [ | |
| "</s>", | |
| "Q:", | |
| "\n\n" | |
| ], | |
| "output_regex": null, | |
| "frozen": false, | |
| "suite": [ | |
| "lighteval" | |
| ], | |
| "original_num_docs": 1750, | |
| "effective_num_docs": 1750, | |
| "trust_dataset": true, | |
| "must_remove_duplicate_docs": null | |
| }, | |
| "lighteval|bigbench:tracking_shuffled_objects_three_objects": { | |
| "name": "bigbench:tracking_shuffled_objects_three_objects", | |
| "prompt_function": "bbh_lighteval", | |
| "hf_repo": "lighteval/bbh", | |
| "hf_subset": "tracking_shuffled_objects_three_objects", | |
| "metric": [ | |
| "loglikelihood_acc_single_token" | |
| ], | |
| "hf_avail_splits": [ | |
| "train" | |
| ], | |
| "evaluation_splits": [ | |
| "train" | |
| ], | |
| "few_shots_split": null, | |
| "few_shots_select": null, | |
| "generation_size": -1, | |
| "stop_sequence": [ | |
| "</s>", | |
| "Q:", | |
| "\n\n" | |
| ], | |
| "output_regex": null, | |
| "frozen": false, | |
| "suite": [ | |
| "lighteval" | |
| ], | |
| "original_num_docs": 300, | |
| "effective_num_docs": 300, | |
| "trust_dataset": true, | |
| "must_remove_duplicate_docs": null | |
| } | |
| }, | |
| "summary_tasks": { | |
| "lighteval|bigbench:causal_judgment|0": { | |
| "hashes": { | |
| "hash_examples": "dfb1ae47218f2850", | |
| "hash_full_prompts": "fc6cc504d4ca599e", | |
| "hash_input_tokens": "58fb6d0b04d33e40", | |
| "hash_cont_tokens": "f60cdfb6764beda9" | |
| }, | |
| "truncated": 0, | |
| "non_truncated": 190, | |
| "padded": 189, | |
| "non_padded": 1, | |
| "effective_few_shots": 0.0, | |
| "num_truncated_few_shots": 0 | |
| }, | |
| "lighteval|bigbench:date_understanding|0": { | |
| "hashes": { | |
| "hash_examples": "2b823c41500a6ec2", | |
| "hash_full_prompts": "65896778991c35e0", | |
| "hash_input_tokens": "3551d37c672894be", | |
| "hash_cont_tokens": "9ba8f8f33248c263" | |
| }, | |
| "truncated": 0, | |
| "non_truncated": 369, | |
| "padded": 369, | |
| "non_padded": 0, | |
| "effective_few_shots": 0.0, | |
| "num_truncated_few_shots": 0 | |
| }, | |
| "lighteval|bigbench:disambiguation_qa|0": { | |
| "hashes": { | |
| "hash_examples": "2a4c3d41db198cea", | |
| "hash_full_prompts": "9152cdaa3cb483e2", | |
| "hash_input_tokens": "b52371edddaa8656", | |
| "hash_cont_tokens": "d8d0e8c885a30065" | |
| }, | |
| "truncated": 0, | |
| "non_truncated": 258, | |
| "padded": 258, | |
| "non_padded": 0, | |
| "effective_few_shots": 0.0, | |
| "num_truncated_few_shots": 0 | |
| }, | |
| "lighteval|bigbench:geometric_shapes|0": { | |
| "hashes": { | |
| "hash_examples": "24aa261103911b72", | |
| "hash_full_prompts": "47061a00712fe06f", | |
| "hash_input_tokens": "557e5eaaa210b045", | |
| "hash_cont_tokens": "998701c8daf93883" | |
| }, | |
| "truncated": 0, | |
| "non_truncated": 360, | |
| "padded": 360, | |
| "non_padded": 0, | |
| "effective_few_shots": 0.0, | |
| "num_truncated_few_shots": 0 | |
| }, | |
| "lighteval|bigbench:logical_deduction_five_objects|0": { | |
| "hashes": { | |
| "hash_examples": "cb5bdc92afc41f83", | |
| "hash_full_prompts": "405e95b6be9a22dc", | |
| "hash_input_tokens": "3dc203ea2157c330", | |
| "hash_cont_tokens": "de5444578c42e943" | |
| }, | |
| "truncated": 0, | |
| "non_truncated": 500, | |
| "padded": 500, | |
| "non_padded": 0, | |
| "effective_few_shots": 0.0, | |
| "num_truncated_few_shots": 0 | |
| }, | |
| "lighteval|bigbench:logical_deduction_seven_objects|0": { | |
| "hashes": { | |
| "hash_examples": "b6805ea696739f9f", | |
| "hash_full_prompts": "43a3a3420b7115ec", | |
| "hash_input_tokens": "94f1bb7583276239", | |
| "hash_cont_tokens": "6e6be500a1fcdf22" | |
| }, | |
| "truncated": 0, | |
| "non_truncated": 700, | |
| "padded": 687, | |
| "non_padded": 13, | |
| "effective_few_shots": 0.0, | |
| "num_truncated_few_shots": 0 | |
| }, | |
| "lighteval|bigbench:logical_deduction_three_objects|0": { | |
| "hashes": { | |
| "hash_examples": "0509e5712ab9bcdb", | |
| "hash_full_prompts": "ea9281587cbe257d", | |
| "hash_input_tokens": "36014b30521c248a", | |
| "hash_cont_tokens": "bcbe9729575c6976" | |
| }, | |
| "truncated": 0, | |
| "non_truncated": 300, | |
| "padded": 297, | |
| "non_padded": 3, | |
| "effective_few_shots": 0.0, | |
| "num_truncated_few_shots": 0 | |
| }, | |
| "lighteval|bigbench:movie_recommendation|0": { | |
| "hashes": { | |
| "hash_examples": "530cc6f737830f45", | |
| "hash_full_prompts": "7781cede4b566228", | |
| "hash_input_tokens": "02b0aa21ec52a994", | |
| "hash_cont_tokens": "6b81fe39250b92d6" | |
| }, | |
| "truncated": 0, | |
| "non_truncated": 500, | |
| "padded": 494, | |
| "non_padded": 6, | |
| "effective_few_shots": 0.0, | |
| "num_truncated_few_shots": 0 | |
| }, | |
| "lighteval|bigbench:navigate|0": { | |
| "hashes": { | |
| "hash_examples": "7962ef85d0058b9a", | |
| "hash_full_prompts": "03dc193c000dabf8", | |
| "hash_input_tokens": "e5af8fd8c52b0f84", | |
| "hash_cont_tokens": "022ad4720d477fe8" | |
| }, | |
| "truncated": 0, | |
| "non_truncated": 1000, | |
| "padded": 981, | |
| "non_padded": 19, | |
| "effective_few_shots": 0.0, | |
| "num_truncated_few_shots": 0 | |
| }, | |
| "lighteval|bigbench:reasoning_about_colored_objects|0": { | |
| "hashes": { | |
| "hash_examples": "39be1ab1677a651d", | |
| "hash_full_prompts": "359c8723dd8cf505", | |
| "hash_input_tokens": "d6e0260b84823dcf", | |
| "hash_cont_tokens": "0e1f5086642305e6" | |
| }, | |
| "truncated": 0, | |
| "non_truncated": 2000, | |
| "padded": 1991, | |
| "non_padded": 9, | |
| "effective_few_shots": 0.0, | |
| "num_truncated_few_shots": 0 | |
| }, | |
| "lighteval|bigbench:ruin_names|0": { | |
| "hashes": { | |
| "hash_examples": "e9b96b31d2154941", | |
| "hash_full_prompts": "c76b3c2dd07ddb04", | |
| "hash_input_tokens": "aa9269e724bec8cd", | |
| "hash_cont_tokens": "7e46b30c5970694d" | |
| }, | |
| "truncated": 0, | |
| "non_truncated": 448, | |
| "padded": 447, | |
| "non_padded": 1, | |
| "effective_few_shots": 0.0, | |
| "num_truncated_few_shots": 0 | |
| }, | |
| "lighteval|bigbench:salient_translation_error_detection|0": { | |
| "hashes": { | |
| "hash_examples": "951ac59f7ad0427d", | |
| "hash_full_prompts": "911539e1fbe765ac", | |
| "hash_input_tokens": "af7c908dd5649806", | |
| "hash_cont_tokens": "c2d00eb5b415f14b" | |
| }, | |
| "truncated": 0, | |
| "non_truncated": 998, | |
| "padded": 998, | |
| "non_padded": 0, | |
| "effective_few_shots": 0.0, | |
| "num_truncated_few_shots": 0 | |
| }, | |
| "lighteval|bigbench:snarks|0": { | |
| "hashes": { | |
| "hash_examples": "3a53eb9b9d758534", | |
| "hash_full_prompts": "83a88ab30ec943fe", | |
| "hash_input_tokens": "438ce37d0598df83", | |
| "hash_cont_tokens": "a9eee5dc9fe75a8f" | |
| }, | |
| "truncated": 0, | |
| "non_truncated": 181, | |
| "padded": 173, | |
| "non_padded": 8, | |
| "effective_few_shots": 0.0, | |
| "num_truncated_few_shots": 0 | |
| }, | |
| "lighteval|bigbench:sports_understanding|0": { | |
| "hashes": { | |
| "hash_examples": "bd65741f00770373", | |
| "hash_full_prompts": "198e075d6d7f5329", | |
| "hash_input_tokens": "e200ccf0fc92e620", | |
| "hash_cont_tokens": "873ce08c23c5beb9" | |
| }, | |
| "truncated": 0, | |
| "non_truncated": 1000, | |
| "padded": 1000, | |
| "non_padded": 0, | |
| "effective_few_shots": 0.0, | |
| "num_truncated_few_shots": 0 | |
| }, | |
| "lighteval|bigbench:temporal_sequences|0": { | |
| "hashes": { | |
| "hash_examples": "1d13139f47cb2df7", | |
| "hash_full_prompts": "b234ca31e3cec55a", | |
| "hash_input_tokens": "0c011e1e08917b0d", | |
| "hash_cont_tokens": "a99963662702fcb8" | |
| }, | |
| "truncated": 0, | |
| "non_truncated": 1000, | |
| "padded": 988, | |
| "non_padded": 12, | |
| "effective_few_shots": 0.0, | |
| "num_truncated_few_shots": 0 | |
| }, | |
| "lighteval|bigbench:tracking_shuffled_objects_five_objects|0": { | |
| "hashes": { | |
| "hash_examples": "8770a702a9646648", | |
| "hash_full_prompts": "cfbf754293d3037f", | |
| "hash_input_tokens": "a61bc551e64ec248", | |
| "hash_cont_tokens": "fcde41fda82cbfbd" | |
| }, | |
| "truncated": 0, | |
| "non_truncated": 1250, | |
| "padded": 1240, | |
| "non_padded": 10, | |
| "effective_few_shots": 0.0, | |
| "num_truncated_few_shots": 0 | |
| }, | |
| "lighteval|bigbench:tracking_shuffled_objects_seven_objects|0": { | |
| "hashes": { | |
| "hash_examples": "b469b7d073824a59", | |
| "hash_full_prompts": "31722bad2cef8da1", | |
| "hash_input_tokens": "89f187294404fdb2", | |
| "hash_cont_tokens": "d31d39fd0432e625" | |
| }, | |
| "truncated": 0, | |
| "non_truncated": 1750, | |
| "padded": 1675, | |
| "non_padded": 75, | |
| "effective_few_shots": 0.0, | |
| "num_truncated_few_shots": 0 | |
| }, | |
| "lighteval|bigbench:tracking_shuffled_objects_three_objects|0": { | |
| "hashes": { | |
| "hash_examples": "0509e5712ab9bcdb", | |
| "hash_full_prompts": "ea9281587cbe257d", | |
| "hash_input_tokens": "e863a6df2896b953", | |
| "hash_cont_tokens": "0bb87f6cc907c01f" | |
| }, | |
| "truncated": 0, | |
| "non_truncated": 300, | |
| "padded": 294, | |
| "non_padded": 6, | |
| "effective_few_shots": 0.0, | |
| "num_truncated_few_shots": 0 | |
| } | |
| }, | |
| "summary_general": { | |
| "hashes": { | |
| "hash_examples": "51a30c4501ba4586", | |
| "hash_full_prompts": "d20cce6e7e01d8d5", | |
| "hash_input_tokens": "5f9ef4639840f47f", | |
| "hash_cont_tokens": "9cf40ca479e12372" | |
| }, | |
| "truncated": 0, | |
| "non_truncated": 13104, | |
| "padded": 12941, | |
| "non_padded": 163, | |
| "num_truncated_few_shots": 0 | |
| } | |
| } |