open-r1-eval-leaderboard
/
eval_results
/meta-llama
/Meta-Llama-3-70B-Instruct
/main
/bbh
/results_2024-04-21T19-55-44.505259.json
| { | |
| "config_general": { | |
| "lighteval_sha": "?", | |
| "num_fewshot_seeds": 1, | |
| "override_batch_size": 4, | |
| "max_samples": null, | |
| "job_id": "", | |
| "start_time": 796129.532502433, | |
| "end_time": 797008.24915019, | |
| "total_evaluation_time_secondes": "878.7166477569845", | |
| "model_name": "meta-llama/Meta-Llama-3-70B-Instruct", | |
| "model_sha": "5fcb2901844dde3111159f24205b71c25900ffbd", | |
| "model_dtype": "torch.bfloat16", | |
| "model_size": "131.73 GB", | |
| "config": null | |
| }, | |
| "results": { | |
| "lighteval|bigbench:causal_judgment|0": { | |
| "acc": 0.5210526315789473, | |
| "acc_stderr": 0.03633739504773335 | |
| }, | |
| "lighteval|bigbench:date_understanding|0": { | |
| "acc": 0.986449864498645, | |
| "acc_stderr": 0.006026781111722431 | |
| }, | |
| "lighteval|bigbench:disambiguation_qa|0": { | |
| "acc": 0.31007751937984496, | |
| "acc_stderr": 0.028851521910581428 | |
| }, | |
| "lighteval|bigbench:geometric_shapes|0": { | |
| "acc": 0.1, | |
| "acc_stderr": 0.01583339441907784 | |
| }, | |
| "lighteval|bigbench:logical_deduction_five_objects|0": { | |
| "acc": 0.2, | |
| "acc_stderr": 0.01790645924143384 | |
| }, | |
| "lighteval|bigbench:logical_deduction_seven_objects|0": { | |
| "acc": 0.14285714285714285, | |
| "acc_stderr": 0.013235458703202278 | |
| }, | |
| "lighteval|bigbench:logical_deduction_three_objects|0": { | |
| "acc": 0.3333333333333333, | |
| "acc_stderr": 0.027262027336984396 | |
| }, | |
| "lighteval|bigbench:movie_recommendation|0": { | |
| "acc": 0.244, | |
| "acc_stderr": 0.019226734893614587 | |
| }, | |
| "lighteval|bigbench:navigate|0": { | |
| "acc": 0.5, | |
| "acc_stderr": 0.015819299929208316 | |
| }, | |
| "lighteval|bigbench:reasoning_about_colored_objects|0": { | |
| "acc": 0.178, | |
| "acc_stderr": 0.0085553841116954 | |
| }, | |
| "lighteval|bigbench:ruin_names|0": { | |
| "acc": 0.27901785714285715, | |
| "acc_stderr": 0.021214094157265963 | |
| }, | |
| "lighteval|bigbench:salient_translation_error_detection|0": { | |
| "acc": 0.14829659318637275, | |
| "acc_stderr": 0.011255432818843761 | |
| }, | |
| "lighteval|bigbench:snarks|0": { | |
| "acc": 0.46408839779005523, | |
| "acc_stderr": 0.03717155101139654 | |
| }, | |
| "lighteval|bigbench:sports_understanding|0": { | |
| "acc": 0.728, | |
| "acc_stderr": 0.014078856992462623 | |
| }, | |
| "lighteval|bigbench:temporal_sequences|0": { | |
| "acc": 1.0, | |
| "acc_stderr": 0.0 | |
| }, | |
| "lighteval|bigbench:tracking_shuffled_objects_five_objects|0": { | |
| "acc": 0.2, | |
| "acc_stderr": 0.011318236699485788 | |
| }, | |
| "lighteval|bigbench:tracking_shuffled_objects_seven_objects|0": { | |
| "acc": 0.14285714285714285, | |
| "acc_stderr": 0.008367248752248818 | |
| }, | |
| "lighteval|bigbench:tracking_shuffled_objects_three_objects|0": { | |
| "acc": 0.3333333333333333, | |
| "acc_stderr": 0.027262027336984396 | |
| }, | |
| "lighteval|bigbench:_average|0": { | |
| "acc": 0.37840910088653756, | |
| "acc_stderr": 0.0177623280263301 | |
| }, | |
| "all": { | |
| "acc": 0.37840910088653756, | |
| "acc_stderr": 0.0177623280263301 | |
| } | |
| }, | |
| "versions": { | |
| "lighteval|bigbench:causal_judgment|0": 0, | |
| "lighteval|bigbench:date_understanding|0": 0, | |
| "lighteval|bigbench:disambiguation_qa|0": 0, | |
| "lighteval|bigbench:geometric_shapes|0": 0, | |
| "lighteval|bigbench:logical_deduction_five_objects|0": 0, | |
| "lighteval|bigbench:logical_deduction_seven_objects|0": 0, | |
| "lighteval|bigbench:logical_deduction_three_objects|0": 0, | |
| "lighteval|bigbench:movie_recommendation|0": 0, | |
| "lighteval|bigbench:navigate|0": 0, | |
| "lighteval|bigbench:reasoning_about_colored_objects|0": 0, | |
| "lighteval|bigbench:ruin_names|0": 0, | |
| "lighteval|bigbench:salient_translation_error_detection|0": 0, | |
| "lighteval|bigbench:snarks|0": 0, | |
| "lighteval|bigbench:sports_understanding|0": 0, | |
| "lighteval|bigbench:temporal_sequences|0": 0, | |
| "lighteval|bigbench:tracking_shuffled_objects_five_objects|0": 0, | |
| "lighteval|bigbench:tracking_shuffled_objects_seven_objects|0": 0, | |
| "lighteval|bigbench:tracking_shuffled_objects_three_objects|0": 0 | |
| }, | |
| "config_tasks": { | |
| "lighteval|bigbench:causal_judgment": { | |
| "name": "bigbench:causal_judgment", | |
| "prompt_function": "bbh_lighteval", | |
| "hf_repo": "lighteval/bbh", | |
| "hf_subset": "causal_judgement", | |
| "metric": [ | |
| "loglikelihood_acc_single_token" | |
| ], | |
| "hf_avail_splits": [ | |
| "train" | |
| ], | |
| "evaluation_splits": [ | |
| "train" | |
| ], | |
| "few_shots_split": null, | |
| "few_shots_select": null, | |
| "generation_size": -1, | |
| "stop_sequence": [ | |
| "</s>", | |
| "Q:", | |
| "\n\n" | |
| ], | |
| "output_regex": null, | |
| "frozen": false, | |
| "suite": [ | |
| "lighteval" | |
| ], | |
| "original_num_docs": 190, | |
| "effective_num_docs": 190, | |
| "trust_dataset": true, | |
| "must_remove_duplicate_docs": null | |
| }, | |
| "lighteval|bigbench:date_understanding": { | |
| "name": "bigbench:date_understanding", | |
| "prompt_function": "bbh_lighteval", | |
| "hf_repo": "lighteval/bbh", | |
| "hf_subset": "date_understanding", | |
| "metric": [ | |
| "loglikelihood_acc_single_token" | |
| ], | |
| "hf_avail_splits": [ | |
| "train" | |
| ], | |
| "evaluation_splits": [ | |
| "train" | |
| ], | |
| "few_shots_split": null, | |
| "few_shots_select": null, | |
| "generation_size": -1, | |
| "stop_sequence": [ | |
| "</s>", | |
| "Q:", | |
| "\n\n" | |
| ], | |
| "output_regex": null, | |
| "frozen": false, | |
| "suite": [ | |
| "lighteval" | |
| ], | |
| "original_num_docs": 369, | |
| "effective_num_docs": 369, | |
| "trust_dataset": true, | |
| "must_remove_duplicate_docs": null | |
| }, | |
| "lighteval|bigbench:disambiguation_qa": { | |
| "name": "bigbench:disambiguation_qa", | |
| "prompt_function": "bbh_lighteval", | |
| "hf_repo": "lighteval/bbh", | |
| "hf_subset": "disambiguation_qa", | |
| "metric": [ | |
| "loglikelihood_acc_single_token" | |
| ], | |
| "hf_avail_splits": [ | |
| "train" | |
| ], | |
| "evaluation_splits": [ | |
| "train" | |
| ], | |
| "few_shots_split": null, | |
| "few_shots_select": null, | |
| "generation_size": -1, | |
| "stop_sequence": [ | |
| "</s>", | |
| "Q:", | |
| "\n\n" | |
| ], | |
| "output_regex": null, | |
| "frozen": false, | |
| "suite": [ | |
| "lighteval" | |
| ], | |
| "original_num_docs": 258, | |
| "effective_num_docs": 258, | |
| "trust_dataset": true, | |
| "must_remove_duplicate_docs": null | |
| }, | |
| "lighteval|bigbench:geometric_shapes": { | |
| "name": "bigbench:geometric_shapes", | |
| "prompt_function": "bbh_lighteval", | |
| "hf_repo": "lighteval/bbh", | |
| "hf_subset": "geometric_shapes", | |
| "metric": [ | |
| "loglikelihood_acc_single_token" | |
| ], | |
| "hf_avail_splits": [ | |
| "train" | |
| ], | |
| "evaluation_splits": [ | |
| "train" | |
| ], | |
| "few_shots_split": null, | |
| "few_shots_select": null, | |
| "generation_size": -1, | |
| "stop_sequence": [ | |
| "</s>", | |
| "Q:", | |
| "\n\n" | |
| ], | |
| "output_regex": null, | |
| "frozen": false, | |
| "suite": [ | |
| "lighteval" | |
| ], | |
| "original_num_docs": 360, | |
| "effective_num_docs": 360, | |
| "trust_dataset": true, | |
| "must_remove_duplicate_docs": null | |
| }, | |
| "lighteval|bigbench:logical_deduction_five_objects": { | |
| "name": "bigbench:logical_deduction_five_objects", | |
| "prompt_function": "bbh_lighteval", | |
| "hf_repo": "lighteval/bbh", | |
| "hf_subset": "logical_deduction_five_objects", | |
| "metric": [ | |
| "loglikelihood_acc_single_token" | |
| ], | |
| "hf_avail_splits": [ | |
| "train" | |
| ], | |
| "evaluation_splits": [ | |
| "train" | |
| ], | |
| "few_shots_split": null, | |
| "few_shots_select": null, | |
| "generation_size": -1, | |
| "stop_sequence": [ | |
| "</s>", | |
| "Q:", | |
| "\n\n" | |
| ], | |
| "output_regex": null, | |
| "frozen": false, | |
| "suite": [ | |
| "lighteval" | |
| ], | |
| "original_num_docs": 500, | |
| "effective_num_docs": 500, | |
| "trust_dataset": true, | |
| "must_remove_duplicate_docs": null | |
| }, | |
| "lighteval|bigbench:logical_deduction_seven_objects": { | |
| "name": "bigbench:logical_deduction_seven_objects", | |
| "prompt_function": "bbh_lighteval", | |
| "hf_repo": "lighteval/bbh", | |
| "hf_subset": "logical_deduction_seven_objects", | |
| "metric": [ | |
| "loglikelihood_acc_single_token" | |
| ], | |
| "hf_avail_splits": [ | |
| "train" | |
| ], | |
| "evaluation_splits": [ | |
| "train" | |
| ], | |
| "few_shots_split": null, | |
| "few_shots_select": null, | |
| "generation_size": -1, | |
| "stop_sequence": [ | |
| "</s>", | |
| "Q:", | |
| "\n\n" | |
| ], | |
| "output_regex": null, | |
| "frozen": false, | |
| "suite": [ | |
| "lighteval" | |
| ], | |
| "original_num_docs": 700, | |
| "effective_num_docs": 700, | |
| "trust_dataset": true, | |
| "must_remove_duplicate_docs": null | |
| }, | |
| "lighteval|bigbench:logical_deduction_three_objects": { | |
| "name": "bigbench:logical_deduction_three_objects", | |
| "prompt_function": "bbh_lighteval", | |
| "hf_repo": "lighteval/bbh", | |
| "hf_subset": "logical_deduction_three_objects", | |
| "metric": [ | |
| "loglikelihood_acc_single_token" | |
| ], | |
| "hf_avail_splits": [ | |
| "train" | |
| ], | |
| "evaluation_splits": [ | |
| "train" | |
| ], | |
| "few_shots_split": null, | |
| "few_shots_select": null, | |
| "generation_size": -1, | |
| "stop_sequence": [ | |
| "</s>", | |
| "Q:", | |
| "\n\n" | |
| ], | |
| "output_regex": null, | |
| "frozen": false, | |
| "suite": [ | |
| "lighteval" | |
| ], | |
| "original_num_docs": 300, | |
| "effective_num_docs": 300, | |
| "trust_dataset": true, | |
| "must_remove_duplicate_docs": null | |
| }, | |
| "lighteval|bigbench:movie_recommendation": { | |
| "name": "bigbench:movie_recommendation", | |
| "prompt_function": "bbh_lighteval", | |
| "hf_repo": "lighteval/bbh", | |
| "hf_subset": "movie_recommendation", | |
| "metric": [ | |
| "loglikelihood_acc_single_token" | |
| ], | |
| "hf_avail_splits": [ | |
| "train" | |
| ], | |
| "evaluation_splits": [ | |
| "train" | |
| ], | |
| "few_shots_split": null, | |
| "few_shots_select": null, | |
| "generation_size": -1, | |
| "stop_sequence": [ | |
| "</s>", | |
| "Q:", | |
| "\n\n" | |
| ], | |
| "output_regex": null, | |
| "frozen": false, | |
| "suite": [ | |
| "lighteval" | |
| ], | |
| "original_num_docs": 500, | |
| "effective_num_docs": 500, | |
| "trust_dataset": true, | |
| "must_remove_duplicate_docs": null | |
| }, | |
| "lighteval|bigbench:navigate": { | |
| "name": "bigbench:navigate", | |
| "prompt_function": "bbh_lighteval", | |
| "hf_repo": "lighteval/bbh", | |
| "hf_subset": "navigate", | |
| "metric": [ | |
| "loglikelihood_acc_single_token" | |
| ], | |
| "hf_avail_splits": [ | |
| "train" | |
| ], | |
| "evaluation_splits": [ | |
| "train" | |
| ], | |
| "few_shots_split": null, | |
| "few_shots_select": null, | |
| "generation_size": -1, | |
| "stop_sequence": [ | |
| "</s>", | |
| "Q:", | |
| "\n\n" | |
| ], | |
| "output_regex": null, | |
| "frozen": false, | |
| "suite": [ | |
| "lighteval" | |
| ], | |
| "original_num_docs": 1000, | |
| "effective_num_docs": 1000, | |
| "trust_dataset": true, | |
| "must_remove_duplicate_docs": null | |
| }, | |
| "lighteval|bigbench:reasoning_about_colored_objects": { | |
| "name": "bigbench:reasoning_about_colored_objects", | |
| "prompt_function": "bbh_lighteval", | |
| "hf_repo": "lighteval/bbh", | |
| "hf_subset": "reasoning_about_colored_objects", | |
| "metric": [ | |
| "loglikelihood_acc_single_token" | |
| ], | |
| "hf_avail_splits": [ | |
| "train" | |
| ], | |
| "evaluation_splits": [ | |
| "train" | |
| ], | |
| "few_shots_split": null, | |
| "few_shots_select": null, | |
| "generation_size": -1, | |
| "stop_sequence": [ | |
| "</s>", | |
| "Q:", | |
| "\n\n" | |
| ], | |
| "output_regex": null, | |
| "frozen": false, | |
| "suite": [ | |
| "lighteval" | |
| ], | |
| "original_num_docs": 2000, | |
| "effective_num_docs": 2000, | |
| "trust_dataset": true, | |
| "must_remove_duplicate_docs": null | |
| }, | |
| "lighteval|bigbench:ruin_names": { | |
| "name": "bigbench:ruin_names", | |
| "prompt_function": "bbh_lighteval", | |
| "hf_repo": "lighteval/bbh", | |
| "hf_subset": "ruin_names", | |
| "metric": [ | |
| "loglikelihood_acc_single_token" | |
| ], | |
| "hf_avail_splits": [ | |
| "train" | |
| ], | |
| "evaluation_splits": [ | |
| "train" | |
| ], | |
| "few_shots_split": null, | |
| "few_shots_select": null, | |
| "generation_size": -1, | |
| "stop_sequence": [ | |
| "</s>", | |
| "Q:", | |
| "\n\n" | |
| ], | |
| "output_regex": null, | |
| "frozen": false, | |
| "suite": [ | |
| "lighteval" | |
| ], | |
| "original_num_docs": 448, | |
| "effective_num_docs": 448, | |
| "trust_dataset": true, | |
| "must_remove_duplicate_docs": null | |
| }, | |
| "lighteval|bigbench:salient_translation_error_detection": { | |
| "name": "bigbench:salient_translation_error_detection", | |
| "prompt_function": "bbh_lighteval", | |
| "hf_repo": "lighteval/bbh", | |
| "hf_subset": "salient_translation_error_detection", | |
| "metric": [ | |
| "loglikelihood_acc_single_token" | |
| ], | |
| "hf_avail_splits": [ | |
| "train" | |
| ], | |
| "evaluation_splits": [ | |
| "train" | |
| ], | |
| "few_shots_split": null, | |
| "few_shots_select": null, | |
| "generation_size": -1, | |
| "stop_sequence": [ | |
| "</s>", | |
| "Q:", | |
| "\n\n" | |
| ], | |
| "output_regex": null, | |
| "frozen": false, | |
| "suite": [ | |
| "lighteval" | |
| ], | |
| "original_num_docs": 998, | |
| "effective_num_docs": 998, | |
| "trust_dataset": true, | |
| "must_remove_duplicate_docs": null | |
| }, | |
| "lighteval|bigbench:snarks": { | |
| "name": "bigbench:snarks", | |
| "prompt_function": "bbh_lighteval", | |
| "hf_repo": "lighteval/bbh", | |
| "hf_subset": "snarks", | |
| "metric": [ | |
| "loglikelihood_acc_single_token" | |
| ], | |
| "hf_avail_splits": [ | |
| "train" | |
| ], | |
| "evaluation_splits": [ | |
| "train" | |
| ], | |
| "few_shots_split": null, | |
| "few_shots_select": null, | |
| "generation_size": -1, | |
| "stop_sequence": [ | |
| "</s>", | |
| "Q:", | |
| "\n\n" | |
| ], | |
| "output_regex": null, | |
| "frozen": false, | |
| "suite": [ | |
| "lighteval" | |
| ], | |
| "original_num_docs": 181, | |
| "effective_num_docs": 181, | |
| "trust_dataset": true, | |
| "must_remove_duplicate_docs": null | |
| }, | |
| "lighteval|bigbench:sports_understanding": { | |
| "name": "bigbench:sports_understanding", | |
| "prompt_function": "bbh_lighteval", | |
| "hf_repo": "lighteval/bbh", | |
| "hf_subset": "sports_understanding", | |
| "metric": [ | |
| "loglikelihood_acc_single_token" | |
| ], | |
| "hf_avail_splits": [ | |
| "train" | |
| ], | |
| "evaluation_splits": [ | |
| "train" | |
| ], | |
| "few_shots_split": null, | |
| "few_shots_select": null, | |
| "generation_size": -1, | |
| "stop_sequence": [ | |
| "</s>", | |
| "Q:", | |
| "\n\n" | |
| ], | |
| "output_regex": null, | |
| "frozen": false, | |
| "suite": [ | |
| "lighteval" | |
| ], | |
| "original_num_docs": 1000, | |
| "effective_num_docs": 1000, | |
| "trust_dataset": true, | |
| "must_remove_duplicate_docs": null | |
| }, | |
| "lighteval|bigbench:temporal_sequences": { | |
| "name": "bigbench:temporal_sequences", | |
| "prompt_function": "bbh_lighteval", | |
| "hf_repo": "lighteval/bbh", | |
| "hf_subset": "temporal_sequences", | |
| "metric": [ | |
| "loglikelihood_acc_single_token" | |
| ], | |
| "hf_avail_splits": [ | |
| "train" | |
| ], | |
| "evaluation_splits": [ | |
| "train" | |
| ], | |
| "few_shots_split": null, | |
| "few_shots_select": null, | |
| "generation_size": -1, | |
| "stop_sequence": [ | |
| "</s>", | |
| "Q:", | |
| "\n\n" | |
| ], | |
| "output_regex": null, | |
| "frozen": false, | |
| "suite": [ | |
| "lighteval" | |
| ], | |
| "original_num_docs": 1000, | |
| "effective_num_docs": 1000, | |
| "trust_dataset": true, | |
| "must_remove_duplicate_docs": null | |
| }, | |
| "lighteval|bigbench:tracking_shuffled_objects_five_objects": { | |
| "name": "bigbench:tracking_shuffled_objects_five_objects", | |
| "prompt_function": "bbh_lighteval", | |
| "hf_repo": "lighteval/bbh", | |
| "hf_subset": "tracking_shuffled_objects_five_objects", | |
| "metric": [ | |
| "loglikelihood_acc_single_token" | |
| ], | |
| "hf_avail_splits": [ | |
| "train" | |
| ], | |
| "evaluation_splits": [ | |
| "train" | |
| ], | |
| "few_shots_split": null, | |
| "few_shots_select": null, | |
| "generation_size": -1, | |
| "stop_sequence": [ | |
| "</s>", | |
| "Q:", | |
| "\n\n" | |
| ], | |
| "output_regex": null, | |
| "frozen": false, | |
| "suite": [ | |
| "lighteval" | |
| ], | |
| "original_num_docs": 1250, | |
| "effective_num_docs": 1250, | |
| "trust_dataset": true, | |
| "must_remove_duplicate_docs": null | |
| }, | |
| "lighteval|bigbench:tracking_shuffled_objects_seven_objects": { | |
| "name": "bigbench:tracking_shuffled_objects_seven_objects", | |
| "prompt_function": "bbh_lighteval", | |
| "hf_repo": "lighteval/bbh", | |
| "hf_subset": "tracking_shuffled_objects_seven_objects", | |
| "metric": [ | |
| "loglikelihood_acc_single_token" | |
| ], | |
| "hf_avail_splits": [ | |
| "train" | |
| ], | |
| "evaluation_splits": [ | |
| "train" | |
| ], | |
| "few_shots_split": null, | |
| "few_shots_select": null, | |
| "generation_size": -1, | |
| "stop_sequence": [ | |
| "</s>", | |
| "Q:", | |
| "\n\n" | |
| ], | |
| "output_regex": null, | |
| "frozen": false, | |
| "suite": [ | |
| "lighteval" | |
| ], | |
| "original_num_docs": 1750, | |
| "effective_num_docs": 1750, | |
| "trust_dataset": true, | |
| "must_remove_duplicate_docs": null | |
| }, | |
| "lighteval|bigbench:tracking_shuffled_objects_three_objects": { | |
| "name": "bigbench:tracking_shuffled_objects_three_objects", | |
| "prompt_function": "bbh_lighteval", | |
| "hf_repo": "lighteval/bbh", | |
| "hf_subset": "tracking_shuffled_objects_three_objects", | |
| "metric": [ | |
| "loglikelihood_acc_single_token" | |
| ], | |
| "hf_avail_splits": [ | |
| "train" | |
| ], | |
| "evaluation_splits": [ | |
| "train" | |
| ], | |
| "few_shots_split": null, | |
| "few_shots_select": null, | |
| "generation_size": -1, | |
| "stop_sequence": [ | |
| "</s>", | |
| "Q:", | |
| "\n\n" | |
| ], | |
| "output_regex": null, | |
| "frozen": false, | |
| "suite": [ | |
| "lighteval" | |
| ], | |
| "original_num_docs": 300, | |
| "effective_num_docs": 300, | |
| "trust_dataset": true, | |
| "must_remove_duplicate_docs": null | |
| } | |
| }, | |
| "summary_tasks": { | |
| "lighteval|bigbench:causal_judgment|0": { | |
| "hashes": { | |
| "hash_examples": "dfb1ae47218f2850", | |
| "hash_full_prompts": "59c75adfe9a31a36", | |
| "hash_input_tokens": "d367bb45f59d1c1a", | |
| "hash_cont_tokens": "7abd56823b0f881d" | |
| }, | |
| "truncated": 0, | |
| "non_truncated": 190, | |
| "padded": 189, | |
| "non_padded": 1, | |
| "effective_few_shots": 0.0, | |
| "num_truncated_few_shots": 0 | |
| }, | |
| "lighteval|bigbench:date_understanding|0": { | |
| "hashes": { | |
| "hash_examples": "2b823c41500a6ec2", | |
| "hash_full_prompts": "884a091accfcb273", | |
| "hash_input_tokens": "a7ab94b84b243a48", | |
| "hash_cont_tokens": "2200d35a17fcfadf" | |
| }, | |
| "truncated": 0, | |
| "non_truncated": 369, | |
| "padded": 369, | |
| "non_padded": 0, | |
| "effective_few_shots": 0.0, | |
| "num_truncated_few_shots": 0 | |
| }, | |
| "lighteval|bigbench:disambiguation_qa|0": { | |
| "hashes": { | |
| "hash_examples": "2a4c3d41db198cea", | |
| "hash_full_prompts": "4e7bae13576ed64d", | |
| "hash_input_tokens": "595bd87275fcf560", | |
| "hash_cont_tokens": "dd2e010c817c594d" | |
| }, | |
| "truncated": 0, | |
| "non_truncated": 258, | |
| "padded": 258, | |
| "non_padded": 0, | |
| "effective_few_shots": 0.0, | |
| "num_truncated_few_shots": 0 | |
| }, | |
| "lighteval|bigbench:geometric_shapes|0": { | |
| "hashes": { | |
| "hash_examples": "24aa261103911b72", | |
| "hash_full_prompts": "581dc240081a5d44", | |
| "hash_input_tokens": "429977eeea021e91", | |
| "hash_cont_tokens": "bd1e625b921480c9" | |
| }, | |
| "truncated": 0, | |
| "non_truncated": 360, | |
| "padded": 360, | |
| "non_padded": 0, | |
| "effective_few_shots": 0.0, | |
| "num_truncated_few_shots": 0 | |
| }, | |
| "lighteval|bigbench:logical_deduction_five_objects|0": { | |
| "hashes": { | |
| "hash_examples": "cb5bdc92afc41f83", | |
| "hash_full_prompts": "467c64e62f5268c8", | |
| "hash_input_tokens": "fd02498d81f88010", | |
| "hash_cont_tokens": "b859a88ce2f1987b" | |
| }, | |
| "truncated": 0, | |
| "non_truncated": 500, | |
| "padded": 500, | |
| "non_padded": 0, | |
| "effective_few_shots": 0.0, | |
| "num_truncated_few_shots": 0 | |
| }, | |
| "lighteval|bigbench:logical_deduction_seven_objects|0": { | |
| "hashes": { | |
| "hash_examples": "b6805ea696739f9f", | |
| "hash_full_prompts": "0ecc606738c5ca4a", | |
| "hash_input_tokens": "87c52da26faaae85", | |
| "hash_cont_tokens": "92b6a4d8c0f0f7c2" | |
| }, | |
| "truncated": 0, | |
| "non_truncated": 700, | |
| "padded": 700, | |
| "non_padded": 0, | |
| "effective_few_shots": 0.0, | |
| "num_truncated_few_shots": 0 | |
| }, | |
| "lighteval|bigbench:logical_deduction_three_objects|0": { | |
| "hashes": { | |
| "hash_examples": "0509e5712ab9bcdb", | |
| "hash_full_prompts": "92218dd1c7cb33a8", | |
| "hash_input_tokens": "b229650974dcd7a7", | |
| "hash_cont_tokens": "41fe514efb632853" | |
| }, | |
| "truncated": 0, | |
| "non_truncated": 300, | |
| "padded": 267, | |
| "non_padded": 33, | |
| "effective_few_shots": 0.0, | |
| "num_truncated_few_shots": 0 | |
| }, | |
| "lighteval|bigbench:movie_recommendation|0": { | |
| "hashes": { | |
| "hash_examples": "530cc6f737830f45", | |
| "hash_full_prompts": "d5861db561509a2b", | |
| "hash_input_tokens": "1f4ceca0a58f8fe0", | |
| "hash_cont_tokens": "d0089dce2f90542e" | |
| }, | |
| "truncated": 0, | |
| "non_truncated": 500, | |
| "padded": 500, | |
| "non_padded": 0, | |
| "effective_few_shots": 0.0, | |
| "num_truncated_few_shots": 0 | |
| }, | |
| "lighteval|bigbench:navigate|0": { | |
| "hashes": { | |
| "hash_examples": "7962ef85d0058b9a", | |
| "hash_full_prompts": "ec163d66b9f3d03c", | |
| "hash_input_tokens": "08d0397109277cc1", | |
| "hash_cont_tokens": "989b10dafac90407" | |
| }, | |
| "truncated": 0, | |
| "non_truncated": 1000, | |
| "padded": 990, | |
| "non_padded": 10, | |
| "effective_few_shots": 0.0, | |
| "num_truncated_few_shots": 0 | |
| }, | |
| "lighteval|bigbench:reasoning_about_colored_objects|0": { | |
| "hashes": { | |
| "hash_examples": "39be1ab1677a651d", | |
| "hash_full_prompts": "8d0f1eb86028a5d6", | |
| "hash_input_tokens": "b7965cad3396a6b4", | |
| "hash_cont_tokens": "bea580eabc54d6a4" | |
| }, | |
| "truncated": 0, | |
| "non_truncated": 2000, | |
| "padded": 2000, | |
| "non_padded": 0, | |
| "effective_few_shots": 0.0, | |
| "num_truncated_few_shots": 0 | |
| }, | |
| "lighteval|bigbench:ruin_names|0": { | |
| "hashes": { | |
| "hash_examples": "e9b96b31d2154941", | |
| "hash_full_prompts": "901d7ff41fc0ee8e", | |
| "hash_input_tokens": "1f76f54b792ebe2f", | |
| "hash_cont_tokens": "67949635d7b35df4" | |
| }, | |
| "truncated": 0, | |
| "non_truncated": 448, | |
| "padded": 443, | |
| "non_padded": 5, | |
| "effective_few_shots": 0.0, | |
| "num_truncated_few_shots": 0 | |
| }, | |
| "lighteval|bigbench:salient_translation_error_detection|0": { | |
| "hashes": { | |
| "hash_examples": "951ac59f7ad0427d", | |
| "hash_full_prompts": "bf3ca7d2f21c2a2f", | |
| "hash_input_tokens": "1f2fa9aa4d92db9c", | |
| "hash_cont_tokens": "7091834cc929babc" | |
| }, | |
| "truncated": 0, | |
| "non_truncated": 998, | |
| "padded": 998, | |
| "non_padded": 0, | |
| "effective_few_shots": 0.0, | |
| "num_truncated_few_shots": 0 | |
| }, | |
| "lighteval|bigbench:snarks|0": { | |
| "hashes": { | |
| "hash_examples": "3a53eb9b9d758534", | |
| "hash_full_prompts": "b6a540690bddada6", | |
| "hash_input_tokens": "123c54819600d026", | |
| "hash_cont_tokens": "445b91a8f6611775" | |
| }, | |
| "truncated": 0, | |
| "non_truncated": 181, | |
| "padded": 180, | |
| "non_padded": 1, | |
| "effective_few_shots": 0.0, | |
| "num_truncated_few_shots": 0 | |
| }, | |
| "lighteval|bigbench:sports_understanding|0": { | |
| "hashes": { | |
| "hash_examples": "bd65741f00770373", | |
| "hash_full_prompts": "bf934b4518e08adf", | |
| "hash_input_tokens": "f8861cd8326e12a4", | |
| "hash_cont_tokens": "61b7586b7d2997a9" | |
| }, | |
| "truncated": 0, | |
| "non_truncated": 1000, | |
| "padded": 1000, | |
| "non_padded": 0, | |
| "effective_few_shots": 0.0, | |
| "num_truncated_few_shots": 0 | |
| }, | |
| "lighteval|bigbench:temporal_sequences|0": { | |
| "hashes": { | |
| "hash_examples": "1d13139f47cb2df7", | |
| "hash_full_prompts": "f1b38a6175096665", | |
| "hash_input_tokens": "9b2630aff90471b8", | |
| "hash_cont_tokens": "58a836d9f1cd718b" | |
| }, | |
| "truncated": 0, | |
| "non_truncated": 1000, | |
| "padded": 1000, | |
| "non_padded": 0, | |
| "effective_few_shots": 0.0, | |
| "num_truncated_few_shots": 0 | |
| }, | |
| "lighteval|bigbench:tracking_shuffled_objects_five_objects|0": { | |
| "hashes": { | |
| "hash_examples": "8770a702a9646648", | |
| "hash_full_prompts": "5d6ffe9b6461b651", | |
| "hash_input_tokens": "989a3385867007b8", | |
| "hash_cont_tokens": "ea6242ff1afdd5d4" | |
| }, | |
| "truncated": 0, | |
| "non_truncated": 1250, | |
| "padded": 1239, | |
| "non_padded": 11, | |
| "effective_few_shots": 0.0, | |
| "num_truncated_few_shots": 0 | |
| }, | |
| "lighteval|bigbench:tracking_shuffled_objects_seven_objects|0": { | |
| "hashes": { | |
| "hash_examples": "b469b7d073824a59", | |
| "hash_full_prompts": "a61210e25dbf195d", | |
| "hash_input_tokens": "416406a726b3b787", | |
| "hash_cont_tokens": "9aae23ebb1e0f579" | |
| }, | |
| "truncated": 0, | |
| "non_truncated": 1750, | |
| "padded": 1709, | |
| "non_padded": 41, | |
| "effective_few_shots": 0.0, | |
| "num_truncated_few_shots": 0 | |
| }, | |
| "lighteval|bigbench:tracking_shuffled_objects_three_objects|0": { | |
| "hashes": { | |
| "hash_examples": "0509e5712ab9bcdb", | |
| "hash_full_prompts": "92218dd1c7cb33a8", | |
| "hash_input_tokens": "24d85c6c80ecfad4", | |
| "hash_cont_tokens": "419f5cf4c2029190" | |
| }, | |
| "truncated": 0, | |
| "non_truncated": 300, | |
| "padded": 264, | |
| "non_padded": 36, | |
| "effective_few_shots": 0.0, | |
| "num_truncated_few_shots": 0 | |
| } | |
| }, | |
| "summary_general": { | |
| "hashes": { | |
| "hash_examples": "51a30c4501ba4586", | |
| "hash_full_prompts": "b40f7b2153556dee", | |
| "hash_input_tokens": "547b2506c8d2a0b5", | |
| "hash_cont_tokens": "dad86a65dc1c0fc2" | |
| }, | |
| "truncated": 0, | |
| "non_truncated": 13104, | |
| "padded": 12966, | |
| "non_padded": 138, | |
| "num_truncated_few_shots": 0 | |
| } | |
| } |