| { | |
| "results": { | |
| "ifeval": { | |
| "prompt_level_strict_acc,none": 0.0036968576709796672, | |
| "prompt_level_strict_acc_stderr,none": 0.0026116515685375786, | |
| "inst_level_strict_acc,none": 0.004796163069544364, | |
| "inst_level_strict_acc_stderr,none": "N/A", | |
| "prompt_level_loose_acc,none": 0.0036968576709796672, | |
| "prompt_level_loose_acc_stderr,none": 0.0026116515685375786, | |
| "inst_level_loose_acc,none": 0.005995203836930456, | |
| "inst_level_loose_acc_stderr,none": "N/A", | |
| "alias": "ifeval" | |
| } | |
| }, | |
| "configs": { | |
| "ifeval": { | |
| "task": "ifeval", | |
| "dataset_path": "wis-k/instruction-following-eval", | |
| "test_split": "train", | |
| "doc_to_text": "prompt", | |
| "doc_to_target": 0, | |
| "process_results": "def process_results(doc, results):\n eval_logger.warning(\n \"This task is meant for chat-finetuned models, and may not give meaningful results for models other than `openai` or `anthropic` if `doc_to_text` in its YAML is not wrapped in the appropriate chat template string. This warning will be removed when chat templating support is added natively to local models\"\n )\n\n inp = InputExample(\n key=doc[\"key\"],\n instruction_id_list=doc[\"instruction_id_list\"],\n prompt=doc[\"prompt\"],\n kwargs=doc[\"kwargs\"],\n )\n response = results[0]\n\n out_strict = test_instruction_following_strict(inp, response)\n out_loose = test_instruction_following_loose(inp, response)\n\n return {\n \"prompt_level_strict_acc\": out_strict.follow_all_instructions,\n \"inst_level_strict_acc\": out_strict.follow_instruction_list,\n \"prompt_level_loose_acc\": out_loose.follow_all_instructions,\n \"inst_level_loose_acc\": out_loose.follow_instruction_list,\n }\n", | |
| "description": "", | |
| "target_delimiter": " ", | |
| "fewshot_delimiter": "\n\n", | |
| "num_fewshot": 0, | |
| "metric_list": [ | |
| { | |
| "metric": "prompt_level_strict_acc", | |
| "aggregation": "mean", | |
| "higher_is_better": true | |
| }, | |
| { | |
| "metric": "inst_level_strict_acc", | |
| "aggregation": "def agg_inst_level_acc(items):\n flat_items = [item for sublist in items for item in sublist]\n inst_level_acc = sum(flat_items) / len(flat_items)\n return inst_level_acc\n", | |
| "higher_is_better": true | |
| }, | |
| { | |
| "metric": "prompt_level_loose_acc", | |
| "aggregation": "mean", | |
| "higher_is_better": true | |
| }, | |
| { | |
| "metric": "inst_level_loose_acc", | |
| "aggregation": "def agg_inst_level_acc(items):\n flat_items = [item for sublist in items for item in sublist]\n inst_level_acc = sum(flat_items) / len(flat_items)\n return inst_level_acc\n", | |
| "higher_is_better": true | |
| } | |
| ], | |
| "output_type": "generate_until", | |
| "generation_kwargs": { | |
| "until": [], | |
| "do_sample": false, | |
| "temperature": 0.0, | |
| "max_gen_toks": 1280 | |
| }, | |
| "repeats": 1, | |
| "should_decontaminate": false, | |
| "metadata": { | |
| "version": 2.0 | |
| } | |
| } | |
| }, | |
| "versions": { | |
| "ifeval": 2.0 | |
| }, | |
| "n-shot": { | |
| "ifeval": 0 | |
| }, | |
| "config": { | |
| "model": "hf", | |
| "model_args": "pretrained=Qwen/Qwen1.5-0.5B-Chat,revision=main,dtype=bfloat16", | |
| "batch_size": "auto", | |
| "batch_sizes": [], | |
| "device": null, | |
| "use_cache": null, | |
| "limit": null, | |
| "bootstrap_iters": 100000, | |
| "gen_kwargs": null | |
| }, | |
| "git_hash": "2d2e67f" | |
| } |