Spaces:
Runtime error
Runtime error
File size: 3,378 Bytes
c87c295 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 |
"""
This script aims at quickly initialize a sketch for HumanEvalPlus. It's not going to be
perfect, but we will either manually or automatically fix/complete it later.
+ CHANGE 1: Adds "contract", "base_input", "atol" in addition to HumanEval.
"""
import json
import os
import pathlib
from importlib import import_module
from inspect import getsource
from typing import Tuple
from tempdir import TempDir
from evalplus.data.humaneval import get_human_eval
HUMANEVAL_PLUS_PATH = (
pathlib.Path(__file__).parent.parent.parent / "HumanEvalPlus.jsonl"
)
def _ret(entry_point) -> str:
"""This is a hacky function to return some garbages so that we can
successfully run the function .
"""
if entry_point == "sort_third" or entry_point == "sort_even":
return [1, 2, 3]
elif entry_point == "bf":
return ()
return "1"
def instrument_inputs(entry_point, prompt, test) -> str:
globals()["_inputs"] = []
fn_text = f"""{prompt.split(f"def {entry_point}")[0]}
def {entry_point}(*args):
_inputs.append(args)
return {_ret(entry_point)}
"""
exec(fn_text, globals())
exec(test.replace("assert ", ""), globals())
exec(f"check({entry_point})", globals())
exec(fn_text, globals())
return globals()["_inputs"]
def get_contract_and_ref(task_id: int, entry_point) -> Tuple[str, str]:
mod = import_module(f"groundtruth.humaneval.{str(task_id).zfill(3)}_{entry_point}")
fn = getattr(mod, entry_point)
doc = fn.__doc__
if task_id == 51:
doc = doc.replace("bcdf\nghjklm", r"bcdf\nghjklm").replace(
"abcdef\nghijklm", r"abcdef\nghijklm"
)
code = (
getsource(fn).replace(doc, "").replace("''''''", '""""""').split('""""""\n')[-1]
)
assert code, f"Something wrong with {task_id}!"
assert code[:3] != "def", f"Something wrong with the {task_id}!"
# split code to contract and impl
contract = ""
impl = ""
reading_contract = True
for line in code.strip("\n").split("\n"):
if reading_contract and "$_CONTRACT_$" in line:
contract += line + "\n"
else:
reading_contract = False
impl += line + "\n"
if contract:
contract = "\n" + contract
return contract, "\n" + impl + "\n"
def get_atol(task_id: int) -> float:
if task_id == 2 or task_id == 4:
return 1e-6
elif task_id == 32:
return 1e-4
return 0
if __name__ == "__main__":
assert not HUMANEVAL_PLUS_PATH.exists(), f"{HUMANEVAL_PLUS_PATH} already exists!"
human_eval = get_human_eval()
with TempDir() as temp_dir:
tmp_file = os.path.join(temp_dir, HUMANEVAL_PLUS_PATH)
with open(tmp_file, "w") as writer:
for task in human_eval:
task_id = int(task["task_id"].split("/")[-1])
task["contract"], task["canonical_solution"] = get_contract_and_ref(
task_id, task["entry_point"]
)
task["base_input"] = instrument_inputs(
task["entry_point"], task["prompt"], task["test"]
)
task["atol"] = get_atol(task_id)
task["task_id"] = task["task_id"]
writer.write(json.dumps(task) + "\n")
# move tmp_file to HUMANEVAL_PLUS_PATH
os.rename(tmp_file, HUMANEVAL_PLUS_PATH)
|