Spaces:
Runtime error
Runtime error
File size: 1,099 Bytes
c87c295 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 |
"""This script checks:
1. Independence of "contract" and "canonical_solution" in groundtruth. (i.e., it should work without the "contract" part)
"""
import multiprocessing as mp
import pathlib
from rich.progress import track
from evalplus.data import get_human_eval_plus
if __name__ == "__main__":
human_eval_plus = get_human_eval_plus().values()
for i, task in track(enumerate(human_eval_plus)):
fname = (
pathlib.Path(__file__).parent.parent
/ "groundtruth"
/ "humaneval"
/ (str(i).zfill(3) + "_" + task["entry_point"] + ".py")
)
print(fname)
code = open(fname, "r").read()
if task["contract"]:
assert task["contract"] in code
code = code.replace(task["contract"], "\n")
# run the code in a subprocess
p = mp.Process(target=exec, args=(code, globals()))
p.start()
p.join(timeout=2)
assert not p.is_alive(), f"Timeout for {fname}!"
p.terminate()
p.join()
assert p.exitcode == 0, f"Error for {fname}! {code}"
|