Llama2-Code-Interpreter-main/OpenCodeInterpreter/evaluation/evalplus/tools/humaneval/check_ground_truth.py

"""This script checks:
1. Independence of "contract" and "canonical_solution" in groundtruth. (i.e., it should work without the "contract" part)
"""

import multiprocessing as mp
import pathlib

from rich.progress import track

from evalplus.data import get_human_eval_plus

if __name__ == "__main__":
    human_eval_plus = get_human_eval_plus().values()

    for i, task in track(enumerate(human_eval_plus)):
        fname = (
            pathlib.Path(__file__).parent.parent
            / "groundtruth"
            / "humaneval"
            / (str(i).zfill(3) + "_" + task["entry_point"] + ".py")
        )
        print(fname)
        code = open(fname, "r").read()
        if task["contract"]:
            assert task["contract"] in code
            code = code.replace(task["contract"], "\n")

        # run the code in a subprocess
        p = mp.Process(target=exec, args=(code, globals()))
        p.start()
        p.join(timeout=2)
        assert not p.is_alive(), f"Timeout for {fname}!"
        p.terminate()
        p.join()
        assert p.exitcode == 0, f"Error for {fname}! {code}"