Spaces:
Runtime error
Runtime error
File size: 2,836 Bytes
c87c295 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 |
import math
def check_id(data, task_id):
assert data[task_id]["task_id"] == f"HumanEval/{task_id}"
def fix(data):
# https://github.com/evalplus/evalplus/issues/44
check_id(data, 115)
data[115]["prompt"] = "import math\n" + data[115]["prompt"].replace(
" import math\n", ""
)
check_id(data, 114)
data[114]["prompt"] = data[114]["prompt"].replace("import math\n", "")
# https://github.com/evalplus/evalplus/issues/30#issue-1944054257
check_id(data, 35)
data[35][
"contract"
] += ' assert all(type(x) in [int, float] for x in l), "invalid inputs" # $_CONTRACT_$\n'
data[35]["canonical_solution"] = data[35]["canonical_solution"].replace(
' assert all(type(x) in [int, float] for x in l), "invalid inputs"\n', ""
)
# https://github.com/evalplus/evalplus/issues/30#issuecomment-1763502126
check_id(data, 28)
data[28][
"contract"
] += ' assert isinstance(strings, list), "invalid inputs" # $_CONTRACT_$\n'
# https://github.com/evalplus/evalplus/issues/34
check_id(data, 32)
terms = data[32]["contract"].splitlines()
terms.insert(2, ' assert xs[-1] != 0, "invalid inputs" # $_CONTRACT_$')
data[32]["contract"] = "\n".join(terms)
# https://github.com/evalplus/evalplus/issues/35
check_id(data, 160)
terms = data[160]["contract"].splitlines()
terms.insert(
len(terms),
' assert not any([operand[i-1] == 0 and operator[i] == "//" for i in range(1, len(operand))]), "invalid inputs" # $_CONTRACT_$',
)
data[160]["contract"] = "\n".join(terms)
return data
if __name__ == "__main__":
import json
TASK_INSPECT = [28, 32, 35, 114, 115, 160]
SOURCE_VERSION = "v0.1.8"
TARGET_VERSION = "v0.1.9"
def evolve(src_file, tgt_file):
with open(src_file) as f:
data = [json.loads(line) for line in f.readlines() if line]
data = fix(data)
with open(tgt_file, "wb") as f:
for x in data:
f.write((json.dumps(x) + "\n").encode("utf-8"))
evolve(
f"HumanEvalPlus-{SOURCE_VERSION}.jsonl", f"HumanEvalPlus-{TARGET_VERSION}.jsonl"
)
evolve(
f"HumanEvalPlus-Mini-{SOURCE_VERSION}.jsonl",
f"HumanEvalPlus-Mini-{TARGET_VERSION}.jsonl",
)
# Debug the output of jsonl
with open(f"HumanEvalPlus-{TARGET_VERSION}.jsonl") as f:
data = [json.loads(line) for line in f.readlines() if line]
data = {x["task_id"]: x for x in data}
for tid in TASK_INSPECT:
code = (
data[f"HumanEval/{tid}"]["prompt"]
+ data[f"HumanEval/{tid}"]["contract"]
+ data[f"HumanEval/{tid}"]["canonical_solution"]
)
print(code)
exec(code)
print("====================================")
|