File size: 4,647 Bytes
e9b694b |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 |
from dataclasses import dataclass
from typing import Any, ClassVar
from llmdataparser.base_parser import HuggingFaceDatasetParser, HuggingFaceParseEntry
from llmdataparser.prompts import HUMANEVAL_SYSTEM_PROMPT
@dataclass(frozen=True, kw_only=True, slots=True)
class HumanEvalParseEntry(HuggingFaceParseEntry):
"""Custom entry class for HumanEval, with fields specific to this dataset parser."""
task_id: str
task_name: str
entry_point: str
test: str
@classmethod
def create(
cls,
prompt: str,
answer: str,
raw_question: str,
task_id: str,
entry_point: str,
test: str,
task_name: str,
) -> "HumanEvalParseEntry":
if not task_id:
raise ValueError("Task ID cannot be empty")
if not entry_point:
raise ValueError("Entry point cannot be empty")
return cls(
prompt=prompt,
answer=answer,
raw_question=raw_question,
raw_answer=answer, # In HumanEval, the canonical solution is the raw answer
task_id=task_id,
entry_point=entry_point,
test=test,
task_name=task_name,
)
class HumanEvalDatasetParser(HuggingFaceDatasetParser[HumanEvalParseEntry]):
"""Parser for the HumanEval dataset."""
_data_source: ClassVar[str] = "openai/openai_humaneval"
_default_task: ClassVar[str] = "openai_humaneval"
_task_names: ClassVar[list[str]] = ["openai_humaneval"]
_default_system_prompt: ClassVar[str] = HUMANEVAL_SYSTEM_PROMPT
def process_entry(
self, row: dict[str, Any], task_name: str | None = None, **kwargs: Any
) -> HumanEvalParseEntry:
"""Process a single HumanEval entry."""
raw_question = row["prompt"]
answer = row["canonical_solution"]
task_id = row["task_id"]
entry_point = row["entry_point"]
test = row["test"]
# Combine system prompt with the function signature and docstring
prompt = f"{self._system_prompt}\n\n{raw_question}"
# Use task_name if provided, otherwise use default
task = task_name or self._get_current_task(row)
return HumanEvalParseEntry.create(
prompt=prompt,
answer=answer,
raw_question=raw_question,
task_id=task_id,
entry_point=entry_point,
test=test,
task_name=task, # Guarantee non-None
)
class HumanEvalDatasetPlusParser(HumanEvalDatasetParser):
"""Parser for the HumanEval dataset."""
_data_source: ClassVar[str] = "evalplus/humanevalplus"
_default_task: ClassVar[str] = "default"
_task_names: ClassVar[list[str]] = ["default"]
_default_system_prompt: ClassVar[str] = HUMANEVAL_SYSTEM_PROMPT
def process_entry(
self, row: dict[str, Any], task_name: str | None = None, **kwargs: Any
) -> HumanEvalParseEntry:
"""Process a single HumanEval entry."""
raw_question = row["prompt"]
answer = row["canonical_solution"]
task_id = row["task_id"]
entry_point = row["entry_point"]
test = row["test"]
# Combine system prompt with the function signature and docstring
prompt = f"{self._system_prompt}\n\n{raw_question}"
# Use task_name if provided, otherwise use default
task = task_name or self._get_current_task(row)
return HumanEvalParseEntry.create(
prompt=prompt,
answer=answer,
raw_question=raw_question,
task_id=task_id,
entry_point=entry_point,
test=test,
task_name=task, # task is guaranteed to be str from _get_current_task
)
if __name__ == "__main__":
# Example usage
parser = HumanEvalDatasetParser()
# Load the dataset
parser.load()
# Parse all splits
parser.parse()
# Get parsed data
parsed_data = parser.get_parsed_data
# Print example entry
if parsed_data:
example = parsed_data[0]
print("\nExample parsed entry:")
print(f"Task ID: {example.task_id}")
print(f"Entry Point: {example.entry_point}")
print(f"Prompt:\n{example.prompt}")
print(f"Solution:\n{example.answer}")
parser = HumanEvalDatasetPlusParser()
parser.load()
parser.parse()
parsed_data = parser.get_parsed_data
if parsed_data:
example = parsed_data[0]
print("\nExample parsed entry:")
print(f"Task: {example.task_name}")
print(f"Question: {example.raw_question}")
print(f"Correct Answer: {example.answer}")
|