feat: add tmlu parser

llmdataparser/__init__.py

@@ -15,6 +15,7 @@ from .mmlu_parser import (
     MMLUReduxDatasetParser,
     TMMLUPlusDatasetParser,
 )
+from .tmlu_parser import TMLUDatasetParser
 from .tw_legal_parser import TWLegalDatasetParser
 
 
@@ -56,3 +57,4 @@ ParserRegistry.register_parser("bbh", BBHDatasetParser)
 ParserRegistry.register_parser("mbpp", MBPPDatasetParser)
 ParserRegistry.register_parser("ifeval", IFEvalDatasetParser)
 ParserRegistry.register_parser("twlegal", TWLegalDatasetParser)
+ParserRegistry.register_parser("tmlu", TMLUDatasetParser)

llmdataparser/prompts.py

@@ -118,7 +118,7 @@ BBH_SYSTEM_PROMPT: Final[str] = textwrap.dedent(
     3. Consider all relationships and constraints mentioned in the problem
     4. Apply structured thinking to reach a valid conclusion
     5. Choose the answer that logically follows from the given information
-    6. Respond with ONLY the letter (A, B, C, etc.) or "True"/"False" or "Yes"/"No" - no explanations or additional text
+    6. Respond with ONLY the letter (A, B, C, etc.) or "True"/"False" or "Yes"/"No" and so on - no explanations or additional text
 """
 )
 
@@ -141,7 +141,7 @@ MBPP_SYSTEM_PROMPT: Final[str] = textwrap.dedent(
 
 TW_LEGAL_SYSTEM_PROMPT: Final[str] = textwrap.dedent(
     """\
-    You are an expert lawyer with deep knowledge of Taiwan's legal system. You are taking the Taiwan Bar Examination (臺灣律師資格考試). For each question, you will analyze legal scenarios or concepts based on Taiwan's laws and regulations. Your task is to select the most appropriate answer that aligns with Taiwan's legal principles.
+    You are an expert lawyer with deep knowledge of Taiwan's legal system. For each question, you will analyze legal scenarios or concepts based on Taiwan's laws and regulations. Your task is to select the most appropriate answer that aligns with Taiwan's legal principles.
 
     Instructions:
     1. Carefully analyze the legal question and all options
@@ -151,3 +151,16 @@ TW_LEGAL_SYSTEM_PROMPT: Final[str] = textwrap.dedent(
     5. Respond with ONLY the letter (A, B, C, or D) - no explanations or additional text
 """
 )
+
+TMLU_SYSTEM_PROMPT: Final[str] = textwrap.dedent(
+    """\
+    You are an expert evaluator with deep knowledge of Taiwan's educational system and professional fields. For each question, analyze it carefully and select the most appropriate answer based on your understanding of the subject matter.
+
+    Instructions:
+    1. Carefully read and understand the question
+    2. Consider all answer options thoroughly
+    3. Apply subject-specific knowledge and reasoning
+    4. Select the single most accurate answer
+    5. Respond with ONLY the letter (A, B, C, or D) - no explanations or additional text
+"""
+)

llmdataparser/tmlu_parser.py

@@ -0,0 +1,202 @@
+from dataclasses import dataclass
+from typing import Any, Dict, Final, List
+
+from llmdataparser.base_parser import HuggingFaceDatasetParser, HuggingFaceParseEntry
+from llmdataparser.prompts import TMLU_SYSTEM_PROMPT
+
+TMLU_VALID_ANSWERS: Final[set[str]] = {"A", "B", "C", "D"}
+TMLU_VALID_ANSWER_STR: Final[str] = ", ".join(sorted(TMLU_VALID_ANSWERS))
+
+
+@dataclass(frozen=True, kw_only=True, slots=True)
+class TMLUParseEntry(HuggingFaceParseEntry):
+    """Custom entry class for TMLU, with fields specific to this dataset parser."""
+
+    raw_choices: list[str]
+    explanation: str
+    metadata: dict[str, Any]
+
+    @classmethod
+    def create(
+        cls,
+        prompt: str,
+        answer: str,
+        raw_question: str,
+        raw_choices: list[str],
+        raw_answer: str,
+        task_name: str,
+        explanation: str = "",
+        metadata: dict[str, Any] = {},
+    ) -> "TMLUParseEntry":
+        if answer not in TMLU_VALID_ANSWERS:
+            raise ValueError(
+                f"Invalid answer_letter '{answer}'; must be one of {TMLU_VALID_ANSWER_STR}"
+            )
+        return cls(
+            prompt=prompt,
+            answer=answer,
+            raw_question=raw_question,
+            raw_answer=raw_answer,
+            raw_choices=raw_choices,
+            task_name=task_name,
+            explanation=explanation,
+            metadata=metadata,
+        )
+
+
+class TMLUDatasetParser(HuggingFaceDatasetParser[TMLUParseEntry]):
+    """Parser for the TMLU dataset."""
+
+    _data_source = "miulab/tmlu"
+    _default_task = "AST_chinese"
+    _task_names = [
+        "AST_chinese",
+        "AST_mathematics",
+        "AST_biology",
+        "AST_chemistry",
+        "AST_physics",
+        "AST_civics",
+        "AST_geography",
+        "AST_history",
+        "GSAT_chinese",
+        "GSAT_chemistry",
+        "GSAT_biology",
+        "GSAT_physics",
+        "GSAT_earth_science",
+        "GSAT_mathematics",
+        "GSAT_geography",
+        "GSAT_history",
+        "GSAT_civics",
+        "CAP_mathematics",
+        "CAP_biology",
+        "CAP_physics",
+        "CAP_chemistry",
+        "CAP_earth_science",
+        "CAP_civics",
+        "CAP_history",
+        "CAP_geography",
+        "CAP_chinese",
+        "driving_rule",
+        "basic_traditional_chinese_medicine",
+        "clinical_traditional_chinese_medicine",
+        "lawyer_qualification",
+        "nutritionist",
+        "tour_leader",
+        "tour_guide",
+        "taiwan_tourist_resources",
+        "clinical_psychologist",
+        "teacher_qualification",
+        "accountant",
+    ]
+    _default_system_prompt = TMLU_SYSTEM_PROMPT
+
+    def process_entry(
+        self, row: dict[str, Any], task_name: str | None = None, **kwargs: Any
+    ) -> TMLUParseEntry:
+        """Process a single TMLU entry."""
+        task = task_name or self._get_current_task(row)
+        # Extract choices in order
+        raw_choices = [row["A"], row["B"], row["C"], row["D"]]
+        choices = "\n".join(
+            f"{chr(65 + i)}. {choice}" for i, choice in enumerate(raw_choices)
+        )
+        raw_question = row["question"]
+        raw_answer = row["answer"]
+        explanation = row.get("explanation", "")
+        metadata = row.get("metadata", {})
+
+        prompt = f"{self._system_prompt}\nQuestion: {raw_question}\n{choices}\nAnswer:"
+
+        return TMLUParseEntry.create(
+            prompt=prompt,
+            answer=raw_answer,
+            raw_question=raw_question,
+            raw_choices=raw_choices,
+            raw_answer=raw_answer,
+            task_name=task,
+            explanation=explanation,
+            metadata=metadata,
+        )
+
+    def get_dataset_description(self) -> Dict[str, str]:
+        """Returns description of the TMLU dataset."""
+        return {
+            "name": "Taiwan Multiple-choice Language Understanding (TMLU)",
+            "version": "1.0",
+            "language": "Traditional Chinese",
+            "purpose": "Evaluate models on Taiwan-specific educational and professional knowledge",
+            "source": "Various Taiwan standardized tests and professional certifications",
+            "format": "Multiple choice questions (A/B/C/D)",
+            "size": "Multiple subjects across different test types",
+            "domain": "Education and Professional Certification",
+            "characteristics": (
+                "Covers various subjects including Advanced Subjects Test (AST), "
+                "General Scholastic Ability Test (GSAT), College Admission Practice (CAP), "
+                "and professional certifications"
+            ),
+            "reference": "https://huggingface.co/datasets/miulab/tmlu",
+        }
+
+    def get_evaluation_metrics(self) -> List[Dict[str, Any]]:
+        """Returns recommended evaluation metrics for TMLU."""
+        return [
+            {
+                "name": "accuracy",
+                "type": "classification",
+                "description": "Overall percentage of correctly answered questions",
+                "implementation": "datasets.load_metric('accuracy')",
+                "primary": True,
+            },
+            {
+                "name": "per_subject_accuracy",
+                "type": "classification",
+                "description": "Accuracy broken down by subject areas (AST, GSAT, CAP, etc.)",
+                "implementation": "custom_subject_accuracy",
+                "primary": True,
+            },
+            {
+                "name": "per_difficulty_accuracy",
+                "type": "classification",
+                "description": "Accuracy broken down by test difficulty levels",
+                "implementation": "custom_difficulty_accuracy",
+                "primary": False,
+            },
+            {
+                "name": "confusion_matrix",
+                "type": "classification",
+                "description": "Distribution of predicted vs actual answers",
+                "implementation": "datasets.load_metric('confusion_matrix')",
+                "primary": False,
+            },
+            {
+                "name": "explanation_quality",
+                "type": "text",
+                "description": "Quality assessment of model explanations when available",
+                "implementation": "custom_explanation_metric",
+                "primary": False,
+            },
+        ]
+
+
+if __name__ == "__main__":
+    # Example usage
+    parser = TMLUDatasetParser()
+    parser.load()
+    parser.parse()
+
+    # Get parsed data with correct type
+    parsed_data = parser.get_parsed_data
+
+    # Print example entry
+    if parsed_data:
+        example = parsed_data[0]
+        print("\nExample parsed entry:")
+        print(f"Task: {example.task_name}")
+        print(f"Question: {example.raw_question}")
+        print("Choices:")
+        for i, choice in enumerate(example.raw_choices):
+            print(f"{chr(65 + i)}. {choice}")
+        print(f"Correct Answer: {example.answer}")
+        if example.explanation:
+            print(f"Explanation: {example.explanation}")
+        print(f"Metadata: {example.metadata}")

tests/test_tmlu_parser.py

@@ -0,0 +1,245 @@
+import pytest
+
+from llmdataparser.tmlu_parser import TMLUDatasetParser, TMLUParseEntry
+
+
+@pytest.fixture
+def tmlu_parser():
+    """Create a TMLU parser instance for testing."""
+    return TMLUDatasetParser()
+
+
+@pytest.fixture
+def sample_tmlu_entries():
+    """Create sample TMLU dataset entries for testing."""
+    return [
+        {
+            "question": "閱讀下文，選出依序最適合填入□內的選項：",
+            "A": "張揚／綢繆未雨／奏疏",
+            "B": "抽搐／煮繭抽絲／奏疏",
+            "C": "張揚／煮繭抽絲／進貢",
+            "D": "抽搐／綢繆未雨／進貢",
+            "answer": "B",
+            "explanation": "根據文意，選項B最為恰當。",
+            "metadata": {
+                "timestamp": "2023-10-09T18:27:20.304623",
+                "source": "AST chinese - 108",
+                "explanation_source": "",
+            },
+        },
+        {
+            "question": "下列何者是質數？",
+            "A": "21",
+            "B": "27",
+            "C": "31",
+            "D": "33",
+            "answer": "C",
+            "explanation": "31是質數，其他選項都是合數。",
+            "metadata": {
+                "timestamp": "2023-10-09T18:27:20.304623",
+                "source": "AST mathematics - 108",
+                "explanation_source": "",
+            },
+        },
+    ]
+
+
+def test_tmlu_parse_entry_creation_valid():
+    """Test valid creation of TMLUParseEntry."""
+    entry = TMLUParseEntry.create(
+        prompt="Test prompt",
+        answer="A",
+        raw_question="Test question",
+        raw_choices=["choice1", "choice2", "choice3", "choice4"],
+        raw_answer="A",
+        task_name="AST_chinese",
+        explanation="Test explanation",
+        metadata={"source": "test"},
+    )
+    assert isinstance(entry, TMLUParseEntry)
+    assert entry.prompt == "Test prompt"
+    assert entry.answer == "A"
+    assert entry.raw_choices == ["choice1", "choice2", "choice3", "choice4"]
+    assert entry.explanation == "Test explanation"
+    assert entry.metadata == {"source": "test"}
+
+
+@pytest.mark.parametrize("invalid_answer", ["E", "F", "1", "", None])
+def test_tmlu_parse_entry_creation_invalid(invalid_answer):
+    """Test invalid answer handling in TMLUParseEntry creation."""
+    with pytest.raises(
+        ValueError, match="Invalid answer_letter.*must be one of A, B, C, D"
+    ):
+        TMLUParseEntry.create(
+            prompt="Test prompt",
+            answer=invalid_answer,
+            raw_question="Test question",
+            raw_choices=["choice1", "choice2", "choice3", "choice4"],
+            raw_answer=invalid_answer,
+            task_name="AST_chinese",
+        )
+
+
+def test_process_entry(tmlu_parser, sample_tmlu_entries):
+    """Test processing entries in TMLU parser."""
+    entry = tmlu_parser.process_entry(sample_tmlu_entries[0], task_name="AST_chinese")
+
+    assert isinstance(entry, TMLUParseEntry)
+    assert entry.answer == "B"
+    assert entry.task_name == "AST_chinese"
+    assert len(entry.raw_choices) == 4
+    assert entry.explanation == "根據文意，選項B最為恰當。"
+    assert "AST chinese - 108" in entry.metadata["source"]
+
+
+def test_tmlu_parser_initialization(tmlu_parser):
+    """Test TMLU parser initialization and properties."""
+    assert isinstance(tmlu_parser.task_names, list)
+    assert len(tmlu_parser.task_names) == 37  # Total number of tasks
+    assert tmlu_parser._data_source == "miulab/tmlu"
+    assert tmlu_parser._default_task == "AST_chinese"
+    assert "AST_chinese" in tmlu_parser.task_names
+    assert "GSAT_mathematics" in tmlu_parser.task_names
+    assert (
+        tmlu_parser.get_huggingface_link
+        == "https://huggingface.co/datasets/miulab/tmlu"
+    )
+
+
+@pytest.mark.integration
+def test_load_dataset(tmlu_parser):
+    """Test loading the TMLU dataset."""
+    tmlu_parser.load(task_name="AST_chinese", split="test")
+    assert tmlu_parser.raw_data is not None
+    assert tmlu_parser.split_names == ["test"]
+    assert tmlu_parser._current_task == "AST_chinese"
+
+
+def test_parser_string_representation(tmlu_parser):
+    """Test string representation of TMLU parser."""
+    repr_str = str(tmlu_parser)
+    assert "TMLUDatasetParser" in repr_str
+    assert "miulab/tmlu" in repr_str
+    assert "not loaded" in repr_str
+
+
+@pytest.mark.integration
+def test_different_tasks_parsing(tmlu_parser):
+    """Test parsing different tasks of the dataset."""
+    # Load and parse AST_chinese
+    tmlu_parser.load(task_name="AST_chinese", split="test")
+    tmlu_parser.parse(split_names="test", force=True)
+    chinese_count = len(tmlu_parser.get_parsed_data)
+
+    # Load and parse AST_mathematics
+    tmlu_parser.load(task_name="AST_mathematics", split="test")
+    tmlu_parser.parse(split_names="test", force=True)
+    math_count = len(tmlu_parser.get_parsed_data)
+
+    assert chinese_count > 0
+    assert math_count > 0
+
+
+def test_system_prompt_override(tmlu_parser):
+    """Test overriding the default system prompt."""
+    custom_prompt = "Custom system prompt for testing"
+    parser = TMLUDatasetParser(system_prompt=custom_prompt)
+
+    test_entry = {
+        "question": "Test question",
+        "A": "Choice A",
+        "B": "Choice B",
+        "C": "Choice C",
+        "D": "Choice D",
+        "answer": "A",
+        "explanation": "Test explanation",
+        "metadata": {"source": "test"},
+    }
+
+    entry = parser.process_entry(test_entry)
+    assert custom_prompt in entry.prompt
+
+
+def test_metadata_handling(tmlu_parser, sample_tmlu_entries):
+    """Test proper handling of metadata in entries."""
+    entry = tmlu_parser.process_entry(sample_tmlu_entries[0])
+
+    assert "timestamp" in entry.metadata
+    assert "source" in entry.metadata
+    assert "explanation_source" in entry.metadata
+    assert entry.metadata["source"] == "AST chinese - 108"
+
+
+def test_dataset_description(tmlu_parser):
+    """Test dataset description contains all required fields."""
+    description = tmlu_parser.get_dataset_description()
+
+    required_fields = [
+        "name",
+        "version",
+        "language",
+        "purpose",
+        "source",
+        "format",
+        "size",
+        "domain",
+        "characteristics",
+        "reference",
+    ]
+
+    for field in required_fields:
+        assert field in description, f"Missing required field: {field}"
+
+    assert description["language"] == "Traditional Chinese"
+    assert "TMLU" in description["name"]
+    assert "miulab/tmlu" in description["reference"]
+    assert "AST" in description["characteristics"]
+    assert "GSAT" in description["characteristics"]
+
+
+def test_evaluation_metrics(tmlu_parser):
+    """Test evaluation metrics structure and content."""
+    metrics = tmlu_parser.get_evaluation_metrics()
+
+    # Check if we have metrics defined
+    assert len(metrics) > 0
+
+    # Check structure of each metric
+    required_metric_fields = [
+        "name",
+        "type",
+        "description",
+        "implementation",
+        "primary",
+    ]
+
+    for metric in metrics:
+        for field in required_metric_fields:
+            assert field in metric, f"Missing required field in metric: {field}"
+
+        # Type checks
+        assert isinstance(metric["name"], str)
+        assert isinstance(metric["type"], str)
+        assert isinstance(metric["description"], str)
+        assert isinstance(metric["implementation"], str)
+        assert isinstance(metric["primary"], bool)
+
+    # Check for TMLU-specific metrics
+    metric_names = {m["name"] for m in metrics}
+    expected_metrics = {
+        "accuracy",
+        "per_subject_accuracy",
+        "per_difficulty_accuracy",
+        "explanation_quality",
+    }
+
+    for expected in expected_metrics:
+        assert expected in metric_names, f"Missing expected metric: {expected}"
+
+    # Verify primary metrics
+    primary_metrics = [m for m in metrics if m["primary"]]
+    assert (
+        len(primary_metrics) >= 2
+    )  # Should have at least accuracy and per_subject_accuracy
+    assert any(m["name"] == "accuracy" for m in primary_metrics)
+    assert any(m["name"] == "per_subject_accuracy" for m in primary_metrics)