File size: 1,279 Bytes
78a5823 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 |
import json
import os
from typing import Dict, List, Union
import sys
project_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
sys.path.append(project_dir)
from utils import structure_data
def load_dataset(dataset_name: str) -> Dict[str, Union[str, List[str]]]:
"""
Load training dataset or validation dataset.
Args:
dataset_name (str): The name of the dataset. Should be either 'training_dataset' or 'validation_dataset'.
Returns:
dataset (Dict[str, Union[str. List[str]]]): A dictionary representing the
loaded dataset with keys 'text', 'ner', and 'intent'.
Raises:
ValueError: If the provided dataset_name is not one of the valid_names.
FileNotFoundError: If the dataset file is not found in the specified path.
"""
valid_names = ["training_dataset", "validation_dataset"]
if dataset_name not in valid_names:
raise ValueError(f"Invalid dataset name. Expected one of {valid_names}, got {dataset_name}")
path = f"{dataset_name}.json"
if not os.path.exists(path):
raise FileNotFoundError(f"Dataset file not found at {path}")
with open(path, 'r') as f:
dataset = json.load(f)
dataset = structure_data(dataset)
return dataset |