File size: 393 Bytes
0e0a1bb
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
import json
from pathlib import Path
from utils.data_to_jsonl import data_to_jsonl

def load_data(jsonl_file):
    data = []
    script_dir = Path(__file__).parent
    input_dir = script_dir.parent / "data/questions"
    data_to_jsonl(input_dir, jsonl_file)
    with open(jsonl_file, "r", encoding="utf-8") as f:
        for line in f:
            data.append(json.loads(line))
    return data