LINYINGHAO commited on
Commit
013e703
·
verified ·
1 Parent(s): fee6421

Create app.py

Browse files
Files changed (1) hide show
  1. app.py +26 -0
app.py ADDED
@@ -0,0 +1,26 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # 读取文件,解析JSON
2
+ import json
3
+ import os
4
+ from datasets import Dataset
5
+
6
+ training_file_path = os.path.join(".", "training_data.json")
7
+ print("file path:", training_file_path)
8
+
9
+ training_data_json = {}
10
+ with open(training_file_path, "r", encoding='utf-8') as training_file:
11
+ training_data_json = json.load(training_file)
12
+
13
+ # 转载数据集,划分训练/评估/测试
14
+ from datasets import Dataset
15
+
16
+ training_dataset = Dataset.from_list(training_data_json)
17
+
18
+ from datasets import DatasetDict
19
+
20
+ train_test_split = training_dataset.train_test_split(test_size=0.2)
21
+ dataset = DatasetDict({
22
+ 'train': train_test_split['train'],
23
+ 'test': train_test_split['test']
24
+ })
25
+
26
+ print(dataset)