Spaces:
Running
Running
Fix bug
Browse files- dataset.py +4 -5
dataset.py
CHANGED
@@ -56,11 +56,10 @@ class ParallelTextReader(IterableDataset):
|
|
56 |
return pred, [gold]
|
57 |
|
58 |
def __iter__(self):
|
59 |
-
|
60 |
-
|
61 |
-
|
62 |
-
|
63 |
-
return mapped_itr
|
64 |
|
65 |
def __len__(self):
|
66 |
return self.num_sentences
|
|
|
56 |
return pred, [gold]
|
57 |
|
58 |
def __iter__(self):
|
59 |
+
pred_itr = open(self.pred_path, "r", encoding="utf8")
|
60 |
+
gold_itr = open(self.gold_path, "r", encoding="utf8")
|
61 |
+
mapped_itr = map(self.preprocess, pred_itr, gold_itr)
|
62 |
+
return mapped_itr
|
|
|
63 |
|
64 |
def __len__(self):
|
65 |
return self.num_sentences
|