Iker commited on
Commit
f016c88
·
1 Parent(s): 3d26fd2

Fix output if translator predicts new line in Windows format

Browse files
Files changed (1) hide show
  1. translate.py +5 -4
translate.py CHANGED
@@ -20,6 +20,10 @@ from dataset import DatasetReader, count_lines
20
  from accelerate import Accelerator, DistributedType, find_executable_batch_size
21
 
22
 
 
 
 
 
23
  def get_dataloader(
24
  accelerator: Accelerator,
25
  filename: str,
@@ -211,10 +215,7 @@ def main(
211
 
212
  print(
213
  "\n".join(
214
- [
215
- sentence.replace("\n", r"\n").replace("\t", r"\t")
216
- for sentence in tgt_text
217
- ]
218
  ),
219
  file=output_file,
220
  )
 
20
  from accelerate import Accelerator, DistributedType, find_executable_batch_size
21
 
22
 
23
+ def encode_string(text):
24
+ return text.replace("\r", r"\r").replace("\n", r"\n").replace("\t", r"\t")
25
+
26
+
27
  def get_dataloader(
28
  accelerator: Accelerator,
29
  filename: str,
 
215
 
216
  print(
217
  "\n".join(
218
+ [encode_string(sentence) for sentence in tgt_text]
 
 
 
219
  ),
220
  file=output_file,
221
  )