We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
the raw output data can not be trained
when I use preprocess.py to ouput raw data by use `--output-format raw`, the output data file can not be used to train.py which I also use `--raw-text`. Through looking the source code, I change the output data file mentioned above in this way: rename 'train.src' to 'train.src-tgt.src' and  'train.tgt' to 'train.src-tgt.tgt' (assume I use `--source-lang=src --target-lang=tgt` ) and this can run.
I think it's a bug and is easy to fix :)

--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `preprocess.py`
Content:
```
1 #!/usr/bin/env python3
2 # Copyright (c) 2017-present, Facebook, Inc.
3 # All rights reserved.
4 #
5 # This source code is licensed under the license found in the LICENSE file in
6 # the root directory of this source tree. An additional grant of patent rights
7 # can be found in the PATENTS file in the same directory.
8 #
9 
10 import argparse
11 from itertools import zip_longest
12 import os
13 import shutil
14 
15 from fairseq.data import indexed_dataset, dictionary
16 from fairseq.tokenizer import Tokenizer, tokenize_line
17 
18 
19 def get_parser():
20     parser = argparse.ArgumentParser(
21         description='Data pre-processing: Create dictionary and store data in binary format')
22     parser.add_argument('-s', '--source-lang', default=None, metavar='SRC', help='source language')
23     parser.add_argument('-t', '--target-lang', default=None, metavar='TARGET', help='target language')
24     parser.add_argument('--trainpref', metavar='FP', default=None, help='target language')
25     parser.add_argument('--validpref', metavar='FP', default=None, help='comma separated, valid language prefixes')
26     parser.add_argument('--testpref', metavar='FP', default=None, help='comma separated, test language prefixes')
27     parser.add_argument('--destdir', metavar='DIR', default='data-bin', help='destination dir')
28     parser.add_argument('--thresholdtgt', metavar='N', default=0, type=int,
29                         help='map words appearing less than threshold times to unknown')
30     parser.add_argument('--thresholdsrc', metavar='N', default=0, type=int,
31                         help='map words appearing less than threshold times to unknown')
32     parser.add_argument('--tgtdict', metavar='FP', help='reuse given target dictionary')
33     parser.add_argument('--srcdict', metavar='FP', help='reuse given source dictionary')
34     parser.add_argument('--nwordstgt', metavar='N', default=-1, type=int, help='number of target words to retain')
35     parser.add_argument('--nwordssrc', metavar='N', default=-1, type=int, help='number of source words to retain')
36     parser.add_argument('--alignfile', metavar='ALIGN', default=None, help='an alignment file (optional)')
37     parser.add_argument('--output-format', metavar='FORMAT', default='binary', choices=['binary', 'raw'],
38                         help='output format (optional)')
39     parser.add_argument('--joined-dictionary', action='store_true', help='Generate joined dictionary')
40     parser.add_argument('--only-source', action='store_true', help='Only process the source language')
41     parser.add_argument('--padding-factor', metavar='N', default=8, type=int,
42                         help='Pad dictionary size to be multiple of N')
43     return parser
44 
45 
46 def main(args):
47     print(args)
48     os.makedirs(args.destdir, exist_ok=True)
49     target = not args.only_source
50 
51     def build_dictionary(filenames):
52         d = dictionary.Dictionary()
53         for filename in filenames:
54             Tokenizer.add_file_to_dictionary(filename, d, tokenize_line)
55         return d
56 
57     def train_path(lang):
58         return '{}{}'.format(args.trainpref, ('.' + lang) if lang else '')
59 
60     def file_name(prefix, lang):
61         fname = prefix
62         if lang is not None:
63             fname += f'.{lang}'
64         return fname
65 
66     def dest_path(prefix, lang):
67         return os.path.join(args.destdir, file_name(prefix, lang))
68 
69     def dict_path(lang):
70         return dest_path('dict', lang) + '.txt'
71 
72     def dataset_dest_path(output_prefix, lang, extension):
73         base = f'{args.destdir}/{output_prefix}'
74         lang_part = f'.{args.source_lang}-{args.target_lang}.{lang}' if lang is not None else ''
75         return f'{base}{lang_part}.{extension}'
76 
77     if args.joined_dictionary:
78         assert not args.srcdict, 'cannot combine --srcdict and --joined-dictionary'
79         assert not args.tgtdict, 'cannot combine --tgtdict and --joined-dictionary'
80         src_dict = build_dictionary(set([
81             train_path(lang)
82             for lang in [args.source_lang, args.target_lang]
83         ]))
84         tgt_dict = src_dict
85     else:
86         if args.srcdict:
87             src_dict = dictionary.Dictionary.load(args.srcdict)
88         else:
89             assert args.trainpref, "--trainpref must be set if --srcdict is not specified"
90             src_dict = build_dictionary([train_path(args.source_lang)])
91         if target:
92             if args.tgtdict:
93                 tgt_dict = dictionary.Dictionary.load(args.tgtdict)
94             else:
95                 assert args.trainpref, "--trainpref must be set if --tgtdict is not specified"
96                 tgt_dict = build_dictionary([train_path(args.target_lang)])
97 
98     src_dict.finalize(
99         threshold=args.thresholdsrc,
100         nwords=args.nwordssrc,
101         padding_factor=args.padding_factor,
102     )
103     src_dict.save(dict_path(args.source_lang))
104     if target:
105         if not args.joined_dictionary:
106             tgt_dict.finalize(
107                 threshold=args.thresholdtgt,
108                 nwords=args.nwordstgt,
109                 padding_factor=args.padding_factor,
110             )
111         tgt_dict.save(dict_path(args.target_lang))
112 
113     def make_binary_dataset(input_prefix, output_prefix, lang):
114         dict = dictionary.Dictionary.load(dict_path(lang))
115         print('| [{}] Dictionary: {} types'.format(lang, len(dict) - 1))
116 
117         ds = indexed_dataset.IndexedDatasetBuilder(dataset_dest_path(output_prefix, lang, 'bin'))
118 
119         def consumer(tensor):
120             ds.add_item(tensor)
121 
122         input_file = '{}{}'.format(input_prefix, ('.' + lang) if lang is not None else '')
123         res = Tokenizer.binarize(input_file, dict, consumer)
124         print('| [{}] {}: {} sents, {} tokens, {:.3}% replaced by {}'.format(
125             lang, input_file, res['nseq'], res['ntok'],
126             100 * res['nunk'] / res['ntok'], dict.unk_word))
127         ds.finalize(dataset_dest_path(output_prefix, lang, 'idx'))
128 
129     def make_dataset(input_prefix, output_prefix, lang, output_format='binary'):
130         if output_format == 'binary':
131             make_binary_dataset(input_prefix, output_prefix, lang)
132         elif output_format == 'raw':
133             # Copy original text file to destination folder
134             output_text_file = dest_path(output_prefix, lang)
135             shutil.copyfile(file_name(input_prefix, lang), output_text_file)
136 
137     def make_all(args, make_dataset, lang):
138         if args.trainpref:
139             make_dataset(args.trainpref, 'train', lang, args.output_format)
140         if args.validpref:
141             for k, validpref in enumerate(args.validpref.split(',')):
142                 outprefix = 'valid{}'.format(k) if k > 0 else 'valid'
143                 make_dataset(validpref, outprefix, lang, args.output_format)
144         if args.testpref:
145             for k, testpref in enumerate(args.testpref.split(',')):
146                 outprefix = 'test{}'.format(k) if k > 0 else 'test'
147                 make_dataset(testpref, outprefix, lang, args.output_format)
148 
149     make_all(args, make_dataset, args.source_lang)
150     if target:
151         make_all(args, make_dataset, args.target_lang)
152 
153     print('| Wrote preprocessed data to {}'.format(args.destdir))
154 
155     if args.alignfile:
156         assert args.trainpref, "--trainpref must be set if --alignfile is specified"
157         src_file_name = train_path(args.source_lang)
158         tgt_file_name = train_path(args.target_lang)
159         src_dict = dictionary.Dictionary.load(dict_path(args.source_lang))
160         tgt_dict = dictionary.Dictionary.load(dict_path(args.target_lang))
161         freq_map = {}
162         with open(args.alignfile, 'r') as align_file:
163             with open(src_file_name, 'r') as src_file:
164                 with open(tgt_file_name, 'r') as tgt_file:
165                     for a, s, t in zip_longest(align_file, src_file, tgt_file):
166                         si = Tokenizer.tokenize(s, src_dict, add_if_not_exist=False)
167                         ti = Tokenizer.tokenize(t, tgt_dict, add_if_not_exist=False)
168                         ai = list(map(lambda x: tuple(x.split('-')), a.split()))
169                         for sai, tai in ai:
170                             srcidx = si[int(sai)]
171                             tgtidx = ti[int(tai)]
172                             if srcidx != src_dict.unk() and tgtidx != tgt_dict.unk():
173                                 assert srcidx != src_dict.pad()
174                                 assert srcidx != src_dict.eos()
175                                 assert tgtidx != tgt_dict.pad()
176                                 assert tgtidx != tgt_dict.eos()
177 
178                                 if srcidx not in freq_map:
179                                     freq_map[srcidx] = {}
180                                 if tgtidx not in freq_map[srcidx]:
181                                     freq_map[srcidx][tgtidx] = 1
182                                 else:
183                                     freq_map[srcidx][tgtidx] += 1
184 
185         align_dict = {}
186         for srcidx in freq_map.keys():
187             align_dict[srcidx] = max(freq_map[srcidx], key=freq_map[srcidx].get)
188 
189         with open(os.path.join(args.destdir, 'alignment.{}-{}.txt'.format(
190                 args.source_lang, args.target_lang)), 'w') as f:
191             for k, v in align_dict.items():
192                 print('{} {}'.format(src_dict[k], tgt_dict[v]), file=f)
193 
194 
195 if __name__ == '__main__':
196     parser = get_parser()
197     args = parser.parse_args()
198     main(args)
199
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/preprocess.py b/preprocess.py
--- a/preprocess.py
+++ b/preprocess.py
@@ -126,29 +126,32 @@
             100 * res['nunk'] / res['ntok'], dict.unk_word))
         ds.finalize(dataset_dest_path(output_prefix, lang, 'idx'))
 
-    def make_dataset(input_prefix, output_prefix, lang, output_format='binary'):
-        if output_format == 'binary':
+    def make_dataset(input_prefix, output_prefix, lang):
+        if args.output_format == 'binary':
             make_binary_dataset(input_prefix, output_prefix, lang)
-        elif output_format == 'raw':
+        elif args.output_format == 'raw':
             # Copy original text file to destination folder
-            output_text_file = dest_path(output_prefix, lang)
+            output_text_file = dest_path(
+                output_prefix + '.{}-{}'.format(args.source_lang, args.target_lang),
+                lang,
+            )
             shutil.copyfile(file_name(input_prefix, lang), output_text_file)
 
-    def make_all(args, make_dataset, lang):
+    def make_all(lang):
         if args.trainpref:
-            make_dataset(args.trainpref, 'train', lang, args.output_format)
+            make_dataset(args.trainpref, 'train', lang)
         if args.validpref:
             for k, validpref in enumerate(args.validpref.split(',')):
                 outprefix = 'valid{}'.format(k) if k > 0 else 'valid'
-                make_dataset(validpref, outprefix, lang, args.output_format)
+                make_dataset(validpref, outprefix, lang)
         if args.testpref:
             for k, testpref in enumerate(args.testpref.split(',')):
                 outprefix = 'test{}'.format(k) if k > 0 else 'test'
-                make_dataset(testpref, outprefix, lang, args.output_format)
+                make_dataset(testpref, outprefix, lang)
 
-    make_all(args, make_dataset, args.source_lang)
+    make_all(args.source_lang)
     if target:
-        make_all(args, make_dataset, args.target_lang)
+        make_all(args.target_lang)
 
     print('| Wrote preprocessed data to {}'.format(args.destdir))

{"golden_diff": "diff --git a/preprocess.py b/preprocess.py\n--- a/preprocess.py\n+++ b/preprocess.py\n@@ -126,29 +126,32 @@\n             100 * res['nunk'] / res['ntok'], dict.unk_word))\n         ds.finalize(dataset_dest_path(output_prefix, lang, 'idx'))\n \n-    def make_dataset(input_prefix, output_prefix, lang, output_format='binary'):\n-        if output_format == 'binary':\n+    def make_dataset(input_prefix, output_prefix, lang):\n+        if args.output_format == 'binary':\n             make_binary_dataset(input_prefix, output_prefix, lang)\n-        elif output_format == 'raw':\n+        elif args.output_format == 'raw':\n             # Copy original text file to destination folder\n-            output_text_file = dest_path(output_prefix, lang)\n+            output_text_file = dest_path(\n+                output_prefix + '.{}-{}'.format(args.source_lang, args.target_lang),\n+                lang,\n+            )\n             shutil.copyfile(file_name(input_prefix, lang), output_text_file)\n \n-    def make_all(args, make_dataset, lang):\n+    def make_all(lang):\n         if args.trainpref:\n-            make_dataset(args.trainpref, 'train', lang, args.output_format)\n+            make_dataset(args.trainpref, 'train', lang)\n         if args.validpref:\n             for k, validpref in enumerate(args.validpref.split(',')):\n                 outprefix = 'valid{}'.format(k) if k > 0 else 'valid'\n-                make_dataset(validpref, outprefix, lang, args.output_format)\n+                make_dataset(validpref, outprefix, lang)\n         if args.testpref:\n             for k, testpref in enumerate(args.testpref.split(',')):\n                 outprefix = 'test{}'.format(k) if k > 0 else 'test'\n-                make_dataset(testpref, outprefix, lang, args.output_format)\n+                make_dataset(testpref, outprefix, lang)\n \n-    make_all(args, make_dataset, args.source_lang)\n+    make_all(args.source_lang)\n     if target:\n-        make_all(args, make_dataset, args.target_lang)\n+        make_all(args.target_lang)\n \n     print('| Wrote preprocessed data to {}'.format(args.destdir))\n", "issue": "the raw output data can not be trained\nwhen I use preprocess.py to ouput raw data by use `--output-format raw`, the output data file can not be used to train.py which I also use `--raw-text`. Through looking the source code, I change the output data file mentioned above in this way: rename 'train.src' to 'train.src-tgt.src' and  'train.tgt' to 'train.src-tgt.tgt' (assume I use `--source-lang=src --target-lang=tgt` ) and this can run.\r\nI think it's a bug and is easy to fix :)\n", "before_files": [{"content": "#!/usr/bin/env python3\n# Copyright (c) 2017-present, Facebook, Inc.\n# All rights reserved.\n#\n# This source code is licensed under the license found in the LICENSE file in\n# the root directory of this source tree. An additional grant of patent rights\n# can be found in the PATENTS file in the same directory.\n#\n\nimport argparse\nfrom itertools import zip_longest\nimport os\nimport shutil\n\nfrom fairseq.data import indexed_dataset, dictionary\nfrom fairseq.tokenizer import Tokenizer, tokenize_line\n\n\ndef get_parser():\n    parser = argparse.ArgumentParser(\n        description='Data pre-processing: Create dictionary and store data in binary format')\n    parser.add_argument('-s', '--source-lang', default=None, metavar='SRC', help='source language')\n    parser.add_argument('-t', '--target-lang', default=None, metavar='TARGET', help='target language')\n    parser.add_argument('--trainpref', metavar='FP', default=None, help='target language')\n    parser.add_argument('--validpref', metavar='FP', default=None, help='comma separated, valid language prefixes')\n    parser.add_argument('--testpref', metavar='FP', default=None, help='comma separated, test language prefixes')\n    parser.add_argument('--destdir', metavar='DIR', default='data-bin', help='destination dir')\n    parser.add_argument('--thresholdtgt', metavar='N', default=0, type=int,\n                        help='map words appearing less than threshold times to unknown')\n    parser.add_argument('--thresholdsrc', metavar='N', default=0, type=int,\n                        help='map words appearing less than threshold times to unknown')\n    parser.add_argument('--tgtdict', metavar='FP', help='reuse given target dictionary')\n    parser.add_argument('--srcdict', metavar='FP', help='reuse given source dictionary')\n    parser.add_argument('--nwordstgt', metavar='N', default=-1, type=int, help='number of target words to retain')\n    parser.add_argument('--nwordssrc', metavar='N', default=-1, type=int, help='number of source words to retain')\n    parser.add_argument('--alignfile', metavar='ALIGN', default=None, help='an alignment file (optional)')\n    parser.add_argument('--output-format', metavar='FORMAT', default='binary', choices=['binary', 'raw'],\n                        help='output format (optional)')\n    parser.add_argument('--joined-dictionary', action='store_true', help='Generate joined dictionary')\n    parser.add_argument('--only-source', action='store_true', help='Only process the source language')\n    parser.add_argument('--padding-factor', metavar='N', default=8, type=int,\n                        help='Pad dictionary size to be multiple of N')\n    return parser\n\n\ndef main(args):\n    print(args)\n    os.makedirs(args.destdir, exist_ok=True)\n    target = not args.only_source\n\n    def build_dictionary(filenames):\n        d = dictionary.Dictionary()\n        for filename in filenames:\n            Tokenizer.add_file_to_dictionary(filename, d, tokenize_line)\n        return d\n\n    def train_path(lang):\n        return '{}{}'.format(args.trainpref, ('.' + lang) if lang else '')\n\n    def file_name(prefix, lang):\n        fname = prefix\n        if lang is not None:\n            fname += f'.{lang}'\n        return fname\n\n    def dest_path(prefix, lang):\n        return os.path.join(args.destdir, file_name(prefix, lang))\n\n    def dict_path(lang):\n        return dest_path('dict', lang) + '.txt'\n\n    def dataset_dest_path(output_prefix, lang, extension):\n        base = f'{args.destdir}/{output_prefix}'\n        lang_part = f'.{args.source_lang}-{args.target_lang}.{lang}' if lang is not None else ''\n        return f'{base}{lang_part}.{extension}'\n\n    if args.joined_dictionary:\n        assert not args.srcdict, 'cannot combine --srcdict and --joined-dictionary'\n        assert not args.tgtdict, 'cannot combine --tgtdict and --joined-dictionary'\n        src_dict = build_dictionary(set([\n            train_path(lang)\n            for lang in [args.source_lang, args.target_lang]\n        ]))\n        tgt_dict = src_dict\n    else:\n        if args.srcdict:\n            src_dict = dictionary.Dictionary.load(args.srcdict)\n        else:\n            assert args.trainpref, \"--trainpref must be set if --srcdict is not specified\"\n            src_dict = build_dictionary([train_path(args.source_lang)])\n        if target:\n            if args.tgtdict:\n                tgt_dict = dictionary.Dictionary.load(args.tgtdict)\n            else:\n                assert args.trainpref, \"--trainpref must be set if --tgtdict is not specified\"\n                tgt_dict = build_dictionary([train_path(args.target_lang)])\n\n    src_dict.finalize(\n        threshold=args.thresholdsrc,\n        nwords=args.nwordssrc,\n        padding_factor=args.padding_factor,\n    )\n    src_dict.save(dict_path(args.source_lang))\n    if target:\n        if not args.joined_dictionary:\n            tgt_dict.finalize(\n                threshold=args.thresholdtgt,\n                nwords=args.nwordstgt,\n                padding_factor=args.padding_factor,\n            )\n        tgt_dict.save(dict_path(args.target_lang))\n\n    def make_binary_dataset(input_prefix, output_prefix, lang):\n        dict = dictionary.Dictionary.load(dict_path(lang))\n        print('| [{}] Dictionary: {} types'.format(lang, len(dict) - 1))\n\n        ds = indexed_dataset.IndexedDatasetBuilder(dataset_dest_path(output_prefix, lang, 'bin'))\n\n        def consumer(tensor):\n            ds.add_item(tensor)\n\n        input_file = '{}{}'.format(input_prefix, ('.' + lang) if lang is not None else '')\n        res = Tokenizer.binarize(input_file, dict, consumer)\n        print('| [{}] {}: {} sents, {} tokens, {:.3}% replaced by {}'.format(\n            lang, input_file, res['nseq'], res['ntok'],\n            100 * res['nunk'] / res['ntok'], dict.unk_word))\n        ds.finalize(dataset_dest_path(output_prefix, lang, 'idx'))\n\n    def make_dataset(input_prefix, output_prefix, lang, output_format='binary'):\n        if output_format == 'binary':\n            make_binary_dataset(input_prefix, output_prefix, lang)\n        elif output_format == 'raw':\n            # Copy original text file to destination folder\n            output_text_file = dest_path(output_prefix, lang)\n            shutil.copyfile(file_name(input_prefix, lang), output_text_file)\n\n    def make_all(args, make_dataset, lang):\n        if args.trainpref:\n            make_dataset(args.trainpref, 'train', lang, args.output_format)\n        if args.validpref:\n            for k, validpref in enumerate(args.validpref.split(',')):\n                outprefix = 'valid{}'.format(k) if k > 0 else 'valid'\n                make_dataset(validpref, outprefix, lang, args.output_format)\n        if args.testpref:\n            for k, testpref in enumerate(args.testpref.split(',')):\n                outprefix = 'test{}'.format(k) if k > 0 else 'test'\n                make_dataset(testpref, outprefix, lang, args.output_format)\n\n    make_all(args, make_dataset, args.source_lang)\n    if target:\n        make_all(args, make_dataset, args.target_lang)\n\n    print('| Wrote preprocessed data to {}'.format(args.destdir))\n\n    if args.alignfile:\n        assert args.trainpref, \"--trainpref must be set if --alignfile is specified\"\n        src_file_name = train_path(args.source_lang)\n        tgt_file_name = train_path(args.target_lang)\n        src_dict = dictionary.Dictionary.load(dict_path(args.source_lang))\n        tgt_dict = dictionary.Dictionary.load(dict_path(args.target_lang))\n        freq_map = {}\n        with open(args.alignfile, 'r') as align_file:\n            with open(src_file_name, 'r') as src_file:\n                with open(tgt_file_name, 'r') as tgt_file:\n                    for a, s, t in zip_longest(align_file, src_file, tgt_file):\n                        si = Tokenizer.tokenize(s, src_dict, add_if_not_exist=False)\n                        ti = Tokenizer.tokenize(t, tgt_dict, add_if_not_exist=False)\n                        ai = list(map(lambda x: tuple(x.split('-')), a.split()))\n                        for sai, tai in ai:\n                            srcidx = si[int(sai)]\n                            tgtidx = ti[int(tai)]\n                            if srcidx != src_dict.unk() and tgtidx != tgt_dict.unk():\n                                assert srcidx != src_dict.pad()\n                                assert srcidx != src_dict.eos()\n                                assert tgtidx != tgt_dict.pad()\n                                assert tgtidx != tgt_dict.eos()\n\n                                if srcidx not in freq_map:\n                                    freq_map[srcidx] = {}\n                                if tgtidx not in freq_map[srcidx]:\n                                    freq_map[srcidx][tgtidx] = 1\n                                else:\n                                    freq_map[srcidx][tgtidx] += 1\n\n        align_dict = {}\n        for srcidx in freq_map.keys():\n            align_dict[srcidx] = max(freq_map[srcidx], key=freq_map[srcidx].get)\n\n        with open(os.path.join(args.destdir, 'alignment.{}-{}.txt'.format(\n                args.source_lang, args.target_lang)), 'w') as f:\n            for k, v in align_dict.items():\n                print('{} {}'.format(src_dict[k], tgt_dict[v]), file=f)\n\n\nif __name__ == '__main__':\n    parser = get_parser()\n    args = parser.parse_args()\n    main(args)\n", "path": "preprocess.py"}], "after_files": [{"content": "#!/usr/bin/env python3\n# Copyright (c) 2017-present, Facebook, Inc.\n# All rights reserved.\n#\n# This source code is licensed under the license found in the LICENSE file in\n# the root directory of this source tree. An additional grant of patent rights\n# can be found in the PATENTS file in the same directory.\n#\n\nimport argparse\nfrom itertools import zip_longest\nimport os\nimport shutil\n\nfrom fairseq.data import indexed_dataset, dictionary\nfrom fairseq.tokenizer import Tokenizer, tokenize_line\n\n\ndef get_parser():\n    parser = argparse.ArgumentParser(\n        description='Data pre-processing: Create dictionary and store data in binary format')\n    parser.add_argument('-s', '--source-lang', default=None, metavar='SRC', help='source language')\n    parser.add_argument('-t', '--target-lang', default=None, metavar='TARGET', help='target language')\n    parser.add_argument('--trainpref', metavar='FP', default=None, help='target language')\n    parser.add_argument('--validpref', metavar='FP', default=None, help='comma separated, valid language prefixes')\n    parser.add_argument('--testpref', metavar='FP', default=None, help='comma separated, test language prefixes')\n    parser.add_argument('--destdir', metavar='DIR', default='data-bin', help='destination dir')\n    parser.add_argument('--thresholdtgt', metavar='N', default=0, type=int,\n                        help='map words appearing less than threshold times to unknown')\n    parser.add_argument('--thresholdsrc', metavar='N', default=0, type=int,\n                        help='map words appearing less than threshold times to unknown')\n    parser.add_argument('--tgtdict', metavar='FP', help='reuse given target dictionary')\n    parser.add_argument('--srcdict', metavar='FP', help='reuse given source dictionary')\n    parser.add_argument('--nwordstgt', metavar='N', default=-1, type=int, help='number of target words to retain')\n    parser.add_argument('--nwordssrc', metavar='N', default=-1, type=int, help='number of source words to retain')\n    parser.add_argument('--alignfile', metavar='ALIGN', default=None, help='an alignment file (optional)')\n    parser.add_argument('--output-format', metavar='FORMAT', default='binary', choices=['binary', 'raw'],\n                        help='output format (optional)')\n    parser.add_argument('--joined-dictionary', action='store_true', help='Generate joined dictionary')\n    parser.add_argument('--only-source', action='store_true', help='Only process the source language')\n    parser.add_argument('--padding-factor', metavar='N', default=8, type=int,\n                        help='Pad dictionary size to be multiple of N')\n    return parser\n\n\ndef main(args):\n    print(args)\n    os.makedirs(args.destdir, exist_ok=True)\n    target = not args.only_source\n\n    def build_dictionary(filenames):\n        d = dictionary.Dictionary()\n        for filename in filenames:\n            Tokenizer.add_file_to_dictionary(filename, d, tokenize_line)\n        return d\n\n    def train_path(lang):\n        return '{}{}'.format(args.trainpref, ('.' + lang) if lang else '')\n\n    def file_name(prefix, lang):\n        fname = prefix\n        if lang is not None:\n            fname += f'.{lang}'\n        return fname\n\n    def dest_path(prefix, lang):\n        return os.path.join(args.destdir, file_name(prefix, lang))\n\n    def dict_path(lang):\n        return dest_path('dict', lang) + '.txt'\n\n    def dataset_dest_path(output_prefix, lang, extension):\n        base = f'{args.destdir}/{output_prefix}'\n        lang_part = f'.{args.source_lang}-{args.target_lang}.{lang}' if lang is not None else ''\n        return f'{base}{lang_part}.{extension}'\n\n    if args.joined_dictionary:\n        assert not args.srcdict, 'cannot combine --srcdict and --joined-dictionary'\n        assert not args.tgtdict, 'cannot combine --tgtdict and --joined-dictionary'\n        src_dict = build_dictionary(set([\n            train_path(lang)\n            for lang in [args.source_lang, args.target_lang]\n        ]))\n        tgt_dict = src_dict\n    else:\n        if args.srcdict:\n            src_dict = dictionary.Dictionary.load(args.srcdict)\n        else:\n            assert args.trainpref, \"--trainpref must be set if --srcdict is not specified\"\n            src_dict = build_dictionary([train_path(args.source_lang)])\n        if target:\n            if args.tgtdict:\n                tgt_dict = dictionary.Dictionary.load(args.tgtdict)\n            else:\n                assert args.trainpref, \"--trainpref must be set if --tgtdict is not specified\"\n                tgt_dict = build_dictionary([train_path(args.target_lang)])\n\n    src_dict.finalize(\n        threshold=args.thresholdsrc,\n        nwords=args.nwordssrc,\n        padding_factor=args.padding_factor,\n    )\n    src_dict.save(dict_path(args.source_lang))\n    if target:\n        if not args.joined_dictionary:\n            tgt_dict.finalize(\n                threshold=args.thresholdtgt,\n                nwords=args.nwordstgt,\n                padding_factor=args.padding_factor,\n            )\n        tgt_dict.save(dict_path(args.target_lang))\n\n    def make_binary_dataset(input_prefix, output_prefix, lang):\n        dict = dictionary.Dictionary.load(dict_path(lang))\n        print('| [{}] Dictionary: {} types'.format(lang, len(dict) - 1))\n\n        ds = indexed_dataset.IndexedDatasetBuilder(dataset_dest_path(output_prefix, lang, 'bin'))\n\n        def consumer(tensor):\n            ds.add_item(tensor)\n\n        input_file = '{}{}'.format(input_prefix, ('.' + lang) if lang is not None else '')\n        res = Tokenizer.binarize(input_file, dict, consumer)\n        print('| [{}] {}: {} sents, {} tokens, {:.3}% replaced by {}'.format(\n            lang, input_file, res['nseq'], res['ntok'],\n            100 * res['nunk'] / res['ntok'], dict.unk_word))\n        ds.finalize(dataset_dest_path(output_prefix, lang, 'idx'))\n\n    def make_dataset(input_prefix, output_prefix, lang):\n        if args.output_format == 'binary':\n            make_binary_dataset(input_prefix, output_prefix, lang)\n        elif args.output_format == 'raw':\n            # Copy original text file to destination folder\n            output_text_file = dest_path(\n                output_prefix + '.{}-{}'.format(args.source_lang, args.target_lang),\n                lang,\n            )\n            shutil.copyfile(file_name(input_prefix, lang), output_text_file)\n\n    def make_all(lang):\n        if args.trainpref:\n            make_dataset(args.trainpref, 'train', lang)\n        if args.validpref:\n            for k, validpref in enumerate(args.validpref.split(',')):\n                outprefix = 'valid{}'.format(k) if k > 0 else 'valid'\n                make_dataset(validpref, outprefix, lang)\n        if args.testpref:\n            for k, testpref in enumerate(args.testpref.split(',')):\n                outprefix = 'test{}'.format(k) if k > 0 else 'test'\n                make_dataset(testpref, outprefix, lang)\n\n    make_all(args.source_lang)\n    if target:\n        make_all(args.target_lang)\n\n    print('| Wrote preprocessed data to {}'.format(args.destdir))\n\n    if args.alignfile:\n        assert args.trainpref, \"--trainpref must be set if --alignfile is specified\"\n        src_file_name = train_path(args.source_lang)\n        tgt_file_name = train_path(args.target_lang)\n        src_dict = dictionary.Dictionary.load(dict_path(args.source_lang))\n        tgt_dict = dictionary.Dictionary.load(dict_path(args.target_lang))\n        freq_map = {}\n        with open(args.alignfile, 'r') as align_file:\n            with open(src_file_name, 'r') as src_file:\n                with open(tgt_file_name, 'r') as tgt_file:\n                    for a, s, t in zip_longest(align_file, src_file, tgt_file):\n                        si = Tokenizer.tokenize(s, src_dict, add_if_not_exist=False)\n                        ti = Tokenizer.tokenize(t, tgt_dict, add_if_not_exist=False)\n                        ai = list(map(lambda x: tuple(x.split('-')), a.split()))\n                        for sai, tai in ai:\n                            srcidx = si[int(sai)]\n                            tgtidx = ti[int(tai)]\n                            if srcidx != src_dict.unk() and tgtidx != tgt_dict.unk():\n                                assert srcidx != src_dict.pad()\n                                assert srcidx != src_dict.eos()\n                                assert tgtidx != tgt_dict.pad()\n                                assert tgtidx != tgt_dict.eos()\n\n                                if srcidx not in freq_map:\n                                    freq_map[srcidx] = {}\n                                if tgtidx not in freq_map[srcidx]:\n                                    freq_map[srcidx][tgtidx] = 1\n                                else:\n                                    freq_map[srcidx][tgtidx] += 1\n\n        align_dict = {}\n        for srcidx in freq_map.keys():\n            align_dict[srcidx] = max(freq_map[srcidx], key=freq_map[srcidx].get)\n\n        with open(os.path.join(args.destdir, 'alignment.{}-{}.txt'.format(\n                args.source_lang, args.target_lang)), 'w') as f:\n            for k, v in align_dict.items():\n                print('{} {}'.format(src_dict[k], tgt_dict[v]), file=f)\n\n\nif __name__ == '__main__':\n    parser = get_parser()\n    args = parser.parse_args()\n    main(args)\n", "path": "preprocess.py"}]}