We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
Transformer BLEU of the current code base is worse than previous one
I recently retrained the transformer, and found that it will diverge with the default setting. The divergence issue is caused by use_layer_norm_before_dropout=True and is fixed in #543. However, the final testing BLEU is still worse than the previous result. Compare to the version in #413, the testing BLEU is about 0.8 worse.

--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `scripts/machine_translation/dataprocessor.py`
Content:
```
1 # coding: utf-8
2 
3 # Licensed to the Apache Software Foundation (ASF) under one
4 # or more contributor license agreements.  See the NOTICE file
5 # distributed with this work for additional information
6 # regarding copyright ownership.  The ASF licenses this file
7 # to you under the Apache License, Version 2.0 (the
8 # "License"); you may not use this file except in compliance
9 # with the License.  You may obtain a copy of the License at
10 #
11 #   http://www.apache.org/licenses/LICENSE-2.0
12 #
13 # Unless required by applicable law or agreed to in writing,
14 # software distributed under the License is distributed on an
15 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
16 # KIND, either express or implied.  See the License for the
17 # specific language governing permissions and limitations
18 # under the License.
19 """Data preprocessing for transformer."""
20 
21 import os
22 import io
23 import time
24 import logging
25 import numpy as np
26 from mxnet import gluon
27 import gluonnlp as nlp
28 import gluonnlp.data.batchify as btf
29 import _constants
30 import dataset as _dataset
31 
32 
33 def _cache_dataset(dataset, prefix):
34     """Cache the processed npy dataset the dataset into a npz
35 
36     Parameters
37     ----------
38     dataset : SimpleDataset
39     file_path : str
40     """
41     if not os.path.exists(_constants.CACHE_PATH):
42         os.makedirs(_constants.CACHE_PATH)
43     src_data = np.concatenate([e[0] for e in dataset])
44     tgt_data = np.concatenate([e[1] for e in dataset])
45     src_cumlen = np.cumsum([0]+[len(e[0]) for e in dataset])
46     tgt_cumlen = np.cumsum([0]+[len(e[1]) for e in dataset])
47     np.savez(os.path.join(_constants.CACHE_PATH, prefix + '.npz'),
48              src_data=src_data, tgt_data=tgt_data,
49              src_cumlen=src_cumlen, tgt_cumlen=tgt_cumlen)
50 
51 
52 def _load_cached_dataset(prefix):
53     cached_file_path = os.path.join(_constants.CACHE_PATH, prefix + '.npz')
54     if os.path.exists(cached_file_path):
55         print('Loading dataset...')
56         npz_data = np.load(cached_file_path)
57         src_data, tgt_data, src_cumlen, tgt_cumlen = \
58                 [npz_data[n] for n in ['src_data', 'tgt_data', 'src_cumlen', 'tgt_cumlen']]
59         src_data = np.array([src_data[low:high] for low, high
60                              in zip(src_cumlen[:-1], src_cumlen[1:])])
61         tgt_data = np.array([tgt_data[low:high] for low, high
62                              in zip(tgt_cumlen[:-1], tgt_cumlen[1:])])
63         return gluon.data.ArrayDataset(np.array(src_data), np.array(tgt_data))
64     else:
65         return None
66 
67 
68 class TrainValDataTransform(object):
69     """Transform the machine translation dataset.
70 
71     Clip source and the target sentences to the maximum length. For the source sentence, append the
72     EOS. For the target sentence, append BOS and EOS.
73 
74     Parameters
75     ----------
76     src_vocab : Vocab
77     tgt_vocab : Vocab
78     src_max_len : int
79     tgt_max_len : int
80     """
81 
82     def __init__(self, src_vocab, tgt_vocab, src_max_len=None, tgt_max_len=None):
83         self._src_vocab = src_vocab
84         self._tgt_vocab = tgt_vocab
85         self._src_max_len = src_max_len
86         self._tgt_max_len = tgt_max_len
87 
88     def __call__(self, src, tgt):
89         if self._src_max_len:
90             src_sentence = self._src_vocab[src.split()[:self._src_max_len]]
91         else:
92             src_sentence = self._src_vocab[src.split()]
93         if self._tgt_max_len:
94             tgt_sentence = self._tgt_vocab[tgt.split()[:self._tgt_max_len]]
95         else:
96             tgt_sentence = self._tgt_vocab[tgt.split()]
97         src_sentence.append(self._src_vocab[self._src_vocab.eos_token])
98         tgt_sentence.insert(0, self._tgt_vocab[self._tgt_vocab.bos_token])
99         tgt_sentence.append(self._tgt_vocab[self._tgt_vocab.eos_token])
100         src_npy = np.array(src_sentence, dtype=np.int32)
101         tgt_npy = np.array(tgt_sentence, dtype=np.int32)
102         return src_npy, tgt_npy
103 
104 
105 def process_dataset(dataset, src_vocab, tgt_vocab, src_max_len=-1, tgt_max_len=-1):
106     start = time.time()
107     dataset_processed = dataset.transform(TrainValDataTransform(src_vocab, tgt_vocab,
108                                                                 src_max_len,
109                                                                 tgt_max_len), lazy=False)
110     end = time.time()
111     print('Processing Time spent: {}'.format(end - start))
112     return dataset_processed
113 
114 
115 def load_translation_data(dataset, bleu, args):
116     """Load translation dataset
117 
118     Parameters
119     ----------
120     dataset : str
121     args : argparse result
122 
123     Returns
124     -------
125 
126     """
127     src_lang, tgt_lang = args.src_lang, args.tgt_lang
128     if dataset == 'IWSLT2015':
129         common_prefix = 'IWSLT2015_{}_{}_{}_{}'.format(src_lang, tgt_lang,
130                                                        args.src_max_len, args.tgt_max_len)
131         data_train = nlp.data.IWSLT2015('train', src_lang=src_lang, tgt_lang=tgt_lang)
132         data_val = nlp.data.IWSLT2015('val', src_lang=src_lang, tgt_lang=tgt_lang)
133         data_test = nlp.data.IWSLT2015('test', src_lang=src_lang, tgt_lang=tgt_lang)
134     elif dataset == 'WMT2016BPE':
135         common_prefix = 'WMT2016BPE_{}_{}_{}_{}'.format(src_lang, tgt_lang,
136                                                         args.src_max_len, args.tgt_max_len)
137         data_train = nlp.data.WMT2016BPE('train', src_lang=src_lang, tgt_lang=tgt_lang)
138         data_val = nlp.data.WMT2016BPE('newstest2013', src_lang=src_lang, tgt_lang=tgt_lang)
139         data_test = nlp.data.WMT2016BPE('newstest2014', src_lang=src_lang, tgt_lang=tgt_lang)
140     elif dataset == 'WMT2014BPE':
141         common_prefix = 'WMT2014BPE_{}_{}_{}_{}'.format(src_lang, tgt_lang,
142                                                         args.src_max_len, args.tgt_max_len)
143         data_train = nlp.data.WMT2014BPE('train', src_lang=src_lang, tgt_lang=tgt_lang)
144         data_val = nlp.data.WMT2014BPE('newstest2013', src_lang=src_lang, tgt_lang=tgt_lang)
145         data_test = nlp.data.WMT2014BPE('newstest2014', src_lang=src_lang, tgt_lang=tgt_lang,
146                                         full=args.full)
147     elif dataset == 'TOY':
148         common_prefix = 'TOY_{}_{}_{}_{}'.format(src_lang, tgt_lang,
149                                                  args.src_max_len, args.tgt_max_len)
150         data_train = _dataset.TOY('train', src_lang=src_lang, tgt_lang=tgt_lang)
151         data_val = _dataset.TOY('val', src_lang=src_lang, tgt_lang=tgt_lang)
152         data_test = _dataset.TOY('test', src_lang=src_lang, tgt_lang=tgt_lang)
153     else:
154         raise NotImplementedError
155     src_vocab, tgt_vocab = data_train.src_vocab, data_train.tgt_vocab
156     data_train_processed = _load_cached_dataset(common_prefix + '_train')
157     if not data_train_processed:
158         data_train_processed = process_dataset(data_train, src_vocab, tgt_vocab,
159                                                args.src_max_len, args.tgt_max_len)
160         _cache_dataset(data_train_processed, common_prefix + '_train')
161     data_val_processed = _load_cached_dataset(common_prefix + '_val')
162     if not data_val_processed:
163         data_val_processed = process_dataset(data_val, src_vocab, tgt_vocab)
164         _cache_dataset(data_val_processed, common_prefix + '_val')
165     data_test_processed = _load_cached_dataset(common_prefix + '_' + str(False) + '_test')
166     if not data_test_processed:
167         data_test_processed = process_dataset(data_test, src_vocab, tgt_vocab)
168         _cache_dataset(data_test_processed, common_prefix + '_' + str(False) + '_test')
169     if bleu == 'tweaked':
170         fetch_tgt_sentence = lambda src, tgt: tgt.split()
171         val_tgt_sentences = list(data_val.transform(fetch_tgt_sentence))
172         test_tgt_sentences = list(data_test.transform(fetch_tgt_sentence))
173     elif bleu == '13a' or bleu == 'intl':
174         fetch_tgt_sentence = lambda src, tgt: tgt
175         if dataset == 'WMT2016BPE':
176             val_text = nlp.data.WMT2016('newstest2013', src_lang=src_lang, tgt_lang=tgt_lang)
177             test_text = nlp.data.WMT2016('newstest2014', src_lang=src_lang, tgt_lang=tgt_lang)
178         elif dataset == 'WMT2014BPE':
179             val_text = nlp.data.WMT2014('newstest2013', src_lang=src_lang, tgt_lang=tgt_lang)
180             test_text = nlp.data.WMT2014('newstest2014', src_lang=src_lang, tgt_lang=tgt_lang,
181                                          full=args.full)
182         elif dataset == 'IWSLT2015' or dataset == 'TOY':
183             val_text = data_val
184             test_text = data_test
185         else:
186             raise NotImplementedError
187         val_tgt_sentences = list(val_text.transform(fetch_tgt_sentence))
188         test_tgt_sentences = list(test_text.transform(fetch_tgt_sentence))
189     else:
190         raise NotImplementedError
191     return data_train_processed, data_val_processed, data_test_processed, \
192            val_tgt_sentences, test_tgt_sentences, src_vocab, tgt_vocab
193 
194 
195 def get_data_lengths(dataset):
196     get_lengths = lambda *args: (args[2], args[3])
197     return list(dataset.transform(get_lengths))
198 
199 
200 def make_dataloader(data_train, data_val, data_test, args,
201                     use_average_length=False, num_shards=0, num_workers=8):
202     """Create data loaders for training/validation/test."""
203     data_train_lengths = get_data_lengths(data_train)
204     data_val_lengths = get_data_lengths(data_val)
205     data_test_lengths = get_data_lengths(data_test)
206     train_batchify_fn = btf.Tuple(btf.Pad(), btf.Pad(),
207                                   btf.Stack(dtype='float32'), btf.Stack(dtype='float32'))
208     test_batchify_fn = btf.Tuple(btf.Pad(), btf.Pad(),
209                                  btf.Stack(dtype='float32'), btf.Stack(dtype='float32'),
210                                  btf.Stack())
211     target_val_lengths = list(map(lambda x: x[-1], data_val_lengths))
212     target_test_lengths = list(map(lambda x: x[-1], data_test_lengths))
213     if args.bucket_scheme == 'constant':
214         bucket_scheme = nlp.data.ConstWidthBucket()
215     elif args.bucket_scheme == 'linear':
216         bucket_scheme = nlp.data.LinearWidthBucket()
217     elif args.bucket_scheme == 'exp':
218         bucket_scheme = nlp.data.ExpWidthBucket(bucket_len_step=1.2)
219     else:
220         raise NotImplementedError
221     train_batch_sampler = nlp.data.FixedBucketSampler(lengths=data_train_lengths,
222                                                       batch_size=args.batch_size,
223                                                       num_buckets=args.num_buckets,
224                                                       ratio=args.bucket_ratio,
225                                                       shuffle=True,
226                                                       use_average_length=use_average_length,
227                                                       num_shards=num_shards,
228                                                       bucket_scheme=bucket_scheme)
229     logging.info('Train Batch Sampler:\n%s', train_batch_sampler.stats())
230     train_data_loader = nlp.data.ShardedDataLoader(data_train,
231                                                    batch_sampler=train_batch_sampler,
232                                                    batchify_fn=train_batchify_fn,
233                                                    num_workers=num_workers)
234 
235     val_batch_sampler = nlp.data.FixedBucketSampler(lengths=target_val_lengths,
236                                                     batch_size=args.test_batch_size,
237                                                     num_buckets=args.num_buckets,
238                                                     ratio=args.bucket_ratio,
239                                                     shuffle=False,
240                                                     use_average_length=use_average_length,
241                                                     bucket_scheme=bucket_scheme)
242     logging.info('Valid Batch Sampler:\n%s', val_batch_sampler.stats())
243     val_data_loader = gluon.data.DataLoader(data_val,
244                                             batch_sampler=val_batch_sampler,
245                                             batchify_fn=test_batchify_fn,
246                                             num_workers=num_workers)
247     test_batch_sampler = nlp.data.FixedBucketSampler(lengths=target_test_lengths,
248                                                      batch_size=args.test_batch_size,
249                                                      num_buckets=args.num_buckets,
250                                                      ratio=args.bucket_ratio,
251                                                      shuffle=False,
252                                                      use_average_length=use_average_length,
253                                                      bucket_scheme=bucket_scheme)
254     logging.info('Test Batch Sampler:\n%s', test_batch_sampler.stats())
255     test_data_loader = gluon.data.DataLoader(data_test,
256                                              batch_sampler=test_batch_sampler,
257                                              batchify_fn=test_batchify_fn,
258                                              num_workers=num_workers)
259     return train_data_loader, val_data_loader, test_data_loader
260 
261 
262 def write_sentences(sentences, file_path):
263     with io.open(file_path, 'w', encoding='utf-8') as of:
264         for sent in sentences:
265             if isinstance(sent, (list, tuple)):
266                 of.write(u' '.join(sent) + u'\n')
267             else:
268                 of.write(sent + u'\n')
269
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/scripts/machine_translation/dataprocessor.py b/scripts/machine_translation/dataprocessor.py
--- a/scripts/machine_translation/dataprocessor.py
+++ b/scripts/machine_translation/dataprocessor.py
@@ -86,11 +86,13 @@
         self._tgt_max_len = tgt_max_len
 
     def __call__(self, src, tgt):
-        if self._src_max_len:
+        # For src_max_len < 0, we do not clip the sequence
+        if self._src_max_len >= 0:
             src_sentence = self._src_vocab[src.split()[:self._src_max_len]]
         else:
             src_sentence = self._src_vocab[src.split()]
-        if self._tgt_max_len:
+        # For tgt_max_len < 0, we do not clip the sequence
+        if self._tgt_max_len >= 0:
             tgt_sentence = self._tgt_vocab[tgt.split()[:self._tgt_max_len]]
         else:
             tgt_sentence = self._tgt_vocab[tgt.split()]
@@ -162,10 +164,14 @@
     if not data_val_processed:
         data_val_processed = process_dataset(data_val, src_vocab, tgt_vocab)
         _cache_dataset(data_val_processed, common_prefix + '_val')
-    data_test_processed = _load_cached_dataset(common_prefix + '_' + str(False) + '_test')
+    if dataset == 'WMT2014BPE':
+        filename = common_prefix + '_' + str(args.full) + '_test'
+    else:
+        filename = common_prefix + '_test'
+    data_test_processed = _load_cached_dataset(filename)
     if not data_test_processed:
         data_test_processed = process_dataset(data_test, src_vocab, tgt_vocab)
-        _cache_dataset(data_test_processed, common_prefix + '_' + str(False) + '_test')
+        _cache_dataset(data_test_processed, filename)
     if bleu == 'tweaked':
         fetch_tgt_sentence = lambda src, tgt: tgt.split()
         val_tgt_sentences = list(data_val.transform(fetch_tgt_sentence))

{"golden_diff": "diff --git a/scripts/machine_translation/dataprocessor.py b/scripts/machine_translation/dataprocessor.py\n--- a/scripts/machine_translation/dataprocessor.py\n+++ b/scripts/machine_translation/dataprocessor.py\n@@ -86,11 +86,13 @@\n         self._tgt_max_len = tgt_max_len\n \n     def __call__(self, src, tgt):\n-        if self._src_max_len:\n+        # For src_max_len < 0, we do not clip the sequence\n+        if self._src_max_len >= 0:\n             src_sentence = self._src_vocab[src.split()[:self._src_max_len]]\n         else:\n             src_sentence = self._src_vocab[src.split()]\n-        if self._tgt_max_len:\n+        # For tgt_max_len < 0, we do not clip the sequence\n+        if self._tgt_max_len >= 0:\n             tgt_sentence = self._tgt_vocab[tgt.split()[:self._tgt_max_len]]\n         else:\n             tgt_sentence = self._tgt_vocab[tgt.split()]\n@@ -162,10 +164,14 @@\n     if not data_val_processed:\n         data_val_processed = process_dataset(data_val, src_vocab, tgt_vocab)\n         _cache_dataset(data_val_processed, common_prefix + '_val')\n-    data_test_processed = _load_cached_dataset(common_prefix + '_' + str(False) + '_test')\n+    if dataset == 'WMT2014BPE':\n+        filename = common_prefix + '_' + str(args.full) + '_test'\n+    else:\n+        filename = common_prefix + '_test'\n+    data_test_processed = _load_cached_dataset(filename)\n     if not data_test_processed:\n         data_test_processed = process_dataset(data_test, src_vocab, tgt_vocab)\n-        _cache_dataset(data_test_processed, common_prefix + '_' + str(False) + '_test')\n+        _cache_dataset(data_test_processed, filename)\n     if bleu == 'tweaked':\n         fetch_tgt_sentence = lambda src, tgt: tgt.split()\n         val_tgt_sentences = list(data_val.transform(fetch_tgt_sentence))\n", "issue": "Transformer BLEU of the current code base is worse than previous one\nI recently retrained the transformer, and found that it will diverge with the default setting. The divergence issue is caused by use_layer_norm_before_dropout=True and is fixed in #543. However, the final testing BLEU is still worse than the previous result. Compare to the version in #413, the testing BLEU is about 0.8 worse.\n", "before_files": [{"content": "# coding: utf-8\n\n# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\"\"\"Data preprocessing for transformer.\"\"\"\n\nimport os\nimport io\nimport time\nimport logging\nimport numpy as np\nfrom mxnet import gluon\nimport gluonnlp as nlp\nimport gluonnlp.data.batchify as btf\nimport _constants\nimport dataset as _dataset\n\n\ndef _cache_dataset(dataset, prefix):\n    \"\"\"Cache the processed npy dataset the dataset into a npz\n\n    Parameters\n    ----------\n    dataset : SimpleDataset\n    file_path : str\n    \"\"\"\n    if not os.path.exists(_constants.CACHE_PATH):\n        os.makedirs(_constants.CACHE_PATH)\n    src_data = np.concatenate([e[0] for e in dataset])\n    tgt_data = np.concatenate([e[1] for e in dataset])\n    src_cumlen = np.cumsum([0]+[len(e[0]) for e in dataset])\n    tgt_cumlen = np.cumsum([0]+[len(e[1]) for e in dataset])\n    np.savez(os.path.join(_constants.CACHE_PATH, prefix + '.npz'),\n             src_data=src_data, tgt_data=tgt_data,\n             src_cumlen=src_cumlen, tgt_cumlen=tgt_cumlen)\n\n\ndef _load_cached_dataset(prefix):\n    cached_file_path = os.path.join(_constants.CACHE_PATH, prefix + '.npz')\n    if os.path.exists(cached_file_path):\n        print('Loading dataset...')\n        npz_data = np.load(cached_file_path)\n        src_data, tgt_data, src_cumlen, tgt_cumlen = \\\n                [npz_data[n] for n in ['src_data', 'tgt_data', 'src_cumlen', 'tgt_cumlen']]\n        src_data = np.array([src_data[low:high] for low, high\n                             in zip(src_cumlen[:-1], src_cumlen[1:])])\n        tgt_data = np.array([tgt_data[low:high] for low, high\n                             in zip(tgt_cumlen[:-1], tgt_cumlen[1:])])\n        return gluon.data.ArrayDataset(np.array(src_data), np.array(tgt_data))\n    else:\n        return None\n\n\nclass TrainValDataTransform(object):\n    \"\"\"Transform the machine translation dataset.\n\n    Clip source and the target sentences to the maximum length. For the source sentence, append the\n    EOS. For the target sentence, append BOS and EOS.\n\n    Parameters\n    ----------\n    src_vocab : Vocab\n    tgt_vocab : Vocab\n    src_max_len : int\n    tgt_max_len : int\n    \"\"\"\n\n    def __init__(self, src_vocab, tgt_vocab, src_max_len=None, tgt_max_len=None):\n        self._src_vocab = src_vocab\n        self._tgt_vocab = tgt_vocab\n        self._src_max_len = src_max_len\n        self._tgt_max_len = tgt_max_len\n\n    def __call__(self, src, tgt):\n        if self._src_max_len:\n            src_sentence = self._src_vocab[src.split()[:self._src_max_len]]\n        else:\n            src_sentence = self._src_vocab[src.split()]\n        if self._tgt_max_len:\n            tgt_sentence = self._tgt_vocab[tgt.split()[:self._tgt_max_len]]\n        else:\n            tgt_sentence = self._tgt_vocab[tgt.split()]\n        src_sentence.append(self._src_vocab[self._src_vocab.eos_token])\n        tgt_sentence.insert(0, self._tgt_vocab[self._tgt_vocab.bos_token])\n        tgt_sentence.append(self._tgt_vocab[self._tgt_vocab.eos_token])\n        src_npy = np.array(src_sentence, dtype=np.int32)\n        tgt_npy = np.array(tgt_sentence, dtype=np.int32)\n        return src_npy, tgt_npy\n\n\ndef process_dataset(dataset, src_vocab, tgt_vocab, src_max_len=-1, tgt_max_len=-1):\n    start = time.time()\n    dataset_processed = dataset.transform(TrainValDataTransform(src_vocab, tgt_vocab,\n                                                                src_max_len,\n                                                                tgt_max_len), lazy=False)\n    end = time.time()\n    print('Processing Time spent: {}'.format(end - start))\n    return dataset_processed\n\n\ndef load_translation_data(dataset, bleu, args):\n    \"\"\"Load translation dataset\n\n    Parameters\n    ----------\n    dataset : str\n    args : argparse result\n\n    Returns\n    -------\n\n    \"\"\"\n    src_lang, tgt_lang = args.src_lang, args.tgt_lang\n    if dataset == 'IWSLT2015':\n        common_prefix = 'IWSLT2015_{}_{}_{}_{}'.format(src_lang, tgt_lang,\n                                                       args.src_max_len, args.tgt_max_len)\n        data_train = nlp.data.IWSLT2015('train', src_lang=src_lang, tgt_lang=tgt_lang)\n        data_val = nlp.data.IWSLT2015('val', src_lang=src_lang, tgt_lang=tgt_lang)\n        data_test = nlp.data.IWSLT2015('test', src_lang=src_lang, tgt_lang=tgt_lang)\n    elif dataset == 'WMT2016BPE':\n        common_prefix = 'WMT2016BPE_{}_{}_{}_{}'.format(src_lang, tgt_lang,\n                                                        args.src_max_len, args.tgt_max_len)\n        data_train = nlp.data.WMT2016BPE('train', src_lang=src_lang, tgt_lang=tgt_lang)\n        data_val = nlp.data.WMT2016BPE('newstest2013', src_lang=src_lang, tgt_lang=tgt_lang)\n        data_test = nlp.data.WMT2016BPE('newstest2014', src_lang=src_lang, tgt_lang=tgt_lang)\n    elif dataset == 'WMT2014BPE':\n        common_prefix = 'WMT2014BPE_{}_{}_{}_{}'.format(src_lang, tgt_lang,\n                                                        args.src_max_len, args.tgt_max_len)\n        data_train = nlp.data.WMT2014BPE('train', src_lang=src_lang, tgt_lang=tgt_lang)\n        data_val = nlp.data.WMT2014BPE('newstest2013', src_lang=src_lang, tgt_lang=tgt_lang)\n        data_test = nlp.data.WMT2014BPE('newstest2014', src_lang=src_lang, tgt_lang=tgt_lang,\n                                        full=args.full)\n    elif dataset == 'TOY':\n        common_prefix = 'TOY_{}_{}_{}_{}'.format(src_lang, tgt_lang,\n                                                 args.src_max_len, args.tgt_max_len)\n        data_train = _dataset.TOY('train', src_lang=src_lang, tgt_lang=tgt_lang)\n        data_val = _dataset.TOY('val', src_lang=src_lang, tgt_lang=tgt_lang)\n        data_test = _dataset.TOY('test', src_lang=src_lang, tgt_lang=tgt_lang)\n    else:\n        raise NotImplementedError\n    src_vocab, tgt_vocab = data_train.src_vocab, data_train.tgt_vocab\n    data_train_processed = _load_cached_dataset(common_prefix + '_train')\n    if not data_train_processed:\n        data_train_processed = process_dataset(data_train, src_vocab, tgt_vocab,\n                                               args.src_max_len, args.tgt_max_len)\n        _cache_dataset(data_train_processed, common_prefix + '_train')\n    data_val_processed = _load_cached_dataset(common_prefix + '_val')\n    if not data_val_processed:\n        data_val_processed = process_dataset(data_val, src_vocab, tgt_vocab)\n        _cache_dataset(data_val_processed, common_prefix + '_val')\n    data_test_processed = _load_cached_dataset(common_prefix + '_' + str(False) + '_test')\n    if not data_test_processed:\n        data_test_processed = process_dataset(data_test, src_vocab, tgt_vocab)\n        _cache_dataset(data_test_processed, common_prefix + '_' + str(False) + '_test')\n    if bleu == 'tweaked':\n        fetch_tgt_sentence = lambda src, tgt: tgt.split()\n        val_tgt_sentences = list(data_val.transform(fetch_tgt_sentence))\n        test_tgt_sentences = list(data_test.transform(fetch_tgt_sentence))\n    elif bleu == '13a' or bleu == 'intl':\n        fetch_tgt_sentence = lambda src, tgt: tgt\n        if dataset == 'WMT2016BPE':\n            val_text = nlp.data.WMT2016('newstest2013', src_lang=src_lang, tgt_lang=tgt_lang)\n            test_text = nlp.data.WMT2016('newstest2014', src_lang=src_lang, tgt_lang=tgt_lang)\n        elif dataset == 'WMT2014BPE':\n            val_text = nlp.data.WMT2014('newstest2013', src_lang=src_lang, tgt_lang=tgt_lang)\n            test_text = nlp.data.WMT2014('newstest2014', src_lang=src_lang, tgt_lang=tgt_lang,\n                                         full=args.full)\n        elif dataset == 'IWSLT2015' or dataset == 'TOY':\n            val_text = data_val\n            test_text = data_test\n        else:\n            raise NotImplementedError\n        val_tgt_sentences = list(val_text.transform(fetch_tgt_sentence))\n        test_tgt_sentences = list(test_text.transform(fetch_tgt_sentence))\n    else:\n        raise NotImplementedError\n    return data_train_processed, data_val_processed, data_test_processed, \\\n           val_tgt_sentences, test_tgt_sentences, src_vocab, tgt_vocab\n\n\ndef get_data_lengths(dataset):\n    get_lengths = lambda *args: (args[2], args[3])\n    return list(dataset.transform(get_lengths))\n\n\ndef make_dataloader(data_train, data_val, data_test, args,\n                    use_average_length=False, num_shards=0, num_workers=8):\n    \"\"\"Create data loaders for training/validation/test.\"\"\"\n    data_train_lengths = get_data_lengths(data_train)\n    data_val_lengths = get_data_lengths(data_val)\n    data_test_lengths = get_data_lengths(data_test)\n    train_batchify_fn = btf.Tuple(btf.Pad(), btf.Pad(),\n                                  btf.Stack(dtype='float32'), btf.Stack(dtype='float32'))\n    test_batchify_fn = btf.Tuple(btf.Pad(), btf.Pad(),\n                                 btf.Stack(dtype='float32'), btf.Stack(dtype='float32'),\n                                 btf.Stack())\n    target_val_lengths = list(map(lambda x: x[-1], data_val_lengths))\n    target_test_lengths = list(map(lambda x: x[-1], data_test_lengths))\n    if args.bucket_scheme == 'constant':\n        bucket_scheme = nlp.data.ConstWidthBucket()\n    elif args.bucket_scheme == 'linear':\n        bucket_scheme = nlp.data.LinearWidthBucket()\n    elif args.bucket_scheme == 'exp':\n        bucket_scheme = nlp.data.ExpWidthBucket(bucket_len_step=1.2)\n    else:\n        raise NotImplementedError\n    train_batch_sampler = nlp.data.FixedBucketSampler(lengths=data_train_lengths,\n                                                      batch_size=args.batch_size,\n                                                      num_buckets=args.num_buckets,\n                                                      ratio=args.bucket_ratio,\n                                                      shuffle=True,\n                                                      use_average_length=use_average_length,\n                                                      num_shards=num_shards,\n                                                      bucket_scheme=bucket_scheme)\n    logging.info('Train Batch Sampler:\\n%s', train_batch_sampler.stats())\n    train_data_loader = nlp.data.ShardedDataLoader(data_train,\n                                                   batch_sampler=train_batch_sampler,\n                                                   batchify_fn=train_batchify_fn,\n                                                   num_workers=num_workers)\n\n    val_batch_sampler = nlp.data.FixedBucketSampler(lengths=target_val_lengths,\n                                                    batch_size=args.test_batch_size,\n                                                    num_buckets=args.num_buckets,\n                                                    ratio=args.bucket_ratio,\n                                                    shuffle=False,\n                                                    use_average_length=use_average_length,\n                                                    bucket_scheme=bucket_scheme)\n    logging.info('Valid Batch Sampler:\\n%s', val_batch_sampler.stats())\n    val_data_loader = gluon.data.DataLoader(data_val,\n                                            batch_sampler=val_batch_sampler,\n                                            batchify_fn=test_batchify_fn,\n                                            num_workers=num_workers)\n    test_batch_sampler = nlp.data.FixedBucketSampler(lengths=target_test_lengths,\n                                                     batch_size=args.test_batch_size,\n                                                     num_buckets=args.num_buckets,\n                                                     ratio=args.bucket_ratio,\n                                                     shuffle=False,\n                                                     use_average_length=use_average_length,\n                                                     bucket_scheme=bucket_scheme)\n    logging.info('Test Batch Sampler:\\n%s', test_batch_sampler.stats())\n    test_data_loader = gluon.data.DataLoader(data_test,\n                                             batch_sampler=test_batch_sampler,\n                                             batchify_fn=test_batchify_fn,\n                                             num_workers=num_workers)\n    return train_data_loader, val_data_loader, test_data_loader\n\n\ndef write_sentences(sentences, file_path):\n    with io.open(file_path, 'w', encoding='utf-8') as of:\n        for sent in sentences:\n            if isinstance(sent, (list, tuple)):\n                of.write(u' '.join(sent) + u'\\n')\n            else:\n                of.write(sent + u'\\n')\n", "path": "scripts/machine_translation/dataprocessor.py"}], "after_files": [{"content": "# coding: utf-8\n\n# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\"\"\"Data preprocessing for transformer.\"\"\"\n\nimport os\nimport io\nimport time\nimport logging\nimport numpy as np\nfrom mxnet import gluon\nimport gluonnlp as nlp\nimport gluonnlp.data.batchify as btf\nimport _constants\nimport dataset as _dataset\n\n\ndef _cache_dataset(dataset, prefix):\n    \"\"\"Cache the processed npy dataset the dataset into a npz\n\n    Parameters\n    ----------\n    dataset : SimpleDataset\n    file_path : str\n    \"\"\"\n    if not os.path.exists(_constants.CACHE_PATH):\n        os.makedirs(_constants.CACHE_PATH)\n    src_data = np.concatenate([e[0] for e in dataset])\n    tgt_data = np.concatenate([e[1] for e in dataset])\n    src_cumlen = np.cumsum([0]+[len(e[0]) for e in dataset])\n    tgt_cumlen = np.cumsum([0]+[len(e[1]) for e in dataset])\n    np.savez(os.path.join(_constants.CACHE_PATH, prefix + '.npz'),\n             src_data=src_data, tgt_data=tgt_data,\n             src_cumlen=src_cumlen, tgt_cumlen=tgt_cumlen)\n\n\ndef _load_cached_dataset(prefix):\n    cached_file_path = os.path.join(_constants.CACHE_PATH, prefix + '.npz')\n    if os.path.exists(cached_file_path):\n        print('Loading dataset...')\n        npz_data = np.load(cached_file_path)\n        src_data, tgt_data, src_cumlen, tgt_cumlen = \\\n                [npz_data[n] for n in ['src_data', 'tgt_data', 'src_cumlen', 'tgt_cumlen']]\n        src_data = np.array([src_data[low:high] for low, high\n                             in zip(src_cumlen[:-1], src_cumlen[1:])])\n        tgt_data = np.array([tgt_data[low:high] for low, high\n                             in zip(tgt_cumlen[:-1], tgt_cumlen[1:])])\n        return gluon.data.ArrayDataset(np.array(src_data), np.array(tgt_data))\n    else:\n        return None\n\n\nclass TrainValDataTransform(object):\n    \"\"\"Transform the machine translation dataset.\n\n    Clip source and the target sentences to the maximum length. For the source sentence, append the\n    EOS. For the target sentence, append BOS and EOS.\n\n    Parameters\n    ----------\n    src_vocab : Vocab\n    tgt_vocab : Vocab\n    src_max_len : int\n    tgt_max_len : int\n    \"\"\"\n\n    def __init__(self, src_vocab, tgt_vocab, src_max_len=None, tgt_max_len=None):\n        self._src_vocab = src_vocab\n        self._tgt_vocab = tgt_vocab\n        self._src_max_len = src_max_len\n        self._tgt_max_len = tgt_max_len\n\n    def __call__(self, src, tgt):\n        # For src_max_len < 0, we do not clip the sequence\n        if self._src_max_len >= 0:\n            src_sentence = self._src_vocab[src.split()[:self._src_max_len]]\n        else:\n            src_sentence = self._src_vocab[src.split()]\n        # For tgt_max_len < 0, we do not clip the sequence\n        if self._tgt_max_len >= 0:\n            tgt_sentence = self._tgt_vocab[tgt.split()[:self._tgt_max_len]]\n        else:\n            tgt_sentence = self._tgt_vocab[tgt.split()]\n        src_sentence.append(self._src_vocab[self._src_vocab.eos_token])\n        tgt_sentence.insert(0, self._tgt_vocab[self._tgt_vocab.bos_token])\n        tgt_sentence.append(self._tgt_vocab[self._tgt_vocab.eos_token])\n        src_npy = np.array(src_sentence, dtype=np.int32)\n        tgt_npy = np.array(tgt_sentence, dtype=np.int32)\n        return src_npy, tgt_npy\n\n\ndef process_dataset(dataset, src_vocab, tgt_vocab, src_max_len=-1, tgt_max_len=-1):\n    start = time.time()\n    dataset_processed = dataset.transform(TrainValDataTransform(src_vocab, tgt_vocab,\n                                                                src_max_len,\n                                                                tgt_max_len), lazy=False)\n    end = time.time()\n    print('Processing Time spent: {}'.format(end - start))\n    return dataset_processed\n\n\ndef load_translation_data(dataset, bleu, args):\n    \"\"\"Load translation dataset\n\n    Parameters\n    ----------\n    dataset : str\n    args : argparse result\n\n    Returns\n    -------\n\n    \"\"\"\n    src_lang, tgt_lang = args.src_lang, args.tgt_lang\n    if dataset == 'IWSLT2015':\n        common_prefix = 'IWSLT2015_{}_{}_{}_{}'.format(src_lang, tgt_lang,\n                                                       args.src_max_len, args.tgt_max_len)\n        data_train = nlp.data.IWSLT2015('train', src_lang=src_lang, tgt_lang=tgt_lang)\n        data_val = nlp.data.IWSLT2015('val', src_lang=src_lang, tgt_lang=tgt_lang)\n        data_test = nlp.data.IWSLT2015('test', src_lang=src_lang, tgt_lang=tgt_lang)\n    elif dataset == 'WMT2016BPE':\n        common_prefix = 'WMT2016BPE_{}_{}_{}_{}'.format(src_lang, tgt_lang,\n                                                        args.src_max_len, args.tgt_max_len)\n        data_train = nlp.data.WMT2016BPE('train', src_lang=src_lang, tgt_lang=tgt_lang)\n        data_val = nlp.data.WMT2016BPE('newstest2013', src_lang=src_lang, tgt_lang=tgt_lang)\n        data_test = nlp.data.WMT2016BPE('newstest2014', src_lang=src_lang, tgt_lang=tgt_lang)\n    elif dataset == 'WMT2014BPE':\n        common_prefix = 'WMT2014BPE_{}_{}_{}_{}'.format(src_lang, tgt_lang,\n                                                        args.src_max_len, args.tgt_max_len)\n        data_train = nlp.data.WMT2014BPE('train', src_lang=src_lang, tgt_lang=tgt_lang)\n        data_val = nlp.data.WMT2014BPE('newstest2013', src_lang=src_lang, tgt_lang=tgt_lang)\n        data_test = nlp.data.WMT2014BPE('newstest2014', src_lang=src_lang, tgt_lang=tgt_lang,\n                                        full=args.full)\n    elif dataset == 'TOY':\n        common_prefix = 'TOY_{}_{}_{}_{}'.format(src_lang, tgt_lang,\n                                                 args.src_max_len, args.tgt_max_len)\n        data_train = _dataset.TOY('train', src_lang=src_lang, tgt_lang=tgt_lang)\n        data_val = _dataset.TOY('val', src_lang=src_lang, tgt_lang=tgt_lang)\n        data_test = _dataset.TOY('test', src_lang=src_lang, tgt_lang=tgt_lang)\n    else:\n        raise NotImplementedError\n    src_vocab, tgt_vocab = data_train.src_vocab, data_train.tgt_vocab\n    data_train_processed = _load_cached_dataset(common_prefix + '_train')\n    if not data_train_processed:\n        data_train_processed = process_dataset(data_train, src_vocab, tgt_vocab,\n                                               args.src_max_len, args.tgt_max_len)\n        _cache_dataset(data_train_processed, common_prefix + '_train')\n    data_val_processed = _load_cached_dataset(common_prefix + '_val')\n    if not data_val_processed:\n        data_val_processed = process_dataset(data_val, src_vocab, tgt_vocab)\n        _cache_dataset(data_val_processed, common_prefix + '_val')\n    if dataset == 'WMT2014BPE':\n        filename = common_prefix + '_' + str(args.full) + '_test'\n    else:\n        filename = common_prefix + '_test'\n    data_test_processed = _load_cached_dataset(filename)\n    if not data_test_processed:\n        data_test_processed = process_dataset(data_test, src_vocab, tgt_vocab)\n        _cache_dataset(data_test_processed, filename)\n    if bleu == 'tweaked':\n        fetch_tgt_sentence = lambda src, tgt: tgt.split()\n        val_tgt_sentences = list(data_val.transform(fetch_tgt_sentence))\n        test_tgt_sentences = list(data_test.transform(fetch_tgt_sentence))\n    elif bleu == '13a' or bleu == 'intl':\n        fetch_tgt_sentence = lambda src, tgt: tgt\n        if dataset == 'WMT2016BPE':\n            val_text = nlp.data.WMT2016('newstest2013', src_lang=src_lang, tgt_lang=tgt_lang)\n            test_text = nlp.data.WMT2016('newstest2014', src_lang=src_lang, tgt_lang=tgt_lang)\n        elif dataset == 'WMT2014BPE':\n            val_text = nlp.data.WMT2014('newstest2013', src_lang=src_lang, tgt_lang=tgt_lang)\n            test_text = nlp.data.WMT2014('newstest2014', src_lang=src_lang, tgt_lang=tgt_lang,\n                                         full=args.full)\n        elif dataset == 'IWSLT2015' or dataset == 'TOY':\n            val_text = data_val\n            test_text = data_test\n        else:\n            raise NotImplementedError\n        val_tgt_sentences = list(val_text.transform(fetch_tgt_sentence))\n        test_tgt_sentences = list(test_text.transform(fetch_tgt_sentence))\n    else:\n        raise NotImplementedError\n    return data_train_processed, data_val_processed, data_test_processed, \\\n           val_tgt_sentences, test_tgt_sentences, src_vocab, tgt_vocab\n\n\ndef get_data_lengths(dataset):\n    get_lengths = lambda *args: (args[2], args[3])\n    return list(dataset.transform(get_lengths))\n\n\ndef make_dataloader(data_train, data_val, data_test, args,\n                    use_average_length=False, num_shards=0, num_workers=8):\n    \"\"\"Create data loaders for training/validation/test.\"\"\"\n    data_train_lengths = get_data_lengths(data_train)\n    data_val_lengths = get_data_lengths(data_val)\n    data_test_lengths = get_data_lengths(data_test)\n    train_batchify_fn = btf.Tuple(btf.Pad(), btf.Pad(),\n                                  btf.Stack(dtype='float32'), btf.Stack(dtype='float32'))\n    test_batchify_fn = btf.Tuple(btf.Pad(), btf.Pad(),\n                                 btf.Stack(dtype='float32'), btf.Stack(dtype='float32'),\n                                 btf.Stack())\n    target_val_lengths = list(map(lambda x: x[-1], data_val_lengths))\n    target_test_lengths = list(map(lambda x: x[-1], data_test_lengths))\n    if args.bucket_scheme == 'constant':\n        bucket_scheme = nlp.data.ConstWidthBucket()\n    elif args.bucket_scheme == 'linear':\n        bucket_scheme = nlp.data.LinearWidthBucket()\n    elif args.bucket_scheme == 'exp':\n        bucket_scheme = nlp.data.ExpWidthBucket(bucket_len_step=1.2)\n    else:\n        raise NotImplementedError\n    train_batch_sampler = nlp.data.FixedBucketSampler(lengths=data_train_lengths,\n                                                      batch_size=args.batch_size,\n                                                      num_buckets=args.num_buckets,\n                                                      ratio=args.bucket_ratio,\n                                                      shuffle=True,\n                                                      use_average_length=use_average_length,\n                                                      num_shards=num_shards,\n                                                      bucket_scheme=bucket_scheme)\n    logging.info('Train Batch Sampler:\\n%s', train_batch_sampler.stats())\n    train_data_loader = nlp.data.ShardedDataLoader(data_train,\n                                                   batch_sampler=train_batch_sampler,\n                                                   batchify_fn=train_batchify_fn,\n                                                   num_workers=num_workers)\n\n    val_batch_sampler = nlp.data.FixedBucketSampler(lengths=target_val_lengths,\n                                                    batch_size=args.test_batch_size,\n                                                    num_buckets=args.num_buckets,\n                                                    ratio=args.bucket_ratio,\n                                                    shuffle=False,\n                                                    use_average_length=use_average_length,\n                                                    bucket_scheme=bucket_scheme)\n    logging.info('Valid Batch Sampler:\\n%s', val_batch_sampler.stats())\n    val_data_loader = gluon.data.DataLoader(data_val,\n                                            batch_sampler=val_batch_sampler,\n                                            batchify_fn=test_batchify_fn,\n                                            num_workers=num_workers)\n    test_batch_sampler = nlp.data.FixedBucketSampler(lengths=target_test_lengths,\n                                                     batch_size=args.test_batch_size,\n                                                     num_buckets=args.num_buckets,\n                                                     ratio=args.bucket_ratio,\n                                                     shuffle=False,\n                                                     use_average_length=use_average_length,\n                                                     bucket_scheme=bucket_scheme)\n    logging.info('Test Batch Sampler:\\n%s', test_batch_sampler.stats())\n    test_data_loader = gluon.data.DataLoader(data_test,\n                                             batch_sampler=test_batch_sampler,\n                                             batchify_fn=test_batchify_fn,\n                                             num_workers=num_workers)\n    return train_data_loader, val_data_loader, test_data_loader\n\n\ndef write_sentences(sentences, file_path):\n    with io.open(file_path, 'w', encoding='utf-8') as of:\n        for sent in sentences:\n            if isinstance(sent, (list, tuple)):\n                of.write(u' '.join(sent) + u'\\n')\n            else:\n                of.write(sent + u'\\n')\n", "path": "scripts/machine_translation/dataprocessor.py"}]}