We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
Translation datasets not automatically downloading
Code:
``` python

from torchtext.data import Field
from torchtext.datasets import Multi30k

DE = Field(init_token='<sos>', eos_token='<eos>')
EN = Field(init_token='<sos>', eos_token='<eos>')

train, val, test = Multi30k.splits(exts=('.de', '.en'), fields=(DE, EN))
```

Error:
```
---------------------------------------------------------------------------
FileNotFoundError                         Traceback (most recent call last)
<ipython-input-3-637d49b65435> in <module>()
----> 1 train, val, test = Multi30k.splits(exts=('.de', '.en'), fields=(DE, EN))

~/miniconda3/envs/pytorch/lib/python3.6/site-packages/torchtext/datasets/translation.py in splits(cls, exts, fields, root, train, validation, test, **kwargs)
     99         """
    100         return super(Multi30k, cls).splits(
--> 101             exts, fields, root, train, validation, test, **kwargs)
    102 
    103 

~/miniconda3/envs/pytorch/lib/python3.6/site-packages/torchtext/datasets/translation.py in splits(cls, exts, fields, path, root, train, validation, test, **kwargs)
     62 
     63         train_data = None if train is None else cls(
---> 64             os.path.join(path, train), exts, fields, **kwargs)
     65         val_data = None if validation is None else cls(
     66             os.path.join(path, validation), exts, fields, **kwargs)

~/miniconda3/envs/pytorch/lib/python3.6/site-packages/torchtext/datasets/translation.py in __init__(self, path, exts, fields, **kwargs)
     31 
     32         examples = []
---> 33         with open(src_path) as src_file, open(trg_path) as trg_file:
     34             for src_line, trg_line in zip(src_file, trg_file):
     35                 src_line, trg_line = src_line.strip(), trg_line.strip()

FileNotFoundError: [Errno 2] No such file or directory: '.data/val.de'
```

It just doesn't seem to automatically download the data for both the Multi30k and WMT14 datasets. 

PyTorch version: 0.3.1
TorchText version 0.2.3

**EDIT**

I have downgraded my TorchText to version 0.2.1 and I do not get the error, had a quick look at the commits between 0.2.1 and 0.2.3 and couldn't figure out which commit introduced the break.

--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `torchtext/datasets/translation.py`
Content:
```
1 import os
2 import xml.etree.ElementTree as ET
3 import glob
4 import io
5 
6 from .. import data
7 
8 
9 class TranslationDataset(data.Dataset):
10     """Defines a dataset for machine translation."""
11 
12     @staticmethod
13     def sort_key(ex):
14         return data.interleave_keys(len(ex.src), len(ex.trg))
15 
16     def __init__(self, path, exts, fields, **kwargs):
17         """Create a TranslationDataset given paths and fields.
18 
19         Arguments:
20             path: Common prefix of paths to the data files for both languages.
21             exts: A tuple containing the extension to path for each language.
22             fields: A tuple containing the fields that will be used for data
23                 in each language.
24             Remaining keyword arguments: Passed to the constructor of
25                 data.Dataset.
26         """
27         if not isinstance(fields[0], (tuple, list)):
28             fields = [('src', fields[0]), ('trg', fields[1])]
29 
30         src_path, trg_path = tuple(os.path.expanduser(path + x) for x in exts)
31 
32         examples = []
33         with open(src_path) as src_file, open(trg_path) as trg_file:
34             for src_line, trg_line in zip(src_file, trg_file):
35                 src_line, trg_line = src_line.strip(), trg_line.strip()
36                 if src_line != '' and trg_line != '':
37                     examples.append(data.Example.fromlist(
38                         [src_line, trg_line], fields))
39 
40         super(TranslationDataset, self).__init__(examples, fields, **kwargs)
41 
42     @classmethod
43     def splits(cls, exts, fields, path=None, root='.data',
44                train='train', validation='val', test='test', **kwargs):
45         """Create dataset objects for splits of a TranslationDataset.
46 
47         Arguments:
48             path (str): Common prefix of the splits' file paths, or None to use
49                 the result of cls.download(root).
50             root: Root dataset storage directory. Default is '.data'.
51             exts: A tuple containing the extension to path for each language.
52             fields: A tuple containing the fields that will be used for data
53                 in each language.
54             train: The prefix of the train data. Default: 'train'.
55             validation: The prefix of the validation data. Default: 'val'.
56             test: The prefix of the test data. Default: 'test'.
57             Remaining keyword arguments: Passed to the splits method of
58                 Dataset.
59         """
60         if path is None:
61             path = cls.download(root)
62 
63         train_data = None if train is None else cls(
64             os.path.join(path, train), exts, fields, **kwargs)
65         val_data = None if validation is None else cls(
66             os.path.join(path, validation), exts, fields, **kwargs)
67         test_data = None if test is None else cls(
68             os.path.join(path, test), exts, fields, **kwargs)
69         return tuple(d for d in (train_data, val_data, test_data)
70                      if d is not None)
71 
72 
73 class Multi30k(TranslationDataset):
74     """The small-dataset WMT 2016 multimodal task, also known as Flickr30k"""
75 
76     urls = ['http://www.quest.dcs.shef.ac.uk/wmt16_files_mmt/training.tar.gz',
77             'http://www.quest.dcs.shef.ac.uk/wmt16_files_mmt/validation.tar.gz',
78             'http://www.quest.dcs.shef.ac.uk/'
79             'wmt17_files_mmt/mmt_task1_test2016.tar.gz']
80     name = 'multi30k'
81     dirname = ''
82 
83     @classmethod
84     def splits(cls, exts, fields, root='.data',
85                train='train', validation='val', test='test2016', **kwargs):
86         """Create dataset objects for splits of the Multi30k dataset.
87 
88         Arguments:
89 
90             root: Root dataset storage directory. Default is '.data'.
91             exts: A tuple containing the extension to path for each language.
92             fields: A tuple containing the fields that will be used for data
93                 in each language.
94             train: The prefix of the train data. Default: 'train'.
95             validation: The prefix of the validation data. Default: 'val'.
96             test: The prefix of the test data. Default: 'test'.
97             Remaining keyword arguments: Passed to the splits method of
98                 Dataset.
99         """
100         return super(Multi30k, cls).splits(
101             exts, fields, root, train, validation, test, **kwargs)
102 
103 
104 class IWSLT(TranslationDataset):
105     """The IWSLT 2016 TED talk translation task"""
106 
107     base_url = 'https://wit3.fbk.eu/archive/2016-01//texts/{}/{}/{}.tgz'
108     name = 'iwslt'
109     base_dirname = '{}-{}'
110 
111     @classmethod
112     def splits(cls, exts, fields, root='.data',
113                train='train', validation='IWSLT16.TED.tst2013',
114                test='IWSLT16.TED.tst2014', **kwargs):
115         """Create dataset objects for splits of the IWSLT dataset.
116 
117         Arguments:
118 
119             root: Root dataset storage directory. Default is '.data'.
120             exts: A tuple containing the extension to path for each language.
121             fields: A tuple containing the fields that will be used for data
122                 in each language.
123             train: The prefix of the train data. Default: 'train'.
124             validation: The prefix of the validation data. Default: 'val'.
125             test: The prefix of the test data. Default: 'test'.
126             Remaining keyword arguments: Passed to the splits method of
127                 Dataset.
128         """
129         cls.dirname = cls.base_dirname.format(exts[0][1:], exts[1][1:])
130         cls.urls = [cls.base_url.format(exts[0][1:], exts[1][1:], cls.dirname)]
131         check = os.path.join(root, cls.name, cls.dirname)
132         path = cls.download(root, check=check)
133 
134         train = '.'.join([train, cls.dirname])
135         validation = '.'.join([validation, cls.dirname])
136         if test is not None:
137             test = '.'.join([test, cls.dirname])
138 
139         if not os.path.exists(os.path.join(path, train) + exts[0]):
140             cls.clean(path)
141 
142         train_data = None if train is None else cls(
143             os.path.join(path, train), exts, fields, **kwargs)
144         val_data = None if validation is None else cls(
145             os.path.join(path, validation), exts, fields, **kwargs)
146         test_data = None if test is None else cls(
147             os.path.join(path, test), exts, fields, **kwargs)
148         return tuple(d for d in (train_data, val_data, test_data)
149                      if d is not None)
150 
151     @staticmethod
152     def clean(path):
153         for f_xml in glob.iglob(os.path.join(path, '*.xml')):
154             print(f_xml)
155             f_txt = os.path.splitext(f_xml)[0]
156             with io.open(f_txt, mode='w', encoding='utf-8') as fd_txt:
157                 root = ET.parse(f_xml).getroot()[0]
158                 for doc in root.findall('doc'):
159                     for e in doc.findall('seg'):
160                         fd_txt.write(e.text.strip() + '\n')
161 
162         xml_tags = ['<url', '<keywords', '<talkid', '<description',
163                     '<reviewer', '<translator', '<title', '<speaker']
164         for f_orig in glob.iglob(os.path.join(path, 'train.tags*')):
165             print(f_orig)
166             f_txt = f_orig.replace('.tags', '')
167             with io.open(f_txt, mode='w', encoding='utf-8') as fd_txt, \
168                     io.open(f_orig, mode='r', encoding='utf-8') as fd_orig:
169                 for l in fd_orig:
170                     if not any(tag in l for tag in xml_tags):
171                         fd_txt.write(l.strip() + '\n')
172 
173 
174 class WMT14(TranslationDataset):
175     """The WMT 2014 English-German dataset, as preprocessed by Google Brain.
176 
177     Though this download contains test sets from 2015 and 2016, the train set
178     differs slightly from WMT 2015 and 2016 and significantly from WMT 2017."""
179 
180     urls = [('https://drive.google.com/uc?export=download&'
181              'id=0B_bZck-ksdkpM25jRUN2X2UxMm8', 'wmt16_en_de.tar.gz')]
182     name = 'wmt14'
183     dirname = ''
184 
185     @classmethod
186     def splits(cls, exts, fields, root='.data',
187                train='train.tok.clean.bpe.32000',
188                validation='newstest2013.tok.bpe.32000',
189                test='newstest2014.tok.bpe.32000', **kwargs):
190         """Create dataset objects for splits of the WMT 2014 dataset.
191 
192         Arguments:
193 
194             root: Root dataset storage directory. Default is '.data'.
195             exts: A tuple containing the extensions for each language. Must be
196                 either ('.en', '.de') or the reverse.
197             fields: A tuple containing the fields that will be used for data
198                 in each language.
199             train: The prefix of the train data. Default:
200                 'train.tok.clean.bpe.32000'.
201             validation: The prefix of the validation data. Default:
202                 'newstest2013.tok.bpe.32000'.
203             test: The prefix of the test data. Default:
204                 'newstest2014.tok.bpe.32000'.
205             Remaining keyword arguments: Passed to the splits method of
206                 Dataset.
207         """
208         return super(WMT14, cls).splits(
209             exts, fields, root, train, validation, test, **kwargs)
210
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/torchtext/datasets/translation.py b/torchtext/datasets/translation.py
--- a/torchtext/datasets/translation.py
+++ b/torchtext/datasets/translation.py
@@ -97,8 +97,9 @@
             Remaining keyword arguments: Passed to the splits method of
                 Dataset.
         """
+        path = os.path.join('data', cls.name)
         return super(Multi30k, cls).splits(
-            exts, fields, root, train, validation, test, **kwargs)
+            exts, fields, path, root, train, validation, test, **kwargs)
 
 
 class IWSLT(TranslationDataset):
@@ -205,5 +206,6 @@
             Remaining keyword arguments: Passed to the splits method of
                 Dataset.
         """
+        path = os.path.join('data', cls.name)
         return super(WMT14, cls).splits(
-            exts, fields, root, train, validation, test, **kwargs)
+            exts, fields, path, root, train, validation, test, **kwargs)

{"golden_diff": "diff --git a/torchtext/datasets/translation.py b/torchtext/datasets/translation.py\n--- a/torchtext/datasets/translation.py\n+++ b/torchtext/datasets/translation.py\n@@ -97,8 +97,9 @@\n             Remaining keyword arguments: Passed to the splits method of\n                 Dataset.\n         \"\"\"\n+        path = os.path.join('data', cls.name)\n         return super(Multi30k, cls).splits(\n-            exts, fields, root, train, validation, test, **kwargs)\n+            exts, fields, path, root, train, validation, test, **kwargs)\n \n \n class IWSLT(TranslationDataset):\n@@ -205,5 +206,6 @@\n             Remaining keyword arguments: Passed to the splits method of\n                 Dataset.\n         \"\"\"\n+        path = os.path.join('data', cls.name)\n         return super(WMT14, cls).splits(\n-            exts, fields, root, train, validation, test, **kwargs)\n+            exts, fields, path, root, train, validation, test, **kwargs)\n", "issue": "Translation datasets not automatically downloading\nCode:\r\n``` python\r\n\r\nfrom torchtext.data import Field\r\nfrom torchtext.datasets import Multi30k\r\n\r\nDE = Field(init_token='<sos>', eos_token='<eos>')\r\nEN = Field(init_token='<sos>', eos_token='<eos>')\r\n\r\ntrain, val, test = Multi30k.splits(exts=('.de', '.en'), fields=(DE, EN))\r\n```\r\n\r\nError:\r\n```\r\n---------------------------------------------------------------------------\r\nFileNotFoundError                         Traceback (most recent call last)\r\n<ipython-input-3-637d49b65435> in <module>()\r\n----> 1 train, val, test = Multi30k.splits(exts=('.de', '.en'), fields=(DE, EN))\r\n\r\n~/miniconda3/envs/pytorch/lib/python3.6/site-packages/torchtext/datasets/translation.py in splits(cls, exts, fields, root, train, validation, test, **kwargs)\r\n     99         \"\"\"\r\n    100         return super(Multi30k, cls).splits(\r\n--> 101             exts, fields, root, train, validation, test, **kwargs)\r\n    102 \r\n    103 \r\n\r\n~/miniconda3/envs/pytorch/lib/python3.6/site-packages/torchtext/datasets/translation.py in splits(cls, exts, fields, path, root, train, validation, test, **kwargs)\r\n     62 \r\n     63         train_data = None if train is None else cls(\r\n---> 64             os.path.join(path, train), exts, fields, **kwargs)\r\n     65         val_data = None if validation is None else cls(\r\n     66             os.path.join(path, validation), exts, fields, **kwargs)\r\n\r\n~/miniconda3/envs/pytorch/lib/python3.6/site-packages/torchtext/datasets/translation.py in __init__(self, path, exts, fields, **kwargs)\r\n     31 \r\n     32         examples = []\r\n---> 33         with open(src_path) as src_file, open(trg_path) as trg_file:\r\n     34             for src_line, trg_line in zip(src_file, trg_file):\r\n     35                 src_line, trg_line = src_line.strip(), trg_line.strip()\r\n\r\nFileNotFoundError: [Errno 2] No such file or directory: '.data/val.de'\r\n```\r\n\r\nIt just doesn't seem to automatically download the data for both the Multi30k and WMT14 datasets. \r\n\r\nPyTorch version: 0.3.1\r\nTorchText version 0.2.3\r\n\r\n**EDIT**\r\n\r\nI have downgraded my TorchText to version 0.2.1 and I do not get the error, had a quick look at the commits between 0.2.1 and 0.2.3 and couldn't figure out which commit introduced the break.\n", "before_files": [{"content": "import os\nimport xml.etree.ElementTree as ET\nimport glob\nimport io\n\nfrom .. import data\n\n\nclass TranslationDataset(data.Dataset):\n    \"\"\"Defines a dataset for machine translation.\"\"\"\n\n    @staticmethod\n    def sort_key(ex):\n        return data.interleave_keys(len(ex.src), len(ex.trg))\n\n    def __init__(self, path, exts, fields, **kwargs):\n        \"\"\"Create a TranslationDataset given paths and fields.\n\n        Arguments:\n            path: Common prefix of paths to the data files for both languages.\n            exts: A tuple containing the extension to path for each language.\n            fields: A tuple containing the fields that will be used for data\n                in each language.\n            Remaining keyword arguments: Passed to the constructor of\n                data.Dataset.\n        \"\"\"\n        if not isinstance(fields[0], (tuple, list)):\n            fields = [('src', fields[0]), ('trg', fields[1])]\n\n        src_path, trg_path = tuple(os.path.expanduser(path + x) for x in exts)\n\n        examples = []\n        with open(src_path) as src_file, open(trg_path) as trg_file:\n            for src_line, trg_line in zip(src_file, trg_file):\n                src_line, trg_line = src_line.strip(), trg_line.strip()\n                if src_line != '' and trg_line != '':\n                    examples.append(data.Example.fromlist(\n                        [src_line, trg_line], fields))\n\n        super(TranslationDataset, self).__init__(examples, fields, **kwargs)\n\n    @classmethod\n    def splits(cls, exts, fields, path=None, root='.data',\n               train='train', validation='val', test='test', **kwargs):\n        \"\"\"Create dataset objects for splits of a TranslationDataset.\n\n        Arguments:\n            path (str): Common prefix of the splits' file paths, or None to use\n                the result of cls.download(root).\n            root: Root dataset storage directory. Default is '.data'.\n            exts: A tuple containing the extension to path for each language.\n            fields: A tuple containing the fields that will be used for data\n                in each language.\n            train: The prefix of the train data. Default: 'train'.\n            validation: The prefix of the validation data. Default: 'val'.\n            test: The prefix of the test data. Default: 'test'.\n            Remaining keyword arguments: Passed to the splits method of\n                Dataset.\n        \"\"\"\n        if path is None:\n            path = cls.download(root)\n\n        train_data = None if train is None else cls(\n            os.path.join(path, train), exts, fields, **kwargs)\n        val_data = None if validation is None else cls(\n            os.path.join(path, validation), exts, fields, **kwargs)\n        test_data = None if test is None else cls(\n            os.path.join(path, test), exts, fields, **kwargs)\n        return tuple(d for d in (train_data, val_data, test_data)\n                     if d is not None)\n\n\nclass Multi30k(TranslationDataset):\n    \"\"\"The small-dataset WMT 2016 multimodal task, also known as Flickr30k\"\"\"\n\n    urls = ['http://www.quest.dcs.shef.ac.uk/wmt16_files_mmt/training.tar.gz',\n            'http://www.quest.dcs.shef.ac.uk/wmt16_files_mmt/validation.tar.gz',\n            'http://www.quest.dcs.shef.ac.uk/'\n            'wmt17_files_mmt/mmt_task1_test2016.tar.gz']\n    name = 'multi30k'\n    dirname = ''\n\n    @classmethod\n    def splits(cls, exts, fields, root='.data',\n               train='train', validation='val', test='test2016', **kwargs):\n        \"\"\"Create dataset objects for splits of the Multi30k dataset.\n\n        Arguments:\n\n            root: Root dataset storage directory. Default is '.data'.\n            exts: A tuple containing the extension to path for each language.\n            fields: A tuple containing the fields that will be used for data\n                in each language.\n            train: The prefix of the train data. Default: 'train'.\n            validation: The prefix of the validation data. Default: 'val'.\n            test: The prefix of the test data. Default: 'test'.\n            Remaining keyword arguments: Passed to the splits method of\n                Dataset.\n        \"\"\"\n        return super(Multi30k, cls).splits(\n            exts, fields, root, train, validation, test, **kwargs)\n\n\nclass IWSLT(TranslationDataset):\n    \"\"\"The IWSLT 2016 TED talk translation task\"\"\"\n\n    base_url = 'https://wit3.fbk.eu/archive/2016-01//texts/{}/{}/{}.tgz'\n    name = 'iwslt'\n    base_dirname = '{}-{}'\n\n    @classmethod\n    def splits(cls, exts, fields, root='.data',\n               train='train', validation='IWSLT16.TED.tst2013',\n               test='IWSLT16.TED.tst2014', **kwargs):\n        \"\"\"Create dataset objects for splits of the IWSLT dataset.\n\n        Arguments:\n\n            root: Root dataset storage directory. Default is '.data'.\n            exts: A tuple containing the extension to path for each language.\n            fields: A tuple containing the fields that will be used for data\n                in each language.\n            train: The prefix of the train data. Default: 'train'.\n            validation: The prefix of the validation data. Default: 'val'.\n            test: The prefix of the test data. Default: 'test'.\n            Remaining keyword arguments: Passed to the splits method of\n                Dataset.\n        \"\"\"\n        cls.dirname = cls.base_dirname.format(exts[0][1:], exts[1][1:])\n        cls.urls = [cls.base_url.format(exts[0][1:], exts[1][1:], cls.dirname)]\n        check = os.path.join(root, cls.name, cls.dirname)\n        path = cls.download(root, check=check)\n\n        train = '.'.join([train, cls.dirname])\n        validation = '.'.join([validation, cls.dirname])\n        if test is not None:\n            test = '.'.join([test, cls.dirname])\n\n        if not os.path.exists(os.path.join(path, train) + exts[0]):\n            cls.clean(path)\n\n        train_data = None if train is None else cls(\n            os.path.join(path, train), exts, fields, **kwargs)\n        val_data = None if validation is None else cls(\n            os.path.join(path, validation), exts, fields, **kwargs)\n        test_data = None if test is None else cls(\n            os.path.join(path, test), exts, fields, **kwargs)\n        return tuple(d for d in (train_data, val_data, test_data)\n                     if d is not None)\n\n    @staticmethod\n    def clean(path):\n        for f_xml in glob.iglob(os.path.join(path, '*.xml')):\n            print(f_xml)\n            f_txt = os.path.splitext(f_xml)[0]\n            with io.open(f_txt, mode='w', encoding='utf-8') as fd_txt:\n                root = ET.parse(f_xml).getroot()[0]\n                for doc in root.findall('doc'):\n                    for e in doc.findall('seg'):\n                        fd_txt.write(e.text.strip() + '\\n')\n\n        xml_tags = ['<url', '<keywords', '<talkid', '<description',\n                    '<reviewer', '<translator', '<title', '<speaker']\n        for f_orig in glob.iglob(os.path.join(path, 'train.tags*')):\n            print(f_orig)\n            f_txt = f_orig.replace('.tags', '')\n            with io.open(f_txt, mode='w', encoding='utf-8') as fd_txt, \\\n                    io.open(f_orig, mode='r', encoding='utf-8') as fd_orig:\n                for l in fd_orig:\n                    if not any(tag in l for tag in xml_tags):\n                        fd_txt.write(l.strip() + '\\n')\n\n\nclass WMT14(TranslationDataset):\n    \"\"\"The WMT 2014 English-German dataset, as preprocessed by Google Brain.\n\n    Though this download contains test sets from 2015 and 2016, the train set\n    differs slightly from WMT 2015 and 2016 and significantly from WMT 2017.\"\"\"\n\n    urls = [('https://drive.google.com/uc?export=download&'\n             'id=0B_bZck-ksdkpM25jRUN2X2UxMm8', 'wmt16_en_de.tar.gz')]\n    name = 'wmt14'\n    dirname = ''\n\n    @classmethod\n    def splits(cls, exts, fields, root='.data',\n               train='train.tok.clean.bpe.32000',\n               validation='newstest2013.tok.bpe.32000',\n               test='newstest2014.tok.bpe.32000', **kwargs):\n        \"\"\"Create dataset objects for splits of the WMT 2014 dataset.\n\n        Arguments:\n\n            root: Root dataset storage directory. Default is '.data'.\n            exts: A tuple containing the extensions for each language. Must be\n                either ('.en', '.de') or the reverse.\n            fields: A tuple containing the fields that will be used for data\n                in each language.\n            train: The prefix of the train data. Default:\n                'train.tok.clean.bpe.32000'.\n            validation: The prefix of the validation data. Default:\n                'newstest2013.tok.bpe.32000'.\n            test: The prefix of the test data. Default:\n                'newstest2014.tok.bpe.32000'.\n            Remaining keyword arguments: Passed to the splits method of\n                Dataset.\n        \"\"\"\n        return super(WMT14, cls).splits(\n            exts, fields, root, train, validation, test, **kwargs)\n", "path": "torchtext/datasets/translation.py"}], "after_files": [{"content": "import os\nimport xml.etree.ElementTree as ET\nimport glob\nimport io\n\nfrom .. import data\n\n\nclass TranslationDataset(data.Dataset):\n    \"\"\"Defines a dataset for machine translation.\"\"\"\n\n    @staticmethod\n    def sort_key(ex):\n        return data.interleave_keys(len(ex.src), len(ex.trg))\n\n    def __init__(self, path, exts, fields, **kwargs):\n        \"\"\"Create a TranslationDataset given paths and fields.\n\n        Arguments:\n            path: Common prefix of paths to the data files for both languages.\n            exts: A tuple containing the extension to path for each language.\n            fields: A tuple containing the fields that will be used for data\n                in each language.\n            Remaining keyword arguments: Passed to the constructor of\n                data.Dataset.\n        \"\"\"\n        if not isinstance(fields[0], (tuple, list)):\n            fields = [('src', fields[0]), ('trg', fields[1])]\n\n        src_path, trg_path = tuple(os.path.expanduser(path + x) for x in exts)\n\n        examples = []\n        with open(src_path) as src_file, open(trg_path) as trg_file:\n            for src_line, trg_line in zip(src_file, trg_file):\n                src_line, trg_line = src_line.strip(), trg_line.strip()\n                if src_line != '' and trg_line != '':\n                    examples.append(data.Example.fromlist(\n                        [src_line, trg_line], fields))\n\n        super(TranslationDataset, self).__init__(examples, fields, **kwargs)\n\n    @classmethod\n    def splits(cls, exts, fields, path=None, root='.data',\n               train='train', validation='val', test='test', **kwargs):\n        \"\"\"Create dataset objects for splits of a TranslationDataset.\n\n        Arguments:\n            path (str): Common prefix of the splits' file paths, or None to use\n                the result of cls.download(root).\n            root: Root dataset storage directory. Default is '.data'.\n            exts: A tuple containing the extension to path for each language.\n            fields: A tuple containing the fields that will be used for data\n                in each language.\n            train: The prefix of the train data. Default: 'train'.\n            validation: The prefix of the validation data. Default: 'val'.\n            test: The prefix of the test data. Default: 'test'.\n            Remaining keyword arguments: Passed to the splits method of\n                Dataset.\n        \"\"\"\n        if path is None:\n            path = cls.download(root)\n\n        train_data = None if train is None else cls(\n            os.path.join(path, train), exts, fields, **kwargs)\n        val_data = None if validation is None else cls(\n            os.path.join(path, validation), exts, fields, **kwargs)\n        test_data = None if test is None else cls(\n            os.path.join(path, test), exts, fields, **kwargs)\n        return tuple(d for d in (train_data, val_data, test_data)\n                     if d is not None)\n\n\nclass Multi30k(TranslationDataset):\n    \"\"\"The small-dataset WMT 2016 multimodal task, also known as Flickr30k\"\"\"\n\n    urls = ['http://www.quest.dcs.shef.ac.uk/wmt16_files_mmt/training.tar.gz',\n            'http://www.quest.dcs.shef.ac.uk/wmt16_files_mmt/validation.tar.gz',\n            'http://www.quest.dcs.shef.ac.uk/'\n            'wmt17_files_mmt/mmt_task1_test2016.tar.gz']\n    name = 'multi30k'\n    dirname = ''\n\n    @classmethod\n    def splits(cls, exts, fields, root='.data',\n               train='train', validation='val', test='test2016', **kwargs):\n        \"\"\"Create dataset objects for splits of the Multi30k dataset.\n\n        Arguments:\n\n            root: Root dataset storage directory. Default is '.data'.\n            exts: A tuple containing the extension to path for each language.\n            fields: A tuple containing the fields that will be used for data\n                in each language.\n            train: The prefix of the train data. Default: 'train'.\n            validation: The prefix of the validation data. Default: 'val'.\n            test: The prefix of the test data. Default: 'test'.\n            Remaining keyword arguments: Passed to the splits method of\n                Dataset.\n        \"\"\"\n        path = os.path.join('data', cls.name)\n        return super(Multi30k, cls).splits(\n            exts, fields, path, root, train, validation, test, **kwargs)\n\n\nclass IWSLT(TranslationDataset):\n    \"\"\"The IWSLT 2016 TED talk translation task\"\"\"\n\n    base_url = 'https://wit3.fbk.eu/archive/2016-01//texts/{}/{}/{}.tgz'\n    name = 'iwslt'\n    base_dirname = '{}-{}'\n\n    @classmethod\n    def splits(cls, exts, fields, root='.data',\n               train='train', validation='IWSLT16.TED.tst2013',\n               test='IWSLT16.TED.tst2014', **kwargs):\n        \"\"\"Create dataset objects for splits of the IWSLT dataset.\n\n        Arguments:\n\n            root: Root dataset storage directory. Default is '.data'.\n            exts: A tuple containing the extension to path for each language.\n            fields: A tuple containing the fields that will be used for data\n                in each language.\n            train: The prefix of the train data. Default: 'train'.\n            validation: The prefix of the validation data. Default: 'val'.\n            test: The prefix of the test data. Default: 'test'.\n            Remaining keyword arguments: Passed to the splits method of\n                Dataset.\n        \"\"\"\n        cls.dirname = cls.base_dirname.format(exts[0][1:], exts[1][1:])\n        cls.urls = [cls.base_url.format(exts[0][1:], exts[1][1:], cls.dirname)]\n        check = os.path.join(root, cls.name, cls.dirname)\n        path = cls.download(root, check=check)\n\n        train = '.'.join([train, cls.dirname])\n        validation = '.'.join([validation, cls.dirname])\n        if test is not None:\n            test = '.'.join([test, cls.dirname])\n\n        if not os.path.exists(os.path.join(path, train) + exts[0]):\n            cls.clean(path)\n\n        train_data = None if train is None else cls(\n            os.path.join(path, train), exts, fields, **kwargs)\n        val_data = None if validation is None else cls(\n            os.path.join(path, validation), exts, fields, **kwargs)\n        test_data = None if test is None else cls(\n            os.path.join(path, test), exts, fields, **kwargs)\n        return tuple(d for d in (train_data, val_data, test_data)\n                     if d is not None)\n\n    @staticmethod\n    def clean(path):\n        for f_xml in glob.iglob(os.path.join(path, '*.xml')):\n            print(f_xml)\n            f_txt = os.path.splitext(f_xml)[0]\n            with io.open(f_txt, mode='w', encoding='utf-8') as fd_txt:\n                root = ET.parse(f_xml).getroot()[0]\n                for doc in root.findall('doc'):\n                    for e in doc.findall('seg'):\n                        fd_txt.write(e.text.strip() + '\\n')\n\n        xml_tags = ['<url', '<keywords', '<talkid', '<description',\n                    '<reviewer', '<translator', '<title', '<speaker']\n        for f_orig in glob.iglob(os.path.join(path, 'train.tags*')):\n            print(f_orig)\n            f_txt = f_orig.replace('.tags', '')\n            with io.open(f_txt, mode='w', encoding='utf-8') as fd_txt, \\\n                    io.open(f_orig, mode='r', encoding='utf-8') as fd_orig:\n                for l in fd_orig:\n                    if not any(tag in l for tag in xml_tags):\n                        fd_txt.write(l.strip() + '\\n')\n\n\nclass WMT14(TranslationDataset):\n    \"\"\"The WMT 2014 English-German dataset, as preprocessed by Google Brain.\n\n    Though this download contains test sets from 2015 and 2016, the train set\n    differs slightly from WMT 2015 and 2016 and significantly from WMT 2017.\"\"\"\n\n    urls = [('https://drive.google.com/uc?export=download&'\n             'id=0B_bZck-ksdkpM25jRUN2X2UxMm8', 'wmt16_en_de.tar.gz')]\n    name = 'wmt14'\n    dirname = ''\n\n    @classmethod\n    def splits(cls, exts, fields, root='.data',\n               train='train.tok.clean.bpe.32000',\n               validation='newstest2013.tok.bpe.32000',\n               test='newstest2014.tok.bpe.32000', **kwargs):\n        \"\"\"Create dataset objects for splits of the WMT 2014 dataset.\n\n        Arguments:\n\n            root: Root dataset storage directory. Default is '.data'.\n            exts: A tuple containing the extensions for each language. Must be\n                either ('.en', '.de') or the reverse.\n            fields: A tuple containing the fields that will be used for data\n                in each language.\n            train: The prefix of the train data. Default:\n                'train.tok.clean.bpe.32000'.\n            validation: The prefix of the validation data. Default:\n                'newstest2013.tok.bpe.32000'.\n            test: The prefix of the test data. Default:\n                'newstest2014.tok.bpe.32000'.\n            Remaining keyword arguments: Passed to the splits method of\n                Dataset.\n        \"\"\"\n        path = os.path.join('data', cls.name)\n        return super(WMT14, cls).splits(\n            exts, fields, path, root, train, validation, test, **kwargs)\n", "path": "torchtext/datasets/translation.py"}]}