We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
Translation splits error when not downloading dataset first
Thanks @AngusMonroe for finding this! The problem is that the absence of dataset is not addressed when creating splits. Minimal example:

```

from torchtext.datasets import Multi30k
from torchtext.data import Field
EN = Field()
DE = Field()
ds = Multi30k.splits(('.de','.en'),[('de',DE),('en',EN)],'data/multi30k')
```


--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `torchtext/datasets/translation.py`
Content:
```
1 import os
2 import xml.etree.ElementTree as ET
3 import glob
4 import io
5 
6 from .. import data
7 
8 
9 class TranslationDataset(data.Dataset):
10     """Defines a dataset for machine translation."""
11 
12     @staticmethod
13     def sort_key(ex):
14         return data.interleave_keys(len(ex.src), len(ex.trg))
15 
16     def __init__(self, path, exts, fields, **kwargs):
17         """Create a TranslationDataset given paths and fields.
18 
19         Arguments:
20             path: Common prefix of paths to the data files for both languages.
21             exts: A tuple containing the extension to path for each language.
22             fields: A tuple containing the fields that will be used for data
23                 in each language.
24             Remaining keyword arguments: Passed to the constructor of
25                 data.Dataset.
26         """
27         if not isinstance(fields[0], (tuple, list)):
28             fields = [('src', fields[0]), ('trg', fields[1])]
29 
30         src_path, trg_path = tuple(os.path.expanduser(path + x) for x in exts)
31 
32         examples = []
33         with open(src_path) as src_file, open(trg_path) as trg_file:
34             for src_line, trg_line in zip(src_file, trg_file):
35                 src_line, trg_line = src_line.strip(), trg_line.strip()
36                 if src_line != '' and trg_line != '':
37                     examples.append(data.Example.fromlist(
38                         [src_line, trg_line], fields))
39 
40         super(TranslationDataset, self).__init__(examples, fields, **kwargs)
41 
42     @classmethod
43     def splits(cls, exts, fields, path=None, root='.data',
44                train='train', validation='val', test='test', **kwargs):
45         """Create dataset objects for splits of a TranslationDataset.
46 
47         Arguments:
48             path (str): Common prefix of the splits' file paths, or None to use
49                 the result of cls.download(root).
50             root: Root dataset storage directory. Default is '.data'.
51             exts: A tuple containing the extension to path for each language.
52             fields: A tuple containing the fields that will be used for data
53                 in each language.
54             train: The prefix of the train data. Default: 'train'.
55             validation: The prefix of the validation data. Default: 'val'.
56             test: The prefix of the test data. Default: 'test'.
57             Remaining keyword arguments: Passed to the splits method of
58                 Dataset.
59         """
60         if path is None:
61             path = cls.download(root)
62 
63         train_data = None if train is None else cls(
64             os.path.join(path, train), exts, fields, **kwargs)
65         val_data = None if validation is None else cls(
66             os.path.join(path, validation), exts, fields, **kwargs)
67         test_data = None if test is None else cls(
68             os.path.join(path, test), exts, fields, **kwargs)
69         return tuple(d for d in (train_data, val_data, test_data)
70                      if d is not None)
71 
72 
73 class Multi30k(TranslationDataset):
74     """The small-dataset WMT 2016 multimodal task, also known as Flickr30k"""
75 
76     urls = ['http://www.quest.dcs.shef.ac.uk/wmt16_files_mmt/training.tar.gz',
77             'http://www.quest.dcs.shef.ac.uk/wmt16_files_mmt/validation.tar.gz',
78             'http://www.quest.dcs.shef.ac.uk/'
79             'wmt17_files_mmt/mmt_task1_test2016.tar.gz']
80     name = 'multi30k'
81     dirname = ''
82 
83     @classmethod
84     def splits(cls, exts, fields, root='.data',
85                train='train', validation='val', test='test2016', **kwargs):
86         """Create dataset objects for splits of the Multi30k dataset.
87 
88         Arguments:
89 
90             root: Root dataset storage directory. Default is '.data'.
91             exts: A tuple containing the extension to path for each language.
92             fields: A tuple containing the fields that will be used for data
93                 in each language.
94             train: The prefix of the train data. Default: 'train'.
95             validation: The prefix of the validation data. Default: 'val'.
96             test: The prefix of the test data. Default: 'test'.
97             Remaining keyword arguments: Passed to the splits method of
98                 Dataset.
99         """
100         path = os.path.join('data', cls.name)
101         return super(Multi30k, cls).splits(
102             exts, fields, path, root, train, validation, test, **kwargs)
103 
104 
105 class IWSLT(TranslationDataset):
106     """The IWSLT 2016 TED talk translation task"""
107 
108     base_url = 'https://wit3.fbk.eu/archive/2016-01//texts/{}/{}/{}.tgz'
109     name = 'iwslt'
110     base_dirname = '{}-{}'
111 
112     @classmethod
113     def splits(cls, exts, fields, root='.data',
114                train='train', validation='IWSLT16.TED.tst2013',
115                test='IWSLT16.TED.tst2014', **kwargs):
116         """Create dataset objects for splits of the IWSLT dataset.
117 
118         Arguments:
119 
120             root: Root dataset storage directory. Default is '.data'.
121             exts: A tuple containing the extension to path for each language.
122             fields: A tuple containing the fields that will be used for data
123                 in each language.
124             train: The prefix of the train data. Default: 'train'.
125             validation: The prefix of the validation data. Default: 'val'.
126             test: The prefix of the test data. Default: 'test'.
127             Remaining keyword arguments: Passed to the splits method of
128                 Dataset.
129         """
130         cls.dirname = cls.base_dirname.format(exts[0][1:], exts[1][1:])
131         cls.urls = [cls.base_url.format(exts[0][1:], exts[1][1:], cls.dirname)]
132         check = os.path.join(root, cls.name, cls.dirname)
133         path = cls.download(root, check=check)
134 
135         train = '.'.join([train, cls.dirname])
136         validation = '.'.join([validation, cls.dirname])
137         if test is not None:
138             test = '.'.join([test, cls.dirname])
139 
140         if not os.path.exists(os.path.join(path, train) + exts[0]):
141             cls.clean(path)
142 
143         train_data = None if train is None else cls(
144             os.path.join(path, train), exts, fields, **kwargs)
145         val_data = None if validation is None else cls(
146             os.path.join(path, validation), exts, fields, **kwargs)
147         test_data = None if test is None else cls(
148             os.path.join(path, test), exts, fields, **kwargs)
149         return tuple(d for d in (train_data, val_data, test_data)
150                      if d is not None)
151 
152     @staticmethod
153     def clean(path):
154         for f_xml in glob.iglob(os.path.join(path, '*.xml')):
155             print(f_xml)
156             f_txt = os.path.splitext(f_xml)[0]
157             with io.open(f_txt, mode='w', encoding='utf-8') as fd_txt:
158                 root = ET.parse(f_xml).getroot()[0]
159                 for doc in root.findall('doc'):
160                     for e in doc.findall('seg'):
161                         fd_txt.write(e.text.strip() + '\n')
162 
163         xml_tags = ['<url', '<keywords', '<talkid', '<description',
164                     '<reviewer', '<translator', '<title', '<speaker']
165         for f_orig in glob.iglob(os.path.join(path, 'train.tags*')):
166             print(f_orig)
167             f_txt = f_orig.replace('.tags', '')
168             with io.open(f_txt, mode='w', encoding='utf-8') as fd_txt, \
169                     io.open(f_orig, mode='r', encoding='utf-8') as fd_orig:
170                 for l in fd_orig:
171                     if not any(tag in l for tag in xml_tags):
172                         fd_txt.write(l.strip() + '\n')
173 
174 
175 class WMT14(TranslationDataset):
176     """The WMT 2014 English-German dataset, as preprocessed by Google Brain.
177 
178     Though this download contains test sets from 2015 and 2016, the train set
179     differs slightly from WMT 2015 and 2016 and significantly from WMT 2017."""
180 
181     urls = [('https://drive.google.com/uc?export=download&'
182              'id=0B_bZck-ksdkpM25jRUN2X2UxMm8', 'wmt16_en_de.tar.gz')]
183     name = 'wmt14'
184     dirname = ''
185 
186     @classmethod
187     def splits(cls, exts, fields, root='.data',
188                train='train.tok.clean.bpe.32000',
189                validation='newstest2013.tok.bpe.32000',
190                test='newstest2014.tok.bpe.32000', **kwargs):
191         """Create dataset objects for splits of the WMT 2014 dataset.
192 
193         Arguments:
194 
195             root: Root dataset storage directory. Default is '.data'.
196             exts: A tuple containing the extensions for each language. Must be
197                 either ('.en', '.de') or the reverse.
198             fields: A tuple containing the fields that will be used for data
199                 in each language.
200             train: The prefix of the train data. Default:
201                 'train.tok.clean.bpe.32000'.
202             validation: The prefix of the validation data. Default:
203                 'newstest2013.tok.bpe.32000'.
204             test: The prefix of the test data. Default:
205                 'newstest2014.tok.bpe.32000'.
206             Remaining keyword arguments: Passed to the splits method of
207                 Dataset.
208         """
209         path = os.path.join('data', cls.name)
210         return super(WMT14, cls).splits(
211             exts, fields, path, root, train, validation, test, **kwargs)
212
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/torchtext/datasets/translation.py b/torchtext/datasets/translation.py
--- a/torchtext/datasets/translation.py
+++ b/torchtext/datasets/translation.py
@@ -97,7 +97,9 @@
             Remaining keyword arguments: Passed to the splits method of
                 Dataset.
         """
-        path = os.path.join('data', cls.name)
+        expected_folder = os.path.join(root, cls.name)
+        path = expected_folder if os.path.exists(expected_folder) else None
+
         return super(Multi30k, cls).splits(
             exts, fields, path, root, train, validation, test, **kwargs)
 
@@ -206,6 +208,8 @@
             Remaining keyword arguments: Passed to the splits method of
                 Dataset.
         """
-        path = os.path.join('data', cls.name)
+        expected_folder = os.path.join(root, cls.name)
+        path = expected_folder if os.path.exists(expected_folder) else None
+
         return super(WMT14, cls).splits(
             exts, fields, path, root, train, validation, test, **kwargs)

{"golden_diff": "diff --git a/torchtext/datasets/translation.py b/torchtext/datasets/translation.py\n--- a/torchtext/datasets/translation.py\n+++ b/torchtext/datasets/translation.py\n@@ -97,7 +97,9 @@\n             Remaining keyword arguments: Passed to the splits method of\n                 Dataset.\n         \"\"\"\n-        path = os.path.join('data', cls.name)\n+        expected_folder = os.path.join(root, cls.name)\n+        path = expected_folder if os.path.exists(expected_folder) else None\n+\n         return super(Multi30k, cls).splits(\n             exts, fields, path, root, train, validation, test, **kwargs)\n \n@@ -206,6 +208,8 @@\n             Remaining keyword arguments: Passed to the splits method of\n                 Dataset.\n         \"\"\"\n-        path = os.path.join('data', cls.name)\n+        expected_folder = os.path.join(root, cls.name)\n+        path = expected_folder if os.path.exists(expected_folder) else None\n+\n         return super(WMT14, cls).splits(\n             exts, fields, path, root, train, validation, test, **kwargs)\n", "issue": "Translation splits error when not downloading dataset first\nThanks @AngusMonroe for finding this! The problem is that the absence of dataset is not addressed when creating splits. Minimal example:\r\n\r\n```\r\n\r\nfrom torchtext.datasets import Multi30k\r\nfrom torchtext.data import Field\r\nEN = Field()\r\nDE = Field()\r\nds = Multi30k.splits(('.de','.en'),[('de',DE),('en',EN)],'data/multi30k')\r\n```\r\n\n", "before_files": [{"content": "import os\nimport xml.etree.ElementTree as ET\nimport glob\nimport io\n\nfrom .. import data\n\n\nclass TranslationDataset(data.Dataset):\n    \"\"\"Defines a dataset for machine translation.\"\"\"\n\n    @staticmethod\n    def sort_key(ex):\n        return data.interleave_keys(len(ex.src), len(ex.trg))\n\n    def __init__(self, path, exts, fields, **kwargs):\n        \"\"\"Create a TranslationDataset given paths and fields.\n\n        Arguments:\n            path: Common prefix of paths to the data files for both languages.\n            exts: A tuple containing the extension to path for each language.\n            fields: A tuple containing the fields that will be used for data\n                in each language.\n            Remaining keyword arguments: Passed to the constructor of\n                data.Dataset.\n        \"\"\"\n        if not isinstance(fields[0], (tuple, list)):\n            fields = [('src', fields[0]), ('trg', fields[1])]\n\n        src_path, trg_path = tuple(os.path.expanduser(path + x) for x in exts)\n\n        examples = []\n        with open(src_path) as src_file, open(trg_path) as trg_file:\n            for src_line, trg_line in zip(src_file, trg_file):\n                src_line, trg_line = src_line.strip(), trg_line.strip()\n                if src_line != '' and trg_line != '':\n                    examples.append(data.Example.fromlist(\n                        [src_line, trg_line], fields))\n\n        super(TranslationDataset, self).__init__(examples, fields, **kwargs)\n\n    @classmethod\n    def splits(cls, exts, fields, path=None, root='.data',\n               train='train', validation='val', test='test', **kwargs):\n        \"\"\"Create dataset objects for splits of a TranslationDataset.\n\n        Arguments:\n            path (str): Common prefix of the splits' file paths, or None to use\n                the result of cls.download(root).\n            root: Root dataset storage directory. Default is '.data'.\n            exts: A tuple containing the extension to path for each language.\n            fields: A tuple containing the fields that will be used for data\n                in each language.\n            train: The prefix of the train data. Default: 'train'.\n            validation: The prefix of the validation data. Default: 'val'.\n            test: The prefix of the test data. Default: 'test'.\n            Remaining keyword arguments: Passed to the splits method of\n                Dataset.\n        \"\"\"\n        if path is None:\n            path = cls.download(root)\n\n        train_data = None if train is None else cls(\n            os.path.join(path, train), exts, fields, **kwargs)\n        val_data = None if validation is None else cls(\n            os.path.join(path, validation), exts, fields, **kwargs)\n        test_data = None if test is None else cls(\n            os.path.join(path, test), exts, fields, **kwargs)\n        return tuple(d for d in (train_data, val_data, test_data)\n                     if d is not None)\n\n\nclass Multi30k(TranslationDataset):\n    \"\"\"The small-dataset WMT 2016 multimodal task, also known as Flickr30k\"\"\"\n\n    urls = ['http://www.quest.dcs.shef.ac.uk/wmt16_files_mmt/training.tar.gz',\n            'http://www.quest.dcs.shef.ac.uk/wmt16_files_mmt/validation.tar.gz',\n            'http://www.quest.dcs.shef.ac.uk/'\n            'wmt17_files_mmt/mmt_task1_test2016.tar.gz']\n    name = 'multi30k'\n    dirname = ''\n\n    @classmethod\n    def splits(cls, exts, fields, root='.data',\n               train='train', validation='val', test='test2016', **kwargs):\n        \"\"\"Create dataset objects for splits of the Multi30k dataset.\n\n        Arguments:\n\n            root: Root dataset storage directory. Default is '.data'.\n            exts: A tuple containing the extension to path for each language.\n            fields: A tuple containing the fields that will be used for data\n                in each language.\n            train: The prefix of the train data. Default: 'train'.\n            validation: The prefix of the validation data. Default: 'val'.\n            test: The prefix of the test data. Default: 'test'.\n            Remaining keyword arguments: Passed to the splits method of\n                Dataset.\n        \"\"\"\n        path = os.path.join('data', cls.name)\n        return super(Multi30k, cls).splits(\n            exts, fields, path, root, train, validation, test, **kwargs)\n\n\nclass IWSLT(TranslationDataset):\n    \"\"\"The IWSLT 2016 TED talk translation task\"\"\"\n\n    base_url = 'https://wit3.fbk.eu/archive/2016-01//texts/{}/{}/{}.tgz'\n    name = 'iwslt'\n    base_dirname = '{}-{}'\n\n    @classmethod\n    def splits(cls, exts, fields, root='.data',\n               train='train', validation='IWSLT16.TED.tst2013',\n               test='IWSLT16.TED.tst2014', **kwargs):\n        \"\"\"Create dataset objects for splits of the IWSLT dataset.\n\n        Arguments:\n\n            root: Root dataset storage directory. Default is '.data'.\n            exts: A tuple containing the extension to path for each language.\n            fields: A tuple containing the fields that will be used for data\n                in each language.\n            train: The prefix of the train data. Default: 'train'.\n            validation: The prefix of the validation data. Default: 'val'.\n            test: The prefix of the test data. Default: 'test'.\n            Remaining keyword arguments: Passed to the splits method of\n                Dataset.\n        \"\"\"\n        cls.dirname = cls.base_dirname.format(exts[0][1:], exts[1][1:])\n        cls.urls = [cls.base_url.format(exts[0][1:], exts[1][1:], cls.dirname)]\n        check = os.path.join(root, cls.name, cls.dirname)\n        path = cls.download(root, check=check)\n\n        train = '.'.join([train, cls.dirname])\n        validation = '.'.join([validation, cls.dirname])\n        if test is not None:\n            test = '.'.join([test, cls.dirname])\n\n        if not os.path.exists(os.path.join(path, train) + exts[0]):\n            cls.clean(path)\n\n        train_data = None if train is None else cls(\n            os.path.join(path, train), exts, fields, **kwargs)\n        val_data = None if validation is None else cls(\n            os.path.join(path, validation), exts, fields, **kwargs)\n        test_data = None if test is None else cls(\n            os.path.join(path, test), exts, fields, **kwargs)\n        return tuple(d for d in (train_data, val_data, test_data)\n                     if d is not None)\n\n    @staticmethod\n    def clean(path):\n        for f_xml in glob.iglob(os.path.join(path, '*.xml')):\n            print(f_xml)\n            f_txt = os.path.splitext(f_xml)[0]\n            with io.open(f_txt, mode='w', encoding='utf-8') as fd_txt:\n                root = ET.parse(f_xml).getroot()[0]\n                for doc in root.findall('doc'):\n                    for e in doc.findall('seg'):\n                        fd_txt.write(e.text.strip() + '\\n')\n\n        xml_tags = ['<url', '<keywords', '<talkid', '<description',\n                    '<reviewer', '<translator', '<title', '<speaker']\n        for f_orig in glob.iglob(os.path.join(path, 'train.tags*')):\n            print(f_orig)\n            f_txt = f_orig.replace('.tags', '')\n            with io.open(f_txt, mode='w', encoding='utf-8') as fd_txt, \\\n                    io.open(f_orig, mode='r', encoding='utf-8') as fd_orig:\n                for l in fd_orig:\n                    if not any(tag in l for tag in xml_tags):\n                        fd_txt.write(l.strip() + '\\n')\n\n\nclass WMT14(TranslationDataset):\n    \"\"\"The WMT 2014 English-German dataset, as preprocessed by Google Brain.\n\n    Though this download contains test sets from 2015 and 2016, the train set\n    differs slightly from WMT 2015 and 2016 and significantly from WMT 2017.\"\"\"\n\n    urls = [('https://drive.google.com/uc?export=download&'\n             'id=0B_bZck-ksdkpM25jRUN2X2UxMm8', 'wmt16_en_de.tar.gz')]\n    name = 'wmt14'\n    dirname = ''\n\n    @classmethod\n    def splits(cls, exts, fields, root='.data',\n               train='train.tok.clean.bpe.32000',\n               validation='newstest2013.tok.bpe.32000',\n               test='newstest2014.tok.bpe.32000', **kwargs):\n        \"\"\"Create dataset objects for splits of the WMT 2014 dataset.\n\n        Arguments:\n\n            root: Root dataset storage directory. Default is '.data'.\n            exts: A tuple containing the extensions for each language. Must be\n                either ('.en', '.de') or the reverse.\n            fields: A tuple containing the fields that will be used for data\n                in each language.\n            train: The prefix of the train data. Default:\n                'train.tok.clean.bpe.32000'.\n            validation: The prefix of the validation data. Default:\n                'newstest2013.tok.bpe.32000'.\n            test: The prefix of the test data. Default:\n                'newstest2014.tok.bpe.32000'.\n            Remaining keyword arguments: Passed to the splits method of\n                Dataset.\n        \"\"\"\n        path = os.path.join('data', cls.name)\n        return super(WMT14, cls).splits(\n            exts, fields, path, root, train, validation, test, **kwargs)\n", "path": "torchtext/datasets/translation.py"}], "after_files": [{"content": "import os\nimport xml.etree.ElementTree as ET\nimport glob\nimport io\n\nfrom .. import data\n\n\nclass TranslationDataset(data.Dataset):\n    \"\"\"Defines a dataset for machine translation.\"\"\"\n\n    @staticmethod\n    def sort_key(ex):\n        return data.interleave_keys(len(ex.src), len(ex.trg))\n\n    def __init__(self, path, exts, fields, **kwargs):\n        \"\"\"Create a TranslationDataset given paths and fields.\n\n        Arguments:\n            path: Common prefix of paths to the data files for both languages.\n            exts: A tuple containing the extension to path for each language.\n            fields: A tuple containing the fields that will be used for data\n                in each language.\n            Remaining keyword arguments: Passed to the constructor of\n                data.Dataset.\n        \"\"\"\n        if not isinstance(fields[0], (tuple, list)):\n            fields = [('src', fields[0]), ('trg', fields[1])]\n\n        src_path, trg_path = tuple(os.path.expanduser(path + x) for x in exts)\n\n        examples = []\n        with open(src_path) as src_file, open(trg_path) as trg_file:\n            for src_line, trg_line in zip(src_file, trg_file):\n                src_line, trg_line = src_line.strip(), trg_line.strip()\n                if src_line != '' and trg_line != '':\n                    examples.append(data.Example.fromlist(\n                        [src_line, trg_line], fields))\n\n        super(TranslationDataset, self).__init__(examples, fields, **kwargs)\n\n    @classmethod\n    def splits(cls, exts, fields, path=None, root='.data',\n               train='train', validation='val', test='test', **kwargs):\n        \"\"\"Create dataset objects for splits of a TranslationDataset.\n\n        Arguments:\n            path (str): Common prefix of the splits' file paths, or None to use\n                the result of cls.download(root).\n            root: Root dataset storage directory. Default is '.data'.\n            exts: A tuple containing the extension to path for each language.\n            fields: A tuple containing the fields that will be used for data\n                in each language.\n            train: The prefix of the train data. Default: 'train'.\n            validation: The prefix of the validation data. Default: 'val'.\n            test: The prefix of the test data. Default: 'test'.\n            Remaining keyword arguments: Passed to the splits method of\n                Dataset.\n        \"\"\"\n        if path is None:\n            path = cls.download(root)\n\n        train_data = None if train is None else cls(\n            os.path.join(path, train), exts, fields, **kwargs)\n        val_data = None if validation is None else cls(\n            os.path.join(path, validation), exts, fields, **kwargs)\n        test_data = None if test is None else cls(\n            os.path.join(path, test), exts, fields, **kwargs)\n        return tuple(d for d in (train_data, val_data, test_data)\n                     if d is not None)\n\n\nclass Multi30k(TranslationDataset):\n    \"\"\"The small-dataset WMT 2016 multimodal task, also known as Flickr30k\"\"\"\n\n    urls = ['http://www.quest.dcs.shef.ac.uk/wmt16_files_mmt/training.tar.gz',\n            'http://www.quest.dcs.shef.ac.uk/wmt16_files_mmt/validation.tar.gz',\n            'http://www.quest.dcs.shef.ac.uk/'\n            'wmt17_files_mmt/mmt_task1_test2016.tar.gz']\n    name = 'multi30k'\n    dirname = ''\n\n    @classmethod\n    def splits(cls, exts, fields, root='.data',\n               train='train', validation='val', test='test2016', **kwargs):\n        \"\"\"Create dataset objects for splits of the Multi30k dataset.\n\n        Arguments:\n\n            root: Root dataset storage directory. Default is '.data'.\n            exts: A tuple containing the extension to path for each language.\n            fields: A tuple containing the fields that will be used for data\n                in each language.\n            train: The prefix of the train data. Default: 'train'.\n            validation: The prefix of the validation data. Default: 'val'.\n            test: The prefix of the test data. Default: 'test'.\n            Remaining keyword arguments: Passed to the splits method of\n                Dataset.\n        \"\"\"\n        expected_folder = os.path.join(root, cls.name)\n        path = expected_folder if os.path.exists(expected_folder) else None\n\n        return super(Multi30k, cls).splits(\n            exts, fields, path, root, train, validation, test, **kwargs)\n\n\nclass IWSLT(TranslationDataset):\n    \"\"\"The IWSLT 2016 TED talk translation task\"\"\"\n\n    base_url = 'https://wit3.fbk.eu/archive/2016-01//texts/{}/{}/{}.tgz'\n    name = 'iwslt'\n    base_dirname = '{}-{}'\n\n    @classmethod\n    def splits(cls, exts, fields, root='.data',\n               train='train', validation='IWSLT16.TED.tst2013',\n               test='IWSLT16.TED.tst2014', **kwargs):\n        \"\"\"Create dataset objects for splits of the IWSLT dataset.\n\n        Arguments:\n\n            root: Root dataset storage directory. Default is '.data'.\n            exts: A tuple containing the extension to path for each language.\n            fields: A tuple containing the fields that will be used for data\n                in each language.\n            train: The prefix of the train data. Default: 'train'.\n            validation: The prefix of the validation data. Default: 'val'.\n            test: The prefix of the test data. Default: 'test'.\n            Remaining keyword arguments: Passed to the splits method of\n                Dataset.\n        \"\"\"\n        cls.dirname = cls.base_dirname.format(exts[0][1:], exts[1][1:])\n        cls.urls = [cls.base_url.format(exts[0][1:], exts[1][1:], cls.dirname)]\n        check = os.path.join(root, cls.name, cls.dirname)\n        path = cls.download(root, check=check)\n\n        train = '.'.join([train, cls.dirname])\n        validation = '.'.join([validation, cls.dirname])\n        if test is not None:\n            test = '.'.join([test, cls.dirname])\n\n        if not os.path.exists(os.path.join(path, train) + exts[0]):\n            cls.clean(path)\n\n        train_data = None if train is None else cls(\n            os.path.join(path, train), exts, fields, **kwargs)\n        val_data = None if validation is None else cls(\n            os.path.join(path, validation), exts, fields, **kwargs)\n        test_data = None if test is None else cls(\n            os.path.join(path, test), exts, fields, **kwargs)\n        return tuple(d for d in (train_data, val_data, test_data)\n                     if d is not None)\n\n    @staticmethod\n    def clean(path):\n        for f_xml in glob.iglob(os.path.join(path, '*.xml')):\n            print(f_xml)\n            f_txt = os.path.splitext(f_xml)[0]\n            with io.open(f_txt, mode='w', encoding='utf-8') as fd_txt:\n                root = ET.parse(f_xml).getroot()[0]\n                for doc in root.findall('doc'):\n                    for e in doc.findall('seg'):\n                        fd_txt.write(e.text.strip() + '\\n')\n\n        xml_tags = ['<url', '<keywords', '<talkid', '<description',\n                    '<reviewer', '<translator', '<title', '<speaker']\n        for f_orig in glob.iglob(os.path.join(path, 'train.tags*')):\n            print(f_orig)\n            f_txt = f_orig.replace('.tags', '')\n            with io.open(f_txt, mode='w', encoding='utf-8') as fd_txt, \\\n                    io.open(f_orig, mode='r', encoding='utf-8') as fd_orig:\n                for l in fd_orig:\n                    if not any(tag in l for tag in xml_tags):\n                        fd_txt.write(l.strip() + '\\n')\n\n\nclass WMT14(TranslationDataset):\n    \"\"\"The WMT 2014 English-German dataset, as preprocessed by Google Brain.\n\n    Though this download contains test sets from 2015 and 2016, the train set\n    differs slightly from WMT 2015 and 2016 and significantly from WMT 2017.\"\"\"\n\n    urls = [('https://drive.google.com/uc?export=download&'\n             'id=0B_bZck-ksdkpM25jRUN2X2UxMm8', 'wmt16_en_de.tar.gz')]\n    name = 'wmt14'\n    dirname = ''\n\n    @classmethod\n    def splits(cls, exts, fields, root='.data',\n               train='train.tok.clean.bpe.32000',\n               validation='newstest2013.tok.bpe.32000',\n               test='newstest2014.tok.bpe.32000', **kwargs):\n        \"\"\"Create dataset objects for splits of the WMT 2014 dataset.\n\n        Arguments:\n\n            root: Root dataset storage directory. Default is '.data'.\n            exts: A tuple containing the extensions for each language. Must be\n                either ('.en', '.de') or the reverse.\n            fields: A tuple containing the fields that will be used for data\n                in each language.\n            train: The prefix of the train data. Default:\n                'train.tok.clean.bpe.32000'.\n            validation: The prefix of the validation data. Default:\n                'newstest2013.tok.bpe.32000'.\n            test: The prefix of the test data. Default:\n                'newstest2014.tok.bpe.32000'.\n            Remaining keyword arguments: Passed to the splits method of\n                Dataset.\n        \"\"\"\n        expected_folder = os.path.join(root, cls.name)\n        path = expected_folder if os.path.exists(expected_folder) else None\n\n        return super(WMT14, cls).splits(\n            exts, fields, path, root, train, validation, test, **kwargs)\n", "path": "torchtext/datasets/translation.py"}]}