We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
multi30k dataset url not avaliable


--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `mindnlp/dataset/machine_translation/multi30k.py`
Content:
```
1 # Copyright 2022 Huawei Technologies Co., Ltd
2 #
3 # Licensed under the Apache License, Version 2.0 (the "License");
4 # you may not use this file except in compliance with the License.
5 # You may obtain a copy of the License at
6 #
7 # http://www.apache.org/licenses/LICENSE-2.0
8 #
9 # Unless required by applicable law or agreed to in writing, software
10 # distributed under the License is distributed on an "AS IS" BASIS,
11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 # See the License for the specific language governing permissions and
13 # limitations under the License.
14 # ============================================================================
15 """
16 Multi30k load function
17 """
18 # pylint: disable=C0103
19 
20 import os
21 import re
22 from operator import itemgetter
23 from typing import Union, Tuple
24 from mindspore.dataset import TextFileDataset, transforms
25 from mindspore.dataset import text
26 from mindnlp.utils.download import cache_file
27 from mindnlp.dataset.register import load_dataset, process
28 from mindnlp.configs import DEFAULT_ROOT
29 from mindnlp.utils import untar
30 
31 URL = {
32     "train": "http://www.quest.dcs.shef.ac.uk/wmt16_files_mmt/training.tar.gz",
33     "valid": "http://www.quest.dcs.shef.ac.uk/wmt16_files_mmt/validation.tar.gz",
34     "test": "http://www.quest.dcs.shef.ac.uk/wmt16_files_mmt/mmt16_task1_test.tar.gz",
35 }
36 
37 MD5 = {
38     "train": "8ebce33f4ebeb71dcb617f62cba077b7",
39     "valid": "2a46f18dbae0df0becc56e33d4e28e5d",
40     "test": "f63b12fc6f95beb3bfca2c393e861063",
41 }
42 
43 
44 @load_dataset.register
45 def Multi30k(root: str = DEFAULT_ROOT, split: Union[Tuple[str], str] = ('train', 'valid', 'test'),
46              language_pair: Tuple[str] = ('de', 'en'), proxies=None):
47     r"""
48     Load the Multi30k dataset
49 
50     Args:
51         root (str): Directory where the datasets are saved.
52             Default:~/.mindnlp
53         split (str|Tuple[str]): Split or splits to be returned.
54             Default:('train', 'valid', 'test').
55         language_pair (Tuple[str]): Tuple containing src and tgt language.
56             Default: ('de', 'en').
57         proxies (dict): a dict to identify proxies,for example: {"https": "https://127.0.0.1:7890"}.
58 
59     Returns:
60         - **datasets_list** (list) -A list of loaded datasets.
61           If only one type of dataset is specified,such as 'trian',
62           this dataset is returned instead of a list of datasets.
63 
64     Raises:
65         TypeError: If `root` is not a string.
66         TypeError: If `split` is not a string or Tuple[str].
67         TypeError: If `language_pair` is not a Tuple[str].
68         RuntimeError: If the length of `language_pair` is not 2.
69         RuntimeError: If `language_pair` is neither ('de', 'en') nor ('en', 'de').
70 
71     Examples:
72         >>> root = os.path.join(os.path.expanduser('~'), ".mindnlp")
73         >>> split = ('train', 'valid', 'test')
74         >>> language_pair = ('de', 'en')
75         >>> dataset_train, dataset_valid, dataset_test = Multi30k(root, split, language_pair)
76         >>> train_iter = dataset_train.create_tuple_iterator()
77         >>> print(next(train_iter))
78         [Tensor(shape=[], dtype=String, value=\
79             'Ein Mann mit einem orangefarbenen Hut, der etwas anstarrt.'),
80         Tensor(shape=[], dtype=String, value= 'A man in an orange hat starring at something.')]
81 
82     """
83 
84     assert len(
85         language_pair) == 2, "language_pair must contain only 2 elements:\
86             src and tgt language respectively"
87     assert tuple(sorted(language_pair)) == (
88         "de",
89         "en",
90     ), "language_pair must be either ('de','en') or ('en', 'de')"
91 
92     if root == DEFAULT_ROOT:
93         cache_dir = os.path.join(root, "datasets", "Multi30k")
94     else:
95         cache_dir = root
96 
97     file_list = []
98 
99     untar_files = []
100     source_files = []
101     target_files = []
102 
103     datasets_list = []
104 
105     if isinstance(split, str):
106         file_path, _ = cache_file(
107             None, cache_dir=cache_dir, url=URL[split], md5sum=MD5[split], proxies=proxies)
108         file_list.append(file_path)
109 
110     else:
111         urls = itemgetter(*split)(URL)
112         md5s = itemgetter(*split)(MD5)
113         for i, url in enumerate(urls):
114             file_path, _ = cache_file(
115                 None, cache_dir=cache_dir, url=url, md5sum=md5s[i], proxies=proxies)
116             file_list.append(file_path)
117 
118     for file in file_list:
119         untar_files.append(untar(file, os.path.dirname(file)))
120 
121     regexp = r".de"
122     if language_pair == ("en", "de"):
123         regexp = r".en"
124 
125     for file_pair in untar_files:
126         for file in file_pair:
127             match = re.search(regexp, file)
128             if match:
129                 source_files.append(file)
130             else:
131                 target_files.append(file)
132 
133     for i in range(len(untar_files)):
134         source_dataset = TextFileDataset(
135             os.path.join(cache_dir, source_files[i]), shuffle=False)
136         source_dataset = source_dataset.rename(["text"], [language_pair[0]])
137         target_dataset = TextFileDataset(
138             os.path.join(cache_dir, target_files[i]), shuffle=False)
139         target_dataset = target_dataset.rename(["text"], [language_pair[1]])
140         datasets = source_dataset.zip(target_dataset)
141         datasets_list.append(datasets)
142 
143     if len(datasets_list) == 1:
144         return datasets_list[0]
145     return datasets_list
146 
147 @process.register
148 def Multi30k_Process(dataset, vocab, batch_size=64, max_len=500, \
149                 drop_remainder=False):
150     """
151     the process of the Multi30k dataset
152 
153     Args:
154         dataset (GeneratorDataset): Multi30k dataset.
155         vocab (Vocab): vocabulary object, used to store the mapping of token and index.
156         batch_size (int): The number of rows each batch is created with. Default: 64.
157         max_len (int): The max length of the sentence. Default: 500.
158         drop_remainder (bool): When the last batch of data contains a data entry smaller than batch_size, whether
159             to discard the batch and not pass it to the next operation. Default: False.
160 
161     Returns:
162         - **dataset** (MapDataset) - dataset after transforms.
163 
164     Raises:
165         TypeError: If `input_column` is not a string.
166 
167     Examples:
168         >>> train_dataset = Multi30k(
169         >>>     root=self.root,
170         >>>     split="train",
171         >>>     language_pair=("de", "en")
172         >>> )
173         >>> tokenizer = BasicTokenizer(True)
174         >>> train_dataset = train_dataset.map([tokenizer], 'en')
175         >>> train_dataset = train_dataset.map([tokenizer], 'de')
176         >>> en_vocab = text.Vocab.from_dataset(train_dataset, 'en', special_tokens=
177         >>>   ['<pad>', '<unk>'], special_first= True)
178         >>> de_vocab = text.Vocab.from_dataset(train_dataset, 'de', special_tokens=
179         >>>   ['<pad>', '<unk>'], special_first= True)
180         >>> vocab = {'en':en_vocab, 'de':de_vocab}
181         >>> train_dataset = process('Multi30k', train_dataset, vocab = vocab)
182     """
183 
184     en_pad_value = vocab['en'].tokens_to_ids('<pad>')
185     de_pad_value = vocab['de'].tokens_to_ids('<pad>')
186 
187     en_lookup_op = text.Lookup(vocab['en'], unknown_token='<unk>')
188     de_lookup_op = text.Lookup(vocab['de'], unknown_token='<unk>')
189 
190     dataset = dataset.map([en_lookup_op], 'en')
191     dataset = dataset.map([de_lookup_op], 'de')
192 
193     en_pad_op = transforms.PadEnd([max_len], en_pad_value)
194     de_pad_op = transforms.PadEnd([max_len], de_pad_value)
195 
196     dataset = dataset.map([en_pad_op], 'en')
197     dataset = dataset.map([de_pad_op], 'de')
198 
199     dataset = dataset.batch(batch_size, drop_remainder=drop_remainder)
200     return dataset
201
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/mindnlp/dataset/machine_translation/multi30k.py b/mindnlp/dataset/machine_translation/multi30k.py
--- a/mindnlp/dataset/machine_translation/multi30k.py
+++ b/mindnlp/dataset/machine_translation/multi30k.py
@@ -29,15 +29,15 @@
 from mindnlp.utils import untar
 
 URL = {
-    "train": "http://www.quest.dcs.shef.ac.uk/wmt16_files_mmt/training.tar.gz",
-    "valid": "http://www.quest.dcs.shef.ac.uk/wmt16_files_mmt/validation.tar.gz",
-    "test": "http://www.quest.dcs.shef.ac.uk/wmt16_files_mmt/mmt16_task1_test.tar.gz",
+    "train": "https://openi.pcl.ac.cn/lvyufeng/multi30k/raw/branch/master/training.tar.gz",
+    "valid": "https://openi.pcl.ac.cn/lvyufeng/multi30k/raw/branch/master/validation.tar.gz",
+    "test": "https://openi.pcl.ac.cn/lvyufeng/multi30k/raw/branch/master/mmt16_task1_test.tar.gz",
 }
 
 MD5 = {
     "train": "8ebce33f4ebeb71dcb617f62cba077b7",
     "valid": "2a46f18dbae0df0becc56e33d4e28e5d",
-    "test": "f63b12fc6f95beb3bfca2c393e861063",
+    "test": "1586ce11f70cba049e9ed3d64db08843",
 }

{"golden_diff": "diff --git a/mindnlp/dataset/machine_translation/multi30k.py b/mindnlp/dataset/machine_translation/multi30k.py\n--- a/mindnlp/dataset/machine_translation/multi30k.py\n+++ b/mindnlp/dataset/machine_translation/multi30k.py\n@@ -29,15 +29,15 @@\n from mindnlp.utils import untar\n \n URL = {\n-    \"train\": \"http://www.quest.dcs.shef.ac.uk/wmt16_files_mmt/training.tar.gz\",\n-    \"valid\": \"http://www.quest.dcs.shef.ac.uk/wmt16_files_mmt/validation.tar.gz\",\n-    \"test\": \"http://www.quest.dcs.shef.ac.uk/wmt16_files_mmt/mmt16_task1_test.tar.gz\",\n+    \"train\": \"https://openi.pcl.ac.cn/lvyufeng/multi30k/raw/branch/master/training.tar.gz\",\n+    \"valid\": \"https://openi.pcl.ac.cn/lvyufeng/multi30k/raw/branch/master/validation.tar.gz\",\n+    \"test\": \"https://openi.pcl.ac.cn/lvyufeng/multi30k/raw/branch/master/mmt16_task1_test.tar.gz\",\n }\n \n MD5 = {\n     \"train\": \"8ebce33f4ebeb71dcb617f62cba077b7\",\n     \"valid\": \"2a46f18dbae0df0becc56e33d4e28e5d\",\n-    \"test\": \"f63b12fc6f95beb3bfca2c393e861063\",\n+    \"test\": \"1586ce11f70cba049e9ed3d64db08843\",\n }\n", "issue": "multi30k dataset url not avaliable\n\n", "before_files": [{"content": "# Copyright 2022 Huawei Technologies Co., Ltd\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n# ============================================================================\n\"\"\"\nMulti30k load function\n\"\"\"\n# pylint: disable=C0103\n\nimport os\nimport re\nfrom operator import itemgetter\nfrom typing import Union, Tuple\nfrom mindspore.dataset import TextFileDataset, transforms\nfrom mindspore.dataset import text\nfrom mindnlp.utils.download import cache_file\nfrom mindnlp.dataset.register import load_dataset, process\nfrom mindnlp.configs import DEFAULT_ROOT\nfrom mindnlp.utils import untar\n\nURL = {\n    \"train\": \"http://www.quest.dcs.shef.ac.uk/wmt16_files_mmt/training.tar.gz\",\n    \"valid\": \"http://www.quest.dcs.shef.ac.uk/wmt16_files_mmt/validation.tar.gz\",\n    \"test\": \"http://www.quest.dcs.shef.ac.uk/wmt16_files_mmt/mmt16_task1_test.tar.gz\",\n}\n\nMD5 = {\n    \"train\": \"8ebce33f4ebeb71dcb617f62cba077b7\",\n    \"valid\": \"2a46f18dbae0df0becc56e33d4e28e5d\",\n    \"test\": \"f63b12fc6f95beb3bfca2c393e861063\",\n}\n\n\n@load_dataset.register\ndef Multi30k(root: str = DEFAULT_ROOT, split: Union[Tuple[str], str] = ('train', 'valid', 'test'),\n             language_pair: Tuple[str] = ('de', 'en'), proxies=None):\n    r\"\"\"\n    Load the Multi30k dataset\n\n    Args:\n        root (str): Directory where the datasets are saved.\n            Default:~/.mindnlp\n        split (str|Tuple[str]): Split or splits to be returned.\n            Default:('train', 'valid', 'test').\n        language_pair (Tuple[str]): Tuple containing src and tgt language.\n            Default: ('de', 'en').\n        proxies (dict): a dict to identify proxies,for example: {\"https\": \"https://127.0.0.1:7890\"}.\n\n    Returns:\n        - **datasets_list** (list) -A list of loaded datasets.\n          If only one type of dataset is specified,such as 'trian',\n          this dataset is returned instead of a list of datasets.\n\n    Raises:\n        TypeError: If `root` is not a string.\n        TypeError: If `split` is not a string or Tuple[str].\n        TypeError: If `language_pair` is not a Tuple[str].\n        RuntimeError: If the length of `language_pair` is not 2.\n        RuntimeError: If `language_pair` is neither ('de', 'en') nor ('en', 'de').\n\n    Examples:\n        >>> root = os.path.join(os.path.expanduser('~'), \".mindnlp\")\n        >>> split = ('train', 'valid', 'test')\n        >>> language_pair = ('de', 'en')\n        >>> dataset_train, dataset_valid, dataset_test = Multi30k(root, split, language_pair)\n        >>> train_iter = dataset_train.create_tuple_iterator()\n        >>> print(next(train_iter))\n        [Tensor(shape=[], dtype=String, value=\\\n            'Ein Mann mit einem orangefarbenen Hut, der etwas anstarrt.'),\n        Tensor(shape=[], dtype=String, value= 'A man in an orange hat starring at something.')]\n\n    \"\"\"\n\n    assert len(\n        language_pair) == 2, \"language_pair must contain only 2 elements:\\\n            src and tgt language respectively\"\n    assert tuple(sorted(language_pair)) == (\n        \"de\",\n        \"en\",\n    ), \"language_pair must be either ('de','en') or ('en', 'de')\"\n\n    if root == DEFAULT_ROOT:\n        cache_dir = os.path.join(root, \"datasets\", \"Multi30k\")\n    else:\n        cache_dir = root\n\n    file_list = []\n\n    untar_files = []\n    source_files = []\n    target_files = []\n\n    datasets_list = []\n\n    if isinstance(split, str):\n        file_path, _ = cache_file(\n            None, cache_dir=cache_dir, url=URL[split], md5sum=MD5[split], proxies=proxies)\n        file_list.append(file_path)\n\n    else:\n        urls = itemgetter(*split)(URL)\n        md5s = itemgetter(*split)(MD5)\n        for i, url in enumerate(urls):\n            file_path, _ = cache_file(\n                None, cache_dir=cache_dir, url=url, md5sum=md5s[i], proxies=proxies)\n            file_list.append(file_path)\n\n    for file in file_list:\n        untar_files.append(untar(file, os.path.dirname(file)))\n\n    regexp = r\".de\"\n    if language_pair == (\"en\", \"de\"):\n        regexp = r\".en\"\n\n    for file_pair in untar_files:\n        for file in file_pair:\n            match = re.search(regexp, file)\n            if match:\n                source_files.append(file)\n            else:\n                target_files.append(file)\n\n    for i in range(len(untar_files)):\n        source_dataset = TextFileDataset(\n            os.path.join(cache_dir, source_files[i]), shuffle=False)\n        source_dataset = source_dataset.rename([\"text\"], [language_pair[0]])\n        target_dataset = TextFileDataset(\n            os.path.join(cache_dir, target_files[i]), shuffle=False)\n        target_dataset = target_dataset.rename([\"text\"], [language_pair[1]])\n        datasets = source_dataset.zip(target_dataset)\n        datasets_list.append(datasets)\n\n    if len(datasets_list) == 1:\n        return datasets_list[0]\n    return datasets_list\n\n@process.register\ndef Multi30k_Process(dataset, vocab, batch_size=64, max_len=500, \\\n                drop_remainder=False):\n    \"\"\"\n    the process of the Multi30k dataset\n\n    Args:\n        dataset (GeneratorDataset): Multi30k dataset.\n        vocab (Vocab): vocabulary object, used to store the mapping of token and index.\n        batch_size (int): The number of rows each batch is created with. Default: 64.\n        max_len (int): The max length of the sentence. Default: 500.\n        drop_remainder (bool): When the last batch of data contains a data entry smaller than batch_size, whether\n            to discard the batch and not pass it to the next operation. Default: False.\n\n    Returns:\n        - **dataset** (MapDataset) - dataset after transforms.\n\n    Raises:\n        TypeError: If `input_column` is not a string.\n\n    Examples:\n        >>> train_dataset = Multi30k(\n        >>>     root=self.root,\n        >>>     split=\"train\",\n        >>>     language_pair=(\"de\", \"en\")\n        >>> )\n        >>> tokenizer = BasicTokenizer(True)\n        >>> train_dataset = train_dataset.map([tokenizer], 'en')\n        >>> train_dataset = train_dataset.map([tokenizer], 'de')\n        >>> en_vocab = text.Vocab.from_dataset(train_dataset, 'en', special_tokens=\n        >>>   ['<pad>', '<unk>'], special_first= True)\n        >>> de_vocab = text.Vocab.from_dataset(train_dataset, 'de', special_tokens=\n        >>>   ['<pad>', '<unk>'], special_first= True)\n        >>> vocab = {'en':en_vocab, 'de':de_vocab}\n        >>> train_dataset = process('Multi30k', train_dataset, vocab = vocab)\n    \"\"\"\n\n    en_pad_value = vocab['en'].tokens_to_ids('<pad>')\n    de_pad_value = vocab['de'].tokens_to_ids('<pad>')\n\n    en_lookup_op = text.Lookup(vocab['en'], unknown_token='<unk>')\n    de_lookup_op = text.Lookup(vocab['de'], unknown_token='<unk>')\n\n    dataset = dataset.map([en_lookup_op], 'en')\n    dataset = dataset.map([de_lookup_op], 'de')\n\n    en_pad_op = transforms.PadEnd([max_len], en_pad_value)\n    de_pad_op = transforms.PadEnd([max_len], de_pad_value)\n\n    dataset = dataset.map([en_pad_op], 'en')\n    dataset = dataset.map([de_pad_op], 'de')\n\n    dataset = dataset.batch(batch_size, drop_remainder=drop_remainder)\n    return dataset\n", "path": "mindnlp/dataset/machine_translation/multi30k.py"}], "after_files": [{"content": "# Copyright 2022 Huawei Technologies Co., Ltd\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n# ============================================================================\n\"\"\"\nMulti30k load function\n\"\"\"\n# pylint: disable=C0103\n\nimport os\nimport re\nfrom operator import itemgetter\nfrom typing import Union, Tuple\nfrom mindspore.dataset import TextFileDataset, transforms\nfrom mindspore.dataset import text\nfrom mindnlp.utils.download import cache_file\nfrom mindnlp.dataset.register import load_dataset, process\nfrom mindnlp.configs import DEFAULT_ROOT\nfrom mindnlp.utils import untar\n\nURL = {\n    \"train\": \"https://openi.pcl.ac.cn/lvyufeng/multi30k/raw/branch/master/training.tar.gz\",\n    \"valid\": \"https://openi.pcl.ac.cn/lvyufeng/multi30k/raw/branch/master/validation.tar.gz\",\n    \"test\": \"https://openi.pcl.ac.cn/lvyufeng/multi30k/raw/branch/master/mmt16_task1_test.tar.gz\",\n}\n\nMD5 = {\n    \"train\": \"8ebce33f4ebeb71dcb617f62cba077b7\",\n    \"valid\": \"2a46f18dbae0df0becc56e33d4e28e5d\",\n    \"test\": \"1586ce11f70cba049e9ed3d64db08843\",\n}\n\n\n@load_dataset.register\ndef Multi30k(root: str = DEFAULT_ROOT, split: Union[Tuple[str], str] = ('train', 'valid', 'test'),\n             language_pair: Tuple[str] = ('de', 'en'), proxies=None):\n    r\"\"\"\n    Load the Multi30k dataset\n\n    Args:\n        root (str): Directory where the datasets are saved.\n            Default:~/.mindnlp\n        split (str|Tuple[str]): Split or splits to be returned.\n            Default:('train', 'valid', 'test').\n        language_pair (Tuple[str]): Tuple containing src and tgt language.\n            Default: ('de', 'en').\n        proxies (dict): a dict to identify proxies,for example: {\"https\": \"https://127.0.0.1:7890\"}.\n\n    Returns:\n        - **datasets_list** (list) -A list of loaded datasets.\n          If only one type of dataset is specified,such as 'trian',\n          this dataset is returned instead of a list of datasets.\n\n    Raises:\n        TypeError: If `root` is not a string.\n        TypeError: If `split` is not a string or Tuple[str].\n        TypeError: If `language_pair` is not a Tuple[str].\n        RuntimeError: If the length of `language_pair` is not 2.\n        RuntimeError: If `language_pair` is neither ('de', 'en') nor ('en', 'de').\n\n    Examples:\n        >>> root = os.path.join(os.path.expanduser('~'), \".mindnlp\")\n        >>> split = ('train', 'valid', 'test')\n        >>> language_pair = ('de', 'en')\n        >>> dataset_train, dataset_valid, dataset_test = Multi30k(root, split, language_pair)\n        >>> train_iter = dataset_train.create_tuple_iterator()\n        >>> print(next(train_iter))\n        [Tensor(shape=[], dtype=String, value=\\\n            'Ein Mann mit einem orangefarbenen Hut, der etwas anstarrt.'),\n        Tensor(shape=[], dtype=String, value= 'A man in an orange hat starring at something.')]\n\n    \"\"\"\n\n    assert len(\n        language_pair) == 2, \"language_pair must contain only 2 elements:\\\n            src and tgt language respectively\"\n    assert tuple(sorted(language_pair)) == (\n        \"de\",\n        \"en\",\n    ), \"language_pair must be either ('de','en') or ('en', 'de')\"\n\n    if root == DEFAULT_ROOT:\n        cache_dir = os.path.join(root, \"datasets\", \"Multi30k\")\n    else:\n        cache_dir = root\n\n    file_list = []\n\n    untar_files = []\n    source_files = []\n    target_files = []\n\n    datasets_list = []\n\n    if isinstance(split, str):\n        file_path, _ = cache_file(\n            None, cache_dir=cache_dir, url=URL[split], md5sum=MD5[split], proxies=proxies)\n        file_list.append(file_path)\n\n    else:\n        urls = itemgetter(*split)(URL)\n        md5s = itemgetter(*split)(MD5)\n        for i, url in enumerate(urls):\n            file_path, _ = cache_file(\n                None, cache_dir=cache_dir, url=url, md5sum=md5s[i], proxies=proxies)\n            file_list.append(file_path)\n\n    for file in file_list:\n        untar_files.append(untar(file, os.path.dirname(file)))\n\n    regexp = r\".de\"\n    if language_pair == (\"en\", \"de\"):\n        regexp = r\".en\"\n\n    for file_pair in untar_files:\n        for file in file_pair:\n            match = re.search(regexp, file)\n            if match:\n                source_files.append(file)\n            else:\n                target_files.append(file)\n\n    for i in range(len(untar_files)):\n        source_dataset = TextFileDataset(\n            os.path.join(cache_dir, source_files[i]), shuffle=False)\n        source_dataset = source_dataset.rename([\"text\"], [language_pair[0]])\n        target_dataset = TextFileDataset(\n            os.path.join(cache_dir, target_files[i]), shuffle=False)\n        target_dataset = target_dataset.rename([\"text\"], [language_pair[1]])\n        datasets = source_dataset.zip(target_dataset)\n        datasets_list.append(datasets)\n\n    if len(datasets_list) == 1:\n        return datasets_list[0]\n    return datasets_list\n\n@process.register\ndef Multi30k_Process(dataset, vocab, batch_size=64, max_len=500, \\\n                drop_remainder=False):\n    \"\"\"\n    the process of the Multi30k dataset\n\n    Args:\n        dataset (GeneratorDataset): Multi30k dataset.\n        vocab (Vocab): vocabulary object, used to store the mapping of token and index.\n        batch_size (int): The number of rows each batch is created with. Default: 64.\n        max_len (int): The max length of the sentence. Default: 500.\n        drop_remainder (bool): When the last batch of data contains a data entry smaller than batch_size, whether\n            to discard the batch and not pass it to the next operation. Default: False.\n\n    Returns:\n        - **dataset** (MapDataset) - dataset after transforms.\n\n    Raises:\n        TypeError: If `input_column` is not a string.\n\n    Examples:\n        >>> train_dataset = Multi30k(\n        >>>     root=self.root,\n        >>>     split=\"train\",\n        >>>     language_pair=(\"de\", \"en\")\n        >>> )\n        >>> tokenizer = BasicTokenizer(True)\n        >>> train_dataset = train_dataset.map([tokenizer], 'en')\n        >>> train_dataset = train_dataset.map([tokenizer], 'de')\n        >>> en_vocab = text.Vocab.from_dataset(train_dataset, 'en', special_tokens=\n        >>>   ['<pad>', '<unk>'], special_first= True)\n        >>> de_vocab = text.Vocab.from_dataset(train_dataset, 'de', special_tokens=\n        >>>   ['<pad>', '<unk>'], special_first= True)\n        >>> vocab = {'en':en_vocab, 'de':de_vocab}\n        >>> train_dataset = process('Multi30k', train_dataset, vocab = vocab)\n    \"\"\"\n\n    en_pad_value = vocab['en'].tokens_to_ids('<pad>')\n    de_pad_value = vocab['de'].tokens_to_ids('<pad>')\n\n    en_lookup_op = text.Lookup(vocab['en'], unknown_token='<unk>')\n    de_lookup_op = text.Lookup(vocab['de'], unknown_token='<unk>')\n\n    dataset = dataset.map([en_lookup_op], 'en')\n    dataset = dataset.map([de_lookup_op], 'de')\n\n    en_pad_op = transforms.PadEnd([max_len], en_pad_value)\n    de_pad_op = transforms.PadEnd([max_len], de_pad_value)\n\n    dataset = dataset.map([en_pad_op], 'en')\n    dataset = dataset.map([de_pad_op], 'de')\n\n    dataset = dataset.batch(batch_size, drop_remainder=drop_remainder)\n    return dataset\n", "path": "mindnlp/dataset/machine_translation/multi30k.py"}]}