We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
Include Moses Tokenizer


--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `torchtext/data/example.py`
Content:
```
1 import csv
2 import json
3 
4 
5 class Example(object):
6     """Defines a single training or test example.
7 
8     Stores each column of the example as an attribute.
9     """
10 
11     @classmethod
12     def fromJSON(cls, data, fields):
13         return cls.fromdict(json.loads(data), fields)
14 
15     @classmethod
16     def fromdict(cls, data, fields):
17         ex = cls()
18         for key, vals in fields.items():
19             if key in data and vals is not None:
20                 if not isinstance(vals, list):
21                     vals = [vals]
22                 for val in vals:
23                     name, field = val
24                     setattr(ex, name, field.preprocess(data[key]))
25         return ex
26 
27     @classmethod
28     def fromTSV(cls, data, fields):
29         if data[-1] == '\n':
30             data = data[:-1]
31         return cls.fromlist(data.split('\t'), fields)
32 
33     @classmethod
34     def fromCSV(cls, data, fields):
35         if data[-1] == '\n':
36             data = data[:-1]
37         return cls.fromlist(list(csv.reader([data]))[0], fields)
38 
39     @classmethod
40     def fromlist(cls, data, fields):
41         ex = cls()
42         for (name, field), val in zip(fields, data):
43             if field is not None:
44                 setattr(ex, name, field.preprocess(val))
45         return ex
46 
47     @classmethod
48     def fromtree(cls, data, fields, subtrees=False):
49         try:
50             from nltk.tree import Tree
51         except ImportError:
52             print('''Please install NLTK:
53     $ pip install nltk''')
54             raise
55         tree = Tree.fromstring(data)
56         if subtrees:
57             return [cls.fromlist(
58                 [t.leaves(), t.label()], fields) for t in tree.subtrees()]
59         return cls.fromlist([tree.leaves(), tree.label()], fields)
60
```

Path: `torchtext/data/utils.py`
Content:
```
1 def get_tokenizer(tokenizer):
2     if callable(tokenizer):
3         return tokenizer
4     if tokenizer == 'spacy':
5         try:
6             import spacy
7             spacy_en = spacy.load('en')
8             return lambda s: [tok.text for tok in spacy_en.tokenizer(s)]
9         except ImportError:
10             print("Please install SpaCy and the SpaCy English tokenizer. "
11                   "See the docs at https://spacy.io for more information.")
12             raise
13         except AttributeError:
14             print("Please install SpaCy and the SpaCy English tokenizer. "
15                   "See the docs at https://spacy.io for more information.")
16             raise
17     raise ValueError("Requested tokenizer {}, valid choices are a "
18                      "callable that takes a single string as input "
19                      "and \"spacy\" for the SpaCy English "
20                      "tokenizer.".format(tokenizer))
21 
22 
23 def interleave_keys(a, b):
24     """Interleave bits from two sort keys to form a joint sort key.
25 
26     Examples that are similar in both of the provided keys will have similar
27     values for the key defined by this function. Useful for tasks with two
28     text fields like machine translation or natural language inference.
29     """
30     def interleave(args):
31         return ''.join([x for t in zip(*args) for x in t])
32     return int(''.join(interleave(format(x, '016b') for x in (a, b))), base=2)
33
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/torchtext/data/example.py b/torchtext/data/example.py
--- a/torchtext/data/example.py
+++ b/torchtext/data/example.py
@@ -49,8 +49,8 @@
         try:
             from nltk.tree import Tree
         except ImportError:
-            print('''Please install NLTK:
-    $ pip install nltk''')
+            print("Please install NLTK. "
+                  "See the docs at http://nltk.org for more information.")
             raise
         tree = Tree.fromstring(data)
         if subtrees:
diff --git a/torchtext/data/utils.py b/torchtext/data/utils.py
--- a/torchtext/data/utils.py
+++ b/torchtext/data/utils.py
@@ -1,7 +1,7 @@
 def get_tokenizer(tokenizer):
     if callable(tokenizer):
         return tokenizer
-    if tokenizer == 'spacy':
+    if tokenizer == "spacy":
         try:
             import spacy
             spacy_en = spacy.load('en')
@@ -14,10 +14,24 @@
             print("Please install SpaCy and the SpaCy English tokenizer. "
                   "See the docs at https://spacy.io for more information.")
             raise
+    elif tokenizer == "moses":
+        try:
+            from nltk.tokenize.moses import MosesTokenizer
+            moses_tokenizer = MosesTokenizer()
+            return moses_tokenizer.tokenize
+        except ImportError:
+            print("Please install NLTK. "
+                  "See the docs at http://nltk.org for more information.")
+            raise
+        except LookupError:
+            print("Please install the necessary NLTK corpora. "
+                  "See the docs at http://nltk.org for more information.")
+            raise
     raise ValueError("Requested tokenizer {}, valid choices are a "
-                     "callable that takes a single string as input "
-                     "and \"spacy\" for the SpaCy English "
-                     "tokenizer.".format(tokenizer))
+                     "callable that takes a single string as input, "
+                     "\"spacy\" for the SpaCy English tokenizer, or "
+                     "\"moses\" for the NLTK port of the Moses tokenization "
+                     "script.".format(tokenizer))
 
 
 def interleave_keys(a, b):

{"golden_diff": "diff --git a/torchtext/data/example.py b/torchtext/data/example.py\n--- a/torchtext/data/example.py\n+++ b/torchtext/data/example.py\n@@ -49,8 +49,8 @@\n         try:\n             from nltk.tree import Tree\n         except ImportError:\n-            print('''Please install NLTK:\n-    $ pip install nltk''')\n+            print(\"Please install NLTK. \"\n+                  \"See the docs at http://nltk.org for more information.\")\n             raise\n         tree = Tree.fromstring(data)\n         if subtrees:\ndiff --git a/torchtext/data/utils.py b/torchtext/data/utils.py\n--- a/torchtext/data/utils.py\n+++ b/torchtext/data/utils.py\n@@ -1,7 +1,7 @@\n def get_tokenizer(tokenizer):\n     if callable(tokenizer):\n         return tokenizer\n-    if tokenizer == 'spacy':\n+    if tokenizer == \"spacy\":\n         try:\n             import spacy\n             spacy_en = spacy.load('en')\n@@ -14,10 +14,24 @@\n             print(\"Please install SpaCy and the SpaCy English tokenizer. \"\n                   \"See the docs at https://spacy.io for more information.\")\n             raise\n+    elif tokenizer == \"moses\":\n+        try:\n+            from nltk.tokenize.moses import MosesTokenizer\n+            moses_tokenizer = MosesTokenizer()\n+            return moses_tokenizer.tokenize\n+        except ImportError:\n+            print(\"Please install NLTK. \"\n+                  \"See the docs at http://nltk.org for more information.\")\n+            raise\n+        except LookupError:\n+            print(\"Please install the necessary NLTK corpora. \"\n+                  \"See the docs at http://nltk.org for more information.\")\n+            raise\n     raise ValueError(\"Requested tokenizer {}, valid choices are a \"\n-                     \"callable that takes a single string as input \"\n-                     \"and \\\"spacy\\\" for the SpaCy English \"\n-                     \"tokenizer.\".format(tokenizer))\n+                     \"callable that takes a single string as input, \"\n+                     \"\\\"spacy\\\" for the SpaCy English tokenizer, or \"\n+                     \"\\\"moses\\\" for the NLTK port of the Moses tokenization \"\n+                     \"script.\".format(tokenizer))\n \n \n def interleave_keys(a, b):\n", "issue": "Include Moses Tokenizer\n\n", "before_files": [{"content": "import csv\nimport json\n\n\nclass Example(object):\n    \"\"\"Defines a single training or test example.\n\n    Stores each column of the example as an attribute.\n    \"\"\"\n\n    @classmethod\n    def fromJSON(cls, data, fields):\n        return cls.fromdict(json.loads(data), fields)\n\n    @classmethod\n    def fromdict(cls, data, fields):\n        ex = cls()\n        for key, vals in fields.items():\n            if key in data and vals is not None:\n                if not isinstance(vals, list):\n                    vals = [vals]\n                for val in vals:\n                    name, field = val\n                    setattr(ex, name, field.preprocess(data[key]))\n        return ex\n\n    @classmethod\n    def fromTSV(cls, data, fields):\n        if data[-1] == '\\n':\n            data = data[:-1]\n        return cls.fromlist(data.split('\\t'), fields)\n\n    @classmethod\n    def fromCSV(cls, data, fields):\n        if data[-1] == '\\n':\n            data = data[:-1]\n        return cls.fromlist(list(csv.reader([data]))[0], fields)\n\n    @classmethod\n    def fromlist(cls, data, fields):\n        ex = cls()\n        for (name, field), val in zip(fields, data):\n            if field is not None:\n                setattr(ex, name, field.preprocess(val))\n        return ex\n\n    @classmethod\n    def fromtree(cls, data, fields, subtrees=False):\n        try:\n            from nltk.tree import Tree\n        except ImportError:\n            print('''Please install NLTK:\n    $ pip install nltk''')\n            raise\n        tree = Tree.fromstring(data)\n        if subtrees:\n            return [cls.fromlist(\n                [t.leaves(), t.label()], fields) for t in tree.subtrees()]\n        return cls.fromlist([tree.leaves(), tree.label()], fields)\n", "path": "torchtext/data/example.py"}, {"content": "def get_tokenizer(tokenizer):\n    if callable(tokenizer):\n        return tokenizer\n    if tokenizer == 'spacy':\n        try:\n            import spacy\n            spacy_en = spacy.load('en')\n            return lambda s: [tok.text for tok in spacy_en.tokenizer(s)]\n        except ImportError:\n            print(\"Please install SpaCy and the SpaCy English tokenizer. \"\n                  \"See the docs at https://spacy.io for more information.\")\n            raise\n        except AttributeError:\n            print(\"Please install SpaCy and the SpaCy English tokenizer. \"\n                  \"See the docs at https://spacy.io for more information.\")\n            raise\n    raise ValueError(\"Requested tokenizer {}, valid choices are a \"\n                     \"callable that takes a single string as input \"\n                     \"and \\\"spacy\\\" for the SpaCy English \"\n                     \"tokenizer.\".format(tokenizer))\n\n\ndef interleave_keys(a, b):\n    \"\"\"Interleave bits from two sort keys to form a joint sort key.\n\n    Examples that are similar in both of the provided keys will have similar\n    values for the key defined by this function. Useful for tasks with two\n    text fields like machine translation or natural language inference.\n    \"\"\"\n    def interleave(args):\n        return ''.join([x for t in zip(*args) for x in t])\n    return int(''.join(interleave(format(x, '016b') for x in (a, b))), base=2)\n", "path": "torchtext/data/utils.py"}], "after_files": [{"content": "import csv\nimport json\n\n\nclass Example(object):\n    \"\"\"Defines a single training or test example.\n\n    Stores each column of the example as an attribute.\n    \"\"\"\n\n    @classmethod\n    def fromJSON(cls, data, fields):\n        return cls.fromdict(json.loads(data), fields)\n\n    @classmethod\n    def fromdict(cls, data, fields):\n        ex = cls()\n        for key, vals in fields.items():\n            if key in data and vals is not None:\n                if not isinstance(vals, list):\n                    vals = [vals]\n                for val in vals:\n                    name, field = val\n                    setattr(ex, name, field.preprocess(data[key]))\n        return ex\n\n    @classmethod\n    def fromTSV(cls, data, fields):\n        if data[-1] == '\\n':\n            data = data[:-1]\n        return cls.fromlist(data.split('\\t'), fields)\n\n    @classmethod\n    def fromCSV(cls, data, fields):\n        if data[-1] == '\\n':\n            data = data[:-1]\n        return cls.fromlist(list(csv.reader([data]))[0], fields)\n\n    @classmethod\n    def fromlist(cls, data, fields):\n        ex = cls()\n        for (name, field), val in zip(fields, data):\n            if field is not None:\n                setattr(ex, name, field.preprocess(val))\n        return ex\n\n    @classmethod\n    def fromtree(cls, data, fields, subtrees=False):\n        try:\n            from nltk.tree import Tree\n        except ImportError:\n            print(\"Please install NLTK. \"\n                  \"See the docs at http://nltk.org for more information.\")\n            raise\n        tree = Tree.fromstring(data)\n        if subtrees:\n            return [cls.fromlist(\n                [t.leaves(), t.label()], fields) for t in tree.subtrees()]\n        return cls.fromlist([tree.leaves(), tree.label()], fields)\n", "path": "torchtext/data/example.py"}, {"content": "def get_tokenizer(tokenizer):\n    if callable(tokenizer):\n        return tokenizer\n    if tokenizer == \"spacy\":\n        try:\n            import spacy\n            spacy_en = spacy.load('en')\n            return lambda s: [tok.text for tok in spacy_en.tokenizer(s)]\n        except ImportError:\n            print(\"Please install SpaCy and the SpaCy English tokenizer. \"\n                  \"See the docs at https://spacy.io for more information.\")\n            raise\n        except AttributeError:\n            print(\"Please install SpaCy and the SpaCy English tokenizer. \"\n                  \"See the docs at https://spacy.io for more information.\")\n            raise\n    elif tokenizer == \"moses\":\n        try:\n            from nltk.tokenize.moses import MosesTokenizer\n            moses_tokenizer = MosesTokenizer()\n            return moses_tokenizer.tokenize\n        except ImportError:\n            print(\"Please install NLTK. \"\n                  \"See the docs at http://nltk.org for more information.\")\n            raise\n        except LookupError:\n            print(\"Please install the necessary NLTK corpora. \"\n                  \"See the docs at http://nltk.org for more information.\")\n            raise\n    raise ValueError(\"Requested tokenizer {}, valid choices are a \"\n                     \"callable that takes a single string as input, \"\n                     \"\\\"spacy\\\" for the SpaCy English tokenizer, or \"\n                     \"\\\"moses\\\" for the NLTK port of the Moses tokenization \"\n                     \"script.\".format(tokenizer))\n\n\ndef interleave_keys(a, b):\n    \"\"\"Interleave bits from two sort keys to form a joint sort key.\n\n    Examples that are similar in both of the provided keys will have similar\n    values for the key defined by this function. Useful for tasks with two\n    text fields like machine translation or natural language inference.\n    \"\"\"\n    def interleave(args):\n        return ''.join([x for t in zip(*args) for x in t])\n    return int(''.join(interleave(format(x, '016b') for x in (a, b))), base=2)\n", "path": "torchtext/data/utils.py"}]}