prompt: We are currently solving the following issue within our repository. Here is the issue text:
  --- BEGIN ISSUE ---
  MosesTokenizer has been moved out of NLTK due to licensing issues
  @jekbradbury great work here!
  
  Due to https://github.com/nltk/nltk/issues/2000, we had to remove MosesTokenizer out of NLTK but now it's hosted on https://github.com/alvations/sacremoses 
  
  ```
  pip install sacremoses
  ```
  
  The silver lining is that the package comes with the data needed for tokenization so there's no need to keep the `nltk_data` directory =)
  
  ----
  
  I would propose adding `sacremoses` on top of `nltk` because NLTK has another port of a nice tokenizer (by @jonsafari) that people overlook, https://github.com/nltk/nltk/blob/develop/nltk/tokenize/toktok.py (I think it's fast too)
  
  --- END ISSUE ---
  
  Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
  --- BEGIN FILES ---
  Path: `torchtext/data/utils.py`
  Content:
  ```
  1 import random
  2 from contextlib import contextmanager
  3 from copy import deepcopy
  4 
  5 
  6 def get_tokenizer(tokenizer):
  7     if callable(tokenizer):
  8         return tokenizer
  9     if tokenizer == "spacy":
  10         try:
  11             import spacy
  12             spacy_en = spacy.load('en')
  13             return lambda s: [tok.text for tok in spacy_en.tokenizer(s)]
  14         except ImportError:
  15             print("Please install SpaCy and the SpaCy English tokenizer. "
  16                   "See the docs at https://spacy.io for more information.")
  17             raise
  18         except AttributeError:
  19             print("Please install SpaCy and the SpaCy English tokenizer. "
  20                   "See the docs at https://spacy.io for more information.")
  21             raise
  22     elif tokenizer == "moses":
  23         try:
  24             from nltk.tokenize.moses import MosesTokenizer
  25             moses_tokenizer = MosesTokenizer()
  26             return moses_tokenizer.tokenize
  27         except ImportError:
  28             print("Please install NLTK. "
  29                   "See the docs at http://nltk.org for more information.")
  30             raise
  31         except LookupError:
  32             print("Please install the necessary NLTK corpora. "
  33                   "See the docs at http://nltk.org for more information.")
  34             raise
  35     elif tokenizer == 'revtok':
  36         try:
  37             import revtok
  38             return revtok.tokenize
  39         except ImportError:
  40             print("Please install revtok.")
  41             raise
  42     elif tokenizer == 'subword':
  43         try:
  44             import revtok
  45             return lambda x: revtok.tokenize(x, decap=True)
  46         except ImportError:
  47             print("Please install revtok.")
  48             raise
  49     raise ValueError("Requested tokenizer {}, valid choices are a "
  50                      "callable that takes a single string as input, "
  51                      "\"revtok\" for the revtok reversible tokenizer, "
  52                      "\"subword\" for the revtok caps-aware tokenizer, "
  53                      "\"spacy\" for the SpaCy English tokenizer, or "
  54                      "\"moses\" for the NLTK port of the Moses tokenization "
  55                      "script.".format(tokenizer))
  56 
  57 
  58 def interleave_keys(a, b):
  59     """Interleave bits from two sort keys to form a joint sort key.
  60 
  61     Examples that are similar in both of the provided keys will have similar
  62     values for the key defined by this function. Useful for tasks with two
  63     text fields like machine translation or natural language inference.
  64     """
  65     def interleave(args):
  66         return ''.join([x for t in zip(*args) for x in t])
  67     return int(''.join(interleave(format(x, '016b') for x in (a, b))), base=2)
  68 
  69 
  70 def get_torch_version():
  71     import torch
  72     v = torch.__version__
  73     version_substrings = v.split('.')
  74     major, minor = version_substrings[0], version_substrings[1]
  75     return int(major), int(minor)
  76 
  77 
  78 class RandomShuffler(object):
  79     """Use random functions while keeping track of the random state to make it
  80     reproducible and deterministic."""
  81 
  82     def __init__(self, random_state=None):
  83         self._random_state = random_state
  84         if self._random_state is None:
  85             self._random_state = random.getstate()
  86 
  87     @contextmanager
  88     def use_internal_state(self):
  89         """Use a specific RNG state."""
  90         old_state = random.getstate()
  91         random.setstate(self._random_state)
  92         yield
  93         self._random_state = random.getstate()
  94         random.setstate(old_state)
  95 
  96     @property
  97     def random_state(self):
  98         return deepcopy(self._random_state)
  99 
  100     @random_state.setter
  101     def random_state(self, s):
  102         self._random_state = s
  103 
  104     def __call__(self, data):
  105         """Shuffle and return a new list."""
  106         with self.use_internal_state():
  107             return random.sample(data, len(data))
  108
  ```
  --- END FILES ---
  
  Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.
  
  Here is an example:
  
  ```diff
  diff --git a/examples/server_async.py b/examples/server_async.py
  --- a/examples/server_async.py
  +++ b/examples/server_async.py
  @@ -313,4 +313,4 @@
   
   
   if __name__ == "__main__":
  -    asyncio.run(run_async_server("."), debug=True)
  +    asyncio.run(run_async_server(), debug=True)
  diff --git a/examples/server_sync.py b/examples/server_sync.py
  --- a/examples/server_sync.py
  +++ b/examples/server_sync.py
  @@ -313,5 +313,5 @@
   
   
   if __name__ == "__main__":
  -    server = run_sync_server(".")
  +    server = run_sync_server()
       server.shutdown()
  
  ```
  
golden_diff: diff --git a/torchtext/data/utils.py b/torchtext/data/utils.py
  --- a/torchtext/data/utils.py
  +++ b/torchtext/data/utils.py
  @@ -21,16 +21,22 @@
               raise
       elif tokenizer == "moses":
           try:
  -            from nltk.tokenize.moses import MosesTokenizer
  +            from sacremoses import MosesTokenizer
               moses_tokenizer = MosesTokenizer()
               return moses_tokenizer.tokenize
           except ImportError:
  -            print("Please install NLTK. "
  -                  "See the docs at http://nltk.org for more information.")
  +            print("Please install SacreMoses. "
  +                  "See the docs at https://github.com/alvations/sacremoses "
  +                  "for more information.")
               raise
  -        except LookupError:
  -            print("Please install the necessary NLTK corpora. "
  -                  "See the docs at http://nltk.org for more information.")
  +    elif tokenizer == "toktok":
  +        try:
  +            from nltk.tokenize.toktok import ToktokTokenizer
  +            toktok = ToktokTokenizer()
  +            return toktok.tokenize
  +        except ImportError:
  +            print("Please install NLTK. "
  +                  "See the docs at https://nltk.org  for more information.")
               raise
       elif tokenizer == 'revtok':
           try:
  
verification_info: {"golden_diff": "diff --git a/torchtext/data/utils.py b/torchtext/data/utils.py\n--- a/torchtext/data/utils.py\n+++ b/torchtext/data/utils.py\n@@ -21,16 +21,22 @@\n             raise\n     elif tokenizer == \"moses\":\n         try:\n-            from nltk.tokenize.moses import MosesTokenizer\n+            from sacremoses import MosesTokenizer\n             moses_tokenizer = MosesTokenizer()\n             return moses_tokenizer.tokenize\n         except ImportError:\n-            print(\"Please install NLTK. \"\n-                  \"See the docs at http://nltk.org for more information.\")\n+            print(\"Please install SacreMoses. \"\n+                  \"See the docs at https://github.com/alvations/sacremoses \"\n+                  \"for more information.\")\n             raise\n-        except LookupError:\n-            print(\"Please install the necessary NLTK corpora. \"\n-                  \"See the docs at http://nltk.org for more information.\")\n+    elif tokenizer == \"toktok\":\n+        try:\n+            from nltk.tokenize.toktok import ToktokTokenizer\n+            toktok = ToktokTokenizer()\n+            return toktok.tokenize\n+        except ImportError:\n+            print(\"Please install NLTK. \"\n+                  \"See the docs at https://nltk.org  for more information.\")\n             raise\n     elif tokenizer == 'revtok':\n         try:\n", "issue": "MosesTokenizer has been moved out of NLTK due to licensing issues\n@jekbradbury great work here!\r\n\r\nDue to https://github.com/nltk/nltk/issues/2000, we had to remove MosesTokenizer out of NLTK but now it's hosted on https://github.com/alvations/sacremoses \r\n\r\n```\r\npip install sacremoses\r\n```\r\n\r\nThe silver lining is that the package comes with the data needed for tokenization so there's no need to keep the `nltk_data` directory =)\r\n\r\n----\r\n\r\nI would propose adding `sacremoses` on top of `nltk` because NLTK has another port of a nice tokenizer (by @jonsafari) that people overlook, https://github.com/nltk/nltk/blob/develop/nltk/tokenize/toktok.py (I think it's fast too)\n", "before_files": [{"content": "import random\nfrom contextlib import contextmanager\nfrom copy import deepcopy\n\n\ndef get_tokenizer(tokenizer):\n    if callable(tokenizer):\n        return tokenizer\n    if tokenizer == \"spacy\":\n        try:\n            import spacy\n            spacy_en = spacy.load('en')\n            return lambda s: [tok.text for tok in spacy_en.tokenizer(s)]\n        except ImportError:\n            print(\"Please install SpaCy and the SpaCy English tokenizer. \"\n                  \"See the docs at https://spacy.io for more information.\")\n            raise\n        except AttributeError:\n            print(\"Please install SpaCy and the SpaCy English tokenizer. \"\n                  \"See the docs at https://spacy.io for more information.\")\n            raise\n    elif tokenizer == \"moses\":\n        try:\n            from nltk.tokenize.moses import MosesTokenizer\n            moses_tokenizer = MosesTokenizer()\n            return moses_tokenizer.tokenize\n        except ImportError:\n            print(\"Please install NLTK. \"\n                  \"See the docs at http://nltk.org for more information.\")\n            raise\n        except LookupError:\n            print(\"Please install the necessary NLTK corpora. \"\n                  \"See the docs at http://nltk.org for more information.\")\n            raise\n    elif tokenizer == 'revtok':\n        try:\n            import revtok\n            return revtok.tokenize\n        except ImportError:\n            print(\"Please install revtok.\")\n            raise\n    elif tokenizer == 'subword':\n        try:\n            import revtok\n            return lambda x: revtok.tokenize(x, decap=True)\n        except ImportError:\n            print(\"Please install revtok.\")\n            raise\n    raise ValueError(\"Requested tokenizer {}, valid choices are a \"\n                     \"callable that takes a single string as input, \"\n                     \"\\\"revtok\\\" for the revtok reversible tokenizer, \"\n                     \"\\\"subword\\\" for the revtok caps-aware tokenizer, \"\n                     \"\\\"spacy\\\" for the SpaCy English tokenizer, or \"\n                     \"\\\"moses\\\" for the NLTK port of the Moses tokenization \"\n                     \"script.\".format(tokenizer))\n\n\ndef interleave_keys(a, b):\n    \"\"\"Interleave bits from two sort keys to form a joint sort key.\n\n    Examples that are similar in both of the provided keys will have similar\n    values for the key defined by this function. Useful for tasks with two\n    text fields like machine translation or natural language inference.\n    \"\"\"\n    def interleave(args):\n        return ''.join([x for t in zip(*args) for x in t])\n    return int(''.join(interleave(format(x, '016b') for x in (a, b))), base=2)\n\n\ndef get_torch_version():\n    import torch\n    v = torch.__version__\n    version_substrings = v.split('.')\n    major, minor = version_substrings[0], version_substrings[1]\n    return int(major), int(minor)\n\n\nclass RandomShuffler(object):\n    \"\"\"Use random functions while keeping track of the random state to make it\n    reproducible and deterministic.\"\"\"\n\n    def __init__(self, random_state=None):\n        self._random_state = random_state\n        if self._random_state is None:\n            self._random_state = random.getstate()\n\n    @contextmanager\n    def use_internal_state(self):\n        \"\"\"Use a specific RNG state.\"\"\"\n        old_state = random.getstate()\n        random.setstate(self._random_state)\n        yield\n        self._random_state = random.getstate()\n        random.setstate(old_state)\n\n    @property\n    def random_state(self):\n        return deepcopy(self._random_state)\n\n    @random_state.setter\n    def random_state(self, s):\n        self._random_state = s\n\n    def __call__(self, data):\n        \"\"\"Shuffle and return a new list.\"\"\"\n        with self.use_internal_state():\n            return random.sample(data, len(data))\n", "path": "torchtext/data/utils.py"}], "after_files": [{"content": "import random\nfrom contextlib import contextmanager\nfrom copy import deepcopy\n\n\ndef get_tokenizer(tokenizer):\n    if callable(tokenizer):\n        return tokenizer\n    if tokenizer == \"spacy\":\n        try:\n            import spacy\n            spacy_en = spacy.load('en')\n            return lambda s: [tok.text for tok in spacy_en.tokenizer(s)]\n        except ImportError:\n            print(\"Please install SpaCy and the SpaCy English tokenizer. \"\n                  \"See the docs at https://spacy.io for more information.\")\n            raise\n        except AttributeError:\n            print(\"Please install SpaCy and the SpaCy English tokenizer. \"\n                  \"See the docs at https://spacy.io for more information.\")\n            raise\n    elif tokenizer == \"moses\":\n        try:\n            from sacremoses import MosesTokenizer\n            moses_tokenizer = MosesTokenizer()\n            return moses_tokenizer.tokenize\n        except ImportError:\n            print(\"Please install SacreMoses. \"\n                  \"See the docs at https://github.com/alvations/sacremoses \"\n                  \"for more information.\")\n            raise\n    elif tokenizer == \"toktok\":\n        try:\n            from nltk.tokenize.toktok import ToktokTokenizer\n            toktok = ToktokTokenizer()\n            return toktok.tokenize\n        except ImportError:\n            print(\"Please install NLTK. \"\n                  \"See the docs at https://nltk.org  for more information.\")\n            raise\n    elif tokenizer == 'revtok':\n        try:\n            import revtok\n            return revtok.tokenize\n        except ImportError:\n            print(\"Please install revtok.\")\n            raise\n    elif tokenizer == 'subword':\n        try:\n            import revtok\n            return lambda x: revtok.tokenize(x, decap=True)\n        except ImportError:\n            print(\"Please install revtok.\")\n            raise\n    raise ValueError(\"Requested tokenizer {}, valid choices are a \"\n                     \"callable that takes a single string as input, \"\n                     \"\\\"revtok\\\" for the revtok reversible tokenizer, \"\n                     \"\\\"subword\\\" for the revtok caps-aware tokenizer, \"\n                     \"\\\"spacy\\\" for the SpaCy English tokenizer, or \"\n                     \"\\\"moses\\\" for the NLTK port of the Moses tokenization \"\n                     \"script.\".format(tokenizer))\n\n\ndef interleave_keys(a, b):\n    \"\"\"Interleave bits from two sort keys to form a joint sort key.\n\n    Examples that are similar in both of the provided keys will have similar\n    values for the key defined by this function. Useful for tasks with two\n    text fields like machine translation or natural language inference.\n    \"\"\"\n    def interleave(args):\n        return ''.join([x for t in zip(*args) for x in t])\n    return int(''.join(interleave(format(x, '016b') for x in (a, b))), base=2)\n\n\ndef get_torch_version():\n    import torch\n    v = torch.__version__\n    version_substrings = v.split('.')\n    major, minor = version_substrings[0], version_substrings[1]\n    return int(major), int(minor)\n\n\nclass RandomShuffler(object):\n    \"\"\"Use random functions while keeping track of the random state to make it\n    reproducible and deterministic.\"\"\"\n\n    def __init__(self, random_state=None):\n        self._random_state = random_state\n        if self._random_state is None:\n            self._random_state = random.getstate()\n\n    @contextmanager\n    def use_internal_state(self):\n        \"\"\"Use a specific RNG state.\"\"\"\n        old_state = random.getstate()\n        random.setstate(self._random_state)\n        yield\n        self._random_state = random.getstate()\n        random.setstate(old_state)\n\n    @property\n    def random_state(self):\n        return deepcopy(self._random_state)\n\n    @random_state.setter\n    def random_state(self, s):\n        self._random_state = s\n\n    def __call__(self, data):\n        \"\"\"Shuffle and return a new list.\"\"\"\n        with self.use_internal_state():\n            return random.sample(data, len(data))\n", "path": "torchtext/data/utils.py"}]}