prompt: We are currently solving the following issue within our repository. Here is the issue text:
  --- BEGIN ISSUE ---
  Tokenization not working using v2.1
  ## How to reproduce the behaviour
  I found a bug where tokenization is completely not working with version 2.1.0a10 on python 2.7. I have reproduced this on three of my machines.
  
  ```
  $ conda create -n py27_spacy2 python=2.7
  $ source activate py27_spacy2
  $ pip install -U spacy-nightly
  $ python -m spacy download en_core_web_sm
  $ python -c "import spacy; nlp=spacy.load('en_core_web_sm'); doc=nlp(u'hello world'); print ','.join([t.text for t in doc])"
  h,e,ll,o,w,o,r,l,d
  ```
  
  ## Your Environment
  * Operating System: Ubuntu
  * Python Version Used: 2.7
  * spaCy Version Used: 2.1.0a10
  
  
  
  --- END ISSUE ---
  
  Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
  --- BEGIN FILES ---
  Path: `spacy/compat.py`
  Content:
  ```
  1 # coding: utf8
  2 """
  3 Helpers for Python and platform compatibility. To distinguish them from
  4 the builtin functions, replacement functions are suffixed with an underscore,
  5 e.g. `unicode_`.
  6 
  7 DOCS: https://spacy.io/api/top-level#compat
  8 """
  9 from __future__ import unicode_literals
  10 
  11 import os
  12 import sys
  13 import itertools
  14 
  15 from thinc.neural.util import copy_array
  16 
  17 try:
  18     import cPickle as pickle
  19 except ImportError:
  20     import pickle
  21 
  22 try:
  23     import copy_reg
  24 except ImportError:
  25     import copyreg as copy_reg
  26 
  27 try:
  28     from cupy.cuda.stream import Stream as CudaStream
  29 except ImportError:
  30     CudaStream = None
  31 
  32 try:
  33     import cupy
  34 except ImportError:
  35     cupy = None
  36 
  37 try:
  38     from thinc.neural.optimizers import Optimizer  # noqa: F401
  39 except ImportError:
  40     from thinc.neural.optimizers import Adam as Optimizer  # noqa: F401
  41 
  42 pickle = pickle
  43 copy_reg = copy_reg
  44 CudaStream = CudaStream
  45 cupy = cupy
  46 copy_array = copy_array
  47 izip = getattr(itertools, "izip", zip)
  48 
  49 is_windows = sys.platform.startswith("win")
  50 is_linux = sys.platform.startswith("linux")
  51 is_osx = sys.platform == "darwin"
  52 
  53 # See: https://github.com/benjaminp/six/blob/master/six.py
  54 is_python2 = sys.version_info[0] == 2
  55 is_python3 = sys.version_info[0] == 3
  56 is_python_pre_3_5 = is_python2 or (is_python3 and sys.version_info[1] < 5)
  57 
  58 if is_python2:
  59     bytes_ = str
  60     unicode_ = unicode  # noqa: F821
  61     basestring_ = basestring  # noqa: F821
  62     input_ = raw_input  # noqa: F821
  63     path2str = lambda path: str(path).decode("utf8")
  64 
  65 elif is_python3:
  66     bytes_ = bytes
  67     unicode_ = str
  68     basestring_ = str
  69     input_ = input
  70     path2str = lambda path: str(path)
  71 
  72 
  73 def b_to_str(b_str):
  74     """Convert a bytes object to a string.
  75 
  76     b_str (bytes): The object to convert.
  77     RETURNS (unicode): The converted string.
  78     """
  79     if is_python2:
  80         return b_str
  81     # Important: if no encoding is set, string becomes "b'...'"
  82     return str(b_str, encoding="utf8")
  83 
  84 
  85 def symlink_to(orig, dest):
  86     """Create a symlink. Used for model shortcut links.
  87 
  88     orig (unicode / Path): The origin path.
  89     dest (unicode / Path): The destination path of the symlink.
  90     """
  91     if is_windows:
  92         import subprocess
  93 
  94         subprocess.call(["mklink", "/d", path2str(orig), path2str(dest)], shell=True)
  95     else:
  96         orig.symlink_to(dest)
  97 
  98 
  99 def symlink_remove(link):
  100     """Remove a symlink. Used for model shortcut links.
  101 
  102     link (unicode / Path): The path to the symlink.
  103     """
  104     # https://stackoverflow.com/q/26554135/6400719
  105     if os.path.isdir(path2str(link)) and is_windows:
  106         # this should only be on Py2.7 and windows
  107         os.rmdir(path2str(link))
  108     else:
  109         os.unlink(path2str(link))
  110 
  111 
  112 def is_config(python2=None, python3=None, windows=None, linux=None, osx=None):
  113     """Check if a specific configuration of Python version and operating system
  114     matches the user's setup. Mostly used to display targeted error messages.
  115 
  116     python2 (bool): spaCy is executed with Python 2.x.
  117     python3 (bool): spaCy is executed with Python 3.x.
  118     windows (bool): spaCy is executed on Windows.
  119     linux (bool): spaCy is executed on Linux.
  120     osx (bool): spaCy is executed on OS X or macOS.
  121     RETURNS (bool): Whether the configuration matches the user's platform.
  122 
  123     DOCS: https://spacy.io/api/top-level#compat.is_config
  124     """
  125     return (
  126         python2 in (None, is_python2)
  127         and python3 in (None, is_python3)
  128         and windows in (None, is_windows)
  129         and linux in (None, is_linux)
  130         and osx in (None, is_osx)
  131     )
  132 
  133 
  134 def import_file(name, loc):
  135     """Import module from a file. Used to load models from a directory.
  136 
  137     name (unicode): Name of module to load.
  138     loc (unicode / Path): Path to the file.
  139     RETURNS: The loaded module.
  140     """
  141     loc = path2str(loc)
  142     if is_python_pre_3_5:
  143         import imp
  144 
  145         return imp.load_source(name, loc)
  146     else:
  147         import importlib.util
  148 
  149         spec = importlib.util.spec_from_file_location(name, str(loc))
  150         module = importlib.util.module_from_spec(spec)
  151         spec.loader.exec_module(module)
  152         return module
  153
  ```
  --- END FILES ---
  
  Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.
  
  Here is an example:
  
  ```diff
  diff --git a/examples/server_async.py b/examples/server_async.py
  --- a/examples/server_async.py
  +++ b/examples/server_async.py
  @@ -313,4 +313,4 @@
   
   
   if __name__ == "__main__":
  -    asyncio.run(run_async_server("."), debug=True)
  +    asyncio.run(run_async_server(), debug=True)
  diff --git a/examples/server_sync.py b/examples/server_sync.py
  --- a/examples/server_sync.py
  +++ b/examples/server_sync.py
  @@ -313,5 +313,5 @@
   
   
   if __name__ == "__main__":
  -    server = run_sync_server(".")
  +    server = run_sync_server()
       server.shutdown()
  
  ```
  
golden_diff: diff --git a/spacy/compat.py b/spacy/compat.py
  --- a/spacy/compat.py
  +++ b/spacy/compat.py
  @@ -11,6 +11,7 @@
   import os
   import sys
   import itertools
  +import ast
   
   from thinc.neural.util import copy_array
   
  @@ -150,3 +151,26 @@
           module = importlib.util.module_from_spec(spec)
           spec.loader.exec_module(module)
           return module
  +
  +
  +def unescape_unicode(string):
  +    """Python2.7's re module chokes when compiling patterns that have ranges
  +    between escaped unicode codepoints if the two codepoints are unrecognised
  +    in the unicode database. For instance:
  +
  +        re.compile('[\\uAA77-\\uAA79]').findall("hello")
  +
  +    Ends up matching every character (on Python 2). This problem doesn't occur
  +    if we're dealing with unicode literals.
  +    """
  +    if string is None:
  +        return string
  +    # We only want to unescape the unicode, so we first must protect the other
  +    # backslashes.
  +    string = string.replace("\\", "\\\\")
  +    # Now we remove that protection for the unicode.
  +    string = string.replace("\\\\u", "\\u")
  +    string = string.replace("\\\\U", "\\U")
  +    # Now we unescape by evaling the string with the AST. This can't execute
  +    # code -- it only does the representational level.
  +    return ast.literal_eval("u'''" + string + "'''")
  
verification_info: {"golden_diff": "diff --git a/spacy/compat.py b/spacy/compat.py\n--- a/spacy/compat.py\n+++ b/spacy/compat.py\n@@ -11,6 +11,7 @@\n import os\n import sys\n import itertools\n+import ast\n \n from thinc.neural.util import copy_array\n \n@@ -150,3 +151,26 @@\n         module = importlib.util.module_from_spec(spec)\n         spec.loader.exec_module(module)\n         return module\n+\n+\n+def unescape_unicode(string):\n+    \"\"\"Python2.7's re module chokes when compiling patterns that have ranges\n+    between escaped unicode codepoints if the two codepoints are unrecognised\n+    in the unicode database. For instance:\n+\n+        re.compile('[\\\\uAA77-\\\\uAA79]').findall(\"hello\")\n+\n+    Ends up matching every character (on Python 2). This problem doesn't occur\n+    if we're dealing with unicode literals.\n+    \"\"\"\n+    if string is None:\n+        return string\n+    # We only want to unescape the unicode, so we first must protect the other\n+    # backslashes.\n+    string = string.replace(\"\\\\\", \"\\\\\\\\\")\n+    # Now we remove that protection for the unicode.\n+    string = string.replace(\"\\\\\\\\u\", \"\\\\u\")\n+    string = string.replace(\"\\\\\\\\U\", \"\\\\U\")\n+    # Now we unescape by evaling the string with the AST. This can't execute\n+    # code -- it only does the representational level.\n+    return ast.literal_eval(\"u'''\" + string + \"'''\")\n", "issue": "Tokenization not working using v2.1\n## How to reproduce the behaviour\r\nI found a bug where tokenization is completely not working with version 2.1.0a10 on python 2.7. I have reproduced this on three of my machines.\r\n\r\n```\r\n$ conda create -n py27_spacy2 python=2.7\r\n$ source activate py27_spacy2\r\n$ pip install -U spacy-nightly\r\n$ python -m spacy download en_core_web_sm\r\n$ python -c \"import spacy; nlp=spacy.load('en_core_web_sm'); doc=nlp(u'hello world'); print ','.join([t.text for t in doc])\"\r\nh,e,ll,o,w,o,r,l,d\r\n```\r\n\r\n## Your Environment\r\n* Operating System: Ubuntu\r\n* Python Version Used: 2.7\r\n* spaCy Version Used: 2.1.0a10\r\n\r\n\n", "before_files": [{"content": "# coding: utf8\n\"\"\"\nHelpers for Python and platform compatibility. To distinguish them from\nthe builtin functions, replacement functions are suffixed with an underscore,\ne.g. `unicode_`.\n\nDOCS: https://spacy.io/api/top-level#compat\n\"\"\"\nfrom __future__ import unicode_literals\n\nimport os\nimport sys\nimport itertools\n\nfrom thinc.neural.util import copy_array\n\ntry:\n    import cPickle as pickle\nexcept ImportError:\n    import pickle\n\ntry:\n    import copy_reg\nexcept ImportError:\n    import copyreg as copy_reg\n\ntry:\n    from cupy.cuda.stream import Stream as CudaStream\nexcept ImportError:\n    CudaStream = None\n\ntry:\n    import cupy\nexcept ImportError:\n    cupy = None\n\ntry:\n    from thinc.neural.optimizers import Optimizer  # noqa: F401\nexcept ImportError:\n    from thinc.neural.optimizers import Adam as Optimizer  # noqa: F401\n\npickle = pickle\ncopy_reg = copy_reg\nCudaStream = CudaStream\ncupy = cupy\ncopy_array = copy_array\nizip = getattr(itertools, \"izip\", zip)\n\nis_windows = sys.platform.startswith(\"win\")\nis_linux = sys.platform.startswith(\"linux\")\nis_osx = sys.platform == \"darwin\"\n\n# See: https://github.com/benjaminp/six/blob/master/six.py\nis_python2 = sys.version_info[0] == 2\nis_python3 = sys.version_info[0] == 3\nis_python_pre_3_5 = is_python2 or (is_python3 and sys.version_info[1] < 5)\n\nif is_python2:\n    bytes_ = str\n    unicode_ = unicode  # noqa: F821\n    basestring_ = basestring  # noqa: F821\n    input_ = raw_input  # noqa: F821\n    path2str = lambda path: str(path).decode(\"utf8\")\n\nelif is_python3:\n    bytes_ = bytes\n    unicode_ = str\n    basestring_ = str\n    input_ = input\n    path2str = lambda path: str(path)\n\n\ndef b_to_str(b_str):\n    \"\"\"Convert a bytes object to a string.\n\n    b_str (bytes): The object to convert.\n    RETURNS (unicode): The converted string.\n    \"\"\"\n    if is_python2:\n        return b_str\n    # Important: if no encoding is set, string becomes \"b'...'\"\n    return str(b_str, encoding=\"utf8\")\n\n\ndef symlink_to(orig, dest):\n    \"\"\"Create a symlink. Used for model shortcut links.\n\n    orig (unicode / Path): The origin path.\n    dest (unicode / Path): The destination path of the symlink.\n    \"\"\"\n    if is_windows:\n        import subprocess\n\n        subprocess.call([\"mklink\", \"/d\", path2str(orig), path2str(dest)], shell=True)\n    else:\n        orig.symlink_to(dest)\n\n\ndef symlink_remove(link):\n    \"\"\"Remove a symlink. Used for model shortcut links.\n\n    link (unicode / Path): The path to the symlink.\n    \"\"\"\n    # https://stackoverflow.com/q/26554135/6400719\n    if os.path.isdir(path2str(link)) and is_windows:\n        # this should only be on Py2.7 and windows\n        os.rmdir(path2str(link))\n    else:\n        os.unlink(path2str(link))\n\n\ndef is_config(python2=None, python3=None, windows=None, linux=None, osx=None):\n    \"\"\"Check if a specific configuration of Python version and operating system\n    matches the user's setup. Mostly used to display targeted error messages.\n\n    python2 (bool): spaCy is executed with Python 2.x.\n    python3 (bool): spaCy is executed with Python 3.x.\n    windows (bool): spaCy is executed on Windows.\n    linux (bool): spaCy is executed on Linux.\n    osx (bool): spaCy is executed on OS X or macOS.\n    RETURNS (bool): Whether the configuration matches the user's platform.\n\n    DOCS: https://spacy.io/api/top-level#compat.is_config\n    \"\"\"\n    return (\n        python2 in (None, is_python2)\n        and python3 in (None, is_python3)\n        and windows in (None, is_windows)\n        and linux in (None, is_linux)\n        and osx in (None, is_osx)\n    )\n\n\ndef import_file(name, loc):\n    \"\"\"Import module from a file. Used to load models from a directory.\n\n    name (unicode): Name of module to load.\n    loc (unicode / Path): Path to the file.\n    RETURNS: The loaded module.\n    \"\"\"\n    loc = path2str(loc)\n    if is_python_pre_3_5:\n        import imp\n\n        return imp.load_source(name, loc)\n    else:\n        import importlib.util\n\n        spec = importlib.util.spec_from_file_location(name, str(loc))\n        module = importlib.util.module_from_spec(spec)\n        spec.loader.exec_module(module)\n        return module\n", "path": "spacy/compat.py"}], "after_files": [{"content": "# coding: utf8\n\"\"\"\nHelpers for Python and platform compatibility. To distinguish them from\nthe builtin functions, replacement functions are suffixed with an underscore,\ne.g. `unicode_`.\n\nDOCS: https://spacy.io/api/top-level#compat\n\"\"\"\nfrom __future__ import unicode_literals\n\nimport os\nimport sys\nimport itertools\nimport ast\n\nfrom thinc.neural.util import copy_array\n\ntry:\n    import cPickle as pickle\nexcept ImportError:\n    import pickle\n\ntry:\n    import copy_reg\nexcept ImportError:\n    import copyreg as copy_reg\n\ntry:\n    from cupy.cuda.stream import Stream as CudaStream\nexcept ImportError:\n    CudaStream = None\n\ntry:\n    import cupy\nexcept ImportError:\n    cupy = None\n\ntry:\n    from thinc.neural.optimizers import Optimizer  # noqa: F401\nexcept ImportError:\n    from thinc.neural.optimizers import Adam as Optimizer  # noqa: F401\n\npickle = pickle\ncopy_reg = copy_reg\nCudaStream = CudaStream\ncupy = cupy\ncopy_array = copy_array\nizip = getattr(itertools, \"izip\", zip)\n\nis_windows = sys.platform.startswith(\"win\")\nis_linux = sys.platform.startswith(\"linux\")\nis_osx = sys.platform == \"darwin\"\n\n# See: https://github.com/benjaminp/six/blob/master/six.py\nis_python2 = sys.version_info[0] == 2\nis_python3 = sys.version_info[0] == 3\nis_python_pre_3_5 = is_python2 or (is_python3 and sys.version_info[1] < 5)\n\nif is_python2:\n    bytes_ = str\n    unicode_ = unicode  # noqa: F821\n    basestring_ = basestring  # noqa: F821\n    input_ = raw_input  # noqa: F821\n    path2str = lambda path: str(path).decode(\"utf8\")\n\nelif is_python3:\n    bytes_ = bytes\n    unicode_ = str\n    basestring_ = str\n    input_ = input\n    path2str = lambda path: str(path)\n\n\ndef b_to_str(b_str):\n    \"\"\"Convert a bytes object to a string.\n\n    b_str (bytes): The object to convert.\n    RETURNS (unicode): The converted string.\n    \"\"\"\n    if is_python2:\n        return b_str\n    # Important: if no encoding is set, string becomes \"b'...'\"\n    return str(b_str, encoding=\"utf8\")\n\n\ndef symlink_to(orig, dest):\n    \"\"\"Create a symlink. Used for model shortcut links.\n\n    orig (unicode / Path): The origin path.\n    dest (unicode / Path): The destination path of the symlink.\n    \"\"\"\n    if is_windows:\n        import subprocess\n\n        subprocess.call([\"mklink\", \"/d\", path2str(orig), path2str(dest)], shell=True)\n    else:\n        orig.symlink_to(dest)\n\n\ndef symlink_remove(link):\n    \"\"\"Remove a symlink. Used for model shortcut links.\n\n    link (unicode / Path): The path to the symlink.\n    \"\"\"\n    # https://stackoverflow.com/q/26554135/6400719\n    if os.path.isdir(path2str(link)) and is_windows:\n        # this should only be on Py2.7 and windows\n        os.rmdir(path2str(link))\n    else:\n        os.unlink(path2str(link))\n\n\ndef is_config(python2=None, python3=None, windows=None, linux=None, osx=None):\n    \"\"\"Check if a specific configuration of Python version and operating system\n    matches the user's setup. Mostly used to display targeted error messages.\n\n    python2 (bool): spaCy is executed with Python 2.x.\n    python3 (bool): spaCy is executed with Python 3.x.\n    windows (bool): spaCy is executed on Windows.\n    linux (bool): spaCy is executed on Linux.\n    osx (bool): spaCy is executed on OS X or macOS.\n    RETURNS (bool): Whether the configuration matches the user's platform.\n\n    DOCS: https://spacy.io/api/top-level#compat.is_config\n    \"\"\"\n    return (\n        python2 in (None, is_python2)\n        and python3 in (None, is_python3)\n        and windows in (None, is_windows)\n        and linux in (None, is_linux)\n        and osx in (None, is_osx)\n    )\n\n\ndef import_file(name, loc):\n    \"\"\"Import module from a file. Used to load models from a directory.\n\n    name (unicode): Name of module to load.\n    loc (unicode / Path): Path to the file.\n    RETURNS: The loaded module.\n    \"\"\"\n    loc = path2str(loc)\n    if is_python_pre_3_5:\n        import imp\n\n        return imp.load_source(name, loc)\n    else:\n        import importlib.util\n\n        spec = importlib.util.spec_from_file_location(name, str(loc))\n        module = importlib.util.module_from_spec(spec)\n        spec.loader.exec_module(module)\n        return module\n\n\ndef unescape_unicode(string):\n    \"\"\"Python2.7's re module chokes when compiling patterns that have ranges\n    between escaped unicode codepoints if the two codepoints are unrecognised\n    in the unicode database. For instance:\n\n        re.compile('[\\\\uAA77-\\\\uAA79]').findall(\"hello\")\n\n    Ends up matching every character (on Python 2). This problem doesn't occur\n    if we're dealing with unicode literals.\n    \"\"\"\n    if string is None:\n        return string\n    # We only want to unescape the unicode, so we first must protect the other\n    # backslashes.\n    string = string.replace(\"\\\\\", \"\\\\\\\\\")\n    # Now we remove that protection for the unicode.\n    string = string.replace(\"\\\\\\\\u\", \"\\\\u\")\n    string = string.replace(\"\\\\\\\\U\", \"\\\\U\")\n    # Now we unescape by evaling the string with the AST. This can't execute\n    # code -- it only does the representational level.\n    return ast.literal_eval(\"u'''\" + string + \"'''\")\n", "path": "spacy/compat.py"}]}