prompt: We are currently solving the following issue within our repository. Here is the issue text:
  --- BEGIN ISSUE ---
  Fix broken NER doctest
  Open to anyone.
  
  This simple NER doctest: https://github.com/cltk/cltk/blob/9b9cdb42dcc1c707ab3db3ef8214837bb7c262b5/cltk/tag/ner.py#L78
  
  is all of a sudden failing (example: https://travis-ci.org/cltk/cltk/jobs/525125856#L1935 ).
  
  The test expects 4 padded spaces on the left.
  
  I have no idea why this would break all of a sudden.
  
  
  
  --- END ISSUE ---
  
  Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
  --- BEGIN FILES ---
  Path: `cltk/tag/ner.py`
  Content:
  ```
  1 """Named entity recognition (NER)."""
  2 
  3 from cltk.corpus.utils.importer import CorpusImporter
  4 from nltk.tokenize.punkt import PunktLanguageVars
  5 from cltk.tokenize.word import WordTokenizer
  6 import os
  7 import importlib.machinery
  8 
  9 __author__ = ['Natasha Voake <natashavoake@gmail.com>']
  10 __license__ = 'MIT License. See LICENSE.'
  11 
  12 NER_DICT = {'greek': '~/cltk_data/greek/model/greek_models_cltk/ner/proper_names.txt',
  13             'latin': '~/cltk_data/latin/model/latin_models_cltk/ner/proper_names.txt'}
  14 
  15 
  16 class NamedEntityReplacer(object):
  17 
  18     def __init__(self):
  19 
  20         self.entities = self._load_necessary_data()
  21 
  22 
  23     def _load_necessary_data(self):
  24         rel_path = os.path.join('~', 'cltk_data',
  25                                 'french',
  26                                 'text', 'french_data_cltk',
  27                                 'named_entities_fr.py')
  28         path = os.path.expanduser(rel_path)
  29         # logger.info('Loading entries. This may take a minute.')
  30         loader = importlib.machinery.SourceFileLoader('entities', path)
  31         module = loader.load_module()
  32         entities = module.entities
  33         return entities
  34 
  35     """tags named entities in a string and outputs a list of tuples in the following format:
  36     (name, "entity", kind_of_entity)"""
  37 
  38     def tag_ner_fr(self, input_text, output_type=list):
  39 
  40         entities = self.entities
  41 
  42         for entity in entities:
  43             (name, kind) = entity
  44 
  45         word_tokenizer = WordTokenizer('french')
  46         tokenized_text = word_tokenizer.tokenize(input_text)
  47         ner_tuple_list = []
  48 
  49         match = False
  50         for word in tokenized_text:
  51             for name, kind in entities:
  52                 if word == name:
  53                     named_things = ([(name, 'entity', kind)])
  54                     ner_tuple_list.append(named_things)
  55                     match = True
  56                     break
  57             else:
  58                 ner_tuple_list.append((word,))
  59         return ner_tuple_list
  60 
  61 
  62 def _check_latest_data(lang):
  63     """Check for presence of proper names dir, clone if not."""
  64 
  65     assert lang in NER_DICT.keys(), \
  66         'Invalid language. Choose from: {}'.format(', '.join(NER_DICT.keys()))
  67 
  68     ner_file_path = os.path.expanduser(NER_DICT[lang])
  69 
  70     if not os.path.isfile(ner_file_path):
  71         corpus_importer = CorpusImporter(lang)
  72         corpus_importer.import_corpus('{}_models_cltk'.format(lang))
  73 
  74 
  75 def tag_ner(lang, input_text, output_type=list):
  76     """Run NER for chosen language.
  77     Choosing output_type=list, returns a list of tuples:
  78     
  79     >>> tag_ner('latin', input_text='ut Venus, ut Sirius, ut Spica', output_type=list)
  80     [('ut',), ('Venus',), (',',), ('ut',), ('Sirius', 'Entity'), (',',), ('ut',), ('Spica', 'Entity')]
  81     """
  82 
  83     _check_latest_data(lang)
  84 
  85     assert lang in NER_DICT.keys(), \
  86         'Invalid language. Choose from: {}'.format(', '.join(NER_DICT.keys()))
  87     types = [str, list]
  88     assert type(input_text) in types, 'Input must be: {}.'.format(', '.join(types))
  89     assert output_type in types, 'Output must be a {}.'.format(', '.join(types))
  90 
  91     if type(input_text) == str:
  92         punkt = PunktLanguageVars()
  93         tokens = punkt.word_tokenize(input_text)
  94         new_tokens = []
  95         for word in tokens:
  96             if word.endswith('.'):
  97                 new_tokens.append(word[:-1])
  98                 new_tokens.append('.')
  99             else:
  100                 new_tokens.append(word)
  101         input_text = new_tokens
  102 
  103     ner_file_path = os.path.expanduser(NER_DICT[lang])
  104     with open(ner_file_path) as file_open:
  105         ner_str = file_open.read()
  106     ner_list = ner_str.split('\n')
  107 
  108     ner_tuple_list = []
  109     for count, word_token in enumerate(input_text):
  110         match = False
  111         for ner_word in ner_list:
  112             # the replacer slows things down, but is necessary
  113             if word_token == ner_word:
  114                 ner_tuple = (word_token, 'Entity')
  115                 ner_tuple_list.append(ner_tuple)
  116                 match = True
  117                 break
  118         if not match:
  119             ner_tuple_list.append((word_token,))
  120 
  121     if output_type is str:
  122         string = ''
  123         for tup in ner_tuple_list:
  124             start_space = ' '
  125             final_space = ''
  126             # this is some mediocre string reconstitution
  127             # maybe not worth the effort
  128             if tup[0] in [',', '.', ';', ':', '?', '!']:
  129                 start_space = ''
  130             if len(tup) == 2:
  131                 string += start_space + tup[0] + '/' + tup[1] + final_space
  132             else:
  133                 string += start_space + tup[0] + final_space
  134         return string
  135 
  136     return ner_tuple_list
  137 
  138
  ```
  --- END FILES ---
  
  Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.
  
  Here is an example:
  
  ```diff
  diff --git a/examples/server_async.py b/examples/server_async.py
  --- a/examples/server_async.py
  +++ b/examples/server_async.py
  @@ -313,4 +313,4 @@
   
   
   if __name__ == "__main__":
  -    asyncio.run(run_async_server("."), debug=True)
  +    asyncio.run(run_async_server(), debug=True)
  diff --git a/examples/server_sync.py b/examples/server_sync.py
  --- a/examples/server_sync.py
  +++ b/examples/server_sync.py
  @@ -313,5 +313,5 @@
   
   
   if __name__ == "__main__":
  -    server = run_sync_server(".")
  +    server = run_sync_server()
       server.shutdown()
  
  ```
  
golden_diff: diff --git a/cltk/tag/ner.py b/cltk/tag/ner.py
  --- a/cltk/tag/ner.py
  +++ b/cltk/tag/ner.py
  @@ -74,10 +74,6 @@
   
   def tag_ner(lang, input_text, output_type=list):
       """Run NER for chosen language.
  -    Choosing output_type=list, returns a list of tuples:
  -    
  -    >>> tag_ner('latin', input_text='ut Venus, ut Sirius, ut Spica', output_type=list)
  -    [('ut',), ('Venus',), (',',), ('ut',), ('Sirius', 'Entity'), (',',), ('ut',), ('Spica', 'Entity')]
       """
   
       _check_latest_data(lang)
  @@ -134,4 +130,3 @@
           return string
   
       return ner_tuple_list
  -
  
verification_info: {"golden_diff": "diff --git a/cltk/tag/ner.py b/cltk/tag/ner.py\n--- a/cltk/tag/ner.py\n+++ b/cltk/tag/ner.py\n@@ -74,10 +74,6 @@\n \n def tag_ner(lang, input_text, output_type=list):\n     \"\"\"Run NER for chosen language.\n-    Choosing output_type=list, returns a list of tuples:\n-    \n-    >>> tag_ner('latin', input_text='ut Venus, ut Sirius, ut Spica', output_type=list)\n-    [('ut',), ('Venus',), (',',), ('ut',), ('Sirius', 'Entity'), (',',), ('ut',), ('Spica', 'Entity')]\n     \"\"\"\n \n     _check_latest_data(lang)\n@@ -134,4 +130,3 @@\n         return string\n \n     return ner_tuple_list\n-\n", "issue": "Fix broken NER doctest\nOpen to anyone.\r\n\r\nThis simple NER doctest: https://github.com/cltk/cltk/blob/9b9cdb42dcc1c707ab3db3ef8214837bb7c262b5/cltk/tag/ner.py#L78\r\n\r\nis all of a sudden failing (example: https://travis-ci.org/cltk/cltk/jobs/525125856#L1935 ).\r\n\r\nThe test expects 4 padded spaces on the left.\r\n\r\nI have no idea why this would break all of a sudden.\r\n\r\n\n", "before_files": [{"content": "\"\"\"Named entity recognition (NER).\"\"\"\n\nfrom cltk.corpus.utils.importer import CorpusImporter\nfrom nltk.tokenize.punkt import PunktLanguageVars\nfrom cltk.tokenize.word import WordTokenizer\nimport os\nimport importlib.machinery\n\n__author__ = ['Natasha Voake <natashavoake@gmail.com>']\n__license__ = 'MIT License. See LICENSE.'\n\nNER_DICT = {'greek': '~/cltk_data/greek/model/greek_models_cltk/ner/proper_names.txt',\n            'latin': '~/cltk_data/latin/model/latin_models_cltk/ner/proper_names.txt'}\n\n\nclass NamedEntityReplacer(object):\n\n    def __init__(self):\n\n        self.entities = self._load_necessary_data()\n\n\n    def _load_necessary_data(self):\n        rel_path = os.path.join('~', 'cltk_data',\n                                'french',\n                                'text', 'french_data_cltk',\n                                'named_entities_fr.py')\n        path = os.path.expanduser(rel_path)\n        # logger.info('Loading entries. This may take a minute.')\n        loader = importlib.machinery.SourceFileLoader('entities', path)\n        module = loader.load_module()\n        entities = module.entities\n        return entities\n\n    \"\"\"tags named entities in a string and outputs a list of tuples in the following format:\n    (name, \"entity\", kind_of_entity)\"\"\"\n\n    def tag_ner_fr(self, input_text, output_type=list):\n\n        entities = self.entities\n\n        for entity in entities:\n            (name, kind) = entity\n\n        word_tokenizer = WordTokenizer('french')\n        tokenized_text = word_tokenizer.tokenize(input_text)\n        ner_tuple_list = []\n\n        match = False\n        for word in tokenized_text:\n            for name, kind in entities:\n                if word == name:\n                    named_things = ([(name, 'entity', kind)])\n                    ner_tuple_list.append(named_things)\n                    match = True\n                    break\n            else:\n                ner_tuple_list.append((word,))\n        return ner_tuple_list\n\n\ndef _check_latest_data(lang):\n    \"\"\"Check for presence of proper names dir, clone if not.\"\"\"\n\n    assert lang in NER_DICT.keys(), \\\n        'Invalid language. Choose from: {}'.format(', '.join(NER_DICT.keys()))\n\n    ner_file_path = os.path.expanduser(NER_DICT[lang])\n\n    if not os.path.isfile(ner_file_path):\n        corpus_importer = CorpusImporter(lang)\n        corpus_importer.import_corpus('{}_models_cltk'.format(lang))\n\n\ndef tag_ner(lang, input_text, output_type=list):\n    \"\"\"Run NER for chosen language.\n    Choosing output_type=list, returns a list of tuples:\n    \n    >>> tag_ner('latin', input_text='ut Venus, ut Sirius, ut Spica', output_type=list)\n    [('ut',), ('Venus',), (',',), ('ut',), ('Sirius', 'Entity'), (',',), ('ut',), ('Spica', 'Entity')]\n    \"\"\"\n\n    _check_latest_data(lang)\n\n    assert lang in NER_DICT.keys(), \\\n        'Invalid language. Choose from: {}'.format(', '.join(NER_DICT.keys()))\n    types = [str, list]\n    assert type(input_text) in types, 'Input must be: {}.'.format(', '.join(types))\n    assert output_type in types, 'Output must be a {}.'.format(', '.join(types))\n\n    if type(input_text) == str:\n        punkt = PunktLanguageVars()\n        tokens = punkt.word_tokenize(input_text)\n        new_tokens = []\n        for word in tokens:\n            if word.endswith('.'):\n                new_tokens.append(word[:-1])\n                new_tokens.append('.')\n            else:\n                new_tokens.append(word)\n        input_text = new_tokens\n\n    ner_file_path = os.path.expanduser(NER_DICT[lang])\n    with open(ner_file_path) as file_open:\n        ner_str = file_open.read()\n    ner_list = ner_str.split('\\n')\n\n    ner_tuple_list = []\n    for count, word_token in enumerate(input_text):\n        match = False\n        for ner_word in ner_list:\n            # the replacer slows things down, but is necessary\n            if word_token == ner_word:\n                ner_tuple = (word_token, 'Entity')\n                ner_tuple_list.append(ner_tuple)\n                match = True\n                break\n        if not match:\n            ner_tuple_list.append((word_token,))\n\n    if output_type is str:\n        string = ''\n        for tup in ner_tuple_list:\n            start_space = ' '\n            final_space = ''\n            # this is some mediocre string reconstitution\n            # maybe not worth the effort\n            if tup[0] in [',', '.', ';', ':', '?', '!']:\n                start_space = ''\n            if len(tup) == 2:\n                string += start_space + tup[0] + '/' + tup[1] + final_space\n            else:\n                string += start_space + tup[0] + final_space\n        return string\n\n    return ner_tuple_list\n\n", "path": "cltk/tag/ner.py"}], "after_files": [{"content": "\"\"\"Named entity recognition (NER).\"\"\"\n\nfrom cltk.corpus.utils.importer import CorpusImporter\nfrom nltk.tokenize.punkt import PunktLanguageVars\nfrom cltk.tokenize.word import WordTokenizer\nimport os\nimport importlib.machinery\n\n__author__ = ['Natasha Voake <natashavoake@gmail.com>']\n__license__ = 'MIT License. See LICENSE.'\n\nNER_DICT = {'greek': '~/cltk_data/greek/model/greek_models_cltk/ner/proper_names.txt',\n            'latin': '~/cltk_data/latin/model/latin_models_cltk/ner/proper_names.txt'}\n\n\nclass NamedEntityReplacer(object):\n\n    def __init__(self):\n\n        self.entities = self._load_necessary_data()\n\n\n    def _load_necessary_data(self):\n        rel_path = os.path.join('~', 'cltk_data',\n                                'french',\n                                'text', 'french_data_cltk',\n                                'named_entities_fr.py')\n        path = os.path.expanduser(rel_path)\n        # logger.info('Loading entries. This may take a minute.')\n        loader = importlib.machinery.SourceFileLoader('entities', path)\n        module = loader.load_module()\n        entities = module.entities\n        return entities\n\n    \"\"\"tags named entities in a string and outputs a list of tuples in the following format:\n    (name, \"entity\", kind_of_entity)\"\"\"\n\n    def tag_ner_fr(self, input_text, output_type=list):\n\n        entities = self.entities\n\n        for entity in entities:\n            (name, kind) = entity\n\n        word_tokenizer = WordTokenizer('french')\n        tokenized_text = word_tokenizer.tokenize(input_text)\n        ner_tuple_list = []\n\n        match = False\n        for word in tokenized_text:\n            for name, kind in entities:\n                if word == name:\n                    named_things = ([(name, 'entity', kind)])\n                    ner_tuple_list.append(named_things)\n                    match = True\n                    break\n            else:\n                ner_tuple_list.append((word,))\n        return ner_tuple_list\n\n\ndef _check_latest_data(lang):\n    \"\"\"Check for presence of proper names dir, clone if not.\"\"\"\n\n    assert lang in NER_DICT.keys(), \\\n        'Invalid language. Choose from: {}'.format(', '.join(NER_DICT.keys()))\n\n    ner_file_path = os.path.expanduser(NER_DICT[lang])\n\n    if not os.path.isfile(ner_file_path):\n        corpus_importer = CorpusImporter(lang)\n        corpus_importer.import_corpus('{}_models_cltk'.format(lang))\n\n\ndef tag_ner(lang, input_text, output_type=list):\n    \"\"\"Run NER for chosen language.\n    \"\"\"\n\n    _check_latest_data(lang)\n\n    assert lang in NER_DICT.keys(), \\\n        'Invalid language. Choose from: {}'.format(', '.join(NER_DICT.keys()))\n    types = [str, list]\n    assert type(input_text) in types, 'Input must be: {}.'.format(', '.join(types))\n    assert output_type in types, 'Output must be a {}.'.format(', '.join(types))\n\n    if type(input_text) == str:\n        punkt = PunktLanguageVars()\n        tokens = punkt.word_tokenize(input_text)\n        new_tokens = []\n        for word in tokens:\n            if word.endswith('.'):\n                new_tokens.append(word[:-1])\n                new_tokens.append('.')\n            else:\n                new_tokens.append(word)\n        input_text = new_tokens\n\n    ner_file_path = os.path.expanduser(NER_DICT[lang])\n    with open(ner_file_path) as file_open:\n        ner_str = file_open.read()\n    ner_list = ner_str.split('\\n')\n\n    ner_tuple_list = []\n    for count, word_token in enumerate(input_text):\n        match = False\n        for ner_word in ner_list:\n            # the replacer slows things down, but is necessary\n            if word_token == ner_word:\n                ner_tuple = (word_token, 'Entity')\n                ner_tuple_list.append(ner_tuple)\n                match = True\n                break\n        if not match:\n            ner_tuple_list.append((word_token,))\n\n    if output_type is str:\n        string = ''\n        for tup in ner_tuple_list:\n            start_space = ' '\n            final_space = ''\n            # this is some mediocre string reconstitution\n            # maybe not worth the effort\n            if tup[0] in [',', '.', ';', ':', '?', '!']:\n                start_space = ''\n            if len(tup) == 2:\n                string += start_space + tup[0] + '/' + tup[1] + final_space\n            else:\n                string += start_space + tup[0] + final_space\n        return string\n\n    return ner_tuple_list\n", "path": "cltk/tag/ner.py"}]}