prompt: We are currently solving the following issue within our repository. Here is the issue text:
  --- BEGIN ISSUE ---
  External punctuation stopped working on Latin sent tokenizer
  Recently reviewing the tokenizer, and it is not capturing exclamation points. I'll look to see the NLTK has changed anything.
  ``` python
  In [12]: text = """quam penitus maestas exedit cura medullas! ut tibi tunc toto 
      ...: pectore sollicitae sensibus ereptis mens excidit! at ego certe cognoram
      ...:  a parva virgine magnanimam. Mam. Aemilius ad castra venit."""
  
  In [13]:  tokenizer.tokenize_sentences(text)
  Out[13]: 
  ['quam penitus maestas exedit cura medullas! ut tibi tunc toto pectore sollicitae sensibus ereptis mens excidit! at ego certe cognoram a parva virgine magnanimam.',
   'Mam. Aemilius ad castra venit.']
  ```
  
  --- END ISSUE ---
  
  Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
  --- BEGIN FILES ---
  Path: `cltk/tokenize/sentence.py`
  Content:
  ```
  1 """Tokenize sentences."""
  2 
  3 __author__ = 'Kyle P. Johnson <kyle@kyle-p-johnson.com>'
  4 __license__ = 'MIT License. See LICENSE.'
  5 
  6 
  7 from cltk.utils.file_operations import open_pickle
  8 from nltk.tokenize.punkt import PunktLanguageVars
  9 from nltk.tokenize.punkt import PunktSentenceTokenizer
  10 import os
  11 
  12 
  13 PUNCTUATION = {'greek':
  14                    {'external': ('.', ';'),
  15                     'internal': (',', '·'),
  16                     'file': 'greek.pickle', },
  17                'latin':
  18                    {'external': ('.', '?', ':'),
  19                     'internal': (',', ';'),
  20                     'file': 'latin.pickle', }}
  21 
  22 
  23 class TokenizeSentence():  # pylint: disable=R0903
  24     """Tokenize sentences for the language given as argument, e.g.,
  25     ``TokenizeSentence('greek')``.
  26     """
  27 
  28     def __init__(self: object, language: str):
  29         """Lower incoming language name and assemble variables.
  30         :type language: str
  31         :param language : Language for sentence tokenization.
  32         """
  33         self.language = language.lower()
  34         self.internal_punctuation, self.external_punctuation, self.tokenizer_path = \
  35             self._setup_language_variables(self.language)
  36 
  37     def _setup_language_variables(self, lang: str):
  38         """Check for language availability and presence of tokenizer file,
  39         then read punctuation characters for language and build tokenizer file
  40         path.
  41         :param lang: The language argument given to the class.
  42         :type lang: str
  43         :rtype (str, str, str)
  44         """
  45         assert lang in PUNCTUATION.keys(), \
  46             'Sentence tokenizer not available for {0} language.'.format(lang)
  47         internal_punctuation = PUNCTUATION[lang]['internal']
  48         external_punctuation = PUNCTUATION[lang]['external']
  49         file = PUNCTUATION[lang]['file']
  50         rel_path = os.path.join('~/cltk_data',
  51                                 lang,
  52                                 'model/' + lang + '_models_cltk/tokenizers/sentence')  # pylint: disable=C0301
  53         path = os.path.expanduser(rel_path)
  54         tokenizer_path = os.path.join(path, file)
  55         assert os.path.isfile(tokenizer_path), \
  56             'CLTK linguistics data not found for language {0}'.format(lang)
  57         return internal_punctuation, external_punctuation, tokenizer_path
  58 
  59     def _setup_tokenizer(self, tokenizer: object):
  60         """Add tokenizer and punctuation variables.
  61         :type tokenizer: object
  62         :param tokenizer : Unpickled tokenizer object.
  63         :rtype : object
  64         """
  65         language_punkt_vars = PunktLanguageVars
  66         language_punkt_vars.sent_end_chars = self.external_punctuation
  67         language_punkt_vars.internal_punctuation = self.internal_punctuation
  68         tokenizer.INCLUDE_ALL_COLLOCS = True
  69         tokenizer.INCLUDE_ABBREV_COLLOCS = True
  70         params = tokenizer.get_params()
  71         return PunktSentenceTokenizer(params)
  72 
  73     def tokenize_sentences(self: object, untokenized_string: str):
  74         """Tokenize sentences by reading trained tokenizer and invoking
  75         ``PunktSentenceTokenizer()``.
  76         :type untokenized_string: str
  77         :param untokenized_string: A string containing one of more sentences.
  78         :rtype : list of strings
  79         """
  80         # load tokenizer
  81         assert isinstance(untokenized_string, str), \
  82             'Incoming argument must be a string.'
  83         tokenizer = open_pickle(self.tokenizer_path)
  84         tokenizer = self._setup_tokenizer(tokenizer)
  85 
  86         # mk list of tokenized sentences
  87         tokenized_sentences = []
  88         for sentence in tokenizer.sentences_from_text(untokenized_string, realign_boundaries=True):  # pylint: disable=C0301
  89             tokenized_sentences.append(sentence)
  90         return tokenized_sentences
  91         
  92     def tokenize(self: object, untokenized_string: str):
  93         # NLTK's PlaintextCorpusReader needs a function called tokenize
  94         # in functions used as a parameter for sentence tokenization.
  95         # So this is an alias for tokenize_sentences().
  96         return self.tokenize_sentences(untokenized_string)
  97
  ```
  --- END FILES ---
  
  Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.
  
  Here is an example:
  
  ```diff
  diff --git a/examples/server_async.py b/examples/server_async.py
  --- a/examples/server_async.py
  +++ b/examples/server_async.py
  @@ -313,4 +313,4 @@
   
   
   if __name__ == "__main__":
  -    asyncio.run(run_async_server("."), debug=True)
  +    asyncio.run(run_async_server(), debug=True)
  diff --git a/examples/server_sync.py b/examples/server_sync.py
  --- a/examples/server_sync.py
  +++ b/examples/server_sync.py
  @@ -313,5 +313,5 @@
   
   
   if __name__ == "__main__":
  -    server = run_sync_server(".")
  +    server = run_sync_server()
       server.shutdown()
  
  ```
  
golden_diff: diff --git a/cltk/tokenize/sentence.py b/cltk/tokenize/sentence.py
  --- a/cltk/tokenize/sentence.py
  +++ b/cltk/tokenize/sentence.py
  @@ -15,7 +15,7 @@
                       'internal': (',', '·'),
                       'file': 'greek.pickle', },
                  'latin':
  -                   {'external': ('.', '?', ':'),
  +                   {'external': ('.', '?', '!', ':'),
                       'internal': (',', ';'),
                       'file': 'latin.pickle', }}
  
verification_info: {"golden_diff": "diff --git a/cltk/tokenize/sentence.py b/cltk/tokenize/sentence.py\n--- a/cltk/tokenize/sentence.py\n+++ b/cltk/tokenize/sentence.py\n@@ -15,7 +15,7 @@\n                     'internal': (',', '\u00b7'),\n                     'file': 'greek.pickle', },\n                'latin':\n-                   {'external': ('.', '?', ':'),\n+                   {'external': ('.', '?', '!', ':'),\n                     'internal': (',', ';'),\n                     'file': 'latin.pickle', }}\n", "issue": "External punctuation stopped working on Latin sent tokenizer\nRecently reviewing the tokenizer, and it is not capturing exclamation points. I'll look to see the NLTK has changed anything.\r\n``` python\r\nIn [12]: text = \"\"\"quam penitus maestas exedit cura medullas! ut tibi tunc toto \r\n    ...: pectore sollicitae sensibus ereptis mens excidit! at ego certe cognoram\r\n    ...:  a parva virgine magnanimam. Mam. Aemilius ad castra venit.\"\"\"\r\n\r\nIn [13]:  tokenizer.tokenize_sentences(text)\r\nOut[13]: \r\n['quam penitus maestas exedit cura medullas! ut tibi tunc toto pectore sollicitae sensibus ereptis mens excidit! at ego certe cognoram a parva virgine magnanimam.',\r\n 'Mam. Aemilius ad castra venit.']\r\n```\n", "before_files": [{"content": "\"\"\"Tokenize sentences.\"\"\"\n\n__author__ = 'Kyle P. Johnson <kyle@kyle-p-johnson.com>'\n__license__ = 'MIT License. See LICENSE.'\n\n\nfrom cltk.utils.file_operations import open_pickle\nfrom nltk.tokenize.punkt import PunktLanguageVars\nfrom nltk.tokenize.punkt import PunktSentenceTokenizer\nimport os\n\n\nPUNCTUATION = {'greek':\n                   {'external': ('.', ';'),\n                    'internal': (',', '\u00b7'),\n                    'file': 'greek.pickle', },\n               'latin':\n                   {'external': ('.', '?', ':'),\n                    'internal': (',', ';'),\n                    'file': 'latin.pickle', }}\n\n\nclass TokenizeSentence():  # pylint: disable=R0903\n    \"\"\"Tokenize sentences for the language given as argument, e.g.,\n    ``TokenizeSentence('greek')``.\n    \"\"\"\n\n    def __init__(self: object, language: str):\n        \"\"\"Lower incoming language name and assemble variables.\n        :type language: str\n        :param language : Language for sentence tokenization.\n        \"\"\"\n        self.language = language.lower()\n        self.internal_punctuation, self.external_punctuation, self.tokenizer_path = \\\n            self._setup_language_variables(self.language)\n\n    def _setup_language_variables(self, lang: str):\n        \"\"\"Check for language availability and presence of tokenizer file,\n        then read punctuation characters for language and build tokenizer file\n        path.\n        :param lang: The language argument given to the class.\n        :type lang: str\n        :rtype (str, str, str)\n        \"\"\"\n        assert lang in PUNCTUATION.keys(), \\\n            'Sentence tokenizer not available for {0} language.'.format(lang)\n        internal_punctuation = PUNCTUATION[lang]['internal']\n        external_punctuation = PUNCTUATION[lang]['external']\n        file = PUNCTUATION[lang]['file']\n        rel_path = os.path.join('~/cltk_data',\n                                lang,\n                                'model/' + lang + '_models_cltk/tokenizers/sentence')  # pylint: disable=C0301\n        path = os.path.expanduser(rel_path)\n        tokenizer_path = os.path.join(path, file)\n        assert os.path.isfile(tokenizer_path), \\\n            'CLTK linguistics data not found for language {0}'.format(lang)\n        return internal_punctuation, external_punctuation, tokenizer_path\n\n    def _setup_tokenizer(self, tokenizer: object):\n        \"\"\"Add tokenizer and punctuation variables.\n        :type tokenizer: object\n        :param tokenizer : Unpickled tokenizer object.\n        :rtype : object\n        \"\"\"\n        language_punkt_vars = PunktLanguageVars\n        language_punkt_vars.sent_end_chars = self.external_punctuation\n        language_punkt_vars.internal_punctuation = self.internal_punctuation\n        tokenizer.INCLUDE_ALL_COLLOCS = True\n        tokenizer.INCLUDE_ABBREV_COLLOCS = True\n        params = tokenizer.get_params()\n        return PunktSentenceTokenizer(params)\n\n    def tokenize_sentences(self: object, untokenized_string: str):\n        \"\"\"Tokenize sentences by reading trained tokenizer and invoking\n        ``PunktSentenceTokenizer()``.\n        :type untokenized_string: str\n        :param untokenized_string: A string containing one of more sentences.\n        :rtype : list of strings\n        \"\"\"\n        # load tokenizer\n        assert isinstance(untokenized_string, str), \\\n            'Incoming argument must be a string.'\n        tokenizer = open_pickle(self.tokenizer_path)\n        tokenizer = self._setup_tokenizer(tokenizer)\n\n        # mk list of tokenized sentences\n        tokenized_sentences = []\n        for sentence in tokenizer.sentences_from_text(untokenized_string, realign_boundaries=True):  # pylint: disable=C0301\n            tokenized_sentences.append(sentence)\n        return tokenized_sentences\n        \n    def tokenize(self: object, untokenized_string: str):\n        # NLTK's PlaintextCorpusReader needs a function called tokenize\n        # in functions used as a parameter for sentence tokenization.\n        # So this is an alias for tokenize_sentences().\n        return self.tokenize_sentences(untokenized_string)\n", "path": "cltk/tokenize/sentence.py"}], "after_files": [{"content": "\"\"\"Tokenize sentences.\"\"\"\n\n__author__ = 'Kyle P. Johnson <kyle@kyle-p-johnson.com>'\n__license__ = 'MIT License. See LICENSE.'\n\n\nfrom cltk.utils.file_operations import open_pickle\nfrom nltk.tokenize.punkt import PunktLanguageVars\nfrom nltk.tokenize.punkt import PunktSentenceTokenizer\nimport os\n\n\nPUNCTUATION = {'greek':\n                   {'external': ('.', ';'),\n                    'internal': (',', '\u00b7'),\n                    'file': 'greek.pickle', },\n               'latin':\n                   {'external': ('.', '?', '!', ':'),\n                    'internal': (',', ';'),\n                    'file': 'latin.pickle', }}\n\n\nclass TokenizeSentence():  # pylint: disable=R0903\n    \"\"\"Tokenize sentences for the language given as argument, e.g.,\n    ``TokenizeSentence('greek')``.\n    \"\"\"\n\n    def __init__(self: object, language: str):\n        \"\"\"Lower incoming language name and assemble variables.\n        :type language: str\n        :param language : Language for sentence tokenization.\n        \"\"\"\n        self.language = language.lower()\n        self.internal_punctuation, self.external_punctuation, self.tokenizer_path = \\\n            self._setup_language_variables(self.language)\n\n    def _setup_language_variables(self, lang: str):\n        \"\"\"Check for language availability and presence of tokenizer file,\n        then read punctuation characters for language and build tokenizer file\n        path.\n        :param lang: The language argument given to the class.\n        :type lang: str\n        :rtype (str, str, str)\n        \"\"\"\n        assert lang in PUNCTUATION.keys(), \\\n            'Sentence tokenizer not available for {0} language.'.format(lang)\n        internal_punctuation = PUNCTUATION[lang]['internal']\n        external_punctuation = PUNCTUATION[lang]['external']\n        file = PUNCTUATION[lang]['file']\n        rel_path = os.path.join('~/cltk_data',\n                                lang,\n                                'model/' + lang + '_models_cltk/tokenizers/sentence')  # pylint: disable=C0301\n        path = os.path.expanduser(rel_path)\n        tokenizer_path = os.path.join(path, file)\n        assert os.path.isfile(tokenizer_path), \\\n            'CLTK linguistics data not found for language {0}'.format(lang)\n        return internal_punctuation, external_punctuation, tokenizer_path\n\n    def _setup_tokenizer(self, tokenizer: object):\n        \"\"\"Add tokenizer and punctuation variables.\n        :type tokenizer: object\n        :param tokenizer : Unpickled tokenizer object.\n        :rtype : object\n        \"\"\"\n        language_punkt_vars = PunktLanguageVars\n        language_punkt_vars.sent_end_chars = self.external_punctuation\n        language_punkt_vars.internal_punctuation = self.internal_punctuation\n        tokenizer.INCLUDE_ALL_COLLOCS = True\n        tokenizer.INCLUDE_ABBREV_COLLOCS = True\n        params = tokenizer.get_params()\n        return PunktSentenceTokenizer(params)\n\n    def tokenize_sentences(self: object, untokenized_string: str):\n        \"\"\"Tokenize sentences by reading trained tokenizer and invoking\n        ``PunktSentenceTokenizer()``.\n        :type untokenized_string: str\n        :param untokenized_string: A string containing one of more sentences.\n        :rtype : list of strings\n        \"\"\"\n        # load tokenizer\n        assert isinstance(untokenized_string, str), \\\n            'Incoming argument must be a string.'\n        tokenizer = open_pickle(self.tokenizer_path)\n        tokenizer = self._setup_tokenizer(tokenizer)\n\n        # mk list of tokenized sentences\n        tokenized_sentences = []\n        for sentence in tokenizer.sentences_from_text(untokenized_string, realign_boundaries=True):  # pylint: disable=C0301\n            tokenized_sentences.append(sentence)\n        return tokenized_sentences\n        \n    def tokenize(self: object, untokenized_string: str):\n        # NLTK's PlaintextCorpusReader needs a function called tokenize\n        # in functions used as a parameter for sentence tokenization.\n        # So this is an alias for tokenize_sentences().\n        return self.tokenize_sentences(untokenized_string)\n", "path": "cltk/tokenize/sentence.py"}]}