We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
Hyphen added when separating 'ne' enclitics at sentence start
**Describe the bug**
Hyphen added when separating '-ne' enclitics at sentence start

**To Reproduce**
Python 3.9.9; CLTK 1.0.22

```
test = 'nihilne te nocturnum praesidium Palati'
t = LatinWordTokenizer()
print(t.tokenize(test))
```
Current output:
['nihil', '-', '-ne', 'te', 'nocturnum', 'praesidium', 'Palati']

4. See error (include literal copy-paste)

Expected output:
['nihil', '-ne', 'te', 'nocturnum', 'praesidium', 'Palati']

**Desktop (please complete the following information):**
 - MacOS 12.1

**Additional context**
Issue is that there is a redundant hyphen being added in cases where '-ne' is separated as part of the first token in a sentence. I will submit a PR to fix this.


--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `src/cltk/tokenizers/lat/lat.py`
Content:
```
1 """ Latin word tokenization - handles enclitics and abbreviations."""
2 
3 __author__ = [
4     "Patrick J. Burns <patrick@diyclassics.org>",
5     "Todd Cook <todd.g.cook@gmail.com",
6 ]
7 __license__ = "MIT License."
8 
9 import re
10 from typing import List, Tuple
11 
12 from nltk.tokenize.punkt import PunktLanguageVars, PunktParameters
13 
14 from cltk.sentence.lat import LatinPunktSentenceTokenizer
15 from cltk.tokenizers.lat.params import ABBREVIATIONS, latin_exceptions
16 from cltk.tokenizers.lat.params import latin_replacements as REPLACEMENTS
17 from cltk.tokenizers.word import WordTokenizer
18 
19 
20 class LatinLanguageVars(PunktLanguageVars):
21     _re_non_word_chars = PunktLanguageVars()._re_non_word_chars.replace("'", "")
22 
23 
24 class LatinWordTokenizer(WordTokenizer):
25     """Tokenize according to rules specific to a given language."""
26 
27     ENCLITICS = ["que", "n", "ne", "ue", "ve", "st"]
28 
29     EXCEPTIONS = list(set(ENCLITICS + latin_exceptions))
30 
31     def __init__(self):
32         self.punkt_param = PunktParameters()
33         self.punkt_param.abbrev_types = set(ABBREVIATIONS)
34         self.sent_tokenizer = LatinPunktSentenceTokenizer()
35         self.word_tokenizer = LatinLanguageVars()
36 
37     def tokenize(
38         self,
39         text: str,
40         replacements: List[Tuple[str, str]] = REPLACEMENTS,
41         enclitics_exceptions: List[str] = EXCEPTIONS,
42         enclitics: List[str] = ENCLITICS,
43     ) -> List[str]:
44         """
45         Tokenizer divides the text into a list of substrings
46 
47         :param text: This accepts the string value that needs to be tokenized
48         :param replacements: List of replacements to apply to tokens such as "mecum" -> ["cum", "me"]
49         :param enclitics_exceptions: List of words that look likes they end with an enclitic but are not.
50         :param enclitics: List of enclitics to check for in tokenization
51 
52         :returns: A list of substrings extracted from the text
53 
54         >>> toker = LatinWordTokenizer()
55         >>> text = 'atque haec abuterque puerve paterne nihil'
56         >>> toker.tokenize(text)
57         ['atque', 'haec', 'abuter', '-que', 'puer', '-ve', 'pater', '-ne', 'nihil']
58 
59         >>> toker.tokenize('Cicero dixit orationem pro Sex. Roscio')
60         ['Cicero', 'dixit', 'orationem', 'pro', 'Sex.', 'Roscio']
61 
62         >>> toker.tokenize('Cenavin ego heri in navi in portu Persico?')
63         ['Cenavi', '-ne', 'ego', 'heri', 'in', 'navi', 'in', 'portu', 'Persico', '?']
64 
65         >>> toker.tokenize('Dic si audes mihi, bellan videtur specie mulier?')
66         ['Dic', 'si', 'audes', 'mihi', ',', 'bella', '-ne', 'videtur', 'specie', 'mulier', '?']
67 
68         >>> toker.tokenize("mecum")
69         ['cum', 'me']
70 
71         You can specify how replacements are made using replacements
72 
73         >>> toker.tokenize("mecum", replacements=[(r"mecum", "me cum")])
74         ['me', 'cum']
75 
76         Or change enclitics and enclitics exception:
77         >>> toker.tokenize("atque haec abuterque puerve paterne nihil", enclitics=["que"])
78         ['atque', 'haec', 'abuter', '-que', 'puerve', 'paterne', 'nihil']
79 
80         >>> toker.tokenize("atque haec abuterque puerve paterne nihil", enclitics=["que", "ve", "ne"],
81         ...    enclitics_exceptions=('paterne', 'atque'))
82         ['atque', 'haec', 'abuter', '-que', 'puer', '-ve', 'paterne', 'nihil']
83 
84         """
85 
86         def matchcase(word):
87             """helper function From Python Cookbook"""
88 
89             def replace(matching):
90                 text = matching.group()
91                 if text.isupper():
92                     return word.upper()
93                 elif text.islower():
94                     return word.lower()
95                 elif text[0].isupper():
96                     return word.capitalize()
97                 return word
98 
99             return replace
100 
101         for replacement in replacements:
102             text = re.sub(
103                 replacement[0], matchcase(replacement[1]), text, flags=re.IGNORECASE
104             )
105 
106         sents = self.sent_tokenizer.tokenize(text)
107         tokens = []  # type: List[str]
108 
109         for sent in sents:
110             temp_tokens = self.word_tokenizer.word_tokenize(sent)
111             # Need to check that tokens exist before handling them;
112             # needed to make stream.readlines work in PlaintextCorpusReader
113             if temp_tokens:
114                 if temp_tokens[0].endswith("ne"):
115                     if temp_tokens[0].lower() not in enclitics_exceptions:
116                         temp = [temp_tokens[0][:-2], "-ne"]
117                         temp_tokens = temp + temp_tokens[1:]
118                 if temp_tokens[-1].endswith("."):
119                     final_word = temp_tokens[-1][:-1]
120                     del temp_tokens[-1]
121                     temp_tokens += [final_word, "."]
122 
123                 for token in temp_tokens:
124                     tokens.append(token)
125 
126         # Break enclitic handling into own function?
127         specific_tokens = []  # type: List[str]
128 
129         for token in tokens:
130             is_enclitic = False
131             if token.lower() not in enclitics_exceptions:
132                 for enclitic in enclitics:
133                     if token.endswith(enclitic):
134                         if enclitic == "n":
135                             specific_tokens += [token[: -len(enclitic)]] + ["-ne"]
136                         elif enclitic == "st":
137                             if token.endswith("ust"):
138                                 specific_tokens += [token[: -len(enclitic) + 1]] + [
139                                     "est"
140                                 ]
141                             else:
142                                 specific_tokens += [token[: -len(enclitic)]] + ["est"]
143                         else:
144                             specific_tokens += [token[: -len(enclitic)]] + [
145                                 "-" + enclitic
146                             ]
147                         is_enclitic = True
148                         break
149             if not is_enclitic:
150                 specific_tokens.append(token)
151 
152         # collapse abbreviations
153         abbrev_idx = []
154         for idx, token in enumerate(specific_tokens):
155             if token.lower() in self.punkt_param.abbrev_types:
156                 abbrev_idx.append(idx)
157         for val in reversed(abbrev_idx):
158             if val + 1 < len(specific_tokens) and specific_tokens[val + 1] == ".":
159                 specific_tokens[val] = specific_tokens[val] + "."
160                 specific_tokens[val + 1] = ""
161         specific_tokens = [tmp for tmp in specific_tokens if tmp]
162         return specific_tokens
163 
164     @staticmethod
165     def compute_indices(text: str, tokens):
166         indices = []
167         for i, token in enumerate(tokens):
168             if 1 <= i:
169                 current_index = indices[-1] + len(tokens[i - 1])
170                 if token == "-ne":
171                     indices.append(current_index + text[current_index:].find(token[1:]))
172                 else:
173                     indices.append(current_index + text[current_index:].find(token))
174             else:
175                 indices.append(text.find(token))
176         return indices
177
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/src/cltk/tokenizers/lat/lat.py b/src/cltk/tokenizers/lat/lat.py
--- a/src/cltk/tokenizers/lat/lat.py
+++ b/src/cltk/tokenizers/lat/lat.py
@@ -59,6 +59,9 @@
         >>> toker.tokenize('Cicero dixit orationem pro Sex. Roscio')
         ['Cicero', 'dixit', 'orationem', 'pro', 'Sex.', 'Roscio']
 
+        >>> toker.tokenize('nihilne te nocturnum praesidium Palati')
+        ['nihil', '-ne', 'te', 'nocturnum', 'praesidium', 'Palati']
+
         >>> toker.tokenize('Cenavin ego heri in navi in portu Persico?')
         ['Cenavi', '-ne', 'ego', 'heri', 'in', 'navi', 'in', 'portu', 'Persico', '?']
 
@@ -141,9 +144,7 @@
                             else:
                                 specific_tokens += [token[: -len(enclitic)]] + ["est"]
                         else:
-                            specific_tokens += [token[: -len(enclitic)]] + [
-                                "-" + enclitic
-                            ]
+                            specific_tokens += [token]
                         is_enclitic = True
                         break
             if not is_enclitic:

{"golden_diff": "diff --git a/src/cltk/tokenizers/lat/lat.py b/src/cltk/tokenizers/lat/lat.py\n--- a/src/cltk/tokenizers/lat/lat.py\n+++ b/src/cltk/tokenizers/lat/lat.py\n@@ -59,6 +59,9 @@\n         >>> toker.tokenize('Cicero dixit orationem pro Sex. Roscio')\n         ['Cicero', 'dixit', 'orationem', 'pro', 'Sex.', 'Roscio']\n \n+        >>> toker.tokenize('nihilne te nocturnum praesidium Palati')\n+        ['nihil', '-ne', 'te', 'nocturnum', 'praesidium', 'Palati']\n+\n         >>> toker.tokenize('Cenavin ego heri in navi in portu Persico?')\n         ['Cenavi', '-ne', 'ego', 'heri', 'in', 'navi', 'in', 'portu', 'Persico', '?']\n \n@@ -141,9 +144,7 @@\n                             else:\n                                 specific_tokens += [token[: -len(enclitic)]] + [\"est\"]\n                         else:\n-                            specific_tokens += [token[: -len(enclitic)]] + [\n-                                \"-\" + enclitic\n-                            ]\n+                            specific_tokens += [token]\n                         is_enclitic = True\n                         break\n             if not is_enclitic:\n", "issue": "Hyphen added when separating 'ne' enclitics at sentence start\n**Describe the bug**\r\nHyphen added when separating '-ne' enclitics at sentence start\r\n\r\n**To Reproduce**\r\nPython 3.9.9; CLTK 1.0.22\r\n\r\n```\r\ntest = 'nihilne te nocturnum praesidium Palati'\r\nt = LatinWordTokenizer()\r\nprint(t.tokenize(test))\r\n```\r\nCurrent output:\r\n['nihil', '-', '-ne', 'te', 'nocturnum', 'praesidium', 'Palati']\r\n\r\n4. See error (include literal copy-paste)\r\n\r\nExpected output:\r\n['nihil', '-ne', 'te', 'nocturnum', 'praesidium', 'Palati']\r\n\r\n**Desktop (please complete the following information):**\r\n - MacOS 12.1\r\n\r\n**Additional context**\r\nIssue is that there is a redundant hyphen being added in cases where '-ne' is separated as part of the first token in a sentence. I will submit a PR to fix this.\r\n\n", "before_files": [{"content": "\"\"\" Latin word tokenization - handles enclitics and abbreviations.\"\"\"\n\n__author__ = [\n    \"Patrick J. Burns <patrick@diyclassics.org>\",\n    \"Todd Cook <todd.g.cook@gmail.com\",\n]\n__license__ = \"MIT License.\"\n\nimport re\nfrom typing import List, Tuple\n\nfrom nltk.tokenize.punkt import PunktLanguageVars, PunktParameters\n\nfrom cltk.sentence.lat import LatinPunktSentenceTokenizer\nfrom cltk.tokenizers.lat.params import ABBREVIATIONS, latin_exceptions\nfrom cltk.tokenizers.lat.params import latin_replacements as REPLACEMENTS\nfrom cltk.tokenizers.word import WordTokenizer\n\n\nclass LatinLanguageVars(PunktLanguageVars):\n    _re_non_word_chars = PunktLanguageVars()._re_non_word_chars.replace(\"'\", \"\")\n\n\nclass LatinWordTokenizer(WordTokenizer):\n    \"\"\"Tokenize according to rules specific to a given language.\"\"\"\n\n    ENCLITICS = [\"que\", \"n\", \"ne\", \"ue\", \"ve\", \"st\"]\n\n    EXCEPTIONS = list(set(ENCLITICS + latin_exceptions))\n\n    def __init__(self):\n        self.punkt_param = PunktParameters()\n        self.punkt_param.abbrev_types = set(ABBREVIATIONS)\n        self.sent_tokenizer = LatinPunktSentenceTokenizer()\n        self.word_tokenizer = LatinLanguageVars()\n\n    def tokenize(\n        self,\n        text: str,\n        replacements: List[Tuple[str, str]] = REPLACEMENTS,\n        enclitics_exceptions: List[str] = EXCEPTIONS,\n        enclitics: List[str] = ENCLITICS,\n    ) -> List[str]:\n        \"\"\"\n        Tokenizer divides the text into a list of substrings\n\n        :param text: This accepts the string value that needs to be tokenized\n        :param replacements: List of replacements to apply to tokens such as \"mecum\" -> [\"cum\", \"me\"]\n        :param enclitics_exceptions: List of words that look likes they end with an enclitic but are not.\n        :param enclitics: List of enclitics to check for in tokenization\n\n        :returns: A list of substrings extracted from the text\n\n        >>> toker = LatinWordTokenizer()\n        >>> text = 'atque haec abuterque puerve paterne nihil'\n        >>> toker.tokenize(text)\n        ['atque', 'haec', 'abuter', '-que', 'puer', '-ve', 'pater', '-ne', 'nihil']\n\n        >>> toker.tokenize('Cicero dixit orationem pro Sex. Roscio')\n        ['Cicero', 'dixit', 'orationem', 'pro', 'Sex.', 'Roscio']\n\n        >>> toker.tokenize('Cenavin ego heri in navi in portu Persico?')\n        ['Cenavi', '-ne', 'ego', 'heri', 'in', 'navi', 'in', 'portu', 'Persico', '?']\n\n        >>> toker.tokenize('Dic si audes mihi, bellan videtur specie mulier?')\n        ['Dic', 'si', 'audes', 'mihi', ',', 'bella', '-ne', 'videtur', 'specie', 'mulier', '?']\n\n        >>> toker.tokenize(\"mecum\")\n        ['cum', 'me']\n\n        You can specify how replacements are made using replacements\n\n        >>> toker.tokenize(\"mecum\", replacements=[(r\"mecum\", \"me cum\")])\n        ['me', 'cum']\n\n        Or change enclitics and enclitics exception:\n        >>> toker.tokenize(\"atque haec abuterque puerve paterne nihil\", enclitics=[\"que\"])\n        ['atque', 'haec', 'abuter', '-que', 'puerve', 'paterne', 'nihil']\n\n        >>> toker.tokenize(\"atque haec abuterque puerve paterne nihil\", enclitics=[\"que\", \"ve\", \"ne\"],\n        ...    enclitics_exceptions=('paterne', 'atque'))\n        ['atque', 'haec', 'abuter', '-que', 'puer', '-ve', 'paterne', 'nihil']\n\n        \"\"\"\n\n        def matchcase(word):\n            \"\"\"helper function From Python Cookbook\"\"\"\n\n            def replace(matching):\n                text = matching.group()\n                if text.isupper():\n                    return word.upper()\n                elif text.islower():\n                    return word.lower()\n                elif text[0].isupper():\n                    return word.capitalize()\n                return word\n\n            return replace\n\n        for replacement in replacements:\n            text = re.sub(\n                replacement[0], matchcase(replacement[1]), text, flags=re.IGNORECASE\n            )\n\n        sents = self.sent_tokenizer.tokenize(text)\n        tokens = []  # type: List[str]\n\n        for sent in sents:\n            temp_tokens = self.word_tokenizer.word_tokenize(sent)\n            # Need to check that tokens exist before handling them;\n            # needed to make stream.readlines work in PlaintextCorpusReader\n            if temp_tokens:\n                if temp_tokens[0].endswith(\"ne\"):\n                    if temp_tokens[0].lower() not in enclitics_exceptions:\n                        temp = [temp_tokens[0][:-2], \"-ne\"]\n                        temp_tokens = temp + temp_tokens[1:]\n                if temp_tokens[-1].endswith(\".\"):\n                    final_word = temp_tokens[-1][:-1]\n                    del temp_tokens[-1]\n                    temp_tokens += [final_word, \".\"]\n\n                for token in temp_tokens:\n                    tokens.append(token)\n\n        # Break enclitic handling into own function?\n        specific_tokens = []  # type: List[str]\n\n        for token in tokens:\n            is_enclitic = False\n            if token.lower() not in enclitics_exceptions:\n                for enclitic in enclitics:\n                    if token.endswith(enclitic):\n                        if enclitic == \"n\":\n                            specific_tokens += [token[: -len(enclitic)]] + [\"-ne\"]\n                        elif enclitic == \"st\":\n                            if token.endswith(\"ust\"):\n                                specific_tokens += [token[: -len(enclitic) + 1]] + [\n                                    \"est\"\n                                ]\n                            else:\n                                specific_tokens += [token[: -len(enclitic)]] + [\"est\"]\n                        else:\n                            specific_tokens += [token[: -len(enclitic)]] + [\n                                \"-\" + enclitic\n                            ]\n                        is_enclitic = True\n                        break\n            if not is_enclitic:\n                specific_tokens.append(token)\n\n        # collapse abbreviations\n        abbrev_idx = []\n        for idx, token in enumerate(specific_tokens):\n            if token.lower() in self.punkt_param.abbrev_types:\n                abbrev_idx.append(idx)\n        for val in reversed(abbrev_idx):\n            if val + 1 < len(specific_tokens) and specific_tokens[val + 1] == \".\":\n                specific_tokens[val] = specific_tokens[val] + \".\"\n                specific_tokens[val + 1] = \"\"\n        specific_tokens = [tmp for tmp in specific_tokens if tmp]\n        return specific_tokens\n\n    @staticmethod\n    def compute_indices(text: str, tokens):\n        indices = []\n        for i, token in enumerate(tokens):\n            if 1 <= i:\n                current_index = indices[-1] + len(tokens[i - 1])\n                if token == \"-ne\":\n                    indices.append(current_index + text[current_index:].find(token[1:]))\n                else:\n                    indices.append(current_index + text[current_index:].find(token))\n            else:\n                indices.append(text.find(token))\n        return indices\n", "path": "src/cltk/tokenizers/lat/lat.py"}], "after_files": [{"content": "\"\"\" Latin word tokenization - handles enclitics and abbreviations.\"\"\"\n\n__author__ = [\n    \"Patrick J. Burns <patrick@diyclassics.org>\",\n    \"Todd Cook <todd.g.cook@gmail.com\",\n]\n__license__ = \"MIT License.\"\n\nimport re\nfrom typing import List, Tuple\n\nfrom nltk.tokenize.punkt import PunktLanguageVars, PunktParameters\n\nfrom cltk.sentence.lat import LatinPunktSentenceTokenizer\nfrom cltk.tokenizers.lat.params import ABBREVIATIONS, latin_exceptions\nfrom cltk.tokenizers.lat.params import latin_replacements as REPLACEMENTS\nfrom cltk.tokenizers.word import WordTokenizer\n\n\nclass LatinLanguageVars(PunktLanguageVars):\n    _re_non_word_chars = PunktLanguageVars()._re_non_word_chars.replace(\"'\", \"\")\n\n\nclass LatinWordTokenizer(WordTokenizer):\n    \"\"\"Tokenize according to rules specific to a given language.\"\"\"\n\n    ENCLITICS = [\"que\", \"n\", \"ne\", \"ue\", \"ve\", \"st\"]\n\n    EXCEPTIONS = list(set(ENCLITICS + latin_exceptions))\n\n    def __init__(self):\n        self.punkt_param = PunktParameters()\n        self.punkt_param.abbrev_types = set(ABBREVIATIONS)\n        self.sent_tokenizer = LatinPunktSentenceTokenizer()\n        self.word_tokenizer = LatinLanguageVars()\n\n    def tokenize(\n        self,\n        text: str,\n        replacements: List[Tuple[str, str]] = REPLACEMENTS,\n        enclitics_exceptions: List[str] = EXCEPTIONS,\n        enclitics: List[str] = ENCLITICS,\n    ) -> List[str]:\n        \"\"\"\n        Tokenizer divides the text into a list of substrings\n\n        :param text: This accepts the string value that needs to be tokenized\n        :param replacements: List of replacements to apply to tokens such as \"mecum\" -> [\"cum\", \"me\"]\n        :param enclitics_exceptions: List of words that look likes they end with an enclitic but are not.\n        :param enclitics: List of enclitics to check for in tokenization\n\n        :returns: A list of substrings extracted from the text\n\n        >>> toker = LatinWordTokenizer()\n        >>> text = 'atque haec abuterque puerve paterne nihil'\n        >>> toker.tokenize(text)\n        ['atque', 'haec', 'abuter', '-que', 'puer', '-ve', 'pater', '-ne', 'nihil']\n\n        >>> toker.tokenize('Cicero dixit orationem pro Sex. Roscio')\n        ['Cicero', 'dixit', 'orationem', 'pro', 'Sex.', 'Roscio']\n\n        >>> toker.tokenize('nihilne te nocturnum praesidium Palati')\n        ['nihil', '-ne', 'te', 'nocturnum', 'praesidium', 'Palati']\n\n        >>> toker.tokenize('Cenavin ego heri in navi in portu Persico?')\n        ['Cenavi', '-ne', 'ego', 'heri', 'in', 'navi', 'in', 'portu', 'Persico', '?']\n\n        >>> toker.tokenize('Dic si audes mihi, bellan videtur specie mulier?')\n        ['Dic', 'si', 'audes', 'mihi', ',', 'bella', '-ne', 'videtur', 'specie', 'mulier', '?']\n\n        >>> toker.tokenize(\"mecum\")\n        ['cum', 'me']\n\n        You can specify how replacements are made using replacements\n\n        >>> toker.tokenize(\"mecum\", replacements=[(r\"mecum\", \"me cum\")])\n        ['me', 'cum']\n\n        Or change enclitics and enclitics exception:\n        >>> toker.tokenize(\"atque haec abuterque puerve paterne nihil\", enclitics=[\"que\"])\n        ['atque', 'haec', 'abuter', '-que', 'puerve', 'paterne', 'nihil']\n\n        >>> toker.tokenize(\"atque haec abuterque puerve paterne nihil\", enclitics=[\"que\", \"ve\", \"ne\"],\n        ...    enclitics_exceptions=('paterne', 'atque'))\n        ['atque', 'haec', 'abuter', '-que', 'puer', '-ve', 'paterne', 'nihil']\n\n        \"\"\"\n\n        def matchcase(word):\n            \"\"\"helper function From Python Cookbook\"\"\"\n\n            def replace(matching):\n                text = matching.group()\n                if text.isupper():\n                    return word.upper()\n                elif text.islower():\n                    return word.lower()\n                elif text[0].isupper():\n                    return word.capitalize()\n                return word\n\n            return replace\n\n        for replacement in replacements:\n            text = re.sub(\n                replacement[0], matchcase(replacement[1]), text, flags=re.IGNORECASE\n            )\n\n        sents = self.sent_tokenizer.tokenize(text)\n        tokens = []  # type: List[str]\n\n        for sent in sents:\n            temp_tokens = self.word_tokenizer.word_tokenize(sent)\n            # Need to check that tokens exist before handling them;\n            # needed to make stream.readlines work in PlaintextCorpusReader\n            if temp_tokens:\n                if temp_tokens[0].endswith(\"ne\"):\n                    if temp_tokens[0].lower() not in enclitics_exceptions:\n                        temp = [temp_tokens[0][:-2], \"-ne\"]\n                        temp_tokens = temp + temp_tokens[1:]\n                if temp_tokens[-1].endswith(\".\"):\n                    final_word = temp_tokens[-1][:-1]\n                    del temp_tokens[-1]\n                    temp_tokens += [final_word, \".\"]\n\n                for token in temp_tokens:\n                    tokens.append(token)\n\n        # Break enclitic handling into own function?\n        specific_tokens = []  # type: List[str]\n\n        for token in tokens:\n            is_enclitic = False\n            if token.lower() not in enclitics_exceptions:\n                for enclitic in enclitics:\n                    if token.endswith(enclitic):\n                        if enclitic == \"n\":\n                            specific_tokens += [token[: -len(enclitic)]] + [\"-ne\"]\n                        elif enclitic == \"st\":\n                            if token.endswith(\"ust\"):\n                                specific_tokens += [token[: -len(enclitic) + 1]] + [\n                                    \"est\"\n                                ]\n                            else:\n                                specific_tokens += [token[: -len(enclitic)]] + [\"est\"]\n                        else:\n                            specific_tokens += [token]\n                        is_enclitic = True\n                        break\n            if not is_enclitic:\n                specific_tokens.append(token)\n\n        # collapse abbreviations\n        abbrev_idx = []\n        for idx, token in enumerate(specific_tokens):\n            if token.lower() in self.punkt_param.abbrev_types:\n                abbrev_idx.append(idx)\n        for val in reversed(abbrev_idx):\n            if val + 1 < len(specific_tokens) and specific_tokens[val + 1] == \".\":\n                specific_tokens[val] = specific_tokens[val] + \".\"\n                specific_tokens[val + 1] = \"\"\n        specific_tokens = [tmp for tmp in specific_tokens if tmp]\n        return specific_tokens\n\n    @staticmethod\n    def compute_indices(text: str, tokens):\n        indices = []\n        for i, token in enumerate(tokens):\n            if 1 <= i:\n                current_index = indices[-1] + len(tokens[i - 1])\n                if token == \"-ne\":\n                    indices.append(current_index + text[current_index:].find(token[1:]))\n                else:\n                    indices.append(current_index + text[current_index:].find(token))\n            else:\n                indices.append(text.find(token))\n        return indices\n", "path": "src/cltk/tokenizers/lat/lat.py"}]}