We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
Unused imports in word tokenizer module
The word tokenizer module has a large number of imports from NLTK that are not used anywhere in the module. Removing them 1. cleans up the code, and 2. speeds up testing.

--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `cltk/tokenize/word.py`
Content:
```
1 # -*-coding:utf-8-*-
2 """Language-specific word tokenizers. Primary purpose is to handle enclitics."""
3 
4 import re
5 
6 from nltk.tokenize.punkt import PunktLanguageVars
7 from nltk.tokenize.punkt import PunktSentenceTokenizer, PunktParameters
8 
9 import re
10 
11 # Cleanup these imports—most are not used!
12 from nltk.data              import load
13 from nltk.tokenize.casual   import (TweetTokenizer, casual_tokenize)
14 from nltk.tokenize.mwe      import MWETokenizer
15 from nltk.tokenize.punkt    import PunktSentenceTokenizer
16 from nltk.tokenize.regexp   import (RegexpTokenizer, WhitespaceTokenizer,
17                                     BlanklineTokenizer, WordPunctTokenizer,
18                                     wordpunct_tokenize, regexp_tokenize,
19                                     blankline_tokenize)
20 #from nltk.tokenize.repp     import ReppTokenizer
21 from nltk.tokenize.sexpr    import SExprTokenizer, sexpr_tokenize
22 from nltk.tokenize.simple   import (SpaceTokenizer, TabTokenizer, LineTokenizer,
23                                     line_tokenize)
24 from nltk.tokenize.stanford import StanfordTokenizer
25 from nltk.tokenize.texttiling import TextTilingTokenizer
26 #from nltk.tokenize.toktok   import ToktokTokenizer
27 from nltk.tokenize.treebank import TreebankWordTokenizer
28 from nltk.tokenize.util     import string_span_tokenize, regexp_span_tokenize
29 from nltk.tokenize.stanford_segmenter import StanfordSegmenter
30 
31 import cltk.corpus.arabic.utils.pyarabic.araby as araby
32 
33 __author__ = ['Patrick J. Burns <patrick@diyclassics.org>', 'Kyle P. Johnson <kyle@kyle-p-johnson.com>',
34               'Natasha Voake <natashavoake@gmail.com>']
35 __license__ = 'MIT License. See LICENSE.'
36 
37 
38 class WordTokenizer:  # pylint: disable=too-few-public-methods
39     """Tokenize according to rules specific to a given language."""
40 
41     def __init__(self, language):
42         """Take language as argument to the class. Check availability and
43         setup class variables."""
44         self.language = language
45         self.available_languages = ['arabic', 
46                                     'french',
47                                     'greek',
48                                     'latin',
49                                     'old_norse']
50         assert self.language in self.available_languages, \
51             "Specific tokenizer not available for '{0}'. Only available for: '{1}'.".format(self.language,  # pylint: disable=line-too-long
52             self.available_languages)  # pylint: disable=line-too-long
53         # ^^^ Necessary? since we have an 'else' in `tokenize`
54         
55 
56     def tokenize(self, string):
57         """Tokenize incoming string."""
58         
59         if self.language == 'arabic':
60             tokens = tokenize_arabic_words(string)
61         elif self.language == 'french':
62             tokens = tokenize_french_words(string)
63         elif self.language == 'greek':
64             tokens = tokenize_greek_words(string)
65         elif self.language == 'latin':
66             tokens = tokenize_latin_words(string)
67         elif self.language == 'old_norse':
68             tokens = tokenize_old_norse_words(string)
69         else:
70             tokens = nltk_tokenize_words(string)
71 
72         return tokens
73 
74 
75 def nltk_tokenize_words(string, attached_period=False, language=None):
76     """Wrap NLTK's tokenizer PunktLanguageVars(), but make final period
77     its own token.
78 
79     >>> nltk_tokenize_words("Sentence 1. Sentence 2.")
80     ['Sentence', '1', '.', 'Sentence', '2', '.']
81 
82     >>> #Optionally keep the NLTK's output:
83 
84     >>> nltk_tokenize_words("Sentence 1. Sentence 2.", attached_period=True)
85     ['Sentence', '1.', 'Sentence', '2.']
86 
87     TODO: Run some tests to determine whether there is a large penalty for
88     re-calling PunktLanguageVars() for each use of this function. If so, this
89     will need to become a class, perhaps inheriting from the PunktLanguageVars
90     object. Maybe integrate with WordTokenizer.
91     """
92     assert isinstance(string, str), "Incoming string must be type str."
93     if language == 'sanskrit':
94         periods = ['.', '।','॥']
95     else:
96         periods = ['.']
97     punkt = PunktLanguageVars()
98     tokens = punkt.word_tokenize(string)
99     if attached_period:
100         return tokens
101     new_tokens = []
102     for word in tokens:
103         for char in periods:
104             if word.endswith(char):
105                 new_tokens.append(word[:-1])
106                 new_tokens.append(char)
107                 break
108         else:
109             new_tokens.append(word)
110     return new_tokens
111 
112 
113 def tokenize_arabic_words(text):
114 
115     """
116         Tokenize text into words
117         @param text: the input text.
118         @type text: unicode.
119         @return: list of words.
120         @rtype: list.
121     """
122     specific_tokens = []
123     if not text:
124         return specific_tokens
125     else:
126         specific_tokens = araby.tokenize(text)
127         return specific_tokens
128         
129 
130 def tokenize_french_words(string):
131     assert isinstance(string, str), "Incoming string must be type str."
132 
133     # normalize apostrophes
134 
135     text = re.sub(r"’", r"'", string)
136 
137     # Dealing with punctuation
138     text = re.sub(r"\'", r"' ", text)
139     text = re.sub("(?<=.)(?=[.!?)(\";:,«»\-])", " ", text)
140 
141     results = str.split(text)
142     return (results)
143     
144     
145 def tokenize_greek_words(text):
146     """
147     Tokenizer divides the string into a list of substrings. This is a placeholder
148     function that returns the default NLTK word tokenizer until
149     Greek-specific options are added.
150     
151     Example:
152     >>> text = 'Θουκυδίδης Ἀθηναῖος ξυνέγραψε τὸν πόλεμον τῶν Πελοποννησίων καὶ Ἀθηναίων,'
153     >>> tokenize_greek_words(text)
154     ['Θουκυδίδης', 'Ἀθηναῖος', 'ξυνέγραψε', 'τὸν', 'πόλεμον', 'τῶν', 'Πελοποννησίων', 'καὶ', 'Ἀθηναίων', ',']
155       
156     :param string: This accepts the string value that needs to be tokenized
157     :returns: A list of substrings extracted from the string
158     """
159     
160     return nltk_tokenize_words(text) # Simplest implementation to start
161         
162 
163 def tokenize_latin_words(string):
164     """
165     Tokenizer divides the string into a list of substrings
166   
167     >>> from cltk.corpus.utils.formatter import remove_non_ascii
168     >>> text =  'Dices ἐστιν ἐμός pulchrum esse inimicos ulcisci.'
169     >>> tokenize_latin_words(text)
170     ['Dices', 'ἐστιν', 'ἐμός', 'pulchrum', 'esse', 'inimicos', 'ulcisci', '.']
171   
172     :param string: This accepts the string value that needs to be tokenized
173     :returns: A list of substrings extracted from the string
174     """
175     from cltk.tokenize.latin_exceptions import latin_exceptions
176 
177     assert isinstance(string, str), "Incoming string must be type str."
178 
179     def matchcase(word):
180         # From Python Cookbook
181         def replace(m):
182             text = m.group()
183             if text.isupper():
184                 return word.upper()
185             elif text.islower():
186                 return word.lower()
187             elif text[0].isupper():
188                 return word.capitalize()
189             else:
190                 return word
191 
192         return replace
193 
194     replacements = [(r'mecum', 'cum me'),
195                     (r'tecum', 'cum te'),
196                     (r'secum', 'cum se'),
197                     (r'nobiscum', 'cum nobis'),
198                     (r'vobiscum', 'cum vobis'),
199                     (r'quocum', 'cum quo'),
200                     (r'quacum', 'cum qua'),
201                     (r'quicum', 'cum qui'),
202                     (r'quibuscum', 'cum quibus'),
203                     (r'sodes', 'si audes'),
204                     (r'satin', 'satis ne'),
205                     (r'scin', 'scis ne'),
206                     (r'sultis', 'si vultis'),
207                     (r'similist', 'similis est'),
208                     (r'qualist', 'qualis est')
209                     ]
210 
211     for replacement in replacements:
212         string = re.sub(replacement[0], matchcase(replacement[1]), string, flags=re.IGNORECASE)
213 
214     punkt_param = PunktParameters()
215     abbreviations = ['c', 'l', 'm', 'p', 'q', 't', 'ti', 'sex', 'a', 'd', 'cn', 'sp', "m'", 'ser', 'ap', 'n',
216                      'v', 'k', 'mam', 'post', 'f', 'oct', 'opet', 'paul', 'pro', 'sert', 'st', 'sta', 'v', 'vol', 'vop']
217     punkt_param.abbrev_types = set(abbreviations)
218     sent_tokenizer = PunktSentenceTokenizer(punkt_param)
219 
220     word_tokenizer = PunktLanguageVars()
221     sents = sent_tokenizer.tokenize(string)
222 
223     enclitics = ['que', 'n', 'ue', 've', 'st']
224     exceptions = enclitics
225     exceptions = list(set(exceptions + latin_exceptions))
226 
227     tokens = []
228 
229     for sent in sents:
230         temp_tokens = word_tokenizer.word_tokenize(sent)
231         # Need to check that tokens exist before handling them;
232         # needed to make stream.readlines work in PlaintextCorpusReader
233         
234         if temp_tokens:
235             if temp_tokens[0].endswith('ne'):
236                 if temp_tokens[0].lower() not in exceptions:
237                     temp = [temp_tokens[0][:-2], '-ne']
238                     temp_tokens = temp + temp_tokens[1:]
239 
240             if temp_tokens[-1].endswith('.'):
241                 final_word = temp_tokens[-1][:-1]
242                 del temp_tokens[-1]
243                 temp_tokens += [final_word, '.']
244 
245             for token in temp_tokens:
246                 tokens.append(token)
247 
248     # Break enclitic handling into own function?
249     specific_tokens = []
250 
251     for token in tokens:
252         is_enclitic = False
253         if token.lower() not in exceptions:
254             for enclitic in enclitics:
255                 if token.endswith(enclitic):
256                     if enclitic == 'n':
257                         specific_tokens += [token[:-len(enclitic)]] + ['-ne']
258                     elif enclitic == 'st':
259                         if token.endswith('ust'):
260                             specific_tokens += [token[:-len(enclitic) + 1]] + ['est']
261                         else:
262                             specific_tokens += [token[:-len(enclitic)]] + ['est']
263                     else:
264                         specific_tokens += [token[:-len(enclitic)]] + ['-' + enclitic]
265                     is_enclitic = True
266                     break
267         if not is_enclitic:
268             specific_tokens.append(token)
269 
270     return specific_tokens
271 
272 
273 def tokenize_old_norse_words(text):
274     """
275 
276     :param text: a text or a sentence
277     :return:
278     """
279     assert isinstance(text, str)
280 
281     # punctuation
282     text = re.sub(r"\'", r"' ", text)
283     text = re.sub("(?<=.)(?=[.!?)(\";:,«»\-])", " ", text)
284 
285     # TODO dealing with merges between verbs at the second person of the present tense and þú
286     # -> -tu, -ðu, -du, -u : question
287 
288     # TODO dealing with merges between verbs and sik -> st : middle voice
289 
290     results = str.split(text)
291     return results
292
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/cltk/tokenize/word.py b/cltk/tokenize/word.py
--- a/cltk/tokenize/word.py
+++ b/cltk/tokenize/word.py
@@ -1,40 +1,19 @@
-# -*-coding:utf-8-*-
 """Language-specific word tokenizers. Primary purpose is to handle enclitics."""
 
-import re
+__author__ = ['Patrick J. Burns <patrick@diyclassics.org>', 
+              'Kyle P. Johnson <kyle@kyle-p-johnson.com>', 
+              'Natasha Voake <natashavoake@gmail.com>']
+# Author info for Arabic, Old Norse?
 
-from nltk.tokenize.punkt import PunktLanguageVars
-from nltk.tokenize.punkt import PunktSentenceTokenizer, PunktParameters
+__license__ = 'MIT License. See LICENSE.'
 
 import re
 
-# Cleanup these imports—most are not used!
-from nltk.data              import load
-from nltk.tokenize.casual   import (TweetTokenizer, casual_tokenize)
-from nltk.tokenize.mwe      import MWETokenizer
-from nltk.tokenize.punkt    import PunktSentenceTokenizer
-from nltk.tokenize.regexp   import (RegexpTokenizer, WhitespaceTokenizer,
-                                    BlanklineTokenizer, WordPunctTokenizer,
-                                    wordpunct_tokenize, regexp_tokenize,
-                                    blankline_tokenize)
-#from nltk.tokenize.repp     import ReppTokenizer
-from nltk.tokenize.sexpr    import SExprTokenizer, sexpr_tokenize
-from nltk.tokenize.simple   import (SpaceTokenizer, TabTokenizer, LineTokenizer,
-                                    line_tokenize)
-from nltk.tokenize.stanford import StanfordTokenizer
-from nltk.tokenize.texttiling import TextTilingTokenizer
-#from nltk.tokenize.toktok   import ToktokTokenizer
-from nltk.tokenize.treebank import TreebankWordTokenizer
-from nltk.tokenize.util     import string_span_tokenize, regexp_span_tokenize
-from nltk.tokenize.stanford_segmenter import StanfordSegmenter
+from nltk.tokenize.punkt import PunktLanguageVars
+from nltk.tokenize.punkt import PunktSentenceTokenizer, PunktParameters
 
 import cltk.corpus.arabic.utils.pyarabic.araby as araby
 
-__author__ = ['Patrick J. Burns <patrick@diyclassics.org>', 'Kyle P. Johnson <kyle@kyle-p-johnson.com>',
-              'Natasha Voake <natashavoake@gmail.com>']
-__license__ = 'MIT License. See LICENSE.'
-
-
 class WordTokenizer:  # pylint: disable=too-few-public-methods
     """Tokenize according to rules specific to a given language."""

{"golden_diff": "diff --git a/cltk/tokenize/word.py b/cltk/tokenize/word.py\n--- a/cltk/tokenize/word.py\n+++ b/cltk/tokenize/word.py\n@@ -1,40 +1,19 @@\n-# -*-coding:utf-8-*-\n \"\"\"Language-specific word tokenizers. Primary purpose is to handle enclitics.\"\"\"\n \n-import re\n+__author__ = ['Patrick J. Burns <patrick@diyclassics.org>', \n+              'Kyle P. Johnson <kyle@kyle-p-johnson.com>', \n+              'Natasha Voake <natashavoake@gmail.com>']\n+# Author info for Arabic, Old Norse?\n \n-from nltk.tokenize.punkt import PunktLanguageVars\n-from nltk.tokenize.punkt import PunktSentenceTokenizer, PunktParameters\n+__license__ = 'MIT License. See LICENSE.'\n \n import re\n \n-# Cleanup these imports\u2014most are not used!\n-from nltk.data              import load\n-from nltk.tokenize.casual   import (TweetTokenizer, casual_tokenize)\n-from nltk.tokenize.mwe      import MWETokenizer\n-from nltk.tokenize.punkt    import PunktSentenceTokenizer\n-from nltk.tokenize.regexp   import (RegexpTokenizer, WhitespaceTokenizer,\n-                                    BlanklineTokenizer, WordPunctTokenizer,\n-                                    wordpunct_tokenize, regexp_tokenize,\n-                                    blankline_tokenize)\n-#from nltk.tokenize.repp     import ReppTokenizer\n-from nltk.tokenize.sexpr    import SExprTokenizer, sexpr_tokenize\n-from nltk.tokenize.simple   import (SpaceTokenizer, TabTokenizer, LineTokenizer,\n-                                    line_tokenize)\n-from nltk.tokenize.stanford import StanfordTokenizer\n-from nltk.tokenize.texttiling import TextTilingTokenizer\n-#from nltk.tokenize.toktok   import ToktokTokenizer\n-from nltk.tokenize.treebank import TreebankWordTokenizer\n-from nltk.tokenize.util     import string_span_tokenize, regexp_span_tokenize\n-from nltk.tokenize.stanford_segmenter import StanfordSegmenter\n+from nltk.tokenize.punkt import PunktLanguageVars\n+from nltk.tokenize.punkt import PunktSentenceTokenizer, PunktParameters\n \n import cltk.corpus.arabic.utils.pyarabic.araby as araby\n \n-__author__ = ['Patrick J. Burns <patrick@diyclassics.org>', 'Kyle P. Johnson <kyle@kyle-p-johnson.com>',\n-              'Natasha Voake <natashavoake@gmail.com>']\n-__license__ = 'MIT License. See LICENSE.'\n-\n-\n class WordTokenizer:  # pylint: disable=too-few-public-methods\n     \"\"\"Tokenize according to rules specific to a given language.\"\"\"\n", "issue": "Unused imports in word tokenizer module\nThe word tokenizer module has a large number of imports from NLTK that are not used anywhere in the module. Removing them 1. cleans up the code, and 2. speeds up testing.\n", "before_files": [{"content": "# -*-coding:utf-8-*-\n\"\"\"Language-specific word tokenizers. Primary purpose is to handle enclitics.\"\"\"\n\nimport re\n\nfrom nltk.tokenize.punkt import PunktLanguageVars\nfrom nltk.tokenize.punkt import PunktSentenceTokenizer, PunktParameters\n\nimport re\n\n# Cleanup these imports\u2014most are not used!\nfrom nltk.data              import load\nfrom nltk.tokenize.casual   import (TweetTokenizer, casual_tokenize)\nfrom nltk.tokenize.mwe      import MWETokenizer\nfrom nltk.tokenize.punkt    import PunktSentenceTokenizer\nfrom nltk.tokenize.regexp   import (RegexpTokenizer, WhitespaceTokenizer,\n                                    BlanklineTokenizer, WordPunctTokenizer,\n                                    wordpunct_tokenize, regexp_tokenize,\n                                    blankline_tokenize)\n#from nltk.tokenize.repp     import ReppTokenizer\nfrom nltk.tokenize.sexpr    import SExprTokenizer, sexpr_tokenize\nfrom nltk.tokenize.simple   import (SpaceTokenizer, TabTokenizer, LineTokenizer,\n                                    line_tokenize)\nfrom nltk.tokenize.stanford import StanfordTokenizer\nfrom nltk.tokenize.texttiling import TextTilingTokenizer\n#from nltk.tokenize.toktok   import ToktokTokenizer\nfrom nltk.tokenize.treebank import TreebankWordTokenizer\nfrom nltk.tokenize.util     import string_span_tokenize, regexp_span_tokenize\nfrom nltk.tokenize.stanford_segmenter import StanfordSegmenter\n\nimport cltk.corpus.arabic.utils.pyarabic.araby as araby\n\n__author__ = ['Patrick J. Burns <patrick@diyclassics.org>', 'Kyle P. Johnson <kyle@kyle-p-johnson.com>',\n              'Natasha Voake <natashavoake@gmail.com>']\n__license__ = 'MIT License. See LICENSE.'\n\n\nclass WordTokenizer:  # pylint: disable=too-few-public-methods\n    \"\"\"Tokenize according to rules specific to a given language.\"\"\"\n\n    def __init__(self, language):\n        \"\"\"Take language as argument to the class. Check availability and\n        setup class variables.\"\"\"\n        self.language = language\n        self.available_languages = ['arabic', \n                                    'french',\n                                    'greek',\n                                    'latin',\n                                    'old_norse']\n        assert self.language in self.available_languages, \\\n            \"Specific tokenizer not available for '{0}'. Only available for: '{1}'.\".format(self.language,  # pylint: disable=line-too-long\n            self.available_languages)  # pylint: disable=line-too-long\n        # ^^^ Necessary? since we have an 'else' in `tokenize`\n        \n\n    def tokenize(self, string):\n        \"\"\"Tokenize incoming string.\"\"\"\n        \n        if self.language == 'arabic':\n            tokens = tokenize_arabic_words(string)\n        elif self.language == 'french':\n            tokens = tokenize_french_words(string)\n        elif self.language == 'greek':\n            tokens = tokenize_greek_words(string)\n        elif self.language == 'latin':\n            tokens = tokenize_latin_words(string)\n        elif self.language == 'old_norse':\n            tokens = tokenize_old_norse_words(string)\n        else:\n            tokens = nltk_tokenize_words(string)\n\n        return tokens\n\n\ndef nltk_tokenize_words(string, attached_period=False, language=None):\n    \"\"\"Wrap NLTK's tokenizer PunktLanguageVars(), but make final period\n    its own token.\n\n    >>> nltk_tokenize_words(\"Sentence 1. Sentence 2.\")\n    ['Sentence', '1', '.', 'Sentence', '2', '.']\n\n    >>> #Optionally keep the NLTK's output:\n\n    >>> nltk_tokenize_words(\"Sentence 1. Sentence 2.\", attached_period=True)\n    ['Sentence', '1.', 'Sentence', '2.']\n\n    TODO: Run some tests to determine whether there is a large penalty for\n    re-calling PunktLanguageVars() for each use of this function. If so, this\n    will need to become a class, perhaps inheriting from the PunktLanguageVars\n    object. Maybe integrate with WordTokenizer.\n    \"\"\"\n    assert isinstance(string, str), \"Incoming string must be type str.\"\n    if language == 'sanskrit':\n        periods = ['.', '\u0964','\u0965']\n    else:\n        periods = ['.']\n    punkt = PunktLanguageVars()\n    tokens = punkt.word_tokenize(string)\n    if attached_period:\n        return tokens\n    new_tokens = []\n    for word in tokens:\n        for char in periods:\n            if word.endswith(char):\n                new_tokens.append(word[:-1])\n                new_tokens.append(char)\n                break\n        else:\n            new_tokens.append(word)\n    return new_tokens\n\n\ndef tokenize_arabic_words(text):\n\n    \"\"\"\n        Tokenize text into words\n        @param text: the input text.\n        @type text: unicode.\n        @return: list of words.\n        @rtype: list.\n    \"\"\"\n    specific_tokens = []\n    if not text:\n        return specific_tokens\n    else:\n        specific_tokens = araby.tokenize(text)\n        return specific_tokens\n        \n\ndef tokenize_french_words(string):\n    assert isinstance(string, str), \"Incoming string must be type str.\"\n\n    # normalize apostrophes\n\n    text = re.sub(r\"\u2019\", r\"'\", string)\n\n    # Dealing with punctuation\n    text = re.sub(r\"\\'\", r\"' \", text)\n    text = re.sub(\"(?<=.)(?=[.!?)(\\\";:,\u00ab\u00bb\\-])\", \" \", text)\n\n    results = str.split(text)\n    return (results)\n    \n    \ndef tokenize_greek_words(text):\n    \"\"\"\n    Tokenizer divides the string into a list of substrings. This is a placeholder\n    function that returns the default NLTK word tokenizer until\n    Greek-specific options are added.\n    \n    Example:\n    >>> text = '\u0398\u03bf\u03c5\u03ba\u03c5\u03b4\u03af\u03b4\u03b7\u03c2 \u1f08\u03b8\u03b7\u03bd\u03b1\u1fd6\u03bf\u03c2 \u03be\u03c5\u03bd\u03ad\u03b3\u03c1\u03b1\u03c8\u03b5 \u03c4\u1f78\u03bd \u03c0\u03cc\u03bb\u03b5\u03bc\u03bf\u03bd \u03c4\u1ff6\u03bd \u03a0\u03b5\u03bb\u03bf\u03c0\u03bf\u03bd\u03bd\u03b7\u03c3\u03af\u03c9\u03bd \u03ba\u03b1\u1f76 \u1f08\u03b8\u03b7\u03bd\u03b1\u03af\u03c9\u03bd,'\n    >>> tokenize_greek_words(text)\n    ['\u0398\u03bf\u03c5\u03ba\u03c5\u03b4\u03af\u03b4\u03b7\u03c2', '\u1f08\u03b8\u03b7\u03bd\u03b1\u1fd6\u03bf\u03c2', '\u03be\u03c5\u03bd\u03ad\u03b3\u03c1\u03b1\u03c8\u03b5', '\u03c4\u1f78\u03bd', '\u03c0\u03cc\u03bb\u03b5\u03bc\u03bf\u03bd', '\u03c4\u1ff6\u03bd', '\u03a0\u03b5\u03bb\u03bf\u03c0\u03bf\u03bd\u03bd\u03b7\u03c3\u03af\u03c9\u03bd', '\u03ba\u03b1\u1f76', '\u1f08\u03b8\u03b7\u03bd\u03b1\u03af\u03c9\u03bd', ',']\n      \n    :param string: This accepts the string value that needs to be tokenized\n    :returns: A list of substrings extracted from the string\n    \"\"\"\n    \n    return nltk_tokenize_words(text) # Simplest implementation to start\n        \n\ndef tokenize_latin_words(string):\n    \"\"\"\n    Tokenizer divides the string into a list of substrings\n  \n    >>> from cltk.corpus.utils.formatter import remove_non_ascii\n    >>> text =  'Dices \u1f10\u03c3\u03c4\u03b9\u03bd \u1f10\u03bc\u03cc\u03c2 pulchrum esse inimicos ulcisci.'\n    >>> tokenize_latin_words(text)\n    ['Dices', '\u1f10\u03c3\u03c4\u03b9\u03bd', '\u1f10\u03bc\u03cc\u03c2', 'pulchrum', 'esse', 'inimicos', 'ulcisci', '.']\n  \n    :param string: This accepts the string value that needs to be tokenized\n    :returns: A list of substrings extracted from the string\n    \"\"\"\n    from cltk.tokenize.latin_exceptions import latin_exceptions\n\n    assert isinstance(string, str), \"Incoming string must be type str.\"\n\n    def matchcase(word):\n        # From Python Cookbook\n        def replace(m):\n            text = m.group()\n            if text.isupper():\n                return word.upper()\n            elif text.islower():\n                return word.lower()\n            elif text[0].isupper():\n                return word.capitalize()\n            else:\n                return word\n\n        return replace\n\n    replacements = [(r'mecum', 'cum me'),\n                    (r'tecum', 'cum te'),\n                    (r'secum', 'cum se'),\n                    (r'nobiscum', 'cum nobis'),\n                    (r'vobiscum', 'cum vobis'),\n                    (r'quocum', 'cum quo'),\n                    (r'quacum', 'cum qua'),\n                    (r'quicum', 'cum qui'),\n                    (r'quibuscum', 'cum quibus'),\n                    (r'sodes', 'si audes'),\n                    (r'satin', 'satis ne'),\n                    (r'scin', 'scis ne'),\n                    (r'sultis', 'si vultis'),\n                    (r'similist', 'similis est'),\n                    (r'qualist', 'qualis est')\n                    ]\n\n    for replacement in replacements:\n        string = re.sub(replacement[0], matchcase(replacement[1]), string, flags=re.IGNORECASE)\n\n    punkt_param = PunktParameters()\n    abbreviations = ['c', 'l', 'm', 'p', 'q', 't', 'ti', 'sex', 'a', 'd', 'cn', 'sp', \"m'\", 'ser', 'ap', 'n',\n                     'v', 'k', 'mam', 'post', 'f', 'oct', 'opet', 'paul', 'pro', 'sert', 'st', 'sta', 'v', 'vol', 'vop']\n    punkt_param.abbrev_types = set(abbreviations)\n    sent_tokenizer = PunktSentenceTokenizer(punkt_param)\n\n    word_tokenizer = PunktLanguageVars()\n    sents = sent_tokenizer.tokenize(string)\n\n    enclitics = ['que', 'n', 'ue', 've', 'st']\n    exceptions = enclitics\n    exceptions = list(set(exceptions + latin_exceptions))\n\n    tokens = []\n\n    for sent in sents:\n        temp_tokens = word_tokenizer.word_tokenize(sent)\n        # Need to check that tokens exist before handling them;\n        # needed to make stream.readlines work in PlaintextCorpusReader\n        \n        if temp_tokens:\n            if temp_tokens[0].endswith('ne'):\n                if temp_tokens[0].lower() not in exceptions:\n                    temp = [temp_tokens[0][:-2], '-ne']\n                    temp_tokens = temp + temp_tokens[1:]\n\n            if temp_tokens[-1].endswith('.'):\n                final_word = temp_tokens[-1][:-1]\n                del temp_tokens[-1]\n                temp_tokens += [final_word, '.']\n\n            for token in temp_tokens:\n                tokens.append(token)\n\n    # Break enclitic handling into own function?\n    specific_tokens = []\n\n    for token in tokens:\n        is_enclitic = False\n        if token.lower() not in exceptions:\n            for enclitic in enclitics:\n                if token.endswith(enclitic):\n                    if enclitic == 'n':\n                        specific_tokens += [token[:-len(enclitic)]] + ['-ne']\n                    elif enclitic == 'st':\n                        if token.endswith('ust'):\n                            specific_tokens += [token[:-len(enclitic) + 1]] + ['est']\n                        else:\n                            specific_tokens += [token[:-len(enclitic)]] + ['est']\n                    else:\n                        specific_tokens += [token[:-len(enclitic)]] + ['-' + enclitic]\n                    is_enclitic = True\n                    break\n        if not is_enclitic:\n            specific_tokens.append(token)\n\n    return specific_tokens\n\n\ndef tokenize_old_norse_words(text):\n    \"\"\"\n\n    :param text: a text or a sentence\n    :return:\n    \"\"\"\n    assert isinstance(text, str)\n\n    # punctuation\n    text = re.sub(r\"\\'\", r\"' \", text)\n    text = re.sub(\"(?<=.)(?=[.!?)(\\\";:,\u00ab\u00bb\\-])\", \" \", text)\n\n    # TODO dealing with merges between verbs at the second person of the present tense and \u00fe\u00fa\n    # -> -tu, -\u00f0u, -du, -u : question\n\n    # TODO dealing with merges between verbs and sik -> st : middle voice\n\n    results = str.split(text)\n    return results\n", "path": "cltk/tokenize/word.py"}], "after_files": [{"content": "\"\"\"Language-specific word tokenizers. Primary purpose is to handle enclitics.\"\"\"\n\n__author__ = ['Patrick J. Burns <patrick@diyclassics.org>', \n              'Kyle P. Johnson <kyle@kyle-p-johnson.com>', \n              'Natasha Voake <natashavoake@gmail.com>']\n# Author info for Arabic, Old Norse?\n\n__license__ = 'MIT License. See LICENSE.'\n\nimport re\n\nfrom nltk.tokenize.punkt import PunktLanguageVars\nfrom nltk.tokenize.punkt import PunktSentenceTokenizer, PunktParameters\n\nimport cltk.corpus.arabic.utils.pyarabic.araby as araby\n\nclass WordTokenizer:  # pylint: disable=too-few-public-methods\n    \"\"\"Tokenize according to rules specific to a given language.\"\"\"\n\n    def __init__(self, language):\n        \"\"\"Take language as argument to the class. Check availability and\n        setup class variables.\"\"\"\n        self.language = language\n        self.available_languages = ['arabic', \n                                    'french',\n                                    'greek',\n                                    'latin',\n                                    'old_norse']\n        assert self.language in self.available_languages, \\\n            \"Specific tokenizer not available for '{0}'. Only available for: '{1}'.\".format(self.language,  # pylint: disable=line-too-long\n            self.available_languages)  # pylint: disable=line-too-long\n        # ^^^ Necessary? since we have an 'else' in `tokenize`\n        \n\n    def tokenize(self, string):\n        \"\"\"Tokenize incoming string.\"\"\"\n        \n        if self.language == 'arabic':\n            tokens = tokenize_arabic_words(string)\n        elif self.language == 'french':\n            tokens = tokenize_french_words(string)\n        elif self.language == 'greek':\n            tokens = tokenize_greek_words(string)\n        elif self.language == 'latin':\n            tokens = tokenize_latin_words(string)\n        elif self.language == 'old_norse':\n            tokens = tokenize_old_norse_words(string)\n        else:\n            tokens = nltk_tokenize_words(string)\n\n        return tokens\n\n\ndef nltk_tokenize_words(string, attached_period=False, language=None):\n    \"\"\"Wrap NLTK's tokenizer PunktLanguageVars(), but make final period\n    its own token.\n\n    >>> nltk_tokenize_words(\"Sentence 1. Sentence 2.\")\n    ['Sentence', '1', '.', 'Sentence', '2', '.']\n\n    >>> #Optionally keep the NLTK's output:\n\n    >>> nltk_tokenize_words(\"Sentence 1. Sentence 2.\", attached_period=True)\n    ['Sentence', '1.', 'Sentence', '2.']\n\n    TODO: Run some tests to determine whether there is a large penalty for\n    re-calling PunktLanguageVars() for each use of this function. If so, this\n    will need to become a class, perhaps inheriting from the PunktLanguageVars\n    object. Maybe integrate with WordTokenizer.\n    \"\"\"\n    assert isinstance(string, str), \"Incoming string must be type str.\"\n    if language == 'sanskrit':\n        periods = ['.', '\u0964','\u0965']\n    else:\n        periods = ['.']\n    punkt = PunktLanguageVars()\n    tokens = punkt.word_tokenize(string)\n    if attached_period:\n        return tokens\n    new_tokens = []\n    for word in tokens:\n        for char in periods:\n            if word.endswith(char):\n                new_tokens.append(word[:-1])\n                new_tokens.append(char)\n                break\n        else:\n            new_tokens.append(word)\n    return new_tokens\n\n\ndef tokenize_arabic_words(text):\n\n    \"\"\"\n        Tokenize text into words\n        @param text: the input text.\n        @type text: unicode.\n        @return: list of words.\n        @rtype: list.\n    \"\"\"\n    specific_tokens = []\n    if not text:\n        return specific_tokens\n    else:\n        specific_tokens = araby.tokenize(text)\n        return specific_tokens\n        \n\ndef tokenize_french_words(string):\n    assert isinstance(string, str), \"Incoming string must be type str.\"\n\n    # normalize apostrophes\n\n    text = re.sub(r\"\u2019\", r\"'\", string)\n\n    # Dealing with punctuation\n    text = re.sub(r\"\\'\", r\"' \", text)\n    text = re.sub(\"(?<=.)(?=[.!?)(\\\";:,\u00ab\u00bb\\-])\", \" \", text)\n\n    results = str.split(text)\n    return (results)\n    \n    \ndef tokenize_greek_words(text):\n    \"\"\"\n    Tokenizer divides the string into a list of substrings. This is a placeholder\n    function that returns the default NLTK word tokenizer until\n    Greek-specific options are added.\n    \n    Example:\n    >>> text = '\u0398\u03bf\u03c5\u03ba\u03c5\u03b4\u03af\u03b4\u03b7\u03c2 \u1f08\u03b8\u03b7\u03bd\u03b1\u1fd6\u03bf\u03c2 \u03be\u03c5\u03bd\u03ad\u03b3\u03c1\u03b1\u03c8\u03b5 \u03c4\u1f78\u03bd \u03c0\u03cc\u03bb\u03b5\u03bc\u03bf\u03bd \u03c4\u1ff6\u03bd \u03a0\u03b5\u03bb\u03bf\u03c0\u03bf\u03bd\u03bd\u03b7\u03c3\u03af\u03c9\u03bd \u03ba\u03b1\u1f76 \u1f08\u03b8\u03b7\u03bd\u03b1\u03af\u03c9\u03bd,'\n    >>> tokenize_greek_words(text)\n    ['\u0398\u03bf\u03c5\u03ba\u03c5\u03b4\u03af\u03b4\u03b7\u03c2', '\u1f08\u03b8\u03b7\u03bd\u03b1\u1fd6\u03bf\u03c2', '\u03be\u03c5\u03bd\u03ad\u03b3\u03c1\u03b1\u03c8\u03b5', '\u03c4\u1f78\u03bd', '\u03c0\u03cc\u03bb\u03b5\u03bc\u03bf\u03bd', '\u03c4\u1ff6\u03bd', '\u03a0\u03b5\u03bb\u03bf\u03c0\u03bf\u03bd\u03bd\u03b7\u03c3\u03af\u03c9\u03bd', '\u03ba\u03b1\u1f76', '\u1f08\u03b8\u03b7\u03bd\u03b1\u03af\u03c9\u03bd', ',']\n      \n    :param string: This accepts the string value that needs to be tokenized\n    :returns: A list of substrings extracted from the string\n    \"\"\"\n    \n    return nltk_tokenize_words(text) # Simplest implementation to start\n        \n\ndef tokenize_latin_words(string):\n    \"\"\"\n    Tokenizer divides the string into a list of substrings\n  \n    >>> from cltk.corpus.utils.formatter import remove_non_ascii\n    >>> text =  'Dices \u1f10\u03c3\u03c4\u03b9\u03bd \u1f10\u03bc\u03cc\u03c2 pulchrum esse inimicos ulcisci.'\n    >>> tokenize_latin_words(text)\n    ['Dices', '\u1f10\u03c3\u03c4\u03b9\u03bd', '\u1f10\u03bc\u03cc\u03c2', 'pulchrum', 'esse', 'inimicos', 'ulcisci', '.']\n  \n    :param string: This accepts the string value that needs to be tokenized\n    :returns: A list of substrings extracted from the string\n    \"\"\"\n    from cltk.tokenize.latin_exceptions import latin_exceptions\n\n    assert isinstance(string, str), \"Incoming string must be type str.\"\n\n    def matchcase(word):\n        # From Python Cookbook\n        def replace(m):\n            text = m.group()\n            if text.isupper():\n                return word.upper()\n            elif text.islower():\n                return word.lower()\n            elif text[0].isupper():\n                return word.capitalize()\n            else:\n                return word\n\n        return replace\n\n    replacements = [(r'mecum', 'cum me'),\n                    (r'tecum', 'cum te'),\n                    (r'secum', 'cum se'),\n                    (r'nobiscum', 'cum nobis'),\n                    (r'vobiscum', 'cum vobis'),\n                    (r'quocum', 'cum quo'),\n                    (r'quacum', 'cum qua'),\n                    (r'quicum', 'cum qui'),\n                    (r'quibuscum', 'cum quibus'),\n                    (r'sodes', 'si audes'),\n                    (r'satin', 'satis ne'),\n                    (r'scin', 'scis ne'),\n                    (r'sultis', 'si vultis'),\n                    (r'similist', 'similis est'),\n                    (r'qualist', 'qualis est')\n                    ]\n\n    for replacement in replacements:\n        string = re.sub(replacement[0], matchcase(replacement[1]), string, flags=re.IGNORECASE)\n\n    punkt_param = PunktParameters()\n    abbreviations = ['c', 'l', 'm', 'p', 'q', 't', 'ti', 'sex', 'a', 'd', 'cn', 'sp', \"m'\", 'ser', 'ap', 'n',\n                     'v', 'k', 'mam', 'post', 'f', 'oct', 'opet', 'paul', 'pro', 'sert', 'st', 'sta', 'v', 'vol', 'vop']\n    punkt_param.abbrev_types = set(abbreviations)\n    sent_tokenizer = PunktSentenceTokenizer(punkt_param)\n\n    word_tokenizer = PunktLanguageVars()\n    sents = sent_tokenizer.tokenize(string)\n\n    enclitics = ['que', 'n', 'ue', 've', 'st']\n    exceptions = enclitics\n    exceptions = list(set(exceptions + latin_exceptions))\n\n    tokens = []\n\n    for sent in sents:\n        temp_tokens = word_tokenizer.word_tokenize(sent)\n        # Need to check that tokens exist before handling them;\n        # needed to make stream.readlines work in PlaintextCorpusReader\n        \n        if temp_tokens:\n            if temp_tokens[0].endswith('ne'):\n                if temp_tokens[0].lower() not in exceptions:\n                    temp = [temp_tokens[0][:-2], '-ne']\n                    temp_tokens = temp + temp_tokens[1:]\n\n            if temp_tokens[-1].endswith('.'):\n                final_word = temp_tokens[-1][:-1]\n                del temp_tokens[-1]\n                temp_tokens += [final_word, '.']\n\n            for token in temp_tokens:\n                tokens.append(token)\n\n    # Break enclitic handling into own function?\n    specific_tokens = []\n\n    for token in tokens:\n        is_enclitic = False\n        if token.lower() not in exceptions:\n            for enclitic in enclitics:\n                if token.endswith(enclitic):\n                    if enclitic == 'n':\n                        specific_tokens += [token[:-len(enclitic)]] + ['-ne']\n                    elif enclitic == 'st':\n                        if token.endswith('ust'):\n                            specific_tokens += [token[:-len(enclitic) + 1]] + ['est']\n                        else:\n                            specific_tokens += [token[:-len(enclitic)]] + ['est']\n                    else:\n                        specific_tokens += [token[:-len(enclitic)]] + ['-' + enclitic]\n                    is_enclitic = True\n                    break\n        if not is_enclitic:\n            specific_tokens.append(token)\n\n    return specific_tokens\n\n\ndef tokenize_old_norse_words(text):\n    \"\"\"\n\n    :param text: a text or a sentence\n    :return:\n    \"\"\"\n    assert isinstance(text, str)\n\n    # punctuation\n    text = re.sub(r\"\\'\", r\"' \", text)\n    text = re.sub(\"(?<=.)(?=[.!?)(\\\";:,\u00ab\u00bb\\-])\", \" \", text)\n\n    # TODO dealing with merges between verbs at the second person of the present tense and \u00fe\u00fa\n    # -> -tu, -\u00f0u, -du, -u : question\n\n    # TODO dealing with merges between verbs and sik -> st : middle voice\n\n    results = str.split(text)\n    return results\n", "path": "cltk/tokenize/word.py"}]}