We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
Ensure unit tests work with termcolor v2.1.0
tests/test_token_classification.py currently fails after termcolor just upgraded to v2.1.0, specifically `test_color_sentence` 

- [ ] update unit test code to make them pass with termcolor v2.1.0
- [ ] ensure new unit test code also works with older versions of termcolor pre v2.1.0 (suboptimal but ok if the unit test only works with versions post v2.0.0, as long as the package works with all termcolor versions currently supported).
- [ ] remove version upper bound on termcolor if it has been added to package in the meantime.

https://pypi.org/project/termcolor/
https://github.com/termcolor/termcolor/pull/25/files
https://github.com/cleanlab/cleanlab/actions/runs/3357515340/jobs/5563372689


--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `cleanlab/internal/token_classification_utils.py`
Content:
```
1 # Copyright (C) 2017-2022  Cleanlab Inc.
2 # This file is part of cleanlab.
3 #
4 # cleanlab is free software: you can redistribute it and/or modify
5 # it under the terms of the GNU Affero General Public License as published
6 # by the Free Software Foundation, either version 3 of the License, or
7 # (at your option) any later version.
8 #
9 # cleanlab is distributed in the hope that it will be useful,
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
12 # GNU Affero General Public License for more details.
13 #
14 # You should have received a copy of the GNU Affero General Public License
15 # along with cleanlab.  If not, see <https://www.gnu.org/licenses/>.
16 
17 """
18 Helper methods used internally in cleanlab.token_classification
19 """
20 
21 import re
22 import string
23 import numpy as np
24 from termcolor import colored
25 from typing import List, Optional, Callable, Tuple
26 
27 
28 def get_sentence(words: List[str]) -> str:
29     """
30     Get sentence formed by a list of words with minor processing for readability
31 
32     Parameters
33     ----------
34     words:
35         list of word-level tokens
36 
37     Returns
38     ----------
39     sentence:
40         sentence formed by list of word-level tokens
41 
42     Examples
43     --------
44     >>> from cleanlab.internal.token_classification_utils import get_sentence
45     >>> words = ["This", "is", "a", "sentence", "."]
46     >>> get_sentence(words)
47     'This is a sentence.'
48     """
49     sentence = ""
50     for word in words:
51         if word not in string.punctuation or word in ["-", "("]:
52             word = " " + word
53         sentence += word
54     sentence = sentence.replace(" '", "'").replace("( ", "(").strip()
55     return sentence
56 
57 
58 def filter_sentence(
59     sentences: List[str],
60     condition: Optional[Callable[[str], bool]] = None,
61 ) -> Tuple[List[str], List[bool]]:
62     """
63     Filter sentence based on some condition, and returns filter mask
64 
65     Parameters
66     ----------
67     sentences:
68         list of sentences
69 
70     condition:
71         sentence filtering condition
72 
73     Returns
74     ---------
75     sentences:
76         list of sentences filtered
77 
78     mask:
79         boolean mask such that `mask[i] == True` if the i'th sentence is included in the
80         filtered sentence, otherwise `mask[i] == False`
81 
82     Examples
83     --------
84     >>> from cleanlab.internal.token_classification_utils import filter_sentence
85     >>> sentences = ["Short sentence.", "This is a longer sentence."]
86     >>> condition = lambda x: len(x.split()) > 2
87     >>> long_sentences, _ = filter_sentence(sentences, condition)
88     >>> long_sentences
89     ['This is a longer sentence.']
90     >>> document = ["# Headline", "Sentence 1.", "&", "Sentence 2."]
91     >>> sentences, mask = filter_sentence(document)
92     >>> sentences, mask
93     (['Sentence 1.', 'Sentence 2.'], [False, True, False, True])
94     """
95     if not condition:
96         condition = lambda sentence: len(sentence) > 1 and "#" not in sentence
97     mask = list(map(condition, sentences))
98     sentences = [sentence for m, sentence in zip(mask, sentences) if m]
99     return sentences, mask
100 
101 
102 def process_token(token: str, replace: List[Tuple[str, str]] = [("#", "")]) -> str:
103     """
104     Replaces special characters in the tokens
105 
106     Parameters
107     ----------
108     token:
109         token which potentially contains special characters
110 
111     replace:
112         list of tuples `(s1, s2)`, where all occurances of s1 are replaced by s2
113 
114     Returns
115     ---------
116     processed_token:
117         processed token whose special character has been replaced
118 
119     Note
120     ----
121         Only applies to characters in the original input token.
122 
123     Examples
124     --------
125     >>> from cleanlab.internal.token_classification_utils import process_token
126     >>> token = "#Comment"
127     >>> process_token("#Comment")
128     'Comment'
129 
130     Specify custom replacement rules
131 
132     >>> replace = [("C", "a"), ("a", "C")]
133     >>> process_token("Cleanlab", replace)
134     'aleCnlCb'
135     """
136     replace_dict = {re.escape(k): v for (k, v) in replace}
137     pattern = "|".join(replace_dict.keys())
138     compiled_pattern = re.compile(pattern)
139     replacement = lambda match: replace_dict[re.escape(match.group(0))]
140     processed_token = compiled_pattern.sub(replacement, token)
141     return processed_token
142 
143 
144 def mapping(entities: List[int], maps: List[int]) -> List[int]:
145     """
146     Map a list of entities to its corresponding entities
147 
148     Parameters
149     ----------
150     entities:
151         a list of given entities
152 
153     maps:
154         a list of mapped entities, such that the i'th indexed token should be mapped to `maps[i]`
155 
156     Returns
157     ---------
158     mapped_entities:
159         a list of mapped entities
160 
161     Examples
162     --------
163     >>> unique_identities = [0, 1, 2, 3, 4]  # ["O", "B-PER", "I-PER", "B-LOC", "I-LOC"]
164     >>> maps = [0, 1, 1, 2, 2]  # ["O", "PER", "PER", "LOC", "LOC"]
165     >>> mapping(unique_identities, maps)
166     [0, 1, 1, 2, 2]  # ["O", "PER", "PER", "LOC", "LOC"]
167     >>> mapping([0, 0, 4, 4, 3, 4, 0, 2], maps)
168     [0, 0, 2, 2, 2, 2, 0, 1]  # ["O", "O", "LOC", "LOC", "LOC", "LOC", "O", "PER"]
169     """
170     f = lambda x: maps[x]
171     return list(map(f, entities))
172 
173 
174 def merge_probs(probs: np.ndarray, maps: List[int]) -> np.ndarray:
175     """
176     Merges model-predictive probabilities with desired mapping
177 
178     Parameters
179     ----------
180     probs:
181         np.array of shape `(N, K)`, where N is the number of tokens, and K is the number of classes for the model
182 
183     maps:
184         a list of mapped index, such that the probability of the token being in the i'th class is mapped to the
185         `maps[i]` index. If `maps[i] == -1`, the i'th column of `probs` is ignored. If `np.any(maps == -1)`, the
186         returned probability is re-normalized.
187 
188     Returns
189     ---------
190     probs_merged:
191         np.array of shape ``(N, K')``, where `K` is the number of new classes. Probabilities are merged and
192         re-normalized if necessary.
193 
194     Examples
195     --------
196     >>> import numpy as np
197     >>> from cleanlab.internal.token_classification_utils import merge_probs
198     >>> probs = np.array([
199     ...     [0.55, 0.0125, 0.0375, 0.1, 0.3],
200     ...     [0.1, 0.8, 0, 0.075, 0.025],
201     ... ])
202     >>> maps = [0, 1, 1, 2, 2]
203     >>> merge_probs(probs, maps)
204     array([[0.55, 0.05, 0.4 ],
205            [0.1 , 0.8 , 0.1 ]])
206     """
207     old_classes = probs.shape[1]
208     map_size = np.max(maps) + 1
209     probs_merged = np.zeros([len(probs), map_size], dtype=probs.dtype.type)
210 
211     for i in range(old_classes):
212         if maps[i] >= 0:
213             probs_merged[:, maps[i]] += probs[:, i]
214     if -1 in maps:
215         row_sums = probs_merged.sum(axis=1)
216         probs_merged /= row_sums[:, np.newaxis]
217     return probs_merged
218 
219 
220 def color_sentence(sentence: str, word: str) -> str:
221     """
222     Searches for a given token in the sentence and returns the sentence where the given token is colored red
223 
224     Parameters
225     ----------
226     sentence:
227         a sentence where the word is searched
228 
229     word:
230         keyword to find in `sentence`. Assumes the word exists in the sentence.
231     Returns
232     ---------
233     colored_sentence:
234         `sentence` where the every occurrence of the word is colored red, using ``termcolor.colored``
235 
236     Examples
237     --------
238     >>> from cleanlab.internal.token_classification_utils import color_sentence
239     >>> sentence = "This is a sentence."
240     >>> word = "sentence"
241     >>> color_sentence(sentence, word)
242     'This is a \x1b[31msentence\x1b[0m.'
243 
244     Also works for multiple occurrences of the word
245 
246     >>> document = "This is a sentence. This is another sentence."
247     >>> word = "sentence"
248     >>> color_sentence(document, word)
249     'This is a \x1b[31msentence\x1b[0m. This is another \x1b[31msentence\x1b[0m.'
250     """
251     colored_word = colored(word, "red")
252     colored_sentence, number_of_substitions = re.subn(
253         r"\b{}\b".format(re.escape(word)), colored_word, sentence
254     )
255     if number_of_substitions == 0:
256         # Use basic string manipulation if regex fails
257         colored_sentence = sentence.replace(word, colored_word)
258     return colored_sentence
259
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/cleanlab/internal/token_classification_utils.py b/cleanlab/internal/token_classification_utils.py
--- a/cleanlab/internal/token_classification_utils.py
+++ b/cleanlab/internal/token_classification_utils.py
@@ -249,10 +249,35 @@
     'This is a \x1b[31msentence\x1b[0m. This is another \x1b[31msentence\x1b[0m.'
     """
     colored_word = colored(word, "red")
-    colored_sentence, number_of_substitions = re.subn(
-        r"\b{}\b".format(re.escape(word)), colored_word, sentence
+    return _replace_sentence(sentence=sentence, word=word, new_word=colored_word)
+
+
+def _replace_sentence(sentence: str, word: str, new_word: str) -> str:
+    """
+    Searches for a given token in the sentence and returns the sentence where the given token has been replaced by
+    `new_word`.
+
+    Parameters
+    ----------
+    sentence:
+        a sentence where the word is searched
+
+    word:
+        keyword to find in `sentence`. Assumes the word exists in the sentence.
+
+    new_word:
+        the word to replace the keyword with
+
+    Returns
+    ---------
+    new_sentence:
+        `sentence` where the every occurrence of the word is replaced by `colored_word`
+    """
+
+    new_sentence, number_of_substitions = re.subn(
+        r"\b{}\b".format(re.escape(word)), new_word, sentence
     )
     if number_of_substitions == 0:
         # Use basic string manipulation if regex fails
-        colored_sentence = sentence.replace(word, colored_word)
-    return colored_sentence
+        new_sentence = sentence.replace(word, new_word)
+    return new_sentence

{"golden_diff": "diff --git a/cleanlab/internal/token_classification_utils.py b/cleanlab/internal/token_classification_utils.py\n--- a/cleanlab/internal/token_classification_utils.py\n+++ b/cleanlab/internal/token_classification_utils.py\n@@ -249,10 +249,35 @@\n     'This is a \\x1b[31msentence\\x1b[0m. This is another \\x1b[31msentence\\x1b[0m.'\n     \"\"\"\n     colored_word = colored(word, \"red\")\n-    colored_sentence, number_of_substitions = re.subn(\n-        r\"\\b{}\\b\".format(re.escape(word)), colored_word, sentence\n+    return _replace_sentence(sentence=sentence, word=word, new_word=colored_word)\n+\n+\n+def _replace_sentence(sentence: str, word: str, new_word: str) -> str:\n+    \"\"\"\n+    Searches for a given token in the sentence and returns the sentence where the given token has been replaced by\n+    `new_word`.\n+\n+    Parameters\n+    ----------\n+    sentence:\n+        a sentence where the word is searched\n+\n+    word:\n+        keyword to find in `sentence`. Assumes the word exists in the sentence.\n+\n+    new_word:\n+        the word to replace the keyword with\n+\n+    Returns\n+    ---------\n+    new_sentence:\n+        `sentence` where the every occurrence of the word is replaced by `colored_word`\n+    \"\"\"\n+\n+    new_sentence, number_of_substitions = re.subn(\n+        r\"\\b{}\\b\".format(re.escape(word)), new_word, sentence\n     )\n     if number_of_substitions == 0:\n         # Use basic string manipulation if regex fails\n-        colored_sentence = sentence.replace(word, colored_word)\n-    return colored_sentence\n+        new_sentence = sentence.replace(word, new_word)\n+    return new_sentence\n", "issue": "Ensure unit tests\u00a0work with termcolor v2.1.0\ntests/test_token_classification.py currently fails after termcolor just upgraded to v2.1.0, specifically `test_color_sentence` \n\n- [ ] update unit test code to make them pass with termcolor v2.1.0\n- [ ] ensure new unit test code also works with older versions of termcolor pre v2.1.0 (suboptimal but ok if the unit test only works with versions post v2.0.0, as long as the package works with all termcolor versions currently supported).\n- [ ] remove version upper bound on termcolor if it has been added to package in the meantime.\n\nhttps://pypi.org/project/termcolor/\nhttps://github.com/termcolor/termcolor/pull/25/files\nhttps://github.com/cleanlab/cleanlab/actions/runs/3357515340/jobs/5563372689\n\n", "before_files": [{"content": "# Copyright (C) 2017-2022  Cleanlab Inc.\n# This file is part of cleanlab.\n#\n# cleanlab is free software: you can redistribute it and/or modify\n# it under the terms of the GNU Affero General Public License as published\n# by the Free Software Foundation, either version 3 of the License, or\n# (at your option) any later version.\n#\n# cleanlab is distributed in the hope that it will be useful,\n# but WITHOUT ANY WARRANTY; without even the implied warranty of\n# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the\n# GNU Affero General Public License for more details.\n#\n# You should have received a copy of the GNU Affero General Public License\n# along with cleanlab.  If not, see <https://www.gnu.org/licenses/>.\n\n\"\"\"\nHelper methods used internally in cleanlab.token_classification\n\"\"\"\n\nimport re\nimport string\nimport numpy as np\nfrom termcolor import colored\nfrom typing import List, Optional, Callable, Tuple\n\n\ndef get_sentence(words: List[str]) -> str:\n    \"\"\"\n    Get sentence formed by a list of words with minor processing for readability\n\n    Parameters\n    ----------\n    words:\n        list of word-level tokens\n\n    Returns\n    ----------\n    sentence:\n        sentence formed by list of word-level tokens\n\n    Examples\n    --------\n    >>> from cleanlab.internal.token_classification_utils import get_sentence\n    >>> words = [\"This\", \"is\", \"a\", \"sentence\", \".\"]\n    >>> get_sentence(words)\n    'This is a sentence.'\n    \"\"\"\n    sentence = \"\"\n    for word in words:\n        if word not in string.punctuation or word in [\"-\", \"(\"]:\n            word = \" \" + word\n        sentence += word\n    sentence = sentence.replace(\" '\", \"'\").replace(\"( \", \"(\").strip()\n    return sentence\n\n\ndef filter_sentence(\n    sentences: List[str],\n    condition: Optional[Callable[[str], bool]] = None,\n) -> Tuple[List[str], List[bool]]:\n    \"\"\"\n    Filter sentence based on some condition, and returns filter mask\n\n    Parameters\n    ----------\n    sentences:\n        list of sentences\n\n    condition:\n        sentence filtering condition\n\n    Returns\n    ---------\n    sentences:\n        list of sentences filtered\n\n    mask:\n        boolean mask such that `mask[i] == True` if the i'th sentence is included in the\n        filtered sentence, otherwise `mask[i] == False`\n\n    Examples\n    --------\n    >>> from cleanlab.internal.token_classification_utils import filter_sentence\n    >>> sentences = [\"Short sentence.\", \"This is a longer sentence.\"]\n    >>> condition = lambda x: len(x.split()) > 2\n    >>> long_sentences, _ = filter_sentence(sentences, condition)\n    >>> long_sentences\n    ['This is a longer sentence.']\n    >>> document = [\"# Headline\", \"Sentence 1.\", \"&\", \"Sentence 2.\"]\n    >>> sentences, mask = filter_sentence(document)\n    >>> sentences, mask\n    (['Sentence 1.', 'Sentence 2.'], [False, True, False, True])\n    \"\"\"\n    if not condition:\n        condition = lambda sentence: len(sentence) > 1 and \"#\" not in sentence\n    mask = list(map(condition, sentences))\n    sentences = [sentence for m, sentence in zip(mask, sentences) if m]\n    return sentences, mask\n\n\ndef process_token(token: str, replace: List[Tuple[str, str]] = [(\"#\", \"\")]) -> str:\n    \"\"\"\n    Replaces special characters in the tokens\n\n    Parameters\n    ----------\n    token:\n        token which potentially contains special characters\n\n    replace:\n        list of tuples `(s1, s2)`, where all occurances of s1 are replaced by s2\n\n    Returns\n    ---------\n    processed_token:\n        processed token whose special character has been replaced\n\n    Note\n    ----\n        Only applies to characters in the original input token.\n\n    Examples\n    --------\n    >>> from cleanlab.internal.token_classification_utils import process_token\n    >>> token = \"#Comment\"\n    >>> process_token(\"#Comment\")\n    'Comment'\n\n    Specify custom replacement rules\n\n    >>> replace = [(\"C\", \"a\"), (\"a\", \"C\")]\n    >>> process_token(\"Cleanlab\", replace)\n    'aleCnlCb'\n    \"\"\"\n    replace_dict = {re.escape(k): v for (k, v) in replace}\n    pattern = \"|\".join(replace_dict.keys())\n    compiled_pattern = re.compile(pattern)\n    replacement = lambda match: replace_dict[re.escape(match.group(0))]\n    processed_token = compiled_pattern.sub(replacement, token)\n    return processed_token\n\n\ndef mapping(entities: List[int], maps: List[int]) -> List[int]:\n    \"\"\"\n    Map a list of entities to its corresponding entities\n\n    Parameters\n    ----------\n    entities:\n        a list of given entities\n\n    maps:\n        a list of mapped entities, such that the i'th indexed token should be mapped to `maps[i]`\n\n    Returns\n    ---------\n    mapped_entities:\n        a list of mapped entities\n\n    Examples\n    --------\n    >>> unique_identities = [0, 1, 2, 3, 4]  # [\"O\", \"B-PER\", \"I-PER\", \"B-LOC\", \"I-LOC\"]\n    >>> maps = [0, 1, 1, 2, 2]  # [\"O\", \"PER\", \"PER\", \"LOC\", \"LOC\"]\n    >>> mapping(unique_identities, maps)\n    [0, 1, 1, 2, 2]  # [\"O\", \"PER\", \"PER\", \"LOC\", \"LOC\"]\n    >>> mapping([0, 0, 4, 4, 3, 4, 0, 2], maps)\n    [0, 0, 2, 2, 2, 2, 0, 1]  # [\"O\", \"O\", \"LOC\", \"LOC\", \"LOC\", \"LOC\", \"O\", \"PER\"]\n    \"\"\"\n    f = lambda x: maps[x]\n    return list(map(f, entities))\n\n\ndef merge_probs(probs: np.ndarray, maps: List[int]) -> np.ndarray:\n    \"\"\"\n    Merges model-predictive probabilities with desired mapping\n\n    Parameters\n    ----------\n    probs:\n        np.array of shape `(N, K)`, where N is the number of tokens, and K is the number of classes for the model\n\n    maps:\n        a list of mapped index, such that the probability of the token being in the i'th class is mapped to the\n        `maps[i]` index. If `maps[i] == -1`, the i'th column of `probs` is ignored. If `np.any(maps == -1)`, the\n        returned probability is re-normalized.\n\n    Returns\n    ---------\n    probs_merged:\n        np.array of shape ``(N, K')``, where `K` is the number of new classes. Probabilities are merged and\n        re-normalized if necessary.\n\n    Examples\n    --------\n    >>> import numpy as np\n    >>> from cleanlab.internal.token_classification_utils import merge_probs\n    >>> probs = np.array([\n    ...     [0.55, 0.0125, 0.0375, 0.1, 0.3],\n    ...     [0.1, 0.8, 0, 0.075, 0.025],\n    ... ])\n    >>> maps = [0, 1, 1, 2, 2]\n    >>> merge_probs(probs, maps)\n    array([[0.55, 0.05, 0.4 ],\n           [0.1 , 0.8 , 0.1 ]])\n    \"\"\"\n    old_classes = probs.shape[1]\n    map_size = np.max(maps) + 1\n    probs_merged = np.zeros([len(probs), map_size], dtype=probs.dtype.type)\n\n    for i in range(old_classes):\n        if maps[i] >= 0:\n            probs_merged[:, maps[i]] += probs[:, i]\n    if -1 in maps:\n        row_sums = probs_merged.sum(axis=1)\n        probs_merged /= row_sums[:, np.newaxis]\n    return probs_merged\n\n\ndef color_sentence(sentence: str, word: str) -> str:\n    \"\"\"\n    Searches for a given token in the sentence and returns the sentence where the given token is colored red\n\n    Parameters\n    ----------\n    sentence:\n        a sentence where the word is searched\n\n    word:\n        keyword to find in `sentence`. Assumes the word exists in the sentence.\n    Returns\n    ---------\n    colored_sentence:\n        `sentence` where the every occurrence of the word is colored red, using ``termcolor.colored``\n\n    Examples\n    --------\n    >>> from cleanlab.internal.token_classification_utils import color_sentence\n    >>> sentence = \"This is a sentence.\"\n    >>> word = \"sentence\"\n    >>> color_sentence(sentence, word)\n    'This is a \\x1b[31msentence\\x1b[0m.'\n\n    Also works for multiple occurrences of the word\n\n    >>> document = \"This is a sentence. This is another sentence.\"\n    >>> word = \"sentence\"\n    >>> color_sentence(document, word)\n    'This is a \\x1b[31msentence\\x1b[0m. This is another \\x1b[31msentence\\x1b[0m.'\n    \"\"\"\n    colored_word = colored(word, \"red\")\n    colored_sentence, number_of_substitions = re.subn(\n        r\"\\b{}\\b\".format(re.escape(word)), colored_word, sentence\n    )\n    if number_of_substitions == 0:\n        # Use basic string manipulation if regex fails\n        colored_sentence = sentence.replace(word, colored_word)\n    return colored_sentence\n", "path": "cleanlab/internal/token_classification_utils.py"}], "after_files": [{"content": "# Copyright (C) 2017-2022  Cleanlab Inc.\n# This file is part of cleanlab.\n#\n# cleanlab is free software: you can redistribute it and/or modify\n# it under the terms of the GNU Affero General Public License as published\n# by the Free Software Foundation, either version 3 of the License, or\n# (at your option) any later version.\n#\n# cleanlab is distributed in the hope that it will be useful,\n# but WITHOUT ANY WARRANTY; without even the implied warranty of\n# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the\n# GNU Affero General Public License for more details.\n#\n# You should have received a copy of the GNU Affero General Public License\n# along with cleanlab.  If not, see <https://www.gnu.org/licenses/>.\n\n\"\"\"\nHelper methods used internally in cleanlab.token_classification\n\"\"\"\n\nimport re\nimport string\nimport numpy as np\nfrom termcolor import colored\nfrom typing import List, Optional, Callable, Tuple\n\n\ndef get_sentence(words: List[str]) -> str:\n    \"\"\"\n    Get sentence formed by a list of words with minor processing for readability\n\n    Parameters\n    ----------\n    words:\n        list of word-level tokens\n\n    Returns\n    ----------\n    sentence:\n        sentence formed by list of word-level tokens\n\n    Examples\n    --------\n    >>> from cleanlab.internal.token_classification_utils import get_sentence\n    >>> words = [\"This\", \"is\", \"a\", \"sentence\", \".\"]\n    >>> get_sentence(words)\n    'This is a sentence.'\n    \"\"\"\n    sentence = \"\"\n    for word in words:\n        if word not in string.punctuation or word in [\"-\", \"(\"]:\n            word = \" \" + word\n        sentence += word\n    sentence = sentence.replace(\" '\", \"'\").replace(\"( \", \"(\").strip()\n    return sentence\n\n\ndef filter_sentence(\n    sentences: List[str],\n    condition: Optional[Callable[[str], bool]] = None,\n) -> Tuple[List[str], List[bool]]:\n    \"\"\"\n    Filter sentence based on some condition, and returns filter mask\n\n    Parameters\n    ----------\n    sentences:\n        list of sentences\n\n    condition:\n        sentence filtering condition\n\n    Returns\n    ---------\n    sentences:\n        list of sentences filtered\n\n    mask:\n        boolean mask such that `mask[i] == True` if the i'th sentence is included in the\n        filtered sentence, otherwise `mask[i] == False`\n\n    Examples\n    --------\n    >>> from cleanlab.internal.token_classification_utils import filter_sentence\n    >>> sentences = [\"Short sentence.\", \"This is a longer sentence.\"]\n    >>> condition = lambda x: len(x.split()) > 2\n    >>> long_sentences, _ = filter_sentence(sentences, condition)\n    >>> long_sentences\n    ['This is a longer sentence.']\n    >>> document = [\"# Headline\", \"Sentence 1.\", \"&\", \"Sentence 2.\"]\n    >>> sentences, mask = filter_sentence(document)\n    >>> sentences, mask\n    (['Sentence 1.', 'Sentence 2.'], [False, True, False, True])\n    \"\"\"\n    if not condition:\n        condition = lambda sentence: len(sentence) > 1 and \"#\" not in sentence\n    mask = list(map(condition, sentences))\n    sentences = [sentence for m, sentence in zip(mask, sentences) if m]\n    return sentences, mask\n\n\ndef process_token(token: str, replace: List[Tuple[str, str]] = [(\"#\", \"\")]) -> str:\n    \"\"\"\n    Replaces special characters in the tokens\n\n    Parameters\n    ----------\n    token:\n        token which potentially contains special characters\n\n    replace:\n        list of tuples `(s1, s2)`, where all occurances of s1 are replaced by s2\n\n    Returns\n    ---------\n    processed_token:\n        processed token whose special character has been replaced\n\n    Note\n    ----\n        Only applies to characters in the original input token.\n\n    Examples\n    --------\n    >>> from cleanlab.internal.token_classification_utils import process_token\n    >>> token = \"#Comment\"\n    >>> process_token(\"#Comment\")\n    'Comment'\n\n    Specify custom replacement rules\n\n    >>> replace = [(\"C\", \"a\"), (\"a\", \"C\")]\n    >>> process_token(\"Cleanlab\", replace)\n    'aleCnlCb'\n    \"\"\"\n    replace_dict = {re.escape(k): v for (k, v) in replace}\n    pattern = \"|\".join(replace_dict.keys())\n    compiled_pattern = re.compile(pattern)\n    replacement = lambda match: replace_dict[re.escape(match.group(0))]\n    processed_token = compiled_pattern.sub(replacement, token)\n    return processed_token\n\n\ndef mapping(entities: List[int], maps: List[int]) -> List[int]:\n    \"\"\"\n    Map a list of entities to its corresponding entities\n\n    Parameters\n    ----------\n    entities:\n        a list of given entities\n\n    maps:\n        a list of mapped entities, such that the i'th indexed token should be mapped to `maps[i]`\n\n    Returns\n    ---------\n    mapped_entities:\n        a list of mapped entities\n\n    Examples\n    --------\n    >>> unique_identities = [0, 1, 2, 3, 4]  # [\"O\", \"B-PER\", \"I-PER\", \"B-LOC\", \"I-LOC\"]\n    >>> maps = [0, 1, 1, 2, 2]  # [\"O\", \"PER\", \"PER\", \"LOC\", \"LOC\"]\n    >>> mapping(unique_identities, maps)\n    [0, 1, 1, 2, 2]  # [\"O\", \"PER\", \"PER\", \"LOC\", \"LOC\"]\n    >>> mapping([0, 0, 4, 4, 3, 4, 0, 2], maps)\n    [0, 0, 2, 2, 2, 2, 0, 1]  # [\"O\", \"O\", \"LOC\", \"LOC\", \"LOC\", \"LOC\", \"O\", \"PER\"]\n    \"\"\"\n    f = lambda x: maps[x]\n    return list(map(f, entities))\n\n\ndef merge_probs(probs: np.ndarray, maps: List[int]) -> np.ndarray:\n    \"\"\"\n    Merges model-predictive probabilities with desired mapping\n\n    Parameters\n    ----------\n    probs:\n        np.array of shape `(N, K)`, where N is the number of tokens, and K is the number of classes for the model\n\n    maps:\n        a list of mapped index, such that the probability of the token being in the i'th class is mapped to the\n        `maps[i]` index. If `maps[i] == -1`, the i'th column of `probs` is ignored. If `np.any(maps == -1)`, the\n        returned probability is re-normalized.\n\n    Returns\n    ---------\n    probs_merged:\n        np.array of shape ``(N, K')``, where `K` is the number of new classes. Probabilities are merged and\n        re-normalized if necessary.\n\n    Examples\n    --------\n    >>> import numpy as np\n    >>> from cleanlab.internal.token_classification_utils import merge_probs\n    >>> probs = np.array([\n    ...     [0.55, 0.0125, 0.0375, 0.1, 0.3],\n    ...     [0.1, 0.8, 0, 0.075, 0.025],\n    ... ])\n    >>> maps = [0, 1, 1, 2, 2]\n    >>> merge_probs(probs, maps)\n    array([[0.55, 0.05, 0.4 ],\n           [0.1 , 0.8 , 0.1 ]])\n    \"\"\"\n    old_classes = probs.shape[1]\n    map_size = np.max(maps) + 1\n    probs_merged = np.zeros([len(probs), map_size], dtype=probs.dtype.type)\n\n    for i in range(old_classes):\n        if maps[i] >= 0:\n            probs_merged[:, maps[i]] += probs[:, i]\n    if -1 in maps:\n        row_sums = probs_merged.sum(axis=1)\n        probs_merged /= row_sums[:, np.newaxis]\n    return probs_merged\n\n\ndef color_sentence(sentence: str, word: str) -> str:\n    \"\"\"\n    Searches for a given token in the sentence and returns the sentence where the given token is colored red\n\n    Parameters\n    ----------\n    sentence:\n        a sentence where the word is searched\n\n    word:\n        keyword to find in `sentence`. Assumes the word exists in the sentence.\n    Returns\n    ---------\n    colored_sentence:\n        `sentence` where the every occurrence of the word is colored red, using ``termcolor.colored``\n\n    Examples\n    --------\n    >>> from cleanlab.internal.token_classification_utils import color_sentence\n    >>> sentence = \"This is a sentence.\"\n    >>> word = \"sentence\"\n    >>> color_sentence(sentence, word)\n    'This is a \\x1b[31msentence\\x1b[0m.'\n\n    Also works for multiple occurrences of the word\n\n    >>> document = \"This is a sentence. This is another sentence.\"\n    >>> word = \"sentence\"\n    >>> color_sentence(document, word)\n    'This is a \\x1b[31msentence\\x1b[0m. This is another \\x1b[31msentence\\x1b[0m.'\n    \"\"\"\n    colored_word = colored(word, \"red\")\n    return _replace_sentence(sentence=sentence, word=word, new_word=colored_word)\n\n\ndef _replace_sentence(sentence: str, word: str, new_word: str) -> str:\n    \"\"\"\n    Searches for a given token in the sentence and returns the sentence where the given token has been replaced by\n    `new_word`.\n\n    Parameters\n    ----------\n    sentence:\n        a sentence where the word is searched\n\n    word:\n        keyword to find in `sentence`. Assumes the word exists in the sentence.\n\n    new_word:\n        the word to replace the keyword with\n\n    Returns\n    ---------\n    new_sentence:\n        `sentence` where the every occurrence of the word is replaced by `colored_word`\n    \"\"\"\n\n    new_sentence, number_of_substitions = re.subn(\n        r\"\\b{}\\b\".format(re.escape(word)), new_word, sentence\n    )\n    if number_of_substitions == 0:\n        # Use basic string manipulation if regex fails\n        new_sentence = sentence.replace(word, new_word)\n    return new_sentence\n", "path": "cleanlab/internal/token_classification_utils.py"}]}