We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
Token Classification: given_labels -> labels
Rename argument `given_labels` to just `labels` throughout the module to be more consistent with rest of package.

Example:  https://github.com/cleanlab/cleanlab/blob/master/cleanlab/token_classification/summary.py#L33

Need to correspondingly update tutorial notebook as well, possibly also example notebook but I don't think so.

--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `cleanlab/token_classification/summary.py`
Content:
```
1 # Copyright (C) 2017-2022  Cleanlab Inc.
2 # This file is part of cleanlab.
3 #
4 # cleanlab is free software: you can redistribute it and/or modify
5 # it under the terms of the GNU Affero General Public License as published
6 # by the Free Software Foundation, either version 3 of the License, or
7 # (at your option) any later version.
8 #
9 # cleanlab is distributed in the hope that it will be useful,
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
12 # GNU Affero General Public License for more details.
13 #
14 # You should have received a copy of the GNU Affero General Public License
15 # along with cleanlab.  If not, see <https://www.gnu.org/licenses/>.
16 
17 """
18 Methods to display sentences and their label issues in a token classification dataset (text data), as well as summarize the types of issues identified.
19 """
20 
21 import numpy as np
22 import pandas as pd
23 from typing import List, Optional, Tuple, Dict, Any
24 
25 from cleanlab.internal.token_classification_utils import get_sentence, color_sentence
26 
27 
28 def display_issues(
29     issues: list,
30     given_words: List[List[str]],
31     *,
32     pred_probs: Optional[list] = None,
33     given_labels: Optional[list] = None,
34     exclude: List[Tuple[int, int]] = [],
35     class_names: Optional[List[str]] = None,
36     top: int = 20
37 ) -> None:
38     """
39     Display issues, including sentence with issue token highlighted. Also shows given and predicted label
40     if possible.
41 
42     Parameters
43     ----------
44     issues:
45         list of tuples `(i, j)`, which represents the j'th token of the i'th sentence.
46 
47     given_words:
48         tokens in a nested-list format, such that `given_words[i]` contains the words of the i'th sentence from
49         the original file.
50 
51     pred_probs:
52         list of model-predicted probability, such that `pred_probs[i]` contains the model-predicted probability of
53         the tokens in the i'th sentence, and has shape `(N, K)`, where `N` is the number of given tokens of the i'th
54         sentence, and `K` is the number of classes predicted by the model. If provided, also displays the predicted
55         label of the token.
56 
57     given_labels:
58         list of given labels, such that `given_labels[i]` is a list containing the given labels of the tokens in the
59         i'th sentence, and has length equal to the number of given tokens of the i'th sentence. If provided, also
60         displays the given label of the token.
61 
62     exclude:
63         list of given/predicted label swaps to be excluded. For example, if `exclude=[(0, 1), (1, 0)]`, swaps between
64         class 0 and 1 are not displayed.
65 
66     class_names:
67         name of classes. If not provided, display the integer index for predicted and given labels.
68 
69     top: int, default=20
70         maximum number of outputs to be printed.
71 
72     """
73     if not class_names:
74         print(
75             "Classes will be printed in terms of their integer index since `class_names` was not provided. "
76         )
77         print("Specify this argument to see the string names of each class. \n")
78 
79     shown = 0
80     is_tuple = isinstance(issues[0], tuple)
81 
82     for issue in issues:
83         if is_tuple:
84             i, j = issue
85             sentence = get_sentence(given_words[i])
86             word = given_words[i][j]
87 
88             if pred_probs:
89                 prediction = pred_probs[i][j].argmax()
90             if given_labels:
91                 given = given_labels[i][j]
92             if pred_probs and given_labels:
93                 if (given, prediction) in exclude:
94                     continue
95 
96             if pred_probs and class_names:
97                 prediction = class_names[prediction]
98             if given_labels and class_names:
99                 given = class_names[given]
100 
101             shown += 1
102             print("Sentence %d, token %d: \n%s" % (i, j, color_sentence(sentence, word)))
103             if given_labels and not pred_probs:
104                 print("Given label: %s\n" % str(given))
105             elif not given_labels and pred_probs:
106                 print("Predicted label according to provided pred_probs: %s\n" % str(prediction))
107             elif given_labels and pred_probs:
108                 print(
109                     "Given label: %s, predicted label according to provided pred_probs: %s\n"
110                     % (str(given), str(prediction))
111                 )
112             else:
113                 print()
114         else:
115             shown += 1
116             sentence = get_sentence(given_words[issue])
117             print("Sentence %d: %s\n" % (issue, sentence))
118         if shown == top:
119             break
120 
121 
122 def common_label_issues(
123     issues: List[Tuple[int, int]],
124     given_words: List[List[str]],
125     *,
126     labels: Optional[list] = None,
127     pred_probs: Optional[list] = None,
128     class_names: Optional[List[str]] = None,
129     top: int = 10,
130     exclude: List[Tuple[int, int]] = [],
131     verbose: bool = True
132 ) -> pd.DataFrame:
133     """
134     Display the most common tokens that are potentially mislabeled.
135 
136     Parameters
137     ----------
138     issues:
139         list of tuples `(i, j)`, which represents the j'th token of the i'th sentence.
140 
141     given_words:
142         tokens in a nested-list format, such that `given_words[i]` contains the words of the i'th sentence from
143         the original file.
144 
145     labels:
146         list of given labels, such that `labels[i]` is a list containing the given labels of the tokens in the i'th
147     sentence, and has length equal to the number of given tokens of the i'th sentence. If provided, also
148     displays the given label of the token.
149 
150     pred_probs:
151         list of model-predicted probability, such that `pred_probs[i]` contains the model-predicted probability of
152     the tokens in the i'th sentence, and has shape `(N, K)`, where `N` is the number of given tokens of the i'th
153     sentence, and `K` is the number of classes predicted by the model. If both `labels` and `pred_probs` are
154     provided, also evaluate each type of given/predicted label swap.
155 
156     class_names:
157         name of classes. If not provided, display the integer index for predicted and given labels.
158 
159     top:
160         maximum number of outputs to be printed.
161 
162     exclude:
163         list of given/predicted label swaps to be excluded. For example, if `exclude=[(0, 1), (1, 0)]`, swaps between
164         class 0 and 1 are not displayed.
165 
166     verbose:
167         if set to True, also display each type of given/predicted label swap for each token.
168 
169     Returns
170     ---------
171     df:
172         if both `labels` and `pred_probs` are provided, return a data frame with columns ['token', 'given_label',
173         'predicted_label', 'num_label_issues'], and each row contains the information for a specific token and
174         given/predicted label swap, ordered by the number of label issues in descending order. Otherwise, return
175         a data frame with columns ['token', 'num_label_issues'], and each row contains the information for a specific
176         token, ordered by the number of label issues in descending order.
177 
178     """
179     count: Dict[str, Any] = {}
180     if not labels or not pred_probs:
181         for issue in issues:
182             i, j = issue
183             word = given_words[i][j]
184             if word not in count:
185                 count[word] = 0
186             count[word] += 1
187 
188         words = [word for word in count.keys()]
189         freq = [count[word] for word in words]
190         rank = np.argsort(freq)[::-1][:top]
191 
192         for r in rank:
193             print(
194                 "Token '%s' is potentially mislabeled %d times throughout the dataset\n"
195                 % (words[r], freq[r])
196             )
197 
198         info = [[word, f] for word, f in zip(words, freq)]
199         info = sorted(info, key=lambda x: x[1], reverse=True)
200         return pd.DataFrame(info, columns=["token", "num_label_issues"])
201 
202     if not class_names:
203         print(
204             "Classes will be printed in terms of their integer index since `class_names` was not provided. "
205         )
206         print("Specify this argument to see the string names of each class. \n")
207 
208     n = pred_probs[0].shape[1]
209     for issue in issues:
210         i, j = issue
211         word = given_words[i][j]
212         label = labels[i][j]
213         pred = pred_probs[i][j].argmax()
214         if word not in count:
215             count[word] = np.zeros([n, n], dtype=int)
216         if (label, pred) not in exclude:
217             count[word][label][pred] += 1
218     words = [word for word in count.keys()]
219     freq = [np.sum(count[word]) for word in words]
220     rank = np.argsort(freq)[::-1][:top]
221 
222     for r in rank:
223         matrix = count[words[r]]
224         most_frequent = np.argsort(count[words[r]].flatten())[::-1]
225         print(
226             "Token '%s' is potentially mislabeled %d times throughout the dataset"
227             % (words[r], freq[r])
228         )
229         if verbose:
230             print(
231                 "---------------------------------------------------------------------------------------"
232             )
233             for f in most_frequent:
234                 i, j = f // n, f % n
235                 if matrix[i][j] == 0:
236                     break
237                 if class_names:
238                     print(
239                         "labeled as class `%s` but predicted to actually be class `%s` %d times"
240                         % (class_names[i], class_names[j], matrix[i][j])
241                     )
242                 else:
243                     print(
244                         "labeled as class %d but predicted to actually be class %d %d times"
245                         % (i, j, matrix[i][j])
246                     )
247         print()
248     info = []
249     for word in words:
250         for i in range(n):
251             for j in range(n):
252                 num = count[word][i][j]
253                 if num > 0:
254                     if not class_names:
255                         info.append([word, i, j, num])
256                     else:
257                         info.append([word, class_names[i], class_names[j], num])
258     info = sorted(info, key=lambda x: x[3], reverse=True)
259     return pd.DataFrame(
260         info, columns=["token", "given_label", "predicted_label", "num_label_issues"]
261     )
262 
263 
264 def filter_by_token(
265     token: str, issues: List[Tuple[int, int]], given_words: List[List[str]]
266 ) -> List[Tuple[int, int]]:
267     """
268     Searches a specific token within all issue tokens
269 
270     Parameters
271     ----------
272         token:
273             the specific token the user is looking for
274 
275         issues:
276             list of tuples `(i, j)`, which represents the j'th token of the i'th sentence.
277 
278         given_words:
279             tokens in a nested-list format, such that `given_words[i]` contains the words of the i'th sentence from
280         the original file.
281 
282     Returns
283     ----------
284     returned_issues:
285         list of tuples `(i, j)`, which represents the j'th token of the i'th sentence.
286 
287     """
288     returned_issues = []
289     for issue in issues:
290         i, j = issue
291         if token.lower() == given_words[i][j].lower():
292             returned_issues.append(issue)
293     return returned_issues
294
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/cleanlab/token_classification/summary.py b/cleanlab/token_classification/summary.py
--- a/cleanlab/token_classification/summary.py
+++ b/cleanlab/token_classification/summary.py
@@ -9,7 +9,7 @@
     given_words: List[List[str]],
     *,
     pred_probs: Optional[list] = None,
-    given_labels: Optional[list] = None,
+    labels: Optional[list] = None,
     exclude: List[Tuple[int, int]] = [],
     class_names: Optional[List[str]] = None,
     top: int = 20
@@ -33,8 +33,8 @@
         sentence, and `K` is the number of classes predicted by the model. If provided, also displays the predicted
         label of the token.
 
-    given_labels:
-        list of given labels, such that `given_labels[i]` is a list containing the given labels of the tokens in the
+    labels:
+        list of given labels, such that `labels[i]` is a list containing the given labels of the tokens in the
         i'th sentence, and has length equal to the number of given tokens of the i'th sentence. If provided, also
         displays the given label of the token.
 
@@ -66,24 +66,24 @@
 
             if pred_probs:
                 prediction = pred_probs[i][j].argmax()
-            if given_labels:
-                given = given_labels[i][j]
-            if pred_probs and given_labels:
+            if labels:
+                given = labels[i][j]
+            if pred_probs and labels:
                 if (given, prediction) in exclude:
                     continue
 
             if pred_probs and class_names:
                 prediction = class_names[prediction]
-            if given_labels and class_names:
+            if labels and class_names:
                 given = class_names[given]
 
             shown += 1
             print("Sentence %d, token %d: \n%s" % (i, j, color_sentence(sentence, word)))
-            if given_labels and not pred_probs:
+            if labels and not pred_probs:
                 print("Given label: %s\n" % str(given))
-            elif not given_labels and pred_probs:
+            elif not labels and pred_probs:
                 print("Predicted label according to provided pred_probs: %s\n" % str(prediction))
-            elif given_labels and pred_probs:
+            elif labels and pred_probs:
                 print(
                     "Given label: %s, predicted label according to provided pred_probs: %s\n"
                     % (str(given), str(prediction))

{"golden_diff": "diff --git a/cleanlab/token_classification/summary.py b/cleanlab/token_classification/summary.py\n--- a/cleanlab/token_classification/summary.py\n+++ b/cleanlab/token_classification/summary.py\n@@ -9,7 +9,7 @@\n     given_words: List[List[str]],\n     *,\n     pred_probs: Optional[list] = None,\n-    given_labels: Optional[list] = None,\n+    labels: Optional[list] = None,\n     exclude: List[Tuple[int, int]] = [],\n     class_names: Optional[List[str]] = None,\n     top: int = 20\n@@ -33,8 +33,8 @@\n         sentence, and `K` is the number of classes predicted by the model. If provided, also displays the predicted\n         label of the token.\n \n-    given_labels:\n-        list of given labels, such that `given_labels[i]` is a list containing the given labels of the tokens in the\n+    labels:\n+        list of given labels, such that `labels[i]` is a list containing the given labels of the tokens in the\n         i'th sentence, and has length equal to the number of given tokens of the i'th sentence. If provided, also\n         displays the given label of the token.\n \n@@ -66,24 +66,24 @@\n \n             if pred_probs:\n                 prediction = pred_probs[i][j].argmax()\n-            if given_labels:\n-                given = given_labels[i][j]\n-            if pred_probs and given_labels:\n+            if labels:\n+                given = labels[i][j]\n+            if pred_probs and labels:\n                 if (given, prediction) in exclude:\n                     continue\n \n             if pred_probs and class_names:\n                 prediction = class_names[prediction]\n-            if given_labels and class_names:\n+            if labels and class_names:\n                 given = class_names[given]\n \n             shown += 1\n             print(\"Sentence %d, token %d: \\n%s\" % (i, j, color_sentence(sentence, word)))\n-            if given_labels and not pred_probs:\n+            if labels and not pred_probs:\n                 print(\"Given label: %s\\n\" % str(given))\n-            elif not given_labels and pred_probs:\n+            elif not labels and pred_probs:\n                 print(\"Predicted label according to provided pred_probs: %s\\n\" % str(prediction))\n-            elif given_labels and pred_probs:\n+            elif labels and pred_probs:\n                 print(\n                     \"Given label: %s, predicted label according to provided pred_probs: %s\\n\"\n                     % (str(given), str(prediction))\n", "issue": "Token Classification: given_labels -> labels\nRename argument `given_labels` to just `labels` throughout the module to be more consistent with rest of package.\n\nExample:  https://github.com/cleanlab/cleanlab/blob/master/cleanlab/token_classification/summary.py#L33\n\nNeed to correspondingly update tutorial notebook as well, possibly also example notebook but I don't think so.\n", "before_files": [{"content": "# Copyright (C) 2017-2022  Cleanlab Inc.\n# This file is part of cleanlab.\n#\n# cleanlab is free software: you can redistribute it and/or modify\n# it under the terms of the GNU Affero General Public License as published\n# by the Free Software Foundation, either version 3 of the License, or\n# (at your option) any later version.\n#\n# cleanlab is distributed in the hope that it will be useful,\n# but WITHOUT ANY WARRANTY; without even the implied warranty of\n# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the\n# GNU Affero General Public License for more details.\n#\n# You should have received a copy of the GNU Affero General Public License\n# along with cleanlab.  If not, see <https://www.gnu.org/licenses/>.\n\n\"\"\"\nMethods to display sentences and their label issues in a token classification dataset (text data), as well as summarize the types of issues identified.\n\"\"\"\n\nimport numpy as np\nimport pandas as pd\nfrom typing import List, Optional, Tuple, Dict, Any\n\nfrom cleanlab.internal.token_classification_utils import get_sentence, color_sentence\n\n\ndef display_issues(\n    issues: list,\n    given_words: List[List[str]],\n    *,\n    pred_probs: Optional[list] = None,\n    given_labels: Optional[list] = None,\n    exclude: List[Tuple[int, int]] = [],\n    class_names: Optional[List[str]] = None,\n    top: int = 20\n) -> None:\n    \"\"\"\n    Display issues, including sentence with issue token highlighted. Also shows given and predicted label\n    if possible.\n\n    Parameters\n    ----------\n    issues:\n        list of tuples `(i, j)`, which represents the j'th token of the i'th sentence.\n\n    given_words:\n        tokens in a nested-list format, such that `given_words[i]` contains the words of the i'th sentence from\n        the original file.\n\n    pred_probs:\n        list of model-predicted probability, such that `pred_probs[i]` contains the model-predicted probability of\n        the tokens in the i'th sentence, and has shape `(N, K)`, where `N` is the number of given tokens of the i'th\n        sentence, and `K` is the number of classes predicted by the model. If provided, also displays the predicted\n        label of the token.\n\n    given_labels:\n        list of given labels, such that `given_labels[i]` is a list containing the given labels of the tokens in the\n        i'th sentence, and has length equal to the number of given tokens of the i'th sentence. If provided, also\n        displays the given label of the token.\n\n    exclude:\n        list of given/predicted label swaps to be excluded. For example, if `exclude=[(0, 1), (1, 0)]`, swaps between\n        class 0 and 1 are not displayed.\n\n    class_names:\n        name of classes. If not provided, display the integer index for predicted and given labels.\n\n    top: int, default=20\n        maximum number of outputs to be printed.\n\n    \"\"\"\n    if not class_names:\n        print(\n            \"Classes will be printed in terms of their integer index since `class_names` was not provided. \"\n        )\n        print(\"Specify this argument to see the string names of each class. \\n\")\n\n    shown = 0\n    is_tuple = isinstance(issues[0], tuple)\n\n    for issue in issues:\n        if is_tuple:\n            i, j = issue\n            sentence = get_sentence(given_words[i])\n            word = given_words[i][j]\n\n            if pred_probs:\n                prediction = pred_probs[i][j].argmax()\n            if given_labels:\n                given = given_labels[i][j]\n            if pred_probs and given_labels:\n                if (given, prediction) in exclude:\n                    continue\n\n            if pred_probs and class_names:\n                prediction = class_names[prediction]\n            if given_labels and class_names:\n                given = class_names[given]\n\n            shown += 1\n            print(\"Sentence %d, token %d: \\n%s\" % (i, j, color_sentence(sentence, word)))\n            if given_labels and not pred_probs:\n                print(\"Given label: %s\\n\" % str(given))\n            elif not given_labels and pred_probs:\n                print(\"Predicted label according to provided pred_probs: %s\\n\" % str(prediction))\n            elif given_labels and pred_probs:\n                print(\n                    \"Given label: %s, predicted label according to provided pred_probs: %s\\n\"\n                    % (str(given), str(prediction))\n                )\n            else:\n                print()\n        else:\n            shown += 1\n            sentence = get_sentence(given_words[issue])\n            print(\"Sentence %d: %s\\n\" % (issue, sentence))\n        if shown == top:\n            break\n\n\ndef common_label_issues(\n    issues: List[Tuple[int, int]],\n    given_words: List[List[str]],\n    *,\n    labels: Optional[list] = None,\n    pred_probs: Optional[list] = None,\n    class_names: Optional[List[str]] = None,\n    top: int = 10,\n    exclude: List[Tuple[int, int]] = [],\n    verbose: bool = True\n) -> pd.DataFrame:\n    \"\"\"\n    Display the most common tokens that are potentially mislabeled.\n\n    Parameters\n    ----------\n    issues:\n        list of tuples `(i, j)`, which represents the j'th token of the i'th sentence.\n\n    given_words:\n        tokens in a nested-list format, such that `given_words[i]` contains the words of the i'th sentence from\n        the original file.\n\n    labels:\n        list of given labels, such that `labels[i]` is a list containing the given labels of the tokens in the i'th\n    sentence, and has length equal to the number of given tokens of the i'th sentence. If provided, also\n    displays the given label of the token.\n\n    pred_probs:\n        list of model-predicted probability, such that `pred_probs[i]` contains the model-predicted probability of\n    the tokens in the i'th sentence, and has shape `(N, K)`, where `N` is the number of given tokens of the i'th\n    sentence, and `K` is the number of classes predicted by the model. If both `labels` and `pred_probs` are\n    provided, also evaluate each type of given/predicted label swap.\n\n    class_names:\n        name of classes. If not provided, display the integer index for predicted and given labels.\n\n    top:\n        maximum number of outputs to be printed.\n\n    exclude:\n        list of given/predicted label swaps to be excluded. For example, if `exclude=[(0, 1), (1, 0)]`, swaps between\n        class 0 and 1 are not displayed.\n\n    verbose:\n        if set to True, also display each type of given/predicted label swap for each token.\n\n    Returns\n    ---------\n    df:\n        if both `labels` and `pred_probs` are provided, return a data frame with columns ['token', 'given_label',\n        'predicted_label', 'num_label_issues'], and each row contains the information for a specific token and\n        given/predicted label swap, ordered by the number of label issues in descending order. Otherwise, return\n        a data frame with columns ['token', 'num_label_issues'], and each row contains the information for a specific\n        token, ordered by the number of label issues in descending order.\n\n    \"\"\"\n    count: Dict[str, Any] = {}\n    if not labels or not pred_probs:\n        for issue in issues:\n            i, j = issue\n            word = given_words[i][j]\n            if word not in count:\n                count[word] = 0\n            count[word] += 1\n\n        words = [word for word in count.keys()]\n        freq = [count[word] for word in words]\n        rank = np.argsort(freq)[::-1][:top]\n\n        for r in rank:\n            print(\n                \"Token '%s' is potentially mislabeled %d times throughout the dataset\\n\"\n                % (words[r], freq[r])\n            )\n\n        info = [[word, f] for word, f in zip(words, freq)]\n        info = sorted(info, key=lambda x: x[1], reverse=True)\n        return pd.DataFrame(info, columns=[\"token\", \"num_label_issues\"])\n\n    if not class_names:\n        print(\n            \"Classes will be printed in terms of their integer index since `class_names` was not provided. \"\n        )\n        print(\"Specify this argument to see the string names of each class. \\n\")\n\n    n = pred_probs[0].shape[1]\n    for issue in issues:\n        i, j = issue\n        word = given_words[i][j]\n        label = labels[i][j]\n        pred = pred_probs[i][j].argmax()\n        if word not in count:\n            count[word] = np.zeros([n, n], dtype=int)\n        if (label, pred) not in exclude:\n            count[word][label][pred] += 1\n    words = [word for word in count.keys()]\n    freq = [np.sum(count[word]) for word in words]\n    rank = np.argsort(freq)[::-1][:top]\n\n    for r in rank:\n        matrix = count[words[r]]\n        most_frequent = np.argsort(count[words[r]].flatten())[::-1]\n        print(\n            \"Token '%s' is potentially mislabeled %d times throughout the dataset\"\n            % (words[r], freq[r])\n        )\n        if verbose:\n            print(\n                \"---------------------------------------------------------------------------------------\"\n            )\n            for f in most_frequent:\n                i, j = f // n, f % n\n                if matrix[i][j] == 0:\n                    break\n                if class_names:\n                    print(\n                        \"labeled as class `%s` but predicted to actually be class `%s` %d times\"\n                        % (class_names[i], class_names[j], matrix[i][j])\n                    )\n                else:\n                    print(\n                        \"labeled as class %d but predicted to actually be class %d %d times\"\n                        % (i, j, matrix[i][j])\n                    )\n        print()\n    info = []\n    for word in words:\n        for i in range(n):\n            for j in range(n):\n                num = count[word][i][j]\n                if num > 0:\n                    if not class_names:\n                        info.append([word, i, j, num])\n                    else:\n                        info.append([word, class_names[i], class_names[j], num])\n    info = sorted(info, key=lambda x: x[3], reverse=True)\n    return pd.DataFrame(\n        info, columns=[\"token\", \"given_label\", \"predicted_label\", \"num_label_issues\"]\n    )\n\n\ndef filter_by_token(\n    token: str, issues: List[Tuple[int, int]], given_words: List[List[str]]\n) -> List[Tuple[int, int]]:\n    \"\"\"\n    Searches a specific token within all issue tokens\n\n    Parameters\n    ----------\n        token:\n            the specific token the user is looking for\n\n        issues:\n            list of tuples `(i, j)`, which represents the j'th token of the i'th sentence.\n\n        given_words:\n            tokens in a nested-list format, such that `given_words[i]` contains the words of the i'th sentence from\n        the original file.\n\n    Returns\n    ----------\n    returned_issues:\n        list of tuples `(i, j)`, which represents the j'th token of the i'th sentence.\n\n    \"\"\"\n    returned_issues = []\n    for issue in issues:\n        i, j = issue\n        if token.lower() == given_words[i][j].lower():\n            returned_issues.append(issue)\n    return returned_issues\n", "path": "cleanlab/token_classification/summary.py"}], "after_files": [{"content": "import numpy as np\nimport pandas as pd\nfrom cleanlab.internal.token_classification_utils import get_sentence, color_sentence\nfrom typing import List, Optional, Tuple, Dict, Any\n\n\ndef display_issues(\n    issues: list,\n    given_words: List[List[str]],\n    *,\n    pred_probs: Optional[list] = None,\n    labels: Optional[list] = None,\n    exclude: List[Tuple[int, int]] = [],\n    class_names: Optional[List[str]] = None,\n    top: int = 20\n) -> None:\n    \"\"\"\n    Display issues, including sentence with issue token highlighted. Also shows given and predicted label\n    if possible.\n\n    Parameters\n    ----------\n    issues:\n        list of tuples `(i, j)`, which represents the j'th token of the i'th sentence.\n\n    given_words:\n        tokens in a nested-list format, such that `given_words[i]` contains the words of the i'th sentence from\n        the original file.\n\n    pred_probs:\n        list of model-predicted probability, such that `pred_probs[i]` contains the model-predicted probability of\n        the tokens in the i'th sentence, and has shape `(N, K)`, where `N` is the number of given tokens of the i'th\n        sentence, and `K` is the number of classes predicted by the model. If provided, also displays the predicted\n        label of the token.\n\n    labels:\n        list of given labels, such that `labels[i]` is a list containing the given labels of the tokens in the\n        i'th sentence, and has length equal to the number of given tokens of the i'th sentence. If provided, also\n        displays the given label of the token.\n\n    exclude:\n        list of given/predicted label swaps to be excluded. For example, if `exclude=[(0, 1), (1, 0)]`, swaps between\n        class 0 and 1 are not displayed.\n\n    class_names:\n        name of classes. If not provided, display the integer index for predicted and given labels.\n\n    top: int, default=20\n        maximum number of outputs to be printed.\n\n    \"\"\"\n    if not class_names:\n        print(\n            \"Classes will be printed in terms of their integer index since `class_names` was not provided. \"\n        )\n        print(\"Specify this argument to see the string names of each class. \\n\")\n\n    shown = 0\n    is_tuple = isinstance(issues[0], tuple)\n\n    for issue in issues:\n        if is_tuple:\n            i, j = issue\n            sentence = get_sentence(given_words[i])\n            word = given_words[i][j]\n\n            if pred_probs:\n                prediction = pred_probs[i][j].argmax()\n            if labels:\n                given = labels[i][j]\n            if pred_probs and labels:\n                if (given, prediction) in exclude:\n                    continue\n\n            if pred_probs and class_names:\n                prediction = class_names[prediction]\n            if labels and class_names:\n                given = class_names[given]\n\n            shown += 1\n            print(\"Sentence %d, token %d: \\n%s\" % (i, j, color_sentence(sentence, word)))\n            if labels and not pred_probs:\n                print(\"Given label: %s\\n\" % str(given))\n            elif not labels and pred_probs:\n                print(\"Predicted label according to provided pred_probs: %s\\n\" % str(prediction))\n            elif labels and pred_probs:\n                print(\n                    \"Given label: %s, predicted label according to provided pred_probs: %s\\n\"\n                    % (str(given), str(prediction))\n                )\n            else:\n                print()\n        else:\n            shown += 1\n            sentence = get_sentence(given_words[issue])\n            print(\"Sentence %d: %s\\n\" % (issue, sentence))\n        if shown == top:\n            break\n\n\ndef common_label_issues(\n    issues: List[Tuple[int, int]],\n    given_words: List[List[str]],\n    *,\n    labels: Optional[list] = None,\n    pred_probs: Optional[list] = None,\n    class_names: Optional[List[str]] = None,\n    top: int = 10,\n    exclude: List[Tuple[int, int]] = [],\n    verbose: bool = True\n) -> pd.DataFrame:\n    \"\"\"\n    Display the most common tokens that are potentially mislabeled.\n\n    Parameters\n    ----------\n    issues:\n        list of tuples `(i, j)`, which represents the j'th token of the i'th sentence.\n\n    given_words:\n        tokens in a nested-list format, such that `given_words[i]` contains the words of the i'th sentence from\n        the original file.\n\n    labels:\n        list of given labels, such that `labels[i]` is a list containing the given labels of the tokens in the i'th\n    sentence, and has length equal to the number of given tokens of the i'th sentence. If provided, also\n    displays the given label of the token.\n\n    pred_probs:\n        list of model-predicted probability, such that `pred_probs[i]` contains the model-predicted probability of\n    the tokens in the i'th sentence, and has shape `(N, K)`, where `N` is the number of given tokens of the i'th\n    sentence, and `K` is the number of classes predicted by the model. If both `labels` and `pred_probs` are\n    provided, also evaluate each type of given/predicted label swap.\n\n    class_names:\n        name of classes. If not provided, display the integer index for predicted and given labels.\n\n    top:\n        maximum number of outputs to be printed.\n\n    exclude:\n        list of given/predicted label swaps to be excluded. For example, if `exclude=[(0, 1), (1, 0)]`, swaps between\n        class 0 and 1 are not displayed.\n\n    verbose:\n        if set to True, also display each type of given/predicted label swap for each token.\n\n    Returns\n    ---------\n    df:\n        if both `labels` and `pred_probs` are provided, return a data frame with columns ['token', 'given_label',\n        'predicted_label', 'num_label_issues'], and each row contains the information for a specific token and\n        given/predicted label swap, ordered by the number of label issues in descending order. Otherwise, return\n        a data frame with columns ['token', 'num_label_issues'], and each row contains the information for a specific\n        token, ordered by the number of label issues in descending order.\n\n    \"\"\"\n    count: Dict[str, Any] = {}\n    if not labels or not pred_probs:\n        for issue in issues:\n            i, j = issue\n            word = given_words[i][j]\n            if word not in count:\n                count[word] = 0\n            count[word] += 1\n\n        words = [word for word in count.keys()]\n        freq = [count[word] for word in words]\n        rank = np.argsort(freq)[::-1][:top]\n\n        for r in rank:\n            print(\n                \"Token '%s' is potentially mislabeled %d times throughout the dataset\\n\"\n                % (words[r], freq[r])\n            )\n\n        info = [[word, f] for word, f in zip(words, freq)]\n        info = sorted(info, key=lambda x: x[1], reverse=True)\n        return pd.DataFrame(info, columns=[\"token\", \"num_label_issues\"])\n\n    if not class_names:\n        print(\n            \"Classes will be printed in terms of their integer index since `class_names` was not provided. \"\n        )\n        print(\"Specify this argument to see the string names of each class. \\n\")\n\n    n = pred_probs[0].shape[1]\n    for issue in issues:\n        i, j = issue\n        word = given_words[i][j]\n        label = labels[i][j]\n        pred = pred_probs[i][j].argmax()\n        if word not in count:\n            count[word] = np.zeros([n, n], dtype=int)\n        if (label, pred) not in exclude:\n            count[word][label][pred] += 1\n    words = [word for word in count.keys()]\n    freq = [np.sum(count[word]) for word in words]\n    rank = np.argsort(freq)[::-1][:top]\n\n    for r in rank:\n        matrix = count[words[r]]\n        most_frequent = np.argsort(count[words[r]].flatten())[::-1]\n        print(\n            \"Token '%s' is potentially mislabeled %d times throughout the dataset\"\n            % (words[r], freq[r])\n        )\n        if verbose:\n            print(\n                \"---------------------------------------------------------------------------------------\"\n            )\n            for f in most_frequent:\n                i, j = f // n, f % n\n                if matrix[i][j] == 0:\n                    break\n                if class_names:\n                    print(\n                        \"labeled as class `%s` but predicted to actually be class `%s` %d times\"\n                        % (class_names[i], class_names[j], matrix[i][j])\n                    )\n                else:\n                    print(\n                        \"labeled as class %d but predicted to actually be class %d %d times\"\n                        % (i, j, matrix[i][j])\n                    )\n        print()\n    info = []\n    for word in words:\n        for i in range(n):\n            for j in range(n):\n                num = count[word][i][j]\n                if num > 0:\n                    if not class_names:\n                        info.append([word, i, j, num])\n                    else:\n                        info.append([word, class_names[i], class_names[j], num])\n    info = sorted(info, key=lambda x: x[3], reverse=True)\n    return pd.DataFrame(\n        info, columns=[\"token\", \"given_label\", \"predicted_label\", \"num_label_issues\"]\n    )\n\n\ndef filter_by_token(\n    token: str, issues: List[Tuple[int, int]], given_words: List[List[str]]\n) -> List[Tuple[int, int]]:\n    \"\"\"\n    Searches a specific token within all issue tokens\n\n    Parameters\n    ----------\n        token:\n            the specific token the user is looking for\n\n        issues:\n            list of tuples `(i, j)`, which represents the j'th token of the i'th sentence.\n\n        given_words:\n            tokens in a nested-list format, such that `given_words[i]` contains the words of the i'th sentence from\n        the original file.\n\n    Returns\n    ----------\n    returned_issues:\n        list of tuples `(i, j)`, which represents the j'th token of the i'th sentence.\n\n    \"\"\"\n    returned_issues = []\n    for issue in issues:\n        i, j = issue\n        if token.lower() == given_words[i][j].lower():\n            returned_issues.append(issue)\n    return returned_issues\n", "path": "cleanlab/token_classification/summary.py"}]}