We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
[model:bugtype] ValueError: y should be a 1d array, got an array of shape (198635, 5) instead.
> I'm wondering if the error encountered during the training of the `bugtype` model is something we should investigate.
>
> ```bash
> Traceback (most recent call last):
>   File "/home/promisefru/anaconda3/envs/bugbug/lib/python3.10/runpy.py", line 196, in _run_module_as_main
>     return _run_code(code, main_globals, None,
>   File "/home/promisefru/anaconda3/envs/bugbug/lib/python3.10/runpy.py", line 86, in _run_code
>     exec(code, run_globals)
>   File "/home/promisefru/mozilla/bugbug/scripts/trainer.py", line 145, in <module>
>     main()
>   File "/home/promisefru/mozilla/bugbug/scripts/trainer.py", line 141, in main
>     retriever.go(args)
>   File "/home/promisefru/mozilla/bugbug/scripts/trainer.py", line 41, in go
>     metrics = model_obj.train(limit=args.limit)
>   File "/home/promisefru/mozilla/bugbug/bugbug/model.py", line 377, in train
>     self.le.fit(y)
>   File "/home/promisefru/anaconda3/envs/bugbug/lib/python3.10/site-packages/sklearn/preprocessing/_label.py", line 98, in fit
>     y = column_or_1d(y, warn=True)
>   File "/home/promisefru/anaconda3/envs/bugbug/lib/python3.10/site-packages/sklearn/utils/validation.py", line 1156, in column_or_1d
>     raise ValueError(
> ValueError: y should be a 1d array, got an array of shape (198635, 5) instead.
> ```

_Originally posted by @PromiseFru in https://github.com/mozilla/bugbug/issues/3928#issuecomment-1875673580_

See also: https://github.com/mozilla/bugbug/pull/3823#pullrequestreview-1746981626

--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `bugbug/models/rcatype.py`
Content:
```
1 # -*- coding: utf-8 -*-
2 # This Source Code Form is subject to the terms of the Mozilla Public
3 # License, v. 2.0. If a copy of the MPL was not distributed with this file,
4 # You can obtain one at http://mozilla.org/MPL/2.0/.
5 
6 import logging
7 import re
8 
9 import numpy as np
10 import xgboost
11 from sklearn.compose import ColumnTransformer
12 from sklearn.feature_extraction import DictVectorizer
13 from sklearn.multiclass import OneVsRestClassifier
14 from sklearn.pipeline import Pipeline
15 
16 from bugbug import bug_features, bugzilla, feature_cleanup, utils
17 from bugbug.model import BugModel
18 
19 # For the moment, rca - XYZ is treated of bugtype XYZ,
20 # so we don't need to store it in a dictionary.
21 RCA_CATEGORIES = [
22     "requirementerror",
23     "poorarchitecture",
24     "designerror",
25     "codingerror",
26     "testingerror",
27     "externalsoftwareaffectingfirefox",
28     "performanceerror",
29     "standards",
30     "systemerror",
31     "localizationerror",
32     "memory",
33     "infrastructure/builderror",
34     "communicationissues",
35     "productdecision",
36     "undocumentedchange",
37     "cornercase",
38 ]
39 
40 RCA_SUBCATEGORIES = [
41     "codingerror-syntaxerror",
42     "codingerror-logicalerror",
43     "codingerror-semanticerror",
44     "codingerror-runtimeerror",
45     "codingerror-unhandledexceptions",
46     "codingerror-internalapiissue",
47     "codingerror-networkissue",
48     "codingerror-compatibilityissue",
49     "codingerror-other",
50 ]
51 
52 logger = logging.getLogger(__name__)
53 
54 
55 class RCATypeModel(BugModel):
56     def __init__(
57         self, lemmatization=False, historical=False, rca_subcategories_enabled=False
58     ):
59         BugModel.__init__(self, lemmatization)
60 
61         self.calculate_importance = False
62         self.rca_subcategories_enabled = rca_subcategories_enabled
63 
64         # should we consider only the main category or all sub categories
65         self.RCA_TYPES = (
66             RCA_SUBCATEGORIES + RCA_CATEGORIES
67             if rca_subcategories_enabled
68             else RCA_CATEGORIES
69         )
70 
71         self.RCA_LIST = sorted(set(self.RCA_TYPES))
72 
73         feature_extractors = [
74             bug_features.HasSTR(),
75             bug_features.Severity(),
76             bug_features.IsCoverityIssue(),
77             bug_features.HasCrashSignature(),
78             bug_features.HasURL(),
79             bug_features.HasW3CURL(),
80             bug_features.HasGithubURL(),
81             # Ignore whiteboards that would make the ML completely skewed
82             # bug_features.whiteboard(),
83             bug_features.Patches(),
84             bug_features.Landings(),
85             bug_features.BlockedBugsNumber(),
86             bug_features.EverAffected(),
87             bug_features.AffectedThenUnaffected(),
88             bug_features.Product(),
89             bug_features.Component(),
90         ]
91 
92         cleanup_functions = [
93             feature_cleanup.url(),
94             feature_cleanup.fileref(),
95             feature_cleanup.synonyms(),
96         ]
97 
98         self.extraction_pipeline = Pipeline(
99             [
100                 (
101                     "bug_extractor",
102                     bug_features.BugExtractor(feature_extractors, cleanup_functions),
103                 ),
104             ]
105         )
106 
107         self.clf = Pipeline(
108             [
109                 (
110                     "union",
111                     ColumnTransformer(
112                         [
113                             ("data", DictVectorizer(), "data"),
114                             ("title", self.text_vectorizer(min_df=0.001), "title"),
115                             (
116                                 "first_comment",
117                                 self.text_vectorizer(min_df=0.001),
118                                 "first_comment",
119                             ),
120                             (
121                                 "comments",
122                                 self.text_vectorizer(min_df=0.001),
123                                 "comments",
124                             ),
125                         ]
126                     ),
127                 ),
128                 (
129                     "estimator",
130                     OneVsRestClassifier(
131                         xgboost.XGBClassifier(n_jobs=utils.get_physical_cpu_count())
132                     ),
133                 ),
134             ]
135         )
136 
137     # return rca from a whiteboard string
138     def get_rca_from_whiteboard(self, whiteboard_data):
139         rca = []
140         whiteboard_data = re.sub(" ", "", whiteboard_data).lower()
141         for whiteboard in whiteboard_data.split("["):
142             if not any(whiteboard.startswith(s) for s in ("rca-", "rca:")):
143                 continue
144 
145             rca_whiteboard = re.sub("]", "", whiteboard)
146 
147             # Hybrid cases: rca:X-Y
148             rca_whiteboard = re.sub(":", "-", rca_whiteboard)
149 
150             rca_whiteboard_split = (
151                 rca_whiteboard.split("-", 1)
152                 if self.rca_subcategories_enabled
153                 else rca_whiteboard.split("-")
154             )
155 
156             if rca_whiteboard_split[1] not in self.RCA_LIST:
157                 logger.warning(rca_whiteboard_split[1] + " not in RCA_LIST")
158             else:
159                 rca.append(rca_whiteboard_split[1])
160         return rca
161 
162     def get_labels(self):
163         classes = {}
164         for bug in bugzilla.get_bugs():
165             target = np.zeros(len(self.RCA_LIST))
166             for rca in self.get_rca_from_whiteboard(bug["whiteboard"]):
167                 target[self.RCA_LIST.index(rca)] = 1
168             classes[bug["id"]] = target
169         return classes, self.RCA_LIST
170 
171     def get_feature_names(self):
172         return self.clf.named_steps["union"].get_feature_names_out()
173 
174     def overwrite_classes(self, bugs, classes, probabilities):
175         rca_values = self.get_rca(bugs)
176         for i in len(classes):
177             for rca in rca_values[i]:
178                 if rca in self.RCA_LIST:
179                     if probabilities:
180                         classes[i][self.RCA_LIST.index(rca)] = 1.0
181                     else:
182                         classes[i][self.RCA_LIST.index(rca)] = 1
183 
184         return classes
185
```

Path: `bugbug/models/bugtype.py`
Content:
```
1 # -*- coding: utf-8 -*-
2 # This Source Code Form is subject to the terms of the Mozilla Public
3 # License, v. 2.0. If a copy of the MPL was not distributed with this file,
4 # You can obtain one at http://mozilla.org/MPL/2.0/.
5 
6 import logging
7 from typing import Iterable
8 
9 import numpy as np
10 import xgboost
11 from sklearn.compose import ColumnTransformer
12 from sklearn.feature_extraction import DictVectorizer
13 from sklearn.multiclass import OneVsRestClassifier
14 from sklearn.pipeline import Pipeline
15 
16 from bugbug import bug_features, bugzilla, feature_cleanup, utils
17 from bugbug.model import BugModel
18 
19 logger = logging.getLogger(__name__)
20 
21 
22 class BugTypeModel(BugModel):
23     def __init__(self, lemmatization=False, historical=False):
24         BugModel.__init__(self, lemmatization)
25 
26         self.calculate_importance = False
27 
28         self.bug_type_extractors = bug_features.BugTypes.bug_type_extractors
29 
30         label_keyword_prefixes = {
31             keyword
32             for extractor in self.bug_type_extractors
33             for keyword in extractor.keyword_prefixes
34         }
35 
36         feature_extractors = [
37             bug_features.HasSTR(),
38             bug_features.Severity(),
39             # Ignore keywords that would make the ML completely skewed
40             # (we are going to use them as 100% rules in the evaluation phase).
41             bug_features.Keywords(label_keyword_prefixes),
42             bug_features.IsCoverityIssue(),
43             bug_features.HasCrashSignature(),
44             bug_features.HasURL(),
45             bug_features.HasW3CURL(),
46             bug_features.HasGithubURL(),
47             bug_features.Whiteboard(),
48             bug_features.Patches(),
49             bug_features.Landings(),
50             bug_features.BlockedBugsNumber(),
51             bug_features.EverAffected(),
52             bug_features.AffectedThenUnaffected(),
53             bug_features.Product(),
54             bug_features.Component(),
55         ]
56 
57         cleanup_functions = [
58             feature_cleanup.url(),
59             feature_cleanup.fileref(),
60             feature_cleanup.synonyms(),
61         ]
62 
63         self.extraction_pipeline = Pipeline(
64             [
65                 (
66                     "bug_extractor",
67                     bug_features.BugExtractor(feature_extractors, cleanup_functions),
68                 ),
69             ]
70         )
71 
72         self.clf = Pipeline(
73             [
74                 (
75                     "union",
76                     ColumnTransformer(
77                         [
78                             ("data", DictVectorizer(), "data"),
79                             ("title", self.text_vectorizer(min_df=0.001), "title"),
80                             (
81                                 "first_comment",
82                                 self.text_vectorizer(min_df=0.001),
83                                 "first_comment",
84                             ),
85                             (
86                                 "comments",
87                                 self.text_vectorizer(min_df=0.001),
88                                 "comments",
89                             ),
90                         ]
91                     ),
92                 ),
93                 (
94                     "estimator",
95                     OneVsRestClassifier(
96                         xgboost.XGBClassifier(n_jobs=utils.get_physical_cpu_count())
97                     ),
98                 ),
99             ]
100         )
101 
102     def get_labels(self) -> tuple[dict[int, np.ndarray], list[str]]:
103         classes = {}
104 
105         bug_map = {bug["id"]: bug for bug in bugzilla.get_bugs()}
106 
107         for bug_data in bug_map.values():
108             target = np.zeros(len(self.bug_type_extractors))
109             for i, is_type in enumerate(self.bug_type_extractors):
110                 if is_type(bug_data, bug_map):
111                     target[i] = 1
112 
113             classes[int(bug_data["id"])] = target
114 
115         bug_types = [extractor.type_name for extractor in self.bug_type_extractors]
116 
117         for i, bug_type in enumerate(bug_types):
118             logger.info(
119                 "%d %s bugs",
120                 sum(target[i] for target in classes.values()),
121                 bug_type,
122             )
123 
124         return classes, bug_types
125 
126     def get_feature_names(self):
127         return self.clf.named_steps["union"].get_feature_names_out()
128 
129     def overwrite_classes(
130         self,
131         bugs: Iterable[bugzilla.BugDict],
132         classes: dict[int, np.ndarray],
133         probabilities: bool,
134     ):
135         bug_map = {bug["id"]: bug for bug in bugs}
136 
137         for i, bug in enumerate(bugs):
138             for j, is_type_applicable in enumerate(self.bug_type_extractors):
139                 if is_type_applicable(bug, bug_map):
140                     if probabilities:
141                         classes[i][j] = 1.0
142                     else:
143                         classes[i][j] = 1
144 
145         return classes
146
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/bugbug/models/bugtype.py b/bugbug/models/bugtype.py
--- a/bugbug/models/bugtype.py
+++ b/bugbug/models/bugtype.py
@@ -12,6 +12,7 @@
 from sklearn.feature_extraction import DictVectorizer
 from sklearn.multiclass import OneVsRestClassifier
 from sklearn.pipeline import Pipeline
+from sklearn.preprocessing import LabelBinarizer
 
 from bugbug import bug_features, bugzilla, feature_cleanup, utils
 from bugbug.model import BugModel
@@ -25,6 +26,8 @@
 
         self.calculate_importance = False
 
+        self.le = LabelBinarizer()
+
         self.bug_type_extractors = bug_features.BugTypes.bug_type_extractors
 
         label_keyword_prefixes = {
diff --git a/bugbug/models/rcatype.py b/bugbug/models/rcatype.py
--- a/bugbug/models/rcatype.py
+++ b/bugbug/models/rcatype.py
@@ -12,6 +12,7 @@
 from sklearn.feature_extraction import DictVectorizer
 from sklearn.multiclass import OneVsRestClassifier
 from sklearn.pipeline import Pipeline
+from sklearn.preprocessing import LabelBinarizer
 
 from bugbug import bug_features, bugzilla, feature_cleanup, utils
 from bugbug.model import BugModel
@@ -61,6 +62,8 @@
         self.calculate_importance = False
         self.rca_subcategories_enabled = rca_subcategories_enabled
 
+        self.le = LabelBinarizer()
+
         # should we consider only the main category or all sub categories
         self.RCA_TYPES = (
             RCA_SUBCATEGORIES + RCA_CATEGORIES

{"golden_diff": "diff --git a/bugbug/models/bugtype.py b/bugbug/models/bugtype.py\n--- a/bugbug/models/bugtype.py\n+++ b/bugbug/models/bugtype.py\n@@ -12,6 +12,7 @@\n from sklearn.feature_extraction import DictVectorizer\n from sklearn.multiclass import OneVsRestClassifier\n from sklearn.pipeline import Pipeline\n+from sklearn.preprocessing import LabelBinarizer\n \n from bugbug import bug_features, bugzilla, feature_cleanup, utils\n from bugbug.model import BugModel\n@@ -25,6 +26,8 @@\n \n         self.calculate_importance = False\n \n+        self.le = LabelBinarizer()\n+\n         self.bug_type_extractors = bug_features.BugTypes.bug_type_extractors\n \n         label_keyword_prefixes = {\ndiff --git a/bugbug/models/rcatype.py b/bugbug/models/rcatype.py\n--- a/bugbug/models/rcatype.py\n+++ b/bugbug/models/rcatype.py\n@@ -12,6 +12,7 @@\n from sklearn.feature_extraction import DictVectorizer\n from sklearn.multiclass import OneVsRestClassifier\n from sklearn.pipeline import Pipeline\n+from sklearn.preprocessing import LabelBinarizer\n \n from bugbug import bug_features, bugzilla, feature_cleanup, utils\n from bugbug.model import BugModel\n@@ -61,6 +62,8 @@\n         self.calculate_importance = False\n         self.rca_subcategories_enabled = rca_subcategories_enabled\n \n+        self.le = LabelBinarizer()\n+\n         # should we consider only the main category or all sub categories\n         self.RCA_TYPES = (\n             RCA_SUBCATEGORIES + RCA_CATEGORIES\n", "issue": "[model:bugtype] ValueError: y should be a 1d array, got an array of shape (198635, 5) instead.\n> I'm wondering if the error encountered during the training of the `bugtype` model is something we should investigate.\r\n>\r\n> ```bash\r\n> Traceback (most recent call last):\r\n>   File \"/home/promisefru/anaconda3/envs/bugbug/lib/python3.10/runpy.py\", line 196, in _run_module_as_main\r\n>     return _run_code(code, main_globals, None,\r\n>   File \"/home/promisefru/anaconda3/envs/bugbug/lib/python3.10/runpy.py\", line 86, in _run_code\r\n>     exec(code, run_globals)\r\n>   File \"/home/promisefru/mozilla/bugbug/scripts/trainer.py\", line 145, in <module>\r\n>     main()\r\n>   File \"/home/promisefru/mozilla/bugbug/scripts/trainer.py\", line 141, in main\r\n>     retriever.go(args)\r\n>   File \"/home/promisefru/mozilla/bugbug/scripts/trainer.py\", line 41, in go\r\n>     metrics = model_obj.train(limit=args.limit)\r\n>   File \"/home/promisefru/mozilla/bugbug/bugbug/model.py\", line 377, in train\r\n>     self.le.fit(y)\r\n>   File \"/home/promisefru/anaconda3/envs/bugbug/lib/python3.10/site-packages/sklearn/preprocessing/_label.py\", line 98, in fit\r\n>     y = column_or_1d(y, warn=True)\r\n>   File \"/home/promisefru/anaconda3/envs/bugbug/lib/python3.10/site-packages/sklearn/utils/validation.py\", line 1156, in column_or_1d\r\n>     raise ValueError(\r\n> ValueError: y should be a 1d array, got an array of shape (198635, 5) instead.\r\n> ```\r\n\r\n_Originally posted by @PromiseFru in https://github.com/mozilla/bugbug/issues/3928#issuecomment-1875673580_\r\n\r\nSee also: https://github.com/mozilla/bugbug/pull/3823#pullrequestreview-1746981626\n", "before_files": [{"content": "# -*- coding: utf-8 -*-\n# This Source Code Form is subject to the terms of the Mozilla Public\n# License, v. 2.0. If a copy of the MPL was not distributed with this file,\n# You can obtain one at http://mozilla.org/MPL/2.0/.\n\nimport logging\nimport re\n\nimport numpy as np\nimport xgboost\nfrom sklearn.compose import ColumnTransformer\nfrom sklearn.feature_extraction import DictVectorizer\nfrom sklearn.multiclass import OneVsRestClassifier\nfrom sklearn.pipeline import Pipeline\n\nfrom bugbug import bug_features, bugzilla, feature_cleanup, utils\nfrom bugbug.model import BugModel\n\n# For the moment, rca - XYZ is treated of bugtype XYZ,\n# so we don't need to store it in a dictionary.\nRCA_CATEGORIES = [\n    \"requirementerror\",\n    \"poorarchitecture\",\n    \"designerror\",\n    \"codingerror\",\n    \"testingerror\",\n    \"externalsoftwareaffectingfirefox\",\n    \"performanceerror\",\n    \"standards\",\n    \"systemerror\",\n    \"localizationerror\",\n    \"memory\",\n    \"infrastructure/builderror\",\n    \"communicationissues\",\n    \"productdecision\",\n    \"undocumentedchange\",\n    \"cornercase\",\n]\n\nRCA_SUBCATEGORIES = [\n    \"codingerror-syntaxerror\",\n    \"codingerror-logicalerror\",\n    \"codingerror-semanticerror\",\n    \"codingerror-runtimeerror\",\n    \"codingerror-unhandledexceptions\",\n    \"codingerror-internalapiissue\",\n    \"codingerror-networkissue\",\n    \"codingerror-compatibilityissue\",\n    \"codingerror-other\",\n]\n\nlogger = logging.getLogger(__name__)\n\n\nclass RCATypeModel(BugModel):\n    def __init__(\n        self, lemmatization=False, historical=False, rca_subcategories_enabled=False\n    ):\n        BugModel.__init__(self, lemmatization)\n\n        self.calculate_importance = False\n        self.rca_subcategories_enabled = rca_subcategories_enabled\n\n        # should we consider only the main category or all sub categories\n        self.RCA_TYPES = (\n            RCA_SUBCATEGORIES + RCA_CATEGORIES\n            if rca_subcategories_enabled\n            else RCA_CATEGORIES\n        )\n\n        self.RCA_LIST = sorted(set(self.RCA_TYPES))\n\n        feature_extractors = [\n            bug_features.HasSTR(),\n            bug_features.Severity(),\n            bug_features.IsCoverityIssue(),\n            bug_features.HasCrashSignature(),\n            bug_features.HasURL(),\n            bug_features.HasW3CURL(),\n            bug_features.HasGithubURL(),\n            # Ignore whiteboards that would make the ML completely skewed\n            # bug_features.whiteboard(),\n            bug_features.Patches(),\n            bug_features.Landings(),\n            bug_features.BlockedBugsNumber(),\n            bug_features.EverAffected(),\n            bug_features.AffectedThenUnaffected(),\n            bug_features.Product(),\n            bug_features.Component(),\n        ]\n\n        cleanup_functions = [\n            feature_cleanup.url(),\n            feature_cleanup.fileref(),\n            feature_cleanup.synonyms(),\n        ]\n\n        self.extraction_pipeline = Pipeline(\n            [\n                (\n                    \"bug_extractor\",\n                    bug_features.BugExtractor(feature_extractors, cleanup_functions),\n                ),\n            ]\n        )\n\n        self.clf = Pipeline(\n            [\n                (\n                    \"union\",\n                    ColumnTransformer(\n                        [\n                            (\"data\", DictVectorizer(), \"data\"),\n                            (\"title\", self.text_vectorizer(min_df=0.001), \"title\"),\n                            (\n                                \"first_comment\",\n                                self.text_vectorizer(min_df=0.001),\n                                \"first_comment\",\n                            ),\n                            (\n                                \"comments\",\n                                self.text_vectorizer(min_df=0.001),\n                                \"comments\",\n                            ),\n                        ]\n                    ),\n                ),\n                (\n                    \"estimator\",\n                    OneVsRestClassifier(\n                        xgboost.XGBClassifier(n_jobs=utils.get_physical_cpu_count())\n                    ),\n                ),\n            ]\n        )\n\n    # return rca from a whiteboard string\n    def get_rca_from_whiteboard(self, whiteboard_data):\n        rca = []\n        whiteboard_data = re.sub(\" \", \"\", whiteboard_data).lower()\n        for whiteboard in whiteboard_data.split(\"[\"):\n            if not any(whiteboard.startswith(s) for s in (\"rca-\", \"rca:\")):\n                continue\n\n            rca_whiteboard = re.sub(\"]\", \"\", whiteboard)\n\n            # Hybrid cases: rca:X-Y\n            rca_whiteboard = re.sub(\":\", \"-\", rca_whiteboard)\n\n            rca_whiteboard_split = (\n                rca_whiteboard.split(\"-\", 1)\n                if self.rca_subcategories_enabled\n                else rca_whiteboard.split(\"-\")\n            )\n\n            if rca_whiteboard_split[1] not in self.RCA_LIST:\n                logger.warning(rca_whiteboard_split[1] + \" not in RCA_LIST\")\n            else:\n                rca.append(rca_whiteboard_split[1])\n        return rca\n\n    def get_labels(self):\n        classes = {}\n        for bug in bugzilla.get_bugs():\n            target = np.zeros(len(self.RCA_LIST))\n            for rca in self.get_rca_from_whiteboard(bug[\"whiteboard\"]):\n                target[self.RCA_LIST.index(rca)] = 1\n            classes[bug[\"id\"]] = target\n        return classes, self.RCA_LIST\n\n    def get_feature_names(self):\n        return self.clf.named_steps[\"union\"].get_feature_names_out()\n\n    def overwrite_classes(self, bugs, classes, probabilities):\n        rca_values = self.get_rca(bugs)\n        for i in len(classes):\n            for rca in rca_values[i]:\n                if rca in self.RCA_LIST:\n                    if probabilities:\n                        classes[i][self.RCA_LIST.index(rca)] = 1.0\n                    else:\n                        classes[i][self.RCA_LIST.index(rca)] = 1\n\n        return classes\n", "path": "bugbug/models/rcatype.py"}, {"content": "# -*- coding: utf-8 -*-\n# This Source Code Form is subject to the terms of the Mozilla Public\n# License, v. 2.0. If a copy of the MPL was not distributed with this file,\n# You can obtain one at http://mozilla.org/MPL/2.0/.\n\nimport logging\nfrom typing import Iterable\n\nimport numpy as np\nimport xgboost\nfrom sklearn.compose import ColumnTransformer\nfrom sklearn.feature_extraction import DictVectorizer\nfrom sklearn.multiclass import OneVsRestClassifier\nfrom sklearn.pipeline import Pipeline\n\nfrom bugbug import bug_features, bugzilla, feature_cleanup, utils\nfrom bugbug.model import BugModel\n\nlogger = logging.getLogger(__name__)\n\n\nclass BugTypeModel(BugModel):\n    def __init__(self, lemmatization=False, historical=False):\n        BugModel.__init__(self, lemmatization)\n\n        self.calculate_importance = False\n\n        self.bug_type_extractors = bug_features.BugTypes.bug_type_extractors\n\n        label_keyword_prefixes = {\n            keyword\n            for extractor in self.bug_type_extractors\n            for keyword in extractor.keyword_prefixes\n        }\n\n        feature_extractors = [\n            bug_features.HasSTR(),\n            bug_features.Severity(),\n            # Ignore keywords that would make the ML completely skewed\n            # (we are going to use them as 100% rules in the evaluation phase).\n            bug_features.Keywords(label_keyword_prefixes),\n            bug_features.IsCoverityIssue(),\n            bug_features.HasCrashSignature(),\n            bug_features.HasURL(),\n            bug_features.HasW3CURL(),\n            bug_features.HasGithubURL(),\n            bug_features.Whiteboard(),\n            bug_features.Patches(),\n            bug_features.Landings(),\n            bug_features.BlockedBugsNumber(),\n            bug_features.EverAffected(),\n            bug_features.AffectedThenUnaffected(),\n            bug_features.Product(),\n            bug_features.Component(),\n        ]\n\n        cleanup_functions = [\n            feature_cleanup.url(),\n            feature_cleanup.fileref(),\n            feature_cleanup.synonyms(),\n        ]\n\n        self.extraction_pipeline = Pipeline(\n            [\n                (\n                    \"bug_extractor\",\n                    bug_features.BugExtractor(feature_extractors, cleanup_functions),\n                ),\n            ]\n        )\n\n        self.clf = Pipeline(\n            [\n                (\n                    \"union\",\n                    ColumnTransformer(\n                        [\n                            (\"data\", DictVectorizer(), \"data\"),\n                            (\"title\", self.text_vectorizer(min_df=0.001), \"title\"),\n                            (\n                                \"first_comment\",\n                                self.text_vectorizer(min_df=0.001),\n                                \"first_comment\",\n                            ),\n                            (\n                                \"comments\",\n                                self.text_vectorizer(min_df=0.001),\n                                \"comments\",\n                            ),\n                        ]\n                    ),\n                ),\n                (\n                    \"estimator\",\n                    OneVsRestClassifier(\n                        xgboost.XGBClassifier(n_jobs=utils.get_physical_cpu_count())\n                    ),\n                ),\n            ]\n        )\n\n    def get_labels(self) -> tuple[dict[int, np.ndarray], list[str]]:\n        classes = {}\n\n        bug_map = {bug[\"id\"]: bug for bug in bugzilla.get_bugs()}\n\n        for bug_data in bug_map.values():\n            target = np.zeros(len(self.bug_type_extractors))\n            for i, is_type in enumerate(self.bug_type_extractors):\n                if is_type(bug_data, bug_map):\n                    target[i] = 1\n\n            classes[int(bug_data[\"id\"])] = target\n\n        bug_types = [extractor.type_name for extractor in self.bug_type_extractors]\n\n        for i, bug_type in enumerate(bug_types):\n            logger.info(\n                \"%d %s bugs\",\n                sum(target[i] for target in classes.values()),\n                bug_type,\n            )\n\n        return classes, bug_types\n\n    def get_feature_names(self):\n        return self.clf.named_steps[\"union\"].get_feature_names_out()\n\n    def overwrite_classes(\n        self,\n        bugs: Iterable[bugzilla.BugDict],\n        classes: dict[int, np.ndarray],\n        probabilities: bool,\n    ):\n        bug_map = {bug[\"id\"]: bug for bug in bugs}\n\n        for i, bug in enumerate(bugs):\n            for j, is_type_applicable in enumerate(self.bug_type_extractors):\n                if is_type_applicable(bug, bug_map):\n                    if probabilities:\n                        classes[i][j] = 1.0\n                    else:\n                        classes[i][j] = 1\n\n        return classes\n", "path": "bugbug/models/bugtype.py"}], "after_files": [{"content": "# -*- coding: utf-8 -*-\n# This Source Code Form is subject to the terms of the Mozilla Public\n# License, v. 2.0. If a copy of the MPL was not distributed with this file,\n# You can obtain one at http://mozilla.org/MPL/2.0/.\n\nimport logging\nimport re\n\nimport numpy as np\nimport xgboost\nfrom sklearn.compose import ColumnTransformer\nfrom sklearn.feature_extraction import DictVectorizer\nfrom sklearn.multiclass import OneVsRestClassifier\nfrom sklearn.pipeline import Pipeline\nfrom sklearn.preprocessing import LabelBinarizer\n\nfrom bugbug import bug_features, bugzilla, feature_cleanup, utils\nfrom bugbug.model import BugModel\n\n# For the moment, rca - XYZ is treated of bugtype XYZ,\n# so we don't need to store it in a dictionary.\nRCA_CATEGORIES = [\n    \"requirementerror\",\n    \"poorarchitecture\",\n    \"designerror\",\n    \"codingerror\",\n    \"testingerror\",\n    \"externalsoftwareaffectingfirefox\",\n    \"performanceerror\",\n    \"standards\",\n    \"systemerror\",\n    \"localizationerror\",\n    \"memory\",\n    \"infrastructure/builderror\",\n    \"communicationissues\",\n    \"productdecision\",\n    \"undocumentedchange\",\n    \"cornercase\",\n]\n\nRCA_SUBCATEGORIES = [\n    \"codingerror-syntaxerror\",\n    \"codingerror-logicalerror\",\n    \"codingerror-semanticerror\",\n    \"codingerror-runtimeerror\",\n    \"codingerror-unhandledexceptions\",\n    \"codingerror-internalapiissue\",\n    \"codingerror-networkissue\",\n    \"codingerror-compatibilityissue\",\n    \"codingerror-other\",\n]\n\nlogger = logging.getLogger(__name__)\n\n\nclass RCATypeModel(BugModel):\n    def __init__(\n        self, lemmatization=False, historical=False, rca_subcategories_enabled=False\n    ):\n        BugModel.__init__(self, lemmatization)\n\n        self.calculate_importance = False\n        self.rca_subcategories_enabled = rca_subcategories_enabled\n\n        self.le = LabelBinarizer()\n\n        # should we consider only the main category or all sub categories\n        self.RCA_TYPES = (\n            RCA_SUBCATEGORIES + RCA_CATEGORIES\n            if rca_subcategories_enabled\n            else RCA_CATEGORIES\n        )\n\n        self.RCA_LIST = sorted(set(self.RCA_TYPES))\n\n        feature_extractors = [\n            bug_features.HasSTR(),\n            bug_features.Severity(),\n            bug_features.IsCoverityIssue(),\n            bug_features.HasCrashSignature(),\n            bug_features.HasURL(),\n            bug_features.HasW3CURL(),\n            bug_features.HasGithubURL(),\n            # Ignore whiteboards that would make the ML completely skewed\n            # bug_features.whiteboard(),\n            bug_features.Patches(),\n            bug_features.Landings(),\n            bug_features.BlockedBugsNumber(),\n            bug_features.EverAffected(),\n            bug_features.AffectedThenUnaffected(),\n            bug_features.Product(),\n            bug_features.Component(),\n        ]\n\n        cleanup_functions = [\n            feature_cleanup.url(),\n            feature_cleanup.fileref(),\n            feature_cleanup.synonyms(),\n        ]\n\n        self.extraction_pipeline = Pipeline(\n            [\n                (\n                    \"bug_extractor\",\n                    bug_features.BugExtractor(feature_extractors, cleanup_functions),\n                ),\n            ]\n        )\n\n        self.clf = Pipeline(\n            [\n                (\n                    \"union\",\n                    ColumnTransformer(\n                        [\n                            (\"data\", DictVectorizer(), \"data\"),\n                            (\"title\", self.text_vectorizer(min_df=0.001), \"title\"),\n                            (\n                                \"first_comment\",\n                                self.text_vectorizer(min_df=0.001),\n                                \"first_comment\",\n                            ),\n                            (\n                                \"comments\",\n                                self.text_vectorizer(min_df=0.001),\n                                \"comments\",\n                            ),\n                        ]\n                    ),\n                ),\n                (\n                    \"estimator\",\n                    OneVsRestClassifier(\n                        xgboost.XGBClassifier(n_jobs=utils.get_physical_cpu_count())\n                    ),\n                ),\n            ]\n        )\n\n    # return rca from a whiteboard string\n    def get_rca_from_whiteboard(self, whiteboard_data):\n        rca = []\n        whiteboard_data = re.sub(\" \", \"\", whiteboard_data).lower()\n        for whiteboard in whiteboard_data.split(\"[\"):\n            if not any(whiteboard.startswith(s) for s in (\"rca-\", \"rca:\")):\n                continue\n\n            rca_whiteboard = re.sub(\"]\", \"\", whiteboard)\n\n            # Hybrid cases: rca:X-Y\n            rca_whiteboard = re.sub(\":\", \"-\", rca_whiteboard)\n\n            rca_whiteboard_split = (\n                rca_whiteboard.split(\"-\", 1)\n                if self.rca_subcategories_enabled\n                else rca_whiteboard.split(\"-\")\n            )\n\n            if rca_whiteboard_split[1] not in self.RCA_LIST:\n                logger.warning(rca_whiteboard_split[1] + \" not in RCA_LIST\")\n            else:\n                rca.append(rca_whiteboard_split[1])\n        return rca\n\n    def get_labels(self):\n        classes = {}\n        for bug in bugzilla.get_bugs():\n            target = np.zeros(len(self.RCA_LIST))\n            for rca in self.get_rca_from_whiteboard(bug[\"whiteboard\"]):\n                target[self.RCA_LIST.index(rca)] = 1\n            classes[bug[\"id\"]] = target\n        return classes, self.RCA_LIST\n\n    def get_feature_names(self):\n        return self.clf.named_steps[\"union\"].get_feature_names_out()\n\n    def overwrite_classes(self, bugs, classes, probabilities):\n        rca_values = self.get_rca(bugs)\n        for i in len(classes):\n            for rca in rca_values[i]:\n                if rca in self.RCA_LIST:\n                    if probabilities:\n                        classes[i][self.RCA_LIST.index(rca)] = 1.0\n                    else:\n                        classes[i][self.RCA_LIST.index(rca)] = 1\n\n        return classes\n", "path": "bugbug/models/rcatype.py"}, {"content": "# -*- coding: utf-8 -*-\n# This Source Code Form is subject to the terms of the Mozilla Public\n# License, v. 2.0. If a copy of the MPL was not distributed with this file,\n# You can obtain one at http://mozilla.org/MPL/2.0/.\n\nimport logging\nfrom typing import Iterable\n\nimport numpy as np\nimport xgboost\nfrom sklearn.compose import ColumnTransformer\nfrom sklearn.feature_extraction import DictVectorizer\nfrom sklearn.multiclass import OneVsRestClassifier\nfrom sklearn.pipeline import Pipeline\nfrom sklearn.preprocessing import LabelBinarizer\n\nfrom bugbug import bug_features, bugzilla, feature_cleanup, utils\nfrom bugbug.model import BugModel\n\nlogger = logging.getLogger(__name__)\n\n\nclass BugTypeModel(BugModel):\n    def __init__(self, lemmatization=False, historical=False):\n        BugModel.__init__(self, lemmatization)\n\n        self.calculate_importance = False\n\n        self.le = LabelBinarizer()\n\n        self.bug_type_extractors = bug_features.BugTypes.bug_type_extractors\n\n        label_keyword_prefixes = {\n            keyword\n            for extractor in self.bug_type_extractors\n            for keyword in extractor.keyword_prefixes\n        }\n\n        feature_extractors = [\n            bug_features.HasSTR(),\n            bug_features.Severity(),\n            # Ignore keywords that would make the ML completely skewed\n            # (we are going to use them as 100% rules in the evaluation phase).\n            bug_features.Keywords(label_keyword_prefixes),\n            bug_features.IsCoverityIssue(),\n            bug_features.HasCrashSignature(),\n            bug_features.HasURL(),\n            bug_features.HasW3CURL(),\n            bug_features.HasGithubURL(),\n            bug_features.Whiteboard(),\n            bug_features.Patches(),\n            bug_features.Landings(),\n            bug_features.BlockedBugsNumber(),\n            bug_features.EverAffected(),\n            bug_features.AffectedThenUnaffected(),\n            bug_features.Product(),\n            bug_features.Component(),\n        ]\n\n        cleanup_functions = [\n            feature_cleanup.url(),\n            feature_cleanup.fileref(),\n            feature_cleanup.synonyms(),\n        ]\n\n        self.extraction_pipeline = Pipeline(\n            [\n                (\n                    \"bug_extractor\",\n                    bug_features.BugExtractor(feature_extractors, cleanup_functions),\n                ),\n            ]\n        )\n\n        self.clf = Pipeline(\n            [\n                (\n                    \"union\",\n                    ColumnTransformer(\n                        [\n                            (\"data\", DictVectorizer(), \"data\"),\n                            (\"title\", self.text_vectorizer(min_df=0.001), \"title\"),\n                            (\n                                \"first_comment\",\n                                self.text_vectorizer(min_df=0.001),\n                                \"first_comment\",\n                            ),\n                            (\n                                \"comments\",\n                                self.text_vectorizer(min_df=0.001),\n                                \"comments\",\n                            ),\n                        ]\n                    ),\n                ),\n                (\n                    \"estimator\",\n                    OneVsRestClassifier(\n                        xgboost.XGBClassifier(n_jobs=utils.get_physical_cpu_count())\n                    ),\n                ),\n            ]\n        )\n\n    def get_labels(self) -> tuple[dict[int, np.ndarray], list[str]]:\n        classes = {}\n\n        bug_map = {bug[\"id\"]: bug for bug in bugzilla.get_bugs()}\n\n        for bug_data in bug_map.values():\n            target = np.zeros(len(self.bug_type_extractors))\n            for i, is_type in enumerate(self.bug_type_extractors):\n                if is_type(bug_data, bug_map):\n                    target[i] = 1\n\n            classes[int(bug_data[\"id\"])] = target\n\n        bug_types = [extractor.type_name for extractor in self.bug_type_extractors]\n\n        for i, bug_type in enumerate(bug_types):\n            logger.info(\n                \"%d %s bugs\",\n                sum(target[i] for target in classes.values()),\n                bug_type,\n            )\n\n        return classes, bug_types\n\n    def get_feature_names(self):\n        return self.clf.named_steps[\"union\"].get_feature_names_out()\n\n    def overwrite_classes(\n        self,\n        bugs: Iterable[bugzilla.BugDict],\n        classes: dict[int, np.ndarray],\n        probabilities: bool,\n    ):\n        bug_map = {bug[\"id\"]: bug for bug in bugs}\n\n        for i, bug in enumerate(bugs):\n            for j, is_type_applicable in enumerate(self.bug_type_extractors):\n                if is_type_applicable(bug, bug_map):\n                    if probabilities:\n                        classes[i][j] = 1.0\n                    else:\n                        classes[i][j] = 1\n\n        return classes\n", "path": "bugbug/models/bugtype.py"}]}