We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
Use bug reporter as a feature


--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `bugbug/models/tracking.py`
Content:
```
1 # -*- coding: utf-8 -*-
2 # This Source Code Form is subject to the terms of the Mozilla Public
3 # License, v. 2.0. If a copy of the MPL was not distributed with this file,
4 # You can obtain one at http://mozilla.org/MPL/2.0/.
5 
6 import xgboost
7 from sklearn.compose import ColumnTransformer
8 from sklearn.feature_extraction import DictVectorizer
9 from sklearn.pipeline import Pipeline
10 
11 from bugbug import bug_features
12 from bugbug import bugzilla
13 from bugbug.model import Model
14 
15 
16 class TrackingModel(Model):
17     def __init__(self, lemmatization=False):
18         Model.__init__(self, lemmatization)
19 
20         feature_extractors = [
21             bug_features.has_str(),
22             bug_features.has_regression_range(),
23             bug_features.severity(),
24             bug_features.keywords(),
25             bug_features.is_coverity_issue(),
26             bug_features.has_crash_signature(),
27             bug_features.has_url(),
28             bug_features.has_w3c_url(),
29             bug_features.has_github_url(),
30             bug_features.whiteboard(),
31             bug_features.patches(),
32             bug_features.landings(),
33             bug_features.title(),
34             bug_features.priority(),
35         ]
36 
37         cleanup_functions = [
38             bug_features.cleanup_fileref,
39             bug_features.cleanup_url,
40             bug_features.cleanup_synonyms,
41         ]
42 
43         self.extraction_pipeline = Pipeline([
44             ('bug_extractor', bug_features.BugExtractor(feature_extractors, cleanup_functions, rollback=True, rollback_when=self.rollback)),
45             ('union', ColumnTransformer([
46                 ('data', DictVectorizer(), 'data'),
47 
48                 ('title', self.text_vectorizer(stop_words='english'), 'title'),
49 
50                 ('comments', self.text_vectorizer(stop_words='english'), 'comments'),
51             ])),
52         ])
53 
54         self.clf = xgboost.XGBClassifier(n_jobs=16)
55         self.clf.set_params(predictor='cpu_predictor')
56 
57     def rollback(self, change):
58         return change['field_name'].startswith('cf_tracking_firefox')
59 
60     def get_labels(self):
61         classes = {}
62 
63         for bug_data in bugzilla.get_bugs():
64             bug_id = int(bug_data['id'])
65 
66             for entry in bug_data['history']:
67                 for change in entry['changes']:
68                     if change['field_name'].startswith('cf_tracking_firefox'):
69                         if change['added'] in ['blocking', '+']:
70                             classes[bug_id] = 1
71                         elif change['added'] == '-':
72                             classes[bug_id] = 0
73 
74             if bug_id not in classes:
75                 classes[bug_id] = 0
76 
77         return classes
78 
79     def get_feature_names(self):
80         return self.extraction_pipeline.named_steps['union'].get_feature_names()
81
```

Path: `bugbug/bug_features.py`
Content:
```
1 # -*- coding: utf-8 -*-
2 # This Source Code Form is subject to the terms of the Mozilla Public
3 # License, v. 2.0. If a copy of the MPL was not distributed with this file,
4 # You can obtain one at http://mozilla.org/MPL/2.0/.
5 
6 import re
7 from datetime import datetime
8 from datetime import timezone
9 
10 import pandas as pd
11 from libmozdata import versions
12 from sklearn.base import BaseEstimator
13 from sklearn.base import TransformerMixin
14 
15 from bugbug import bug_snapshot
16 from bugbug import repository
17 
18 
19 def field(bug, field):
20     if field in bug and bug[field] != '---':
21         return bug[field]
22 
23     return None
24 
25 
26 class has_str(object):
27     def __call__(self, bug):
28         return field(bug, 'cf_has_str')
29 
30 
31 class has_regression_range(object):
32     def __call__(self, bug):
33         return field(bug, 'cf_has_regression_range')
34 
35 
36 class has_crash_signature(object):
37     def __call__(self, bug):
38         return 'cf_crash_signature' in bug and bug['cf_crash_signature'] != ''
39 
40 
41 class keywords(object):
42     def __init__(self, to_ignore=set()):
43         self.to_ignore = to_ignore
44 
45     def __call__(self, bug):
46         keywords = []
47         subkeywords = []
48         for keyword in bug['keywords']:
49             if keyword in self.to_ignore:
50                 continue
51 
52             keywords.append(keyword)
53 
54             if keyword.startswith('sec-'):
55                 subkeywords.append('sec-')
56             elif keyword.startswith('csectype-'):
57                 subkeywords.append('csectype-')
58         return keywords + subkeywords
59 
60 
61 class severity(object):
62     def __call__(self, bug):
63         return field(bug, 'severity')
64 
65 
66 class is_coverity_issue(object):
67     def __call__(self, bug):
68         return re.search('[CID ?[0-9]+]', bug['summary']) is not None or re.search('[CID ?[0-9]+]', bug['whiteboard']) is not None
69 
70 
71 class has_url(object):
72     def __call__(self, bug):
73         return bug['url'] != ''
74 
75 
76 class has_w3c_url(object):
77     def __call__(self, bug):
78         return 'w3c' in bug['url']
79 
80 
81 class has_github_url(object):
82     def __call__(self, bug):
83         return 'github' in bug['url']
84 
85 
86 class whiteboard(object):
87     def __call__(self, bug):
88 
89         # Split by '['
90         paren_splits = bug['whiteboard'].lower().split('[')
91 
92         # Split splits by space if they weren't in [ and ].
93         splits = []
94         for paren_split in paren_splits:
95             if ']' in paren_split:
96                 paren_split = paren_split.split(']')
97                 splits += paren_split
98             else:
99                 splits += paren_split.split(' ')
100 
101         # Remove empty splits and strip
102         splits = [split.strip() for split in splits if split.strip() != '']
103 
104         # For splits which contain ':', return both the whole string and the string before ':'.
105         splits += [split.split(':', 1)[0] for split in splits if ':' in split]
106 
107         return splits
108 
109 
110 class patches(object):
111     def __call__(self, bug):
112         return sum(1 for a in bug['attachments'] if a['is_patch'] or a['content_type'] in ['text/x-review-board-request', 'text/x-phabricator-request'])
113 
114 
115 class landings(object):
116     def __call__(self, bug):
117         return sum(1 for c in bug['comments'] if '://hg.mozilla.org/' in c['text'])
118 
119 
120 class title(object):
121     def __call__(self, bug):
122         ret = []
123 
124         keywords = [
125             'fail',
126         ]
127         for keyword in keywords:
128             if keyword in bug['summary'].lower():
129                 ret.append(keyword)
130 
131         return ret
132 
133 
134 class product(object):
135     def __call__(self, bug):
136         return bug['product']
137 
138 
139 class component(object):
140     def __call__(self, bug):
141         return bug['component']
142 
143 
144 class is_mozillian(object):
145     def __call__(self, bug):
146         return any(bug['creator_detail']['email'].endswith(domain) for domain in ['@mozilla.com', '@mozilla.org'])
147 
148 
149 class delta_request_merge(object):
150     def __call__(self, bug):
151         for history in bug['history']:
152             for change in history['changes']:
153                 if change['added'].startswith('approval-mozilla'):
154                     uplift_request_datetime = datetime.strptime(history['when'], '%Y-%m-%dT%H:%M:%SZ').replace(tzinfo=timezone.utc)
155                     timedelta = versions.getCloserRelease(uplift_request_datetime)[1] - uplift_request_datetime
156                     return timedelta.days + timedelta.seconds / (24 * 60 * 60)
157 
158         return None
159 
160 
161 class commit_added(object):
162     def __call__(self, bug):
163         return sum(commit['added'] for commit in bug['commits'])
164 
165 
166 class commit_deleted(object):
167     def __call__(self, bug):
168         return sum(commit['deleted'] for commit in bug['commits'])
169 
170 
171 class commit_types(object):
172     def __call__(self, bug):
173         return sum((commit['types'] for commit in bug['commits']), [])
174 
175 
176 class blocked_bugs_number(object):
177     def __call__(self, bug):
178         return len(bug['blocks'])
179 
180 
181 class priority(object):
182     def __call__(self, bug):
183         return bug['priority']
184 
185 
186 class bug_has_cve_in_alias(object):
187     def __call__(self, bug):
188         return bug['alias'] is not None and 'CVE' in bug['alias']
189 
190 
191 class commit_files_modified_num(object):
192     def __call__(self, bug):
193         return sum(commit['files_modified_num'] for commit in bug['commits'])
194 
195 
196 class comment_count(object):
197     def __call__(self, bug):
198         return field(bug, 'comment_count')
199 
200 
201 class comment_length(object):
202     def __call__(self, bug):
203         return sum(len(x['text']) for x in bug['comments'])
204 
205 
206 def cleanup_url(text):
207     text = re.sub(r'http[s]?://(hg.mozilla|searchfox|dxr.mozilla)\S+', '__CODE_REFERENCE_URL__', text)
208     return re.sub(r'http\S+', '__URL__', text)
209 
210 
211 def cleanup_fileref(text):
212     return re.sub(r'\w+\.py\b|\w+\.json\b|\w+\.js\b|\w+\.jsm\b|\w+\.html\b|\w+\.css\b|\w+\.c\b|\w+\.cpp\b|\w+\.h\b', '__FILE_REFERENCE__', text)
213 
214 
215 def cleanup_responses(text):
216     return re.sub('>[^\n]+', ' ', text)
217 
218 
219 def cleanup_hex(text):
220     return re.sub(r'\b0[xX][0-9a-fA-F]+\b', '__HEX_NUMBER__', text)
221 
222 
223 def cleanup_dll(text):
224     return re.sub(r'\w+(\.dll|\.so|\.dylib)\b', '__DLL_NAME__', text)
225 
226 
227 def cleanup_synonyms(text):
228     synonyms = [
229         ('safemode', ['safemode', 'safe mode']),
230         ('str', ['str', 'steps to reproduce', 'repro steps']),
231         ('uaf', ['uaf', 'use after free', 'use-after-free']),
232         ('asan', ['asan', 'address sanitizer', 'addresssanitizer']),
233         ('permafailure', ['permafailure', 'permafailing', 'permafail', 'perma failure', 'perma failing', 'perma fail', 'perma-failure', 'perma-failing', 'perma-fail']),
234         ('spec', ['spec', 'specification']),
235     ]
236 
237     for synonym_group, synonym_list in synonyms:
238         text = re.sub('|'.join(fr'\b{synonym}\b' for synonym in synonym_list), synonym_group, text, flags=re.IGNORECASE)
239 
240     return text
241 
242 
243 def cleanup_crash(text):
244     return re.sub(r'bp-[a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{6}[0-9]{6}\b', '__CRASH_STATS_LINK__', text)
245 
246 
247 class BugExtractor(BaseEstimator, TransformerMixin):
248     def __init__(self, feature_extractors, cleanup_functions, rollback=False, rollback_when=None, commit_data=False):
249         self.feature_extractors = feature_extractors
250         self.cleanup_functions = cleanup_functions
251         self.rollback = rollback
252         self.rollback_when = rollback_when
253         self.commit_map = repository.get_commit_map() if commit_data else None
254 
255     def fit(self, x, y=None):
256         return self
257 
258     def transform(self, bugs):
259         results = []
260 
261         for bug in bugs:
262             bug_id = bug['id']
263 
264             if self.rollback:
265                 bug = bug_snapshot.rollback(bug, self.rollback_when)
266 
267             data = {}
268 
269             if self.commit_map is not None:
270                 if bug_id in self.commit_map:
271                     bug['commits'] = self.commit_map[bug_id]
272                 else:
273                     bug['commits'] = []
274 
275             for f in self.feature_extractors:
276                 res = f(bug)
277 
278                 if res is None:
279                     continue
280 
281                 if isinstance(res, list):
282                     for item in res:
283                         data[f.__class__.__name__ + '-' + item] = 'True'
284                     continue
285 
286                 if isinstance(res, bool):
287                     res = str(res)
288 
289                 data[f.__class__.__name__] = res
290 
291             # TODO: Try simply using all possible fields instead of extracting features manually.
292 
293             for cleanup_function in self.cleanup_functions:
294                 bug['summary'] = cleanup_function(bug['summary'])
295                 for c in bug['comments']:
296                     c['text'] = cleanup_function(c['text'])
297 
298             result = {
299                 'data': data,
300                 'title': bug['summary'],
301                 'first_comment': bug['comments'][0]['text'],
302                 'comments': ' '.join([c['text'] for c in bug['comments']]),
303             }
304 
305             results.append(result)
306 
307         return pd.DataFrame(results)
308
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/bugbug/bug_features.py b/bugbug/bug_features.py
--- a/bugbug/bug_features.py
+++ b/bugbug/bug_features.py
@@ -146,6 +146,11 @@
         return any(bug['creator_detail']['email'].endswith(domain) for domain in ['@mozilla.com', '@mozilla.org'])
 
 
+class bug_reporter(object):
+    def __call__(self, bug):
+        return bug['creator_detail']['email']
+
+
 class delta_request_merge(object):
     def __call__(self, bug):
         for history in bug['history']:
diff --git a/bugbug/models/tracking.py b/bugbug/models/tracking.py
--- a/bugbug/models/tracking.py
+++ b/bugbug/models/tracking.py
@@ -32,6 +32,7 @@
             bug_features.landings(),
             bug_features.title(),
             bug_features.priority(),
+            bug_features.bug_reporter()
         ]
 
         cleanup_functions = [

{"golden_diff": "diff --git a/bugbug/bug_features.py b/bugbug/bug_features.py\n--- a/bugbug/bug_features.py\n+++ b/bugbug/bug_features.py\n@@ -146,6 +146,11 @@\n         return any(bug['creator_detail']['email'].endswith(domain) for domain in ['@mozilla.com', '@mozilla.org'])\n \n \n+class bug_reporter(object):\n+    def __call__(self, bug):\n+        return bug['creator_detail']['email']\n+\n+\n class delta_request_merge(object):\n     def __call__(self, bug):\n         for history in bug['history']:\ndiff --git a/bugbug/models/tracking.py b/bugbug/models/tracking.py\n--- a/bugbug/models/tracking.py\n+++ b/bugbug/models/tracking.py\n@@ -32,6 +32,7 @@\n             bug_features.landings(),\n             bug_features.title(),\n             bug_features.priority(),\n+            bug_features.bug_reporter()\n         ]\n \n         cleanup_functions = [\n", "issue": "Use bug reporter as a feature\n\n", "before_files": [{"content": "# -*- coding: utf-8 -*-\n# This Source Code Form is subject to the terms of the Mozilla Public\n# License, v. 2.0. If a copy of the MPL was not distributed with this file,\n# You can obtain one at http://mozilla.org/MPL/2.0/.\n\nimport xgboost\nfrom sklearn.compose import ColumnTransformer\nfrom sklearn.feature_extraction import DictVectorizer\nfrom sklearn.pipeline import Pipeline\n\nfrom bugbug import bug_features\nfrom bugbug import bugzilla\nfrom bugbug.model import Model\n\n\nclass TrackingModel(Model):\n    def __init__(self, lemmatization=False):\n        Model.__init__(self, lemmatization)\n\n        feature_extractors = [\n            bug_features.has_str(),\n            bug_features.has_regression_range(),\n            bug_features.severity(),\n            bug_features.keywords(),\n            bug_features.is_coverity_issue(),\n            bug_features.has_crash_signature(),\n            bug_features.has_url(),\n            bug_features.has_w3c_url(),\n            bug_features.has_github_url(),\n            bug_features.whiteboard(),\n            bug_features.patches(),\n            bug_features.landings(),\n            bug_features.title(),\n            bug_features.priority(),\n        ]\n\n        cleanup_functions = [\n            bug_features.cleanup_fileref,\n            bug_features.cleanup_url,\n            bug_features.cleanup_synonyms,\n        ]\n\n        self.extraction_pipeline = Pipeline([\n            ('bug_extractor', bug_features.BugExtractor(feature_extractors, cleanup_functions, rollback=True, rollback_when=self.rollback)),\n            ('union', ColumnTransformer([\n                ('data', DictVectorizer(), 'data'),\n\n                ('title', self.text_vectorizer(stop_words='english'), 'title'),\n\n                ('comments', self.text_vectorizer(stop_words='english'), 'comments'),\n            ])),\n        ])\n\n        self.clf = xgboost.XGBClassifier(n_jobs=16)\n        self.clf.set_params(predictor='cpu_predictor')\n\n    def rollback(self, change):\n        return change['field_name'].startswith('cf_tracking_firefox')\n\n    def get_labels(self):\n        classes = {}\n\n        for bug_data in bugzilla.get_bugs():\n            bug_id = int(bug_data['id'])\n\n            for entry in bug_data['history']:\n                for change in entry['changes']:\n                    if change['field_name'].startswith('cf_tracking_firefox'):\n                        if change['added'] in ['blocking', '+']:\n                            classes[bug_id] = 1\n                        elif change['added'] == '-':\n                            classes[bug_id] = 0\n\n            if bug_id not in classes:\n                classes[bug_id] = 0\n\n        return classes\n\n    def get_feature_names(self):\n        return self.extraction_pipeline.named_steps['union'].get_feature_names()\n", "path": "bugbug/models/tracking.py"}, {"content": "# -*- coding: utf-8 -*-\n# This Source Code Form is subject to the terms of the Mozilla Public\n# License, v. 2.0. If a copy of the MPL was not distributed with this file,\n# You can obtain one at http://mozilla.org/MPL/2.0/.\n\nimport re\nfrom datetime import datetime\nfrom datetime import timezone\n\nimport pandas as pd\nfrom libmozdata import versions\nfrom sklearn.base import BaseEstimator\nfrom sklearn.base import TransformerMixin\n\nfrom bugbug import bug_snapshot\nfrom bugbug import repository\n\n\ndef field(bug, field):\n    if field in bug and bug[field] != '---':\n        return bug[field]\n\n    return None\n\n\nclass has_str(object):\n    def __call__(self, bug):\n        return field(bug, 'cf_has_str')\n\n\nclass has_regression_range(object):\n    def __call__(self, bug):\n        return field(bug, 'cf_has_regression_range')\n\n\nclass has_crash_signature(object):\n    def __call__(self, bug):\n        return 'cf_crash_signature' in bug and bug['cf_crash_signature'] != ''\n\n\nclass keywords(object):\n    def __init__(self, to_ignore=set()):\n        self.to_ignore = to_ignore\n\n    def __call__(self, bug):\n        keywords = []\n        subkeywords = []\n        for keyword in bug['keywords']:\n            if keyword in self.to_ignore:\n                continue\n\n            keywords.append(keyword)\n\n            if keyword.startswith('sec-'):\n                subkeywords.append('sec-')\n            elif keyword.startswith('csectype-'):\n                subkeywords.append('csectype-')\n        return keywords + subkeywords\n\n\nclass severity(object):\n    def __call__(self, bug):\n        return field(bug, 'severity')\n\n\nclass is_coverity_issue(object):\n    def __call__(self, bug):\n        return re.search('[CID ?[0-9]+]', bug['summary']) is not None or re.search('[CID ?[0-9]+]', bug['whiteboard']) is not None\n\n\nclass has_url(object):\n    def __call__(self, bug):\n        return bug['url'] != ''\n\n\nclass has_w3c_url(object):\n    def __call__(self, bug):\n        return 'w3c' in bug['url']\n\n\nclass has_github_url(object):\n    def __call__(self, bug):\n        return 'github' in bug['url']\n\n\nclass whiteboard(object):\n    def __call__(self, bug):\n\n        # Split by '['\n        paren_splits = bug['whiteboard'].lower().split('[')\n\n        # Split splits by space if they weren't in [ and ].\n        splits = []\n        for paren_split in paren_splits:\n            if ']' in paren_split:\n                paren_split = paren_split.split(']')\n                splits += paren_split\n            else:\n                splits += paren_split.split(' ')\n\n        # Remove empty splits and strip\n        splits = [split.strip() for split in splits if split.strip() != '']\n\n        # For splits which contain ':', return both the whole string and the string before ':'.\n        splits += [split.split(':', 1)[0] for split in splits if ':' in split]\n\n        return splits\n\n\nclass patches(object):\n    def __call__(self, bug):\n        return sum(1 for a in bug['attachments'] if a['is_patch'] or a['content_type'] in ['text/x-review-board-request', 'text/x-phabricator-request'])\n\n\nclass landings(object):\n    def __call__(self, bug):\n        return sum(1 for c in bug['comments'] if '://hg.mozilla.org/' in c['text'])\n\n\nclass title(object):\n    def __call__(self, bug):\n        ret = []\n\n        keywords = [\n            'fail',\n        ]\n        for keyword in keywords:\n            if keyword in bug['summary'].lower():\n                ret.append(keyword)\n\n        return ret\n\n\nclass product(object):\n    def __call__(self, bug):\n        return bug['product']\n\n\nclass component(object):\n    def __call__(self, bug):\n        return bug['component']\n\n\nclass is_mozillian(object):\n    def __call__(self, bug):\n        return any(bug['creator_detail']['email'].endswith(domain) for domain in ['@mozilla.com', '@mozilla.org'])\n\n\nclass delta_request_merge(object):\n    def __call__(self, bug):\n        for history in bug['history']:\n            for change in history['changes']:\n                if change['added'].startswith('approval-mozilla'):\n                    uplift_request_datetime = datetime.strptime(history['when'], '%Y-%m-%dT%H:%M:%SZ').replace(tzinfo=timezone.utc)\n                    timedelta = versions.getCloserRelease(uplift_request_datetime)[1] - uplift_request_datetime\n                    return timedelta.days + timedelta.seconds / (24 * 60 * 60)\n\n        return None\n\n\nclass commit_added(object):\n    def __call__(self, bug):\n        return sum(commit['added'] for commit in bug['commits'])\n\n\nclass commit_deleted(object):\n    def __call__(self, bug):\n        return sum(commit['deleted'] for commit in bug['commits'])\n\n\nclass commit_types(object):\n    def __call__(self, bug):\n        return sum((commit['types'] for commit in bug['commits']), [])\n\n\nclass blocked_bugs_number(object):\n    def __call__(self, bug):\n        return len(bug['blocks'])\n\n\nclass priority(object):\n    def __call__(self, bug):\n        return bug['priority']\n\n\nclass bug_has_cve_in_alias(object):\n    def __call__(self, bug):\n        return bug['alias'] is not None and 'CVE' in bug['alias']\n\n\nclass commit_files_modified_num(object):\n    def __call__(self, bug):\n        return sum(commit['files_modified_num'] for commit in bug['commits'])\n\n\nclass comment_count(object):\n    def __call__(self, bug):\n        return field(bug, 'comment_count')\n\n\nclass comment_length(object):\n    def __call__(self, bug):\n        return sum(len(x['text']) for x in bug['comments'])\n\n\ndef cleanup_url(text):\n    text = re.sub(r'http[s]?://(hg.mozilla|searchfox|dxr.mozilla)\\S+', '__CODE_REFERENCE_URL__', text)\n    return re.sub(r'http\\S+', '__URL__', text)\n\n\ndef cleanup_fileref(text):\n    return re.sub(r'\\w+\\.py\\b|\\w+\\.json\\b|\\w+\\.js\\b|\\w+\\.jsm\\b|\\w+\\.html\\b|\\w+\\.css\\b|\\w+\\.c\\b|\\w+\\.cpp\\b|\\w+\\.h\\b', '__FILE_REFERENCE__', text)\n\n\ndef cleanup_responses(text):\n    return re.sub('>[^\\n]+', ' ', text)\n\n\ndef cleanup_hex(text):\n    return re.sub(r'\\b0[xX][0-9a-fA-F]+\\b', '__HEX_NUMBER__', text)\n\n\ndef cleanup_dll(text):\n    return re.sub(r'\\w+(\\.dll|\\.so|\\.dylib)\\b', '__DLL_NAME__', text)\n\n\ndef cleanup_synonyms(text):\n    synonyms = [\n        ('safemode', ['safemode', 'safe mode']),\n        ('str', ['str', 'steps to reproduce', 'repro steps']),\n        ('uaf', ['uaf', 'use after free', 'use-after-free']),\n        ('asan', ['asan', 'address sanitizer', 'addresssanitizer']),\n        ('permafailure', ['permafailure', 'permafailing', 'permafail', 'perma failure', 'perma failing', 'perma fail', 'perma-failure', 'perma-failing', 'perma-fail']),\n        ('spec', ['spec', 'specification']),\n    ]\n\n    for synonym_group, synonym_list in synonyms:\n        text = re.sub('|'.join(fr'\\b{synonym}\\b' for synonym in synonym_list), synonym_group, text, flags=re.IGNORECASE)\n\n    return text\n\n\ndef cleanup_crash(text):\n    return re.sub(r'bp-[a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{6}[0-9]{6}\\b', '__CRASH_STATS_LINK__', text)\n\n\nclass BugExtractor(BaseEstimator, TransformerMixin):\n    def __init__(self, feature_extractors, cleanup_functions, rollback=False, rollback_when=None, commit_data=False):\n        self.feature_extractors = feature_extractors\n        self.cleanup_functions = cleanup_functions\n        self.rollback = rollback\n        self.rollback_when = rollback_when\n        self.commit_map = repository.get_commit_map() if commit_data else None\n\n    def fit(self, x, y=None):\n        return self\n\n    def transform(self, bugs):\n        results = []\n\n        for bug in bugs:\n            bug_id = bug['id']\n\n            if self.rollback:\n                bug = bug_snapshot.rollback(bug, self.rollback_when)\n\n            data = {}\n\n            if self.commit_map is not None:\n                if bug_id in self.commit_map:\n                    bug['commits'] = self.commit_map[bug_id]\n                else:\n                    bug['commits'] = []\n\n            for f in self.feature_extractors:\n                res = f(bug)\n\n                if res is None:\n                    continue\n\n                if isinstance(res, list):\n                    for item in res:\n                        data[f.__class__.__name__ + '-' + item] = 'True'\n                    continue\n\n                if isinstance(res, bool):\n                    res = str(res)\n\n                data[f.__class__.__name__] = res\n\n            # TODO: Try simply using all possible fields instead of extracting features manually.\n\n            for cleanup_function in self.cleanup_functions:\n                bug['summary'] = cleanup_function(bug['summary'])\n                for c in bug['comments']:\n                    c['text'] = cleanup_function(c['text'])\n\n            result = {\n                'data': data,\n                'title': bug['summary'],\n                'first_comment': bug['comments'][0]['text'],\n                'comments': ' '.join([c['text'] for c in bug['comments']]),\n            }\n\n            results.append(result)\n\n        return pd.DataFrame(results)\n", "path": "bugbug/bug_features.py"}], "after_files": [{"content": "# -*- coding: utf-8 -*-\n# This Source Code Form is subject to the terms of the Mozilla Public\n# License, v. 2.0. If a copy of the MPL was not distributed with this file,\n# You can obtain one at http://mozilla.org/MPL/2.0/.\n\nimport xgboost\nfrom sklearn.compose import ColumnTransformer\nfrom sklearn.feature_extraction import DictVectorizer\nfrom sklearn.pipeline import Pipeline\n\nfrom bugbug import bug_features\nfrom bugbug import bugzilla\nfrom bugbug.model import Model\n\n\nclass TrackingModel(Model):\n    def __init__(self, lemmatization=False):\n        Model.__init__(self, lemmatization)\n\n        feature_extractors = [\n            bug_features.has_str(),\n            bug_features.has_regression_range(),\n            bug_features.severity(),\n            bug_features.keywords(),\n            bug_features.is_coverity_issue(),\n            bug_features.has_crash_signature(),\n            bug_features.has_url(),\n            bug_features.has_w3c_url(),\n            bug_features.has_github_url(),\n            bug_features.whiteboard(),\n            bug_features.patches(),\n            bug_features.landings(),\n            bug_features.title(),\n            bug_features.priority(),\n            bug_features.bug_reporter()\n        ]\n\n        cleanup_functions = [\n            bug_features.cleanup_fileref,\n            bug_features.cleanup_url,\n            bug_features.cleanup_synonyms,\n        ]\n\n        self.extraction_pipeline = Pipeline([\n            ('bug_extractor', bug_features.BugExtractor(feature_extractors, cleanup_functions, rollback=True, rollback_when=self.rollback)),\n            ('union', ColumnTransformer([\n                ('data', DictVectorizer(), 'data'),\n\n                ('title', self.text_vectorizer(stop_words='english'), 'title'),\n\n                ('comments', self.text_vectorizer(stop_words='english'), 'comments'),\n            ])),\n        ])\n\n        self.clf = xgboost.XGBClassifier(n_jobs=16)\n        self.clf.set_params(predictor='cpu_predictor')\n\n    def rollback(self, change):\n        return change['field_name'].startswith('cf_tracking_firefox')\n\n    def get_labels(self):\n        classes = {}\n\n        for bug_data in bugzilla.get_bugs():\n            bug_id = int(bug_data['id'])\n\n            for entry in bug_data['history']:\n                for change in entry['changes']:\n                    if change['field_name'].startswith('cf_tracking_firefox'):\n                        if change['added'] in ['blocking', '+']:\n                            classes[bug_id] = 1\n                        elif change['added'] == '-':\n                            classes[bug_id] = 0\n\n            if bug_id not in classes:\n                classes[bug_id] = 0\n\n        return classes\n\n    def get_feature_names(self):\n        return self.extraction_pipeline.named_steps['union'].get_feature_names()\n", "path": "bugbug/models/tracking.py"}, {"content": "# -*- coding: utf-8 -*-\n# This Source Code Form is subject to the terms of the Mozilla Public\n# License, v. 2.0. If a copy of the MPL was not distributed with this file,\n# You can obtain one at http://mozilla.org/MPL/2.0/.\n\nimport re\nfrom datetime import datetime\nfrom datetime import timezone\n\nimport pandas as pd\nfrom libmozdata import versions\nfrom sklearn.base import BaseEstimator\nfrom sklearn.base import TransformerMixin\n\nfrom bugbug import bug_snapshot\nfrom bugbug import repository\n\n\ndef field(bug, field):\n    if field in bug and bug[field] != '---':\n        return bug[field]\n\n    return None\n\n\nclass has_str(object):\n    def __call__(self, bug):\n        return field(bug, 'cf_has_str')\n\n\nclass has_regression_range(object):\n    def __call__(self, bug):\n        return field(bug, 'cf_has_regression_range')\n\n\nclass has_crash_signature(object):\n    def __call__(self, bug):\n        return 'cf_crash_signature' in bug and bug['cf_crash_signature'] != ''\n\n\nclass keywords(object):\n    def __init__(self, to_ignore=set()):\n        self.to_ignore = to_ignore\n\n    def __call__(self, bug):\n        keywords = []\n        subkeywords = []\n        for keyword in bug['keywords']:\n            if keyword in self.to_ignore:\n                continue\n\n            keywords.append(keyword)\n\n            if keyword.startswith('sec-'):\n                subkeywords.append('sec-')\n            elif keyword.startswith('csectype-'):\n                subkeywords.append('csectype-')\n        return keywords + subkeywords\n\n\nclass severity(object):\n    def __call__(self, bug):\n        return field(bug, 'severity')\n\n\nclass is_coverity_issue(object):\n    def __call__(self, bug):\n        return re.search('[CID ?[0-9]+]', bug['summary']) is not None or re.search('[CID ?[0-9]+]', bug['whiteboard']) is not None\n\n\nclass has_url(object):\n    def __call__(self, bug):\n        return bug['url'] != ''\n\n\nclass has_w3c_url(object):\n    def __call__(self, bug):\n        return 'w3c' in bug['url']\n\n\nclass has_github_url(object):\n    def __call__(self, bug):\n        return 'github' in bug['url']\n\n\nclass whiteboard(object):\n    def __call__(self, bug):\n\n        # Split by '['\n        paren_splits = bug['whiteboard'].lower().split('[')\n\n        # Split splits by space if they weren't in [ and ].\n        splits = []\n        for paren_split in paren_splits:\n            if ']' in paren_split:\n                paren_split = paren_split.split(']')\n                splits += paren_split\n            else:\n                splits += paren_split.split(' ')\n\n        # Remove empty splits and strip\n        splits = [split.strip() for split in splits if split.strip() != '']\n\n        # For splits which contain ':', return both the whole string and the string before ':'.\n        splits += [split.split(':', 1)[0] for split in splits if ':' in split]\n\n        return splits\n\n\nclass patches(object):\n    def __call__(self, bug):\n        return sum(1 for a in bug['attachments'] if a['is_patch'] or a['content_type'] in ['text/x-review-board-request', 'text/x-phabricator-request'])\n\n\nclass landings(object):\n    def __call__(self, bug):\n        return sum(1 for c in bug['comments'] if '://hg.mozilla.org/' in c['text'])\n\n\nclass title(object):\n    def __call__(self, bug):\n        ret = []\n\n        keywords = [\n            'fail',\n        ]\n        for keyword in keywords:\n            if keyword in bug['summary'].lower():\n                ret.append(keyword)\n\n        return ret\n\n\nclass product(object):\n    def __call__(self, bug):\n        return bug['product']\n\n\nclass component(object):\n    def __call__(self, bug):\n        return bug['component']\n\n\nclass is_mozillian(object):\n    def __call__(self, bug):\n        return any(bug['creator_detail']['email'].endswith(domain) for domain in ['@mozilla.com', '@mozilla.org'])\n\n\nclass bug_reporter(object):\n    def __call__(self, bug):\n        return bug['creator_detail']['email']\n\n\nclass delta_request_merge(object):\n    def __call__(self, bug):\n        for history in bug['history']:\n            for change in history['changes']:\n                if change['added'].startswith('approval-mozilla'):\n                    uplift_request_datetime = datetime.strptime(history['when'], '%Y-%m-%dT%H:%M:%SZ').replace(tzinfo=timezone.utc)\n                    timedelta = versions.getCloserRelease(uplift_request_datetime)[1] - uplift_request_datetime\n                    return timedelta.days + timedelta.seconds / (24 * 60 * 60)\n\n        return None\n\n\nclass commit_added(object):\n    def __call__(self, bug):\n        return sum(commit['added'] for commit in bug['commits'])\n\n\nclass commit_deleted(object):\n    def __call__(self, bug):\n        return sum(commit['deleted'] for commit in bug['commits'])\n\n\nclass commit_types(object):\n    def __call__(self, bug):\n        return sum((commit['types'] for commit in bug['commits']), [])\n\n\nclass blocked_bugs_number(object):\n    def __call__(self, bug):\n        return len(bug['blocks'])\n\n\nclass priority(object):\n    def __call__(self, bug):\n        return bug['priority']\n\n\nclass bug_has_cve_in_alias(object):\n    def __call__(self, bug):\n        return bug['alias'] is not None and 'CVE' in bug['alias']\n\n\nclass commit_files_modified_num(object):\n    def __call__(self, bug):\n        return sum(commit['files_modified_num'] for commit in bug['commits'])\n\n\nclass comment_count(object):\n    def __call__(self, bug):\n        return field(bug, 'comment_count')\n\n\nclass comment_length(object):\n    def __call__(self, bug):\n        return sum(len(x['text']) for x in bug['comments'])\n\n\ndef cleanup_url(text):\n    text = re.sub(r'http[s]?://(hg.mozilla|searchfox|dxr.mozilla)\\S+', '__CODE_REFERENCE_URL__', text)\n    return re.sub(r'http\\S+', '__URL__', text)\n\n\ndef cleanup_fileref(text):\n    return re.sub(r'\\w+\\.py\\b|\\w+\\.json\\b|\\w+\\.js\\b|\\w+\\.jsm\\b|\\w+\\.html\\b|\\w+\\.css\\b|\\w+\\.c\\b|\\w+\\.cpp\\b|\\w+\\.h\\b', '__FILE_REFERENCE__', text)\n\n\ndef cleanup_responses(text):\n    return re.sub('>[^\\n]+', ' ', text)\n\n\ndef cleanup_hex(text):\n    return re.sub(r'\\b0[xX][0-9a-fA-F]+\\b', '__HEX_NUMBER__', text)\n\n\ndef cleanup_dll(text):\n    return re.sub(r'\\w+(\\.dll|\\.so|\\.dylib)\\b', '__DLL_NAME__', text)\n\n\ndef cleanup_synonyms(text):\n    synonyms = [\n        ('safemode', ['safemode', 'safe mode']),\n        ('str', ['str', 'steps to reproduce', 'repro steps']),\n        ('uaf', ['uaf', 'use after free', 'use-after-free']),\n        ('asan', ['asan', 'address sanitizer', 'addresssanitizer']),\n        ('permafailure', ['permafailure', 'permafailing', 'permafail', 'perma failure', 'perma failing', 'perma fail', 'perma-failure', 'perma-failing', 'perma-fail']),\n        ('spec', ['spec', 'specification']),\n    ]\n\n    for synonym_group, synonym_list in synonyms:\n        text = re.sub('|'.join(fr'\\b{synonym}\\b' for synonym in synonym_list), synonym_group, text, flags=re.IGNORECASE)\n\n    return text\n\n\ndef cleanup_crash(text):\n    return re.sub(r'bp-[a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{6}[0-9]{6}\\b', '__CRASH_STATS_LINK__', text)\n\n\nclass BugExtractor(BaseEstimator, TransformerMixin):\n    def __init__(self, feature_extractors, cleanup_functions, rollback=False, rollback_when=None, commit_data=False):\n        self.feature_extractors = feature_extractors\n        self.cleanup_functions = cleanup_functions\n        self.rollback = rollback\n        self.rollback_when = rollback_when\n        self.commit_map = repository.get_commit_map() if commit_data else None\n\n    def fit(self, x, y=None):\n        return self\n\n    def transform(self, bugs):\n        results = []\n\n        for bug in bugs:\n            bug_id = bug['id']\n\n            if self.rollback:\n                bug = bug_snapshot.rollback(bug, self.rollback_when)\n\n            data = {}\n\n            if self.commit_map is not None:\n                if bug_id in self.commit_map:\n                    bug['commits'] = self.commit_map[bug_id]\n                else:\n                    bug['commits'] = []\n\n            for f in self.feature_extractors:\n                res = f(bug)\n\n                if res is None:\n                    continue\n\n                if isinstance(res, list):\n                    for item in res:\n                        data[f.__class__.__name__ + '-' + item] = 'True'\n                    continue\n\n                if isinstance(res, bool):\n                    res = str(res)\n\n                data[f.__class__.__name__] = res\n\n            # TODO: Try simply using all possible fields instead of extracting features manually.\n\n            for cleanup_function in self.cleanup_functions:\n                bug['summary'] = cleanup_function(bug['summary'])\n                for c in bug['comments']:\n                    c['text'] = cleanup_function(c['text'])\n\n            result = {\n                'data': data,\n                'title': bug['summary'],\n                'first_comment': bug['comments'][0]['text'],\n                'comments': ' '.join([c['text'] for c in bug['comments']]),\n            }\n\n            results.append(result)\n\n        return pd.DataFrame(results)\n", "path": "bugbug/bug_features.py"}]}