We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
GitHubPullrequestPoller token secret gives builtins.TypeError:
When attempting the following in GitHubPullrequestPoller:
```
token=util.Secret("githubToken"),
```
I receive: builtins.TypeError: must be str, not Secret

and similarly via Interpolate:
```
token=util.Interpolate("%(secret:githubToken)s"),
```
builtins.TypeError: must be str, not Interpolate

Apologies if I'm doing this wrong but given:
https://docs.buildbot.net/latest/manual/secretsmanagement.html#as-argument-to-services
I had expected this to work. I got the impression that the secret manager was as yet not supported across the board and so if this is an example of an unsupported use then it seems like a nice one to have.

Thanks all for a great project.

Incidentally in both cases I can cast via str() and avoid this issue but then run into another issue which I will report separately.

--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `master/buildbot/changes/github.py`
Content:
```
1 # This file is part of Buildbot.  Buildbot is free software: you can
2 # redistribute it and/or modify it under the terms of the GNU General Public
3 # License as published by the Free Software Foundation, version 2.
4 #
5 # This program is distributed in the hope that it will be useful, but WITHOUT
6 # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
7 # FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
8 # details.
9 #
10 # You should have received a copy of the GNU General Public License along with
11 # this program; if not, write to the Free Software Foundation, Inc., 51
12 # Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
13 #
14 # Copyright Buildbot Team Members
15 
16 from datetime import datetime
17 from fnmatch import fnmatch
18 
19 from twisted.internet import defer
20 
21 from buildbot import config
22 from buildbot.changes import base
23 from buildbot.util import bytes2unicode
24 from buildbot.util import datetime2epoch
25 from buildbot.util import httpclientservice
26 from buildbot.util.logger import Logger
27 from buildbot.util.state import StateMixin
28 
29 log = Logger()
30 
31 HOSTED_BASE_URL = "https://api.github.com"
32 link_urls = {
33     "https": "clone_url",
34     "svn": "svn_url",
35     "git": "git_url",
36     "ssh": "ssh_url"
37 }
38 
39 
40 class PullRequestMixin:
41     def extractProperties(self, payload):
42         def flatten(properties, base, info_dict):
43             for k, v in info_dict.items():
44                 name = ".".join([base, k])
45                 if isinstance(v, dict):
46                     flatten(properties, name, v)
47                 elif any([fnmatch(name, expr)
48                           for expr in self.github_property_whitelist]):
49                     properties[name] = v
50 
51         properties = {}
52         flatten(properties, "github", payload)
53         return properties
54 
55 
56 class GitHubPullrequestPoller(base.ReconfigurablePollingChangeSource,
57                               StateMixin, PullRequestMixin):
58     compare_attrs = ("owner", "repo", "token", "branches", "pollInterval",
59                      "category", "pollAtLaunch", "name")
60     db_class_name = 'GitHubPullrequestPoller'
61 
62     def __init__(self, owner, repo, **kwargs):
63         name = kwargs.get("name")
64         if not name:
65             kwargs["name"] = "GitHubPullrequestPoller:" + owner + "/" + repo
66         super().__init__(owner, repo, **kwargs)
67 
68     def checkConfig(self,
69                     owner,
70                     repo,
71                     branches=None,
72                     category='pull',
73                     baseURL=None,
74                     pullrequest_filter=True,
75                     token=None,
76                     magic_link=False,
77                     repository_type="https",
78                     github_property_whitelist=None,
79                     **kwargs):
80         if repository_type not in ["https", "svn", "git", "ssh"]:
81             config.error(
82                 "repository_type must be one of {https, svn, git, ssh}")
83         super().checkConfig(name=self.name, **kwargs)
84 
85     @defer.inlineCallbacks
86     def reconfigService(self,
87                         owner,
88                         repo,
89                         branches=None,
90                         pollInterval=10 * 60,
91                         category=None,
92                         baseURL=None,
93                         pullrequest_filter=True,
94                         token=None,
95                         pollAtLaunch=False,
96                         magic_link=False,
97                         repository_type="https",
98                         github_property_whitelist=None,
99                         **kwargs):
100         yield super().reconfigService(name=self.name, **kwargs)
101 
102         if baseURL is None:
103             baseURL = HOSTED_BASE_URL
104         if baseURL.endswith('/'):
105             baseURL = baseURL[:-1]
106 
107         http_headers = {'User-Agent': 'Buildbot'}
108         if token is not None:
109             http_headers.update({'Authorization': 'token ' + token})
110 
111         self._http = yield httpclientservice.HTTPClientService.getService(
112             self.master, baseURL, headers=http_headers)
113 
114         self.token = token
115         self.owner = owner
116         self.repo = repo
117         self.branches = branches
118         self.github_property_whitelist = github_property_whitelist
119         self.pollInterval = pollInterval
120         self.pollAtLaunch = pollAtLaunch
121         self.repository_type = link_urls[repository_type]
122         self.magic_link = magic_link
123 
124         if github_property_whitelist is None:
125             self.github_property_whitelist = []
126 
127         if callable(pullrequest_filter):
128             self.pullrequest_filter = pullrequest_filter
129         else:
130             self.pullrequest_filter = (lambda _: pullrequest_filter)
131 
132         self.category = category if callable(category) else bytes2unicode(
133             category)
134 
135     def describe(self):
136         return ("GitHubPullrequestPoller watching the "
137                 "GitHub repository {}/{}").format(self.owner, self.repo)
138 
139     @defer.inlineCallbacks
140     def _getPullInformation(self, pull_number):
141         result = yield self._http.get('/'.join(
142             ['/repos', self.owner, self.repo, 'pulls', str(pull_number)]))
143         my_json = yield result.json()
144         return my_json
145 
146     @defer.inlineCallbacks
147     def _getPulls(self):
148         log.debug(("GitHubPullrequestPoller: polling "
149                    "GitHub repository {}/{}, branches: {}").format(self.owner, self.repo,
150                                                                    self.branches))
151         result = yield self._http.get('/'.join(
152             ['/repos', self.owner, self.repo, 'pulls']))
153         my_json = yield result.json()
154         if result.code != 200:
155             message = my_json.get('message', 'unknown')
156             log.error("GitHubPullrequestPoller error {0.code} '{1}' "
157                       "while loading {0.url}".format(result, message))
158             return []
159         return my_json
160 
161     @defer.inlineCallbacks
162     def _getFiles(self, prnumber):
163         result = yield self._http.get("/".join([
164             '/repos', self.owner, self.repo, 'pulls', str(prnumber), 'files'
165         ]))
166         my_json = yield result.json()
167 
168         return [f["filename"] for f in my_json]
169 
170     @defer.inlineCallbacks
171     def _getCommitters(self, prnumber):
172         result = yield self._http.get("/".join([
173             '/repos', self.owner, self.repo, 'pulls', str(prnumber), 'commits'
174         ]))
175         my_json = yield result.json()
176 
177         return [[c["commit"]["committer"]["name"],
178                  c["commit"]["committer"]["email"]] for c in my_json]
179 
180     @defer.inlineCallbacks
181     def _getAuthors(self, prnumber):
182         result = yield self._http.get("/".join([
183             '/repos', self.owner, self.repo, 'pulls', str(prnumber), 'commits'
184         ]))
185         my_json = yield result.json()
186 
187         return [[a["commit"]["author"]["name"], a["commit"]["author"]["email"]] for a in my_json]
188 
189     @defer.inlineCallbacks
190     def _getCurrentRev(self, prnumber):
191         # Get currently assigned revision of PR number
192 
193         result = yield self._getStateObjectId()
194         rev = yield self.master.db.state.getState(result, 'pull_request%d' %
195                                                   prnumber, None)
196         return rev
197 
198     @defer.inlineCallbacks
199     def _setCurrentRev(self, prnumber, rev):
200         # Set the updated revision for PR number.
201 
202         result = yield self._getStateObjectId()
203         yield self.master.db.state.setState(result,
204                                             'pull_request%d' % prnumber, rev)
205 
206     @defer.inlineCallbacks
207     def _getStateObjectId(self):
208         # Return a deferred for object id in state db.
209         result = yield self.master.db.state.getObjectId('{}/{}'.format(self.owner, self.repo),
210                                                         self.db_class_name)
211         return result
212 
213     @defer.inlineCallbacks
214     def _processChanges(self, github_result):
215         for pr in github_result:
216             # Track PRs for specified branches
217             base_branch = pr['base']['ref']
218             prnumber = pr['number']
219             revision = pr['head']['sha']
220 
221             # Check to see if the branch is set or matches
222             if self.branches is not None and base_branch not in self.branches:
223                 continue
224             if (self.pullrequest_filter is not None and
225                     not self.pullrequest_filter(pr)):
226                 continue
227             current = yield self._getCurrentRev(prnumber)
228             if not current or current[0:12] != revision[0:12]:
229                 # Access title, repo, html link, and comments
230                 pr = yield self._getPullInformation(prnumber)
231                 title = pr['title']
232                 if self.magic_link:
233                     branch = 'refs/pull/{:d}/merge'.format(prnumber)
234                     repo = pr['base']['repo'][self.repository_type]
235                 else:
236                     branch = pr['head']['ref']
237                     repo = pr['head']['repo'][self.repository_type]
238                 revlink = pr['html_url']
239                 comments = pr['body']
240                 updated = datetime.strptime(pr['updated_at'],
241                                             '%Y-%m-%dT%H:%M:%SZ')
242                 # update database
243                 yield self._setCurrentRev(prnumber, revision)
244 
245                 project = pr['base']['repo']['full_name']
246                 commits = pr['commits']
247 
248                 dl = defer.DeferredList(
249                      [self._getAuthors(prnumber), self._getCommitters(prnumber),
250                       self._getFiles(prnumber)],
251                      consumeErrors=True)
252 
253                 results = yield dl
254                 failures = [r[1] for r in results if not r[0]]
255                 if failures:
256                     for failure in failures:
257                         log.error("while processing changes for "
258                                   "Pullrequest {} revision {}".format(
259                                       prnumber, revision))
260                         # Fail on the first error!
261                         failures[0].raiseException()
262                 [authors, committers, files] = [r[1] for r in results]
263 
264                 author = authors[0][0] + " <" + authors[0][1] + ">"
265 
266                 committer = committers[0][0] + " <" + committers[0][1] + ">"
267 
268                 properties = self.extractProperties(pr)
269 
270                 # emit the change
271                 yield self.master.data.updates.addChange(
272                     author=author,
273                     committer=committer,
274                     revision=bytes2unicode(revision),
275                     revlink=bytes2unicode(revlink),
276                     comments='GitHub Pull Request #{0} ({1} commit{2})\n{3}\n{4}'.
277                     format(prnumber, commits, 's'
278                            if commits > 0 else '', title, comments),
279                     when_timestamp=datetime2epoch(updated),
280                     branch=bytes2unicode(branch),
281                     category=self.category,
282                     project=project,
283                     repository=bytes2unicode(repo),
284                     files=files,
285                     properties=properties,
286                     src='git')
287 
288     @defer.inlineCallbacks
289     def poll(self):
290         result = yield self._getPulls()
291         yield self._processChanges(result)
292
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/master/buildbot/changes/github.py b/master/buildbot/changes/github.py
--- a/master/buildbot/changes/github.py
+++ b/master/buildbot/changes/github.py
@@ -106,6 +106,7 @@
 
         http_headers = {'User-Agent': 'Buildbot'}
         if token is not None:
+            token = yield self.renderSecrets(token)
             http_headers.update({'Authorization': 'token ' + token})
 
         self._http = yield httpclientservice.HTTPClientService.getService(

{"golden_diff": "diff --git a/master/buildbot/changes/github.py b/master/buildbot/changes/github.py\n--- a/master/buildbot/changes/github.py\n+++ b/master/buildbot/changes/github.py\n@@ -106,6 +106,7 @@\n \n         http_headers = {'User-Agent': 'Buildbot'}\n         if token is not None:\n+            token = yield self.renderSecrets(token)\n             http_headers.update({'Authorization': 'token ' + token})\n \n         self._http = yield httpclientservice.HTTPClientService.getService(\n", "issue": "GitHubPullrequestPoller token secret gives builtins.TypeError:\nWhen attempting the following in GitHubPullrequestPoller:\r\n```\r\ntoken=util.Secret(\"githubToken\"),\r\n```\r\nI receive: builtins.TypeError: must be str, not Secret\r\n\r\nand similarly via Interpolate:\r\n```\r\ntoken=util.Interpolate(\"%(secret:githubToken)s\"),\r\n```\r\nbuiltins.TypeError: must be str, not Interpolate\r\n\r\nApologies if I'm doing this wrong but given:\r\nhttps://docs.buildbot.net/latest/manual/secretsmanagement.html#as-argument-to-services\r\nI had expected this to work. I got the impression that the secret manager was as yet not supported across the board and so if this is an example of an unsupported use then it seems like a nice one to have.\r\n\r\nThanks all for a great project.\r\n\r\nIncidentally in both cases I can cast via str() and avoid this issue but then run into another issue which I will report separately.\n", "before_files": [{"content": "# This file is part of Buildbot.  Buildbot is free software: you can\n# redistribute it and/or modify it under the terms of the GNU General Public\n# License as published by the Free Software Foundation, version 2.\n#\n# This program is distributed in the hope that it will be useful, but WITHOUT\n# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS\n# FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more\n# details.\n#\n# You should have received a copy of the GNU General Public License along with\n# this program; if not, write to the Free Software Foundation, Inc., 51\n# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.\n#\n# Copyright Buildbot Team Members\n\nfrom datetime import datetime\nfrom fnmatch import fnmatch\n\nfrom twisted.internet import defer\n\nfrom buildbot import config\nfrom buildbot.changes import base\nfrom buildbot.util import bytes2unicode\nfrom buildbot.util import datetime2epoch\nfrom buildbot.util import httpclientservice\nfrom buildbot.util.logger import Logger\nfrom buildbot.util.state import StateMixin\n\nlog = Logger()\n\nHOSTED_BASE_URL = \"https://api.github.com\"\nlink_urls = {\n    \"https\": \"clone_url\",\n    \"svn\": \"svn_url\",\n    \"git\": \"git_url\",\n    \"ssh\": \"ssh_url\"\n}\n\n\nclass PullRequestMixin:\n    def extractProperties(self, payload):\n        def flatten(properties, base, info_dict):\n            for k, v in info_dict.items():\n                name = \".\".join([base, k])\n                if isinstance(v, dict):\n                    flatten(properties, name, v)\n                elif any([fnmatch(name, expr)\n                          for expr in self.github_property_whitelist]):\n                    properties[name] = v\n\n        properties = {}\n        flatten(properties, \"github\", payload)\n        return properties\n\n\nclass GitHubPullrequestPoller(base.ReconfigurablePollingChangeSource,\n                              StateMixin, PullRequestMixin):\n    compare_attrs = (\"owner\", \"repo\", \"token\", \"branches\", \"pollInterval\",\n                     \"category\", \"pollAtLaunch\", \"name\")\n    db_class_name = 'GitHubPullrequestPoller'\n\n    def __init__(self, owner, repo, **kwargs):\n        name = kwargs.get(\"name\")\n        if not name:\n            kwargs[\"name\"] = \"GitHubPullrequestPoller:\" + owner + \"/\" + repo\n        super().__init__(owner, repo, **kwargs)\n\n    def checkConfig(self,\n                    owner,\n                    repo,\n                    branches=None,\n                    category='pull',\n                    baseURL=None,\n                    pullrequest_filter=True,\n                    token=None,\n                    magic_link=False,\n                    repository_type=\"https\",\n                    github_property_whitelist=None,\n                    **kwargs):\n        if repository_type not in [\"https\", \"svn\", \"git\", \"ssh\"]:\n            config.error(\n                \"repository_type must be one of {https, svn, git, ssh}\")\n        super().checkConfig(name=self.name, **kwargs)\n\n    @defer.inlineCallbacks\n    def reconfigService(self,\n                        owner,\n                        repo,\n                        branches=None,\n                        pollInterval=10 * 60,\n                        category=None,\n                        baseURL=None,\n                        pullrequest_filter=True,\n                        token=None,\n                        pollAtLaunch=False,\n                        magic_link=False,\n                        repository_type=\"https\",\n                        github_property_whitelist=None,\n                        **kwargs):\n        yield super().reconfigService(name=self.name, **kwargs)\n\n        if baseURL is None:\n            baseURL = HOSTED_BASE_URL\n        if baseURL.endswith('/'):\n            baseURL = baseURL[:-1]\n\n        http_headers = {'User-Agent': 'Buildbot'}\n        if token is not None:\n            http_headers.update({'Authorization': 'token ' + token})\n\n        self._http = yield httpclientservice.HTTPClientService.getService(\n            self.master, baseURL, headers=http_headers)\n\n        self.token = token\n        self.owner = owner\n        self.repo = repo\n        self.branches = branches\n        self.github_property_whitelist = github_property_whitelist\n        self.pollInterval = pollInterval\n        self.pollAtLaunch = pollAtLaunch\n        self.repository_type = link_urls[repository_type]\n        self.magic_link = magic_link\n\n        if github_property_whitelist is None:\n            self.github_property_whitelist = []\n\n        if callable(pullrequest_filter):\n            self.pullrequest_filter = pullrequest_filter\n        else:\n            self.pullrequest_filter = (lambda _: pullrequest_filter)\n\n        self.category = category if callable(category) else bytes2unicode(\n            category)\n\n    def describe(self):\n        return (\"GitHubPullrequestPoller watching the \"\n                \"GitHub repository {}/{}\").format(self.owner, self.repo)\n\n    @defer.inlineCallbacks\n    def _getPullInformation(self, pull_number):\n        result = yield self._http.get('/'.join(\n            ['/repos', self.owner, self.repo, 'pulls', str(pull_number)]))\n        my_json = yield result.json()\n        return my_json\n\n    @defer.inlineCallbacks\n    def _getPulls(self):\n        log.debug((\"GitHubPullrequestPoller: polling \"\n                   \"GitHub repository {}/{}, branches: {}\").format(self.owner, self.repo,\n                                                                   self.branches))\n        result = yield self._http.get('/'.join(\n            ['/repos', self.owner, self.repo, 'pulls']))\n        my_json = yield result.json()\n        if result.code != 200:\n            message = my_json.get('message', 'unknown')\n            log.error(\"GitHubPullrequestPoller error {0.code} '{1}' \"\n                      \"while loading {0.url}\".format(result, message))\n            return []\n        return my_json\n\n    @defer.inlineCallbacks\n    def _getFiles(self, prnumber):\n        result = yield self._http.get(\"/\".join([\n            '/repos', self.owner, self.repo, 'pulls', str(prnumber), 'files'\n        ]))\n        my_json = yield result.json()\n\n        return [f[\"filename\"] for f in my_json]\n\n    @defer.inlineCallbacks\n    def _getCommitters(self, prnumber):\n        result = yield self._http.get(\"/\".join([\n            '/repos', self.owner, self.repo, 'pulls', str(prnumber), 'commits'\n        ]))\n        my_json = yield result.json()\n\n        return [[c[\"commit\"][\"committer\"][\"name\"],\n                 c[\"commit\"][\"committer\"][\"email\"]] for c in my_json]\n\n    @defer.inlineCallbacks\n    def _getAuthors(self, prnumber):\n        result = yield self._http.get(\"/\".join([\n            '/repos', self.owner, self.repo, 'pulls', str(prnumber), 'commits'\n        ]))\n        my_json = yield result.json()\n\n        return [[a[\"commit\"][\"author\"][\"name\"], a[\"commit\"][\"author\"][\"email\"]] for a in my_json]\n\n    @defer.inlineCallbacks\n    def _getCurrentRev(self, prnumber):\n        # Get currently assigned revision of PR number\n\n        result = yield self._getStateObjectId()\n        rev = yield self.master.db.state.getState(result, 'pull_request%d' %\n                                                  prnumber, None)\n        return rev\n\n    @defer.inlineCallbacks\n    def _setCurrentRev(self, prnumber, rev):\n        # Set the updated revision for PR number.\n\n        result = yield self._getStateObjectId()\n        yield self.master.db.state.setState(result,\n                                            'pull_request%d' % prnumber, rev)\n\n    @defer.inlineCallbacks\n    def _getStateObjectId(self):\n        # Return a deferred for object id in state db.\n        result = yield self.master.db.state.getObjectId('{}/{}'.format(self.owner, self.repo),\n                                                        self.db_class_name)\n        return result\n\n    @defer.inlineCallbacks\n    def _processChanges(self, github_result):\n        for pr in github_result:\n            # Track PRs for specified branches\n            base_branch = pr['base']['ref']\n            prnumber = pr['number']\n            revision = pr['head']['sha']\n\n            # Check to see if the branch is set or matches\n            if self.branches is not None and base_branch not in self.branches:\n                continue\n            if (self.pullrequest_filter is not None and\n                    not self.pullrequest_filter(pr)):\n                continue\n            current = yield self._getCurrentRev(prnumber)\n            if not current or current[0:12] != revision[0:12]:\n                # Access title, repo, html link, and comments\n                pr = yield self._getPullInformation(prnumber)\n                title = pr['title']\n                if self.magic_link:\n                    branch = 'refs/pull/{:d}/merge'.format(prnumber)\n                    repo = pr['base']['repo'][self.repository_type]\n                else:\n                    branch = pr['head']['ref']\n                    repo = pr['head']['repo'][self.repository_type]\n                revlink = pr['html_url']\n                comments = pr['body']\n                updated = datetime.strptime(pr['updated_at'],\n                                            '%Y-%m-%dT%H:%M:%SZ')\n                # update database\n                yield self._setCurrentRev(prnumber, revision)\n\n                project = pr['base']['repo']['full_name']\n                commits = pr['commits']\n\n                dl = defer.DeferredList(\n                     [self._getAuthors(prnumber), self._getCommitters(prnumber),\n                      self._getFiles(prnumber)],\n                     consumeErrors=True)\n\n                results = yield dl\n                failures = [r[1] for r in results if not r[0]]\n                if failures:\n                    for failure in failures:\n                        log.error(\"while processing changes for \"\n                                  \"Pullrequest {} revision {}\".format(\n                                      prnumber, revision))\n                        # Fail on the first error!\n                        failures[0].raiseException()\n                [authors, committers, files] = [r[1] for r in results]\n\n                author = authors[0][0] + \" <\" + authors[0][1] + \">\"\n\n                committer = committers[0][0] + \" <\" + committers[0][1] + \">\"\n\n                properties = self.extractProperties(pr)\n\n                # emit the change\n                yield self.master.data.updates.addChange(\n                    author=author,\n                    committer=committer,\n                    revision=bytes2unicode(revision),\n                    revlink=bytes2unicode(revlink),\n                    comments='GitHub Pull Request #{0} ({1} commit{2})\\n{3}\\n{4}'.\n                    format(prnumber, commits, 's'\n                           if commits > 0 else '', title, comments),\n                    when_timestamp=datetime2epoch(updated),\n                    branch=bytes2unicode(branch),\n                    category=self.category,\n                    project=project,\n                    repository=bytes2unicode(repo),\n                    files=files,\n                    properties=properties,\n                    src='git')\n\n    @defer.inlineCallbacks\n    def poll(self):\n        result = yield self._getPulls()\n        yield self._processChanges(result)\n", "path": "master/buildbot/changes/github.py"}], "after_files": [{"content": "# This file is part of Buildbot.  Buildbot is free software: you can\n# redistribute it and/or modify it under the terms of the GNU General Public\n# License as published by the Free Software Foundation, version 2.\n#\n# This program is distributed in the hope that it will be useful, but WITHOUT\n# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS\n# FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more\n# details.\n#\n# You should have received a copy of the GNU General Public License along with\n# this program; if not, write to the Free Software Foundation, Inc., 51\n# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.\n#\n# Copyright Buildbot Team Members\n\nfrom datetime import datetime\nfrom fnmatch import fnmatch\n\nfrom twisted.internet import defer\n\nfrom buildbot import config\nfrom buildbot.changes import base\nfrom buildbot.util import bytes2unicode\nfrom buildbot.util import datetime2epoch\nfrom buildbot.util import httpclientservice\nfrom buildbot.util.logger import Logger\nfrom buildbot.util.state import StateMixin\n\nlog = Logger()\n\nHOSTED_BASE_URL = \"https://api.github.com\"\nlink_urls = {\n    \"https\": \"clone_url\",\n    \"svn\": \"svn_url\",\n    \"git\": \"git_url\",\n    \"ssh\": \"ssh_url\"\n}\n\n\nclass PullRequestMixin:\n    def extractProperties(self, payload):\n        def flatten(properties, base, info_dict):\n            for k, v in info_dict.items():\n                name = \".\".join([base, k])\n                if isinstance(v, dict):\n                    flatten(properties, name, v)\n                elif any([fnmatch(name, expr)\n                          for expr in self.github_property_whitelist]):\n                    properties[name] = v\n\n        properties = {}\n        flatten(properties, \"github\", payload)\n        return properties\n\n\nclass GitHubPullrequestPoller(base.ReconfigurablePollingChangeSource,\n                              StateMixin, PullRequestMixin):\n    compare_attrs = (\"owner\", \"repo\", \"token\", \"branches\", \"pollInterval\",\n                     \"category\", \"pollAtLaunch\", \"name\")\n    db_class_name = 'GitHubPullrequestPoller'\n\n    def __init__(self, owner, repo, **kwargs):\n        name = kwargs.get(\"name\")\n        if not name:\n            kwargs[\"name\"] = \"GitHubPullrequestPoller:\" + owner + \"/\" + repo\n        super().__init__(owner, repo, **kwargs)\n\n    def checkConfig(self,\n                    owner,\n                    repo,\n                    branches=None,\n                    category='pull',\n                    baseURL=None,\n                    pullrequest_filter=True,\n                    token=None,\n                    magic_link=False,\n                    repository_type=\"https\",\n                    github_property_whitelist=None,\n                    **kwargs):\n        if repository_type not in [\"https\", \"svn\", \"git\", \"ssh\"]:\n            config.error(\n                \"repository_type must be one of {https, svn, git, ssh}\")\n        super().checkConfig(name=self.name, **kwargs)\n\n    @defer.inlineCallbacks\n    def reconfigService(self,\n                        owner,\n                        repo,\n                        branches=None,\n                        pollInterval=10 * 60,\n                        category=None,\n                        baseURL=None,\n                        pullrequest_filter=True,\n                        token=None,\n                        pollAtLaunch=False,\n                        magic_link=False,\n                        repository_type=\"https\",\n                        github_property_whitelist=None,\n                        **kwargs):\n        yield super().reconfigService(name=self.name, **kwargs)\n\n        if baseURL is None:\n            baseURL = HOSTED_BASE_URL\n        if baseURL.endswith('/'):\n            baseURL = baseURL[:-1]\n\n        http_headers = {'User-Agent': 'Buildbot'}\n        if token is not None:\n            token = yield self.renderSecrets(token)\n            http_headers.update({'Authorization': 'token ' + token})\n\n        self._http = yield httpclientservice.HTTPClientService.getService(\n            self.master, baseURL, headers=http_headers)\n\n        self.token = token\n        self.owner = owner\n        self.repo = repo\n        self.branches = branches\n        self.github_property_whitelist = github_property_whitelist\n        self.pollInterval = pollInterval\n        self.pollAtLaunch = pollAtLaunch\n        self.repository_type = link_urls[repository_type]\n        self.magic_link = magic_link\n\n        if github_property_whitelist is None:\n            self.github_property_whitelist = []\n\n        if callable(pullrequest_filter):\n            self.pullrequest_filter = pullrequest_filter\n        else:\n            self.pullrequest_filter = (lambda _: pullrequest_filter)\n\n        self.category = category if callable(category) else bytes2unicode(\n            category)\n\n    def describe(self):\n        return (\"GitHubPullrequestPoller watching the \"\n                \"GitHub repository {}/{}\").format(self.owner, self.repo)\n\n    @defer.inlineCallbacks\n    def _getPullInformation(self, pull_number):\n        result = yield self._http.get('/'.join(\n            ['/repos', self.owner, self.repo, 'pulls', str(pull_number)]))\n        my_json = yield result.json()\n        return my_json\n\n    @defer.inlineCallbacks\n    def _getPulls(self):\n        log.debug((\"GitHubPullrequestPoller: polling \"\n                   \"GitHub repository {}/{}, branches: {}\").format(self.owner, self.repo,\n                                                                   self.branches))\n        result = yield self._http.get('/'.join(\n            ['/repos', self.owner, self.repo, 'pulls']))\n        my_json = yield result.json()\n        if result.code != 200:\n            message = my_json.get('message', 'unknown')\n            log.error(\"GitHubPullrequestPoller error {0.code} '{1}' \"\n                      \"while loading {0.url}\".format(result, message))\n            return []\n        return my_json\n\n    @defer.inlineCallbacks\n    def _getFiles(self, prnumber):\n        result = yield self._http.get(\"/\".join([\n            '/repos', self.owner, self.repo, 'pulls', str(prnumber), 'files'\n        ]))\n        my_json = yield result.json()\n\n        return [f[\"filename\"] for f in my_json]\n\n    @defer.inlineCallbacks\n    def _getCommitters(self, prnumber):\n        result = yield self._http.get(\"/\".join([\n            '/repos', self.owner, self.repo, 'pulls', str(prnumber), 'commits'\n        ]))\n        my_json = yield result.json()\n\n        return [[c[\"commit\"][\"committer\"][\"name\"],\n                 c[\"commit\"][\"committer\"][\"email\"]] for c in my_json]\n\n    @defer.inlineCallbacks\n    def _getAuthors(self, prnumber):\n        result = yield self._http.get(\"/\".join([\n            '/repos', self.owner, self.repo, 'pulls', str(prnumber), 'commits'\n        ]))\n        my_json = yield result.json()\n\n        return [[a[\"commit\"][\"author\"][\"name\"], a[\"commit\"][\"author\"][\"email\"]] for a in my_json]\n\n    @defer.inlineCallbacks\n    def _getCurrentRev(self, prnumber):\n        # Get currently assigned revision of PR number\n\n        result = yield self._getStateObjectId()\n        rev = yield self.master.db.state.getState(result, 'pull_request%d' %\n                                                  prnumber, None)\n        return rev\n\n    @defer.inlineCallbacks\n    def _setCurrentRev(self, prnumber, rev):\n        # Set the updated revision for PR number.\n\n        result = yield self._getStateObjectId()\n        yield self.master.db.state.setState(result,\n                                            'pull_request%d' % prnumber, rev)\n\n    @defer.inlineCallbacks\n    def _getStateObjectId(self):\n        # Return a deferred for object id in state db.\n        result = yield self.master.db.state.getObjectId('{}/{}'.format(self.owner, self.repo),\n                                                        self.db_class_name)\n        return result\n\n    @defer.inlineCallbacks\n    def _processChanges(self, github_result):\n        for pr in github_result:\n            # Track PRs for specified branches\n            base_branch = pr['base']['ref']\n            prnumber = pr['number']\n            revision = pr['head']['sha']\n\n            # Check to see if the branch is set or matches\n            if self.branches is not None and base_branch not in self.branches:\n                continue\n            if (self.pullrequest_filter is not None and\n                    not self.pullrequest_filter(pr)):\n                continue\n            current = yield self._getCurrentRev(prnumber)\n            if not current or current[0:12] != revision[0:12]:\n                # Access title, repo, html link, and comments\n                pr = yield self._getPullInformation(prnumber)\n                title = pr['title']\n                if self.magic_link:\n                    branch = 'refs/pull/{:d}/merge'.format(prnumber)\n                    repo = pr['base']['repo'][self.repository_type]\n                else:\n                    branch = pr['head']['ref']\n                    repo = pr['head']['repo'][self.repository_type]\n                revlink = pr['html_url']\n                comments = pr['body']\n                updated = datetime.strptime(pr['updated_at'],\n                                            '%Y-%m-%dT%H:%M:%SZ')\n                # update database\n                yield self._setCurrentRev(prnumber, revision)\n\n                project = pr['base']['repo']['full_name']\n                commits = pr['commits']\n\n                dl = defer.DeferredList(\n                     [self._getAuthors(prnumber), self._getCommitters(prnumber),\n                      self._getFiles(prnumber)],\n                     consumeErrors=True)\n\n                results = yield dl\n                failures = [r[1] for r in results if not r[0]]\n                if failures:\n                    for failure in failures:\n                        log.error(\"while processing changes for \"\n                                  \"Pullrequest {} revision {}\".format(\n                                      prnumber, revision))\n                        # Fail on the first error!\n                        failures[0].raiseException()\n                [authors, committers, files] = [r[1] for r in results]\n\n                author = authors[0][0] + \" <\" + authors[0][1] + \">\"\n\n                committer = committers[0][0] + \" <\" + committers[0][1] + \">\"\n\n                properties = self.extractProperties(pr)\n\n                # emit the change\n                yield self.master.data.updates.addChange(\n                    author=author,\n                    committer=committer,\n                    revision=bytes2unicode(revision),\n                    revlink=bytes2unicode(revlink),\n                    comments='GitHub Pull Request #{0} ({1} commit{2})\\n{3}\\n{4}'.\n                    format(prnumber, commits, 's'\n                           if commits > 0 else '', title, comments),\n                    when_timestamp=datetime2epoch(updated),\n                    branch=bytes2unicode(branch),\n                    category=self.category,\n                    project=project,\n                    repository=bytes2unicode(repo),\n                    files=files,\n                    properties=properties,\n                    src='git')\n\n    @defer.inlineCallbacks\n    def poll(self):\n        result = yield self._getPulls()\n        yield self._processChanges(result)\n", "path": "master/buildbot/changes/github.py"}]}