We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
Buildbot should check for duplicates in addURL 
In a step if we call self.addURL() multiple times with same URL, buildbot doesn't check if URL is already added, and adds the duplicate URL. Buildbot should check if the URL is already added before adding the URL.

This happens when a script parses a lot of output and generate links for various things. We can manually maintain a list ourself and check for duplicates before calling addURL(), but it feels so basic that buildbot should be doing this check instead of callers of addURL() checking for duplicates.


--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `master/buildbot/db/steps.py`
Content:
```
1 # This file is part of Buildbot.  Buildbot is free software: you can
2 # redistribute it and/or modify it under the terms of the GNU General Public
3 # License as published by the Free Software Foundation, version 2.
4 #
5 # This program is distributed in the hope that it will be useful, but WITHOUT
6 # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
7 # FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
8 # details.
9 #
10 # You should have received a copy of the GNU General Public License along with
11 # this program; if not, write to the Free Software Foundation, Inc., 51
12 # Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
13 #
14 # Copyright Buildbot Team Members
15 
16 from __future__ import absolute_import
17 from __future__ import print_function
18 
19 import json
20 
21 import sqlalchemy as sa
22 
23 from twisted.internet import defer
24 from twisted.internet import reactor
25 
26 from buildbot.db import base
27 from buildbot.util import epoch2datetime
28 
29 
30 class StepsConnectorComponent(base.DBConnectorComponent):
31     # Documentation is in developer/db.rst
32     url_lock = None
33 
34     def getStep(self, stepid=None, buildid=None, number=None, name=None):
35         tbl = self.db.model.steps
36         if stepid is not None:
37             wc = (tbl.c.id == stepid)
38         else:
39             if buildid is None:
40                 return defer.fail(RuntimeError('must supply either stepid or buildid'))
41             if number is not None:
42                 wc = (tbl.c.number == number)
43             elif name is not None:
44                 wc = (tbl.c.name == name)
45             else:
46                 return defer.fail(RuntimeError('must supply either number or name'))
47             wc = wc & (tbl.c.buildid == buildid)
48 
49         def thd(conn):
50             q = self.db.model.steps.select(whereclause=wc)
51             res = conn.execute(q)
52             row = res.fetchone()
53 
54             rv = None
55             if row:
56                 rv = self._stepdictFromRow(row)
57             res.close()
58             return rv
59         return self.db.pool.do(thd)
60 
61     def getSteps(self, buildid):
62         def thd(conn):
63             tbl = self.db.model.steps
64             q = tbl.select()
65             q = q.where(tbl.c.buildid == buildid)
66             q = q.order_by(tbl.c.number)
67             res = conn.execute(q)
68             return [self._stepdictFromRow(row) for row in res.fetchall()]
69         return self.db.pool.do(thd)
70 
71     def addStep(self, buildid, name, state_string):
72         def thd(conn):
73             tbl = self.db.model.steps
74             # get the highest current number
75             r = conn.execute(sa.select([sa.func.max(tbl.c.number)],
76                                        whereclause=(tbl.c.buildid == buildid)))
77             number = r.scalar()
78             number = 0 if number is None else number + 1
79 
80             # note that there is no chance for a race condition here,
81             # since only one master is inserting steps.  If there is a
82             # conflict, then the name is likely already taken.
83             insert_row = dict(buildid=buildid, number=number,
84                               started_at=None, complete_at=None,
85                               state_string=state_string,
86                               urls_json='[]', name=name)
87             try:
88                 r = conn.execute(self.db.model.steps.insert(), insert_row)
89                 got_id = r.inserted_primary_key[0]
90             except (sa.exc.IntegrityError, sa.exc.ProgrammingError):
91                 got_id = None
92 
93             if got_id:
94                 return (got_id, number, name)
95 
96             # we didn't get an id, so calculate a unique name and use that
97             # instead.  Because names are truncated at the right to fit in a
98             # 50-character identifier, this isn't a simple query.
99             res = conn.execute(sa.select([tbl.c.name],
100                                          whereclause=((tbl.c.buildid == buildid))))
101             names = set([row[0] for row in res])
102             num = 1
103             while True:
104                 numstr = '_%d' % num
105                 newname = name[:50 - len(numstr)] + numstr
106                 if newname not in names:
107                     break
108                 num += 1
109             insert_row['name'] = newname
110             r = conn.execute(self.db.model.steps.insert(), insert_row)
111             got_id = r.inserted_primary_key[0]
112             return (got_id, number, newname)
113         return self.db.pool.do(thd)
114 
115     def startStep(self, stepid, _reactor=reactor):
116         started_at = _reactor.seconds()
117 
118         def thd(conn):
119             tbl = self.db.model.steps
120             q = tbl.update(whereclause=(tbl.c.id == stepid))
121             conn.execute(q, started_at=started_at)
122         return self.db.pool.do(thd)
123 
124     def setStepStateString(self, stepid, state_string):
125         def thd(conn):
126             tbl = self.db.model.steps
127             q = tbl.update(whereclause=(tbl.c.id == stepid))
128             conn.execute(q, state_string=state_string)
129         return self.db.pool.do(thd)
130 
131     def addURL(self, stepid, name, url, _racehook=None):
132         # This methods adds an URL to the db
133         # This is a read modify write and thus there is a possibility
134         # that several urls are added at the same time (e.g with a deferredlist
135         # at the end of a step)
136         # this race condition is only inside the same master, as only one master
137         # is supposed to add urls to a buildstep.
138         # so threading.lock is used, as we are in the thread pool
139         if self.url_lock is None:
140             # this runs in reactor thread, so no race here..
141             self.url_lock = defer.DeferredLock()
142 
143         def thd(conn):
144 
145             tbl = self.db.model.steps
146             wc = (tbl.c.id == stepid)
147             q = sa.select([tbl.c.urls_json],
148                           whereclause=wc)
149             res = conn.execute(q)
150             row = res.fetchone()
151             if _racehook is not None:
152                 _racehook()
153             urls = json.loads(row.urls_json)
154             urls.append(dict(name=name, url=url))
155 
156             q = tbl.update(whereclause=wc)
157             conn.execute(q, urls_json=json.dumps(urls))
158 
159         return self.url_lock.run(lambda: self.db.pool.do(thd))
160 
161     def finishStep(self, stepid, results, hidden, _reactor=reactor):
162         def thd(conn):
163             tbl = self.db.model.steps
164             q = tbl.update(whereclause=(tbl.c.id == stepid))
165             conn.execute(q,
166                          complete_at=_reactor.seconds(),
167                          results=results,
168                          hidden=1 if hidden else 0)
169         return self.db.pool.do(thd)
170 
171     def _stepdictFromRow(self, row):
172         def mkdt(epoch):
173             if epoch:
174                 return epoch2datetime(epoch)
175 
176         return dict(
177             id=row.id,
178             number=row.number,
179             name=row.name,
180             buildid=row.buildid,
181             started_at=mkdt(row.started_at),
182             complete_at=mkdt(row.complete_at),
183             state_string=row.state_string,
184             results=row.results,
185             urls=json.loads(row.urls_json),
186             hidden=bool(row.hidden))
187
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/master/buildbot/db/steps.py b/master/buildbot/db/steps.py
--- a/master/buildbot/db/steps.py
+++ b/master/buildbot/db/steps.py
@@ -151,10 +151,13 @@
             if _racehook is not None:
                 _racehook()
             urls = json.loads(row.urls_json)
-            urls.append(dict(name=name, url=url))
 
-            q = tbl.update(whereclause=wc)
-            conn.execute(q, urls_json=json.dumps(urls))
+            url_item = dict(name=name, url=url)
+
+            if url_item not in urls:
+                urls.append(url_item)
+                q = tbl.update(whereclause=wc)
+                conn.execute(q, urls_json=json.dumps(urls))
 
         return self.url_lock.run(lambda: self.db.pool.do(thd))

{"golden_diff": "diff --git a/master/buildbot/db/steps.py b/master/buildbot/db/steps.py\n--- a/master/buildbot/db/steps.py\n+++ b/master/buildbot/db/steps.py\n@@ -151,10 +151,13 @@\n             if _racehook is not None:\n                 _racehook()\n             urls = json.loads(row.urls_json)\n-            urls.append(dict(name=name, url=url))\n \n-            q = tbl.update(whereclause=wc)\n-            conn.execute(q, urls_json=json.dumps(urls))\n+            url_item = dict(name=name, url=url)\n+\n+            if url_item not in urls:\n+                urls.append(url_item)\n+                q = tbl.update(whereclause=wc)\n+                conn.execute(q, urls_json=json.dumps(urls))\n \n         return self.url_lock.run(lambda: self.db.pool.do(thd))\n", "issue": "Buildbot should check for duplicates in addURL \nIn a step if we call self.addURL() multiple times with same URL, buildbot doesn't check if URL is already added, and adds the duplicate URL. Buildbot should check if the URL is already added before adding the URL.\r\n\r\nThis happens when a script parses a lot of output and generate links for various things. We can manually maintain a list ourself and check for duplicates before calling addURL(), but it feels so basic that buildbot should be doing this check instead of callers of addURL() checking for duplicates.\r\n\n", "before_files": [{"content": "# This file is part of Buildbot.  Buildbot is free software: you can\n# redistribute it and/or modify it under the terms of the GNU General Public\n# License as published by the Free Software Foundation, version 2.\n#\n# This program is distributed in the hope that it will be useful, but WITHOUT\n# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS\n# FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more\n# details.\n#\n# You should have received a copy of the GNU General Public License along with\n# this program; if not, write to the Free Software Foundation, Inc., 51\n# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.\n#\n# Copyright Buildbot Team Members\n\nfrom __future__ import absolute_import\nfrom __future__ import print_function\n\nimport json\n\nimport sqlalchemy as sa\n\nfrom twisted.internet import defer\nfrom twisted.internet import reactor\n\nfrom buildbot.db import base\nfrom buildbot.util import epoch2datetime\n\n\nclass StepsConnectorComponent(base.DBConnectorComponent):\n    # Documentation is in developer/db.rst\n    url_lock = None\n\n    def getStep(self, stepid=None, buildid=None, number=None, name=None):\n        tbl = self.db.model.steps\n        if stepid is not None:\n            wc = (tbl.c.id == stepid)\n        else:\n            if buildid is None:\n                return defer.fail(RuntimeError('must supply either stepid or buildid'))\n            if number is not None:\n                wc = (tbl.c.number == number)\n            elif name is not None:\n                wc = (tbl.c.name == name)\n            else:\n                return defer.fail(RuntimeError('must supply either number or name'))\n            wc = wc & (tbl.c.buildid == buildid)\n\n        def thd(conn):\n            q = self.db.model.steps.select(whereclause=wc)\n            res = conn.execute(q)\n            row = res.fetchone()\n\n            rv = None\n            if row:\n                rv = self._stepdictFromRow(row)\n            res.close()\n            return rv\n        return self.db.pool.do(thd)\n\n    def getSteps(self, buildid):\n        def thd(conn):\n            tbl = self.db.model.steps\n            q = tbl.select()\n            q = q.where(tbl.c.buildid == buildid)\n            q = q.order_by(tbl.c.number)\n            res = conn.execute(q)\n            return [self._stepdictFromRow(row) for row in res.fetchall()]\n        return self.db.pool.do(thd)\n\n    def addStep(self, buildid, name, state_string):\n        def thd(conn):\n            tbl = self.db.model.steps\n            # get the highest current number\n            r = conn.execute(sa.select([sa.func.max(tbl.c.number)],\n                                       whereclause=(tbl.c.buildid == buildid)))\n            number = r.scalar()\n            number = 0 if number is None else number + 1\n\n            # note that there is no chance for a race condition here,\n            # since only one master is inserting steps.  If there is a\n            # conflict, then the name is likely already taken.\n            insert_row = dict(buildid=buildid, number=number,\n                              started_at=None, complete_at=None,\n                              state_string=state_string,\n                              urls_json='[]', name=name)\n            try:\n                r = conn.execute(self.db.model.steps.insert(), insert_row)\n                got_id = r.inserted_primary_key[0]\n            except (sa.exc.IntegrityError, sa.exc.ProgrammingError):\n                got_id = None\n\n            if got_id:\n                return (got_id, number, name)\n\n            # we didn't get an id, so calculate a unique name and use that\n            # instead.  Because names are truncated at the right to fit in a\n            # 50-character identifier, this isn't a simple query.\n            res = conn.execute(sa.select([tbl.c.name],\n                                         whereclause=((tbl.c.buildid == buildid))))\n            names = set([row[0] for row in res])\n            num = 1\n            while True:\n                numstr = '_%d' % num\n                newname = name[:50 - len(numstr)] + numstr\n                if newname not in names:\n                    break\n                num += 1\n            insert_row['name'] = newname\n            r = conn.execute(self.db.model.steps.insert(), insert_row)\n            got_id = r.inserted_primary_key[0]\n            return (got_id, number, newname)\n        return self.db.pool.do(thd)\n\n    def startStep(self, stepid, _reactor=reactor):\n        started_at = _reactor.seconds()\n\n        def thd(conn):\n            tbl = self.db.model.steps\n            q = tbl.update(whereclause=(tbl.c.id == stepid))\n            conn.execute(q, started_at=started_at)\n        return self.db.pool.do(thd)\n\n    def setStepStateString(self, stepid, state_string):\n        def thd(conn):\n            tbl = self.db.model.steps\n            q = tbl.update(whereclause=(tbl.c.id == stepid))\n            conn.execute(q, state_string=state_string)\n        return self.db.pool.do(thd)\n\n    def addURL(self, stepid, name, url, _racehook=None):\n        # This methods adds an URL to the db\n        # This is a read modify write and thus there is a possibility\n        # that several urls are added at the same time (e.g with a deferredlist\n        # at the end of a step)\n        # this race condition is only inside the same master, as only one master\n        # is supposed to add urls to a buildstep.\n        # so threading.lock is used, as we are in the thread pool\n        if self.url_lock is None:\n            # this runs in reactor thread, so no race here..\n            self.url_lock = defer.DeferredLock()\n\n        def thd(conn):\n\n            tbl = self.db.model.steps\n            wc = (tbl.c.id == stepid)\n            q = sa.select([tbl.c.urls_json],\n                          whereclause=wc)\n            res = conn.execute(q)\n            row = res.fetchone()\n            if _racehook is not None:\n                _racehook()\n            urls = json.loads(row.urls_json)\n            urls.append(dict(name=name, url=url))\n\n            q = tbl.update(whereclause=wc)\n            conn.execute(q, urls_json=json.dumps(urls))\n\n        return self.url_lock.run(lambda: self.db.pool.do(thd))\n\n    def finishStep(self, stepid, results, hidden, _reactor=reactor):\n        def thd(conn):\n            tbl = self.db.model.steps\n            q = tbl.update(whereclause=(tbl.c.id == stepid))\n            conn.execute(q,\n                         complete_at=_reactor.seconds(),\n                         results=results,\n                         hidden=1 if hidden else 0)\n        return self.db.pool.do(thd)\n\n    def _stepdictFromRow(self, row):\n        def mkdt(epoch):\n            if epoch:\n                return epoch2datetime(epoch)\n\n        return dict(\n            id=row.id,\n            number=row.number,\n            name=row.name,\n            buildid=row.buildid,\n            started_at=mkdt(row.started_at),\n            complete_at=mkdt(row.complete_at),\n            state_string=row.state_string,\n            results=row.results,\n            urls=json.loads(row.urls_json),\n            hidden=bool(row.hidden))\n", "path": "master/buildbot/db/steps.py"}], "after_files": [{"content": "# This file is part of Buildbot.  Buildbot is free software: you can\n# redistribute it and/or modify it under the terms of the GNU General Public\n# License as published by the Free Software Foundation, version 2.\n#\n# This program is distributed in the hope that it will be useful, but WITHOUT\n# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS\n# FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more\n# details.\n#\n# You should have received a copy of the GNU General Public License along with\n# this program; if not, write to the Free Software Foundation, Inc., 51\n# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.\n#\n# Copyright Buildbot Team Members\n\nfrom __future__ import absolute_import\nfrom __future__ import print_function\n\nimport json\n\nimport sqlalchemy as sa\n\nfrom twisted.internet import defer\nfrom twisted.internet import reactor\n\nfrom buildbot.db import base\nfrom buildbot.util import epoch2datetime\n\n\nclass StepsConnectorComponent(base.DBConnectorComponent):\n    # Documentation is in developer/db.rst\n    url_lock = None\n\n    def getStep(self, stepid=None, buildid=None, number=None, name=None):\n        tbl = self.db.model.steps\n        if stepid is not None:\n            wc = (tbl.c.id == stepid)\n        else:\n            if buildid is None:\n                return defer.fail(RuntimeError('must supply either stepid or buildid'))\n            if number is not None:\n                wc = (tbl.c.number == number)\n            elif name is not None:\n                wc = (tbl.c.name == name)\n            else:\n                return defer.fail(RuntimeError('must supply either number or name'))\n            wc = wc & (tbl.c.buildid == buildid)\n\n        def thd(conn):\n            q = self.db.model.steps.select(whereclause=wc)\n            res = conn.execute(q)\n            row = res.fetchone()\n\n            rv = None\n            if row:\n                rv = self._stepdictFromRow(row)\n            res.close()\n            return rv\n        return self.db.pool.do(thd)\n\n    def getSteps(self, buildid):\n        def thd(conn):\n            tbl = self.db.model.steps\n            q = tbl.select()\n            q = q.where(tbl.c.buildid == buildid)\n            q = q.order_by(tbl.c.number)\n            res = conn.execute(q)\n            return [self._stepdictFromRow(row) for row in res.fetchall()]\n        return self.db.pool.do(thd)\n\n    def addStep(self, buildid, name, state_string):\n        def thd(conn):\n            tbl = self.db.model.steps\n            # get the highest current number\n            r = conn.execute(sa.select([sa.func.max(tbl.c.number)],\n                                       whereclause=(tbl.c.buildid == buildid)))\n            number = r.scalar()\n            number = 0 if number is None else number + 1\n\n            # note that there is no chance for a race condition here,\n            # since only one master is inserting steps.  If there is a\n            # conflict, then the name is likely already taken.\n            insert_row = dict(buildid=buildid, number=number,\n                              started_at=None, complete_at=None,\n                              state_string=state_string,\n                              urls_json='[]', name=name)\n            try:\n                r = conn.execute(self.db.model.steps.insert(), insert_row)\n                got_id = r.inserted_primary_key[0]\n            except (sa.exc.IntegrityError, sa.exc.ProgrammingError):\n                got_id = None\n\n            if got_id:\n                return (got_id, number, name)\n\n            # we didn't get an id, so calculate a unique name and use that\n            # instead.  Because names are truncated at the right to fit in a\n            # 50-character identifier, this isn't a simple query.\n            res = conn.execute(sa.select([tbl.c.name],\n                                         whereclause=((tbl.c.buildid == buildid))))\n            names = set([row[0] for row in res])\n            num = 1\n            while True:\n                numstr = '_%d' % num\n                newname = name[:50 - len(numstr)] + numstr\n                if newname not in names:\n                    break\n                num += 1\n            insert_row['name'] = newname\n            r = conn.execute(self.db.model.steps.insert(), insert_row)\n            got_id = r.inserted_primary_key[0]\n            return (got_id, number, newname)\n        return self.db.pool.do(thd)\n\n    def startStep(self, stepid, _reactor=reactor):\n        started_at = _reactor.seconds()\n\n        def thd(conn):\n            tbl = self.db.model.steps\n            q = tbl.update(whereclause=(tbl.c.id == stepid))\n            conn.execute(q, started_at=started_at)\n        return self.db.pool.do(thd)\n\n    def setStepStateString(self, stepid, state_string):\n        def thd(conn):\n            tbl = self.db.model.steps\n            q = tbl.update(whereclause=(tbl.c.id == stepid))\n            conn.execute(q, state_string=state_string)\n        return self.db.pool.do(thd)\n\n    def addURL(self, stepid, name, url, _racehook=None):\n        # This methods adds an URL to the db\n        # This is a read modify write and thus there is a possibility\n        # that several urls are added at the same time (e.g with a deferredlist\n        # at the end of a step)\n        # this race condition is only inside the same master, as only one master\n        # is supposed to add urls to a buildstep.\n        # so threading.lock is used, as we are in the thread pool\n        if self.url_lock is None:\n            # this runs in reactor thread, so no race here..\n            self.url_lock = defer.DeferredLock()\n\n        def thd(conn):\n\n            tbl = self.db.model.steps\n            wc = (tbl.c.id == stepid)\n            q = sa.select([tbl.c.urls_json],\n                          whereclause=wc)\n            res = conn.execute(q)\n            row = res.fetchone()\n            if _racehook is not None:\n                _racehook()\n            urls = json.loads(row.urls_json)\n\n            url_item = dict(name=name, url=url)\n\n            if url_item not in urls:\n                urls.append(url_item)\n                q = tbl.update(whereclause=wc)\n                conn.execute(q, urls_json=json.dumps(urls))\n\n        return self.url_lock.run(lambda: self.db.pool.do(thd))\n\n    def finishStep(self, stepid, results, hidden, _reactor=reactor):\n        def thd(conn):\n            tbl = self.db.model.steps\n            q = tbl.update(whereclause=(tbl.c.id == stepid))\n            conn.execute(q,\n                         complete_at=_reactor.seconds(),\n                         results=results,\n                         hidden=1 if hidden else 0)\n        return self.db.pool.do(thd)\n\n    def _stepdictFromRow(self, row):\n        def mkdt(epoch):\n            if epoch:\n                return epoch2datetime(epoch)\n\n        return dict(\n            id=row.id,\n            number=row.number,\n            name=row.name,\n            buildid=row.buildid,\n            started_at=mkdt(row.started_at),\n            complete_at=mkdt(row.complete_at),\n            state_string=row.state_string,\n            results=row.results,\n            urls=json.loads(row.urls_json),\n            hidden=bool(row.hidden))\n", "path": "master/buildbot/db/steps.py"}]}