prompt: We are currently solving the following issue within our repository. Here is the issue text:
  --- BEGIN ISSUE ---
  Updates sometimes get stuck in pending state
  From quick look, it seems that the composer does:
  1. move from `f*-updates-candidate` to `f*-updates-testing`
  2. do stuff
  3. untag from `f*-updates-testing-pending`
  4. mark update state as testing
  
  If the composer hang on 2 the update remains stuck in pending as the builds are not tagged anymore in `f*-updates-candidate`. We should find a solution.
  
  --- END ISSUE ---
  
  Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
  --- BEGIN FILES ---
  Path: `bodhi-server/bodhi/server/tasks/check_signed_builds.py`
  Content:
  ```
  1 # Copyright © 2017 Red Hat, Inc.
  2 #
  3 # This file is part of Bodhi.
  4 #
  5 # This program is free software; you can redistribute it and/or
  6 # modify it under the terms of the GNU General Public License
  7 # as published by the Free Software Foundation; either version 2
  8 # of the License, or (at your option) any later version.
  9 #
  10 # This program is distributed in the hope that it will be useful,
  11 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  12 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  13 # GNU General Public License for more details.
  14 #
  15 # You should have received a copy of the GNU General Public License
  16 # along with this program; if not, write to the Free Software
  17 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
  18 
  19 """
  20 Avoid Updates being stuck in pending.
  21 
  22 It may happen that Bodhi misses fedora-messaging messages announcing builds
  23 have been signed.
  24 In these cases, the Update remain stuck in pending until a manual intervention.
  25 
  26 This script will cycle through builds of Updates in pending status and update
  27 the signed status in the db to match the tags found in Koji.
  28 """
  29 
  30 import logging
  31 from datetime import datetime, timedelta
  32 
  33 from bodhi.server import buildsys, models
  34 from bodhi.server.config import config
  35 from bodhi.server.util import transactional_session_maker
  36 
  37 
  38 log = logging.getLogger(__name__)
  39 
  40 
  41 def main():
  42     """Check build tags and sign those we missed."""
  43     db_factory = transactional_session_maker()
  44     older_than = datetime.utcnow() - timedelta(days=config.get('check_signed_builds_delay'))
  45     with db_factory() as session:
  46         updates = models.Update.query.filter(
  47             models.Update.status == models.UpdateStatus.pending
  48         ).filter(
  49             models.Update.release_id == models.Release.id
  50         ).filter(
  51             models.Release.state.in_([
  52                 models.ReleaseState.current,
  53                 models.ReleaseState.pending,
  54                 models.ReleaseState.frozen,
  55             ])
  56         ).all()
  57 
  58         if len(updates) == 0:
  59             log.debug('No stuck Updates found')
  60             return
  61 
  62         kc = buildsys.get_session()
  63         stuck_builds = []
  64         overlooked_builds = []
  65 
  66         for update in updates:
  67             # Let Bodhi have its times
  68             if update.date_submitted >= older_than:
  69                 continue
  70             builds = update.builds
  71             # Clean Updates with no builds
  72             if len(builds) == 0:
  73                 log.debug(f'Obsoleting empty update {update.alias}')
  74                 update.obsolete(session)
  75                 session.flush()
  76                 continue
  77             pending_signing_tag = update.release.pending_signing_tag
  78             pending_testing_tag = update.release.pending_testing_tag
  79             for build in builds:
  80                 if build.signed:
  81                     log.debug(f'{build.nvr} already marked as signed')
  82                     continue
  83                 build_tags = [t['name'] for t in kc.listTags(build=build.nvr)]
  84                 if pending_signing_tag not in build_tags and pending_testing_tag in build_tags:
  85                     # Our composer missed the message that the build got signed
  86                     log.debug(f'Changing signed status of {build.nvr}')
  87                     build.signed = True
  88                 elif pending_signing_tag in build_tags and pending_testing_tag not in build_tags:
  89                     # autosign missed the message that the build is waiting to be signed
  90                     log.debug(f'{build.nvr} is stuck waiting to be signed, let\'s try again')
  91                     stuck_builds.append((build.nvr, pending_signing_tag))
  92                 elif (pending_signing_tag not in build_tags
  93                       and pending_testing_tag not in build_tags):
  94                     # this means that an update has been created but we never tagged the build
  95                     # as pending-signing
  96                     log.debug(f'Oh, no! We\'ve never sent {build.nvr} for signing, let\'s fix it')
  97                     overlooked_builds.append((build.nvr, pending_signing_tag))
  98             session.flush()
  99 
  100         if stuck_builds:
  101             kc.multicall = True
  102             for b, t in stuck_builds:
  103                 kc.untagBuild(t, b, force=True)
  104             kc.multiCall()
  105             for b, t in stuck_builds:
  106                 kc.tagBuild(t, b, force=True)
  107             kc.multiCall()
  108 
  109         if overlooked_builds:
  110             kc.multicall = True
  111             for b, t in overlooked_builds:
  112                 kc.tagBuild(t, b, force=True)
  113             kc.multiCall()
  114
  ```
  --- END FILES ---
  
  Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.
  
  Here is an example:
  
  ```diff
  diff --git a/examples/server_async.py b/examples/server_async.py
  --- a/examples/server_async.py
  +++ b/examples/server_async.py
  @@ -313,4 +313,4 @@
   
   
   if __name__ == "__main__":
  -    asyncio.run(run_async_server("."), debug=True)
  +    asyncio.run(run_async_server(), debug=True)
  diff --git a/examples/server_sync.py b/examples/server_sync.py
  --- a/examples/server_sync.py
  +++ b/examples/server_sync.py
  @@ -313,5 +313,5 @@
   
   
   if __name__ == "__main__":
  -    server = run_sync_server(".")
  +    server = run_sync_server()
       server.shutdown()
  
  ```
  
golden_diff: diff --git a/bodhi-server/bodhi/server/tasks/check_signed_builds.py b/bodhi-server/bodhi/server/tasks/check_signed_builds.py
  --- a/bodhi-server/bodhi/server/tasks/check_signed_builds.py
  +++ b/bodhi-server/bodhi/server/tasks/check_signed_builds.py
  @@ -77,10 +77,16 @@
               pending_signing_tag = update.release.pending_signing_tag
               pending_testing_tag = update.release.pending_testing_tag
               for build in builds:
  +                build_tags = [t['name'] for t in kc.listTags(build=build.nvr)]
                   if build.signed:
                       log.debug(f'{build.nvr} already marked as signed')
  +                    if (update.release.testing_tag in build_tags
  +                            and update.release.candidate_tag not in build_tags):
  +                        # The update was probably ejected from a compose and is stuck
  +                        log.debug(f'Resubmitting {update.alias} to testing')
  +                        update.set_request(session, models.UpdateRequest.testing, 'bodhi')
  +                        break
                       continue
  -                build_tags = [t['name'] for t in kc.listTags(build=build.nvr)]
                   if pending_signing_tag not in build_tags and pending_testing_tag in build_tags:
                       # Our composer missed the message that the build got signed
                       log.debug(f'Changing signed status of {build.nvr}')
  
verification_info: {"golden_diff": "diff --git a/bodhi-server/bodhi/server/tasks/check_signed_builds.py b/bodhi-server/bodhi/server/tasks/check_signed_builds.py\n--- a/bodhi-server/bodhi/server/tasks/check_signed_builds.py\n+++ b/bodhi-server/bodhi/server/tasks/check_signed_builds.py\n@@ -77,10 +77,16 @@\n             pending_signing_tag = update.release.pending_signing_tag\n             pending_testing_tag = update.release.pending_testing_tag\n             for build in builds:\n+                build_tags = [t['name'] for t in kc.listTags(build=build.nvr)]\n                 if build.signed:\n                     log.debug(f'{build.nvr} already marked as signed')\n+                    if (update.release.testing_tag in build_tags\n+                            and update.release.candidate_tag not in build_tags):\n+                        # The update was probably ejected from a compose and is stuck\n+                        log.debug(f'Resubmitting {update.alias} to testing')\n+                        update.set_request(session, models.UpdateRequest.testing, 'bodhi')\n+                        break\n                     continue\n-                build_tags = [t['name'] for t in kc.listTags(build=build.nvr)]\n                 if pending_signing_tag not in build_tags and pending_testing_tag in build_tags:\n                     # Our composer missed the message that the build got signed\n                     log.debug(f'Changing signed status of {build.nvr}')\n", "issue": "Updates sometimes get stuck in pending state\nFrom quick look, it seems that the composer does:\r\n1. move from `f*-updates-candidate` to `f*-updates-testing`\r\n2. do stuff\r\n3. untag from `f*-updates-testing-pending`\r\n4. mark update state as testing\r\n\r\nIf the composer hang on 2 the update remains stuck in pending as the builds are not tagged anymore in `f*-updates-candidate`. We should find a solution.\n", "before_files": [{"content": "# Copyright \u00a9 2017 Red Hat, Inc.\n#\n# This file is part of Bodhi.\n#\n# This program is free software; you can redistribute it and/or\n# modify it under the terms of the GNU General Public License\n# as published by the Free Software Foundation; either version 2\n# of the License, or (at your option) any later version.\n#\n# This program is distributed in the hope that it will be useful,\n# but WITHOUT ANY WARRANTY; without even the implied warranty of\n# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the\n# GNU General Public License for more details.\n#\n# You should have received a copy of the GNU General Public License\n# along with this program; if not, write to the Free Software\n# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.\n\n\"\"\"\nAvoid Updates being stuck in pending.\n\nIt may happen that Bodhi misses fedora-messaging messages announcing builds\nhave been signed.\nIn these cases, the Update remain stuck in pending until a manual intervention.\n\nThis script will cycle through builds of Updates in pending status and update\nthe signed status in the db to match the tags found in Koji.\n\"\"\"\n\nimport logging\nfrom datetime import datetime, timedelta\n\nfrom bodhi.server import buildsys, models\nfrom bodhi.server.config import config\nfrom bodhi.server.util import transactional_session_maker\n\n\nlog = logging.getLogger(__name__)\n\n\ndef main():\n    \"\"\"Check build tags and sign those we missed.\"\"\"\n    db_factory = transactional_session_maker()\n    older_than = datetime.utcnow() - timedelta(days=config.get('check_signed_builds_delay'))\n    with db_factory() as session:\n        updates = models.Update.query.filter(\n            models.Update.status == models.UpdateStatus.pending\n        ).filter(\n            models.Update.release_id == models.Release.id\n        ).filter(\n            models.Release.state.in_([\n                models.ReleaseState.current,\n                models.ReleaseState.pending,\n                models.ReleaseState.frozen,\n            ])\n        ).all()\n\n        if len(updates) == 0:\n            log.debug('No stuck Updates found')\n            return\n\n        kc = buildsys.get_session()\n        stuck_builds = []\n        overlooked_builds = []\n\n        for update in updates:\n            # Let Bodhi have its times\n            if update.date_submitted >= older_than:\n                continue\n            builds = update.builds\n            # Clean Updates with no builds\n            if len(builds) == 0:\n                log.debug(f'Obsoleting empty update {update.alias}')\n                update.obsolete(session)\n                session.flush()\n                continue\n            pending_signing_tag = update.release.pending_signing_tag\n            pending_testing_tag = update.release.pending_testing_tag\n            for build in builds:\n                if build.signed:\n                    log.debug(f'{build.nvr} already marked as signed')\n                    continue\n                build_tags = [t['name'] for t in kc.listTags(build=build.nvr)]\n                if pending_signing_tag not in build_tags and pending_testing_tag in build_tags:\n                    # Our composer missed the message that the build got signed\n                    log.debug(f'Changing signed status of {build.nvr}')\n                    build.signed = True\n                elif pending_signing_tag in build_tags and pending_testing_tag not in build_tags:\n                    # autosign missed the message that the build is waiting to be signed\n                    log.debug(f'{build.nvr} is stuck waiting to be signed, let\\'s try again')\n                    stuck_builds.append((build.nvr, pending_signing_tag))\n                elif (pending_signing_tag not in build_tags\n                      and pending_testing_tag not in build_tags):\n                    # this means that an update has been created but we never tagged the build\n                    # as pending-signing\n                    log.debug(f'Oh, no! We\\'ve never sent {build.nvr} for signing, let\\'s fix it')\n                    overlooked_builds.append((build.nvr, pending_signing_tag))\n            session.flush()\n\n        if stuck_builds:\n            kc.multicall = True\n            for b, t in stuck_builds:\n                kc.untagBuild(t, b, force=True)\n            kc.multiCall()\n            for b, t in stuck_builds:\n                kc.tagBuild(t, b, force=True)\n            kc.multiCall()\n\n        if overlooked_builds:\n            kc.multicall = True\n            for b, t in overlooked_builds:\n                kc.tagBuild(t, b, force=True)\n            kc.multiCall()\n", "path": "bodhi-server/bodhi/server/tasks/check_signed_builds.py"}], "after_files": [{"content": "# Copyright \u00a9 2017 Red Hat, Inc.\n#\n# This file is part of Bodhi.\n#\n# This program is free software; you can redistribute it and/or\n# modify it under the terms of the GNU General Public License\n# as published by the Free Software Foundation; either version 2\n# of the License, or (at your option) any later version.\n#\n# This program is distributed in the hope that it will be useful,\n# but WITHOUT ANY WARRANTY; without even the implied warranty of\n# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the\n# GNU General Public License for more details.\n#\n# You should have received a copy of the GNU General Public License\n# along with this program; if not, write to the Free Software\n# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.\n\n\"\"\"\nAvoid Updates being stuck in pending.\n\nIt may happen that Bodhi misses fedora-messaging messages announcing builds\nhave been signed.\nIn these cases, the Update remain stuck in pending until a manual intervention.\n\nThis script will cycle through builds of Updates in pending status and update\nthe signed status in the db to match the tags found in Koji.\n\"\"\"\n\nimport logging\nfrom datetime import datetime, timedelta\n\nfrom bodhi.server import buildsys, models\nfrom bodhi.server.config import config\nfrom bodhi.server.util import transactional_session_maker\n\n\nlog = logging.getLogger(__name__)\n\n\ndef main():\n    \"\"\"Check build tags and sign those we missed.\"\"\"\n    db_factory = transactional_session_maker()\n    older_than = datetime.utcnow() - timedelta(days=config.get('check_signed_builds_delay'))\n    with db_factory() as session:\n        updates = models.Update.query.filter(\n            models.Update.status == models.UpdateStatus.pending\n        ).filter(\n            models.Update.release_id == models.Release.id\n        ).filter(\n            models.Release.state.in_([\n                models.ReleaseState.current,\n                models.ReleaseState.pending,\n                models.ReleaseState.frozen,\n            ])\n        ).all()\n\n        if len(updates) == 0:\n            log.debug('No stuck Updates found')\n            return\n\n        kc = buildsys.get_session()\n        stuck_builds = []\n        overlooked_builds = []\n\n        for update in updates:\n            # Let Bodhi have its times\n            if update.date_submitted >= older_than:\n                continue\n            builds = update.builds\n            # Clean Updates with no builds\n            if len(builds) == 0:\n                log.debug(f'Obsoleting empty update {update.alias}')\n                update.obsolete(session)\n                session.flush()\n                continue\n            pending_signing_tag = update.release.pending_signing_tag\n            pending_testing_tag = update.release.pending_testing_tag\n            for build in builds:\n                build_tags = [t['name'] for t in kc.listTags(build=build.nvr)]\n                if build.signed:\n                    log.debug(f'{build.nvr} already marked as signed')\n                    if (update.release.testing_tag in build_tags\n                            and update.release.candidate_tag not in build_tags):\n                        # The update was probably ejected from a compose and is stuck\n                        log.debug(f'Resubmitting {update.alias} to testing')\n                        update.set_request(session, models.UpdateRequest.testing, 'bodhi')\n                        break\n                    continue\n                if pending_signing_tag not in build_tags and pending_testing_tag in build_tags:\n                    # Our composer missed the message that the build got signed\n                    log.debug(f'Changing signed status of {build.nvr}')\n                    build.signed = True\n                elif pending_signing_tag in build_tags and pending_testing_tag not in build_tags:\n                    # autosign missed the message that the build is waiting to be signed\n                    log.debug(f'{build.nvr} is stuck waiting to be signed, let\\'s try again')\n                    stuck_builds.append((build.nvr, pending_signing_tag))\n                elif (pending_signing_tag not in build_tags\n                      and pending_testing_tag not in build_tags):\n                    # this means that an update has been created but we never tagged the build\n                    # as pending-signing\n                    log.debug(f'Oh, no! We\\'ve never sent {build.nvr} for signing, let\\'s fix it')\n                    overlooked_builds.append((build.nvr, pending_signing_tag))\n            session.flush()\n\n        if stuck_builds:\n            kc.multicall = True\n            for b, t in stuck_builds:\n                kc.untagBuild(t, b, force=True)\n            kc.multiCall()\n            for b, t in stuck_builds:\n                kc.tagBuild(t, b, force=True)\n            kc.multiCall()\n\n        if overlooked_builds:\n            kc.multicall = True\n            for b, t in overlooked_builds:\n                kc.tagBuild(t, b, force=True)\n            kc.multiCall()\n", "path": "bodhi-server/bodhi/server/tasks/check_signed_builds.py"}]}