We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
epel5 needs more createrepo_c compatibility for security plugin
From: https://bugzilla.redhat.com/show_bug.cgi?id=1256336

The rhel5 yum security plugin is unable to deal with the updateinfo being .bz2. It wants it to be .gz apparently. 

Not sure if this is something we need to change in createrepo_c, or in bodhi2 when it injects the security info in.


--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `bodhi/metadata.py`
Content:
```
1 # This program is free software; you can redistribute it and/or modify
2 # it under the terms of the GNU General Public License as published by
3 # the Free Software Foundation; either version 2 of the License, or
4 # (at your option) any later version.
5 #
6 # This program is distributed in the hope that it will be useful,
7 # but WITHOUT ANY WARRANTY; without even the implied warranty of
8 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
9 # GNU General Public License for more details.
10 #
11 # You should have received a copy of the GNU General Public License along
12 # with this program; if not, write to the Free Software Foundation, Inc.,
13 # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
14 
15 __version__ = '2.0'
16 
17 import os
18 import logging
19 import shutil
20 import tempfile
21 
22 from urlgrabber.grabber import urlgrab
23 from kitchen.text.converters import to_bytes
24 
25 import createrepo_c as cr
26 
27 from bodhi.config import config
28 from bodhi.models import Build, UpdateStatus, UpdateRequest, UpdateSuggestion
29 from bodhi.buildsys import get_session
30 
31 log = logging.getLogger(__name__)
32 
33 
34 class ExtendedMetadata(object):
35     """This class represents the updateinfo.xml yum metadata.
36 
37     It is generated during push time by the bodhi masher based on koji tags
38     and is injected into the yum repodata using the `modifyrepo_c` tool,
39     which is included in the `createrepo_c` package.
40 
41     """
42     def __init__(self, release, request, db, path):
43         self.repo = path
44         log.debug('repo = %r' % self.repo)
45         self.request = request
46         if request is UpdateRequest.stable:
47             self.tag = release.stable_tag
48         else:
49             self.tag = release.testing_tag
50         self.repo_path = os.path.join(self.repo, self.tag)
51 
52         self.db = db
53         self.updates = set()
54         self.builds = {}
55         self.missing_ids = []
56         self._from = config.get('bodhi_email')
57         self.koji = get_session()
58         self._fetch_updates()
59 
60         self.uinfo = cr.UpdateInfo()
61 
62         self.hash_type = cr.SHA256
63         self.comp_type = cr.XZ
64 
65         if release.id_prefix == u'FEDORA-EPEL':
66             # yum on py2.4 doesn't support sha256 (#1080373)
67             if 'el5' in self.repo or '5E' in self.repo:
68                 self.hash_type = cr.SHA1
69 
70             # FIXME: I'm not sure which versions of RHEL support xz metadata
71             # compression, so use the lowest common denominator for now.
72             self.comp_type = cr.BZ2
73 
74         # Load from the cache if it exists
75         self.cached_repodata = os.path.join(self.repo, '..', self.tag +
76                                             '.repocache', 'repodata/')
77         if os.path.isdir(self.cached_repodata):
78             self._load_cached_updateinfo()
79         else:
80             log.debug("Generating new updateinfo.xml")
81             self.uinfo = cr.UpdateInfo()
82             for update in self.updates:
83                 if update.alias:
84                     self.add_update(update)
85                 else:
86                     self.missing_ids.append(update.title)
87 
88         if self.missing_ids:
89             log.error("%d updates with missing ID!" % len(self.missing_ids))
90             log.error(self.missing_ids)
91 
92     def _load_cached_updateinfo(self):
93         """
94         Load the cached updateinfo.xml from '../{tag}.repocache/repodata'
95         """
96         seen_ids = set()
97         from_cache = set()
98         existing_ids = set()
99 
100         # Parse the updateinfo out of the repomd
101         updateinfo = None
102         repomd_xml = os.path.join(self.cached_repodata, 'repomd.xml')
103         repomd = cr.Repomd()
104         cr.xml_parse_repomd(repomd_xml, repomd)
105         for record in repomd.records:
106             if record.type == 'updateinfo':
107                 updateinfo = os.path.join(os.path.dirname(
108                     os.path.dirname(self.cached_repodata)),
109                     record.location_href)
110                 break
111 
112         assert updateinfo, 'Unable to find updateinfo'
113 
114         # Load the metadata with createrepo_c
115         log.info('Loading cached updateinfo: %s', updateinfo)
116         uinfo = cr.UpdateInfo(updateinfo)
117 
118         # Determine which updates are present in the cache
119         for update in uinfo.updates:
120             existing_ids.add(update.id)
121 
122         # Generate metadata for any new builds
123         for update in self.updates:
124             seen_ids.add(update.alias)
125             if not update.alias:
126                 self.missing_ids.append(update.title)
127                 continue
128             if update.alias in existing_ids:
129                 notice = None
130                 for value in uinfo.updates:
131                     if value.title == update.title:
132                         notice = value
133                         break
134                 if not notice:
135                     log.warn('%s ID in cache but notice cannot be found', update.title)
136                     self.add_update(update)
137                     continue
138                 if notice.updated_date:
139                     if notice.updated_date < update.date_modified:
140                         log.debug('Update modified, generating new notice: %s' % update.title)
141                         self.add_update(update)
142                     else:
143                         log.debug('Loading updated %s from cache' % update.title)
144                         from_cache.add(update.alias)
145                 elif update.date_modified:
146                     log.debug('Update modified, generating new notice: %s' % update.title)
147                     self.add_update(update)
148                 else:
149                     log.debug('Loading %s from cache' % update.title)
150                     from_cache.add(update.alias)
151             else:
152                 log.debug('Adding new update notice: %s' % update.title)
153                 self.add_update(update)
154 
155         # Add all relevant notices from the cache to this document
156         for notice in uinfo.updates:
157             if notice.id in from_cache:
158                 log.debug('Keeping existing notice: %s', notice.title)
159                 self.uinfo.append(notice)
160             else:
161                 # Keep all security notices in the stable repo
162                 if self.request is not UpdateRequest.testing:
163                     if notice.type == 'security':
164                         if notice.id not in seen_ids:
165                             log.debug('Keeping existing security notice: %s',
166                                       notice.title)
167                             self.uinfo.append(notice)
168                         else:
169                             log.debug('%s already added?', notice.title)
170                     else:
171                         log.debug('Purging cached stable notice %s', notice.title)
172                 else:
173                     log.debug('Purging cached testing update %s', notice.title)
174 
175     def _fetch_updates(self):
176         """Based on our given koji tag, populate a list of Update objects"""
177         log.debug("Fetching builds tagged with '%s'" % self.tag)
178         kojiBuilds = self.koji.listTagged(self.tag, latest=True)
179         nonexistent = []
180         log.debug("%d builds found" % len(kojiBuilds))
181         for build in kojiBuilds:
182             self.builds[build['nvr']] = build
183             build_obj = self.db.query(Build).filter_by(nvr=unicode(build['nvr'])).first()
184             if build_obj:
185                 self.updates.add(build_obj.update)
186             else:
187                 nonexistent.append(build['nvr'])
188         if nonexistent:
189             log.warning("Couldn't find the following koji builds tagged as "
190                         "%s in bodhi: %s" % (self.tag, nonexistent))
191 
192     def add_update(self, update):
193         """Generate the extended metadata for a given update"""
194         rec = cr.UpdateRecord()
195         rec.version = __version__
196         rec.fromstr = config.get('bodhi_email')
197         rec.status = update.status.value
198         rec.type = update.type.value
199         rec.id = to_bytes(update.alias)
200         rec.title = to_bytes(update.title)
201         rec.summary = to_bytes('%s %s update' % (update.get_title(),
202                                                  update.type.value))
203         rec.description = to_bytes(update.notes)
204         rec.release = to_bytes(update.release.long_name)
205         rec.rights = config.get('updateinfo_rights')
206 
207         if update.date_pushed:
208             rec.issued_date = update.date_pushed
209         if update.date_modified:
210             rec.updated_date = update.date_modified
211 
212         col = cr.UpdateCollection()
213         col.name = to_bytes(update.release.long_name)
214         col.shortname = to_bytes(update.release.name)
215 
216         for build in update.builds:
217             try:
218                 kojiBuild = self.builds[build.nvr]
219             except:
220                 kojiBuild = self.koji.getBuild(build.nvr)
221 
222             rpms = self.koji.listBuildRPMs(kojiBuild['id'])
223             for rpm in rpms:
224                 pkg = cr.UpdateCollectionPackage()
225                 pkg.name = rpm['name']
226                 pkg.version = rpm['version']
227                 pkg.release = rpm['release']
228                 if rpm['epoch'] is not None:
229                     pkg.epoch = str(rpm['epoch'])
230                 else:
231                     pkg.epoch = '0'
232                 pkg.arch = rpm['arch']
233 
234                 # TODO: how do we handle UpdateSuggestion.logout, etc?
235                 pkg.reboot_suggested = update.suggest is UpdateSuggestion.reboot
236 
237                 filename = '%s.%s.rpm' % (rpm['nvr'], rpm['arch'])
238                 pkg.filename = filename
239 
240                 # Build the URL
241                 if rpm['arch'] == 'src':
242                     arch = 'SRPMS'
243                 elif rpm['arch'] in ('noarch', 'i686'):
244                     arch = 'i386'
245                 else:
246                     arch = rpm['arch']
247 
248                 pkg.src = os.path.join(config.get('file_url'), update.status is
249                         UpdateStatus.testing and 'testing' or '',
250                         str(update.release.version), arch, filename[0], filename)
251 
252                 col.append(pkg)
253 
254         rec.append_collection(col)
255 
256         # Create references for each bug
257         for bug in update.bugs:
258             ref = cr.UpdateReference()
259             ref.type = 'bugzilla'
260             ref.id = to_bytes(bug.bug_id)
261             ref.href = to_bytes(bug.url)
262             ref.title = to_bytes(bug.title)
263             rec.append_reference(ref)
264 
265         # Create references for each CVE
266         for cve in update.cves:
267             ref = cr.UpdateReference()
268             ref.type = 'cve'
269             ref.id = to_bytes(cve.cve_id)
270             ref.href = to_bytes(cve.url)
271             rec.append_reference(ref)
272 
273         self.uinfo.append(rec)
274 
275     def insert_updateinfo(self):
276         fd, name = tempfile.mkstemp()
277         os.write(fd, self.uinfo.xml_dump())
278         os.close(fd)
279         self.modifyrepo(name)
280         os.unlink(name)
281 
282     def modifyrepo(self, filename):
283         """Inject a file into the repodata for each architecture"""
284         for arch in os.listdir(self.repo_path):
285             repodata = os.path.join(self.repo_path, arch, 'repodata')
286             log.info('Inserting %s into %s', filename, repodata)
287             uinfo_xml = os.path.join(repodata, 'updateinfo.xml')
288             shutil.copyfile(filename, uinfo_xml)
289             repomd_xml = os.path.join(repodata, 'repomd.xml')
290             repomd = cr.Repomd(repomd_xml)
291             uinfo_rec = cr.RepomdRecord('updateinfo', uinfo_xml)
292             uinfo_rec_comp = uinfo_rec.compress_and_fill(self.hash_type, self.comp_type)
293             uinfo_rec_comp.rename_file()
294             uinfo_rec_comp.type = 'updateinfo'
295             repomd.set_record(uinfo_rec_comp)
296             with file(repomd_xml, 'w') as repomd_file:
297                 repomd_file.write(repomd.xml_dump())
298             os.unlink(uinfo_xml)
299 
300     def insert_pkgtags(self):
301         """Download and inject the pkgtags sqlite from fedora-tagger"""
302         if config.get('pkgtags_url'):
303             try:
304                 tags_url = config.get('pkgtags_url')
305                 tempdir = tempfile.mkdtemp('bodhi')
306                 local_tags = os.path.join(tempdir, 'pkgtags.sqlite')
307                 log.info('Downloading %s' % tags_url)
308                 urlgrab(tags_url, filename=local_tags)
309                 self.modifyrepo(local_tags)
310             except:
311                 log.exception("There was a problem injecting pkgtags")
312             finally:
313                 shutil.rmtree(tempdir)
314 
315     def cache_repodata(self):
316         arch = os.listdir(self.repo_path)[0]  # Take the first arch
317         repodata = os.path.join(self.repo_path, arch, 'repodata')
318         if not os.path.isdir(repodata):
319             log.warning('Cannot find repodata to cache: %s' % repodata)
320             return
321         cache = self.cached_repodata
322         if os.path.isdir(cache):
323             shutil.rmtree(cache)
324         shutil.copytree(repodata, cache)
325         log.info('%s cached to %s' % (repodata, cache))
326
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/bodhi/metadata.py b/bodhi/metadata.py
--- a/bodhi/metadata.py
+++ b/bodhi/metadata.py
@@ -66,10 +66,11 @@
             # yum on py2.4 doesn't support sha256 (#1080373)
             if 'el5' in self.repo or '5E' in self.repo:
                 self.hash_type = cr.SHA1
-
-            # FIXME: I'm not sure which versions of RHEL support xz metadata
-            # compression, so use the lowest common denominator for now.
-            self.comp_type = cr.BZ2
+                self.comp_type = cr.GZ
+            else:
+                # FIXME: I'm not sure which versions of RHEL support xz metadata
+                # compression, so use the lowest common denominator for now.
+                self.comp_type = cr.BZ2
 
         # Load from the cache if it exists
         self.cached_repodata = os.path.join(self.repo, '..', self.tag +

{"golden_diff": "diff --git a/bodhi/metadata.py b/bodhi/metadata.py\n--- a/bodhi/metadata.py\n+++ b/bodhi/metadata.py\n@@ -66,10 +66,11 @@\n             # yum on py2.4 doesn't support sha256 (#1080373)\n             if 'el5' in self.repo or '5E' in self.repo:\n                 self.hash_type = cr.SHA1\n-\n-            # FIXME: I'm not sure which versions of RHEL support xz metadata\n-            # compression, so use the lowest common denominator for now.\n-            self.comp_type = cr.BZ2\n+                self.comp_type = cr.GZ\n+            else:\n+                # FIXME: I'm not sure which versions of RHEL support xz metadata\n+                # compression, so use the lowest common denominator for now.\n+                self.comp_type = cr.BZ2\n \n         # Load from the cache if it exists\n         self.cached_repodata = os.path.join(self.repo, '..', self.tag +\n", "issue": "epel5 needs more createrepo_c compatibility for security plugin\nFrom: https://bugzilla.redhat.com/show_bug.cgi?id=1256336\n\nThe rhel5 yum security plugin is unable to deal with the updateinfo being .bz2. It wants it to be .gz apparently. \n\nNot sure if this is something we need to change in createrepo_c, or in bodhi2 when it injects the security info in.\n\n", "before_files": [{"content": "# This program is free software; you can redistribute it and/or modify\n# it under the terms of the GNU General Public License as published by\n# the Free Software Foundation; either version 2 of the License, or\n# (at your option) any later version.\n#\n# This program is distributed in the hope that it will be useful,\n# but WITHOUT ANY WARRANTY; without even the implied warranty of\n# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the\n# GNU General Public License for more details.\n#\n# You should have received a copy of the GNU General Public License along\n# with this program; if not, write to the Free Software Foundation, Inc.,\n# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.\n\n__version__ = '2.0'\n\nimport os\nimport logging\nimport shutil\nimport tempfile\n\nfrom urlgrabber.grabber import urlgrab\nfrom kitchen.text.converters import to_bytes\n\nimport createrepo_c as cr\n\nfrom bodhi.config import config\nfrom bodhi.models import Build, UpdateStatus, UpdateRequest, UpdateSuggestion\nfrom bodhi.buildsys import get_session\n\nlog = logging.getLogger(__name__)\n\n\nclass ExtendedMetadata(object):\n    \"\"\"This class represents the updateinfo.xml yum metadata.\n\n    It is generated during push time by the bodhi masher based on koji tags\n    and is injected into the yum repodata using the `modifyrepo_c` tool,\n    which is included in the `createrepo_c` package.\n\n    \"\"\"\n    def __init__(self, release, request, db, path):\n        self.repo = path\n        log.debug('repo = %r' % self.repo)\n        self.request = request\n        if request is UpdateRequest.stable:\n            self.tag = release.stable_tag\n        else:\n            self.tag = release.testing_tag\n        self.repo_path = os.path.join(self.repo, self.tag)\n\n        self.db = db\n        self.updates = set()\n        self.builds = {}\n        self.missing_ids = []\n        self._from = config.get('bodhi_email')\n        self.koji = get_session()\n        self._fetch_updates()\n\n        self.uinfo = cr.UpdateInfo()\n\n        self.hash_type = cr.SHA256\n        self.comp_type = cr.XZ\n\n        if release.id_prefix == u'FEDORA-EPEL':\n            # yum on py2.4 doesn't support sha256 (#1080373)\n            if 'el5' in self.repo or '5E' in self.repo:\n                self.hash_type = cr.SHA1\n\n            # FIXME: I'm not sure which versions of RHEL support xz metadata\n            # compression, so use the lowest common denominator for now.\n            self.comp_type = cr.BZ2\n\n        # Load from the cache if it exists\n        self.cached_repodata = os.path.join(self.repo, '..', self.tag +\n                                            '.repocache', 'repodata/')\n        if os.path.isdir(self.cached_repodata):\n            self._load_cached_updateinfo()\n        else:\n            log.debug(\"Generating new updateinfo.xml\")\n            self.uinfo = cr.UpdateInfo()\n            for update in self.updates:\n                if update.alias:\n                    self.add_update(update)\n                else:\n                    self.missing_ids.append(update.title)\n\n        if self.missing_ids:\n            log.error(\"%d updates with missing ID!\" % len(self.missing_ids))\n            log.error(self.missing_ids)\n\n    def _load_cached_updateinfo(self):\n        \"\"\"\n        Load the cached updateinfo.xml from '../{tag}.repocache/repodata'\n        \"\"\"\n        seen_ids = set()\n        from_cache = set()\n        existing_ids = set()\n\n        # Parse the updateinfo out of the repomd\n        updateinfo = None\n        repomd_xml = os.path.join(self.cached_repodata, 'repomd.xml')\n        repomd = cr.Repomd()\n        cr.xml_parse_repomd(repomd_xml, repomd)\n        for record in repomd.records:\n            if record.type == 'updateinfo':\n                updateinfo = os.path.join(os.path.dirname(\n                    os.path.dirname(self.cached_repodata)),\n                    record.location_href)\n                break\n\n        assert updateinfo, 'Unable to find updateinfo'\n\n        # Load the metadata with createrepo_c\n        log.info('Loading cached updateinfo: %s', updateinfo)\n        uinfo = cr.UpdateInfo(updateinfo)\n\n        # Determine which updates are present in the cache\n        for update in uinfo.updates:\n            existing_ids.add(update.id)\n\n        # Generate metadata for any new builds\n        for update in self.updates:\n            seen_ids.add(update.alias)\n            if not update.alias:\n                self.missing_ids.append(update.title)\n                continue\n            if update.alias in existing_ids:\n                notice = None\n                for value in uinfo.updates:\n                    if value.title == update.title:\n                        notice = value\n                        break\n                if not notice:\n                    log.warn('%s ID in cache but notice cannot be found', update.title)\n                    self.add_update(update)\n                    continue\n                if notice.updated_date:\n                    if notice.updated_date < update.date_modified:\n                        log.debug('Update modified, generating new notice: %s' % update.title)\n                        self.add_update(update)\n                    else:\n                        log.debug('Loading updated %s from cache' % update.title)\n                        from_cache.add(update.alias)\n                elif update.date_modified:\n                    log.debug('Update modified, generating new notice: %s' % update.title)\n                    self.add_update(update)\n                else:\n                    log.debug('Loading %s from cache' % update.title)\n                    from_cache.add(update.alias)\n            else:\n                log.debug('Adding new update notice: %s' % update.title)\n                self.add_update(update)\n\n        # Add all relevant notices from the cache to this document\n        for notice in uinfo.updates:\n            if notice.id in from_cache:\n                log.debug('Keeping existing notice: %s', notice.title)\n                self.uinfo.append(notice)\n            else:\n                # Keep all security notices in the stable repo\n                if self.request is not UpdateRequest.testing:\n                    if notice.type == 'security':\n                        if notice.id not in seen_ids:\n                            log.debug('Keeping existing security notice: %s',\n                                      notice.title)\n                            self.uinfo.append(notice)\n                        else:\n                            log.debug('%s already added?', notice.title)\n                    else:\n                        log.debug('Purging cached stable notice %s', notice.title)\n                else:\n                    log.debug('Purging cached testing update %s', notice.title)\n\n    def _fetch_updates(self):\n        \"\"\"Based on our given koji tag, populate a list of Update objects\"\"\"\n        log.debug(\"Fetching builds tagged with '%s'\" % self.tag)\n        kojiBuilds = self.koji.listTagged(self.tag, latest=True)\n        nonexistent = []\n        log.debug(\"%d builds found\" % len(kojiBuilds))\n        for build in kojiBuilds:\n            self.builds[build['nvr']] = build\n            build_obj = self.db.query(Build).filter_by(nvr=unicode(build['nvr'])).first()\n            if build_obj:\n                self.updates.add(build_obj.update)\n            else:\n                nonexistent.append(build['nvr'])\n        if nonexistent:\n            log.warning(\"Couldn't find the following koji builds tagged as \"\n                        \"%s in bodhi: %s\" % (self.tag, nonexistent))\n\n    def add_update(self, update):\n        \"\"\"Generate the extended metadata for a given update\"\"\"\n        rec = cr.UpdateRecord()\n        rec.version = __version__\n        rec.fromstr = config.get('bodhi_email')\n        rec.status = update.status.value\n        rec.type = update.type.value\n        rec.id = to_bytes(update.alias)\n        rec.title = to_bytes(update.title)\n        rec.summary = to_bytes('%s %s update' % (update.get_title(),\n                                                 update.type.value))\n        rec.description = to_bytes(update.notes)\n        rec.release = to_bytes(update.release.long_name)\n        rec.rights = config.get('updateinfo_rights')\n\n        if update.date_pushed:\n            rec.issued_date = update.date_pushed\n        if update.date_modified:\n            rec.updated_date = update.date_modified\n\n        col = cr.UpdateCollection()\n        col.name = to_bytes(update.release.long_name)\n        col.shortname = to_bytes(update.release.name)\n\n        for build in update.builds:\n            try:\n                kojiBuild = self.builds[build.nvr]\n            except:\n                kojiBuild = self.koji.getBuild(build.nvr)\n\n            rpms = self.koji.listBuildRPMs(kojiBuild['id'])\n            for rpm in rpms:\n                pkg = cr.UpdateCollectionPackage()\n                pkg.name = rpm['name']\n                pkg.version = rpm['version']\n                pkg.release = rpm['release']\n                if rpm['epoch'] is not None:\n                    pkg.epoch = str(rpm['epoch'])\n                else:\n                    pkg.epoch = '0'\n                pkg.arch = rpm['arch']\n\n                # TODO: how do we handle UpdateSuggestion.logout, etc?\n                pkg.reboot_suggested = update.suggest is UpdateSuggestion.reboot\n\n                filename = '%s.%s.rpm' % (rpm['nvr'], rpm['arch'])\n                pkg.filename = filename\n\n                # Build the URL\n                if rpm['arch'] == 'src':\n                    arch = 'SRPMS'\n                elif rpm['arch'] in ('noarch', 'i686'):\n                    arch = 'i386'\n                else:\n                    arch = rpm['arch']\n\n                pkg.src = os.path.join(config.get('file_url'), update.status is\n                        UpdateStatus.testing and 'testing' or '',\n                        str(update.release.version), arch, filename[0], filename)\n\n                col.append(pkg)\n\n        rec.append_collection(col)\n\n        # Create references for each bug\n        for bug in update.bugs:\n            ref = cr.UpdateReference()\n            ref.type = 'bugzilla'\n            ref.id = to_bytes(bug.bug_id)\n            ref.href = to_bytes(bug.url)\n            ref.title = to_bytes(bug.title)\n            rec.append_reference(ref)\n\n        # Create references for each CVE\n        for cve in update.cves:\n            ref = cr.UpdateReference()\n            ref.type = 'cve'\n            ref.id = to_bytes(cve.cve_id)\n            ref.href = to_bytes(cve.url)\n            rec.append_reference(ref)\n\n        self.uinfo.append(rec)\n\n    def insert_updateinfo(self):\n        fd, name = tempfile.mkstemp()\n        os.write(fd, self.uinfo.xml_dump())\n        os.close(fd)\n        self.modifyrepo(name)\n        os.unlink(name)\n\n    def modifyrepo(self, filename):\n        \"\"\"Inject a file into the repodata for each architecture\"\"\"\n        for arch in os.listdir(self.repo_path):\n            repodata = os.path.join(self.repo_path, arch, 'repodata')\n            log.info('Inserting %s into %s', filename, repodata)\n            uinfo_xml = os.path.join(repodata, 'updateinfo.xml')\n            shutil.copyfile(filename, uinfo_xml)\n            repomd_xml = os.path.join(repodata, 'repomd.xml')\n            repomd = cr.Repomd(repomd_xml)\n            uinfo_rec = cr.RepomdRecord('updateinfo', uinfo_xml)\n            uinfo_rec_comp = uinfo_rec.compress_and_fill(self.hash_type, self.comp_type)\n            uinfo_rec_comp.rename_file()\n            uinfo_rec_comp.type = 'updateinfo'\n            repomd.set_record(uinfo_rec_comp)\n            with file(repomd_xml, 'w') as repomd_file:\n                repomd_file.write(repomd.xml_dump())\n            os.unlink(uinfo_xml)\n\n    def insert_pkgtags(self):\n        \"\"\"Download and inject the pkgtags sqlite from fedora-tagger\"\"\"\n        if config.get('pkgtags_url'):\n            try:\n                tags_url = config.get('pkgtags_url')\n                tempdir = tempfile.mkdtemp('bodhi')\n                local_tags = os.path.join(tempdir, 'pkgtags.sqlite')\n                log.info('Downloading %s' % tags_url)\n                urlgrab(tags_url, filename=local_tags)\n                self.modifyrepo(local_tags)\n            except:\n                log.exception(\"There was a problem injecting pkgtags\")\n            finally:\n                shutil.rmtree(tempdir)\n\n    def cache_repodata(self):\n        arch = os.listdir(self.repo_path)[0]  # Take the first arch\n        repodata = os.path.join(self.repo_path, arch, 'repodata')\n        if not os.path.isdir(repodata):\n            log.warning('Cannot find repodata to cache: %s' % repodata)\n            return\n        cache = self.cached_repodata\n        if os.path.isdir(cache):\n            shutil.rmtree(cache)\n        shutil.copytree(repodata, cache)\n        log.info('%s cached to %s' % (repodata, cache))\n", "path": "bodhi/metadata.py"}], "after_files": [{"content": "# This program is free software; you can redistribute it and/or modify\n# it under the terms of the GNU General Public License as published by\n# the Free Software Foundation; either version 2 of the License, or\n# (at your option) any later version.\n#\n# This program is distributed in the hope that it will be useful,\n# but WITHOUT ANY WARRANTY; without even the implied warranty of\n# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the\n# GNU General Public License for more details.\n#\n# You should have received a copy of the GNU General Public License along\n# with this program; if not, write to the Free Software Foundation, Inc.,\n# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.\n\n__version__ = '2.0'\n\nimport os\nimport logging\nimport shutil\nimport tempfile\n\nfrom urlgrabber.grabber import urlgrab\nfrom kitchen.text.converters import to_bytes\n\nimport createrepo_c as cr\n\nfrom bodhi.config import config\nfrom bodhi.models import Build, UpdateStatus, UpdateRequest, UpdateSuggestion\nfrom bodhi.buildsys import get_session\n\nlog = logging.getLogger(__name__)\n\n\nclass ExtendedMetadata(object):\n    \"\"\"This class represents the updateinfo.xml yum metadata.\n\n    It is generated during push time by the bodhi masher based on koji tags\n    and is injected into the yum repodata using the `modifyrepo_c` tool,\n    which is included in the `createrepo_c` package.\n\n    \"\"\"\n    def __init__(self, release, request, db, path):\n        self.repo = path\n        log.debug('repo = %r' % self.repo)\n        self.request = request\n        if request is UpdateRequest.stable:\n            self.tag = release.stable_tag\n        else:\n            self.tag = release.testing_tag\n        self.repo_path = os.path.join(self.repo, self.tag)\n\n        self.db = db\n        self.updates = set()\n        self.builds = {}\n        self.missing_ids = []\n        self._from = config.get('bodhi_email')\n        self.koji = get_session()\n        self._fetch_updates()\n\n        self.uinfo = cr.UpdateInfo()\n\n        self.hash_type = cr.SHA256\n        self.comp_type = cr.XZ\n\n        if release.id_prefix == u'FEDORA-EPEL':\n            # yum on py2.4 doesn't support sha256 (#1080373)\n            if 'el5' in self.repo or '5E' in self.repo:\n                self.hash_type = cr.SHA1\n                self.comp_type = cr.GZ\n            else:\n                # FIXME: I'm not sure which versions of RHEL support xz metadata\n                # compression, so use the lowest common denominator for now.\n                self.comp_type = cr.BZ2\n\n        # Load from the cache if it exists\n        self.cached_repodata = os.path.join(self.repo, '..', self.tag +\n                                            '.repocache', 'repodata/')\n        if os.path.isdir(self.cached_repodata):\n            self._load_cached_updateinfo()\n        else:\n            log.debug(\"Generating new updateinfo.xml\")\n            self.uinfo = cr.UpdateInfo()\n            for update in self.updates:\n                if update.alias:\n                    self.add_update(update)\n                else:\n                    self.missing_ids.append(update.title)\n\n        if self.missing_ids:\n            log.error(\"%d updates with missing ID!\" % len(self.missing_ids))\n            log.error(self.missing_ids)\n\n    def _load_cached_updateinfo(self):\n        \"\"\"\n        Load the cached updateinfo.xml from '../{tag}.repocache/repodata'\n        \"\"\"\n        seen_ids = set()\n        from_cache = set()\n        existing_ids = set()\n\n        # Parse the updateinfo out of the repomd\n        updateinfo = None\n        repomd_xml = os.path.join(self.cached_repodata, 'repomd.xml')\n        repomd = cr.Repomd()\n        cr.xml_parse_repomd(repomd_xml, repomd)\n        for record in repomd.records:\n            if record.type == 'updateinfo':\n                updateinfo = os.path.join(os.path.dirname(\n                    os.path.dirname(self.cached_repodata)),\n                    record.location_href)\n                break\n\n        assert updateinfo, 'Unable to find updateinfo'\n\n        # Load the metadata with createrepo_c\n        log.info('Loading cached updateinfo: %s', updateinfo)\n        uinfo = cr.UpdateInfo(updateinfo)\n\n        # Determine which updates are present in the cache\n        for update in uinfo.updates:\n            existing_ids.add(update.id)\n\n        # Generate metadata for any new builds\n        for update in self.updates:\n            seen_ids.add(update.alias)\n            if not update.alias:\n                self.missing_ids.append(update.title)\n                continue\n            if update.alias in existing_ids:\n                notice = None\n                for value in uinfo.updates:\n                    if value.title == update.title:\n                        notice = value\n                        break\n                if not notice:\n                    log.warn('%s ID in cache but notice cannot be found', update.title)\n                    self.add_update(update)\n                    continue\n                if notice.updated_date:\n                    if notice.updated_date < update.date_modified:\n                        log.debug('Update modified, generating new notice: %s' % update.title)\n                        self.add_update(update)\n                    else:\n                        log.debug('Loading updated %s from cache' % update.title)\n                        from_cache.add(update.alias)\n                elif update.date_modified:\n                    log.debug('Update modified, generating new notice: %s' % update.title)\n                    self.add_update(update)\n                else:\n                    log.debug('Loading %s from cache' % update.title)\n                    from_cache.add(update.alias)\n            else:\n                log.debug('Adding new update notice: %s' % update.title)\n                self.add_update(update)\n\n        # Add all relevant notices from the cache to this document\n        for notice in uinfo.updates:\n            if notice.id in from_cache:\n                log.debug('Keeping existing notice: %s', notice.title)\n                self.uinfo.append(notice)\n            else:\n                # Keep all security notices in the stable repo\n                if self.request is not UpdateRequest.testing:\n                    if notice.type == 'security':\n                        if notice.id not in seen_ids:\n                            log.debug('Keeping existing security notice: %s',\n                                      notice.title)\n                            self.uinfo.append(notice)\n                        else:\n                            log.debug('%s already added?', notice.title)\n                    else:\n                        log.debug('Purging cached stable notice %s', notice.title)\n                else:\n                    log.debug('Purging cached testing update %s', notice.title)\n\n    def _fetch_updates(self):\n        \"\"\"Based on our given koji tag, populate a list of Update objects\"\"\"\n        log.debug(\"Fetching builds tagged with '%s'\" % self.tag)\n        kojiBuilds = self.koji.listTagged(self.tag, latest=True)\n        nonexistent = []\n        log.debug(\"%d builds found\" % len(kojiBuilds))\n        for build in kojiBuilds:\n            self.builds[build['nvr']] = build\n            build_obj = self.db.query(Build).filter_by(nvr=unicode(build['nvr'])).first()\n            if build_obj:\n                self.updates.add(build_obj.update)\n            else:\n                nonexistent.append(build['nvr'])\n        if nonexistent:\n            log.warning(\"Couldn't find the following koji builds tagged as \"\n                        \"%s in bodhi: %s\" % (self.tag, nonexistent))\n\n    def add_update(self, update):\n        \"\"\"Generate the extended metadata for a given update\"\"\"\n        rec = cr.UpdateRecord()\n        rec.version = __version__\n        rec.fromstr = config.get('bodhi_email')\n        rec.status = update.status.value\n        rec.type = update.type.value\n        rec.id = to_bytes(update.alias)\n        rec.title = to_bytes(update.title)\n        rec.summary = to_bytes('%s %s update' % (update.get_title(),\n                                                 update.type.value))\n        rec.description = to_bytes(update.notes)\n        rec.release = to_bytes(update.release.long_name)\n        rec.rights = config.get('updateinfo_rights')\n\n        if update.date_pushed:\n            rec.issued_date = update.date_pushed\n        if update.date_modified:\n            rec.updated_date = update.date_modified\n\n        col = cr.UpdateCollection()\n        col.name = to_bytes(update.release.long_name)\n        col.shortname = to_bytes(update.release.name)\n\n        for build in update.builds:\n            try:\n                kojiBuild = self.builds[build.nvr]\n            except:\n                kojiBuild = self.koji.getBuild(build.nvr)\n\n            rpms = self.koji.listBuildRPMs(kojiBuild['id'])\n            for rpm in rpms:\n                pkg = cr.UpdateCollectionPackage()\n                pkg.name = rpm['name']\n                pkg.version = rpm['version']\n                pkg.release = rpm['release']\n                if rpm['epoch'] is not None:\n                    pkg.epoch = str(rpm['epoch'])\n                else:\n                    pkg.epoch = '0'\n                pkg.arch = rpm['arch']\n\n                # TODO: how do we handle UpdateSuggestion.logout, etc?\n                pkg.reboot_suggested = update.suggest is UpdateSuggestion.reboot\n\n                filename = '%s.%s.rpm' % (rpm['nvr'], rpm['arch'])\n                pkg.filename = filename\n\n                # Build the URL\n                if rpm['arch'] == 'src':\n                    arch = 'SRPMS'\n                elif rpm['arch'] in ('noarch', 'i686'):\n                    arch = 'i386'\n                else:\n                    arch = rpm['arch']\n\n                pkg.src = os.path.join(config.get('file_url'), update.status is\n                        UpdateStatus.testing and 'testing' or '',\n                        str(update.release.version), arch, filename[0], filename)\n\n                col.append(pkg)\n\n        rec.append_collection(col)\n\n        # Create references for each bug\n        for bug in update.bugs:\n            ref = cr.UpdateReference()\n            ref.type = 'bugzilla'\n            ref.id = to_bytes(bug.bug_id)\n            ref.href = to_bytes(bug.url)\n            ref.title = to_bytes(bug.title)\n            rec.append_reference(ref)\n\n        # Create references for each CVE\n        for cve in update.cves:\n            ref = cr.UpdateReference()\n            ref.type = 'cve'\n            ref.id = to_bytes(cve.cve_id)\n            ref.href = to_bytes(cve.url)\n            rec.append_reference(ref)\n\n        self.uinfo.append(rec)\n\n    def insert_updateinfo(self):\n        fd, name = tempfile.mkstemp()\n        os.write(fd, self.uinfo.xml_dump())\n        os.close(fd)\n        self.modifyrepo(name)\n        os.unlink(name)\n\n    def modifyrepo(self, filename):\n        \"\"\"Inject a file into the repodata for each architecture\"\"\"\n        for arch in os.listdir(self.repo_path):\n            repodata = os.path.join(self.repo_path, arch, 'repodata')\n            log.info('Inserting %s into %s', filename, repodata)\n            uinfo_xml = os.path.join(repodata, 'updateinfo.xml')\n            shutil.copyfile(filename, uinfo_xml)\n            repomd_xml = os.path.join(repodata, 'repomd.xml')\n            repomd = cr.Repomd(repomd_xml)\n            uinfo_rec = cr.RepomdRecord('updateinfo', uinfo_xml)\n            uinfo_rec_comp = uinfo_rec.compress_and_fill(self.hash_type, self.comp_type)\n            uinfo_rec_comp.rename_file()\n            uinfo_rec_comp.type = 'updateinfo'\n            repomd.set_record(uinfo_rec_comp)\n            with file(repomd_xml, 'w') as repomd_file:\n                repomd_file.write(repomd.xml_dump())\n            os.unlink(uinfo_xml)\n\n    def insert_pkgtags(self):\n        \"\"\"Download and inject the pkgtags sqlite from fedora-tagger\"\"\"\n        if config.get('pkgtags_url'):\n            try:\n                tags_url = config.get('pkgtags_url')\n                tempdir = tempfile.mkdtemp('bodhi')\n                local_tags = os.path.join(tempdir, 'pkgtags.sqlite')\n                log.info('Downloading %s' % tags_url)\n                urlgrab(tags_url, filename=local_tags)\n                self.modifyrepo(local_tags)\n            except:\n                log.exception(\"There was a problem injecting pkgtags\")\n            finally:\n                shutil.rmtree(tempdir)\n\n    def cache_repodata(self):\n        arch = os.listdir(self.repo_path)[0]  # Take the first arch\n        repodata = os.path.join(self.repo_path, arch, 'repodata')\n        if not os.path.isdir(repodata):\n            log.warning('Cannot find repodata to cache: %s' % repodata)\n            return\n        cache = self.cached_repodata\n        if os.path.isdir(cache):\n            shutil.rmtree(cache)\n        shutil.copytree(repodata, cache)\n        log.info('%s cached to %s' % (repodata, cache))\n", "path": "bodhi/metadata.py"}]}