prompt: We are currently solving the following issue within our repository. Here is the issue text:
  --- BEGIN ISSUE ---
  "withdrawn" entries not getting exported correctly
  Identified by @andrewpollock : 
  
  https://github.com/google/osv.dev/blob/26050deb42785bc5a4dc7d802eac8e7f95135509/docker/exporter/exporter.py#L94
  
  withdrawn entries are marked as status = INVALID in our DB, so they're not included.
  
  They should be included when we export. 
  
  --- END ISSUE ---
  
  Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
  --- BEGIN FILES ---
  Path: `docker/exporter/exporter.py`
  Content:
  ```
  1 #!/usr/bin/env python3
  2 # Copyright 2021 Google LLC
  3 #
  4 # Licensed under the Apache License, Version 2.0 (the "License");
  5 # you may not use this file except in compliance with the License.
  6 # You may obtain a copy of the License at
  7 #
  8 #      http://www.apache.org/licenses/LICENSE-2.0
  9 #
  10 # Unless required by applicable law or agreed to in writing, software
  11 # distributed under the License is distributed on an "AS IS" BASIS,
  12 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13 # See the License for the specific language governing permissions and
  14 # limitations under the License.
  15 """OSV Exporter."""
  16 import argparse
  17 import concurrent.futures
  18 import logging
  19 import os
  20 import tempfile
  21 import zipfile
  22 from typing import List
  23 
  24 from google.cloud import ndb
  25 from google.cloud import storage
  26 from google.cloud import logging as google_logging
  27 
  28 import osv
  29 
  30 DEFAULT_WORK_DIR = '/work'
  31 
  32 DEFAULT_EXPORT_BUCKET = 'osv-vulnerabilities'
  33 _EXPORT_WORKERS = 32
  34 ECOSYSTEMS_FILE = 'ecosystems.txt'
  35 
  36 
  37 class Exporter:
  38   """Exporter."""
  39 
  40   def __init__(self, work_dir, export_bucket):
  41     self._work_dir = work_dir
  42     self._export_bucket = export_bucket
  43 
  44   def run(self):
  45     """Run exporter."""
  46     query = osv.Bug.query(projection=[osv.Bug.ecosystem], distinct=True)
  47     ecosystems = [bug.ecosystem[0] for bug in query if bug.ecosystem]
  48 
  49     for ecosystem in ecosystems:
  50       with tempfile.TemporaryDirectory() as tmp_dir:
  51         self._export_ecosystem_to_bucket(ecosystem, tmp_dir)
  52 
  53     with tempfile.TemporaryDirectory() as tmp_dir:
  54       self._export_ecosystem_list_to_bucket(ecosystems, tmp_dir)
  55 
  56   def upload_single(self, bucket, source_path, target_path):
  57     """Upload a single file to a bucket."""
  58     logging.info('Uploading %s', target_path)
  59     try:
  60       blob = bucket.blob(target_path)
  61       blob.upload_from_filename(source_path)
  62     except Exception as e:
  63       logging.error('Failed to export: %s', e)
  64 
  65   def _export_ecosystem_list_to_bucket(self, ecosystems: List[str],
  66                                        tmp_dir: str):
  67     """Export an ecosystems.txt file with all of the ecosystem names.
  68 
  69     See https://github.com/google/osv.dev/issues/619
  70 
  71     Args:
  72       ecosystems: the list of ecosystem names
  73       tmp_dir: temporary directory for scratch
  74     """
  75 
  76     logging.info('Exporting ecosystem list to %s', ECOSYSTEMS_FILE)
  77     storage_client = storage.Client()
  78     bucket = storage_client.get_bucket(self._export_bucket)
  79     ecosystems_file_path = os.path.join(tmp_dir, ECOSYSTEMS_FILE)
  80     with open(ecosystems_file_path, "w") as ecosystems_file:
  81       ecosystems_file.writelines([e + "\n" for e in ecosystems])
  82 
  83     self.upload_single(bucket, ecosystems_file_path, ECOSYSTEMS_FILE)
  84 
  85   def _export_ecosystem_to_bucket(self, ecosystem, tmp_dir):
  86     """Export ecosystem vulns to bucket."""
  87     logging.info('Exporting vulnerabilities for ecosystem %s', ecosystem)
  88     storage_client = storage.Client()
  89     bucket = storage_client.get_bucket(self._export_bucket)
  90 
  91     zip_path = os.path.join(tmp_dir, 'all.zip')
  92     with zipfile.ZipFile(zip_path, 'w', zipfile.ZIP_DEFLATED) as zip_file:
  93       for bug in osv.Bug.query(osv.Bug.ecosystem == ecosystem):
  94         if not bug.public or not bug.status == osv.BugStatus.PROCESSED:
  95           continue
  96 
  97         file_path = os.path.join(tmp_dir, bug.id() + '.json')
  98         osv.write_vulnerability(
  99             bug.to_vulnerability(include_source=True), file_path)
  100         zip_file.write(file_path, os.path.basename(file_path))
  101 
  102     with concurrent.futures.ThreadPoolExecutor(
  103         max_workers=_EXPORT_WORKERS) as executor:
  104       for filename in os.listdir(tmp_dir):
  105         executor.submit(self.upload_single, bucket,
  106                         os.path.join(tmp_dir, filename),
  107                         f'{ecosystem}/{filename}')
  108 
  109 
  110 def main():
  111   logging.getLogger().setLevel(logging.INFO)
  112   parser = argparse.ArgumentParser(description='Exporter')
  113   parser.add_argument(
  114       '--work_dir', help='Working directory', default=DEFAULT_WORK_DIR)
  115   parser.add_argument(
  116       '--bucket',
  117       help='Bucket name to export to',
  118       default=DEFAULT_EXPORT_BUCKET)
  119   args = parser.parse_args()
  120 
  121   tmp_dir = os.path.join(args.work_dir, 'tmp')
  122   os.makedirs(tmp_dir, exist_ok=True)
  123   os.environ['TMPDIR'] = tmp_dir
  124 
  125   exporter = Exporter(args.work_dir, args.bucket)
  126   exporter.run()
  127 
  128 
  129 if __name__ == '__main__':
  130   _ndb_client = ndb.Client()
  131   logging_client = google_logging.Client()
  132   logging_client.setup_logging()
  133   with _ndb_client.context():
  134     main()
  135
  ```
  --- END FILES ---
  
  Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.
  
  Here is an example:
  
  ```diff
  diff --git a/examples/server_async.py b/examples/server_async.py
  --- a/examples/server_async.py
  +++ b/examples/server_async.py
  @@ -313,4 +313,4 @@
   
   
   if __name__ == "__main__":
  -    asyncio.run(run_async_server("."), debug=True)
  +    asyncio.run(run_async_server(), debug=True)
  diff --git a/examples/server_sync.py b/examples/server_sync.py
  --- a/examples/server_sync.py
  +++ b/examples/server_sync.py
  @@ -313,5 +313,5 @@
   
   
   if __name__ == "__main__":
  -    server = run_sync_server(".")
  +    server = run_sync_server()
       server.shutdown()
  
  ```
  
golden_diff: diff --git a/docker/exporter/exporter.py b/docker/exporter/exporter.py
  --- a/docker/exporter/exporter.py
  +++ b/docker/exporter/exporter.py
  @@ -91,7 +91,7 @@
       zip_path = os.path.join(tmp_dir, 'all.zip')
       with zipfile.ZipFile(zip_path, 'w', zipfile.ZIP_DEFLATED) as zip_file:
         for bug in osv.Bug.query(osv.Bug.ecosystem == ecosystem):
  -        if not bug.public or not bug.status == osv.BugStatus.PROCESSED:
  +        if not bug.public or bug.status == osv.BugStatus.UNPROCESSED:
             continue
   
           file_path = os.path.join(tmp_dir, bug.id() + '.json')
  
verification_info: {"golden_diff": "diff --git a/docker/exporter/exporter.py b/docker/exporter/exporter.py\n--- a/docker/exporter/exporter.py\n+++ b/docker/exporter/exporter.py\n@@ -91,7 +91,7 @@\n     zip_path = os.path.join(tmp_dir, 'all.zip')\n     with zipfile.ZipFile(zip_path, 'w', zipfile.ZIP_DEFLATED) as zip_file:\n       for bug in osv.Bug.query(osv.Bug.ecosystem == ecosystem):\n-        if not bug.public or not bug.status == osv.BugStatus.PROCESSED:\n+        if not bug.public or bug.status == osv.BugStatus.UNPROCESSED:\n           continue\n \n         file_path = os.path.join(tmp_dir, bug.id() + '.json')\n", "issue": "\"withdrawn\" entries not getting exported correctly\nIdentified by @andrewpollock : \r\n\r\nhttps://github.com/google/osv.dev/blob/26050deb42785bc5a4dc7d802eac8e7f95135509/docker/exporter/exporter.py#L94\r\n\r\nwithdrawn entries are marked as status = INVALID in our DB, so they're not included.\r\n\r\nThey should be included when we export. \n", "before_files": [{"content": "#!/usr/bin/env python3\n# Copyright 2021 Google LLC\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#      http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\"\"\"OSV Exporter.\"\"\"\nimport argparse\nimport concurrent.futures\nimport logging\nimport os\nimport tempfile\nimport zipfile\nfrom typing import List\n\nfrom google.cloud import ndb\nfrom google.cloud import storage\nfrom google.cloud import logging as google_logging\n\nimport osv\n\nDEFAULT_WORK_DIR = '/work'\n\nDEFAULT_EXPORT_BUCKET = 'osv-vulnerabilities'\n_EXPORT_WORKERS = 32\nECOSYSTEMS_FILE = 'ecosystems.txt'\n\n\nclass Exporter:\n  \"\"\"Exporter.\"\"\"\n\n  def __init__(self, work_dir, export_bucket):\n    self._work_dir = work_dir\n    self._export_bucket = export_bucket\n\n  def run(self):\n    \"\"\"Run exporter.\"\"\"\n    query = osv.Bug.query(projection=[osv.Bug.ecosystem], distinct=True)\n    ecosystems = [bug.ecosystem[0] for bug in query if bug.ecosystem]\n\n    for ecosystem in ecosystems:\n      with tempfile.TemporaryDirectory() as tmp_dir:\n        self._export_ecosystem_to_bucket(ecosystem, tmp_dir)\n\n    with tempfile.TemporaryDirectory() as tmp_dir:\n      self._export_ecosystem_list_to_bucket(ecosystems, tmp_dir)\n\n  def upload_single(self, bucket, source_path, target_path):\n    \"\"\"Upload a single file to a bucket.\"\"\"\n    logging.info('Uploading %s', target_path)\n    try:\n      blob = bucket.blob(target_path)\n      blob.upload_from_filename(source_path)\n    except Exception as e:\n      logging.error('Failed to export: %s', e)\n\n  def _export_ecosystem_list_to_bucket(self, ecosystems: List[str],\n                                       tmp_dir: str):\n    \"\"\"Export an ecosystems.txt file with all of the ecosystem names.\n\n    See https://github.com/google/osv.dev/issues/619\n\n    Args:\n      ecosystems: the list of ecosystem names\n      tmp_dir: temporary directory for scratch\n    \"\"\"\n\n    logging.info('Exporting ecosystem list to %s', ECOSYSTEMS_FILE)\n    storage_client = storage.Client()\n    bucket = storage_client.get_bucket(self._export_bucket)\n    ecosystems_file_path = os.path.join(tmp_dir, ECOSYSTEMS_FILE)\n    with open(ecosystems_file_path, \"w\") as ecosystems_file:\n      ecosystems_file.writelines([e + \"\\n\" for e in ecosystems])\n\n    self.upload_single(bucket, ecosystems_file_path, ECOSYSTEMS_FILE)\n\n  def _export_ecosystem_to_bucket(self, ecosystem, tmp_dir):\n    \"\"\"Export ecosystem vulns to bucket.\"\"\"\n    logging.info('Exporting vulnerabilities for ecosystem %s', ecosystem)\n    storage_client = storage.Client()\n    bucket = storage_client.get_bucket(self._export_bucket)\n\n    zip_path = os.path.join(tmp_dir, 'all.zip')\n    with zipfile.ZipFile(zip_path, 'w', zipfile.ZIP_DEFLATED) as zip_file:\n      for bug in osv.Bug.query(osv.Bug.ecosystem == ecosystem):\n        if not bug.public or not bug.status == osv.BugStatus.PROCESSED:\n          continue\n\n        file_path = os.path.join(tmp_dir, bug.id() + '.json')\n        osv.write_vulnerability(\n            bug.to_vulnerability(include_source=True), file_path)\n        zip_file.write(file_path, os.path.basename(file_path))\n\n    with concurrent.futures.ThreadPoolExecutor(\n        max_workers=_EXPORT_WORKERS) as executor:\n      for filename in os.listdir(tmp_dir):\n        executor.submit(self.upload_single, bucket,\n                        os.path.join(tmp_dir, filename),\n                        f'{ecosystem}/{filename}')\n\n\ndef main():\n  logging.getLogger().setLevel(logging.INFO)\n  parser = argparse.ArgumentParser(description='Exporter')\n  parser.add_argument(\n      '--work_dir', help='Working directory', default=DEFAULT_WORK_DIR)\n  parser.add_argument(\n      '--bucket',\n      help='Bucket name to export to',\n      default=DEFAULT_EXPORT_BUCKET)\n  args = parser.parse_args()\n\n  tmp_dir = os.path.join(args.work_dir, 'tmp')\n  os.makedirs(tmp_dir, exist_ok=True)\n  os.environ['TMPDIR'] = tmp_dir\n\n  exporter = Exporter(args.work_dir, args.bucket)\n  exporter.run()\n\n\nif __name__ == '__main__':\n  _ndb_client = ndb.Client()\n  logging_client = google_logging.Client()\n  logging_client.setup_logging()\n  with _ndb_client.context():\n    main()\n", "path": "docker/exporter/exporter.py"}], "after_files": [{"content": "#!/usr/bin/env python3\n# Copyright 2021 Google LLC\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#      http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\"\"\"OSV Exporter.\"\"\"\nimport argparse\nimport concurrent.futures\nimport logging\nimport os\nimport tempfile\nimport zipfile\nfrom typing import List\n\nfrom google.cloud import ndb\nfrom google.cloud import storage\nfrom google.cloud import logging as google_logging\n\nimport osv\n\nDEFAULT_WORK_DIR = '/work'\n\nDEFAULT_EXPORT_BUCKET = 'osv-vulnerabilities'\n_EXPORT_WORKERS = 32\nECOSYSTEMS_FILE = 'ecosystems.txt'\n\n\nclass Exporter:\n  \"\"\"Exporter.\"\"\"\n\n  def __init__(self, work_dir, export_bucket):\n    self._work_dir = work_dir\n    self._export_bucket = export_bucket\n\n  def run(self):\n    \"\"\"Run exporter.\"\"\"\n    query = osv.Bug.query(projection=[osv.Bug.ecosystem], distinct=True)\n    ecosystems = [bug.ecosystem[0] for bug in query if bug.ecosystem]\n\n    for ecosystem in ecosystems:\n      with tempfile.TemporaryDirectory() as tmp_dir:\n        self._export_ecosystem_to_bucket(ecosystem, tmp_dir)\n\n    with tempfile.TemporaryDirectory() as tmp_dir:\n      self._export_ecosystem_list_to_bucket(ecosystems, tmp_dir)\n\n  def upload_single(self, bucket, source_path, target_path):\n    \"\"\"Upload a single file to a bucket.\"\"\"\n    logging.info('Uploading %s', target_path)\n    try:\n      blob = bucket.blob(target_path)\n      blob.upload_from_filename(source_path)\n    except Exception as e:\n      logging.error('Failed to export: %s', e)\n\n  def _export_ecosystem_list_to_bucket(self, ecosystems: List[str],\n                                       tmp_dir: str):\n    \"\"\"Export an ecosystems.txt file with all of the ecosystem names.\n\n    See https://github.com/google/osv.dev/issues/619\n\n    Args:\n      ecosystems: the list of ecosystem names\n      tmp_dir: temporary directory for scratch\n    \"\"\"\n\n    logging.info('Exporting ecosystem list to %s', ECOSYSTEMS_FILE)\n    storage_client = storage.Client()\n    bucket = storage_client.get_bucket(self._export_bucket)\n    ecosystems_file_path = os.path.join(tmp_dir, ECOSYSTEMS_FILE)\n    with open(ecosystems_file_path, \"w\") as ecosystems_file:\n      ecosystems_file.writelines([e + \"\\n\" for e in ecosystems])\n\n    self.upload_single(bucket, ecosystems_file_path, ECOSYSTEMS_FILE)\n\n  def _export_ecosystem_to_bucket(self, ecosystem, tmp_dir):\n    \"\"\"Export ecosystem vulns to bucket.\"\"\"\n    logging.info('Exporting vulnerabilities for ecosystem %s', ecosystem)\n    storage_client = storage.Client()\n    bucket = storage_client.get_bucket(self._export_bucket)\n\n    zip_path = os.path.join(tmp_dir, 'all.zip')\n    with zipfile.ZipFile(zip_path, 'w', zipfile.ZIP_DEFLATED) as zip_file:\n      for bug in osv.Bug.query(osv.Bug.ecosystem == ecosystem):\n        if not bug.public or bug.status == osv.BugStatus.UNPROCESSED:\n          continue\n\n        file_path = os.path.join(tmp_dir, bug.id() + '.json')\n        osv.write_vulnerability(\n            bug.to_vulnerability(include_source=True), file_path)\n        zip_file.write(file_path, os.path.basename(file_path))\n\n    with concurrent.futures.ThreadPoolExecutor(\n        max_workers=_EXPORT_WORKERS) as executor:\n      for filename in os.listdir(tmp_dir):\n        executor.submit(self.upload_single, bucket,\n                        os.path.join(tmp_dir, filename),\n                        f'{ecosystem}/{filename}')\n\n\ndef main():\n  logging.getLogger().setLevel(logging.INFO)\n  parser = argparse.ArgumentParser(description='Exporter')\n  parser.add_argument(\n      '--work_dir', help='Working directory', default=DEFAULT_WORK_DIR)\n  parser.add_argument(\n      '--bucket',\n      help='Bucket name to export to',\n      default=DEFAULT_EXPORT_BUCKET)\n  args = parser.parse_args()\n\n  tmp_dir = os.path.join(args.work_dir, 'tmp')\n  os.makedirs(tmp_dir, exist_ok=True)\n  os.environ['TMPDIR'] = tmp_dir\n\n  exporter = Exporter(args.work_dir, args.bucket)\n  exporter.run()\n\n\nif __name__ == '__main__':\n  _ndb_client = ndb.Client()\n  logging_client = google_logging.Client()\n  logging_client.setup_logging()\n  with _ndb_client.context():\n    main()\n", "path": "docker/exporter/exporter.py"}]}