prompt: We are currently solving the following issue within our repository. Here is the issue text:
  --- BEGIN ISSUE ---
  Include crawl date in data
  I'm looking at an old output directory, trying to workout which release it is.
  
  I think we could add the crawl time and/or build id to the dataset attributes easily.
  
  I think @rjw62 asked for this before. Which I promptly forgot. Sorry.
  
  I'll look at this later or Monday.
  
  --- END ISSUE ---
  
  Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
  --- BEGIN FILES ---
  Path: `locations/exporters/geojson.py`
  Content:
  ```
  1 import base64
  2 import hashlib
  3 import io
  4 import json
  5 import logging
  6 import uuid
  7 
  8 from scrapy.exporters import JsonItemExporter
  9 from scrapy.utils.misc import walk_modules
  10 from scrapy.utils.python import to_bytes
  11 from scrapy.utils.spider import iter_spider_classes
  12 
  13 from locations.settings import SPIDER_MODULES
  14 
  15 mapping = (
  16     ("addr_full", "addr:full"),
  17     ("housenumber", "addr:housenumber"),
  18     ("street", "addr:street"),
  19     ("street_address", "addr:street_address"),
  20     ("city", "addr:city"),
  21     ("state", "addr:state"),
  22     ("postcode", "addr:postcode"),
  23     ("country", "addr:country"),
  24     ("name", "name"),
  25     ("phone", "phone"),
  26     ("website", "website"),
  27     ("twitter", "contact:twitter"),
  28     ("facebook", "contact:facebook"),
  29     ("email", "contact:email"),
  30     ("opening_hours", "opening_hours"),
  31     ("image", "image"),
  32     ("brand", "brand"),
  33     ("brand_wikidata", "brand:wikidata"),
  34     ("located_in", "located_in"),
  35     ("located_in_wikidata", "located_in:wikidata"),
  36     ("nsi_id", "nsi_id"),
  37 )
  38 
  39 
  40 def item_to_properties(item):
  41     props = {}
  42 
  43     # Ref is required, unless `no_refs = True` is set in spider
  44     if ref := item.get("ref"):
  45         props["ref"] = str(ref)
  46 
  47     # Add in the extra bits
  48     if extras := item.get("extras"):
  49         for key, value in extras.items():
  50             if value:
  51                 # Only export populated values
  52                 props[key] = value
  53 
  54     # Bring in the optional stuff
  55     for map_from, map_to in mapping:
  56         if item_value := item.get(map_from):
  57             props[map_to] = item_value
  58 
  59     return props
  60 
  61 
  62 def compute_hash(item):
  63     ref = str(item.get("ref") or uuid.uuid1()).encode("utf8")
  64     sha1 = hashlib.sha1(ref)
  65 
  66     if spider_name := item.get("extras", {}).get("@spider"):
  67         sha1.update(spider_name.encode("utf8"))
  68 
  69     return base64.urlsafe_b64encode(sha1.digest()).decode("utf8")
  70 
  71 
  72 def find_spider_class(spider_name):
  73     if not spider_name:
  74         return None
  75     for mod in SPIDER_MODULES:
  76         for module in walk_modules(mod):
  77             for spider_class in iter_spider_classes(module):
  78                 if spider_name == spider_class.name:
  79                     return spider_class
  80     return None
  81 
  82 
  83 def get_dataset_attributes(spider_name) -> {}:
  84     spider_class = find_spider_class(spider_name)
  85     dataset_attributes = getattr(spider_class, "dataset_attributes", {})
  86     settings = getattr(spider_class, "custom_settings", {}) or {}
  87     if not settings.get("ROBOTSTXT_OBEY", True):
  88         # See https://github.com/alltheplaces/alltheplaces/issues/4537
  89         dataset_attributes["spider:robots_txt"] = "ignored"
  90     dataset_attributes["@spider"] = spider_name
  91 
  92     return dataset_attributes
  93 
  94 
  95 class GeoJsonExporter(JsonItemExporter):
  96     def __init__(self, file, **kwargs):
  97         super().__init__(file, **kwargs)
  98         self.spider_name = None
  99 
  100     def start_exporting(self):
  101         pass
  102 
  103     def export_item(self, item):
  104         spider_name = item.get("extras", {}).get("@spider")
  105         if self.first_item:
  106             self.spider_name = spider_name
  107             self.write_geojson_header()
  108         if spider_name != self.spider_name:
  109             # It really should not happen that a single exporter instance
  110             # handles output from different spiders. If it does happen,
  111             # we rather crash than emit GeoJSON with the wrong dataset
  112             # properties, which may include legally relevant license tags.
  113             raise ValueError(
  114                 f"harvest from multiple spiders ({spider_name, self.spider_name}) cannot be written to same GeoJSON file"
  115             )
  116 
  117         super().export_item(item)
  118 
  119     def _get_serialized_fields(self, item, default_value=None, include_empty=None):
  120         feature = []
  121         feature.append(("type", "Feature"))
  122         feature.append(("id", compute_hash(item)))
  123         feature.append(("properties", item_to_properties(item)))
  124 
  125         lat = item.get("lat")
  126         lon = item.get("lon")
  127         geometry = item.get("geometry")
  128         if lat and lon and not geometry:
  129             try:
  130                 geometry = {
  131                     "type": "Point",
  132                     "coordinates": [float(item["lon"]), float(item["lat"])],
  133                 }
  134             except ValueError:
  135                 logging.warning("Couldn't convert lat (%s) and lon (%s) to float", lat, lon)
  136         feature.append(("geometry", geometry))
  137 
  138         return feature
  139 
  140     def write_geojson_header(self):
  141         header = io.StringIO()
  142         header.write('{"type":"FeatureCollection","dataset_attributes":')
  143         json.dump(
  144             get_dataset_attributes(self.spider_name), header, ensure_ascii=False, separators=(",", ":"), sort_keys=True
  145         )
  146         header.write(',"features":[\n')
  147         self.file.write(to_bytes(header.getvalue(), self.encoding))
  148 
  149     def finish_exporting(self):
  150         self.file.write(b"\n]}\n")
  151
  ```
  --- END FILES ---
  
  Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.
  
  Here is an example:
  
  ```diff
  diff --git a/examples/server_async.py b/examples/server_async.py
  --- a/examples/server_async.py
  +++ b/examples/server_async.py
  @@ -313,4 +313,4 @@
   
   
   if __name__ == "__main__":
  -    asyncio.run(run_async_server("."), debug=True)
  +    asyncio.run(run_async_server(), debug=True)
  diff --git a/examples/server_sync.py b/examples/server_sync.py
  --- a/examples/server_sync.py
  +++ b/examples/server_sync.py
  @@ -313,5 +313,5 @@
   
   
   if __name__ == "__main__":
  -    server = run_sync_server(".")
  +    server = run_sync_server()
       server.shutdown()
  
  ```
  
golden_diff: diff --git a/locations/exporters/geojson.py b/locations/exporters/geojson.py
  --- a/locations/exporters/geojson.py
  +++ b/locations/exporters/geojson.py
  @@ -1,4 +1,5 @@
   import base64
  +import datetime
   import hashlib
   import io
   import json
  @@ -88,6 +89,7 @@
           # See https://github.com/alltheplaces/alltheplaces/issues/4537
           dataset_attributes["spider:robots_txt"] = "ignored"
       dataset_attributes["@spider"] = spider_name
  +    dataset_attributes["spider:collection_time"] = datetime.datetime.now().isoformat()
   
       return dataset_attributes
  
verification_info: {"golden_diff": "diff --git a/locations/exporters/geojson.py b/locations/exporters/geojson.py\n--- a/locations/exporters/geojson.py\n+++ b/locations/exporters/geojson.py\n@@ -1,4 +1,5 @@\n import base64\n+import datetime\n import hashlib\n import io\n import json\n@@ -88,6 +89,7 @@\n         # See https://github.com/alltheplaces/alltheplaces/issues/4537\n         dataset_attributes[\"spider:robots_txt\"] = \"ignored\"\n     dataset_attributes[\"@spider\"] = spider_name\n+    dataset_attributes[\"spider:collection_time\"] = datetime.datetime.now().isoformat()\n \n     return dataset_attributes\n", "issue": "Include crawl date in data\nI'm looking at an old output directory, trying to workout which release it is.\r\n\r\nI think we could add the crawl time and/or build id to the dataset attributes easily.\r\n\r\nI think @rjw62 asked for this before. Which I promptly forgot. Sorry.\r\n\r\nI'll look at this later or Monday.\n", "before_files": [{"content": "import base64\nimport hashlib\nimport io\nimport json\nimport logging\nimport uuid\n\nfrom scrapy.exporters import JsonItemExporter\nfrom scrapy.utils.misc import walk_modules\nfrom scrapy.utils.python import to_bytes\nfrom scrapy.utils.spider import iter_spider_classes\n\nfrom locations.settings import SPIDER_MODULES\n\nmapping = (\n    (\"addr_full\", \"addr:full\"),\n    (\"housenumber\", \"addr:housenumber\"),\n    (\"street\", \"addr:street\"),\n    (\"street_address\", \"addr:street_address\"),\n    (\"city\", \"addr:city\"),\n    (\"state\", \"addr:state\"),\n    (\"postcode\", \"addr:postcode\"),\n    (\"country\", \"addr:country\"),\n    (\"name\", \"name\"),\n    (\"phone\", \"phone\"),\n    (\"website\", \"website\"),\n    (\"twitter\", \"contact:twitter\"),\n    (\"facebook\", \"contact:facebook\"),\n    (\"email\", \"contact:email\"),\n    (\"opening_hours\", \"opening_hours\"),\n    (\"image\", \"image\"),\n    (\"brand\", \"brand\"),\n    (\"brand_wikidata\", \"brand:wikidata\"),\n    (\"located_in\", \"located_in\"),\n    (\"located_in_wikidata\", \"located_in:wikidata\"),\n    (\"nsi_id\", \"nsi_id\"),\n)\n\n\ndef item_to_properties(item):\n    props = {}\n\n    # Ref is required, unless `no_refs = True` is set in spider\n    if ref := item.get(\"ref\"):\n        props[\"ref\"] = str(ref)\n\n    # Add in the extra bits\n    if extras := item.get(\"extras\"):\n        for key, value in extras.items():\n            if value:\n                # Only export populated values\n                props[key] = value\n\n    # Bring in the optional stuff\n    for map_from, map_to in mapping:\n        if item_value := item.get(map_from):\n            props[map_to] = item_value\n\n    return props\n\n\ndef compute_hash(item):\n    ref = str(item.get(\"ref\") or uuid.uuid1()).encode(\"utf8\")\n    sha1 = hashlib.sha1(ref)\n\n    if spider_name := item.get(\"extras\", {}).get(\"@spider\"):\n        sha1.update(spider_name.encode(\"utf8\"))\n\n    return base64.urlsafe_b64encode(sha1.digest()).decode(\"utf8\")\n\n\ndef find_spider_class(spider_name):\n    if not spider_name:\n        return None\n    for mod in SPIDER_MODULES:\n        for module in walk_modules(mod):\n            for spider_class in iter_spider_classes(module):\n                if spider_name == spider_class.name:\n                    return spider_class\n    return None\n\n\ndef get_dataset_attributes(spider_name) -> {}:\n    spider_class = find_spider_class(spider_name)\n    dataset_attributes = getattr(spider_class, \"dataset_attributes\", {})\n    settings = getattr(spider_class, \"custom_settings\", {}) or {}\n    if not settings.get(\"ROBOTSTXT_OBEY\", True):\n        # See https://github.com/alltheplaces/alltheplaces/issues/4537\n        dataset_attributes[\"spider:robots_txt\"] = \"ignored\"\n    dataset_attributes[\"@spider\"] = spider_name\n\n    return dataset_attributes\n\n\nclass GeoJsonExporter(JsonItemExporter):\n    def __init__(self, file, **kwargs):\n        super().__init__(file, **kwargs)\n        self.spider_name = None\n\n    def start_exporting(self):\n        pass\n\n    def export_item(self, item):\n        spider_name = item.get(\"extras\", {}).get(\"@spider\")\n        if self.first_item:\n            self.spider_name = spider_name\n            self.write_geojson_header()\n        if spider_name != self.spider_name:\n            # It really should not happen that a single exporter instance\n            # handles output from different spiders. If it does happen,\n            # we rather crash than emit GeoJSON with the wrong dataset\n            # properties, which may include legally relevant license tags.\n            raise ValueError(\n                f\"harvest from multiple spiders ({spider_name, self.spider_name}) cannot be written to same GeoJSON file\"\n            )\n\n        super().export_item(item)\n\n    def _get_serialized_fields(self, item, default_value=None, include_empty=None):\n        feature = []\n        feature.append((\"type\", \"Feature\"))\n        feature.append((\"id\", compute_hash(item)))\n        feature.append((\"properties\", item_to_properties(item)))\n\n        lat = item.get(\"lat\")\n        lon = item.get(\"lon\")\n        geometry = item.get(\"geometry\")\n        if lat and lon and not geometry:\n            try:\n                geometry = {\n                    \"type\": \"Point\",\n                    \"coordinates\": [float(item[\"lon\"]), float(item[\"lat\"])],\n                }\n            except ValueError:\n                logging.warning(\"Couldn't convert lat (%s) and lon (%s) to float\", lat, lon)\n        feature.append((\"geometry\", geometry))\n\n        return feature\n\n    def write_geojson_header(self):\n        header = io.StringIO()\n        header.write('{\"type\":\"FeatureCollection\",\"dataset_attributes\":')\n        json.dump(\n            get_dataset_attributes(self.spider_name), header, ensure_ascii=False, separators=(\",\", \":\"), sort_keys=True\n        )\n        header.write(',\"features\":[\\n')\n        self.file.write(to_bytes(header.getvalue(), self.encoding))\n\n    def finish_exporting(self):\n        self.file.write(b\"\\n]}\\n\")\n", "path": "locations/exporters/geojson.py"}], "after_files": [{"content": "import base64\nimport datetime\nimport hashlib\nimport io\nimport json\nimport logging\nimport uuid\n\nfrom scrapy.exporters import JsonItemExporter\nfrom scrapy.utils.misc import walk_modules\nfrom scrapy.utils.python import to_bytes\nfrom scrapy.utils.spider import iter_spider_classes\n\nfrom locations.settings import SPIDER_MODULES\n\nmapping = (\n    (\"addr_full\", \"addr:full\"),\n    (\"housenumber\", \"addr:housenumber\"),\n    (\"street\", \"addr:street\"),\n    (\"street_address\", \"addr:street_address\"),\n    (\"city\", \"addr:city\"),\n    (\"state\", \"addr:state\"),\n    (\"postcode\", \"addr:postcode\"),\n    (\"country\", \"addr:country\"),\n    (\"name\", \"name\"),\n    (\"phone\", \"phone\"),\n    (\"website\", \"website\"),\n    (\"twitter\", \"contact:twitter\"),\n    (\"facebook\", \"contact:facebook\"),\n    (\"email\", \"contact:email\"),\n    (\"opening_hours\", \"opening_hours\"),\n    (\"image\", \"image\"),\n    (\"brand\", \"brand\"),\n    (\"brand_wikidata\", \"brand:wikidata\"),\n    (\"located_in\", \"located_in\"),\n    (\"located_in_wikidata\", \"located_in:wikidata\"),\n    (\"nsi_id\", \"nsi_id\"),\n)\n\n\ndef item_to_properties(item):\n    props = {}\n\n    # Ref is required, unless `no_refs = True` is set in spider\n    if ref := item.get(\"ref\"):\n        props[\"ref\"] = str(ref)\n\n    # Add in the extra bits\n    if extras := item.get(\"extras\"):\n        for key, value in extras.items():\n            if value:\n                # Only export populated values\n                props[key] = value\n\n    # Bring in the optional stuff\n    for map_from, map_to in mapping:\n        if item_value := item.get(map_from):\n            props[map_to] = item_value\n\n    return props\n\n\ndef compute_hash(item):\n    ref = str(item.get(\"ref\") or uuid.uuid1()).encode(\"utf8\")\n    sha1 = hashlib.sha1(ref)\n\n    if spider_name := item.get(\"extras\", {}).get(\"@spider\"):\n        sha1.update(spider_name.encode(\"utf8\"))\n\n    return base64.urlsafe_b64encode(sha1.digest()).decode(\"utf8\")\n\n\ndef find_spider_class(spider_name):\n    if not spider_name:\n        return None\n    for mod in SPIDER_MODULES:\n        for module in walk_modules(mod):\n            for spider_class in iter_spider_classes(module):\n                if spider_name == spider_class.name:\n                    return spider_class\n    return None\n\n\ndef get_dataset_attributes(spider_name) -> {}:\n    spider_class = find_spider_class(spider_name)\n    dataset_attributes = getattr(spider_class, \"dataset_attributes\", {})\n    settings = getattr(spider_class, \"custom_settings\", {}) or {}\n    if not settings.get(\"ROBOTSTXT_OBEY\", True):\n        # See https://github.com/alltheplaces/alltheplaces/issues/4537\n        dataset_attributes[\"spider:robots_txt\"] = \"ignored\"\n    dataset_attributes[\"@spider\"] = spider_name\n    dataset_attributes[\"spider:collection_time\"] = datetime.datetime.now().isoformat()\n\n    return dataset_attributes\n\n\nclass GeoJsonExporter(JsonItemExporter):\n    def __init__(self, file, **kwargs):\n        super().__init__(file, **kwargs)\n        self.spider_name = None\n\n    def start_exporting(self):\n        pass\n\n    def export_item(self, item):\n        spider_name = item.get(\"extras\", {}).get(\"@spider\")\n        if self.first_item:\n            self.spider_name = spider_name\n            self.write_geojson_header()\n        if spider_name != self.spider_name:\n            # It really should not happen that a single exporter instance\n            # handles output from different spiders. If it does happen,\n            # we rather crash than emit GeoJSON with the wrong dataset\n            # properties, which may include legally relevant license tags.\n            raise ValueError(\n                f\"harvest from multiple spiders ({spider_name, self.spider_name}) cannot be written to same GeoJSON file\"\n            )\n\n        super().export_item(item)\n\n    def _get_serialized_fields(self, item, default_value=None, include_empty=None):\n        feature = []\n        feature.append((\"type\", \"Feature\"))\n        feature.append((\"id\", compute_hash(item)))\n        feature.append((\"properties\", item_to_properties(item)))\n\n        lat = item.get(\"lat\")\n        lon = item.get(\"lon\")\n        geometry = item.get(\"geometry\")\n        if lat and lon and not geometry:\n            try:\n                geometry = {\n                    \"type\": \"Point\",\n                    \"coordinates\": [float(item[\"lon\"]), float(item[\"lat\"])],\n                }\n            except ValueError:\n                logging.warning(\"Couldn't convert lat (%s) and lon (%s) to float\", lat, lon)\n        feature.append((\"geometry\", geometry))\n\n        return feature\n\n    def write_geojson_header(self):\n        header = io.StringIO()\n        header.write('{\"type\":\"FeatureCollection\",\"dataset_attributes\":')\n        json.dump(\n            get_dataset_attributes(self.spider_name), header, ensure_ascii=False, separators=(\",\", \":\"), sort_keys=True\n        )\n        header.write(',\"features\":[\\n')\n        self.file.write(to_bytes(header.getvalue(), self.encoding))\n\n    def finish_exporting(self):\n        self.file.write(b\"\\n]}\\n\")\n", "path": "locations/exporters/geojson.py"}]}