prompt: We are currently solving the following issue within our repository. Here is the issue text:
  --- BEGIN ISSUE ---
  toolstation spider now not returning any UK stores
  The most recent run of the toolstation.py spider from 2023-05-15 has lost about 550 stores compared to the previous run from 2023-04-15. This corresponds to all the UK branches (on the toolstation.com website).
  
  It looks like toolstation.com has changed its mapping provider (from Google to Woosmap, which ironically uses an OSM basemap) and as a result the machine readable store info formatting has changed. The ATP spider now fails to find the expected JS script fragment, throws an error, and doesn't return the branch. The .fr and .nl branches still use a Google map on their branch pages, so the spider still works for them (at least for now).
  
  I think the data we need for the UK branches is still there in a structured form, see e.g. a UK branch page at https://www.toolstation.com/branches/aldridge , but it will need some custom code to parse it out.
  
  --- END ISSUE ---
  
  Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
  --- BEGIN FILES ---
  Path: `locations/spiders/toolstation.py`
  Content:
  ```
  1 import json
  2 import re
  3 
  4 import scrapy
  5 
  6 from locations.dict_parser import DictParser
  7 
  8 
  9 class ToolstationSpider(scrapy.spiders.SitemapSpider):
  10     name = "toolstation"
  11     item_attributes = {"brand": "Toolstation", "brand_wikidata": "Q7824103"}
  12     sitemap_urls = [
  13         "https://www.toolstation.com/sitemap/branches.xml",
  14         "https://www.toolstation.fr/sitemap/branches.xml",
  15         "https://www.toolstation.nl/sitemap/branches.xml",
  16     ]
  17 
  18     def parse(self, response):
  19         pattern = re.compile(r"var store = (.*?)\n", re.MULTILINE | re.DOTALL)
  20         store = json.loads(response.xpath('//script[contains(., "var store")]/text()').re(pattern)[0])[0]
  21         item = DictParser.parse(store)
  22         item["website"] = response.url
  23         item["addr_full"] = store["address_text"].split("<br /><br />")[0]
  24         yield item
  25
  ```
  --- END FILES ---
  
  Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.
  
  Here is an example:
  
  ```diff
  diff --git a/examples/server_async.py b/examples/server_async.py
  --- a/examples/server_async.py
  +++ b/examples/server_async.py
  @@ -313,4 +313,4 @@
   
   
   if __name__ == "__main__":
  -    asyncio.run(run_async_server("."), debug=True)
  +    asyncio.run(run_async_server(), debug=True)
  diff --git a/examples/server_sync.py b/examples/server_sync.py
  --- a/examples/server_sync.py
  +++ b/examples/server_sync.py
  @@ -313,5 +313,5 @@
   
   
   if __name__ == "__main__":
  -    server = run_sync_server(".")
  +    server = run_sync_server()
       server.shutdown()
  
  ```
  
golden_diff: diff --git a/locations/spiders/toolstation.py b/locations/spiders/toolstation.py
  --- a/locations/spiders/toolstation.py
  +++ b/locations/spiders/toolstation.py
  @@ -1,9 +1,12 @@
   import json
   import re
   
  +import chompjs
   import scrapy
   
   from locations.dict_parser import DictParser
  +from locations.hours import OpeningHours, day_range, sanitise_day
  +from locations.spiders.vapestore_gb import clean_address
   
   
   class ToolstationSpider(scrapy.spiders.SitemapSpider):
  @@ -14,11 +17,64 @@
           "https://www.toolstation.fr/sitemap/branches.xml",
           "https://www.toolstation.nl/sitemap/branches.xml",
       ]
  +    gm_pattern = re.compile(r"var store = (.*?)\n", re.MULTILINE | re.DOTALL)
  +    params_pattern = re.compile(r"function\(([_$\w,\s]+)\)")
  +    values_pattern = re.compile(r"}\((.+)\)\);")
  +    stores_pattern = re.compile(r"data:(\[.+\]),fe")
   
       def parse(self, response):
  -        pattern = re.compile(r"var store = (.*?)\n", re.MULTILINE | re.DOTALL)
  -        store = json.loads(response.xpath('//script[contains(., "var store")]/text()').re(pattern)[0])[0]
  -        item = DictParser.parse(store)
  -        item["website"] = response.url
  -        item["addr_full"] = store["address_text"].split("<br /><br />")[0]
  -        yield item
  +        if js := response.xpath('//script[contains(., "var store")]/text()').get():
  +            store = json.loads(re.search(self.gm_pattern, js).group(1))[0]
  +            item = DictParser.parse(store)
  +            item["website"] = response.url
  +            item["addr_full"] = clean_address(store["address_text"].split("<br /><br />")[0])
  +            yield item
  +        elif js := response.xpath('//script[contains(text(), "__NUXT__")]/text()').get():
  +            # stores is actually a JS function, so we have to parse the parameters and values
  +            params = re.search(self.params_pattern, js).group(1).split(",")
  +            values = chompjs.parse_js_object("[" + re.search(self.values_pattern, js).group(1) + "]")
  +            args = {}
  +            for i in range(0, len(params)):
  +                args[params[i]] = values[i]
  +
  +            store = chompjs.parse_js_object(re.search(self.stores_pattern, js).group(1))[0]["branch"]
  +            self.populate(store, args)
  +
  +            if store["status"] != 1:
  +                return
  +
  +            item = DictParser.parse(store)
  +            item["website"] = response.url
  +            item["addr_full"] = store["address_text"]
  +
  +            item["opening_hours"] = OpeningHours()
  +            for rule in store["opening_hours"]:
  +                days, times = rule.split(": ", 1)
  +                if "-" in days:
  +                    start_day, end_day = days.split("-")
  +                else:
  +                    start_day = end_day = days
  +                start_day = sanitise_day(start_day)
  +                end_day = sanitise_day(end_day)
  +                if start_day and end_day:
  +                    start_time, end_time = times.strip().split("-")
  +                    item["opening_hours"].add_days_range(
  +                        day_range(start_day, end_day), start_time, end_time, time_format="%H%M"
  +                    )
  +
  +            yield item
  +
  +    @staticmethod
  +    def populate(data: dict, args: dict):
  +        for key, value in data.items():
  +            if isinstance(value, str):
  +                if value in args:
  +                    data[key] = args[value]
  +            elif isinstance(value, list):
  +                for i, x in enumerate(value):
  +                    if isinstance(x, dict):
  +                        ToolstationSpider.populate(x, args)
  +                    elif x in args:
  +                        value[i] = args[x]
  +            elif isinstance(value, dict):
  +                ToolstationSpider.populate(value, args)
  
verification_info: {"golden_diff": "diff --git a/locations/spiders/toolstation.py b/locations/spiders/toolstation.py\n--- a/locations/spiders/toolstation.py\n+++ b/locations/spiders/toolstation.py\n@@ -1,9 +1,12 @@\n import json\n import re\n \n+import chompjs\n import scrapy\n \n from locations.dict_parser import DictParser\n+from locations.hours import OpeningHours, day_range, sanitise_day\n+from locations.spiders.vapestore_gb import clean_address\n \n \n class ToolstationSpider(scrapy.spiders.SitemapSpider):\n@@ -14,11 +17,64 @@\n         \"https://www.toolstation.fr/sitemap/branches.xml\",\n         \"https://www.toolstation.nl/sitemap/branches.xml\",\n     ]\n+    gm_pattern = re.compile(r\"var store = (.*?)\\n\", re.MULTILINE | re.DOTALL)\n+    params_pattern = re.compile(r\"function\\(([_$\\w,\\s]+)\\)\")\n+    values_pattern = re.compile(r\"}\\((.+)\\)\\);\")\n+    stores_pattern = re.compile(r\"data:(\\[.+\\]),fe\")\n \n     def parse(self, response):\n-        pattern = re.compile(r\"var store = (.*?)\\n\", re.MULTILINE | re.DOTALL)\n-        store = json.loads(response.xpath('//script[contains(., \"var store\")]/text()').re(pattern)[0])[0]\n-        item = DictParser.parse(store)\n-        item[\"website\"] = response.url\n-        item[\"addr_full\"] = store[\"address_text\"].split(\"<br /><br />\")[0]\n-        yield item\n+        if js := response.xpath('//script[contains(., \"var store\")]/text()').get():\n+            store = json.loads(re.search(self.gm_pattern, js).group(1))[0]\n+            item = DictParser.parse(store)\n+            item[\"website\"] = response.url\n+            item[\"addr_full\"] = clean_address(store[\"address_text\"].split(\"<br /><br />\")[0])\n+            yield item\n+        elif js := response.xpath('//script[contains(text(), \"__NUXT__\")]/text()').get():\n+            # stores is actually a JS function, so we have to parse the parameters and values\n+            params = re.search(self.params_pattern, js).group(1).split(\",\")\n+            values = chompjs.parse_js_object(\"[\" + re.search(self.values_pattern, js).group(1) + \"]\")\n+            args = {}\n+            for i in range(0, len(params)):\n+                args[params[i]] = values[i]\n+\n+            store = chompjs.parse_js_object(re.search(self.stores_pattern, js).group(1))[0][\"branch\"]\n+            self.populate(store, args)\n+\n+            if store[\"status\"] != 1:\n+                return\n+\n+            item = DictParser.parse(store)\n+            item[\"website\"] = response.url\n+            item[\"addr_full\"] = store[\"address_text\"]\n+\n+            item[\"opening_hours\"] = OpeningHours()\n+            for rule in store[\"opening_hours\"]:\n+                days, times = rule.split(\": \", 1)\n+                if \"-\" in days:\n+                    start_day, end_day = days.split(\"-\")\n+                else:\n+                    start_day = end_day = days\n+                start_day = sanitise_day(start_day)\n+                end_day = sanitise_day(end_day)\n+                if start_day and end_day:\n+                    start_time, end_time = times.strip().split(\"-\")\n+                    item[\"opening_hours\"].add_days_range(\n+                        day_range(start_day, end_day), start_time, end_time, time_format=\"%H%M\"\n+                    )\n+\n+            yield item\n+\n+    @staticmethod\n+    def populate(data: dict, args: dict):\n+        for key, value in data.items():\n+            if isinstance(value, str):\n+                if value in args:\n+                    data[key] = args[value]\n+            elif isinstance(value, list):\n+                for i, x in enumerate(value):\n+                    if isinstance(x, dict):\n+                        ToolstationSpider.populate(x, args)\n+                    elif x in args:\n+                        value[i] = args[x]\n+            elif isinstance(value, dict):\n+                ToolstationSpider.populate(value, args)\n", "issue": "toolstation spider now not returning any UK stores\nThe most recent run of the toolstation.py spider from 2023-05-15 has lost about 550 stores compared to the previous run from 2023-04-15. This corresponds to all the UK branches (on the toolstation.com website).\r\n\r\nIt looks like toolstation.com has changed its mapping provider (from Google to Woosmap, which ironically uses an OSM basemap) and as a result the machine readable store info formatting has changed. The ATP spider now fails to find the expected JS script fragment, throws an error, and doesn't return the branch. The .fr and .nl branches still use a Google map on their branch pages, so the spider still works for them (at least for now).\r\n\r\nI think the data we need for the UK branches is still there in a structured form, see e.g. a UK branch page at https://www.toolstation.com/branches/aldridge , but it will need some custom code to parse it out.\n", "before_files": [{"content": "import json\nimport re\n\nimport scrapy\n\nfrom locations.dict_parser import DictParser\n\n\nclass ToolstationSpider(scrapy.spiders.SitemapSpider):\n    name = \"toolstation\"\n    item_attributes = {\"brand\": \"Toolstation\", \"brand_wikidata\": \"Q7824103\"}\n    sitemap_urls = [\n        \"https://www.toolstation.com/sitemap/branches.xml\",\n        \"https://www.toolstation.fr/sitemap/branches.xml\",\n        \"https://www.toolstation.nl/sitemap/branches.xml\",\n    ]\n\n    def parse(self, response):\n        pattern = re.compile(r\"var store = (.*?)\\n\", re.MULTILINE | re.DOTALL)\n        store = json.loads(response.xpath('//script[contains(., \"var store\")]/text()').re(pattern)[0])[0]\n        item = DictParser.parse(store)\n        item[\"website\"] = response.url\n        item[\"addr_full\"] = store[\"address_text\"].split(\"<br /><br />\")[0]\n        yield item\n", "path": "locations/spiders/toolstation.py"}], "after_files": [{"content": "import json\nimport re\n\nimport chompjs\nimport scrapy\n\nfrom locations.dict_parser import DictParser\nfrom locations.hours import OpeningHours, day_range, sanitise_day\nfrom locations.spiders.vapestore_gb import clean_address\n\n\nclass ToolstationSpider(scrapy.spiders.SitemapSpider):\n    name = \"toolstation\"\n    item_attributes = {\"brand\": \"Toolstation\", \"brand_wikidata\": \"Q7824103\"}\n    sitemap_urls = [\n        \"https://www.toolstation.com/sitemap/branches.xml\",\n        \"https://www.toolstation.fr/sitemap/branches.xml\",\n        \"https://www.toolstation.nl/sitemap/branches.xml\",\n    ]\n    gm_pattern = re.compile(r\"var store = (.*?)\\n\", re.MULTILINE | re.DOTALL)\n    params_pattern = re.compile(r\"function\\(([_$\\w,\\s]+)\\)\")\n    values_pattern = re.compile(r\"}\\((.+)\\)\\);\")\n    stores_pattern = re.compile(r\"data:(\\[.+\\]),fe\")\n\n    def parse(self, response):\n        if js := response.xpath('//script[contains(., \"var store\")]/text()').get():\n            store = json.loads(re.search(self.gm_pattern, js).group(1))[0]\n            item = DictParser.parse(store)\n            item[\"website\"] = response.url\n            item[\"addr_full\"] = clean_address(store[\"address_text\"].split(\"<br /><br />\")[0])\n            yield item\n        elif js := response.xpath('//script[contains(text(), \"__NUXT__\")]/text()').get():\n            # stores is actually a JS function, so we have to parse the parameters and values\n            params = re.search(self.params_pattern, js).group(1).split(\",\")\n            values = chompjs.parse_js_object(\"[\" + re.search(self.values_pattern, js).group(1) + \"]\")\n            args = {}\n            for i in range(0, len(params)):\n                args[params[i]] = values[i]\n\n            store = chompjs.parse_js_object(re.search(self.stores_pattern, js).group(1))[0][\"branch\"]\n            self.populate(store, args)\n\n            if store[\"status\"] != 1:\n                return\n\n            item = DictParser.parse(store)\n            item[\"website\"] = response.url\n            item[\"addr_full\"] = store[\"address_text\"]\n\n            item[\"opening_hours\"] = OpeningHours()\n            for rule in store[\"opening_hours\"]:\n                days, times = rule.split(\": \", 1)\n                if \"-\" in days:\n                    start_day, end_day = days.split(\"-\")\n                else:\n                    start_day = end_day = days\n                start_day = sanitise_day(start_day)\n                end_day = sanitise_day(end_day)\n                if start_day and end_day:\n                    start_time, end_time = times.strip().split(\"-\")\n                    item[\"opening_hours\"].add_days_range(\n                        day_range(start_day, end_day), start_time, end_time, time_format=\"%H%M\"\n                    )\n\n            yield item\n\n    @staticmethod\n    def populate(data: dict, args: dict):\n        for key, value in data.items():\n            if isinstance(value, str):\n                if value in args:\n                    data[key] = args[value]\n            elif isinstance(value, list):\n                for i, x in enumerate(value):\n                    if isinstance(x, dict):\n                        ToolstationSpider.populate(x, args)\n                    elif x in args:\n                        value[i] = args[x]\n            elif isinstance(value, dict):\n                ToolstationSpider.populate(value, args)\n", "path": "locations/spiders/toolstation.py"}]}