We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
StructuredDataSpider's wanted_types duplicates items
If a JSON-LD contains e.g. `"@type":["HomeGoodsStore","FurnitureStore"]`, then StructuredDataSpider should only consider the item once when given `wanted_types = ["HomeGoodsStore", "FurnitureStore"]`.

(If there's really some reason to handle the item once for each of the types, LinkedDataParser can be invoked directly; if the site contains two distinct items, e.g. "Store" and "Pharmacy", then those are expressed as separate items, not two types on the same item.)

--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `locations/structured_data_spider.py`
Content:
```
1 import re
2 
3 from scrapy import Spider
4 
5 from locations.linked_data_parser import LinkedDataParser
6 from locations.microdata_parser import MicrodataParser
7 
8 
9 def extract_email(item, response):
10     for link in response.xpath("//a[contains(@href, 'mailto')]/@href").getall():
11         link = link.strip()
12         if link.startswith("mailto:"):
13             item["email"] = link.replace("mailto:", "")
14             return
15 
16 
17 def extract_phone(item, response):
18     for link in response.xpath("//a[contains(@href, 'tel')]/@href").getall():
19         link = link.strip()
20         if link.startswith("tel:"):
21 
22             item["phone"] = link.replace("tel:", "")
23             return
24 
25 
26 def extract_twitter(item, response):
27     if twitter := response.xpath('//meta[@name="twitter:site"]/@content').get():
28         item["twitter"] = twitter.strip()
29 
30 
31 def extract_image(item, response):
32     if image := response.xpath('//meta[@name="twitter:image"]/@content').get():
33         item["image"] = image.strip()
34         return
35     if image := response.xpath('//meta[@name="og:image"]/@content').get():
36         item["image"] = image.strip()
37 
38 
39 class StructuredDataSpider(Spider):
40 
41     wanted_types = []
42     search_for_email = True
43     search_for_phone = True
44     search_for_twitter = True
45     search_for_image = True
46 
47     def parse_sd(self, response):
48         MicrodataParser.convert_to_json_ld(response)
49         for wanted_type in self.wanted_types:
50             if item := LinkedDataParser.parse(response, wanted_type):
51 
52                 if item["ref"] is None:
53                     if hasattr(self, "rules"):
54                         # Attempt to pull a match from CrawlSpider.rules
55                         for rule in getattr(self, "rules"):
56                             for allow in rule.link_extractor.allow_res:
57                                 if match := re.match(allow, response.url):
58                                     if len(match.groups()) > 0:
59                                         item["ref"] = match.group(1)
60                     elif hasattr(self, "sitemap_rules"):
61                         # Attempt to pull a match from SitemapSpider.sitemap_rules
62                         for rule in getattr(self, "sitemap_rules"):
63                             if match := re.match(rule[0], response.url):
64                                 if len(match.groups()) > 0:
65                                     item["ref"] = match.group(1)
66 
67                     if item["ref"] is None:
68                         item["ref"] = response.url
69 
70                 if self.search_for_email and item["email"] is None:
71                     extract_email(item, response)
72 
73                 if self.search_for_phone and item["phone"] is None:
74                     extract_phone(item, response)
75 
76                 if self.search_for_twitter and item.get("twitter") is None:
77                     extract_twitter(item, response)
78 
79                 if self.search_for_image and item.get("image") is None:
80                     extract_image(item, response)
81 
82                 yield from self.inspect_item(item, response)
83 
84     def inspect_item(self, item, response):
85         """Override with any additional processing on the item."""
86         yield item
87
```

Path: `locations/linked_data_parser.py`
Content:
```
1 import json
2 
3 from locations.hours import OpeningHours
4 from locations.items import GeojsonPointItem
5 
6 
7 class LinkedDataParser(object):
8     @staticmethod
9     def iter_linked_data(response):
10         lds = response.xpath('//script[@type="application/ld+json"]//text()').getall()
11         for ld in lds:
12             try:
13                 ld_obj = json.loads(ld, strict=False)
14             except json.decoder.JSONDecodeError:
15                 continue
16 
17             if isinstance(ld_obj, dict):
18                 if "@graph" in ld_obj:
19                     yield from ld_obj["@graph"]
20                 else:
21                     yield ld_obj
22             elif isinstance(ld_obj, list):
23                 yield from ld_obj
24             else:
25                 raise TypeError(ld_obj)
26 
27     @staticmethod
28     def find_linked_data(response, wanted_type) -> {}:
29         for ld_obj in LinkedDataParser.iter_linked_data(response):
30             if not ld_obj.get("@type"):
31                 continue
32 
33             types = ld_obj["@type"]
34 
35             if not isinstance(types, list):
36                 types = [types]
37 
38             for t in types:
39                 if LinkedDataParser.check_type(t, wanted_type, default=False):
40                     return ld_obj
41 
42     @staticmethod
43     def parse_ld(ld) -> GeojsonPointItem:
44         item = GeojsonPointItem()
45 
46         if (
47             (geo := ld.get("geo"))
48             or "location" in ld
49             and (geo := ld["location"].get("geo"))
50         ):
51             if isinstance(geo, list):
52                 geo = geo[0]
53 
54             if LinkedDataParser.check_type(geo.get("@type"), "GeoCoordinates"):
55                 item["lat"] = LinkedDataParser.get_clean(geo, "latitude")
56                 item["lon"] = LinkedDataParser.get_clean(geo, "longitude")
57 
58         item["name"] = LinkedDataParser.get_clean(ld, "name")
59 
60         if addr := LinkedDataParser.get_clean(ld, "address"):
61             if isinstance(addr, list):
62                 addr = addr[0]
63 
64             if isinstance(addr, str):
65                 item["addr_full"] = addr
66             elif isinstance(addr, dict):
67                 if LinkedDataParser.check_type(addr.get("@type"), "PostalAddress"):
68                     item["street_address"] = LinkedDataParser.get_case_insensitive(
69                         addr, "streetAddress"
70                     )
71                     item["city"] = LinkedDataParser.get_case_insensitive(
72                         addr, "addressLocality"
73                     )
74                     item["state"] = LinkedDataParser.get_case_insensitive(
75                         addr, "addressRegion"
76                     )
77                     item["postcode"] = LinkedDataParser.get_case_insensitive(
78                         addr, "postalCode"
79                     )
80                     country = LinkedDataParser.get_case_insensitive(
81                         addr, "addressCountry"
82                     )
83 
84                     if isinstance(country, str):
85                         item["country"] = country
86                     elif isinstance(country, dict):
87                         if LinkedDataParser.check_type(country.get("@type"), "Country"):
88                             item["country"] = country.get("name")
89 
90                     # Common mistake to put "telephone" in "address"
91                     item["phone"] = LinkedDataParser.get_clean(addr, "telephone")
92 
93         if item.get("phone") is None:
94             item["phone"] = LinkedDataParser.get_clean(ld, "telephone")
95 
96         if isinstance(item["phone"], list):
97             item["phone"] = item["phone"][0]
98 
99         if isinstance(item["phone"], str):
100             item["phone"] = item["phone"].replace("tel:", "")
101 
102         item["email"] = LinkedDataParser.get_clean(ld, "email")
103 
104         if isinstance(item["email"], str):
105             item["email"] = item["email"].replace("mailto:", "")
106 
107         item["website"] = ld.get("url")
108 
109         try:
110             oh = OpeningHours()
111             oh.from_linked_data(ld)
112             item["opening_hours"] = oh.as_opening_hours()
113         except:
114             pass
115 
116         if image := ld.get("image"):
117             if isinstance(image, list):
118                 image = image[0]
119 
120             if isinstance(image, str):
121                 item["image"] = image
122             elif isinstance(image, dict):
123                 if LinkedDataParser.check_type(image.get("@type"), "ImageObject"):
124                     item["image"] = image.get("contentUrl")
125 
126         item["ref"] = ld.get("branchCode")
127         if item["ref"] is None or item["ref"] == "":
128             item["ref"] = ld.get("@id")
129 
130         if item["ref"] == "":
131             item["ref"] = None
132 
133         return item
134 
135     @staticmethod
136     def parse(response, wanted_type) -> GeojsonPointItem:
137         ld_item = LinkedDataParser.find_linked_data(response, wanted_type)
138         if ld_item:
139             item = LinkedDataParser.parse_ld(ld_item)
140 
141             if item["website"] is None:
142                 item["website"] = response.url
143             elif item["website"] == "":
144                 item["website"] = response.url
145             elif item["website"][0] == "/":
146                 item["website"] = response.url
147             elif item["website"].startswith("www"):
148                 item["website"] = "https://" + item["website"]
149 
150             return item
151 
152     @staticmethod
153     def get_clean(obj, key):
154         if value := obj.get(key):
155             if isinstance(value, str):
156                 if value == "null":
157                     return None
158                 return value.strip()
159             return value
160 
161     @staticmethod
162     def get_case_insensitive(obj, key):
163         # Prioritise the case correct key
164         if value := LinkedDataParser.get_clean(obj, key):
165             return value
166 
167         for real_key in obj:
168             if real_key.lower() == key.lower():
169                 return LinkedDataParser.get_clean(obj, real_key)
170 
171     @staticmethod
172     def check_type(type: str, wanted_type: str, default: bool = True) -> bool:
173         if default and type is None:
174             return True
175 
176         return (
177             type.lower()
178             .replace("http://", "")
179             .replace("https://", "")
180             .replace("schema.org/", "")
181             == wanted_type.lower()
182         )
183
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/locations/linked_data_parser.py b/locations/linked_data_parser.py
--- a/locations/linked_data_parser.py
+++ b/locations/linked_data_parser.py
@@ -35,8 +35,22 @@
             if not isinstance(types, list):
                 types = [types]
 
-            for t in types:
-                if LinkedDataParser.check_type(t, wanted_type, default=False):
+            types = [LinkedDataParser.clean_type(t) for t in types]
+
+            if isinstance(wanted_type, list):
+                wanted_types = wanted_type
+            else:
+                wanted_types = [wanted_type]
+
+            wanted_types = [LinkedDataParser.clean_type(t) for t in wanted_types]
+
+            for wanted_type in wanted_types:
+                valid_type = True
+                for t in types:
+                    if not t in wanted_types:
+                        valid_type = False
+
+                if valid_type:
                     return ld_obj
 
     @staticmethod
@@ -173,10 +187,13 @@
         if default and type is None:
             return True
 
+        return LinkedDataParser.clean_type(type) == wanted_type.lower()
+
+    @staticmethod
+    def clean_type(type: str) -> str:
         return (
             type.lower()
             .replace("http://", "")
             .replace("https://", "")
             .replace("schema.org/", "")
-            == wanted_type.lower()
         )
diff --git a/locations/structured_data_spider.py b/locations/structured_data_spider.py
--- a/locations/structured_data_spider.py
+++ b/locations/structured_data_spider.py
@@ -38,7 +38,19 @@
 
 class StructuredDataSpider(Spider):
 
-    wanted_types = []
+    wanted_types = [
+        "LocalBusiness",
+        "Store",
+        "Restaurant",
+        "BankOrCreditUnion",
+        "GroceryStore",
+        "FastFoodRestaurant",
+        "Hotel",
+        "Place",
+        "ClothingStore",
+        "DepartmentStore",
+        "HardwareStore",
+    ]
     search_for_email = True
     search_for_phone = True
     search_for_twitter = True
@@ -47,7 +59,10 @@
     def parse_sd(self, response):
         MicrodataParser.convert_to_json_ld(response)
         for wanted_type in self.wanted_types:
-            if item := LinkedDataParser.parse(response, wanted_type):
+            if ld_item := LinkedDataParser.find_linked_data(response, wanted_type):
+                self.pre_process_data(ld_item)
+
+                item = LinkedDataParser.parse_ld(ld_item)
 
                 if item["ref"] is None:
                     if hasattr(self, "rules"):
@@ -79,8 +94,16 @@
                 if self.search_for_image and item.get("image") is None:
                     extract_image(item, response)
 
-                yield from self.inspect_item(item, response)
+                yield from self.post_process_item(item, response, ld_item)
+
+    def pre_process_data(self, ld_data, **kwargs):
+        """Override with any pre-processing on the item."""
+        pass
+
+    def post_process_item(self, item, response, ld_data, **kwargs):
+        """Override with any post-processing on the item."""
+        yield from self.inspect_item(item, response)
 
     def inspect_item(self, item, response):
-        """Override with any additional processing on the item."""
+        """Deprecated, please use post_process_item(self, item, response, ld_data):"""
         yield item

{"golden_diff": "diff --git a/locations/linked_data_parser.py b/locations/linked_data_parser.py\n--- a/locations/linked_data_parser.py\n+++ b/locations/linked_data_parser.py\n@@ -35,8 +35,22 @@\n             if not isinstance(types, list):\n                 types = [types]\n \n-            for t in types:\n-                if LinkedDataParser.check_type(t, wanted_type, default=False):\n+            types = [LinkedDataParser.clean_type(t) for t in types]\n+\n+            if isinstance(wanted_type, list):\n+                wanted_types = wanted_type\n+            else:\n+                wanted_types = [wanted_type]\n+\n+            wanted_types = [LinkedDataParser.clean_type(t) for t in wanted_types]\n+\n+            for wanted_type in wanted_types:\n+                valid_type = True\n+                for t in types:\n+                    if not t in wanted_types:\n+                        valid_type = False\n+\n+                if valid_type:\n                     return ld_obj\n \n     @staticmethod\n@@ -173,10 +187,13 @@\n         if default and type is None:\n             return True\n \n+        return LinkedDataParser.clean_type(type) == wanted_type.lower()\n+\n+    @staticmethod\n+    def clean_type(type: str) -> str:\n         return (\n             type.lower()\n             .replace(\"http://\", \"\")\n             .replace(\"https://\", \"\")\n             .replace(\"schema.org/\", \"\")\n-            == wanted_type.lower()\n         )\ndiff --git a/locations/structured_data_spider.py b/locations/structured_data_spider.py\n--- a/locations/structured_data_spider.py\n+++ b/locations/structured_data_spider.py\n@@ -38,7 +38,19 @@\n \n class StructuredDataSpider(Spider):\n \n-    wanted_types = []\n+    wanted_types = [\n+        \"LocalBusiness\",\n+        \"Store\",\n+        \"Restaurant\",\n+        \"BankOrCreditUnion\",\n+        \"GroceryStore\",\n+        \"FastFoodRestaurant\",\n+        \"Hotel\",\n+        \"Place\",\n+        \"ClothingStore\",\n+        \"DepartmentStore\",\n+        \"HardwareStore\",\n+    ]\n     search_for_email = True\n     search_for_phone = True\n     search_for_twitter = True\n@@ -47,7 +59,10 @@\n     def parse_sd(self, response):\n         MicrodataParser.convert_to_json_ld(response)\n         for wanted_type in self.wanted_types:\n-            if item := LinkedDataParser.parse(response, wanted_type):\n+            if ld_item := LinkedDataParser.find_linked_data(response, wanted_type):\n+                self.pre_process_data(ld_item)\n+\n+                item = LinkedDataParser.parse_ld(ld_item)\n \n                 if item[\"ref\"] is None:\n                     if hasattr(self, \"rules\"):\n@@ -79,8 +94,16 @@\n                 if self.search_for_image and item.get(\"image\") is None:\n                     extract_image(item, response)\n \n-                yield from self.inspect_item(item, response)\n+                yield from self.post_process_item(item, response, ld_item)\n+\n+    def pre_process_data(self, ld_data, **kwargs):\n+        \"\"\"Override with any pre-processing on the item.\"\"\"\n+        pass\n+\n+    def post_process_item(self, item, response, ld_data, **kwargs):\n+        \"\"\"Override with any post-processing on the item.\"\"\"\n+        yield from self.inspect_item(item, response)\n \n     def inspect_item(self, item, response):\n-        \"\"\"Override with any additional processing on the item.\"\"\"\n+        \"\"\"Deprecated, please use post_process_item(self, item, response, ld_data):\"\"\"\n         yield item\n", "issue": "StructuredDataSpider's wanted_types duplicates items\nIf a JSON-LD contains e.g. `\"@type\":[\"HomeGoodsStore\",\"FurnitureStore\"]`, then StructuredDataSpider should only consider the item once when given `wanted_types = [\"HomeGoodsStore\", \"FurnitureStore\"]`.\r\n\r\n(If there's really some reason to handle the item once for each of the types, LinkedDataParser can be invoked directly; if the site contains two distinct items, e.g. \"Store\" and \"Pharmacy\", then those are expressed as separate items, not two types on the same item.)\n", "before_files": [{"content": "import re\n\nfrom scrapy import Spider\n\nfrom locations.linked_data_parser import LinkedDataParser\nfrom locations.microdata_parser import MicrodataParser\n\n\ndef extract_email(item, response):\n    for link in response.xpath(\"//a[contains(@href, 'mailto')]/@href\").getall():\n        link = link.strip()\n        if link.startswith(\"mailto:\"):\n            item[\"email\"] = link.replace(\"mailto:\", \"\")\n            return\n\n\ndef extract_phone(item, response):\n    for link in response.xpath(\"//a[contains(@href, 'tel')]/@href\").getall():\n        link = link.strip()\n        if link.startswith(\"tel:\"):\n\n            item[\"phone\"] = link.replace(\"tel:\", \"\")\n            return\n\n\ndef extract_twitter(item, response):\n    if twitter := response.xpath('//meta[@name=\"twitter:site\"]/@content').get():\n        item[\"twitter\"] = twitter.strip()\n\n\ndef extract_image(item, response):\n    if image := response.xpath('//meta[@name=\"twitter:image\"]/@content').get():\n        item[\"image\"] = image.strip()\n        return\n    if image := response.xpath('//meta[@name=\"og:image\"]/@content').get():\n        item[\"image\"] = image.strip()\n\n\nclass StructuredDataSpider(Spider):\n\n    wanted_types = []\n    search_for_email = True\n    search_for_phone = True\n    search_for_twitter = True\n    search_for_image = True\n\n    def parse_sd(self, response):\n        MicrodataParser.convert_to_json_ld(response)\n        for wanted_type in self.wanted_types:\n            if item := LinkedDataParser.parse(response, wanted_type):\n\n                if item[\"ref\"] is None:\n                    if hasattr(self, \"rules\"):\n                        # Attempt to pull a match from CrawlSpider.rules\n                        for rule in getattr(self, \"rules\"):\n                            for allow in rule.link_extractor.allow_res:\n                                if match := re.match(allow, response.url):\n                                    if len(match.groups()) > 0:\n                                        item[\"ref\"] = match.group(1)\n                    elif hasattr(self, \"sitemap_rules\"):\n                        # Attempt to pull a match from SitemapSpider.sitemap_rules\n                        for rule in getattr(self, \"sitemap_rules\"):\n                            if match := re.match(rule[0], response.url):\n                                if len(match.groups()) > 0:\n                                    item[\"ref\"] = match.group(1)\n\n                    if item[\"ref\"] is None:\n                        item[\"ref\"] = response.url\n\n                if self.search_for_email and item[\"email\"] is None:\n                    extract_email(item, response)\n\n                if self.search_for_phone and item[\"phone\"] is None:\n                    extract_phone(item, response)\n\n                if self.search_for_twitter and item.get(\"twitter\") is None:\n                    extract_twitter(item, response)\n\n                if self.search_for_image and item.get(\"image\") is None:\n                    extract_image(item, response)\n\n                yield from self.inspect_item(item, response)\n\n    def inspect_item(self, item, response):\n        \"\"\"Override with any additional processing on the item.\"\"\"\n        yield item\n", "path": "locations/structured_data_spider.py"}, {"content": "import json\n\nfrom locations.hours import OpeningHours\nfrom locations.items import GeojsonPointItem\n\n\nclass LinkedDataParser(object):\n    @staticmethod\n    def iter_linked_data(response):\n        lds = response.xpath('//script[@type=\"application/ld+json\"]//text()').getall()\n        for ld in lds:\n            try:\n                ld_obj = json.loads(ld, strict=False)\n            except json.decoder.JSONDecodeError:\n                continue\n\n            if isinstance(ld_obj, dict):\n                if \"@graph\" in ld_obj:\n                    yield from ld_obj[\"@graph\"]\n                else:\n                    yield ld_obj\n            elif isinstance(ld_obj, list):\n                yield from ld_obj\n            else:\n                raise TypeError(ld_obj)\n\n    @staticmethod\n    def find_linked_data(response, wanted_type) -> {}:\n        for ld_obj in LinkedDataParser.iter_linked_data(response):\n            if not ld_obj.get(\"@type\"):\n                continue\n\n            types = ld_obj[\"@type\"]\n\n            if not isinstance(types, list):\n                types = [types]\n\n            for t in types:\n                if LinkedDataParser.check_type(t, wanted_type, default=False):\n                    return ld_obj\n\n    @staticmethod\n    def parse_ld(ld) -> GeojsonPointItem:\n        item = GeojsonPointItem()\n\n        if (\n            (geo := ld.get(\"geo\"))\n            or \"location\" in ld\n            and (geo := ld[\"location\"].get(\"geo\"))\n        ):\n            if isinstance(geo, list):\n                geo = geo[0]\n\n            if LinkedDataParser.check_type(geo.get(\"@type\"), \"GeoCoordinates\"):\n                item[\"lat\"] = LinkedDataParser.get_clean(geo, \"latitude\")\n                item[\"lon\"] = LinkedDataParser.get_clean(geo, \"longitude\")\n\n        item[\"name\"] = LinkedDataParser.get_clean(ld, \"name\")\n\n        if addr := LinkedDataParser.get_clean(ld, \"address\"):\n            if isinstance(addr, list):\n                addr = addr[0]\n\n            if isinstance(addr, str):\n                item[\"addr_full\"] = addr\n            elif isinstance(addr, dict):\n                if LinkedDataParser.check_type(addr.get(\"@type\"), \"PostalAddress\"):\n                    item[\"street_address\"] = LinkedDataParser.get_case_insensitive(\n                        addr, \"streetAddress\"\n                    )\n                    item[\"city\"] = LinkedDataParser.get_case_insensitive(\n                        addr, \"addressLocality\"\n                    )\n                    item[\"state\"] = LinkedDataParser.get_case_insensitive(\n                        addr, \"addressRegion\"\n                    )\n                    item[\"postcode\"] = LinkedDataParser.get_case_insensitive(\n                        addr, \"postalCode\"\n                    )\n                    country = LinkedDataParser.get_case_insensitive(\n                        addr, \"addressCountry\"\n                    )\n\n                    if isinstance(country, str):\n                        item[\"country\"] = country\n                    elif isinstance(country, dict):\n                        if LinkedDataParser.check_type(country.get(\"@type\"), \"Country\"):\n                            item[\"country\"] = country.get(\"name\")\n\n                    # Common mistake to put \"telephone\" in \"address\"\n                    item[\"phone\"] = LinkedDataParser.get_clean(addr, \"telephone\")\n\n        if item.get(\"phone\") is None:\n            item[\"phone\"] = LinkedDataParser.get_clean(ld, \"telephone\")\n\n        if isinstance(item[\"phone\"], list):\n            item[\"phone\"] = item[\"phone\"][0]\n\n        if isinstance(item[\"phone\"], str):\n            item[\"phone\"] = item[\"phone\"].replace(\"tel:\", \"\")\n\n        item[\"email\"] = LinkedDataParser.get_clean(ld, \"email\")\n\n        if isinstance(item[\"email\"], str):\n            item[\"email\"] = item[\"email\"].replace(\"mailto:\", \"\")\n\n        item[\"website\"] = ld.get(\"url\")\n\n        try:\n            oh = OpeningHours()\n            oh.from_linked_data(ld)\n            item[\"opening_hours\"] = oh.as_opening_hours()\n        except:\n            pass\n\n        if image := ld.get(\"image\"):\n            if isinstance(image, list):\n                image = image[0]\n\n            if isinstance(image, str):\n                item[\"image\"] = image\n            elif isinstance(image, dict):\n                if LinkedDataParser.check_type(image.get(\"@type\"), \"ImageObject\"):\n                    item[\"image\"] = image.get(\"contentUrl\")\n\n        item[\"ref\"] = ld.get(\"branchCode\")\n        if item[\"ref\"] is None or item[\"ref\"] == \"\":\n            item[\"ref\"] = ld.get(\"@id\")\n\n        if item[\"ref\"] == \"\":\n            item[\"ref\"] = None\n\n        return item\n\n    @staticmethod\n    def parse(response, wanted_type) -> GeojsonPointItem:\n        ld_item = LinkedDataParser.find_linked_data(response, wanted_type)\n        if ld_item:\n            item = LinkedDataParser.parse_ld(ld_item)\n\n            if item[\"website\"] is None:\n                item[\"website\"] = response.url\n            elif item[\"website\"] == \"\":\n                item[\"website\"] = response.url\n            elif item[\"website\"][0] == \"/\":\n                item[\"website\"] = response.url\n            elif item[\"website\"].startswith(\"www\"):\n                item[\"website\"] = \"https://\" + item[\"website\"]\n\n            return item\n\n    @staticmethod\n    def get_clean(obj, key):\n        if value := obj.get(key):\n            if isinstance(value, str):\n                if value == \"null\":\n                    return None\n                return value.strip()\n            return value\n\n    @staticmethod\n    def get_case_insensitive(obj, key):\n        # Prioritise the case correct key\n        if value := LinkedDataParser.get_clean(obj, key):\n            return value\n\n        for real_key in obj:\n            if real_key.lower() == key.lower():\n                return LinkedDataParser.get_clean(obj, real_key)\n\n    @staticmethod\n    def check_type(type: str, wanted_type: str, default: bool = True) -> bool:\n        if default and type is None:\n            return True\n\n        return (\n            type.lower()\n            .replace(\"http://\", \"\")\n            .replace(\"https://\", \"\")\n            .replace(\"schema.org/\", \"\")\n            == wanted_type.lower()\n        )\n", "path": "locations/linked_data_parser.py"}], "after_files": [{"content": "import re\n\nfrom scrapy import Spider\n\nfrom locations.linked_data_parser import LinkedDataParser\nfrom locations.microdata_parser import MicrodataParser\n\n\ndef extract_email(item, response):\n    for link in response.xpath(\"//a[contains(@href, 'mailto')]/@href\").getall():\n        link = link.strip()\n        if link.startswith(\"mailto:\"):\n            item[\"email\"] = link.replace(\"mailto:\", \"\")\n            return\n\n\ndef extract_phone(item, response):\n    for link in response.xpath(\"//a[contains(@href, 'tel')]/@href\").getall():\n        link = link.strip()\n        if link.startswith(\"tel:\"):\n\n            item[\"phone\"] = link.replace(\"tel:\", \"\")\n            return\n\n\ndef extract_twitter(item, response):\n    if twitter := response.xpath('//meta[@name=\"twitter:site\"]/@content').get():\n        item[\"twitter\"] = twitter.strip()\n\n\ndef extract_image(item, response):\n    if image := response.xpath('//meta[@name=\"twitter:image\"]/@content').get():\n        item[\"image\"] = image.strip()\n        return\n    if image := response.xpath('//meta[@name=\"og:image\"]/@content').get():\n        item[\"image\"] = image.strip()\n\n\nclass StructuredDataSpider(Spider):\n\n    wanted_types = [\n        \"LocalBusiness\",\n        \"Store\",\n        \"Restaurant\",\n        \"BankOrCreditUnion\",\n        \"GroceryStore\",\n        \"FastFoodRestaurant\",\n        \"Hotel\",\n        \"Place\",\n        \"ClothingStore\",\n        \"DepartmentStore\",\n        \"HardwareStore\",\n    ]\n    search_for_email = True\n    search_for_phone = True\n    search_for_twitter = True\n    search_for_image = True\n\n    def parse_sd(self, response):\n        MicrodataParser.convert_to_json_ld(response)\n        for wanted_type in self.wanted_types:\n            if ld_item := LinkedDataParser.find_linked_data(response, wanted_type):\n                self.pre_process_data(ld_item)\n\n                item = LinkedDataParser.parse_ld(ld_item)\n\n                if item[\"ref\"] is None:\n                    if hasattr(self, \"rules\"):\n                        # Attempt to pull a match from CrawlSpider.rules\n                        for rule in getattr(self, \"rules\"):\n                            for allow in rule.link_extractor.allow_res:\n                                if match := re.match(allow, response.url):\n                                    if len(match.groups()) > 0:\n                                        item[\"ref\"] = match.group(1)\n                    elif hasattr(self, \"sitemap_rules\"):\n                        # Attempt to pull a match from SitemapSpider.sitemap_rules\n                        for rule in getattr(self, \"sitemap_rules\"):\n                            if match := re.match(rule[0], response.url):\n                                if len(match.groups()) > 0:\n                                    item[\"ref\"] = match.group(1)\n\n                    if item[\"ref\"] is None:\n                        item[\"ref\"] = response.url\n\n                if self.search_for_email and item[\"email\"] is None:\n                    extract_email(item, response)\n\n                if self.search_for_phone and item[\"phone\"] is None:\n                    extract_phone(item, response)\n\n                if self.search_for_twitter and item.get(\"twitter\") is None:\n                    extract_twitter(item, response)\n\n                if self.search_for_image and item.get(\"image\") is None:\n                    extract_image(item, response)\n\n                yield from self.post_process_item(item, response, ld_item)\n\n    def pre_process_data(self, ld_data, **kwargs):\n        \"\"\"Override with any pre-processing on the item.\"\"\"\n        pass\n\n    def post_process_item(self, item, response, ld_data, **kwargs):\n        \"\"\"Override with any post-processing on the item.\"\"\"\n        yield from self.inspect_item(item, response)\n\n    def inspect_item(self, item, response):\n        \"\"\"Deprecated, please use post_process_item(self, item, response, ld_data):\"\"\"\n        yield item\n", "path": "locations/structured_data_spider.py"}, {"content": "import json\n\nfrom locations.hours import OpeningHours\nfrom locations.items import GeojsonPointItem\n\n\nclass LinkedDataParser(object):\n    @staticmethod\n    def iter_linked_data(response):\n        lds = response.xpath('//script[@type=\"application/ld+json\"]//text()').getall()\n        for ld in lds:\n            try:\n                ld_obj = json.loads(ld, strict=False)\n            except json.decoder.JSONDecodeError:\n                continue\n\n            if isinstance(ld_obj, dict):\n                if \"@graph\" in ld_obj:\n                    yield from ld_obj[\"@graph\"]\n                else:\n                    yield ld_obj\n            elif isinstance(ld_obj, list):\n                yield from ld_obj\n            else:\n                raise TypeError(ld_obj)\n\n    @staticmethod\n    def find_linked_data(response, wanted_type) -> {}:\n        for ld_obj in LinkedDataParser.iter_linked_data(response):\n            if not ld_obj.get(\"@type\"):\n                continue\n\n            types = ld_obj[\"@type\"]\n\n            if not isinstance(types, list):\n                types = [types]\n\n            types = [LinkedDataParser.clean_type(t) for t in types]\n\n            if isinstance(wanted_type, list):\n                wanted_types = wanted_type\n            else:\n                wanted_types = [wanted_type]\n\n            wanted_types = [LinkedDataParser.clean_type(t) for t in wanted_types]\n\n            for wanted_type in wanted_types:\n                valid_type = True\n                for t in types:\n                    if not t in wanted_types:\n                        valid_type = False\n\n                if valid_type:\n                    return ld_obj\n\n    @staticmethod\n    def parse_ld(ld) -> GeojsonPointItem:\n        item = GeojsonPointItem()\n\n        if (\n            (geo := ld.get(\"geo\"))\n            or \"location\" in ld\n            and (geo := ld[\"location\"].get(\"geo\"))\n        ):\n            if isinstance(geo, list):\n                geo = geo[0]\n\n            if LinkedDataParser.check_type(geo.get(\"@type\"), \"GeoCoordinates\"):\n                item[\"lat\"] = LinkedDataParser.get_clean(geo, \"latitude\")\n                item[\"lon\"] = LinkedDataParser.get_clean(geo, \"longitude\")\n\n        item[\"name\"] = LinkedDataParser.get_clean(ld, \"name\")\n\n        if addr := LinkedDataParser.get_clean(ld, \"address\"):\n            if isinstance(addr, list):\n                addr = addr[0]\n\n            if isinstance(addr, str):\n                item[\"addr_full\"] = addr\n            elif isinstance(addr, dict):\n                if LinkedDataParser.check_type(addr.get(\"@type\"), \"PostalAddress\"):\n                    item[\"street_address\"] = LinkedDataParser.get_case_insensitive(\n                        addr, \"streetAddress\"\n                    )\n                    item[\"city\"] = LinkedDataParser.get_case_insensitive(\n                        addr, \"addressLocality\"\n                    )\n                    item[\"state\"] = LinkedDataParser.get_case_insensitive(\n                        addr, \"addressRegion\"\n                    )\n                    item[\"postcode\"] = LinkedDataParser.get_case_insensitive(\n                        addr, \"postalCode\"\n                    )\n                    country = LinkedDataParser.get_case_insensitive(\n                        addr, \"addressCountry\"\n                    )\n\n                    if isinstance(country, str):\n                        item[\"country\"] = country\n                    elif isinstance(country, dict):\n                        if LinkedDataParser.check_type(country.get(\"@type\"), \"Country\"):\n                            item[\"country\"] = country.get(\"name\")\n\n                    # Common mistake to put \"telephone\" in \"address\"\n                    item[\"phone\"] = LinkedDataParser.get_clean(addr, \"telephone\")\n\n        if item.get(\"phone\") is None:\n            item[\"phone\"] = LinkedDataParser.get_clean(ld, \"telephone\")\n\n        if isinstance(item[\"phone\"], list):\n            item[\"phone\"] = item[\"phone\"][0]\n\n        if isinstance(item[\"phone\"], str):\n            item[\"phone\"] = item[\"phone\"].replace(\"tel:\", \"\")\n\n        item[\"email\"] = LinkedDataParser.get_clean(ld, \"email\")\n\n        if isinstance(item[\"email\"], str):\n            item[\"email\"] = item[\"email\"].replace(\"mailto:\", \"\")\n\n        item[\"website\"] = ld.get(\"url\")\n\n        try:\n            oh = OpeningHours()\n            oh.from_linked_data(ld)\n            item[\"opening_hours\"] = oh.as_opening_hours()\n        except:\n            pass\n\n        if image := ld.get(\"image\"):\n            if isinstance(image, list):\n                image = image[0]\n\n            if isinstance(image, str):\n                item[\"image\"] = image\n            elif isinstance(image, dict):\n                if LinkedDataParser.check_type(image.get(\"@type\"), \"ImageObject\"):\n                    item[\"image\"] = image.get(\"contentUrl\")\n\n        item[\"ref\"] = ld.get(\"branchCode\")\n        if item[\"ref\"] is None or item[\"ref\"] == \"\":\n            item[\"ref\"] = ld.get(\"@id\")\n\n        if item[\"ref\"] == \"\":\n            item[\"ref\"] = None\n\n        return item\n\n    @staticmethod\n    def parse(response, wanted_type) -> GeojsonPointItem:\n        ld_item = LinkedDataParser.find_linked_data(response, wanted_type)\n        if ld_item:\n            item = LinkedDataParser.parse_ld(ld_item)\n\n            if item[\"website\"] is None:\n                item[\"website\"] = response.url\n            elif item[\"website\"] == \"\":\n                item[\"website\"] = response.url\n            elif item[\"website\"][0] == \"/\":\n                item[\"website\"] = response.url\n            elif item[\"website\"].startswith(\"www\"):\n                item[\"website\"] = \"https://\" + item[\"website\"]\n\n            return item\n\n    @staticmethod\n    def get_clean(obj, key):\n        if value := obj.get(key):\n            if isinstance(value, str):\n                if value == \"null\":\n                    return None\n                return value.strip()\n            return value\n\n    @staticmethod\n    def get_case_insensitive(obj, key):\n        # Prioritise the case correct key\n        if value := LinkedDataParser.get_clean(obj, key):\n            return value\n\n        for real_key in obj:\n            if real_key.lower() == key.lower():\n                return LinkedDataParser.get_clean(obj, real_key)\n\n    @staticmethod\n    def check_type(type: str, wanted_type: str, default: bool = True) -> bool:\n        if default and type is None:\n            return True\n\n        return LinkedDataParser.clean_type(type) == wanted_type.lower()\n\n    @staticmethod\n    def clean_type(type: str) -> str:\n        return (\n            type.lower()\n            .replace(\"http://\", \"\")\n            .replace(\"https://\", \"\")\n            .replace(\"schema.org/\", \"\")\n        )\n", "path": "locations/linked_data_parser.py"}]}