We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
test-documentation: Fix output spam from external links
Currently, test-documentation run in a development environment (i.e. without `--skip-external-links`) prints like 2 screenfuls of errors like this:
```
2021-04-01 10:20:38 [documentation_crawler] ERROR: Please check link: https://github.com/zulip/zulip/commit/49dbd85a8985b12666087f9ea36acb6f7da0aa4f
2021-04-01 10:20:38 [documentation_crawler] ERROR: Please check link: https://github.com/zulip/zulip-desktop
2021-04-01 10:20:38 [documentation_crawler] ERROR: Please check link: https://github.com/zulip/zulip/issues/10976
```

I imagine this is really confusing for anyone new to our ReadTheDocs documentation.

Most of these are 429 errors because GitHub doesn't want automation hitting their servers all the time; we could probably just suppress most of them that fit a pattern that we expect to be statically correct (E.g. anything in github.com/zulip/ that is not a tree path).

--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `tools/documentation_crawler/documentation_crawler/spiders/common/spiders.py`
Content:
```
1 import json
2 import os
3 import re
4 from typing import Callable, Iterator, List, Optional, Union
5 
6 import scrapy
7 from scrapy.http import Request, Response
8 from scrapy.linkextractors import IGNORED_EXTENSIONS
9 from scrapy.linkextractors.lxmlhtml import LxmlLinkExtractor
10 from scrapy.spidermiddlewares.httperror import HttpError
11 from scrapy.utils.url import url_has_any_extension
12 from twisted.python.failure import Failure
13 
14 EXCLUDED_URLS = [
15     # Google calendar returns 404s on HEAD requests unconditionally
16     "https://calendar.google.com/calendar/embed?src=ktiduof4eoh47lmgcl2qunnc0o@group.calendar.google.com",
17     # Returns 409 errors to HEAD requests frequently
18     "https://medium.freecodecamp.org/",
19     # Returns 404 to HEAD requests unconditionally
20     "https://www.git-tower.com/blog/command-line-cheat-sheet/",
21     "https://marketplace.visualstudio.com/items?itemName=rafaelmaiolla.remote-vscode",
22     # Requires authentication
23     "https://circleci.com/gh/zulip/zulip/tree/master",
24     "https://circleci.com/gh/zulip/zulip/16617",
25     "https://www.linkedin.com/company/zulip-project",
26     # Returns 403 errors to HEAD requests
27     "https://giphy.com",
28     "https://giphy.com/apps/giphycapture",
29     "https://www.udemy.com/course/the-complete-react-native-and-redux-course/",
30 ]
31 
32 VNU_IGNORE = [
33     # Real errors that should be fixed.
34     r"Duplicate ID “[^”]*”\.",
35     r"The first occurrence of ID “[^”]*” was here\.",
36     r"Attribute “markdown” not allowed on element “div” at this point\.",
37     r"No “p” element in scope but a “p” end tag seen\.",
38     r"Element “div” not allowed as child of element “ul” in this context\. "
39     + r"\(Suppressing further errors from this subtree\.\)",
40     # Warnings that are probably less important.
41     r"The “type” attribute is unnecessary for JavaScript resources\.",
42 ]
43 VNU_IGNORE_REGEX = re.compile(r"|".join(VNU_IGNORE))
44 
45 DEPLOY_ROOT = os.path.abspath(os.path.join(__file__, "../../../../../.."))
46 
47 ZULIP_SERVER_GITHUB_FILE_URL_PREFIX = "https://github.com/zulip/zulip/blob/master"
48 ZULIP_SERVER_GITHUB_DIRECTORY_URL_PREFIX = "https://github.com/zulip/zulip/tree/master"
49 
50 
51 class BaseDocumentationSpider(scrapy.Spider):
52     name: Optional[str] = None
53     # Exclude domain address.
54     deny_domains: List[str] = []
55     start_urls: List[str] = []
56     deny: List[str] = []
57     file_extensions: List[str] = ["." + ext for ext in IGNORED_EXTENSIONS]
58     tags = ("a", "area", "img")
59     attrs = ("href", "src")
60 
61     def _has_extension(self, url: str) -> bool:
62         return url_has_any_extension(url, self.file_extensions)
63 
64     def _is_external_url(self, url: str) -> bool:
65         return url.startswith("http") or self._has_extension(url)
66 
67     def check_existing(self, response: Response) -> None:
68         self.log(response)
69 
70     def _is_external_link(self, url: str) -> bool:
71         if url.startswith("https://chat.zulip.org"):
72             # Since most chat.zulip.org URLs will be links to specific
73             # logged-in content that the spider cannot verify, or the
74             # homepage, there's no need to check those (which can
75             # cause errors when chat.zulip.org is being updated).
76             return True
77         if "zulip.readthedocs" in url or "zulip.com" in url or "zulip.org" in url:
78             # We want CI to check any links to Zulip sites.
79             return False
80         if (len(url) > 4 and url[:4] == "file") or ("localhost" in url):
81             # We also want CI to check any links to built documentation.
82             return False
83         if url.startswith(ZULIP_SERVER_GITHUB_FILE_URL_PREFIX) or url.startswith(
84             ZULIP_SERVER_GITHUB_DIRECTORY_URL_PREFIX
85         ):
86             # We can verify these links directly in the local git repo without making any requests to GitHub servers.
87             return False
88         if "github.com/zulip" in url:
89             # We want to check these links but due to rate limiting from GitHub, these checks often
90             # fail in the CI. Thus, we should treat these as external links for now.
91             # TODO: Figure out how to test github.com/zulip links in CI.
92             return True
93         return True
94 
95     def check_fragment(self, response: Response) -> None:
96         self.log(response)
97         xpath_template = "//*[@id='{fragment}' or @name='{fragment}']"
98         m = re.match(r".+\#(?P<fragment>.*)$", response.request.url)  # Get fragment value.
99         if not m:
100             return
101         fragment = m.group("fragment")
102         # Check fragment existing on response page.
103         if not response.selector.xpath(xpath_template.format(fragment=fragment)):
104             self.logger.error(
105                 "Fragment #%s is not found on page %s", fragment, response.request.url
106             )
107 
108     def _vnu_callback(self, url: str) -> Callable[[Response], None]:
109         def callback(response: Response) -> None:
110             vnu_out = json.loads(response.text)
111             for message in vnu_out["messages"]:
112                 if not VNU_IGNORE_REGEX.fullmatch(message["message"]):
113                     self.logger.error(
114                         '"%s":%d.%d-%d.%d: %s: %s',
115                         url,
116                         message.get("firstLine", message["lastLine"]),
117                         message.get("firstColumn", message["lastColumn"]),
118                         message["lastLine"],
119                         message["lastColumn"],
120                         message["type"],
121                         message["message"],
122                     )
123 
124         return callback
125 
126     def _make_requests(self, url: str) -> Iterator[Request]:
127         # These URLs are for Zulip's webapp, which with recent changes
128         # can be accessible without login an account.  While we do
129         # crawl documentation served by the webapp (E.g. /help/), we
130         # don't want to crawl the webapp itself, so we exclude these.
131         if (
132             url in ["http://localhost:9981/", "http://localhost:9981"]
133             or url.startswith("http://localhost:9981/#")
134             or url.startswith("http://localhost:9981#")
135         ):
136             return
137 
138         callback: Callable[[Response], Optional[Iterator[Request]]] = self.parse
139         dont_filter = False
140         method = "GET"
141         if self._is_external_url(url):
142             callback = self.check_existing
143             method = "HEAD"
144 
145             if url.startswith(ZULIP_SERVER_GITHUB_FILE_URL_PREFIX):
146                 file_path = url.replace(ZULIP_SERVER_GITHUB_FILE_URL_PREFIX, DEPLOY_ROOT)
147                 hash_index = file_path.find("#")
148                 if hash_index != -1:
149                     file_path = file_path[:hash_index]
150                 if not os.path.isfile(file_path):
151                     self.logger.error(
152                         "There is no local file associated with the GitHub URL: %s", url
153                     )
154                 return
155             elif url.startswith(ZULIP_SERVER_GITHUB_DIRECTORY_URL_PREFIX):
156                 dir_path = url.replace(ZULIP_SERVER_GITHUB_DIRECTORY_URL_PREFIX, DEPLOY_ROOT)
157                 if not os.path.isdir(dir_path):
158                     self.logger.error(
159                         "There is no local directory associated with the GitHub URL: %s", url
160                     )
161                 return
162         elif "#" in url:
163             dont_filter = True
164             callback = self.check_fragment
165         if getattr(self, "skip_external", False) and self._is_external_link(url):
166             return
167         yield Request(
168             url,
169             method=method,
170             callback=callback,
171             dont_filter=dont_filter,
172             errback=self.error_callback,
173         )
174 
175     def start_requests(self) -> Iterator[Request]:
176         for url in self.start_urls:
177             yield from self._make_requests(url)
178 
179     def parse(self, response: Response) -> Iterator[Request]:
180         self.log(response)
181 
182         if getattr(self, "validate_html", False):
183             yield Request(
184                 "http://127.0.0.1:9988/?out=json",
185                 method="POST",
186                 headers={"Content-Type": response.headers["Content-Type"]},
187                 body=response.body,
188                 callback=self._vnu_callback(response.url),
189                 errback=self.error_callback,
190             )
191 
192         for link in LxmlLinkExtractor(
193             deny_domains=self.deny_domains,
194             deny_extensions=["doc"],
195             tags=self.tags,
196             attrs=self.attrs,
197             deny=self.deny,
198             canonicalize=False,
199         ).extract_links(response):
200             yield from self._make_requests(link.url)
201 
202     def retry_request_with_get(self, request: Request) -> Iterator[Request]:
203         request.method = "GET"
204         request.dont_filter = True
205         yield request
206 
207     def exclude_error(self, url: str) -> bool:
208         return url in EXCLUDED_URLS
209 
210     def error_callback(self, failure: Failure) -> Optional[Union[Failure, Iterator[Request]]]:
211         if isinstance(failure.value, HttpError):
212             response = failure.value.response
213             if self.exclude_error(response.url):
214                 return None
215             if response.status == 405 and response.request.method == "HEAD":
216                 # Method 'HEAD' not allowed, repeat request with 'GET'
217                 return self.retry_request_with_get(response.request)
218             self.logger.error("Please check link: %s", response.request.url)
219 
220         return failure
221
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/tools/documentation_crawler/documentation_crawler/spiders/common/spiders.py b/tools/documentation_crawler/documentation_crawler/spiders/common/spiders.py
--- a/tools/documentation_crawler/documentation_crawler/spiders/common/spiders.py
+++ b/tools/documentation_crawler/documentation_crawler/spiders/common/spiders.py
@@ -2,6 +2,7 @@
 import os
 import re
 from typing import Callable, Iterator, List, Optional, Union
+from urllib.parse import urlparse
 
 import scrapy
 from scrapy.http import Request, Response
@@ -11,6 +12,15 @@
 from scrapy.utils.url import url_has_any_extension
 from twisted.python.failure import Failure
 
+EXCLUDED_DOMAINS = [
+    # Returns 429 Rate-Limited Errors
+    "github.com",
+    "gist.github.com",
+    # Returns 503 Errors
+    "www.amazon.com",
+    "gitlab.com",
+]
+
 EXCLUDED_URLS = [
     # Google calendar returns 404s on HEAD requests unconditionally
     "https://calendar.google.com/calendar/embed?src=ktiduof4eoh47lmgcl2qunnc0o@group.calendar.google.com",
@@ -19,6 +29,8 @@
     # Returns 404 to HEAD requests unconditionally
     "https://www.git-tower.com/blog/command-line-cheat-sheet/",
     "https://marketplace.visualstudio.com/items?itemName=rafaelmaiolla.remote-vscode",
+    "https://www.transifex.com/zulip/zulip/announcements/",
+    "https://marketplace.visualstudio.com/items?itemName=ms-vscode-remote.remote-ssh",
     # Requires authentication
     "https://circleci.com/gh/zulip/zulip/tree/master",
     "https://circleci.com/gh/zulip/zulip/16617",
@@ -164,6 +176,10 @@
             callback = self.check_fragment
         if getattr(self, "skip_external", False) and self._is_external_link(url):
             return
+        if urlparse(url).netloc in EXCLUDED_DOMAINS:
+            return
+        if url in EXCLUDED_URLS:
+            return
         yield Request(
             url,
             method=method,
@@ -204,13 +220,12 @@
         request.dont_filter = True
         yield request
 
-    def exclude_error(self, url: str) -> bool:
-        return url in EXCLUDED_URLS
-
     def error_callback(self, failure: Failure) -> Optional[Union[Failure, Iterator[Request]]]:
         if isinstance(failure.value, HttpError):
             response = failure.value.response
-            if self.exclude_error(response.url):
+            # Hack: The filtering above does not catch this URL,
+            # likely due to a redirect.
+            if urlparse(response.url).netloc == "idmsa.apple.com":
                 return None
             if response.status == 405 and response.request.method == "HEAD":
                 # Method 'HEAD' not allowed, repeat request with 'GET'

{"golden_diff": "diff --git a/tools/documentation_crawler/documentation_crawler/spiders/common/spiders.py b/tools/documentation_crawler/documentation_crawler/spiders/common/spiders.py\n--- a/tools/documentation_crawler/documentation_crawler/spiders/common/spiders.py\n+++ b/tools/documentation_crawler/documentation_crawler/spiders/common/spiders.py\n@@ -2,6 +2,7 @@\n import os\n import re\n from typing import Callable, Iterator, List, Optional, Union\n+from urllib.parse import urlparse\n \n import scrapy\n from scrapy.http import Request, Response\n@@ -11,6 +12,15 @@\n from scrapy.utils.url import url_has_any_extension\n from twisted.python.failure import Failure\n \n+EXCLUDED_DOMAINS = [\n+    # Returns 429 Rate-Limited Errors\n+    \"github.com\",\n+    \"gist.github.com\",\n+    # Returns 503 Errors\n+    \"www.amazon.com\",\n+    \"gitlab.com\",\n+]\n+\n EXCLUDED_URLS = [\n     # Google calendar returns 404s on HEAD requests unconditionally\n     \"https://calendar.google.com/calendar/embed?src=ktiduof4eoh47lmgcl2qunnc0o@group.calendar.google.com\",\n@@ -19,6 +29,8 @@\n     # Returns 404 to HEAD requests unconditionally\n     \"https://www.git-tower.com/blog/command-line-cheat-sheet/\",\n     \"https://marketplace.visualstudio.com/items?itemName=rafaelmaiolla.remote-vscode\",\n+    \"https://www.transifex.com/zulip/zulip/announcements/\",\n+    \"https://marketplace.visualstudio.com/items?itemName=ms-vscode-remote.remote-ssh\",\n     # Requires authentication\n     \"https://circleci.com/gh/zulip/zulip/tree/master\",\n     \"https://circleci.com/gh/zulip/zulip/16617\",\n@@ -164,6 +176,10 @@\n             callback = self.check_fragment\n         if getattr(self, \"skip_external\", False) and self._is_external_link(url):\n             return\n+        if urlparse(url).netloc in EXCLUDED_DOMAINS:\n+            return\n+        if url in EXCLUDED_URLS:\n+            return\n         yield Request(\n             url,\n             method=method,\n@@ -204,13 +220,12 @@\n         request.dont_filter = True\n         yield request\n \n-    def exclude_error(self, url: str) -> bool:\n-        return url in EXCLUDED_URLS\n-\n     def error_callback(self, failure: Failure) -> Optional[Union[Failure, Iterator[Request]]]:\n         if isinstance(failure.value, HttpError):\n             response = failure.value.response\n-            if self.exclude_error(response.url):\n+            # Hack: The filtering above does not catch this URL,\n+            # likely due to a redirect.\n+            if urlparse(response.url).netloc == \"idmsa.apple.com\":\n                 return None\n             if response.status == 405 and response.request.method == \"HEAD\":\n                 # Method 'HEAD' not allowed, repeat request with 'GET'\n", "issue": "test-documentation: Fix output spam from external links\nCurrently, test-documentation run in a development environment (i.e. without `--skip-external-links`) prints like 2 screenfuls of errors like this:\r\n```\r\n2021-04-01 10:20:38 [documentation_crawler] ERROR: Please check link: https://github.com/zulip/zulip/commit/49dbd85a8985b12666087f9ea36acb6f7da0aa4f\r\n2021-04-01 10:20:38 [documentation_crawler] ERROR: Please check link: https://github.com/zulip/zulip-desktop\r\n2021-04-01 10:20:38 [documentation_crawler] ERROR: Please check link: https://github.com/zulip/zulip/issues/10976\r\n```\r\n\r\nI imagine this is really confusing for anyone new to our ReadTheDocs documentation.\r\n\r\nMost of these are 429 errors because GitHub doesn't want automation hitting their servers all the time; we could probably just suppress most of them that fit a pattern that we expect to be statically correct (E.g. anything in github.com/zulip/ that is not a tree path).\n", "before_files": [{"content": "import json\nimport os\nimport re\nfrom typing import Callable, Iterator, List, Optional, Union\n\nimport scrapy\nfrom scrapy.http import Request, Response\nfrom scrapy.linkextractors import IGNORED_EXTENSIONS\nfrom scrapy.linkextractors.lxmlhtml import LxmlLinkExtractor\nfrom scrapy.spidermiddlewares.httperror import HttpError\nfrom scrapy.utils.url import url_has_any_extension\nfrom twisted.python.failure import Failure\n\nEXCLUDED_URLS = [\n    # Google calendar returns 404s on HEAD requests unconditionally\n    \"https://calendar.google.com/calendar/embed?src=ktiduof4eoh47lmgcl2qunnc0o@group.calendar.google.com\",\n    # Returns 409 errors to HEAD requests frequently\n    \"https://medium.freecodecamp.org/\",\n    # Returns 404 to HEAD requests unconditionally\n    \"https://www.git-tower.com/blog/command-line-cheat-sheet/\",\n    \"https://marketplace.visualstudio.com/items?itemName=rafaelmaiolla.remote-vscode\",\n    # Requires authentication\n    \"https://circleci.com/gh/zulip/zulip/tree/master\",\n    \"https://circleci.com/gh/zulip/zulip/16617\",\n    \"https://www.linkedin.com/company/zulip-project\",\n    # Returns 403 errors to HEAD requests\n    \"https://giphy.com\",\n    \"https://giphy.com/apps/giphycapture\",\n    \"https://www.udemy.com/course/the-complete-react-native-and-redux-course/\",\n]\n\nVNU_IGNORE = [\n    # Real errors that should be fixed.\n    r\"Duplicate ID \u201c[^\u201d]*\u201d\\.\",\n    r\"The first occurrence of ID \u201c[^\u201d]*\u201d was here\\.\",\n    r\"Attribute \u201cmarkdown\u201d not allowed on element \u201cdiv\u201d at this point\\.\",\n    r\"No \u201cp\u201d element in scope but a \u201cp\u201d end tag seen\\.\",\n    r\"Element \u201cdiv\u201d not allowed as child of element \u201cul\u201d in this context\\. \"\n    + r\"\\(Suppressing further errors from this subtree\\.\\)\",\n    # Warnings that are probably less important.\n    r\"The \u201ctype\u201d attribute is unnecessary for JavaScript resources\\.\",\n]\nVNU_IGNORE_REGEX = re.compile(r\"|\".join(VNU_IGNORE))\n\nDEPLOY_ROOT = os.path.abspath(os.path.join(__file__, \"../../../../../..\"))\n\nZULIP_SERVER_GITHUB_FILE_URL_PREFIX = \"https://github.com/zulip/zulip/blob/master\"\nZULIP_SERVER_GITHUB_DIRECTORY_URL_PREFIX = \"https://github.com/zulip/zulip/tree/master\"\n\n\nclass BaseDocumentationSpider(scrapy.Spider):\n    name: Optional[str] = None\n    # Exclude domain address.\n    deny_domains: List[str] = []\n    start_urls: List[str] = []\n    deny: List[str] = []\n    file_extensions: List[str] = [\".\" + ext for ext in IGNORED_EXTENSIONS]\n    tags = (\"a\", \"area\", \"img\")\n    attrs = (\"href\", \"src\")\n\n    def _has_extension(self, url: str) -> bool:\n        return url_has_any_extension(url, self.file_extensions)\n\n    def _is_external_url(self, url: str) -> bool:\n        return url.startswith(\"http\") or self._has_extension(url)\n\n    def check_existing(self, response: Response) -> None:\n        self.log(response)\n\n    def _is_external_link(self, url: str) -> bool:\n        if url.startswith(\"https://chat.zulip.org\"):\n            # Since most chat.zulip.org URLs will be links to specific\n            # logged-in content that the spider cannot verify, or the\n            # homepage, there's no need to check those (which can\n            # cause errors when chat.zulip.org is being updated).\n            return True\n        if \"zulip.readthedocs\" in url or \"zulip.com\" in url or \"zulip.org\" in url:\n            # We want CI to check any links to Zulip sites.\n            return False\n        if (len(url) > 4 and url[:4] == \"file\") or (\"localhost\" in url):\n            # We also want CI to check any links to built documentation.\n            return False\n        if url.startswith(ZULIP_SERVER_GITHUB_FILE_URL_PREFIX) or url.startswith(\n            ZULIP_SERVER_GITHUB_DIRECTORY_URL_PREFIX\n        ):\n            # We can verify these links directly in the local git repo without making any requests to GitHub servers.\n            return False\n        if \"github.com/zulip\" in url:\n            # We want to check these links but due to rate limiting from GitHub, these checks often\n            # fail in the CI. Thus, we should treat these as external links for now.\n            # TODO: Figure out how to test github.com/zulip links in CI.\n            return True\n        return True\n\n    def check_fragment(self, response: Response) -> None:\n        self.log(response)\n        xpath_template = \"//*[@id='{fragment}' or @name='{fragment}']\"\n        m = re.match(r\".+\\#(?P<fragment>.*)$\", response.request.url)  # Get fragment value.\n        if not m:\n            return\n        fragment = m.group(\"fragment\")\n        # Check fragment existing on response page.\n        if not response.selector.xpath(xpath_template.format(fragment=fragment)):\n            self.logger.error(\n                \"Fragment #%s is not found on page %s\", fragment, response.request.url\n            )\n\n    def _vnu_callback(self, url: str) -> Callable[[Response], None]:\n        def callback(response: Response) -> None:\n            vnu_out = json.loads(response.text)\n            for message in vnu_out[\"messages\"]:\n                if not VNU_IGNORE_REGEX.fullmatch(message[\"message\"]):\n                    self.logger.error(\n                        '\"%s\":%d.%d-%d.%d: %s: %s',\n                        url,\n                        message.get(\"firstLine\", message[\"lastLine\"]),\n                        message.get(\"firstColumn\", message[\"lastColumn\"]),\n                        message[\"lastLine\"],\n                        message[\"lastColumn\"],\n                        message[\"type\"],\n                        message[\"message\"],\n                    )\n\n        return callback\n\n    def _make_requests(self, url: str) -> Iterator[Request]:\n        # These URLs are for Zulip's webapp, which with recent changes\n        # can be accessible without login an account.  While we do\n        # crawl documentation served by the webapp (E.g. /help/), we\n        # don't want to crawl the webapp itself, so we exclude these.\n        if (\n            url in [\"http://localhost:9981/\", \"http://localhost:9981\"]\n            or url.startswith(\"http://localhost:9981/#\")\n            or url.startswith(\"http://localhost:9981#\")\n        ):\n            return\n\n        callback: Callable[[Response], Optional[Iterator[Request]]] = self.parse\n        dont_filter = False\n        method = \"GET\"\n        if self._is_external_url(url):\n            callback = self.check_existing\n            method = \"HEAD\"\n\n            if url.startswith(ZULIP_SERVER_GITHUB_FILE_URL_PREFIX):\n                file_path = url.replace(ZULIP_SERVER_GITHUB_FILE_URL_PREFIX, DEPLOY_ROOT)\n                hash_index = file_path.find(\"#\")\n                if hash_index != -1:\n                    file_path = file_path[:hash_index]\n                if not os.path.isfile(file_path):\n                    self.logger.error(\n                        \"There is no local file associated with the GitHub URL: %s\", url\n                    )\n                return\n            elif url.startswith(ZULIP_SERVER_GITHUB_DIRECTORY_URL_PREFIX):\n                dir_path = url.replace(ZULIP_SERVER_GITHUB_DIRECTORY_URL_PREFIX, DEPLOY_ROOT)\n                if not os.path.isdir(dir_path):\n                    self.logger.error(\n                        \"There is no local directory associated with the GitHub URL: %s\", url\n                    )\n                return\n        elif \"#\" in url:\n            dont_filter = True\n            callback = self.check_fragment\n        if getattr(self, \"skip_external\", False) and self._is_external_link(url):\n            return\n        yield Request(\n            url,\n            method=method,\n            callback=callback,\n            dont_filter=dont_filter,\n            errback=self.error_callback,\n        )\n\n    def start_requests(self) -> Iterator[Request]:\n        for url in self.start_urls:\n            yield from self._make_requests(url)\n\n    def parse(self, response: Response) -> Iterator[Request]:\n        self.log(response)\n\n        if getattr(self, \"validate_html\", False):\n            yield Request(\n                \"http://127.0.0.1:9988/?out=json\",\n                method=\"POST\",\n                headers={\"Content-Type\": response.headers[\"Content-Type\"]},\n                body=response.body,\n                callback=self._vnu_callback(response.url),\n                errback=self.error_callback,\n            )\n\n        for link in LxmlLinkExtractor(\n            deny_domains=self.deny_domains,\n            deny_extensions=[\"doc\"],\n            tags=self.tags,\n            attrs=self.attrs,\n            deny=self.deny,\n            canonicalize=False,\n        ).extract_links(response):\n            yield from self._make_requests(link.url)\n\n    def retry_request_with_get(self, request: Request) -> Iterator[Request]:\n        request.method = \"GET\"\n        request.dont_filter = True\n        yield request\n\n    def exclude_error(self, url: str) -> bool:\n        return url in EXCLUDED_URLS\n\n    def error_callback(self, failure: Failure) -> Optional[Union[Failure, Iterator[Request]]]:\n        if isinstance(failure.value, HttpError):\n            response = failure.value.response\n            if self.exclude_error(response.url):\n                return None\n            if response.status == 405 and response.request.method == \"HEAD\":\n                # Method 'HEAD' not allowed, repeat request with 'GET'\n                return self.retry_request_with_get(response.request)\n            self.logger.error(\"Please check link: %s\", response.request.url)\n\n        return failure\n", "path": "tools/documentation_crawler/documentation_crawler/spiders/common/spiders.py"}], "after_files": [{"content": "import json\nimport os\nimport re\nfrom typing import Callable, Iterator, List, Optional, Union\nfrom urllib.parse import urlparse\n\nimport scrapy\nfrom scrapy.http import Request, Response\nfrom scrapy.linkextractors import IGNORED_EXTENSIONS\nfrom scrapy.linkextractors.lxmlhtml import LxmlLinkExtractor\nfrom scrapy.spidermiddlewares.httperror import HttpError\nfrom scrapy.utils.url import url_has_any_extension\nfrom twisted.python.failure import Failure\n\nEXCLUDED_DOMAINS = [\n    # Returns 429 Rate-Limited Errors\n    \"github.com\",\n    \"gist.github.com\",\n    # Returns 503 Errors\n    \"www.amazon.com\",\n    \"gitlab.com\",\n]\n\nEXCLUDED_URLS = [\n    # Google calendar returns 404s on HEAD requests unconditionally\n    \"https://calendar.google.com/calendar/embed?src=ktiduof4eoh47lmgcl2qunnc0o@group.calendar.google.com\",\n    # Returns 409 errors to HEAD requests frequently\n    \"https://medium.freecodecamp.org/\",\n    # Returns 404 to HEAD requests unconditionally\n    \"https://www.git-tower.com/blog/command-line-cheat-sheet/\",\n    \"https://marketplace.visualstudio.com/items?itemName=rafaelmaiolla.remote-vscode\",\n    \"https://www.transifex.com/zulip/zulip/announcements/\",\n    \"https://marketplace.visualstudio.com/items?itemName=ms-vscode-remote.remote-ssh\",\n    # Requires authentication\n    \"https://circleci.com/gh/zulip/zulip/tree/master\",\n    \"https://circleci.com/gh/zulip/zulip/16617\",\n    \"https://www.linkedin.com/company/zulip-project\",\n    # Returns 403 errors to HEAD requests\n    \"https://giphy.com\",\n    \"https://giphy.com/apps/giphycapture\",\n    \"https://www.udemy.com/course/the-complete-react-native-and-redux-course/\",\n]\n\nVNU_IGNORE = [\n    # Real errors that should be fixed.\n    r\"Duplicate ID \u201c[^\u201d]*\u201d\\.\",\n    r\"The first occurrence of ID \u201c[^\u201d]*\u201d was here\\.\",\n    r\"Attribute \u201cmarkdown\u201d not allowed on element \u201cdiv\u201d at this point\\.\",\n    r\"No \u201cp\u201d element in scope but a \u201cp\u201d end tag seen\\.\",\n    r\"Element \u201cdiv\u201d not allowed as child of element \u201cul\u201d in this context\\. \"\n    + r\"\\(Suppressing further errors from this subtree\\.\\)\",\n    # Warnings that are probably less important.\n    r\"The \u201ctype\u201d attribute is unnecessary for JavaScript resources\\.\",\n]\nVNU_IGNORE_REGEX = re.compile(r\"|\".join(VNU_IGNORE))\n\nDEPLOY_ROOT = os.path.abspath(os.path.join(__file__, \"../../../../../..\"))\n\nZULIP_SERVER_GITHUB_FILE_URL_PREFIX = \"https://github.com/zulip/zulip/blob/master\"\nZULIP_SERVER_GITHUB_DIRECTORY_URL_PREFIX = \"https://github.com/zulip/zulip/tree/master\"\n\n\nclass BaseDocumentationSpider(scrapy.Spider):\n    name: Optional[str] = None\n    # Exclude domain address.\n    deny_domains: List[str] = []\n    start_urls: List[str] = []\n    deny: List[str] = []\n    file_extensions: List[str] = [\".\" + ext for ext in IGNORED_EXTENSIONS]\n    tags = (\"a\", \"area\", \"img\")\n    attrs = (\"href\", \"src\")\n\n    def _has_extension(self, url: str) -> bool:\n        return url_has_any_extension(url, self.file_extensions)\n\n    def _is_external_url(self, url: str) -> bool:\n        return url.startswith(\"http\") or self._has_extension(url)\n\n    def check_existing(self, response: Response) -> None:\n        self.log(response)\n\n    def _is_external_link(self, url: str) -> bool:\n        if url.startswith(\"https://chat.zulip.org\"):\n            # Since most chat.zulip.org URLs will be links to specific\n            # logged-in content that the spider cannot verify, or the\n            # homepage, there's no need to check those (which can\n            # cause errors when chat.zulip.org is being updated).\n            return True\n        if \"zulip.readthedocs\" in url or \"zulip.com\" in url or \"zulip.org\" in url:\n            # We want CI to check any links to Zulip sites.\n            return False\n        if (len(url) > 4 and url[:4] == \"file\") or (\"localhost\" in url):\n            # We also want CI to check any links to built documentation.\n            return False\n        if url.startswith(ZULIP_SERVER_GITHUB_FILE_URL_PREFIX) or url.startswith(\n            ZULIP_SERVER_GITHUB_DIRECTORY_URL_PREFIX\n        ):\n            # We can verify these links directly in the local git repo without making any requests to GitHub servers.\n            return False\n        if \"github.com/zulip\" in url:\n            # We want to check these links but due to rate limiting from GitHub, these checks often\n            # fail in the CI. Thus, we should treat these as external links for now.\n            # TODO: Figure out how to test github.com/zulip links in CI.\n            return True\n        return True\n\n    def check_fragment(self, response: Response) -> None:\n        self.log(response)\n        xpath_template = \"//*[@id='{fragment}' or @name='{fragment}']\"\n        m = re.match(r\".+\\#(?P<fragment>.*)$\", response.request.url)  # Get fragment value.\n        if not m:\n            return\n        fragment = m.group(\"fragment\")\n        # Check fragment existing on response page.\n        if not response.selector.xpath(xpath_template.format(fragment=fragment)):\n            self.logger.error(\n                \"Fragment #%s is not found on page %s\", fragment, response.request.url\n            )\n\n    def _vnu_callback(self, url: str) -> Callable[[Response], None]:\n        def callback(response: Response) -> None:\n            vnu_out = json.loads(response.text)\n            for message in vnu_out[\"messages\"]:\n                if not VNU_IGNORE_REGEX.fullmatch(message[\"message\"]):\n                    self.logger.error(\n                        '\"%s\":%d.%d-%d.%d: %s: %s',\n                        url,\n                        message.get(\"firstLine\", message[\"lastLine\"]),\n                        message.get(\"firstColumn\", message[\"lastColumn\"]),\n                        message[\"lastLine\"],\n                        message[\"lastColumn\"],\n                        message[\"type\"],\n                        message[\"message\"],\n                    )\n\n        return callback\n\n    def _make_requests(self, url: str) -> Iterator[Request]:\n        # These URLs are for Zulip's webapp, which with recent changes\n        # can be accessible without login an account.  While we do\n        # crawl documentation served by the webapp (E.g. /help/), we\n        # don't want to crawl the webapp itself, so we exclude these.\n        if (\n            url in [\"http://localhost:9981/\", \"http://localhost:9981\"]\n            or url.startswith(\"http://localhost:9981/#\")\n            or url.startswith(\"http://localhost:9981#\")\n        ):\n            return\n\n        callback: Callable[[Response], Optional[Iterator[Request]]] = self.parse\n        dont_filter = False\n        method = \"GET\"\n        if self._is_external_url(url):\n            callback = self.check_existing\n            method = \"HEAD\"\n\n            if url.startswith(ZULIP_SERVER_GITHUB_FILE_URL_PREFIX):\n                file_path = url.replace(ZULIP_SERVER_GITHUB_FILE_URL_PREFIX, DEPLOY_ROOT)\n                hash_index = file_path.find(\"#\")\n                if hash_index != -1:\n                    file_path = file_path[:hash_index]\n                if not os.path.isfile(file_path):\n                    self.logger.error(\n                        \"There is no local file associated with the GitHub URL: %s\", url\n                    )\n                return\n            elif url.startswith(ZULIP_SERVER_GITHUB_DIRECTORY_URL_PREFIX):\n                dir_path = url.replace(ZULIP_SERVER_GITHUB_DIRECTORY_URL_PREFIX, DEPLOY_ROOT)\n                if not os.path.isdir(dir_path):\n                    self.logger.error(\n                        \"There is no local directory associated with the GitHub URL: %s\", url\n                    )\n                return\n        elif \"#\" in url:\n            dont_filter = True\n            callback = self.check_fragment\n        if getattr(self, \"skip_external\", False) and self._is_external_link(url):\n            return\n        if urlparse(url).netloc in EXCLUDED_DOMAINS:\n            return\n        if url in EXCLUDED_URLS:\n            return\n        yield Request(\n            url,\n            method=method,\n            callback=callback,\n            dont_filter=dont_filter,\n            errback=self.error_callback,\n        )\n\n    def start_requests(self) -> Iterator[Request]:\n        for url in self.start_urls:\n            yield from self._make_requests(url)\n\n    def parse(self, response: Response) -> Iterator[Request]:\n        self.log(response)\n\n        if getattr(self, \"validate_html\", False):\n            yield Request(\n                \"http://127.0.0.1:9988/?out=json\",\n                method=\"POST\",\n                headers={\"Content-Type\": response.headers[\"Content-Type\"]},\n                body=response.body,\n                callback=self._vnu_callback(response.url),\n                errback=self.error_callback,\n            )\n\n        for link in LxmlLinkExtractor(\n            deny_domains=self.deny_domains,\n            deny_extensions=[\"doc\"],\n            tags=self.tags,\n            attrs=self.attrs,\n            deny=self.deny,\n            canonicalize=False,\n        ).extract_links(response):\n            yield from self._make_requests(link.url)\n\n    def retry_request_with_get(self, request: Request) -> Iterator[Request]:\n        request.method = \"GET\"\n        request.dont_filter = True\n        yield request\n\n    def error_callback(self, failure: Failure) -> Optional[Union[Failure, Iterator[Request]]]:\n        if isinstance(failure.value, HttpError):\n            response = failure.value.response\n            # Hack: The filtering above does not catch this URL,\n            # likely due to a redirect.\n            if urlparse(response.url).netloc == \"idmsa.apple.com\":\n                return None\n            if response.status == 405 and response.request.method == \"HEAD\":\n                # Method 'HEAD' not allowed, repeat request with 'GET'\n                return self.retry_request_with_get(response.request)\n            self.logger.error(\"Please check link: %s\", response.request.url)\n\n        return failure\n", "path": "tools/documentation_crawler/documentation_crawler/spiders/common/spiders.py"}]}