prompt: You will be provided with a partial code base and an issue statement explaining a problem to resolve.
  <issue>
  Pull GCI docs into main Zulip repo.
  Some of our assets, like git cheatsheets, currently live in the zulip-gci repo.  We should move them to the main Zulip repo and link from appropriate places.
  
  cc @synicalsyntax @lonerz @tommyip 
  
  </issue>
  <code>
  [start of tools/documentation_crawler/documentation_crawler/spiders/common/spiders.py]
  1 #!/usr/bin/env python
  2 from __future__ import print_function
  3 
  4 import logging
  5 import re
  6 import scrapy
  7 
  8 from scrapy import Request
  9 from scrapy.linkextractors import IGNORED_EXTENSIONS
  10 from scrapy.linkextractors.lxmlhtml import LxmlLinkExtractor
  11 from scrapy.utils.url import url_has_any_extension
  12 
  13 from typing import Any, Generator, List, Optional, Tuple
  14 
  15 EXCLUDED_URLS = [
  16     # Google calendar returns 404s on HEAD requests unconditionally
  17     'https://calendar.google.com/calendar/embed?src=ktiduof4eoh47lmgcl2qunnc0o@group.calendar.google.com',
  18     # Returns 409 errors to HEAD requests frequently
  19     'https://medium.freecodecamp.com',
  20 ]
  21 
  22 
  23 class BaseDocumentationSpider(scrapy.Spider):
  24     name = None  # type: Optional[str]
  25     # Exclude domain address.
  26     deny_domains = []  # type: List[str]
  27     start_urls = []  # type: List[str]
  28     deny = []  # type: List[str]
  29     file_extensions = ['.' + ext for ext in IGNORED_EXTENSIONS]  # type: List[str]
  30     tags = ('a', 'area', 'img')
  31     attrs = ('href', 'src')
  32 
  33     def __init__(self, *args, **kwargs):
  34         # type: (*Any, **Any) -> None
  35         super(BaseDocumentationSpider, self).__init__(*args, **kwargs)
  36         self.has_error = False
  37 
  38     def _set_error_state(self):
  39         # type: () -> None
  40         self.has_error = True
  41 
  42     def _has_extension(self, url):
  43         # type: (str) -> bool
  44         return url_has_any_extension(url, self.file_extensions)
  45 
  46     def _is_external_url(self, url):
  47         # type: (str) -> bool
  48         return url.startswith('http') or self._has_extension(url)
  49 
  50     def check_existing(self, response):
  51         # type: (Any) -> None
  52         self.log(response)
  53 
  54     def check_permalink(self, response):
  55         # type: (Any) -> None
  56         self.log(response)
  57         xpath_template = "//*[@id='{permalink}' or @name='{permalink}']"
  58         m = re.match(r".+\#(?P<permalink>.*)$", response.request.url)  # Get anchor value.
  59         if not m:
  60             return
  61         permalink = m.group('permalink')
  62         # Check permalink existing on response page.
  63         if not response.selector.xpath(xpath_template.format(permalink=permalink)):
  64             self._set_error_state()
  65             raise Exception(
  66                 "Permalink #{} is not found on page {}".format(permalink, response.request.url))
  67 
  68     def parse(self, response):
  69         # type: (Any) -> Generator[Request, None, None]
  70         self.log(response)
  71         for link in LxmlLinkExtractor(deny_domains=self.deny_domains, deny_extensions=['doc'],
  72                                       tags=self.tags, attrs=self.attrs, deny=self.deny,
  73                                       canonicalize=False).extract_links(response):
  74             callback = self.parse  # type: Any
  75             dont_filter = False
  76             method = 'GET'
  77             if self._is_external_url(link.url):
  78                 callback = self.check_existing
  79                 method = 'HEAD'
  80             elif '#' in link.url:
  81                 dont_filter = True
  82                 callback = self.check_permalink
  83             yield Request(link.url, method=method, callback=callback, dont_filter=dont_filter,
  84                           errback=self.error_callback)
  85 
  86     def retry_request_with_get(self, request):
  87         # type: (Request) -> Generator[Request, None, None]
  88         request.method = 'GET'
  89         request.dont_filter = True
  90         yield request
  91 
  92     def exclude_error(self, url):
  93         # type: (str) -> bool
  94         if url in EXCLUDED_URLS:
  95             return True
  96         return False
  97 
  98     def error_callback(self, failure):
  99         # type: (Any) -> Optional[Generator[Any, None, None]]
  100         if hasattr(failure.value, 'response') and failure.value.response:
  101             response = failure.value.response
  102             if self.exclude_error(response.url):
  103                 return None
  104             if response.status == 404:
  105                 self._set_error_state()
  106                 raise Exception('Page not found: {}'.format(response))
  107             if response.status == 405 and response.request.method == 'HEAD':
  108                 # Method 'HEAD' not allowed, repeat request with 'GET'
  109                 return self.retry_request_with_get(response.request)
  110             self.log("Error! Please check link: {}".format(response), logging.ERROR)
  111         elif isinstance(failure.type, IOError):
  112             self._set_error_state()
  113         else:
  114             raise Exception(failure.value)
  115         return None
  116
  [end of tools/documentation_crawler/documentation_crawler/spiders/common/spiders.py]
  </code>
  
  I need you to solve this issue by generating a single patch file that I can apply directly to this repository using git apply. Please respond with a single patch file in the following format.
  <patch>
  diff --git a/file.py b/file.py
  --- a/file.py
  +++ b/file.py
  @@ -1,27 +1,35 @@
   def euclidean(a, b):
  -    while b:
  -        a, b = b, a % b
  -    return a
  +    if b == 0:
  +        return a
  +    return euclidean(b, a % b)
   
   
   def bresenham(x0, y0, x1, y1):
       points = []
       dx = abs(x1 - x0)
       dy = abs(y1 - y0)
  -    sx = 1 if x0 < x1 else -1
  -    sy = 1 if y0 < y1 else -1
  -    err = dx - dy
  +    x, y = x0, y0
  +    sx = -1 if x0 > x1 else 1
  +    sy = -1 if y0 > y1 else 1
   
  -    while True:
  -        points.append((x0, y0))
  -        if x0 == x1 and y0 == y1:
  -            break
  -        e2 = 2 * err
  -        if e2 > -dy:
  -            err -= dy
  -            x0 += sx
  -        if e2 < dx:
  -            err += dx
  -            y0 += sy
  +    if dx > dy:
  +        err = dx / 2.0
  +        while x != x1:
  +            points.append((x, y))
  +            err -= dy
  +            if err < 0:
  +                y += sy
  +                err += dx
  +            x += sx
  +    else:
  +        err = dy / 2.0
  +        while y != y1:
  +            points.append((x, y))
  +            err -= dx
  +            if err < 0:
  +                x += sx
  +                err += dy
  +            y += sy
  +
  +    points.append((x, y))
       return points
  
  </patch>
golden_diff: diff --git a/tools/documentation_crawler/documentation_crawler/spiders/common/spiders.py b/tools/documentation_crawler/documentation_crawler/spiders/common/spiders.py
  --- a/tools/documentation_crawler/documentation_crawler/spiders/common/spiders.py
  +++ b/tools/documentation_crawler/documentation_crawler/spiders/common/spiders.py
  @@ -17,6 +17,8 @@
       'https://calendar.google.com/calendar/embed?src=ktiduof4eoh47lmgcl2qunnc0o@group.calendar.google.com',
       # Returns 409 errors to HEAD requests frequently
       'https://medium.freecodecamp.com',
  +    # Returns 404 to HEAD requests unconditionally
  +    'https://www.git-tower.com/blog/command-line-cheat-sheet/',
   ]
  
verification_info: {"golden_diff": "diff --git a/tools/documentation_crawler/documentation_crawler/spiders/common/spiders.py b/tools/documentation_crawler/documentation_crawler/spiders/common/spiders.py\n--- a/tools/documentation_crawler/documentation_crawler/spiders/common/spiders.py\n+++ b/tools/documentation_crawler/documentation_crawler/spiders/common/spiders.py\n@@ -17,6 +17,8 @@\n     'https://calendar.google.com/calendar/embed?src=ktiduof4eoh47lmgcl2qunnc0o@group.calendar.google.com',\n     # Returns 409 errors to HEAD requests frequently\n     'https://medium.freecodecamp.com',\n+    # Returns 404 to HEAD requests unconditionally\n+    'https://www.git-tower.com/blog/command-line-cheat-sheet/',\n ]\n", "issue": "Pull GCI docs into main Zulip repo.\nSome of our assets, like git cheatsheets, currently live in the zulip-gci repo.  We should move them to the main Zulip repo and link from appropriate places.\r\n\r\ncc @synicalsyntax @lonerz @tommyip \n", "before_files": [{"content": "#!/usr/bin/env python\nfrom __future__ import print_function\n\nimport logging\nimport re\nimport scrapy\n\nfrom scrapy import Request\nfrom scrapy.linkextractors import IGNORED_EXTENSIONS\nfrom scrapy.linkextractors.lxmlhtml import LxmlLinkExtractor\nfrom scrapy.utils.url import url_has_any_extension\n\nfrom typing import Any, Generator, List, Optional, Tuple\n\nEXCLUDED_URLS = [\n    # Google calendar returns 404s on HEAD requests unconditionally\n    'https://calendar.google.com/calendar/embed?src=ktiduof4eoh47lmgcl2qunnc0o@group.calendar.google.com',\n    # Returns 409 errors to HEAD requests frequently\n    'https://medium.freecodecamp.com',\n]\n\n\nclass BaseDocumentationSpider(scrapy.Spider):\n    name = None  # type: Optional[str]\n    # Exclude domain address.\n    deny_domains = []  # type: List[str]\n    start_urls = []  # type: List[str]\n    deny = []  # type: List[str]\n    file_extensions = ['.' + ext for ext in IGNORED_EXTENSIONS]  # type: List[str]\n    tags = ('a', 'area', 'img')\n    attrs = ('href', 'src')\n\n    def __init__(self, *args, **kwargs):\n        # type: (*Any, **Any) -> None\n        super(BaseDocumentationSpider, self).__init__(*args, **kwargs)\n        self.has_error = False\n\n    def _set_error_state(self):\n        # type: () -> None\n        self.has_error = True\n\n    def _has_extension(self, url):\n        # type: (str) -> bool\n        return url_has_any_extension(url, self.file_extensions)\n\n    def _is_external_url(self, url):\n        # type: (str) -> bool\n        return url.startswith('http') or self._has_extension(url)\n\n    def check_existing(self, response):\n        # type: (Any) -> None\n        self.log(response)\n\n    def check_permalink(self, response):\n        # type: (Any) -> None\n        self.log(response)\n        xpath_template = \"//*[@id='{permalink}' or @name='{permalink}']\"\n        m = re.match(r\".+\\#(?P<permalink>.*)$\", response.request.url)  # Get anchor value.\n        if not m:\n            return\n        permalink = m.group('permalink')\n        # Check permalink existing on response page.\n        if not response.selector.xpath(xpath_template.format(permalink=permalink)):\n            self._set_error_state()\n            raise Exception(\n                \"Permalink #{} is not found on page {}\".format(permalink, response.request.url))\n\n    def parse(self, response):\n        # type: (Any) -> Generator[Request, None, None]\n        self.log(response)\n        for link in LxmlLinkExtractor(deny_domains=self.deny_domains, deny_extensions=['doc'],\n                                      tags=self.tags, attrs=self.attrs, deny=self.deny,\n                                      canonicalize=False).extract_links(response):\n            callback = self.parse  # type: Any\n            dont_filter = False\n            method = 'GET'\n            if self._is_external_url(link.url):\n                callback = self.check_existing\n                method = 'HEAD'\n            elif '#' in link.url:\n                dont_filter = True\n                callback = self.check_permalink\n            yield Request(link.url, method=method, callback=callback, dont_filter=dont_filter,\n                          errback=self.error_callback)\n\n    def retry_request_with_get(self, request):\n        # type: (Request) -> Generator[Request, None, None]\n        request.method = 'GET'\n        request.dont_filter = True\n        yield request\n\n    def exclude_error(self, url):\n        # type: (str) -> bool\n        if url in EXCLUDED_URLS:\n            return True\n        return False\n\n    def error_callback(self, failure):\n        # type: (Any) -> Optional[Generator[Any, None, None]]\n        if hasattr(failure.value, 'response') and failure.value.response:\n            response = failure.value.response\n            if self.exclude_error(response.url):\n                return None\n            if response.status == 404:\n                self._set_error_state()\n                raise Exception('Page not found: {}'.format(response))\n            if response.status == 405 and response.request.method == 'HEAD':\n                # Method 'HEAD' not allowed, repeat request with 'GET'\n                return self.retry_request_with_get(response.request)\n            self.log(\"Error! Please check link: {}\".format(response), logging.ERROR)\n        elif isinstance(failure.type, IOError):\n            self._set_error_state()\n        else:\n            raise Exception(failure.value)\n        return None\n", "path": "tools/documentation_crawler/documentation_crawler/spiders/common/spiders.py"}]}