prompt: You will be provided with a partial code base and an issue statement explaining a problem to resolve.
  <issue>
  KeyError in robotstxt middleware
  I'm getting these errors in robots.txt middleware:
  
  ```
  2016-01-27 16:18:21 [scrapy.core.scraper] ERROR: Error downloading <GET http://yellowpages.co.th>
  Traceback (most recent call last):
    File "/Users/kmike/envs/scraping/lib/python2.7/site-packages/twisted/internet/defer.py", line 150, in maybeDeferred
      result = f(*args, **kw)
    File "/Users/kmike/svn/scrapy/scrapy/downloadermiddlewares/robotstxt.py", line 65, in robot_parser
      if isinstance(self._parsers[netloc], Deferred):
  KeyError: 'yellowpages.co.th'
  ```
  
  It looks like https://github.com/scrapy/scrapy/pull/1473 caused it (I can't get this issue in Scrapy 1.0.4, but it present in Scrapy master). It happens when page failed to download and HTTP cache is enabled. I haven't debugged it further.
  
  
  </issue>
  <code>
  [start of scrapy/downloadermiddlewares/robotstxt.py]
  1 """
  2 This is a middleware to respect robots.txt policies. To activate it you must
  3 enable this middleware and enable the ROBOTSTXT_OBEY setting.
  4 
  5 """
  6 
  7 import logging
  8 
  9 from six.moves.urllib import robotparser
  10 
  11 from twisted.internet.defer import Deferred, maybeDeferred
  12 from scrapy.exceptions import NotConfigured, IgnoreRequest
  13 from scrapy.http import Request
  14 from scrapy.utils.httpobj import urlparse_cached
  15 from scrapy.utils.log import failure_to_exc_info
  16 
  17 logger = logging.getLogger(__name__)
  18 
  19 
  20 class RobotsTxtMiddleware(object):
  21     DOWNLOAD_PRIORITY = 1000
  22 
  23     def __init__(self, crawler):
  24         if not crawler.settings.getbool('ROBOTSTXT_OBEY'):
  25             raise NotConfigured
  26 
  27         self.crawler = crawler
  28         self._useragent = crawler.settings.get('USER_AGENT')
  29         self._parsers = {}
  30 
  31     @classmethod
  32     def from_crawler(cls, crawler):
  33         return cls(crawler)
  34 
  35     def process_request(self, request, spider):
  36         if request.meta.get('dont_obey_robotstxt'):
  37             return
  38         d = maybeDeferred(self.robot_parser, request, spider)
  39         d.addCallback(self.process_request_2, request, spider)
  40         return d
  41 
  42     def process_request_2(self, rp, request, spider):
  43         if rp is not None and not rp.can_fetch(self._useragent, request.url):
  44             logger.debug("Forbidden by robots.txt: %(request)s",
  45                          {'request': request}, extra={'spider': spider})
  46             raise IgnoreRequest()
  47 
  48     def robot_parser(self, request, spider):
  49         url = urlparse_cached(request)
  50         netloc = url.netloc
  51 
  52         if netloc not in self._parsers:
  53             self._parsers[netloc] = Deferred()
  54             robotsurl = "%s://%s/robots.txt" % (url.scheme, url.netloc)
  55             robotsreq = Request(
  56                 robotsurl,
  57                 priority=self.DOWNLOAD_PRIORITY,
  58                 meta={'dont_obey_robotstxt': True}
  59             )
  60             dfd = self.crawler.engine.download(robotsreq, spider)
  61             dfd.addCallback(self._parse_robots, netloc)
  62             dfd.addErrback(self._logerror, robotsreq, spider)
  63             dfd.addErrback(self._robots_error, netloc)
  64 
  65         if isinstance(self._parsers[netloc], Deferred):
  66             d = Deferred()
  67             def cb(result):
  68                 d.callback(result)
  69                 return result
  70             self._parsers[netloc].addCallback(cb)
  71             return d
  72         else:
  73             return self._parsers[netloc]
  74 
  75     def _logerror(self, failure, request, spider):
  76         if failure.type is not IgnoreRequest:
  77             logger.error("Error downloading %(request)s: %(f_exception)s",
  78                          {'request': request, 'f_exception': failure.value},
  79                          exc_info=failure_to_exc_info(failure),
  80                          extra={'spider': spider})
  81         return failure
  82 
  83     def _parse_robots(self, response, netloc):
  84         rp = robotparser.RobotFileParser(response.url)
  85         body = ''
  86         if hasattr(response, 'text'):
  87             body = response.text
  88         else: # last effort try
  89             try:
  90                 body = response.body.decode('utf-8')
  91             except UnicodeDecodeError:
  92                 # If we found garbage, disregard it:,
  93                 # but keep the lookup cached (in self._parsers)
  94                 # Running rp.parse() will set rp state from
  95                 # 'disallow all' to 'allow any'.
  96                 pass
  97         rp.parse(body.splitlines())
  98 
  99         rp_dfd = self._parsers[netloc]
  100         self._parsers[netloc] = rp
  101         rp_dfd.callback(rp)
  102 
  103     def _robots_error(self, failure, netloc):
  104         self._parsers.pop(netloc).callback(None)
  105
  [end of scrapy/downloadermiddlewares/robotstxt.py]
  </code>
  
  I need you to solve this issue by generating a single patch file that I can apply directly to this repository using git apply. Please respond with a single patch file in the following format.
  <patch>
  diff --git a/file.py b/file.py
  --- a/file.py
  +++ b/file.py
  @@ -1,27 +1,35 @@
   def euclidean(a, b):
  -    while b:
  -        a, b = b, a % b
  -    return a
  +    if b == 0:
  +        return a
  +    return euclidean(b, a % b)
   
   
   def bresenham(x0, y0, x1, y1):
       points = []
       dx = abs(x1 - x0)
       dy = abs(y1 - y0)
  -    sx = 1 if x0 < x1 else -1
  -    sy = 1 if y0 < y1 else -1
  -    err = dx - dy
  +    x, y = x0, y0
  +    sx = -1 if x0 > x1 else 1
  +    sy = -1 if y0 > y1 else 1
   
  -    while True:
  -        points.append((x0, y0))
  -        if x0 == x1 and y0 == y1:
  -            break
  -        e2 = 2 * err
  -        if e2 > -dy:
  -            err -= dy
  -            x0 += sx
  -        if e2 < dx:
  -            err += dx
  -            y0 += sy
  +    if dx > dy:
  +        err = dx / 2.0
  +        while x != x1:
  +            points.append((x, y))
  +            err -= dy
  +            if err < 0:
  +                y += sy
  +                err += dx
  +            x += sx
  +    else:
  +        err = dy / 2.0
  +        while y != y1:
  +            points.append((x, y))
  +            err -= dx
  +            if err < 0:
  +                x += sx
  +                err += dy
  +            y += sy
  +
  +    points.append((x, y))
       return points
  
  </patch>
golden_diff: diff --git a/scrapy/downloadermiddlewares/robotstxt.py b/scrapy/downloadermiddlewares/robotstxt.py
  --- a/scrapy/downloadermiddlewares/robotstxt.py
  +++ b/scrapy/downloadermiddlewares/robotstxt.py
  @@ -101,4 +101,6 @@
           rp_dfd.callback(rp)
   
       def _robots_error(self, failure, netloc):
  -        self._parsers.pop(netloc).callback(None)
  +        rp_dfd = self._parsers[netloc]
  +        self._parsers[netloc] = None
  +        rp_dfd.callback(None)
  
verification_info: {"golden_diff": "diff --git a/scrapy/downloadermiddlewares/robotstxt.py b/scrapy/downloadermiddlewares/robotstxt.py\n--- a/scrapy/downloadermiddlewares/robotstxt.py\n+++ b/scrapy/downloadermiddlewares/robotstxt.py\n@@ -101,4 +101,6 @@\n         rp_dfd.callback(rp)\n \n     def _robots_error(self, failure, netloc):\n-        self._parsers.pop(netloc).callback(None)\n+        rp_dfd = self._parsers[netloc]\n+        self._parsers[netloc] = None\n+        rp_dfd.callback(None)\n", "issue": "KeyError in robotstxt middleware\nI'm getting these errors in robots.txt middleware:\n\n```\n2016-01-27 16:18:21 [scrapy.core.scraper] ERROR: Error downloading <GET http://yellowpages.co.th>\nTraceback (most recent call last):\n  File \"/Users/kmike/envs/scraping/lib/python2.7/site-packages/twisted/internet/defer.py\", line 150, in maybeDeferred\n    result = f(*args, **kw)\n  File \"/Users/kmike/svn/scrapy/scrapy/downloadermiddlewares/robotstxt.py\", line 65, in robot_parser\n    if isinstance(self._parsers[netloc], Deferred):\nKeyError: 'yellowpages.co.th'\n```\n\nIt looks like https://github.com/scrapy/scrapy/pull/1473 caused it (I can't get this issue in Scrapy 1.0.4, but it present in Scrapy master). It happens when page failed to download and HTTP cache is enabled. I haven't debugged it further.\n\n", "before_files": [{"content": "\"\"\"\nThis is a middleware to respect robots.txt policies. To activate it you must\nenable this middleware and enable the ROBOTSTXT_OBEY setting.\n\n\"\"\"\n\nimport logging\n\nfrom six.moves.urllib import robotparser\n\nfrom twisted.internet.defer import Deferred, maybeDeferred\nfrom scrapy.exceptions import NotConfigured, IgnoreRequest\nfrom scrapy.http import Request\nfrom scrapy.utils.httpobj import urlparse_cached\nfrom scrapy.utils.log import failure_to_exc_info\n\nlogger = logging.getLogger(__name__)\n\n\nclass RobotsTxtMiddleware(object):\n    DOWNLOAD_PRIORITY = 1000\n\n    def __init__(self, crawler):\n        if not crawler.settings.getbool('ROBOTSTXT_OBEY'):\n            raise NotConfigured\n\n        self.crawler = crawler\n        self._useragent = crawler.settings.get('USER_AGENT')\n        self._parsers = {}\n\n    @classmethod\n    def from_crawler(cls, crawler):\n        return cls(crawler)\n\n    def process_request(self, request, spider):\n        if request.meta.get('dont_obey_robotstxt'):\n            return\n        d = maybeDeferred(self.robot_parser, request, spider)\n        d.addCallback(self.process_request_2, request, spider)\n        return d\n\n    def process_request_2(self, rp, request, spider):\n        if rp is not None and not rp.can_fetch(self._useragent, request.url):\n            logger.debug(\"Forbidden by robots.txt: %(request)s\",\n                         {'request': request}, extra={'spider': spider})\n            raise IgnoreRequest()\n\n    def robot_parser(self, request, spider):\n        url = urlparse_cached(request)\n        netloc = url.netloc\n\n        if netloc not in self._parsers:\n            self._parsers[netloc] = Deferred()\n            robotsurl = \"%s://%s/robots.txt\" % (url.scheme, url.netloc)\n            robotsreq = Request(\n                robotsurl,\n                priority=self.DOWNLOAD_PRIORITY,\n                meta={'dont_obey_robotstxt': True}\n            )\n            dfd = self.crawler.engine.download(robotsreq, spider)\n            dfd.addCallback(self._parse_robots, netloc)\n            dfd.addErrback(self._logerror, robotsreq, spider)\n            dfd.addErrback(self._robots_error, netloc)\n\n        if isinstance(self._parsers[netloc], Deferred):\n            d = Deferred()\n            def cb(result):\n                d.callback(result)\n                return result\n            self._parsers[netloc].addCallback(cb)\n            return d\n        else:\n            return self._parsers[netloc]\n\n    def _logerror(self, failure, request, spider):\n        if failure.type is not IgnoreRequest:\n            logger.error(\"Error downloading %(request)s: %(f_exception)s\",\n                         {'request': request, 'f_exception': failure.value},\n                         exc_info=failure_to_exc_info(failure),\n                         extra={'spider': spider})\n        return failure\n\n    def _parse_robots(self, response, netloc):\n        rp = robotparser.RobotFileParser(response.url)\n        body = ''\n        if hasattr(response, 'text'):\n            body = response.text\n        else: # last effort try\n            try:\n                body = response.body.decode('utf-8')\n            except UnicodeDecodeError:\n                # If we found garbage, disregard it:,\n                # but keep the lookup cached (in self._parsers)\n                # Running rp.parse() will set rp state from\n                # 'disallow all' to 'allow any'.\n                pass\n        rp.parse(body.splitlines())\n\n        rp_dfd = self._parsers[netloc]\n        self._parsers[netloc] = rp\n        rp_dfd.callback(rp)\n\n    def _robots_error(self, failure, netloc):\n        self._parsers.pop(netloc).callback(None)\n", "path": "scrapy/downloadermiddlewares/robotstxt.py"}]}