prompt: We are currently solving the following issue within our repository. Here is the issue text:
  --- BEGIN ISSUE ---
  Unhandled error in Deferred (RobotsTxtMiddleware)
  **Dev story**:
  Let's say spider downloads all .zip files from http://habrahabr.ru/post/212029/ page
  Url with .zip files looks like this: http://layer6.jenkins.tox.im/job/qt_gui_win32/lastSuccessfulBuild/artifact/qt/build/release/TOX-Qt-GUI.zip
  
  It's a polite spider, so settings file contains:
  `ROBOTSTXT_OBEY = True`
  
  Middleware parses habrahabr.ru robots.txt file as well as 'external' robots.txt file from layer6.jenkins.tox.im. It's expected behaviour. 
  But if request will be returned with error then the output would be:
  
  ```
  2015-04-02 17:06:16+0300 [habrahabr] DEBUG: Gave up retrying <GET http://layer6.jenkins.tox.im/robots.txt> (failed 1 times): DNS lookup failed: address 'layer6.jenkins.tox.im' not found: [Errno 8] nodename nor servname provided, or not known.
  
  2015-04-02 17:06:16+0300 [-] ERROR: Unhandled error in Deferred:
  2015-04-02 17:06:16+0300 [-] Unhandled Error
      Traceback (most recent call last):
      Failure: twisted.internet.error.DNSLookupError: DNS lookup failed: address 'layer6.jenkins.tox.im' not found: [Errno 8] nodename nor servname provided, or not known.
  ```
  
  
  --- END ISSUE ---
  
  Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
  --- BEGIN FILES ---
  Path: `scrapy/contrib/downloadermiddleware/robotstxt.py`
  Content:
  ```
  1 """
  2 This is a middleware to respect robots.txt policies. To activate it you must
  3 enable this middleware and enable the ROBOTSTXT_OBEY setting.
  4 
  5 """
  6 
  7 from six.moves.urllib import robotparser
  8 
  9 from scrapy import signals, log
  10 from scrapy.exceptions import NotConfigured, IgnoreRequest
  11 from scrapy.http import Request
  12 from scrapy.utils.httpobj import urlparse_cached
  13 
  14 
  15 class RobotsTxtMiddleware(object):
  16     DOWNLOAD_PRIORITY = 1000
  17 
  18     def __init__(self, crawler):
  19         if not crawler.settings.getbool('ROBOTSTXT_OBEY'):
  20             raise NotConfigured
  21 
  22         self.crawler = crawler
  23         self._useragent = crawler.settings.get('USER_AGENT')
  24         self._parsers = {}
  25 
  26     @classmethod
  27     def from_crawler(cls, crawler):
  28         return cls(crawler)
  29 
  30     def process_request(self, request, spider):
  31         if request.meta.get('dont_obey_robotstxt'):
  32             return
  33         rp = self.robot_parser(request, spider)
  34         if rp and not rp.can_fetch(self._useragent, request.url):
  35             log.msg(format="Forbidden by robots.txt: %(request)s",
  36                     level=log.DEBUG, request=request)
  37             raise IgnoreRequest
  38 
  39     def robot_parser(self, request, spider):
  40         url = urlparse_cached(request)
  41         netloc = url.netloc
  42         if netloc not in self._parsers:
  43             self._parsers[netloc] = None
  44             robotsurl = "%s://%s/robots.txt" % (url.scheme, url.netloc)
  45             robotsreq = Request(
  46                 robotsurl,
  47                 priority=self.DOWNLOAD_PRIORITY,
  48                 meta={'dont_obey_robotstxt': True}
  49             )
  50             dfd = self.crawler.engine.download(robotsreq, spider)
  51             dfd.addCallback(self._parse_robots)
  52         return self._parsers[netloc]
  53 
  54     def _parse_robots(self, response):
  55         rp = robotparser.RobotFileParser(response.url)
  56         rp.parse(response.body.splitlines())
  57         self._parsers[urlparse_cached(response).netloc] = rp
  58
  ```
  --- END FILES ---
  
  Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.
  
  Here is an example:
  
  ```diff
  diff --git a/examples/server_async.py b/examples/server_async.py
  --- a/examples/server_async.py
  +++ b/examples/server_async.py
  @@ -313,4 +313,4 @@
   
   
   if __name__ == "__main__":
  -    asyncio.run(run_async_server("."), debug=True)
  +    asyncio.run(run_async_server(), debug=True)
  diff --git a/examples/server_sync.py b/examples/server_sync.py
  --- a/examples/server_sync.py
  +++ b/examples/server_sync.py
  @@ -313,5 +313,5 @@
   
   
   if __name__ == "__main__":
  -    server = run_sync_server(".")
  +    server = run_sync_server()
       server.shutdown()
  
  ```
  
golden_diff: diff --git a/scrapy/contrib/downloadermiddleware/robotstxt.py b/scrapy/contrib/downloadermiddleware/robotstxt.py
  --- a/scrapy/contrib/downloadermiddleware/robotstxt.py
  +++ b/scrapy/contrib/downloadermiddleware/robotstxt.py
  @@ -49,8 +49,14 @@
               )
               dfd = self.crawler.engine.download(robotsreq, spider)
               dfd.addCallback(self._parse_robots)
  +            dfd.addErrback(self._logerror, robotsreq, spider)
           return self._parsers[netloc]
   
  +    def _logerror(self, failure, request, spider):
  +        if failure.type is not IgnoreRequest:
  +            log.msg(format="Error downloading %%(request)s: %s" % failure.value,
  +                    level=log.ERROR, request=request, spider=spider)
  +
       def _parse_robots(self, response):
           rp = robotparser.RobotFileParser(response.url)
           rp.parse(response.body.splitlines())
  
verification_info: {"golden_diff": "diff --git a/scrapy/contrib/downloadermiddleware/robotstxt.py b/scrapy/contrib/downloadermiddleware/robotstxt.py\n--- a/scrapy/contrib/downloadermiddleware/robotstxt.py\n+++ b/scrapy/contrib/downloadermiddleware/robotstxt.py\n@@ -49,8 +49,14 @@\n             )\n             dfd = self.crawler.engine.download(robotsreq, spider)\n             dfd.addCallback(self._parse_robots)\n+            dfd.addErrback(self._logerror, robotsreq, spider)\n         return self._parsers[netloc]\n \n+    def _logerror(self, failure, request, spider):\n+        if failure.type is not IgnoreRequest:\n+            log.msg(format=\"Error downloading %%(request)s: %s\" % failure.value,\n+                    level=log.ERROR, request=request, spider=spider)\n+\n     def _parse_robots(self, response):\n         rp = robotparser.RobotFileParser(response.url)\n         rp.parse(response.body.splitlines())\n", "issue": "Unhandled error in Deferred (RobotsTxtMiddleware)\n**Dev story**:\nLet's say spider downloads all .zip files from http://habrahabr.ru/post/212029/ page\nUrl with .zip files looks like this: http://layer6.jenkins.tox.im/job/qt_gui_win32/lastSuccessfulBuild/artifact/qt/build/release/TOX-Qt-GUI.zip\n\nIt's a polite spider, so settings file contains:\n`ROBOTSTXT_OBEY = True`\n\nMiddleware parses habrahabr.ru robots.txt file as well as 'external' robots.txt file from layer6.jenkins.tox.im. It's expected behaviour. \nBut if request will be returned with error then the output would be:\n\n```\n2015-04-02 17:06:16+0300 [habrahabr] DEBUG: Gave up retrying <GET http://layer6.jenkins.tox.im/robots.txt> (failed 1 times): DNS lookup failed: address 'layer6.jenkins.tox.im' not found: [Errno 8] nodename nor servname provided, or not known.\n\n2015-04-02 17:06:16+0300 [-] ERROR: Unhandled error in Deferred:\n2015-04-02 17:06:16+0300 [-] Unhandled Error\n    Traceback (most recent call last):\n    Failure: twisted.internet.error.DNSLookupError: DNS lookup failed: address 'layer6.jenkins.tox.im' not found: [Errno 8] nodename nor servname provided, or not known.\n```\n\n", "before_files": [{"content": "\"\"\"\nThis is a middleware to respect robots.txt policies. To activate it you must\nenable this middleware and enable the ROBOTSTXT_OBEY setting.\n\n\"\"\"\n\nfrom six.moves.urllib import robotparser\n\nfrom scrapy import signals, log\nfrom scrapy.exceptions import NotConfigured, IgnoreRequest\nfrom scrapy.http import Request\nfrom scrapy.utils.httpobj import urlparse_cached\n\n\nclass RobotsTxtMiddleware(object):\n    DOWNLOAD_PRIORITY = 1000\n\n    def __init__(self, crawler):\n        if not crawler.settings.getbool('ROBOTSTXT_OBEY'):\n            raise NotConfigured\n\n        self.crawler = crawler\n        self._useragent = crawler.settings.get('USER_AGENT')\n        self._parsers = {}\n\n    @classmethod\n    def from_crawler(cls, crawler):\n        return cls(crawler)\n\n    def process_request(self, request, spider):\n        if request.meta.get('dont_obey_robotstxt'):\n            return\n        rp = self.robot_parser(request, spider)\n        if rp and not rp.can_fetch(self._useragent, request.url):\n            log.msg(format=\"Forbidden by robots.txt: %(request)s\",\n                    level=log.DEBUG, request=request)\n            raise IgnoreRequest\n\n    def robot_parser(self, request, spider):\n        url = urlparse_cached(request)\n        netloc = url.netloc\n        if netloc not in self._parsers:\n            self._parsers[netloc] = None\n            robotsurl = \"%s://%s/robots.txt\" % (url.scheme, url.netloc)\n            robotsreq = Request(\n                robotsurl,\n                priority=self.DOWNLOAD_PRIORITY,\n                meta={'dont_obey_robotstxt': True}\n            )\n            dfd = self.crawler.engine.download(robotsreq, spider)\n            dfd.addCallback(self._parse_robots)\n        return self._parsers[netloc]\n\n    def _parse_robots(self, response):\n        rp = robotparser.RobotFileParser(response.url)\n        rp.parse(response.body.splitlines())\n        self._parsers[urlparse_cached(response).netloc] = rp\n", "path": "scrapy/contrib/downloadermiddleware/robotstxt.py"}], "after_files": [{"content": "\"\"\"\nThis is a middleware to respect robots.txt policies. To activate it you must\nenable this middleware and enable the ROBOTSTXT_OBEY setting.\n\n\"\"\"\n\nfrom six.moves.urllib import robotparser\n\nfrom scrapy import signals, log\nfrom scrapy.exceptions import NotConfigured, IgnoreRequest\nfrom scrapy.http import Request\nfrom scrapy.utils.httpobj import urlparse_cached\n\n\nclass RobotsTxtMiddleware(object):\n    DOWNLOAD_PRIORITY = 1000\n\n    def __init__(self, crawler):\n        if not crawler.settings.getbool('ROBOTSTXT_OBEY'):\n            raise NotConfigured\n\n        self.crawler = crawler\n        self._useragent = crawler.settings.get('USER_AGENT')\n        self._parsers = {}\n\n    @classmethod\n    def from_crawler(cls, crawler):\n        return cls(crawler)\n\n    def process_request(self, request, spider):\n        if request.meta.get('dont_obey_robotstxt'):\n            return\n        rp = self.robot_parser(request, spider)\n        if rp and not rp.can_fetch(self._useragent, request.url):\n            log.msg(format=\"Forbidden by robots.txt: %(request)s\",\n                    level=log.DEBUG, request=request)\n            raise IgnoreRequest\n\n    def robot_parser(self, request, spider):\n        url = urlparse_cached(request)\n        netloc = url.netloc\n        if netloc not in self._parsers:\n            self._parsers[netloc] = None\n            robotsurl = \"%s://%s/robots.txt\" % (url.scheme, url.netloc)\n            robotsreq = Request(\n                robotsurl,\n                priority=self.DOWNLOAD_PRIORITY,\n                meta={'dont_obey_robotstxt': True}\n            )\n            dfd = self.crawler.engine.download(robotsreq, spider)\n            dfd.addCallback(self._parse_robots)\n            dfd.addErrback(self._logerror, robotsreq, spider)\n        return self._parsers[netloc]\n\n    def _logerror(self, failure, request, spider):\n        if failure.type is not IgnoreRequest:\n            log.msg(format=\"Error downloading %%(request)s: %s\" % failure.value,\n                    level=log.ERROR, request=request, spider=spider)\n\n    def _parse_robots(self, response):\n        rp = robotparser.RobotFileParser(response.url)\n        rp.parse(response.body.splitlines())\n        self._parsers[urlparse_cached(response).netloc] = rp\n", "path": "scrapy/contrib/downloadermiddleware/robotstxt.py"}]}