We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
Sitemap spider not robust against wrong sitemap URLs in robots.txt
[The "specs"](http://www.sitemaps.org/protocol.html#submit_robots) do say that the URL should be a "full URL":

> You can specify the location of the Sitemap using a robots.txt file. To do this, simply add the following line including the full URL to the sitemap:
> `Sitemap: http://www.example.com/sitemap.xml`

But some robots.txt use relative ones.

Example: http://www.asos.com/robots.txt

```
User-agent: *
Sitemap: /sitemap.ashx
Sitemap: http://www.asos.com/sitemap.xml
Disallow: /basket/
(...)
```

Spider:
```
from scrapy.spiders import SitemapSpider


class TestSpider(SitemapSpider):
    name = "test"
    sitemap_urls = [
        'http://www.asos.com/robots.txt',
    ]

    def parse(self, response):
        self.logger.info('parsing %r' % response.url)
```
Logs:

```
$ scrapy runspider spider.py
Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.90 Safari/537.36'
2016-11-09 17:46:19 [scrapy] INFO: Scrapy 1.2.1 started (bot: scrapybot)
(...)
2016-11-09 17:46:19 [scrapy] DEBUG: Crawled (200) <GET http://www.asos.com/robots.txt> (referer: None)
2016-11-09 17:46:19 [scrapy] ERROR: Spider error processing <GET http://www.asos.com/robots.txt> (referer: None)
Traceback (most recent call last):
  File "/home/paul/.virtualenvs/scrapy12/local/lib/python2.7/site-packages/scrapy/utils/defer.py", line 102, in iter_errback
    yield next(it)
  File "/home/paul/.virtualenvs/scrapy12/local/lib/python2.7/site-packages/scrapy/spidermiddlewares/offsite.py", line 29, in process_spider_output
    for x in result:
  File "/home/paul/.virtualenvs/scrapy12/local/lib/python2.7/site-packages/scrapy/spidermiddlewares/referer.py", line 22, in <genexpr>
    return (_set_referer(r) for r in result or ())
  File "/home/paul/.virtualenvs/scrapy12/local/lib/python2.7/site-packages/scrapy/spidermiddlewares/urllength.py", line 37, in <genexpr>
    return (r for r in result or () if _filter(r))
  File "/home/paul/.virtualenvs/scrapy12/local/lib/python2.7/site-packages/scrapy/spidermiddlewares/depth.py", line 58, in <genexpr>
    return (r for r in result or () if _filter(r))
  File "/home/paul/.virtualenvs/scrapy12/local/lib/python2.7/site-packages/scrapy/spiders/sitemap.py", line 36, in _parse_sitemap
    yield Request(url, callback=self._parse_sitemap)
  File "/home/paul/.virtualenvs/scrapy12/local/lib/python2.7/site-packages/scrapy/http/request/__init__.py", line 25, in __init__
    self._set_url(url)
  File "/home/paul/.virtualenvs/scrapy12/local/lib/python2.7/site-packages/scrapy/http/request/__init__.py", line 57, in _set_url
    raise ValueError('Missing scheme in request url: %s' % self._url)
ValueError: Missing scheme in request url: /sitemap.ashx
2016-11-09 17:46:19 [scrapy] INFO: Closing spider (finished)
2016-11-09 17:46:19 [scrapy] INFO: Dumping Scrapy stats:
{'downloader/request_bytes': 291,
 'downloader/request_count': 1,
 'downloader/request_method_count/GET': 1,
 'downloader/response_bytes': 1857,
 'downloader/response_count': 1,
 'downloader/response_status_count/200': 1,
 'finish_reason': 'finished',
 'finish_time': datetime.datetime(2016, 11, 9, 16, 46, 19, 332383),
 'log_count/DEBUG': 2,
 'log_count/ERROR': 1,
 'log_count/INFO': 7,
 'response_received_count': 1,
 'scheduler/dequeued': 1,
 'scheduler/dequeued/memory': 1,
 'scheduler/enqueued': 1,
 'scheduler/enqueued/memory': 1,
 'spider_exceptions/ValueError': 1,
 'start_time': datetime.datetime(2016, 11, 9, 16, 46, 19, 71714)}
2016-11-09 17:46:19 [scrapy] INFO: Spider closed (finished)
```

--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `scrapy/spiders/sitemap.py`
Content:
```
1 import re
2 import logging
3 import six
4 
5 from scrapy.spiders import Spider
6 from scrapy.http import Request, XmlResponse
7 from scrapy.utils.sitemap import Sitemap, sitemap_urls_from_robots
8 from scrapy.utils.gz import gunzip, is_gzipped
9 
10 logger = logging.getLogger(__name__)
11 
12 
13 class SitemapSpider(Spider):
14 
15     sitemap_urls = ()
16     sitemap_rules = [('', 'parse')]
17     sitemap_follow = ['']
18     sitemap_alternate_links = False
19 
20     def __init__(self, *a, **kw):
21         super(SitemapSpider, self).__init__(*a, **kw)
22         self._cbs = []
23         for r, c in self.sitemap_rules:
24             if isinstance(c, six.string_types):
25                 c = getattr(self, c)
26             self._cbs.append((regex(r), c))
27         self._follow = [regex(x) for x in self.sitemap_follow]
28 
29     def start_requests(self):
30         for url in self.sitemap_urls:
31             yield Request(url, self._parse_sitemap)
32 
33     def _parse_sitemap(self, response):
34         if response.url.endswith('/robots.txt'):
35             for url in sitemap_urls_from_robots(response.text):
36                 yield Request(url, callback=self._parse_sitemap)
37         else:
38             body = self._get_sitemap_body(response)
39             if body is None:
40                 logger.warning("Ignoring invalid sitemap: %(response)s",
41                                {'response': response}, extra={'spider': self})
42                 return
43 
44             s = Sitemap(body)
45             if s.type == 'sitemapindex':
46                 for loc in iterloc(s, self.sitemap_alternate_links):
47                     if any(x.search(loc) for x in self._follow):
48                         yield Request(loc, callback=self._parse_sitemap)
49             elif s.type == 'urlset':
50                 for loc in iterloc(s):
51                     for r, c in self._cbs:
52                         if r.search(loc):
53                             yield Request(loc, callback=c)
54                             break
55 
56     def _get_sitemap_body(self, response):
57         """Return the sitemap body contained in the given response,
58         or None if the response is not a sitemap.
59         """
60         if isinstance(response, XmlResponse):
61             return response.body
62         elif is_gzipped(response):
63             return gunzip(response.body)
64         elif response.url.endswith('.xml'):
65             return response.body
66         elif response.url.endswith('.xml.gz'):
67             return gunzip(response.body)
68 
69 
70 def regex(x):
71     if isinstance(x, six.string_types):
72         return re.compile(x)
73     return x
74 
75 
76 def iterloc(it, alt=False):
77     for d in it:
78         yield d['loc']
79 
80         # Also consider alternate URLs (xhtml:link rel="alternate")
81         if alt and 'alternate' in d:
82             for l in d['alternate']:
83                 yield l
84
```

Path: `scrapy/utils/sitemap.py`
Content:
```
1 """
2 Module for processing Sitemaps.
3 
4 Note: The main purpose of this module is to provide support for the
5 SitemapSpider, its API is subject to change without notice.
6 """
7 import lxml.etree
8 
9 
10 class Sitemap(object):
11     """Class to parse Sitemap (type=urlset) and Sitemap Index
12     (type=sitemapindex) files"""
13 
14     def __init__(self, xmltext):
15         xmlp = lxml.etree.XMLParser(recover=True, remove_comments=True, resolve_entities=False)
16         self._root = lxml.etree.fromstring(xmltext, parser=xmlp)
17         rt = self._root.tag
18         self.type = self._root.tag.split('}', 1)[1] if '}' in rt else rt
19 
20     def __iter__(self):
21         for elem in self._root.getchildren():
22             d = {}
23             for el in elem.getchildren():
24                 tag = el.tag
25                 name = tag.split('}', 1)[1] if '}' in tag else tag
26 
27                 if name == 'link':
28                     if 'href' in el.attrib:
29                         d.setdefault('alternate', []).append(el.get('href'))
30                 else:
31                     d[name] = el.text.strip() if el.text else ''
32 
33             if 'loc' in d:
34                 yield d
35 
36 
37 def sitemap_urls_from_robots(robots_text):
38     """Return an iterator over all sitemap urls contained in the given
39     robots.txt file
40     """
41     for line in robots_text.splitlines():
42         if line.lstrip().lower().startswith('sitemap:'):
43             yield line.split(':', 1)[1].strip()
44
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/scrapy/spiders/sitemap.py b/scrapy/spiders/sitemap.py
--- a/scrapy/spiders/sitemap.py
+++ b/scrapy/spiders/sitemap.py
@@ -32,7 +32,7 @@
 
     def _parse_sitemap(self, response):
         if response.url.endswith('/robots.txt'):
-            for url in sitemap_urls_from_robots(response.text):
+            for url in sitemap_urls_from_robots(response.text, base_url=response.url):
                 yield Request(url, callback=self._parse_sitemap)
         else:
             body = self._get_sitemap_body(response)
diff --git a/scrapy/utils/sitemap.py b/scrapy/utils/sitemap.py
--- a/scrapy/utils/sitemap.py
+++ b/scrapy/utils/sitemap.py
@@ -4,7 +4,9 @@
 Note: The main purpose of this module is to provide support for the
 SitemapSpider, its API is subject to change without notice.
 """
+
 import lxml.etree
+from six.moves.urllib.parse import urljoin
 
 
 class Sitemap(object):
@@ -34,10 +36,11 @@
                 yield d
 
 
-def sitemap_urls_from_robots(robots_text):
+def sitemap_urls_from_robots(robots_text, base_url=None):
     """Return an iterator over all sitemap urls contained in the given
     robots.txt file
     """
     for line in robots_text.splitlines():
         if line.lstrip().lower().startswith('sitemap:'):
-            yield line.split(':', 1)[1].strip()
+            url = line.split(':', 1)[1].strip()
+            yield urljoin(base_url, url)

{"golden_diff": "diff --git a/scrapy/spiders/sitemap.py b/scrapy/spiders/sitemap.py\n--- a/scrapy/spiders/sitemap.py\n+++ b/scrapy/spiders/sitemap.py\n@@ -32,7 +32,7 @@\n \n     def _parse_sitemap(self, response):\n         if response.url.endswith('/robots.txt'):\n-            for url in sitemap_urls_from_robots(response.text):\n+            for url in sitemap_urls_from_robots(response.text, base_url=response.url):\n                 yield Request(url, callback=self._parse_sitemap)\n         else:\n             body = self._get_sitemap_body(response)\ndiff --git a/scrapy/utils/sitemap.py b/scrapy/utils/sitemap.py\n--- a/scrapy/utils/sitemap.py\n+++ b/scrapy/utils/sitemap.py\n@@ -4,7 +4,9 @@\n Note: The main purpose of this module is to provide support for the\n SitemapSpider, its API is subject to change without notice.\n \"\"\"\n+\n import lxml.etree\n+from six.moves.urllib.parse import urljoin\n \n \n class Sitemap(object):\n@@ -34,10 +36,11 @@\n                 yield d\n \n \n-def sitemap_urls_from_robots(robots_text):\n+def sitemap_urls_from_robots(robots_text, base_url=None):\n     \"\"\"Return an iterator over all sitemap urls contained in the given\n     robots.txt file\n     \"\"\"\n     for line in robots_text.splitlines():\n         if line.lstrip().lower().startswith('sitemap:'):\n-            yield line.split(':', 1)[1].strip()\n+            url = line.split(':', 1)[1].strip()\n+            yield urljoin(base_url, url)\n", "issue": "Sitemap spider not robust against wrong sitemap URLs in robots.txt\n[The \"specs\"](http://www.sitemaps.org/protocol.html#submit_robots) do say that the URL should be a \"full URL\":\r\n\r\n> You can specify the location of the Sitemap using a robots.txt file. To do this, simply add the following line including the full URL to the sitemap:\r\n> `Sitemap: http://www.example.com/sitemap.xml`\r\n\r\nBut some robots.txt use relative ones.\r\n\r\nExample: http://www.asos.com/robots.txt\r\n\r\n```\r\nUser-agent: *\r\nSitemap: /sitemap.ashx\r\nSitemap: http://www.asos.com/sitemap.xml\r\nDisallow: /basket/\r\n(...)\r\n```\r\n\r\nSpider:\r\n```\r\nfrom scrapy.spiders import SitemapSpider\r\n\r\n\r\nclass TestSpider(SitemapSpider):\r\n    name = \"test\"\r\n    sitemap_urls = [\r\n        'http://www.asos.com/robots.txt',\r\n    ]\r\n\r\n    def parse(self, response):\r\n        self.logger.info('parsing %r' % response.url)\r\n```\r\nLogs:\r\n\r\n```\r\n$ scrapy runspider spider.py\r\nLinux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.90 Safari/537.36'\r\n2016-11-09 17:46:19 [scrapy] INFO: Scrapy 1.2.1 started (bot: scrapybot)\r\n(...)\r\n2016-11-09 17:46:19 [scrapy] DEBUG: Crawled (200) <GET http://www.asos.com/robots.txt> (referer: None)\r\n2016-11-09 17:46:19 [scrapy] ERROR: Spider error processing <GET http://www.asos.com/robots.txt> (referer: None)\r\nTraceback (most recent call last):\r\n  File \"/home/paul/.virtualenvs/scrapy12/local/lib/python2.7/site-packages/scrapy/utils/defer.py\", line 102, in iter_errback\r\n    yield next(it)\r\n  File \"/home/paul/.virtualenvs/scrapy12/local/lib/python2.7/site-packages/scrapy/spidermiddlewares/offsite.py\", line 29, in process_spider_output\r\n    for x in result:\r\n  File \"/home/paul/.virtualenvs/scrapy12/local/lib/python2.7/site-packages/scrapy/spidermiddlewares/referer.py\", line 22, in <genexpr>\r\n    return (_set_referer(r) for r in result or ())\r\n  File \"/home/paul/.virtualenvs/scrapy12/local/lib/python2.7/site-packages/scrapy/spidermiddlewares/urllength.py\", line 37, in <genexpr>\r\n    return (r for r in result or () if _filter(r))\r\n  File \"/home/paul/.virtualenvs/scrapy12/local/lib/python2.7/site-packages/scrapy/spidermiddlewares/depth.py\", line 58, in <genexpr>\r\n    return (r for r in result or () if _filter(r))\r\n  File \"/home/paul/.virtualenvs/scrapy12/local/lib/python2.7/site-packages/scrapy/spiders/sitemap.py\", line 36, in _parse_sitemap\r\n    yield Request(url, callback=self._parse_sitemap)\r\n  File \"/home/paul/.virtualenvs/scrapy12/local/lib/python2.7/site-packages/scrapy/http/request/__init__.py\", line 25, in __init__\r\n    self._set_url(url)\r\n  File \"/home/paul/.virtualenvs/scrapy12/local/lib/python2.7/site-packages/scrapy/http/request/__init__.py\", line 57, in _set_url\r\n    raise ValueError('Missing scheme in request url: %s' % self._url)\r\nValueError: Missing scheme in request url: /sitemap.ashx\r\n2016-11-09 17:46:19 [scrapy] INFO: Closing spider (finished)\r\n2016-11-09 17:46:19 [scrapy] INFO: Dumping Scrapy stats:\r\n{'downloader/request_bytes': 291,\r\n 'downloader/request_count': 1,\r\n 'downloader/request_method_count/GET': 1,\r\n 'downloader/response_bytes': 1857,\r\n 'downloader/response_count': 1,\r\n 'downloader/response_status_count/200': 1,\r\n 'finish_reason': 'finished',\r\n 'finish_time': datetime.datetime(2016, 11, 9, 16, 46, 19, 332383),\r\n 'log_count/DEBUG': 2,\r\n 'log_count/ERROR': 1,\r\n 'log_count/INFO': 7,\r\n 'response_received_count': 1,\r\n 'scheduler/dequeued': 1,\r\n 'scheduler/dequeued/memory': 1,\r\n 'scheduler/enqueued': 1,\r\n 'scheduler/enqueued/memory': 1,\r\n 'spider_exceptions/ValueError': 1,\r\n 'start_time': datetime.datetime(2016, 11, 9, 16, 46, 19, 71714)}\r\n2016-11-09 17:46:19 [scrapy] INFO: Spider closed (finished)\r\n```\n", "before_files": [{"content": "import re\nimport logging\nimport six\n\nfrom scrapy.spiders import Spider\nfrom scrapy.http import Request, XmlResponse\nfrom scrapy.utils.sitemap import Sitemap, sitemap_urls_from_robots\nfrom scrapy.utils.gz import gunzip, is_gzipped\n\nlogger = logging.getLogger(__name__)\n\n\nclass SitemapSpider(Spider):\n\n    sitemap_urls = ()\n    sitemap_rules = [('', 'parse')]\n    sitemap_follow = ['']\n    sitemap_alternate_links = False\n\n    def __init__(self, *a, **kw):\n        super(SitemapSpider, self).__init__(*a, **kw)\n        self._cbs = []\n        for r, c in self.sitemap_rules:\n            if isinstance(c, six.string_types):\n                c = getattr(self, c)\n            self._cbs.append((regex(r), c))\n        self._follow = [regex(x) for x in self.sitemap_follow]\n\n    def start_requests(self):\n        for url in self.sitemap_urls:\n            yield Request(url, self._parse_sitemap)\n\n    def _parse_sitemap(self, response):\n        if response.url.endswith('/robots.txt'):\n            for url in sitemap_urls_from_robots(response.text):\n                yield Request(url, callback=self._parse_sitemap)\n        else:\n            body = self._get_sitemap_body(response)\n            if body is None:\n                logger.warning(\"Ignoring invalid sitemap: %(response)s\",\n                               {'response': response}, extra={'spider': self})\n                return\n\n            s = Sitemap(body)\n            if s.type == 'sitemapindex':\n                for loc in iterloc(s, self.sitemap_alternate_links):\n                    if any(x.search(loc) for x in self._follow):\n                        yield Request(loc, callback=self._parse_sitemap)\n            elif s.type == 'urlset':\n                for loc in iterloc(s):\n                    for r, c in self._cbs:\n                        if r.search(loc):\n                            yield Request(loc, callback=c)\n                            break\n\n    def _get_sitemap_body(self, response):\n        \"\"\"Return the sitemap body contained in the given response,\n        or None if the response is not a sitemap.\n        \"\"\"\n        if isinstance(response, XmlResponse):\n            return response.body\n        elif is_gzipped(response):\n            return gunzip(response.body)\n        elif response.url.endswith('.xml'):\n            return response.body\n        elif response.url.endswith('.xml.gz'):\n            return gunzip(response.body)\n\n\ndef regex(x):\n    if isinstance(x, six.string_types):\n        return re.compile(x)\n    return x\n\n\ndef iterloc(it, alt=False):\n    for d in it:\n        yield d['loc']\n\n        # Also consider alternate URLs (xhtml:link rel=\"alternate\")\n        if alt and 'alternate' in d:\n            for l in d['alternate']:\n                yield l\n", "path": "scrapy/spiders/sitemap.py"}, {"content": "\"\"\"\nModule for processing Sitemaps.\n\nNote: The main purpose of this module is to provide support for the\nSitemapSpider, its API is subject to change without notice.\n\"\"\"\nimport lxml.etree\n\n\nclass Sitemap(object):\n    \"\"\"Class to parse Sitemap (type=urlset) and Sitemap Index\n    (type=sitemapindex) files\"\"\"\n\n    def __init__(self, xmltext):\n        xmlp = lxml.etree.XMLParser(recover=True, remove_comments=True, resolve_entities=False)\n        self._root = lxml.etree.fromstring(xmltext, parser=xmlp)\n        rt = self._root.tag\n        self.type = self._root.tag.split('}', 1)[1] if '}' in rt else rt\n\n    def __iter__(self):\n        for elem in self._root.getchildren():\n            d = {}\n            for el in elem.getchildren():\n                tag = el.tag\n                name = tag.split('}', 1)[1] if '}' in tag else tag\n\n                if name == 'link':\n                    if 'href' in el.attrib:\n                        d.setdefault('alternate', []).append(el.get('href'))\n                else:\n                    d[name] = el.text.strip() if el.text else ''\n\n            if 'loc' in d:\n                yield d\n\n\ndef sitemap_urls_from_robots(robots_text):\n    \"\"\"Return an iterator over all sitemap urls contained in the given\n    robots.txt file\n    \"\"\"\n    for line in robots_text.splitlines():\n        if line.lstrip().lower().startswith('sitemap:'):\n            yield line.split(':', 1)[1].strip()\n", "path": "scrapy/utils/sitemap.py"}], "after_files": [{"content": "import re\nimport logging\nimport six\n\nfrom scrapy.spiders import Spider\nfrom scrapy.http import Request, XmlResponse\nfrom scrapy.utils.sitemap import Sitemap, sitemap_urls_from_robots\nfrom scrapy.utils.gz import gunzip, is_gzipped\n\nlogger = logging.getLogger(__name__)\n\n\nclass SitemapSpider(Spider):\n\n    sitemap_urls = ()\n    sitemap_rules = [('', 'parse')]\n    sitemap_follow = ['']\n    sitemap_alternate_links = False\n\n    def __init__(self, *a, **kw):\n        super(SitemapSpider, self).__init__(*a, **kw)\n        self._cbs = []\n        for r, c in self.sitemap_rules:\n            if isinstance(c, six.string_types):\n                c = getattr(self, c)\n            self._cbs.append((regex(r), c))\n        self._follow = [regex(x) for x in self.sitemap_follow]\n\n    def start_requests(self):\n        for url in self.sitemap_urls:\n            yield Request(url, self._parse_sitemap)\n\n    def _parse_sitemap(self, response):\n        if response.url.endswith('/robots.txt'):\n            for url in sitemap_urls_from_robots(response.text, base_url=response.url):\n                yield Request(url, callback=self._parse_sitemap)\n        else:\n            body = self._get_sitemap_body(response)\n            if body is None:\n                logger.warning(\"Ignoring invalid sitemap: %(response)s\",\n                               {'response': response}, extra={'spider': self})\n                return\n\n            s = Sitemap(body)\n            if s.type == 'sitemapindex':\n                for loc in iterloc(s, self.sitemap_alternate_links):\n                    if any(x.search(loc) for x in self._follow):\n                        yield Request(loc, callback=self._parse_sitemap)\n            elif s.type == 'urlset':\n                for loc in iterloc(s):\n                    for r, c in self._cbs:\n                        if r.search(loc):\n                            yield Request(loc, callback=c)\n                            break\n\n    def _get_sitemap_body(self, response):\n        \"\"\"Return the sitemap body contained in the given response,\n        or None if the response is not a sitemap.\n        \"\"\"\n        if isinstance(response, XmlResponse):\n            return response.body\n        elif is_gzipped(response):\n            return gunzip(response.body)\n        elif response.url.endswith('.xml'):\n            return response.body\n        elif response.url.endswith('.xml.gz'):\n            return gunzip(response.body)\n\n\ndef regex(x):\n    if isinstance(x, six.string_types):\n        return re.compile(x)\n    return x\n\n\ndef iterloc(it, alt=False):\n    for d in it:\n        yield d['loc']\n\n        # Also consider alternate URLs (xhtml:link rel=\"alternate\")\n        if alt and 'alternate' in d:\n            for l in d['alternate']:\n                yield l\n", "path": "scrapy/spiders/sitemap.py"}, {"content": "\"\"\"\nModule for processing Sitemaps.\n\nNote: The main purpose of this module is to provide support for the\nSitemapSpider, its API is subject to change without notice.\n\"\"\"\n\nimport lxml.etree\nfrom six.moves.urllib.parse import urljoin\n\n\nclass Sitemap(object):\n    \"\"\"Class to parse Sitemap (type=urlset) and Sitemap Index\n    (type=sitemapindex) files\"\"\"\n\n    def __init__(self, xmltext):\n        xmlp = lxml.etree.XMLParser(recover=True, remove_comments=True, resolve_entities=False)\n        self._root = lxml.etree.fromstring(xmltext, parser=xmlp)\n        rt = self._root.tag\n        self.type = self._root.tag.split('}', 1)[1] if '}' in rt else rt\n\n    def __iter__(self):\n        for elem in self._root.getchildren():\n            d = {}\n            for el in elem.getchildren():\n                tag = el.tag\n                name = tag.split('}', 1)[1] if '}' in tag else tag\n\n                if name == 'link':\n                    if 'href' in el.attrib:\n                        d.setdefault('alternate', []).append(el.get('href'))\n                else:\n                    d[name] = el.text.strip() if el.text else ''\n\n            if 'loc' in d:\n                yield d\n\n\ndef sitemap_urls_from_robots(robots_text, base_url=None):\n    \"\"\"Return an iterator over all sitemap urls contained in the given\n    robots.txt file\n    \"\"\"\n    for line in robots_text.splitlines():\n        if line.lstrip().lower().startswith('sitemap:'):\n            url = line.split(':', 1)[1].strip()\n            yield urljoin(base_url, url)\n", "path": "scrapy/utils/sitemap.py"}]}