prompt: We are currently solving the following issue within our repository. Here is the issue text:
  --- BEGIN ISSUE ---
  _nons function is not used in scrapy.contrib.linkextractors.lxmlhtml
  There is either a bug or some stray code in https://github.com/scrapy/scrapy/blob/master/scrapy/contrib/linkextractors/lxmlhtml.py#L37: `tag = _nons(el.tag)` local variable is not used, and so `_nons` function is also unused. @redapple - what was the intended behavior?
  
  
  --- END ISSUE ---
  
  Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
  --- BEGIN FILES ---
  Path: `scrapy/contrib/linkextractors/lxmlhtml.py`
  Content:
  ```
  1 """
  2 Link extractor based on lxml.html
  3 """
  4 
  5 import re
  6 from urlparse import urlparse, urljoin
  7 
  8 import lxml.etree as etree
  9 
  10 from scrapy.selector import Selector
  11 from scrapy.link import Link
  12 from scrapy.utils.misc import arg_to_iter
  13 from scrapy.utils.python import unique as unique_list, str_to_unicode
  14 from scrapy.linkextractor import FilteringLinkExtractor
  15 from scrapy.utils.response import get_base_url
  16 
  17 
  18 # from lxml/src/lxml/html/__init__.py
  19 XHTML_NAMESPACE = "http://www.w3.org/1999/xhtml"
  20 
  21 _collect_string_content = etree.XPath("string()")
  22 
  23 def _nons(tag):
  24     if isinstance(tag, basestring):
  25         if tag[0] == '{' and tag[1:len(XHTML_NAMESPACE)+1] == XHTML_NAMESPACE:
  26             return tag.split('}')[-1]
  27     return tag
  28 
  29 
  30 class LxmlParserLinkExtractor(object):
  31     def __init__(self, tag="a", attr="href", process=None, unique=False):
  32         self.scan_tag = tag if callable(tag) else lambda t: t == tag
  33         self.scan_attr = attr if callable(attr) else lambda a: a == attr
  34         self.process_attr = process if callable(process) else lambda v: v
  35         self.unique = unique
  36 
  37     def _iter_links(self, document):
  38         for el in document.iter(etree.Element):
  39             tag = _nons(el.tag)
  40             if not self.scan_tag(el.tag):
  41                 continue
  42             attribs = el.attrib
  43             for attrib in attribs:
  44                 yield (el, attrib, attribs[attrib])
  45 
  46     def _extract_links(self, selector, response_url, response_encoding, base_url):
  47         links = []
  48         # hacky way to get the underlying lxml parsed document
  49         for el, attr, attr_val in self._iter_links(selector._root):
  50             if self.scan_tag(el.tag) and self.scan_attr(attr):
  51                 # pseudo _root.make_links_absolute(base_url)
  52                 attr_val = urljoin(base_url, attr_val)
  53                 url = self.process_attr(attr_val)
  54                 if url is None:
  55                     continue
  56                 if isinstance(url, unicode):
  57                     url = url.encode(response_encoding)
  58                 # to fix relative links after process_value
  59                 url = urljoin(response_url, url)
  60                 link = Link(url, _collect_string_content(el) or u'',
  61                     nofollow=True if el.get('rel') == 'nofollow' else False)
  62                 links.append(link)
  63 
  64         return unique_list(links, key=lambda link: link.url) \
  65                 if self.unique else links
  66 
  67     def extract_links(self, response):
  68         html = Selector(response)
  69         base_url = get_base_url(response)
  70         return self._extract_links(html, response.url, response.encoding, base_url)
  71 
  72     def _process_links(self, links):
  73         """ Normalize and filter extracted links
  74 
  75         The subclass should override it if neccessary
  76         """
  77         links = unique_list(links, key=lambda link: link.url) if self.unique else links
  78         return links
  79 
  80 
  81 class LxmlLinkExtractor(FilteringLinkExtractor):
  82 
  83     def __init__(self, allow=(), deny=(), allow_domains=(), deny_domains=(), restrict_xpaths=(),
  84                  tags=('a', 'area'), attrs=('href',), canonicalize=True, unique=True, process_value=None,
  85                  deny_extensions=None):
  86         tags, attrs = set(arg_to_iter(tags)), set(arg_to_iter(attrs))
  87         tag_func = lambda x: x in tags
  88         attr_func = lambda x: x in attrs
  89         lx = LxmlParserLinkExtractor(tag=tag_func, attr=attr_func,
  90             unique=unique, process=process_value)
  91 
  92         super(LxmlLinkExtractor, self).__init__(lx, allow, deny,
  93             allow_domains, deny_domains, restrict_xpaths, canonicalize,
  94             deny_extensions)
  95 
  96     def extract_links(self, response):
  97         html = Selector(response)
  98         base_url = get_base_url(response)
  99         if self.restrict_xpaths:
  100             docs = [subdoc
  101                     for x in self.restrict_xpaths
  102                     for subdoc in html.xpath(x)]
  103         else:
  104             docs = [html]
  105         all_links = []
  106         for doc in docs:
  107             links = self._extract_links(doc, response.url, response.encoding, base_url)
  108             all_links.extend(self._process_links(links))
  109         return unique_list(all_links)
  110 
  111
  ```
  --- END FILES ---
  
  Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.
  
  Here is an example:
  
  ```diff
  diff --git a/examples/server_async.py b/examples/server_async.py
  --- a/examples/server_async.py
  +++ b/examples/server_async.py
  @@ -313,4 +313,4 @@
   
   
   if __name__ == "__main__":
  -    asyncio.run(run_async_server("."), debug=True)
  +    asyncio.run(run_async_server(), debug=True)
  diff --git a/examples/server_sync.py b/examples/server_sync.py
  --- a/examples/server_sync.py
  +++ b/examples/server_sync.py
  @@ -313,5 +313,5 @@
   
   
   if __name__ == "__main__":
  -    server = run_sync_server(".")
  +    server = run_sync_server()
       server.shutdown()
  
  ```
  
golden_diff: diff --git a/scrapy/contrib/linkextractors/lxmlhtml.py b/scrapy/contrib/linkextractors/lxmlhtml.py
  --- a/scrapy/contrib/linkextractors/lxmlhtml.py
  +++ b/scrapy/contrib/linkextractors/lxmlhtml.py
  @@ -36,30 +36,30 @@
   
       def _iter_links(self, document):
           for el in document.iter(etree.Element):
  -            tag = _nons(el.tag)
  -            if not self.scan_tag(el.tag):
  +            if not self.scan_tag(_nons(el.tag)):
                   continue
               attribs = el.attrib
               for attrib in attribs:
  +                if not self.scan_attr(attrib):
  +                    continue
                   yield (el, attrib, attribs[attrib])
   
       def _extract_links(self, selector, response_url, response_encoding, base_url):
           links = []
           # hacky way to get the underlying lxml parsed document
           for el, attr, attr_val in self._iter_links(selector._root):
  -            if self.scan_tag(el.tag) and self.scan_attr(attr):
  -                # pseudo _root.make_links_absolute(base_url)
  -                attr_val = urljoin(base_url, attr_val)
  -                url = self.process_attr(attr_val)
  -                if url is None:
  -                    continue
  -                if isinstance(url, unicode):
  -                    url = url.encode(response_encoding)
  -                # to fix relative links after process_value
  -                url = urljoin(response_url, url)
  -                link = Link(url, _collect_string_content(el) or u'',
  -                    nofollow=True if el.get('rel') == 'nofollow' else False)
  -                links.append(link)
  +            # pseudo lxml.html.HtmlElement.make_links_absolute(base_url)
  +            attr_val = urljoin(base_url, attr_val)
  +            url = self.process_attr(attr_val)
  +            if url is None:
  +                continue
  +            if isinstance(url, unicode):
  +                url = url.encode(response_encoding)
  +            # to fix relative links after process_value
  +            url = urljoin(response_url, url)
  +            link = Link(url, _collect_string_content(el) or u'',
  +                nofollow=True if el.get('rel') == 'nofollow' else False)
  +            links.append(link)
   
           return unique_list(links, key=lambda link: link.url) \
                   if self.unique else links
  
verification_info: {"golden_diff": "diff --git a/scrapy/contrib/linkextractors/lxmlhtml.py b/scrapy/contrib/linkextractors/lxmlhtml.py\n--- a/scrapy/contrib/linkextractors/lxmlhtml.py\n+++ b/scrapy/contrib/linkextractors/lxmlhtml.py\n@@ -36,30 +36,30 @@\n \n     def _iter_links(self, document):\n         for el in document.iter(etree.Element):\n-            tag = _nons(el.tag)\n-            if not self.scan_tag(el.tag):\n+            if not self.scan_tag(_nons(el.tag)):\n                 continue\n             attribs = el.attrib\n             for attrib in attribs:\n+                if not self.scan_attr(attrib):\n+                    continue\n                 yield (el, attrib, attribs[attrib])\n \n     def _extract_links(self, selector, response_url, response_encoding, base_url):\n         links = []\n         # hacky way to get the underlying lxml parsed document\n         for el, attr, attr_val in self._iter_links(selector._root):\n-            if self.scan_tag(el.tag) and self.scan_attr(attr):\n-                # pseudo _root.make_links_absolute(base_url)\n-                attr_val = urljoin(base_url, attr_val)\n-                url = self.process_attr(attr_val)\n-                if url is None:\n-                    continue\n-                if isinstance(url, unicode):\n-                    url = url.encode(response_encoding)\n-                # to fix relative links after process_value\n-                url = urljoin(response_url, url)\n-                link = Link(url, _collect_string_content(el) or u'',\n-                    nofollow=True if el.get('rel') == 'nofollow' else False)\n-                links.append(link)\n+            # pseudo lxml.html.HtmlElement.make_links_absolute(base_url)\n+            attr_val = urljoin(base_url, attr_val)\n+            url = self.process_attr(attr_val)\n+            if url is None:\n+                continue\n+            if isinstance(url, unicode):\n+                url = url.encode(response_encoding)\n+            # to fix relative links after process_value\n+            url = urljoin(response_url, url)\n+            link = Link(url, _collect_string_content(el) or u'',\n+                nofollow=True if el.get('rel') == 'nofollow' else False)\n+            links.append(link)\n \n         return unique_list(links, key=lambda link: link.url) \\\n                 if self.unique else links\n", "issue": "_nons function is not used in scrapy.contrib.linkextractors.lxmlhtml\nThere is either a bug or some stray code in https://github.com/scrapy/scrapy/blob/master/scrapy/contrib/linkextractors/lxmlhtml.py#L37: `tag = _nons(el.tag)` local variable is not used, and so `_nons` function is also unused. @redapple - what was the intended behavior?\n\n", "before_files": [{"content": "\"\"\"\nLink extractor based on lxml.html\n\"\"\"\n\nimport re\nfrom urlparse import urlparse, urljoin\n\nimport lxml.etree as etree\n\nfrom scrapy.selector import Selector\nfrom scrapy.link import Link\nfrom scrapy.utils.misc import arg_to_iter\nfrom scrapy.utils.python import unique as unique_list, str_to_unicode\nfrom scrapy.linkextractor import FilteringLinkExtractor\nfrom scrapy.utils.response import get_base_url\n\n\n# from lxml/src/lxml/html/__init__.py\nXHTML_NAMESPACE = \"http://www.w3.org/1999/xhtml\"\n\n_collect_string_content = etree.XPath(\"string()\")\n\ndef _nons(tag):\n    if isinstance(tag, basestring):\n        if tag[0] == '{' and tag[1:len(XHTML_NAMESPACE)+1] == XHTML_NAMESPACE:\n            return tag.split('}')[-1]\n    return tag\n\n\nclass LxmlParserLinkExtractor(object):\n    def __init__(self, tag=\"a\", attr=\"href\", process=None, unique=False):\n        self.scan_tag = tag if callable(tag) else lambda t: t == tag\n        self.scan_attr = attr if callable(attr) else lambda a: a == attr\n        self.process_attr = process if callable(process) else lambda v: v\n        self.unique = unique\n\n    def _iter_links(self, document):\n        for el in document.iter(etree.Element):\n            tag = _nons(el.tag)\n            if not self.scan_tag(el.tag):\n                continue\n            attribs = el.attrib\n            for attrib in attribs:\n                yield (el, attrib, attribs[attrib])\n\n    def _extract_links(self, selector, response_url, response_encoding, base_url):\n        links = []\n        # hacky way to get the underlying lxml parsed document\n        for el, attr, attr_val in self._iter_links(selector._root):\n            if self.scan_tag(el.tag) and self.scan_attr(attr):\n                # pseudo _root.make_links_absolute(base_url)\n                attr_val = urljoin(base_url, attr_val)\n                url = self.process_attr(attr_val)\n                if url is None:\n                    continue\n                if isinstance(url, unicode):\n                    url = url.encode(response_encoding)\n                # to fix relative links after process_value\n                url = urljoin(response_url, url)\n                link = Link(url, _collect_string_content(el) or u'',\n                    nofollow=True if el.get('rel') == 'nofollow' else False)\n                links.append(link)\n\n        return unique_list(links, key=lambda link: link.url) \\\n                if self.unique else links\n\n    def extract_links(self, response):\n        html = Selector(response)\n        base_url = get_base_url(response)\n        return self._extract_links(html, response.url, response.encoding, base_url)\n\n    def _process_links(self, links):\n        \"\"\" Normalize and filter extracted links\n\n        The subclass should override it if neccessary\n        \"\"\"\n        links = unique_list(links, key=lambda link: link.url) if self.unique else links\n        return links\n\n\nclass LxmlLinkExtractor(FilteringLinkExtractor):\n\n    def __init__(self, allow=(), deny=(), allow_domains=(), deny_domains=(), restrict_xpaths=(),\n                 tags=('a', 'area'), attrs=('href',), canonicalize=True, unique=True, process_value=None,\n                 deny_extensions=None):\n        tags, attrs = set(arg_to_iter(tags)), set(arg_to_iter(attrs))\n        tag_func = lambda x: x in tags\n        attr_func = lambda x: x in attrs\n        lx = LxmlParserLinkExtractor(tag=tag_func, attr=attr_func,\n            unique=unique, process=process_value)\n\n        super(LxmlLinkExtractor, self).__init__(lx, allow, deny,\n            allow_domains, deny_domains, restrict_xpaths, canonicalize,\n            deny_extensions)\n\n    def extract_links(self, response):\n        html = Selector(response)\n        base_url = get_base_url(response)\n        if self.restrict_xpaths:\n            docs = [subdoc\n                    for x in self.restrict_xpaths\n                    for subdoc in html.xpath(x)]\n        else:\n            docs = [html]\n        all_links = []\n        for doc in docs:\n            links = self._extract_links(doc, response.url, response.encoding, base_url)\n            all_links.extend(self._process_links(links))\n        return unique_list(all_links)\n\n", "path": "scrapy/contrib/linkextractors/lxmlhtml.py"}], "after_files": [{"content": "\"\"\"\nLink extractor based on lxml.html\n\"\"\"\n\nimport re\nfrom urlparse import urlparse, urljoin\n\nimport lxml.etree as etree\n\nfrom scrapy.selector import Selector\nfrom scrapy.link import Link\nfrom scrapy.utils.misc import arg_to_iter\nfrom scrapy.utils.python import unique as unique_list, str_to_unicode\nfrom scrapy.linkextractor import FilteringLinkExtractor\nfrom scrapy.utils.response import get_base_url\n\n\n# from lxml/src/lxml/html/__init__.py\nXHTML_NAMESPACE = \"http://www.w3.org/1999/xhtml\"\n\n_collect_string_content = etree.XPath(\"string()\")\n\ndef _nons(tag):\n    if isinstance(tag, basestring):\n        if tag[0] == '{' and tag[1:len(XHTML_NAMESPACE)+1] == XHTML_NAMESPACE:\n            return tag.split('}')[-1]\n    return tag\n\n\nclass LxmlParserLinkExtractor(object):\n    def __init__(self, tag=\"a\", attr=\"href\", process=None, unique=False):\n        self.scan_tag = tag if callable(tag) else lambda t: t == tag\n        self.scan_attr = attr if callable(attr) else lambda a: a == attr\n        self.process_attr = process if callable(process) else lambda v: v\n        self.unique = unique\n\n    def _iter_links(self, document):\n        for el in document.iter(etree.Element):\n            if not self.scan_tag(_nons(el.tag)):\n                continue\n            attribs = el.attrib\n            for attrib in attribs:\n                if not self.scan_attr(attrib):\n                    continue\n                yield (el, attrib, attribs[attrib])\n\n    def _extract_links(self, selector, response_url, response_encoding, base_url):\n        links = []\n        # hacky way to get the underlying lxml parsed document\n        for el, attr, attr_val in self._iter_links(selector._root):\n            # pseudo lxml.html.HtmlElement.make_links_absolute(base_url)\n            attr_val = urljoin(base_url, attr_val)\n            url = self.process_attr(attr_val)\n            if url is None:\n                continue\n            if isinstance(url, unicode):\n                url = url.encode(response_encoding)\n            # to fix relative links after process_value\n            url = urljoin(response_url, url)\n            link = Link(url, _collect_string_content(el) or u'',\n                nofollow=True if el.get('rel') == 'nofollow' else False)\n            links.append(link)\n\n        return unique_list(links, key=lambda link: link.url) \\\n                if self.unique else links\n\n    def extract_links(self, response):\n        html = Selector(response)\n        base_url = get_base_url(response)\n        return self._extract_links(html, response.url, response.encoding, base_url)\n\n    def _process_links(self, links):\n        \"\"\" Normalize and filter extracted links\n\n        The subclass should override it if neccessary\n        \"\"\"\n        links = unique_list(links, key=lambda link: link.url) if self.unique else links\n        return links\n\n\nclass LxmlLinkExtractor(FilteringLinkExtractor):\n\n    def __init__(self, allow=(), deny=(), allow_domains=(), deny_domains=(), restrict_xpaths=(),\n                 tags=('a', 'area'), attrs=('href',), canonicalize=True, unique=True, process_value=None,\n                 deny_extensions=None):\n        tags, attrs = set(arg_to_iter(tags)), set(arg_to_iter(attrs))\n        tag_func = lambda x: x in tags\n        attr_func = lambda x: x in attrs\n        lx = LxmlParserLinkExtractor(tag=tag_func, attr=attr_func,\n            unique=unique, process=process_value)\n\n        super(LxmlLinkExtractor, self).__init__(lx, allow, deny,\n            allow_domains, deny_domains, restrict_xpaths, canonicalize,\n            deny_extensions)\n\n    def extract_links(self, response):\n        html = Selector(response)\n        base_url = get_base_url(response)\n        if self.restrict_xpaths:\n            docs = [subdoc\n                    for x in self.restrict_xpaths\n                    for subdoc in html.xpath(x)]\n        else:\n            docs = [html]\n        all_links = []\n        for doc in docs:\n            links = self._extract_links(doc, response.url, response.encoding, base_url)\n            all_links.extend(self._process_links(links))\n        return unique_list(all_links)\n\n", "path": "scrapy/contrib/linkextractors/lxmlhtml.py"}]}