We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
Deprecate htmlparser link extractor
Let's add a deprecation warning like we did for SGML Link extractor


--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `scrapy/linkextractors/sgml.py`
Content:
```
1 """
2 SGMLParser-based Link extractors
3 """
4 from six.moves.urllib.parse import urljoin
5 import warnings
6 from sgmllib import SGMLParser
7 
8 from w3lib.url import safe_url_string
9 from scrapy.selector import Selector
10 from scrapy.link import Link
11 from scrapy.linkextractor import FilteringLinkExtractor
12 from scrapy.utils.misc import arg_to_iter
13 from scrapy.utils.python import unique as unique_list, str_to_unicode
14 from scrapy.utils.response import get_base_url
15 from scrapy.exceptions import ScrapyDeprecationWarning
16 
17 
18 class BaseSgmlLinkExtractor(SGMLParser):
19 
20     def __init__(self, tag="a", attr="href", unique=False, process_value=None):
21         warnings.warn(
22             "BaseSgmlLinkExtractor is deprecated and will be removed in future releases. "
23             "Please use scrapy.linkextractors.LinkExtractor",
24             ScrapyDeprecationWarning
25         )
26         SGMLParser.__init__(self)
27         self.scan_tag = tag if callable(tag) else lambda t: t == tag
28         self.scan_attr = attr if callable(attr) else lambda a: a == attr
29         self.process_value = (lambda v: v) if process_value is None else process_value
30         self.current_link = None
31         self.unique = unique
32 
33     def _extract_links(self, response_text, response_url, response_encoding, base_url=None):
34         """ Do the real extraction work """
35         self.reset()
36         self.feed(response_text)
37         self.close()
38 
39         ret = []
40         if base_url is None:
41             base_url = urljoin(response_url, self.base_url) if self.base_url else response_url
42         for link in self.links:
43             if isinstance(link.url, unicode):
44                 link.url = link.url.encode(response_encoding)
45             link.url = urljoin(base_url, link.url)
46             link.url = safe_url_string(link.url, response_encoding)
47             link.text = str_to_unicode(link.text, response_encoding, errors='replace').strip()
48             ret.append(link)
49 
50         return ret
51 
52     def _process_links(self, links):
53         """ Normalize and filter extracted links
54 
55         The subclass should override it if necessary
56         """
57         links = unique_list(links, key=lambda link: link.url) if self.unique else links
58         return links
59 
60     def extract_links(self, response):
61         # wrapper needed to allow to work directly with text
62         links = self._extract_links(response.body, response.url, response.encoding)
63         links = self._process_links(links)
64         return links
65 
66     def reset(self):
67         SGMLParser.reset(self)
68         self.links = []
69         self.base_url = None
70         self.current_link = None
71 
72     def unknown_starttag(self, tag, attrs):
73         if tag == 'base':
74             self.base_url = dict(attrs).get('href')
75         if self.scan_tag(tag):
76             for attr, value in attrs:
77                 if self.scan_attr(attr):
78                     url = self.process_value(value)
79                     if url is not None:
80                         link = Link(url=url, nofollow=True if dict(attrs).get('rel') == 'nofollow' else False)
81                         self.links.append(link)
82                         self.current_link = link
83 
84     def unknown_endtag(self, tag):
85         if self.scan_tag(tag):
86             self.current_link = None
87 
88     def handle_data(self, data):
89         if self.current_link:
90             self.current_link.text = self.current_link.text + data
91 
92     def matches(self, url):
93         """This extractor matches with any url, since
94         it doesn't contain any patterns"""
95         return True
96 
97 
98 class SgmlLinkExtractor(FilteringLinkExtractor):
99 
100     def __init__(self, allow=(), deny=(), allow_domains=(), deny_domains=(), restrict_xpaths=(),
101                  tags=('a', 'area'), attrs=('href',), canonicalize=True, unique=True,
102                  process_value=None, deny_extensions=None, restrict_css=()):
103 
104         warnings.warn(
105             "SgmlLinkExtractor is deprecated and will be removed in future releases. "
106             "Please use scrapy.linkextractors.LinkExtractor",
107             ScrapyDeprecationWarning
108         )
109 
110         tags, attrs = set(arg_to_iter(tags)), set(arg_to_iter(attrs))
111         tag_func = lambda x: x in tags
112         attr_func = lambda x: x in attrs
113 
114         with warnings.catch_warnings(record=True):
115             lx = BaseSgmlLinkExtractor(tag=tag_func, attr=attr_func,
116                 unique=unique, process_value=process_value)
117 
118         super(SgmlLinkExtractor, self).__init__(lx, allow=allow, deny=deny,
119             allow_domains=allow_domains, deny_domains=deny_domains,
120             restrict_xpaths=restrict_xpaths, restrict_css=restrict_css,
121             canonicalize=canonicalize, deny_extensions=deny_extensions)
122 
123         # FIXME: was added to fix a RegexLinkExtractor testcase
124         self.base_url = None
125 
126     def extract_links(self, response):
127         base_url = None
128         if self.restrict_xpaths:
129             sel = Selector(response)
130             base_url = get_base_url(response)
131             body = u''.join(f
132                             for x in self.restrict_xpaths
133                             for f in sel.xpath(x).extract()
134                             ).encode(response.encoding, errors='xmlcharrefreplace')
135         else:
136             body = response.body
137 
138         links = self._extract_links(body, response.url, response.encoding, base_url)
139         links = self._process_links(links)
140         return links
141
```

Path: `scrapy/linkextractors/htmlparser.py`
Content:
```
1 """
2 HTMLParser-based link extractor
3 """
4 
5 from HTMLParser import HTMLParser
6 from six.moves.urllib.parse import urljoin
7 
8 from w3lib.url import safe_url_string
9 
10 from scrapy.link import Link
11 from scrapy.utils.python import unique as unique_list
12 
13 class HtmlParserLinkExtractor(HTMLParser):
14 
15     def __init__(self, tag="a", attr="href", process=None, unique=False):
16         HTMLParser.__init__(self)
17 
18         self.scan_tag = tag if callable(tag) else lambda t: t == tag
19         self.scan_attr = attr if callable(attr) else lambda a: a == attr
20         self.process_attr = process if callable(process) else lambda v: v
21         self.unique = unique
22 
23     def _extract_links(self, response_text, response_url, response_encoding):
24         self.reset()
25         self.feed(response_text)
26         self.close()
27 
28         links = unique_list(self.links, key=lambda link: link.url) if self.unique else self.links
29 
30         ret = []
31         base_url = urljoin(response_url, self.base_url) if self.base_url else response_url
32         for link in links:
33             if isinstance(link.url, unicode):
34                 link.url = link.url.encode(response_encoding)
35             link.url = urljoin(base_url, link.url)
36             link.url = safe_url_string(link.url, response_encoding)
37             link.text = link.text.decode(response_encoding)
38             ret.append(link)
39 
40         return ret
41 
42     def extract_links(self, response):
43         # wrapper needed to allow to work directly with text
44         return self._extract_links(response.body, response.url, response.encoding)
45 
46     def reset(self):
47         HTMLParser.reset(self)
48 
49         self.base_url = None
50         self.current_link = None
51         self.links = []
52 
53     def handle_starttag(self, tag, attrs):
54         if tag == 'base':
55             self.base_url = dict(attrs).get('href')
56         if self.scan_tag(tag):
57             for attr, value in attrs:
58                 if self.scan_attr(attr):
59                     url = self.process_attr(value)
60                     link = Link(url=url)
61                     self.links.append(link)
62                     self.current_link = link
63 
64     def handle_endtag(self, tag):
65         if self.scan_tag(tag):
66             self.current_link = None
67 
68     def handle_data(self, data):
69         if self.current_link:
70             self.current_link.text = self.current_link.text + data
71 
72     def matches(self, url):
73         """This extractor matches with any url, since
74         it doesn't contain any patterns"""
75         return True
76
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/scrapy/linkextractors/htmlparser.py b/scrapy/linkextractors/htmlparser.py
--- a/scrapy/linkextractors/htmlparser.py
+++ b/scrapy/linkextractors/htmlparser.py
@@ -2,6 +2,7 @@
 HTMLParser-based link extractor
 """
 
+import warnings
 from HTMLParser import HTMLParser
 from six.moves.urllib.parse import urljoin
 
@@ -9,12 +10,20 @@
 
 from scrapy.link import Link
 from scrapy.utils.python import unique as unique_list
+from scrapy.exceptions import ScrapyDeprecationWarning
+
 
 class HtmlParserLinkExtractor(HTMLParser):
 
     def __init__(self, tag="a", attr="href", process=None, unique=False):
         HTMLParser.__init__(self)
 
+        warnings.warn(
+            "HtmlParserLinkExtractor is deprecated and will be removed in "
+            "future releases. Please use scrapy.linkextractors.LinkExtractor",
+            ScrapyDeprecationWarning, stacklevel=2,
+        )
+
         self.scan_tag = tag if callable(tag) else lambda t: t == tag
         self.scan_attr = attr if callable(attr) else lambda a: a == attr
         self.process_attr = process if callable(process) else lambda v: v
diff --git a/scrapy/linkextractors/sgml.py b/scrapy/linkextractors/sgml.py
--- a/scrapy/linkextractors/sgml.py
+++ b/scrapy/linkextractors/sgml.py
@@ -21,7 +21,7 @@
         warnings.warn(
             "BaseSgmlLinkExtractor is deprecated and will be removed in future releases. "
             "Please use scrapy.linkextractors.LinkExtractor",
-            ScrapyDeprecationWarning
+            ScrapyDeprecationWarning, stacklevel=2,
         )
         SGMLParser.__init__(self)
         self.scan_tag = tag if callable(tag) else lambda t: t == tag
@@ -104,7 +104,7 @@
         warnings.warn(
             "SgmlLinkExtractor is deprecated and will be removed in future releases. "
             "Please use scrapy.linkextractors.LinkExtractor",
-            ScrapyDeprecationWarning
+            ScrapyDeprecationWarning, stacklevel=2,
         )
 
         tags, attrs = set(arg_to_iter(tags)), set(arg_to_iter(attrs))

{"golden_diff": "diff --git a/scrapy/linkextractors/htmlparser.py b/scrapy/linkextractors/htmlparser.py\n--- a/scrapy/linkextractors/htmlparser.py\n+++ b/scrapy/linkextractors/htmlparser.py\n@@ -2,6 +2,7 @@\n HTMLParser-based link extractor\n \"\"\"\n \n+import warnings\n from HTMLParser import HTMLParser\n from six.moves.urllib.parse import urljoin\n \n@@ -9,12 +10,20 @@\n \n from scrapy.link import Link\n from scrapy.utils.python import unique as unique_list\n+from scrapy.exceptions import ScrapyDeprecationWarning\n+\n \n class HtmlParserLinkExtractor(HTMLParser):\n \n     def __init__(self, tag=\"a\", attr=\"href\", process=None, unique=False):\n         HTMLParser.__init__(self)\n \n+        warnings.warn(\n+            \"HtmlParserLinkExtractor is deprecated and will be removed in \"\n+            \"future releases. Please use scrapy.linkextractors.LinkExtractor\",\n+            ScrapyDeprecationWarning, stacklevel=2,\n+        )\n+\n         self.scan_tag = tag if callable(tag) else lambda t: t == tag\n         self.scan_attr = attr if callable(attr) else lambda a: a == attr\n         self.process_attr = process if callable(process) else lambda v: v\ndiff --git a/scrapy/linkextractors/sgml.py b/scrapy/linkextractors/sgml.py\n--- a/scrapy/linkextractors/sgml.py\n+++ b/scrapy/linkextractors/sgml.py\n@@ -21,7 +21,7 @@\n         warnings.warn(\n             \"BaseSgmlLinkExtractor is deprecated and will be removed in future releases. \"\n             \"Please use scrapy.linkextractors.LinkExtractor\",\n-            ScrapyDeprecationWarning\n+            ScrapyDeprecationWarning, stacklevel=2,\n         )\n         SGMLParser.__init__(self)\n         self.scan_tag = tag if callable(tag) else lambda t: t == tag\n@@ -104,7 +104,7 @@\n         warnings.warn(\n             \"SgmlLinkExtractor is deprecated and will be removed in future releases. \"\n             \"Please use scrapy.linkextractors.LinkExtractor\",\n-            ScrapyDeprecationWarning\n+            ScrapyDeprecationWarning, stacklevel=2,\n         )\n \n         tags, attrs = set(arg_to_iter(tags)), set(arg_to_iter(attrs))\n", "issue": "Deprecate htmlparser link extractor\nLet's add a deprecation warning like we did for SGML Link extractor\n\n", "before_files": [{"content": "\"\"\"\nSGMLParser-based Link extractors\n\"\"\"\nfrom six.moves.urllib.parse import urljoin\nimport warnings\nfrom sgmllib import SGMLParser\n\nfrom w3lib.url import safe_url_string\nfrom scrapy.selector import Selector\nfrom scrapy.link import Link\nfrom scrapy.linkextractor import FilteringLinkExtractor\nfrom scrapy.utils.misc import arg_to_iter\nfrom scrapy.utils.python import unique as unique_list, str_to_unicode\nfrom scrapy.utils.response import get_base_url\nfrom scrapy.exceptions import ScrapyDeprecationWarning\n\n\nclass BaseSgmlLinkExtractor(SGMLParser):\n\n    def __init__(self, tag=\"a\", attr=\"href\", unique=False, process_value=None):\n        warnings.warn(\n            \"BaseSgmlLinkExtractor is deprecated and will be removed in future releases. \"\n            \"Please use scrapy.linkextractors.LinkExtractor\",\n            ScrapyDeprecationWarning\n        )\n        SGMLParser.__init__(self)\n        self.scan_tag = tag if callable(tag) else lambda t: t == tag\n        self.scan_attr = attr if callable(attr) else lambda a: a == attr\n        self.process_value = (lambda v: v) if process_value is None else process_value\n        self.current_link = None\n        self.unique = unique\n\n    def _extract_links(self, response_text, response_url, response_encoding, base_url=None):\n        \"\"\" Do the real extraction work \"\"\"\n        self.reset()\n        self.feed(response_text)\n        self.close()\n\n        ret = []\n        if base_url is None:\n            base_url = urljoin(response_url, self.base_url) if self.base_url else response_url\n        for link in self.links:\n            if isinstance(link.url, unicode):\n                link.url = link.url.encode(response_encoding)\n            link.url = urljoin(base_url, link.url)\n            link.url = safe_url_string(link.url, response_encoding)\n            link.text = str_to_unicode(link.text, response_encoding, errors='replace').strip()\n            ret.append(link)\n\n        return ret\n\n    def _process_links(self, links):\n        \"\"\" Normalize and filter extracted links\n\n        The subclass should override it if necessary\n        \"\"\"\n        links = unique_list(links, key=lambda link: link.url) if self.unique else links\n        return links\n\n    def extract_links(self, response):\n        # wrapper needed to allow to work directly with text\n        links = self._extract_links(response.body, response.url, response.encoding)\n        links = self._process_links(links)\n        return links\n\n    def reset(self):\n        SGMLParser.reset(self)\n        self.links = []\n        self.base_url = None\n        self.current_link = None\n\n    def unknown_starttag(self, tag, attrs):\n        if tag == 'base':\n            self.base_url = dict(attrs).get('href')\n        if self.scan_tag(tag):\n            for attr, value in attrs:\n                if self.scan_attr(attr):\n                    url = self.process_value(value)\n                    if url is not None:\n                        link = Link(url=url, nofollow=True if dict(attrs).get('rel') == 'nofollow' else False)\n                        self.links.append(link)\n                        self.current_link = link\n\n    def unknown_endtag(self, tag):\n        if self.scan_tag(tag):\n            self.current_link = None\n\n    def handle_data(self, data):\n        if self.current_link:\n            self.current_link.text = self.current_link.text + data\n\n    def matches(self, url):\n        \"\"\"This extractor matches with any url, since\n        it doesn't contain any patterns\"\"\"\n        return True\n\n\nclass SgmlLinkExtractor(FilteringLinkExtractor):\n\n    def __init__(self, allow=(), deny=(), allow_domains=(), deny_domains=(), restrict_xpaths=(),\n                 tags=('a', 'area'), attrs=('href',), canonicalize=True, unique=True,\n                 process_value=None, deny_extensions=None, restrict_css=()):\n\n        warnings.warn(\n            \"SgmlLinkExtractor is deprecated and will be removed in future releases. \"\n            \"Please use scrapy.linkextractors.LinkExtractor\",\n            ScrapyDeprecationWarning\n        )\n\n        tags, attrs = set(arg_to_iter(tags)), set(arg_to_iter(attrs))\n        tag_func = lambda x: x in tags\n        attr_func = lambda x: x in attrs\n\n        with warnings.catch_warnings(record=True):\n            lx = BaseSgmlLinkExtractor(tag=tag_func, attr=attr_func,\n                unique=unique, process_value=process_value)\n\n        super(SgmlLinkExtractor, self).__init__(lx, allow=allow, deny=deny,\n            allow_domains=allow_domains, deny_domains=deny_domains,\n            restrict_xpaths=restrict_xpaths, restrict_css=restrict_css,\n            canonicalize=canonicalize, deny_extensions=deny_extensions)\n\n        # FIXME: was added to fix a RegexLinkExtractor testcase\n        self.base_url = None\n\n    def extract_links(self, response):\n        base_url = None\n        if self.restrict_xpaths:\n            sel = Selector(response)\n            base_url = get_base_url(response)\n            body = u''.join(f\n                            for x in self.restrict_xpaths\n                            for f in sel.xpath(x).extract()\n                            ).encode(response.encoding, errors='xmlcharrefreplace')\n        else:\n            body = response.body\n\n        links = self._extract_links(body, response.url, response.encoding, base_url)\n        links = self._process_links(links)\n        return links\n", "path": "scrapy/linkextractors/sgml.py"}, {"content": "\"\"\"\nHTMLParser-based link extractor\n\"\"\"\n\nfrom HTMLParser import HTMLParser\nfrom six.moves.urllib.parse import urljoin\n\nfrom w3lib.url import safe_url_string\n\nfrom scrapy.link import Link\nfrom scrapy.utils.python import unique as unique_list\n\nclass HtmlParserLinkExtractor(HTMLParser):\n\n    def __init__(self, tag=\"a\", attr=\"href\", process=None, unique=False):\n        HTMLParser.__init__(self)\n\n        self.scan_tag = tag if callable(tag) else lambda t: t == tag\n        self.scan_attr = attr if callable(attr) else lambda a: a == attr\n        self.process_attr = process if callable(process) else lambda v: v\n        self.unique = unique\n\n    def _extract_links(self, response_text, response_url, response_encoding):\n        self.reset()\n        self.feed(response_text)\n        self.close()\n\n        links = unique_list(self.links, key=lambda link: link.url) if self.unique else self.links\n\n        ret = []\n        base_url = urljoin(response_url, self.base_url) if self.base_url else response_url\n        for link in links:\n            if isinstance(link.url, unicode):\n                link.url = link.url.encode(response_encoding)\n            link.url = urljoin(base_url, link.url)\n            link.url = safe_url_string(link.url, response_encoding)\n            link.text = link.text.decode(response_encoding)\n            ret.append(link)\n\n        return ret\n\n    def extract_links(self, response):\n        # wrapper needed to allow to work directly with text\n        return self._extract_links(response.body, response.url, response.encoding)\n\n    def reset(self):\n        HTMLParser.reset(self)\n\n        self.base_url = None\n        self.current_link = None\n        self.links = []\n\n    def handle_starttag(self, tag, attrs):\n        if tag == 'base':\n            self.base_url = dict(attrs).get('href')\n        if self.scan_tag(tag):\n            for attr, value in attrs:\n                if self.scan_attr(attr):\n                    url = self.process_attr(value)\n                    link = Link(url=url)\n                    self.links.append(link)\n                    self.current_link = link\n\n    def handle_endtag(self, tag):\n        if self.scan_tag(tag):\n            self.current_link = None\n\n    def handle_data(self, data):\n        if self.current_link:\n            self.current_link.text = self.current_link.text + data\n\n    def matches(self, url):\n        \"\"\"This extractor matches with any url, since\n        it doesn't contain any patterns\"\"\"\n        return True\n", "path": "scrapy/linkextractors/htmlparser.py"}], "after_files": [{"content": "\"\"\"\nSGMLParser-based Link extractors\n\"\"\"\nfrom six.moves.urllib.parse import urljoin\nimport warnings\nfrom sgmllib import SGMLParser\n\nfrom w3lib.url import safe_url_string\nfrom scrapy.selector import Selector\nfrom scrapy.link import Link\nfrom scrapy.linkextractor import FilteringLinkExtractor\nfrom scrapy.utils.misc import arg_to_iter\nfrom scrapy.utils.python import unique as unique_list, str_to_unicode\nfrom scrapy.utils.response import get_base_url\nfrom scrapy.exceptions import ScrapyDeprecationWarning\n\n\nclass BaseSgmlLinkExtractor(SGMLParser):\n\n    def __init__(self, tag=\"a\", attr=\"href\", unique=False, process_value=None):\n        warnings.warn(\n            \"BaseSgmlLinkExtractor is deprecated and will be removed in future releases. \"\n            \"Please use scrapy.linkextractors.LinkExtractor\",\n            ScrapyDeprecationWarning, stacklevel=2,\n        )\n        SGMLParser.__init__(self)\n        self.scan_tag = tag if callable(tag) else lambda t: t == tag\n        self.scan_attr = attr if callable(attr) else lambda a: a == attr\n        self.process_value = (lambda v: v) if process_value is None else process_value\n        self.current_link = None\n        self.unique = unique\n\n    def _extract_links(self, response_text, response_url, response_encoding, base_url=None):\n        \"\"\" Do the real extraction work \"\"\"\n        self.reset()\n        self.feed(response_text)\n        self.close()\n\n        ret = []\n        if base_url is None:\n            base_url = urljoin(response_url, self.base_url) if self.base_url else response_url\n        for link in self.links:\n            if isinstance(link.url, unicode):\n                link.url = link.url.encode(response_encoding)\n            link.url = urljoin(base_url, link.url)\n            link.url = safe_url_string(link.url, response_encoding)\n            link.text = str_to_unicode(link.text, response_encoding, errors='replace').strip()\n            ret.append(link)\n\n        return ret\n\n    def _process_links(self, links):\n        \"\"\" Normalize and filter extracted links\n\n        The subclass should override it if necessary\n        \"\"\"\n        links = unique_list(links, key=lambda link: link.url) if self.unique else links\n        return links\n\n    def extract_links(self, response):\n        # wrapper needed to allow to work directly with text\n        links = self._extract_links(response.body, response.url, response.encoding)\n        links = self._process_links(links)\n        return links\n\n    def reset(self):\n        SGMLParser.reset(self)\n        self.links = []\n        self.base_url = None\n        self.current_link = None\n\n    def unknown_starttag(self, tag, attrs):\n        if tag == 'base':\n            self.base_url = dict(attrs).get('href')\n        if self.scan_tag(tag):\n            for attr, value in attrs:\n                if self.scan_attr(attr):\n                    url = self.process_value(value)\n                    if url is not None:\n                        link = Link(url=url, nofollow=True if dict(attrs).get('rel') == 'nofollow' else False)\n                        self.links.append(link)\n                        self.current_link = link\n\n    def unknown_endtag(self, tag):\n        if self.scan_tag(tag):\n            self.current_link = None\n\n    def handle_data(self, data):\n        if self.current_link:\n            self.current_link.text = self.current_link.text + data\n\n    def matches(self, url):\n        \"\"\"This extractor matches with any url, since\n        it doesn't contain any patterns\"\"\"\n        return True\n\n\nclass SgmlLinkExtractor(FilteringLinkExtractor):\n\n    def __init__(self, allow=(), deny=(), allow_domains=(), deny_domains=(), restrict_xpaths=(),\n                 tags=('a', 'area'), attrs=('href',), canonicalize=True, unique=True,\n                 process_value=None, deny_extensions=None, restrict_css=()):\n\n        warnings.warn(\n            \"SgmlLinkExtractor is deprecated and will be removed in future releases. \"\n            \"Please use scrapy.linkextractors.LinkExtractor\",\n            ScrapyDeprecationWarning, stacklevel=2,\n        )\n\n        tags, attrs = set(arg_to_iter(tags)), set(arg_to_iter(attrs))\n        tag_func = lambda x: x in tags\n        attr_func = lambda x: x in attrs\n\n        with warnings.catch_warnings(record=True):\n            lx = BaseSgmlLinkExtractor(tag=tag_func, attr=attr_func,\n                unique=unique, process_value=process_value)\n\n        super(SgmlLinkExtractor, self).__init__(lx, allow=allow, deny=deny,\n            allow_domains=allow_domains, deny_domains=deny_domains,\n            restrict_xpaths=restrict_xpaths, restrict_css=restrict_css,\n            canonicalize=canonicalize, deny_extensions=deny_extensions)\n\n        # FIXME: was added to fix a RegexLinkExtractor testcase\n        self.base_url = None\n\n    def extract_links(self, response):\n        base_url = None\n        if self.restrict_xpaths:\n            sel = Selector(response)\n            base_url = get_base_url(response)\n            body = u''.join(f\n                            for x in self.restrict_xpaths\n                            for f in sel.xpath(x).extract()\n                            ).encode(response.encoding, errors='xmlcharrefreplace')\n        else:\n            body = response.body\n\n        links = self._extract_links(body, response.url, response.encoding, base_url)\n        links = self._process_links(links)\n        return links\n", "path": "scrapy/linkextractors/sgml.py"}, {"content": "\"\"\"\nHTMLParser-based link extractor\n\"\"\"\n\nimport warnings\nfrom HTMLParser import HTMLParser\nfrom six.moves.urllib.parse import urljoin\n\nfrom w3lib.url import safe_url_string\n\nfrom scrapy.link import Link\nfrom scrapy.utils.python import unique as unique_list\nfrom scrapy.exceptions import ScrapyDeprecationWarning\n\n\nclass HtmlParserLinkExtractor(HTMLParser):\n\n    def __init__(self, tag=\"a\", attr=\"href\", process=None, unique=False):\n        HTMLParser.__init__(self)\n\n        warnings.warn(\n            \"HtmlParserLinkExtractor is deprecated and will be removed in \"\n            \"future releases. Please use scrapy.linkextractors.LinkExtractor\",\n            ScrapyDeprecationWarning, stacklevel=2,\n        )\n\n        self.scan_tag = tag if callable(tag) else lambda t: t == tag\n        self.scan_attr = attr if callable(attr) else lambda a: a == attr\n        self.process_attr = process if callable(process) else lambda v: v\n        self.unique = unique\n\n    def _extract_links(self, response_text, response_url, response_encoding):\n        self.reset()\n        self.feed(response_text)\n        self.close()\n\n        links = unique_list(self.links, key=lambda link: link.url) if self.unique else self.links\n\n        ret = []\n        base_url = urljoin(response_url, self.base_url) if self.base_url else response_url\n        for link in links:\n            if isinstance(link.url, unicode):\n                link.url = link.url.encode(response_encoding)\n            link.url = urljoin(base_url, link.url)\n            link.url = safe_url_string(link.url, response_encoding)\n            link.text = link.text.decode(response_encoding)\n            ret.append(link)\n\n        return ret\n\n    def extract_links(self, response):\n        # wrapper needed to allow to work directly with text\n        return self._extract_links(response.body, response.url, response.encoding)\n\n    def reset(self):\n        HTMLParser.reset(self)\n\n        self.base_url = None\n        self.current_link = None\n        self.links = []\n\n    def handle_starttag(self, tag, attrs):\n        if tag == 'base':\n            self.base_url = dict(attrs).get('href')\n        if self.scan_tag(tag):\n            for attr, value in attrs:\n                if self.scan_attr(attr):\n                    url = self.process_attr(value)\n                    link = Link(url=url)\n                    self.links.append(link)\n                    self.current_link = link\n\n    def handle_endtag(self, tag):\n        if self.scan_tag(tag):\n            self.current_link = None\n\n    def handle_data(self, data):\n        if self.current_link:\n            self.current_link.text = self.current_link.text + data\n\n    def matches(self, url):\n        \"\"\"This extractor matches with any url, since\n        it doesn't contain any patterns\"\"\"\n        return True\n", "path": "scrapy/linkextractors/htmlparser.py"}]}