redapple: Can you add tests for this?
Can you provide example websites showing this issue?
diff --git a/w3lib/html.py b/w3lib/html.py
index a4be054..a31d42b 100644
--- a/w3lib/html.py
+++ b/w3lib/html.py
@@ -311,7 +311,7 @@ def get_base_url(
 
     """
 
-    utext = to_unicode(text, encoding)
+    utext: str = remove_comments(text, encoding=encoding)
     m = _baseurl_re.search(utext)
     if m:
         return urljoin(

It's not a good idead to parse HTML text using regular expressions
In [`w3lib.html`](https://github.com/scrapy/w3lib/blob/master/w3lib/html.py) regular expressions are used to parse HTML texts:

``` python
_ent_re = re.compile(r'&((?P<named>[a-z\d]+)|#(?P<dec>\d+)|#x(?P<hex>[a-f\d]+))(?P<semicolon>;?)', re.IGNORECASE)
_tag_re = re.compile(r'<[a-zA-Z\/!].*?>', re.DOTALL)
_baseurl_re = re.compile(six.u(r'<base\s[^>]*href\s*=\s*[\"\']\s*([^\"\'\s]+)\s*[\"\']'), re.I)
_meta_refresh_re = re.compile(six.u(r'<meta\s[^>]*http-equiv[^>]*refresh[^>]*content\s*=\s*(?P<quote>["\'])(?P<int>(\d*\.)?\d+)\s*;\s*url=\s*(?P<url>.*?)(?P=quote)'), re.DOTALL | re.IGNORECASE)
_cdata_re = re.compile(r'((?P<cdata_s><!\[CDATA\[)(?P<cdata_d>.*?)(?P<cdata_e>\]\]>))', re.DOTALL)
```

However this is definitely incorrect when it involves commented contents, e.g.

``` python
>>> from w3lib import html
>>> html.get_base_url("""<!-- <base href="http://example.com/" /> -->""")
'http://example.com/'
```

Introducing "heavier" utilities like `lxml` would solve this issue easily, but that might be an awful idea as `w3lib` aims to be lightweight & fast.  
Or maybe we could implement some quick parser merely for eliminating the commented parts.

Any ideas?

diff --git a/tests/test_html.py b/tests/test_html.py
index d4861ba..1e637b0 100644
--- a/tests/test_html.py
+++ b/tests/test_html.py
@@ -372,6 +372,30 @@ class GetBaseUrlTest(unittest.TestCase):
             get_base_url(text, baseurl.encode("ascii")), "http://example.org/something"
         )
 
+    def test_base_url_in_comment(self):
+        self.assertEqual(
+            get_base_url("""<!-- <base href="http://example.com/"/> -->"""), ""
+        )
+        self.assertEqual(
+            get_base_url("""<!-- <base href="http://example.com/"/>"""), ""
+        )
+        self.assertEqual(
+            get_base_url("""<!-- <base href="http://example.com/"/> --"""), ""
+        )
+        self.assertEqual(
+            get_base_url(
+                """<!-- <!--  <base href="http://example.com/"/> -- -->  <base href="http://example_2.com/"/> """
+            ),
+            "http://example_2.com/",
+        )
+
+        self.assertEqual(
+            get_base_url(
+                """<!-- <base href="http://example.com/"/> --> <!-- <base href="http://example_2.com/"/> --> <base href="http://example_3.com/"/>"""
+            ),
+            "http://example_3.com/",
+        )
+
     def test_relative_url_with_absolute_path(self):
         baseurl = "https://example.org"
         text = """\

pip install -e .
parse_log_pytest
pytest
pytest
3.9
pytest --no-header -rA --tb=line --color=no -p no:cacheprovider -W ignore::DeprecationWarning
exceptiongroup @ file:///croot/exceptiongroup_1706031385326/work
iniconfig @ file:///home/linux1/recipes/ci/iniconfig_1610983019677/work
packaging @ file:///croot/packaging_1734472117206/work
pluggy @ file:///croot/pluggy_1733169602837/work
pytest @ file:///croot/pytest_1738938843180/work
tomli @ file:///opt/conda/conda-bld/tomli_1657175507142/work
-e git+https://github.com/scrapy/w3lib.git@8e19741b6b004d6248fb70b525255a96a1eb1ee6#egg=w3lib

name: w3lib
channels:
  - defaults
  - https://repo.anaconda.com/pkgs/main
  - https://repo.anaconda.com/pkgs/r
  - conda-forge
dependencies:
  - _libgcc_mutex=0.1=main
  - _openmp_mutex=5.1=1_gnu
  - ca-certificates=2025.2.25=h06a4308_0
  - exceptiongroup=1.2.0=py39h06a4308_0
  - iniconfig=1.1.1=pyhd3eb1b0_0
  - ld_impl_linux-64=2.40=h12ee557_0
  - libffi=3.4.4=h6a678d5_1
  - libgcc-ng=11.2.0=h1234567_1
  - libgomp=11.2.0=h1234567_1
  - libstdcxx-ng=11.2.0=h1234567_1
  - ncurses=6.4=h6a678d5_0
  - openssl=3.0.16=h5eee18b_0
  - packaging=24.2=py39h06a4308_0
  - pip=25.0=py39h06a4308_0
  - pluggy=1.5.0=py39h06a4308_0
  - pytest=8.3.4=py39h06a4308_0
  - python=3.9.21=he870216_1
  - readline=8.2=h5eee18b_0
  - setuptools=75.8.0=py39h06a4308_0
  - sqlite=3.45.3=h5eee18b_0
  - tk=8.6.14=h39e8969_0
  - tomli=2.0.1=py39h06a4308_0
  - tzdata=2025a=h04d1e81_0
  - wheel=0.45.1=py39h06a4308_0
  - xz=5.6.4=h5eee18b_1
  - zlib=1.2.13=h5eee18b_1
prefix: /opt/conda/envs/w3lib

tests/test_html.py::GetBaseUrlTest::test_base_url_in_comment
tests/test_html.py::RemoveEntitiesTest::test_browser_hack tests/test_html.py::RemoveEntitiesTest::test_encoding tests/test_html.py::RemoveEntitiesTest::test_illegal_entities tests/test_html.py::RemoveEntitiesTest::test_keep_entities tests/test_html.py::RemoveEntitiesTest::test_missing_semicolon tests/test_html.py::RemoveEntitiesTest::test_regular tests/test_html.py::RemoveEntitiesTest::test_returns_unicode tests/test_html.py::ReplaceTagsTest::test_replace_tags tests/test_html.py::ReplaceTagsTest::test_replace_tags_multiline tests/test_html.py::ReplaceTagsTest::test_returns_unicode tests/test_html.py::RemoveCommentsTest::test_no_comments tests/test_html.py::RemoveCommentsTest::test_remove_comments tests/test_html.py::RemoveCommentsTest::test_returns_unicode tests/test_html.py::RemoveTagsTest::test_keep_argument tests/test_html.py::RemoveTagsTest::test_remove_empty_tags tests/test_html.py::RemoveTagsTest::test_remove_tags tests/test_html.py::RemoveTagsTest::test_remove_tags_with_attributes tests/test_html.py::RemoveTagsTest::test_remove_tags_without_tags tests/test_html.py::RemoveTagsTest::test_returns_unicode tests/test_html.py::RemoveTagsTest::test_uppercase_tags tests/test_html.py::RemoveTagsWithContentTest::test_empty_tags tests/test_html.py::RemoveTagsWithContentTest::test_returns_unicode tests/test_html.py::RemoveTagsWithContentTest::test_tags_with_shared_prefix tests/test_html.py::RemoveTagsWithContentTest::test_with_tags tests/test_html.py::RemoveTagsWithContentTest::test_without_tags tests/test_html.py::ReplaceEscapeCharsTest::test_returns_unicode tests/test_html.py::ReplaceEscapeCharsTest::test_with_escape_chars tests/test_html.py::ReplaceEscapeCharsTest::test_without_escape_chars tests/test_html.py::UnquoteMarkupTest::test_returns_unicode tests/test_html.py::UnquoteMarkupTest::test_unquote_markup tests/test_html.py::GetBaseUrlTest::test_attributes_before_href tests/test_html.py::GetBaseUrlTest::test_get_base_url tests/test_html.py::GetBaseUrlTest::test_get_base_url_latin1 tests/test_html.py::GetBaseUrlTest::test_get_base_url_latin1_percent tests/test_html.py::GetBaseUrlTest::test_get_base_url_utf8 tests/test_html.py::GetBaseUrlTest::test_no_scheme_url tests/test_html.py::GetBaseUrlTest::test_relative_url_with_absolute_path tests/test_html.py::GetBaseUrlTest::test_tag_name tests/test_html.py::GetMetaRefreshTest::test_commented_meta_refresh tests/test_html.py::GetMetaRefreshTest::test_entities_in_redirect_url tests/test_html.py::GetMetaRefreshTest::test_float_refresh_intervals tests/test_html.py::GetMetaRefreshTest::test_get_meta_refresh tests/test_html.py::GetMetaRefreshTest::test_html_comments_with_uncommented_meta_refresh tests/test_html.py::GetMetaRefreshTest::test_inside_noscript tests/test_html.py::GetMetaRefreshTest::test_inside_script tests/test_html.py::GetMetaRefreshTest::test_leading_newline_in_url tests/test_html.py::GetMetaRefreshTest::test_multiline tests/test_html.py::GetMetaRefreshTest::test_nonascii_url_latin1 tests/test_html.py::GetMetaRefreshTest::test_nonascii_url_latin1_query tests/test_html.py::GetMetaRefreshTest::test_nonascii_url_utf8 tests/test_html.py::GetMetaRefreshTest::test_redirections_in_different_ordering__in_meta_tag tests/test_html.py::GetMetaRefreshTest::test_relative_redirects tests/test_html.py::GetMetaRefreshTest::test_tag_name tests/test_html.py::GetMetaRefreshTest::test_without_url
w3lib/html.py
w3lib/html.py