We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
Improvements for content->host-block-lists
- [ ] Handle local directories (read every file in the directory)
- [ ] Handle local paths without `file://`


--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `qutebrowser/browser/adblock.py`
Content:
```
1 # vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:
2 
3 # Copyright 2014-2018 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
4 #
5 # This file is part of qutebrowser.
6 #
7 # qutebrowser is free software: you can redistribute it and/or modify
8 # it under the terms of the GNU General Public License as published by
9 # the Free Software Foundation, either version 3 of the License, or
10 # (at your option) any later version.
11 #
12 # qutebrowser is distributed in the hope that it will be useful,
13 # but WITHOUT ANY WARRANTY; without even the implied warranty of
14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
15 # GNU General Public License for more details.
16 #
17 # You should have received a copy of the GNU General Public License
18 # along with qutebrowser.  If not, see <http://www.gnu.org/licenses/>.
19 
20 """Functions related to ad blocking."""
21 
22 import io
23 import os.path
24 import functools
25 import posixpath
26 import zipfile
27 
28 from qutebrowser.browser import downloads
29 from qutebrowser.config import config
30 from qutebrowser.utils import objreg, standarddir, log, message
31 from qutebrowser.commands import cmdutils
32 
33 
34 def _guess_zip_filename(zf):
35     """Guess which file to use inside a zip file.
36 
37     Args:
38         zf: A ZipFile instance.
39     """
40     files = zf.namelist()
41     if len(files) == 1:
42         return files[0]
43     else:
44         for e in files:
45             if posixpath.splitext(e)[0].lower() == 'hosts':
46                 return e
47     raise FileNotFoundError("No hosts file found in zip")
48 
49 
50 def get_fileobj(byte_io):
51     """Get a usable file object to read the hosts file from."""
52     byte_io.seek(0)  # rewind downloaded file
53     if zipfile.is_zipfile(byte_io):
54         byte_io.seek(0)  # rewind what zipfile.is_zipfile did
55         zf = zipfile.ZipFile(byte_io)
56         filename = _guess_zip_filename(zf)
57         byte_io = zf.open(filename, mode='r')
58     else:
59         byte_io.seek(0)  # rewind what zipfile.is_zipfile did
60     return byte_io
61 
62 
63 def _is_whitelisted_url(url):
64     """Check if the given URL is on the adblock whitelist.
65 
66     Args:
67         url: The URL to check as QUrl.
68     """
69     for pattern in config.val.content.host_blocking.whitelist:
70         if pattern.matches(url):
71             return True
72     return False
73 
74 
75 class _FakeDownload:
76 
77     """A download stub to use on_download_finished with local files."""
78 
79     def __init__(self, fileobj):
80         self.basename = os.path.basename(fileobj.name)
81         self.fileobj = fileobj
82         self.successful = True
83 
84 
85 class HostBlocker:
86 
87     """Manage blocked hosts based from /etc/hosts-like files.
88 
89     Attributes:
90         _blocked_hosts: A set of blocked hosts.
91         _config_blocked_hosts: A set of blocked hosts from ~/.config.
92         _in_progress: The DownloadItems which are currently downloading.
93         _done_count: How many files have been read successfully.
94         _local_hosts_file: The path to the blocked-hosts file.
95         _config_hosts_file: The path to a blocked-hosts in ~/.config
96     """
97 
98     def __init__(self):
99         self._blocked_hosts = set()
100         self._config_blocked_hosts = set()
101         self._in_progress = []
102         self._done_count = 0
103 
104         data_dir = standarddir.data()
105         self._local_hosts_file = os.path.join(data_dir, 'blocked-hosts')
106         self._update_files()
107 
108         config_dir = standarddir.config()
109         self._config_hosts_file = os.path.join(config_dir, 'blocked-hosts')
110 
111         config.instance.changed.connect(self._update_files)
112 
113     def is_blocked(self, url):
114         """Check if the given URL (as QUrl) is blocked."""
115         if not config.val.content.host_blocking.enabled:
116             return False
117         host = url.host()
118         return ((host in self._blocked_hosts or
119                  host in self._config_blocked_hosts) and
120                 not _is_whitelisted_url(url))
121 
122     def _read_hosts_file(self, filename, target):
123         """Read hosts from the given filename.
124 
125         Args:
126             filename: The file to read.
127             target: The set to store the hosts in.
128 
129         Return:
130             True if a read was attempted, False otherwise
131         """
132         if not os.path.exists(filename):
133             return False
134 
135         try:
136             with open(filename, 'r', encoding='utf-8') as f:
137                 for line in f:
138                     target.add(line.strip())
139         except (OSError, UnicodeDecodeError):
140             log.misc.exception("Failed to read host blocklist!")
141 
142         return True
143 
144     def read_hosts(self):
145         """Read hosts from the existing blocked-hosts file."""
146         self._blocked_hosts = set()
147 
148         self._read_hosts_file(self._config_hosts_file,
149                               self._config_blocked_hosts)
150 
151         found = self._read_hosts_file(self._local_hosts_file,
152                                       self._blocked_hosts)
153 
154         if not found:
155             args = objreg.get('args')
156             if (config.val.content.host_blocking.lists and
157                     args.basedir is None and
158                     config.val.content.host_blocking.enabled):
159                 message.info("Run :adblock-update to get adblock lists.")
160 
161     @cmdutils.register(instance='host-blocker')
162     def adblock_update(self):
163         """Update the adblock block lists.
164 
165         This updates `~/.local/share/qutebrowser/blocked-hosts` with downloaded
166         host lists and re-reads `~/.config/qutebrowser/blocked-hosts`.
167         """
168         self._read_hosts_file(self._config_hosts_file,
169                               self._config_blocked_hosts)
170         self._blocked_hosts = set()
171         self._done_count = 0
172         download_manager = objreg.get('qtnetwork-download-manager')
173         for url in config.val.content.host_blocking.lists:
174             if url.scheme() == 'file':
175                 filename = url.toLocalFile()
176                 try:
177                     fileobj = open(filename, 'rb')
178                 except OSError as e:
179                     message.error("adblock: Error while reading {}: {}".format(
180                         filename, e.strerror))
181                     continue
182                 download = _FakeDownload(fileobj)
183                 self._in_progress.append(download)
184                 self._on_download_finished(download)
185             else:
186                 fobj = io.BytesIO()
187                 fobj.name = 'adblock: ' + url.host()
188                 target = downloads.FileObjDownloadTarget(fobj)
189                 download = download_manager.get(url, target=target,
190                                                 auto_remove=True)
191                 self._in_progress.append(download)
192                 download.finished.connect(
193                     functools.partial(self._on_download_finished, download))
194 
195     def _parse_line(self, line):
196         """Parse a line from a host file.
197 
198         Args:
199             line: The bytes object to parse.
200 
201         Returns:
202             True if parsing succeeded, False otherwise.
203         """
204         if line.startswith(b'#'):
205             # Ignoring comments early so we don't have to care about
206             # encoding errors in them.
207             return True
208 
209         try:
210             line = line.decode('utf-8')
211         except UnicodeDecodeError:
212             log.misc.error("Failed to decode: {!r}".format(line))
213             return False
214 
215         # Remove comments
216         try:
217             hash_idx = line.index('#')
218             line = line[:hash_idx]
219         except ValueError:
220             pass
221 
222         line = line.strip()
223         # Skip empty lines
224         if not line:
225             return True
226 
227         parts = line.split()
228         if len(parts) == 1:
229             # "one host per line" format
230             hosts = [parts[0]]
231         else:
232             # /etc/hosts format
233             hosts = parts[1:]
234 
235         for host in hosts:
236             if ('.' in host and
237                     not host.endswith('.localdomain') and
238                     host != '0.0.0.0'):
239                 self._blocked_hosts.add(host)
240 
241         return True
242 
243     def _merge_file(self, byte_io):
244         """Read and merge host files.
245 
246         Args:
247             byte_io: The BytesIO object of the completed download.
248 
249         Return:
250             A set of the merged hosts.
251         """
252         error_count = 0
253         line_count = 0
254         try:
255             f = get_fileobj(byte_io)
256         except (OSError, zipfile.BadZipFile, zipfile.LargeZipFile,
257                 LookupError) as e:
258             message.error("adblock: Error while reading {}: {} - {}".format(
259                 byte_io.name, e.__class__.__name__, e))
260             return
261 
262         for line in f:
263             line_count += 1
264             ok = self._parse_line(line)
265             if not ok:
266                 error_count += 1
267 
268         log.misc.debug("{}: read {} lines".format(byte_io.name, line_count))
269         if error_count > 0:
270             message.error("adblock: {} read errors for {}".format(
271                 error_count, byte_io.name))
272 
273     def _on_lists_downloaded(self):
274         """Install block lists after files have been downloaded."""
275         with open(self._local_hosts_file, 'w', encoding='utf-8') as f:
276             for host in sorted(self._blocked_hosts):
277                 f.write(host + '\n')
278             message.info("adblock: Read {} hosts from {} sources.".format(
279                 len(self._blocked_hosts), self._done_count))
280 
281     @config.change_filter('content.host_blocking.lists')
282     def _update_files(self):
283         """Update files when the config changed."""
284         if not config.val.content.host_blocking.lists:
285             try:
286                 os.remove(self._local_hosts_file)
287             except FileNotFoundError:
288                 pass
289             except OSError as e:
290                 log.misc.exception("Failed to delete hosts file: {}".format(e))
291 
292     def _on_download_finished(self, download):
293         """Check if all downloads are finished and if so, trigger reading.
294 
295         Arguments:
296             download: The finished DownloadItem.
297         """
298         self._in_progress.remove(download)
299         if download.successful:
300             self._done_count += 1
301             try:
302                 self._merge_file(download.fileobj)
303             finally:
304                 download.fileobj.close()
305         if not self._in_progress:
306             try:
307                 self._on_lists_downloaded()
308             except OSError:
309                 log.misc.exception("Failed to write host block list!")
310
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/qutebrowser/browser/adblock.py b/qutebrowser/browser/adblock.py
--- a/qutebrowser/browser/adblock.py
+++ b/qutebrowser/browser/adblock.py
@@ -173,15 +173,12 @@
         for url in config.val.content.host_blocking.lists:
             if url.scheme() == 'file':
                 filename = url.toLocalFile()
-                try:
-                    fileobj = open(filename, 'rb')
-                except OSError as e:
-                    message.error("adblock: Error while reading {}: {}".format(
-                        filename, e.strerror))
-                    continue
-                download = _FakeDownload(fileobj)
-                self._in_progress.append(download)
-                self._on_download_finished(download)
+                if os.path.isdir(filename):
+                    for filenames in os.scandir(filename):
+                        if filenames.is_file():
+                            self._import_local(filenames.path)
+                else:
+                    self._import_local(filename)
             else:
                 fobj = io.BytesIO()
                 fobj.name = 'adblock: ' + url.host()
@@ -192,6 +189,22 @@
                 download.finished.connect(
                     functools.partial(self._on_download_finished, download))
 
+    def _import_local(self, filename):
+        """Adds the contents of a file to the blocklist.
+
+        Args:
+            filename: path to a local file to import.
+        """
+        try:
+            fileobj = open(filename, 'rb')
+        except OSError as e:
+            message.error("adblock: Error while reading {}: {}".format(
+                filename, e.strerror))
+            return
+        download = _FakeDownload(fileobj)
+        self._in_progress.append(download)
+        self._on_download_finished(download)
+
     def _parse_line(self, line):
         """Parse a line from a host file.

{"golden_diff": "diff --git a/qutebrowser/browser/adblock.py b/qutebrowser/browser/adblock.py\n--- a/qutebrowser/browser/adblock.py\n+++ b/qutebrowser/browser/adblock.py\n@@ -173,15 +173,12 @@\n         for url in config.val.content.host_blocking.lists:\n             if url.scheme() == 'file':\n                 filename = url.toLocalFile()\n-                try:\n-                    fileobj = open(filename, 'rb')\n-                except OSError as e:\n-                    message.error(\"adblock: Error while reading {}: {}\".format(\n-                        filename, e.strerror))\n-                    continue\n-                download = _FakeDownload(fileobj)\n-                self._in_progress.append(download)\n-                self._on_download_finished(download)\n+                if os.path.isdir(filename):\n+                    for filenames in os.scandir(filename):\n+                        if filenames.is_file():\n+                            self._import_local(filenames.path)\n+                else:\n+                    self._import_local(filename)\n             else:\n                 fobj = io.BytesIO()\n                 fobj.name = 'adblock: ' + url.host()\n@@ -192,6 +189,22 @@\n                 download.finished.connect(\n                     functools.partial(self._on_download_finished, download))\n \n+    def _import_local(self, filename):\n+        \"\"\"Adds the contents of a file to the blocklist.\n+\n+        Args:\n+            filename: path to a local file to import.\n+        \"\"\"\n+        try:\n+            fileobj = open(filename, 'rb')\n+        except OSError as e:\n+            message.error(\"adblock: Error while reading {}: {}\".format(\n+                filename, e.strerror))\n+            return\n+        download = _FakeDownload(fileobj)\n+        self._in_progress.append(download)\n+        self._on_download_finished(download)\n+\n     def _parse_line(self, line):\n         \"\"\"Parse a line from a host file.\n", "issue": "Improvements for content->host-block-lists\n- [ ] Handle local directories (read every file in the directory)\n- [ ] Handle local paths without `file://`\n\n", "before_files": [{"content": "# vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:\n\n# Copyright 2014-2018 Florian Bruhin (The Compiler) <mail@qutebrowser.org>\n#\n# This file is part of qutebrowser.\n#\n# qutebrowser is free software: you can redistribute it and/or modify\n# it under the terms of the GNU General Public License as published by\n# the Free Software Foundation, either version 3 of the License, or\n# (at your option) any later version.\n#\n# qutebrowser is distributed in the hope that it will be useful,\n# but WITHOUT ANY WARRANTY; without even the implied warranty of\n# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the\n# GNU General Public License for more details.\n#\n# You should have received a copy of the GNU General Public License\n# along with qutebrowser.  If not, see <http://www.gnu.org/licenses/>.\n\n\"\"\"Functions related to ad blocking.\"\"\"\n\nimport io\nimport os.path\nimport functools\nimport posixpath\nimport zipfile\n\nfrom qutebrowser.browser import downloads\nfrom qutebrowser.config import config\nfrom qutebrowser.utils import objreg, standarddir, log, message\nfrom qutebrowser.commands import cmdutils\n\n\ndef _guess_zip_filename(zf):\n    \"\"\"Guess which file to use inside a zip file.\n\n    Args:\n        zf: A ZipFile instance.\n    \"\"\"\n    files = zf.namelist()\n    if len(files) == 1:\n        return files[0]\n    else:\n        for e in files:\n            if posixpath.splitext(e)[0].lower() == 'hosts':\n                return e\n    raise FileNotFoundError(\"No hosts file found in zip\")\n\n\ndef get_fileobj(byte_io):\n    \"\"\"Get a usable file object to read the hosts file from.\"\"\"\n    byte_io.seek(0)  # rewind downloaded file\n    if zipfile.is_zipfile(byte_io):\n        byte_io.seek(0)  # rewind what zipfile.is_zipfile did\n        zf = zipfile.ZipFile(byte_io)\n        filename = _guess_zip_filename(zf)\n        byte_io = zf.open(filename, mode='r')\n    else:\n        byte_io.seek(0)  # rewind what zipfile.is_zipfile did\n    return byte_io\n\n\ndef _is_whitelisted_url(url):\n    \"\"\"Check if the given URL is on the adblock whitelist.\n\n    Args:\n        url: The URL to check as QUrl.\n    \"\"\"\n    for pattern in config.val.content.host_blocking.whitelist:\n        if pattern.matches(url):\n            return True\n    return False\n\n\nclass _FakeDownload:\n\n    \"\"\"A download stub to use on_download_finished with local files.\"\"\"\n\n    def __init__(self, fileobj):\n        self.basename = os.path.basename(fileobj.name)\n        self.fileobj = fileobj\n        self.successful = True\n\n\nclass HostBlocker:\n\n    \"\"\"Manage blocked hosts based from /etc/hosts-like files.\n\n    Attributes:\n        _blocked_hosts: A set of blocked hosts.\n        _config_blocked_hosts: A set of blocked hosts from ~/.config.\n        _in_progress: The DownloadItems which are currently downloading.\n        _done_count: How many files have been read successfully.\n        _local_hosts_file: The path to the blocked-hosts file.\n        _config_hosts_file: The path to a blocked-hosts in ~/.config\n    \"\"\"\n\n    def __init__(self):\n        self._blocked_hosts = set()\n        self._config_blocked_hosts = set()\n        self._in_progress = []\n        self._done_count = 0\n\n        data_dir = standarddir.data()\n        self._local_hosts_file = os.path.join(data_dir, 'blocked-hosts')\n        self._update_files()\n\n        config_dir = standarddir.config()\n        self._config_hosts_file = os.path.join(config_dir, 'blocked-hosts')\n\n        config.instance.changed.connect(self._update_files)\n\n    def is_blocked(self, url):\n        \"\"\"Check if the given URL (as QUrl) is blocked.\"\"\"\n        if not config.val.content.host_blocking.enabled:\n            return False\n        host = url.host()\n        return ((host in self._blocked_hosts or\n                 host in self._config_blocked_hosts) and\n                not _is_whitelisted_url(url))\n\n    def _read_hosts_file(self, filename, target):\n        \"\"\"Read hosts from the given filename.\n\n        Args:\n            filename: The file to read.\n            target: The set to store the hosts in.\n\n        Return:\n            True if a read was attempted, False otherwise\n        \"\"\"\n        if not os.path.exists(filename):\n            return False\n\n        try:\n            with open(filename, 'r', encoding='utf-8') as f:\n                for line in f:\n                    target.add(line.strip())\n        except (OSError, UnicodeDecodeError):\n            log.misc.exception(\"Failed to read host blocklist!\")\n\n        return True\n\n    def read_hosts(self):\n        \"\"\"Read hosts from the existing blocked-hosts file.\"\"\"\n        self._blocked_hosts = set()\n\n        self._read_hosts_file(self._config_hosts_file,\n                              self._config_blocked_hosts)\n\n        found = self._read_hosts_file(self._local_hosts_file,\n                                      self._blocked_hosts)\n\n        if not found:\n            args = objreg.get('args')\n            if (config.val.content.host_blocking.lists and\n                    args.basedir is None and\n                    config.val.content.host_blocking.enabled):\n                message.info(\"Run :adblock-update to get adblock lists.\")\n\n    @cmdutils.register(instance='host-blocker')\n    def adblock_update(self):\n        \"\"\"Update the adblock block lists.\n\n        This updates `~/.local/share/qutebrowser/blocked-hosts` with downloaded\n        host lists and re-reads `~/.config/qutebrowser/blocked-hosts`.\n        \"\"\"\n        self._read_hosts_file(self._config_hosts_file,\n                              self._config_blocked_hosts)\n        self._blocked_hosts = set()\n        self._done_count = 0\n        download_manager = objreg.get('qtnetwork-download-manager')\n        for url in config.val.content.host_blocking.lists:\n            if url.scheme() == 'file':\n                filename = url.toLocalFile()\n                try:\n                    fileobj = open(filename, 'rb')\n                except OSError as e:\n                    message.error(\"adblock: Error while reading {}: {}\".format(\n                        filename, e.strerror))\n                    continue\n                download = _FakeDownload(fileobj)\n                self._in_progress.append(download)\n                self._on_download_finished(download)\n            else:\n                fobj = io.BytesIO()\n                fobj.name = 'adblock: ' + url.host()\n                target = downloads.FileObjDownloadTarget(fobj)\n                download = download_manager.get(url, target=target,\n                                                auto_remove=True)\n                self._in_progress.append(download)\n                download.finished.connect(\n                    functools.partial(self._on_download_finished, download))\n\n    def _parse_line(self, line):\n        \"\"\"Parse a line from a host file.\n\n        Args:\n            line: The bytes object to parse.\n\n        Returns:\n            True if parsing succeeded, False otherwise.\n        \"\"\"\n        if line.startswith(b'#'):\n            # Ignoring comments early so we don't have to care about\n            # encoding errors in them.\n            return True\n\n        try:\n            line = line.decode('utf-8')\n        except UnicodeDecodeError:\n            log.misc.error(\"Failed to decode: {!r}\".format(line))\n            return False\n\n        # Remove comments\n        try:\n            hash_idx = line.index('#')\n            line = line[:hash_idx]\n        except ValueError:\n            pass\n\n        line = line.strip()\n        # Skip empty lines\n        if not line:\n            return True\n\n        parts = line.split()\n        if len(parts) == 1:\n            # \"one host per line\" format\n            hosts = [parts[0]]\n        else:\n            # /etc/hosts format\n            hosts = parts[1:]\n\n        for host in hosts:\n            if ('.' in host and\n                    not host.endswith('.localdomain') and\n                    host != '0.0.0.0'):\n                self._blocked_hosts.add(host)\n\n        return True\n\n    def _merge_file(self, byte_io):\n        \"\"\"Read and merge host files.\n\n        Args:\n            byte_io: The BytesIO object of the completed download.\n\n        Return:\n            A set of the merged hosts.\n        \"\"\"\n        error_count = 0\n        line_count = 0\n        try:\n            f = get_fileobj(byte_io)\n        except (OSError, zipfile.BadZipFile, zipfile.LargeZipFile,\n                LookupError) as e:\n            message.error(\"adblock: Error while reading {}: {} - {}\".format(\n                byte_io.name, e.__class__.__name__, e))\n            return\n\n        for line in f:\n            line_count += 1\n            ok = self._parse_line(line)\n            if not ok:\n                error_count += 1\n\n        log.misc.debug(\"{}: read {} lines\".format(byte_io.name, line_count))\n        if error_count > 0:\n            message.error(\"adblock: {} read errors for {}\".format(\n                error_count, byte_io.name))\n\n    def _on_lists_downloaded(self):\n        \"\"\"Install block lists after files have been downloaded.\"\"\"\n        with open(self._local_hosts_file, 'w', encoding='utf-8') as f:\n            for host in sorted(self._blocked_hosts):\n                f.write(host + '\\n')\n            message.info(\"adblock: Read {} hosts from {} sources.\".format(\n                len(self._blocked_hosts), self._done_count))\n\n    @config.change_filter('content.host_blocking.lists')\n    def _update_files(self):\n        \"\"\"Update files when the config changed.\"\"\"\n        if not config.val.content.host_blocking.lists:\n            try:\n                os.remove(self._local_hosts_file)\n            except FileNotFoundError:\n                pass\n            except OSError as e:\n                log.misc.exception(\"Failed to delete hosts file: {}\".format(e))\n\n    def _on_download_finished(self, download):\n        \"\"\"Check if all downloads are finished and if so, trigger reading.\n\n        Arguments:\n            download: The finished DownloadItem.\n        \"\"\"\n        self._in_progress.remove(download)\n        if download.successful:\n            self._done_count += 1\n            try:\n                self._merge_file(download.fileobj)\n            finally:\n                download.fileobj.close()\n        if not self._in_progress:\n            try:\n                self._on_lists_downloaded()\n            except OSError:\n                log.misc.exception(\"Failed to write host block list!\")\n", "path": "qutebrowser/browser/adblock.py"}], "after_files": [{"content": "# vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:\n\n# Copyright 2014-2018 Florian Bruhin (The Compiler) <mail@qutebrowser.org>\n#\n# This file is part of qutebrowser.\n#\n# qutebrowser is free software: you can redistribute it and/or modify\n# it under the terms of the GNU General Public License as published by\n# the Free Software Foundation, either version 3 of the License, or\n# (at your option) any later version.\n#\n# qutebrowser is distributed in the hope that it will be useful,\n# but WITHOUT ANY WARRANTY; without even the implied warranty of\n# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the\n# GNU General Public License for more details.\n#\n# You should have received a copy of the GNU General Public License\n# along with qutebrowser.  If not, see <http://www.gnu.org/licenses/>.\n\n\"\"\"Functions related to ad blocking.\"\"\"\n\nimport io\nimport os.path\nimport functools\nimport posixpath\nimport zipfile\n\nfrom qutebrowser.browser import downloads\nfrom qutebrowser.config import config\nfrom qutebrowser.utils import objreg, standarddir, log, message\nfrom qutebrowser.commands import cmdutils\n\n\ndef _guess_zip_filename(zf):\n    \"\"\"Guess which file to use inside a zip file.\n\n    Args:\n        zf: A ZipFile instance.\n    \"\"\"\n    files = zf.namelist()\n    if len(files) == 1:\n        return files[0]\n    else:\n        for e in files:\n            if posixpath.splitext(e)[0].lower() == 'hosts':\n                return e\n    raise FileNotFoundError(\"No hosts file found in zip\")\n\n\ndef get_fileobj(byte_io):\n    \"\"\"Get a usable file object to read the hosts file from.\"\"\"\n    byte_io.seek(0)  # rewind downloaded file\n    if zipfile.is_zipfile(byte_io):\n        byte_io.seek(0)  # rewind what zipfile.is_zipfile did\n        zf = zipfile.ZipFile(byte_io)\n        filename = _guess_zip_filename(zf)\n        byte_io = zf.open(filename, mode='r')\n    else:\n        byte_io.seek(0)  # rewind what zipfile.is_zipfile did\n    return byte_io\n\n\ndef _is_whitelisted_url(url):\n    \"\"\"Check if the given URL is on the adblock whitelist.\n\n    Args:\n        url: The URL to check as QUrl.\n    \"\"\"\n    for pattern in config.val.content.host_blocking.whitelist:\n        if pattern.matches(url):\n            return True\n    return False\n\n\nclass _FakeDownload:\n\n    \"\"\"A download stub to use on_download_finished with local files.\"\"\"\n\n    def __init__(self, fileobj):\n        self.basename = os.path.basename(fileobj.name)\n        self.fileobj = fileobj\n        self.successful = True\n\n\nclass HostBlocker:\n\n    \"\"\"Manage blocked hosts based from /etc/hosts-like files.\n\n    Attributes:\n        _blocked_hosts: A set of blocked hosts.\n        _config_blocked_hosts: A set of blocked hosts from ~/.config.\n        _in_progress: The DownloadItems which are currently downloading.\n        _done_count: How many files have been read successfully.\n        _local_hosts_file: The path to the blocked-hosts file.\n        _config_hosts_file: The path to a blocked-hosts in ~/.config\n    \"\"\"\n\n    def __init__(self):\n        self._blocked_hosts = set()\n        self._config_blocked_hosts = set()\n        self._in_progress = []\n        self._done_count = 0\n\n        data_dir = standarddir.data()\n        self._local_hosts_file = os.path.join(data_dir, 'blocked-hosts')\n        self._update_files()\n\n        config_dir = standarddir.config()\n        self._config_hosts_file = os.path.join(config_dir, 'blocked-hosts')\n\n        config.instance.changed.connect(self._update_files)\n\n    def is_blocked(self, url):\n        \"\"\"Check if the given URL (as QUrl) is blocked.\"\"\"\n        if not config.val.content.host_blocking.enabled:\n            return False\n        host = url.host()\n        return ((host in self._blocked_hosts or\n                 host in self._config_blocked_hosts) and\n                not _is_whitelisted_url(url))\n\n    def _read_hosts_file(self, filename, target):\n        \"\"\"Read hosts from the given filename.\n\n        Args:\n            filename: The file to read.\n            target: The set to store the hosts in.\n\n        Return:\n            True if a read was attempted, False otherwise\n        \"\"\"\n        if not os.path.exists(filename):\n            return False\n\n        try:\n            with open(filename, 'r', encoding='utf-8') as f:\n                for line in f:\n                    target.add(line.strip())\n        except (OSError, UnicodeDecodeError):\n            log.misc.exception(\"Failed to read host blocklist!\")\n\n        return True\n\n    def read_hosts(self):\n        \"\"\"Read hosts from the existing blocked-hosts file.\"\"\"\n        self._blocked_hosts = set()\n\n        self._read_hosts_file(self._config_hosts_file,\n                              self._config_blocked_hosts)\n\n        found = self._read_hosts_file(self._local_hosts_file,\n                                      self._blocked_hosts)\n\n        if not found:\n            args = objreg.get('args')\n            if (config.val.content.host_blocking.lists and\n                    args.basedir is None and\n                    config.val.content.host_blocking.enabled):\n                message.info(\"Run :adblock-update to get adblock lists.\")\n\n    @cmdutils.register(instance='host-blocker')\n    def adblock_update(self):\n        \"\"\"Update the adblock block lists.\n\n        This updates `~/.local/share/qutebrowser/blocked-hosts` with downloaded\n        host lists and re-reads `~/.config/qutebrowser/blocked-hosts`.\n        \"\"\"\n        self._read_hosts_file(self._config_hosts_file,\n                              self._config_blocked_hosts)\n        self._blocked_hosts = set()\n        self._done_count = 0\n        download_manager = objreg.get('qtnetwork-download-manager')\n        for url in config.val.content.host_blocking.lists:\n            if url.scheme() == 'file':\n                filename = url.toLocalFile()\n                if os.path.isdir(filename):\n                    for filenames in os.scandir(filename):\n                        if filenames.is_file():\n                            self._import_local(filenames.path)\n                else:\n                    self._import_local(filename)\n            else:\n                fobj = io.BytesIO()\n                fobj.name = 'adblock: ' + url.host()\n                target = downloads.FileObjDownloadTarget(fobj)\n                download = download_manager.get(url, target=target,\n                                                auto_remove=True)\n                self._in_progress.append(download)\n                download.finished.connect(\n                    functools.partial(self._on_download_finished, download))\n\n    def _import_local(self, filename):\n        \"\"\"Adds the contents of a file to the blocklist.\n\n        Args:\n            filename: path to a local file to import.\n        \"\"\"\n        try:\n            fileobj = open(filename, 'rb')\n        except OSError as e:\n            message.error(\"adblock: Error while reading {}: {}\".format(\n                filename, e.strerror))\n            return\n        download = _FakeDownload(fileobj)\n        self._in_progress.append(download)\n        self._on_download_finished(download)\n\n    def _parse_line(self, line):\n        \"\"\"Parse a line from a host file.\n\n        Args:\n            line: The bytes object to parse.\n\n        Returns:\n            True if parsing succeeded, False otherwise.\n        \"\"\"\n        if line.startswith(b'#'):\n            # Ignoring comments early so we don't have to care about\n            # encoding errors in them.\n            return True\n\n        try:\n            line = line.decode('utf-8')\n        except UnicodeDecodeError:\n            log.misc.error(\"Failed to decode: {!r}\".format(line))\n            return False\n\n        # Remove comments\n        try:\n            hash_idx = line.index('#')\n            line = line[:hash_idx]\n        except ValueError:\n            pass\n\n        line = line.strip()\n        # Skip empty lines\n        if not line:\n            return True\n\n        parts = line.split()\n        if len(parts) == 1:\n            # \"one host per line\" format\n            hosts = [parts[0]]\n        else:\n            # /etc/hosts format\n            hosts = parts[1:]\n\n        for host in hosts:\n            if ('.' in host and\n                    not host.endswith('.localdomain') and\n                    host != '0.0.0.0'):\n                self._blocked_hosts.add(host)\n\n        return True\n\n    def _merge_file(self, byte_io):\n        \"\"\"Read and merge host files.\n\n        Args:\n            byte_io: The BytesIO object of the completed download.\n\n        Return:\n            A set of the merged hosts.\n        \"\"\"\n        error_count = 0\n        line_count = 0\n        try:\n            f = get_fileobj(byte_io)\n        except (OSError, zipfile.BadZipFile, zipfile.LargeZipFile,\n                LookupError) as e:\n            message.error(\"adblock: Error while reading {}: {} - {}\".format(\n                byte_io.name, e.__class__.__name__, e))\n            return\n\n        for line in f:\n            line_count += 1\n            ok = self._parse_line(line)\n            if not ok:\n                error_count += 1\n\n        log.misc.debug(\"{}: read {} lines\".format(byte_io.name, line_count))\n        if error_count > 0:\n            message.error(\"adblock: {} read errors for {}\".format(\n                error_count, byte_io.name))\n\n    def _on_lists_downloaded(self):\n        \"\"\"Install block lists after files have been downloaded.\"\"\"\n        with open(self._local_hosts_file, 'w', encoding='utf-8') as f:\n            for host in sorted(self._blocked_hosts):\n                f.write(host + '\\n')\n            message.info(\"adblock: Read {} hosts from {} sources.\".format(\n                len(self._blocked_hosts), self._done_count))\n\n    @config.change_filter('content.host_blocking.lists')\n    def _update_files(self):\n        \"\"\"Update files when the config changed.\"\"\"\n        if not config.val.content.host_blocking.lists:\n            try:\n                os.remove(self._local_hosts_file)\n            except FileNotFoundError:\n                pass\n            except OSError as e:\n                log.misc.exception(\"Failed to delete hosts file: {}\".format(e))\n\n    def _on_download_finished(self, download):\n        \"\"\"Check if all downloads are finished and if so, trigger reading.\n\n        Arguments:\n            download: The finished DownloadItem.\n        \"\"\"\n        self._in_progress.remove(download)\n        if download.successful:\n            self._done_count += 1\n            try:\n                self._merge_file(download.fileobj)\n            finally:\n                download.fileobj.close()\n        if not self._in_progress:\n            try:\n                self._on_lists_downloaded()\n            except OSError:\n                log.misc.exception(\"Failed to write host block list!\")\n", "path": "qutebrowser/browser/adblock.py"}]}