We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
URL percent-encoding in the SQL history
There are couple of problems:

- The [add_url](https://github.com/qutebrowser/qutebrowser/blob/49b858e3599a361cf5c994358c6b189dddfb522c/qutebrowser/browser/history.py#L162) method does percent-encoding, but the [parse_entry](https://github.com/qutebrowser/qutebrowser/blob/49b858e3599a361cf5c994358c6b189dddfb522c/qutebrowser/browser/history.py#L192) method does not. So the URLs which were imported from the text file are not percent-encoded, but the newly added URLs are, which leads to "duplicates" if I open an old URL.
- The old history completion called [entry.url.toDisplayString()](https://github.com/qutebrowser/qutebrowser/pull/2295/files#diff-d0e110fc69342b1ed59bb5c1a96b0234L101) when the URL was added to the completion. Now the completion shows URLs as they are in the database, which is inconsistent (see the previous point) and a visual regression (I want pretty URLs in the completion).

cc @rcorre 

--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `qutebrowser/browser/history.py`
Content:
```
1 # vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:
2 
3 # Copyright 2015-2017 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
4 #
5 # This file is part of qutebrowser.
6 #
7 # qutebrowser is free software: you can redistribute it and/or modify
8 # it under the terms of the GNU General Public License as published by
9 # the Free Software Foundation, either version 3 of the License, or
10 # (at your option) any later version.
11 #
12 # qutebrowser is distributed in the hope that it will be useful,
13 # but WITHOUT ANY WARRANTY; without even the implied warranty of
14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
15 # GNU General Public License for more details.
16 #
17 # You should have received a copy of the GNU General Public License
18 # along with qutebrowser.  If not, see <http://www.gnu.org/licenses/>.
19 
20 """Simple history which gets written to disk."""
21 
22 import os
23 import time
24 
25 from PyQt5.QtCore import pyqtSlot, QUrl, QTimer
26 
27 from qutebrowser.commands import cmdutils, cmdexc
28 from qutebrowser.utils import (utils, objreg, log, usertypes, message,
29                                debug, standarddir)
30 from qutebrowser.misc import objects, sql
31 
32 
33 class CompletionHistory(sql.SqlTable):
34 
35     """History which only has the newest entry for each URL."""
36 
37     def __init__(self, parent=None):
38         super().__init__("CompletionHistory", ['url', 'title', 'last_atime'],
39                          constraints={'url': 'PRIMARY KEY'}, parent=parent)
40         self.create_index('CompletionHistoryAtimeIndex', 'last_atime')
41 
42 
43 class WebHistory(sql.SqlTable):
44 
45     """The global history of visited pages."""
46 
47     def __init__(self, parent=None):
48         super().__init__("History", ['url', 'title', 'atime', 'redirect'],
49                          parent=parent)
50         self.completion = CompletionHistory(parent=self)
51         self.create_index('HistoryIndex', 'url')
52         self.create_index('HistoryAtimeIndex', 'atime')
53         self._contains_query = self.contains_query('url')
54         self._between_query = sql.Query('SELECT * FROM History '
55                                         'where not redirect '
56                                         'and not url like "qute://%" '
57                                         'and atime > :earliest '
58                                         'and atime <= :latest '
59                                         'ORDER BY atime desc')
60 
61         self._before_query = sql.Query('SELECT * FROM History '
62                                        'where not redirect '
63                                        'and not url like "qute://%" '
64                                        'and atime <= :latest '
65                                        'ORDER BY atime desc '
66                                        'limit :limit offset :offset')
67 
68     def __repr__(self):
69         return utils.get_repr(self, length=len(self))
70 
71     def __contains__(self, url):
72         return self._contains_query.run(val=url).value()
73 
74     def get_recent(self):
75         """Get the most recent history entries."""
76         return self.select(sort_by='atime', sort_order='desc', limit=100)
77 
78     def entries_between(self, earliest, latest):
79         """Iterate non-redirect, non-qute entries between two timestamps.
80 
81         Args:
82             earliest: Omit timestamps earlier than this.
83             latest: Omit timestamps later than this.
84         """
85         self._between_query.run(earliest=earliest, latest=latest)
86         return iter(self._between_query)
87 
88     def entries_before(self, latest, limit, offset):
89         """Iterate non-redirect, non-qute entries occurring before a timestamp.
90 
91         Args:
92             latest: Omit timestamps more recent than this.
93             limit: Max number of entries to include.
94             offset: Number of entries to skip.
95         """
96         self._before_query.run(latest=latest, limit=limit, offset=offset)
97         return iter(self._before_query)
98 
99     @cmdutils.register(name='history-clear', instance='web-history')
100     def clear(self, force=False):
101         """Clear all browsing history.
102 
103         Note this only clears the global history
104         (e.g. `~/.local/share/qutebrowser/history` on Linux) but not cookies,
105         the back/forward history of a tab, cache or other persistent data.
106 
107         Args:
108             force: Don't ask for confirmation.
109         """
110         if force:
111             self._do_clear()
112         else:
113             message.confirm_async(self._do_clear, title="Clear all browsing "
114                                 "history?")
115 
116     def _do_clear(self):
117         self.delete_all()
118         self.completion.delete_all()
119 
120     def delete_url(self, url):
121         """Remove all history entries with the given url.
122 
123         Args:
124             url: URL string to delete.
125         """
126         self.delete('url', url)
127         self.completion.delete('url', url)
128 
129     @pyqtSlot(QUrl, QUrl, str)
130     def add_from_tab(self, url, requested_url, title):
131         """Add a new history entry as slot, called from a BrowserTab."""
132         if url.scheme() == 'data' or requested_url.scheme() == 'data':
133             return
134         if url.isEmpty():
135             # things set via setHtml
136             return
137 
138         no_formatting = QUrl.UrlFormattingOption(0)
139         if (requested_url.isValid() and
140                 not requested_url.matches(url, no_formatting)):
141             # If the url of the page is different than the url of the link
142             # originally clicked, save them both.
143             self.add_url(requested_url, title, redirect=True)
144         self.add_url(url, title)
145 
146     def add_url(self, url, title="", *, redirect=False, atime=None):
147         """Called via add_from_tab when a URL should be added to the history.
148 
149         Args:
150             url: A url (as QUrl) to add to the history.
151             redirect: Whether the entry was redirected to another URL
152                       (hidden in completion)
153             atime: Override the atime used to add the entry
154         """
155         if not url.isValid():  # pragma: no cover
156             # the no cover pragma is a WORKAROUND for this not being covered in
157             # old Qt versions.
158             log.misc.warning("Ignoring invalid URL being added to history")
159             return
160 
161         atime = int(atime) if (atime is not None) else int(time.time())
162         url_str = url.toString(QUrl.FullyEncoded | QUrl.RemovePassword)
163         self.insert({'url': url_str,
164                      'title': title,
165                      'atime': atime,
166                      'redirect': redirect})
167         if not redirect:
168             self.completion.insert({'url': url_str,
169                                     'title': title,
170                                     'last_atime': atime},
171                                    replace=True)
172 
173     def _parse_entry(self, line):
174         """Parse a history line like '12345 http://example.com title'."""
175         if not line or line.startswith('#'):
176             return None
177         data = line.split(maxsplit=2)
178         if len(data) == 2:
179             atime, url = data
180             title = ""
181         elif len(data) == 3:
182             atime, url, title = data
183         else:
184             raise ValueError("2 or 3 fields expected")
185 
186         # http://xn--pple-43d.com/ with
187         # https://bugreports.qt.io/browse/QTBUG-60364
188         if url in ['http://.com/', 'https://.com/',
189                    'http://www..com/', 'https://www..com/']:
190             return None
191 
192         url = QUrl(url)
193         if not url.isValid():
194             raise ValueError("Invalid URL: {}".format(url.errorString()))
195 
196         # https://github.com/qutebrowser/qutebrowser/issues/2646
197         if url.scheme() == 'data':
198             return None
199 
200         # https://github.com/qutebrowser/qutebrowser/issues/670
201         atime = atime.lstrip('\0')
202 
203         if '-' in atime:
204             atime, flags = atime.split('-')
205         else:
206             flags = ''
207 
208         if not set(flags).issubset('r'):
209             raise ValueError("Invalid flags {!r}".format(flags))
210 
211         redirect = 'r' in flags
212         return (url, title, int(atime), redirect)
213 
214     def import_txt(self):
215         """Import a history text file into sqlite if it exists.
216 
217         In older versions of qutebrowser, history was stored in a text format.
218         This converts that file into the new sqlite format and moves it to a
219         backup location.
220         """
221         path = os.path.join(standarddir.data(), 'history')
222         if not os.path.isfile(path):
223             return
224 
225         def action():
226             with debug.log_time(log.init, 'Import old history file to sqlite'):
227                 try:
228                     self._read(path)
229                 except ValueError as ex:
230                     message.error('Failed to import history: {}'.format(ex))
231                 else:
232                     bakpath = path + '.bak'
233                     message.info('History import complete. Moving {} to {}'
234                                  .format(path, bakpath))
235                     os.rename(path, bakpath)
236 
237         # delay to give message time to appear before locking down for import
238         message.info('Converting {} to sqlite...'.format(path))
239         QTimer.singleShot(100, action)
240 
241     def _read(self, path):
242         """Import a text file into the sql database."""
243         with open(path, 'r', encoding='utf-8') as f:
244             data = {'url': [], 'title': [], 'atime': [], 'redirect': []}
245             completion_data = {'url': [], 'title': [], 'last_atime': []}
246             for (i, line) in enumerate(f):
247                 try:
248                     parsed = self._parse_entry(line.strip())
249                     if parsed is None:
250                         continue
251                     url, title, atime, redirect = parsed
252                     data['url'].append(url)
253                     data['title'].append(title)
254                     data['atime'].append(atime)
255                     data['redirect'].append(redirect)
256                     if not redirect:
257                         completion_data['url'].append(url)
258                         completion_data['title'].append(title)
259                         completion_data['last_atime'].append(atime)
260                 except ValueError as ex:
261                     raise ValueError('Failed to parse line #{} of {}: "{}"'
262                                      .format(i, path, ex))
263         self.insert_batch(data)
264         self.completion.insert_batch(completion_data, replace=True)
265 
266     @cmdutils.register(instance='web-history', debug=True)
267     def debug_dump_history(self, dest):
268         """Dump the history to a file in the old pre-SQL format.
269 
270         Args:
271             dest: Where to write the file to.
272         """
273         dest = os.path.expanduser(dest)
274 
275         lines = ('{}{} {} {}'
276                  .format(int(x.atime), '-r' * x.redirect, x.url, x.title)
277                  for x in self.select(sort_by='atime', sort_order='asc'))
278 
279         try:
280             with open(dest, 'w', encoding='utf-8') as f:
281                 f.write('\n'.join(lines))
282             message.info("Dumped history to {}".format(dest))
283         except OSError as e:
284             raise cmdexc.CommandError('Could not write history: {}', e)
285 
286 
287 def init(parent=None):
288     """Initialize the web history.
289 
290     Args:
291         parent: The parent to use for WebHistory.
292     """
293     history = WebHistory(parent=parent)
294     objreg.register('web-history', history)
295 
296     if objects.backend == usertypes.Backend.QtWebKit:
297         from qutebrowser.browser.webkit import webkithistory
298         webkithistory.init(history)
299
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/qutebrowser/browser/history.py b/qutebrowser/browser/history.py
--- a/qutebrowser/browser/history.py
+++ b/qutebrowser/browser/history.py
@@ -30,6 +30,10 @@
 from qutebrowser.misc import objects, sql
 
 
+# increment to indicate that HistoryCompletion must be regenerated
+_USER_VERSION = 1
+
+
 class CompletionHistory(sql.SqlTable):
 
     """History which only has the newest entry for each URL."""
@@ -48,6 +52,11 @@
         super().__init__("History", ['url', 'title', 'atime', 'redirect'],
                          parent=parent)
         self.completion = CompletionHistory(parent=self)
+        if sql.Query('pragma user_version').run().value() < _USER_VERSION:
+            self.completion.delete_all()
+        if not self.completion:
+            # either the table is out-of-date or the user wiped it manually
+            self._rebuild_completion()
         self.create_index('HistoryIndex', 'url')
         self.create_index('HistoryAtimeIndex', 'atime')
         self._contains_query = self.contains_query('url')
@@ -71,6 +80,18 @@
     def __contains__(self, url):
         return self._contains_query.run(val=url).value()
 
+    def _rebuild_completion(self):
+        data = {'url': [], 'title': [], 'last_atime': []}
+        # select the latest entry for each url
+        q = sql.Query('SELECT url, title, max(atime) AS atime FROM History '
+                      'WHERE NOT redirect GROUP BY url ORDER BY atime asc')
+        for entry in q.run():
+            data['url'].append(self._format_completion_url(QUrl(entry.url)))
+            data['title'].append(entry.title)
+            data['last_atime'].append(entry.atime)
+        self.completion.insert_batch(data, replace=True)
+        sql.Query('pragma user_version = {}'.format(_USER_VERSION)).run()
+
     def get_recent(self):
         """Get the most recent history entries."""
         return self.select(sort_by='atime', sort_order='desc', limit=100)
@@ -159,13 +180,12 @@
             return
 
         atime = int(atime) if (atime is not None) else int(time.time())
-        url_str = url.toString(QUrl.FullyEncoded | QUrl.RemovePassword)
-        self.insert({'url': url_str,
+        self.insert({'url': self._format_url(url),
                      'title': title,
                      'atime': atime,
                      'redirect': redirect})
         if not redirect:
-            self.completion.insert({'url': url_str,
+            self.completion.insert({'url': self._format_completion_url(url),
                                     'title': title,
                                     'last_atime': atime},
                                    replace=True)
@@ -249,12 +269,13 @@
                     if parsed is None:
                         continue
                     url, title, atime, redirect = parsed
-                    data['url'].append(url)
+                    data['url'].append(self._format_url(url))
                     data['title'].append(title)
                     data['atime'].append(atime)
                     data['redirect'].append(redirect)
                     if not redirect:
-                        completion_data['url'].append(url)
+                        completion_data['url'].append(
+                            self._format_completion_url(url))
                         completion_data['title'].append(title)
                         completion_data['last_atime'].append(atime)
                 except ValueError as ex:
@@ -263,6 +284,12 @@
         self.insert_batch(data)
         self.completion.insert_batch(completion_data, replace=True)
 
+    def _format_url(self, url):
+        return url.toString(QUrl.RemovePassword | QUrl.FullyEncoded)
+
+    def _format_completion_url(self, url):
+        return url.toString(QUrl.RemovePassword)
+
     @cmdutils.register(instance='web-history', debug=True)
     def debug_dump_history(self, dest):
         """Dump the history to a file in the old pre-SQL format.

{"golden_diff": "diff --git a/qutebrowser/browser/history.py b/qutebrowser/browser/history.py\n--- a/qutebrowser/browser/history.py\n+++ b/qutebrowser/browser/history.py\n@@ -30,6 +30,10 @@\n from qutebrowser.misc import objects, sql\n \n \n+# increment to indicate that HistoryCompletion must be regenerated\n+_USER_VERSION = 1\n+\n+\n class CompletionHistory(sql.SqlTable):\n \n     \"\"\"History which only has the newest entry for each URL.\"\"\"\n@@ -48,6 +52,11 @@\n         super().__init__(\"History\", ['url', 'title', 'atime', 'redirect'],\n                          parent=parent)\n         self.completion = CompletionHistory(parent=self)\n+        if sql.Query('pragma user_version').run().value() < _USER_VERSION:\n+            self.completion.delete_all()\n+        if not self.completion:\n+            # either the table is out-of-date or the user wiped it manually\n+            self._rebuild_completion()\n         self.create_index('HistoryIndex', 'url')\n         self.create_index('HistoryAtimeIndex', 'atime')\n         self._contains_query = self.contains_query('url')\n@@ -71,6 +80,18 @@\n     def __contains__(self, url):\n         return self._contains_query.run(val=url).value()\n \n+    def _rebuild_completion(self):\n+        data = {'url': [], 'title': [], 'last_atime': []}\n+        # select the latest entry for each url\n+        q = sql.Query('SELECT url, title, max(atime) AS atime FROM History '\n+                      'WHERE NOT redirect GROUP BY url ORDER BY atime asc')\n+        for entry in q.run():\n+            data['url'].append(self._format_completion_url(QUrl(entry.url)))\n+            data['title'].append(entry.title)\n+            data['last_atime'].append(entry.atime)\n+        self.completion.insert_batch(data, replace=True)\n+        sql.Query('pragma user_version = {}'.format(_USER_VERSION)).run()\n+\n     def get_recent(self):\n         \"\"\"Get the most recent history entries.\"\"\"\n         return self.select(sort_by='atime', sort_order='desc', limit=100)\n@@ -159,13 +180,12 @@\n             return\n \n         atime = int(atime) if (atime is not None) else int(time.time())\n-        url_str = url.toString(QUrl.FullyEncoded | QUrl.RemovePassword)\n-        self.insert({'url': url_str,\n+        self.insert({'url': self._format_url(url),\n                      'title': title,\n                      'atime': atime,\n                      'redirect': redirect})\n         if not redirect:\n-            self.completion.insert({'url': url_str,\n+            self.completion.insert({'url': self._format_completion_url(url),\n                                     'title': title,\n                                     'last_atime': atime},\n                                    replace=True)\n@@ -249,12 +269,13 @@\n                     if parsed is None:\n                         continue\n                     url, title, atime, redirect = parsed\n-                    data['url'].append(url)\n+                    data['url'].append(self._format_url(url))\n                     data['title'].append(title)\n                     data['atime'].append(atime)\n                     data['redirect'].append(redirect)\n                     if not redirect:\n-                        completion_data['url'].append(url)\n+                        completion_data['url'].append(\n+                            self._format_completion_url(url))\n                         completion_data['title'].append(title)\n                         completion_data['last_atime'].append(atime)\n                 except ValueError as ex:\n@@ -263,6 +284,12 @@\n         self.insert_batch(data)\n         self.completion.insert_batch(completion_data, replace=True)\n \n+    def _format_url(self, url):\n+        return url.toString(QUrl.RemovePassword | QUrl.FullyEncoded)\n+\n+    def _format_completion_url(self, url):\n+        return url.toString(QUrl.RemovePassword)\n+\n     @cmdutils.register(instance='web-history', debug=True)\n     def debug_dump_history(self, dest):\n         \"\"\"Dump the history to a file in the old pre-SQL format.\n", "issue": "URL percent-encoding in the SQL history\nThere are couple of problems:\r\n\r\n- The [add_url](https://github.com/qutebrowser/qutebrowser/blob/49b858e3599a361cf5c994358c6b189dddfb522c/qutebrowser/browser/history.py#L162) method does percent-encoding, but the [parse_entry](https://github.com/qutebrowser/qutebrowser/blob/49b858e3599a361cf5c994358c6b189dddfb522c/qutebrowser/browser/history.py#L192) method does not. So the URLs which were imported from the text file are not percent-encoded, but the newly added URLs are, which leads to \"duplicates\" if I open an old URL.\r\n- The old history completion called [entry.url.toDisplayString()](https://github.com/qutebrowser/qutebrowser/pull/2295/files#diff-d0e110fc69342b1ed59bb5c1a96b0234L101) when the URL was added to the completion. Now the completion shows URLs as they are in the database, which is inconsistent (see the previous point) and a visual regression (I want pretty URLs in the completion).\r\n\r\ncc @rcorre \n", "before_files": [{"content": "# vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:\n\n# Copyright 2015-2017 Florian Bruhin (The Compiler) <mail@qutebrowser.org>\n#\n# This file is part of qutebrowser.\n#\n# qutebrowser is free software: you can redistribute it and/or modify\n# it under the terms of the GNU General Public License as published by\n# the Free Software Foundation, either version 3 of the License, or\n# (at your option) any later version.\n#\n# qutebrowser is distributed in the hope that it will be useful,\n# but WITHOUT ANY WARRANTY; without even the implied warranty of\n# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the\n# GNU General Public License for more details.\n#\n# You should have received a copy of the GNU General Public License\n# along with qutebrowser.  If not, see <http://www.gnu.org/licenses/>.\n\n\"\"\"Simple history which gets written to disk.\"\"\"\n\nimport os\nimport time\n\nfrom PyQt5.QtCore import pyqtSlot, QUrl, QTimer\n\nfrom qutebrowser.commands import cmdutils, cmdexc\nfrom qutebrowser.utils import (utils, objreg, log, usertypes, message,\n                               debug, standarddir)\nfrom qutebrowser.misc import objects, sql\n\n\nclass CompletionHistory(sql.SqlTable):\n\n    \"\"\"History which only has the newest entry for each URL.\"\"\"\n\n    def __init__(self, parent=None):\n        super().__init__(\"CompletionHistory\", ['url', 'title', 'last_atime'],\n                         constraints={'url': 'PRIMARY KEY'}, parent=parent)\n        self.create_index('CompletionHistoryAtimeIndex', 'last_atime')\n\n\nclass WebHistory(sql.SqlTable):\n\n    \"\"\"The global history of visited pages.\"\"\"\n\n    def __init__(self, parent=None):\n        super().__init__(\"History\", ['url', 'title', 'atime', 'redirect'],\n                         parent=parent)\n        self.completion = CompletionHistory(parent=self)\n        self.create_index('HistoryIndex', 'url')\n        self.create_index('HistoryAtimeIndex', 'atime')\n        self._contains_query = self.contains_query('url')\n        self._between_query = sql.Query('SELECT * FROM History '\n                                        'where not redirect '\n                                        'and not url like \"qute://%\" '\n                                        'and atime > :earliest '\n                                        'and atime <= :latest '\n                                        'ORDER BY atime desc')\n\n        self._before_query = sql.Query('SELECT * FROM History '\n                                       'where not redirect '\n                                       'and not url like \"qute://%\" '\n                                       'and atime <= :latest '\n                                       'ORDER BY atime desc '\n                                       'limit :limit offset :offset')\n\n    def __repr__(self):\n        return utils.get_repr(self, length=len(self))\n\n    def __contains__(self, url):\n        return self._contains_query.run(val=url).value()\n\n    def get_recent(self):\n        \"\"\"Get the most recent history entries.\"\"\"\n        return self.select(sort_by='atime', sort_order='desc', limit=100)\n\n    def entries_between(self, earliest, latest):\n        \"\"\"Iterate non-redirect, non-qute entries between two timestamps.\n\n        Args:\n            earliest: Omit timestamps earlier than this.\n            latest: Omit timestamps later than this.\n        \"\"\"\n        self._between_query.run(earliest=earliest, latest=latest)\n        return iter(self._between_query)\n\n    def entries_before(self, latest, limit, offset):\n        \"\"\"Iterate non-redirect, non-qute entries occurring before a timestamp.\n\n        Args:\n            latest: Omit timestamps more recent than this.\n            limit: Max number of entries to include.\n            offset: Number of entries to skip.\n        \"\"\"\n        self._before_query.run(latest=latest, limit=limit, offset=offset)\n        return iter(self._before_query)\n\n    @cmdutils.register(name='history-clear', instance='web-history')\n    def clear(self, force=False):\n        \"\"\"Clear all browsing history.\n\n        Note this only clears the global history\n        (e.g. `~/.local/share/qutebrowser/history` on Linux) but not cookies,\n        the back/forward history of a tab, cache or other persistent data.\n\n        Args:\n            force: Don't ask for confirmation.\n        \"\"\"\n        if force:\n            self._do_clear()\n        else:\n            message.confirm_async(self._do_clear, title=\"Clear all browsing \"\n                                \"history?\")\n\n    def _do_clear(self):\n        self.delete_all()\n        self.completion.delete_all()\n\n    def delete_url(self, url):\n        \"\"\"Remove all history entries with the given url.\n\n        Args:\n            url: URL string to delete.\n        \"\"\"\n        self.delete('url', url)\n        self.completion.delete('url', url)\n\n    @pyqtSlot(QUrl, QUrl, str)\n    def add_from_tab(self, url, requested_url, title):\n        \"\"\"Add a new history entry as slot, called from a BrowserTab.\"\"\"\n        if url.scheme() == 'data' or requested_url.scheme() == 'data':\n            return\n        if url.isEmpty():\n            # things set via setHtml\n            return\n\n        no_formatting = QUrl.UrlFormattingOption(0)\n        if (requested_url.isValid() and\n                not requested_url.matches(url, no_formatting)):\n            # If the url of the page is different than the url of the link\n            # originally clicked, save them both.\n            self.add_url(requested_url, title, redirect=True)\n        self.add_url(url, title)\n\n    def add_url(self, url, title=\"\", *, redirect=False, atime=None):\n        \"\"\"Called via add_from_tab when a URL should be added to the history.\n\n        Args:\n            url: A url (as QUrl) to add to the history.\n            redirect: Whether the entry was redirected to another URL\n                      (hidden in completion)\n            atime: Override the atime used to add the entry\n        \"\"\"\n        if not url.isValid():  # pragma: no cover\n            # the no cover pragma is a WORKAROUND for this not being covered in\n            # old Qt versions.\n            log.misc.warning(\"Ignoring invalid URL being added to history\")\n            return\n\n        atime = int(atime) if (atime is not None) else int(time.time())\n        url_str = url.toString(QUrl.FullyEncoded | QUrl.RemovePassword)\n        self.insert({'url': url_str,\n                     'title': title,\n                     'atime': atime,\n                     'redirect': redirect})\n        if not redirect:\n            self.completion.insert({'url': url_str,\n                                    'title': title,\n                                    'last_atime': atime},\n                                   replace=True)\n\n    def _parse_entry(self, line):\n        \"\"\"Parse a history line like '12345 http://example.com title'.\"\"\"\n        if not line or line.startswith('#'):\n            return None\n        data = line.split(maxsplit=2)\n        if len(data) == 2:\n            atime, url = data\n            title = \"\"\n        elif len(data) == 3:\n            atime, url, title = data\n        else:\n            raise ValueError(\"2 or 3 fields expected\")\n\n        # http://xn--pple-43d.com/ with\n        # https://bugreports.qt.io/browse/QTBUG-60364\n        if url in ['http://.com/', 'https://.com/',\n                   'http://www..com/', 'https://www..com/']:\n            return None\n\n        url = QUrl(url)\n        if not url.isValid():\n            raise ValueError(\"Invalid URL: {}\".format(url.errorString()))\n\n        # https://github.com/qutebrowser/qutebrowser/issues/2646\n        if url.scheme() == 'data':\n            return None\n\n        # https://github.com/qutebrowser/qutebrowser/issues/670\n        atime = atime.lstrip('\\0')\n\n        if '-' in atime:\n            atime, flags = atime.split('-')\n        else:\n            flags = ''\n\n        if not set(flags).issubset('r'):\n            raise ValueError(\"Invalid flags {!r}\".format(flags))\n\n        redirect = 'r' in flags\n        return (url, title, int(atime), redirect)\n\n    def import_txt(self):\n        \"\"\"Import a history text file into sqlite if it exists.\n\n        In older versions of qutebrowser, history was stored in a text format.\n        This converts that file into the new sqlite format and moves it to a\n        backup location.\n        \"\"\"\n        path = os.path.join(standarddir.data(), 'history')\n        if not os.path.isfile(path):\n            return\n\n        def action():\n            with debug.log_time(log.init, 'Import old history file to sqlite'):\n                try:\n                    self._read(path)\n                except ValueError as ex:\n                    message.error('Failed to import history: {}'.format(ex))\n                else:\n                    bakpath = path + '.bak'\n                    message.info('History import complete. Moving {} to {}'\n                                 .format(path, bakpath))\n                    os.rename(path, bakpath)\n\n        # delay to give message time to appear before locking down for import\n        message.info('Converting {} to sqlite...'.format(path))\n        QTimer.singleShot(100, action)\n\n    def _read(self, path):\n        \"\"\"Import a text file into the sql database.\"\"\"\n        with open(path, 'r', encoding='utf-8') as f:\n            data = {'url': [], 'title': [], 'atime': [], 'redirect': []}\n            completion_data = {'url': [], 'title': [], 'last_atime': []}\n            for (i, line) in enumerate(f):\n                try:\n                    parsed = self._parse_entry(line.strip())\n                    if parsed is None:\n                        continue\n                    url, title, atime, redirect = parsed\n                    data['url'].append(url)\n                    data['title'].append(title)\n                    data['atime'].append(atime)\n                    data['redirect'].append(redirect)\n                    if not redirect:\n                        completion_data['url'].append(url)\n                        completion_data['title'].append(title)\n                        completion_data['last_atime'].append(atime)\n                except ValueError as ex:\n                    raise ValueError('Failed to parse line #{} of {}: \"{}\"'\n                                     .format(i, path, ex))\n        self.insert_batch(data)\n        self.completion.insert_batch(completion_data, replace=True)\n\n    @cmdutils.register(instance='web-history', debug=True)\n    def debug_dump_history(self, dest):\n        \"\"\"Dump the history to a file in the old pre-SQL format.\n\n        Args:\n            dest: Where to write the file to.\n        \"\"\"\n        dest = os.path.expanduser(dest)\n\n        lines = ('{}{} {} {}'\n                 .format(int(x.atime), '-r' * x.redirect, x.url, x.title)\n                 for x in self.select(sort_by='atime', sort_order='asc'))\n\n        try:\n            with open(dest, 'w', encoding='utf-8') as f:\n                f.write('\\n'.join(lines))\n            message.info(\"Dumped history to {}\".format(dest))\n        except OSError as e:\n            raise cmdexc.CommandError('Could not write history: {}', e)\n\n\ndef init(parent=None):\n    \"\"\"Initialize the web history.\n\n    Args:\n        parent: The parent to use for WebHistory.\n    \"\"\"\n    history = WebHistory(parent=parent)\n    objreg.register('web-history', history)\n\n    if objects.backend == usertypes.Backend.QtWebKit:\n        from qutebrowser.browser.webkit import webkithistory\n        webkithistory.init(history)\n", "path": "qutebrowser/browser/history.py"}], "after_files": [{"content": "# vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:\n\n# Copyright 2015-2017 Florian Bruhin (The Compiler) <mail@qutebrowser.org>\n#\n# This file is part of qutebrowser.\n#\n# qutebrowser is free software: you can redistribute it and/or modify\n# it under the terms of the GNU General Public License as published by\n# the Free Software Foundation, either version 3 of the License, or\n# (at your option) any later version.\n#\n# qutebrowser is distributed in the hope that it will be useful,\n# but WITHOUT ANY WARRANTY; without even the implied warranty of\n# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the\n# GNU General Public License for more details.\n#\n# You should have received a copy of the GNU General Public License\n# along with qutebrowser.  If not, see <http://www.gnu.org/licenses/>.\n\n\"\"\"Simple history which gets written to disk.\"\"\"\n\nimport os\nimport time\n\nfrom PyQt5.QtCore import pyqtSlot, QUrl, QTimer\n\nfrom qutebrowser.commands import cmdutils, cmdexc\nfrom qutebrowser.utils import (utils, objreg, log, usertypes, message,\n                               debug, standarddir)\nfrom qutebrowser.misc import objects, sql\n\n\n# increment to indicate that HistoryCompletion must be regenerated\n_USER_VERSION = 1\n\n\nclass CompletionHistory(sql.SqlTable):\n\n    \"\"\"History which only has the newest entry for each URL.\"\"\"\n\n    def __init__(self, parent=None):\n        super().__init__(\"CompletionHistory\", ['url', 'title', 'last_atime'],\n                         constraints={'url': 'PRIMARY KEY'}, parent=parent)\n        self.create_index('CompletionHistoryAtimeIndex', 'last_atime')\n\n\nclass WebHistory(sql.SqlTable):\n\n    \"\"\"The global history of visited pages.\"\"\"\n\n    def __init__(self, parent=None):\n        super().__init__(\"History\", ['url', 'title', 'atime', 'redirect'],\n                         parent=parent)\n        self.completion = CompletionHistory(parent=self)\n        if sql.Query('pragma user_version').run().value() < _USER_VERSION:\n            self.completion.delete_all()\n        if not self.completion:\n            # either the table is out-of-date or the user wiped it manually\n            self._rebuild_completion()\n        self.create_index('HistoryIndex', 'url')\n        self.create_index('HistoryAtimeIndex', 'atime')\n        self._contains_query = self.contains_query('url')\n        self._between_query = sql.Query('SELECT * FROM History '\n                                        'where not redirect '\n                                        'and not url like \"qute://%\" '\n                                        'and atime > :earliest '\n                                        'and atime <= :latest '\n                                        'ORDER BY atime desc')\n\n        self._before_query = sql.Query('SELECT * FROM History '\n                                       'where not redirect '\n                                       'and not url like \"qute://%\" '\n                                       'and atime <= :latest '\n                                       'ORDER BY atime desc '\n                                       'limit :limit offset :offset')\n\n    def __repr__(self):\n        return utils.get_repr(self, length=len(self))\n\n    def __contains__(self, url):\n        return self._contains_query.run(val=url).value()\n\n    def _rebuild_completion(self):\n        data = {'url': [], 'title': [], 'last_atime': []}\n        # select the latest entry for each url\n        q = sql.Query('SELECT url, title, max(atime) AS atime FROM History '\n                      'WHERE NOT redirect GROUP BY url ORDER BY atime asc')\n        for entry in q.run():\n            data['url'].append(self._format_completion_url(QUrl(entry.url)))\n            data['title'].append(entry.title)\n            data['last_atime'].append(entry.atime)\n        self.completion.insert_batch(data, replace=True)\n        sql.Query('pragma user_version = {}'.format(_USER_VERSION)).run()\n\n    def get_recent(self):\n        \"\"\"Get the most recent history entries.\"\"\"\n        return self.select(sort_by='atime', sort_order='desc', limit=100)\n\n    def entries_between(self, earliest, latest):\n        \"\"\"Iterate non-redirect, non-qute entries between two timestamps.\n\n        Args:\n            earliest: Omit timestamps earlier than this.\n            latest: Omit timestamps later than this.\n        \"\"\"\n        self._between_query.run(earliest=earliest, latest=latest)\n        return iter(self._between_query)\n\n    def entries_before(self, latest, limit, offset):\n        \"\"\"Iterate non-redirect, non-qute entries occurring before a timestamp.\n\n        Args:\n            latest: Omit timestamps more recent than this.\n            limit: Max number of entries to include.\n            offset: Number of entries to skip.\n        \"\"\"\n        self._before_query.run(latest=latest, limit=limit, offset=offset)\n        return iter(self._before_query)\n\n    @cmdutils.register(name='history-clear', instance='web-history')\n    def clear(self, force=False):\n        \"\"\"Clear all browsing history.\n\n        Note this only clears the global history\n        (e.g. `~/.local/share/qutebrowser/history` on Linux) but not cookies,\n        the back/forward history of a tab, cache or other persistent data.\n\n        Args:\n            force: Don't ask for confirmation.\n        \"\"\"\n        if force:\n            self._do_clear()\n        else:\n            message.confirm_async(self._do_clear, title=\"Clear all browsing \"\n                                \"history?\")\n\n    def _do_clear(self):\n        self.delete_all()\n        self.completion.delete_all()\n\n    def delete_url(self, url):\n        \"\"\"Remove all history entries with the given url.\n\n        Args:\n            url: URL string to delete.\n        \"\"\"\n        self.delete('url', url)\n        self.completion.delete('url', url)\n\n    @pyqtSlot(QUrl, QUrl, str)\n    def add_from_tab(self, url, requested_url, title):\n        \"\"\"Add a new history entry as slot, called from a BrowserTab.\"\"\"\n        if url.scheme() == 'data' or requested_url.scheme() == 'data':\n            return\n        if url.isEmpty():\n            # things set via setHtml\n            return\n\n        no_formatting = QUrl.UrlFormattingOption(0)\n        if (requested_url.isValid() and\n                not requested_url.matches(url, no_formatting)):\n            # If the url of the page is different than the url of the link\n            # originally clicked, save them both.\n            self.add_url(requested_url, title, redirect=True)\n        self.add_url(url, title)\n\n    def add_url(self, url, title=\"\", *, redirect=False, atime=None):\n        \"\"\"Called via add_from_tab when a URL should be added to the history.\n\n        Args:\n            url: A url (as QUrl) to add to the history.\n            redirect: Whether the entry was redirected to another URL\n                      (hidden in completion)\n            atime: Override the atime used to add the entry\n        \"\"\"\n        if not url.isValid():  # pragma: no cover\n            # the no cover pragma is a WORKAROUND for this not being covered in\n            # old Qt versions.\n            log.misc.warning(\"Ignoring invalid URL being added to history\")\n            return\n\n        atime = int(atime) if (atime is not None) else int(time.time())\n        self.insert({'url': self._format_url(url),\n                     'title': title,\n                     'atime': atime,\n                     'redirect': redirect})\n        if not redirect:\n            self.completion.insert({'url': self._format_completion_url(url),\n                                    'title': title,\n                                    'last_atime': atime},\n                                   replace=True)\n\n    def _parse_entry(self, line):\n        \"\"\"Parse a history line like '12345 http://example.com title'.\"\"\"\n        if not line or line.startswith('#'):\n            return None\n        data = line.split(maxsplit=2)\n        if len(data) == 2:\n            atime, url = data\n            title = \"\"\n        elif len(data) == 3:\n            atime, url, title = data\n        else:\n            raise ValueError(\"2 or 3 fields expected\")\n\n        # http://xn--pple-43d.com/ with\n        # https://bugreports.qt.io/browse/QTBUG-60364\n        if url in ['http://.com/', 'https://.com/',\n                   'http://www..com/', 'https://www..com/']:\n            return None\n\n        url = QUrl(url)\n        if not url.isValid():\n            raise ValueError(\"Invalid URL: {}\".format(url.errorString()))\n\n        # https://github.com/qutebrowser/qutebrowser/issues/2646\n        if url.scheme() == 'data':\n            return None\n\n        # https://github.com/qutebrowser/qutebrowser/issues/670\n        atime = atime.lstrip('\\0')\n\n        if '-' in atime:\n            atime, flags = atime.split('-')\n        else:\n            flags = ''\n\n        if not set(flags).issubset('r'):\n            raise ValueError(\"Invalid flags {!r}\".format(flags))\n\n        redirect = 'r' in flags\n        return (url, title, int(atime), redirect)\n\n    def import_txt(self):\n        \"\"\"Import a history text file into sqlite if it exists.\n\n        In older versions of qutebrowser, history was stored in a text format.\n        This converts that file into the new sqlite format and moves it to a\n        backup location.\n        \"\"\"\n        path = os.path.join(standarddir.data(), 'history')\n        if not os.path.isfile(path):\n            return\n\n        def action():\n            with debug.log_time(log.init, 'Import old history file to sqlite'):\n                try:\n                    self._read(path)\n                except ValueError as ex:\n                    message.error('Failed to import history: {}'.format(ex))\n                else:\n                    bakpath = path + '.bak'\n                    message.info('History import complete. Moving {} to {}'\n                                 .format(path, bakpath))\n                    os.rename(path, bakpath)\n\n        # delay to give message time to appear before locking down for import\n        message.info('Converting {} to sqlite...'.format(path))\n        QTimer.singleShot(100, action)\n\n    def _read(self, path):\n        \"\"\"Import a text file into the sql database.\"\"\"\n        with open(path, 'r', encoding='utf-8') as f:\n            data = {'url': [], 'title': [], 'atime': [], 'redirect': []}\n            completion_data = {'url': [], 'title': [], 'last_atime': []}\n            for (i, line) in enumerate(f):\n                try:\n                    parsed = self._parse_entry(line.strip())\n                    if parsed is None:\n                        continue\n                    url, title, atime, redirect = parsed\n                    data['url'].append(self._format_url(url))\n                    data['title'].append(title)\n                    data['atime'].append(atime)\n                    data['redirect'].append(redirect)\n                    if not redirect:\n                        completion_data['url'].append(\n                            self._format_completion_url(url))\n                        completion_data['title'].append(title)\n                        completion_data['last_atime'].append(atime)\n                except ValueError as ex:\n                    raise ValueError('Failed to parse line #{} of {}: \"{}\"'\n                                     .format(i, path, ex))\n        self.insert_batch(data)\n        self.completion.insert_batch(completion_data, replace=True)\n\n    def _format_url(self, url):\n        return url.toString(QUrl.RemovePassword | QUrl.FullyEncoded)\n\n    def _format_completion_url(self, url):\n        return url.toString(QUrl.RemovePassword)\n\n    @cmdutils.register(instance='web-history', debug=True)\n    def debug_dump_history(self, dest):\n        \"\"\"Dump the history to a file in the old pre-SQL format.\n\n        Args:\n            dest: Where to write the file to.\n        \"\"\"\n        dest = os.path.expanduser(dest)\n\n        lines = ('{}{} {} {}'\n                 .format(int(x.atime), '-r' * x.redirect, x.url, x.title)\n                 for x in self.select(sort_by='atime', sort_order='asc'))\n\n        try:\n            with open(dest, 'w', encoding='utf-8') as f:\n                f.write('\\n'.join(lines))\n            message.info(\"Dumped history to {}\".format(dest))\n        except OSError as e:\n            raise cmdexc.CommandError('Could not write history: {}', e)\n\n\ndef init(parent=None):\n    \"\"\"Initialize the web history.\n\n    Args:\n        parent: The parent to use for WebHistory.\n    \"\"\"\n    history = WebHistory(parent=parent)\n    objreg.register('web-history', history)\n\n    if objects.backend == usertypes.Backend.QtWebKit:\n        from qutebrowser.browser.webkit import webkithistory\n        webkithistory.init(history)\n", "path": "qutebrowser/browser/history.py"}]}