We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
[BUG] Selecting date ranges breaks searches
### Description

Searching documents using TWO filters will not work, if one parameter is "last month" or "last year" in the "added" or "created" field. I tried to be more precise than #3203 

Example: 

Selected document type is "Anleitung" -> everything works as expected:
![image](https://user-images.githubusercontent.com/113503248/234883356-1975b4b6-713c-4d93-bb52-459638fa2ed4.png)

Now a second filter "last year" is applied. Year is as expected but selected document type is ignored:
![image](https://user-images.githubusercontent.com/113503248/234883815-39dcd974-a010-46ac-9897-cb326cf036cc.png)


### Steps to reproduce

1. login as a superuser
2. go to web-ui -> documents
3. reset filters (all documents displayed now)
4. select document type.
5. select "created" oder "added" date range "last year"
-> all document types ara displayed.

### Webserver logs

```bash
none
```


### Browser logs

```bash
none
```


### Paperless-ngx version

1.14.0

### Host OS

Linux 3.10.108 #42661 SMP (x86_64)

### Installation method

Docker - official image

### Browser

Chrome

### Configuration changes

_No response_

### Other

_No response_

--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `src/documents/index.py`
Content:
```
1 import logging
2 import math
3 import os
4 from contextlib import contextmanager
5 
6 from dateutil.parser import isoparse
7 from django.conf import settings
8 from django.utils import timezone
9 from guardian.shortcuts import get_users_with_perms
10 from whoosh import classify
11 from whoosh import highlight
12 from whoosh import query
13 from whoosh.fields import BOOLEAN
14 from whoosh.fields import DATETIME
15 from whoosh.fields import KEYWORD
16 from whoosh.fields import NUMERIC
17 from whoosh.fields import TEXT
18 from whoosh.fields import Schema
19 from whoosh.highlight import HtmlFormatter
20 from whoosh.index import create_in
21 from whoosh.index import exists_in
22 from whoosh.index import open_dir
23 from whoosh.qparser import MultifieldParser
24 from whoosh.qparser.dateparse import DateParserPlugin
25 from whoosh.searching import ResultsPage
26 from whoosh.searching import Searcher
27 from whoosh.writing import AsyncWriter
28 
29 from documents.models import Document
30 from documents.models import Note
31 
32 logger = logging.getLogger("paperless.index")
33 
34 
35 def get_schema():
36     return Schema(
37         id=NUMERIC(stored=True, unique=True),
38         title=TEXT(sortable=True),
39         content=TEXT(),
40         asn=NUMERIC(sortable=True, signed=False),
41         correspondent=TEXT(sortable=True),
42         correspondent_id=NUMERIC(),
43         has_correspondent=BOOLEAN(),
44         tag=KEYWORD(commas=True, scorable=True, lowercase=True),
45         tag_id=KEYWORD(commas=True, scorable=True),
46         has_tag=BOOLEAN(),
47         type=TEXT(sortable=True),
48         type_id=NUMERIC(),
49         has_type=BOOLEAN(),
50         created=DATETIME(sortable=True),
51         modified=DATETIME(sortable=True),
52         added=DATETIME(sortable=True),
53         path=TEXT(sortable=True),
54         path_id=NUMERIC(),
55         has_path=BOOLEAN(),
56         notes=TEXT(),
57         owner=TEXT(),
58         owner_id=NUMERIC(),
59         has_owner=BOOLEAN(),
60         viewer_id=KEYWORD(commas=True),
61     )
62 
63 
64 def open_index(recreate=False):
65     try:
66         if exists_in(settings.INDEX_DIR) and not recreate:
67             return open_dir(settings.INDEX_DIR, schema=get_schema())
68     except Exception:
69         logger.exception("Error while opening the index, recreating.")
70 
71     if not os.path.isdir(settings.INDEX_DIR):
72         os.makedirs(settings.INDEX_DIR, exist_ok=True)
73     return create_in(settings.INDEX_DIR, get_schema())
74 
75 
76 @contextmanager
77 def open_index_writer(optimize=False):
78     writer = AsyncWriter(open_index())
79 
80     try:
81         yield writer
82     except Exception as e:
83         logger.exception(str(e))
84         writer.cancel()
85     finally:
86         writer.commit(optimize=optimize)
87 
88 
89 @contextmanager
90 def open_index_searcher():
91     searcher = open_index().searcher()
92 
93     try:
94         yield searcher
95     finally:
96         searcher.close()
97 
98 
99 def update_document(writer: AsyncWriter, doc: Document):
100     tags = ",".join([t.name for t in doc.tags.all()])
101     tags_ids = ",".join([str(t.id) for t in doc.tags.all()])
102     notes = ",".join([str(c.note) for c in Note.objects.filter(document=doc)])
103     asn = doc.archive_serial_number
104     if asn is not None and (
105         asn < Document.ARCHIVE_SERIAL_NUMBER_MIN
106         or asn > Document.ARCHIVE_SERIAL_NUMBER_MAX
107     ):
108         logger.error(
109             f"Not indexing Archive Serial Number {asn} of document {doc.pk}. "
110             f"ASN is out of range "
111             f"[{Document.ARCHIVE_SERIAL_NUMBER_MIN:,}, "
112             f"{Document.ARCHIVE_SERIAL_NUMBER_MAX:,}.",
113         )
114         asn = 0
115     users_with_perms = get_users_with_perms(
116         doc,
117         only_with_perms_in=["view_document"],
118     )
119     viewer_ids = ",".join([str(u.id) for u in users_with_perms])
120     writer.update_document(
121         id=doc.pk,
122         title=doc.title,
123         content=doc.content,
124         correspondent=doc.correspondent.name if doc.correspondent else None,
125         correspondent_id=doc.correspondent.id if doc.correspondent else None,
126         has_correspondent=doc.correspondent is not None,
127         tag=tags if tags else None,
128         tag_id=tags_ids if tags_ids else None,
129         has_tag=len(tags) > 0,
130         type=doc.document_type.name if doc.document_type else None,
131         type_id=doc.document_type.id if doc.document_type else None,
132         has_type=doc.document_type is not None,
133         created=doc.created,
134         added=doc.added,
135         asn=asn,
136         modified=doc.modified,
137         path=doc.storage_path.name if doc.storage_path else None,
138         path_id=doc.storage_path.id if doc.storage_path else None,
139         has_path=doc.storage_path is not None,
140         notes=notes,
141         owner=doc.owner.username if doc.owner else None,
142         owner_id=doc.owner.id if doc.owner else None,
143         has_owner=doc.owner is not None,
144         viewer_id=viewer_ids if viewer_ids else None,
145     )
146 
147 
148 def remove_document(writer, doc):
149     remove_document_by_id(writer, doc.pk)
150 
151 
152 def remove_document_by_id(writer, doc_id):
153     writer.delete_by_term("id", doc_id)
154 
155 
156 def add_or_update_document(document):
157     with open_index_writer() as writer:
158         update_document(writer, document)
159 
160 
161 def remove_document_from_index(document):
162     with open_index_writer() as writer:
163         remove_document(writer, document)
164 
165 
166 class DelayedQuery:
167     def _get_query(self):
168         raise NotImplementedError
169 
170     def _get_query_filter(self):
171         criterias = []
172         for k, v in self.query_params.items():
173             if k == "correspondent__id":
174                 criterias.append(query.Term("correspondent_id", v))
175             elif k == "tags__id__all":
176                 for tag_id in v.split(","):
177                     criterias.append(query.Term("tag_id", tag_id))
178             elif k == "tags__id__none":
179                 for tag_id in v.split(","):
180                     criterias.append(query.Not(query.Term("tag_id", tag_id)))
181             elif k == "document_type__id":
182                 criterias.append(query.Term("type_id", v))
183             elif k == "correspondent__isnull":
184                 criterias.append(query.Term("has_correspondent", v == "false"))
185             elif k == "is_tagged":
186                 criterias.append(query.Term("has_tag", v == "true"))
187             elif k == "document_type__isnull":
188                 criterias.append(query.Term("has_type", v == "false"))
189             elif k == "created__date__lt":
190                 criterias.append(
191                     query.DateRange("created", start=None, end=isoparse(v)),
192                 )
193             elif k == "created__date__gt":
194                 criterias.append(
195                     query.DateRange("created", start=isoparse(v), end=None),
196                 )
197             elif k == "added__date__gt":
198                 criterias.append(query.DateRange("added", start=isoparse(v), end=None))
199             elif k == "added__date__lt":
200                 criterias.append(query.DateRange("added", start=None, end=isoparse(v)))
201             elif k == "storage_path__id":
202                 criterias.append(query.Term("path_id", v))
203             elif k == "storage_path__isnull":
204                 criterias.append(query.Term("has_path", v == "false"))
205 
206         user_criterias = [query.Term("has_owner", False)]
207         if "user" in self.query_params:
208             user_criterias.append(query.Term("owner_id", self.query_params["user"]))
209             user_criterias.append(
210                 query.Term("viewer_id", str(self.query_params["user"])),
211             )
212         if len(criterias) > 0:
213             criterias.append(query.Or(user_criterias))
214             return query.And(criterias)
215         else:
216             return query.Or(user_criterias)
217 
218     def _get_query_sortedby(self):
219         if "ordering" not in self.query_params:
220             return None, False
221 
222         field: str = self.query_params["ordering"]
223 
224         sort_fields_map = {
225             "created": "created",
226             "modified": "modified",
227             "added": "added",
228             "title": "title",
229             "correspondent__name": "correspondent",
230             "document_type__name": "type",
231             "archive_serial_number": "asn",
232         }
233 
234         if field.startswith("-"):
235             field = field[1:]
236             reverse = True
237         else:
238             reverse = False
239 
240         if field not in sort_fields_map:
241             return None, False
242         else:
243             return sort_fields_map[field], reverse
244 
245     def __init__(self, searcher: Searcher, query_params, page_size):
246         self.searcher = searcher
247         self.query_params = query_params
248         self.page_size = page_size
249         self.saved_results = dict()
250         self.first_score = None
251 
252     def __len__(self):
253         page = self[0:1]
254         return len(page)
255 
256     def __getitem__(self, item):
257         if item.start in self.saved_results:
258             return self.saved_results[item.start]
259 
260         q, mask = self._get_query()
261         sortedby, reverse = self._get_query_sortedby()
262 
263         page: ResultsPage = self.searcher.search_page(
264             q,
265             mask=mask,
266             filter=self._get_query_filter(),
267             pagenum=math.floor(item.start / self.page_size) + 1,
268             pagelen=self.page_size,
269             sortedby=sortedby,
270             reverse=reverse,
271         )
272         page.results.fragmenter = highlight.ContextFragmenter(surround=50)
273         page.results.formatter = HtmlFormatter(tagname="span", between=" ... ")
274 
275         if not self.first_score and len(page.results) > 0 and sortedby is None:
276             self.first_score = page.results[0].score
277 
278         page.results.top_n = list(
279             map(
280                 lambda hit: (
281                     (hit[0] / self.first_score) if self.first_score else None,
282                     hit[1],
283                 ),
284                 page.results.top_n,
285             ),
286         )
287 
288         self.saved_results[item.start] = page
289 
290         return page
291 
292 
293 class DelayedFullTextQuery(DelayedQuery):
294     def _get_query(self):
295         q_str = self.query_params["query"]
296         qp = MultifieldParser(
297             ["content", "title", "correspondent", "tag", "type", "notes"],
298             self.searcher.ixreader.schema,
299         )
300         qp.add_plugin(DateParserPlugin(basedate=timezone.now()))
301         q = qp.parse(q_str)
302 
303         corrected = self.searcher.correct_query(q, q_str)
304         if corrected.query != q:
305             corrected.query = corrected.string
306 
307         return q, None
308 
309 
310 class DelayedMoreLikeThisQuery(DelayedQuery):
311     def _get_query(self):
312         more_like_doc_id = int(self.query_params["more_like_id"])
313         content = Document.objects.get(id=more_like_doc_id).content
314 
315         docnum = self.searcher.document_number(id=more_like_doc_id)
316         kts = self.searcher.key_terms_from_text(
317             "content",
318             content,
319             numterms=20,
320             model=classify.Bo1Model,
321             normalize=False,
322         )
323         q = query.Or(
324             [query.Term("content", word, boost=weight) for word, weight in kts],
325         )
326         mask = {docnum}
327 
328         return q, mask
329 
330 
331 def autocomplete(ix, term, limit=10):
332     with ix.reader() as reader:
333         terms = []
334         for score, t in reader.most_distinctive_terms(
335             "content",
336             number=limit,
337             prefix=term.lower(),
338         ):
339             terms.append(t)
340         return terms
341
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/src/documents/index.py b/src/documents/index.py
--- a/src/documents/index.py
+++ b/src/documents/index.py
@@ -172,6 +172,14 @@
         for k, v in self.query_params.items():
             if k == "correspondent__id":
                 criterias.append(query.Term("correspondent_id", v))
+            elif k == "correspondent__id__in":
+                for correspondent_id in v.split(","):
+                    criterias.append(query.Term("correspondent_id", correspondent_id))
+            elif k == "correspondent__id__none":
+                for correspondent_id in v.split(","):
+                    criterias.append(
+                        query.Not(query.Term("correspondent_id", correspondent_id)),
+                    )
             elif k == "tags__id__all":
                 for tag_id in v.split(","):
                     criterias.append(query.Term("tag_id", tag_id))
@@ -180,6 +188,12 @@
                     criterias.append(query.Not(query.Term("tag_id", tag_id)))
             elif k == "document_type__id":
                 criterias.append(query.Term("type_id", v))
+            elif k == "document_type__id__in":
+                for document_type_id in v.split(","):
+                    criterias.append(query.Term("type_id", document_type_id))
+            elif k == "document_type__id__none":
+                for document_type_id in v.split(","):
+                    criterias.append(query.Not(query.Term("type_id", document_type_id)))
             elif k == "correspondent__isnull":
                 criterias.append(query.Term("has_correspondent", v == "false"))
             elif k == "is_tagged":
@@ -200,6 +214,12 @@
                 criterias.append(query.DateRange("added", start=None, end=isoparse(v)))
             elif k == "storage_path__id":
                 criterias.append(query.Term("path_id", v))
+            elif k == "storage_path__id__in":
+                for storage_path_id in v.split(","):
+                    criterias.append(query.Term("path_id", storage_path_id))
+            elif k == "storage_path__id__none":
+                for storage_path_id in v.split(","):
+                    criterias.append(query.Not(query.Term("path_id", storage_path_id)))
             elif k == "storage_path__isnull":
                 criterias.append(query.Term("has_path", v == "false"))

{"golden_diff": "diff --git a/src/documents/index.py b/src/documents/index.py\n--- a/src/documents/index.py\n+++ b/src/documents/index.py\n@@ -172,6 +172,14 @@\n         for k, v in self.query_params.items():\n             if k == \"correspondent__id\":\n                 criterias.append(query.Term(\"correspondent_id\", v))\n+            elif k == \"correspondent__id__in\":\n+                for correspondent_id in v.split(\",\"):\n+                    criterias.append(query.Term(\"correspondent_id\", correspondent_id))\n+            elif k == \"correspondent__id__none\":\n+                for correspondent_id in v.split(\",\"):\n+                    criterias.append(\n+                        query.Not(query.Term(\"correspondent_id\", correspondent_id)),\n+                    )\n             elif k == \"tags__id__all\":\n                 for tag_id in v.split(\",\"):\n                     criterias.append(query.Term(\"tag_id\", tag_id))\n@@ -180,6 +188,12 @@\n                     criterias.append(query.Not(query.Term(\"tag_id\", tag_id)))\n             elif k == \"document_type__id\":\n                 criterias.append(query.Term(\"type_id\", v))\n+            elif k == \"document_type__id__in\":\n+                for document_type_id in v.split(\",\"):\n+                    criterias.append(query.Term(\"type_id\", document_type_id))\n+            elif k == \"document_type__id__none\":\n+                for document_type_id in v.split(\",\"):\n+                    criterias.append(query.Not(query.Term(\"type_id\", document_type_id)))\n             elif k == \"correspondent__isnull\":\n                 criterias.append(query.Term(\"has_correspondent\", v == \"false\"))\n             elif k == \"is_tagged\":\n@@ -200,6 +214,12 @@\n                 criterias.append(query.DateRange(\"added\", start=None, end=isoparse(v)))\n             elif k == \"storage_path__id\":\n                 criterias.append(query.Term(\"path_id\", v))\n+            elif k == \"storage_path__id__in\":\n+                for storage_path_id in v.split(\",\"):\n+                    criterias.append(query.Term(\"path_id\", storage_path_id))\n+            elif k == \"storage_path__id__none\":\n+                for storage_path_id in v.split(\",\"):\n+                    criterias.append(query.Not(query.Term(\"path_id\", storage_path_id)))\n             elif k == \"storage_path__isnull\":\n                 criterias.append(query.Term(\"has_path\", v == \"false\"))\n", "issue": "[BUG] Selecting date ranges breaks searches\n### Description\n\nSearching documents using TWO filters will not work, if one parameter is \"last month\" or \"last year\" in the \"added\" or \"created\" field. I tried to be more precise than #3203 \r\n\r\nExample: \r\n\r\nSelected document type is \"Anleitung\" -> everything works as expected:\r\n![image](https://user-images.githubusercontent.com/113503248/234883356-1975b4b6-713c-4d93-bb52-459638fa2ed4.png)\r\n\r\nNow a second filter \"last year\" is applied. Year is as expected but selected document type is ignored:\r\n![image](https://user-images.githubusercontent.com/113503248/234883815-39dcd974-a010-46ac-9897-cb326cf036cc.png)\r\n\n\n### Steps to reproduce\n\n1. login as a superuser\r\n2. go to web-ui -> documents\r\n3. reset filters (all documents displayed now)\r\n4. select document type.\r\n5. select \"created\" oder \"added\" date range \"last year\"\r\n-> all document types ara displayed.\n\n### Webserver logs\n\n```bash\nnone\n```\n\n\n### Browser logs\n\n```bash\nnone\n```\n\n\n### Paperless-ngx version\n\n1.14.0\n\n### Host OS\n\nLinux 3.10.108 #42661 SMP (x86_64)\n\n### Installation method\n\nDocker - official image\n\n### Browser\n\nChrome\n\n### Configuration changes\n\n_No response_\n\n### Other\n\n_No response_\n", "before_files": [{"content": "import logging\nimport math\nimport os\nfrom contextlib import contextmanager\n\nfrom dateutil.parser import isoparse\nfrom django.conf import settings\nfrom django.utils import timezone\nfrom guardian.shortcuts import get_users_with_perms\nfrom whoosh import classify\nfrom whoosh import highlight\nfrom whoosh import query\nfrom whoosh.fields import BOOLEAN\nfrom whoosh.fields import DATETIME\nfrom whoosh.fields import KEYWORD\nfrom whoosh.fields import NUMERIC\nfrom whoosh.fields import TEXT\nfrom whoosh.fields import Schema\nfrom whoosh.highlight import HtmlFormatter\nfrom whoosh.index import create_in\nfrom whoosh.index import exists_in\nfrom whoosh.index import open_dir\nfrom whoosh.qparser import MultifieldParser\nfrom whoosh.qparser.dateparse import DateParserPlugin\nfrom whoosh.searching import ResultsPage\nfrom whoosh.searching import Searcher\nfrom whoosh.writing import AsyncWriter\n\nfrom documents.models import Document\nfrom documents.models import Note\n\nlogger = logging.getLogger(\"paperless.index\")\n\n\ndef get_schema():\n    return Schema(\n        id=NUMERIC(stored=True, unique=True),\n        title=TEXT(sortable=True),\n        content=TEXT(),\n        asn=NUMERIC(sortable=True, signed=False),\n        correspondent=TEXT(sortable=True),\n        correspondent_id=NUMERIC(),\n        has_correspondent=BOOLEAN(),\n        tag=KEYWORD(commas=True, scorable=True, lowercase=True),\n        tag_id=KEYWORD(commas=True, scorable=True),\n        has_tag=BOOLEAN(),\n        type=TEXT(sortable=True),\n        type_id=NUMERIC(),\n        has_type=BOOLEAN(),\n        created=DATETIME(sortable=True),\n        modified=DATETIME(sortable=True),\n        added=DATETIME(sortable=True),\n        path=TEXT(sortable=True),\n        path_id=NUMERIC(),\n        has_path=BOOLEAN(),\n        notes=TEXT(),\n        owner=TEXT(),\n        owner_id=NUMERIC(),\n        has_owner=BOOLEAN(),\n        viewer_id=KEYWORD(commas=True),\n    )\n\n\ndef open_index(recreate=False):\n    try:\n        if exists_in(settings.INDEX_DIR) and not recreate:\n            return open_dir(settings.INDEX_DIR, schema=get_schema())\n    except Exception:\n        logger.exception(\"Error while opening the index, recreating.\")\n\n    if not os.path.isdir(settings.INDEX_DIR):\n        os.makedirs(settings.INDEX_DIR, exist_ok=True)\n    return create_in(settings.INDEX_DIR, get_schema())\n\n\n@contextmanager\ndef open_index_writer(optimize=False):\n    writer = AsyncWriter(open_index())\n\n    try:\n        yield writer\n    except Exception as e:\n        logger.exception(str(e))\n        writer.cancel()\n    finally:\n        writer.commit(optimize=optimize)\n\n\n@contextmanager\ndef open_index_searcher():\n    searcher = open_index().searcher()\n\n    try:\n        yield searcher\n    finally:\n        searcher.close()\n\n\ndef update_document(writer: AsyncWriter, doc: Document):\n    tags = \",\".join([t.name for t in doc.tags.all()])\n    tags_ids = \",\".join([str(t.id) for t in doc.tags.all()])\n    notes = \",\".join([str(c.note) for c in Note.objects.filter(document=doc)])\n    asn = doc.archive_serial_number\n    if asn is not None and (\n        asn < Document.ARCHIVE_SERIAL_NUMBER_MIN\n        or asn > Document.ARCHIVE_SERIAL_NUMBER_MAX\n    ):\n        logger.error(\n            f\"Not indexing Archive Serial Number {asn} of document {doc.pk}. \"\n            f\"ASN is out of range \"\n            f\"[{Document.ARCHIVE_SERIAL_NUMBER_MIN:,}, \"\n            f\"{Document.ARCHIVE_SERIAL_NUMBER_MAX:,}.\",\n        )\n        asn = 0\n    users_with_perms = get_users_with_perms(\n        doc,\n        only_with_perms_in=[\"view_document\"],\n    )\n    viewer_ids = \",\".join([str(u.id) for u in users_with_perms])\n    writer.update_document(\n        id=doc.pk,\n        title=doc.title,\n        content=doc.content,\n        correspondent=doc.correspondent.name if doc.correspondent else None,\n        correspondent_id=doc.correspondent.id if doc.correspondent else None,\n        has_correspondent=doc.correspondent is not None,\n        tag=tags if tags else None,\n        tag_id=tags_ids if tags_ids else None,\n        has_tag=len(tags) > 0,\n        type=doc.document_type.name if doc.document_type else None,\n        type_id=doc.document_type.id if doc.document_type else None,\n        has_type=doc.document_type is not None,\n        created=doc.created,\n        added=doc.added,\n        asn=asn,\n        modified=doc.modified,\n        path=doc.storage_path.name if doc.storage_path else None,\n        path_id=doc.storage_path.id if doc.storage_path else None,\n        has_path=doc.storage_path is not None,\n        notes=notes,\n        owner=doc.owner.username if doc.owner else None,\n        owner_id=doc.owner.id if doc.owner else None,\n        has_owner=doc.owner is not None,\n        viewer_id=viewer_ids if viewer_ids else None,\n    )\n\n\ndef remove_document(writer, doc):\n    remove_document_by_id(writer, doc.pk)\n\n\ndef remove_document_by_id(writer, doc_id):\n    writer.delete_by_term(\"id\", doc_id)\n\n\ndef add_or_update_document(document):\n    with open_index_writer() as writer:\n        update_document(writer, document)\n\n\ndef remove_document_from_index(document):\n    with open_index_writer() as writer:\n        remove_document(writer, document)\n\n\nclass DelayedQuery:\n    def _get_query(self):\n        raise NotImplementedError\n\n    def _get_query_filter(self):\n        criterias = []\n        for k, v in self.query_params.items():\n            if k == \"correspondent__id\":\n                criterias.append(query.Term(\"correspondent_id\", v))\n            elif k == \"tags__id__all\":\n                for tag_id in v.split(\",\"):\n                    criterias.append(query.Term(\"tag_id\", tag_id))\n            elif k == \"tags__id__none\":\n                for tag_id in v.split(\",\"):\n                    criterias.append(query.Not(query.Term(\"tag_id\", tag_id)))\n            elif k == \"document_type__id\":\n                criterias.append(query.Term(\"type_id\", v))\n            elif k == \"correspondent__isnull\":\n                criterias.append(query.Term(\"has_correspondent\", v == \"false\"))\n            elif k == \"is_tagged\":\n                criterias.append(query.Term(\"has_tag\", v == \"true\"))\n            elif k == \"document_type__isnull\":\n                criterias.append(query.Term(\"has_type\", v == \"false\"))\n            elif k == \"created__date__lt\":\n                criterias.append(\n                    query.DateRange(\"created\", start=None, end=isoparse(v)),\n                )\n            elif k == \"created__date__gt\":\n                criterias.append(\n                    query.DateRange(\"created\", start=isoparse(v), end=None),\n                )\n            elif k == \"added__date__gt\":\n                criterias.append(query.DateRange(\"added\", start=isoparse(v), end=None))\n            elif k == \"added__date__lt\":\n                criterias.append(query.DateRange(\"added\", start=None, end=isoparse(v)))\n            elif k == \"storage_path__id\":\n                criterias.append(query.Term(\"path_id\", v))\n            elif k == \"storage_path__isnull\":\n                criterias.append(query.Term(\"has_path\", v == \"false\"))\n\n        user_criterias = [query.Term(\"has_owner\", False)]\n        if \"user\" in self.query_params:\n            user_criterias.append(query.Term(\"owner_id\", self.query_params[\"user\"]))\n            user_criterias.append(\n                query.Term(\"viewer_id\", str(self.query_params[\"user\"])),\n            )\n        if len(criterias) > 0:\n            criterias.append(query.Or(user_criterias))\n            return query.And(criterias)\n        else:\n            return query.Or(user_criterias)\n\n    def _get_query_sortedby(self):\n        if \"ordering\" not in self.query_params:\n            return None, False\n\n        field: str = self.query_params[\"ordering\"]\n\n        sort_fields_map = {\n            \"created\": \"created\",\n            \"modified\": \"modified\",\n            \"added\": \"added\",\n            \"title\": \"title\",\n            \"correspondent__name\": \"correspondent\",\n            \"document_type__name\": \"type\",\n            \"archive_serial_number\": \"asn\",\n        }\n\n        if field.startswith(\"-\"):\n            field = field[1:]\n            reverse = True\n        else:\n            reverse = False\n\n        if field not in sort_fields_map:\n            return None, False\n        else:\n            return sort_fields_map[field], reverse\n\n    def __init__(self, searcher: Searcher, query_params, page_size):\n        self.searcher = searcher\n        self.query_params = query_params\n        self.page_size = page_size\n        self.saved_results = dict()\n        self.first_score = None\n\n    def __len__(self):\n        page = self[0:1]\n        return len(page)\n\n    def __getitem__(self, item):\n        if item.start in self.saved_results:\n            return self.saved_results[item.start]\n\n        q, mask = self._get_query()\n        sortedby, reverse = self._get_query_sortedby()\n\n        page: ResultsPage = self.searcher.search_page(\n            q,\n            mask=mask,\n            filter=self._get_query_filter(),\n            pagenum=math.floor(item.start / self.page_size) + 1,\n            pagelen=self.page_size,\n            sortedby=sortedby,\n            reverse=reverse,\n        )\n        page.results.fragmenter = highlight.ContextFragmenter(surround=50)\n        page.results.formatter = HtmlFormatter(tagname=\"span\", between=\" ... \")\n\n        if not self.first_score and len(page.results) > 0 and sortedby is None:\n            self.first_score = page.results[0].score\n\n        page.results.top_n = list(\n            map(\n                lambda hit: (\n                    (hit[0] / self.first_score) if self.first_score else None,\n                    hit[1],\n                ),\n                page.results.top_n,\n            ),\n        )\n\n        self.saved_results[item.start] = page\n\n        return page\n\n\nclass DelayedFullTextQuery(DelayedQuery):\n    def _get_query(self):\n        q_str = self.query_params[\"query\"]\n        qp = MultifieldParser(\n            [\"content\", \"title\", \"correspondent\", \"tag\", \"type\", \"notes\"],\n            self.searcher.ixreader.schema,\n        )\n        qp.add_plugin(DateParserPlugin(basedate=timezone.now()))\n        q = qp.parse(q_str)\n\n        corrected = self.searcher.correct_query(q, q_str)\n        if corrected.query != q:\n            corrected.query = corrected.string\n\n        return q, None\n\n\nclass DelayedMoreLikeThisQuery(DelayedQuery):\n    def _get_query(self):\n        more_like_doc_id = int(self.query_params[\"more_like_id\"])\n        content = Document.objects.get(id=more_like_doc_id).content\n\n        docnum = self.searcher.document_number(id=more_like_doc_id)\n        kts = self.searcher.key_terms_from_text(\n            \"content\",\n            content,\n            numterms=20,\n            model=classify.Bo1Model,\n            normalize=False,\n        )\n        q = query.Or(\n            [query.Term(\"content\", word, boost=weight) for word, weight in kts],\n        )\n        mask = {docnum}\n\n        return q, mask\n\n\ndef autocomplete(ix, term, limit=10):\n    with ix.reader() as reader:\n        terms = []\n        for score, t in reader.most_distinctive_terms(\n            \"content\",\n            number=limit,\n            prefix=term.lower(),\n        ):\n            terms.append(t)\n        return terms\n", "path": "src/documents/index.py"}], "after_files": [{"content": "import logging\nimport math\nimport os\nfrom contextlib import contextmanager\n\nfrom dateutil.parser import isoparse\nfrom django.conf import settings\nfrom django.utils import timezone\nfrom guardian.shortcuts import get_users_with_perms\nfrom whoosh import classify\nfrom whoosh import highlight\nfrom whoosh import query\nfrom whoosh.fields import BOOLEAN\nfrom whoosh.fields import DATETIME\nfrom whoosh.fields import KEYWORD\nfrom whoosh.fields import NUMERIC\nfrom whoosh.fields import TEXT\nfrom whoosh.fields import Schema\nfrom whoosh.highlight import HtmlFormatter\nfrom whoosh.index import create_in\nfrom whoosh.index import exists_in\nfrom whoosh.index import open_dir\nfrom whoosh.qparser import MultifieldParser\nfrom whoosh.qparser.dateparse import DateParserPlugin\nfrom whoosh.searching import ResultsPage\nfrom whoosh.searching import Searcher\nfrom whoosh.writing import AsyncWriter\n\nfrom documents.models import Document\nfrom documents.models import Note\n\nlogger = logging.getLogger(\"paperless.index\")\n\n\ndef get_schema():\n    return Schema(\n        id=NUMERIC(stored=True, unique=True),\n        title=TEXT(sortable=True),\n        content=TEXT(),\n        asn=NUMERIC(sortable=True, signed=False),\n        correspondent=TEXT(sortable=True),\n        correspondent_id=NUMERIC(),\n        has_correspondent=BOOLEAN(),\n        tag=KEYWORD(commas=True, scorable=True, lowercase=True),\n        tag_id=KEYWORD(commas=True, scorable=True),\n        has_tag=BOOLEAN(),\n        type=TEXT(sortable=True),\n        type_id=NUMERIC(),\n        has_type=BOOLEAN(),\n        created=DATETIME(sortable=True),\n        modified=DATETIME(sortable=True),\n        added=DATETIME(sortable=True),\n        path=TEXT(sortable=True),\n        path_id=NUMERIC(),\n        has_path=BOOLEAN(),\n        notes=TEXT(),\n        owner=TEXT(),\n        owner_id=NUMERIC(),\n        has_owner=BOOLEAN(),\n        viewer_id=KEYWORD(commas=True),\n    )\n\n\ndef open_index(recreate=False):\n    try:\n        if exists_in(settings.INDEX_DIR) and not recreate:\n            return open_dir(settings.INDEX_DIR, schema=get_schema())\n    except Exception:\n        logger.exception(\"Error while opening the index, recreating.\")\n\n    if not os.path.isdir(settings.INDEX_DIR):\n        os.makedirs(settings.INDEX_DIR, exist_ok=True)\n    return create_in(settings.INDEX_DIR, get_schema())\n\n\n@contextmanager\ndef open_index_writer(optimize=False):\n    writer = AsyncWriter(open_index())\n\n    try:\n        yield writer\n    except Exception as e:\n        logger.exception(str(e))\n        writer.cancel()\n    finally:\n        writer.commit(optimize=optimize)\n\n\n@contextmanager\ndef open_index_searcher():\n    searcher = open_index().searcher()\n\n    try:\n        yield searcher\n    finally:\n        searcher.close()\n\n\ndef update_document(writer: AsyncWriter, doc: Document):\n    tags = \",\".join([t.name for t in doc.tags.all()])\n    tags_ids = \",\".join([str(t.id) for t in doc.tags.all()])\n    notes = \",\".join([str(c.note) for c in Note.objects.filter(document=doc)])\n    asn = doc.archive_serial_number\n    if asn is not None and (\n        asn < Document.ARCHIVE_SERIAL_NUMBER_MIN\n        or asn > Document.ARCHIVE_SERIAL_NUMBER_MAX\n    ):\n        logger.error(\n            f\"Not indexing Archive Serial Number {asn} of document {doc.pk}. \"\n            f\"ASN is out of range \"\n            f\"[{Document.ARCHIVE_SERIAL_NUMBER_MIN:,}, \"\n            f\"{Document.ARCHIVE_SERIAL_NUMBER_MAX:,}.\",\n        )\n        asn = 0\n    users_with_perms = get_users_with_perms(\n        doc,\n        only_with_perms_in=[\"view_document\"],\n    )\n    viewer_ids = \",\".join([str(u.id) for u in users_with_perms])\n    writer.update_document(\n        id=doc.pk,\n        title=doc.title,\n        content=doc.content,\n        correspondent=doc.correspondent.name if doc.correspondent else None,\n        correspondent_id=doc.correspondent.id if doc.correspondent else None,\n        has_correspondent=doc.correspondent is not None,\n        tag=tags if tags else None,\n        tag_id=tags_ids if tags_ids else None,\n        has_tag=len(tags) > 0,\n        type=doc.document_type.name if doc.document_type else None,\n        type_id=doc.document_type.id if doc.document_type else None,\n        has_type=doc.document_type is not None,\n        created=doc.created,\n        added=doc.added,\n        asn=asn,\n        modified=doc.modified,\n        path=doc.storage_path.name if doc.storage_path else None,\n        path_id=doc.storage_path.id if doc.storage_path else None,\n        has_path=doc.storage_path is not None,\n        notes=notes,\n        owner=doc.owner.username if doc.owner else None,\n        owner_id=doc.owner.id if doc.owner else None,\n        has_owner=doc.owner is not None,\n        viewer_id=viewer_ids if viewer_ids else None,\n    )\n\n\ndef remove_document(writer, doc):\n    remove_document_by_id(writer, doc.pk)\n\n\ndef remove_document_by_id(writer, doc_id):\n    writer.delete_by_term(\"id\", doc_id)\n\n\ndef add_or_update_document(document):\n    with open_index_writer() as writer:\n        update_document(writer, document)\n\n\ndef remove_document_from_index(document):\n    with open_index_writer() as writer:\n        remove_document(writer, document)\n\n\nclass DelayedQuery:\n    def _get_query(self):\n        raise NotImplementedError\n\n    def _get_query_filter(self):\n        criterias = []\n        for k, v in self.query_params.items():\n            if k == \"correspondent__id\":\n                criterias.append(query.Term(\"correspondent_id\", v))\n            elif k == \"correspondent__id__in\":\n                for correspondent_id in v.split(\",\"):\n                    criterias.append(query.Term(\"correspondent_id\", correspondent_id))\n            elif k == \"correspondent__id__none\":\n                for correspondent_id in v.split(\",\"):\n                    criterias.append(\n                        query.Not(query.Term(\"correspondent_id\", correspondent_id)),\n                    )\n            elif k == \"tags__id__all\":\n                for tag_id in v.split(\",\"):\n                    criterias.append(query.Term(\"tag_id\", tag_id))\n            elif k == \"tags__id__none\":\n                for tag_id in v.split(\",\"):\n                    criterias.append(query.Not(query.Term(\"tag_id\", tag_id)))\n            elif k == \"document_type__id\":\n                criterias.append(query.Term(\"type_id\", v))\n            elif k == \"document_type__id__in\":\n                for document_type_id in v.split(\",\"):\n                    criterias.append(query.Term(\"type_id\", document_type_id))\n            elif k == \"document_type__id__none\":\n                for document_type_id in v.split(\",\"):\n                    criterias.append(query.Not(query.Term(\"type_id\", document_type_id)))\n            elif k == \"correspondent__isnull\":\n                criterias.append(query.Term(\"has_correspondent\", v == \"false\"))\n            elif k == \"is_tagged\":\n                criterias.append(query.Term(\"has_tag\", v == \"true\"))\n            elif k == \"document_type__isnull\":\n                criterias.append(query.Term(\"has_type\", v == \"false\"))\n            elif k == \"created__date__lt\":\n                criterias.append(\n                    query.DateRange(\"created\", start=None, end=isoparse(v)),\n                )\n            elif k == \"created__date__gt\":\n                criterias.append(\n                    query.DateRange(\"created\", start=isoparse(v), end=None),\n                )\n            elif k == \"added__date__gt\":\n                criterias.append(query.DateRange(\"added\", start=isoparse(v), end=None))\n            elif k == \"added__date__lt\":\n                criterias.append(query.DateRange(\"added\", start=None, end=isoparse(v)))\n            elif k == \"storage_path__id\":\n                criterias.append(query.Term(\"path_id\", v))\n            elif k == \"storage_path__id__in\":\n                for storage_path_id in v.split(\",\"):\n                    criterias.append(query.Term(\"path_id\", storage_path_id))\n            elif k == \"storage_path__id__none\":\n                for storage_path_id in v.split(\",\"):\n                    criterias.append(query.Not(query.Term(\"path_id\", storage_path_id)))\n            elif k == \"storage_path__isnull\":\n                criterias.append(query.Term(\"has_path\", v == \"false\"))\n\n        user_criterias = [query.Term(\"has_owner\", False)]\n        if \"user\" in self.query_params:\n            user_criterias.append(query.Term(\"owner_id\", self.query_params[\"user\"]))\n            user_criterias.append(\n                query.Term(\"viewer_id\", str(self.query_params[\"user\"])),\n            )\n        if len(criterias) > 0:\n            criterias.append(query.Or(user_criterias))\n            return query.And(criterias)\n        else:\n            return query.Or(user_criterias)\n\n    def _get_query_sortedby(self):\n        if \"ordering\" not in self.query_params:\n            return None, False\n\n        field: str = self.query_params[\"ordering\"]\n\n        sort_fields_map = {\n            \"created\": \"created\",\n            \"modified\": \"modified\",\n            \"added\": \"added\",\n            \"title\": \"title\",\n            \"correspondent__name\": \"correspondent\",\n            \"document_type__name\": \"type\",\n            \"archive_serial_number\": \"asn\",\n        }\n\n        if field.startswith(\"-\"):\n            field = field[1:]\n            reverse = True\n        else:\n            reverse = False\n\n        if field not in sort_fields_map:\n            return None, False\n        else:\n            return sort_fields_map[field], reverse\n\n    def __init__(self, searcher: Searcher, query_params, page_size):\n        self.searcher = searcher\n        self.query_params = query_params\n        self.page_size = page_size\n        self.saved_results = dict()\n        self.first_score = None\n\n    def __len__(self):\n        page = self[0:1]\n        return len(page)\n\n    def __getitem__(self, item):\n        if item.start in self.saved_results:\n            return self.saved_results[item.start]\n\n        q, mask = self._get_query()\n        sortedby, reverse = self._get_query_sortedby()\n\n        page: ResultsPage = self.searcher.search_page(\n            q,\n            mask=mask,\n            filter=self._get_query_filter(),\n            pagenum=math.floor(item.start / self.page_size) + 1,\n            pagelen=self.page_size,\n            sortedby=sortedby,\n            reverse=reverse,\n        )\n        page.results.fragmenter = highlight.ContextFragmenter(surround=50)\n        page.results.formatter = HtmlFormatter(tagname=\"span\", between=\" ... \")\n\n        if not self.first_score and len(page.results) > 0 and sortedby is None:\n            self.first_score = page.results[0].score\n\n        page.results.top_n = list(\n            map(\n                lambda hit: (\n                    (hit[0] / self.first_score) if self.first_score else None,\n                    hit[1],\n                ),\n                page.results.top_n,\n            ),\n        )\n\n        self.saved_results[item.start] = page\n\n        return page\n\n\nclass DelayedFullTextQuery(DelayedQuery):\n    def _get_query(self):\n        q_str = self.query_params[\"query\"]\n        qp = MultifieldParser(\n            [\"content\", \"title\", \"correspondent\", \"tag\", \"type\", \"notes\"],\n            self.searcher.ixreader.schema,\n        )\n        qp.add_plugin(DateParserPlugin(basedate=timezone.now()))\n        q = qp.parse(q_str)\n\n        corrected = self.searcher.correct_query(q, q_str)\n        if corrected.query != q:\n            corrected.query = corrected.string\n\n        return q, None\n\n\nclass DelayedMoreLikeThisQuery(DelayedQuery):\n    def _get_query(self):\n        more_like_doc_id = int(self.query_params[\"more_like_id\"])\n        content = Document.objects.get(id=more_like_doc_id).content\n\n        docnum = self.searcher.document_number(id=more_like_doc_id)\n        kts = self.searcher.key_terms_from_text(\n            \"content\",\n            content,\n            numterms=20,\n            model=classify.Bo1Model,\n            normalize=False,\n        )\n        q = query.Or(\n            [query.Term(\"content\", word, boost=weight) for word, weight in kts],\n        )\n        mask = {docnum}\n\n        return q, mask\n\n\ndef autocomplete(ix, term, limit=10):\n    with ix.reader() as reader:\n        terms = []\n        for score, t in reader.most_distinctive_terms(\n            \"content\",\n            number=limit,\n            prefix=term.lower(),\n        ):\n            terms.append(t)\n        return terms\n", "path": "src/documents/index.py"}]}