We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
[BUG] storage_path is missing in manifest.json
### Description

there is no "model": "documents.storage_path" in manifest.json. I found it only in db.sqlite

### Steps to reproduce

open export/manifest.json after `docker compose exec webserver document_exporter ../export -f`

### Webserver logs

_No response_

### Paperless-ngx version

1.8.0

### Host OS

Raspberry Pi 4/arm64

### Installation method

Docker - official image

### Browser

_No response_

### Configuration changes

_No response_

### Other

_No response_

--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `src/documents/management/commands/document_exporter.py`
Content:
```
1 import hashlib
2 import json
3 import os
4 import shutil
5 import time
6 
7 import tqdm
8 from django.conf import settings
9 from django.contrib.auth.models import Group
10 from django.contrib.auth.models import User
11 from django.core import serializers
12 from django.core.management.base import BaseCommand
13 from django.core.management.base import CommandError
14 from django.db import transaction
15 from documents.models import Comment
16 from documents.models import Correspondent
17 from documents.models import Document
18 from documents.models import DocumentType
19 from documents.models import SavedView
20 from documents.models import SavedViewFilterRule
21 from documents.models import Tag
22 from documents.models import UiSettings
23 from documents.settings import EXPORTER_ARCHIVE_NAME
24 from documents.settings import EXPORTER_FILE_NAME
25 from documents.settings import EXPORTER_THUMBNAIL_NAME
26 from filelock import FileLock
27 from paperless import version
28 from paperless.db import GnuPG
29 from paperless_mail.models import MailAccount
30 from paperless_mail.models import MailRule
31 
32 from ...file_handling import delete_empty_directories
33 from ...file_handling import generate_filename
34 
35 
36 class Command(BaseCommand):
37 
38     help = """
39         Decrypt and rename all files in our collection into a given target
40         directory.  And include a manifest file containing document data for
41         easy import.
42     """.replace(
43         "    ",
44         "",
45     )
46 
47     def add_arguments(self, parser):
48         parser.add_argument("target")
49 
50         parser.add_argument(
51             "-c",
52             "--compare-checksums",
53             default=False,
54             action="store_true",
55             help="Compare file checksums when determining whether to export "
56             "a file or not. If not specified, file size and time "
57             "modified is used instead.",
58         )
59 
60         parser.add_argument(
61             "-f",
62             "--use-filename-format",
63             default=False,
64             action="store_true",
65             help="Use PAPERLESS_FILENAME_FORMAT for storing files in the "
66             "export directory, if configured.",
67         )
68 
69         parser.add_argument(
70             "-d",
71             "--delete",
72             default=False,
73             action="store_true",
74             help="After exporting, delete files in the export directory that "
75             "do not belong to the current export, such as files from "
76             "deleted documents.",
77         )
78         parser.add_argument(
79             "--no-progress-bar",
80             default=False,
81             action="store_true",
82             help="If set, the progress bar will not be shown",
83         )
84 
85     def __init__(self, *args, **kwargs):
86         BaseCommand.__init__(self, *args, **kwargs)
87         self.target = None
88         self.files_in_export_dir = []
89         self.exported_files = []
90         self.compare_checksums = False
91         self.use_filename_format = False
92         self.delete = False
93 
94     def handle(self, *args, **options):
95 
96         self.target = options["target"]
97         self.compare_checksums = options["compare_checksums"]
98         self.use_filename_format = options["use_filename_format"]
99         self.delete = options["delete"]
100 
101         if not os.path.exists(self.target):
102             raise CommandError("That path doesn't exist")
103 
104         if not os.access(self.target, os.W_OK):
105             raise CommandError("That path doesn't appear to be writable")
106 
107         with FileLock(settings.MEDIA_LOCK):
108             self.dump(options["no_progress_bar"])
109 
110     def dump(self, progress_bar_disable=False):
111         # 1. Take a snapshot of what files exist in the current export folder
112         for root, dirs, files in os.walk(self.target):
113             self.files_in_export_dir.extend(
114                 map(lambda f: os.path.abspath(os.path.join(root, f)), files),
115             )
116 
117         # 2. Create manifest, containing all correspondents, types, tags,
118         # documents and ui_settings
119         with transaction.atomic():
120             manifest = json.loads(
121                 serializers.serialize("json", Correspondent.objects.all()),
122             )
123 
124             manifest += json.loads(serializers.serialize("json", Tag.objects.all()))
125 
126             manifest += json.loads(
127                 serializers.serialize("json", DocumentType.objects.all()),
128             )
129 
130             manifest += json.loads(
131                 serializers.serialize("json", Comment.objects.all()),
132             )
133 
134             documents = Document.objects.order_by("id")
135             document_map = {d.pk: d for d in documents}
136             document_manifest = json.loads(serializers.serialize("json", documents))
137             manifest += document_manifest
138 
139             manifest += json.loads(
140                 serializers.serialize("json", MailAccount.objects.all()),
141             )
142 
143             manifest += json.loads(
144                 serializers.serialize("json", MailRule.objects.all()),
145             )
146 
147             manifest += json.loads(
148                 serializers.serialize("json", SavedView.objects.all()),
149             )
150 
151             manifest += json.loads(
152                 serializers.serialize("json", SavedViewFilterRule.objects.all()),
153             )
154 
155             manifest += json.loads(serializers.serialize("json", Group.objects.all()))
156 
157             manifest += json.loads(serializers.serialize("json", User.objects.all()))
158 
159             manifest += json.loads(
160                 serializers.serialize("json", UiSettings.objects.all()),
161             )
162 
163         # 3. Export files from each document
164         for index, document_dict in tqdm.tqdm(
165             enumerate(document_manifest),
166             total=len(document_manifest),
167             disable=progress_bar_disable,
168         ):
169             # 3.1. store files unencrypted
170             document_dict["fields"]["storage_type"] = Document.STORAGE_TYPE_UNENCRYPTED
171 
172             document = document_map[document_dict["pk"]]
173 
174             # 3.2. generate a unique filename
175             filename_counter = 0
176             while True:
177                 if self.use_filename_format:
178                     base_name = generate_filename(
179                         document,
180                         counter=filename_counter,
181                         append_gpg=False,
182                     )
183                 else:
184                     base_name = document.get_public_filename(counter=filename_counter)
185 
186                 if base_name not in self.exported_files:
187                     self.exported_files.append(base_name)
188                     break
189                 else:
190                     filename_counter += 1
191 
192             # 3.3. write filenames into manifest
193             original_name = base_name
194             original_target = os.path.join(self.target, original_name)
195             document_dict[EXPORTER_FILE_NAME] = original_name
196 
197             thumbnail_name = base_name + "-thumbnail.webp"
198             thumbnail_target = os.path.join(self.target, thumbnail_name)
199             document_dict[EXPORTER_THUMBNAIL_NAME] = thumbnail_name
200 
201             if document.has_archive_version:
202                 archive_name = base_name + "-archive.pdf"
203                 archive_target = os.path.join(self.target, archive_name)
204                 document_dict[EXPORTER_ARCHIVE_NAME] = archive_name
205             else:
206                 archive_target = None
207 
208             # 3.4. write files to target folder
209             t = int(time.mktime(document.created.timetuple()))
210             if document.storage_type == Document.STORAGE_TYPE_GPG:
211 
212                 os.makedirs(os.path.dirname(original_target), exist_ok=True)
213                 with open(original_target, "wb") as f:
214                     with document.source_file as out_file:
215                         f.write(GnuPG.decrypted(out_file))
216                         os.utime(original_target, times=(t, t))
217 
218                 os.makedirs(os.path.dirname(thumbnail_target), exist_ok=True)
219                 with open(thumbnail_target, "wb") as f:
220                     with document.thumbnail_file as out_file:
221                         f.write(GnuPG.decrypted(out_file))
222                         os.utime(thumbnail_target, times=(t, t))
223 
224                 if archive_target:
225                     os.makedirs(os.path.dirname(archive_target), exist_ok=True)
226                     with open(archive_target, "wb") as f:
227                         with document.archive_path as out_file:
228                             f.write(GnuPG.decrypted(out_file))
229                             os.utime(archive_target, times=(t, t))
230             else:
231                 self.check_and_copy(
232                     document.source_path,
233                     document.checksum,
234                     original_target,
235                 )
236 
237                 self.check_and_copy(document.thumbnail_path, None, thumbnail_target)
238 
239                 if archive_target:
240                     self.check_and_copy(
241                         document.archive_path,
242                         document.archive_checksum,
243                         archive_target,
244                     )
245 
246         # 4.1 write manifest to target folder
247         manifest_path = os.path.abspath(os.path.join(self.target, "manifest.json"))
248 
249         with open(manifest_path, "w") as f:
250             json.dump(manifest, f, indent=2)
251 
252         # 4.2 write version information to target folder
253         version_path = os.path.abspath(os.path.join(self.target, "version.json"))
254 
255         with open(version_path, "w") as f:
256             json.dump({"version": version.__full_version_str__}, f, indent=2)
257 
258         if self.delete:
259             # 5. Remove files which we did not explicitly export in this run
260 
261             if manifest_path in self.files_in_export_dir:
262                 self.files_in_export_dir.remove(manifest_path)
263 
264             for f in self.files_in_export_dir:
265                 os.remove(f)
266 
267                 delete_empty_directories(
268                     os.path.abspath(os.path.dirname(f)),
269                     os.path.abspath(self.target),
270                 )
271 
272     def check_and_copy(self, source, source_checksum, target):
273         if os.path.abspath(target) in self.files_in_export_dir:
274             self.files_in_export_dir.remove(os.path.abspath(target))
275 
276         perform_copy = False
277 
278         if os.path.exists(target):
279             source_stat = os.stat(source)
280             target_stat = os.stat(target)
281             if self.compare_checksums and source_checksum:
282                 with open(target, "rb") as f:
283                     target_checksum = hashlib.md5(f.read()).hexdigest()
284                 perform_copy = target_checksum != source_checksum
285             elif source_stat.st_mtime != target_stat.st_mtime:
286                 perform_copy = True
287             elif source_stat.st_size != target_stat.st_size:
288                 perform_copy = True
289         else:
290             # Copy if it does not exist
291             perform_copy = True
292 
293         if perform_copy:
294             os.makedirs(os.path.dirname(target), exist_ok=True)
295             shutil.copy2(source, target)
296
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/src/documents/management/commands/document_exporter.py b/src/documents/management/commands/document_exporter.py
--- a/src/documents/management/commands/document_exporter.py
+++ b/src/documents/management/commands/document_exporter.py
@@ -18,6 +18,7 @@
 from documents.models import DocumentType
 from documents.models import SavedView
 from documents.models import SavedViewFilterRule
+from documents.models import StoragePath
 from documents.models import Tag
 from documents.models import UiSettings
 from documents.settings import EXPORTER_ARCHIVE_NAME
@@ -114,8 +115,8 @@
                 map(lambda f: os.path.abspath(os.path.join(root, f)), files),
             )
 
-        # 2. Create manifest, containing all correspondents, types, tags,
-        # documents and ui_settings
+        # 2. Create manifest, containing all correspondents, types, tags, storage paths
+        # comments, documents and ui_settings
         with transaction.atomic():
             manifest = json.loads(
                 serializers.serialize("json", Correspondent.objects.all()),
@@ -127,6 +128,10 @@
                 serializers.serialize("json", DocumentType.objects.all()),
             )
 
+            manifest += json.loads(
+                serializers.serialize("json", StoragePath.objects.all()),
+            )
+
             manifest += json.loads(
                 serializers.serialize("json", Comment.objects.all()),
             )

{"golden_diff": "diff --git a/src/documents/management/commands/document_exporter.py b/src/documents/management/commands/document_exporter.py\n--- a/src/documents/management/commands/document_exporter.py\n+++ b/src/documents/management/commands/document_exporter.py\n@@ -18,6 +18,7 @@\n from documents.models import DocumentType\n from documents.models import SavedView\n from documents.models import SavedViewFilterRule\n+from documents.models import StoragePath\n from documents.models import Tag\n from documents.models import UiSettings\n from documents.settings import EXPORTER_ARCHIVE_NAME\n@@ -114,8 +115,8 @@\n                 map(lambda f: os.path.abspath(os.path.join(root, f)), files),\n             )\n \n-        # 2. Create manifest, containing all correspondents, types, tags,\n-        # documents and ui_settings\n+        # 2. Create manifest, containing all correspondents, types, tags, storage paths\n+        # comments, documents and ui_settings\n         with transaction.atomic():\n             manifest = json.loads(\n                 serializers.serialize(\"json\", Correspondent.objects.all()),\n@@ -127,6 +128,10 @@\n                 serializers.serialize(\"json\", DocumentType.objects.all()),\n             )\n \n+            manifest += json.loads(\n+                serializers.serialize(\"json\", StoragePath.objects.all()),\n+            )\n+\n             manifest += json.loads(\n                 serializers.serialize(\"json\", Comment.objects.all()),\n             )\n", "issue": "[BUG] storage_path is missing in manifest.json\n### Description\n\nthere is no \"model\": \"documents.storage_path\" in manifest.json. I found it only in db.sqlite\n\n### Steps to reproduce\n\nopen export/manifest.json after `docker compose exec webserver document_exporter ../export -f`\n\n### Webserver logs\n\n_No response_\n\n### Paperless-ngx version\n\n1.8.0\n\n### Host OS\n\nRaspberry Pi 4/arm64\n\n### Installation method\n\nDocker - official image\n\n### Browser\n\n_No response_\n\n### Configuration changes\n\n_No response_\n\n### Other\n\n_No response_\n", "before_files": [{"content": "import hashlib\nimport json\nimport os\nimport shutil\nimport time\n\nimport tqdm\nfrom django.conf import settings\nfrom django.contrib.auth.models import Group\nfrom django.contrib.auth.models import User\nfrom django.core import serializers\nfrom django.core.management.base import BaseCommand\nfrom django.core.management.base import CommandError\nfrom django.db import transaction\nfrom documents.models import Comment\nfrom documents.models import Correspondent\nfrom documents.models import Document\nfrom documents.models import DocumentType\nfrom documents.models import SavedView\nfrom documents.models import SavedViewFilterRule\nfrom documents.models import Tag\nfrom documents.models import UiSettings\nfrom documents.settings import EXPORTER_ARCHIVE_NAME\nfrom documents.settings import EXPORTER_FILE_NAME\nfrom documents.settings import EXPORTER_THUMBNAIL_NAME\nfrom filelock import FileLock\nfrom paperless import version\nfrom paperless.db import GnuPG\nfrom paperless_mail.models import MailAccount\nfrom paperless_mail.models import MailRule\n\nfrom ...file_handling import delete_empty_directories\nfrom ...file_handling import generate_filename\n\n\nclass Command(BaseCommand):\n\n    help = \"\"\"\n        Decrypt and rename all files in our collection into a given target\n        directory.  And include a manifest file containing document data for\n        easy import.\n    \"\"\".replace(\n        \"    \",\n        \"\",\n    )\n\n    def add_arguments(self, parser):\n        parser.add_argument(\"target\")\n\n        parser.add_argument(\n            \"-c\",\n            \"--compare-checksums\",\n            default=False,\n            action=\"store_true\",\n            help=\"Compare file checksums when determining whether to export \"\n            \"a file or not. If not specified, file size and time \"\n            \"modified is used instead.\",\n        )\n\n        parser.add_argument(\n            \"-f\",\n            \"--use-filename-format\",\n            default=False,\n            action=\"store_true\",\n            help=\"Use PAPERLESS_FILENAME_FORMAT for storing files in the \"\n            \"export directory, if configured.\",\n        )\n\n        parser.add_argument(\n            \"-d\",\n            \"--delete\",\n            default=False,\n            action=\"store_true\",\n            help=\"After exporting, delete files in the export directory that \"\n            \"do not belong to the current export, such as files from \"\n            \"deleted documents.\",\n        )\n        parser.add_argument(\n            \"--no-progress-bar\",\n            default=False,\n            action=\"store_true\",\n            help=\"If set, the progress bar will not be shown\",\n        )\n\n    def __init__(self, *args, **kwargs):\n        BaseCommand.__init__(self, *args, **kwargs)\n        self.target = None\n        self.files_in_export_dir = []\n        self.exported_files = []\n        self.compare_checksums = False\n        self.use_filename_format = False\n        self.delete = False\n\n    def handle(self, *args, **options):\n\n        self.target = options[\"target\"]\n        self.compare_checksums = options[\"compare_checksums\"]\n        self.use_filename_format = options[\"use_filename_format\"]\n        self.delete = options[\"delete\"]\n\n        if not os.path.exists(self.target):\n            raise CommandError(\"That path doesn't exist\")\n\n        if not os.access(self.target, os.W_OK):\n            raise CommandError(\"That path doesn't appear to be writable\")\n\n        with FileLock(settings.MEDIA_LOCK):\n            self.dump(options[\"no_progress_bar\"])\n\n    def dump(self, progress_bar_disable=False):\n        # 1. Take a snapshot of what files exist in the current export folder\n        for root, dirs, files in os.walk(self.target):\n            self.files_in_export_dir.extend(\n                map(lambda f: os.path.abspath(os.path.join(root, f)), files),\n            )\n\n        # 2. Create manifest, containing all correspondents, types, tags,\n        # documents and ui_settings\n        with transaction.atomic():\n            manifest = json.loads(\n                serializers.serialize(\"json\", Correspondent.objects.all()),\n            )\n\n            manifest += json.loads(serializers.serialize(\"json\", Tag.objects.all()))\n\n            manifest += json.loads(\n                serializers.serialize(\"json\", DocumentType.objects.all()),\n            )\n\n            manifest += json.loads(\n                serializers.serialize(\"json\", Comment.objects.all()),\n            )\n\n            documents = Document.objects.order_by(\"id\")\n            document_map = {d.pk: d for d in documents}\n            document_manifest = json.loads(serializers.serialize(\"json\", documents))\n            manifest += document_manifest\n\n            manifest += json.loads(\n                serializers.serialize(\"json\", MailAccount.objects.all()),\n            )\n\n            manifest += json.loads(\n                serializers.serialize(\"json\", MailRule.objects.all()),\n            )\n\n            manifest += json.loads(\n                serializers.serialize(\"json\", SavedView.objects.all()),\n            )\n\n            manifest += json.loads(\n                serializers.serialize(\"json\", SavedViewFilterRule.objects.all()),\n            )\n\n            manifest += json.loads(serializers.serialize(\"json\", Group.objects.all()))\n\n            manifest += json.loads(serializers.serialize(\"json\", User.objects.all()))\n\n            manifest += json.loads(\n                serializers.serialize(\"json\", UiSettings.objects.all()),\n            )\n\n        # 3. Export files from each document\n        for index, document_dict in tqdm.tqdm(\n            enumerate(document_manifest),\n            total=len(document_manifest),\n            disable=progress_bar_disable,\n        ):\n            # 3.1. store files unencrypted\n            document_dict[\"fields\"][\"storage_type\"] = Document.STORAGE_TYPE_UNENCRYPTED\n\n            document = document_map[document_dict[\"pk\"]]\n\n            # 3.2. generate a unique filename\n            filename_counter = 0\n            while True:\n                if self.use_filename_format:\n                    base_name = generate_filename(\n                        document,\n                        counter=filename_counter,\n                        append_gpg=False,\n                    )\n                else:\n                    base_name = document.get_public_filename(counter=filename_counter)\n\n                if base_name not in self.exported_files:\n                    self.exported_files.append(base_name)\n                    break\n                else:\n                    filename_counter += 1\n\n            # 3.3. write filenames into manifest\n            original_name = base_name\n            original_target = os.path.join(self.target, original_name)\n            document_dict[EXPORTER_FILE_NAME] = original_name\n\n            thumbnail_name = base_name + \"-thumbnail.webp\"\n            thumbnail_target = os.path.join(self.target, thumbnail_name)\n            document_dict[EXPORTER_THUMBNAIL_NAME] = thumbnail_name\n\n            if document.has_archive_version:\n                archive_name = base_name + \"-archive.pdf\"\n                archive_target = os.path.join(self.target, archive_name)\n                document_dict[EXPORTER_ARCHIVE_NAME] = archive_name\n            else:\n                archive_target = None\n\n            # 3.4. write files to target folder\n            t = int(time.mktime(document.created.timetuple()))\n            if document.storage_type == Document.STORAGE_TYPE_GPG:\n\n                os.makedirs(os.path.dirname(original_target), exist_ok=True)\n                with open(original_target, \"wb\") as f:\n                    with document.source_file as out_file:\n                        f.write(GnuPG.decrypted(out_file))\n                        os.utime(original_target, times=(t, t))\n\n                os.makedirs(os.path.dirname(thumbnail_target), exist_ok=True)\n                with open(thumbnail_target, \"wb\") as f:\n                    with document.thumbnail_file as out_file:\n                        f.write(GnuPG.decrypted(out_file))\n                        os.utime(thumbnail_target, times=(t, t))\n\n                if archive_target:\n                    os.makedirs(os.path.dirname(archive_target), exist_ok=True)\n                    with open(archive_target, \"wb\") as f:\n                        with document.archive_path as out_file:\n                            f.write(GnuPG.decrypted(out_file))\n                            os.utime(archive_target, times=(t, t))\n            else:\n                self.check_and_copy(\n                    document.source_path,\n                    document.checksum,\n                    original_target,\n                )\n\n                self.check_and_copy(document.thumbnail_path, None, thumbnail_target)\n\n                if archive_target:\n                    self.check_and_copy(\n                        document.archive_path,\n                        document.archive_checksum,\n                        archive_target,\n                    )\n\n        # 4.1 write manifest to target folder\n        manifest_path = os.path.abspath(os.path.join(self.target, \"manifest.json\"))\n\n        with open(manifest_path, \"w\") as f:\n            json.dump(manifest, f, indent=2)\n\n        # 4.2 write version information to target folder\n        version_path = os.path.abspath(os.path.join(self.target, \"version.json\"))\n\n        with open(version_path, \"w\") as f:\n            json.dump({\"version\": version.__full_version_str__}, f, indent=2)\n\n        if self.delete:\n            # 5. Remove files which we did not explicitly export in this run\n\n            if manifest_path in self.files_in_export_dir:\n                self.files_in_export_dir.remove(manifest_path)\n\n            for f in self.files_in_export_dir:\n                os.remove(f)\n\n                delete_empty_directories(\n                    os.path.abspath(os.path.dirname(f)),\n                    os.path.abspath(self.target),\n                )\n\n    def check_and_copy(self, source, source_checksum, target):\n        if os.path.abspath(target) in self.files_in_export_dir:\n            self.files_in_export_dir.remove(os.path.abspath(target))\n\n        perform_copy = False\n\n        if os.path.exists(target):\n            source_stat = os.stat(source)\n            target_stat = os.stat(target)\n            if self.compare_checksums and source_checksum:\n                with open(target, \"rb\") as f:\n                    target_checksum = hashlib.md5(f.read()).hexdigest()\n                perform_copy = target_checksum != source_checksum\n            elif source_stat.st_mtime != target_stat.st_mtime:\n                perform_copy = True\n            elif source_stat.st_size != target_stat.st_size:\n                perform_copy = True\n        else:\n            # Copy if it does not exist\n            perform_copy = True\n\n        if perform_copy:\n            os.makedirs(os.path.dirname(target), exist_ok=True)\n            shutil.copy2(source, target)\n", "path": "src/documents/management/commands/document_exporter.py"}], "after_files": [{"content": "import hashlib\nimport json\nimport os\nimport shutil\nimport time\n\nimport tqdm\nfrom django.conf import settings\nfrom django.contrib.auth.models import Group\nfrom django.contrib.auth.models import User\nfrom django.core import serializers\nfrom django.core.management.base import BaseCommand\nfrom django.core.management.base import CommandError\nfrom django.db import transaction\nfrom documents.models import Comment\nfrom documents.models import Correspondent\nfrom documents.models import Document\nfrom documents.models import DocumentType\nfrom documents.models import SavedView\nfrom documents.models import SavedViewFilterRule\nfrom documents.models import StoragePath\nfrom documents.models import Tag\nfrom documents.models import UiSettings\nfrom documents.settings import EXPORTER_ARCHIVE_NAME\nfrom documents.settings import EXPORTER_FILE_NAME\nfrom documents.settings import EXPORTER_THUMBNAIL_NAME\nfrom filelock import FileLock\nfrom paperless import version\nfrom paperless.db import GnuPG\nfrom paperless_mail.models import MailAccount\nfrom paperless_mail.models import MailRule\n\nfrom ...file_handling import delete_empty_directories\nfrom ...file_handling import generate_filename\n\n\nclass Command(BaseCommand):\n\n    help = \"\"\"\n        Decrypt and rename all files in our collection into a given target\n        directory.  And include a manifest file containing document data for\n        easy import.\n    \"\"\".replace(\n        \"    \",\n        \"\",\n    )\n\n    def add_arguments(self, parser):\n        parser.add_argument(\"target\")\n\n        parser.add_argument(\n            \"-c\",\n            \"--compare-checksums\",\n            default=False,\n            action=\"store_true\",\n            help=\"Compare file checksums when determining whether to export \"\n            \"a file or not. If not specified, file size and time \"\n            \"modified is used instead.\",\n        )\n\n        parser.add_argument(\n            \"-f\",\n            \"--use-filename-format\",\n            default=False,\n            action=\"store_true\",\n            help=\"Use PAPERLESS_FILENAME_FORMAT for storing files in the \"\n            \"export directory, if configured.\",\n        )\n\n        parser.add_argument(\n            \"-d\",\n            \"--delete\",\n            default=False,\n            action=\"store_true\",\n            help=\"After exporting, delete files in the export directory that \"\n            \"do not belong to the current export, such as files from \"\n            \"deleted documents.\",\n        )\n        parser.add_argument(\n            \"--no-progress-bar\",\n            default=False,\n            action=\"store_true\",\n            help=\"If set, the progress bar will not be shown\",\n        )\n\n    def __init__(self, *args, **kwargs):\n        BaseCommand.__init__(self, *args, **kwargs)\n        self.target = None\n        self.files_in_export_dir = []\n        self.exported_files = []\n        self.compare_checksums = False\n        self.use_filename_format = False\n        self.delete = False\n\n    def handle(self, *args, **options):\n\n        self.target = options[\"target\"]\n        self.compare_checksums = options[\"compare_checksums\"]\n        self.use_filename_format = options[\"use_filename_format\"]\n        self.delete = options[\"delete\"]\n\n        if not os.path.exists(self.target):\n            raise CommandError(\"That path doesn't exist\")\n\n        if not os.access(self.target, os.W_OK):\n            raise CommandError(\"That path doesn't appear to be writable\")\n\n        with FileLock(settings.MEDIA_LOCK):\n            self.dump(options[\"no_progress_bar\"])\n\n    def dump(self, progress_bar_disable=False):\n        # 1. Take a snapshot of what files exist in the current export folder\n        for root, dirs, files in os.walk(self.target):\n            self.files_in_export_dir.extend(\n                map(lambda f: os.path.abspath(os.path.join(root, f)), files),\n            )\n\n        # 2. Create manifest, containing all correspondents, types, tags, storage paths\n        # comments, documents and ui_settings\n        with transaction.atomic():\n            manifest = json.loads(\n                serializers.serialize(\"json\", Correspondent.objects.all()),\n            )\n\n            manifest += json.loads(serializers.serialize(\"json\", Tag.objects.all()))\n\n            manifest += json.loads(\n                serializers.serialize(\"json\", DocumentType.objects.all()),\n            )\n\n            manifest += json.loads(\n                serializers.serialize(\"json\", StoragePath.objects.all()),\n            )\n\n            manifest += json.loads(\n                serializers.serialize(\"json\", Comment.objects.all()),\n            )\n\n            documents = Document.objects.order_by(\"id\")\n            document_map = {d.pk: d for d in documents}\n            document_manifest = json.loads(serializers.serialize(\"json\", documents))\n            manifest += document_manifest\n\n            manifest += json.loads(\n                serializers.serialize(\"json\", MailAccount.objects.all()),\n            )\n\n            manifest += json.loads(\n                serializers.serialize(\"json\", MailRule.objects.all()),\n            )\n\n            manifest += json.loads(\n                serializers.serialize(\"json\", SavedView.objects.all()),\n            )\n\n            manifest += json.loads(\n                serializers.serialize(\"json\", SavedViewFilterRule.objects.all()),\n            )\n\n            manifest += json.loads(serializers.serialize(\"json\", Group.objects.all()))\n\n            manifest += json.loads(serializers.serialize(\"json\", User.objects.all()))\n\n            manifest += json.loads(\n                serializers.serialize(\"json\", UiSettings.objects.all()),\n            )\n\n        # 3. Export files from each document\n        for index, document_dict in tqdm.tqdm(\n            enumerate(document_manifest),\n            total=len(document_manifest),\n            disable=progress_bar_disable,\n        ):\n            # 3.1. store files unencrypted\n            document_dict[\"fields\"][\"storage_type\"] = Document.STORAGE_TYPE_UNENCRYPTED\n\n            document = document_map[document_dict[\"pk\"]]\n\n            # 3.2. generate a unique filename\n            filename_counter = 0\n            while True:\n                if self.use_filename_format:\n                    base_name = generate_filename(\n                        document,\n                        counter=filename_counter,\n                        append_gpg=False,\n                    )\n                else:\n                    base_name = document.get_public_filename(counter=filename_counter)\n\n                if base_name not in self.exported_files:\n                    self.exported_files.append(base_name)\n                    break\n                else:\n                    filename_counter += 1\n\n            # 3.3. write filenames into manifest\n            original_name = base_name\n            original_target = os.path.join(self.target, original_name)\n            document_dict[EXPORTER_FILE_NAME] = original_name\n\n            thumbnail_name = base_name + \"-thumbnail.webp\"\n            thumbnail_target = os.path.join(self.target, thumbnail_name)\n            document_dict[EXPORTER_THUMBNAIL_NAME] = thumbnail_name\n\n            if document.has_archive_version:\n                archive_name = base_name + \"-archive.pdf\"\n                archive_target = os.path.join(self.target, archive_name)\n                document_dict[EXPORTER_ARCHIVE_NAME] = archive_name\n            else:\n                archive_target = None\n\n            # 3.4. write files to target folder\n            t = int(time.mktime(document.created.timetuple()))\n            if document.storage_type == Document.STORAGE_TYPE_GPG:\n\n                os.makedirs(os.path.dirname(original_target), exist_ok=True)\n                with open(original_target, \"wb\") as f:\n                    with document.source_file as out_file:\n                        f.write(GnuPG.decrypted(out_file))\n                        os.utime(original_target, times=(t, t))\n\n                os.makedirs(os.path.dirname(thumbnail_target), exist_ok=True)\n                with open(thumbnail_target, \"wb\") as f:\n                    with document.thumbnail_file as out_file:\n                        f.write(GnuPG.decrypted(out_file))\n                        os.utime(thumbnail_target, times=(t, t))\n\n                if archive_target:\n                    os.makedirs(os.path.dirname(archive_target), exist_ok=True)\n                    with open(archive_target, \"wb\") as f:\n                        with document.archive_path as out_file:\n                            f.write(GnuPG.decrypted(out_file))\n                            os.utime(archive_target, times=(t, t))\n            else:\n                self.check_and_copy(\n                    document.source_path,\n                    document.checksum,\n                    original_target,\n                )\n\n                self.check_and_copy(document.thumbnail_path, None, thumbnail_target)\n\n                if archive_target:\n                    self.check_and_copy(\n                        document.archive_path,\n                        document.archive_checksum,\n                        archive_target,\n                    )\n\n        # 4.1 write manifest to target folder\n        manifest_path = os.path.abspath(os.path.join(self.target, \"manifest.json\"))\n\n        with open(manifest_path, \"w\") as f:\n            json.dump(manifest, f, indent=2)\n\n        # 4.2 write version information to target folder\n        version_path = os.path.abspath(os.path.join(self.target, \"version.json\"))\n\n        with open(version_path, \"w\") as f:\n            json.dump({\"version\": version.__full_version_str__}, f, indent=2)\n\n        if self.delete:\n            # 5. Remove files which we did not explicitly export in this run\n\n            if manifest_path in self.files_in_export_dir:\n                self.files_in_export_dir.remove(manifest_path)\n\n            for f in self.files_in_export_dir:\n                os.remove(f)\n\n                delete_empty_directories(\n                    os.path.abspath(os.path.dirname(f)),\n                    os.path.abspath(self.target),\n                )\n\n    def check_and_copy(self, source, source_checksum, target):\n        if os.path.abspath(target) in self.files_in_export_dir:\n            self.files_in_export_dir.remove(os.path.abspath(target))\n\n        perform_copy = False\n\n        if os.path.exists(target):\n            source_stat = os.stat(source)\n            target_stat = os.stat(target)\n            if self.compare_checksums and source_checksum:\n                with open(target, \"rb\") as f:\n                    target_checksum = hashlib.md5(f.read()).hexdigest()\n                perform_copy = target_checksum != source_checksum\n            elif source_stat.st_mtime != target_stat.st_mtime:\n                perform_copy = True\n            elif source_stat.st_size != target_stat.st_size:\n                perform_copy = True\n        else:\n            # Copy if it does not exist\n            perform_copy = True\n\n        if perform_copy:\n            os.makedirs(os.path.dirname(target), exist_ok=True)\n            shutil.copy2(source, target)\n", "path": "src/documents/management/commands/document_exporter.py"}]}