We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
New submission cleanup could delete valid files of replies
## Description

In `securedrop/management/submissions.py`, the `find_disconnected_fs_submissions` returns any file found in the store that is not associated with a `Submission` record in the database. It should only return files that have neither a `Submission` nor `Reply` record, to avoid warning admins about and possibly deleting replies.

## Steps to Reproduce

- Run `make dev`
- In another shell:
  - Run `docker container ls` to get the dev container's ID
  - Run `docker exec -it "container-id" bash`
  - Run `./manage.py list-disconnected-fs-submissions`

## Expected Behavior

With a consistent database and filesystem, no disconnects should be reported. 

## Actual Behavior

The files of all replies in the database are reported as candidates for cleanup.

## Comments

The function simply needs to build a list of reply files as it does for submissions, and check the files under the store against the complete set of valid files.


--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `securedrop/management/submissions.py`
Content:
```
1 from __future__ import print_function
2 
3 import datetime
4 import os
5 import sys
6 import time
7 
8 from six.moves import input
9 
10 from db import db
11 from rm import secure_delete
12 from models import Source, Submission
13 from management import app_context
14 
15 
16 def find_disconnected_db_submissions(path):
17     submissions = db.session.query(Submission).order_by(Submission.id, Submission.filename).all()
18 
19     files_in_fs = {}
20     for directory, subdirs, files in os.walk(path):
21         for f in files:
22             files_in_fs[f] = os.path.abspath(os.path.join(directory, f))
23 
24     disconnected_submissions = [s for s in submissions if s.filename not in files_in_fs]
25 
26     return disconnected_submissions
27 
28 
29 def check_for_disconnected_db_submissions(args):
30     """
31     Check for Submission records whose files are missing.
32     """
33     with app_context():
34         disconnected = find_disconnected_db_submissions(args.store_dir)
35         if disconnected:
36             print(
37                 "There are submissions in the database with no corresponding files. "
38                 'Run "manage.py list-disconnected-db-submissions" for details.'
39             )
40         else:
41             print("No problems were found. All submissions' files are present.")
42 
43 
44 def list_disconnected_db_submissions(args):
45     """
46     List the IDs of Submission records whose files are missing.
47     """
48     with app_context():
49         disconnected_submissions = find_disconnected_db_submissions(args.store_dir)
50         if disconnected_submissions:
51             print(
52                 'Run "manage.py delete-disconnected-db-submissions" to delete these records.',
53                 file=sys.stderr
54             )
55         for s in disconnected_submissions:
56             print(s.id)
57 
58 
59 def delete_disconnected_db_submissions(args):
60     """
61     Delete Submission records whose files are missing.
62     """
63     with app_context():
64         disconnected_submissions = find_disconnected_db_submissions(args.store_dir)
65         ids = [s.id for s in disconnected_submissions]
66 
67         remove = args.force
68         if not args.force:
69             remove = input("Enter 'y' to delete all submissions missing files: ") == "y"
70         if remove:
71             print("Removing submission IDs {}...".format(ids))
72             db.session.query(Submission).filter(Submission.id.in_(ids)).delete(
73                 synchronize_session="fetch"
74             )
75             db.session.commit()
76         else:
77             print("Not removing disconnected submissions in database.")
78 
79 
80 def find_disconnected_fs_submissions(path):
81     submissions = Submission.query.order_by(Submission.id, Submission.filename).all()
82     files_in_db = {s.filename: s for s in submissions}
83 
84     files_in_fs = {}
85     for directory, subdirs, files in os.walk(path):
86         for f in files:
87             files_in_fs[f] = os.path.abspath(os.path.join(directory, f))
88 
89     disconnected_files = []
90     for f, p in files_in_fs.items():
91         if f not in files_in_db:
92             filesize = os.stat(p).st_size
93             disconnected_files.append((p, filesize))
94 
95     disconnected_files = [t[0] for t in sorted(disconnected_files, key=lambda t: t[1])]
96 
97     return disconnected_files
98 
99 
100 def check_for_disconnected_fs_submissions(args):
101     """
102     Check for files without a corresponding Submission record in the database.
103     """
104     with app_context():
105         disconnected = find_disconnected_fs_submissions(args.store_dir)
106         if disconnected:
107             print(
108                 "There are files in the submission area with no corresponding records in the "
109                 'database. Run "manage.py list-disconnected-fs-submissions" for details.'
110             )
111         else:
112             print("No unexpected files were found in the store.")
113 
114 
115 def list_disconnected_fs_submissions(args):
116     """
117     List files without a corresponding Submission record in the database.
118     """
119     with app_context():
120         disconnected_files = find_disconnected_fs_submissions(args.store_dir)
121         if disconnected_files:
122             print(
123                 'Run "manage.py delete-disconnected-fs-submissions" to delete these files.',
124                 file=sys.stderr
125             )
126         for f in disconnected_files:
127             print(f)
128 
129 
130 def delete_disconnected_fs_submissions(args):
131     """
132     Delete files without a corresponding Submission record in the database.
133     """
134     with app_context():
135         disconnected_files = find_disconnected_fs_submissions(args.store_dir)
136         bytes_deleted = 0
137         time_elapsed = 0.0
138         rate = 1.0
139         filecount = len(disconnected_files)
140         eta = 1.0
141         eta_msg = ""
142         for i, f in enumerate(disconnected_files, 1):
143             remove = args.force
144             if not args.force:
145                 remove = input("Enter 'y' to delete {}: ".format(f)) == "y"
146             if remove:
147                 filesize = os.stat(f).st_size
148                 if i > 1:
149                     eta = filesize / rate
150                     eta_msg = " (ETA to remove {:d} bytes: {:.0f}s )".format(filesize, eta)
151                 print("Securely removing file {}/{} {}{}...".format(i, filecount, f, eta_msg))
152                 start = time.time()
153                 secure_delete(f)
154                 file_elapsed = time.time() - start
155                 bytes_deleted += filesize
156                 time_elapsed += file_elapsed
157                 rate = bytes_deleted / time_elapsed
158                 print(
159                     "elapsed: {:.2f}s rate: {:.1f} MB/s overall rate: {:.1f} MB/s".format(
160                         file_elapsed, filesize / 1048576 / file_elapsed, rate / 1048576
161                     )
162                 )
163             else:
164                 print("Not removing {}.".format(f))
165 
166 
167 def were_there_submissions_today(args, context=None):
168     with context or app_context():
169         something = (
170             db.session.query(Source)
171             .filter(Source.last_updated > datetime.datetime.utcnow() - datetime.timedelta(hours=24))
172             .count()
173             > 0
174         )
175         count_file = os.path.join(args.data_root, "submissions_today.txt")
176         open(count_file, "w").write(something and "1" or "0")
177 
178 
179 def add_check_db_disconnect_parser(subps):
180     check_db_disconnect_subp = subps.add_parser(
181         "check-disconnected-db-submissions",
182         help="Check for submissions that exist in the database but not the filesystem.",
183     )
184     check_db_disconnect_subp.set_defaults(func=check_for_disconnected_db_submissions)
185 
186 
187 def add_check_fs_disconnect_parser(subps):
188     check_fs_disconnect_subp = subps.add_parser(
189         "check-disconnected-fs-submissions",
190         help="Check for submissions that exist in the filesystem but not in the database.",
191     )
192     check_fs_disconnect_subp.set_defaults(func=check_for_disconnected_fs_submissions)
193 
194 
195 def add_delete_db_disconnect_parser(subps):
196     delete_db_disconnect_subp = subps.add_parser(
197         "delete-disconnected-db-submissions",
198         help="Delete submissions that exist in the database but not the filesystem.",
199     )
200     delete_db_disconnect_subp.set_defaults(func=delete_disconnected_db_submissions)
201     delete_db_disconnect_subp.add_argument(
202         "--force", action="store_true", help="Do not ask for confirmation."
203     )
204 
205 
206 def add_delete_fs_disconnect_parser(subps):
207     delete_fs_disconnect_subp = subps.add_parser(
208         "delete-disconnected-fs-submissions",
209         help="Delete submissions that exist in the filesystem but not the database.",
210     )
211     delete_fs_disconnect_subp.set_defaults(func=delete_disconnected_fs_submissions)
212     delete_fs_disconnect_subp.add_argument(
213         "--force", action="store_true", help="Do not ask for confirmation."
214     )
215 
216 
217 def add_list_db_disconnect_parser(subps):
218     list_db_disconnect_subp = subps.add_parser(
219         "list-disconnected-db-submissions",
220         help="List submissions that exist in the database but not the filesystem.",
221     )
222     list_db_disconnect_subp.set_defaults(func=list_disconnected_db_submissions)
223 
224 
225 def add_list_fs_disconnect_parser(subps):
226     list_fs_disconnect_subp = subps.add_parser(
227         "list-disconnected-fs-submissions",
228         help="List submissions that exist in the filesystem but not the database.",
229     )
230     list_fs_disconnect_subp.set_defaults(func=list_disconnected_fs_submissions)
231 
232 
233 def add_were_there_submissions_today(subps):
234     parser = subps.add_parser(
235         "were-there-submissions-today",
236         help=("Update the file indicating " "whether submissions were received in the past 24h"),
237     )
238     parser.set_defaults(func=were_there_submissions_today)
239
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/securedrop/management/submissions.py b/securedrop/management/submissions.py
--- a/securedrop/management/submissions.py
+++ b/securedrop/management/submissions.py
@@ -9,11 +9,14 @@
 
 from db import db
 from rm import secure_delete
-from models import Source, Submission
+from models import Reply, Source, Submission
 from management import app_context
 
 
 def find_disconnected_db_submissions(path):
+    """
+    Finds Submission records whose file does not exist.
+    """
     submissions = db.session.query(Submission).order_by(Submission.id, Submission.filename).all()
 
     files_in_fs = {}
@@ -50,7 +53,7 @@
         if disconnected_submissions:
             print(
                 'Run "manage.py delete-disconnected-db-submissions" to delete these records.',
-                file=sys.stderr
+                file=sys.stderr,
             )
         for s in disconnected_submissions:
             print(s.id)
@@ -78,8 +81,14 @@
 
 
 def find_disconnected_fs_submissions(path):
+    """
+    Finds files in the store that lack a Submission or Reply record.
+    """
     submissions = Submission.query.order_by(Submission.id, Submission.filename).all()
-    files_in_db = {s.filename: s for s in submissions}
+    files_in_db = {s.filename: True for s in submissions}
+
+    replies = Reply.query.order_by(Reply.id, Reply.filename).all()
+    files_in_db.update({r.filename: True for r in replies})
 
     files_in_fs = {}
     for directory, subdirs, files in os.walk(path):
@@ -99,7 +108,7 @@
 
 def check_for_disconnected_fs_submissions(args):
     """
-    Check for files without a corresponding Submission record in the database.
+    Check for files without a corresponding Submission or Reply record in the database.
     """
     with app_context():
         disconnected = find_disconnected_fs_submissions(args.store_dir)
@@ -114,14 +123,14 @@
 
 def list_disconnected_fs_submissions(args):
     """
-    List files without a corresponding Submission record in the database.
+    List files without a corresponding Submission or Reply record in the database.
     """
     with app_context():
         disconnected_files = find_disconnected_fs_submissions(args.store_dir)
         if disconnected_files:
             print(
                 'Run "manage.py delete-disconnected-fs-submissions" to delete these files.',
-                file=sys.stderr
+                file=sys.stderr,
             )
         for f in disconnected_files:
             print(f)

{"golden_diff": "diff --git a/securedrop/management/submissions.py b/securedrop/management/submissions.py\n--- a/securedrop/management/submissions.py\n+++ b/securedrop/management/submissions.py\n@@ -9,11 +9,14 @@\n \n from db import db\n from rm import secure_delete\n-from models import Source, Submission\n+from models import Reply, Source, Submission\n from management import app_context\n \n \n def find_disconnected_db_submissions(path):\n+    \"\"\"\n+    Finds Submission records whose file does not exist.\n+    \"\"\"\n     submissions = db.session.query(Submission).order_by(Submission.id, Submission.filename).all()\n \n     files_in_fs = {}\n@@ -50,7 +53,7 @@\n         if disconnected_submissions:\n             print(\n                 'Run \"manage.py delete-disconnected-db-submissions\" to delete these records.',\n-                file=sys.stderr\n+                file=sys.stderr,\n             )\n         for s in disconnected_submissions:\n             print(s.id)\n@@ -78,8 +81,14 @@\n \n \n def find_disconnected_fs_submissions(path):\n+    \"\"\"\n+    Finds files in the store that lack a Submission or Reply record.\n+    \"\"\"\n     submissions = Submission.query.order_by(Submission.id, Submission.filename).all()\n-    files_in_db = {s.filename: s for s in submissions}\n+    files_in_db = {s.filename: True for s in submissions}\n+\n+    replies = Reply.query.order_by(Reply.id, Reply.filename).all()\n+    files_in_db.update({r.filename: True for r in replies})\n \n     files_in_fs = {}\n     for directory, subdirs, files in os.walk(path):\n@@ -99,7 +108,7 @@\n \n def check_for_disconnected_fs_submissions(args):\n     \"\"\"\n-    Check for files without a corresponding Submission record in the database.\n+    Check for files without a corresponding Submission or Reply record in the database.\n     \"\"\"\n     with app_context():\n         disconnected = find_disconnected_fs_submissions(args.store_dir)\n@@ -114,14 +123,14 @@\n \n def list_disconnected_fs_submissions(args):\n     \"\"\"\n-    List files without a corresponding Submission record in the database.\n+    List files without a corresponding Submission or Reply record in the database.\n     \"\"\"\n     with app_context():\n         disconnected_files = find_disconnected_fs_submissions(args.store_dir)\n         if disconnected_files:\n             print(\n                 'Run \"manage.py delete-disconnected-fs-submissions\" to delete these files.',\n-                file=sys.stderr\n+                file=sys.stderr,\n             )\n         for f in disconnected_files:\n             print(f)\n", "issue": "New submission cleanup could delete valid files of replies\n## Description\r\n\r\nIn `securedrop/management/submissions.py`, the `find_disconnected_fs_submissions` returns any file found in the store that is not associated with a `Submission` record in the database. It should only return files that have neither a `Submission` nor `Reply` record, to avoid warning admins about and possibly deleting replies.\r\n\r\n## Steps to Reproduce\r\n\r\n- Run `make dev`\r\n- In another shell:\r\n  - Run `docker container ls` to get the dev container's ID\r\n  - Run `docker exec -it \"container-id\" bash`\r\n  - Run `./manage.py list-disconnected-fs-submissions`\r\n\r\n## Expected Behavior\r\n\r\nWith a consistent database and filesystem, no disconnects should be reported. \r\n\r\n## Actual Behavior\r\n\r\nThe files of all replies in the database are reported as candidates for cleanup.\r\n\r\n## Comments\r\n\r\nThe function simply needs to build a list of reply files as it does for submissions, and check the files under the store against the complete set of valid files.\r\n\n", "before_files": [{"content": "from __future__ import print_function\n\nimport datetime\nimport os\nimport sys\nimport time\n\nfrom six.moves import input\n\nfrom db import db\nfrom rm import secure_delete\nfrom models import Source, Submission\nfrom management import app_context\n\n\ndef find_disconnected_db_submissions(path):\n    submissions = db.session.query(Submission).order_by(Submission.id, Submission.filename).all()\n\n    files_in_fs = {}\n    for directory, subdirs, files in os.walk(path):\n        for f in files:\n            files_in_fs[f] = os.path.abspath(os.path.join(directory, f))\n\n    disconnected_submissions = [s for s in submissions if s.filename not in files_in_fs]\n\n    return disconnected_submissions\n\n\ndef check_for_disconnected_db_submissions(args):\n    \"\"\"\n    Check for Submission records whose files are missing.\n    \"\"\"\n    with app_context():\n        disconnected = find_disconnected_db_submissions(args.store_dir)\n        if disconnected:\n            print(\n                \"There are submissions in the database with no corresponding files. \"\n                'Run \"manage.py list-disconnected-db-submissions\" for details.'\n            )\n        else:\n            print(\"No problems were found. All submissions' files are present.\")\n\n\ndef list_disconnected_db_submissions(args):\n    \"\"\"\n    List the IDs of Submission records whose files are missing.\n    \"\"\"\n    with app_context():\n        disconnected_submissions = find_disconnected_db_submissions(args.store_dir)\n        if disconnected_submissions:\n            print(\n                'Run \"manage.py delete-disconnected-db-submissions\" to delete these records.',\n                file=sys.stderr\n            )\n        for s in disconnected_submissions:\n            print(s.id)\n\n\ndef delete_disconnected_db_submissions(args):\n    \"\"\"\n    Delete Submission records whose files are missing.\n    \"\"\"\n    with app_context():\n        disconnected_submissions = find_disconnected_db_submissions(args.store_dir)\n        ids = [s.id for s in disconnected_submissions]\n\n        remove = args.force\n        if not args.force:\n            remove = input(\"Enter 'y' to delete all submissions missing files: \") == \"y\"\n        if remove:\n            print(\"Removing submission IDs {}...\".format(ids))\n            db.session.query(Submission).filter(Submission.id.in_(ids)).delete(\n                synchronize_session=\"fetch\"\n            )\n            db.session.commit()\n        else:\n            print(\"Not removing disconnected submissions in database.\")\n\n\ndef find_disconnected_fs_submissions(path):\n    submissions = Submission.query.order_by(Submission.id, Submission.filename).all()\n    files_in_db = {s.filename: s for s in submissions}\n\n    files_in_fs = {}\n    for directory, subdirs, files in os.walk(path):\n        for f in files:\n            files_in_fs[f] = os.path.abspath(os.path.join(directory, f))\n\n    disconnected_files = []\n    for f, p in files_in_fs.items():\n        if f not in files_in_db:\n            filesize = os.stat(p).st_size\n            disconnected_files.append((p, filesize))\n\n    disconnected_files = [t[0] for t in sorted(disconnected_files, key=lambda t: t[1])]\n\n    return disconnected_files\n\n\ndef check_for_disconnected_fs_submissions(args):\n    \"\"\"\n    Check for files without a corresponding Submission record in the database.\n    \"\"\"\n    with app_context():\n        disconnected = find_disconnected_fs_submissions(args.store_dir)\n        if disconnected:\n            print(\n                \"There are files in the submission area with no corresponding records in the \"\n                'database. Run \"manage.py list-disconnected-fs-submissions\" for details.'\n            )\n        else:\n            print(\"No unexpected files were found in the store.\")\n\n\ndef list_disconnected_fs_submissions(args):\n    \"\"\"\n    List files without a corresponding Submission record in the database.\n    \"\"\"\n    with app_context():\n        disconnected_files = find_disconnected_fs_submissions(args.store_dir)\n        if disconnected_files:\n            print(\n                'Run \"manage.py delete-disconnected-fs-submissions\" to delete these files.',\n                file=sys.stderr\n            )\n        for f in disconnected_files:\n            print(f)\n\n\ndef delete_disconnected_fs_submissions(args):\n    \"\"\"\n    Delete files without a corresponding Submission record in the database.\n    \"\"\"\n    with app_context():\n        disconnected_files = find_disconnected_fs_submissions(args.store_dir)\n        bytes_deleted = 0\n        time_elapsed = 0.0\n        rate = 1.0\n        filecount = len(disconnected_files)\n        eta = 1.0\n        eta_msg = \"\"\n        for i, f in enumerate(disconnected_files, 1):\n            remove = args.force\n            if not args.force:\n                remove = input(\"Enter 'y' to delete {}: \".format(f)) == \"y\"\n            if remove:\n                filesize = os.stat(f).st_size\n                if i > 1:\n                    eta = filesize / rate\n                    eta_msg = \" (ETA to remove {:d} bytes: {:.0f}s )\".format(filesize, eta)\n                print(\"Securely removing file {}/{} {}{}...\".format(i, filecount, f, eta_msg))\n                start = time.time()\n                secure_delete(f)\n                file_elapsed = time.time() - start\n                bytes_deleted += filesize\n                time_elapsed += file_elapsed\n                rate = bytes_deleted / time_elapsed\n                print(\n                    \"elapsed: {:.2f}s rate: {:.1f} MB/s overall rate: {:.1f} MB/s\".format(\n                        file_elapsed, filesize / 1048576 / file_elapsed, rate / 1048576\n                    )\n                )\n            else:\n                print(\"Not removing {}.\".format(f))\n\n\ndef were_there_submissions_today(args, context=None):\n    with context or app_context():\n        something = (\n            db.session.query(Source)\n            .filter(Source.last_updated > datetime.datetime.utcnow() - datetime.timedelta(hours=24))\n            .count()\n            > 0\n        )\n        count_file = os.path.join(args.data_root, \"submissions_today.txt\")\n        open(count_file, \"w\").write(something and \"1\" or \"0\")\n\n\ndef add_check_db_disconnect_parser(subps):\n    check_db_disconnect_subp = subps.add_parser(\n        \"check-disconnected-db-submissions\",\n        help=\"Check for submissions that exist in the database but not the filesystem.\",\n    )\n    check_db_disconnect_subp.set_defaults(func=check_for_disconnected_db_submissions)\n\n\ndef add_check_fs_disconnect_parser(subps):\n    check_fs_disconnect_subp = subps.add_parser(\n        \"check-disconnected-fs-submissions\",\n        help=\"Check for submissions that exist in the filesystem but not in the database.\",\n    )\n    check_fs_disconnect_subp.set_defaults(func=check_for_disconnected_fs_submissions)\n\n\ndef add_delete_db_disconnect_parser(subps):\n    delete_db_disconnect_subp = subps.add_parser(\n        \"delete-disconnected-db-submissions\",\n        help=\"Delete submissions that exist in the database but not the filesystem.\",\n    )\n    delete_db_disconnect_subp.set_defaults(func=delete_disconnected_db_submissions)\n    delete_db_disconnect_subp.add_argument(\n        \"--force\", action=\"store_true\", help=\"Do not ask for confirmation.\"\n    )\n\n\ndef add_delete_fs_disconnect_parser(subps):\n    delete_fs_disconnect_subp = subps.add_parser(\n        \"delete-disconnected-fs-submissions\",\n        help=\"Delete submissions that exist in the filesystem but not the database.\",\n    )\n    delete_fs_disconnect_subp.set_defaults(func=delete_disconnected_fs_submissions)\n    delete_fs_disconnect_subp.add_argument(\n        \"--force\", action=\"store_true\", help=\"Do not ask for confirmation.\"\n    )\n\n\ndef add_list_db_disconnect_parser(subps):\n    list_db_disconnect_subp = subps.add_parser(\n        \"list-disconnected-db-submissions\",\n        help=\"List submissions that exist in the database but not the filesystem.\",\n    )\n    list_db_disconnect_subp.set_defaults(func=list_disconnected_db_submissions)\n\n\ndef add_list_fs_disconnect_parser(subps):\n    list_fs_disconnect_subp = subps.add_parser(\n        \"list-disconnected-fs-submissions\",\n        help=\"List submissions that exist in the filesystem but not the database.\",\n    )\n    list_fs_disconnect_subp.set_defaults(func=list_disconnected_fs_submissions)\n\n\ndef add_were_there_submissions_today(subps):\n    parser = subps.add_parser(\n        \"were-there-submissions-today\",\n        help=(\"Update the file indicating \" \"whether submissions were received in the past 24h\"),\n    )\n    parser.set_defaults(func=were_there_submissions_today)\n", "path": "securedrop/management/submissions.py"}], "after_files": [{"content": "from __future__ import print_function\n\nimport datetime\nimport os\nimport sys\nimport time\n\nfrom six.moves import input\n\nfrom db import db\nfrom rm import secure_delete\nfrom models import Reply, Source, Submission\nfrom management import app_context\n\n\ndef find_disconnected_db_submissions(path):\n    \"\"\"\n    Finds Submission records whose file does not exist.\n    \"\"\"\n    submissions = db.session.query(Submission).order_by(Submission.id, Submission.filename).all()\n\n    files_in_fs = {}\n    for directory, subdirs, files in os.walk(path):\n        for f in files:\n            files_in_fs[f] = os.path.abspath(os.path.join(directory, f))\n\n    disconnected_submissions = [s for s in submissions if s.filename not in files_in_fs]\n\n    return disconnected_submissions\n\n\ndef check_for_disconnected_db_submissions(args):\n    \"\"\"\n    Check for Submission records whose files are missing.\n    \"\"\"\n    with app_context():\n        disconnected = find_disconnected_db_submissions(args.store_dir)\n        if disconnected:\n            print(\n                \"There are submissions in the database with no corresponding files. \"\n                'Run \"manage.py list-disconnected-db-submissions\" for details.'\n            )\n        else:\n            print(\"No problems were found. All submissions' files are present.\")\n\n\ndef list_disconnected_db_submissions(args):\n    \"\"\"\n    List the IDs of Submission records whose files are missing.\n    \"\"\"\n    with app_context():\n        disconnected_submissions = find_disconnected_db_submissions(args.store_dir)\n        if disconnected_submissions:\n            print(\n                'Run \"manage.py delete-disconnected-db-submissions\" to delete these records.',\n                file=sys.stderr,\n            )\n        for s in disconnected_submissions:\n            print(s.id)\n\n\ndef delete_disconnected_db_submissions(args):\n    \"\"\"\n    Delete Submission records whose files are missing.\n    \"\"\"\n    with app_context():\n        disconnected_submissions = find_disconnected_db_submissions(args.store_dir)\n        ids = [s.id for s in disconnected_submissions]\n\n        remove = args.force\n        if not args.force:\n            remove = input(\"Enter 'y' to delete all submissions missing files: \") == \"y\"\n        if remove:\n            print(\"Removing submission IDs {}...\".format(ids))\n            db.session.query(Submission).filter(Submission.id.in_(ids)).delete(\n                synchronize_session=\"fetch\"\n            )\n            db.session.commit()\n        else:\n            print(\"Not removing disconnected submissions in database.\")\n\n\ndef find_disconnected_fs_submissions(path):\n    \"\"\"\n    Finds files in the store that lack a Submission or Reply record.\n    \"\"\"\n    submissions = Submission.query.order_by(Submission.id, Submission.filename).all()\n    files_in_db = {s.filename: True for s in submissions}\n\n    replies = Reply.query.order_by(Reply.id, Reply.filename).all()\n    files_in_db.update({r.filename: True for r in replies})\n\n    files_in_fs = {}\n    for directory, subdirs, files in os.walk(path):\n        for f in files:\n            files_in_fs[f] = os.path.abspath(os.path.join(directory, f))\n\n    disconnected_files = []\n    for f, p in files_in_fs.items():\n        if f not in files_in_db:\n            filesize = os.stat(p).st_size\n            disconnected_files.append((p, filesize))\n\n    disconnected_files = [t[0] for t in sorted(disconnected_files, key=lambda t: t[1])]\n\n    return disconnected_files\n\n\ndef check_for_disconnected_fs_submissions(args):\n    \"\"\"\n    Check for files without a corresponding Submission or Reply record in the database.\n    \"\"\"\n    with app_context():\n        disconnected = find_disconnected_fs_submissions(args.store_dir)\n        if disconnected:\n            print(\n                \"There are files in the submission area with no corresponding records in the \"\n                'database. Run \"manage.py list-disconnected-fs-submissions\" for details.'\n            )\n        else:\n            print(\"No unexpected files were found in the store.\")\n\n\ndef list_disconnected_fs_submissions(args):\n    \"\"\"\n    List files without a corresponding Submission or Reply record in the database.\n    \"\"\"\n    with app_context():\n        disconnected_files = find_disconnected_fs_submissions(args.store_dir)\n        if disconnected_files:\n            print(\n                'Run \"manage.py delete-disconnected-fs-submissions\" to delete these files.',\n                file=sys.stderr,\n            )\n        for f in disconnected_files:\n            print(f)\n\n\ndef delete_disconnected_fs_submissions(args):\n    \"\"\"\n    Delete files without a corresponding Submission record in the database.\n    \"\"\"\n    with app_context():\n        disconnected_files = find_disconnected_fs_submissions(args.store_dir)\n        bytes_deleted = 0\n        time_elapsed = 0.0\n        rate = 1.0\n        filecount = len(disconnected_files)\n        eta = 1.0\n        eta_msg = \"\"\n        for i, f in enumerate(disconnected_files, 1):\n            remove = args.force\n            if not args.force:\n                remove = input(\"Enter 'y' to delete {}: \".format(f)) == \"y\"\n            if remove:\n                filesize = os.stat(f).st_size\n                if i > 1:\n                    eta = filesize / rate\n                    eta_msg = \" (ETA to remove {:d} bytes: {:.0f}s )\".format(filesize, eta)\n                print(\"Securely removing file {}/{} {}{}...\".format(i, filecount, f, eta_msg))\n                start = time.time()\n                secure_delete(f)\n                file_elapsed = time.time() - start\n                bytes_deleted += filesize\n                time_elapsed += file_elapsed\n                rate = bytes_deleted / time_elapsed\n                print(\n                    \"elapsed: {:.2f}s rate: {:.1f} MB/s overall rate: {:.1f} MB/s\".format(\n                        file_elapsed, filesize / 1048576 / file_elapsed, rate / 1048576\n                    )\n                )\n            else:\n                print(\"Not removing {}.\".format(f))\n\n\ndef were_there_submissions_today(args, context=None):\n    with context or app_context():\n        something = (\n            db.session.query(Source)\n            .filter(Source.last_updated > datetime.datetime.utcnow() - datetime.timedelta(hours=24))\n            .count()\n            > 0\n        )\n        count_file = os.path.join(args.data_root, \"submissions_today.txt\")\n        open(count_file, \"w\").write(something and \"1\" or \"0\")\n\n\ndef add_check_db_disconnect_parser(subps):\n    check_db_disconnect_subp = subps.add_parser(\n        \"check-disconnected-db-submissions\",\n        help=\"Check for submissions that exist in the database but not the filesystem.\",\n    )\n    check_db_disconnect_subp.set_defaults(func=check_for_disconnected_db_submissions)\n\n\ndef add_check_fs_disconnect_parser(subps):\n    check_fs_disconnect_subp = subps.add_parser(\n        \"check-disconnected-fs-submissions\",\n        help=\"Check for submissions that exist in the filesystem but not in the database.\",\n    )\n    check_fs_disconnect_subp.set_defaults(func=check_for_disconnected_fs_submissions)\n\n\ndef add_delete_db_disconnect_parser(subps):\n    delete_db_disconnect_subp = subps.add_parser(\n        \"delete-disconnected-db-submissions\",\n        help=\"Delete submissions that exist in the database but not the filesystem.\",\n    )\n    delete_db_disconnect_subp.set_defaults(func=delete_disconnected_db_submissions)\n    delete_db_disconnect_subp.add_argument(\n        \"--force\", action=\"store_true\", help=\"Do not ask for confirmation.\"\n    )\n\n\ndef add_delete_fs_disconnect_parser(subps):\n    delete_fs_disconnect_subp = subps.add_parser(\n        \"delete-disconnected-fs-submissions\",\n        help=\"Delete submissions that exist in the filesystem but not the database.\",\n    )\n    delete_fs_disconnect_subp.set_defaults(func=delete_disconnected_fs_submissions)\n    delete_fs_disconnect_subp.add_argument(\n        \"--force\", action=\"store_true\", help=\"Do not ask for confirmation.\"\n    )\n\n\ndef add_list_db_disconnect_parser(subps):\n    list_db_disconnect_subp = subps.add_parser(\n        \"list-disconnected-db-submissions\",\n        help=\"List submissions that exist in the database but not the filesystem.\",\n    )\n    list_db_disconnect_subp.set_defaults(func=list_disconnected_db_submissions)\n\n\ndef add_list_fs_disconnect_parser(subps):\n    list_fs_disconnect_subp = subps.add_parser(\n        \"list-disconnected-fs-submissions\",\n        help=\"List submissions that exist in the filesystem but not the database.\",\n    )\n    list_fs_disconnect_subp.set_defaults(func=list_disconnected_fs_submissions)\n\n\ndef add_were_there_submissions_today(subps):\n    parser = subps.add_parser(\n        \"were-there-submissions-today\",\n        help=(\"Update the file indicating \" \"whether submissions were received in the past 24h\"),\n    )\n    parser.set_defaults(func=were_there_submissions_today)\n", "path": "securedrop/management/submissions.py"}]}