We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
`yield_checksumfiles` function doesn't handle `ChecksumFiles` with file paths as their `name` correctly
To reproduce, run [this management script](https://gist.github.com/mvandenburgh/6c9b284b6bf2b684b43236fe0da35a93) and then try to view the `Raster` that it creates. The tile requests will fail with 500 errors with errors about wrong file paths in the server log.

I came across this bug when working on Danesfield. On that project we encode the file path of a given `ChecksumFile` inside the `name` column - for example, `foo/bar/foobar.txt` represents a file hierarchy of
```
foo/
  bar/
    foobar.txt
````

[This line](https://github.com/ResonantGeoData/ResonantGeoData/blob/main/django-rgd/rgd/models/file.py#L290) in RGD is incompatible with this approach, since it assumes the ChecksumFile name is a flat file name instead of a file path. For example, if we have a ChecksumFile with the name `foo.tiff`, the current code would work:
```
path = '/tmp/rgd/file_cache/foo.tiff'
with yield_checksumfiles([self], path.parent):
    yield path
```
Note that `path.parent` would be `/tmp/rgd/file_cache/foo.tiff`, which is correct. But if the file name is `a/b/c/foo.tiff`:
```
path = '/tmp/rgd/file_cache/a/b/c/foo.tiff'
with yield_checksumfiles([self], path.parent):
    yield path
```
the call to `path.parent` would evaluate to `/tmp/rgd/file_cache/a/b/c`, which is incorrect and leads to 500 errors since the server can't find the files in that location.

--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `django-rgd/rgd/models/file.py`
Content:
```
1 from contextlib import contextmanager
2 import logging
3 import os
4 from pathlib import Path
5 from urllib.error import URLError
6 from urllib.parse import urlparse
7 
8 from crum import get_current_user
9 from django.conf import settings
10 from django.contrib.gis.db import models
11 from django_extensions.db.models import TimeStampedModel
12 from rgd.utility import (
13     _link_url,
14     clean_file_cache,
15     compute_checksum_file_field,
16     compute_checksum_url,
17     compute_hash,
18     download_field_file_to_local_path,
19     download_url_file_to_local_path,
20     get_cache_dir,
21     get_file_lock,
22     patch_internal_presign,
23     precheck_fuse,
24     safe_urlopen,
25     url_file_to_fuse_path,
26     uuid_prefix_filename,
27 )
28 from s3_file_field import S3FileField
29 
30 from .. import tasks
31 from .collection import Collection
32 from .fileset import FileSet
33 from .mixins import TaskEventMixin
34 
35 logger = logging.getLogger(__name__)
36 
37 
38 class FileSourceType(models.IntegerChoices):
39     FILE_FIELD = 1, 'FileField'
40     URL = 2, 'URL'
41 
42 
43 class ChecksumFile(TimeStampedModel, TaskEventMixin):
44     """The main class for user-uploaded files.
45 
46     This has support for manually uploading files or specifying a URL to a file
47     (for example in an existing S3 bucket). This broadly supports ``http<s>://``
48     URLs to file resources as well as ``s3://`` as long as the node the app is
49     running on is provisioned to access that S3 bucket.
50 
51     """
52 
53     name = models.CharField(max_length=1000, blank=True)
54     description = models.TextField(null=True, blank=True)
55     checksum = models.CharField(max_length=128)  # sha512
56     validate_checksum = models.BooleanField(
57         default=False
58     )  # a flag to validate the checksum against the saved checksum
59     last_validation = models.BooleanField(default=True)
60     collection = models.ForeignKey(
61         Collection,
62         on_delete=models.SET_NULL,
63         related_name='%(class)ss',
64         related_query_name='%(class)ss',
65         null=True,
66         blank=True,
67     )
68     created_by = models.ForeignKey(
69         settings.AUTH_USER_MODEL, null=True, blank=True, default=None, on_delete=models.SET_NULL
70     )
71     file_set = models.ForeignKey(FileSet, null=True, blank=True, on_delete=models.SET_NULL)
72 
73     type = models.IntegerField(choices=FileSourceType.choices, default=FileSourceType.FILE_FIELD)
74     file = S3FileField(null=True, blank=True, upload_to=uuid_prefix_filename)
75     url = models.TextField(null=True, blank=True)
76 
77     task_funcs = (tasks.task_checksum_file_post_save,)
78 
79     class Meta:
80         constraints = [
81             models.CheckConstraint(
82                 name='%(app_label)s_%(class)s_file_source_value_matches_type',
83                 check=(
84                     models.Q(
85                         models.Q(type=FileSourceType.FILE_FIELD, file__regex=r'.+')
86                         & models.Q(models.Q(url__in=['', None]) | models.Q(url__isnull=True))
87                     )
88                     | models.Q(
89                         models.Q(type=FileSourceType.URL)
90                         & models.Q(models.Q(url__isnull=False) & models.Q(url__regex=r'.+'))
91                         & models.Q(models.Q(file__in=['', None]) | models.Q(file__isnull=True))
92                     )
93                 ),
94             ),
95             models.UniqueConstraint(
96                 fields=['file_set', 'name'],
97                 name='unique_name',
98             ),
99             models.UniqueConstraint(
100                 fields=['collection', 'url'],
101                 name='unique_url_collection',
102                 condition=models.Q(type=FileSourceType.URL),  # Only apply if URL file
103             ),
104         ]
105 
106     @property
107     def basename(self):
108         return os.path.basename(self.name)
109 
110     @property
111     def size(self):
112         # Ensure safe check of self.file
113         try:
114             return self.file.size
115         except ValueError:
116             return None
117 
118     def get_checksum(self):
119         """Compute a new checksum without saving it."""
120         if self.type == FileSourceType.FILE_FIELD:
121             return compute_checksum_file_field(self.file)
122         elif self.type == FileSourceType.URL:
123             parsed = urlparse(self.url)
124             if parsed.scheme in ['https', 'http']:
125                 return compute_checksum_url(self.url)
126             else:
127                 with self.yield_local_path() as path:
128                     with open(path, 'rb') as f:
129                         return compute_hash(f)
130         else:
131             raise NotImplementedError(f'Type ({self.type}) not supported.')
132 
133     def update_checksum(self):
134         self.checksum = self.get_checksum()
135         # Simple update save - not full save
136         super(ChecksumFile, self).save(
137             update_fields=[
138                 'checksum',
139             ]
140         )
141 
142     def validate(self):
143         previous = self.checksum
144         self.update_checksum()
145         self.last_validation = self.checksum == previous
146         # Simple update save - not full save
147         super(ChecksumFile, self).save(
148             update_fields=[
149                 'last_validation',
150             ]
151         )
152         return self.last_validation
153 
154     def post_save_job(self):
155         if not self.checksum or self.validate_checksum:
156             if self.validate_checksum:
157                 self.validate()
158             else:
159                 self.update_checksum()
160             # Reset the user flags
161             self.validate_checksum = False
162             # Simple update save - not full save
163             self.save(
164                 update_fields=[
165                     'checksum',
166                     'last_validation',
167                     'validate_checksum',
168                 ]
169             )
170 
171     def save(self, *args, **kwargs):
172         if not self.name:
173             if self.type == FileSourceType.FILE_FIELD and self.file.name:
174                 self.name = os.path.basename(self.file.name)
175             elif self.type == FileSourceType.URL:
176                 parsed = urlparse(self.url)
177                 if parsed.scheme in ['https', 'http']:
178                     try:
179                         with safe_urlopen(self.url) as r:
180                             self.name = r.info().get_filename()
181                     except (AttributeError, ValueError, URLError):
182                         pass
183                 if not self.name:
184                     # Fallback
185                     self.name = os.path.basename(urlparse(self.url).path)
186         # Handle `created_by` User
187         user = get_current_user()
188         if user and not user.pk:
189             user = None
190         if not self.pk:
191             self.created_by = user
192         # Must save the model with the file before accessing it for the checksum
193         super(ChecksumFile, self).save(*args, **kwargs)
194 
195     def download_to_local_path(self, directory: str = None):
196         """Forcibly download this file to a directory on disk.
197 
198         Cleanup must be handled by caller.
199 
200         This will handle locking to prevent multiple processes/threads
201         from trying to download the file at the same time -- only one thread
202         or process will perform the download and the rest will yield its
203         result.
204 
205         """
206         if directory is None:
207             dest_path = self.get_cache_path()
208         else:
209             dest_path = Path(directory, self.name)
210         dest_path.parent.mkdir(parents=True, exist_ok=True)
211         # Thread/process safe locking for file access
212         lock = get_file_lock(dest_path)
213 
214         with lock:  # TODO: handle timeouts in condition
215             if dest_path.exists() and dest_path.stat().st_size > 0:
216                 # File already exists (is cached)
217                 logger.debug(f'Found cached file ({self.pk}) at: {dest_path}')
218                 # Touch the file so that it moves to the top of the priority list
219                 # when cleaning.
220                 dest_path.touch()
221                 return dest_path
222             else:
223                 logger.debug(f'Downloading file ({self.pk}) to: {dest_path}')
224                 # If downloading to the cache, clean to achieve available free space
225                 if get_cache_dir() in dest_path.parents:
226                     clean_file_cache()
227                 # TODO: handle if these fail (e.g. bad S3 credentials)
228                 if self.type == FileSourceType.FILE_FIELD:
229                     return download_field_file_to_local_path(self.file, dest_path)
230                 elif self.type == FileSourceType.URL:
231                     return download_url_file_to_local_path(self.url, dest_path)
232 
233     def get_cache_path(self):
234         """Generate a predetermined path in the cache directory.
235 
236         This will use the associated FileSet's cache path if this resource
237         has a file_set, otherwise it will place in the top of the cache
238         directory.
239 
240         """
241         if self.file_set is None:
242             # If no file_set, place in the main cache directory
243             directory = get_cache_dir() / f'f-{self.pk}'
244             directory.mkdir(parents=True, exist_ok=True)
245         else:
246             directory = self.file_set.get_cache_path()
247         return directory / f'{self.name}'
248 
249     @contextmanager
250     def yield_local_path(self, try_fuse: bool = True, yield_file_set: bool = False):
251         """Create a local path for this file and all other files in its file_set.
252 
253         This will first attempt to use httpfs to FUSE mount the file's URL if
254         and only if the file does not belong to a FileSet. FUSE with multiple
255         files in a FileSet is not yet supported.
256 
257         If FUSE is unavailable, this will fallback to downloading the entire
258         file (and the other files in this item's FileSet) to local storage.
259 
260         Parameters
261         ----------
262         try_fuse : bool
263             Try to use the FUSE interface. If false, use VSI or download to
264             local storage.
265 
266         yield_file_set : bool
267             Yield all of the files in this file's file_set if available.
268 
269         """
270         # TODO: fix FUSE to handle adjacent files
271         if (
272             self.file_set is None
273             and try_fuse
274             and self.type == FileSourceType.URL
275             and precheck_fuse(self.get_url())
276         ):
277             yield url_file_to_fuse_path(self.get_url(internal=True))
278             return
279         # Fallback to loading entire file locally - this uses `get_temp_path`
280         logger.debug('`yield_local_path` falling back to downloading entire file to local storage.')
281         path = self.get_cache_path()
282         if yield_file_set and self.file_set:
283             # NOTE: This is messy and should be improved but it ensures the directory remains locked
284             with self.file_set.yield_all_to_local_path() as _:
285                 yield path
286             return
287         # Not in file_set. Download to cache dir
288         from .utils import yield_checksumfiles
289 
290         with yield_checksumfiles([self], path.parent):
291             yield path
292 
293     def get_url(self, internal: bool = False):
294         """Get the URL of the stored resource.
295 
296         Parameters
297         ----------
298         internal : bool
299             In most cases this URL will be accessible from anywhere. In some
300             cases, this URL will only be accessible from within the container.
301             This flag is for use with internal processes to make sure the host
302             is correctly set to ``minio`` when needed. See
303             ``patch_internal_presign`` for more details.
304 
305         """
306         if self.type == FileSourceType.FILE_FIELD:
307             if internal:
308                 with patch_internal_presign(self.file):
309                     return self.file.url
310             else:
311                 return self.file.url
312         elif self.type == FileSourceType.URL:
313             return self.url
314 
315     def data_link(self):
316         return _link_url(self, 'get_url')
317 
318     data_link.allow_tags = True
319
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/django-rgd/rgd/models/file.py b/django-rgd/rgd/models/file.py
--- a/django-rgd/rgd/models/file.py
+++ b/django-rgd/rgd/models/file.py
@@ -230,7 +230,7 @@
                 elif self.type == FileSourceType.URL:
                     return download_url_file_to_local_path(self.url, dest_path)
 
-    def get_cache_path(self):
+    def get_cache_path(self, root: bool = False):
         """Generate a predetermined path in the cache directory.
 
         This will use the associated FileSet's cache path if this resource
@@ -244,6 +244,8 @@
             directory.mkdir(parents=True, exist_ok=True)
         else:
             directory = self.file_set.get_cache_path()
+        if root:
+            return directory
         return directory / f'{self.name}'
 
     @contextmanager
@@ -278,6 +280,7 @@
             return
         # Fallback to loading entire file locally - this uses `get_temp_path`
         logger.debug('`yield_local_path` falling back to downloading entire file to local storage.')
+        root = self.get_cache_path(root=True)
         path = self.get_cache_path()
         if yield_file_set and self.file_set:
             # NOTE: This is messy and should be improved but it ensures the directory remains locked
@@ -287,7 +290,7 @@
         # Not in file_set. Download to cache dir
         from .utils import yield_checksumfiles
 
-        with yield_checksumfiles([self], path.parent):
+        with yield_checksumfiles([self], root):
             yield path
 
     def get_url(self, internal: bool = False):

{"golden_diff": "diff --git a/django-rgd/rgd/models/file.py b/django-rgd/rgd/models/file.py\n--- a/django-rgd/rgd/models/file.py\n+++ b/django-rgd/rgd/models/file.py\n@@ -230,7 +230,7 @@\n                 elif self.type == FileSourceType.URL:\n                     return download_url_file_to_local_path(self.url, dest_path)\n \n-    def get_cache_path(self):\n+    def get_cache_path(self, root: bool = False):\n         \"\"\"Generate a predetermined path in the cache directory.\n \n         This will use the associated FileSet's cache path if this resource\n@@ -244,6 +244,8 @@\n             directory.mkdir(parents=True, exist_ok=True)\n         else:\n             directory = self.file_set.get_cache_path()\n+        if root:\n+            return directory\n         return directory / f'{self.name}'\n \n     @contextmanager\n@@ -278,6 +280,7 @@\n             return\n         # Fallback to loading entire file locally - this uses `get_temp_path`\n         logger.debug('`yield_local_path` falling back to downloading entire file to local storage.')\n+        root = self.get_cache_path(root=True)\n         path = self.get_cache_path()\n         if yield_file_set and self.file_set:\n             # NOTE: This is messy and should be improved but it ensures the directory remains locked\n@@ -287,7 +290,7 @@\n         # Not in file_set. Download to cache dir\n         from .utils import yield_checksumfiles\n \n-        with yield_checksumfiles([self], path.parent):\n+        with yield_checksumfiles([self], root):\n             yield path\n \n     def get_url(self, internal: bool = False):\n", "issue": "`yield_checksumfiles` function doesn't handle `ChecksumFiles` with file paths as their `name` correctly\nTo reproduce, run [this management script](https://gist.github.com/mvandenburgh/6c9b284b6bf2b684b43236fe0da35a93) and then try to view the `Raster` that it creates. The tile requests will fail with 500 errors with errors about wrong file paths in the server log.\r\n\r\nI came across this bug when working on Danesfield. On that project we encode the file path of a given `ChecksumFile` inside the `name` column - for example, `foo/bar/foobar.txt` represents a file hierarchy of\r\n```\r\nfoo/\r\n  bar/\r\n    foobar.txt\r\n````\r\n\r\n[This line](https://github.com/ResonantGeoData/ResonantGeoData/blob/main/django-rgd/rgd/models/file.py#L290) in RGD is incompatible with this approach, since it assumes the ChecksumFile name is a flat file name instead of a file path. For example, if we have a ChecksumFile with the name `foo.tiff`, the current code would work:\r\n```\r\npath = '/tmp/rgd/file_cache/foo.tiff'\r\nwith yield_checksumfiles([self], path.parent):\r\n    yield path\r\n```\r\nNote that `path.parent` would be `/tmp/rgd/file_cache/foo.tiff`, which is correct. But if the file name is `a/b/c/foo.tiff`:\r\n```\r\npath = '/tmp/rgd/file_cache/a/b/c/foo.tiff'\r\nwith yield_checksumfiles([self], path.parent):\r\n    yield path\r\n```\r\nthe call to `path.parent` would evaluate to `/tmp/rgd/file_cache/a/b/c`, which is incorrect and leads to 500 errors since the server can't find the files in that location.\n", "before_files": [{"content": "from contextlib import contextmanager\nimport logging\nimport os\nfrom pathlib import Path\nfrom urllib.error import URLError\nfrom urllib.parse import urlparse\n\nfrom crum import get_current_user\nfrom django.conf import settings\nfrom django.contrib.gis.db import models\nfrom django_extensions.db.models import TimeStampedModel\nfrom rgd.utility import (\n    _link_url,\n    clean_file_cache,\n    compute_checksum_file_field,\n    compute_checksum_url,\n    compute_hash,\n    download_field_file_to_local_path,\n    download_url_file_to_local_path,\n    get_cache_dir,\n    get_file_lock,\n    patch_internal_presign,\n    precheck_fuse,\n    safe_urlopen,\n    url_file_to_fuse_path,\n    uuid_prefix_filename,\n)\nfrom s3_file_field import S3FileField\n\nfrom .. import tasks\nfrom .collection import Collection\nfrom .fileset import FileSet\nfrom .mixins import TaskEventMixin\n\nlogger = logging.getLogger(__name__)\n\n\nclass FileSourceType(models.IntegerChoices):\n    FILE_FIELD = 1, 'FileField'\n    URL = 2, 'URL'\n\n\nclass ChecksumFile(TimeStampedModel, TaskEventMixin):\n    \"\"\"The main class for user-uploaded files.\n\n    This has support for manually uploading files or specifying a URL to a file\n    (for example in an existing S3 bucket). This broadly supports ``http<s>://``\n    URLs to file resources as well as ``s3://`` as long as the node the app is\n    running on is provisioned to access that S3 bucket.\n\n    \"\"\"\n\n    name = models.CharField(max_length=1000, blank=True)\n    description = models.TextField(null=True, blank=True)\n    checksum = models.CharField(max_length=128)  # sha512\n    validate_checksum = models.BooleanField(\n        default=False\n    )  # a flag to validate the checksum against the saved checksum\n    last_validation = models.BooleanField(default=True)\n    collection = models.ForeignKey(\n        Collection,\n        on_delete=models.SET_NULL,\n        related_name='%(class)ss',\n        related_query_name='%(class)ss',\n        null=True,\n        blank=True,\n    )\n    created_by = models.ForeignKey(\n        settings.AUTH_USER_MODEL, null=True, blank=True, default=None, on_delete=models.SET_NULL\n    )\n    file_set = models.ForeignKey(FileSet, null=True, blank=True, on_delete=models.SET_NULL)\n\n    type = models.IntegerField(choices=FileSourceType.choices, default=FileSourceType.FILE_FIELD)\n    file = S3FileField(null=True, blank=True, upload_to=uuid_prefix_filename)\n    url = models.TextField(null=True, blank=True)\n\n    task_funcs = (tasks.task_checksum_file_post_save,)\n\n    class Meta:\n        constraints = [\n            models.CheckConstraint(\n                name='%(app_label)s_%(class)s_file_source_value_matches_type',\n                check=(\n                    models.Q(\n                        models.Q(type=FileSourceType.FILE_FIELD, file__regex=r'.+')\n                        & models.Q(models.Q(url__in=['', None]) | models.Q(url__isnull=True))\n                    )\n                    | models.Q(\n                        models.Q(type=FileSourceType.URL)\n                        & models.Q(models.Q(url__isnull=False) & models.Q(url__regex=r'.+'))\n                        & models.Q(models.Q(file__in=['', None]) | models.Q(file__isnull=True))\n                    )\n                ),\n            ),\n            models.UniqueConstraint(\n                fields=['file_set', 'name'],\n                name='unique_name',\n            ),\n            models.UniqueConstraint(\n                fields=['collection', 'url'],\n                name='unique_url_collection',\n                condition=models.Q(type=FileSourceType.URL),  # Only apply if URL file\n            ),\n        ]\n\n    @property\n    def basename(self):\n        return os.path.basename(self.name)\n\n    @property\n    def size(self):\n        # Ensure safe check of self.file\n        try:\n            return self.file.size\n        except ValueError:\n            return None\n\n    def get_checksum(self):\n        \"\"\"Compute a new checksum without saving it.\"\"\"\n        if self.type == FileSourceType.FILE_FIELD:\n            return compute_checksum_file_field(self.file)\n        elif self.type == FileSourceType.URL:\n            parsed = urlparse(self.url)\n            if parsed.scheme in ['https', 'http']:\n                return compute_checksum_url(self.url)\n            else:\n                with self.yield_local_path() as path:\n                    with open(path, 'rb') as f:\n                        return compute_hash(f)\n        else:\n            raise NotImplementedError(f'Type ({self.type}) not supported.')\n\n    def update_checksum(self):\n        self.checksum = self.get_checksum()\n        # Simple update save - not full save\n        super(ChecksumFile, self).save(\n            update_fields=[\n                'checksum',\n            ]\n        )\n\n    def validate(self):\n        previous = self.checksum\n        self.update_checksum()\n        self.last_validation = self.checksum == previous\n        # Simple update save - not full save\n        super(ChecksumFile, self).save(\n            update_fields=[\n                'last_validation',\n            ]\n        )\n        return self.last_validation\n\n    def post_save_job(self):\n        if not self.checksum or self.validate_checksum:\n            if self.validate_checksum:\n                self.validate()\n            else:\n                self.update_checksum()\n            # Reset the user flags\n            self.validate_checksum = False\n            # Simple update save - not full save\n            self.save(\n                update_fields=[\n                    'checksum',\n                    'last_validation',\n                    'validate_checksum',\n                ]\n            )\n\n    def save(self, *args, **kwargs):\n        if not self.name:\n            if self.type == FileSourceType.FILE_FIELD and self.file.name:\n                self.name = os.path.basename(self.file.name)\n            elif self.type == FileSourceType.URL:\n                parsed = urlparse(self.url)\n                if parsed.scheme in ['https', 'http']:\n                    try:\n                        with safe_urlopen(self.url) as r:\n                            self.name = r.info().get_filename()\n                    except (AttributeError, ValueError, URLError):\n                        pass\n                if not self.name:\n                    # Fallback\n                    self.name = os.path.basename(urlparse(self.url).path)\n        # Handle `created_by` User\n        user = get_current_user()\n        if user and not user.pk:\n            user = None\n        if not self.pk:\n            self.created_by = user\n        # Must save the model with the file before accessing it for the checksum\n        super(ChecksumFile, self).save(*args, **kwargs)\n\n    def download_to_local_path(self, directory: str = None):\n        \"\"\"Forcibly download this file to a directory on disk.\n\n        Cleanup must be handled by caller.\n\n        This will handle locking to prevent multiple processes/threads\n        from trying to download the file at the same time -- only one thread\n        or process will perform the download and the rest will yield its\n        result.\n\n        \"\"\"\n        if directory is None:\n            dest_path = self.get_cache_path()\n        else:\n            dest_path = Path(directory, self.name)\n        dest_path.parent.mkdir(parents=True, exist_ok=True)\n        # Thread/process safe locking for file access\n        lock = get_file_lock(dest_path)\n\n        with lock:  # TODO: handle timeouts in condition\n            if dest_path.exists() and dest_path.stat().st_size > 0:\n                # File already exists (is cached)\n                logger.debug(f'Found cached file ({self.pk}) at: {dest_path}')\n                # Touch the file so that it moves to the top of the priority list\n                # when cleaning.\n                dest_path.touch()\n                return dest_path\n            else:\n                logger.debug(f'Downloading file ({self.pk}) to: {dest_path}')\n                # If downloading to the cache, clean to achieve available free space\n                if get_cache_dir() in dest_path.parents:\n                    clean_file_cache()\n                # TODO: handle if these fail (e.g. bad S3 credentials)\n                if self.type == FileSourceType.FILE_FIELD:\n                    return download_field_file_to_local_path(self.file, dest_path)\n                elif self.type == FileSourceType.URL:\n                    return download_url_file_to_local_path(self.url, dest_path)\n\n    def get_cache_path(self):\n        \"\"\"Generate a predetermined path in the cache directory.\n\n        This will use the associated FileSet's cache path if this resource\n        has a file_set, otherwise it will place in the top of the cache\n        directory.\n\n        \"\"\"\n        if self.file_set is None:\n            # If no file_set, place in the main cache directory\n            directory = get_cache_dir() / f'f-{self.pk}'\n            directory.mkdir(parents=True, exist_ok=True)\n        else:\n            directory = self.file_set.get_cache_path()\n        return directory / f'{self.name}'\n\n    @contextmanager\n    def yield_local_path(self, try_fuse: bool = True, yield_file_set: bool = False):\n        \"\"\"Create a local path for this file and all other files in its file_set.\n\n        This will first attempt to use httpfs to FUSE mount the file's URL if\n        and only if the file does not belong to a FileSet. FUSE with multiple\n        files in a FileSet is not yet supported.\n\n        If FUSE is unavailable, this will fallback to downloading the entire\n        file (and the other files in this item's FileSet) to local storage.\n\n        Parameters\n        ----------\n        try_fuse : bool\n            Try to use the FUSE interface. If false, use VSI or download to\n            local storage.\n\n        yield_file_set : bool\n            Yield all of the files in this file's file_set if available.\n\n        \"\"\"\n        # TODO: fix FUSE to handle adjacent files\n        if (\n            self.file_set is None\n            and try_fuse\n            and self.type == FileSourceType.URL\n            and precheck_fuse(self.get_url())\n        ):\n            yield url_file_to_fuse_path(self.get_url(internal=True))\n            return\n        # Fallback to loading entire file locally - this uses `get_temp_path`\n        logger.debug('`yield_local_path` falling back to downloading entire file to local storage.')\n        path = self.get_cache_path()\n        if yield_file_set and self.file_set:\n            # NOTE: This is messy and should be improved but it ensures the directory remains locked\n            with self.file_set.yield_all_to_local_path() as _:\n                yield path\n            return\n        # Not in file_set. Download to cache dir\n        from .utils import yield_checksumfiles\n\n        with yield_checksumfiles([self], path.parent):\n            yield path\n\n    def get_url(self, internal: bool = False):\n        \"\"\"Get the URL of the stored resource.\n\n        Parameters\n        ----------\n        internal : bool\n            In most cases this URL will be accessible from anywhere. In some\n            cases, this URL will only be accessible from within the container.\n            This flag is for use with internal processes to make sure the host\n            is correctly set to ``minio`` when needed. See\n            ``patch_internal_presign`` for more details.\n\n        \"\"\"\n        if self.type == FileSourceType.FILE_FIELD:\n            if internal:\n                with patch_internal_presign(self.file):\n                    return self.file.url\n            else:\n                return self.file.url\n        elif self.type == FileSourceType.URL:\n            return self.url\n\n    def data_link(self):\n        return _link_url(self, 'get_url')\n\n    data_link.allow_tags = True\n", "path": "django-rgd/rgd/models/file.py"}], "after_files": [{"content": "from contextlib import contextmanager\nimport logging\nimport os\nfrom pathlib import Path\nfrom urllib.error import URLError\nfrom urllib.parse import urlparse\n\nfrom crum import get_current_user\nfrom django.conf import settings\nfrom django.contrib.gis.db import models\nfrom django_extensions.db.models import TimeStampedModel\nfrom rgd.utility import (\n    _link_url,\n    clean_file_cache,\n    compute_checksum_file_field,\n    compute_checksum_url,\n    compute_hash,\n    download_field_file_to_local_path,\n    download_url_file_to_local_path,\n    get_cache_dir,\n    get_file_lock,\n    patch_internal_presign,\n    precheck_fuse,\n    safe_urlopen,\n    url_file_to_fuse_path,\n    uuid_prefix_filename,\n)\nfrom s3_file_field import S3FileField\n\nfrom .. import tasks\nfrom .collection import Collection\nfrom .fileset import FileSet\nfrom .mixins import TaskEventMixin\n\nlogger = logging.getLogger(__name__)\n\n\nclass FileSourceType(models.IntegerChoices):\n    FILE_FIELD = 1, 'FileField'\n    URL = 2, 'URL'\n\n\nclass ChecksumFile(TimeStampedModel, TaskEventMixin):\n    \"\"\"The main class for user-uploaded files.\n\n    This has support for manually uploading files or specifying a URL to a file\n    (for example in an existing S3 bucket). This broadly supports ``http<s>://``\n    URLs to file resources as well as ``s3://`` as long as the node the app is\n    running on is provisioned to access that S3 bucket.\n\n    \"\"\"\n\n    name = models.CharField(max_length=1000, blank=True)\n    description = models.TextField(null=True, blank=True)\n    checksum = models.CharField(max_length=128)  # sha512\n    validate_checksum = models.BooleanField(\n        default=False\n    )  # a flag to validate the checksum against the saved checksum\n    last_validation = models.BooleanField(default=True)\n    collection = models.ForeignKey(\n        Collection,\n        on_delete=models.SET_NULL,\n        related_name='%(class)ss',\n        related_query_name='%(class)ss',\n        null=True,\n        blank=True,\n    )\n    created_by = models.ForeignKey(\n        settings.AUTH_USER_MODEL, null=True, blank=True, default=None, on_delete=models.SET_NULL\n    )\n    file_set = models.ForeignKey(FileSet, null=True, blank=True, on_delete=models.SET_NULL)\n\n    type = models.IntegerField(choices=FileSourceType.choices, default=FileSourceType.FILE_FIELD)\n    file = S3FileField(null=True, blank=True, upload_to=uuid_prefix_filename)\n    url = models.TextField(null=True, blank=True)\n\n    task_funcs = (tasks.task_checksum_file_post_save,)\n\n    class Meta:\n        constraints = [\n            models.CheckConstraint(\n                name='%(app_label)s_%(class)s_file_source_value_matches_type',\n                check=(\n                    models.Q(\n                        models.Q(type=FileSourceType.FILE_FIELD, file__regex=r'.+')\n                        & models.Q(models.Q(url__in=['', None]) | models.Q(url__isnull=True))\n                    )\n                    | models.Q(\n                        models.Q(type=FileSourceType.URL)\n                        & models.Q(models.Q(url__isnull=False) & models.Q(url__regex=r'.+'))\n                        & models.Q(models.Q(file__in=['', None]) | models.Q(file__isnull=True))\n                    )\n                ),\n            ),\n            models.UniqueConstraint(\n                fields=['file_set', 'name'],\n                name='unique_name',\n            ),\n            models.UniqueConstraint(\n                fields=['collection', 'url'],\n                name='unique_url_collection',\n                condition=models.Q(type=FileSourceType.URL),  # Only apply if URL file\n            ),\n        ]\n\n    @property\n    def basename(self):\n        return os.path.basename(self.name)\n\n    @property\n    def size(self):\n        # Ensure safe check of self.file\n        try:\n            return self.file.size\n        except ValueError:\n            return None\n\n    def get_checksum(self):\n        \"\"\"Compute a new checksum without saving it.\"\"\"\n        if self.type == FileSourceType.FILE_FIELD:\n            return compute_checksum_file_field(self.file)\n        elif self.type == FileSourceType.URL:\n            parsed = urlparse(self.url)\n            if parsed.scheme in ['https', 'http']:\n                return compute_checksum_url(self.url)\n            else:\n                with self.yield_local_path() as path:\n                    with open(path, 'rb') as f:\n                        return compute_hash(f)\n        else:\n            raise NotImplementedError(f'Type ({self.type}) not supported.')\n\n    def update_checksum(self):\n        self.checksum = self.get_checksum()\n        # Simple update save - not full save\n        super(ChecksumFile, self).save(\n            update_fields=[\n                'checksum',\n            ]\n        )\n\n    def validate(self):\n        previous = self.checksum\n        self.update_checksum()\n        self.last_validation = self.checksum == previous\n        # Simple update save - not full save\n        super(ChecksumFile, self).save(\n            update_fields=[\n                'last_validation',\n            ]\n        )\n        return self.last_validation\n\n    def post_save_job(self):\n        if not self.checksum or self.validate_checksum:\n            if self.validate_checksum:\n                self.validate()\n            else:\n                self.update_checksum()\n            # Reset the user flags\n            self.validate_checksum = False\n            # Simple update save - not full save\n            self.save(\n                update_fields=[\n                    'checksum',\n                    'last_validation',\n                    'validate_checksum',\n                ]\n            )\n\n    def save(self, *args, **kwargs):\n        if not self.name:\n            if self.type == FileSourceType.FILE_FIELD and self.file.name:\n                self.name = os.path.basename(self.file.name)\n            elif self.type == FileSourceType.URL:\n                parsed = urlparse(self.url)\n                if parsed.scheme in ['https', 'http']:\n                    try:\n                        with safe_urlopen(self.url) as r:\n                            self.name = r.info().get_filename()\n                    except (AttributeError, ValueError, URLError):\n                        pass\n                if not self.name:\n                    # Fallback\n                    self.name = os.path.basename(urlparse(self.url).path)\n        # Handle `created_by` User\n        user = get_current_user()\n        if user and not user.pk:\n            user = None\n        if not self.pk:\n            self.created_by = user\n        # Must save the model with the file before accessing it for the checksum\n        super(ChecksumFile, self).save(*args, **kwargs)\n\n    def download_to_local_path(self, directory: str = None):\n        \"\"\"Forcibly download this file to a directory on disk.\n\n        Cleanup must be handled by caller.\n\n        This will handle locking to prevent multiple processes/threads\n        from trying to download the file at the same time -- only one thread\n        or process will perform the download and the rest will yield its\n        result.\n\n        \"\"\"\n        if directory is None:\n            dest_path = self.get_cache_path()\n        else:\n            dest_path = Path(directory, self.name)\n        dest_path.parent.mkdir(parents=True, exist_ok=True)\n        # Thread/process safe locking for file access\n        lock = get_file_lock(dest_path)\n\n        with lock:  # TODO: handle timeouts in condition\n            if dest_path.exists() and dest_path.stat().st_size > 0:\n                # File already exists (is cached)\n                logger.debug(f'Found cached file ({self.pk}) at: {dest_path}')\n                # Touch the file so that it moves to the top of the priority list\n                # when cleaning.\n                dest_path.touch()\n                return dest_path\n            else:\n                logger.debug(f'Downloading file ({self.pk}) to: {dest_path}')\n                # If downloading to the cache, clean to achieve available free space\n                if get_cache_dir() in dest_path.parents:\n                    clean_file_cache()\n                # TODO: handle if these fail (e.g. bad S3 credentials)\n                if self.type == FileSourceType.FILE_FIELD:\n                    return download_field_file_to_local_path(self.file, dest_path)\n                elif self.type == FileSourceType.URL:\n                    return download_url_file_to_local_path(self.url, dest_path)\n\n    def get_cache_path(self, root: bool = False):\n        \"\"\"Generate a predetermined path in the cache directory.\n\n        This will use the associated FileSet's cache path if this resource\n        has a file_set, otherwise it will place in the top of the cache\n        directory.\n\n        \"\"\"\n        if self.file_set is None:\n            # If no file_set, place in the main cache directory\n            directory = get_cache_dir() / f'f-{self.pk}'\n            directory.mkdir(parents=True, exist_ok=True)\n        else:\n            directory = self.file_set.get_cache_path()\n        if root:\n            return directory\n        return directory / f'{self.name}'\n\n    @contextmanager\n    def yield_local_path(self, try_fuse: bool = True, yield_file_set: bool = False):\n        \"\"\"Create a local path for this file and all other files in its file_set.\n\n        This will first attempt to use httpfs to FUSE mount the file's URL if\n        and only if the file does not belong to a FileSet. FUSE with multiple\n        files in a FileSet is not yet supported.\n\n        If FUSE is unavailable, this will fallback to downloading the entire\n        file (and the other files in this item's FileSet) to local storage.\n\n        Parameters\n        ----------\n        try_fuse : bool\n            Try to use the FUSE interface. If false, use VSI or download to\n            local storage.\n\n        yield_file_set : bool\n            Yield all of the files in this file's file_set if available.\n\n        \"\"\"\n        # TODO: fix FUSE to handle adjacent files\n        if (\n            self.file_set is None\n            and try_fuse\n            and self.type == FileSourceType.URL\n            and precheck_fuse(self.get_url())\n        ):\n            yield url_file_to_fuse_path(self.get_url(internal=True))\n            return\n        # Fallback to loading entire file locally - this uses `get_temp_path`\n        logger.debug('`yield_local_path` falling back to downloading entire file to local storage.')\n        root = self.get_cache_path(root=True)\n        path = self.get_cache_path()\n        if yield_file_set and self.file_set:\n            # NOTE: This is messy and should be improved but it ensures the directory remains locked\n            with self.file_set.yield_all_to_local_path() as _:\n                yield path\n            return\n        # Not in file_set. Download to cache dir\n        from .utils import yield_checksumfiles\n\n        with yield_checksumfiles([self], root):\n            yield path\n\n    def get_url(self, internal: bool = False):\n        \"\"\"Get the URL of the stored resource.\n\n        Parameters\n        ----------\n        internal : bool\n            In most cases this URL will be accessible from anywhere. In some\n            cases, this URL will only be accessible from within the container.\n            This flag is for use with internal processes to make sure the host\n            is correctly set to ``minio`` when needed. See\n            ``patch_internal_presign`` for more details.\n\n        \"\"\"\n        if self.type == FileSourceType.FILE_FIELD:\n            if internal:\n                with patch_internal_presign(self.file):\n                    return self.file.url\n            else:\n                return self.file.url\n        elif self.type == FileSourceType.URL:\n            return self.url\n\n    def data_link(self):\n        return _link_url(self, 'get_url')\n\n    data_link.allow_tags = True\n", "path": "django-rgd/rgd/models/file.py"}]}