don't overwrite cache
MODIFY
pangeo_forge/recipe.py
pangeo_forge/recipe.py
@@ -173,6 +173,11 @@ def _fsspec_safe_open(fname, **kwargs):
yield fp2
+def _get_url_size(fname):
+ with fsspec.open(fname, mode='rb') as of:
+ size = of.fs.size(fname)
+ return size
+
# Notes about dataclasses:
# - https://www.python.org/dev/peps/pep-0557/#inheritance
# - https://stackoverflow.com/questions/51575931/class-inheritance-in-python-3-7-dataclasses
@@ -332,7 +337,15 @@ class NetCDFtoZarrRecipe(BaseRecipe):
def cache_func(input_key: Hashable) -> None:
logger.info(f"Caching input {input_key}")
fname = self._inputs[input_key]
- # TODO: check and see if the file already exists in the cache
+
+ # check and see if the file already exists in the cache
+ if self.input_cache.exists(fname):
+ cached_size = self.input_cache.size(fname)
+ remote_size = _get_url_size(fname)
+ if cached_size == remote_size:
+ logger.info(f"Input {input_key} file {fname} is already cached")
+ return
+
input_opener = _fsspec_safe_open(fname, mode="rb", **self.fsspec_open_kwargs)
target_opener = self.input_cache.open(fname, mode="wb")
_copy_btw_filesystems(input_opener, target_opener)

MODIFY
pangeo_forge/storage.py
pangeo_forge/storage.py
@@ -61,6 +61,9 @@ class FSSpecTarget(AbstractTarget):
"""Remove file from the cache."""
self.fs.rm(self._full_path(path))
+ def size(self, path: str) -> int:
+ return self.fs.size(self._full_path(path))
+
@contextmanager
def open(self, path: str, **kwargs) -> Iterator[None]:
"""Open file with a context manager."""

don't overwrite cache