passed open_kwargs to _get_url_size
MODIFY
pangeo_forge_recipes/storage.py
pangeo_forge_recipes/storage.py
@@ -18,8 +18,8 @@ logger = logging.getLogger(__name__)
OpenFileType = Any
-def _get_url_size(fname):
- with fsspec.open(fname, mode="rb") as of:
+def _get_url_size(fname, **open_kwargs):
+ with fsspec.open(fname, mode="rb", **open_kwargs) as of:
size = of.size
return size
@@ -150,7 +150,7 @@ class CacheFSSpecTarget(FlatFSSpecTarget):
logger.info(f"Caching file '{fname}'")
if self.exists(fname):
cached_size = self.size(fname)
- remote_size = _get_url_size(fname)
+ remote_size = _get_url_size(fname, **open_kwargs)
if cached_size == remote_size:
# TODO: add checksumming here
logger.info(f"File '{fname}' is already cached")

passed open_kwargs to _get_url_size