use fs.open instead of fsspec.open
MODIFY
pangeo_forge_recipes/storage.py
pangeo_forge_recipes/storage.py
@@ -26,10 +26,16 @@ def _get_url_size(fname):
@contextmanager
def _fsspec_safe_open(fname: str, **kwargs) -> Iterator[OpenFileType]:
+ fs, _, paths = fsspec.get_fs_token_paths(fname, mode="rb")
+ path = paths[0]
+ logger.debug(f"_fsspec_safe_open opening {path} with fs {fs}")
+ with fs.open(path, mode="rb") as fp:
+ logger.debug("_fsspec_safe_open yielding fs")
+ yield fp
# workaround for inconsistent behavior of fsspec.open
# https://github.com/intake/filesystem_spec/issues/579
- with fsspec.open(fname, **kwargs) as fp:
- yield fp
+ # with fsspec.open(fname, **kwargs) as fp:
+ # yield fp
# with fp as fp2:
# yield fp2

use fs.open instead of fsspec.open