make HDFReferenceRecipe work with with storage and reference API changes
MODIFY
pangeo_forge_recipes/recipes/reference_hdf_zarr.py
pangeo_forge_recipes/recipes/reference_hdf_zarr.py
@@ -9,7 +9,7 @@ import yaml
from fsspec_reference_maker.combine import MultiZarrToZarr
from ..patterns import FilePattern, Index
-from ..reference import create_hdf5_reference
+from ..reference import create_hdf5_reference, unstrip_protocol
from ..storage import FSSpecTarget, MetadataTarget, file_opener
from .base import BaseRecipe, FilePatternRecipeMixin
@@ -151,9 +151,10 @@ def _one_chunk(
):
fname = file_pattern[chunk_key]
ref_fname = os.path.basename(fname + ".json")
- # with fsspec.open(fname, **netcdf_storage_options) as fp:
with file_opener(fname, **netcdf_storage_options) as fp:
- metadata_cache[ref_fname] = create_hdf5_reference(fp, fname)
+ protocol = getattr(getattr(fp, "fs", None), "protocol", None) # make mypy happy
+ target_url = unstrip_protocol(fname, protocol)
+ metadata_cache[ref_fname] = create_hdf5_reference(fp, target_url, fname)
def _finalize(

MODIFY
pangeo_forge_recipes/reference.py
pangeo_forge_recipes/reference.py
Functions related to creating fsspec references.
"""
+from typing import Dict, Tuple, Union
+
from fsspec_reference_maker.hdf import SingleHdf5ToZarr
-def create_hdf5_reference(fp, fname: str, inline_threshold: int = 300, **netcdf_storage_options):
- h5chunks = SingleHdf5ToZarr(fp, _unstrip_protocol(fname, fp.fs), inline_threshold=300)
+def create_hdf5_reference(
+ fp, fname: str, url: str, inline_threshold: int = 300, **netcdf_storage_options
+) -> Dict:
+ h5chunks = SingleHdf5ToZarr(fp, url, inline_threshold=inline_threshold)
reference_data = h5chunks.translate()
return reference_data
-def _unstrip_protocol(name, fs):
+def unstrip_protocol(name: str, protocol: Union[str, Tuple[str, ...]]) -> str:
# should be upstreamed into fsspec and maybe also
# be a method on an OpenFile
- if isinstance(fs.protocol, str):
- if name.startswith(fs.protocol):
+ if isinstance(protocol, str):
+ if name.startswith(protocol):
return name
- return fs.protocol + "://" + name
+ return protocol + "://" + name
else:
- if name.startswith(tuple(fs.protocol)):
+ if name.startswith(tuple(protocol)):
return name
- return fs.protocol[0] + "://" + name
+ return protocol[0] + "://" + name

make HDFReferenceRecipe work with with storage and reference API changes