refactor reference module api
MODIFY
pangeo_forge_recipes/recipes/reference_hdf_zarr.py
pangeo_forge_recipes/recipes/reference_hdf_zarr.py
@@ -10,7 +10,7 @@ from fsspec_reference_maker.combine import MultiZarrToZarr
from ..patterns import FilePattern, Index
from ..reference import create_hdf5_reference
-from ..storage import FSSpecTarget, MetadataTarget
+from ..storage import FSSpecTarget, MetadataTarget, file_opener
from .base import BaseRecipe, FilePatternRecipeMixin
ChunkKey = Index
@@ -151,7 +151,9 @@ def _one_chunk(
):
fname = file_pattern[chunk_key]
ref_fname = os.path.basename(fname + ".json")
- metadata_cache[ref_fname] = create_hdf5_reference(fname, **netcdf_storage_options)
+ # with fsspec.open(fname, **netcdf_storage_options) as fp:
+ with file_opener(fname, **netcdf_storage_options) as fp:
+ metadata_cache[ref_fname] = create_hdf5_reference(fp, fname)
def _finalize(

MODIFY
pangeo_forge_recipes/reference.py
pangeo_forge_recipes/reference.py
Functions related to creating fsspec references.
"""
-import fsspec
from fsspec_reference_maker.hdf import SingleHdf5ToZarr
-def create_hdf5_reference(fname: str, inline_threshold: int = 300, **netcdf_storage_options):
- with fsspec.open(fname, **netcdf_storage_options) as f:
- h5chunks = SingleHdf5ToZarr(f, _unstrip_protocol(fname, f.fs), inline_threshold=300)
+def create_hdf5_reference(fp, fname: str, inline_threshold: int = 300, **netcdf_storage_options):
+ h5chunks = SingleHdf5ToZarr(fp, _unstrip_protocol(fname, fp.fs), inline_threshold=300)
reference_data = h5chunks.translate()
return reference_data

refactor reference module api