start refactoring references to module
MODIFY
pangeo_forge_recipes/recipes/reference_hdf_zarr.py
pangeo_forge_recipes/recipes/reference_hdf_zarr.py
@@ -7,9 +7,9 @@ from typing import Callable, Dict, Hashable, Iterable, Optional
import fsspec
import yaml
from fsspec_reference_maker.combine import MultiZarrToZarr
-from fsspec_reference_maker.hdf import SingleHdf5ToZarr
from ..patterns import FilePattern, Index
+from ..reference import create_hdf5_reference
from ..storage import FSSpecTarget, MetadataTarget
from .base import BaseRecipe, FilePatternRecipeMixin
@@ -150,10 +150,8 @@ def _one_chunk(
metadata_cache: MetadataTarget,
):
fname = file_pattern[chunk_key]
- with fsspec.open(fname, **netcdf_storage_options) as f:
- fn = os.path.basename(fname + ".json")
- h5chunks = SingleHdf5ToZarr(f, _unstrip_protocol(fname, f.fs), inline_threshold=300)
- metadata_cache[fn] = h5chunks.translate()
+ ref_fname = os.path.basename(fname + ".json")
+ metadata_cache[ref_fname] = create_hdf5_reference(fname, **netcdf_storage_options)
def _finalize(
@@ -212,16 +210,3 @@ def _finalize(
}
with out_target.open(output_intake_yaml_fname, mode="wt") as f:
yaml.dump(spec, f, default_flow_style=False)
-
-
-def _unstrip_protocol(name, fs):
- # should be upstreamed into fsspec and maybe also
- # be a method on an OpenFile
- if isinstance(fs.protocol, str):
- if name.startswith(fs.protocol):
- return name
- return fs.protocol + "://" + name
- else:
- if name.startswith(tuple(fs.protocol)):
- return name
- return fs.protocol[0] + "://" + name

ADD
pangeo_forge_recipes/reference.py
+"""
+Functions related to creating fsspec references.
+"""
+
+import fsspec
+from fsspec_reference_maker.hdf import SingleHdf5ToZarr
+
+
+def create_hdf5_reference(fname: str, inline_threshold: int = 300, **netcdf_storage_options):
+ with fsspec.open(fname, **netcdf_storage_options) as f:
+ h5chunks = SingleHdf5ToZarr(f, _unstrip_protocol(fname, f.fs), inline_threshold=300)
+ reference_data = h5chunks.translate()
+ return reference_data
+
+
+def _unstrip_protocol(name, fs):
+ # should be upstreamed into fsspec and maybe also
+ # be a method on an OpenFile
+ if isinstance(fs.protocol, str):
+ if name.startswith(fs.protocol):
+ return name
+ return fs.protocol + "://" + name
+ else:
+ if name.startswith(tuple(fs.protocol)):
+ return name
+ return fs.protocol[0] + "://" + name

start refactoring references to module