Add multi tests
MODIFY
pangeo_forge_recipes/recipes/reference_hdf_zarr.py
pangeo_forge_recipes/recipes/reference_hdf_zarr.py
@@ -100,12 +100,16 @@ class ReferenceHDFRecipe(BaseRecipe):
"""Final step to finish the recipe after data has been written.
Attribute that returns a callable function.
"""
+ if isinstance(self.netcdf_url, str):
+ proto = fsspec.utils.get_protocol(self.netcdf_url)
+ else:
+ proto = fsspec.utils.get_protocol(self.netcdf_url[0])
return functools.partial(
_finalize,
work_dir=self._work_dir,
out_url=self.output_url,
out_so=self.output_storage_options,
- remote_protocol=fsspec.utils.get_protocol(self.netcdf_url),
+ remote_protocol=proto,
remote_options=self.netcdf_storage_options,
xr_open_kwargs=self.xarray_open_kwargs,
xr_concat_kwargs=self.xarray_concat_args,

MODIFY
tests/test_references.py
tests/test_references.py
@@ -4,13 +4,15 @@ import fsspec
import pytest
import xarray as xr
-from pangeo_forge_recipes.recipes.reference_hdf_zarr import ReferenceHDFRecipe
-
try:
import intake
except ImportError:
intake = False
+pytest.importorskip("fsspec_reference_maker")
+reference_hdf_zarr = pytest.importorskip("pangeo_forge_recipes.recipes.reference_hdf_zarr")
+ReferenceHDFRecipe = reference_hdf_zarr.ReferenceHDFRecipe
+
@pytest.mark.parametrize("with_intake", [True, False])
def test_single(netcdf_local_paths, tmpdir, with_intake):
@@ -37,3 +39,32 @@ def test_single(netcdf_local_paths, tmpdir, with_intake):
m = fs.get_mapper("")
ds = xr.open_dataset(m, engine="zarr")
assert (ds.foo == expected.foo).all()
+
+
+@pytest.mark.parametrize("with_intake", [True, False])
+def test_multi(netcdf_local_paths, tmpdir, with_intake):
+ if with_intake and intake is False:
+ pytest.skip("intake not installed")
+ full_paths, items_per_file = netcdf_local_paths
+ paths = [str(f) for f in full_paths]
+ expected = xr.open_mfdataset(paths, engine="h5netcdf")
+
+ out = os.path.join(tmpdir, "out.json")
+
+ concat_kwargs = {"dim": "time"}
+ recipe = ReferenceHDFRecipe(netcdf_url=paths, output_url=out, xarray_concat_args=concat_kwargs)
+
+ recipe.to_dask().compute(scheduler="sync")
+
+ assert os.path.isfile(out)
+ assert os.path.isfile(out + ".yaml")
+
+ if with_intake:
+ cat = intake.open_catalog(out + ".yaml")
+ ds = cat.data.read()
+ else:
+ m = fsspec.get_mapper(
+ "reference://", fo=out, target_protocol="file", remote_protocol="file",
+ )
+ ds = xr.open_dataset(m, engine="zarr")
+ assert (ds.foo == expected.foo).all()

Add multi tests