Actually test temp directories
MODIFY
pangeo_forge_recipes/recipes/reference_hdf_zarr.py
pangeo_forge_recipes/recipes/reference_hdf_zarr.py
@@ -164,7 +164,8 @@ def _finalize(
out = mzz.translate(None, template_count=template_count)
fs = out_target.fs
protocol = fs.protocol if isinstance(fs.protocol, str) else fs.protocol[0]
- fs.pipe(out_url, json.dumps(out).encode("ascii"))
+ with out_target.open(out_url, mode="wt") as f:
+ f.write(json.dumps(out))
fn2 = out_url + ".yaml"
spec = {
"sources": {
@@ -174,7 +175,7 @@ def _finalize(
"args": {
"urlpath": "reference://",
"storage_options": {
- "fo": out_url,
+ "fo": out_target._full_path(out_url),
"target_protocol": protocol,
"target_options": out_so,
"remote_protocol": remote_protocol,
@@ -187,7 +188,7 @@ def _finalize(
}
}
}
- with fs.open(fn2, "w") as f:
+ with out_target.open(fn2, mode="wt") as f:
yaml.dump(spec, f, default_flow_style=False)

MODIFY
tests/test_references.py
tests/test_references.py
-import os.path
import tempfile
import fsspec
@@ -35,14 +34,14 @@ def test_single(netcdf_local_paths, tmpdir, with_intake):
recipe.to_dask().compute(scheduler="sync")
- assert os.path.isfile(out)
- assert os.path.isfile(out + ".yaml")
+ assert out_target.exists(out)
+ assert out_target.exists(out + ".yaml")
if with_intake:
- cat = intake.open_catalog(out + ".yaml")
+ cat = intake.open_catalog(out_target._full_path(out + ".yaml"))
ds = cat.data.read()
else:
- fs = fsspec.filesystem("reference", fo=out, remote_protocol="file")
+ fs = fsspec.filesystem("reference", fo=out_target._full_path(out), remote_protocol="file")
m = fs.get_mapper("")
ds = xr.open_dataset(m, engine="zarr", chunks={}, consolidated=False)
assert (ds.foo == expected.foo).all()
@@ -72,16 +71,16 @@ def test_multi(netcdf_local_paths, tmpdir, with_intake):
recipe.to_dask().compute(scheduler="sync")
- assert os.path.isfile(out)
- assert os.path.isfile(out + ".yaml")
+ assert out_target.exists(out)
+ assert out_target.exists(out + ".yaml")
if with_intake:
- cat = intake.open_catalog(out + ".yaml")
+ cat = intake.open_catalog(out_target._full_path(out + ".yaml"))
ds = cat.data.read()
else:
m = fsspec.get_mapper(
"reference://",
- fo=out,
+ fo=out_target._full_path(out),
target_protocol="file",
remote_protocol="file",
skip_instance_cache=True,

Actually test temp directories