better target testing
MODIFY
.pre-commit-config.yaml
.pre-commit-config.yaml
@@ -26,16 +26,12 @@ repos:
rev: v2.2.0
hooks:
- id: seed-isort-config
+
- repo: https://github.com/pre-commit/mirrors-isort
rev: v5.2.0
hooks:
- id: isort
-- repo: https://github.com/deathbeds/prenotebook
- rev: f5bdb72a400f1a56fe88109936c83aa12cc349fa
- hooks:
- - id: prenotebook
-
- repo: https://github.com/myint/rstcheck
rev: master
hooks:

MODIFY
pangeo_forge/recipe.py
pangeo_forge/recipe.py
@@ -165,7 +165,7 @@ class NetCDFtoZarrSequentialRecipe(BaseRecipe):
ds = self.open_target()
logger.info("Found an existing dataset in target")
logger.debug(f"{ds}")
- except (IOError, zarr.errors.GroupNotFoundError):
+ except (FileNotFoundError, IOError, zarr.errors.GroupNotFoundError):
first_chunk_key = next(self.iter_chunks())
for input_url in self.inputs_for_chunk(first_chunk_key):
self.cache_input(input_url)
@@ -225,7 +225,7 @@ class NetCDFtoZarrSequentialRecipe(BaseRecipe):
with self.input_cache.open(fname, mode="rb") as f:
logger.info(f"Opening '{fname}' from cache")
yield f
- except IOError: # TODO figure out the excpetion to catch
+ except (IOError, FileNotFoundError):
if self.require_cache:
raise
else:

MODIFY
pangeo_forge/storage.py
pangeo_forge/storage.py
@@ -93,3 +93,29 @@ def _slugify(value):
value = unicodedata.normalize("NFKD", value).encode("ascii", "ignore").decode("ascii")
value = re.sub(r"[^\w\s-]+", "_", value.lower())
return re.sub(r"[-\s]+", "-", value).strip("-_")
+
+
+class UninitializedTarget(AbstractTarget):
+ def get_mapper(self):
+ raise UninitializedTargetError
+
+ def exists(self, path) -> bool:
+ raise UninitializedTargetError
+
+ def rm(self, path) -> NoReturn:
+ raise UninitializedTargetError
+
+ def open(self, path, **kwargs) -> BinaryIO:
+ raise UninitializedTargetError
+
+
+class TargetError(Exception):
+ """Base class for exceptions in this module."""
+
+ pass
+
+
+class UninitializedTargetError(TargetError):
+ """Operation on an uninitialized Target."""
+
+ pass

MODIFY
tests/conftest.py
tests/conftest.py
@@ -8,7 +8,7 @@ import pytest
import xarray as xr
from pangeo_forge import recipe
-from pangeo_forge.storage import FlatFSSpecTarget, FSSpecTarget
+from pangeo_forge.storage import FlatFSSpecTarget, FSSpecTarget, UninitializedTarget
# where to run the http server
_PORT = "8080"
@@ -91,6 +91,11 @@ def tmp_cache(tmpdir_factory):
return cache
+@pytest.fixture()
+def uninitialized_target():
+ return UninitializedTarget()
+
+
@pytest.fixture
def netCDFtoZarr_sequential_recipe(daily_xarray_dataset, netcdf_local_paths, tmp_target, tmp_cache):
r = recipe.NetCDFtoZarrSequentialRecipe(

MODIFY
tests/test_fixtures.py
tests/test_fixtures.py
import fsspec
+import pytest
import xarray as xr
+from pangeo_forge.storage import UninitializedTargetError
from pangeo_forge.utils import fix_scalar_attr_encoding
@@ -27,6 +29,24 @@ def test_target(tmp_target):
with open(tmp_target.root_path + "/foo") as f:
res = f.read()
assert res == "bar"
+ with pytest.raises(FileNotFoundError):
+ tmp_target.rm("baz")
+ with pytest.raises(FileNotFoundError):
+ with tmp_target.open("baz"):
+ pass
+
+
+def test_uninitialized_target(uninitialized_target):
+ target = uninitialized_target
+ with pytest.raises(UninitializedTargetError):
+ target.get_mapper()
+ with pytest.raises(UninitializedTargetError):
+ target.exists("foo")
+ with pytest.raises(UninitializedTargetError):
+ target.rm("foo")
+ with pytest.raises(UninitializedTargetError):
+ with target.open("foo"):
+ pass
def test_cache(tmp_cache):

better target testing