do some hyperparametrization!
MODIFY
setup.cfg
setup.cfg
@@ -3,7 +3,7 @@ max-line-length = 100
[isort]
known_first_party=pangeo_forge
-known_third_party=aiohttp,dask,fsspec,numpy,pandas,pkg_resources,prefect,pytest,rechunker,setuptools,sphinx_book_theme,xarray,zarr
+known_third_party=aiohttp,dask,fsspec,numpy,pandas,pkg_resources,prefect,pytest,pytest_lazyfixture,rechunker,setuptools,sphinx_book_theme,xarray,zarr
multi_line_output=3
include_trailing_comma=True
force_grid_wrap=0

MODIFY
tests/conftest.py
tests/conftest.py
@@ -202,7 +202,7 @@ def netCDFtoZarr_sequential_multi_variable_recipe(
return recipe.NetCDFtoZarrMultiVarSequentialRecipe, kwargs, daily_xarray_dataset, tmp_target
-@pytest.fixture(scope="module")
+@pytest.fixture(scope="session")
def dask_cluster():
cluster = LocalCluster()
yield cluster

MODIFY
tests/test_executors.py
tests/test_executors.py
import pytest
import xarray as xr
+# need to import this way (rather than use pytest.lazy_fixture) to make it work with dask
+from pytest_lazyfixture import lazy_fixture
+
+all_recipes = [
+ lazy_fixture("netCDFtoZarr_sequential_recipe"),
+ lazy_fixture("netCDFtoZarr_sequential_multi_variable_recipe"),
+]
+
+
+@pytest.mark.parametrize("recipe_fixture", all_recipes)
+def test_recipe(recipe_fixture, execute_recipe):
+ """The basic recipe test. Use this as a template for other tests."""
+
+ RecipeClass, kwargs, ds_expected, target = recipe_fixture
+ rec = RecipeClass(**kwargs)
+ execute_recipe(rec)
+ ds_actual = xr.open_zarr(target.get_mapper()).load()
+ xr.testing.assert_identical(ds_actual, ds_expected)
+
+
+# function passed to preprocessing
+def incr_date(ds, filename=""):
+ # add one day
+ t = [d + int(24 * 3600e9) for d in ds.time.values]
+ ds = ds.assign_coords(time=t)
+ return ds
+
@pytest.mark.parametrize(
- "recipe_fixture",
- [
- pytest.lazy_fixture("netCDFtoZarr_sequential_recipe"),
- pytest.lazy_fixture("netCDFtoZarr_sequential_multi_variable_recipe"),
- ],
+ "process_input, process_chunk",
+ [(None, None), (incr_date, None), (None, incr_date), (incr_date, incr_date)],
)
-def test_recipe_w_executor(recipe_fixture, execute_recipe):
+@pytest.mark.parametrize("recipe_fixture", all_recipes)
+def test_process(recipe_fixture, execute_recipe, process_input, process_chunk):
+ """Check that the process_chunk and process_input arguments work as expected."""
+
RecipeClass, kwargs, ds_expected, target = recipe_fixture
+ kwargs["process_input"] = process_input
+ kwargs["process_chunk"] = process_chunk
rec = RecipeClass(**kwargs)
execute_recipe(rec)
ds_actual = xr.open_zarr(target.get_mapper()).load()
- assert ds_actual.identical(ds_expected)
+
+ if process_input and process_chunk:
+ assert not ds_actual.identical(ds_expected)
+ ds_expected = incr_date(incr_date(ds_expected))
+ elif process_input or process_chunk:
+ assert not ds_actual.identical(ds_expected)
+ ds_expected = incr_date(ds_expected)
+
+ xr.testing.assert_identical(ds_actual, ds_expected)

do some hyperparametrization!