try to refactor tests a bit
MODIFY
tests/conftest.py
tests/conftest.py
@@ -12,7 +12,6 @@ import xarray as xr
from dask.distributed import Client, LocalCluster
from prefect.executors import DaskExecutor
-from pangeo_forge_recipes import recipes
from pangeo_forge_recipes.executors import (
DaskPipelineExecutor,
PrefectPipelineExecutor,
@@ -74,19 +73,19 @@ def daily_xarray_dataset():
return ds
-def _split_up_files_by_day(ds, day_param):
+def split_up_files_by_day(ds, day_param):
gb = ds.resample(time=day_param)
_, datasets = zip(*gb)
fnames = [f"{n:03d}.nc" for n in range(len(datasets))]
return datasets, fnames
-def _split_up_files_by_variable_and_day(ds, day_param):
+def split_up_files_by_variable_and_day(ds, day_param):
all_dsets = []
all_fnames = []
fnames_by_variable = {}
for varname in ds.data_vars:
- var_dsets, fnames = _split_up_files_by_day(ds[[varname]], day_param)
+ var_dsets, fnames = split_up_files_by_day(ds[[varname]], day_param)
fnames = [f"{varname}_{fname}" for fname in fnames]
all_dsets += var_dsets
all_fnames += fnames
@@ -94,7 +93,7 @@ def _split_up_files_by_variable_and_day(ds, day_param):
return all_dsets, all_fnames, fnames_by_variable
-def _make_file_pattern(netcdf_paths):
+def make_file_pattern(netcdf_paths):
paths, items_per_file, fnames_by_variable, path_format = netcdf_paths
if not fnames_by_variable:
@@ -121,9 +120,7 @@ def items_per_file(request):
return request.param
-@pytest.fixture(
- scope="session", params=[_split_up_files_by_day, _split_up_files_by_variable_and_day]
-)
+@pytest.fixture(scope="session", params=[split_up_files_by_day, split_up_files_by_variable_and_day])
def file_splitter(request):
return request.param
@@ -144,6 +141,11 @@ def netcdf_paths(daily_xarray_dataset, tmpdir_factory, items_per_file, file_spli
return full_paths, items_per_file, fnames_by_variable, path_format
+@pytest.fixture(scope="session")
+def netcdf_local_file_pattern(netcdf_paths):
+ return make_file_pattern(netcdf_paths)
+
+
@pytest.fixture(scope="session")
def netcdf_http_paths(netcdf_paths, request):
paths, items_per_file, fnames_by_variable, path_format = netcdf_paths
@@ -217,41 +219,6 @@ def tmp_metadata_target(tmpdir_factory):
return cache
-@pytest.fixture
-def netCDFtoZarr_recipe(
- daily_xarray_dataset, netcdf_paths, tmp_target, tmp_cache, tmp_metadata_target
-):
- file_pattern = _make_file_pattern(netcdf_paths)
-
- kwargs = dict(
- inputs_per_chunk=1,
- target=tmp_target,
- input_cache=tmp_cache,
- metadata_cache=tmp_metadata_target,
- )
- return recipes.XarrayZarrRecipe, file_pattern, kwargs, daily_xarray_dataset, tmp_target
-
-
-@pytest.fixture
-def netCDFtoZarr_subset_recipe(
- daily_xarray_dataset, netcdf_paths, tmp_target, tmp_cache, tmp_metadata_target
-):
- items_per_file = netcdf_paths[1]
- if items_per_file != 2:
- pytest.skip("This recipe only makes sense with items_per_file == 2.")
-
- file_pattern = _make_file_pattern(netcdf_paths)
-
- kwargs = dict(
- subset_inputs={"time": 2},
- inputs_per_chunk=1,
- target=tmp_target,
- input_cache=tmp_cache,
- metadata_cache=tmp_metadata_target,
- )
- return recipes.XarrayZarrRecipe, file_pattern, kwargs, daily_xarray_dataset, tmp_target
-
-
@pytest.fixture(scope="session")
def dask_cluster(request):
cluster = LocalCluster(n_workers=2, threads_per_worker=1, silence_logs=False)

MODIFY
tests/test_recipes.py
tests/test_recipes.py
@@ -10,6 +10,39 @@ from pytest_lazyfixture import lazy_fixture
from pangeo_forge_recipes.patterns import FilePattern
from pangeo_forge_recipes.recipes.base import BaseRecipe
+from pangeo_forge_recipes.recipes.xarray_zarr import XarrayZarrRecipe
+
+
+@pytest.fixture
+def netCDFtoZarr_recipe(
+ daily_xarray_dataset, netcdf_local_file_pattern, tmp_target, tmp_cache, tmp_metadata_target
+):
+ kwargs = dict(
+ inputs_per_chunk=1,
+ target=tmp_target,
+ input_cache=tmp_cache,
+ metadata_cache=tmp_metadata_target,
+ )
+ return XarrayZarrRecipe, netcdf_local_file_pattern, kwargs, daily_xarray_dataset, tmp_target
+
+
+@pytest.fixture
+def netCDFtoZarr_subset_recipe(
+ daily_xarray_dataset, netcdf_local_file_pattern, tmp_target, tmp_cache, tmp_metadata_target
+):
+ items_per_file = netcdf_local_file_pattern.nitems_per_input.get("time", None)
+ if items_per_file != 2:
+ pytest.skip("This recipe only makes sense with items_per_file == 2.")
+
+ kwargs = dict(
+ subset_inputs={"time": 2},
+ inputs_per_chunk=1,
+ target=tmp_target,
+ input_cache=tmp_cache,
+ metadata_cache=tmp_metadata_target,
+ )
+ return XarrayZarrRecipe, netcdf_local_file_pattern, kwargs, daily_xarray_dataset, tmp_target
+
all_recipes = [
lazy_fixture("netCDFtoZarr_recipe"),
@@ -24,10 +57,6 @@ recipes_no_subset = [
def test_to_pipelines_warns(netCDFtoZarr_recipe):
RecipeClass, file_pattern, kwargs, ds_expected, target = netCDFtoZarr_recipe
- # `netCDFtoZarr_recipe` fixture is parametrized. We don't need to run this test more than once.
- if len(file_pattern.merge_dims) != 0:
- pytest.skip("It's redundant to run this test more than once.")
-
rec = RecipeClass(file_pattern, **kwargs)
with pytest.warns(FutureWarning):
rec.to_pipelines()
@@ -53,7 +82,7 @@ def test_recipe(recipe_fixture, execute_recipe):
@pytest.mark.parametrize("recipe_fixture", all_recipes)
@pytest.mark.parametrize("nkeep", [1, 2])
def test_prune_recipe(recipe_fixture, execute_recipe, nkeep):
- """The basic recipe test. Use this as a template for other tests."""
+ """Check that recipe.copy_pruned works as expected."""
RecipeClass, file_pattern, kwargs, ds_expected, target = recipe_fixture
rec = RecipeClass(file_pattern, **kwargs)
@@ -70,12 +99,9 @@ def test_prune_recipe(recipe_fixture, execute_recipe, nkeep):
def test_recipe_caching_copying(
netCDFtoZarr_recipe, execute_recipe, cache_inputs, copy_input_to_local_file
):
- """The basic recipe test. Use this as a template for other tests."""
- RecipeClass, file_pattern, kwargs, ds_expected, target = netCDFtoZarr_recipe
+ """Test that caching and copying to local file work."""
- # `netCDFtoZarr_recipe` fixture is parametrized. We don't need to run this test more than once.
- if len(file_pattern.merge_dims) != 0:
- pytest.skip("It's redundant to run this test more than once.")
+ RecipeClass, file_pattern, kwargs, ds_expected, target = netCDFtoZarr_recipe
if not cache_inputs:
kwargs.pop("input_cache") # make sure recipe doesn't require input_cache

try to refactor tests a bit