Updated error message to include error values and their locations.
Re-wrote some of the calculate_sequence_length logic to be clearer. In the error message, I've included ways to debug potential causes of the error.
MODIFY
pangeo_forge_recipes/recipes/xarray_zarr.py
pangeo_forge_recipes/recipes/xarray_zarr.py
@@ -16,6 +16,8 @@ from typing import Callable, Dict, Iterator, List, Optional, Sequence, Set, Tupl
import dask
import fsspec
import numpy as np
+import scipy
+import scipy.stats
import xarray as xr
import zarr
@@ -390,14 +392,19 @@ def calculate_sequence_lens(
all_lens = np.array([m["dims"][concat_dim] for m in input_meta.values()])
all_lens.shape = list(file_pattern.dims.values())
- # check that all lens are the same along the concat dim
- concat_dim_axis = list(file_pattern.dims).index(concat_dim)
- selector = [slice(0, 1)] * len(file_pattern.dims)
- selector[concat_dim_axis] = slice(None) # this should broadcast correctly agains all_lens
- sequence_lens = all_lens[tuple(selector)]
- if not (all_lens == sequence_lens).all():
- raise ValueError(f"Inconsistent sequence lengths found: f{all_lens}")
- return np.atleast_1d(sequence_lens.squeeze()).tolist()
+ if len(all_lens.shape) == 1:
+ all_lens = all_lens.reshape((all_lens.shape[0], 1))
+
+ unique_vec, unique_idx = np.unique(all_lens, axis=1, return_index=True)
+ if len(unique_idx) > 1:
+ err_idx = np.where(scipy.stats.mode(all_lens, axis=1)[0] != all_lens)
+ err_pos = list(zip(*err_idx[::-1]))
+ raise ValueError(
+ f"Inconsistent sequence lengths between indicies {unique_idx} of the concat dim."
+ f"\nValue(s) {all_lens[err_idx]} at position(s) {err_pos} are different from the rest."
+ f"\n{all_lens}"
+ )
+ return np.atleast_1d(unique_vec.squeeze()).tolist()
def prepare_target(*, config: XarrayZarrRecipe) -> None:

MODIFY
setup.cfg
setup.cfg
@@ -60,7 +60,7 @@ max-line-length = 100
[isort]
known_first_party=pangeo_forge_recipes
-known_third_party=aiohttp,apache_beam,click,dask,fsspec,fsspec_reference_maker,mypy_extensions,numpy,pandas,prefect,pytest,pytest_lazyfixture,setuptools,xarray,yaml,zarr
+known_third_party=aiohttp,apache_beam,click,dask,fsspec,fsspec_reference_maker,mypy_extensions,numpy,pandas,prefect,pytest,pytest_lazyfixture,scipy,setuptools,xarray,yaml,zarr
multi_line_output=3
include_trailing_comma=True
force_grid_wrap=0

MODIFY
tests/recipe_tests/test_XarrayZarrRecipe.py
tests/recipe_tests/test_XarrayZarrRecipe.py
@@ -473,17 +473,85 @@ def test_calc_sequence_length_errors_no_metadata():
def test_calc_sequence_length_errors_inconsistent_lengths(tmp_metadata_target, monkeypatch):
file_pattern = FilePattern(
lambda time, var: f"tmp/{time.date()!s}_{var}",
- ConcatDim("time", [datetime.datetime(year=2021, month=1, day=d) for d in range(1, 11)]),
- MergeDim("variable", ["foo", "bar"]),
+ ConcatDim("time", [datetime.datetime(year=2021, month=1, day=d) for d in range(1, 5)]),
+ MergeDim("variable", ["foo", "bar", "baz"]),
+ )
+
+ def mock_getitems(*args):
+ return {
+ "a": {"dims": {"time": [1] * 3, "variables": []}},
+ "b": {"dims": {"time": [2] * 3, "variables": []}},
+ "c": {"dims": {"time": [7] + [3] * 2, "variables": []}}, # Inconsistent sequence length
+ "d": {"dims": {"time": [4] * 3, "variables": []}},
+ }
+
+ monkeypatch.setattr(MetadataTarget, "getitems", mock_getitems)
+
+ with pytest.raises(
+ ValueError, match="Inconsistent sequence lengths between indicies"
+ ) as execinfo:
+ calculate_sequence_lens(None, file_pattern, tmp_metadata_target)
+
+ msg = execinfo.value.args[0]
+ assert "[1 0]" in msg
+ assert "[7]" in msg
+ assert "[(0, 2)]" in msg
+
+
+def test_calc_sequence_length_errors_multiple_inconsistent_lengths(
+ tmp_metadata_target, monkeypatch
+):
+ file_pattern = FilePattern(
+ lambda time, var: f"tmp/{time.date()!s}_{var}",
+ ConcatDim("time", [datetime.datetime(year=2021, month=1, day=d) for d in range(1, 5)]),
+ MergeDim("variable", ["foo", "bar", "baz"]),
+ )
+
+ def mock_getitems(*args):
+ return {
+ "a": {"dims": {"time": [1] * 3, "variables": []}},
+ "b": {"dims": {"time": [2] * 3, "variables": []}},
+ "c": {"dims": {"time": [7] + [3] * 2, "variables": []}}, # Inconsistent sequence length
+ "d": {"dims": {"time": [4] * 2 + [10], "variables": []}},
+ }
+
+ monkeypatch.setattr(MetadataTarget, "getitems", mock_getitems)
+
+ with pytest.raises(
+ ValueError, match="Inconsistent sequence lengths between indicies"
+ ) as execinfo:
+ calculate_sequence_lens(None, file_pattern, tmp_metadata_target)
+
+ msg = execinfo.value.args[0]
+ assert "[1 2 0]" in msg
+ assert "[ 7 10]" in msg
+ assert "[(0, 2), (2, 3)]" in msg
+
+
+def test_calc_sequence_length_errors_inconsistent_lengths_reverse_combine_dim_order(
+ tmp_metadata_target, monkeypatch
+):
+ file_pattern = FilePattern(
+ lambda var, time: f"tmp/{time.date()!s}_{var}",
+ MergeDim("variable", ["foo", "bar", "baz"]),
+ ConcatDim("time", [datetime.datetime(year=2021, month=1, day=d) for d in range(1, 5)]),
)
def mock_getitems(*args):
return {
- "a": {"dims": {"time": [2] * 10}},
- "b": {"dims": {"time": [3] + [2] * 9}}, # Inconsistent sequence length
+ "a": {"dims": {"variables": [], "time": [1] * 4}},
+ "b": {"dims": {"variables": [], "time": [2] * 3 + [5]}}, # Inconsistent sequence length
+ "c": {"dims": {"variables": [], "time": [3] * 4}},
}
monkeypatch.setattr(MetadataTarget, "getitems", mock_getitems)
- with pytest.raises(ValueError, match="Inconsistent sequence lengths found: "):
+ with pytest.raises(
+ ValueError, match="Inconsistent sequence lengths between indicies"
+ ) as execinfo:
calculate_sequence_lens(None, file_pattern, tmp_metadata_target)
+
+ msg = execinfo.value.args[0]
+ assert "[0 3]" in msg
+ assert "[5]" in msg
+ assert "[(3, 1)]" in msg

Updated error message to include error values and their locations.

Re-wrote some of the calculate_sequence_length logic to be clearer. In the error message, I've included ways to debug potential causes of the error.