While I'm here, I might as well prevent the large file_pattern error.
MODIFY
pangeo_forge_recipes/recipes/xarray_zarr.py
pangeo_forge_recipes/recipes/xarray_zarr.py
@@ -362,11 +362,11 @@ def open_chunk(chunk_key: ChunkKey, *, config: XarrayZarrRecipe) -> xr.Dataset:
yield ds
-def get_input_meta(metadata_cache: Optional[MetadataTarget], *input_keys: InputKey,) -> Dict:
+def get_input_meta(metadata_cache: Optional[MetadataTarget], file_pattern: FilePattern,) -> Dict:
# getitems should be async; much faster than serial calls
if metadata_cache is None:
raise ValueError("metadata_cache is not set.")
- return metadata_cache.getitems([_input_metadata_fname(k) for k in input_keys])
+ return metadata_cache.getitems([_input_metadata_fname(k) for k in file_pattern])
def calculate_sequence_lens(
@@ -384,8 +384,7 @@ def calculate_sequence_lens(
# read per-input metadata; this is distinct from global metadata
# get the sequence length of every file
- # this line could become problematic for large (> 10_000) lists of files
- input_meta = get_input_meta(metadata_cache, *file_pattern)
+ input_meta = get_input_meta(metadata_cache, file_pattern)
# use a numpy array to allow reshaping
all_lens = np.array([m["dims"][concat_dim] for m in input_meta.values()])
all_lens.shape = list(file_pattern.dims.values())

While I'm here, I might as well prevent the large file_pattern error.