Simple implementation of skipping re-computing metadata cache.
MODIFY
pangeo_forge_recipes/recipes/xarray_zarr.py
pangeo_forge_recipes/recipes/xarray_zarr.py
@@ -162,10 +162,15 @@ def cache_input(input_key: InputKey, *, config: XarrayZarrRecipe) -> None:
if config.cache_metadata:
if config.metadata_cache is None:
raise ValueError("metadata_cache is not set.")
- logger.info(f"Caching metadata for input '{input_key!s}'")
+
+ if not _input_metadata_fname(input_key) in config.metadata_cache:
with open_input(input_key, config=config) as ds:
+ logger.info(f"Caching metadata for input '{input_key!s}'")
input_metadata = ds.to_dict(data=False)
config.metadata_cache[_input_metadata_fname(input_key)] = input_metadata
+ else:
+ logger.info(f"Metadata already ached for input '{input_key!s}'")
+
if config.open_input_with_fsspec_reference:
if config.file_pattern.is_opendap:

MODIFY
pangeo_forge_recipes/storage.py
pangeo_forge_recipes/storage.py
@@ -180,6 +180,9 @@ class MetadataTarget(FSSpecTarget):
def __getitem__(self, key: str) -> dict:
return json.loads(self.get_mapper()[key])
+ def __contains__(self, item: str) -> bool:
+ return item in self.get_mapper()
+
def getitems(self, keys: Sequence[str]) -> dict:
mapper = self.get_mapper()
all_meta_raw = mapper.getitems(keys)

Simple implementation of skipping re-computing metadata cache.