use incremental downloading
MODIFY
pangeo_forge/recipe.py
pangeo_forge/recipe.py
@@ -236,9 +236,16 @@ class NetCDFtoZarrRecipe(BaseRecipe):
def cache_func(input_key: Hashable) -> None:
logger.info(f"Caching input {input_key}")
fname = self._inputs[input_key]
+ # TODO: add a check for whether the input is already cached?
with input_opener(fname, mode="rb", **self.fsspec_open_kwargs) as source:
with self.input_cache.open(fname, mode="wb") as target:
- target.write(source.read())
+ # TODO: make this configurable? Would we ever want to change it?
+ BLOCK_SIZE = 10_000_000 # 10 MB
+ while True:
+ data = source.read(BLOCK_SIZE)
+ if not data:
+ break
+ target.write(data)
return cache_func

use incremental downloading