change default copy_input_to_local_file=True
MODIFY
pangeo_forge_recipes/recipes/xarray_zarr.py
pangeo_forge_recipes/recipes/xarray_zarr.py
@@ -93,7 +93,7 @@ class XarrayZarrRecipe(BaseRecipe):
input_cache: Optional[CacheFSSpecTarget] = None
metadata_cache: Optional[MetadataTarget] = None
cache_inputs: bool = True
- copy_input_to_local_file: bool = False
+ copy_input_to_local_file: bool = True
consolidate_zarr: bool = True
xarray_open_kwargs: dict = field(default_factory=dict)
xarray_concat_kwargs: dict = field(default_factory=dict)
@@ -124,6 +124,12 @@ class XarrayZarrRecipe(BaseRecipe):
"""List of chunks needed to initialize the recipe."""
def __post_init__(self):
+ if not self.copy_input_to_local_file:
+ warnings.warn(
+ "Running recipes with `copy_input_to_local_file=False` may cause hanging. "
+ "Use caution when executing this recipe with Dask. "
+ "See https://github.com/pangeo-forge/pangeo-forge-recipes/pull/146."
+ )
self._validate_file_pattern()
# from here on we know there is at most one merge dim and one concat dim
self._concat_dim = self.file_pattern.concat_dims[0]
@@ -346,7 +352,7 @@ class XarrayZarrRecipe(BaseRecipe):
cache = self.input_cache if self.cache_inputs else None
with file_opener(fname, cache=cache, copy_to_local=self.copy_input_to_local_file) as f:
with dask.config.set(scheduler="single-threaded"): # make sure we don't use a scheduler
- logger.debug("about to call xr.open_dataset")
+ logger.debug(f"about to call xr.open_dataset on {f}")
ds = xr.open_dataset(f, **self.xarray_open_kwargs)
logger.debug("successfully opened dataset")
ds = fix_scalar_attr_encoding(ds)

change default copy_input_to_local_file=True