add a bunch of dask.config context managers
MODIFY
.github/workflows/main.yaml
.github/workflows/main.yaml
@@ -53,7 +53,8 @@ jobs:
- name: Run Tests
shell: bash -l {0}
run: |
- py.test tests/test_storage.py -x -v --redirect-dask-worker-logs-to-stdout=DEBUG \
+ py.test tests/test_storage -x -v --redirect-dask-worker-logs-to-stdout=DEBUG
+ py.test tests -k caching_copying -x -v --redirect-dask-worker-logs-to-stdout=DEBUG \
--cov=pangeo_forge_recipes --cov-config .coveragerc --cov-report term-missing
coverage xml
- name: Codecov

MODIFY
pangeo_forge_recipes/recipes/xarray_zarr.py
pangeo_forge_recipes/recipes/xarray_zarr.py
@@ -345,6 +345,7 @@ class XarrayZarrRecipe(BaseRecipe):
logger.info(f"Opening input with Xarray {input_key}: '{fname}'")
cache = self.input_cache if self.cache_inputs else None
with file_opener(fname, cache=cache, copy_to_local=self.copy_input_to_local_file) as f:
+ with dask.config.set(scheduler="single-threaded"): # make sure we don't use a scheduler
logger.debug("about to call xr.open_dataset")
ds = xr.open_dataset(f, **self.xarray_open_kwargs)
logger.debug("successfully opened dataset")
@@ -382,6 +383,9 @@ class XarrayZarrRecipe(BaseRecipe):
if len(dsets) > 1:
# During concat, attributes and encoding are taken from the first dataset
# https://github.com/pydata/xarray/issues/1614
+ with dask.config.set(
+ scheduler="single-threaded"
+ ): # make sure we don't use a scheduler
ds = xr.concat(dsets, self._concat_dim, **self.xarray_concat_kwargs)
elif len(dsets) == 1:
ds = dsets[0]
@@ -389,8 +393,12 @@ class XarrayZarrRecipe(BaseRecipe):
assert False, "Should never happen"
if self.process_chunk is not None:
+ with dask.config.set(
+ scheduler="single-threaded"
+ ): # make sure we don't use a scheduler
ds = self.process_chunk(ds)
+ with dask.config.set(scheduler="single-threaded"): # make sure we don't use a scheduler
logger.debug(f"{ds}")
# TODO: maybe do some chunking here?

add a bunch of dask.config context managers