test new FilePattern attributes and __init__ ValueErrors
MODIFY
pangeo_forge_recipes/patterns.py
pangeo_forge_recipes/patterns.py
@@ -154,7 +154,7 @@ class FilePattern:
"`is_opendap` must be `False` when passing `fsspec_open_kwargs`."
)
if kwargs.keys():
- raise ValueError(f"{kwargs.keys()[0]} is not a valid keyword argument.")
+ raise ValueError(f"`{list(kwargs.keys())[0]}` is not a supported keyword argument.")
def __repr__(self):
return f"<FilePattern {self.dims}>"

MODIFY
tests/test_patterns.py
tests/test_patterns.py
@@ -29,8 +29,16 @@ def make_concat_merge_pattern(**kwargs):
def format_function(time, variable):
return f"T_{time}_V_{variable}"
+ if "fsspec_open_kwargs" in kwargs.keys() and "is_opendap" in kwargs.keys():
+ with pytest.raises(ValueError):
+ fp = FilePattern(format_function, merge, concat, **kwargs)
+ return
+ elif "unsupported_kwarg" in kwargs.keys():
+ with pytest.raises(ValueError):
+ fp = FilePattern(format_function, merge, concat, **kwargs)
+ return
+ else:
fp = FilePattern(format_function, merge, concat, **kwargs)
-
return fp, times, varnames, format_function, kwargs
@@ -39,6 +47,18 @@ def concat_merge_pattern():
return make_concat_merge_pattern()
+@pytest.fixture(
+ params=[
+ dict(fsspec_open_kwargs={"block_size": "foo"}),
+ dict(is_opendap=True),
+ dict(fsspec_open_kwargs={"block_size": "foo"}, is_opendap=True),
+ dict(unsupported_kwarg="foo"),
+ ]
+)
+def concat_merge_pattern_with_kwargs(request):
+ return make_concat_merge_pattern(**request.param)
+
+
def test_file_pattern_concat(concat_pattern):
fp = concat_pattern
assert fp.dims == {"time": 3}
@@ -65,9 +85,31 @@ def test_pattern_from_file_sequence():
assert fp[key] == file_sequence[key[0].index]
+@pytest.mark.parametrize(
+ "runtime_secrets", [
+ {},
+ dict(fsspec_open_kwargs={"username": "foo", "password": "bar"}),
+ dict(query_string_secrets={"token": "foo"}),
+ ]
+)
@pytest.mark.parametrize("pickle", [False, True])
-def test_file_pattern_concat_merge(pickle, concat_merge_pattern):
- fp, times, varnames, format_function, _ = concat_merge_pattern
+def test_file_pattern_concat_merge(runtime_secrets, pickle, concat_merge_pattern_with_kwargs):
+ if not concat_merge_pattern_with_kwargs:
+ # if `fsspec_open_kwargs` are passed with `is_opendap`, or if an unsupported kwarg is
+ # passed to `FilePattern`, `FilePattern.__init__` raises ValueError and
+ # `concat_merge_pattern_with_kwargs` returns None, so nothing to test in these cases
+ return
+ else:
+ fp, times, varnames, format_function, kwargs = concat_merge_pattern_with_kwargs
+
+ if runtime_secrets:
+ if "fsspec_open_kwargs" in runtime_secrets.keys():
+ if not fp.is_opendap:
+ fp.fsspec_open_kwargs.update(runtime_secrets["fsspec_open_kwargs"])
+ else:
+ return
+ if "query_string_secrets" in runtime_secrets.keys():
+ fp.query_string_secrets.update(runtime_secrets["query_string_secrets"])
if pickle:
# regular pickle doesn't work here because it can't pickle format_function
@@ -95,6 +137,20 @@ def test_file_pattern_concat_merge(pickle, concat_merge_pattern):
# make sure key order doesn't matter
assert fp[key[::-1]] == expected_fname
+ if "fsspec_open_kwargs" in kwargs.keys():
+ assert fp.is_opendap is False
+ if "fsspec_open_kwargs" in runtime_secrets.keys():
+ kwargs["fsspec_open_kwargs"].update(runtime_secrets["fsspec_open_kwargs"])
+ assert fp.fsspec_open_kwargs == kwargs["fsspec_open_kwargs"]
+ else:
+ assert fp.fsspec_open_kwargs == kwargs["fsspec_open_kwargs"]
+ if "query_string_secrets" in runtime_secrets.keys():
+ assert fp.query_string_secrets == runtime_secrets["query_string_secrets"]
+ if "is_opendap" in kwargs.keys():
+ assert fp.is_opendap == kwargs["is_opendap"]
+ assert fp.is_opendap is True
+ assert fp.fsspec_open_kwargs == {}
+
@pytest.mark.parametrize("nkeep", [1, 2])
def test_prune(nkeep, concat_merge_pattern):

test new FilePattern attributes and __init__ ValueErrors