add strict option in validation
MODIFY
fastavro/_validation.pyi
fastavro/_validation.pyi
@@ -3,8 +3,15 @@ from typing import Any, Iterable
from .types import Schema
def validate(
- datum: Any, schema: Schema, field: str = ..., raise_errors: bool = ...
+ datum: Any,
+ schema: Schema,
+ field: str = ...,
+ raise_errors: bool = ...,
+ strict: bool = ...,
) -> bool: ...
def validate_many(
- records: Iterable[Any], schema: Schema, raise_errors: bool = ...
+ records: Iterable[Any],
+ schema: Schema,
+ raise_errors: bool = ...,
+ strict: bool = ...,
) -> bool: ...

MODIFY
fastavro/_validation.pyx
fastavro/_validation.pyx
@@ -22,6 +22,8 @@ cdef int32 INT_MAX_VALUE = const.INT_MAX_VALUE
cdef long64 LONG_MIN_VALUE = const.LONG_MIN_VALUE
cdef long64 LONG_MAX_VALUE = const.LONG_MAX_VALUE
+NoValue = object()
+
cdef inline bint validate_null(datum):
return datum is None
@@ -79,15 +81,20 @@ cdef inline bint validate_array(
dict named_schemas,
str parent_ns="",
bint raise_errors=True,
+ bint strict=False,
) except -1:
if not isinstance(datum, (Sequence, array.array)) or isinstance(datum, str):
return False
for d in datum:
- if not _validate(datum=d, schema=schema["items"],
+ if not _validate(
+ datum=d,
+ schema=schema["items"],
named_schemas=named_schemas,
field=parent_ns,
- raise_errors=raise_errors):
+ raise_errors=raise_errors,
+ strict=strict,
+ ):
return False
return True
@@ -98,6 +105,7 @@ cdef inline bint validate_map(
dict named_schemas,
str parent_ns="",
bint raise_errors=True,
+ bint strict=False,
) except -1:
# initial checks for map type
if not isinstance(datum, Mapping):
@@ -107,10 +115,14 @@ cdef inline bint validate_map(
return False
for v in datum.values():
- if not _validate(datum=v, schema=schema["values"],
+ if not _validate(
+ datum=v,
+ schema=schema["values"],
named_schemas=named_schemas,
field=parent_ns,
- raise_errors=raise_errors):
+ raise_errors=raise_errors,
+ strict=strict,
+ ):
return False
return True
@@ -121,18 +133,29 @@ cdef inline bint validate_record(
dict named_schemas,
str parent_ns="",
bint raise_errors=True,
+ bint strict=False,
) except -1:
if not isinstance(datum, Mapping):
return False
_, fullname = schema_name(schema, parent_ns)
if "-type" in datum and datum["-type"] != fullname:
return False
+
for f in schema["fields"]:
- if not _validate(datum=datum.get(f["name"], f.get("default")),
+ datum_value = datum.get(f["name"], f.get("default", NoValue))
+ if datum_value is NoValue and strict:
+ return False
+ elif datum_value is NoValue:
+ datum_value = None
+
+ if not _validate(
+ datum=datum_value,
schema=f["type"],
named_schemas=named_schemas,
field=f"{fullname}.{f['name']}",
- raise_errors=raise_errors):
+ raise_errors=raise_errors,
+ strict=strict,
+ ):
return False
return True
@@ -143,6 +166,7 @@ cdef inline bint validate_union(
dict named_schemas,
str parent_ns=None,
bint raise_errors=True,
+ bint strict=False,
) except -1:
if isinstance(datum, tuple):
(name, datum) = datum
@@ -152,20 +176,28 @@ cdef inline bint validate_union(
else:
schema_name = candidate
if schema_name == name:
- return _validate(datum, schema=candidate,
+ return _validate(
+ datum,
+ schema=candidate,
named_schemas=named_schemas,
field=parent_ns,
- raise_errors=raise_errors)
+ raise_errors=raise_errors,
+ strict=strict,
+ )
else:
return False
cdef list errors = []
for s in schema:
try:
- ret = _validate(datum, schema=s,
+ ret = _validate(
+ datum,
+ schema=s,
named_schemas=named_schemas,
field=parent_ns,
- raise_errors=raise_errors)
+ raise_errors=raise_errors,
+ strict=strict,
+ )
if ret:
# We exit on the first passing type in Unions
return True
@@ -182,6 +214,7 @@ cpdef _validate(
dict named_schemas,
str field="",
bint raise_errors=True,
+ bint strict=False,
):
record_type = extract_record_type(schema)
result = None
@@ -218,6 +251,7 @@ cpdef _validate(
named_schemas=named_schemas,
parent_ns=field,
raise_errors=raise_errors,
+ strict=strict,
)
elif record_type == "map":
result = validate_map(
@@ -226,6 +260,7 @@ cpdef _validate(
named_schemas=named_schemas,
parent_ns=field,
raise_errors=raise_errors,
+ strict=strict,
)
elif record_type in ("union", "error_union"):
result = validate_union(
@@ -234,6 +269,7 @@ cpdef _validate(
named_schemas=named_schemas,
parent_ns=field,
raise_errors=raise_errors,
+ strict=strict,
)
elif record_type in ("record", "error", "request"):
result = validate_record(
@@ -242,13 +278,17 @@ cpdef _validate(
named_schemas=named_schemas,
parent_ns=field,
raise_errors=raise_errors,
+ strict=strict,
)
elif record_type in named_schemas:
- result = _validate(datum,
+ result = _validate(
+ datum,
schema=named_schemas[record_type],
named_schemas=named_schemas,
field=field,
- raise_errors=raise_errors)
+ raise_errors=raise_errors,
+ strict=strict,
+ )
else:
raise UnknownType(record_type)
@@ -258,14 +298,15 @@ cpdef _validate(
return bool(result)
-cpdef validate(object datum, object schema, str field="",
- bint raise_errors=True):
+cpdef validate(
+ object datum, object schema, str field="", bint raise_errors=True, bint strict=False
+):
named_schemas = {}
parsed_schema = parse_schema(schema, named_schemas)
- return _validate(datum, parsed_schema, named_schemas, field, raise_errors)
+ return _validate(datum, parsed_schema, named_schemas, field, raise_errors, strict)
-cpdef validate_many(records, schema, bint raise_errors=True):
+cpdef validate_many(records, schema, bint raise_errors=True, bint strict=False):
cdef bint result
cdef list errors = []
cdef list results = []
@@ -274,7 +315,11 @@ cpdef validate_many(records, schema, bint raise_errors=True):
for record in records:
try:
result = _validate(
- record, parsed_schema, named_schemas, raise_errors=raise_errors
+ record,
+ parsed_schema,
+ named_schemas,
+ raise_errors=raise_errors,
+ strict=strict,
)
results.append(result)
except ValidationError as e:

MODIFY
fastavro/_validation_py.py
fastavro/_validation_py.py
@@ -10,6 +10,8 @@ from .logical_writers import LOGICAL_WRITERS
from ._schema_common import UnknownType
from .types import Schema, NamedSchemas
+NoValue = object()
+
def validate_null(datum, **kwargs):
"""
@@ -166,7 +168,9 @@ def validate_enum(datum, schema, **kwargs):
return datum in schema["symbols"]
-def validate_array(datum, schema, named_schemas, parent_ns=None, raise_errors=True):
+def validate_array(
+ datum, schema, named_schemas, parent_ns=None, raise_errors=True, strict=False
+):
"""
Check that the data list values all match schema['items'].
@@ -191,13 +195,16 @@ def validate_array(datum, schema, named_schemas, parent_ns=None, raise_errors=Tr
named_schemas=named_schemas,
field=parent_ns,
raise_errors=raise_errors,
+ strict=strict,
)
for d in datum
)
)
-def validate_map(datum, schema, named_schemas, parent_ns=None, raise_errors=True):
+def validate_map(
+ datum, schema, named_schemas, parent_ns=None, raise_errors=True, strict=False
+):
"""
Check that the data is a Map(k,v)
matching values to schema['values'] type.
@@ -223,13 +230,16 @@ def validate_map(datum, schema, named_schemas, parent_ns=None, raise_errors=True
named_schemas=named_schemas,
field=parent_ns,
raise_errors=raise_errors,
+ strict=strict,
)
for v in datum.values()
)
)
-def validate_record(datum, schema, named_schemas, parent_ns=None, raise_errors=True):
+def validate_record(
+ datum, schema, named_schemas, parent_ns=None, raise_errors=True, strict=False
+):
"""
Check that the data is a Mapping type with all schema defined fields
validated as True.
@@ -251,18 +261,21 @@ def validate_record(datum, schema, named_schemas, parent_ns=None, raise_errors=T
and not ("-type" in datum and datum["-type"] != fullname)
and all(
_validate(
- datum=datum.get(f["name"], f.get("default")),
+ datum=datum.get(f["name"], f.get("default", NoValue)),
schema=f["type"],
named_schemas=named_schemas,
field=f"{fullname}.{f['name']}",
raise_errors=raise_errors,
+ strict=strict,
)
for f in schema["fields"]
)
)
-def validate_union(datum, schema, named_schemas, parent_ns=None, raise_errors=True):
+def validate_union(
+ datum, schema, named_schemas, parent_ns=None, raise_errors=True, strict=False
+):
"""
Check that the data is a list type with possible options to
validate as True.
@@ -292,6 +305,7 @@ def validate_union(datum, schema, named_schemas, parent_ns=None, raise_errors=Tr
named_schemas=named_schemas,
field=parent_ns,
raise_errors=raise_errors,
+ strict=strict,
)
else:
return False
@@ -305,6 +319,7 @@ def validate_union(datum, schema, named_schemas, parent_ns=None, raise_errors=Tr
named_schemas=named_schemas,
field=parent_ns,
raise_errors=raise_errors,
+ strict=strict,
)
if ret:
# We exit on the first passing type in Unions
@@ -337,11 +352,17 @@ VALIDATORS = {
}
-def _validate(datum, schema, named_schemas, field="", raise_errors=True):
+def _validate(datum, schema, named_schemas, field="", raise_errors=True, strict=False):
# This function expects the schema to already be parsed
record_type = extract_record_type(schema)
result = None
+ if datum is NoValue and strict:
+ result = False
+ else:
+ if datum is NoValue:
+ datum = None
+
logical_type = extract_logical_type(schema)
if logical_type:
prepare = LOGICAL_WRITERS.get(logical_type)
@@ -356,6 +377,7 @@ def _validate(datum, schema, named_schemas, field="", raise_errors=True):
named_schemas=named_schemas,
parent_ns=field,
raise_errors=raise_errors,
+ strict=strict,
)
elif record_type in named_schemas:
result = _validate(
@@ -364,6 +386,7 @@ def _validate(datum, schema, named_schemas, field="", raise_errors=True):
named_schemas=named_schemas,
field=field,
raise_errors=raise_errors,
+ strict=strict,
)
else:
raise UnknownType(record_type)
@@ -375,7 +398,11 @@ def _validate(datum, schema, named_schemas, field="", raise_errors=True):
def validate(
- datum: Any, schema: Schema, field: str = "", raise_errors: bool = True
+ datum: Any,
+ schema: Schema,
+ field: str = "",
+ raise_errors: bool = True,
+ strict: bool = False,
) -> bool:
"""
Determine if a python datum is an instance of a schema.
@@ -391,6 +418,9 @@ def validate(
raise_errors
If true, errors are raised for invalid data. If false, a simple
True (valid) or False (invalid) result is returned
+ strict
+ If true, fields without values will raise errors rather than implicitly
+ defaulting to None
Example::
@@ -402,11 +432,14 @@ def validate(
"""
named_schemas: NamedSchemas = {}
parsed_schema = parse_schema(schema, named_schemas)
- return _validate(datum, parsed_schema, named_schemas, field, raise_errors)
+ return _validate(datum, parsed_schema, named_schemas, field, raise_errors, strict)
def validate_many(
- records: Iterable[Any], schema: Schema, raise_errors: bool = True
+ records: Iterable[Any],
+ schema: Schema,
+ raise_errors: bool = True,
+ strict: bool = False,
) -> bool:
"""
Validate a list of data!
@@ -420,6 +453,9 @@ def validate_many(
raise_errors
If true, errors are raised for invalid data. If false, a simple
True (valid) or False (invalid) result is returned
+ strict
+ If true, fields without values will raise errors rather than implicitly
+ defaulting to None
Example::
@@ -437,7 +473,11 @@ def validate_many(
try:
results.append(
_validate(
- record, parsed_schema, named_schemas, raise_errors=raise_errors
+ record,
+ parsed_schema,
+ named_schemas,
+ raise_errors=raise_errors,
+ strict=strict,
)
)
except ValidationError as e:

MODIFY
tests/test_validation.py
tests/test_validation.py
@@ -592,3 +592,24 @@ def test_validate_should_not_parse_schema_if_it_was_parsed_already():
validate_many(records, a_schema)
validate(records[0], a_schema)
+
+
+def test_validate_strict():
+ """https://github.com/fastavro/fastavro/issues/423"""
+ schema = {
+ "type": "record",
+ "name": "test_validate_strict",
+ "fields": [
+ {"name": "normal_field", "type": "string"},
+ {"name": "normal_union_field", "type": ["int", "float"]},
+ {"name": "nullable_union_field", "type": ["null", "string"]},
+ ],
+ }
+
+ record = {"normal_field": "a", "normal_union_field": 1}
+ parsed_schema = parse_schema(schema)
+ with pytest.raises(ValidationError):
+ validate(record, parsed_schema, strict=True)
+
+ with pytest.raises(ValidationError):
+ validate_many([record], parsed_schema, strict=True)

add strict option in validation