always parse the schema when calling validate
MODIFY
fastavro/_validation.pyx
fastavro/_validation.pyx
@@ -12,7 +12,9 @@ except ImportError:
from . import const
from ._six import long, is_str, iterkeys, itervalues
-from ._schema import extract_record_type, extract_logical_type, schema_name
+from ._schema import (
+ extract_record_type, extract_logical_type, schema_name, parse_schema
+)
from ._logical_writers import LOGICAL_WRITERS
from ._schema_common import SCHEMA_DEFS, UnknownType
from ._validate_common import ValidationError, ValidationErrorData
@@ -97,7 +99,7 @@ cdef inline bint validate_array(datum, dict schema,
return False
for d in datum:
- if not validate(datum=d, schema=schema['items'],
+ if not _validate(datum=d, schema=schema['items'],
field=parent_ns,
raise_errors=raise_errors):
return False
@@ -114,7 +116,7 @@ cdef inline bint validate_map(object datum, dict schema, str parent_ns='',
return False
for v in itervalues(datum):
- if not validate(datum=v, schema=schema['values'],
+ if not _validate(datum=v, schema=schema['values'],
field=parent_ns,
raise_errors=raise_errors):
return False
@@ -127,7 +129,7 @@ cdef inline bint validate_record(object datum, dict schema, str parent_ns='',
return False
_, namespace = schema_name(schema, parent_ns)
for f in schema['fields']:
- if not validate(datum=datum.get(f['name'], f.get('default')),
+ if not _validate(datum=datum.get(f['name'], f.get('default')),
schema=f['type'],
field='{}.{}'.format(namespace, f['name']),
raise_errors=raise_errors):
@@ -142,7 +144,7 @@ cdef inline bint validate_union(object datum, list schema, str parent_ns=None,
for candidate in schema:
if extract_record_type(candidate) == 'record':
if name == candidate["name"]:
- return validate(datum, schema=candidate,
+ return _validate(datum, schema=candidate,
field=parent_ns,
raise_errors=raise_errors)
else:
@@ -151,7 +153,7 @@ cdef inline bint validate_union(object datum, list schema, str parent_ns=None,
cdef list errors = []
for s in schema:
try:
- ret = validate(datum, schema=s,
+ ret = _validate(datum, schema=s,
field=parent_ns,
raise_errors=raise_errors)
if ret:
@@ -164,7 +166,7 @@ cdef inline bint validate_union(object datum, list schema, str parent_ns=None,
return False
-cpdef validate(object datum, object schema, str field='',
+cpdef _validate(object datum, object schema, str field='',
bint raise_errors=True):
record_type = extract_record_type(schema)
result = None
@@ -216,7 +218,7 @@ cpdef validate(object datum, object schema, str field='',
result = validate_record(datum, schema=schema, parent_ns=field,
raise_errors=raise_errors)
elif record_type in SCHEMA_DEFS:
- result = validate(datum,
+ result = _validate(datum,
schema=SCHEMA_DEFS[record_type],
field=field,
raise_errors=raise_errors)
@@ -229,13 +231,20 @@ cpdef validate(object datum, object schema, str field='',
return bool(result)
+cpdef validate(object datum, object schema, str field='',
+ bint raise_errors=True):
+ parsed_schema = parse_schema(schema, _force=True)
+ return _validate(datum, parsed_schema, field, raise_errors)
+
+
cpdef validate_many(records, schema, bint raise_errors=True):
cdef bint result
cdef list errors = []
cdef list results = []
+ parsed_schema = parse_schema(schema, _force=True)
for record in records:
try:
- result = validate(record, schema, raise_errors=raise_errors)
+ result = _validate(record, parsed_schema, raise_errors=raise_errors)
results.append(result)
except ValidationError as e:
errors.extend(e.errors)

MODIFY
fastavro/_validation_py.py
fastavro/_validation_py.py
@@ -13,11 +13,11 @@ from fastavro.const import (
)
from ._validate_common import ValidationError, ValidationErrorData
from .schema import (
- extract_record_type, extract_logical_type, UnknownType, schema_name
+ extract_record_type, extract_logical_type, schema_name, parse_schema
)
from .logical_writers import LOGICAL_WRITERS
from .six import long, is_str, iterkeys, itervalues
-from ._schema_common import SCHEMA_DEFS
+from ._schema_common import SCHEMA_DEFS, UnknownType
def validate_null(datum, **kwargs):
@@ -212,7 +212,7 @@ def validate_array(datum, schema, parent_ns=None, raise_errors=True):
return (
isinstance(datum, Sequence) and
not is_str(datum) and
- all(validate(datum=d, schema=schema['items'],
+ all(_validate(datum=d, schema=schema['items'],
field=parent_ns,
raise_errors=raise_errors) for d in datum)
)
@@ -237,9 +237,14 @@ def validate_map(datum, schema, parent_ns=None, raise_errors=True):
return (
isinstance(datum, Mapping) and
all(is_str(k) for k in iterkeys(datum)) and
- all(validate(datum=v, schema=schema['values'],
+ all(
+ _validate(
+ datum=v,
+ schema=schema['values'],
field=parent_ns,
- raise_errors=raise_errors) for v in itervalues(datum))
+ raise_errors=raise_errors
+ ) for v in itervalues(datum)
+ )
)
@@ -262,7 +267,7 @@ def validate_record(datum, schema, parent_ns=None, raise_errors=True):
_, namespace = schema_name(schema, parent_ns)
return (
isinstance(datum, Mapping) and
- all(validate(datum=datum.get(f['name'], f.get('default')),
+ all(_validate(datum=datum.get(f['name'], f.get('default')),
schema=f['type'],
field='{}.{}'.format(namespace, f['name']),
raise_errors=raise_errors)
@@ -292,18 +297,24 @@ def validate_union(datum, schema, parent_ns=None, raise_errors=True):
for candidate in schema:
if extract_record_type(candidate) == 'record':
if name == candidate["name"]:
- return validate(datum, schema=candidate,
+ return _validate(
+ datum,
+ schema=candidate,
field=parent_ns,
- raise_errors=raise_errors)
+ raise_errors=raise_errors,
+ )
else:
return False
errors = []
for s in schema:
try:
- ret = validate(datum, schema=s,
+ ret = _validate(
+ datum,
+ schema=s,
field=parent_ns,
- raise_errors=raise_errors)
+ raise_errors=raise_errors,
+ )
if ret:
# We exit on the first passing type in Unions
return True
@@ -335,30 +346,8 @@ VALIDATORS = {
}
-def validate(datum, schema, field=None, raise_errors=True):
- """
- Determine if a python datum is an instance of a schema.
-
- Parameters
- ----------
- datum: Any
- Data being validated
- schema: dict
- Schema
- field: str, optional
- Record field being validated
- raise_errors: bool, optional
- If true, errors are raised for invalid data. If false, a simple
- True (valid) or False (invalid) result is returned
-
-
- Example::
-
- from fastavro.validation import validate
- schema = {...}
- record = {...}
- validate(record, schema)
- """
+def _validate(datum, schema, field=None, raise_errors=True):
+ # This function expects the schema to already be parsed
record_type = extract_record_type(schema)
result = None
@@ -374,10 +363,12 @@ def validate(datum, schema, field=None, raise_errors=True):
parent_ns=field,
raise_errors=raise_errors)
elif record_type in SCHEMA_DEFS:
- result = validate(datum,
+ result = _validate(
+ datum,
schema=SCHEMA_DEFS[record_type],
field=field,
- raise_errors=raise_errors)
+ raise_errors=raise_errors,
+ )
else:
raise UnknownType(record_type)
@@ -387,6 +378,34 @@ def validate(datum, schema, field=None, raise_errors=True):
return result
+def validate(datum, schema, field=None, raise_errors=True):
+ """
+ Determine if a python datum is an instance of a schema.
+
+ Parameters
+ ----------
+ datum: Any
+ Data being validated
+ schema: dict
+ Schema
+ field: str, optional
+ Record field being validated
+ raise_errors: bool, optional
+ If true, errors are raised for invalid data. If false, a simple
+ True (valid) or False (invalid) result is returned
+
+
+ Example::
+
+ from fastavro.validation import validate
+ schema = {...}
+ record = {...}
+ validate(record, schema)
+ """
+ parsed_schema = parse_schema(schema, _force=True)
+ return _validate(datum, parsed_schema, field, raise_errors)
+
+
def validate_many(records, schema, raise_errors=True):
"""
Validate a list of data!
@@ -409,11 +428,14 @@ def validate_many(records, schema, raise_errors=True):
records = [{...}, {...}, ...]
validate_many(records, schema)
"""
+ parsed_schema = parse_schema(schema, _force=True)
errors = []
results = []
for record in records:
try:
- results.append(validate(record, schema, raise_errors=raise_errors))
+ results.append(
+ _validate(record, parsed_schema, raise_errors=raise_errors)
+ )
except ValidationError as e:
errors.extend(e.errors)
if raise_errors and errors:

MODIFY
tests/test_validation.py
tests/test_validation.py
@@ -5,6 +5,7 @@ from fastavro.validation import (
validate,
validate_many
)
+from fastavro import parse_schema
import pytest
import numpy as np
import sys
@@ -373,3 +374,93 @@ def test_validator_numeric_numpy():
for nptype, schema in zip(np_floats, schema_ints):
with pytest.raises(ValidationError):
validate(nptype(1), schema)
+
+
+def test_validate_with_unparsed_schema():
+ """https://github.com/fastavro/fastavro/issues/416"""
+ schema = {
+ "type": "record",
+ "name": "my_schema_v1",
+ "fields": [
+ {
+ "name": "field1",
+ "type": {
+ "type": "record",
+ "name": "my_field_type",
+ "fields": [
+ {
+ "name": "sub_field",
+ "type": "string"
+ }
+ ]
+ }
+ },
+ {
+ "name": "field2",
+ "type": "my_field_type"
+ }
+ ]
+ }
+
+ datum = {
+ "field1": {"sub_field": "foo"}, "field2": {"sub_field": "bar"}
+ }
+
+ validate(datum, schema)
+
+
+def test_validate_clobbering_SCHEMA_DEFS():
+ """https://github.com/fastavro/fastavro/issues/416"""
+ schema_1 = {
+ "type": "record",
+ "name": "my_schema_v1",
+ "fields": [
+ {
+ "name": "field1",
+ "type": {
+ "type": "record",
+ "name": "my_field_type",
+ "fields": [
+ {
+ "name": "sub_field",
+ "type": "string"
+ }
+ ]
+ }
+ },
+ {
+ "name": "field2",
+ "type": "my_field_type"
+ }
+ ]
+ }
+ parsed_schema_1 = parse_schema(schema_1)
+
+ datum_1 = {
+ "field1": {"sub_field": "foo"}, "field2": {"sub_field": "bar"}
+ }
+
+ validate(datum_1, parsed_schema_1)
+
+ schema_2 = {
+ "type": "record",
+ "name": "my_schema_v2",
+ "fields": [
+ {
+ "name": "field1",
+ "type": {
+ "type": "record",
+ "name": "my_field_type",
+ "fields": [
+ {
+ "name": "sub_field",
+ "type": "int"
+ }
+ ]
+ }
+ }
+ ]
+ }
+ parse_schema(schema_2)
+
+ validate(datum_1, parsed_schema_1)

always parse the schema when calling validate (#417)