make validate functions raise exceptions by default
MODIFY
benchmark/benchmark.py
benchmark/benchmark.py
@@ -5,7 +5,7 @@ import time
from fastavro import writer, reader
from fastavro._timezone import utc
-from fastavro.validate import validate, validate_many
+from fastavro.validation import validate, validate_many
def write(schema, records, runs=1):

MODIFY
fastavro/_validation.pyx
fastavro/_validation.pyx
@@ -20,7 +20,7 @@ cdef long64 LONG_MIN_VALUE = const.LONG_MIN_VALUE
cdef long64 LONG_MAX_VALUE = const.LONG_MAX_VALUE
cdef inline bint validate_null(datum, schema=None,
- str parent_ns='', bint raise_errors=False):
+ str parent_ns='', bint raise_errors=True):
"""
Checks that the data value is None.
@@ -33,7 +33,7 @@ cdef inline bint validate_null(datum, schema=None,
return datum is None
cdef inline bint validate_boolean(datum, schema=None,
- str parent_ns='', bint raise_errors=False):
+ str parent_ns='', bint raise_errors=True):
"""Check that the data value is bool instance
:param datum: data to validate as boolean
@@ -45,7 +45,7 @@ cdef inline bint validate_boolean(datum, schema=None,
return isinstance(datum, bool)
cdef inline bint validate_string(datum, schema=None,
- str parent_ns='', bint raise_errors=False):
+ str parent_ns='', bint raise_errors=True):
"""Check that the data value is string type, uses
six for Python version compatibility.
@@ -58,7 +58,7 @@ cdef inline bint validate_string(datum, schema=None,
return is_str(datum)
cdef inline bint validate_bytes(datum, schema=None,
- str parent_ns='', bint raise_errors=False):
+ str parent_ns='', bint raise_errors=True):
"""
Check that the data value is
(bytes or decimal.Decimal)
@@ -72,7 +72,7 @@ cdef inline bint validate_bytes(datum, schema=None,
return isinstance(datum, (bytes, decimal.Decimal))
cdef inline bint validate_int(datum, schema=None,
- str parent_ns='', bint raise_errors=False):
+ str parent_ns='', bint raise_errors=True):
"""
Check that the data value is a non floating
point number with size less that Int32.
@@ -96,7 +96,7 @@ cdef inline bint validate_int(datum, schema=None,
)
cdef inline bint validate_long(datum, schema=None,
- str parent_ns='', bint raise_errors=False):
+ str parent_ns='', bint raise_errors=True):
"""
Check that the data value is a non floating
point number with size less that Int32.
@@ -119,7 +119,7 @@ cdef inline bint validate_long(datum, schema=None,
)
cdef inline bint validate_float(datum, schema=None,
- str parent_ns='', bint raise_errors=False):
+ str parent_ns='', bint raise_errors=True):
"""
Check that the data value is a floating
point number or double precision.
@@ -133,7 +133,7 @@ cdef inline bint validate_float(datum, schema=None,
return isinstance(datum, (int, long, float, numbers.Real))
cdef inline bint validate_fixed(datum, dict schema,
- str parent_ns='', bint raise_errors=False):
+ str parent_ns='', bint raise_errors=True):
"""
Check that the data value is fixed width bytes,
matching the schema['size'] exactly!
@@ -149,7 +149,7 @@ cdef inline bint validate_fixed(datum, dict schema,
(isinstance(datum, decimal.Decimal))
cdef inline bint validate_enum(datum, dict schema,
- str parent_ns='', bint raise_errors=False):
+ str parent_ns='', bint raise_errors=True):
"""
Check that the data value matches one of the enum symbols.
@@ -164,7 +164,7 @@ cdef inline bint validate_enum(datum, dict schema,
return datum in schema['symbols']
cdef inline bint validate_array(datum, dict schema,
- str parent_ns='', bint raise_errors=False):
+ str parent_ns='', bint raise_errors=True):
"""
Check that the data list values all match schema['items'].
@@ -190,7 +190,7 @@ cdef inline bint validate_array(datum, dict schema,
return True
cdef inline bint validate_map(object datum, dict schema, str parent_ns='',
- bint raise_errors=False):
+ bint raise_errors=True):
"""
Check that the data is a Map(k,v)
matching values to schema['values'] type.
@@ -221,7 +221,7 @@ cdef inline bint validate_map(object datum, dict schema, str parent_ns='',
return True
cdef inline bint validate_record(object datum, dict schema, str parent_ns='',
- bint raise_errors=False):
+ bint raise_errors=True):
"""
Check that the data is a Mapping type with all schema defined fields
validated as True.
@@ -248,7 +248,7 @@ cdef inline bint validate_record(object datum, dict schema, str parent_ns='',
return True
cdef inline bint validate_union(object datum, list schema, str parent_ns=None,
- bint raise_errors=False):
+ bint raise_errors=True):
"""
Check that the data is a list type with possible options to
validate as True.
@@ -288,7 +288,7 @@ cdef inline bint validate_union(object datum, list schema, str parent_ns=None,
return False
cpdef validate(object datum, object schema, str field='',
- bint raise_errors=False):
+ bint raise_errors=True):
"""Determine if a python datum is an instance of a schema."""
record_type = extract_record_type(schema)
result = None
@@ -353,18 +353,16 @@ cpdef validate(object datum, object schema, str field='',
return bool(result)
-cpdef validate_many(records, schema, bint raise_errors=False, int stop_count=-1):
+cpdef validate_many(records, schema, bint raise_errors=True):
"""
Validate a list of data!
:param records: Iterable: list of records to validate
:param schema: Avro schema
:param raise_errors: bool: should raise ValidationError
- :param stop_count: int: stop early for error count
:return: bool
:except: ValidationError
"""
- cdef int error_count = 0
cdef bint result
cdef list errors = []
cdef list results = []
@@ -373,10 +371,7 @@ cpdef validate_many(records, schema, bint raise_errors=False, int stop_count=-1)
result = validate(record, schema, raise_errors=raise_errors)
results.append(result)
except ValidationError as e:
- error_count += 1
errors.extend(e.errors)
- if error_count >= stop_count:
- break
if raise_errors:
raise ValidationError(*errors)
return all(results)

MODIFY
fastavro/_validation_py.py
fastavro/_validation_py.py
@@ -150,7 +150,7 @@ def validate_enum(datum, schema, **kwargs):
return datum in schema['symbols']
-def validate_array(datum, schema, parent_ns=None, raise_errors=False):
+def validate_array(datum, schema, parent_ns=None, raise_errors=True):
"""
Check that the data list values all match schema['items'].
@@ -174,7 +174,7 @@ def validate_array(datum, schema, parent_ns=None, raise_errors=False):
)
-def validate_map(datum, schema, parent_ns=None, raise_errors=False):
+def validate_map(datum, schema, parent_ns=None, raise_errors=True):
"""
Check that the data is a Map(k,v)
matching values to schema['values'] type.
@@ -199,7 +199,7 @@ def validate_map(datum, schema, parent_ns=None, raise_errors=False):
)
-def validate_record(datum, schema, parent_ns=None, raise_errors=False):
+def validate_record(datum, schema, parent_ns=None, raise_errors=True):
"""
Check that the data is a Mapping type with all schema defined fields
validated as True.
@@ -226,7 +226,7 @@ def validate_record(datum, schema, parent_ns=None, raise_errors=False):
)
-def validate_union(datum, schema, parent_ns=None, raise_errors=False):
+def validate_union(datum, schema, parent_ns=None, raise_errors=True):
"""
Check that the data is a list type with possible options to
validate as True.
@@ -287,7 +287,7 @@ VALIDATORS = {
}
-def validate(datum, schema, field=None, raise_errors=False):
+def validate(datum, schema, field=None, raise_errors=True):
"""Determine if a python datum is an instance of a schema."""
record_type = extract_record_type(schema)
result = None
@@ -317,28 +317,23 @@ def validate(datum, schema, field=None, raise_errors=False):
return result
-def validate_many(records, schema, raise_errors=False, stop_count=-1):
+def validate_many(records, schema, raise_errors=True):
"""
Validate a list of data!
:param records: Iterable: list of records to validate
:param schema: Avro schema
:param raise_errors: bool: should raise ValidationError
- :param stop_count: int: -1 never stop validation loop
:return: bool
:except: ValidationError
"""
- error_count = 0
errors = []
results = []
for record in records:
try:
results.append(validate(record, schema, raise_errors=raise_errors))
except ValidationError as e:
- error_count += 1
errors.extend(e.errors)
- if error_count == stop_count:
- break
if raise_errors:
raise ValidationError(*errors)
return all(results)

MODIFY
fastavro/_write.pyx
fastavro/_write.pyx
@@ -427,7 +427,7 @@ cpdef write_union(bytearray fo, datum, schema):
best_match_index = -1
most_fields = -1
for index, candidate in enumerate(schema):
- if validate(datum, candidate):
+ if validate(datum, candidate, raise_errors=False):
if extract_record_type(candidate) == 'record':
fields = len(candidate['fields'])
if fields > most_fields:

MODIFY
fastavro/_write_py.py
fastavro/_write_py.py
@@ -327,7 +327,7 @@ def write_union(fo, datum, schema):
best_match_index = -1
most_fields = -1
for index, candidate in enumerate(schema):
- if validate(datum, candidate):
+ if validate(datum, candidate, raise_errors=False):
if extract_record_type(candidate) == 'record':
fields = len(candidate['fields'])
if fields > most_fields:
@@ -553,7 +553,7 @@ def writer(fo,
Header metadata
validator: None, True or a function
Validator function. If None (the default) - no validation. If True then
- then fastavro.writer.validate will be used. If it's a function, it
+ then fastavro.validation.validate will be used. If it's a function, it
should have the same signature as fastavro.writer.validate and raise an
exeption on error.

DELETE
tests/test_validate.py
-from fastavro.validation import validate
-
-
-def test_validator_numeric():
- for datum, schema in [
- (1, 'int'),
- (1, 'long'),
- (1.0, 'float'),
- (1.0, 'double'),
- (1, 'float'),
- (1, 'double'),
- ]:
- assert validate(datum, schema)
-
- for datum, schema in [
- (1.0, 'int'),
- (1.0, 'long'),
- ("1.0", 'float'),
- ("1.0", 'double'),
- ("1", 'float'),
- ("1", 'double'),
- ]:
- assert not validate(datum, schema)
- # and plenty more to add I suppose
-
-
-def test_validator_numeric_numpy():
- import numpy as np
- np_ints = [
- np.int_,
- np.intc,
- np.intp,
- np.int8,
- np.int16,
- np.int32,
- np.int64,
- np.uint8,
- np.uint16,
- np.uint32,
- np.uint64,
- ]
-
- np_floats = [
- np.float_,
- np.float16,
- np.float32,
- np.float64,
- ]
-
- schema_ints = ['int', 'long']
-
- schema_floats = ['float', 'double']
-
- # all these should work
- for nptype, schema in zip(np_ints, schema_ints):
- assert validate(nptype(1), schema)
-
- for nptype, schema in zip(np_ints, schema_floats):
- assert validate(nptype(1), schema)
-
- for nptype, schema in zip(np_floats, schema_floats):
- assert validate(nptype(1), schema)
-
- # these shouldn't work
- for nptype, schema in zip(np_floats, schema_ints):
- assert not validate(nptype(1), schema)

MODIFY
tests/test_validation.py
tests/test_validation.py
from fastavro.validation import (
ValidationError,
ValidationErrorData,
+ validate,
validate_many
)
import pytest
+import numpy as np
schema = {
"fields": [
@@ -170,3 +172,69 @@ def test_validate_error_raises():
# PY2 says <type 'int'> and PY3 says <class 'int'> so just split there
assert "test1 is <10> of type <" in str(error)
assert "'int'> expected string" in str(error)
+
+
+def test_validator_numeric():
+ for datum, schema in [
+ (1, 'int'),
+ (1, 'long'),
+ (1.0, 'float'),
+ (1.0, 'double'),
+ (1, 'float'),
+ (1, 'double'),
+ ]:
+ validate(datum, schema)
+
+ for datum, schema in [
+ (1.0, 'int'),
+ (1.0, 'long'),
+ ("1.0", 'float'),
+ ("1.0", 'double'),
+ ("1", 'float'),
+ ("1", 'double'),
+ ]:
+ with pytest.raises(ValidationError):
+ validate(datum, schema)
+ # and plenty more to add I suppose
+
+
+def test_validator_numeric_numpy():
+ np_ints = [
+ np.int_,
+ np.intc,
+ np.intp,
+ np.int8,
+ np.int16,
+ np.int32,
+ np.int64,
+ np.uint8,
+ np.uint16,
+ np.uint32,
+ np.uint64,
+ ]
+
+ np_floats = [
+ np.float_,
+ np.float16,
+ np.float32,
+ np.float64,
+ ]
+
+ schema_ints = ['int', 'long']
+
+ schema_floats = ['float', 'double']
+
+ # all these should work
+ for nptype, schema in zip(np_ints, schema_ints):
+ validate(nptype(1), schema)
+
+ for nptype, schema in zip(np_ints, schema_floats):
+ validate(nptype(1), schema)
+
+ for nptype, schema in zip(np_floats, schema_floats):
+ validate(nptype(1), schema)
+
+ # these shouldn't work
+ for nptype, schema in zip(np_floats, schema_ints):
+ with pytest.raises(ValidationError):
+ validate(nptype(1), schema)

make validate functions raise exceptions by default