change validate module to validation and misc cleanup
MODIFY
benchmark/benchmark.py
benchmark/benchmark.py
@@ -5,17 +5,7 @@ import time
from fastavro import writer, reader
from fastavro._timezone import utc
-try:
from fastavro.validate import validate, validate_many
-except ImportError:
- try:
- from fastavro._write import validate
- except ImportError:
- from fastavro._write_py import validate
-
-
- def validate_many(records, schema):
- return all([validate(record, schema) for record in records])
def write(schema, records, runs=1):

MODIFY
fastavro/__init__.py
fastavro/__init__.py
@@ -47,7 +47,7 @@ __version__ = '%s.%s.%s' % __version_info__
import fastavro.read
import fastavro.write
import fastavro.schema
-import fastavro.validate
+import fastavro.validation
def _acquaint_schema(schema):
@@ -71,7 +71,7 @@ dump = fastavro.write.dump
acquaint_schema = _acquaint_schema
fastavro.schema.acquaint_schema = _acquaint_schema
is_avro = fastavro.read.is_avro
-validator = fastavro.validate.validate
+validate = fastavro.validation.validate
__all__ = [
n for n in locals().keys() if not n.startswith('_')

MODIFY
fastavro/_schema.pyx
fastavro/_schema.pyx
@@ -28,7 +28,7 @@ cpdef inline str extract_logical_type(schema):
return None
-def schema_name(object schema, parent_ns):
+def schema_name(schema, parent_ns):
name = schema.get('name')
if not name:
return parent_ns, None

MODIFY
fastavro/_validate_common.py
fastavro/_validate_common.py
@@ -9,10 +9,10 @@ class ValidationErrorData(namedtuple('ValidationErrorData',
self.field = ''
if self.datum is None:
- return 'Field({field}) is null' \
+ return 'Field({field}) is None' \
' expected {schema}'.format(field=self.field,
schema=self.schema)
- return '{field} is {datum} of type ' \
+ return '{field} is <{datum}> of type ' \
'{given_type} expected {schema}'. \
format(datum=self.datum, given_type=type(self.datum),
schema=self.schema, field=self.field)

RENAME
fastavro/_validate.pyx
fastavro/_validation.pyx
@@ -19,7 +19,7 @@ cdef int32 INT_MAX_VALUE = const.INT_MAX_VALUE
cdef long64 LONG_MIN_VALUE = const.LONG_MIN_VALUE
cdef long64 LONG_MAX_VALUE = const.LONG_MAX_VALUE
-cpdef bint validate_null(datum, schema=None,
+cdef inline bint validate_null(datum, schema=None,
str parent_ns='', bint raise_errors=False):
"""
Checks that the data value is None.
@@ -32,7 +32,7 @@ cpdef bint validate_null(datum, schema=None,
"""
return datum is None
-cpdef bint validate_boolean(datum, schema=None,
+cdef inline bint validate_boolean(datum, schema=None,
str parent_ns='', bint raise_errors=False):
"""Check that the data value is bool instance
@@ -44,7 +44,7 @@ cpdef bint validate_boolean(datum, schema=None,
"""
return isinstance(datum, bool)
-cpdef bint validate_string(datum, schema=None,
+cdef inline bint validate_string(datum, schema=None,
str parent_ns='', bint raise_errors=False):
"""Check that the data value is string type, uses
six for Python version compatibility.
@@ -57,7 +57,7 @@ cpdef bint validate_string(datum, schema=None,
"""
return is_str(datum)
-cpdef bint validate_bytes(datum, schema=None,
+cdef inline bint validate_bytes(datum, schema=None,
str parent_ns='', bint raise_errors=False):
"""
Check that the data value is
@@ -71,7 +71,7 @@ cpdef bint validate_bytes(datum, schema=None,
"""
return isinstance(datum, (bytes, decimal.Decimal))
-cpdef bint validate_int(datum, schema=None,
+cdef inline bint validate_int(datum, schema=None,
str parent_ns='', bint raise_errors=False):
"""
Check that the data value is a non floating
@@ -95,7 +95,7 @@ cpdef bint validate_int(datum, schema=None,
datetime.time, datetime.datetime, datetime.date))
)
-cpdef bint validate_long(datum, schema=None,
+cdef inline bint validate_long(datum, schema=None,
str parent_ns='', bint raise_errors=False):
"""
Check that the data value is a non floating
@@ -118,7 +118,7 @@ cpdef bint validate_long(datum, schema=None,
datetime.time, datetime.datetime, datetime.date))
)
-cpdef bint validate_float(datum, schema=None,
+cdef inline bint validate_float(datum, schema=None,
str parent_ns='', bint raise_errors=False):
"""
Check that the data value is a floating
@@ -132,7 +132,7 @@ cpdef bint validate_float(datum, schema=None,
"""
return isinstance(datum, (int, long, float, numbers.Real))
-cpdef bint validate_fixed(datum, dict schema,
+cdef inline bint validate_fixed(datum, dict schema,
str parent_ns='', bint raise_errors=False):
"""
Check that the data value is fixed width bytes,
@@ -148,7 +148,7 @@ cpdef bint validate_fixed(datum, dict schema,
len(datum) == schema['size']) or \
(isinstance(datum, decimal.Decimal))
-cpdef bint validate_enum(datum, dict schema,
+cdef inline bint validate_enum(datum, dict schema,
str parent_ns='', bint raise_errors=False):
"""
Check that the data value matches one of the enum symbols.
@@ -163,7 +163,7 @@ cpdef bint validate_enum(datum, dict schema,
"""
return datum in schema['symbols']
-cpdef bint validate_array(datum, dict schema,
+cdef inline bint validate_array(datum, dict schema,
str parent_ns='', bint raise_errors=False):
"""
Check that the data list values all match schema['items'].
@@ -189,7 +189,7 @@ cpdef bint validate_array(datum, dict schema,
return False
return True
-cpdef bint validate_map(object datum, dict schema, str parent_ns='',
+cdef inline bint validate_map(object datum, dict schema, str parent_ns='',
bint raise_errors=False):
"""
Check that the data is a Map(k,v)
@@ -220,7 +220,7 @@ cpdef bint validate_map(object datum, dict schema, str parent_ns='',
return False
return True
-cpdef bint validate_record(object datum, dict schema, str parent_ns='',
+cdef inline bint validate_record(object datum, dict schema, str parent_ns='',
bint raise_errors=False):
"""
Check that the data is a Mapping type with all schema defined fields
@@ -247,7 +247,7 @@ cpdef bint validate_record(object datum, dict schema, str parent_ns='',
return False
return True
-cpdef bint validate_union(object datum, list schema, str parent_ns=None,
+cdef inline bint validate_union(object datum, list schema, str parent_ns=None,
bint raise_errors=False):
"""
Check that the data is a list type with possible options to
@@ -287,36 +287,6 @@ cpdef bint validate_union(object datum, list schema, str parent_ns=None,
raise ValidationError(*errors)
return False
-cpdef BASE_VALIDATORS = {
- 'null': validate_null,
- 'boolean': validate_boolean,
- 'string': validate_string,
- 'int': validate_int,
- 'long': validate_long,
- 'float': validate_float,
- 'double': validate_float,
- 'bytes': validate_bytes,
- 'fixed': validate_fixed,
- 'enum': validate_enum,
- 'array': validate_array,
- 'map': validate_map,
- 'union': validate_union,
- 'error_union': validate_union,
- 'record': validate_record,
- 'error': validate_record,
- 'request': validate_record
-}
-
-cpdef VALIDATORS = BASE_VALIDATORS.copy()
-
-cpdef void register_validator(record_type, validator):
- if record_type in BASE_VALIDATORS:
- raise ValueError("Not allowed to override Base Validators.")
- VALIDATORS[record_type] = validator
-
-cpdef get_validator(record_type):
- return VALIDATORS.get(record_type)
-
cpdef validate(object datum, object schema, str field='',
bint raise_errors=False):
"""Determine if a python datum is an instance of a schema."""
@@ -369,24 +339,17 @@ cpdef validate(object datum, object schema, str field='',
elif record_type in ('record', 'error', 'request'):
result = validate_record(datum, schema=schema, parent_ns=ns_field,
raise_errors=raise_errors)
- else:
- validator = get_validator(record_type)
- if validator:
- result = validator(datum, schema=schema, parent_ns=ns_field,
- raise_errors=raise_errors)
-
- if record_type in SCHEMA_DEFS and result is None:
+ elif record_type in SCHEMA_DEFS:
result = validate(datum,
schema=SCHEMA_DEFS[record_type],
field=ns_field,
raise_errors=raise_errors)
+ else:
+ raise UnknownType(record_type)
if raise_errors and result is False:
raise ValidationError(ValidationErrorData(datum, schema, ns_field))
- if result is None:
- raise UnknownType(record_type)
-
return bool(result)

RENAME
fastavro/_validate_py.py
fastavro/_validation_py.py
@@ -3,8 +3,9 @@ import decimal
import numbers
from collections import Iterable, Mapping
-from fastavro.const import INT_MAX_VALUE, INT_MIN_VALUE, \
- LONG_MAX_VALUE, LONG_MIN_VALUE
+from fastavro.const import (
+ INT_MAX_VALUE, INT_MIN_VALUE, LONG_MAX_VALUE, LONG_MIN_VALUE
+)
from ._validate_common import ValidationError, ValidationErrorData
from .schema import extract_record_type, schema_name, UnknownType
from .six import long, is_str, iterkeys, itervalues
@@ -161,7 +162,7 @@ def validate_array(datum, schema, parent_ns=None, raise_errors=False):
:except: ValidationError
"""
if raise_errors:
- namespace, name = schema_name(schema, parent_ns)
+ _, name = schema_name(schema, parent_ns)
else:
name = parent_ns
return (
@@ -186,7 +187,7 @@ def validate_map(datum, schema, parent_ns=None, raise_errors=False):
:except: ValidationError
"""
if raise_errors:
- namespace, name = schema_name(schema, parent_ns)
+ _, name = schema_name(schema, parent_ns)
else:
name = parent_ns
return (
@@ -265,7 +266,7 @@ def validate_union(datum, schema, parent_ns=None, raise_errors=False):
return False
-BASE_VALIDATORS = {
+VALIDATORS = {
'null': validate_null,
'boolean': validate_boolean,
'string': validate_string,
@@ -285,18 +286,6 @@ BASE_VALIDATORS = {
'request': validate_record
}
-VALIDATORS = BASE_VALIDATORS.copy()
-
-
-def register_validator(record_type, validator):
- if record_type in BASE_VALIDATORS:
- raise ValueError("Not allowed to override Base Validators.")
- VALIDATORS[record_type] = validator
-
-
-def get_validator(record_type):
- return VALIDATORS.get(record_type)
-
def validate(datum, schema, field=None, raise_errors=False):
"""Determine if a python datum is an instance of a schema."""
@@ -309,7 +298,7 @@ def validate(datum, schema, field=None, raise_errors=False):
elif field:
ns_field = field
- validator = get_validator(record_type)
+ validator = VALIDATORS.get(record_type)
if validator:
result = validator(datum, schema=schema,
parent_ns=ns_field,

MODIFY
fastavro/_write.pyx
fastavro/_write.pyx
@@ -16,7 +16,7 @@ from os import urandom
from zlib import compress
from fastavro import const
-from ._validate import validate
+from ._validation import validate
from ._six import utob, long, iteritems, mk_bits
from ._read import HEADER_SCHEMA, SYNC_SIZE, MAGIC
from ._schema import (

MODIFY
fastavro/_write_py.py
fastavro/_write_py.py
@@ -17,7 +17,7 @@ from os import urandom, SEEK_SET
from struct import pack
from zlib import compress
-from .validate import validate
+from .validation import validate
from .const import (
MCS_PER_HOUR, MCS_PER_MINUTE, MCS_PER_SECOND, MLS_PER_HOUR, MLS_PER_MINUTE,
MLS_PER_SECOND, DAYS_SHIFT

DELETE
fastavro/validate.py
-'''Fast Avro file iteration.
-
-Example usage::
-
- # Validating
- from fastavro import validator
-
- schema = {
- 'doc': 'A weather reading.',
- 'name': 'Weather',
- 'namespace': 'test',
- 'type': 'record',
- 'fields': [
- {'name': 'station', 'type': 'string'},
- {'name': 'time', 'type': 'long'},
- {'name': 'temp', 'type': 'int'},
- ],
- }
-
- # 'records' can be an iterable (including generator)
- records = [
- {u'station': u'011990-99999', u'temp': 0, u'time': 1433269388},
- {u'station': u'011990-99999', u'temp': 22, u'time': 1433270389},
- {u'station': u'011990-99999', u'temp': -11, u'time': 1433273379},
- {u'station': u'012650-99999', u'temp': 111, u'time': 1433275478},
- ]
-
- writer(out, schema, records)
-'''
-try:
- from . import _validate
-except ImportError:
- from . import _validate_py as _validate
-from ._validate_common import ValidationErrorData, ValidationError
-
-validate = _validate.validate
-register_validator = _validate.register_validator
-get_validator = _validate.get_validator
-validate_many = _validate.validate_many
-
-__all__ = ['ValidationError', 'ValidationErrorData', 'validate',
- 'validate_many', 'register_validator', 'get_validator']

ADD
fastavro/validation.py
+try:
+ from . import _validation
+except ImportError:
+ from . import _validation_py as _validation
+from ._validate_common import ValidationErrorData, ValidationError
+
+validate = _validation.validate
+validate_many = _validation.validate_many
+
+__all__ = [
+ 'ValidationError', 'ValidationErrorData', 'validate', 'validate_many'
+]

MODIFY
fastavro/write.py
fastavro/write.py
@@ -10,12 +10,11 @@ writer = _write.writer
Writer = _write.Writer
schemaless_writer = _write.schemaless_writer
write_data = _write.write_data
-_validate = _write.validate
WRITERS = _write.WRITERS
LOGICAL_WRITERS = _write.LOGICAL_WRITERS
__all__ = [
'SCHEMA_DEFS', 'acquaint_schema', 'writer', 'schemaless_writer',
- 'write_data', 'WRITERS', 'LOGICAL_WRITERS'
+ 'write_data', 'WRITERS', 'LOGICAL_WRITERS',
]

MODIFY
setup.py
setup.py
@@ -27,7 +27,7 @@ if not hasattr(sys, 'pypy_version_info'):
Extension('fastavro._schema', ["fastavro/_schema" + ext]),
Extension('fastavro._six', ["fastavro/_six" + ext]),
Extension('fastavro._write', ["fastavro/_write" + ext]),
- Extension('fastavro._validate', ["fastavro/_validate" + ext]),
+ Extension('fastavro._validation', ["fastavro/_validation" + ext]),
]

MODIFY
tests/test_validate.py
tests/test_validate.py
-from fastavro.write import _validate
+from fastavro.validation import validate
def test_validator_numeric():
@@ -10,7 +10,7 @@ def test_validator_numeric():
(1, 'float'),
(1, 'double'),
]:
- assert _validate(datum, schema)
+ assert validate(datum, schema)
for datum, schema in [
(1.0, 'int'),
@@ -20,7 +20,7 @@ def test_validator_numeric():
("1", 'float'),
("1", 'double'),
]:
- assert not _validate(datum, schema)
+ assert not validate(datum, schema)
# and plenty more to add I suppose
@@ -53,14 +53,14 @@ def test_validator_numeric_numpy():
# all these should work
for nptype, schema in zip(np_ints, schema_ints):
- assert _validate(nptype(1), schema)
+ assert validate(nptype(1), schema)
for nptype, schema in zip(np_ints, schema_floats):
- assert _validate(nptype(1), schema)
+ assert validate(nptype(1), schema)
for nptype, schema in zip(np_floats, schema_floats):
- assert _validate(nptype(1), schema)
+ assert validate(nptype(1), schema)
# these shouldn't work
for nptype, schema in zip(np_floats, schema_ints):
- assert not _validate(nptype(1), schema)
+ assert not validate(nptype(1), schema)

MODIFY
tests/test_validation.py
tests/test_validation.py
-from fastavro.validate import ValidationError, ValidationErrorData, validate_many
+from fastavro.validation import (
+ ValidationError,
+ ValidationErrorData,
+ validate_many
+)
import pytest
schema = {
@@ -160,7 +164,9 @@ def test_validate_null_in_string_false():
def test_validate_error_raises():
with pytest.raises(ValidationError):
- raise ValidationError(
- ValidationErrorData(10, "string", "test1"),
- ValidationErrorData(10, "bytes", "test1"),
- ValidationErrorData("bad int", "int", "test1.test_obj.test2"))
+ raise ValidationError()
+
+ error = ValidationErrorData(10, "string", "test1")
+ # PY2 says <type 'int'> and PY3 says <class 'int'> so just split there
+ assert "test1 is <10> of type <" in str(error)
+ assert "'int'> expected string" in str(error)

change validate module to validation and misc cleanup