more formatting tweaks
MODIFY
fastavro/_read.pyx
fastavro/_read.pyx
@@ -311,7 +311,12 @@ cdef read_enum(fo, writer_schema, reader_schema):
return symbol
-cdef read_array(fo, writer_schema, reader_schema=None, return_record_name=False):
+cdef read_array(
+ fo,
+ writer_schema,
+ reader_schema=None,
+ return_record_name=False,
+):
"""Arrays are encoded as a series of blocks.
Each block consists of a long count value, followed by that many array
@@ -338,20 +343,31 @@ cdef read_array(fo, writer_schema, reader_schema=None, return_record_name=False)
if reader_schema:
for i in range(block_count):
- read_items.append(_read_data(fo,
+ read_items.append(_read_data(
+ fo,
writer_schema['items'],
reader_schema['items'],
- return_record_name))
+ return_record_name,
+ ))
else:
for i in range(block_count):
- read_items.append(_read_data(fo, writer_schema['items'], None,
- return_record_name))
+ read_items.append(_read_data(
+ fo,
+ writer_schema['items'],
+ None,
+ return_record_name,
+ ))
block_count = read_long(fo)
return read_items
-cdef read_map(fo, writer_schema, reader_schema=None, return_record_name=False):
+cdef read_map(
+ fo,
+ writer_schema,
+ reader_schema=None,
+ return_record_name=False,
+):
"""Maps are encoded as a series of blocks.
Each block consists of a long count value, followed by that many key/value
@@ -378,21 +394,32 @@ cdef read_map(fo, writer_schema, reader_schema=None, return_record_name=False):
if reader_schema:
for i in range(block_count):
key = read_utf8(fo)
- read_items[key] = _read_data(fo,
+ read_items[key] = _read_data(
+ fo,
writer_schema['values'],
reader_schema['values'],
- return_record_name)
+ return_record_name,
+ )
else:
for i in range(block_count):
key = read_utf8(fo)
- read_items[key] = _read_data(fo, writer_schema['values'], None,
- return_record_name)
+ read_items[key] = _read_data(
+ fo,
+ writer_schema['values'],
+ None,
+ return_record_name,
+ )
block_count = read_long(fo)
return read_items
-cdef read_union(fo, writer_schema, reader_schema=None, return_record_name=False):
+cdef read_union(
+ fo,
+ writer_schema,
+ reader_schema=None,
+ return_record_name=False,
+):
"""A union is encoded by first writing a long value indicating the
zero-based position within the union of the schema of its value.
@@ -405,11 +432,21 @@ cdef read_union(fo, writer_schema, reader_schema=None, return_record_name=False)
# Handle case where the reader schema is just a single type (not union)
if not isinstance(reader_schema, list):
if match_types(idx_schema, reader_schema):
- return _read_data(fo, idx_schema, reader_schema, return_record_name)
+ return _read_data(
+ fo,
+ idx_schema,
+ reader_schema,
+ return_record_name,
+ )
else:
for schema in reader_schema:
if match_types(idx_schema, schema):
- return _read_data(fo, idx_schema, schema, return_record_name)
+ return _read_data(
+ fo,
+ idx_schema,
+ schema,
+ return_record_name,
+ )
msg = 'schema mismatch: %s not found in %s' % (writer_schema, reader_schema)
raise SchemaResolutionError(msg)
else:
@@ -429,7 +466,12 @@ cdef read_union(fo, writer_schema, reader_schema=None, return_record_name=False)
return _read_data(fo, idx_schema, None, return_record_name)
-cdef read_record(fo, writer_schema, reader_schema=None, return_record_name=False):
+cdef read_record(
+ fo,
+ writer_schema,
+ reader_schema=None,
+ return_record_name=False,
+):
"""A record is encoded by encoding the values of its fields in the order
that they are declared. In other words, a record is encoded as just the
concatenation of the encodings of its fields. Field values are encoded per
@@ -451,8 +493,12 @@ cdef read_record(fo, writer_schema, reader_schema=None, return_record_name=False
record = {}
if reader_schema is None:
for field in writer_schema['fields']:
- record[field['name']] = _read_data(fo, field['type'], None,
- return_record_name)
+ record[field['name']] = _read_data(
+ fo,
+ field['type'],
+ None,
+ return_record_name,
+ )
else:
readers_field_dict = {}
aliases_field_dict = {}
@@ -467,13 +513,20 @@ cdef read_record(fo, writer_schema, reader_schema=None, return_record_name=False
aliases_field_dict.get(field['name']),
)
if readers_field:
- record[readers_field['name']] = _read_data(fo,
+ record[readers_field['name']] = _read_data(
+ fo,
field['type'],
readers_field['type'],
- return_record_name)
+ return_record_name,
+ )
else:
# should implement skip
- _read_data(fo, field['type'], field['type'], return_record_name)
+ _read_data(
+ fo,
+ field['type'],
+ field['type'],
+ return_record_name,
+ )
# fill in default values
if len(readers_field_dict) > len(record):
@@ -517,7 +570,12 @@ cpdef maybe_promote(data, writer_type, reader_type):
return data
-cpdef _read_data(fo, writer_schema, reader_schema=None, return_record_name=False):
+cpdef _read_data(
+ fo,
+ writer_schema,
+ reader_schema=None,
+ return_record_name=False,
+):
"""Read data from file object according to schema."""
record_type = extract_record_type(writer_schema)
@@ -550,13 +608,33 @@ cpdef _read_data(fo, writer_schema, reader_schema=None, return_record_name=False
elif record_type == 'enum':
data = read_enum(fo, writer_schema, reader_schema)
elif record_type == 'array':
- data = read_array(fo, writer_schema, reader_schema, return_record_name)
+ data = read_array(
+ fo,
+ writer_schema,
+ reader_schema,
+ return_record_name,
+ )
elif record_type == 'map':
- data = read_map(fo, writer_schema, reader_schema, return_record_name)
+ data = read_map(
+ fo,
+ writer_schema,
+ reader_schema,
+ return_record_name,
+ )
elif record_type == 'union' or record_type == 'error_union':
- data = read_union(fo, writer_schema, reader_schema, return_record_name)
+ data = read_union(
+ fo,
+ writer_schema,
+ reader_schema,
+ return_record_name,
+ )
elif record_type == 'record' or record_type == 'error':
- data = read_record(fo, writer_schema, reader_schema, return_record_name)
+ data = read_record(
+ fo,
+ writer_schema,
+ reader_schema,
+ return_record_name,
+ )
else:
return _read_data(
fo,
@@ -677,8 +755,14 @@ else:
BLOCK_READERS["xz"] = xz_read_block
-def _iter_avro_records(fo, header, codec, writer_schema, reader_schema,
- return_record_name=False):
+def _iter_avro_records(
+ fo,
+ header,
+ codec,
+ writer_schema,
+ reader_schema,
+ return_record_name=False,
+):
cdef int32 i
sync_marker = header['sync']
@@ -693,14 +777,24 @@ def _iter_avro_records(fo, header, codec, writer_schema, reader_schema,
block_fo = read_block(fo)
for i in range(block_count):
- yield _read_data(block_fo, writer_schema, reader_schema,
- return_record_name)
+ yield _read_data(
+ block_fo,
+ writer_schema,
+ reader_schema,
+ return_record_name,
+ )
skip_sync(fo, sync_marker)
-def _iter_avro_blocks(fo, header, codec, writer_schema, reader_schema,
- return_record_name=False):
+def _iter_avro_blocks(
+ fo,
+ header,
+ codec,
+ writer_schema,
+ reader_schema,
+ return_record_name=False,
+):
sync_marker = header['sync']
read_block = BLOCK_READERS.get(codec)
@@ -748,8 +842,12 @@ class Block:
def __iter__(self):
for i in range(self.num_records):
- yield _read_data(self.bytes_, self.writer_schema,
- self.reader_schema, self.return_record_name)
+ yield _read_data(
+ self.bytes_,
+ self.writer_schema,
+ self.reader_schema,
+ self.return_record_name,
+ )
def __str__(self):
return ("Avro block: %d bytes, %d records, codec: %s, position %d+%d"

MODIFY
fastavro/_read_py.py
fastavro/_read_py.py
@@ -272,12 +272,20 @@ def read_array(
):
if reader_schema:
def item_reader(decoder, w_schema, r_schema, return_record_name):
- return read_data(decoder, w_schema['items'], r_schema['items'],
- return_record_name)
+ return read_data(
+ decoder,
+ w_schema['items'],
+ r_schema['items'],
+ return_record_name,
+ )
else:
def item_reader(decoder, w_schema, _, return_record_name):
- return read_data(decoder, w_schema['items'], None,
- return_record_name)
+ return read_data(
+ decoder,
+ w_schema['items'],
+ None,
+ return_record_name,
+ )
read_items = []
@@ -300,12 +308,20 @@ def read_map(
):
if reader_schema:
def item_reader(decoder, w_schema, r_schema):
- return read_data(decoder, w_schema['values'], r_schema['values'],
- return_record_name)
+ return read_data(
+ decoder,
+ w_schema['values'],
+ r_schema['values'],
+ return_record_name,
+ )
else:
def item_reader(decoder, w_schema, _):
- return read_data(decoder, w_schema['values'], None,
- return_record_name)
+ return read_data(
+ decoder,
+ w_schema['values'],
+ None,
+ return_record_name,
+ )
read_items = {}
@@ -393,8 +409,9 @@ def read_record(
record = {}
if reader_schema is None:
for field in writer_schema['fields']:
- record[field['name']] = read_data(decoder, field['type'], None,
- return_record_name)
+ record[field['name']] = read_data(
+ decoder, field['type'], None, return_record_name,
+ )
else:
readers_field_dict = {}
aliases_field_dict = {}
@@ -623,8 +640,14 @@ else:
BLOCK_READERS["xz"] = xz_read_block
-def _iter_avro_records(decoder, header, codec, writer_schema, reader_schema,
- return_record_name=False):
+def _iter_avro_records(
+ decoder,
+ header,
+ codec,
+ writer_schema,
+ reader_schema,
+ return_record_name=False,
+):
"""Return iterator over avro records."""
sync_marker = header['sync']
@@ -643,15 +666,23 @@ def _iter_avro_records(decoder, header, codec, writer_schema, reader_schema,
for i in xrange(block_count):
yield read_data(
- BinaryDecoder(block_fo), writer_schema, reader_schema,
- return_record_name
+ BinaryDecoder(block_fo),
+ writer_schema,
+ reader_schema,
+ return_record_name,
)
skip_sync(decoder.fo, sync_marker)
-def _iter_avro_blocks(decoder, header, codec, writer_schema, reader_schema,
- return_record_name=False):
+def _iter_avro_blocks(
+ decoder,
+ header,
+ codec,
+ writer_schema,
+ reader_schema,
+ return_record_name=False,
+):
"""Return iterator over avro blocks."""
sync_marker = header['sync']
@@ -673,8 +704,14 @@ def _iter_avro_blocks(decoder, header, codec, writer_schema, reader_schema,
size = decoder.fo.tell() - offset
yield Block(
- block_bytes, num_block_records, codec, reader_schema,
- writer_schema, offset, size, return_record_name
+ block_bytes,
+ num_block_records,
+ codec,
+ reader_schema,
+ writer_schema,
+ offset,
+ size,
+ return_record_name,
)
@@ -702,8 +739,17 @@ class Block:
Size of the block in bytes
"""
- def __init__(self, bytes_, num_records, codec, reader_schema,
- writer_schema, offset, size, return_record_name=False):
+ def __init__(
+ self,
+ bytes_,
+ num_records,
+ codec,
+ reader_schema,
+ writer_schema,
+ offset,
+ size,
+ return_record_name=False,
+ ):
self.bytes_ = bytes_
self.num_records = num_records
self.codec = codec
@@ -748,8 +794,12 @@ class file_reader(object):
def _read_header(self):
try:
- self._header = read_data(self.decoder, HEADER_SCHEMA, None,
- self.return_record_name)
+ self._header = read_data(
+ self.decoder,
+ HEADER_SCHEMA,
+ None,
+ self.return_record_name,
+ )
except StopIteration:
raise ValueError('cannot read header - is it an avro file?')

MODIFY
fastavro/_validation.pyx
fastavro/_validation.pyx
@@ -27,28 +27,23 @@ cdef long64 LONG_MIN_VALUE = const.LONG_MIN_VALUE
cdef long64 LONG_MAX_VALUE = const.LONG_MAX_VALUE
-cdef inline bint validate_null(datum, schema=None,
- str parent_ns='', bint raise_errors=True):
+cdef inline bint validate_null(datum):
return datum is None
-cdef inline bint validate_boolean(datum, schema=None,
- str parent_ns='', bint raise_errors=True):
+cdef inline bint validate_boolean(datum):
return isinstance(datum, bool)
-cdef inline bint validate_string(datum, schema=None,
- str parent_ns='', bint raise_errors=True):
+cdef inline bint validate_string(datum):
return is_str(datum)
-cdef inline bint validate_bytes(datum, schema=None,
- str parent_ns='', bint raise_errors=True):
+cdef inline bint validate_bytes(datum):
return isinstance(datum, bytes)
-cdef inline bint validate_int(datum, schema=None,
- str parent_ns='', bint raise_errors=True):
+cdef inline bint validate_int(datum):
return (
(isinstance(datum, (int, long, numbers.Integral))
and INT_MIN_VALUE <= datum <= INT_MAX_VALUE
@@ -56,8 +51,7 @@ cdef inline bint validate_int(datum, schema=None,
)
-cdef inline bint validate_long(datum, schema=None,
- str parent_ns='', bint raise_errors=True):
+cdef inline bint validate_long(datum):
return (
(isinstance(datum, (int, long, numbers.Integral))
and LONG_MIN_VALUE <= datum <= LONG_MAX_VALUE
@@ -65,29 +59,30 @@ cdef inline bint validate_long(datum, schema=None,
)
-cdef inline bint validate_float(datum, schema=None,
- str parent_ns='', bint raise_errors=True):
+cdef inline bint validate_float(datum):
return (
isinstance(datum, (int, long, float, numbers.Real))
and not isinstance(datum, bool)
)
-cdef inline bint validate_fixed(datum, dict schema,
- str parent_ns='', bint raise_errors=True):
+cdef inline bint validate_fixed(datum, dict schema):
return (
((isinstance(datum, bytes) or isinstance(datum, bytearray))
and len(datum) == schema['size'])
)
-cdef inline bint validate_enum(datum, dict schema,
- str parent_ns='', bint raise_errors=True):
+cdef inline bint validate_enum(datum, dict schema):
return datum in schema['symbols']
-cdef inline bint validate_array(datum, dict schema,
- str parent_ns='', bint raise_errors=True) except -1:
+cdef inline bint validate_array(
+ datum,
+ dict schema,
+ str parent_ns='',
+ bint raise_errors=True
+) except -1:
if not isinstance(datum, Sequence) or is_str(datum):
return False
@@ -99,8 +94,12 @@ cdef inline bint validate_array(datum, dict schema,
return True
-cdef inline bint validate_map(object datum, dict schema, str parent_ns='',
- bint raise_errors=True) except -1:
+cdef inline bint validate_map(
+ object datum,
+ dict schema,
+ str parent_ns='',
+ bint raise_errors=True,
+) except -1:
# initial checks for map type
if not isinstance(datum, Mapping):
return False
@@ -116,8 +115,12 @@ cdef inline bint validate_map(object datum, dict schema, str parent_ns='',
return True
-cdef inline bint validate_record(object datum, dict schema, str parent_ns='',
- bint raise_errors=True) except -1:
+cdef inline bint validate_record(
+ object datum,
+ dict schema,
+ str parent_ns='',
+ bint raise_errors=True,
+) except -1:
if not isinstance(datum, Mapping):
return False
_, namespace = schema_name(schema, parent_ns)
@@ -130,8 +133,12 @@ cdef inline bint validate_record(object datum, dict schema, str parent_ns='',
return True
-cdef inline bint validate_union(object datum, list schema, str parent_ns=None,
- bint raise_errors=True) except -1:
+cdef inline bint validate_union(
+ object datum,
+ list schema,
+ str parent_ns=None,
+ bint raise_errors=True,
+) except -1:
if isinstance(datum, tuple):
(name, datum) = datum
for candidate in schema:
@@ -159,8 +166,12 @@ cdef inline bint validate_union(object datum, list schema, str parent_ns=None,
return False
-cpdef _validate(object datum, object schema, str field='',
- bint raise_errors=True):
+cpdef _validate(
+ object datum,
+ object schema,
+ str field='',
+ bint raise_errors=True,
+):
record_type = extract_record_type(schema)
result = None
@@ -172,44 +183,51 @@ cpdef _validate(object datum, object schema, str field='',
# explicit, so that cython is faster, but only for Base Validators
if record_type == 'null':
- result = validate_null(datum, schema=schema, parent_ns=field,
- raise_errors=raise_errors)
+ result = validate_null(datum)
elif record_type == 'boolean':
- result = validate_boolean(datum, schema=schema, parent_ns=field,
- raise_errors=raise_errors)
+ result = validate_boolean(datum)
elif record_type == 'string':
- result = validate_string(datum, schema=schema, parent_ns=field,
- raise_errors=raise_errors)
+ result = validate_string(datum)
elif record_type == 'int':
- result = validate_int(datum, schema=schema, parent_ns=field,
- raise_errors=raise_errors)
+ result = validate_int(datum)
elif record_type == 'long':
- result = validate_long(datum, schema=schema, parent_ns=field,
- raise_errors=raise_errors)
+ result = validate_long(datum)
elif record_type in ('float', 'double'):
- result = validate_float(datum, schema=schema, parent_ns=field,
- raise_errors=raise_errors)
+ result = validate_float(datum)
elif record_type == 'bytes':
- result = validate_bytes(datum, schema=schema, parent_ns=field,
- raise_errors=raise_errors)
+ result = validate_bytes(datum)
elif record_type == 'fixed':
- result = validate_fixed(datum, schema=schema, parent_ns=field,
- raise_errors=raise_errors)
+ result = validate_fixed(datum, schema=schema)
elif record_type == 'enum':
- result = validate_enum(datum, schema=schema, parent_ns=field,
- raise_errors=raise_errors)
+ result = validate_enum(datum, schema=schema)
elif record_type == 'array':
- result = validate_array(datum, schema=schema, parent_ns=field,
- raise_errors=raise_errors)
+ result = validate_array(
+ datum,
+ schema=schema,
+ parent_ns=field,
+ raise_errors=raise_errors,
+ )
elif record_type == 'map':
- result = validate_map(datum, schema=schema, parent_ns=field,
- raise_errors=raise_errors)
+ result = validate_map(
+ datum,
+ schema=schema,
+ parent_ns=field,
+ raise_errors=raise_errors,
+ )
elif record_type in ('union', 'error_union'):
- result = validate_union(datum, schema=schema, parent_ns=field,
- raise_errors=raise_errors)
+ result = validate_union(
+ datum,
+ schema=schema,
+ parent_ns=field,
+ raise_errors=raise_errors,
+ )
elif record_type in ('record', 'error', 'request'):
- result = validate_record(datum, schema=schema, parent_ns=field,
- raise_errors=raise_errors)
+ result = validate_record(
+ datum,
+ schema=schema,
+ parent_ns=field,
+ raise_errors=raise_errors,
+ )
elif record_type in SCHEMA_DEFS:
result = _validate(datum,
schema=SCHEMA_DEFS[record_type],

more formatting tweaks