more handling of nulls and unions
MODIFY
fastavro/_validation.pyx
fastavro/_validation.pyx
@@ -26,8 +26,6 @@ cdef int32 INT_MAX_VALUE = const.INT_MAX_VALUE
cdef long64 LONG_MIN_VALUE = const.LONG_MIN_VALUE
cdef long64 LONG_MAX_VALUE = const.LONG_MAX_VALUE
-no_value = object()
-
cdef inline bint validate_null(datum, schema=None,
str parent_ns='', bint raise_errors=True):
@@ -127,7 +125,7 @@ cdef inline bint validate_record(object datum, dict schema, str parent_ns='',
return False
_, namespace = schema_name(schema, parent_ns)
for f in schema['fields']:
- if not validate(datum=datum.get(f['name'], f.get('default', no_value)),
+ if not validate(datum=datum.get(f['name'], f.get('default')),
schema=f['type'],
field='{}.{}'.format(namespace, f['name']),
raise_errors=raise_errors):

MODIFY
fastavro/_validation_py.py
fastavro/_validation_py.py
@@ -16,8 +16,6 @@ from .schema import extract_record_type, UnknownType, schema_name
from .six import long, is_str, iterkeys, itervalues
from ._schema_common import SCHEMA_DEFS
-no_value = object()
-
def validate_null(datum, **kwargs):
"""
@@ -261,7 +259,7 @@ def validate_record(datum, schema, parent_ns=None, raise_errors=True):
_, namespace = schema_name(schema, parent_ns)
return (
isinstance(datum, Mapping) and
- all(validate(datum=datum.get(f['name'], f.get('default', no_value)),
+ all(validate(datum=datum.get(f['name'], f.get('default')),
schema=f['type'],
field='{}.{}'.format(namespace, f['name']),
raise_errors=raise_errors)

MODIFY
fastavro/_write.pyx
fastavro/_write.pyx
@@ -426,7 +426,11 @@ cdef write_union(bytearray fo, datum, schema):
for index, candidate in enumerate(schema):
if validate(datum, candidate, raise_errors=False):
if extract_record_type(candidate) == 'record':
- fields = len(candidate['fields'])
+ candidate_fields = set(
+ f["name"] for f in candidate["fields"]
+ )
+ datum_fields = set(datum)
+ fields = len(candidate_fields.intersection(datum_fields))
if fields > most_fields:
best_match_index = index
most_fields = fields

MODIFY
fastavro/_write_py.py
fastavro/_write_py.py
@@ -324,7 +324,11 @@ def write_union(fo, datum, schema):
for index, candidate in enumerate(schema):
if validate(datum, candidate, raise_errors=False):
if extract_record_type(candidate) == 'record':
- fields = len(candidate['fields'])
+ candidate_fields = set(
+ f["name"] for f in candidate["fields"]
+ )
+ datum_fields = set(datum)
+ fields = len(candidate_fields.intersection(datum_fields))
if fields > most_fields:
best_match_index = index
most_fields = fields

MODIFY
tests/test_fastavro.py
tests/test_fastavro.py
@@ -1859,3 +1859,43 @@ def test_hint_is_not_written_to_the_file():
reader = fastavro.reader(stream)
# By the time it is in the file, it should not
assert "__fastavro_parsed" not in reader._schema
+
+
+def test_more_null_union_issues():
+ """https://github.com/fastavro/fastavro/issues/336"""
+ schema = {
+ "type": "record",
+ "name": "test",
+ "fields": [
+ {"name": "name", "type": "string"},
+ {
+ "name": "address",
+ "type": [
+ "null",
+ {
+ "type": "array",
+ "items": {
+ "type": "record",
+ "name": "whatever",
+ "fields": [
+ {"name": "street", "type": ["null", "string"]},
+ {"name": "zip", "type": ["null", "string"]},
+ ],
+ },
+ },
+ ],
+ },
+ ],
+ }
+
+ records = [
+ {"name": "name1", "address": [{"street": "22st"}]},
+ {"name": "name2"},
+ ]
+
+ expected = [
+ {"name": "name1", "address": [{"street": "22st", "zip": None}]},
+ {"name": "name2", "address": None},
+ ]
+
+ assert expected == roundtrip(schema, records)

more handling of nulls and unions