add options for strict and strict_allow_default
MODIFY
fastavro/_write.pyi
fastavro/_write.pyi
@@ -13,6 +13,8 @@ def writer(
validator: bool = ...,
sync_marker: bytes = ...,
codec_compression_level: Optional[int] = ...,
+ strict: bool = ...,
+ strict_allow_default: bool = ...,
) -> None: ...
class GenericWriter:
@@ -43,4 +45,10 @@ class Writer(GenericWriter):
def write_block(self, block) -> None: ... # type: ignore # Should be a read.Block
def flush(self) -> None: ...
-def schemaless_writer(fo: IO, schema: Schema, record: Any) -> None: ...
+def schemaless_writer(
+ fo: IO,
+ schema: Schema,
+ record: Any,
+ strict: bool = ...,
+ strict_allow_default: bool = ...,
+) -> None: ...

MODIFY
fastavro/_write.pyx
fastavro/_write.pyx
@@ -162,7 +162,7 @@ cdef inline write_enum(bytearray fo, datum, schema, dict named_schemas):
write_int(fo, index)
-cdef write_array(bytearray fo, list datum, schema, dict named_schemas, fname):
+cdef write_array(bytearray fo, list datum, schema, dict named_schemas, fname, dict options):
"""Arrays are encoded as a series of blocks.
Each block consists of a long count value, followed by that many array
@@ -177,7 +177,7 @@ cdef write_array(bytearray fo, list datum, schema, dict named_schemas, fname):
write_long(fo, len(datum))
dtype = schema["items"]
for item in datum:
- write_data(fo, item, dtype, named_schemas, fname)
+ write_data(fo, item, dtype, named_schemas, fname, options)
write_long(fo, 0)
@@ -185,7 +185,8 @@ cdef write_python_array(bytearray fo,
array.array datum,
schema,
named_schemas,
- fname):
+ fname,
+ dict options):
"""Array specialization for python arrays."""
if len(datum) > 0:
write_long(fo, len(datum))
@@ -202,11 +203,11 @@ cdef write_python_array(bytearray fo,
else:
dtype = schema["items"]
for item in datum:
- write_data(fo, item, dtype, named_schemas, fname)
+ write_data(fo, item, dtype, named_schemas, fname, options)
write_long(fo, 0)
-cdef write_map(bytearray fo, object datum, dict schema, dict named_schemas, fname):
+cdef write_map(bytearray fo, object datum, dict schema, dict named_schemas, fname, dict options):
"""Maps are encoded as a series of blocks.
Each block consists of a long count value, followed by that many key/value
@@ -227,7 +228,7 @@ cdef write_map(bytearray fo, object datum, dict schema, dict named_schemas, fnam
vtype = schema["values"]
for key, val in datum.items():
write_utf8(fo, key)
- write_data(fo, val, vtype, named_schemas, fname)
+ write_data(fo, val, vtype, named_schemas, fname, options)
write_long(fo, 0)
else:
# Faster, special-purpose code where datum is a Python dict.
@@ -236,11 +237,11 @@ cdef write_map(bytearray fo, object datum, dict schema, dict named_schemas, fnam
vtype = schema["values"]
for key, val in d_datum.items():
write_utf8(fo, key)
- write_data(fo, val, vtype, named_schemas, fname)
+ write_data(fo, val, vtype, named_schemas, fname, options)
write_long(fo, 0)
-cdef write_union(bytearray fo, datum, schema, dict named_schemas, fname):
+cdef write_union(bytearray fo, datum, schema, dict named_schemas, fname, dict options):
"""A union is encoded by first writing a long value indicating the
zero-based position within the union of the schema of its value. The value
is then encoded per the indicated schema within the union."""
@@ -332,10 +333,10 @@ cdef write_union(bytearray fo, datum, schema, dict named_schemas, fname):
# write data
write_long(fo, index)
- write_data(fo, datum, schema[index], named_schemas, fname)
+ write_data(fo, datum, schema[index], named_schemas, fname, options)
-cdef write_record(bytearray fo, object datum, dict schema, dict named_schemas):
+cdef write_record(bytearray fo, object datum, dict schema, dict named_schemas, dict options):
"""A record is encoded by encoding the values of its fields in the order
that they are declared. In other words, a record is encoded as just the
concatenation of the encodings of its fields. Field values are encoded per
@@ -349,24 +350,53 @@ cdef write_record(bytearray fo, object datum, dict schema, dict named_schemas):
except TypeError:
# Slower, general-purpose code where datum is something besides a dict,
# e.g. a collections.OrderedDict or collections.defaultdict.
+ if (options.get("strict") or options.get("strict_allow_default")) and len(
+ datum
+ ) > len(schema["fields"]):
+ field_names = [field["name"] for field in fields]
+ extras = ", ".join(set(datum) - set(field_names))
+ raise ValueError(
+ f"record contains more fields than the schema specifies: {extras}"
+ )
for field in fields:
name = field["name"]
- if name not in datum and "default" not in field and "null" not in field["type"]:
+ if name not in datum:
+ if options.get("strict") or (
+ options.get("strict_allow_default") and "default" not in field
+ ):
+ raise ValueError(
+ f"Field {name} is specified in the schema but missing from the record"
+ )
+ elif "default" not in field and "null" not in field["type"]:
raise ValueError(f"no value and no default for {name}")
datum_value = datum.get(name, field.get("default"))
- write_data(fo, datum_value, field["type"], named_schemas, name)
+ write_data(fo, datum_value, field["type"], named_schemas, name, options)
else:
# Faster, special-purpose code where datum is a Python dict.
+ if (options.get("strict") or options.get("strict_allow_default")) and len(
+ d_datum
+ ) > len(schema["fields"]):
+ field_names = [field["name"] for field in fields]
+ extras = ", ".join(set(d_datum) - set(field_names))
+ raise ValueError(
+ f"record contains more fields than the schema specifies: {extras}"
+ )
for field in fields:
name = field["name"]
- if name not in d_datum and "default" not in field and \
- "null" not in field["type"]:
+ if name not in d_datum:
+ if options.get("strict") or (
+ options.get("strict_allow_default") and "default" not in field
+ ):
+ raise ValueError(
+ f"Field {name} is specified in the schema but missing from the record"
+ )
+ elif "default" not in field and "null" not in field["type"]:
raise ValueError(f"no value and no default for {name}")
- d_datum_value = datum.get(name, field.get("default"))
- write_data(fo, d_datum_value, field["type"], named_schemas, name)
+ d_datum_value = d_datum.get(name, field.get("default"))
+ write_data(fo, d_datum_value, field["type"], named_schemas, name, options)
-cpdef write_data(bytearray fo, datum, schema, dict named_schemas, fname):
+cpdef write_data(bytearray fo, datum, schema, dict named_schemas, fname, dict options):
"""Write a datum of data to output stream.
Parameters
@@ -409,20 +439,20 @@ cpdef write_data(bytearray fo, datum, schema, dict named_schemas, fname):
elif record_type == "array":
if isinstance(datum, array.array):
return write_python_array(
- fo, datum, schema, named_schemas, fname
+ fo, datum, schema, named_schemas, fname, options
)
elif not isinstance(datum, list):
datum = list(datum)
- return write_array(fo, datum, schema, named_schemas, fname)
+ return write_array(fo, datum, schema, named_schemas, fname, options)
elif record_type == "map":
- return write_map(fo, datum, schema, named_schemas, fname)
+ return write_map(fo, datum, schema, named_schemas, fname, options)
elif record_type == "union" or record_type == "error_union":
- return write_union(fo, datum, schema, named_schemas, fname)
+ return write_union(fo, datum, schema, named_schemas, fname, options)
elif record_type == "record" or record_type == "error":
- return write_record(fo, datum, schema, named_schemas)
+ return write_record(fo, datum, schema, named_schemas, options)
else:
return write_data(
- fo, datum, named_schemas[record_type], named_schemas, fname
+ fo, datum, named_schemas[record_type], named_schemas, fname, options
)
except TypeError as ex:
if fname:
@@ -438,7 +468,7 @@ cpdef write_header(bytearray fo, dict metadata, bytes sync_marker):
"meta": {key: value.encode() for key, value in metadata.items()},
"sync": sync_marker
}
- write_data(fo, header, HEADER_SCHEMA, {}, "")
+ write_data(fo, header, HEADER_SCHEMA, {}, "", {})
cpdef null_write_block(object fo, bytes block_bytes, compression_level):
@@ -590,8 +620,10 @@ cdef class Writer:
cdef public object block_writer
cdef public object compression_level
cdef public dict _named_schemas
+ cdef public dict options
- def __init__(self,
+ def __init__(
+ self,
fo,
schema,
codec="null",
@@ -599,7 +631,9 @@ cdef class Writer:
metadata=None,
validator=None,
sync_marker=None,
- compression_level=None):
+ compression_level=None,
+ options={},
+ ):
cdef bytearray tmp = bytearray()
self.fo = fo
@@ -609,6 +643,7 @@ cdef class Writer:
self.block_count = 0
self.sync_interval = sync_interval
self.compression_level = compression_level
+ self.options = options
if _is_appendable(self.fo):
# Seed to the beginning to read the header
@@ -677,7 +712,7 @@ cdef class Writer:
def write(self, record):
if self.validate_fn:
self.validate_fn(record, self.schema, self._named_schemas)
- write_data(self.io.value, record, self.schema, self._named_schemas, "")
+ write_data(self.io.value, record, self.schema, self._named_schemas, "", self.options)
self.block_count += 1
if self.io.tell() >= self.sync_interval:
self.dump()
@@ -698,7 +733,8 @@ cdef class Writer:
self.fo.flush()
-def writer(fo,
+def writer(
+ fo,
schema,
records,
codec="null",
@@ -706,7 +742,10 @@ def writer(fo,
metadata=None,
validator=None,
sync_marker=None,
- codec_compression_level=None):
+ codec_compression_level=None,
+ strict=False,
+ strict_allow_default=False,
+):
# Sanity check that records is not a single dictionary (as that is a common
# mistake and the exception that gets raised is not helpful)
if isinstance(records, dict):
@@ -721,6 +760,7 @@ def writer(fo,
validator,
sync_marker,
codec_compression_level,
+ options={"strict": strict, "strict_allow_default": strict_allow_default}
)
for record in records:
@@ -728,9 +768,16 @@ def writer(fo,
output.flush()
-def schemaless_writer(fo, schema, record):
+def schemaless_writer(fo, schema, record, strict=False, strict_allow_default=False):
cdef bytearray tmp = bytearray()
named_schemas = {}
schema = parse_schema(schema, named_schemas)
- write_data(tmp, record, schema, named_schemas, "")
+ write_data(
+ tmp,
+ record,
+ schema,
+ named_schemas,
+ "",
+ {"strict": strict, "strict_allow_default": strict_allow_default},
+ )
fo.write(tmp)

MODIFY
fastavro/_write_py.py
fastavro/_write_py.py
@@ -26,47 +26,47 @@ from ._write_common import _is_appendable
from .types import Schema, NamedSchemas
-def write_null(encoder, datum, schema, named_schemas, fname):
+def write_null(encoder, datum, schema, named_schemas, fname, options):
"""null is written as zero bytes"""
encoder.write_null()
-def write_boolean(encoder, datum, schema, named_schemas, fname):
+def write_boolean(encoder, datum, schema, named_schemas, fname, options):
"""A boolean is written as a single byte whose value is either 0 (false) or
1 (true)."""
encoder.write_boolean(datum)
-def write_int(encoder, datum, schema, named_schemas, fname):
+def write_int(encoder, datum, schema, named_schemas, fname, options):
"""int and long values are written using variable-length, zig-zag coding."""
encoder.write_int(datum)
-def write_long(encoder, datum, schema, named_schemas, fname):
+def write_long(encoder, datum, schema, named_schemas, fname, options):
"""int and long values are written using variable-length, zig-zag coding."""
encoder.write_long(datum)
-def write_float(encoder, datum, schema, named_schemas, fname):
+def write_float(encoder, datum, schema, named_schemas, fname, options):
"""A float is written as 4 bytes. The float is converted into a 32-bit
integer using a method equivalent to Java's floatToIntBits and then encoded
in little-endian format."""
encoder.write_float(datum)
-def write_double(encoder, datum, schema, named_schemas, fname):
+def write_double(encoder, datum, schema, named_schemas, fname, options):
"""A double is written as 8 bytes. The double is converted into a 64-bit
integer using a method equivalent to Java's doubleToLongBits and then
encoded in little-endian format."""
encoder.write_double(datum)
-def write_bytes(encoder, datum, schema, named_schemas, fname):
+def write_bytes(encoder, datum, schema, named_schemas, fname, options):
"""Bytes are encoded as a long followed by that many bytes of data."""
encoder.write_bytes(datum)
-def write_utf8(encoder, datum, schema, named_schemas, fname):
+def write_utf8(encoder, datum, schema, named_schemas, fname, options):
"""A string is encoded as a long followed by that many bytes of UTF-8
encoded character data."""
encoder.write_utf8(datum)
@@ -77,7 +77,7 @@ def write_crc32(encoder, datum):
encoder.write_crc32(datum)
-def write_fixed(encoder, datum, schema, named_schemas, fname):
+def write_fixed(encoder, datum, schema, named_schemas, fname, options):
"""Fixed instances are encoded using the number of bytes declared in the
schema."""
if len(datum) != schema["size"]:
@@ -87,14 +87,14 @@ def write_fixed(encoder, datum, schema, named_schemas, fname):
encoder.write_fixed(datum)
-def write_enum(encoder, datum, schema, named_schemas, fname):
+def write_enum(encoder, datum, schema, named_schemas, fname, options):
"""An enum is encoded by a int, representing the zero-based position of
the symbol in the schema."""
index = schema["symbols"].index(datum)
encoder.write_enum(index)
-def write_array(encoder, datum, schema, named_schemas, fname):
+def write_array(encoder, datum, schema, named_schemas, fname, options):
"""Arrays are encoded as a series of blocks.
Each block consists of a long count value, followed by that many array
@@ -109,12 +109,12 @@ def write_array(encoder, datum, schema, named_schemas, fname):
encoder.write_item_count(len(datum))
dtype = schema["items"]
for item in datum:
- write_data(encoder, item, dtype, named_schemas, fname)
+ write_data(encoder, item, dtype, named_schemas, fname, options)
encoder.end_item()
encoder.write_array_end()
-def write_map(encoder, datum, schema, named_schemas, fname):
+def write_map(encoder, datum, schema, named_schemas, fname, options):
"""Maps are encoded as a series of blocks.
Each block consists of a long count value, followed by that many key/value
@@ -130,11 +130,11 @@ def write_map(encoder, datum, schema, named_schemas, fname):
vtype = schema["values"]
for key, val in datum.items():
encoder.write_utf8(key)
- write_data(encoder, val, vtype, named_schemas, fname)
+ write_data(encoder, val, vtype, named_schemas, fname, options)
encoder.write_map_end()
-def write_union(encoder, datum, schema, named_schemas, fname):
+def write_union(encoder, datum, schema, named_schemas, fname, options):
"""A union is encoded by first writing a long value indicating the
zero-based position within the union of the schema of its value. The value
is then encoded per the indicated schema within the union."""
@@ -219,17 +219,32 @@ def write_union(encoder, datum, schema, named_schemas, fname):
# write data
# TODO: There should be a way to give just the index
encoder.write_index(index, schema[index])
- write_data(encoder, datum, schema[index], named_schemas, fname)
+ write_data(encoder, datum, schema[index], named_schemas, fname, options)
-def write_record(encoder, datum, schema, named_schemas, fname):
+def write_record(encoder, datum, schema, named_schemas, fname, options):
"""A record is encoded by encoding the values of its fields in the order
that they are declared. In other words, a record is encoded as just the
concatenation of the encodings of its fields. Field values are encoded per
their schema."""
+ if (options.get("strict") or options.get("strict_allow_default")) and len(
+ datum
+ ) > len(schema["fields"]):
+ field_names = [field["name"] for field in schema["fields"]]
+ extras = ", ".join(set(datum) - set(field_names))
+ raise ValueError(
+ f"record contains more fields than the schema specifies: {extras}"
+ )
for field in schema["fields"]:
name = field["name"]
- if name not in datum and "default" not in field and "null" not in field["type"]:
+ if name not in datum:
+ if options.get("strict") or (
+ options.get("strict_allow_default") and "default" not in field
+ ):
+ raise ValueError(
+ f"Field {name} is specified in the schema but missing from the record"
+ )
+ elif "default" not in field and "null" not in field["type"]:
raise ValueError(f"no value and no default for {name}")
write_data(
encoder,
@@ -237,6 +252,7 @@ def write_record(encoder, datum, schema, named_schemas, fname):
field["type"],
named_schemas,
name,
+ options,
)
@@ -260,7 +276,7 @@ WRITERS = {
}
-def write_data(encoder, datum, schema, named_schemas, fname):
+def write_data(encoder, datum, schema, named_schemas, fname, options):
"""Write a datum of data to output stream.
Parameters
@@ -285,13 +301,15 @@ def write_data(encoder, datum, schema, named_schemas, fname):
if prepare:
datum = prepare(datum, schema)
try:
- return fn(encoder, datum, schema, named_schemas, fname)
+ return fn(encoder, datum, schema, named_schemas, fname, options)
except TypeError as ex:
if fname:
raise TypeError(f"{ex} on field {fname}")
raise
else:
- return write_data(encoder, datum, named_schemas[record_type], named_schemas, "")
+ return write_data(
+ encoder, datum, named_schemas[record_type], named_schemas, "", options
+ )
def write_header(encoder, metadata, sync_marker):
@@ -300,7 +318,7 @@ def write_header(encoder, metadata, sync_marker):
"meta": {key: value.encode() for key, value in metadata.items()},
"sync": sync_marker,
}
- write_data(encoder, header, HEADER_SCHEMA, {}, "")
+ write_data(encoder, header, HEADER_SCHEMA, {}, "", {})
def null_write_block(encoder, block_bytes, compression_level):
@@ -402,10 +420,11 @@ else:
class GenericWriter(ABC):
- def __init__(self, schema, metadata=None, validator=None):
+ def __init__(self, schema, metadata=None, validator=None, options={}):
self._named_schemas = {}
self.validate_fn = _validate if validator else None
self.metadata = metadata or {}
+ self.options = options
# A schema of None is allowed when appending and when doing so the
# self.schema will be updated later
@@ -459,8 +478,9 @@ class Writer(GenericWriter):
validator: bool = False,
sync_marker: bytes = b"",
compression_level: Optional[int] = None,
+ options: Dict[str, bool] = {},
):
- super().__init__(schema, metadata, validator)
+ super().__init__(schema, metadata, validator, options)
self.metadata["avro.codec"] = codec
if isinstance(fo, BinaryEncoder):
@@ -510,7 +530,7 @@ class Writer(GenericWriter):
def write(self, record):
if self.validate_fn:
self.validate_fn(record, self.schema, self._named_schemas)
- write_data(self.io, record, self.schema, self._named_schemas, "")
+ write_data(self.io, record, self.schema, self._named_schemas, "", self.options)
self.block_count += 1
if self.io._fo.tell() >= self.sync_interval:
self.dump()
@@ -540,8 +560,9 @@ class JSONWriter(GenericWriter):
validator: bool = False,
sync_marker: bytes = b"",
codec_compression_level: Optional[int] = None,
+ options: Dict[str, bool] = {},
):
- super().__init__(schema, metadata, validator)
+ super().__init__(schema, metadata, validator, options)
self.encoder = fo
self.encoder.configure(self.schema, self._named_schemas)
@@ -549,7 +570,9 @@ class JSONWriter(GenericWriter):
def write(self, record):
if self.validate_fn:
self.validate_fn(record, self.schema, self._named_schemas)
- write_data(self.encoder, record, self.schema, self._named_schemas, "")
+ write_data(
+ self.encoder, record, self.schema, self._named_schemas, "", self.options
+ )
def flush(self):
self.encoder.flush()
@@ -565,6 +588,8 @@ def writer(
validator: bool = False,
sync_marker: bytes = b"",
codec_compression_level: Optional[int] = None,
+ strict: bool = False,
+ strict_allow_default: bool = False,
):
"""Write records to fo (stream) according to schema
@@ -591,6 +616,13 @@ def writer(
codec_compression_level
Compression level to use with the specified codec (if the codec
supports it)
+ strict
+ If set to True, an error will be raised if records do not contain
+ exactly the same fields that the schema states
+ strict_allow_default
+ If set to True, an error will be raised if records do not contain
+ exactly the same fields that the schema states unless it is a missing
+ field that has a default value in the schema
Example::
@@ -662,6 +694,7 @@ def writer(
validator,
sync_marker,
codec_compression_level,
+ options={"strict": strict, "strict_allow_default": strict_allow_default},
)
else:
output = Writer(
@@ -673,6 +706,7 @@ def writer(
validator,
sync_marker,
codec_compression_level,
+ options={"strict": strict, "strict_allow_default": strict_allow_default},
)
for record in records:
@@ -680,7 +714,13 @@ def writer(
output.flush()
-def schemaless_writer(fo: IO, schema: Schema, record: Any):
+def schemaless_writer(
+ fo: IO,
+ schema: Schema,
+ record: Any,
+ strict: bool = False,
+ strict_allow_default: bool = False,
+):
"""Write a single record without the schema or header information
Parameters
@@ -691,6 +731,13 @@ def schemaless_writer(fo: IO, schema: Schema, record: Any):
Schema
record
Record to write
+ strict
+ If set to True, an error will be raised if records do not contain
+ exactly the same fields that the schema states
+ strict_allow_default
+ If set to True, an error will be raised if records do not contain
+ exactly the same fields that the schema states unless it is a missing
+ field that has a default value in the schema
Example::
@@ -705,5 +752,12 @@ def schemaless_writer(fo: IO, schema: Schema, record: Any):
schema = parse_schema(schema, named_schemas)
encoder = BinaryEncoder(fo)
- write_data(encoder, record, schema, named_schemas, "")
+ write_data(
+ encoder,
+ record,
+ schema,
+ named_schemas,
+ "",
+ {"strict": strict, "strict_allow_default": strict_allow_default},
+ )
encoder.flush()

MODIFY
fastavro/json_write.py
fastavro/json_write.py
@@ -13,6 +13,8 @@ def json_writer(
write_union_type: bool = True,
validator: bool = False,
encoder=AvroJSONEncoder,
+ strict: bool = False,
+ strict_allow_default: bool = False,
) -> None:
"""Write records to fo (stream) according to schema
@@ -34,6 +36,13 @@ def json_writer(
encoder
By default the standard AvroJSONEncoder will be used, but a custom one
could be passed here
+ strict
+ If set to True, an error will be raised if records do not contain
+ exactly the same fields that the schema states
+ strict_allow_default
+ If set to True, an error will be raised if records do not contain
+ exactly the same fields that the schema states unless it is a missing
+ field that has a default value in the schema
Example::
@@ -68,4 +77,6 @@ def json_writer(
schema,
records,
validator=validator,
+ strict=strict,
+ strict_allow_default=strict_allow_default,
)

MODIFY
tests/test_fastavro.py
tests/test_fastavro.py
@@ -30,9 +30,9 @@ NO_DATA = {
}
-def roundtrip(schema, records, **reader_kwargs):
+def roundtrip(schema, records, *, writer_kwargs={}, **reader_kwargs):
new_file = BytesIO()
- fastavro.writer(new_file, schema, records)
+ fastavro.writer(new_file, schema, records, **writer_kwargs)
new_file.seek(0)
reader = fastavro.reader(new_file, **reader_kwargs)
@@ -1032,10 +1032,10 @@ def test_regular_vs_ordered_dict_record_typeerror():
# For the regular dict, fails by reraising an error accessing
# 'd_datum', a variable that only gets a value if the record is an
# actual dict.
- ['write_data(fo, d_datum_value, field["type"], named_schemas, name)'],
+ ['write_data(fo, d_datum_value, field["type"], named_schemas, name, options)'],
# For the OrderedDict, fails directly when accessing 'datum', the
# variable that is used if the record is *not* an actual dict.
- ['write_data(fo, datum_value, field["type"], named_schemas, name)'],
+ ['write_data(fo, datum_value, field["type"], named_schemas, name, options)'],
]
for test_record, expected_write_record_stack_trace in zip(
@@ -3191,3 +3191,54 @@ def test_record_names_must_match():
records = [{"test1": {"field1": "foo"}, "test2": {"field1": "bar"}}]
with pytest.raises(SchemaResolutionError):
roundtrip(writer_schema, records, reader_schema=reader_schema)
+
+
+def test_strict_option():
+ """https://github.com/fastavro/fastavro/issues/549"""
+ schema = {
+ "namespace": "namespace",
+ "name": "name",
+ "type": "record",
+ "fields": [
+ {"name": "field_1", "type": "boolean"},
+ {"name": "field_2", "type": ["null", "string"], "default": None},
+ ],
+ }
+
+ test_record1 = {"field_1": True, "field_2": "foo", "field_3": "something"}
+ test_record2 = {"field_1": True}
+ test_record3 = {"field_2": "foo"}
+
+ with pytest.raises(ValueError, match="field_3"):
+ roundtrip(schema, [test_record1], writer_kwargs={"strict": True})
+
+ with pytest.raises(ValueError, match="field_2 is specified .*? but missing"):
+ roundtrip(schema, [test_record2], writer_kwargs={"strict": True})
+
+ with pytest.raises(ValueError, match="field_1 is specified .*? but missing"):
+ roundtrip(schema, [test_record3], writer_kwargs={"strict": True})
+
+
+def test_strict_allow_default_option():
+ """https://github.com/fastavro/fastavro/issues/549"""
+ schema = {
+ "namespace": "namespace",
+ "name": "name",
+ "type": "record",
+ "fields": [
+ {"name": "field_1", "type": "boolean"},
+ {"name": "field_2", "type": ["null", "string"], "default": None},
+ ],
+ }
+
+ test_record1 = {"field_1": True, "field_2": "foo", "field_3": "something"}
+ test_record2 = {"field_1": True}
+ test_record3 = {"field_2": "foo"}
+
+ with pytest.raises(ValueError, match="field_3"):
+ roundtrip(schema, [test_record1], writer_kwargs={"strict_allow_default": True})
+
+ roundtrip(schema, [test_record2], writer_kwargs={"strict_allow_default": True})
+
+ with pytest.raises(ValueError, match="field_1 is specified .*? but missing"):
+ roundtrip(schema, [test_record3], writer_kwargs={"strict_allow_default": True})

MODIFY
tests/test_json.py
tests/test_json.py
@@ -12,9 +12,9 @@ from fastavro.io.json_encoder import AvroJSONEncoder
from fastavro.io.symbols import MapKeyMarker, String
-def roundtrip(schema, records):
+def roundtrip(schema, records, *, writer_kwargs={}):
new_file = StringIO()
- json_writer(new_file, schema, records)
+ json_writer(new_file, schema, records, **writer_kwargs)
new_file.seek(0)
new_records = list(json_reader(new_file, schema))
@@ -747,3 +747,54 @@ def test_custom_encoder_and_decoder():
new_file.seek(0)
for record in json_reader(new_file, schema, decoder=CustomJSONDecoder):
assert record["station"].startswith("__")
+
+
+def test_strict_option():
+ """https://github.com/fastavro/fastavro/issues/549"""
+ schema = {
+ "namespace": "namespace",
+ "name": "name",
+ "type": "record",
+ "fields": [
+ {"name": "field_1", "type": "boolean"},
+ {"name": "field_2", "type": ["null", "string"], "default": None},
+ ],
+ }
+
+ test_record1 = {"field_1": True, "field_2": "foo", "field_3": "something"}
+ test_record2 = {"field_1": True}
+ test_record3 = {"field_2": "foo"}
+
+ with pytest.raises(ValueError, match="field_3"):
+ roundtrip(schema, [test_record1], writer_kwargs={"strict": True})
+
+ with pytest.raises(ValueError, match="field_2 is specified .*? but missing"):
+ roundtrip(schema, [test_record2], writer_kwargs={"strict": True})
+
+ with pytest.raises(ValueError, match="field_1 is specified .*? but missing"):
+ roundtrip(schema, [test_record3], writer_kwargs={"strict": True})
+
+
+def test_strict_allow_default_option():
+ """https://github.com/fastavro/fastavro/issues/549"""
+ schema = {
+ "namespace": "namespace",
+ "name": "name",
+ "type": "record",
+ "fields": [
+ {"name": "field_1", "type": "boolean"},
+ {"name": "field_2", "type": ["null", "string"], "default": None},
+ ],
+ }
+
+ test_record1 = {"field_1": True, "field_2": "foo", "field_3": "something"}
+ test_record2 = {"field_1": True}
+ test_record3 = {"field_2": "foo"}
+
+ with pytest.raises(ValueError, match="field_3"):
+ roundtrip(schema, [test_record1], writer_kwargs={"strict_allow_default": True})
+
+ roundtrip(schema, [test_record2], writer_kwargs={"strict_allow_default": True})
+
+ with pytest.raises(ValueError, match="field_1 is specified .*? but missing"):
+ roundtrip(schema, [test_record3], writer_kwargs={"strict_allow_default": True})

MODIFY
tests/test_schemaless.py
tests/test_schemaless.py
from io import BytesIO
import fastavro
+import pytest
-def roundtrip(schema, record):
+
+def roundtrip(schema, record, *, writer_kwargs={}):
new_file = BytesIO()
- fastavro.schemaless_writer(new_file, schema, record)
+ fastavro.schemaless_writer(new_file, schema, record, **writer_kwargs)
new_file.seek(0)
new_record = fastavro.schemaless_reader(new_file, schema)
return new_record
@@ -181,3 +183,54 @@ def test_newer_versions_of_named_schemas_2():
parsed_schema = fastavro.parse_schema(schema)
assert example_1 == roundtrip(parsed_schema, example_1)
+
+
+def test_strict_option():
+ """https://github.com/fastavro/fastavro/issues/549"""
+ schema = {
+ "namespace": "namespace",
+ "name": "name",
+ "type": "record",
+ "fields": [
+ {"name": "field_1", "type": "boolean"},
+ {"name": "field_2", "type": ["null", "string"], "default": None},
+ ],
+ }
+
+ test_record1 = {"field_1": True, "field_2": "foo", "field_3": "something"}
+ test_record2 = {"field_1": True}
+ test_record3 = {"field_2": "foo"}
+
+ with pytest.raises(ValueError, match="field_3"):
+ roundtrip(schema, test_record1, writer_kwargs={"strict": True})
+
+ with pytest.raises(ValueError, match="field_2 is specified .*? but missing"):
+ roundtrip(schema, test_record2, writer_kwargs={"strict": True})
+
+ with pytest.raises(ValueError, match="field_1 is specified .*? but missing"):
+ roundtrip(schema, test_record3, writer_kwargs={"strict": True})
+
+
+def test_strict_allow_default_option():
+ """https://github.com/fastavro/fastavro/issues/549"""
+ schema = {
+ "namespace": "namespace",
+ "name": "name",
+ "type": "record",
+ "fields": [
+ {"name": "field_1", "type": "boolean"},
+ {"name": "field_2", "type": ["null", "string"], "default": None},
+ ],
+ }
+
+ test_record1 = {"field_1": True, "field_2": "foo", "field_3": "something"}
+ test_record2 = {"field_1": True}
+ test_record3 = {"field_2": "foo"}
+
+ with pytest.raises(ValueError, match="field_3"):
+ roundtrip(schema, test_record1, writer_kwargs={"strict_allow_default": True})
+
+ roundtrip(schema, test_record2, writer_kwargs={"strict_allow_default": True})
+
+ with pytest.raises(ValueError, match="field_1 is specified .*? but missing"):
+ roundtrip(schema, test_record3, writer_kwargs={"strict_allow_default": True})

add options for strict and strict_allow_default