supply None schema to append file
MODIFY
fastavro/_write.pyx
fastavro/_write.pyx
@@ -604,7 +604,6 @@ cdef class Writer:
self.fo = fo
self._named_schemas = {}
- self.schema = parse_schema(schema, self._named_schemas)
self.validate_fn = _validate if validator is True else validator
self.io = MemoryIO()
self.block_count = 0
@@ -617,12 +616,7 @@ cdef class Writer:
avro_reader = reader(self.fo)
header = avro_reader._header
- file_writer_schema = parse_schema(avro_reader.writer_schema)
- if self.schema != file_writer_schema:
- raise ValueError(
- f"Provided schema {self.schema} does not match file "
- + f"writer_schema {file_writer_schema}"
- )
+ self.schema = parse_schema(avro_reader.writer_schema)
codec = avro_reader.metadata.get("avro.codec", "null")
self.sync_marker = header["sync"]
@@ -632,6 +626,7 @@ cdef class Writer:
self.block_writer = BLOCK_WRITERS[codec]
else:
+ self.schema = parse_schema(schema, self._named_schemas)
self.sync_marker = sync_marker or urandom(SYNC_SIZE)
self.metadata = metadata or {}

MODIFY
fastavro/_write_py.py
fastavro/_write_py.py
@@ -401,10 +401,14 @@ else:
class GenericWriter:
def __init__(self, schema, metadata=None, validator=None):
self._named_schemas = {}
- self.schema = parse_schema(schema, self._named_schemas)
self.validate_fn = _validate if validator is True else validator
self.metadata = metadata or {}
+ # A schema of None is allowed when appending and when doing so the
+ # self.schema will be updated later
+ if schema is not None:
+ self.schema = parse_schema(schema, self._named_schemas)
+
if isinstance(schema, dict):
schema = {
key: value
@@ -463,12 +467,7 @@ class Writer(GenericWriter):
avro_reader = reader(self.encoder._fo)
header = avro_reader._header
- file_writer_schema = parse_schema(avro_reader.writer_schema)
- if self.schema != file_writer_schema:
- raise ValueError(
- f"Provided schema {self.schema} does not match "
- + f"file writer_schema {file_writer_schema}"
- )
+ self.schema = parse_schema(avro_reader.writer_schema)
codec = avro_reader.metadata.get("avro.codec", "null")
@@ -632,7 +631,11 @@ def writer(
# Write some more records
with open('weather.avro', 'a+b') as out:
- writer(out, parsed_schema, more_records)
+ writer(out, None, more_records)
+
+ Note: When appending, any schema provided will be ignored since the schema
+ in the avro file will be re-used. Therefore it is convenient to just use
+ None as the schema.
"""
# Sanity check that records is not a single dictionary (as that is a common
# mistake and the exception that gets raised is not helpful)

MODIFY
tests/test_fastavro.py
tests/test_fastavro.py
@@ -1634,8 +1634,29 @@ def test_appending_records_different_schema_fails(tmpdir):
}
with open(test_file, "a+b") as new_file:
- with pytest.raises(ValueError, match="does not match file writer_schema"):
- fastavro.writer(new_file, different_schema, [{"field": 1}])
+ fastavro.writer(new_file, different_schema, [{"field": "bar"}])
+
+
+def test_appending_records_null_schema_works(tmpdir):
+ """https://github.com/fastavro/fastavro/issues/422"""
+ schema = {
+ "type": "record",
+ "name": "test_appending_records_different_schema_fails",
+ "fields": [
+ {
+ "name": "field",
+ "type": "string",
+ }
+ ],
+ }
+
+ test_file = str(tmpdir.join("test.avro"))
+
+ with open(test_file, "wb") as new_file:
+ fastavro.writer(new_file, schema, [{"field": "foo"}])
+
+ with open(test_file, "a+b") as new_file:
+ fastavro.writer(new_file, None, [{"field": "bar"}])
def test_user_specified_sync():

supply None schema to append file