add option for user to provide a sync_marker value
MODIFY
fastavro/_write.pyx
fastavro/_write.pyx
@@ -629,7 +629,8 @@ cdef class Writer(object):
codec='null',
sync_interval=1000 * SYNC_SIZE,
metadata=None,
- validator=None):
+ validator=None,
+ sync_marker=None):
cdef bytearray tmp = bytearray()
self.fo = fo
@@ -659,7 +660,7 @@ cdef class Writer(object):
self.block_writer = BLOCK_WRITERS[codec]
else:
- self.sync_marker = urandom(SYNC_SIZE)
+ self.sync_marker = sync_marker or urandom(SYNC_SIZE)
self.metadata = metadata or {}
self.metadata['avro.codec'] = codec
@@ -702,7 +703,8 @@ def writer(fo,
codec='null',
sync_interval=1000 * SYNC_SIZE,
metadata=None,
- validator=None):
+ validator=None,
+ sync_marker=None):
# Sanity check that records is not a single dictionary (as that is a common
# mistake and the exception that gets raised is not helpful)
if isinstance(records, dict):
@@ -715,6 +717,7 @@ def writer(fo,
sync_interval,
metadata,
validator,
+ sync_marker,
)
for record in records:

MODIFY
fastavro/_write_py.py
fastavro/_write_py.py
@@ -468,7 +468,8 @@ class Writer(object):
codec='null',
sync_interval=1000 * SYNC_SIZE,
metadata=None,
- validator=None):
+ validator=None,
+ sync_marker=None):
self.fo = fo
self.schema = parse_schema(schema)
self.validate_fn = validate if validator is True else validator
@@ -496,7 +497,7 @@ class Writer(object):
self.block_writer = BLOCK_WRITERS[codec]
else:
- self.sync_marker = urandom(SYNC_SIZE)
+ self.sync_marker = sync_marker or urandom(SYNC_SIZE)
self.metadata = metadata or {}
self.metadata['avro.codec'] = codec
@@ -537,7 +538,8 @@ def writer(fo,
codec='null',
sync_interval=1000 * SYNC_SIZE,
metadata=None,
- validator=None):
+ validator=None,
+ sync_marker=None):
"""Write records to fo (stream) according to schema
Parameters
@@ -558,6 +560,9 @@ def writer(fo,
then fastavro.validation.validate will be used. If it's a function, it
should have the same signature as fastavro.writer.validate and raise an
exeption on error.
+ sync_marker: bytes, optional
+ A byte string used as the avro sync marker. If not provided, a random
+ byte string will be used.
Example::
@@ -612,6 +617,7 @@ def writer(fo,
sync_interval,
metadata,
validator,
+ sync_marker,
)
for record in records:

MODIFY
tests/test_fastavro.py
tests/test_fastavro.py
@@ -1701,3 +1701,22 @@ def test_appending_records_different_schema_fails(tmpdir):
fastavro.writer(new_file, different_schema, [{"field": 1}])
assert "does not match file writer_schema" in str(exc)
+
+
+def test_user_specified_sync():
+ """https://github.com/fastavro/fastavro/issues/300"""
+ schema = {
+ "type": "record",
+ "name": "test_user_specified_sync",
+ "fields": []
+ }
+
+ file1 = MemoryIO()
+ file2 = MemoryIO()
+
+ records = [{}]
+
+ fastavro.writer(file1, schema, records, sync_marker=b'sync')
+ fastavro.writer(file2, schema, records, sync_marker=b'sync')
+
+ assert file1.getvalue() == file2.getvalue()

add option for user to provide a sync_marker value