add support for the xz codec
MODIFY
.travis.yml
.travis.yml
@@ -6,6 +6,7 @@ before_install:
- pip install python-snappy
- pip install zstandard
- pip install lz4
+ - pip install backports.lzma
install:
- pip install -r developer_requirements.txt
script:

MODIFY
README.md
README.md
@@ -33,7 +33,7 @@ encoding/decoding).
* Schemaless Reader
* JSON Writer
* JSON Reader
-* Codecs (Snappy, Deflate, Zstandard, Bzip2, LZ4)
+* Codecs (Snappy, Deflate, Zstandard, Bzip2, LZ4, XZ)
* Schema resolution
* Aliases
* Logical Types

MODIFY
azure-pipelines.yml
azure-pipelines.yml
@@ -41,6 +41,7 @@ jobs:
CPPFLAGS="-I/usr/local/include -L/usr/local/lib" python -m pip install python-snappy
pip install zstandard
pip install lz4
+ pip install backports.lzma
- script: python -m pip install --upgrade -r developer_requirements.txt
displayName: 'Install dependencies'

MODIFY
docs/index.rst
docs/index.rst
@@ -27,7 +27,7 @@ Supported Features
* Schemaless Reader
* JSON Writer
* JSON Reader
-* Codecs (Snappy, Deflate, Zstandard, Bzip2, LZ4)
+* Codecs (Snappy, Deflate, Zstandard, Bzip2, LZ4, XZ)
* Schema resolution
* Aliases
* Logical Types

MODIFY
fastavro/_read.pyx
fastavro/_read.pyx
@@ -660,6 +660,25 @@ else:
BLOCK_READERS["lz4"] = lz4_read_block
+cpdef xz_read_block(fo):
+ length = read_long(fo)
+ data = fo.read(length)
+ return MemoryIO(lzma.decompress(data))
+
+
+try:
+ import lzma
+except ImportError:
+ try:
+ from backports import lzma
+ except ImportError:
+ BLOCK_READERS["xz"] = missing_codec_lib("xz", "backports.lzma")
+ else:
+ BLOCK_READERS["xz"] = xz_read_block
+else:
+ BLOCK_READERS["xz"] = xz_read_block
+
+
def _iter_avro_records(fo, header, codec, writer_schema, reader_schema,
return_record_name=False):
cdef int32 i

MODIFY
fastavro/_read_py.py
fastavro/_read_py.py
@@ -540,6 +540,25 @@ else:
BLOCK_READERS["lz4"] = lz4_read_block
+def xz_read_block(decoder):
+ length = read_long(decoder)
+ data = decoder.read_fixed(length)
+ return MemoryIO(lzma.decompress(data))
+
+
+try:
+ import lzma
+except ImportError:
+ try:
+ from backports import lzma
+ except ImportError:
+ BLOCK_READERS["xz"] = missing_codec_lib("xz", "backports.lzma")
+ else:
+ BLOCK_READERS["xz"] = xz_read_block
+else:
+ BLOCK_READERS["xz"] = xz_read_block
+
+
def _iter_avro_records(decoder, header, codec, writer_schema, reader_schema,
return_record_name=False):
"""Return iterator over avro records."""

MODIFY
fastavro/_write.pyx
fastavro/_write.pyx
@@ -449,6 +449,28 @@ cpdef lz4_write_block(object fo, bytes block_bytes, compression_level):
fo.write(data)
+try:
+ import lzma
+except ImportError:
+ try:
+ from backports import lzma
+ except ImportError:
+ BLOCK_WRITERS["xz"] = _missing_dependency("xz", "backports.lzma")
+ else:
+ BLOCK_WRITERS["xz"] = xz_write_block
+else:
+ BLOCK_WRITERS["xz"] = xz_write_block
+
+
+cpdef xz_write_block(object fo, bytes block_bytes, compression_level):
+ """Write block in "xz" codec."""
+ cdef bytearray tmp = bytearray()
+ data = lzma.compress(block_bytes)
+ write_long(tmp, len(data))
+ fo.write(tmp)
+ fo.write(data)
+
+
cdef class MemoryIO(object):
cdef bytearray value

MODIFY
fastavro/_write_py.py
fastavro/_write_py.py
@@ -333,6 +333,26 @@ else:
BLOCK_WRITERS["lz4"] = lz4_write_block
+def xz_write_block(encoder, block_bytes, compression_level):
+ """Write block in "xz" codec."""
+ data = lzma.compress(block_bytes)
+ encoder.write_long(len(data))
+ encoder._fo.write(data)
+
+
+try:
+ import lzma
+except ImportError:
+ try:
+ from backports import lzma
+ except ImportError:
+ BLOCK_WRITERS["xz"] = _missing_codec_lib("xz", "backports.lzma")
+ else:
+ BLOCK_WRITERS["xz"] = xz_write_block
+else:
+ BLOCK_WRITERS["xz"] = xz_write_block
+
+
class GenericWriter(object):
def __init__(self,

MODIFY
setup.py
setup.py
@@ -95,10 +95,11 @@ setup(
],
install_requires=["pytz"],
extras_require={
- 'codecs': ['python-snappy', 'zstandard', 'lz4'],
+ 'codecs': ['python-snappy', 'zstandard', 'lz4', 'backports.lzma'],
'snappy': ['python-snappy'],
'zstandard': ['zstandard'],
'lz4': ['lz4'],
+ 'xz': ['backports.lzma']
},
tests_require=['pytest', 'flake8', 'check-manifest'],
setup_requires=setup_requires

MODIFY
tests/test_compression.py
tests/test_compression.py
import os
+import sys
import pytest
@@ -35,7 +36,7 @@ def test_builtin_codecs(codec):
assert records == out_records
-@pytest.mark.parametrize("codec", ["snappy", "zstandard", "lz4"])
+@pytest.mark.parametrize("codec", ["snappy", "zstandard", "lz4", "xz"])
@pytest.mark.skipif(os.name == "nt", reason="A pain to set up on windows")
def test_optional_codecs(codec):
schema = {
@@ -95,6 +96,43 @@ def test_optional_codecs_not_installed(codec):
fastavro.writer(file, schema, records, codec=codec)
+@pytest.mark.skipif(os.name != "nt", reason="codec is present")
+def test_xz_works_by_default_on_windows_python3():
+ schema = {
+ "doc": "A weather reading.",
+ "name": "Weather",
+ "namespace": "test",
+ "type": "record",
+ "fields": [
+ {"name": "station", "type": "string"},
+ {"name": "time", "type": "long"},
+ {"name": "temp", "type": "int"},
+ ],
+ }
+
+ records = [
+ {"station": "011990-99999", "temp": 0, "time": 1433269388},
+ {"station": "011990-99999", "temp": 22, "time": 1433270389},
+ {"station": "011990-99999", "temp": -11, "time": 1433273379},
+ {"station": "012650-99999", "temp": 111, "time": 1433275478},
+ ]
+
+ file = MemoryIO()
+
+ if sys.version_info >= (3, 0):
+ fastavro.writer(file, schema, records, codec="xz")
+
+ file.seek(0)
+ out_records = list(fastavro.reader(file))
+ assert records == out_records
+ else:
+ with pytest.raises(
+ ValueError,
+ match="xz codec is supported but you need to install"
+ ):
+ fastavro.writer(file, schema, records, codec="xz")
+
+
def test_unsupported_codec():
schema = {
"doc": "A weather reading.",

add support for the xz codec (#412)