Add optional lz4 codec
MODIFY
.travis.yml
.travis.yml
@@ -5,6 +5,7 @@ before_install:
- sudo apt-get install -y libsnappy-dev
- pip install python-snappy
- pip install zstandard
+ - pip install lz4
install:
- pip install -r developer_requirements.txt
script:

MODIFY
README.md
README.md
@@ -33,7 +33,7 @@ encoding/decoding).
* Schemaless Reader
* JSON Writer
* JSON Reader
-* Codecs (Snappy, Deflate, Zstandard, Bzip2)
+* Codecs (Snappy, Deflate, Zstandard, Bzip2, LZ4)
* Schema resolution
* Aliases
* Logical Types

MODIFY
azure-pipelines.yml
azure-pipelines.yml
@@ -40,6 +40,7 @@ jobs:
brew install snappy
CPPFLAGS="-I/usr/local/include -L/usr/local/lib" python -m pip install python-snappy
pip install zstandard
+ pip install lz4
- script: python -m pip install --upgrade -r developer_requirements.txt
displayName: 'Install dependencies'

MODIFY
docs/index.rst
docs/index.rst
@@ -27,7 +27,7 @@ Supported Features
* Schemaless Reader
* JSON Writer
* JSON Reader
-* Codecs (Snappy, Deflate, Zstandard, Bzip2)
+* Codecs (Snappy, Deflate, Zstandard, Bzip2, LZ4)
* Schema resolution
* Aliases
* Logical Types

MODIFY
fastavro/_read.pyx
fastavro/_read.pyx
@@ -636,6 +636,20 @@ else:
BLOCK_READERS["zstandard"] = zstandard_read_block
+cpdef lz4_read_block(fo):
+ length = read_long(fo)
+ data = fo.read(length)
+ return MemoryIO(lz4.block.decompress(data))
+
+
+try:
+ import lz4.block
+except ImportError:
+ BLOCK_READERS["lz4"] = missing_codec_lib("lz4", "lz4")
+else:
+ BLOCK_READERS["lz4"] = lz4_read_block
+
+
def _iter_avro_records(fo, header, codec, writer_schema, reader_schema,
return_record_name=False):
cdef int32 i

MODIFY
fastavro/_read_py.py
fastavro/_read_py.py
@@ -512,6 +512,20 @@ else:
BLOCK_READERS["zstandard"] = zstandard_read_block
+def lz4_read_block(decoder):
+ length = read_long(decoder)
+ data = decoder.read_fixed(length)
+ return MemoryIO(lz4.block.decompress(data))
+
+
+try:
+ import lz4.block
+except ImportError:
+ BLOCK_READERS["lz4"] = missing_codec_lib("lz4", "lz4")
+else:
+ BLOCK_READERS["lz4"] = lz4_read_block
+
+
def _iter_avro_records(decoder, header, codec, writer_schema, reader_schema,
return_record_name=False):
"""Return iterator over avro records."""

MODIFY
fastavro/_write.pyx
fastavro/_write.pyx
@@ -429,6 +429,23 @@ cpdef zstandard_write_block(object fo, bytes block_bytes):
fo.write(data)
+try:
+ import lz4.block
+except ImportError:
+ BLOCK_WRITERS["lz4"] = _missing_dependency("lz4", "lz4")
+else:
+ BLOCK_WRITERS["lz4"] = lz4_write_block
+
+
+cpdef lz4_write_block(object fo, bytes block_bytes):
+ """Write block in "lz4" codec."""
+ cdef bytearray tmp = bytearray()
+ data = lz4.block.compress(block_bytes)
+ write_long(tmp, len(data))
+ fo.write(tmp)
+ fo.write(data)
+
+
cdef class MemoryIO(object):
cdef bytearray value

MODIFY
fastavro/_write_py.py
fastavro/_write_py.py
@@ -313,6 +313,21 @@ else:
BLOCK_WRITERS["zstandard"] = zstandard_write_block
+def lz4_write_block(encoder, block_bytes):
+ """Write block in "lz4" codec."""
+ data = lz4.block.compress(block_bytes)
+ encoder.write_long(len(data))
+ encoder._fo.write(data)
+
+
+try:
+ import lz4.block
+except ImportError:
+ BLOCK_WRITERS["lz4"] = _missing_codec_lib("lz4", "lz4")
+else:
+ BLOCK_WRITERS["lz4"] = lz4_write_block
+
+
class GenericWriter(object):
def __init__(self,

MODIFY
setup.py
setup.py
@@ -93,9 +93,10 @@ setup(
'Topic :: Scientific/Engineering :: Information Analysis',
],
extras_require={
- 'codecs': ['python-snappy', 'zstandard'],
+ 'codecs': ['python-snappy', 'zstandard', 'lz4'],
'snappy': ['python-snappy'],
'zstandard': ['zstandard'],
+ 'lz4': ['lz4'],
},
tests_require=['pytest', 'flake8', 'check-manifest'],
setup_requires=setup_requires

MODIFY
tests/test_compression.py
tests/test_compression.py
@@ -35,7 +35,7 @@ def test_builtin_codecs(codec):
assert records == out_records
-@pytest.mark.parametrize("codec", ["snappy", "zstandard"])
+@pytest.mark.parametrize("codec", ["snappy", "zstandard", "lz4"])
@pytest.mark.skipif(os.name == "nt", reason="A pain to set up on windows")
def test_optional_codecs(codec):
schema = {
@@ -65,7 +65,7 @@ def test_optional_codecs(codec):
assert records == out_records
-@pytest.mark.parametrize("codec", ["snappy", "zstandard"])
+@pytest.mark.parametrize("codec", ["snappy", "zstandard", "lz4"])
@pytest.mark.skipif(os.name != "nt", reason="codec is present")
def test_optional_codecs_not_installed(codec):
schema = {

Add optional lz4 codec