allow bytes in standalone fastavro output
* allow bytes in standalone fastavro;
simplify by extending JSONEncoder
* add bytes type to test_clean_json_list and _dict
* fix test
MODIFY
fastavro/__main__.py
fastavro/__main__.py
+import json
import datetime
from decimal import Decimal
from sys import stdout
@@ -5,27 +6,21 @@ from uuid import UUID
from platform import python_version_tuple
import fastavro as avro
-from fastavro.six import iteritems, json_dump
+from fastavro.six import json_dump, btou
encoding = stdout.encoding or "UTF-8"
-def _clean_json_value(collection, key, value):
- if isinstance(value, (datetime.date, datetime.datetime)):
- collection[key] = value.isoformat()
- elif isinstance(value, (Decimal, UUID)):
- collection[key] = str(value)
+class CleanJSONEncoder(json.JSONEncoder):
+ def default(self, obj):
+ if isinstance(obj, (datetime.date, datetime.datetime)):
+ return obj.isoformat()
+ elif isinstance(obj, (Decimal, UUID)):
+ return str(obj)
+ elif isinstance(obj, bytes):
+ return btou(obj, encoding='iso-8859-1')
else:
- _clean_json_record(value)
-
-
-def _clean_json_record(data):
- if isinstance(data, dict):
- for k, v in iteritems(data):
- _clean_json_value(data, k, v)
- elif isinstance(data, list):
- for i, v in enumerate(data):
- _clean_json_value(data, i, v)
+ return json.JSONEncoder.default(self, obj)
def main(argv=None):
@@ -67,20 +62,19 @@ def main(argv=None):
reader = avro.reader(fo)
if args.schema:
- json_dump(reader.schema, True)
+ json_dump(reader.schema, indent=4)
sys.stdout.write('\n')
continue
elif args.metadata:
del reader.metadata['avro.schema']
- json_dump(reader.metadata, True)
+ json_dump(reader.metadata, indent=4)
sys.stdout.write('\n')
continue
indent = 4 if args.pretty else None
for record in reader:
- _clean_json_record(record)
- json_dump(record, indent)
+ json_dump(record, indent=indent, cls=CleanJSONEncoder)
sys.stdout.write('\n')
sys.stdout.flush()

MODIFY
fastavro/_six.pyx
fastavro/_six.pyx
@@ -24,8 +24,8 @@ if sys.version_info >= (3, 0):
def py3_utob(n, encoding=_encoding):
return bytes(n, encoding)
- def py3_json_dump(obj, indent):
- json.dump(obj, sys.stdout, indent=indent)
+ def py3_json_dump(obj, **kwargs):
+ json.dump(obj, sys.stdout, **kwargs)
def py3_iterkeys(obj):
return obj.keys()
@@ -88,10 +88,7 @@ else: # Python 2x
_outenc = getattr(sys.stdout, 'encoding', None) or _encoding
- def py2_json_dump(obj, indent):
- kwargs = {}
- if indent is not None:
- kwargs['indent'] = indent
+ def py2_json_dump(obj, **kwargs):
json.dump(obj, sys.stdout, encoding=_outenc, **kwargs)
def py2_iterkeys(obj):

MODIFY
fastavro/six.py
fastavro/six.py
@@ -23,8 +23,8 @@ if sys.version_info >= (3, 0):
def py3_utob(n, encoding=_encoding):
return bytes(n, encoding)
- def py3_json_dump(obj, indent):
- json.dump(obj, sys.stdout, indent=indent)
+ def py3_json_dump(obj, **kwargs):
+ json.dump(obj, sys.stdout, **kwargs)
def py3_iterkeys(obj):
return obj.keys()
@@ -86,10 +86,7 @@ else: # Python 2x
_outenc = getattr(sys.stdout, 'encoding', None) or _encoding
- def py2_json_dump(obj, indent):
- kwargs = {}
- if indent is not None:
- kwargs['indent'] = indent
+ def py2_json_dump(obj, **kwargs):
json.dump(obj, sys.stdout, encoding=_outenc, **kwargs)
def py2_iterkeys(obj):

MODIFY
tests/test_logical_types.py
tests/test_logical_types.py
import fastavro
-from fastavro.__main__ import _clean_json_record
+from fastavro.__main__ import CleanJSONEncoder
+from fastavro.six import btou
from fastavro._timezone import epoch
+import json
import pytest
from decimal import Decimal
@@ -326,15 +328,17 @@ def test_clean_json_list():
datetime.date.today(),
uuid4(),
Decimal('1.23'),
+ bytes(b"5"),
]
str_values = [
values[0].isoformat(),
values[1].isoformat(),
str(values[2]),
str(values[3]),
+ btou(values[4], encoding='iso-8859-1'),
]
- _clean_json_record(values)
- assert values == str_values
+ assert (json.dumps(values, cls=CleanJSONEncoder) ==
+ json.dumps(str_values, cls=CleanJSONEncoder))
def test_clean_json_dict():
@@ -343,15 +347,17 @@ def test_clean_json_dict():
'2': datetime.date.today(),
'3': uuid4(),
'4': Decimal('1.23'),
+ '5': bytes(b"5"),
}
str_values = {
'1': values['1'].isoformat(),
'2': values['2'].isoformat(),
'3': str(values['3']),
'4': str(values['4']),
+ '5': btou(values['5'], encoding='iso-8859-1'),
}
- _clean_json_record(values)
- assert values == str_values
+ assert (json.dumps(values, cls=CleanJSONEncoder) ==
+ json.dumps(str_values, cls=CleanJSONEncoder))
def test_unknown_logical_type():

allow bytes in standalone fastavro output (#465)

* allow bytes in standalone fastavro;
simplify by extending JSONEncoder

* add bytes type to test_clean_json_list and _dict

* fix test

Co-authored-by: scottbelden <scottabelden@gmail.com>