actually change the data type when promoting
MODIFY
fastavro/_read.pyx
fastavro/_read.pyx
@@ -16,7 +16,7 @@ from uuid import UUID
import json
from ._six import (
- btou, iteritems, is_str, str2ints, fstint
+ btou, utob, iteritems, is_str, str2ints, fstint, long
)
from ._schema import extract_record_type, extract_logical_type, parse_schema
from ._schema_common import SCHEMA_DEFS
@@ -75,6 +75,10 @@ cpdef match_types(writer_type, reader_type):
return True
elif writer_type == 'float' and reader_type == 'double':
return True
+ elif writer_type == 'string' and reader_type == 'bytes':
+ return True
+ elif writer_type == 'bytes' and reader_type == 'string':
+ return True
return False
@@ -501,6 +505,22 @@ LOGICAL_READERS = {
}
+cpdef maybe_promote(data, writer_type, reader_type):
+ if writer_type == "int":
+ if reader_type == "long":
+ return long(data)
+ if reader_type == "float" or reader_type == "double":
+ return float(data)
+ if writer_type == "long":
+ if reader_type == "float" or reader_type == "double":
+ return float(data)
+ if writer_type == "string" and reader_type == "bytes":
+ return utob(data)
+ if writer_type == "bytes" and reader_type == "string":
+ return btou(data, 'utf-8')
+ return data
+
+
cpdef _read_data(fo, writer_schema, reader_schema=None):
"""Read data from file object according to schema."""
@@ -556,6 +576,13 @@ cpdef _read_data(fo, writer_schema, reader_schema=None):
if fn:
return fn(data, writer_schema, reader_schema)
+ if reader_schema is not None:
+ return maybe_promote(
+ data,
+ record_type,
+ extract_record_type(reader_schema)
+ )
+ else:
return data

MODIFY
fastavro/_read_py.py
fastavro/_read_py.py
@@ -16,7 +16,7 @@ from uuid import UUID
import json
from .six import (
- xrange, btou, iteritems, is_str, str2ints, fstint
+ xrange, btou, utob, iteritems, is_str, str2ints, fstint, long
)
from .schema import extract_record_type, extract_logical_type, parse_schema
from ._schema_common import SCHEMA_DEFS
@@ -63,6 +63,10 @@ def match_types(writer_type, reader_type):
return True
elif writer_type == 'float' and reader_type == 'double':
return True
+ elif writer_type == 'string' and reader_type == 'bytes':
+ return True
+ elif writer_type == 'bytes' and reader_type == 'string':
+ return True
return False
@@ -454,6 +458,22 @@ READERS = {
}
+def maybe_promote(data, writer_type, reader_type):
+ if writer_type == "int":
+ if reader_type == "long":
+ return long(data)
+ if reader_type == "float" or reader_type == "double":
+ return float(data)
+ if writer_type == "long":
+ if reader_type == "float" or reader_type == "double":
+ return float(data)
+ if writer_type == "string" and reader_type == "bytes":
+ return utob(data)
+ if writer_type == "bytes" and reader_type == "string":
+ return btou(data, 'utf-8')
+ return data
+
+
def read_data(fo, writer_schema, reader_schema=None):
"""Read data from file object according to schema."""
@@ -480,6 +500,13 @@ def read_data(fo, writer_schema, reader_schema=None):
if fn:
return fn(data, writer_schema, reader_schema)
+ if reader_schema is not None:
+ return maybe_promote(
+ data,
+ record_type,
+ extract_record_type(reader_schema)
+ )
+ else:
return data
else:
return read_data(

ADD
tests/test_promotion.py
+import fastavro
+from fastavro.six import MemoryIO, long
+
+
+def roundtrip(record, writer_schema, reader_schema):
+ new_file = MemoryIO()
+ fastavro.writer(new_file, writer_schema, [record])
+ new_file.seek(0)
+
+ new_records = list(fastavro.reader(new_file, reader_schema))
+ return new_records[0]
+
+
+def test_int_promotion():
+ int_schema = {
+ "type": "int",
+ }
+
+ long_schema = {
+ "type": "long",
+ }
+
+ result = roundtrip(1, int_schema, long_schema)
+ assert result == 1
+ assert isinstance(result, long)
+
+ float_schema = {
+ "type": "float",
+ }
+
+ result = roundtrip(1, int_schema, float_schema)
+ assert result == 1.0
+ assert isinstance(result, float)
+
+ double_schema = {
+ "type": "double",
+ }
+
+ result = roundtrip(1, int_schema, double_schema)
+ assert result == 1.0
+ # Python doesn't have a double type, but float is close enough
+ assert isinstance(result, float)
+
+
+def test_long_promotion():
+ long_schema = {
+ "type": "long",
+ }
+
+ float_schema = {
+ "type": "float",
+ }
+
+ result = roundtrip(1, long_schema, float_schema)
+ assert result == 1.0
+ assert isinstance(result, float)
+
+ double_schema = {
+ "type": "double",
+ }
+
+ result = roundtrip(1, long_schema, double_schema)
+ assert result == 1.0
+ # Python doesn't have a double type, but float is close enough
+ assert isinstance(result, float)
+
+
+def test_float_promotion():
+ float_schema = {
+ "type": "float",
+ }
+
+ double_schema = {
+ "type": "double",
+ }
+
+ result = roundtrip(1.0, float_schema, double_schema)
+ assert result == 1.0
+ # Python doesn't have a double type, but float is close enough
+ assert isinstance(result, float)
+
+
+def test_string_promotion():
+ string_schema = {
+ "type": "string",
+ }
+
+ bytes_schema = {
+ "type": "bytes",
+ }
+
+ result = roundtrip(u"foo", string_schema, bytes_schema)
+ assert result == b"foo"
+ assert isinstance(result, bytes)
+
+
+def test_bytes_promotion():
+ bytes_schema = {
+ "type": "bytes",
+ }
+
+ string_schema = {
+ "type": "string",
+ }
+
+ result = roundtrip(b"foo", bytes_schema, string_schema)
+ assert result == u"foo"

actually change the data type when promoting