py2\3 decimal reader
MODIFY
fastavro/reader.py
fastavro/reader.py
@@ -12,18 +12,16 @@ from zlib import decompress
import datetime
from decimal import localcontext, Decimal
-import struct
-
try:
from fastavro._six import MemoryIO, xrange, btou, utob, iteritems,\
- is_str, PY3
+ is_str, str2ints, fstint
from fastavro._schema import (
extract_record_type, acquaint_schema, populate_schema_defs,
extract_logical_type
)
except ImportError:
from fastavro.six import MemoryIO, xrange, btou, utob, iteritems, \
- is_str, PY3
+ is_str, str2ints, fstint
from fastavro.schema import (
extract_record_type, acquaint_schema, populate_schema_defs,
extract_logical_type
@@ -160,65 +158,37 @@ def read_date(data, writer_schema=None, reader_schema=None):
def read_bytes_decimal(data, writer_schema=None, reader_schema=None):
- scale = writer_schema['scale']
- precision = writer_schema['precision']
-
- size = len(data)
-
- unscaled_datum = read_decimal_from_fixed(data, precision, scale, size)
-
- with localcontext() as ctx:
- ctx.prec = precision
- scaled_datum = Decimal(unscaled_datum).scaleb(-scale)
- return scaled_datum
-
-
-def read_decimal_from_fixed3(datum, precision, scale, size):
"""
Decimal is encoded as fixed. Fixed instances are encoded using the
number of bytes declared in the schema.
based on https://github.com/apache/avro/pull/82/
"""
- unscaled_datum = 0
- msb = datum[0]
- leftmost_bit = (msb >> 7) & 1
- if leftmost_bit == 1:
- modified_first_byte = (datum[0]) ^ (1 << 7)
- datum = (modified_first_byte).to_bytes(1, 'big') + datum[1:]
- for offset in xrange(size):
- unscaled_datum <<= 8
- unscaled_datum += datum[offset]
- unscaled_datum += pow(-2, (size * 8) - 1)
- else:
- for offset in xrange(size):
- unscaled_datum <<= 8
- unscaled_datum += datum[offset]
+ scale = writer_schema['scale']
+ precision = writer_schema['precision']
- return unscaled_datum
+ size = len(data)
+ datum_byte = str2ints(data)
-def read_decimal_from_fixed2(datum, precision, scale, size):
- """
- Decimal is encoded as fixed. Fixed instances are encoded using the
- number of bytes declared in the schema.
- based on https://github.com/apache/avro/pull/82/
- """
unscaled_datum = 0
- msb = struct.unpack('!b', datum[0])[0]
+ msb = fstint(data)
leftmost_bit = (msb >> 7) & 1
if leftmost_bit == 1:
- modified_first_byte = ord(datum[0]) ^ (1 << 7)
- datum = chr(modified_first_byte) + datum[1:]
+ modified_first_byte = datum_byte[0] ^ (1 << 7)
+ datum_byte = [modified_first_byte] + datum_byte[1:]
for offset in xrange(size):
unscaled_datum <<= 8
- unscaled_datum += ord(datum[offset])
+ unscaled_datum += datum_byte[offset]
unscaled_datum += pow(-2, (size * 8) - 1)
else:
for offset in xrange(size):
unscaled_datum <<= 8
- unscaled_datum += ord(datum[offset])
+ unscaled_datum += (datum_byte[offset])
- return unscaled_datum
+ with localcontext() as ctx:
+ ctx.prec = precision
+ scaled_datum = Decimal(unscaled_datum).scaleb(-scale)
+ return scaled_datum
def read_long(fo, writer_schema=None, reader_schema=None):
@@ -242,12 +212,6 @@ def read_long(fo, writer_schema=None, reader_schema=None):
return (n >> 1) ^ -(n & 1)
-if PY3:
- read_decimal_from_fixed = read_decimal_from_fixed3
-else:
- read_decimal_from_fixed = read_decimal_from_fixed2
-
-
def read_float(fo, writer_schema=None, reader_schema=None):
"""A float is written as 4 bytes.

MODIFY
fastavro/six.py
fastavro/six.py
@@ -7,14 +7,13 @@ Some of this code is "lifted" from CherryPy.
import sys
import json
from sys import stdout
+from struct import unpack
_encoding = 'UTF-8'
-PY3 = False
if sys.version_info >= (3, 0):
from io import BytesIO as MemoryIO
xrange = range
- PY3 = True
def py3_btou(n, encoding=_encoding):
return n.decode(encoding)
@@ -33,6 +32,12 @@ if sys.version_info >= (3, 0):
def py3_mk_bits(bits):
return bytes([bits & 0xff])
+ def py3_identity(datum):
+ return datum
+
+ def py3_fstint(datum):
+ return datum[0]
+
else: # Python 2x
from cStringIO import StringIO as MemoryIO # flake8: noqa
xrange = xrange
@@ -56,6 +61,13 @@ else: # Python 2x
def py2_mk_bits(bits):
return chr(bits & 0xff)
+
+ def py2_str2ints(datum):
+ return map(lambda x:ord(x), datum)
+
+ def py2_fstint(datum):
+ return unpack('!b', datum[0])[0]
+
# We do it this way and not just redifine function since Cython do not like it
if sys.version_info >= (3, 0):
btou = py3_btou
@@ -65,6 +77,8 @@ if sys.version_info >= (3, 0):
iteritems = py3_iteritems
is_str = py3_is_str
mk_bits = py3_mk_bits
+ str2ints = py3_identity
+ fstint = py3_fstint
else:
btou = py2_btou
utob = py2_utob
@@ -73,3 +87,5 @@ else:
long = long
is_str = py2_is_str
mk_bits = py2_mk_bits
+ str2ints = py2_str2ints
+ fstint = py2_fstint

py2\3 decimal reader