Fix for Timezone & Validate
Addition of validate as well fixing the timezone since benchmark was failing with the timestamp benchmark test.
MODIFY
benchmark/benchmark.py
benchmark/benchmark.py
@@ -3,6 +3,19 @@ import datetime
import time
from fastavro import writer, reader
+try:
+ from datetime.timezone import utc
+except ImportError:
+ from fastavro._timezone import utc
+
+try:
+ from fastavro.validate import validate
+except ImportError:
+ try:
+ from fastavro._write import validate
+ except ImportError:
+ from fastavro._write_py import validate
+
def write(schema, records, runs=1):
times = []
@@ -15,6 +28,19 @@ def write(schema, records, runs=1):
print('... {0} runs averaged {1} seconds'.format(runs, (sum(times) / runs)))
return iostream
+
+def validater(schema, records, runs=1):
+ times = []
+ valid = []
+ for _ in range(runs):
+ start = time.time()
+ valid = [validate(record, schema) for record in records]
+ end = time.time()
+ times.append(end - start)
+ print('... {0} runs averaged {1} seconds'.format(runs, (sum(times) / runs)))
+ return all(valid)
+
+
def read(iostream, runs=1):
times = []
for _ in range(runs):
@@ -26,6 +52,7 @@ def read(iostream, runs=1):
print('... {0} runs averaged {1} seconds'.format(runs, (sum(times) / runs)))
return records
+
small_schema = {
"type": "record",
"name": "Test",
@@ -99,7 +126,6 @@ timestamp_schema = {
"type": "record"
}
-
small_record = {'field': 'foo'}
big_record = {
'username': 'username',
@@ -116,23 +142,23 @@ big_record = {
}
timestamp_record = {
- 'timestamp-micros': datetime.datetime.now().replace(tzinfo=datetime.timezone.utc),
-
+ 'timestamp-micros': datetime.datetime.now().replace(tzinfo=utc),
}
# Configuration is a tuple of (schema, single_record, num_records, num_runs)
configurations = [
- (small_schema, small_record, 1, 100000),
- (small_schema, small_record, 100, 1000),
- (small_schema, small_record, 10000, 10),
- (big_schema, big_record, 1, 100000),
- (big_schema, big_record, 100, 1000),
- (big_schema, big_record, 10000, 10),
- (timestamp_schema, timestamp_record, 100000, 10),
+ ('Small - 1 rec - 100,000 runs', small_schema, small_record, 1, 100000),
+ ('Small - 100 rec - 1,000 runs', small_schema, small_record, 100, 1000),
+ ('Small - 10,000 rec - 10 runs', small_schema, small_record, 10000, 10),
+ ('Big - 1 rec - 100,000 runs', big_schema, big_record, 1, 100000),
+ ('Big - 100 rec - 1,000 runs', big_schema, big_record, 100, 1000),
+ ('Big - 10,000 rec - 10 runs', big_schema, big_record, 10000, 10),
+ ('Timestamp - 100,000 rec - 10 runs', timestamp_schema, timestamp_record, 100000, 10),
]
-for schema, single_record, num_records, num_runs in configurations:
+for desc, schema, single_record, num_records, num_runs in configurations:
print('')
+ print(desc)
original_records = [single_record for _ in range(num_records)]
print('Writing {0} records to one file...'.format(num_records))
bytesio = write(schema, original_records, runs=num_runs)
@@ -140,4 +166,7 @@ for schema, single_record, num_records, num_runs in configurations:
print('Reading {0} records from one file...'.format(num_records))
records = read(bytesio, runs=num_runs)
+ print('Validating {0} records from one file...'.format(num_records))
+ valid = validater(schema, original_records, runs=num_runs)
+
assert records == original_records

Fix for Timezone & Validate

Addition of validate as well fixing the timezone since benchmark was failing with the timestamp benchmark test.