fix issue with resolving named types
MODIFY
fastavro/_read.pyx
fastavro/_read.pyx
@@ -19,7 +19,12 @@ import json
from ._schema import extract_record_type, extract_logical_type, parse_schema
from ._read_common import (
- SchemaResolutionError, MAGIC, SYNC_SIZE, HEADER_SCHEMA, missing_codec_lib
+ SchemaResolutionError,
+ MAGIC,
+ SYNC_SIZE,
+ HEADER_SCHEMA,
+ missing_codec_lib,
+ NAMED_TYPES,
)
from .const import (
MCS_PER_HOUR,
@@ -122,6 +127,11 @@ cpdef match_schemas(w_schema, r_schema):
elif w_type == r_type == "array":
if match_types(w_schema["items"], r_schema["items"]):
return r_schema
+ elif w_type in NAMED_TYPES and r_type in NAMED_TYPES:
+ if w_schema["name"] == r_schema["name"] or w_schema["name"] in r_schema.get(
+ "aliases", []
+ ):
+ return r_schema
elif match_types(w_type, r_type):
return r_schema
raise SchemaResolutionError(error_msg)

MODIFY
fastavro/_read_common.py
fastavro/_read_common.py
@@ -13,6 +13,7 @@ HEADER_SCHEMA = {
{"name": "sync", "type": {"type": "fixed", "name": "sync", "size": SYNC_SIZE}},
],
}
+NAMED_TYPES = {"record", "enum", "fixed", "error"}
class SchemaResolutionError(Exception):

MODIFY
fastavro/_read_py.py
fastavro/_read_py.py
@@ -26,6 +26,7 @@ from ._read_common import (
SYNC_SIZE,
HEADER_SCHEMA,
missing_codec_lib,
+ NAMED_TYPES,
)
from .const import (
MCS_PER_HOUR,
@@ -117,6 +118,11 @@ def match_schemas(w_schema, r_schema):
elif w_type == r_type == "array":
if match_types(w_schema["items"], r_schema["items"]):
return r_schema
+ elif w_type in NAMED_TYPES and r_type in NAMED_TYPES:
+ if w_schema["name"] == r_schema["name"] or w_schema["name"] in r_schema.get(
+ "aliases", []
+ ):
+ return r_schema
elif match_types(w_type, r_type):
return r_schema
raise SchemaResolutionError(error_msg)

MODIFY
tests/test_aliases.py
tests/test_aliases.py
@@ -23,7 +23,7 @@ def test_aliases_not_present():
new_schema = {
"type": "record",
- "name": "test_aliases_not_present_new",
+ "name": "test_aliases_not_present",
"fields": [
{"name": "newtest", "type": "double", "aliases": ["testX"]},
],
@@ -44,7 +44,7 @@ def test_incompatible_aliases():
new_schema = {
"type": "record",
- "name": "test_incompatible_aliases_new",
+ "name": "test_incompatible_aliases",
"fields": [
{"name": "newtest", "type": "int", "aliases": ["test"]},
],
@@ -65,7 +65,7 @@ def test_aliases_in_reader_schema():
new_schema = {
"type": "record",
- "name": "test_aliases_in_reader_schema_new",
+ "name": "test_aliases_in_reader_schema",
"fields": [{"name": "newtest", "type": "int", "aliases": ["test"]}],
}
@@ -95,3 +95,52 @@ def test_aliases_with_default_value_and_field_added():
new_records = roundtrip(schema, records, new_schema)
assert new_records == [{"newtest": 1, "test2": 100}]
+
+
+def test_record_name_alias():
+ schema = {
+ "type": "record",
+ "name": "test_record_name_alias",
+ "fields": [{"name": "test", "type": "int"}],
+ }
+
+ new_schema = {
+ "type": "record",
+ "name": "test_record_name_alias_new",
+ "aliases": "test_record_name_alias",
+ "fields": [{"name": "test", "type": "int"}],
+ }
+
+ records = [{"test": 1}]
+
+ assert roundtrip(schema, records, new_schema) == [{"test": 1}]
+
+
+def test_fixed_name_alias():
+ schema = {"type": "fixed", "name": "test_fixed_name_alias", "size": 4}
+
+ new_schema = {
+ "type": "fixed",
+ "name": "test_fixed_name_alias_new",
+ "aliases": "test_fixed_name_alias",
+ "size": 4,
+ }
+
+ records = [b"1234"]
+
+ assert roundtrip(schema, records, new_schema) == [b"1234"]
+
+
+def test_enum_name_alias():
+ schema = {"type": "enum", "name": "test_enum_name_alias", "symbols": ["FOO"]}
+
+ new_schema = {
+ "type": "enum",
+ "name": "test_enum_name_alias_new",
+ "aliases": "test_enum_name_alias",
+ "symbols": ["FOO"],
+ }
+
+ records = ["FOO"]
+
+ assert roundtrip(schema, records, new_schema) == ["FOO"]

MODIFY
tests/test_fastavro.py
tests/test_fastavro.py
from io import BytesIO
import fastavro
from fastavro.io.binary_decoder import BinaryDecoder
-from fastavro.read import _read as _reader, HEADER_SCHEMA
+from fastavro.read import _read as _reader, HEADER_SCHEMA, SchemaResolutionError
from fastavro.write import _write as _writer, Writer
import pytest
@@ -392,7 +392,7 @@ def test_schema_migration_remove_field():
new_schema = {
"type": "record",
- "name": "test_schema_migration_remove_field_new",
+ "name": "test_schema_migration_remove_field",
"fields": [],
}
@@ -414,7 +414,7 @@ def test_schema_migration_add_default_field():
new_schema = {
"type": "record",
- "name": "test_schema_migration_add_default_field_new",
+ "name": "test_schema_migration_add_default_field",
"fields": [
{
"name": "test",
@@ -447,7 +447,7 @@ def test_schema_migration_type_promotion():
new_schema = {
"type": "record",
- "name": "test_schema_migration_type_promotion_new",
+ "name": "test_schema_migration_type_promotion",
"fields": [
{
"name": "test",
@@ -479,7 +479,7 @@ def test_schema_migration_maps_with_union_promotion():
new_schema = {
"type": "record",
- "name": "test_schema_migration_maps_with_union_promotion_new",
+ "name": "test_schema_migration_maps_with_union_promotion",
"fields": [
{
"name": "test",
@@ -511,7 +511,7 @@ def test_schema_migration_array_with_union_promotion():
new_schema = {
"type": "record",
- "name": "test_schema_migration_array_with_union_promotion_new",
+ "name": "test_schema_migration_array_with_union_promotion",
"fields": [
{
"name": "test",
@@ -538,7 +538,7 @@ def test_schema_migration_writer_union():
new_schema = {
"type": "record",
- "name": "test_schema_migration_writer_union_new",
+ "name": "test_schema_migration_writer_union",
"fields": [{"name": "test", "type": "int"}],
}
@@ -560,7 +560,7 @@ def test_schema_migration_reader_union():
new_schema = {
"type": "record",
- "name": "test_schema_migration_reader_union_new",
+ "name": "test_schema_migration_reader_union",
"fields": [{"name": "test", "type": ["string", "int"]}],
}
@@ -582,7 +582,7 @@ def test_schema_migration_union_failure():
new_schema = {
"type": "record",
- "name": "test_schema_migration_union_failure_new",
+ "name": "test_schema_migration_union_failure",
"fields": [{"name": "test", "type": ["string", "int"]}],
}
@@ -610,7 +610,7 @@ def test_schema_migration_array_failure():
new_schema = {
"type": "record",
- "name": "test_schema_migration_array_failure_new",
+ "name": "test_schema_migration_array_failure",
"fields": [
{
"name": "test",
@@ -643,7 +643,7 @@ def test_schema_migration_maps_failure():
new_schema = {
"type": "record",
- "name": "test_schema_migration_maps_failure_new",
+ "name": "test_schema_migration_maps_failure",
"fields": [
{
"name": "test",
@@ -2537,3 +2537,115 @@ def test_tuple_writer_picks_correct_union_path():
assert expected_roundtrip_value == roundtrip(
parsed_schema, records, return_record_name=True
)
+
+
+def test_schema_migration_should_match_name():
+ """https://github.com/fastavro/fastavro/issues/512"""
+ writer_schema = {
+ "type": "record",
+ "name": "root",
+ "namespace": "example.space",
+ "fields": [
+ {"name": "id", "type": "string"},
+ {
+ "name": "data",
+ "type": [
+ "null",
+ {
+ "type": "record",
+ "name": "SomeData",
+ "fields": [
+ {"name": "somedata_id", "type": "int"},
+ {"name": "somedata_field", "type": "int"},
+ ],
+ },
+ {
+ "type": "record",
+ "name": "OtherData",
+ "fields": [
+ {"name": "otherdata_id", "type": "int"},
+ {"name": "otherdata_field", "type": "string"},
+ ],
+ },
+ ],
+ },
+ ],
+ }
+
+ reader_schema = {
+ "type": "record",
+ "name": "root",
+ "namespace": "example.space",
+ "fields": [
+ {"name": "id", "type": "string"},
+ {
+ "name": "data",
+ "type": [
+ "null",
+ {
+ "type": "record",
+ "name": "SomeData",
+ "fields": [
+ {"name": "somedata_id", "type": "int"},
+ {"name": "somedata_field", "type": "int"},
+ ],
+ },
+ {
+ "type": "record",
+ "name": "OtherData",
+ "fields": [
+ {"name": "otherdata_id", "type": "int"},
+ {"name": "otherdata_field", "type": "string"},
+ # Fully compatible change
+ {
+ "name": "newdata_field",
+ "type": "string",
+ "default": "new stuff",
+ },
+ ],
+ },
+ ],
+ },
+ ],
+ }
+
+ records = [
+ {
+ "id": "example_id_1234",
+ "data": {"otherdata_id": 1234, "otherdata_field": "some example data"},
+ }
+ ]
+
+ roundtrip_records = roundtrip(writer_schema, records, reader_schema=reader_schema)
+
+ expected_roundtrip = [
+ {
+ "id": "example_id_1234",
+ "data": {
+ "otherdata_id": 1234,
+ "otherdata_field": "some example data",
+ "newdata_field": "new stuff",
+ },
+ }
+ ]
+ assert roundtrip_records == expected_roundtrip
+
+
+def test_schema_migration_should_fail_with_different_names():
+ """https://github.com/fastavro/fastavro/issues/515"""
+ writer_schema = {
+ "type": "record",
+ "name": "A",
+ "fields": [{"type": "int", "name": "F"}],
+ }
+
+ reader_schema = {
+ "type": "record",
+ "name": "B",
+ "fields": [{"type": "int", "name": "F"}],
+ }
+
+ records = [{"F": 1}]
+
+ with pytest.raises(SchemaResolutionError):
+ roundtrip(writer_schema, records, reader_schema=reader_schema)

fix issue with resolving named types