AvroUtils { public static Schema dereferencing(Schema schema) { Schema dereferencedSchema = schema; Stream names = getNamedTypes(schema); List<String> flattenNames = flattenStream(names); Set<String> distinctFlattenNames = new HashSet<>(flattenNames); if (distinctFlattenNames.size() != flattenNames.size()) { Map<String, String> namespaces = new HashMap<>(); for (String name : distinctFlattenNames) { namespaces.put(name, "a"); } return buildDereferencedSchema(schema, namespaces); } return dereferencedSchema; }  static Schema createRecordSemanticSchema(Schema sourceSchema, Schema valueLevelMetadataSchema); static Schema copySchema(Schema sourceSchema); static boolean isPrimitiveType(Schema.Type type); static boolean isPrimitiveType(Schema.Type type, boolean enumFixedIncluded); static String itemId(String prefix, String itemId); static Map<String, Object> extractProperties(Schema schema, String propName); static void extractProperties(Schema schema, String propName, Map<String, Object> props, String prefix); static Pair<Stream<IndexedRecord>, Schema> streamAvroFile(File file); static Schema dereferencing(Schema schema); static Schema cleanSchema(Schema schema, List<String> propsToAvoid);  }
@Test public void testDereferencingOfSwitch() throws IOException { String path = AvroUtilsTest.class.getResource("./Switch.avro").getPath(); File fileEntry = new File(path); DataFileReader<GenericRecord> dateAvroReader = new DataFileReader<>(fileEntry, new GenericDatumReader<>()); Schema schemaWithRefType = dateAvroReader.getSchema(); Schema schemaWithoutRefTypes = AvroUtils.dereferencing(schemaWithRefType); assertNotEquals(schemaWithRefType, schemaWithoutRefTypes); dateAvroReader = new DataFileReader<>(fileEntry, new GenericDatumReader<>(schemaWithoutRefTypes)); assertNotNull(dateAvroReader.iterator().next()); }
@Test public void testDereferencingIsIdempotent() throws IOException { String path = AvroUtilsTest.class.getResource("./Switch.avro").getPath(); File fileEntry = new File(path); DataFileReader<GenericRecord> dateAvroReader = new DataFileReader<>(fileEntry, new GenericDatumReader<>()); Schema schemaWithRefType = dateAvroReader.getSchema(); Schema schemaWithoutRefTypes = AvroUtils.dereferencing(schemaWithRefType); assertEquals(schemaWithoutRefTypes, AvroUtils.dereferencing(schemaWithoutRefTypes)); }
@Test public void testDereferencingIsIdentityForSchemaWithoutRef() { Schema schemaWithoutRefTypes = new Schema.Parser().parse( "{\"type\": \"record\",\"name\": \"base\",\"namespace\": \"tdc.qa\",\"fields\": [{ \"name\": \"string1\", \"type\": \"string\" },{ \"name\": \"null1\", \"type\": \"null\" },{ \"name\": \"boolean1\", \"type\": \"boolean\" },{ \"name\": \"int1\", \"type\": \"int\" },{ \"name\": \"long1\", \"type\": \"long\" },{ \"name\": \"float1\", \"type\": \"float\" },{ \"name\": \"double1\", \"type\": \"double\" },{ \"name\": \"bytes1\", \"type\": \"bytes\" },{\"name\": \"enum1\",\"type\": {\"type\": \"enum\",\"name\": \"enum11\",\"symbols\": [\"One\", \"Two\", \"Three\", \"For\"]}},{ \"name\": \"array1\", \"type\": { \"type\": \"array\", \"items\": \"string\" } },{ \"name\": \"map1\", \"type\": { \"type\": \"map\", \"values\": \"long\" } },{ \"name\": \"union1\", \"type\": [\"null\", \"string\", \"boolean\"] },{\"name\": \"fixed1\",\"type\": { \"type\": \"fixed\", \"name\": \"fixed\", \"size\": 15 }}]}"); assertEquals(schemaWithoutRefTypes, AvroUtils.dereferencing(schemaWithoutRefTypes)); }
@Test public void testDereferencingOfnoFancy() throws IOException { String path = AvroUtilsTest.class.getResource("./no-fancy-structures-10.avro").getPath(); File fileEntry = new File(path); DataFileReader<GenericRecord> dateAvroReader = new DataFileReader<>(fileEntry, new GenericDatumReader<>()); Schema schemaWithRefType = dateAvroReader.getSchema(); Schema schemaWithoutRefTypes = AvroUtils.dereferencing(schemaWithRefType); assertNotEquals(schemaWithRefType, schemaWithoutRefTypes); dateAvroReader = new DataFileReader<>(fileEntry, new GenericDatumReader<>(schemaWithoutRefTypes)); assertNotNull(dateAvroReader.iterator().next()); }
@Test public void testDereferencingOfUnionOfComplexRefType() throws IOException { String path = AvroUtilsTest.class.getResource("./UnionOfComplexRefType.avro").getPath(); File fileEntry = new File(path); DataFileReader<GenericRecord> dateAvroReader = new DataFileReader<>(fileEntry, new GenericDatumReader<>()); Schema schemaWithRefType = dateAvroReader.getSchema(); Schema schemaWithoutRefTypes = AvroUtils.dereferencing(schemaWithRefType); assertNotEquals(schemaWithRefType, schemaWithoutRefTypes); dateAvroReader = new DataFileReader<>(fileEntry, new GenericDatumReader<>(schemaWithoutRefTypes)); assertNotNull(dateAvroReader.iterator().next()); }
@Test public void testDereferencingOfUnionOfMapArrayRefType() throws IOException { String path = AvroUtilsTest.class.getResource("./UnionOfMapArrayRefType.avro").getPath(); File fileEntry = new File(path); DataFileReader<GenericRecord> dateAvroReader = new DataFileReader<>(fileEntry, new GenericDatumReader<>()); Schema schemaWithRefType = dateAvroReader.getSchema(); Schema schemaWithoutRefTypes = AvroUtils.dereferencing(schemaWithRefType); assertNotEquals(schemaWithRefType, schemaWithoutRefTypes); dateAvroReader = new DataFileReader<>(fileEntry, new GenericDatumReader<>(schemaWithoutRefTypes)); assertNotNull(dateAvroReader.iterator().next()); }
@Test public void testDereferencingOfExample2() throws IOException { String path = AvroUtilsTest.class.getResource("./example2.avro").getPath(); File fileEntry = new File(path); DataFileReader<GenericRecord> dateAvroReader = new DataFileReader<>(fileEntry, new GenericDatumReader<>()); Schema schemaWithRefType = dateAvroReader.getSchema(); Schema schemaWithoutRefTypes = AvroUtils.dereferencing(schemaWithRefType); assertNotEquals(schemaWithRefType, schemaWithoutRefTypes); dateAvroReader = new DataFileReader<>(fileEntry, new GenericDatumReader<>(schemaWithoutRefTypes)); assertNotNull(dateAvroReader.iterator().next()); assertEquals( "{\"type\":\"record\",\"name\":\"Person\",\"namespace\":\"experiment.sample\",\"fields\":[{\"name\":\"firstName\",\"type\":\"string\"},{\"name\":\"midleName\",\"type\":[\"null\",\"string\"]},{\"name\":\"lastName\",\"type\":\"string\"},{\"name\":\"homeAddress\",\"type\":[\"null\",{\"type\":\"record\",\"name\":\"Address\",\"namespace\":\"experiment.sample.a\",\"fields\":[{\"name\":\"line\",\"type\":\"string\"},{\"name\":\"postalCode\",\"type\":\"string\"},{\"name\":\"city\",\"type\":\"string\"}]}]},{\"name\":\"companyAddress\",\"type\":[\"null\",{\"type\":\"record\",\"name\":\"Address\",\"namespace\":\"experiment.sample.b\",\"fields\":[{\"name\":\"line\",\"type\":\"string\"},{\"name\":\"postalCode\",\"type\":\"string\"},{\"name\":\"city\",\"type\":\"string\"}]}]}]}", schemaWithoutRefTypes.toString()); }
@Test public void testDereferencingArray() throws URISyntaxException, IOException { byte[] avsc = Files.readAllBytes(Paths.get(getClass().getResource("./person.avsc").toURI())); Schema schema = new Schema.Parser().parse(new String(avsc)); Schema schemaWithoutRefTypes = AvroUtils.dereferencing(schema); assertNotEquals(schema, schemaWithoutRefTypes); }
@Test public void testDereferencingoneLevelComplex() throws URISyntaxException, IOException { byte[] avsc = Files.readAllBytes(Paths.get(getClass().getResource("./oneLevelComplex.avsc").toURI())); Schema schema = new Schema.Parser().parse(new String(avsc)); Schema schemaWithoutRefTypes = AvroUtils.dereferencing(schema); assertEquals(schema, schemaWithoutRefTypes); }
@Test public void testDereferencingMultiLevelComplex() throws URISyntaxException, IOException { byte[] avsc = Files.readAllBytes(Paths.get(getClass().getResource("./multiLevelComplex.avsc").toURI())); Schema schema = new Schema.Parser().parse(new String(avsc)); Schema schemaWithoutRefTypes = AvroUtils.dereferencing(schema); assertNotEquals(schema, schemaWithoutRefTypes); }
@Test public void testDereferencingUnionOfComplex() throws URISyntaxException, IOException { byte[] avsc = Files.readAllBytes(Paths.get(getClass().getResource("./unionOfComplex.avsc").toURI())); Schema schema = new Schema.Parser().parse(new String(avsc)); Schema schemaWithoutRefTypes = AvroUtils.dereferencing(schema); assertNotEquals(schema, schemaWithoutRefTypes); }