AvroUtils { public static Schema cleanSchema(Schema schema, List<String> propsToAvoid) { if (schema != null) { switch (schema.getType()) { case RECORD: List<Schema.Field> fields = new ArrayList<>(); for (Schema.Field field : schema.getFields()) { Schema fieldSchema = cleanSchema(field.schema(), propsToAvoid); fields.add(new Schema.Field(field.name(), fieldSchema, field.doc(), field.defaultVal())); } Schema recordSchema = Schema.createRecord(schema.getName(), schema.getDoc(), schema.getNamespace(), schema.isError(), fields); addPropsToSchema(recordSchema, schema.getObjectProps(), propsToAvoid); return recordSchema; case MAP: Schema mapSchema = Schema.createMap(cleanSchema(schema.getValueType(), propsToAvoid)); addPropsToSchema(mapSchema, schema.getObjectProps(), propsToAvoid); return mapSchema; case ARRAY: Schema arraySchema = Schema.createArray(cleanSchema(schema.getElementType(), propsToAvoid)); addPropsToSchema(arraySchema, schema.getObjectProps(), propsToAvoid); return arraySchema; case UNION: List<Schema> types = new ArrayList<>(); for (Schema unionType : schema.getTypes()) { Schema unionTypeSchema = cleanSchema(unionType, propsToAvoid); types.add(unionTypeSchema); } Schema unionSchema = Schema.createUnion(types); addPropsToSchema(unionSchema, schema.getObjectProps(), propsToAvoid); return unionSchema; case ENUM: Schema enumSchema = Schema.createEnum(schema.getName(), schema.getDoc(), schema.getNamespace(), schema.getEnumSymbols()); addPropsToSchema(enumSchema, schema.getObjectProps(), propsToAvoid); return enumSchema; case FIXED: Schema fixedSchema = Schema.createFixed(schema.getName(), schema.getDoc(), schema.getNamespace(), schema.getFixedSize()); addPropsToSchema(fixedSchema, schema.getObjectProps(), propsToAvoid); return fixedSchema; case INT: case LONG: case FLOAT: case BYTES: case STRING: case BOOLEAN: case DOUBLE: case NULL: Schema primitiveSchema = Schema.create(schema.getType()); addPropsToSchema(primitiveSchema, schema.getObjectProps(), propsToAvoid); return primitiveSchema; } } return null; }  static Schema createRecordSemanticSchema(Schema sourceSchema, Schema valueLevelMetadataSchema); static Schema copySchema(Schema sourceSchema); static boolean isPrimitiveType(Schema.Type type); static boolean isPrimitiveType(Schema.Type type, boolean enumFixedIncluded); static String itemId(String prefix, String itemId); static Map<String, Object> extractProperties(Schema schema, String propName); static void extractProperties(Schema schema, String propName, Map<String, Object> props, String prefix); static Pair<Stream<IndexedRecord>, Schema> streamAvroFile(File file); static Schema dereferencing(Schema schema); static Schema cleanSchema(Schema schema, List<String> propsToAvoid);  }
@Test public void testCleanSchemaSimplePrimitive() throws URISyntaxException, IOException { Schema schema = SchemaBuilder .record("record") .fields() .name("int1") .type() .intBuilder() .prop("prop1", "value1") .prop("prop2", "value2") .prop("prop3", "value3") .endInt() .intDefault(2) .endRecord(); Schema cleanSchema1 = AvroUtils.cleanSchema(schema, Collections.emptyList()); Schema cleanSchema2 = AvroUtils.cleanSchema(schema, Collections.singletonList("prop2")); Schema cleanSchema3 = AvroUtils.cleanSchema(schema, Arrays.asList("prop1", "prop3")); assertEquals(schema, cleanSchema1); assertNull(cleanSchema2.getField("int1").schema().getObjectProp("prop2")); assertEquals(2, cleanSchema2.getField("int1").schema().getObjectProps().size()); assertNull(cleanSchema3.getField("int1").schema().getObjectProp("prop1")); assertNull(cleanSchema3.getField("int1").schema().getObjectProp("prop3")); assertEquals(1, cleanSchema3.getField("int1").schema().getObjectProps().size()); schema = SchemaBuilder .record("record") .fields() .name("int1") .type() .optional() .intBuilder() .prop("prop1", "value1") .prop("prop2", "value2") .prop("prop3", "value3") .endInt() .endRecord(); cleanSchema1 = AvroUtils.cleanSchema(schema, Collections.emptyList()); cleanSchema2 = AvroUtils.cleanSchema(schema, Collections.singletonList("prop2")); cleanSchema3 = AvroUtils.cleanSchema(schema, Arrays.asList("prop1", "prop3")); assertEquals(schema, cleanSchema1); assertNull(cleanSchema2.getField("int1").schema().getTypes().get(1).getObjectProp("prop2")); assertEquals(2, cleanSchema2.getField("int1").schema().getTypes().get(1).getObjectProps().size()); assertNull(cleanSchema3.getField("int1").schema().getTypes().get(1).getObjectProp("prop1")); assertNull(cleanSchema3.getField("int1").schema().getTypes().get(1).getObjectProp("prop3")); assertEquals(1, cleanSchema3.getField("int1").schema().getTypes().get(1).getObjectProps().size()); }
@Test public void testCleanSchemaEnum() throws URISyntaxException, IOException { Schema schema = SchemaBuilder .record("record") .fields() .name("enum1") .type() .enumeration("enum1") .prop("prop1", "value1") .prop("prop2", "value2") .prop("prop3", "value3") .symbols("A", "B", "C") .enumDefault("B") .endRecord(); Schema cleanSchema1 = AvroUtils.cleanSchema(schema, Collections.emptyList()); Schema cleanSchema2 = AvroUtils.cleanSchema(schema, Collections.singletonList("prop2")); Schema cleanSchema3 = AvroUtils.cleanSchema(schema, Arrays.asList("prop1", "prop3")); assertEquals(schema, cleanSchema1); assertNull(cleanSchema2.getField("enum1").schema().getObjectProp("prop2")); assertEquals(2, cleanSchema2.getField("enum1").schema().getObjectProps().size()); assertNull(cleanSchema3.getField("enum1").schema().getObjectProp("prop1")); assertNull(cleanSchema3.getField("enum1").schema().getObjectProp("prop3")); assertEquals(1, cleanSchema3.getField("enum1").schema().getObjectProps().size()); schema = SchemaBuilder .record("record") .fields() .name("enum1") .type() .optional() .enumeration("enum1") .prop("prop1", "value1") .prop("prop2", "value2") .prop("prop3", "value3") .symbols("A", "B", "C") .endRecord(); cleanSchema1 = AvroUtils.cleanSchema(schema, Collections.emptyList()); cleanSchema2 = AvroUtils.cleanSchema(schema, Collections.singletonList("prop2")); cleanSchema3 = AvroUtils.cleanSchema(schema, Arrays.asList("prop1", "prop3")); assertEquals(schema, cleanSchema1); assertNull(cleanSchema2.getField("enum1").schema().getTypes().get(1).getObjectProp("prop2")); assertEquals(2, cleanSchema2.getField("enum1").schema().getTypes().get(1).getObjectProps().size()); assertNull(cleanSchema3.getField("enum1").schema().getTypes().get(1).getObjectProp("prop1")); assertNull(cleanSchema3.getField("enum1").schema().getTypes().get(1).getObjectProp("prop3")); assertEquals(1, cleanSchema3.getField("enum1").schema().getTypes().get(1).getObjectProps().size()); }
@Test public void testCleanSchemaFixed() throws URISyntaxException, IOException { Schema schema = SchemaBuilder .record("record") .fields() .name("fixed1") .type() .fixed("fixed1") .prop("prop1", "value1") .prop("prop2", "value2") .prop("prop3", "value3") .size(3) .fixedDefault("ABC") .endRecord(); Schema cleanSchema1 = AvroUtils.cleanSchema(schema, Collections.emptyList()); Schema cleanSchema2 = AvroUtils.cleanSchema(schema, Collections.singletonList("prop2")); Schema cleanSchema3 = AvroUtils.cleanSchema(schema, Arrays.asList("prop1", "prop3")); assertEquals(schema, cleanSchema1); assertNull(cleanSchema2.getField("fixed1").schema().getObjectProp("prop2")); assertEquals(2, cleanSchema2.getField("fixed1").schema().getObjectProps().size()); assertNull(cleanSchema3.getField("fixed1").schema().getObjectProp("prop1")); assertNull(cleanSchema3.getField("fixed1").schema().getObjectProp("prop3")); assertEquals(1, cleanSchema3.getField("fixed1").schema().getObjectProps().size()); schema = SchemaBuilder .record("record") .fields() .name("fixed1") .type() .optional() .fixed("fixed1") .prop("prop1", "value1") .prop("prop2", "value2") .prop("prop3", "value3") .size(3) .endRecord(); cleanSchema1 = AvroUtils.cleanSchema(schema, Collections.emptyList()); cleanSchema2 = AvroUtils.cleanSchema(schema, Collections.singletonList("prop2")); cleanSchema3 = AvroUtils.cleanSchema(schema, Arrays.asList("prop1", "prop3")); assertEquals(schema, cleanSchema1); assertNull(cleanSchema2.getField("fixed1").schema().getTypes().get(1).getObjectProp("prop2")); assertEquals(2, cleanSchema2.getField("fixed1").schema().getTypes().get(1).getObjectProps().size()); assertNull(cleanSchema3.getField("fixed1").schema().getTypes().get(1).getObjectProp("prop1")); assertNull(cleanSchema3.getField("fixed1").schema().getTypes().get(1).getObjectProp("prop3")); assertEquals(1, cleanSchema3.getField("fixed1").schema().getTypes().get(1).getObjectProps().size()); }
@Test public void testCleanSchemaMap() throws URISyntaxException, IOException { Schema schema = SchemaBuilder .record("record") .fields() .name("map1") .type() .map() .prop("prop1", "value1") .prop("prop2", "value2") .prop("prop3", "value3") .values(SchemaBuilder.builder().intType()) .mapDefault(new HashMap<>()) .endRecord(); Schema cleanSchema1 = AvroUtils.cleanSchema(schema, Collections.emptyList()); Schema cleanSchema2 = AvroUtils.cleanSchema(schema, Collections.singletonList("prop2")); Schema cleanSchema3 = AvroUtils.cleanSchema(schema, Arrays.asList("prop1", "prop3")); assertEquals(schema, cleanSchema1); assertNull(cleanSchema2.getField("map1").schema().getObjectProp("prop2")); assertEquals(2, cleanSchema2.getField("map1").schema().getObjectProps().size()); assertNull(cleanSchema3.getField("map1").schema().getObjectProp("prop1")); assertNull(cleanSchema3.getField("map1").schema().getObjectProp("prop3")); assertEquals(1, cleanSchema3.getField("map1").schema().getObjectProps().size()); schema = SchemaBuilder .record("record") .fields() .name("map1") .type() .optional() .map() .prop("prop1", "value1") .prop("prop2", "value2") .prop("prop3", "value3") .values(SchemaBuilder.builder().intType()) .endRecord(); cleanSchema1 = AvroUtils.cleanSchema(schema, Collections.emptyList()); cleanSchema2 = AvroUtils.cleanSchema(schema, Collections.singletonList("prop2")); cleanSchema3 = AvroUtils.cleanSchema(schema, Arrays.asList("prop1", "prop3")); assertEquals(schema, cleanSchema1); assertEquals(schema, cleanSchema1); assertNull(cleanSchema2.getField("map1").schema().getTypes().get(1).getObjectProp("prop2")); assertEquals(2, cleanSchema2.getField("map1").schema().getTypes().get(1).getObjectProps().size()); assertNull(cleanSchema3.getField("map1").schema().getTypes().get(1).getObjectProp("prop1")); assertNull(cleanSchema3.getField("map1").schema().getTypes().get(1).getObjectProp("prop3")); assertEquals(1, cleanSchema3.getField("map1").schema().getTypes().get(1).getObjectProps().size()); }
@Test public void testCleanSchemaArray() throws URISyntaxException, IOException { Schema schema = SchemaBuilder .record("record") .fields() .name("array1") .type() .array() .prop("prop1", "value1") .prop("prop2", "value2") .prop("prop3", "value3") .items(SchemaBuilder.builder().intType()) .arrayDefault(new ArrayList<>()) .endRecord(); Schema cleanSchema1 = AvroUtils.cleanSchema(schema, Collections.emptyList()); Schema cleanSchema2 = AvroUtils.cleanSchema(schema, Collections.singletonList("prop2")); Schema cleanSchema3 = AvroUtils.cleanSchema(schema, Arrays.asList("prop1", "prop3")); assertEquals(schema, cleanSchema1); assertNull(cleanSchema2.getField("array1").schema().getObjectProp("prop2")); assertEquals(2, cleanSchema2.getField("array1").schema().getObjectProps().size()); assertNull(cleanSchema3.getField("array1").schema().getObjectProp("prop1")); assertNull(cleanSchema3.getField("array1").schema().getObjectProp("prop3")); assertEquals(1, cleanSchema3.getField("array1").schema().getObjectProps().size()); schema = SchemaBuilder .record("record") .fields() .name("array1") .type() .optional() .array() .prop("prop1", "value1") .prop("prop2", "value2") .prop("prop3", "value3") .items(SchemaBuilder.builder().intType()) .endRecord(); cleanSchema1 = AvroUtils.cleanSchema(schema, Collections.emptyList()); cleanSchema2 = AvroUtils.cleanSchema(schema, Collections.singletonList("prop2")); cleanSchema3 = AvroUtils.cleanSchema(schema, Arrays.asList("prop1", "prop3")); assertEquals(schema, cleanSchema1); assertEquals(schema, cleanSchema1); assertNull(cleanSchema2.getField("array1").schema().getTypes().get(1).getObjectProp("prop2")); assertEquals(2, cleanSchema2.getField("array1").schema().getTypes().get(1).getObjectProps().size()); assertNull(cleanSchema3.getField("array1").schema().getTypes().get(1).getObjectProp("prop1")); assertNull(cleanSchema3.getField("array1").schema().getTypes().get(1).getObjectProp("prop3")); assertEquals(1, cleanSchema3.getField("array1").schema().getTypes().get(1).getObjectProps().size()); }
@Test public void testCleanSchemaRecord() throws URISyntaxException, IOException { Schema schema = SchemaBuilder .record("record") .fields() .name("record1") .type() .record("record1_1") .prop("prop1", "value1") .prop("prop2", "value2") .prop("prop3", "value3") .fields() .requiredInt("int1") .endRecord() .noDefault() .endRecord(); Schema cleanSchema1 = AvroUtils.cleanSchema(schema, Collections.emptyList()); Schema cleanSchema2 = AvroUtils.cleanSchema(schema, Collections.singletonList("prop2")); Schema cleanSchema3 = AvroUtils.cleanSchema(schema, Arrays.asList("prop1", "prop3")); assertEquals(schema, cleanSchema1); assertNull(cleanSchema2.getField("record1").schema().getObjectProp("prop2")); assertEquals(2, cleanSchema2.getField("record1").schema().getObjectProps().size()); assertNull(cleanSchema3.getField("record1").schema().getObjectProp("prop1")); assertNull(cleanSchema3.getField("record1").schema().getObjectProp("prop3")); assertEquals(1, cleanSchema3.getField("record1").schema().getObjectProps().size()); schema = SchemaBuilder .record("record") .fields() .name("record1") .type() .optional() .record("record1_1") .prop("prop1", "value1") .prop("prop2", "value2") .prop("prop3", "value3") .fields() .requiredInt("int1") .endRecord() .endRecord(); cleanSchema1 = AvroUtils.cleanSchema(schema, Collections.emptyList()); cleanSchema2 = AvroUtils.cleanSchema(schema, Collections.singletonList("prop2")); cleanSchema3 = AvroUtils.cleanSchema(schema, Arrays.asList("prop1", "prop3")); assertEquals(schema, cleanSchema1); assertEquals(schema, cleanSchema1); assertNull(cleanSchema2.getField("record1").schema().getTypes().get(1).getObjectProp("prop2")); assertEquals(2, cleanSchema2.getField("record1").schema().getTypes().get(1).getObjectProps().size()); assertNull(cleanSchema3.getField("record1").schema().getTypes().get(1).getObjectProp("prop1")); assertNull(cleanSchema3.getField("record1").schema().getTypes().get(1).getObjectProp("prop3")); assertEquals(1, cleanSchema3.getField("record1").schema().getTypes().get(1).getObjectProps().size()); }