@Test public void testUnionORCSchemaTranslation() throws Exception { Schema avroUnion = SchemaBuilder.record("test") .fields() .name("test_union") .type(SchemaBuilder.builder().unionOf().stringType().and().intType().and().nullType().endUnion()) .noDefault() .endRecord(); TypeDescription unionSchema = TypeDescription.createUnion() .addUnionChild(TypeDescription.createString()) .addUnionChild(TypeDescription.createInt()); TypeDescription recordSchemaWithUnion = TypeDescription.createStruct().addField("test_union", unionSchema); Assert.assertEquals(AvroOrcSchemaConverter.getOrcSchema(avroUnion), recordSchemaWithUnion); Schema nullableAvroUnion = SchemaBuilder.record("test") .fields() .name("test_union") .type(SchemaBuilder.builder().unionOf().stringType().and().nullType().endUnion()) .noDefault() .endRecord(); Assert.assertEquals(AvroOrcSchemaConverter.getOrcSchema(nullableAvroUnion), TypeDescription.createStruct().addField("test_union", TypeDescription.createString())); Schema nonNullableAvroUnion = SchemaBuilder.record("test") .fields() .name("test_union") .type(SchemaBuilder.builder().unionOf().stringType().endUnion()) .noDefault() .endRecord(); Assert.assertEquals(AvroOrcSchemaConverter.getOrcSchema(nonNullableAvroUnion), TypeDescription.createStruct() .addField("test_union", TypeDescription.createUnion().addUnionChild(TypeDescription.createString()))); }
public static TypeDescription getOrcSchema(Schema avroSchema) { final Schema.Type type = avroSchema.getType(); switch (type) { case NULL: final TypeDescription nullUnion = TypeDescription.createUnion(); return nullUnion; case LONG: return TypeDescription.createLong(); case INT: return TypeDescription.createInt(); case BYTES: return TypeDescription.createBinary(); case ARRAY: return TypeDescription.createList(getOrcSchema(avroSchema.getElementType())); case RECORD: final TypeDescription recordStruct = TypeDescription.createStruct(); for (Schema.Field field2 : avroSchema.getFields()) { final Schema fieldSchema = field2.schema(); final TypeDescription fieldType = getOrcSchema(fieldSchema); if (fieldType != null) { recordStruct.addField(field2.name(), fieldType); } else { throw new IllegalStateException("Should never get a null type as fieldType."); } } return recordStruct; case MAP: return TypeDescription.createMap( TypeDescription.createString(), getOrcSchema(avroSchema.getValueType())); case UNION: final List<Schema> nonNullMembers = getNonNullMembersOfUnion(avroSchema); if (isNullableUnion(avroSchema, nonNullMembers)) { return getOrcSchema(nonNullMembers.get(0)); } else { final TypeDescription union = TypeDescription.createUnion(); for (final Schema childSchema : nonNullMembers) { union.addUnionChild(getOrcSchema(childSchema)); } return union; } case STRING: return TypeDescription.createString(); case FLOAT: return TypeDescription.createFloat(); case DOUBLE: return TypeDescription.createDouble(); case BOOLEAN: return TypeDescription.createBoolean(); case ENUM: return TypeDescription.createString(); case FIXED: return TypeDescription.createBinary(); default: throw new IllegalStateException(String.format("Unrecognized Avro type: %s", type.getName())); } }
AvroOrcSchemaConverter { public static TypeDescription getOrcSchema(Schema avroSchema) { final Schema.Type type = avroSchema.getType(); switch (type) { case NULL: final TypeDescription nullUnion = TypeDescription.createUnion(); return nullUnion; case LONG: return TypeDescription.createLong(); case INT: return TypeDescription.createInt(); case BYTES: return TypeDescription.createBinary(); case ARRAY: return TypeDescription.createList(getOrcSchema(avroSchema.getElementType())); case RECORD: final TypeDescription recordStruct = TypeDescription.createStruct(); for (Schema.Field field2 : avroSchema.getFields()) { final Schema fieldSchema = field2.schema(); final TypeDescription fieldType = getOrcSchema(fieldSchema); if (fieldType != null) { recordStruct.addField(field2.name(), fieldType); } else { throw new IllegalStateException("Should never get a null type as fieldType."); } } return recordStruct; case MAP: return TypeDescription.createMap( TypeDescription.createString(), getOrcSchema(avroSchema.getValueType())); case UNION: final List<Schema> nonNullMembers = getNonNullMembersOfUnion(avroSchema); if (isNullableUnion(avroSchema, nonNullMembers)) { return getOrcSchema(nonNullMembers.get(0)); } else { final TypeDescription union = TypeDescription.createUnion(); for (final Schema childSchema : nonNullMembers) { union.addUnionChild(getOrcSchema(childSchema)); } return union; } case STRING: return TypeDescription.createString(); case FLOAT: return TypeDescription.createFloat(); case DOUBLE: return TypeDescription.createDouble(); case BOOLEAN: return TypeDescription.createBoolean(); case ENUM: return TypeDescription.createString(); case FIXED: return TypeDescription.createBinary(); default: throw new IllegalStateException(String.format("Unrecognized Avro type: %s", type.getName())); } } }
AvroOrcSchemaConverter { public static TypeDescription getOrcSchema(Schema avroSchema) { final Schema.Type type = avroSchema.getType(); switch (type) { case NULL: final TypeDescription nullUnion = TypeDescription.createUnion(); return nullUnion; case LONG: return TypeDescription.createLong(); case INT: return TypeDescription.createInt(); case BYTES: return TypeDescription.createBinary(); case ARRAY: return TypeDescription.createList(getOrcSchema(avroSchema.getElementType())); case RECORD: final TypeDescription recordStruct = TypeDescription.createStruct(); for (Schema.Field field2 : avroSchema.getFields()) { final Schema fieldSchema = field2.schema(); final TypeDescription fieldType = getOrcSchema(fieldSchema); if (fieldType != null) { recordStruct.addField(field2.name(), fieldType); } else { throw new IllegalStateException("Should never get a null type as fieldType."); } } return recordStruct; case MAP: return TypeDescription.createMap( TypeDescription.createString(), getOrcSchema(avroSchema.getValueType())); case UNION: final List<Schema> nonNullMembers = getNonNullMembersOfUnion(avroSchema); if (isNullableUnion(avroSchema, nonNullMembers)) { return getOrcSchema(nonNullMembers.get(0)); } else { final TypeDescription union = TypeDescription.createUnion(); for (final Schema childSchema : nonNullMembers) { union.addUnionChild(getOrcSchema(childSchema)); } return union; } case STRING: return TypeDescription.createString(); case FLOAT: return TypeDescription.createFloat(); case DOUBLE: return TypeDescription.createDouble(); case BOOLEAN: return TypeDescription.createBoolean(); case ENUM: return TypeDescription.createString(); case FIXED: return TypeDescription.createBinary(); default: throw new IllegalStateException(String.format("Unrecognized Avro type: %s", type.getName())); } }  }
AvroOrcSchemaConverter { public static TypeDescription getOrcSchema(Schema avroSchema) { final Schema.Type type = avroSchema.getType(); switch (type) { case NULL: final TypeDescription nullUnion = TypeDescription.createUnion(); return nullUnion; case LONG: return TypeDescription.createLong(); case INT: return TypeDescription.createInt(); case BYTES: return TypeDescription.createBinary(); case ARRAY: return TypeDescription.createList(getOrcSchema(avroSchema.getElementType())); case RECORD: final TypeDescription recordStruct = TypeDescription.createStruct(); for (Schema.Field field2 : avroSchema.getFields()) { final Schema fieldSchema = field2.schema(); final TypeDescription fieldType = getOrcSchema(fieldSchema); if (fieldType != null) { recordStruct.addField(field2.name(), fieldType); } else { throw new IllegalStateException("Should never get a null type as fieldType."); } } return recordStruct; case MAP: return TypeDescription.createMap( TypeDescription.createString(), getOrcSchema(avroSchema.getValueType())); case UNION: final List<Schema> nonNullMembers = getNonNullMembersOfUnion(avroSchema); if (isNullableUnion(avroSchema, nonNullMembers)) { return getOrcSchema(nonNullMembers.get(0)); } else { final TypeDescription union = TypeDescription.createUnion(); for (final Schema childSchema : nonNullMembers) { union.addUnionChild(getOrcSchema(childSchema)); } return union; } case STRING: return TypeDescription.createString(); case FLOAT: return TypeDescription.createFloat(); case DOUBLE: return TypeDescription.createDouble(); case BOOLEAN: return TypeDescription.createBoolean(); case ENUM: return TypeDescription.createString(); case FIXED: return TypeDescription.createBinary(); default: throw new IllegalStateException(String.format("Unrecognized Avro type: %s", type.getName())); } }  static TypeDescription getOrcSchema(Schema avroSchema); static List<Schema> getNonNullMembersOfUnion(Schema unionSchema); static Schema sanitizeNullableSchema(Schema avroSchema); }
AvroOrcSchemaConverter { public static TypeDescription getOrcSchema(Schema avroSchema) { final Schema.Type type = avroSchema.getType(); switch (type) { case NULL: final TypeDescription nullUnion = TypeDescription.createUnion(); return nullUnion; case LONG: return TypeDescription.createLong(); case INT: return TypeDescription.createInt(); case BYTES: return TypeDescription.createBinary(); case ARRAY: return TypeDescription.createList(getOrcSchema(avroSchema.getElementType())); case RECORD: final TypeDescription recordStruct = TypeDescription.createStruct(); for (Schema.Field field2 : avroSchema.getFields()) { final Schema fieldSchema = field2.schema(); final TypeDescription fieldType = getOrcSchema(fieldSchema); if (fieldType != null) { recordStruct.addField(field2.name(), fieldType); } else { throw new IllegalStateException("Should never get a null type as fieldType."); } } return recordStruct; case MAP: return TypeDescription.createMap( TypeDescription.createString(), getOrcSchema(avroSchema.getValueType())); case UNION: final List<Schema> nonNullMembers = getNonNullMembersOfUnion(avroSchema); if (isNullableUnion(avroSchema, nonNullMembers)) { return getOrcSchema(nonNullMembers.get(0)); } else { final TypeDescription union = TypeDescription.createUnion(); for (final Schema childSchema : nonNullMembers) { union.addUnionChild(getOrcSchema(childSchema)); } return union; } case STRING: return TypeDescription.createString(); case FLOAT: return TypeDescription.createFloat(); case DOUBLE: return TypeDescription.createDouble(); case BOOLEAN: return TypeDescription.createBoolean(); case ENUM: return TypeDescription.createString(); case FIXED: return TypeDescription.createBinary(); default: throw new IllegalStateException(String.format("Unrecognized Avro type: %s", type.getName())); } }  static TypeDescription getOrcSchema(Schema avroSchema); static List<Schema> getNonNullMembersOfUnion(Schema unionSchema); static Schema sanitizeNullableSchema(Schema avroSchema);  }