@Test public void toDataTypeBytes() { assertEquals("Invalid DataType", DataTypes.BinaryType, AvroUtils.dataTypeFor(SchemaBuilder.builder().bytesType())); }
public static DataType dataTypeFor(Schema schemaType) { LOG.trace("Converting Schema[{}] to DataType", schemaType); boolean isOptional = isNullable(schemaType); if (isOptional) { if (schemaType.getTypes().size() == 2) { LOG.trace("Unwrapping simple 'optional' union for {}", schemaType); for (Schema s : schemaType.getTypes()) { if (s.getType().equals(NULL)) { continue; } schemaType = s; break; } } } if (null != schemaType.getLogicalType()) { LogicalType logicalType = schemaType.getLogicalType(); switch (logicalType.getName()) { case "date" : return DataTypes.DateType; case "timestamp-millis" : return DataTypes.TimestampType; case "decimal" : LogicalTypes.Decimal decimal = (LogicalTypes.Decimal) logicalType; return DataTypes.createDecimalType(decimal.getPrecision(), decimal.getScale()); default: LOG.warn("Unsupported LogicalType[{}], continuing with underlying base type", logicalType.getName()); } } switch (schemaType.getType()) { case RECORD: List<StructField> structFieldList = Lists.newArrayListWithCapacity(schemaType.getFields().size()); for (Field f : schemaType.getFields()) { structFieldList.add(DataTypes.createStructField(f.name(), dataTypeFor(f.schema()), isNullable(f.schema()))); } return DataTypes.createStructType(structFieldList); case ARRAY: Schema elementType = schemaType.getElementType(); return DataTypes.createArrayType(dataTypeFor(elementType), isNullable(elementType)); case MAP: Schema valueType = schemaType.getValueType(); return DataTypes.createMapType(DataTypes.StringType, dataTypeFor(valueType), isNullable(valueType)); case UNION: List<StructField> unionFieldList = Lists.newArrayListWithCapacity(schemaType.getTypes().size()); int m = 0; for (Schema u : schemaType.getTypes()) { unionFieldList.add(DataTypes.createStructField("member" + m++, dataTypeFor(u), isNullable(u))); } return DataTypes.createStructType(unionFieldList); case FIXED: case BYTES: return DataTypes.BinaryType; case ENUM: case STRING: return DataTypes.StringType; case INT: return DataTypes.IntegerType; case LONG: return DataTypes.LongType; case FLOAT: return DataTypes.FloatType; case DOUBLE: return DataTypes.DoubleType; case BOOLEAN: return DataTypes.BooleanType; case NULL: return DataTypes.NullType; default: throw new RuntimeException(String.format("Unrecognized or unsupported Avro Type conversion: %s", schemaType)); } }
AvroUtils { public static DataType dataTypeFor(Schema schemaType) { LOG.trace("Converting Schema[{}] to DataType", schemaType); boolean isOptional = isNullable(schemaType); if (isOptional) { if (schemaType.getTypes().size() == 2) { LOG.trace("Unwrapping simple 'optional' union for {}", schemaType); for (Schema s : schemaType.getTypes()) { if (s.getType().equals(NULL)) { continue; } schemaType = s; break; } } } if (null != schemaType.getLogicalType()) { LogicalType logicalType = schemaType.getLogicalType(); switch (logicalType.getName()) { case "date" : return DataTypes.DateType; case "timestamp-millis" : return DataTypes.TimestampType; case "decimal" : LogicalTypes.Decimal decimal = (LogicalTypes.Decimal) logicalType; return DataTypes.createDecimalType(decimal.getPrecision(), decimal.getScale()); default: LOG.warn("Unsupported LogicalType[{}], continuing with underlying base type", logicalType.getName()); } } switch (schemaType.getType()) { case RECORD: List<StructField> structFieldList = Lists.newArrayListWithCapacity(schemaType.getFields().size()); for (Field f : schemaType.getFields()) { structFieldList.add(DataTypes.createStructField(f.name(), dataTypeFor(f.schema()), isNullable(f.schema()))); } return DataTypes.createStructType(structFieldList); case ARRAY: Schema elementType = schemaType.getElementType(); return DataTypes.createArrayType(dataTypeFor(elementType), isNullable(elementType)); case MAP: Schema valueType = schemaType.getValueType(); return DataTypes.createMapType(DataTypes.StringType, dataTypeFor(valueType), isNullable(valueType)); case UNION: List<StructField> unionFieldList = Lists.newArrayListWithCapacity(schemaType.getTypes().size()); int m = 0; for (Schema u : schemaType.getTypes()) { unionFieldList.add(DataTypes.createStructField("member" + m++, dataTypeFor(u), isNullable(u))); } return DataTypes.createStructType(unionFieldList); case FIXED: case BYTES: return DataTypes.BinaryType; case ENUM: case STRING: return DataTypes.StringType; case INT: return DataTypes.IntegerType; case LONG: return DataTypes.LongType; case FLOAT: return DataTypes.FloatType; case DOUBLE: return DataTypes.DoubleType; case BOOLEAN: return DataTypes.BooleanType; case NULL: return DataTypes.NullType; default: throw new RuntimeException(String.format("Unrecognized or unsupported Avro Type conversion: %s", schemaType)); } } }
AvroUtils { public static DataType dataTypeFor(Schema schemaType) { LOG.trace("Converting Schema[{}] to DataType", schemaType); boolean isOptional = isNullable(schemaType); if (isOptional) { if (schemaType.getTypes().size() == 2) { LOG.trace("Unwrapping simple 'optional' union for {}", schemaType); for (Schema s : schemaType.getTypes()) { if (s.getType().equals(NULL)) { continue; } schemaType = s; break; } } } if (null != schemaType.getLogicalType()) { LogicalType logicalType = schemaType.getLogicalType(); switch (logicalType.getName()) { case "date" : return DataTypes.DateType; case "timestamp-millis" : return DataTypes.TimestampType; case "decimal" : LogicalTypes.Decimal decimal = (LogicalTypes.Decimal) logicalType; return DataTypes.createDecimalType(decimal.getPrecision(), decimal.getScale()); default: LOG.warn("Unsupported LogicalType[{}], continuing with underlying base type", logicalType.getName()); } } switch (schemaType.getType()) { case RECORD: List<StructField> structFieldList = Lists.newArrayListWithCapacity(schemaType.getFields().size()); for (Field f : schemaType.getFields()) { structFieldList.add(DataTypes.createStructField(f.name(), dataTypeFor(f.schema()), isNullable(f.schema()))); } return DataTypes.createStructType(structFieldList); case ARRAY: Schema elementType = schemaType.getElementType(); return DataTypes.createArrayType(dataTypeFor(elementType), isNullable(elementType)); case MAP: Schema valueType = schemaType.getValueType(); return DataTypes.createMapType(DataTypes.StringType, dataTypeFor(valueType), isNullable(valueType)); case UNION: List<StructField> unionFieldList = Lists.newArrayListWithCapacity(schemaType.getTypes().size()); int m = 0; for (Schema u : schemaType.getTypes()) { unionFieldList.add(DataTypes.createStructField("member" + m++, dataTypeFor(u), isNullable(u))); } return DataTypes.createStructType(unionFieldList); case FIXED: case BYTES: return DataTypes.BinaryType; case ENUM: case STRING: return DataTypes.StringType; case INT: return DataTypes.IntegerType; case LONG: return DataTypes.LongType; case FLOAT: return DataTypes.FloatType; case DOUBLE: return DataTypes.DoubleType; case BOOLEAN: return DataTypes.BooleanType; case NULL: return DataTypes.NullType; default: throw new RuntimeException(String.format("Unrecognized or unsupported Avro Type conversion: %s", schemaType)); } }  }
AvroUtils { public static DataType dataTypeFor(Schema schemaType) { LOG.trace("Converting Schema[{}] to DataType", schemaType); boolean isOptional = isNullable(schemaType); if (isOptional) { if (schemaType.getTypes().size() == 2) { LOG.trace("Unwrapping simple 'optional' union for {}", schemaType); for (Schema s : schemaType.getTypes()) { if (s.getType().equals(NULL)) { continue; } schemaType = s; break; } } } if (null != schemaType.getLogicalType()) { LogicalType logicalType = schemaType.getLogicalType(); switch (logicalType.getName()) { case "date" : return DataTypes.DateType; case "timestamp-millis" : return DataTypes.TimestampType; case "decimal" : LogicalTypes.Decimal decimal = (LogicalTypes.Decimal) logicalType; return DataTypes.createDecimalType(decimal.getPrecision(), decimal.getScale()); default: LOG.warn("Unsupported LogicalType[{}], continuing with underlying base type", logicalType.getName()); } } switch (schemaType.getType()) { case RECORD: List<StructField> structFieldList = Lists.newArrayListWithCapacity(schemaType.getFields().size()); for (Field f : schemaType.getFields()) { structFieldList.add(DataTypes.createStructField(f.name(), dataTypeFor(f.schema()), isNullable(f.schema()))); } return DataTypes.createStructType(structFieldList); case ARRAY: Schema elementType = schemaType.getElementType(); return DataTypes.createArrayType(dataTypeFor(elementType), isNullable(elementType)); case MAP: Schema valueType = schemaType.getValueType(); return DataTypes.createMapType(DataTypes.StringType, dataTypeFor(valueType), isNullable(valueType)); case UNION: List<StructField> unionFieldList = Lists.newArrayListWithCapacity(schemaType.getTypes().size()); int m = 0; for (Schema u : schemaType.getTypes()) { unionFieldList.add(DataTypes.createStructField("member" + m++, dataTypeFor(u), isNullable(u))); } return DataTypes.createStructType(unionFieldList); case FIXED: case BYTES: return DataTypes.BinaryType; case ENUM: case STRING: return DataTypes.StringType; case INT: return DataTypes.IntegerType; case LONG: return DataTypes.LongType; case FLOAT: return DataTypes.FloatType; case DOUBLE: return DataTypes.DoubleType; case BOOLEAN: return DataTypes.BooleanType; case NULL: return DataTypes.NullType; default: throw new RuntimeException(String.format("Unrecognized or unsupported Avro Type conversion: %s", schemaType)); } }  static Schema schemaFor(StructType structType); static Schema schemaFor(StructType structType, String record, String namespace); static Schema typeFor(DataType dataType); static Schema typeFor(DataType dataType, boolean isOptional); static StructType structTypeFor(Schema schema); static DataType dataTypeFor(Schema schemaType); }
AvroUtils { public static DataType dataTypeFor(Schema schemaType) { LOG.trace("Converting Schema[{}] to DataType", schemaType); boolean isOptional = isNullable(schemaType); if (isOptional) { if (schemaType.getTypes().size() == 2) { LOG.trace("Unwrapping simple 'optional' union for {}", schemaType); for (Schema s : schemaType.getTypes()) { if (s.getType().equals(NULL)) { continue; } schemaType = s; break; } } } if (null != schemaType.getLogicalType()) { LogicalType logicalType = schemaType.getLogicalType(); switch (logicalType.getName()) { case "date" : return DataTypes.DateType; case "timestamp-millis" : return DataTypes.TimestampType; case "decimal" : LogicalTypes.Decimal decimal = (LogicalTypes.Decimal) logicalType; return DataTypes.createDecimalType(decimal.getPrecision(), decimal.getScale()); default: LOG.warn("Unsupported LogicalType[{}], continuing with underlying base type", logicalType.getName()); } } switch (schemaType.getType()) { case RECORD: List<StructField> structFieldList = Lists.newArrayListWithCapacity(schemaType.getFields().size()); for (Field f : schemaType.getFields()) { structFieldList.add(DataTypes.createStructField(f.name(), dataTypeFor(f.schema()), isNullable(f.schema()))); } return DataTypes.createStructType(structFieldList); case ARRAY: Schema elementType = schemaType.getElementType(); return DataTypes.createArrayType(dataTypeFor(elementType), isNullable(elementType)); case MAP: Schema valueType = schemaType.getValueType(); return DataTypes.createMapType(DataTypes.StringType, dataTypeFor(valueType), isNullable(valueType)); case UNION: List<StructField> unionFieldList = Lists.newArrayListWithCapacity(schemaType.getTypes().size()); int m = 0; for (Schema u : schemaType.getTypes()) { unionFieldList.add(DataTypes.createStructField("member" + m++, dataTypeFor(u), isNullable(u))); } return DataTypes.createStructType(unionFieldList); case FIXED: case BYTES: return DataTypes.BinaryType; case ENUM: case STRING: return DataTypes.StringType; case INT: return DataTypes.IntegerType; case LONG: return DataTypes.LongType; case FLOAT: return DataTypes.FloatType; case DOUBLE: return DataTypes.DoubleType; case BOOLEAN: return DataTypes.BooleanType; case NULL: return DataTypes.NullType; default: throw new RuntimeException(String.format("Unrecognized or unsupported Avro Type conversion: %s", schemaType)); } }  static Schema schemaFor(StructType structType); static Schema schemaFor(StructType structType, String record, String namespace); static Schema typeFor(DataType dataType); static Schema typeFor(DataType dataType, boolean isOptional); static StructType structTypeFor(Schema schema); static DataType dataTypeFor(Schema schemaType);  }