@Test public void testGetPinotSchemaFromAvroSchemaNullFieldTypeMap() throws IOException { org.apache.avro.Schema avroSchema = new org.apache.avro.Schema.Parser().parse(ClassLoader.getSystemResourceAsStream(AVRO_SCHEMA)); Schema inferredPinotSchema = AvroUtils.getPinotSchemaFromAvroSchema(avroSchema, null, null); Schema expectedSchema = new Schema.SchemaBuilder().addSingleValueDimension("d1", DataType.STRING) .addSingleValueDimension("d2", DataType.LONG).addSingleValueDimension("d3", DataType.STRING) .addSingleValueDimension("m1", DataType.INT).addSingleValueDimension("m2", DataType.INT) .addSingleValueDimension("hoursSinceEpoch", DataType.LONG).build(); Assert.assertEquals(expectedSchema, inferredPinotSchema); }
public static Schema getPinotSchemaFromAvroSchema(org.apache.avro.Schema avroSchema, @Nullable Map<String, FieldSpec.FieldType> fieldTypeMap, @Nullable TimeUnit timeUnit) { Schema pinotSchema = new Schema(); for (Field field : avroSchema.getFields()) { String fieldName = field.name(); FieldSpec.DataType dataType = extractFieldDataType(field); boolean isSingleValueField = isSingleValueField(field); if (fieldTypeMap == null) { pinotSchema.addField(new DimensionFieldSpec(fieldName, dataType, isSingleValueField)); } else { FieldSpec.FieldType fieldType = fieldTypeMap.get(fieldName); Preconditions.checkNotNull(fieldType, "Field type not specified for field: %s", fieldName); switch (fieldType) { case DIMENSION: pinotSchema.addField(new DimensionFieldSpec(fieldName, dataType, isSingleValueField)); break; case METRIC: Preconditions.checkState(isSingleValueField, "Metric field: %s cannot be multi-valued", fieldName); pinotSchema.addField(new MetricFieldSpec(fieldName, dataType)); break; case TIME: Preconditions.checkState(isSingleValueField, "Time field: %s cannot be multi-valued", fieldName); Preconditions.checkNotNull(timeUnit, "Time unit cannot be null"); pinotSchema.addField(new TimeFieldSpec(new TimeGranularitySpec(dataType, timeUnit, field.name()))); break; case DATE_TIME: Preconditions.checkState(isSingleValueField, "Time field: %s cannot be multi-valued", fieldName); Preconditions.checkNotNull(timeUnit, "Time unit cannot be null"); pinotSchema.addField(new DateTimeFieldSpec(field.name(), dataType, new DateTimeFormatSpec(1, timeUnit.toString(), DateTimeFieldSpec.TimeFormat.EPOCH.toString()) .getFormat(), new DateTimeGranularitySpec(1, timeUnit).getGranularity())); break; default: throw new UnsupportedOperationException( "Unsupported field type: " + fieldType + " for field: " + fieldName); } } } return pinotSchema; }
AvroUtils { public static Schema getPinotSchemaFromAvroSchema(org.apache.avro.Schema avroSchema, @Nullable Map<String, FieldSpec.FieldType> fieldTypeMap, @Nullable TimeUnit timeUnit) { Schema pinotSchema = new Schema(); for (Field field : avroSchema.getFields()) { String fieldName = field.name(); FieldSpec.DataType dataType = extractFieldDataType(field); boolean isSingleValueField = isSingleValueField(field); if (fieldTypeMap == null) { pinotSchema.addField(new DimensionFieldSpec(fieldName, dataType, isSingleValueField)); } else { FieldSpec.FieldType fieldType = fieldTypeMap.get(fieldName); Preconditions.checkNotNull(fieldType, "Field type not specified for field: %s", fieldName); switch (fieldType) { case DIMENSION: pinotSchema.addField(new DimensionFieldSpec(fieldName, dataType, isSingleValueField)); break; case METRIC: Preconditions.checkState(isSingleValueField, "Metric field: %s cannot be multi-valued", fieldName); pinotSchema.addField(new MetricFieldSpec(fieldName, dataType)); break; case TIME: Preconditions.checkState(isSingleValueField, "Time field: %s cannot be multi-valued", fieldName); Preconditions.checkNotNull(timeUnit, "Time unit cannot be null"); pinotSchema.addField(new TimeFieldSpec(new TimeGranularitySpec(dataType, timeUnit, field.name()))); break; case DATE_TIME: Preconditions.checkState(isSingleValueField, "Time field: %s cannot be multi-valued", fieldName); Preconditions.checkNotNull(timeUnit, "Time unit cannot be null"); pinotSchema.addField(new DateTimeFieldSpec(field.name(), dataType, new DateTimeFormatSpec(1, timeUnit.toString(), DateTimeFieldSpec.TimeFormat.EPOCH.toString()) .getFormat(), new DateTimeGranularitySpec(1, timeUnit).getGranularity())); break; default: throw new UnsupportedOperationException( "Unsupported field type: " + fieldType + " for field: " + fieldName); } } } return pinotSchema; } }
AvroUtils { public static Schema getPinotSchemaFromAvroSchema(org.apache.avro.Schema avroSchema, @Nullable Map<String, FieldSpec.FieldType> fieldTypeMap, @Nullable TimeUnit timeUnit) { Schema pinotSchema = new Schema(); for (Field field : avroSchema.getFields()) { String fieldName = field.name(); FieldSpec.DataType dataType = extractFieldDataType(field); boolean isSingleValueField = isSingleValueField(field); if (fieldTypeMap == null) { pinotSchema.addField(new DimensionFieldSpec(fieldName, dataType, isSingleValueField)); } else { FieldSpec.FieldType fieldType = fieldTypeMap.get(fieldName); Preconditions.checkNotNull(fieldType, "Field type not specified for field: %s", fieldName); switch (fieldType) { case DIMENSION: pinotSchema.addField(new DimensionFieldSpec(fieldName, dataType, isSingleValueField)); break; case METRIC: Preconditions.checkState(isSingleValueField, "Metric field: %s cannot be multi-valued", fieldName); pinotSchema.addField(new MetricFieldSpec(fieldName, dataType)); break; case TIME: Preconditions.checkState(isSingleValueField, "Time field: %s cannot be multi-valued", fieldName); Preconditions.checkNotNull(timeUnit, "Time unit cannot be null"); pinotSchema.addField(new TimeFieldSpec(new TimeGranularitySpec(dataType, timeUnit, field.name()))); break; case DATE_TIME: Preconditions.checkState(isSingleValueField, "Time field: %s cannot be multi-valued", fieldName); Preconditions.checkNotNull(timeUnit, "Time unit cannot be null"); pinotSchema.addField(new DateTimeFieldSpec(field.name(), dataType, new DateTimeFormatSpec(1, timeUnit.toString(), DateTimeFieldSpec.TimeFormat.EPOCH.toString()) .getFormat(), new DateTimeGranularitySpec(1, timeUnit).getGranularity())); break; default: throw new UnsupportedOperationException( "Unsupported field type: " + fieldType + " for field: " + fieldName); } } } return pinotSchema; } private  AvroUtils(); }
AvroUtils { public static Schema getPinotSchemaFromAvroSchema(org.apache.avro.Schema avroSchema, @Nullable Map<String, FieldSpec.FieldType> fieldTypeMap, @Nullable TimeUnit timeUnit) { Schema pinotSchema = new Schema(); for (Field field : avroSchema.getFields()) { String fieldName = field.name(); FieldSpec.DataType dataType = extractFieldDataType(field); boolean isSingleValueField = isSingleValueField(field); if (fieldTypeMap == null) { pinotSchema.addField(new DimensionFieldSpec(fieldName, dataType, isSingleValueField)); } else { FieldSpec.FieldType fieldType = fieldTypeMap.get(fieldName); Preconditions.checkNotNull(fieldType, "Field type not specified for field: %s", fieldName); switch (fieldType) { case DIMENSION: pinotSchema.addField(new DimensionFieldSpec(fieldName, dataType, isSingleValueField)); break; case METRIC: Preconditions.checkState(isSingleValueField, "Metric field: %s cannot be multi-valued", fieldName); pinotSchema.addField(new MetricFieldSpec(fieldName, dataType)); break; case TIME: Preconditions.checkState(isSingleValueField, "Time field: %s cannot be multi-valued", fieldName); Preconditions.checkNotNull(timeUnit, "Time unit cannot be null"); pinotSchema.addField(new TimeFieldSpec(new TimeGranularitySpec(dataType, timeUnit, field.name()))); break; case DATE_TIME: Preconditions.checkState(isSingleValueField, "Time field: %s cannot be multi-valued", fieldName); Preconditions.checkNotNull(timeUnit, "Time unit cannot be null"); pinotSchema.addField(new DateTimeFieldSpec(field.name(), dataType, new DateTimeFormatSpec(1, timeUnit.toString(), DateTimeFieldSpec.TimeFormat.EPOCH.toString()) .getFormat(), new DateTimeGranularitySpec(1, timeUnit).getGranularity())); break; default: throw new UnsupportedOperationException( "Unsupported field type: " + fieldType + " for field: " + fieldName); } } } return pinotSchema; } private  AvroUtils(); static Schema getPinotSchemaFromAvroSchema(org.apache.avro.Schema avroSchema,
      @Nullable Map<String, FieldSpec.FieldType> fieldTypeMap, @Nullable TimeUnit timeUnit); static Schema getPinotSchemaFromAvroDataFile(File avroDataFile,
      @Nullable Map<String, FieldSpec.FieldType> fieldTypeMap, @Nullable TimeUnit timeUnit); static Schema getPinotSchemaFromAvroDataFile(File avroDataFile); static Schema getPinotSchemaFromAvroSchemaFile(File avroSchemaFile,
      @Nullable Map<String, FieldSpec.FieldType> fieldTypeMap, @Nullable TimeUnit timeUnit); static org.apache.avro.Schema getAvroSchemaFromPinotSchema(Schema pinotSchema); static DataFileStream<GenericRecord> getAvroReader(File avroFile); static boolean isSingleValueField(Field field); static FieldSpec.DataType extractFieldDataType(Field field); static Object convert(Object value); static Object handleMultiValue(Collection values); static Object handleMap(Map map); }
AvroUtils { public static Schema getPinotSchemaFromAvroSchema(org.apache.avro.Schema avroSchema, @Nullable Map<String, FieldSpec.FieldType> fieldTypeMap, @Nullable TimeUnit timeUnit) { Schema pinotSchema = new Schema(); for (Field field : avroSchema.getFields()) { String fieldName = field.name(); FieldSpec.DataType dataType = extractFieldDataType(field); boolean isSingleValueField = isSingleValueField(field); if (fieldTypeMap == null) { pinotSchema.addField(new DimensionFieldSpec(fieldName, dataType, isSingleValueField)); } else { FieldSpec.FieldType fieldType = fieldTypeMap.get(fieldName); Preconditions.checkNotNull(fieldType, "Field type not specified for field: %s", fieldName); switch (fieldType) { case DIMENSION: pinotSchema.addField(new DimensionFieldSpec(fieldName, dataType, isSingleValueField)); break; case METRIC: Preconditions.checkState(isSingleValueField, "Metric field: %s cannot be multi-valued", fieldName); pinotSchema.addField(new MetricFieldSpec(fieldName, dataType)); break; case TIME: Preconditions.checkState(isSingleValueField, "Time field: %s cannot be multi-valued", fieldName); Preconditions.checkNotNull(timeUnit, "Time unit cannot be null"); pinotSchema.addField(new TimeFieldSpec(new TimeGranularitySpec(dataType, timeUnit, field.name()))); break; case DATE_TIME: Preconditions.checkState(isSingleValueField, "Time field: %s cannot be multi-valued", fieldName); Preconditions.checkNotNull(timeUnit, "Time unit cannot be null"); pinotSchema.addField(new DateTimeFieldSpec(field.name(), dataType, new DateTimeFormatSpec(1, timeUnit.toString(), DateTimeFieldSpec.TimeFormat.EPOCH.toString()) .getFormat(), new DateTimeGranularitySpec(1, timeUnit).getGranularity())); break; default: throw new UnsupportedOperationException( "Unsupported field type: " + fieldType + " for field: " + fieldName); } } } return pinotSchema; } private  AvroUtils(); static Schema getPinotSchemaFromAvroSchema(org.apache.avro.Schema avroSchema,
      @Nullable Map<String, FieldSpec.FieldType> fieldTypeMap, @Nullable TimeUnit timeUnit); static Schema getPinotSchemaFromAvroDataFile(File avroDataFile,
      @Nullable Map<String, FieldSpec.FieldType> fieldTypeMap, @Nullable TimeUnit timeUnit); static Schema getPinotSchemaFromAvroDataFile(File avroDataFile); static Schema getPinotSchemaFromAvroSchemaFile(File avroSchemaFile,
      @Nullable Map<String, FieldSpec.FieldType> fieldTypeMap, @Nullable TimeUnit timeUnit); static org.apache.avro.Schema getAvroSchemaFromPinotSchema(Schema pinotSchema); static DataFileStream<GenericRecord> getAvroReader(File avroFile); static boolean isSingleValueField(Field field); static FieldSpec.DataType extractFieldDataType(Field field); static Object convert(Object value); static Object handleMultiValue(Collection values); static Object handleMap(Map map);  }