@Test public void testExtractFieldsSchema() { Schema schema; schema = new Schema(); DimensionFieldSpec dimensionFieldSpec = new DimensionFieldSpec("d1", FieldSpec.DataType.STRING, true); dimensionFieldSpec.setTransformFunction("Groovy({function}, argument1, argument2)"); schema.addField(dimensionFieldSpec); List<String> extract = new ArrayList<>(IngestionUtils.getFieldsForRecordExtractor(null, schema)); Assert.assertEquals(extract.size(), 3); Assert.assertTrue(extract.containsAll(Arrays.asList("d1", "argument1", "argument2"))); schema = new Schema(); dimensionFieldSpec = new DimensionFieldSpec("d1", FieldSpec.DataType.STRING, true); dimensionFieldSpec.setTransformFunction("Groovy({function})"); schema.addField(dimensionFieldSpec); extract = new ArrayList<>(IngestionUtils.getFieldsForRecordExtractor(null, schema)); Assert.assertEquals(extract.size(), 1); Assert.assertTrue(extract.contains("d1")); schema = new Schema(); dimensionFieldSpec = new DimensionFieldSpec("map__KEYS", FieldSpec.DataType.INT, false); schema.addField(dimensionFieldSpec); extract = new ArrayList<>(IngestionUtils.getFieldsForRecordExtractor(null, schema)); Assert.assertEquals(extract.size(), 2); Assert.assertTrue(extract.containsAll(Arrays.asList("map", "map__KEYS"))); schema = new Schema(); dimensionFieldSpec = new DimensionFieldSpec("map__VALUES", FieldSpec.DataType.LONG, false); schema.addField(dimensionFieldSpec); extract = new ArrayList<>(IngestionUtils.getFieldsForRecordExtractor(null, schema)); Assert.assertEquals(extract.size(), 2); Assert.assertTrue(extract.containsAll(Arrays.asList("map", "map__VALUES"))); schema = new Schema.SchemaBuilder() .addTime(new TimeGranularitySpec(FieldSpec.DataType.LONG, TimeUnit.MILLISECONDS, "time"), null).build(); extract = new ArrayList<>(IngestionUtils.getFieldsForRecordExtractor(null, schema)); Assert.assertEquals(extract.size(), 1); Assert.assertTrue(extract.contains("time")); schema = new Schema.SchemaBuilder() .addTime(new TimeGranularitySpec(FieldSpec.DataType.LONG, TimeUnit.MILLISECONDS, "in"), new TimeGranularitySpec(FieldSpec.DataType.LONG, TimeUnit.MILLISECONDS, "out")).build(); extract = new ArrayList<>(IngestionUtils.getFieldsForRecordExtractor(null, schema)); Assert.assertEquals(extract.size(), 2); Assert.assertTrue(extract.containsAll(Arrays.asList("in", "out"))); schema = new Schema(); dimensionFieldSpec = new DimensionFieldSpec("hoursSinceEpoch", FieldSpec.DataType.LONG, true); dimensionFieldSpec.setTransformFunction("toEpochHours(timestamp)"); schema.addField(dimensionFieldSpec); extract = new ArrayList<>(IngestionUtils.getFieldsForRecordExtractor(null, schema)); Assert.assertEquals(extract.size(), 2); Assert.assertTrue(extract.containsAll(Arrays.asList("timestamp", "hoursSinceEpoch"))); schema = new Schema(); dimensionFieldSpec = new DimensionFieldSpec("tenMinutesSinceEpoch", FieldSpec.DataType.LONG, true); dimensionFieldSpec.setTransformFunction("toEpochMinutesBucket(timestamp, 10)"); schema.addField(dimensionFieldSpec); extract = new ArrayList<>(IngestionUtils.getFieldsForRecordExtractor(null, schema)); Assert.assertEquals(extract.size(), 2); Assert.assertTrue(extract.containsAll(Lists.newArrayList("tenMinutesSinceEpoch", "timestamp"))); schema = new Schema(); DateTimeFieldSpec dateTimeFieldSpec = new DateTimeFieldSpec("date", FieldSpec.DataType.STRING, "1:DAYS:SIMPLE_DATE_FORMAT:yyyy-MM-dd", "1:DAYS"); dateTimeFieldSpec.setTransformFunction("toDateTime(timestamp, 'yyyy-MM-dd')"); schema.addField(dateTimeFieldSpec); extract = new ArrayList<>(IngestionUtils.getFieldsForRecordExtractor(null, schema)); Assert.assertEquals(extract.size(), 2); Assert.assertTrue(extract.containsAll(Lists.newArrayList("date", "timestamp"))); }
public static Set<String> getFieldsForRecordExtractor(@Nullable IngestionConfig ingestionConfig, Schema schema) { Set<String> fieldsForRecordExtractor = new HashSet<>(); extractFieldsFromIngestionConfig(ingestionConfig, fieldsForRecordExtractor); extractFieldsFromSchema(schema, fieldsForRecordExtractor); return fieldsForRecordExtractor; }
IngestionUtils { public static Set<String> getFieldsForRecordExtractor(@Nullable IngestionConfig ingestionConfig, Schema schema) { Set<String> fieldsForRecordExtractor = new HashSet<>(); extractFieldsFromIngestionConfig(ingestionConfig, fieldsForRecordExtractor); extractFieldsFromSchema(schema, fieldsForRecordExtractor); return fieldsForRecordExtractor; } }
IngestionUtils { public static Set<String> getFieldsForRecordExtractor(@Nullable IngestionConfig ingestionConfig, Schema schema) { Set<String> fieldsForRecordExtractor = new HashSet<>(); extractFieldsFromIngestionConfig(ingestionConfig, fieldsForRecordExtractor); extractFieldsFromSchema(schema, fieldsForRecordExtractor); return fieldsForRecordExtractor; }  }
IngestionUtils { public static Set<String> getFieldsForRecordExtractor(@Nullable IngestionConfig ingestionConfig, Schema schema) { Set<String> fieldsForRecordExtractor = new HashSet<>(); extractFieldsFromIngestionConfig(ingestionConfig, fieldsForRecordExtractor); extractFieldsFromSchema(schema, fieldsForRecordExtractor); return fieldsForRecordExtractor; }  static Set<String> getFieldsForRecordExtractor(@Nullable IngestionConfig ingestionConfig, Schema schema); static boolean shouldIngestRow(GenericRow genericRow); }
IngestionUtils { public static Set<String> getFieldsForRecordExtractor(@Nullable IngestionConfig ingestionConfig, Schema schema) { Set<String> fieldsForRecordExtractor = new HashSet<>(); extractFieldsFromIngestionConfig(ingestionConfig, fieldsForRecordExtractor); extractFieldsFromSchema(schema, fieldsForRecordExtractor); return fieldsForRecordExtractor; }  static Set<String> getFieldsForRecordExtractor(@Nullable IngestionConfig ingestionConfig, Schema schema); static boolean shouldIngestRow(GenericRow genericRow);  }