@Test public void testTimestampBasedKeyGenerator() throws IOException { baseRecord.put("createTime", 1578283932000L); properties = getBaseKeyConfig("EPOCHMILLISECONDS", "yyyy-MM-dd hh", "GMT+8:00", null); TimestampBasedKeyGenerator keyGen = new TimestampBasedKeyGenerator(properties); HoodieKey hk1 = keyGen.getKey(baseRecord); assertEquals("2020-01-06 12", hk1.getPartitionPath()); baseRow = genericRecordToRow(baseRecord); assertEquals("2020-01-06 12", keyGen.getPartitionPath(baseRow)); properties = getBaseKeyConfig("EPOCHMILLISECONDS", "yyyy-MM-dd hh", "GMT", null); keyGen = new TimestampBasedKeyGenerator(properties); HoodieKey hk2 = keyGen.getKey(baseRecord); assertEquals("2020-01-06 04", hk2.getPartitionPath()); assertEquals("2020-01-06 04", keyGen.getPartitionPath(baseRow)); baseRecord.put("createTime", "2020-01-06 12:12:12"); properties = getBaseKeyConfig("DATE_STRING", "yyyy-MM-dd hh", "GMT+8:00", null); properties.setProperty("hoodie.deltastreamer.keygen.timebased.input.dateformat", "yyyy-MM-dd hh:mm:ss"); keyGen = new TimestampBasedKeyGenerator(properties); HoodieKey hk3 = keyGen.getKey(baseRecord); assertEquals("2020-01-06 12", hk3.getPartitionPath()); baseRow = genericRecordToRow(baseRecord); assertEquals("2020-01-06 12", keyGen.getPartitionPath(baseRow)); properties = getBaseKeyConfig("DATE_STRING", "yyyy-MM-dd hh", "GMT", null); keyGen = new TimestampBasedKeyGenerator(properties); HoodieKey hk4 = keyGen.getKey(baseRecord); assertEquals("2020-01-06 12", hk4.getPartitionPath()); assertEquals("2020-01-06 12", keyGen.getPartitionPath(baseRow)); baseRecord.put("createTime", null); properties = getBaseKeyConfig("EPOCHMILLISECONDS", "yyyy-MM-dd hh", "GMT+8:00", null); keyGen = new TimestampBasedKeyGenerator(properties); HoodieKey hk5 = keyGen.getKey(baseRecord); assertEquals("1970-01-01 08", hk5.getPartitionPath()); baseRow = genericRecordToRow(baseRecord); assertEquals("1970-01-01 08", keyGen.getPartitionPath(baseRow)); baseRecord.put("createTime", null); properties = getBaseKeyConfig("DATE_STRING", "yyyy-MM-dd hh:mm:ss", "GMT", null); properties.setProperty("hoodie.deltastreamer.keygen.timebased.input.dateformat", "yyyy-MM-dd hh:mm:ss"); keyGen = new TimestampBasedKeyGenerator(properties); HoodieKey hk6 = keyGen.getKey(baseRecord); assertEquals("1970-01-01 12:00:00", hk6.getPartitionPath()); baseRow = genericRecordToRow(baseRecord); assertEquals("1970-01-01 12:00:00", keyGen.getPartitionPath(baseRow)); }
public TimestampBasedKeyGenerator(TypedProperties config) throws IOException { this(config, config.getString(DataSourceWriteOptions.RECORDKEY_FIELD_OPT_KEY()), config.getString(DataSourceWriteOptions.PARTITIONPATH_FIELD_OPT_KEY())); }
TimestampBasedKeyGenerator extends SimpleKeyGenerator { public TimestampBasedKeyGenerator(TypedProperties config) throws IOException { this(config, config.getString(DataSourceWriteOptions.RECORDKEY_FIELD_OPT_KEY()), config.getString(DataSourceWriteOptions.PARTITIONPATH_FIELD_OPT_KEY())); } }
TimestampBasedKeyGenerator extends SimpleKeyGenerator { public TimestampBasedKeyGenerator(TypedProperties config) throws IOException { this(config, config.getString(DataSourceWriteOptions.RECORDKEY_FIELD_OPT_KEY()), config.getString(DataSourceWriteOptions.PARTITIONPATH_FIELD_OPT_KEY())); } TimestampBasedKeyGenerator(TypedProperties config);   TimestampBasedKeyGenerator(TypedProperties config, String partitionPathField);   TimestampBasedKeyGenerator(TypedProperties config, String recordKeyField, String partitionPathField); }
TimestampBasedKeyGenerator extends SimpleKeyGenerator { public TimestampBasedKeyGenerator(TypedProperties config) throws IOException { this(config, config.getString(DataSourceWriteOptions.RECORDKEY_FIELD_OPT_KEY()), config.getString(DataSourceWriteOptions.PARTITIONPATH_FIELD_OPT_KEY())); } TimestampBasedKeyGenerator(TypedProperties config);   TimestampBasedKeyGenerator(TypedProperties config, String partitionPathField);   TimestampBasedKeyGenerator(TypedProperties config, String recordKeyField, String partitionPathField); @Override String getPartitionPath(GenericRecord record); @Override String getRecordKey(Row row); @Override String getPartitionPath(Row row); }
TimestampBasedKeyGenerator extends SimpleKeyGenerator { public TimestampBasedKeyGenerator(TypedProperties config) throws IOException { this(config, config.getString(DataSourceWriteOptions.RECORDKEY_FIELD_OPT_KEY()), config.getString(DataSourceWriteOptions.PARTITIONPATH_FIELD_OPT_KEY())); } TimestampBasedKeyGenerator(TypedProperties config);   TimestampBasedKeyGenerator(TypedProperties config, String partitionPathField);   TimestampBasedKeyGenerator(TypedProperties config, String recordKeyField, String partitionPathField); @Override String getPartitionPath(GenericRecord record); @Override String getRecordKey(Row row); @Override String getPartitionPath(Row row);  }