ParseJSONDeriver implements Deriver, ProvidesAlias, ProvidesValidations, InstantiatesComponents { @Override public String getAlias() { return "parse-json"; }  @Override void configure(Config config); @Override Dataset<Row> derive(Map<String, Dataset<Row>> dependencies); @Override String getAlias(); @Override Validations getValidations(); @Override Set<InstantiatedComponent> getComponents(Config config, boolean configure); static final String STEP_NAME_CONFIG; static final String FIELD_NAME_CONFIG; static final String AS_STRUCT_CONFIG; static final String STRUCT_FIELD_NAME_CONFIG; static final String SCHEMA_CONFIG; static final String OPTION_CONFIG_PREFIX; }
@Test public void testDeriveAsFields() { Config config = ConfigFactory.empty() .withValue(ParseJSONDeriver.STEP_NAME_CONFIG, ConfigValueFactory.fromAnyRef("source")) .withValue(ParseJSONDeriver.FIELD_NAME_CONFIG, ConfigValueFactory.fromAnyRef("value")) .withValue(ParseJSONDeriver.SCHEMA_CONFIG + "." + ComponentFactory.TYPE_CONFIG_NAME, ConfigValueFactory.fromAnyRef(new AvroSchema().getAlias())) .withValue(ParseJSONDeriver.SCHEMA_CONFIG + "." + AvroSchema.AVRO_FILE_CONFIG, ConfigValueFactory.fromAnyRef( getClass().getResource("/json/json-schema.avsc").getFile())); Row row = deriveRow(config); assertEquals(6, row.length()); assertEquals("hello", row.getAs("f1")); assertEquals(1.2f, row.getAs("f2")); assertEquals(true, row.getAs("f3")); assertEquals("world", row.getStruct(row.fieldIndex("f4")).getAs("f41")); assertEquals("v5", row.getList(row.fieldIndex("f5")).get(0)); assertEquals(false, ((Row)row.getList(row.fieldIndex("f6")).get(0)).getList(0).get(0)); assertEquals(false, ((Row)row.getList(row.fieldIndex("f6")).get(0)).getList(0).get(1)); }
@Test public void testDeriveAsStruct() { Config config = ConfigFactory.empty() .withValue(ParseJSONDeriver.STEP_NAME_CONFIG, ConfigValueFactory.fromAnyRef("source")) .withValue(ParseJSONDeriver.FIELD_NAME_CONFIG, ConfigValueFactory.fromAnyRef("value")) .withValue(ParseJSONDeriver.AS_STRUCT_CONFIG, ConfigValueFactory.fromAnyRef(true)) .withValue(ParseJSONDeriver.STRUCT_FIELD_NAME_CONFIG, ConfigValueFactory.fromAnyRef("parsed")) .withValue(ParseJSONDeriver.SCHEMA_CONFIG + "." + ComponentFactory.TYPE_CONFIG_NAME, ConfigValueFactory.fromAnyRef(new AvroSchema().getAlias())) .withValue(ParseJSONDeriver.SCHEMA_CONFIG + "." + AvroSchema.AVRO_FILE_CONFIG, ConfigValueFactory.fromAnyRef( getClass().getResource("/json/json-schema.avsc").getFile())); Row row = deriveRow(config); Row struct = row.getStruct(row.fieldIndex("parsed")); assertEquals(1, row.length()); assertEquals(6, struct.length()); assertEquals("hello", struct.getAs("f1")); assertEquals(1.2f, struct.getAs("f2")); assertEquals(true, struct.getAs("f3")); assertEquals("world", struct.getStruct(struct.fieldIndex("f4")).getAs("f41")); assertEquals("v5", struct.getList(struct.fieldIndex("f5")).get(0)); assertEquals(false, ((Row)struct.getList(struct.fieldIndex("f6")).get(0)).getList(0).get(0)); assertEquals(false, ((Row)struct.getList(struct.fieldIndex("f6")).get(0)).getList(0).get(1)); }
@Test (expected = Exception.class) public void testDeriveWithOptions() { Config config = ConfigFactory.empty() .withValue(ParseJSONDeriver.STEP_NAME_CONFIG, ConfigValueFactory.fromAnyRef("source")) .withValue(ParseJSONDeriver.FIELD_NAME_CONFIG, ConfigValueFactory.fromAnyRef("value")) .withValue(ParseJSONDeriver.AS_STRUCT_CONFIG, ConfigValueFactory.fromAnyRef(true)) .withValue(ParseJSONDeriver.STRUCT_FIELD_NAME_CONFIG, ConfigValueFactory.fromAnyRef("parsed")) .withValue(ParseJSONDeriver.SCHEMA_CONFIG + "." + ComponentFactory.TYPE_CONFIG_NAME, ConfigValueFactory.fromAnyRef(new AvroSchema().getAlias())) .withValue(ParseJSONDeriver.SCHEMA_CONFIG + "." + AvroSchema.AVRO_FILE_CONFIG, ConfigValueFactory.fromAnyRef( getClass().getResource("/json/json-schema.avsc").getFile())) .withValue(ParseJSONDeriver.OPTION_CONFIG_PREFIX + "." + "allowSingleQuotes", ConfigValueFactory.fromAnyRef(false)) .withValue(ParseJSONDeriver.OPTION_CONFIG_PREFIX + "." + "mode", ConfigValueFactory.fromAnyRef("FAILFAST")); Row row = deriveRow(config); row.getStruct(row.fieldIndex("parsed")).getAs("f1"); }