@Test public void testFieldsPickWithNestedRecord() throws Exception { Schema inputSchema = new Schema.Parser().parse(getClass().getResourceAsStream("/converter/pickfields_nested_with_union.avsc")); WorkUnitState workUnitState = new WorkUnitState(); workUnitState.setProp(ConfigurationKeys.CONVERTER_AVRO_FIELD_PICK_FIELDS, "name,favorite_number,nested1.nested1_string,nested1.nested2_union.nested2_string"); try (AvroFieldsPickConverter converter = new AvroFieldsPickConverter()) { Schema convertedSchema = converter.convertSchema(inputSchema, workUnitState); Schema expectedSchema = new Schema.Parser().parse(getClass().getResourceAsStream("/converter/converted_pickfields_nested_with_union.avsc")); JSONAssert.assertEquals(expectedSchema.toString(), convertedSchema.toString(), false); try (DataFileReader<GenericRecord> srcDataFileReader = new DataFileReader<GenericRecord>( new File(getClass().getResource("/converter/pickfields_nested_with_union.avro").toURI()), new GenericDatumReader<GenericRecord>(inputSchema)); DataFileReader<GenericRecord> expectedDataFileReader = new DataFileReader<GenericRecord>( new File(getClass().getResource("/converter/converted_pickfields_nested_with_union.avro").toURI()), new GenericDatumReader<GenericRecord>(expectedSchema));) { while (expectedDataFileReader.hasNext()) { GenericRecord expected = expectedDataFileReader.next(); GenericRecord actual = converter.convertRecord(convertedSchema, srcDataFileReader.next(), workUnitState).iterator().next(); Assert.assertEquals(actual, expected); } Assert.assertTrue(!srcDataFileReader.hasNext()); } } }
@Override public Schema convertSchema(Schema inputSchema, WorkUnitState workUnit) throws SchemaConversionException { LOG.info("Converting schema " + inputSchema); String fieldsStr = workUnit.getProp(ConfigurationKeys.CONVERTER_AVRO_FIELD_PICK_FIELDS); Preconditions.checkNotNull(fieldsStr, ConfigurationKeys.CONVERTER_AVRO_FIELD_PICK_FIELDS + " is required for converter " + this.getClass().getSimpleName()); LOG.info("Converting schema to selected fields: " + fieldsStr); try { return createSchema(inputSchema, fieldsStr); } catch (Exception e) { throw new SchemaConversionException(e); } }
AvroFieldsPickConverter extends AvroToAvroConverterBase { @Override public Schema convertSchema(Schema inputSchema, WorkUnitState workUnit) throws SchemaConversionException { LOG.info("Converting schema " + inputSchema); String fieldsStr = workUnit.getProp(ConfigurationKeys.CONVERTER_AVRO_FIELD_PICK_FIELDS); Preconditions.checkNotNull(fieldsStr, ConfigurationKeys.CONVERTER_AVRO_FIELD_PICK_FIELDS + " is required for converter " + this.getClass().getSimpleName()); LOG.info("Converting schema to selected fields: " + fieldsStr); try { return createSchema(inputSchema, fieldsStr); } catch (Exception e) { throw new SchemaConversionException(e); } } }
AvroFieldsPickConverter extends AvroToAvroConverterBase { @Override public Schema convertSchema(Schema inputSchema, WorkUnitState workUnit) throws SchemaConversionException { LOG.info("Converting schema " + inputSchema); String fieldsStr = workUnit.getProp(ConfigurationKeys.CONVERTER_AVRO_FIELD_PICK_FIELDS); Preconditions.checkNotNull(fieldsStr, ConfigurationKeys.CONVERTER_AVRO_FIELD_PICK_FIELDS + " is required for converter " + this.getClass().getSimpleName()); LOG.info("Converting schema to selected fields: " + fieldsStr); try { return createSchema(inputSchema, fieldsStr); } catch (Exception e) { throw new SchemaConversionException(e); } }  }
AvroFieldsPickConverter extends AvroToAvroConverterBase { @Override public Schema convertSchema(Schema inputSchema, WorkUnitState workUnit) throws SchemaConversionException { LOG.info("Converting schema " + inputSchema); String fieldsStr = workUnit.getProp(ConfigurationKeys.CONVERTER_AVRO_FIELD_PICK_FIELDS); Preconditions.checkNotNull(fieldsStr, ConfigurationKeys.CONVERTER_AVRO_FIELD_PICK_FIELDS + " is required for converter " + this.getClass().getSimpleName()); LOG.info("Converting schema to selected fields: " + fieldsStr); try { return createSchema(inputSchema, fieldsStr); } catch (Exception e) { throw new SchemaConversionException(e); } }  @Override Schema convertSchema(Schema inputSchema, WorkUnitState workUnit); @Override Iterable<GenericRecord> convertRecordImpl(Schema outputSchema, GenericRecord inputRecord, WorkUnitState workUnit); }
AvroFieldsPickConverter extends AvroToAvroConverterBase { @Override public Schema convertSchema(Schema inputSchema, WorkUnitState workUnit) throws SchemaConversionException { LOG.info("Converting schema " + inputSchema); String fieldsStr = workUnit.getProp(ConfigurationKeys.CONVERTER_AVRO_FIELD_PICK_FIELDS); Preconditions.checkNotNull(fieldsStr, ConfigurationKeys.CONVERTER_AVRO_FIELD_PICK_FIELDS + " is required for converter " + this.getClass().getSimpleName()); LOG.info("Converting schema to selected fields: " + fieldsStr); try { return createSchema(inputSchema, fieldsStr); } catch (Exception e) { throw new SchemaConversionException(e); } }  @Override Schema convertSchema(Schema inputSchema, WorkUnitState workUnit); @Override Iterable<GenericRecord> convertRecordImpl(Schema outputSchema, GenericRecord inputRecord, WorkUnitState workUnit);  }