DropFields { public static <T> Inner<T> fields(String... fields) { return fields(FieldAccessDescriptor.withFieldNames(fields)); }  static Inner<T> fields(String... fields); static Inner<T> fields(Integer... fieldIds); static Inner<T> fields(FieldAccessDescriptor fieldsToDrop);  }
@Test @Category(NeedsRunner.class) public void testDropTopLevelField() { Schema expectedSchema = Schema.builder().addStringField("field2").build(); PCollection<Row> result = pipeline .apply( Create.of(simpleRow(1, "one"), simpleRow(2, "two"), simpleRow(3, "three")) .withRowSchema(SIMPLE_SCHEMA)) .apply(DropFields.fields("field1")); assertEquals(expectedSchema, result.getSchema()); List<Row> expectedRows = Lists.newArrayList( Row.withSchema(expectedSchema).addValue("one").build(), Row.withSchema(expectedSchema).addValue("two").build(), Row.withSchema(expectedSchema).addValue("three").build()); PAssert.that(result).containsInAnyOrder(expectedRows); pipeline.run(); }
@Test @Category(NeedsRunner.class) public void testDropNestedField() { Schema expectedSchema = Schema.builder().addStringField("string").addStringField("field2").build(); PCollection<Row> result = pipeline .apply( Create.of( nestedRow(simpleRow(1, "one")), nestedRow(simpleRow(2, "two")), nestedRow(simpleRow(3, "three"))) .withRowSchema(NESTED_SCHEMA)) .apply(DropFields.fields("nested.field1")); assertEquals(expectedSchema, result.getSchema()); List<Row> expectedRows = Lists.newArrayList( Row.withSchema(expectedSchema).addValues("foo", "one").build(), Row.withSchema(expectedSchema).addValues("foo", "two").build(), Row.withSchema(expectedSchema).addValues("foo", "three").build()); PAssert.that(result).containsInAnyOrder(expectedRows); pipeline.run(); }
@Test @Category(NeedsRunner.class) public void testDropNestedFieldKeepingOnlyNested() { Schema expectedSchema = Schema.builder().addStringField("field2").build(); PCollection<Row> result = pipeline .apply( Create.of( nestedRow(simpleRow(1, "one")), nestedRow(simpleRow(2, "two")), nestedRow(simpleRow(3, "three"))) .withRowSchema(NESTED_SCHEMA)) .apply(DropFields.fields("string", "nested.field1")); assertEquals(expectedSchema, result.getSchema()); List<Row> expectedRows = Lists.newArrayList( Row.withSchema(expectedSchema).addValue("one").build(), Row.withSchema(expectedSchema).addValue("two").build(), Row.withSchema(expectedSchema).addValue("three").build()); PAssert.that(result).containsInAnyOrder(expectedRows); pipeline.run(); }
@Test @Category(NeedsRunner.class) public void testDropNestedArrayField() { Schema expectedSchema = Schema.builder().addArrayField("field2", FieldType.STRING).build(); PCollection<Row> result = pipeline .apply( Create.of( nestedArray(simpleRow(1, "one1"), simpleRow(1, "one2")), nestedArray(simpleRow(2, "two1"), simpleRow(2, "two2")), nestedArray(simpleRow(3, "three1"), simpleRow(3, "three2"))) .withRowSchema(NESTED_ARRAY_SCHEMA)) .apply(DropFields.fields("array[].field1")); assertEquals(expectedSchema, result.getSchema()); List<Row> expectedRows = Lists.newArrayList( Row.withSchema(expectedSchema).addArray("one1", "one2").build(), Row.withSchema(expectedSchema).addArray("two1", "two2").build(), Row.withSchema(expectedSchema).addArray("three1", "three2").build()); PAssert.that(result).containsInAnyOrder(expectedRows); pipeline.run(); }
@Test @Category(NeedsRunner.class) public void testMaintainsOriginalSchemaOrder() { Schema expectedSchema = Schema.builder() .addFields(intFieldsRange(1, 10)) .addFields(intFieldsRange(11, 19)) .addFields(intFieldsRange(21, 55)) .addFields(intFieldsRange(56, 100)) .build(); PCollection<Row> result = pipeline .apply( Create.of( multipleIntRow(MULTIPLE_INT_SCHEMA, 1), multipleIntRow(MULTIPLE_INT_SCHEMA, 2)) .withRowSchema(MULTIPLE_INT_SCHEMA)) .apply(DropFields.fields("field0", "field10", "field19", "field20", "field55")); assertEquals(expectedSchema, result.getSchema()); List<Row> expectedRows = Lists.newArrayList(multipleIntRow(expectedSchema, 1), multipleIntRow(expectedSchema, 2)); PAssert.that(result).containsInAnyOrder(expectedRows); pipeline.run(); }