DistinctDeriver implements Deriver, ProvidesAlias, ProvidesValidations { @Override public Dataset<Row> derive(Map<String, Dataset<Row>> dependencies) throws Exception { validate(dependencies); return dependencies.get(stepName).distinct(); }  @Override void configure(Config config); @Override Dataset<Row> derive(Map<String, Dataset<Row>> dependencies); @Override String getAlias(); @Override Validations getValidations(); static final String DISTINCT_DERIVER_ALIAS; static final String DISTINCT_STEP_CONFIG; }
@Test public void derive() throws Exception { Dataset<Row> source = createTestDataframe(); Map<String, Dataset<Row>> dependencies = Maps.newHashMap(); dependencies.put("df1", source); DistinctDeriver deriver = new DistinctDeriver(); Config config = ConfigFactory.empty(); assertNoValidationFailures(deriver, config); deriver.configure(config); List<Row> results = deriver.derive(dependencies).collectAsList(); assertEquals(results.size(), 11); assertTrue(results.containsAll(createTestData())); config = ConfigFactory.empty().withValue(DistinctDeriver.DISTINCT_STEP_CONFIG, ConfigValueFactory.fromAnyRef("df1")); deriver.configure(config); results = deriver.derive(dependencies).collectAsList(); assertEquals(results.size(), 11); assertTrue(results.containsAll(createTestData())); dependencies.put("df2", null); dependencies.put("df3", null); results = deriver.derive(dependencies).collectAsList(); assertEquals(results.size(), 11); assertTrue(results.containsAll(createTestData())); }