@Test public void testExtractedOutputForIndependentTransformers() { Transformer t1 = createTransformer(Arrays.asList("a", "b", "c"), Arrays.asList("a1", "b1", "c")); Transformer t2 = createTransformer(Arrays.asList("d", "e", "f"), Arrays.asList("d1", "e1", "f1")); Set<String> outputs = PipelineUtils.extractRequiredOutputColumns(new Transformer[] {t1, t2}); assertEquals(outputs.size(), 6); assertTrue(outputs.containsAll(Arrays.asList("a1","b1","c","d1","e1","f1"))); assertFalse(outputs.contains("a")); assertFalse(outputs.contains("b")); assertFalse(outputs.contains("d")); assertFalse(outputs.contains("e")); assertFalse(outputs.contains("f")); }
public static Set<String> extractRequiredOutputColumns(Transformer[] transformers) { Set<String> outputColumns = new HashSet<>(); for(int i = transformers.length-1; i>=0; i--) { outputColumns.addAll(transformers[i].getOutputKeys()); } for(int i = transformers.length-1; i>=0; i--) { Set<String> setDifference = new HashSet<>(transformers[i].getInputKeys()); setDifference.removeAll(transformers[i].getOutputKeys()); outputColumns.removeAll(setDifference); } return outputColumns; }
PipelineUtils implements Serializable { public static Set<String> extractRequiredOutputColumns(Transformer[] transformers) { Set<String> outputColumns = new HashSet<>(); for(int i = transformers.length-1; i>=0; i--) { outputColumns.addAll(transformers[i].getOutputKeys()); } for(int i = transformers.length-1; i>=0; i--) { Set<String> setDifference = new HashSet<>(transformers[i].getInputKeys()); setDifference.removeAll(transformers[i].getOutputKeys()); outputColumns.removeAll(setDifference); } return outputColumns; } }
PipelineUtils implements Serializable { public static Set<String> extractRequiredOutputColumns(Transformer[] transformers) { Set<String> outputColumns = new HashSet<>(); for(int i = transformers.length-1; i>=0; i--) { outputColumns.addAll(transformers[i].getOutputKeys()); } for(int i = transformers.length-1; i>=0; i--) { Set<String> setDifference = new HashSet<>(transformers[i].getInputKeys()); setDifference.removeAll(transformers[i].getOutputKeys()); outputColumns.removeAll(setDifference); } return outputColumns; }  }
PipelineUtils implements Serializable { public static Set<String> extractRequiredOutputColumns(Transformer[] transformers) { Set<String> outputColumns = new HashSet<>(); for(int i = transformers.length-1; i>=0; i--) { outputColumns.addAll(transformers[i].getOutputKeys()); } for(int i = transformers.length-1; i>=0; i--) { Set<String> setDifference = new HashSet<>(transformers[i].getInputKeys()); setDifference.removeAll(transformers[i].getOutputKeys()); outputColumns.removeAll(setDifference); } return outputColumns; }  static Set<String> extractRequiredInputColumns(Transformer[] transformers); static Set<String> extractRequiredOutputColumns(Transformer[] transformers); }
PipelineUtils implements Serializable { public static Set<String> extractRequiredOutputColumns(Transformer[] transformers) { Set<String> outputColumns = new HashSet<>(); for(int i = transformers.length-1; i>=0; i--) { outputColumns.addAll(transformers[i].getOutputKeys()); } for(int i = transformers.length-1; i>=0; i--) { Set<String> setDifference = new HashSet<>(transformers[i].getInputKeys()); setDifference.removeAll(transformers[i].getOutputKeys()); outputColumns.removeAll(setDifference); } return outputColumns; }  static Set<String> extractRequiredInputColumns(Transformer[] transformers); static Set<String> extractRequiredOutputColumns(Transformer[] transformers);  }