StreamingStep extends DataStep implements CanRecordProgress, ProvidesValidations, InstantiatesComponents { @Override public void configure(Config config) { super.configure(config); } StreamingStep(String name); @Override void configure(Config config); @SuppressWarnings("rawtypes") JavaDStream<?> getStream(); @Override void recordProgress(JavaRDD<?> batch); BatchStep createErrorStep(String stepName, Dataset<Row> data); @SuppressWarnings({"unchecked","rawtypes"}) Dataset<Row> translate(JavaRDD raw); @Override Step copy(); @Override Validations getValidations(); @Override Set<InstantiatedComponent> getComponents(Config config, boolean configure); static final String TRANSLATOR_WITHIN_INPUT_PROPERTY; static final String TRANSLATOR_PROPERTY; static final String REPARTITION_NUM_PARTITIONS_PROPERTY; }
@Test public void testConfigure() { Config config = ConfigFactory.empty() .withValue(StreamingStep.INPUT_TYPE, ConfigValueFactory.fromMap(new HashMap<String, Object>())) .withValue("input.type", ConfigValueFactory.fromAnyRef("com.cloudera.labs.envelope.run.TestStreamingStep$PrepartitionedStreamInput")) .withValue("input.translator", ConfigValueFactory.fromMap(new HashMap<String, Object>()) .withValue("type", ConfigValueFactory.fromAnyRef("com.cloudera.labs.envelope.translate.DummyTranslator"))); assertNoValidationFailures(new StreamingStep("ingest"), config); }