@Test public void testDense() { Row[] testArray = new Row[]{ Row.of("1.0 2.0 4.0", "a"), Row.of("-1.0 -3.0 4.0", "a"), Row.of("4.0 2.0 3.0", "b"), Row.of("3.4 5.1 5.0", "b") }; String[] colNames = new String[]{"vec", "label"}; MemSourceBatchOp source = new MemSourceBatchOp(Arrays.asList(testArray), colNames); VectorChiSqSelectorBatchOp selector = new VectorChiSqSelectorBatchOp() .setSelectedCol("vec") .setLabelCol("label") .setNumTopFeatures(2); selector.linkFrom(source); selector.lazyPrintModelInfo(); selector.lazyCollectModelInfo( new Consumer<ChisqSelectorModelInfo>() { @Override public void accept(ChisqSelectorModelInfo chisqSelectorSummary) { Assert.assertEquals(chisqSelectorSummary.chisq("0"), 4.0, 10e-10); Assert.assertEquals(chisqSelectorSummary.chisq("1"), 2.0, 10e-10); Assert.assertEquals(chisqSelectorSummary.chisq("2"), 4.0, 10e-10); } } ); }
@Override public VectorChiSqSelectorBatchOp linkFrom(BatchOperator<?>... inputs) { BatchOperator<?> in = checkAndGetFirst(inputs); String vectorColName = getSelectedCol(); String labelColName = getLabelCol(); SelectorType selectorType = getParams().get(SELECTOR_TYPE); int numTopFeatures = getParams().get(NUM_TOP_FEATURES); double percentile = getParams().get(PERCENTILE); double fpr = getParams().get(FPR); double fdr = getParams().get(FDR); double fwe = getParams().get(FWE); DataSet<Row> chiSquareTest = ChiSquareTestUtil.vectorTest(in, vectorColName, labelColName); DataSet<Row> model = chiSquareTest.mapPartition( new ChisqSelectorUtil.ChiSquareSelector(null, selectorType, numTopFeatures, percentile, fpr, fdr, fwe)) .name("FilterFeature") .setParallelism(1); setOutputTable(DataSetConversionUtil.toTable(in.getMLEnvironmentId(), model, new ChiSqSelectorModelDataConverter().getModelSchema())); return this; }
VectorChiSqSelectorBatchOp extends BatchOperator<VectorChiSqSelectorBatchOp> implements VectorChiSqSelectorParams<VectorChiSqSelectorBatchOp>,
    WithModelInfoBatchOp<ChisqSelectorModelInfo, VectorChiSqSelectorBatchOp, ChisqSelectorModelInfoBatchOp> { @Override public VectorChiSqSelectorBatchOp linkFrom(BatchOperator<?>... inputs) { BatchOperator<?> in = checkAndGetFirst(inputs); String vectorColName = getSelectedCol(); String labelColName = getLabelCol(); SelectorType selectorType = getParams().get(SELECTOR_TYPE); int numTopFeatures = getParams().get(NUM_TOP_FEATURES); double percentile = getParams().get(PERCENTILE); double fpr = getParams().get(FPR); double fdr = getParams().get(FDR); double fwe = getParams().get(FWE); DataSet<Row> chiSquareTest = ChiSquareTestUtil.vectorTest(in, vectorColName, labelColName); DataSet<Row> model = chiSquareTest.mapPartition( new ChisqSelectorUtil.ChiSquareSelector(null, selectorType, numTopFeatures, percentile, fpr, fdr, fwe)) .name("FilterFeature") .setParallelism(1); setOutputTable(DataSetConversionUtil.toTable(in.getMLEnvironmentId(), model, new ChiSqSelectorModelDataConverter().getModelSchema())); return this; } }
VectorChiSqSelectorBatchOp extends BatchOperator<VectorChiSqSelectorBatchOp> implements VectorChiSqSelectorParams<VectorChiSqSelectorBatchOp>,
    WithModelInfoBatchOp<ChisqSelectorModelInfo, VectorChiSqSelectorBatchOp, ChisqSelectorModelInfoBatchOp> { @Override public VectorChiSqSelectorBatchOp linkFrom(BatchOperator<?>... inputs) { BatchOperator<?> in = checkAndGetFirst(inputs); String vectorColName = getSelectedCol(); String labelColName = getLabelCol(); SelectorType selectorType = getParams().get(SELECTOR_TYPE); int numTopFeatures = getParams().get(NUM_TOP_FEATURES); double percentile = getParams().get(PERCENTILE); double fpr = getParams().get(FPR); double fdr = getParams().get(FDR); double fwe = getParams().get(FWE); DataSet<Row> chiSquareTest = ChiSquareTestUtil.vectorTest(in, vectorColName, labelColName); DataSet<Row> model = chiSquareTest.mapPartition( new ChisqSelectorUtil.ChiSquareSelector(null, selectorType, numTopFeatures, percentile, fpr, fdr, fwe)) .name("FilterFeature") .setParallelism(1); setOutputTable(DataSetConversionUtil.toTable(in.getMLEnvironmentId(), model, new ChiSqSelectorModelDataConverter().getModelSchema())); return this; } VectorChiSqSelectorBatchOp();  VectorChiSqSelectorBatchOp(Params params); }
VectorChiSqSelectorBatchOp extends BatchOperator<VectorChiSqSelectorBatchOp> implements VectorChiSqSelectorParams<VectorChiSqSelectorBatchOp>,
    WithModelInfoBatchOp<ChisqSelectorModelInfo, VectorChiSqSelectorBatchOp, ChisqSelectorModelInfoBatchOp> { @Override public VectorChiSqSelectorBatchOp linkFrom(BatchOperator<?>... inputs) { BatchOperator<?> in = checkAndGetFirst(inputs); String vectorColName = getSelectedCol(); String labelColName = getLabelCol(); SelectorType selectorType = getParams().get(SELECTOR_TYPE); int numTopFeatures = getParams().get(NUM_TOP_FEATURES); double percentile = getParams().get(PERCENTILE); double fpr = getParams().get(FPR); double fdr = getParams().get(FDR); double fwe = getParams().get(FWE); DataSet<Row> chiSquareTest = ChiSquareTestUtil.vectorTest(in, vectorColName, labelColName); DataSet<Row> model = chiSquareTest.mapPartition( new ChisqSelectorUtil.ChiSquareSelector(null, selectorType, numTopFeatures, percentile, fpr, fdr, fwe)) .name("FilterFeature") .setParallelism(1); setOutputTable(DataSetConversionUtil.toTable(in.getMLEnvironmentId(), model, new ChiSqSelectorModelDataConverter().getModelSchema())); return this; } VectorChiSqSelectorBatchOp();  VectorChiSqSelectorBatchOp(Params params); @Override VectorChiSqSelectorBatchOp linkFrom(BatchOperator<?>... inputs); @Override ChisqSelectorModelInfoBatchOp getModelInfoBatchOp(); }
VectorChiSqSelectorBatchOp extends BatchOperator<VectorChiSqSelectorBatchOp> implements VectorChiSqSelectorParams<VectorChiSqSelectorBatchOp>,
    WithModelInfoBatchOp<ChisqSelectorModelInfo, VectorChiSqSelectorBatchOp, ChisqSelectorModelInfoBatchOp> { @Override public VectorChiSqSelectorBatchOp linkFrom(BatchOperator<?>... inputs) { BatchOperator<?> in = checkAndGetFirst(inputs); String vectorColName = getSelectedCol(); String labelColName = getLabelCol(); SelectorType selectorType = getParams().get(SELECTOR_TYPE); int numTopFeatures = getParams().get(NUM_TOP_FEATURES); double percentile = getParams().get(PERCENTILE); double fpr = getParams().get(FPR); double fdr = getParams().get(FDR); double fwe = getParams().get(FWE); DataSet<Row> chiSquareTest = ChiSquareTestUtil.vectorTest(in, vectorColName, labelColName); DataSet<Row> model = chiSquareTest.mapPartition( new ChisqSelectorUtil.ChiSquareSelector(null, selectorType, numTopFeatures, percentile, fpr, fdr, fwe)) .name("FilterFeature") .setParallelism(1); setOutputTable(DataSetConversionUtil.toTable(in.getMLEnvironmentId(), model, new ChiSqSelectorModelDataConverter().getModelSchema())); return this; } VectorChiSqSelectorBatchOp();  VectorChiSqSelectorBatchOp(Params params); @Override VectorChiSqSelectorBatchOp linkFrom(BatchOperator<?>... inputs); @Override ChisqSelectorModelInfoBatchOp getModelInfoBatchOp();  }