@Test public void test() throws Exception { Row[] data = new Row[] { Row.of("prefix1", "{\"prefix1\": 0.9, \"prefix0\": 0.1}"), Row.of("prefix1", "{\"prefix1\": 0.8, \"prefix0\": 0.2}"), Row.of("prefix1", "{\"prefix1\": 0.7, \"prefix0\": 0.3}"), Row.of("prefix0", "{\"prefix1\": 0.75, \"prefix0\": 0.25}"), Row.of("prefix0", "{\"prefix1\": 0.6, \"prefix0\": 0.4}"), Row.of("prefix1", "{\"prefix1\": 0.65, \"prefix0\": 0.35}"), Row.of("prefix0", null), Row.of("prefix1", "{\"prefix1\": 0.55, \"prefix0\": 0.45}"), Row.of("prefix0", "{\"prefix1\": 0.4, \"prefix0\": 0.6}"), Row.of("prefix0", "{\"prefix1\": 0.3, \"prefix0\": 0.7}"), Row.of("prefix1", "{\"prefix1\": 0.35, \"prefix0\": 0.65}"), Row.of("prefix0", "{\"prefix1\": 0.2, \"prefix0\": 0.8}"), Row.of("prefix1", "{\"prefix1\": 0.1, \"prefix0\": 0.9}") }; MemSourceBatchOp source = new MemSourceBatchOp(data, new String[] {"label", "detailInput"}); BinaryClassMetrics metrics = new EvalBinaryClassBatchOp() .setLabelCol("label") .setPredictionDetailCol("detailInput") .linkFrom(source) .collectMetrics(); Assert.assertEquals(0.769, metrics.getPrc(), 0.01); Assert.assertEquals(0.371, metrics.getKs(), 0.01); Assert.assertEquals(0.657, metrics.getAuc(), 0.01); Assert.assertEquals(0.666, metrics.getAccuracy(), 0.01); Assert.assertEquals(0.314, metrics.getMacroKappa(), 0.01); Assert.assertEquals(0.666, metrics.getMicroPrecision(), 0.01); Assert.assertEquals(0.666, metrics.getWeightedRecall(), 0.01); metrics.saveRocCurveAsImage(folder.getRoot().toPath() + "rocCurve.png", true); metrics.saveKSAsImage(folder.getRoot().toPath() + "ks.png", true); metrics.saveLiftChartAsImage(folder.getRoot().toPath() + "liftchart.png", true); metrics.saveRecallPrecisionCurveAsImage(folder.getRoot().toPath() + "recallPrecision.png", true); metrics.saveLorenzCurveAsImage(folder.getRoot().toPath() + "lorenzCurve.png", true); }
@Override public EvalBinaryClassBatchOp linkFrom(BatchOperator<?>... inputs){ BatchOperator<?> in = checkAndGetFirst(inputs); String labelColName = this.get(EvalMultiClassParams.LABEL_COL); TypeInformation labelType = TableUtil.findColTypeWithAssertAndHint(in.getSchema(), labelColName); String positiveValue = this.get(EvalBinaryClassParams.POS_LABEL_VAL_STR); Preconditions.checkArgument(getParams().contains(EvalBinaryClassParams.PREDICTION_DETAIL_COL), "Binary Evaluation must give predictionDetailCol!"); String predDetailColName = this.get(EvalMultiClassParams.PREDICTION_DETAIL_COL); TableUtil.assertSelectedColExist(in.getColNames(), labelColName, predDetailColName); DataSet<Row> data = in.select(new String[] {labelColName, predDetailColName}).getDataSet(); DataSet<BaseMetricsSummary> res = calLabelPredDetailLocal(data, positiveValue, labelType); DataSet<BaseMetricsSummary> metrics = res .reduce(new EvaluationUtil.ReduceBaseMetrics()); this.setOutput(metrics.flatMap(new EvaluationUtil.SaveDataAsParams()), new String[] {"Data"}, new TypeInformation[] {Types.STRING}); return this; }
EvalBinaryClassBatchOp extends BatchOperator<EvalBinaryClassBatchOp> implements
	EvalBinaryClassParams<EvalBinaryClassBatchOp>, EvaluationMetricsCollector<BinaryClassMetrics, EvalBinaryClassBatchOp> { @Override public EvalBinaryClassBatchOp linkFrom(BatchOperator<?>... inputs){ BatchOperator<?> in = checkAndGetFirst(inputs); String labelColName = this.get(EvalMultiClassParams.LABEL_COL); TypeInformation labelType = TableUtil.findColTypeWithAssertAndHint(in.getSchema(), labelColName); String positiveValue = this.get(EvalBinaryClassParams.POS_LABEL_VAL_STR); Preconditions.checkArgument(getParams().contains(EvalBinaryClassParams.PREDICTION_DETAIL_COL), "Binary Evaluation must give predictionDetailCol!"); String predDetailColName = this.get(EvalMultiClassParams.PREDICTION_DETAIL_COL); TableUtil.assertSelectedColExist(in.getColNames(), labelColName, predDetailColName); DataSet<Row> data = in.select(new String[] {labelColName, predDetailColName}).getDataSet(); DataSet<BaseMetricsSummary> res = calLabelPredDetailLocal(data, positiveValue, labelType); DataSet<BaseMetricsSummary> metrics = res .reduce(new EvaluationUtil.ReduceBaseMetrics()); this.setOutput(metrics.flatMap(new EvaluationUtil.SaveDataAsParams()), new String[] {"Data"}, new TypeInformation[] {Types.STRING}); return this; } }
EvalBinaryClassBatchOp extends BatchOperator<EvalBinaryClassBatchOp> implements
	EvalBinaryClassParams<EvalBinaryClassBatchOp>, EvaluationMetricsCollector<BinaryClassMetrics, EvalBinaryClassBatchOp> { @Override public EvalBinaryClassBatchOp linkFrom(BatchOperator<?>... inputs){ BatchOperator<?> in = checkAndGetFirst(inputs); String labelColName = this.get(EvalMultiClassParams.LABEL_COL); TypeInformation labelType = TableUtil.findColTypeWithAssertAndHint(in.getSchema(), labelColName); String positiveValue = this.get(EvalBinaryClassParams.POS_LABEL_VAL_STR); Preconditions.checkArgument(getParams().contains(EvalBinaryClassParams.PREDICTION_DETAIL_COL), "Binary Evaluation must give predictionDetailCol!"); String predDetailColName = this.get(EvalMultiClassParams.PREDICTION_DETAIL_COL); TableUtil.assertSelectedColExist(in.getColNames(), labelColName, predDetailColName); DataSet<Row> data = in.select(new String[] {labelColName, predDetailColName}).getDataSet(); DataSet<BaseMetricsSummary> res = calLabelPredDetailLocal(data, positiveValue, labelType); DataSet<BaseMetricsSummary> metrics = res .reduce(new EvaluationUtil.ReduceBaseMetrics()); this.setOutput(metrics.flatMap(new EvaluationUtil.SaveDataAsParams()), new String[] {"Data"}, new TypeInformation[] {Types.STRING}); return this; } EvalBinaryClassBatchOp();  EvalBinaryClassBatchOp(Params params); }
EvalBinaryClassBatchOp extends BatchOperator<EvalBinaryClassBatchOp> implements
	EvalBinaryClassParams<EvalBinaryClassBatchOp>, EvaluationMetricsCollector<BinaryClassMetrics, EvalBinaryClassBatchOp> { @Override public EvalBinaryClassBatchOp linkFrom(BatchOperator<?>... inputs){ BatchOperator<?> in = checkAndGetFirst(inputs); String labelColName = this.get(EvalMultiClassParams.LABEL_COL); TypeInformation labelType = TableUtil.findColTypeWithAssertAndHint(in.getSchema(), labelColName); String positiveValue = this.get(EvalBinaryClassParams.POS_LABEL_VAL_STR); Preconditions.checkArgument(getParams().contains(EvalBinaryClassParams.PREDICTION_DETAIL_COL), "Binary Evaluation must give predictionDetailCol!"); String predDetailColName = this.get(EvalMultiClassParams.PREDICTION_DETAIL_COL); TableUtil.assertSelectedColExist(in.getColNames(), labelColName, predDetailColName); DataSet<Row> data = in.select(new String[] {labelColName, predDetailColName}).getDataSet(); DataSet<BaseMetricsSummary> res = calLabelPredDetailLocal(data, positiveValue, labelType); DataSet<BaseMetricsSummary> metrics = res .reduce(new EvaluationUtil.ReduceBaseMetrics()); this.setOutput(metrics.flatMap(new EvaluationUtil.SaveDataAsParams()), new String[] {"Data"}, new TypeInformation[] {Types.STRING}); return this; } EvalBinaryClassBatchOp();  EvalBinaryClassBatchOp(Params params); @Override BinaryClassMetrics createMetrics(List<Row> rows); @Override EvalBinaryClassBatchOp linkFrom(BatchOperator<?>... inputs); }
EvalBinaryClassBatchOp extends BatchOperator<EvalBinaryClassBatchOp> implements
	EvalBinaryClassParams<EvalBinaryClassBatchOp>, EvaluationMetricsCollector<BinaryClassMetrics, EvalBinaryClassBatchOp> { @Override public EvalBinaryClassBatchOp linkFrom(BatchOperator<?>... inputs){ BatchOperator<?> in = checkAndGetFirst(inputs); String labelColName = this.get(EvalMultiClassParams.LABEL_COL); TypeInformation labelType = TableUtil.findColTypeWithAssertAndHint(in.getSchema(), labelColName); String positiveValue = this.get(EvalBinaryClassParams.POS_LABEL_VAL_STR); Preconditions.checkArgument(getParams().contains(EvalBinaryClassParams.PREDICTION_DETAIL_COL), "Binary Evaluation must give predictionDetailCol!"); String predDetailColName = this.get(EvalMultiClassParams.PREDICTION_DETAIL_COL); TableUtil.assertSelectedColExist(in.getColNames(), labelColName, predDetailColName); DataSet<Row> data = in.select(new String[] {labelColName, predDetailColName}).getDataSet(); DataSet<BaseMetricsSummary> res = calLabelPredDetailLocal(data, positiveValue, labelType); DataSet<BaseMetricsSummary> metrics = res .reduce(new EvaluationUtil.ReduceBaseMetrics()); this.setOutput(metrics.flatMap(new EvaluationUtil.SaveDataAsParams()), new String[] {"Data"}, new TypeInformation[] {Types.STRING}); return this; } EvalBinaryClassBatchOp();  EvalBinaryClassBatchOp(Params params); @Override BinaryClassMetrics createMetrics(List<Row> rows); @Override EvalBinaryClassBatchOp linkFrom(BatchOperator<?>... inputs);  }