@Test public void updateAccurateBinaryMetricsSummaryTest() { long[] countValues = new long[] {0, 0, 3, 2}; Row[] rows = new Row[] { Row.of("prefix1", "{\"prefix1\": 0.9, \"prefix0\": 0.1}"), Row.of("prefix1", "{\"prefix1\": 0.8, \"prefix0\": 0.2}"), Row.of("prefix1", "{\"prefix1\": 0.7, \"prefix0\": 0.3}"), Row.of("prefix0", "{\"prefix1\": 0.75, \"prefix0\": 0.25}"), Row.of("prefix0", "{\"prefix1\": 0.6, \"prefix0\": 0.4}") }; Object[] labels = new Object[] {"prefix1", "prefix0"}; TypeInformation labelType = Types.STRING; AccurateBinaryMetricsSummary summary = new AccurateBinaryMetricsSummary(labels, 0.0, 0L, 0.); double[] tprFprPrecision = new double[ClassificationEvaluationUtil.RECORD_LEN]; List<Tuple3<Double, Boolean, Double>> list = new ArrayList<>(); for (Row row : rows) { list.add(ClassificationEvaluationUtil.getBinaryDetailStatistics(row, labels, labelType)); } Collections.sort(list, Comparator.comparingDouble(t -> -t.f0)); list.forEach(t -> ClassificationEvaluationUtil.updateAccurateBinaryMetricsSummary(t, summary, countValues, tprFprPrecision, true)); BinaryClassMetrics summary1 = (BinaryClassMetrics)getDetailStatistics(Arrays.asList(rows), null, true, Types.STRING).toMetrics(); BinaryClassMetrics summary2 = summary.toMetrics(); Assert.assertEquals(summary1.getTotalSamples(), summary2.getTotalSamples()); Assert.assertEquals(summary1.getKs(), summary2.getKs()); Assert.assertEquals(summary1.getPrc(), summary2.getPrc()); Assert.assertEquals(summary1.getGini(), summary2.getGini()); }
public static void updateAccurateBinaryMetricsSummary(Tuple3<Double, Boolean, Double> cur, AccurateBinaryMetricsSummary binaryMetricsSummary, long[] countValues, double[] recordValues, boolean first) { if(binaryMetricsSummary.total == 0) { recordValues[TPR] = countValues[TOTAL_TRUE] == 0 ? 1.0 : 1.0 * countValues[CUR_TRUE] / countValues[TOTAL_TRUE]; recordValues[FPR] = countValues[TOTAL_FALSE] == 0 ? 1.0 : 1.0 * countValues[CUR_FALSE] / countValues[TOTAL_FALSE]; recordValues[PRECISION] = countValues[CUR_TRUE] + countValues[CUR_FALSE] == 0 ? 1.0 : 1.0 * countValues[CUR_TRUE] / (countValues[CUR_TRUE] + countValues[CUR_FALSE]); recordValues[POSITIVE_RATE] = 1.0 * (countValues[CUR_TRUE] + countValues[CUR_FALSE]) / (countValues[TOTAL_TRUE] + countValues[TOTAL_FALSE]); } if(!isMiddlePoint(cur)) { binaryMetricsSummary.total++; binaryMetricsSummary.logLoss += cur.f2; if (cur.f1) { countValues[CUR_TRUE]++; } else { countValues[CUR_FALSE]++; } } double threshold = cur.f0; double tpr = countValues[TOTAL_TRUE] == 0 ? 1.0 : 1.0 * countValues[CUR_TRUE] / countValues[TOTAL_TRUE]; double fpr = countValues[TOTAL_FALSE] == 0 ? 1.0 : 1.0 * countValues[CUR_FALSE] / countValues[TOTAL_FALSE]; double precision = countValues[CUR_TRUE] + countValues[CUR_FALSE] == 0 ? 1.0 : 1.0 * countValues[CUR_TRUE] / (countValues[CUR_TRUE] + countValues[CUR_FALSE]); double positiveRate = 1.0 * (countValues[CUR_TRUE] + countValues[CUR_FALSE]) / (countValues[TOTAL_TRUE] + countValues[TOTAL_FALSE]); if(binaryMetricsSummary.total == 1 && first){ recordValues[PRECISION] = precision; ConfusionMatrix confusionMatrix = new ConfusionMatrix( new long[][] {{0, 0}, {countValues[TOTAL_TRUE], countValues[TOTAL_FALSE]}}); binaryMetricsSummary.metricsInfoList.add(Tuple2.of(1.0, confusionMatrix)); } binaryMetricsSummary.gini += ((positiveRate - recordValues[POSITIVE_RATE]) * (tpr + recordValues[TPR]) / 2); binaryMetricsSummary.prc += ((tpr - recordValues[TPR]) * (precision + recordValues[PRECISION]) / 2); binaryMetricsSummary.ks = Math.max(Math.abs(fpr - tpr), binaryMetricsSummary.ks); recordValues[TPR] = tpr; recordValues[FPR] = fpr; recordValues[PRECISION] = precision; recordValues[POSITIVE_RATE] = positiveRate; ConfusionMatrix confusionMatrix = new ConfusionMatrix(new long[][] {{countValues[CUR_TRUE], countValues[CUR_FALSE]}, {countValues[TOTAL_TRUE] - countValues[CUR_TRUE], countValues[TOTAL_FALSE] - countValues[CUR_FALSE]}}); if(binaryMetricsSummary.metricsInfoList.isEmpty() || (binaryMetricsSummary.total == 1 && first) || isMiddlePoint(cur) || Math.abs(threshold - binaryMetricsSummary.metricsInfoList.get(binaryMetricsSummary.metricsInfoList.size() - 1).f0) >= PROBABILITY_ERROR) { binaryMetricsSummary.metricsInfoList.add(Tuple2.of(threshold, confusionMatrix)); }else{ Tuple2<Double, ConfusionMatrix> metricsInfo = binaryMetricsSummary.metricsInfoList .get(binaryMetricsSummary.metricsInfoList.size() - 1); metricsInfo.f1 = confusionMatrix; } }
ClassificationEvaluationUtil implements Serializable { public static void updateAccurateBinaryMetricsSummary(Tuple3<Double, Boolean, Double> cur, AccurateBinaryMetricsSummary binaryMetricsSummary, long[] countValues, double[] recordValues, boolean first) { if(binaryMetricsSummary.total == 0) { recordValues[TPR] = countValues[TOTAL_TRUE] == 0 ? 1.0 : 1.0 * countValues[CUR_TRUE] / countValues[TOTAL_TRUE]; recordValues[FPR] = countValues[TOTAL_FALSE] == 0 ? 1.0 : 1.0 * countValues[CUR_FALSE] / countValues[TOTAL_FALSE]; recordValues[PRECISION] = countValues[CUR_TRUE] + countValues[CUR_FALSE] == 0 ? 1.0 : 1.0 * countValues[CUR_TRUE] / (countValues[CUR_TRUE] + countValues[CUR_FALSE]); recordValues[POSITIVE_RATE] = 1.0 * (countValues[CUR_TRUE] + countValues[CUR_FALSE]) / (countValues[TOTAL_TRUE] + countValues[TOTAL_FALSE]); } if(!isMiddlePoint(cur)) { binaryMetricsSummary.total++; binaryMetricsSummary.logLoss += cur.f2; if (cur.f1) { countValues[CUR_TRUE]++; } else { countValues[CUR_FALSE]++; } } double threshold = cur.f0; double tpr = countValues[TOTAL_TRUE] == 0 ? 1.0 : 1.0 * countValues[CUR_TRUE] / countValues[TOTAL_TRUE]; double fpr = countValues[TOTAL_FALSE] == 0 ? 1.0 : 1.0 * countValues[CUR_FALSE] / countValues[TOTAL_FALSE]; double precision = countValues[CUR_TRUE] + countValues[CUR_FALSE] == 0 ? 1.0 : 1.0 * countValues[CUR_TRUE] / (countValues[CUR_TRUE] + countValues[CUR_FALSE]); double positiveRate = 1.0 * (countValues[CUR_TRUE] + countValues[CUR_FALSE]) / (countValues[TOTAL_TRUE] + countValues[TOTAL_FALSE]); if(binaryMetricsSummary.total == 1 && first){ recordValues[PRECISION] = precision; ConfusionMatrix confusionMatrix = new ConfusionMatrix( new long[][] {{0, 0}, {countValues[TOTAL_TRUE], countValues[TOTAL_FALSE]}}); binaryMetricsSummary.metricsInfoList.add(Tuple2.of(1.0, confusionMatrix)); } binaryMetricsSummary.gini += ((positiveRate - recordValues[POSITIVE_RATE]) * (tpr + recordValues[TPR]) / 2); binaryMetricsSummary.prc += ((tpr - recordValues[TPR]) * (precision + recordValues[PRECISION]) / 2); binaryMetricsSummary.ks = Math.max(Math.abs(fpr - tpr), binaryMetricsSummary.ks); recordValues[TPR] = tpr; recordValues[FPR] = fpr; recordValues[PRECISION] = precision; recordValues[POSITIVE_RATE] = positiveRate; ConfusionMatrix confusionMatrix = new ConfusionMatrix(new long[][] {{countValues[CUR_TRUE], countValues[CUR_FALSE]}, {countValues[TOTAL_TRUE] - countValues[CUR_TRUE], countValues[TOTAL_FALSE] - countValues[CUR_FALSE]}}); if(binaryMetricsSummary.metricsInfoList.isEmpty() || (binaryMetricsSummary.total == 1 && first) || isMiddlePoint(cur) || Math.abs(threshold - binaryMetricsSummary.metricsInfoList.get(binaryMetricsSummary.metricsInfoList.size() - 1).f0) >= PROBABILITY_ERROR) { binaryMetricsSummary.metricsInfoList.add(Tuple2.of(threshold, confusionMatrix)); }else{ Tuple2<Double, ConfusionMatrix> metricsInfo = binaryMetricsSummary.metricsInfoList .get(binaryMetricsSummary.metricsInfoList.size() - 1); metricsInfo.f1 = confusionMatrix; } } }
ClassificationEvaluationUtil implements Serializable { public static void updateAccurateBinaryMetricsSummary(Tuple3<Double, Boolean, Double> cur, AccurateBinaryMetricsSummary binaryMetricsSummary, long[] countValues, double[] recordValues, boolean first) { if(binaryMetricsSummary.total == 0) { recordValues[TPR] = countValues[TOTAL_TRUE] == 0 ? 1.0 : 1.0 * countValues[CUR_TRUE] / countValues[TOTAL_TRUE]; recordValues[FPR] = countValues[TOTAL_FALSE] == 0 ? 1.0 : 1.0 * countValues[CUR_FALSE] / countValues[TOTAL_FALSE]; recordValues[PRECISION] = countValues[CUR_TRUE] + countValues[CUR_FALSE] == 0 ? 1.0 : 1.0 * countValues[CUR_TRUE] / (countValues[CUR_TRUE] + countValues[CUR_FALSE]); recordValues[POSITIVE_RATE] = 1.0 * (countValues[CUR_TRUE] + countValues[CUR_FALSE]) / (countValues[TOTAL_TRUE] + countValues[TOTAL_FALSE]); } if(!isMiddlePoint(cur)) { binaryMetricsSummary.total++; binaryMetricsSummary.logLoss += cur.f2; if (cur.f1) { countValues[CUR_TRUE]++; } else { countValues[CUR_FALSE]++; } } double threshold = cur.f0; double tpr = countValues[TOTAL_TRUE] == 0 ? 1.0 : 1.0 * countValues[CUR_TRUE] / countValues[TOTAL_TRUE]; double fpr = countValues[TOTAL_FALSE] == 0 ? 1.0 : 1.0 * countValues[CUR_FALSE] / countValues[TOTAL_FALSE]; double precision = countValues[CUR_TRUE] + countValues[CUR_FALSE] == 0 ? 1.0 : 1.0 * countValues[CUR_TRUE] / (countValues[CUR_TRUE] + countValues[CUR_FALSE]); double positiveRate = 1.0 * (countValues[CUR_TRUE] + countValues[CUR_FALSE]) / (countValues[TOTAL_TRUE] + countValues[TOTAL_FALSE]); if(binaryMetricsSummary.total == 1 && first){ recordValues[PRECISION] = precision; ConfusionMatrix confusionMatrix = new ConfusionMatrix( new long[][] {{0, 0}, {countValues[TOTAL_TRUE], countValues[TOTAL_FALSE]}}); binaryMetricsSummary.metricsInfoList.add(Tuple2.of(1.0, confusionMatrix)); } binaryMetricsSummary.gini += ((positiveRate - recordValues[POSITIVE_RATE]) * (tpr + recordValues[TPR]) / 2); binaryMetricsSummary.prc += ((tpr - recordValues[TPR]) * (precision + recordValues[PRECISION]) / 2); binaryMetricsSummary.ks = Math.max(Math.abs(fpr - tpr), binaryMetricsSummary.ks); recordValues[TPR] = tpr; recordValues[FPR] = fpr; recordValues[PRECISION] = precision; recordValues[POSITIVE_RATE] = positiveRate; ConfusionMatrix confusionMatrix = new ConfusionMatrix(new long[][] {{countValues[CUR_TRUE], countValues[CUR_FALSE]}, {countValues[TOTAL_TRUE] - countValues[CUR_TRUE], countValues[TOTAL_FALSE] - countValues[CUR_FALSE]}}); if(binaryMetricsSummary.metricsInfoList.isEmpty() || (binaryMetricsSummary.total == 1 && first) || isMiddlePoint(cur) || Math.abs(threshold - binaryMetricsSummary.metricsInfoList.get(binaryMetricsSummary.metricsInfoList.size() - 1).f0) >= PROBABILITY_ERROR) { binaryMetricsSummary.metricsInfoList.add(Tuple2.of(threshold, confusionMatrix)); }else{ Tuple2<Double, ConfusionMatrix> metricsInfo = binaryMetricsSummary.metricsInfoList .get(binaryMetricsSummary.metricsInfoList.size() - 1); metricsInfo.f1 = confusionMatrix; } }  }
ClassificationEvaluationUtil implements Serializable { public static void updateAccurateBinaryMetricsSummary(Tuple3<Double, Boolean, Double> cur, AccurateBinaryMetricsSummary binaryMetricsSummary, long[] countValues, double[] recordValues, boolean first) { if(binaryMetricsSummary.total == 0) { recordValues[TPR] = countValues[TOTAL_TRUE] == 0 ? 1.0 : 1.0 * countValues[CUR_TRUE] / countValues[TOTAL_TRUE]; recordValues[FPR] = countValues[TOTAL_FALSE] == 0 ? 1.0 : 1.0 * countValues[CUR_FALSE] / countValues[TOTAL_FALSE]; recordValues[PRECISION] = countValues[CUR_TRUE] + countValues[CUR_FALSE] == 0 ? 1.0 : 1.0 * countValues[CUR_TRUE] / (countValues[CUR_TRUE] + countValues[CUR_FALSE]); recordValues[POSITIVE_RATE] = 1.0 * (countValues[CUR_TRUE] + countValues[CUR_FALSE]) / (countValues[TOTAL_TRUE] + countValues[TOTAL_FALSE]); } if(!isMiddlePoint(cur)) { binaryMetricsSummary.total++; binaryMetricsSummary.logLoss += cur.f2; if (cur.f1) { countValues[CUR_TRUE]++; } else { countValues[CUR_FALSE]++; } } double threshold = cur.f0; double tpr = countValues[TOTAL_TRUE] == 0 ? 1.0 : 1.0 * countValues[CUR_TRUE] / countValues[TOTAL_TRUE]; double fpr = countValues[TOTAL_FALSE] == 0 ? 1.0 : 1.0 * countValues[CUR_FALSE] / countValues[TOTAL_FALSE]; double precision = countValues[CUR_TRUE] + countValues[CUR_FALSE] == 0 ? 1.0 : 1.0 * countValues[CUR_TRUE] / (countValues[CUR_TRUE] + countValues[CUR_FALSE]); double positiveRate = 1.0 * (countValues[CUR_TRUE] + countValues[CUR_FALSE]) / (countValues[TOTAL_TRUE] + countValues[TOTAL_FALSE]); if(binaryMetricsSummary.total == 1 && first){ recordValues[PRECISION] = precision; ConfusionMatrix confusionMatrix = new ConfusionMatrix( new long[][] {{0, 0}, {countValues[TOTAL_TRUE], countValues[TOTAL_FALSE]}}); binaryMetricsSummary.metricsInfoList.add(Tuple2.of(1.0, confusionMatrix)); } binaryMetricsSummary.gini += ((positiveRate - recordValues[POSITIVE_RATE]) * (tpr + recordValues[TPR]) / 2); binaryMetricsSummary.prc += ((tpr - recordValues[TPR]) * (precision + recordValues[PRECISION]) / 2); binaryMetricsSummary.ks = Math.max(Math.abs(fpr - tpr), binaryMetricsSummary.ks); recordValues[TPR] = tpr; recordValues[FPR] = fpr; recordValues[PRECISION] = precision; recordValues[POSITIVE_RATE] = positiveRate; ConfusionMatrix confusionMatrix = new ConfusionMatrix(new long[][] {{countValues[CUR_TRUE], countValues[CUR_FALSE]}, {countValues[TOTAL_TRUE] - countValues[CUR_TRUE], countValues[TOTAL_FALSE] - countValues[CUR_FALSE]}}); if(binaryMetricsSummary.metricsInfoList.isEmpty() || (binaryMetricsSummary.total == 1 && first) || isMiddlePoint(cur) || Math.abs(threshold - binaryMetricsSummary.metricsInfoList.get(binaryMetricsSummary.metricsInfoList.size() - 1).f0) >= PROBABILITY_ERROR) { binaryMetricsSummary.metricsInfoList.add(Tuple2.of(threshold, confusionMatrix)); }else{ Tuple2<Double, ConfusionMatrix> metricsInfo = binaryMetricsSummary.metricsInfoList .get(binaryMetricsSummary.metricsInfoList.size() - 1); metricsInfo.f1 = confusionMatrix; } }  static Tuple3<Double, Boolean, Double> getBinaryDetailStatistics(Row row,
                                                                            Object[] tuple,
                                                                            TypeInformation labelType); static Tuple2<Map<Object, Integer>, Object[]> buildLabelIndexLabelArray(HashSet<Object> set,
                                                                                   boolean binary,
                                                                                   String positiveValue,
                                                                                   TypeInformation labelType); static Tuple2<Boolean, long[]> reduceBinaryPartitionSummary(List<BinaryPartitionSummary> values, int taskId); static boolean isMiddlePoint(Tuple3<Double, Boolean, Double> t); static void updateBinaryPartitionSummary(BinaryPartitionSummary statistics,
                                                    Tuple3<Double, Boolean, Double> t); static void updateAccurateBinaryMetricsSummary(Tuple3<Double, Boolean, Double> cur,
                                                          AccurateBinaryMetricsSummary binaryMetricsSummary,
                                                          long[] countValues,
                                                          double[] recordValues,
                                                          boolean first); }
ClassificationEvaluationUtil implements Serializable { public static void updateAccurateBinaryMetricsSummary(Tuple3<Double, Boolean, Double> cur, AccurateBinaryMetricsSummary binaryMetricsSummary, long[] countValues, double[] recordValues, boolean first) { if(binaryMetricsSummary.total == 0) { recordValues[TPR] = countValues[TOTAL_TRUE] == 0 ? 1.0 : 1.0 * countValues[CUR_TRUE] / countValues[TOTAL_TRUE]; recordValues[FPR] = countValues[TOTAL_FALSE] == 0 ? 1.0 : 1.0 * countValues[CUR_FALSE] / countValues[TOTAL_FALSE]; recordValues[PRECISION] = countValues[CUR_TRUE] + countValues[CUR_FALSE] == 0 ? 1.0 : 1.0 * countValues[CUR_TRUE] / (countValues[CUR_TRUE] + countValues[CUR_FALSE]); recordValues[POSITIVE_RATE] = 1.0 * (countValues[CUR_TRUE] + countValues[CUR_FALSE]) / (countValues[TOTAL_TRUE] + countValues[TOTAL_FALSE]); } if(!isMiddlePoint(cur)) { binaryMetricsSummary.total++; binaryMetricsSummary.logLoss += cur.f2; if (cur.f1) { countValues[CUR_TRUE]++; } else { countValues[CUR_FALSE]++; } } double threshold = cur.f0; double tpr = countValues[TOTAL_TRUE] == 0 ? 1.0 : 1.0 * countValues[CUR_TRUE] / countValues[TOTAL_TRUE]; double fpr = countValues[TOTAL_FALSE] == 0 ? 1.0 : 1.0 * countValues[CUR_FALSE] / countValues[TOTAL_FALSE]; double precision = countValues[CUR_TRUE] + countValues[CUR_FALSE] == 0 ? 1.0 : 1.0 * countValues[CUR_TRUE] / (countValues[CUR_TRUE] + countValues[CUR_FALSE]); double positiveRate = 1.0 * (countValues[CUR_TRUE] + countValues[CUR_FALSE]) / (countValues[TOTAL_TRUE] + countValues[TOTAL_FALSE]); if(binaryMetricsSummary.total == 1 && first){ recordValues[PRECISION] = precision; ConfusionMatrix confusionMatrix = new ConfusionMatrix( new long[][] {{0, 0}, {countValues[TOTAL_TRUE], countValues[TOTAL_FALSE]}}); binaryMetricsSummary.metricsInfoList.add(Tuple2.of(1.0, confusionMatrix)); } binaryMetricsSummary.gini += ((positiveRate - recordValues[POSITIVE_RATE]) * (tpr + recordValues[TPR]) / 2); binaryMetricsSummary.prc += ((tpr - recordValues[TPR]) * (precision + recordValues[PRECISION]) / 2); binaryMetricsSummary.ks = Math.max(Math.abs(fpr - tpr), binaryMetricsSummary.ks); recordValues[TPR] = tpr; recordValues[FPR] = fpr; recordValues[PRECISION] = precision; recordValues[POSITIVE_RATE] = positiveRate; ConfusionMatrix confusionMatrix = new ConfusionMatrix(new long[][] {{countValues[CUR_TRUE], countValues[CUR_FALSE]}, {countValues[TOTAL_TRUE] - countValues[CUR_TRUE], countValues[TOTAL_FALSE] - countValues[CUR_FALSE]}}); if(binaryMetricsSummary.metricsInfoList.isEmpty() || (binaryMetricsSummary.total == 1 && first) || isMiddlePoint(cur) || Math.abs(threshold - binaryMetricsSummary.metricsInfoList.get(binaryMetricsSummary.metricsInfoList.size() - 1).f0) >= PROBABILITY_ERROR) { binaryMetricsSummary.metricsInfoList.add(Tuple2.of(threshold, confusionMatrix)); }else{ Tuple2<Double, ConfusionMatrix> metricsInfo = binaryMetricsSummary.metricsInfoList .get(binaryMetricsSummary.metricsInfoList.size() - 1); metricsInfo.f1 = confusionMatrix; } }  static Tuple3<Double, Boolean, Double> getBinaryDetailStatistics(Row row,
                                                                            Object[] tuple,
                                                                            TypeInformation labelType); static Tuple2<Map<Object, Integer>, Object[]> buildLabelIndexLabelArray(HashSet<Object> set,
                                                                                   boolean binary,
                                                                                   String positiveValue,
                                                                                   TypeInformation labelType); static Tuple2<Boolean, long[]> reduceBinaryPartitionSummary(List<BinaryPartitionSummary> values, int taskId); static boolean isMiddlePoint(Tuple3<Double, Boolean, Double> t); static void updateBinaryPartitionSummary(BinaryPartitionSummary statistics,
                                                    Tuple3<Double, Boolean, Double> t); static void updateAccurateBinaryMetricsSummary(Tuple3<Double, Boolean, Double> cur,
                                                          AccurateBinaryMetricsSummary binaryMetricsSummary,
                                                          long[] countValues,
                                                          double[] recordValues,
                                                          boolean first); static int DETAIL_BIN_NUMBER; static int TOTAL_TRUE; static int TOTAL_FALSE; static int CUR_TRUE; static int CUR_FALSE; static int RECORD_LEN; static int BINARY_LABEL_NUMBER; static Tuple3<Double, Boolean, Double> middlePoint; }