EvaluationUtil implements Serializable { public static BaseMetricsSummary getDetailStatistics(Iterable<Row> rows, String positiveValue, boolean binary, TypeInformation labelType) { return getDetailStatistics(rows, positiveValue, binary, null, labelType); }  static boolean labelCompare(Object label1, Object label2, TypeInformation type); static int compare(Object o1, Object o2); static Object castTo(Object x, TypeInformation t); static BaseMetricsSummary getDetailStatistics(Iterable<Row> rows, String positiveValue, boolean binary, TypeInformation labelType); static BaseMetricsSummary getDetailStatistics(Iterable<Row> rows,
                                                         boolean binary,
                                                         Tuple2<Map<Object, Integer>, Object[]> labels,
                                                         TypeInformation labelType); static boolean checkRowFieldNotNull(Row row); static double extractLogloss(TreeMap<Object, Double> labelProbMap, Object label); static TreeMap<Object, Double> extractLabelProbMap(Row row, TypeInformation labelType); static void updateBinaryMetricsSummary(TreeMap<Object, Double> labelProbMap,
                                                  Object label,
                                                  BinaryMetricsSummary binaryMetricsSummary); static void updateMultiMetricsSummary(TreeMap<Object, Double> labelProbMap,
                                                 Object label,
                                                 Map<Object, Integer> labels,
                                                 MultiMetricsSummary multiMetricsSummary); static RegressionMetricsSummary getRegressionStatistics(Iterable<Row> rows); static MultiMetricsSummary getMultiClassMetrics(Iterable<Row> rows,
                                                           Tuple2<Map<Object, Integer>, Object[]>
                                                               labelIndexLabelArray);  }
@Test public void getDetailStatisticsBinary(){ Row[] rows = new Row[] { Row.of("prefix1", "{\"prefix1\": 0.9, \"prefix0\": 0.1}"), Row.of("prefix1", "{\"prefix1\": 0.8, \"prefix0\": 0.2}"), Row.of("prefix1", "{\"prefix1\": 0.7, \"prefix0\": 0.3}"), Row.of("prefix0", "{\"prefix1\": 0.75, \"prefix0\": 0.25}"), Row.of("prefix0", "{\"prefix1\": 0.6, \"prefix0\": 0.4}"), Row.of(null, "{\"prefix1\": 0.75, \"prefix0\": 0.25}"), }; BaseMetricsSummary baseMetric = getDetailStatistics(Arrays.asList(rows), null, true, Types.STRING); assertBinaryMetrics(baseMetric); Map<Object, Integer> map = new HashMap<>(); map.put("prefix0", 1); map.put("prefix1", 0); baseMetric = getDetailStatistics(Arrays.asList(rows), true, Tuple2.of(map, new Object[]{"prefix1", "prefix0"}), Types.STRING); assertBinaryMetrics(baseMetric); baseMetric = getDetailStatistics(Arrays.asList(rows), "prefix0", true, Types.STRING); Assert.assertTrue(baseMetric instanceof BinaryMetricsSummary); BinaryMetricsSummary metrics = (BinaryMetricsSummary) baseMetric; Assert.assertArrayEquals(new Object[] {"prefix0", "prefix1"}, metrics.labels); thrown.expect(RuntimeException.class); thrown.expectMessage("Not contain positiveValue"); getDetailStatistics(Arrays.asList(rows), "0", true, Types.STRING); }
@Test public void getDetailStatisticsBinaryDouble(){ Row[] rows = new Row[] { Row.of(1.0, "{\"1.00\": 0.9, \"0.00\": 0.1}"), Row.of(1.00, "{\"1.0\": 0.8, \"0.00\": 0.2}"), Row.of(1.0, "{\"1.00\": 0.7, \"0.000\": 0.3}"), Row.of(0.0, "{\"1.0\": 0.75, \"0.0\": 0.25}"), Row.of(null, "{\"1.0\": 0.75, \"0.0\": 0.25}"), Row.of(0.00, "{\"1.0\": 0.6, \"0.00\": 0.4}") }; BaseMetricsSummary baseMetric = getDetailStatistics(Arrays.asList(rows), null, true, Types.DOUBLE); assertBinaryMetrics(baseMetric); Map<Object, Integer> map = new HashMap<>(); map.put(0.0, 1); map.put(1.0, 0); baseMetric = getDetailStatistics(Arrays.asList(rows), true, Tuple2.of(map, new Object[]{1.0, 0.0}), Types.DOUBLE); assertBinaryMetrics(baseMetric); baseMetric = getDetailStatistics(Arrays.asList(rows), "0.000", true, Types.DOUBLE); Assert.assertTrue(baseMetric instanceof BinaryMetricsSummary); BinaryMetricsSummary metrics = (BinaryMetricsSummary) baseMetric; Assert.assertArrayEquals(new Object[] {0.0, 1.0}, metrics.labels); }
@Test public void getDetailStatisticsMulti(){ Row[] rows = new Row[] { Row.of("prefix0", "{\"prefix0\": 0.3, \"prefix1\": 0.2, \"prefix2\": 0.5}"), Row.of("prefix0", "{\"prefix0\": 0.4, \"prefix1\": 0.3, \"prefix2\": 0.3}"), Row.of("prefix1", "{\"prefix0\": 0.3, \"prefix1\": 0.4, \"prefix2\": 0.3}"), Row.of("prefix2", "{\"prefix0\": 0.5, \"prefix1\": 0.2, \"prefix2\": 0.3}"), Row.of("prefix2", "{\"prefix0\": 0.4, \"prefix1\": 0.3, \"prefix2\": 0.3}"), Row.of("prefix2", "{\"prefix0\": 0.4, \"prefix1\": 0.1, \"prefix2\": 0.5}"), Row.of("prefix2", "{\"prefix0\": 0.6, \"prefix1\": 0.1, \"prefix2\": 0.3}"), Row.of("prefix1", "{\"prefix0\": 0.4, \"prefix1\": 0.1, \"prefix2\": 0.5}"), Row.of("prefix1", "{\"prefix0\": 0.2, \"prefix1\": 0.5, \"prefix2\": 0.3}"), Row.of("prefix1", "{\"prefix0\": 0.2, \"prefix1\": 0.5, \"prefix2\": 0.3}") }; BaseMetricsSummary baseMetric = getDetailStatistics(Arrays.asList(rows), null, false, Types.STRING); assertMultiMetrics(baseMetric); Map<Object, Integer> map = new HashMap<>(); map.put("prefix0", 2); map.put("prefix1", 1); map.put("prefix2", 0); baseMetric = getDetailStatistics(Arrays.asList(rows), false, Tuple2.of(map, new Object[]{"prefix2", "prefix1", "prefix0"}), Types.STRING); assertMultiMetrics(baseMetric); }