@Test public void getRegressionStatisticsTest(){ Row[] rows = new Row[] { Row.of(0.4, 0.5), Row.of(0.3, 0.5), Row.of(0.4, null), Row.of(0.2, 0.6), Row.of(0.6, 0.7), Row.of(0.1, 0.5) }; RegressionMetricsSummary metricsSummary = getRegressionStatistics(Arrays.asList(rows)); Assert.assertEquals(metricsSummary.total, 5); Assert.assertEquals(metricsSummary.ySumLocal, 1.6, 0.001); Assert.assertEquals(metricsSummary.ySum2Local, 0.66, 0.001); Assert.assertEquals(metricsSummary.predSumLocal, 2.8, 0.001); Assert.assertEquals(metricsSummary.predSum2Local, 1.599, 0.001); Assert.assertEquals(metricsSummary.sseLocal, 0.38, 0.001); Assert.assertEquals(metricsSummary.maeLocal, 1.2, 0.001); Assert.assertEquals(metricsSummary.mapeLocal, 7.083, 0.001); }
public static RegressionMetricsSummary getRegressionStatistics(Iterable<Row> rows) { RegressionMetricsSummary regressionSummary = new RegressionMetricsSummary(); for (Row row : rows) { if (checkRowFieldNotNull(row)) { double yVal = ((Number)row.getField(0)).doubleValue(); double predictVal = ((Number)row.getField(1)).doubleValue(); double diff = Math.abs(yVal - predictVal); regressionSummary.ySumLocal += yVal; regressionSummary.ySum2Local += yVal * yVal; regressionSummary.predSumLocal += predictVal; regressionSummary.predSum2Local += predictVal * predictVal; regressionSummary.maeLocal += diff; regressionSummary.sseLocal += diff * diff; regressionSummary.mapeLocal += Math.abs(diff / yVal); regressionSummary.total++; } } return regressionSummary.total == 0 ? null : regressionSummary; }
EvaluationUtil implements Serializable { public static RegressionMetricsSummary getRegressionStatistics(Iterable<Row> rows) { RegressionMetricsSummary regressionSummary = new RegressionMetricsSummary(); for (Row row : rows) { if (checkRowFieldNotNull(row)) { double yVal = ((Number)row.getField(0)).doubleValue(); double predictVal = ((Number)row.getField(1)).doubleValue(); double diff = Math.abs(yVal - predictVal); regressionSummary.ySumLocal += yVal; regressionSummary.ySum2Local += yVal * yVal; regressionSummary.predSumLocal += predictVal; regressionSummary.predSum2Local += predictVal * predictVal; regressionSummary.maeLocal += diff; regressionSummary.sseLocal += diff * diff; regressionSummary.mapeLocal += Math.abs(diff / yVal); regressionSummary.total++; } } return regressionSummary.total == 0 ? null : regressionSummary; } }
EvaluationUtil implements Serializable { public static RegressionMetricsSummary getRegressionStatistics(Iterable<Row> rows) { RegressionMetricsSummary regressionSummary = new RegressionMetricsSummary(); for (Row row : rows) { if (checkRowFieldNotNull(row)) { double yVal = ((Number)row.getField(0)).doubleValue(); double predictVal = ((Number)row.getField(1)).doubleValue(); double diff = Math.abs(yVal - predictVal); regressionSummary.ySumLocal += yVal; regressionSummary.ySum2Local += yVal * yVal; regressionSummary.predSumLocal += predictVal; regressionSummary.predSum2Local += predictVal * predictVal; regressionSummary.maeLocal += diff; regressionSummary.sseLocal += diff * diff; regressionSummary.mapeLocal += Math.abs(diff / yVal); regressionSummary.total++; } } return regressionSummary.total == 0 ? null : regressionSummary; }  }
EvaluationUtil implements Serializable { public static RegressionMetricsSummary getRegressionStatistics(Iterable<Row> rows) { RegressionMetricsSummary regressionSummary = new RegressionMetricsSummary(); for (Row row : rows) { if (checkRowFieldNotNull(row)) { double yVal = ((Number)row.getField(0)).doubleValue(); double predictVal = ((Number)row.getField(1)).doubleValue(); double diff = Math.abs(yVal - predictVal); regressionSummary.ySumLocal += yVal; regressionSummary.ySum2Local += yVal * yVal; regressionSummary.predSumLocal += predictVal; regressionSummary.predSum2Local += predictVal * predictVal; regressionSummary.maeLocal += diff; regressionSummary.sseLocal += diff * diff; regressionSummary.mapeLocal += Math.abs(diff / yVal); regressionSummary.total++; } } return regressionSummary.total == 0 ? null : regressionSummary; }  static boolean labelCompare(Object label1, Object label2, TypeInformation type); static int compare(Object o1, Object o2); static Object castTo(Object x, TypeInformation t); static BaseMetricsSummary getDetailStatistics(Iterable<Row> rows, String positiveValue, boolean binary, TypeInformation labelType); static BaseMetricsSummary getDetailStatistics(Iterable<Row> rows,
                                                         boolean binary,
                                                         Tuple2<Map<Object, Integer>, Object[]> labels,
                                                         TypeInformation labelType); static boolean checkRowFieldNotNull(Row row); static double extractLogloss(TreeMap<Object, Double> labelProbMap, Object label); static TreeMap<Object, Double> extractLabelProbMap(Row row, TypeInformation labelType); static void updateBinaryMetricsSummary(TreeMap<Object, Double> labelProbMap,
                                                  Object label,
                                                  BinaryMetricsSummary binaryMetricsSummary); static void updateMultiMetricsSummary(TreeMap<Object, Double> labelProbMap,
                                                 Object label,
                                                 Map<Object, Integer> labels,
                                                 MultiMetricsSummary multiMetricsSummary); static RegressionMetricsSummary getRegressionStatistics(Iterable<Row> rows); static MultiMetricsSummary getMultiClassMetrics(Iterable<Row> rows,
                                                           Tuple2<Map<Object, Integer>, Object[]>
                                                               labelIndexLabelArray); }
EvaluationUtil implements Serializable { public static RegressionMetricsSummary getRegressionStatistics(Iterable<Row> rows) { RegressionMetricsSummary regressionSummary = new RegressionMetricsSummary(); for (Row row : rows) { if (checkRowFieldNotNull(row)) { double yVal = ((Number)row.getField(0)).doubleValue(); double predictVal = ((Number)row.getField(1)).doubleValue(); double diff = Math.abs(yVal - predictVal); regressionSummary.ySumLocal += yVal; regressionSummary.ySum2Local += yVal * yVal; regressionSummary.predSumLocal += predictVal; regressionSummary.predSum2Local += predictVal * predictVal; regressionSummary.maeLocal += diff; regressionSummary.sseLocal += diff * diff; regressionSummary.mapeLocal += Math.abs(diff / yVal); regressionSummary.total++; } } return regressionSummary.total == 0 ? null : regressionSummary; }  static boolean labelCompare(Object label1, Object label2, TypeInformation type); static int compare(Object o1, Object o2); static Object castTo(Object x, TypeInformation t); static BaseMetricsSummary getDetailStatistics(Iterable<Row> rows, String positiveValue, boolean binary, TypeInformation labelType); static BaseMetricsSummary getDetailStatistics(Iterable<Row> rows,
                                                         boolean binary,
                                                         Tuple2<Map<Object, Integer>, Object[]> labels,
                                                         TypeInformation labelType); static boolean checkRowFieldNotNull(Row row); static double extractLogloss(TreeMap<Object, Double> labelProbMap, Object label); static TreeMap<Object, Double> extractLabelProbMap(Row row, TypeInformation labelType); static void updateBinaryMetricsSummary(TreeMap<Object, Double> labelProbMap,
                                                  Object label,
                                                  BinaryMetricsSummary binaryMetricsSummary); static void updateMultiMetricsSummary(TreeMap<Object, Double> labelProbMap,
                                                 Object label,
                                                 Map<Object, Integer> labels,
                                                 MultiMetricsSummary multiMetricsSummary); static RegressionMetricsSummary getRegressionStatistics(Iterable<Row> rows); static MultiMetricsSummary getMultiClassMetrics(Iterable<Row> rows,
                                                           Tuple2<Map<Object, Integer>, Object[]>
                                                               labelIndexLabelArray);  }