@Test public void shouldAggregateNumericStatistics() { FeatureNameStatistics stat1 = FeatureNameStatistics.newBuilder() .setNumStats( NumericStatistics.newBuilder() .setMax(20) .setMin(1) .setMean(6) .setNumZeros(0) .setStdDev(7.90569415) .setCommonStats( CommonStatistics.newBuilder() .setTotNumValues(5) .setNumNonMissing(5) .setAvgNumValues(1) .setMaxNumValues(1) .setMinNumValues(1) .setNumMissing(0))) .setType(Type.INT) .setPath(Path.newBuilder().addStep("feature").build()) .build(); FeatureNameStatistics stat2 = FeatureNameStatistics.newBuilder() .setNumStats( NumericStatistics.newBuilder() .setMax(10) .setMin(0) .setMean(4) .setNumZeros(1) .setStdDev(3.807886553) .setCommonStats( CommonStatistics.newBuilder() .setTotNumValues(5) .setNumNonMissing(5) .setAvgNumValues(1) .setMaxNumValues(1) .setMinNumValues(1) .setNumMissing(1))) .setPath(Path.newBuilder().addStep("feature").build()) .setType(Type.INT) .build(); FeatureNameStatistics expected = FeatureNameStatistics.newBuilder() .setNumStats( NumericStatistics.newBuilder() .setMax(20) .setMin(0) .setMean(5) .setNumZeros(1) .setStdDev(5.944184833146219) .setCommonStats( CommonStatistics.newBuilder() .setTotNumValues(10) .setNumNonMissing(10) .setAvgNumValues(1) .setMaxNumValues(1) .setMinNumValues(1) .setNumMissing(1))) .setPath(Path.newBuilder().addStep("feature").build()) .setType(Type.INT) .build(); assertThat( statsService.mergeStatistics(Arrays.asList(Arrays.asList(stat1, stat2))), equalTo(Arrays.asList(expected))); }
@VisibleForTesting public List<FeatureNameStatistics> mergeStatistics( List<List<FeatureNameStatistics>> featureNameStatistics) { List<FeatureNameStatistics> unnestedList = new ArrayList<>(); featureNameStatistics.forEach(unnestedList::addAll); Map<Path, List<FeatureNameStatistics>> groupByPath = unnestedList.stream() .collect(Collectors.groupingBy(FeatureNameStatistics::getPath, Collectors.toList())); List<FeatureNameStatistics> merged = new ArrayList<>(); for (Path key : groupByPath.keySet()) { List<FeatureNameStatistics> featureNameStatisticsForKey = groupByPath.get(key); if (featureNameStatisticsForKey.size() == 1) { merged.add(featureNameStatisticsForKey.get(0)); } else { switch (featureNameStatisticsForKey.get(0).getType()) { case INT: case FLOAT: merged.add(mergeNumStatistics(featureNameStatisticsForKey)); break; case STRING: merged.add(mergeCategoricalStatistics(groupByPath.get(key))); break; case BYTES: merged.add(mergeByteStatistics(groupByPath.get(key))); break; case STRUCT: merged.add(mergeStructStats(groupByPath.get(key))); break; default: throw new IllegalArgumentException( "Statistics are only supported for string, boolean, bytes and numeric features"); } } } return merged; }
StatsService { @VisibleForTesting public List<FeatureNameStatistics> mergeStatistics( List<List<FeatureNameStatistics>> featureNameStatistics) { List<FeatureNameStatistics> unnestedList = new ArrayList<>(); featureNameStatistics.forEach(unnestedList::addAll); Map<Path, List<FeatureNameStatistics>> groupByPath = unnestedList.stream() .collect(Collectors.groupingBy(FeatureNameStatistics::getPath, Collectors.toList())); List<FeatureNameStatistics> merged = new ArrayList<>(); for (Path key : groupByPath.keySet()) { List<FeatureNameStatistics> featureNameStatisticsForKey = groupByPath.get(key); if (featureNameStatisticsForKey.size() == 1) { merged.add(featureNameStatisticsForKey.get(0)); } else { switch (featureNameStatisticsForKey.get(0).getType()) { case INT: case FLOAT: merged.add(mergeNumStatistics(featureNameStatisticsForKey)); break; case STRING: merged.add(mergeCategoricalStatistics(groupByPath.get(key))); break; case BYTES: merged.add(mergeByteStatistics(groupByPath.get(key))); break; case STRUCT: merged.add(mergeStructStats(groupByPath.get(key))); break; default: throw new IllegalArgumentException( "Statistics are only supported for string, boolean, bytes and numeric features"); } } } return merged; } }
StatsService { @VisibleForTesting public List<FeatureNameStatistics> mergeStatistics( List<List<FeatureNameStatistics>> featureNameStatistics) { List<FeatureNameStatistics> unnestedList = new ArrayList<>(); featureNameStatistics.forEach(unnestedList::addAll); Map<Path, List<FeatureNameStatistics>> groupByPath = unnestedList.stream() .collect(Collectors.groupingBy(FeatureNameStatistics::getPath, Collectors.toList())); List<FeatureNameStatistics> merged = new ArrayList<>(); for (Path key : groupByPath.keySet()) { List<FeatureNameStatistics> featureNameStatisticsForKey = groupByPath.get(key); if (featureNameStatisticsForKey.size() == 1) { merged.add(featureNameStatisticsForKey.get(0)); } else { switch (featureNameStatisticsForKey.get(0).getType()) { case INT: case FLOAT: merged.add(mergeNumStatistics(featureNameStatisticsForKey)); break; case STRING: merged.add(mergeCategoricalStatistics(groupByPath.get(key))); break; case BYTES: merged.add(mergeByteStatistics(groupByPath.get(key))); break; case STRUCT: merged.add(mergeStructStats(groupByPath.get(key))); break; default: throw new IllegalArgumentException( "Statistics are only supported for string, boolean, bytes and numeric features"); } } } return merged; } @Autowired  StatsService(
      StoreRepository storeRepository,
      FeatureStatisticsRepository featureStatisticsRepository,
      FeatureSetRepository featureSetRepository); }
StatsService { @VisibleForTesting public List<FeatureNameStatistics> mergeStatistics( List<List<FeatureNameStatistics>> featureNameStatistics) { List<FeatureNameStatistics> unnestedList = new ArrayList<>(); featureNameStatistics.forEach(unnestedList::addAll); Map<Path, List<FeatureNameStatistics>> groupByPath = unnestedList.stream() .collect(Collectors.groupingBy(FeatureNameStatistics::getPath, Collectors.toList())); List<FeatureNameStatistics> merged = new ArrayList<>(); for (Path key : groupByPath.keySet()) { List<FeatureNameStatistics> featureNameStatisticsForKey = groupByPath.get(key); if (featureNameStatisticsForKey.size() == 1) { merged.add(featureNameStatisticsForKey.get(0)); } else { switch (featureNameStatisticsForKey.get(0).getType()) { case INT: case FLOAT: merged.add(mergeNumStatistics(featureNameStatisticsForKey)); break; case STRING: merged.add(mergeCategoricalStatistics(groupByPath.get(key))); break; case BYTES: merged.add(mergeByteStatistics(groupByPath.get(key))); break; case STRUCT: merged.add(mergeStructStats(groupByPath.get(key))); break; default: throw new IllegalArgumentException( "Statistics are only supported for string, boolean, bytes and numeric features"); } } } return merged; } @Autowired  StatsService(
      StoreRepository storeRepository,
      FeatureStatisticsRepository featureStatisticsRepository,
      FeatureSetRepository featureSetRepository); @Transactional GetFeatureStatisticsResponse getFeatureStatistics(GetFeatureStatisticsRequest request); @VisibleForTesting List<FeatureNameStatistics> mergeStatistics(
      List<List<FeatureNameStatistics>> featureNameStatistics); }
StatsService { @VisibleForTesting public List<FeatureNameStatistics> mergeStatistics( List<List<FeatureNameStatistics>> featureNameStatistics) { List<FeatureNameStatistics> unnestedList = new ArrayList<>(); featureNameStatistics.forEach(unnestedList::addAll); Map<Path, List<FeatureNameStatistics>> groupByPath = unnestedList.stream() .collect(Collectors.groupingBy(FeatureNameStatistics::getPath, Collectors.toList())); List<FeatureNameStatistics> merged = new ArrayList<>(); for (Path key : groupByPath.keySet()) { List<FeatureNameStatistics> featureNameStatisticsForKey = groupByPath.get(key); if (featureNameStatisticsForKey.size() == 1) { merged.add(featureNameStatisticsForKey.get(0)); } else { switch (featureNameStatisticsForKey.get(0).getType()) { case INT: case FLOAT: merged.add(mergeNumStatistics(featureNameStatisticsForKey)); break; case STRING: merged.add(mergeCategoricalStatistics(groupByPath.get(key))); break; case BYTES: merged.add(mergeByteStatistics(groupByPath.get(key))); break; case STRUCT: merged.add(mergeStructStats(groupByPath.get(key))); break; default: throw new IllegalArgumentException( "Statistics are only supported for string, boolean, bytes and numeric features"); } } } return merged; } @Autowired  StatsService(
      StoreRepository storeRepository,
      FeatureStatisticsRepository featureStatisticsRepository,
      FeatureSetRepository featureSetRepository); @Transactional GetFeatureStatisticsResponse getFeatureStatistics(GetFeatureStatisticsRequest request); @VisibleForTesting List<FeatureNameStatistics> mergeStatistics(
      List<List<FeatureNameStatistics>> featureNameStatistics);  }