CardinalityHLLAnalyzer implements Analyzer<CardinalityHLLStatistics> { @Override public boolean analyze(String... record) { if (record == null) { return true; } cardinalityStatistics.resize(record.length); for (int i = 0; i < record.length; i++) { final CardinalityHLLStatistics cardStats = cardinalityStatistics.get(i); if (cardStats.getHyperLogLog() == null) { cardStats.setHyperLogLog(new HyperLogLog(rsd)); } cardStats.add(record[i]); cardStats.incrementCount(); } return true; }  @Override void init(); void setRelativeStandardDeviation(int rsd); @Override boolean analyze(String... record); @Override void end(); @Override List<CardinalityHLLStatistics> getResult(); @Override void close();  }
@Test public void testAnalyze() { String[] data = new String[] { "0", "1", "2", "3", "16", "17", "18", "19", "19" }; distinctHLLAna.init(); for (String col : data) { distinctHLLAna.analyze(col); } Assert.assertEquals(8, distinctHLLAna.getResult().get(0).getDistinctCount(), 0); Assert.assertEquals(1, distinctHLLAna.getResult().get(0).getDuplicateCount(), 0); int size = 10000000; distinctHLLAna.init(); for (int i = 0; i < size; i++) { distinctHLLAna.analyze(streamElement(i)); } long estimate = distinctHLLAna.getResult().get(0).getDistinctCount(); double err = Math.abs(estimate - size) / (double) size; Assert.assertTrue(err < .1); data = new String[] { "" }; distinctHLLAna.init(); for (String col : data) { distinctHLLAna.analyze(col); } Assert.assertEquals(1, distinctHLLAna.getResult().get(0).getDistinctCount(), 0); Assert.assertEquals(0, distinctHLLAna.getResult().get(0).getDuplicateCount(), 0); distinctHLLAna.init(); Assert.assertTrue(distinctHLLAna.getResult().size() == 0); }