@Test public void testMetaBasic() throws Exception { String[] directives = new String[] { "parse-as-csv body ,", "drop body" }; List<Row> rows = Arrays.asList( new Row("body", "1234.45,650-897-3839,111-11-1111,32826,02/29/2000,\"$1234.56\",http: new Row("body", "45.56,670-897-3839,111-12-1111,32826,02/01/2011,\"$56,789\",http: new Row("body", "45.56,670-897-3839,222,32826,9/14/2016,\"\",http: ); rows = TestingRig.execute(directives, rows); Statistics meta = new BasicStatistics(); Row summary = meta.aggregate(rows); Assert.assertTrue(rows.size() > 1); Assert.assertEquals(3, summary.width()); Assert.assertEquals(3.0, summary.getValue("total")); Row stats = (Row) summary.getValue("stats"); Row types = (Row) summary.getValue("types"); Assert.assertEquals(7, stats.width()); Assert.assertEquals(7, types.width()); }
@Override public Row aggregate(List<Row> rows) { ColumnMetric types = new ColumnMetric(); ColumnMetric stats = new ColumnMetric(); Double count = new Double(0); for (Row row : rows) { ++count; for (int i = 0; i < row.width(); ++i) { String column = row.getColumn(i); Object object = row.getValue(i); if (object == null) { stats.increment(column, "null"); } else { stats.increment(column, "non-null"); } if (object instanceof String) { String value = ((String) object); if (value.isEmpty()) { stats.increment(column, "empty"); } else { Map<String, List<String>> finds = engine.findWithType(value); for (String find : finds.keySet()) { types.increment(column, find); } } } } } Row rowTypes = new Row(); for (String column : types.getColumns()) { rowTypes.add(column, types.percentage(column, count)); } Row rowStats = new Row(); for (String column : stats.getColumns()) { rowStats.add(column, stats.percentage(column, count)); } Row row = new Row(); row.add("types", rowTypes); row.add("stats", rowStats); row.add("total", count); return row; }
BasicStatistics implements Statistics { @Override public Row aggregate(List<Row> rows) { ColumnMetric types = new ColumnMetric(); ColumnMetric stats = new ColumnMetric(); Double count = new Double(0); for (Row row : rows) { ++count; for (int i = 0; i < row.width(); ++i) { String column = row.getColumn(i); Object object = row.getValue(i); if (object == null) { stats.increment(column, "null"); } else { stats.increment(column, "non-null"); } if (object instanceof String) { String value = ((String) object); if (value.isEmpty()) { stats.increment(column, "empty"); } else { Map<String, List<String>> finds = engine.findWithType(value); for (String find : finds.keySet()) { types.increment(column, find); } } } } } Row rowTypes = new Row(); for (String column : types.getColumns()) { rowTypes.add(column, types.percentage(column, count)); } Row rowStats = new Row(); for (String column : stats.getColumns()) { rowStats.add(column, stats.percentage(column, count)); } Row row = new Row(); row.add("types", rowTypes); row.add("stats", rowStats); row.add("total", count); return row; } }
BasicStatistics implements Statistics { @Override public Row aggregate(List<Row> rows) { ColumnMetric types = new ColumnMetric(); ColumnMetric stats = new ColumnMetric(); Double count = new Double(0); for (Row row : rows) { ++count; for (int i = 0; i < row.width(); ++i) { String column = row.getColumn(i); Object object = row.getValue(i); if (object == null) { stats.increment(column, "null"); } else { stats.increment(column, "non-null"); } if (object instanceof String) { String value = ((String) object); if (value.isEmpty()) { stats.increment(column, "empty"); } else { Map<String, List<String>> finds = engine.findWithType(value); for (String find : finds.keySet()) { types.increment(column, find); } } } } } Row rowTypes = new Row(); for (String column : types.getColumns()) { rowTypes.add(column, types.percentage(column, count)); } Row rowStats = new Row(); for (String column : stats.getColumns()) { rowStats.add(column, stats.percentage(column, count)); } Row row = new Row(); row.add("types", rowTypes); row.add("stats", rowStats); row.add("total", count); return row; } BasicStatistics(); }
BasicStatistics implements Statistics { @Override public Row aggregate(List<Row> rows) { ColumnMetric types = new ColumnMetric(); ColumnMetric stats = new ColumnMetric(); Double count = new Double(0); for (Row row : rows) { ++count; for (int i = 0; i < row.width(); ++i) { String column = row.getColumn(i); Object object = row.getValue(i); if (object == null) { stats.increment(column, "null"); } else { stats.increment(column, "non-null"); } if (object instanceof String) { String value = ((String) object); if (value.isEmpty()) { stats.increment(column, "empty"); } else { Map<String, List<String>> finds = engine.findWithType(value); for (String find : finds.keySet()) { types.increment(column, find); } } } } } Row rowTypes = new Row(); for (String column : types.getColumns()) { rowTypes.add(column, types.percentage(column, count)); } Row rowStats = new Row(); for (String column : stats.getColumns()) { rowStats.add(column, stats.percentage(column, count)); } Row row = new Row(); row.add("types", rowTypes); row.add("stats", rowStats); row.add("total", count); return row; } BasicStatistics(); @Override Row aggregate(List<Row> rows); }
BasicStatistics implements Statistics { @Override public Row aggregate(List<Row> rows) { ColumnMetric types = new ColumnMetric(); ColumnMetric stats = new ColumnMetric(); Double count = new Double(0); for (Row row : rows) { ++count; for (int i = 0; i < row.width(); ++i) { String column = row.getColumn(i); Object object = row.getValue(i); if (object == null) { stats.increment(column, "null"); } else { stats.increment(column, "non-null"); } if (object instanceof String) { String value = ((String) object); if (value.isEmpty()) { stats.increment(column, "empty"); } else { Map<String, List<String>> finds = engine.findWithType(value); for (String find : finds.keySet()) { types.increment(column, find); } } } } } Row rowTypes = new Row(); for (String column : types.getColumns()) { rowTypes.add(column, types.percentage(column, count)); } Row rowStats = new Row(); for (String column : stats.getColumns()) { rowStats.add(column, stats.percentage(column, count)); } Row row = new Row(); row.add("types", rowTypes); row.add("stats", rowStats); row.add("total", count); return row; } BasicStatistics(); @Override Row aggregate(List<Row> rows);  }