Flattener { public FlattenedTable flattenTable(Table inputTable, Collection<TableShard> inputTableShards, String flattenByField, UUID flattenId) throws IllegalArgumentException, IllegalStateException, PatternException, LengthColumnMissingException { if (inputTable instanceof FlattenedTable) throw new IllegalArgumentException("Cannot flatten an already flattened table."); if (!flattenByField.endsWith(repeatedColNameGen.allEntriesIdentifyingSubstr())) throw new IllegalArgumentException( "Flatten-By field does not end with '" + repeatedColNameGen.allEntriesIdentifyingSubstr() + "'"); String resultTableName = flattenedTableNameGen.createFlattenedTableName(inputTable.getName(), flattenByField, flattenId); if (inputTableShards == null) inputTableShards = inputTable.getShards(); List<TableShard> flattenedTableShards = new ArrayList<>(); for (TableShard shard : inputTableShards) flattenedTableShards.add(flattenTableShard(resultTableName, shard, flattenByField)); Set<Long> firstRowIdsOfInputShards = inputTableShards.stream().map(shard -> shard.getLowestRowId()).collect(Collectors.toSet()); return factory.createFlattenedTable(resultTableName, flattenedTableShards, firstRowIdsOfInputShards); }  FlattenedTable flattenTable(Table inputTable, Collection<TableShard> inputTableShards,
      String flattenByField, UUID flattenId);  }
@Test public void simpleTest() throws LoadException { String json = "[ { " + "\"a\": [ " + "{ \"b\": 1, \"d\":[99, 100] }, " + "{ \"b\": 2, \"d\":[] }" + "]" + ",\"c\" : [ 9, 10 ] }," + "{ " + "\"a\": [ " + "{ \"b\": 3, \"d\":[300,301,302] }, " + "{ \"b\": 4, \"d\":[303,304,305] }, " + "{ \"b\": 5, \"d\":[306,307,308] } " + "]" + ",\"c\" : [ 0 ]}" + " ]"; Table t = loadFromJson(json); FlattenedTable flattenedTable = flattener.flattenTable(t, null, "a[*]", UUID.randomUUID()); Assert.assertEquals(flattenedTable.getShards().size(), 1, "Expected correct table shard count"); TableShard tableShard = flattenedTable.getShards().iterator().next(); Assert.assertEquals(tableShard.getLowestRowId(), 0L, "Expected correct lowest row ID"); Assert.assertEquals(tableShard.getNumberOfRowsInShard(), 5, "Expected correct number of rows."); Assert.assertEquals(tableShard.getColumns().keySet(), new HashSet<>(Arrays.asList("a.b", "a.d[0]", "a.d[1]", "a.d[2]", "a.d[length]", "c[0]", "c[1]", "c[length]")), "Expected correct columns."); SortedSet<SortedMap<String, Long>> expectedRows = new TreeSet<>(MAP_COMPARATOR); SortedMap<String, Long> row = new TreeMap<>(); row.put("a.b", 1L); row.put("a.d[0]", 99L); row.put("a.d[1]", 100L); row.put("a.d[2]", LoaderColumnInfo.DEFAULT_LONG); row.put("a.d[length]", 2L); row.put("c[0]", 9L); row.put("c[1]", 10L); row.put("c[length]", 2L); expectedRows.add(row); row = new TreeMap<>(); row.put("a.b", 2L); row.put("a.d[0]", LoaderColumnInfo.DEFAULT_LONG); row.put("a.d[1]", LoaderColumnInfo.DEFAULT_LONG); row.put("a.d[2]", LoaderColumnInfo.DEFAULT_LONG); row.put("a.d[length]", 0L); row.put("c[0]", 9L); row.put("c[1]", 10L); row.put("c[length]", 2L); expectedRows.add(row); row = new TreeMap<>(); row.put("a.b", 3L); row.put("a.d[0]", 300L); row.put("a.d[1]", 301L); row.put("a.d[2]", 302L); row.put("a.d[length]", 3L); row.put("c[0]", 0L); row.put("c[1]", LoaderColumnInfo.DEFAULT_LONG); row.put("c[length]", 1L); expectedRows.add(row); row = new TreeMap<>(); row.put("a.b", 4L); row.put("a.d[0]", 303L); row.put("a.d[1]", 304L); row.put("a.d[2]", 305L); row.put("a.d[length]", 3L); row.put("c[0]", 0L); row.put("c[1]", LoaderColumnInfo.DEFAULT_LONG); row.put("c[length]", 1L); expectedRows.add(row); row = new TreeMap<>(); row.put("a.b", 5L); row.put("a.d[0]", 306L); row.put("a.d[1]", 307L); row.put("a.d[2]", 308L); row.put("a.d[length]", 3L); row.put("c[0]", 0L); row.put("c[1]", LoaderColumnInfo.DEFAULT_LONG); row.put("c[length]", 1L); expectedRows.add(row); Assert.assertEquals(getAllRows(tableShard), expectedRows, "Expected to have correct rows."); }
@Test public void repeatedMiddleMissingTest() throws LoadException { Table t = loadFromJson("[ { " + "\"a\": [ " + "{ \"b\": [ " + "{ \"c\": 1 }," + "{ \"d\": 1 }," + "{ \"c\": 3 }" + "] }, " + "{ \"b\": [ " + "{ \"c\": 2 }," + "{ \"d\": 2 }," + "{ \"c\": 4 }" + "] } " + "]" + "}," + "{ " + "\"a\": [ " + "{ \"b\": [ " + "{ \"c\": 5 }," + "{ \"d\": 5 }," + "{ \"c\": 6 }" + "] }, " + "{ \"b\": [ " + "{ \"c\": 7 }," + "{ \"d\": 7 }," + "{ \"c\": 8 }" + "] } " + "]" + "}" + " ]"); FlattenedTable flattenedTable = flattener.flattenTable(t, null, "a[*].b[*]", UUID.randomUUID()); Assert.assertEquals(flattenedTable.getShards().size(), 1, "Expected correct table shard count"); TableShard tableShard = flattenedTable.getShards().iterator().next(); Assert.assertEquals(tableShard.getLowestRowId(), 0L, "Expected correct lowest row ID"); Assert.assertEquals(tableShard.getNumberOfRowsInShard(), 12, "Expected correct number of rows."); Assert.assertEquals(tableShard.getColumns().keySet(), new HashSet<>(Arrays.asList("a.b.c", "a.b.d")), "Expected correct columns."); SortedSet<SortedMap<String, Long>> expectedRows = new TreeSet<>(MAP_COMPARATOR); SortedMap<String, Long> row = new TreeMap<>(); row.put("a.b.c", 1L); row.put("a.b.d", LoaderColumnInfo.DEFAULT_LONG); expectedRows.add(row); row = new TreeMap<>(); row.put("a.b.c", LoaderColumnInfo.DEFAULT_LONG); row.put("a.b.d", 1L); expectedRows.add(row); row = new TreeMap<>(); row.put("a.b.c", 3L); row.put("a.b.d", LoaderColumnInfo.DEFAULT_LONG); expectedRows.add(row); row = new TreeMap<>(); row.put("a.b.c", 2L); row.put("a.b.d", LoaderColumnInfo.DEFAULT_LONG); expectedRows.add(row); row = new TreeMap<>(); row.put("a.b.c", LoaderColumnInfo.DEFAULT_LONG); row.put("a.b.d", 2L); expectedRows.add(row); row = new TreeMap<>(); row.put("a.b.c", 4L); row.put("a.b.d", LoaderColumnInfo.DEFAULT_LONG); expectedRows.add(row); row = new TreeMap<>(); row.put("a.b.c", 5L); row.put("a.b.d", LoaderColumnInfo.DEFAULT_LONG); expectedRows.add(row); row = new TreeMap<>(); row.put("a.b.c", LoaderColumnInfo.DEFAULT_LONG); row.put("a.b.d", 5L); expectedRows.add(row); row = new TreeMap<>(); row.put("a.b.c", 6L); row.put("a.b.d", LoaderColumnInfo.DEFAULT_LONG); expectedRows.add(row); row = new TreeMap<>(); row.put("a.b.c", 7L); row.put("a.b.d", LoaderColumnInfo.DEFAULT_LONG); expectedRows.add(row); row = new TreeMap<>(); row.put("a.b.c", LoaderColumnInfo.DEFAULT_LONG); row.put("a.b.d", 7L); expectedRows.add(row); row = new TreeMap<>(); row.put("a.b.c", 8L); row.put("a.b.d", LoaderColumnInfo.DEFAULT_LONG); expectedRows.add(row); Assert.assertEquals(getAllRows(tableShard), expectedRows, "Expected to have correct rows."); }
@Test public void simpleTestHighFirstRowId() throws LoadException { String json = "[ { " + "\"a\": [ " + "{ \"b\": 1, \"d\":[99, 100] }, " + "{ \"b\": 2, \"d\":[] }" + "]" + ",\"c\" : [ 9, 10 ] }," + "{ " + "\"a\": [ " + "{ \"b\": 3, \"d\":[300,301,302] }, " + "{ \"b\": 4, \"d\":[303,304,305] }, " + "{ \"b\": 5, \"d\":[306,307,308] } " + "]" + ",\"c\" : [ 0 ]}" + " ]"; Table t = loadFromJson(100, json); FlattenedTable flattenedTable = flattener.flattenTable(t, null, "a[*]", UUID.randomUUID()); Assert.assertEquals(flattenedTable.getShards().size(), 1, "Expected correct table shard count"); TableShard tableShard = flattenedTable.getShards().iterator().next(); Assert.assertEquals(tableShard.getLowestRowId(), 100L, "Expected correct lowest row ID"); Assert.assertEquals(tableShard.getNumberOfRowsInShard(), 5, "Expected correct number of rows."); Assert.assertEquals(tableShard.getColumns().keySet(), new HashSet<>(Arrays.asList("a.b", "a.d[0]", "a.d[1]", "a.d[2]", "a.d[length]", "c[0]", "c[1]", "c[length]")), "Expected correct columns."); SortedSet<SortedMap<String, Long>> expectedRows = new TreeSet<>(MAP_COMPARATOR); SortedMap<String, Long> row = new TreeMap<>(); row.put("a.b", 1L); row.put("a.d[0]", 99L); row.put("a.d[1]", 100L); row.put("a.d[2]", LoaderColumnInfo.DEFAULT_LONG); row.put("a.d[length]", 2L); row.put("c[0]", 9L); row.put("c[1]", 10L); row.put("c[length]", 2L); expectedRows.add(row); row = new TreeMap<>(); row.put("a.b", 2L); row.put("a.d[0]", LoaderColumnInfo.DEFAULT_LONG); row.put("a.d[1]", LoaderColumnInfo.DEFAULT_LONG); row.put("a.d[2]", LoaderColumnInfo.DEFAULT_LONG); row.put("a.d[length]", 0L); row.put("c[0]", 9L); row.put("c[1]", 10L); row.put("c[length]", 2L); expectedRows.add(row); row = new TreeMap<>(); row.put("a.b", 3L); row.put("a.d[0]", 300L); row.put("a.d[1]", 301L); row.put("a.d[2]", 302L); row.put("a.d[length]", 3L); row.put("c[0]", 0L); row.put("c[1]", LoaderColumnInfo.DEFAULT_LONG); row.put("c[length]", 1L); expectedRows.add(row); row = new TreeMap<>(); row.put("a.b", 4L); row.put("a.d[0]", 303L); row.put("a.d[1]", 304L); row.put("a.d[2]", 305L); row.put("a.d[length]", 3L); row.put("c[0]", 0L); row.put("c[1]", LoaderColumnInfo.DEFAULT_LONG); row.put("c[length]", 1L); expectedRows.add(row); row = new TreeMap<>(); row.put("a.b", 5L); row.put("a.d[0]", 306L); row.put("a.d[1]", 307L); row.put("a.d[2]", 308L); row.put("a.d[length]", 3L); row.put("c[0]", 0L); row.put("c[1]", LoaderColumnInfo.DEFAULT_LONG); row.put("c[length]", 1L); expectedRows.add(row); Assert.assertEquals(getAllRows(tableShard), expectedRows, "Expected to have correct rows."); }
@Test public void simpleTestForceHalfRemoval() throws LoadException { String json = "[ { " + "\"a\": [ " + "{ \"b\": 1 }, " + "{ \"b\": 2 }" + "]}," + "{ \"a\": [ " + "{ \"b\": 3 }, " + "{ \"b\": 4 } " + "]}," + "{ \"a\": [ " + "{ \"b\": 5 }, " + "{ \"b\": 6 } " + "]}," + "{ \"a\": [ " + "{ \"b\": 7 }, " + "{ \"b\": 8 } " + "]}," + "{ \"a\": [ " + "{ \"b\": 9 }, " + "{ \"b\": 10 } " + "]}," + "{ \"a\": [ " + "{ \"b\": 11 } " + "]}," + "{ \"a\": [ " + "{ \"b\": 12 } " + "]}," + "{ \"a\": [ " + "{ \"b\": 13 } " + "]}," + "{ \"a\": [ " + "{ \"b\": 14 } " + "]}" + " ]"; Table t = loadFromJson(100, json); FlattenedTable flattenedTable = flattener.flattenTable(t, null, "a[*]", UUID.randomUUID()); Assert.assertEquals(flattenedTable.getShards().size(), 1, "Expected correct table shard count"); TableShard tableShard = flattenedTable.getShards().iterator().next(); Assert.assertEquals(tableShard.getLowestRowId(), 100L, "Expected correct lowest row ID"); Assert.assertEquals(tableShard.getNumberOfRowsInShard(), 14, "Expected correct number of rows."); Assert.assertEquals(tableShard.getColumns().keySet(), new HashSet<>(Arrays.asList("a.b")), "Expected correct columns."); SortedSet<SortedMap<String, Long>> expectedRows = new TreeSet<>(MAP_COMPARATOR); SortedMap<String, Long> row; for (long l = 1L; l <= 14; l++) { row = new TreeMap<>(); row.put("a.b", l); expectedRows.add(row); } Assert.assertEquals(getAllRows(tableShard), expectedRows, "Expected to have correct rows."); }
@Test public void simpleTestForceLittleRemoval() throws LoadException { String json = "[ { " + "\"a\": [ " + "{ \"b\": 1 }, " + "{ \"b\": 2 }" + "]}," + "{ \"a\": [ " + "{ \"b\": 3 }, " + "{ \"b\": 4 } " + "]}," + "{ \"a\": [ " + "{ \"b\": 5 }, " + "{ \"b\": 6 } " + "]}," + "{ \"a\": [ " + "{ \"b\": 7 }, " + "{ \"b\": 8 } " + "]}," + "{ \"a\": [ " + "{ \"b\": 9 }, " + "{ \"b\": 10 } " + "]}," + "{ \"a\": [ " + "{ \"b\": 11 }, " + "{ \"b\": 12 } " + "]}," + "{ \"a\": [ " + "{ \"b\": 13 }, " + "{ \"b\": 14 } " + "]}," + "{ \"a\": [ " + "{ \"b\": 15 }, " + "{ \"b\": 16 } " + "]}," + "{ \"a\": [ " + "{ \"b\": 17 } " + "]}" + " ]"; Table t = loadFromJson(100, json); FlattenedTable flattenedTable = flattener.flattenTable(t, null, "a[*]", UUID.randomUUID()); Assert.assertEquals(flattenedTable.getShards().size(), 1, "Expected correct table shard count"); TableShard tableShard = flattenedTable.getShards().iterator().next(); Assert.assertEquals(tableShard.getLowestRowId(), 100L, "Expected correct lowest row ID"); Assert.assertEquals(tableShard.getNumberOfRowsInShard(), 17, "Expected correct number of rows."); Assert.assertEquals(tableShard.getColumns().keySet(), new HashSet<>(Arrays.asList("a.b")), "Expected correct columns."); SortedSet<SortedMap<String, Long>> expectedRows = new TreeSet<>(MAP_COMPARATOR); SortedMap<String, Long> row; for (long l = 1L; l <= 17; l++) { row = new TreeMap<>(); row.put("a.b", l); expectedRows.add(row); } Assert.assertEquals(getAllRows(tableShard), expectedRows, "Expected to have correct rows."); }
@Test public void simpleTestForceMostRemoval() throws LoadException { String json = "[ { " + "\"a\": [ " + "{ \"b\": 1 }, " + "{ \"b\": 2 }" + "]}," + "{ \"a\": [ " + "{ \"b\": 3 } " + "]}," + "{ \"a\": [ " + "{ \"b\": 4 } " + "]}," + "{ \"a\": [ " + "{ \"b\": 5 } " + "]}," + "{ \"a\": [ " + "{ \"b\": 6 } " + "]}," + "{ \"a\": [ " + "{ \"b\": 7 } " + "]}," + "{ \"a\": [ " + "{ \"b\": 8 } " + "]}," + "{ \"a\": [ " + "{ \"b\": 9 } " + "]}," + "{ \"a\": [ " + "{ \"b\": 10 } " + "]}" + " ]"; Table t = loadFromJson(100, json); FlattenedTable flattenedTable = flattener.flattenTable(t, null, "a[*]", UUID.randomUUID()); Assert.assertEquals(flattenedTable.getShards().size(), 1, "Expected correct table shard count"); TableShard tableShard = flattenedTable.getShards().iterator().next(); Assert.assertEquals(tableShard.getLowestRowId(), 100L, "Expected correct lowest row ID"); Assert.assertEquals(tableShard.getNumberOfRowsInShard(), 10, "Expected correct number of rows."); Assert.assertEquals(tableShard.getColumns().keySet(), new HashSet<>(Arrays.asList("a.b")), "Expected correct columns."); SortedSet<SortedMap<String, Long>> expectedRows = new TreeSet<>(MAP_COMPARATOR); SortedMap<String, Long> row; for (long l = 1L; l <= 10; l++) { row = new TreeMap<>(); row.put("a.b", l); expectedRows.add(row); } Assert.assertEquals(getAllRows(tableShard), expectedRows, "Expected to have correct rows."); }