JsonLoader implements Loader { @Override public Collection<TableShard> load(long firstRowId, String filename, String tableName, LoaderColumnInfo columnInfo) throws LoadException { logger.info("Reading data for table '{}' from '{}'.", new Object[] { tableName, filename }); try (RandomAccessFile f = new RandomAccessFile(filename, "r")) { BigByteBuffer buf = new BigByteBuffer(f.getChannel(), MapMode.READ_ONLY, b -> b.load()); return load(firstRowId, buf, tableName, columnInfo); } catch (IOException e) { throw new LoadException("Could not load " + filename, e); } }  @Override Collection<TableShard> load(long firstRowId, String filename, String tableName, LoaderColumnInfo columnInfo); @Override Collection<TableShard> load(long firstRowId, BigByteBuffer jsonBuffer, String tableName,
      LoaderColumnInfo columnInfo); static final int BUCKET_SIZE; }
@Test public void simpleArrayJson() throws LoadException { String json = "[ { \"a\": 1, \"b\": 1, \"c\": [4, 5,6]},{\"a\": 2, \"b\": 3, \"c\": [7, 8, 9 ]}]"; TableShard tableShard = Iterables.getOnlyElement(loader.load(0L, new BigByteBuffer(json.getBytes()), TABLE, colInfo)); Assert.assertEquals(tableShard.getLongColumns().size(), 6, "Expected all long columns to be available"); Set<Triple<Long, Long, List<Long>>> expectedValues = new HashSet<>(); expectedValues.add(new Triple<>(1L, 1L, Arrays.asList(new Long[] { 4L, 5L, 6L }))); expectedValues.add(new Triple<>(2L, 3L, Arrays.asList(new Long[] { 7L, 8L, 9L }))); Set<Triple<Long, Long, List<Long>>> actualValues = new HashSet<>(); LongStandardColumnShard colA = tableShard.getLongColumns().get("a"); LongStandardColumnShard colB = tableShard.getLongColumns().get("b"); LongStandardColumnShard colC0 = tableShard.getLongColumns().get(repeatedColNames.repeatedAtIndex("c", 0)); LongStandardColumnShard colC1 = tableShard.getLongColumns().get(repeatedColNames.repeatedAtIndex("c", 1)); LongStandardColumnShard colC2 = tableShard.getLongColumns().get(repeatedColNames.repeatedAtIndex("c", 2)); LongStandardColumnShard colCLength = tableShard.getLongColumns().get(repeatedColNames.repeatedLength("c")); for (long i = tableShard.getLowestRowId(); i < tableShard.getLowestRowId() + tableShard.getNumberOfRowsInShard(); i++) { Long valueA = resolveSingleRowValue(colA, i); Long valueB = resolveSingleRowValue(colB, i); Long valueC0 = resolveSingleRowValue(colC0, i); Long valueC1 = resolveSingleRowValue(colC1, i); Long valueC2 = resolveSingleRowValue(colC2, i); actualValues.add(new Triple<>(valueA, valueB, Arrays.asList(new Long[] { valueC0, valueC1, valueC2 }))); Assert.assertEquals((long) resolveSingleRowValue(colCLength, i), 3L, "Correct value of length col expected for row " + i); } Assert.assertEquals(actualValues, expectedValues, "Expected correct values to be encoded"); }
@Test public void arrayContainingObjectsJson() throws LoadException { String json = "[ { \"a\": 1, \"c\": [{ \"d\" : 4, \"e\" : 4 }, { \"d\" : 5, \"e\" : 5 }]}," + "{\"a\": 2, \"c\": [ { \"d\" : 7, \"e\" : 7 }, { \"d\" : 8, \"e\" : 8 }] } ]"; TableShard tableShard = Iterables.getOnlyElement(loader.load(0L, new BigByteBuffer(json.getBytes()), TABLE, colInfo)); Assert.assertEquals(tableShard.getLongColumns().size(), 6, "Expected all long columns to be available"); Set<Triple<Long, List<Long>, List<Long>>> expectedValues = new HashSet<>(); expectedValues.add(new Triple<>(1L, Arrays.asList(new Long[] { 4L, 5L }), Arrays.asList(new Long[] { 4L, 5L }))); expectedValues.add(new Triple<>(2L, Arrays.asList(new Long[] { 7L, 8L }), Arrays.asList(new Long[] { 7L, 8L }))); Set<Triple<Long, List<Long>, List<Long>>> actualValues = new HashSet<>(); LongStandardColumnShard colA = tableShard.getLongColumns().get("a"); LongStandardColumnShard colC0D = tableShard.getLongColumns().get(repeatedColNames.repeatedAtIndex("c", 0) + ".d"); LongStandardColumnShard colC1D = tableShard.getLongColumns().get(repeatedColNames.repeatedAtIndex("c", 1) + ".d"); LongStandardColumnShard colC0E = tableShard.getLongColumns().get(repeatedColNames.repeatedAtIndex("c", 0) + ".e"); LongStandardColumnShard colC1E = tableShard.getLongColumns().get(repeatedColNames.repeatedAtIndex("c", 1) + ".e"); LongStandardColumnShard colCLength = tableShard.getLongColumns().get(repeatedColNames.repeatedLength("c")); for (long i = tableShard.getLowestRowId(); i < tableShard.getLowestRowId() + tableShard.getNumberOfRowsInShard(); i++) { Long valueA = resolveSingleRowValue(colA, i); Long valueC0D = resolveSingleRowValue(colC0D, i); Long valueC1D = resolveSingleRowValue(colC1D, i); Long valueC0E = resolveSingleRowValue(colC0E, i); Long valueC1E = resolveSingleRowValue(colC1E, i); actualValues.add(new Triple<>(valueA, Arrays.asList(new Long[] { valueC0D, valueC1D }), Arrays.asList(new Long[] { valueC0E, valueC1E }))); Assert.assertEquals((long) resolveSingleRowValue(colCLength, i), 2L, "Correct value of length col expected for row " + i); } Assert.assertEquals(actualValues, expectedValues, "Expected correct values to be encoded"); }
@Test public void arrayContainingDifferentLengthLongsJson() throws LoadException { String json = "[ { \"a\": 1, \"c\": [4, 5, 6]}," + "{\"a\": 2, \"c\": [ 1 ] } ]"; TableShard tableShard = Iterables.getOnlyElement(loader.load(0L, new BigByteBuffer(json.getBytes()), TABLE, colInfo)); Assert.assertEquals(tableShard.getLongColumns().size(), 5, "Expected all long columns to be available"); Set<Pair<Long, List<Long>>> expectedValues = new HashSet<>(); expectedValues.add(new Pair<>(1L, Arrays.asList(new Long[] { 4L, 5L, 6L }))); expectedValues.add(new Pair<>(2L, Arrays.asList(new Long[] { 1L }))); Set<Pair<Long, List<Long>>> actualValues = new HashSet<>(); LongStandardColumnShard colA = tableShard.getLongColumns().get("a"); LongStandardColumnShard[] colC = new LongStandardColumnShard[] { tableShard.getLongColumns().get(repeatedColNames.repeatedAtIndex("c", 0)), tableShard.getLongColumns().get(repeatedColNames.repeatedAtIndex("c", 1)), tableShard.getLongColumns().get(repeatedColNames.repeatedAtIndex("c", 2)) }; LongStandardColumnShard colCLength = tableShard.getLongColumns().get(repeatedColNames.repeatedLength("c")); for (long i = tableShard.getLowestRowId(); i < tableShard.getLowestRowId() + tableShard.getNumberOfRowsInShard(); i++) { Long valueA = resolveSingleRowValue(colA, i); List<Long> valueC = new ArrayList<>(); for (int c = 0; c < resolveSingleRowValue(colCLength, i); c++) valueC.add(resolveSingleRowValue(colC[c], i)); actualValues.add(new Pair<>(valueA, valueC)); } Assert.assertEquals(actualValues, expectedValues, "Expected correct values to be encoded"); }
@Test public void arrayContainingDifferentLengthObjectsJson() throws LoadException { String json = "[ { \"a\": 1, \"c\": [ { \"d\": 4 }, { \"d\": 5}, { \"d\": 6 } ]}," + "{\"a\": 2, \"c\": [ { \"d\": 1 } ] } ]"; TableShard tableShard = Iterables.getOnlyElement(loader.load(0L, new BigByteBuffer(json.getBytes()), TABLE, colInfo)); Assert.assertEquals(tableShard.getLongColumns().size(), 5, "Expected all long columns to be available"); Set<Pair<Long, List<Long>>> expectedValues = new HashSet<>(); expectedValues.add(new Pair<>(1L, Arrays.asList(new Long[] { 4L, 5L, 6L }))); expectedValues.add(new Pair<>(2L, Arrays.asList(new Long[] { 1L }))); Set<Pair<Long, List<Long>>> actualValues = new HashSet<>(); LongStandardColumnShard colA = tableShard.getLongColumns().get("a"); LongStandardColumnShard[] colD = new LongStandardColumnShard[] { tableShard.getLongColumns().get(repeatedColNames.repeatedAtIndex("c", 0) + ".d"), tableShard.getLongColumns().get(repeatedColNames.repeatedAtIndex("c", 1) + ".d"), tableShard.getLongColumns().get(repeatedColNames.repeatedAtIndex("c", 2) + ".d") }; LongStandardColumnShard colCLength = tableShard.getLongColumns().get(repeatedColNames.repeatedLength("c")); for (long i = tableShard.getLowestRowId(); i < tableShard.getLowestRowId() + tableShard.getNumberOfRowsInShard(); i++) { Long valueA = resolveSingleRowValue(colA, i); List<Long> valueD = new ArrayList<>(); for (int c = 0; c < resolveSingleRowValue(colCLength, i); c++) valueD.add(resolveSingleRowValue(colD[c], i)); actualValues.add(new Pair<>(valueA, valueD)); } Assert.assertEquals(actualValues, expectedValues, "Expected correct values to be encoded"); }
@Test public void longJson() throws LoadException { String json = "[ { \"a\": 0,\"b\": 30 }," + "{ \"a\": 1,\"b\": 23 }," + "{ \"a\": 2,\"b\": 55 }," + "{ \"a\": 3,\"b\": 26 }," + "{ \"a\": 4,\"b\": 39 }," + "{ \"a\": 5,\"b\": 36 }," + "{ \"a\": 6,\"b\": 55 }," + "{ \"a\": 7,\"b\": 39 }," + "{ \"a\": 8,\"b\": 10 }," + "{ \"a\": 9,\"b\": 25 }]"; TableShard tableShard = Iterables.getOnlyElement(loader.load(0L, new BigByteBuffer(json.getBytes()), TABLE, colInfo)); Assert.assertEquals(tableShard.getLongColumns().size(), 2, "Expected both long columns to be available"); Set<Pair<Long, Long>> expectedValues = new HashSet<>(); expectedValues.add(new Pair<>(0L, 30L)); expectedValues.add(new Pair<>(1L, 23L)); expectedValues.add(new Pair<>(2L, 55L)); expectedValues.add(new Pair<>(3L, 26L)); expectedValues.add(new Pair<>(4L, 39L)); expectedValues.add(new Pair<>(5L, 36L)); expectedValues.add(new Pair<>(6L, 55L)); expectedValues.add(new Pair<>(7L, 39L)); expectedValues.add(new Pair<>(8L, 10L)); expectedValues.add(new Pair<>(9L, 25L)); Set<Pair<Long, Long>> actualValues = new HashSet<>(); LongStandardColumnShard colA = tableShard.getLongColumns().get("a"); LongStandardColumnShard colB = tableShard.getLongColumns().get("b"); for (long i = tableShard.getLowestRowId(); i < tableShard.getLowestRowId() + tableShard.getNumberOfRowsInShard(); i++) { Long valueA = resolveSingleRowValue(colA, i); Long valueB = resolveSingleRowValue(colB, i); actualValues.add(new Pair<>(valueA, valueB)); } Assert.assertEquals(actualValues, expectedValues, "Expected correct values to be encoded"); }
@Test(expectedExceptions = LoadException.class) public void unparsableJson() throws LoadException { String json = "[ { \"a\": 0,\"b\": " + Long.MAX_VALUE + "9 }]"; loader.load(0L, new BigByteBuffer(json.getBytes()), TABLE, colInfo); }
@Test public void simpleJson() throws LoadException { String json = "[ { \"a\": 1, \"b\": 1},{\"a\": 2, \"b\": 3}]"; TableShard tableShard = Iterables.getOnlyElement(loader.load(0L, new BigByteBuffer(json.getBytes()), TABLE, colInfo)); Assert.assertEquals(tableShard.getLongColumns().size(), 2, "Expected both long columns to be available"); Set<Pair<Long, Long>> expectedValues = new HashSet<>(); expectedValues.add(new Pair<>(1L, 1L)); expectedValues.add(new Pair<>(2L, 3L)); Set<Pair<Long, Long>> actualValues = new HashSet<>(); LongStandardColumnShard colA = tableShard.getLongColumns().get("a"); LongStandardColumnShard colB = tableShard.getLongColumns().get("b"); for (long i = tableShard.getLowestRowId(); i < tableShard.getLowestRowId() + tableShard.getNumberOfRowsInShard(); i++) { Long valueA = resolveSingleRowValue(colA, i); Long valueB = resolveSingleRowValue(colB, i); actualValues.add(new Pair<>(valueA, valueB)); } Assert.assertEquals(actualValues, expectedValues, "Expected correct values to be encoded"); }