CsvLoader implements Loader { @Override public Collection<TableShard> load(long firstRowId, String filename, String tableName, LoaderColumnInfo columnInfo) throws LoadException { ColumnShardBuilderManager columnManager; logger.info("Reading data for new table '{}' from '{}'.", new Object[] { tableName, filename }); try (RandomAccessFile f = new RandomAccessFile(filename, "r")) { BigByteBuffer buf = new BigByteBuffer(f.getChannel(), MapMode.READ_ONLY, b -> b.load()); columnManager = readColumnData(firstRowId, buf, tableName, columnInfo); buf = null; } catch (IOException e) { throw new LoadException("Could not load " + filename, e); } return createTableShard(columnManager, tableName); }  @Override Collection<TableShard> load(long firstRowId, String filename, String tableName, LoaderColumnInfo columnInfo); @Override Collection<TableShard> load(long firstRowId, BigByteBuffer csvBuffer, String tableName,
      LoaderColumnInfo columnInfo); static String[] parseCsvLine(String line);  }
@Test public void smallSimpleCsv() throws LoadException { BigByteBuffer buf = IoUtils.inputStreamToBigByteBuffer(getClass().getResourceAsStream(CSV_SIMPLE_CLASSPATH)); colInfo.registerColumnType(CSV_COL_B, ColumnType.LONG); colInfo.registerColumnType(CSV_COL_C, ColumnType.DOUBLE); TableShard table = Iterables.getOnlyElement(csvLoader.load(0L, buf, "Test", colInfo)); Assert.assertEquals(table.getNumberOfRowsInShard(), 4, "Expected 4 rows"); Assert.assertNotNull(table.getStringColumns().get(CSV_COL_A), "Expected String column"); Assert.assertNotNull(table.getLongColumns().get(CSV_COL_B), "Expected Long column"); Assert.assertNotNull(table.getDoubleColumns().get(CSV_COL_C), "Expected Double column"); @SuppressWarnings("unchecked") List<String> colAValues = resolveAllValues(table.getStringColumns().get(CSV_COL_A)); @SuppressWarnings("unchecked") List<Long> colBValues = resolveAllValues(table.getLongColumns().get(CSV_COL_B)); @SuppressWarnings("unchecked") List<Double> colCValues = resolveAllValues(table.getDoubleColumns().get(CSV_COL_C)); Assert.assertEquals(new HashSet<String>(colAValues).size(), colAValues.size(), "Duplicate values in column not expected"); Assert.assertEquals(new HashSet<Long>(colBValues).size(), colBValues.size(), "Duplicate values in column not expected"); Assert.assertEquals(new HashSet<Double>(colCValues).size(), colCValues.size(), "Duplicate values in column not expected"); Assert.assertEquals(new HashSet<String>(colAValues), new HashSet<String>(Arrays.asList(new String[] { "1", "2", "3", "4" })), "Different values expected (inspect data type!)"); Assert.assertEquals(new HashSet<Long>(colBValues), new HashSet<Long>(Arrays.asList(new Long[] { 1L, 2L, 3L, 4L })), "Different values expected (inspect data type!)"); Assert.assertEquals(new HashSet<Double>(colCValues), new HashSet<Double>(Arrays.asList(new Double[] { 1., 2., 3., 4. })), "Different values expected (inspect data type!)"); }
@Test public void longCsvTestProposalRows() throws LoadException { int rows = ColumnShardBuilder.PROPOSAL_ROWS; BigByteBuffer buf = generateCsvOneColumn(CSV_COL_A, rows); colInfo.registerColumnType(CSV_COL_A, ColumnType.LONG); TableShard shard = Iterables.getOnlyElement(csvLoader.load(0L, buf, "Test", colInfo)); Assert.assertEquals(shard.getNumberOfRowsInShard(), rows, "Expected " + rows + " rows"); @SuppressWarnings("unchecked") List<Long> colAValues = resolveAllValues(shard.getLongColumns().get(CSV_COL_A)); Assert.assertEquals(new HashSet<>(colAValues), generateLongSetRange(0, rows), "Expected correct values"); Assert.assertEquals(shard.getLongColumns().get(CSV_COL_A).getPages().size(), 1, "Only one ColumnPage expected"); }
@Test public void longCsvTestProposalRowsPlusOne() throws LoadException { int rows = ColumnShardBuilder.PROPOSAL_ROWS + 1; BigByteBuffer buf = generateCsvOneColumn(CSV_COL_A, rows); colInfo.registerColumnType(CSV_COL_A, ColumnType.LONG); TableShard shard = Iterables.getOnlyElement(csvLoader.load(0L, buf, "Test", colInfo)); Assert.assertEquals(shard.getNumberOfRowsInShard(), rows, "Expected " + rows + " rows"); @SuppressWarnings("unchecked") List<Long> colAValues = resolveAllValues(shard.getLongColumns().get(CSV_COL_A)); Assert.assertEquals(new HashSet<>(colAValues), generateLongSetRange(0, rows), "Expected correct values"); Assert.assertEquals(shard.getLongColumns().get(CSV_COL_A).getPages().size(), 2, "Two ColumnPages expected"); }
@Test(expectedExceptions = LoadException.class) public void unparsableCsv() throws LoadException { String csv = CSV_COL_A + "," + CSV_COL_B + "\n" + "1," + Long.MAX_VALUE + "9\n"; colInfo.registerColumnType(CSV_COL_A, ColumnType.LONG); colInfo.registerColumnType(CSV_COL_B, ColumnType.LONG); csvLoader.load(0L, new BigByteBuffer(csv.getBytes()), "Test", colInfo); }