@Test public void endToEndTest() throws IOException { HoodieWriteConfig cfg = SparkDatasetTestUtils.getConfigBuilder(basePath).build(); for (int i = 0; i < 5; i++) { HoodieRowParquetWriteSupport writeSupport = getWriteSupport(cfg, hadoopConf); HoodieRowParquetConfig parquetConfig = new HoodieRowParquetConfig(writeSupport, CompressionCodecName.SNAPPY, cfg.getParquetBlockSize(), cfg.getParquetPageSize(), cfg.getParquetMaxFileSize(), writeSupport.getHadoopConf(), cfg.getParquetCompressionRatio()); String fileId = UUID.randomUUID().toString(); Path filePath = new Path(basePath + "/" + fileId); String partitionPath = HoodieTestDataGenerator.DEFAULT_FIRST_PARTITION_PATH; metaClient.getFs().mkdirs(new Path(basePath)); HoodieInternalRowParquetWriter writer = new HoodieInternalRowParquetWriter(filePath, parquetConfig); int size = 10 + RANDOM.nextInt(100); Dataset<Row> inputRows = SparkDatasetTestUtils.getRandomRows(sqlContext, size, partitionPath, false); List<InternalRow> internalRows = SparkDatasetTestUtils.toInternalRows(inputRows, SparkDatasetTestUtils.ENCODER); for (InternalRow internalRow : internalRows) { writer.write(internalRow); } writer.close(); Dataset<Row> result = sqlContext.read().parquet(basePath); assertEquals(0, inputRows.except(result).count()); } }
@Override public void close() throws IOException { super.close(); }
HoodieInternalRowParquetWriter extends ParquetWriter<InternalRow> implements HoodieInternalRowFileWriter { @Override public void close() throws IOException { super.close(); } }
HoodieInternalRowParquetWriter extends ParquetWriter<InternalRow> implements HoodieInternalRowFileWriter { @Override public void close() throws IOException { super.close(); } HoodieInternalRowParquetWriter(Path file, HoodieRowParquetConfig parquetConfig); }
HoodieInternalRowParquetWriter extends ParquetWriter<InternalRow> implements HoodieInternalRowFileWriter { @Override public void close() throws IOException { super.close(); } HoodieInternalRowParquetWriter(Path file, HoodieRowParquetConfig parquetConfig); @Override boolean canWrite(); @Override void writeRow(String key, InternalRow row); @Override void close(); }
HoodieInternalRowParquetWriter extends ParquetWriter<InternalRow> implements HoodieInternalRowFileWriter { @Override public void close() throws IOException { super.close(); } HoodieInternalRowParquetWriter(Path file, HoodieRowParquetConfig parquetConfig); @Override boolean canWrite(); @Override void writeRow(String key, InternalRow row); @Override void close();  }