@Test public void testTarNoCompressionImportCreateDataset() throws Exception { removeData(); command.targets = Lists.newArrayList(TAR_TEST_FILE, datasetUri); assertEquals(0, command.run()); verify(console).info("Using {} compression", TarImportCommand.CompressionType.NONE); verify(console).info("Creating new dataset: {}", datasetUri); verify(console).info("Added {} records to \"{}\"", NUM_TEST_FILES, TEST_DATASET_NAME); }
@Override public int run() throws IOException { Preconditions.checkArgument(targets != null && targets.size() == 2, "Tar path and target dataset URI are required."); Preconditions.checkArgument( SUPPORTED_TAR_COMPRESSION_TYPES.contains(compressionType), "Compression type " + compressionType + " is not supported"); String source = targets.get(0); String datasetUri = targets.get(1); long blockSize = getConf().getLong("dfs.blocksize", DEFAULT_BLOCK_SIZE); int success = 0; View<TarFileEntry> targetDataset; if (Datasets.exists(datasetUri)) { console.debug("Using existing dataset: {}", datasetUri); targetDataset = Datasets.load(datasetUri, TarFileEntry.class); } else { console.info("Creating new dataset: {}", datasetUri); DatasetDescriptor.Builder descriptorBuilder = new DatasetDescriptor.Builder(); descriptorBuilder.format(Formats.AVRO); descriptorBuilder.schema(TarFileEntry.class); targetDataset = Datasets.create(datasetUri, descriptorBuilder.build(), TarFileEntry.class); } DatasetWriter<TarFileEntry> writer = targetDataset.newWriter(); TarArchiveInputStream tis; CompressionType tarCompressionType = CompressionType.NONE; if (compressionType.isEmpty()) { if (source.endsWith(".tar")) { tarCompressionType = CompressionType.NONE; } else if (source.endsWith(".tar.gz")) { tarCompressionType = CompressionType.GZIP; } else if (source.endsWith(".tar.bz2")) { tarCompressionType = CompressionType.BZIP2; } } else if (compressionType.equals("gzip")) { tarCompressionType = CompressionType.GZIP; } else if (compressionType.equals("bzip2")) { tarCompressionType = CompressionType.BZIP2; } else { tarCompressionType = CompressionType.NONE; } console.info("Using {} compression", tarCompressionType); switch (tarCompressionType) { case GZIP: tis = new TarArchiveInputStream( new GzipCompressorInputStream(open(source))); break; case BZIP2: tis = new TarArchiveInputStream( new BZip2CompressorInputStream(open(source))); break; case NONE: default: tis = new TarArchiveInputStream(open(source)); } TarArchiveEntry entry; try { int count = 0; while ((entry = tis.getNextTarEntry()) != null) { if (!entry.isDirectory()) { long size = entry.getSize(); if (size >= blockSize) { console.warn("Entry \"{}\" (size {}) is larger than the " + "HDFS block size of {}. This may result in remote block reads", new Object[] { entry.getName(), size, blockSize }); } byte[] buf = new byte[(int) size]; try { IOUtils.readFully(tis, buf, 0, (int) size); } catch (IOException e) { console.error("Did not read entry {} successfully (entry size {})", entry.getName(), size); success = 1; throw e; } writer.write( TarFileEntry.newBuilder().setFilename(entry.getName()) .setFilecontent(ByteBuffer.wrap(buf)).build() ); count++; } } console.info("Added {} records to \"{}\"", count, targetDataset.getDataset().getName()); } finally { IOUtils.closeStream(writer); IOUtils.closeStream(tis); } return success; }
TarImportCommand extends BaseDatasetCommand { @Override public int run() throws IOException { Preconditions.checkArgument(targets != null && targets.size() == 2, "Tar path and target dataset URI are required."); Preconditions.checkArgument( SUPPORTED_TAR_COMPRESSION_TYPES.contains(compressionType), "Compression type " + compressionType + " is not supported"); String source = targets.get(0); String datasetUri = targets.get(1); long blockSize = getConf().getLong("dfs.blocksize", DEFAULT_BLOCK_SIZE); int success = 0; View<TarFileEntry> targetDataset; if (Datasets.exists(datasetUri)) { console.debug("Using existing dataset: {}", datasetUri); targetDataset = Datasets.load(datasetUri, TarFileEntry.class); } else { console.info("Creating new dataset: {}", datasetUri); DatasetDescriptor.Builder descriptorBuilder = new DatasetDescriptor.Builder(); descriptorBuilder.format(Formats.AVRO); descriptorBuilder.schema(TarFileEntry.class); targetDataset = Datasets.create(datasetUri, descriptorBuilder.build(), TarFileEntry.class); } DatasetWriter<TarFileEntry> writer = targetDataset.newWriter(); TarArchiveInputStream tis; CompressionType tarCompressionType = CompressionType.NONE; if (compressionType.isEmpty()) { if (source.endsWith(".tar")) { tarCompressionType = CompressionType.NONE; } else if (source.endsWith(".tar.gz")) { tarCompressionType = CompressionType.GZIP; } else if (source.endsWith(".tar.bz2")) { tarCompressionType = CompressionType.BZIP2; } } else if (compressionType.equals("gzip")) { tarCompressionType = CompressionType.GZIP; } else if (compressionType.equals("bzip2")) { tarCompressionType = CompressionType.BZIP2; } else { tarCompressionType = CompressionType.NONE; } console.info("Using {} compression", tarCompressionType); switch (tarCompressionType) { case GZIP: tis = new TarArchiveInputStream( new GzipCompressorInputStream(open(source))); break; case BZIP2: tis = new TarArchiveInputStream( new BZip2CompressorInputStream(open(source))); break; case NONE: default: tis = new TarArchiveInputStream(open(source)); } TarArchiveEntry entry; try { int count = 0; while ((entry = tis.getNextTarEntry()) != null) { if (!entry.isDirectory()) { long size = entry.getSize(); if (size >= blockSize) { console.warn("Entry \"{}\" (size {}) is larger than the " + "HDFS block size of {}. This may result in remote block reads", new Object[] { entry.getName(), size, blockSize }); } byte[] buf = new byte[(int) size]; try { IOUtils.readFully(tis, buf, 0, (int) size); } catch (IOException e) { console.error("Did not read entry {} successfully (entry size {})", entry.getName(), size); success = 1; throw e; } writer.write( TarFileEntry.newBuilder().setFilename(entry.getName()) .setFilecontent(ByteBuffer.wrap(buf)).build() ); count++; } } console.info("Added {} records to \"{}\"", count, targetDataset.getDataset().getName()); } finally { IOUtils.closeStream(writer); IOUtils.closeStream(tis); } return success; } }
TarImportCommand extends BaseDatasetCommand { @Override public int run() throws IOException { Preconditions.checkArgument(targets != null && targets.size() == 2, "Tar path and target dataset URI are required."); Preconditions.checkArgument( SUPPORTED_TAR_COMPRESSION_TYPES.contains(compressionType), "Compression type " + compressionType + " is not supported"); String source = targets.get(0); String datasetUri = targets.get(1); long blockSize = getConf().getLong("dfs.blocksize", DEFAULT_BLOCK_SIZE); int success = 0; View<TarFileEntry> targetDataset; if (Datasets.exists(datasetUri)) { console.debug("Using existing dataset: {}", datasetUri); targetDataset = Datasets.load(datasetUri, TarFileEntry.class); } else { console.info("Creating new dataset: {}", datasetUri); DatasetDescriptor.Builder descriptorBuilder = new DatasetDescriptor.Builder(); descriptorBuilder.format(Formats.AVRO); descriptorBuilder.schema(TarFileEntry.class); targetDataset = Datasets.create(datasetUri, descriptorBuilder.build(), TarFileEntry.class); } DatasetWriter<TarFileEntry> writer = targetDataset.newWriter(); TarArchiveInputStream tis; CompressionType tarCompressionType = CompressionType.NONE; if (compressionType.isEmpty()) { if (source.endsWith(".tar")) { tarCompressionType = CompressionType.NONE; } else if (source.endsWith(".tar.gz")) { tarCompressionType = CompressionType.GZIP; } else if (source.endsWith(".tar.bz2")) { tarCompressionType = CompressionType.BZIP2; } } else if (compressionType.equals("gzip")) { tarCompressionType = CompressionType.GZIP; } else if (compressionType.equals("bzip2")) { tarCompressionType = CompressionType.BZIP2; } else { tarCompressionType = CompressionType.NONE; } console.info("Using {} compression", tarCompressionType); switch (tarCompressionType) { case GZIP: tis = new TarArchiveInputStream( new GzipCompressorInputStream(open(source))); break; case BZIP2: tis = new TarArchiveInputStream( new BZip2CompressorInputStream(open(source))); break; case NONE: default: tis = new TarArchiveInputStream(open(source)); } TarArchiveEntry entry; try { int count = 0; while ((entry = tis.getNextTarEntry()) != null) { if (!entry.isDirectory()) { long size = entry.getSize(); if (size >= blockSize) { console.warn("Entry \"{}\" (size {}) is larger than the " + "HDFS block size of {}. This may result in remote block reads", new Object[] { entry.getName(), size, blockSize }); } byte[] buf = new byte[(int) size]; try { IOUtils.readFully(tis, buf, 0, (int) size); } catch (IOException e) { console.error("Did not read entry {} successfully (entry size {})", entry.getName(), size); success = 1; throw e; } writer.write( TarFileEntry.newBuilder().setFilename(entry.getName()) .setFilecontent(ByteBuffer.wrap(buf)).build() ); count++; } } console.info("Added {} records to \"{}\"", count, targetDataset.getDataset().getName()); } finally { IOUtils.closeStream(writer); IOUtils.closeStream(tis); } return success; } TarImportCommand(Logger console); }
TarImportCommand extends BaseDatasetCommand { @Override public int run() throws IOException { Preconditions.checkArgument(targets != null && targets.size() == 2, "Tar path and target dataset URI are required."); Preconditions.checkArgument( SUPPORTED_TAR_COMPRESSION_TYPES.contains(compressionType), "Compression type " + compressionType + " is not supported"); String source = targets.get(0); String datasetUri = targets.get(1); long blockSize = getConf().getLong("dfs.blocksize", DEFAULT_BLOCK_SIZE); int success = 0; View<TarFileEntry> targetDataset; if (Datasets.exists(datasetUri)) { console.debug("Using existing dataset: {}", datasetUri); targetDataset = Datasets.load(datasetUri, TarFileEntry.class); } else { console.info("Creating new dataset: {}", datasetUri); DatasetDescriptor.Builder descriptorBuilder = new DatasetDescriptor.Builder(); descriptorBuilder.format(Formats.AVRO); descriptorBuilder.schema(TarFileEntry.class); targetDataset = Datasets.create(datasetUri, descriptorBuilder.build(), TarFileEntry.class); } DatasetWriter<TarFileEntry> writer = targetDataset.newWriter(); TarArchiveInputStream tis; CompressionType tarCompressionType = CompressionType.NONE; if (compressionType.isEmpty()) { if (source.endsWith(".tar")) { tarCompressionType = CompressionType.NONE; } else if (source.endsWith(".tar.gz")) { tarCompressionType = CompressionType.GZIP; } else if (source.endsWith(".tar.bz2")) { tarCompressionType = CompressionType.BZIP2; } } else if (compressionType.equals("gzip")) { tarCompressionType = CompressionType.GZIP; } else if (compressionType.equals("bzip2")) { tarCompressionType = CompressionType.BZIP2; } else { tarCompressionType = CompressionType.NONE; } console.info("Using {} compression", tarCompressionType); switch (tarCompressionType) { case GZIP: tis = new TarArchiveInputStream( new GzipCompressorInputStream(open(source))); break; case BZIP2: tis = new TarArchiveInputStream( new BZip2CompressorInputStream(open(source))); break; case NONE: default: tis = new TarArchiveInputStream(open(source)); } TarArchiveEntry entry; try { int count = 0; while ((entry = tis.getNextTarEntry()) != null) { if (!entry.isDirectory()) { long size = entry.getSize(); if (size >= blockSize) { console.warn("Entry \"{}\" (size {}) is larger than the " + "HDFS block size of {}. This may result in remote block reads", new Object[] { entry.getName(), size, blockSize }); } byte[] buf = new byte[(int) size]; try { IOUtils.readFully(tis, buf, 0, (int) size); } catch (IOException e) { console.error("Did not read entry {} successfully (entry size {})", entry.getName(), size); success = 1; throw e; } writer.write( TarFileEntry.newBuilder().setFilename(entry.getName()) .setFilecontent(ByteBuffer.wrap(buf)).build() ); count++; } } console.info("Added {} records to \"{}\"", count, targetDataset.getDataset().getName()); } finally { IOUtils.closeStream(writer); IOUtils.closeStream(tis); } return success; } TarImportCommand(Logger console); @Override int run(); @Override List<String> getExamples(); }
TarImportCommand extends BaseDatasetCommand { @Override public int run() throws IOException { Preconditions.checkArgument(targets != null && targets.size() == 2, "Tar path and target dataset URI are required."); Preconditions.checkArgument( SUPPORTED_TAR_COMPRESSION_TYPES.contains(compressionType), "Compression type " + compressionType + " is not supported"); String source = targets.get(0); String datasetUri = targets.get(1); long blockSize = getConf().getLong("dfs.blocksize", DEFAULT_BLOCK_SIZE); int success = 0; View<TarFileEntry> targetDataset; if (Datasets.exists(datasetUri)) { console.debug("Using existing dataset: {}", datasetUri); targetDataset = Datasets.load(datasetUri, TarFileEntry.class); } else { console.info("Creating new dataset: {}", datasetUri); DatasetDescriptor.Builder descriptorBuilder = new DatasetDescriptor.Builder(); descriptorBuilder.format(Formats.AVRO); descriptorBuilder.schema(TarFileEntry.class); targetDataset = Datasets.create(datasetUri, descriptorBuilder.build(), TarFileEntry.class); } DatasetWriter<TarFileEntry> writer = targetDataset.newWriter(); TarArchiveInputStream tis; CompressionType tarCompressionType = CompressionType.NONE; if (compressionType.isEmpty()) { if (source.endsWith(".tar")) { tarCompressionType = CompressionType.NONE; } else if (source.endsWith(".tar.gz")) { tarCompressionType = CompressionType.GZIP; } else if (source.endsWith(".tar.bz2")) { tarCompressionType = CompressionType.BZIP2; } } else if (compressionType.equals("gzip")) { tarCompressionType = CompressionType.GZIP; } else if (compressionType.equals("bzip2")) { tarCompressionType = CompressionType.BZIP2; } else { tarCompressionType = CompressionType.NONE; } console.info("Using {} compression", tarCompressionType); switch (tarCompressionType) { case GZIP: tis = new TarArchiveInputStream( new GzipCompressorInputStream(open(source))); break; case BZIP2: tis = new TarArchiveInputStream( new BZip2CompressorInputStream(open(source))); break; case NONE: default: tis = new TarArchiveInputStream(open(source)); } TarArchiveEntry entry; try { int count = 0; while ((entry = tis.getNextTarEntry()) != null) { if (!entry.isDirectory()) { long size = entry.getSize(); if (size >= blockSize) { console.warn("Entry \"{}\" (size {}) is larger than the " + "HDFS block size of {}. This may result in remote block reads", new Object[] { entry.getName(), size, blockSize }); } byte[] buf = new byte[(int) size]; try { IOUtils.readFully(tis, buf, 0, (int) size); } catch (IOException e) { console.error("Did not read entry {} successfully (entry size {})", entry.getName(), size); success = 1; throw e; } writer.write( TarFileEntry.newBuilder().setFilename(entry.getName()) .setFilecontent(ByteBuffer.wrap(buf)).build() ); count++; } } console.info("Added {} records to \"{}\"", count, targetDataset.getDataset().getName()); } finally { IOUtils.closeStream(writer); IOUtils.closeStream(tis); } return success; } TarImportCommand(Logger console); @Override int run(); @Override List<String> getExamples();  }