@Test public void testSupportsGzip() throws IOException { properties.setProp(ConfigurationKeys.WRITER_CODEC_TYPE, "gzip"); properties.setProp(ConfigurationKeys.SIMPLE_WRITER_DELIMITER, ""); byte[] toWrite = new byte[] { 'a', 'b', 'c', 'd'}; SimpleDataWriter writer = buildSimpleDataWriter(); writer.write(toWrite); writer.close(); writer.commit(); File outputFile = new File(writer.getOutputFilePath()); InputStream in = new GZIPInputStream(new FileInputStream(outputFile)); byte[] contents = IOUtils.toByteArray(in); Assert.assertEquals(contents, toWrite, "Expected gzip'd content to be written out"); Assert.assertTrue(outputFile.getName().endsWith(".gzip"), "Expected gzip'd file to end in .gzip"); }
@Override public void write(byte[] record) throws IOException { Preconditions.checkNotNull(record); byte[] toWrite = record; if (this.recordDelimiter.isPresent()) { toWrite = Arrays.copyOf(record, record.length + 1); toWrite[toWrite.length - 1] = this.recordDelimiter.get(); } if (this.prependSize) { long recordSize = toWrite.length; ByteBuffer buf = ByteBuffer.allocate(Longs.BYTES); buf.putLong(recordSize); toWrite = ArrayUtils.addAll(buf.array(), toWrite); } this.stagingFileOutputStream.write(toWrite); this.bytesWritten += toWrite.length; this.recordsWritten++; }
SimpleDataWriter extends FsDataWriter<byte[]> { @Override public void write(byte[] record) throws IOException { Preconditions.checkNotNull(record); byte[] toWrite = record; if (this.recordDelimiter.isPresent()) { toWrite = Arrays.copyOf(record, record.length + 1); toWrite[toWrite.length - 1] = this.recordDelimiter.get(); } if (this.prependSize) { long recordSize = toWrite.length; ByteBuffer buf = ByteBuffer.allocate(Longs.BYTES); buf.putLong(recordSize); toWrite = ArrayUtils.addAll(buf.array(), toWrite); } this.stagingFileOutputStream.write(toWrite); this.bytesWritten += toWrite.length; this.recordsWritten++; } }
SimpleDataWriter extends FsDataWriter<byte[]> { @Override public void write(byte[] record) throws IOException { Preconditions.checkNotNull(record); byte[] toWrite = record; if (this.recordDelimiter.isPresent()) { toWrite = Arrays.copyOf(record, record.length + 1); toWrite[toWrite.length - 1] = this.recordDelimiter.get(); } if (this.prependSize) { long recordSize = toWrite.length; ByteBuffer buf = ByteBuffer.allocate(Longs.BYTES); buf.putLong(recordSize); toWrite = ArrayUtils.addAll(buf.array(), toWrite); } this.stagingFileOutputStream.write(toWrite); this.bytesWritten += toWrite.length; this.recordsWritten++; } SimpleDataWriter(SimpleDataWriterBuilder builder, State properties); }
SimpleDataWriter extends FsDataWriter<byte[]> { @Override public void write(byte[] record) throws IOException { Preconditions.checkNotNull(record); byte[] toWrite = record; if (this.recordDelimiter.isPresent()) { toWrite = Arrays.copyOf(record, record.length + 1); toWrite[toWrite.length - 1] = this.recordDelimiter.get(); } if (this.prependSize) { long recordSize = toWrite.length; ByteBuffer buf = ByteBuffer.allocate(Longs.BYTES); buf.putLong(recordSize); toWrite = ArrayUtils.addAll(buf.array(), toWrite); } this.stagingFileOutputStream.write(toWrite); this.bytesWritten += toWrite.length; this.recordsWritten++; } SimpleDataWriter(SimpleDataWriterBuilder builder, State properties); @Override void write(byte[] record); @Override long recordsWritten(); @Override long bytesWritten(); @Override boolean isSpeculativeAttemptSafe(); @Override void flush(); }
SimpleDataWriter extends FsDataWriter<byte[]> { @Override public void write(byte[] record) throws IOException { Preconditions.checkNotNull(record); byte[] toWrite = record; if (this.recordDelimiter.isPresent()) { toWrite = Arrays.copyOf(record, record.length + 1); toWrite[toWrite.length - 1] = this.recordDelimiter.get(); } if (this.prependSize) { long recordSize = toWrite.length; ByteBuffer buf = ByteBuffer.allocate(Longs.BYTES); buf.putLong(recordSize); toWrite = ArrayUtils.addAll(buf.array(), toWrite); } this.stagingFileOutputStream.write(toWrite); this.bytesWritten += toWrite.length; this.recordsWritten++; } SimpleDataWriter(SimpleDataWriterBuilder builder, State properties); @Override void write(byte[] record); @Override long recordsWritten(); @Override long bytesWritten(); @Override boolean isSpeculativeAttemptSafe(); @Override void flush();  }