@Test public void write_large() throws Exception { Path path = new Path("testing"); LongWritable key = new LongWritable(); LongWritable value = new LongWritable(); try (OutputStream out = new FileOutputStream(fs.pathToFile(path)); SequenceFile.Writer writer = SequenceFileUtil.openWriter( new BufferedOutputStream(out), conf, key.getClass(), value.getClass(), null)) { for (long i = 0; i < 300000; i++) { key.set(i); value.set(i + 1); writer.append(key, value); } } try (SequenceFile.Reader reader = new SequenceFile.Reader(conf, SequenceFile.Reader.file(fs.makeQualified(path)))) { for (long i = 0; i < 300000; i++) { assertThat(reader.next(key, value), is(true)); assertThat(key.get(), is(i)); assertThat(value.get(), is(i + 1)); } assertThat(reader.next(key, value), is(false)); } }
public static SequenceFile.Writer openWriter( OutputStream out, Configuration conf, Class<?> keyClass, Class<?> valueClass, CompressionCodec codec) throws IOException { if (out == null) { throw new IllegalArgumentException("out must not be null"); } if (conf == null) { throw new IllegalArgumentException("conf must not be null"); } if (keyClass == null) { throw new IllegalArgumentException("keyClass must not be null"); } if (valueClass == null) { throw new IllegalArgumentException("valueClass must not be null"); } if (LOG.isDebugEnabled()) { LOG.debug(MessageFormat.format( "Creating sequence file writer for output (key={0}, value={0})", keyClass.getName(), valueClass.getName())); } return SequenceFile.createWriter(conf, SequenceFile.Writer.stream(new FSDataOutputStream(out, null)), SequenceFile.Writer.keyClass(keyClass), SequenceFile.Writer.valueClass(valueClass), SequenceFile.Writer.compression(codec == null ? CompressionType.NONE : CompressionType.BLOCK, codec)); }
SequenceFileUtil { public static SequenceFile.Writer openWriter( OutputStream out, Configuration conf, Class<?> keyClass, Class<?> valueClass, CompressionCodec codec) throws IOException { if (out == null) { throw new IllegalArgumentException("out must not be null"); } if (conf == null) { throw new IllegalArgumentException("conf must not be null"); } if (keyClass == null) { throw new IllegalArgumentException("keyClass must not be null"); } if (valueClass == null) { throw new IllegalArgumentException("valueClass must not be null"); } if (LOG.isDebugEnabled()) { LOG.debug(MessageFormat.format( "Creating sequence file writer for output (key={0}, value={0})", keyClass.getName(), valueClass.getName())); } return SequenceFile.createWriter(conf, SequenceFile.Writer.stream(new FSDataOutputStream(out, null)), SequenceFile.Writer.keyClass(keyClass), SequenceFile.Writer.valueClass(valueClass), SequenceFile.Writer.compression(codec == null ? CompressionType.NONE : CompressionType.BLOCK, codec)); } }
SequenceFileUtil { public static SequenceFile.Writer openWriter( OutputStream out, Configuration conf, Class<?> keyClass, Class<?> valueClass, CompressionCodec codec) throws IOException { if (out == null) { throw new IllegalArgumentException("out must not be null"); } if (conf == null) { throw new IllegalArgumentException("conf must not be null"); } if (keyClass == null) { throw new IllegalArgumentException("keyClass must not be null"); } if (valueClass == null) { throw new IllegalArgumentException("valueClass must not be null"); } if (LOG.isDebugEnabled()) { LOG.debug(MessageFormat.format( "Creating sequence file writer for output (key={0}, value={0})", keyClass.getName(), valueClass.getName())); } return SequenceFile.createWriter(conf, SequenceFile.Writer.stream(new FSDataOutputStream(out, null)), SequenceFile.Writer.keyClass(keyClass), SequenceFile.Writer.valueClass(valueClass), SequenceFile.Writer.compression(codec == null ? CompressionType.NONE : CompressionType.BLOCK, codec)); } private  SequenceFileUtil(); }
SequenceFileUtil { public static SequenceFile.Writer openWriter( OutputStream out, Configuration conf, Class<?> keyClass, Class<?> valueClass, CompressionCodec codec) throws IOException { if (out == null) { throw new IllegalArgumentException("out must not be null"); } if (conf == null) { throw new IllegalArgumentException("conf must not be null"); } if (keyClass == null) { throw new IllegalArgumentException("keyClass must not be null"); } if (valueClass == null) { throw new IllegalArgumentException("valueClass must not be null"); } if (LOG.isDebugEnabled()) { LOG.debug(MessageFormat.format( "Creating sequence file writer for output (key={0}, value={0})", keyClass.getName(), valueClass.getName())); } return SequenceFile.createWriter(conf, SequenceFile.Writer.stream(new FSDataOutputStream(out, null)), SequenceFile.Writer.keyClass(keyClass), SequenceFile.Writer.valueClass(valueClass), SequenceFile.Writer.compression(codec == null ? CompressionType.NONE : CompressionType.BLOCK, codec)); } private  SequenceFileUtil(); static SequenceFile.Reader openReader(
            InputStream in,
            FileStatus status,
            Configuration conf); static SequenceFile.Reader openReader(
            InputStream in,
            long length,
            Configuration conf); static SequenceFile.Writer openWriter(
            OutputStream out, Configuration conf,
            Class<?> keyClass, Class<?> valueClass,
            CompressionCodec codec); }
SequenceFileUtil { public static SequenceFile.Writer openWriter( OutputStream out, Configuration conf, Class<?> keyClass, Class<?> valueClass, CompressionCodec codec) throws IOException { if (out == null) { throw new IllegalArgumentException("out must not be null"); } if (conf == null) { throw new IllegalArgumentException("conf must not be null"); } if (keyClass == null) { throw new IllegalArgumentException("keyClass must not be null"); } if (valueClass == null) { throw new IllegalArgumentException("valueClass must not be null"); } if (LOG.isDebugEnabled()) { LOG.debug(MessageFormat.format( "Creating sequence file writer for output (key={0}, value={0})", keyClass.getName(), valueClass.getName())); } return SequenceFile.createWriter(conf, SequenceFile.Writer.stream(new FSDataOutputStream(out, null)), SequenceFile.Writer.keyClass(keyClass), SequenceFile.Writer.valueClass(valueClass), SequenceFile.Writer.compression(codec == null ? CompressionType.NONE : CompressionType.BLOCK, codec)); } private  SequenceFileUtil(); static SequenceFile.Reader openReader(
            InputStream in,
            FileStatus status,
            Configuration conf); static SequenceFile.Reader openReader(
            InputStream in,
            long length,
            Configuration conf); static SequenceFile.Writer openWriter(
            OutputStream out, Configuration conf,
            Class<?> keyClass, Class<?> valueClass,
            CompressionCodec codec);  }