SequenceFile { public static Writer createWriter(Configuration conf, Writer.Option... opts ) throws IOException { Writer.CompressionOption compressionOption = Options.getOption(Writer.CompressionOption.class, opts); CompressionType kind; if (compressionOption != null) { kind = compressionOption.getValue(); } else { kind = getDefaultCompressionType(conf); opts = Options.prependOptions(opts, Writer.compression(kind)); } switch (kind) { default: case NONE: return new Writer(conf, opts); case RECORD: return new RecordCompressWriter(conf, opts); case BLOCK: return new BlockCompressWriter(conf, opts); } } private  SequenceFile(); static CompressionType getDefaultCompressionType(Configuration job); static void setDefaultCompressionType(Configuration job, 
                                               CompressionType val); static Writer createWriter(Configuration conf, Writer.Option... opts
                                    ); @Deprecated static Writer createWriter(FileSystem fs, Configuration conf, Path name, 
                 Class keyClass, Class valClass); @Deprecated static Writer createWriter(FileSystem fs, Configuration conf, Path name, 
                 Class keyClass, Class valClass, 
                 CompressionType compressionType); @Deprecated static Writer createWriter(FileSystem fs, Configuration conf, Path name, 
                 Class keyClass, Class valClass, CompressionType compressionType,
                 Progressable progress); @Deprecated static Writer createWriter(FileSystem fs, Configuration conf, Path name, 
                 Class keyClass, Class valClass, CompressionType compressionType, 
                 CompressionCodec codec); @Deprecated static Writer createWriter(FileSystem fs, Configuration conf, Path name, 
                 Class keyClass, Class valClass, 
                 CompressionType compressionType, CompressionCodec codec,
                 Progressable progress, Metadata metadata); @Deprecated static Writer createWriter(FileSystem fs, Configuration conf, Path name,
                 Class keyClass, Class valClass, int bufferSize,
                 short replication, long blockSize,
                 CompressionType compressionType, CompressionCodec codec,
                 Progressable progress, Metadata metadata); @Deprecated static Writer createWriter(FileSystem fs, Configuration conf, Path name,
               Class keyClass, Class valClass, int bufferSize,
               short replication, long blockSize, boolean createParent,
               CompressionType compressionType, CompressionCodec codec,
               Metadata metadata); static Writer createWriter(FileContext fc, Configuration conf, Path name,
               Class keyClass, Class valClass,
               CompressionType compressionType, CompressionCodec codec,
               Metadata metadata,
               final EnumSet<CreateFlag> createFlag, CreateOpts... opts); @Deprecated static Writer createWriter(FileSystem fs, Configuration conf, Path name, 
                 Class keyClass, Class valClass, 
                 CompressionType compressionType, CompressionCodec codec,
                 Progressable progress); @Deprecated static Writer createWriter(Configuration conf, FSDataOutputStream out, 
                 Class keyClass, Class valClass,
                 CompressionType compressionType,
                 CompressionCodec codec, Metadata metadata); @Deprecated static Writer createWriter(Configuration conf, FSDataOutputStream out, 
                 Class keyClass, Class valClass, CompressionType compressionType,
                 CompressionCodec codec); static final int SYNC_INTERVAL; }
@SuppressWarnings("deprecation") @Test public void testRecursiveSeqFileCreate() throws IOException { FileSystem fs = FileSystem.getLocal(conf); Path name = new Path(new Path(GenericTestUtils.getTempPath( "recursiveCreateDir")), "file"); boolean createParent = false; try { SequenceFile.createWriter(fs, conf, name, RandomDatum.class, RandomDatum.class, 512, (short) 1, 4096, createParent, CompressionType.NONE, null, new Metadata()); fail("Expected an IOException due to missing parent"); } catch (IOException ioe) { } createParent = true; SequenceFile.createWriter(fs, conf, name, RandomDatum.class, RandomDatum.class, 512, (short) 1, 4096, createParent, CompressionType.NONE, null, new Metadata()); }
@SuppressWarnings("deprecation") @Test public void testClose() throws IOException { Configuration conf = new Configuration(); LocalFileSystem fs = FileSystem.getLocal(conf); Path path1 = new Path(System.getProperty("test.build.data",".")+"/test1.seq"); SequenceFile.Writer writer = SequenceFile.createWriter(fs, conf, path1, Text.class, NullWritable.class, CompressionType.BLOCK); writer.append(new Text("file1-1"), NullWritable.get()); writer.append(new Text("file1-2"), NullWritable.get()); writer.close(); Path path2 = new Path(System.getProperty("test.build.data",".")+"/test2.seq"); writer = SequenceFile.createWriter(fs, conf, path2, Text.class, NullWritable.class, CompressionType.BLOCK); writer.append(new Text("file2-1"), NullWritable.get()); writer.append(new Text("file2-2"), NullWritable.get()); writer.close(); SequenceFile.Reader reader = new SequenceFile.Reader(fs, path1, conf); reader.close(); reader.close(); SequenceFile.Reader reader1 = new SequenceFile.Reader(fs, path1, conf); Text text = new Text(); reader1.next(text); assertEquals("file1-1", text.toString()); SequenceFile.Reader reader2 = new SequenceFile.Reader(fs, path2, conf); reader2.next(text); assertEquals("file2-1", text.toString()); reader1.next(text); assertEquals("file1-2", text.toString()); reader2.next(text); assertEquals("file2-2", text.toString()); assertFalse(reader1.next(text)); assertFalse(reader2.next(text)); }
@SuppressWarnings("deprecation") @Test public void testCreateUsesFsArg() throws Exception { FileSystem fs = FileSystem.getLocal(conf); FileSystem spyFs = Mockito.spy(fs); Path p = new Path(System.getProperty("test.build.data", ".")+"/testCreateUsesFSArg.seq"); SequenceFile.Writer writer = SequenceFile.createWriter( spyFs, conf, p, NullWritable.class, NullWritable.class); writer.close(); Mockito.verify(spyFs).getDefaultReplication(p); }
@SuppressWarnings("deprecation") @Test public void testCreateWriterOnExistingFile() throws IOException { Configuration conf = new Configuration(); FileSystem fs = FileSystem.getLocal(conf); Path name = new Path(new Path(System.getProperty("test.build.data","."), "createWriterOnExistingFile") , "file"); fs.create(name); SequenceFile.createWriter(fs, conf, name, RandomDatum.class, RandomDatum.class, 512, (short) 1, 4096, false, CompressionType.NONE, null, new Metadata()); }
@SuppressWarnings("deprecation") @Test public void testRecursiveSeqFileCreate() throws IOException { FileSystem fs = FileSystem.getLocal(conf); Path name = new Path(new Path(System.getProperty("test.build.data","."), "recursiveCreateDir") , "file"); boolean createParent = false; try { SequenceFile.createWriter(fs, conf, name, RandomDatum.class, RandomDatum.class, 512, (short) 1, 4096, createParent, CompressionType.NONE, null, new Metadata()); fail("Expected an IOException due to missing parent"); } catch (IOException ioe) { } createParent = true; SequenceFile.createWriter(fs, conf, name, RandomDatum.class, RandomDatum.class, 512, (short) 1, 4096, createParent, CompressionType.NONE, null, new Metadata()); }
@Test public void testSerializationAvailability() throws IOException { Configuration conf = new Configuration(); Path path = new Path(System.getProperty("test.build.data", "."), "serializationAvailability"); try { SequenceFile.createWriter( conf, SequenceFile.Writer.file(path), SequenceFile.Writer.keyClass(String.class), SequenceFile.Writer.valueClass(NullWritable.class)); fail("Must throw IOException for missing serializer for the Key class"); } catch (IOException e) { assertTrue(e.getMessage().startsWith( "Could not find a serializer for the Key class: '" + String.class.getName() + "'.")); } try { SequenceFile.createWriter( conf, SequenceFile.Writer.file(path), SequenceFile.Writer.keyClass(NullWritable.class), SequenceFile.Writer.valueClass(String.class)); fail("Must throw IOException for missing serializer for the Value class"); } catch (IOException e) { assertTrue(e.getMessage().startsWith( "Could not find a serializer for the Value class: '" + String.class.getName() + "'.")); } writeTest(FileSystem.get(conf), 1, 1, path, CompressionType.NONE, null); conf.setStrings(CommonConfigurationKeys.IO_SERIALIZATIONS_KEY, AvroReflectSerialization.class.getName()); try { new SequenceFile.Reader( conf, SequenceFile.Reader.file(path)); fail("Must throw IOException for missing deserializer for the Key class"); } catch (IOException e) { assertTrue(e.getMessage().startsWith( "Could not find a deserializer for the Key class: '" + RandomDatum.class.getName() + "'.")); } }