SequenceFilesFromDirectory extends AbstractJob { public static void main(String[] args) throws Exception { ToolRunner.run(new SequenceFilesFromDirectory(), args); }  void run(Configuration conf,
                  String keyPrefix,
                  Map<String, String> options,
                  Path input,
                  Path output); static void main(String[] args); @Override int run(String[] args); static final String[] CHUNK_SIZE_OPTION; static final String[] FILE_FILTER_CLASS_OPTION; static final String[] KEY_PREFIX_OPTION; static final String[] CHARSET_OPTION; }
@Test public void testSequenceFileFromDirectoryBasic() throws Exception { Configuration conf = new Configuration(); FileSystem fs = FileSystem.get(conf); Path tmpDir = this.getTestTempDirPath(); Path inputDir = new Path(tmpDir, "inputDir"); fs.mkdirs(inputDir); Path outputDir = new Path(tmpDir, "outputDir"); createFilesFromArrays(conf, inputDir, DATA1); String prefix = "UID"; SequenceFilesFromDirectory.main(new String[] {"--input", inputDir.toString(), "--output", outputDir.toString(), "--chunkSize", "64", "--charset", Charsets.UTF_8.name(), "--keyPrefix", prefix}); checkChunkFiles(conf, outputDir, DATA1, prefix, ParserType.TEXT); }
@Test public void testSequnceFileFromDirectoryTsv() throws Exception { Configuration conf = new Configuration(); FileSystem fs = FileSystem.get(conf); Path tmpDir = this.getTestTempDirPath(); Path inputDir = new Path(tmpDir, "inputDir"); fs.mkdirs(inputDir); Path outputDir = new Path(tmpDir, "outputDir"); createTsvFilesFromArrays(conf, inputDir, DATA1); String prefix = "UID"; int chunkSizeInMB = 64; int keyColumn = 0; int valueColumn = 1; SequenceFilesFromCsvFilter.main(new String[] {"--input", inputDir.toString(), "--output", outputDir.toString(), "--charset", Charsets.UTF_8.name(), "--chunkSize", Integer.toString(chunkSizeInMB), "--keyPrefix", prefix, "--keyColumn", Integer.toString(keyColumn), "--valueColumn", Integer.toString(valueColumn)}); checkChunkFiles(conf, outputDir, DATA1, prefix, ParserType.CSV); }