SequenceFilesFromMailArchives { public static void main(String[] args) throws Exception { DefaultOptionBuilder obuilder = new DefaultOptionBuilder(); ArgumentBuilder abuilder = new ArgumentBuilder(); GroupBuilder gbuilder = new GroupBuilder(); Option inputOpt = DefaultOptionCreator.inputOption().create(); Option outputDirOpt = DefaultOptionCreator.outputOption().create(); Option chunkSizeOpt = obuilder.withLongName("chunkSize").withArgument( abuilder.withName("chunkSize").withMinimum(1).withMaximum(1).create()).withDescription( "The chunkSize in MegaBytes. Defaults to 64").withShortName("chunk").create(); Option keyPrefixOpt = obuilder.withLongName("keyPrefix").withArgument( abuilder.withName("keyPrefix").withMinimum(1).withMaximum(1).create()).withDescription( "The prefix to be prepended to the key").withShortName("prefix").create(); Option charsetOpt = obuilder.withLongName("charset").withRequired(true).withArgument( abuilder.withName("charset").withMinimum(1).withMaximum(1).create()).withDescription( "The name of the character encoding of the input files").withShortName("c").create(); Option subjectOpt = obuilder.withLongName("subject").withRequired(false). withDescription("Include the Mail subject as part of the text. Default is false").withShortName("s").create(); Option toOpt = obuilder.withLongName("to").withRequired(false). withDescription("Include the to field in the text. Default is false").withShortName("to").create(); Option fromOpt = obuilder.withLongName("from").withRequired(false). withDescription("Include the from field in the text. Default is false").withShortName("from").create(); Option refsOpt = obuilder.withLongName("references").withRequired(false). withDescription("Include the references field in the text. Default is false").withShortName("refs").create(); Option bodyOpt = obuilder.withLongName("body").withRequired(false). withDescription("Include the body in the output. Default is false").withShortName("b").create(); Option quotedOpt = obuilder.withLongName("stripQuoted").withRequired(false). withDescription("Strip (remove) quoted email text in the body. Default is false").withShortName("q").create(); Option quotedRegexOpt = obuilder.withLongName("quotedRegex").withRequired(false).withArgument(abuilder.withName("regex").withMinimum(1).withMaximum(1).create()) .withDescription("Specify the regex that identifies quoted text. Default is to look for > or | at the beginning of the line.").withShortName("q").create(); Option separatorOpt = obuilder.withLongName("separator").withRequired(false).withArgument( abuilder.withName("separator").withMinimum(1).withMaximum(1).create()). withDescription("The separator to use between metadata items (to, from, etc.). Default is \\n").withShortName("sep").create(); Option bodySeparatorOpt = obuilder.withLongName("bodySeparator").withRequired(false).withArgument( abuilder.withName("bodySeparator").withMinimum(1).withMaximum(1).create()). withDescription("The separator to use between lines in the body. Default is \\n. Useful to change if you wish to have the message be on one line").withShortName("bodySep").create(); Option helpOpt = DefaultOptionCreator.helpOption(); Group group = gbuilder.withName("Options").withOption(keyPrefixOpt).withOption(chunkSizeOpt).withOption( charsetOpt).withOption(outputDirOpt).withOption(helpOpt).withOption(inputOpt).withOption(subjectOpt).withOption(toOpt) .withOption(fromOpt).withOption(bodyOpt).withOption(quotedOpt).withOption(refsOpt).withOption(bodySeparatorOpt) .withOption(quotedRegexOpt) .withOption(separatorOpt).create(); try { Parser parser = new Parser(); parser.setGroup(group); parser.setHelpOption(helpOpt); CommandLine cmdLine = parser.parse(args); if (cmdLine.hasOption(helpOpt)) { CommandLineUtil.printHelp(group); return; } File input = new File((String) cmdLine.getValue(inputOpt)); String outputDir = (String) cmdLine.getValue(outputDirOpt); int chunkSize = 64; if (cmdLine.hasOption(chunkSizeOpt)) { chunkSize = Integer.parseInt((String) cmdLine.getValue(chunkSizeOpt)); } String prefix = ""; if (cmdLine.hasOption(keyPrefixOpt)) { prefix = (String) cmdLine.getValue(keyPrefixOpt); } Charset charset = Charset.forName((String) cmdLine.getValue(charsetOpt)); SequenceFilesFromMailArchives dir = new SequenceFilesFromMailArchives(); MailOptions options = new MailOptions(); options.setInput(input); options.setOutputDir(outputDir); options.setPrefix(prefix); options.setChunkSize(chunkSize); options.setCharset(charset); List<Pattern> patterns = new ArrayList<Pattern>(5); Map<String, Integer> patternOrder = new HashMap<String, Integer>(); int order = 0; if (cmdLine.hasOption(fromOpt)) { patterns.add(MailProcessor.FROM_PREFIX); patternOrder.put(MailOptions.FROM, order++); } if (cmdLine.hasOption(toOpt)) { patterns.add(MailProcessor.TO_PREFIX); patternOrder.put(MailOptions.TO, order++); } if (cmdLine.hasOption(refsOpt)) { patterns.add(MailProcessor.REFS_PREFIX); patternOrder.put(MailOptions.REFS, order++); } if (cmdLine.hasOption(subjectOpt)) { patterns.add(MailProcessor.SUBJECT_PREFIX); patternOrder.put(MailOptions.SUBJECT, order++); } options.setStripQuotedText(cmdLine.hasOption(quotedOpt)); options.setPatternsToMatch(patterns.toArray(new Pattern[patterns.size()])); options.setPatternOrder(patternOrder); options.setIncludeBody(cmdLine.hasOption(bodyOpt)); options.setSeparator("\n"); if (cmdLine.hasOption(separatorOpt)) { options.setSeparator(cmdLine.getValue(separatorOpt).toString()); } if (cmdLine.hasOption(bodySeparatorOpt)) { options.setBodySeparator(cmdLine.getValue(bodySeparatorOpt).toString()); } if (cmdLine.hasOption(quotedRegexOpt)){ options.setQuotedTextPattern(Pattern.compile(cmdLine.getValue(quotedRegexOpt).toString())); } long start = System.currentTimeMillis(); dir.createSequenceFiles(options); long finish = System.currentTimeMillis(); log.info("Conversion took {}ms", finish - start); } catch (OptionException e) { log.error("Exception", e); CommandLineUtil.printHelp(group); } }  void createSequenceFiles(MailOptions options); static void main(String[] args);  }
@Test public void testMain() throws Exception { String[] args = { "--input", inputDir.getAbsolutePath(), "--output", outputDir.getAbsolutePath(), "--charset", "UTF-8", "--keyPrefix", "TEST", "--body", "--subject", "--separator", "" }; SequenceFilesFromMailArchives.main(args); File expectedChunkFile = new File(outputDir, "chunk-0"); String expectedChunkPath = expectedChunkFile.getAbsolutePath(); Assert.assertTrue("Expected chunk file "+expectedChunkPath+" not found!", expectedChunkFile.isFile()); Configuration conf = new Configuration(); SequenceFileIterator<Text,Text> iterator = new SequenceFileIterator<Text,Text>(new Path(expectedChunkPath), true, conf); Assert.assertTrue("First key/value pair not found!", iterator.hasNext()); Pair<Text,Text> record = iterator.next(); File parentFile = new File(new File(new File("TEST"), "subdir"), "mail-messages.gz"); Assert.assertEquals(new File(parentFile, testVars[0][0]).toString(), record.getFirst().toString()); Assert.assertEquals(testVars[0][1]+testVars[0][2], record.getSecond().toString()); Assert.assertTrue("Second key/value pair not found!", iterator.hasNext()); record = iterator.next(); Assert.assertEquals(new File(parentFile, testVars[1][0]).toString(), record.getFirst().toString()); Assert.assertEquals(testVars[1][1]+testVars[1][2], record.getSecond().toString()); Assert.assertFalse("Only two key/value pairs expected!", iterator.hasNext()); }