Docs2Tokens { public static void main(String[] args) throws Exception { Options options = CliFactory.parseArguments(Options.class, args); CollectionReader reader = UriCollectionReader.getCollectionReaderFromDirectory(options.getInputDirectory()); AnalysisEngineDescription uriToText = UriToDocumentTextAnnotator.getDescription(); AnalysisEngineDescription sentences = SentenceAnnotator.getDescription(); AnalysisEngineDescription tokenizer = TokenAnnotator.getDescription(); AnalysisEngineDescription posTagger = PosTaggerAnnotator.getDescription(); AnalysisEngineDescription lineWriter = AnalysisEngineFactory.createEngineDescription( LineWriter.class, LineWriter.PARAM_OUTPUT_FILE_NAME, options.getOutputFile(), LineWriter.PARAM_OUTPUT_ANNOTATION_CLASS_NAME, Token.class.getName(), LineWriter.PARAM_ANNOTATION_WRITER_CLASS_NAME, TokenAnnotationWriter.class.getName()); SimplePipeline.runPipeline(reader, uriToText, sentences, tokenizer, posTagger, lineWriter); System.out.println("results written to " + options.getOutputFile()); }  static void main(String[] args);  }
@Test public void testDocs2Tokens() throws Exception { Docs2Sentences.main(new String[] { "-i", "src/test/resources/data/twain", "-o", "target/test/twain-tokens.txt" }); String[] lines = FileUtil.loadListOfStrings(new File("target/test/twain-tokens.txt")); assertTrue(lines.length > 20); }