Tokenizer { public static AnalysisEngineDescription getDescription(String languageCode) throws ResourceInitializationException { String modelPath = String.format("/models/%s-token.bin", languageCode); return AnalysisEngineFactory.createEngineDescription( opennlp.uima.tokenize.Tokenizer.class, opennlp.uima.util.UimaUtil.MODEL_PARAMETER, ExternalResourceFactory.createExternalResourceDescription( TokenizerModelResourceImpl.class, Tokenizer.class.getResource(modelPath).toString()), opennlp.uima.util.UimaUtil.SENTENCE_TYPE_PARAMETER, Sentence.class.getName(), opennlp.uima.util.UimaUtil.TOKEN_TYPE_PARAMETER, Token.class.getName()); }  static AnalysisEngineDescription getDescription(String languageCode);  }
@Test public void test() throws Exception { this.jCas.setDocumentText("\"John & Mary's dog,\" Jane thought (to herself).\n" + "\"What a #$%!\n" + "a- ``I like AT&T''.\""); new Sentence(this.jCas, 0, 47).addToIndexes(); new Sentence(this.jCas, 48, 60).addToIndexes(); new Sentence(this.jCas, 61, 81).addToIndexes(); Level level = DisableLogging.disableLogging(); AnalysisEngine engine = AnalysisEngineFactory.createEngine(Tokenizer.getDescription("en")); engine.process(this.jCas); DisableLogging.enableLogging(level); List<String> expected = Arrays.asList( "\"", "John", "&", "Mary", "'s", "dog", ",", "\"", "Jane", "thought", "(", "to", "herself", ")", ".", "\"", "What", "a", "#", "$", "%", "!", "a", "-", "``", "I", "like", "AT&T", "''", ".", "\""); List<String> actual = JCasUtil.toText(JCasUtil.select(this.jCas, Token.class)); Assert.assertEquals(expected, actual); }