BreakIteratorSegmenter extends SegmenterBase { @Override protected void process(JCas aJCas, String text, int zoneBegin) throws AnalysisEngineProcessException { BreakIterator bi = BreakIterator.getSentenceInstance(getLocale(aJCas)); bi.setText(text); int last = bi.first() + zoneBegin; int cur = bi.next(); while (cur != BreakIterator.DONE) { cur += zoneBegin; if (isWriteSentence()) { Annotation segment = createSentence(aJCas, last, cur); if (segment != null) { processSentence(aJCas, segment.getCoveredText(), segment.getBegin()); } } else { int[] span = new int[] { last, cur }; TrimUtils.trim(aJCas.getDocumentText(), span); processSentence(aJCas, aJCas.getDocumentText().substring(span[0], span[1]), span[0]); } last = cur; cur = bi.next(); } }   static final String PARAM_SPLIT_AT_APOSTROPHE; }
@Test public void testJapanese() throws Exception { JCas jcas = JCasFactory.createText("滧の べ滦榥榜ぶ 廤ま楺獣お 䨣みゅ騪", "ja"); AnalysisEngine aed = createEngine(BreakIteratorSegmenter.class); aed.process(jcas); String[] tokens = { "滧", "の", "べ", "滦榥榜", "ぶ", "廤", "ま", "楺獣", "お", "䨣", "みゅ", "騪" }; AssertAnnotations.assertToken(tokens, select(jcas, Token.class)); }