KuromojiSegmenter extends SegmenterBase { @Override protected void process(JCas aJCas, String text, int zoneBegin) throws AnalysisEngineProcessException { int sentenceBegin = 0; int sentenceEnd = text.indexOf("。"); while (sentenceEnd > sentenceBegin) { String stext = text.substring(sentenceBegin, sentenceEnd + 1); processSentence(aJCas, stext, zoneBegin + sentenceBegin); sentenceBegin = sentenceEnd + 1; sentenceEnd = text.indexOf("。", sentenceBegin); } if (sentenceBegin < text.length()) { String stext = text.substring(sentenceBegin, text.length()); processSentence(aJCas, stext, zoneBegin + sentenceBegin); } }  @Override void initialize(UimaContext aContext);  }
@Test public void testJapanese() throws Exception { JCas jcas = JCasFactory.createText("滧の べ滦榥榜ぶ 廤ま楺獣お 䨣みゅ騪", "ja"); AnalysisEngine aed = createEngine(KuromojiSegmenter.class); aed.process(jcas); String[] tokens = { "滧", "の", "べ", "滦", "榥", "榜", "ぶ", "廤", "ま", "楺", "獣", "お", "䨣", "み", "ゅ", "騪" }; AssertAnnotations.assertToken(tokens, select(jcas, Token.class)); }
@Test public void testJapanese2() throws Exception { JCas jcas = JCasFactory.createText("1993年（平成5年）12月にはユネスコの世界遺産（文化遺産）" + "に登録された[13]。この他、「国宝五城」[注釈 1]や「三名城」、" + "「三大平山城・三大連立式平山城」の一つにも数えられている。", "ja"); AnalysisEngine aed = createEngine(KuromojiSegmenter.class); aed.process(jcas); String[] sentences = { "1993年（平成5年）12月にはユネスコの世界遺産（文化遺産）に登録された[13]。", "この他、「国宝五城」[注釈 1]や「三名城」、「三大平山城・三大連立式平山城」の一つにも数えられている。" }; String[] tokens = { "1993", "年", "（", "平成", "5", "年", "）", "12", "月", "に", "は", "ユネスコ", "の", "世界", "遺産", "（", "文化", "遺産", "）", "に", "登録", "さ", "れ", "た", "[", "13", "]", "。", "この", "他", "、", "「", "国宝", "五", "城", "」[", "注釈", "1", "]", "や", "「", "三", "名城", "」", "、", "「", "三", "大平山", "城", "・", "三", "大", "連立", "式", "平山", "城", "」", "の", "一つ", "に", "も", "数え", "られ", "て", "いる", "。" }; AssertAnnotations.assertSentence(sentences, select(jcas, Sentence.class)); AssertAnnotations.assertToken(tokens, select(jcas, Token.class)); }