LanguageToolSegmenter extends SegmenterBase { @Override protected void process(JCas aJCas, String aText, int aZoneBegin) throws AnalysisEngineProcessException { Language lang = Languages.getLanguageForShortCode(getLanguage(aJCas)); Language defaultVariant = lang.getDefaultLanguageVariant(); if (defaultVariant != null) { getLogger().debug( "Using default variant [" + defaultVariant.getShortCodeWithCountryAndVariant() + "] for language [" + getLanguage(aJCas) + "]"); lang = defaultVariant; } List<String> sentences = lang.getSentenceTokenizer().tokenize(aText); int lastSStart = 0; for (String s : sentences) { int sStart = aText.indexOf(s, lastSStart); int sEnd = sStart + s.length(); lastSStart = sEnd; sStart += aZoneBegin; sEnd += aZoneBegin; createSentence(aJCas, sStart, sEnd); List<String> tokens = lang.getWordTokenizer().tokenize(s); int lastTStart = 0; for (String t : tokens) { int tStart = s.indexOf(t, lastTStart); if ("zh".equals(getLanguage(aJCas)) && tStart == -1) { int suffix = t.indexOf('/'); if (suffix != -1) { t = t.substring(0, suffix); } tStart = s.indexOf(t, lastTStart); } if ("zh".equals(getLanguage(aJCas)) && tStart == -1) { String trad = CJFBeanFactory.getChineseJF().chineseJan2Fan(t); tStart = s.indexOf(trad, lastTStart); } if (tStart == -1) { throw new IllegalStateException("Token [" + t + "] not found in sentence [" + s + "]"); } int tEnd = tStart + t.length(); lastTStart = tEnd; tStart += sStart; tEnd += sStart; createToken(aJCas, tStart, tEnd); } } }    }
@Test public void testTwoSentences() throws Exception { JCas jcas = JCasFactory.createJCas(); jcas.setDocumentLanguage("en"); jcas.setDocumentText("This is a test. This is another one."); AnalysisEngine aed = createEngine(LanguageToolSegmenter.class); aed.process(jcas); String[] sentences = { "This is a test.", "This is another one." }; AssertAnnotations.assertSentence(sentences, select(jcas, Sentence.class)); }
@Test public void testTwoSentences2() throws Exception { JCas jcas = JCasFactory.createJCas(); jcas.setDocumentLanguage("en"); jcas.setDocumentText("I bought a car for my little brother. He said, he likes it a lot."); AnalysisEngine aed = createEngine(LanguageToolSegmenter.class); aed.process(jcas); String[] sentences = { "I bought a car for my little brother.", "He said, he likes it a lot." }; AssertAnnotations.assertSentence(sentences, select(jcas, Sentence.class)); }
@Test public void testTraditionalChinese() throws Exception { JCas jcas = JCasFactory.createJCas(); jcas.setDocumentLanguage("zh"); jcas.setDocumentText("毛澤東住在北京"); AnalysisEngine aed = createEngine(LanguageToolSegmenter.class); aed.process(jcas); String[] tokens = { "毛", "澤東", "住", "在", "北京" }; AssertAnnotations.assertToken(tokens, select(jcas, Token.class)); }