ClearNlpSegmenter extends SegmenterBase { @Override protected void process(JCas aJCas, String aText, int aZoneBegin) throws AnalysisEngineProcessException { modelProvider.configure(aJCas.getCas()); AbstractSegmenter segmenter = modelProvider.getResource(); List<List<String>> sentences = segmenter .getSentences(new BufferedReader(new StringReader(aText))); int sBegin = 0; int sEnd = 0; int tBegin = 0; int tEnd = 0; for (List<String> sentence : sentences) { sBegin = -1; for (String token : sentence) { tBegin = aText.indexOf(token, tEnd); tEnd = tBegin + token.length(); if (sBegin == -1) { sBegin = tBegin; } createToken(aJCas, aZoneBegin + tBegin, aZoneBegin + tEnd); } sEnd = tEnd; createSentence(aJCas, aZoneBegin + sBegin, aZoneBegin + sEnd); } }  @Override void initialize(UimaContext aContext); static final String PARAM_VARIANT; static final String PARAM_MODEL_ARTIFACT_URI; static final String PARAM_MODEL_LOCATION; }
@Test public void testSuffix() throws Exception { JCas jcas = JCasFactory.createJCas(); jcas.setDocumentLanguage("en"); jcas.setDocumentText("this is is this is is"); AnalysisEngine aed = createEngine(ClearNlpSegmenter.class); aed.process(jcas); List<Token> tokens = new ArrayList<>(select(jcas, Token.class)); assertEquals(5, tokens.get(1).getBegin()); assertEquals(7, tokens.get(1).getEnd()); for (Token t : tokens) { System.out.printf("%d %d %s%n", t.getBegin(), t.getEnd(), t.getCoveredText()); } }