TreeTaggerPosTagger extends JCasAnnotator_ImplBase { @Override public void process(final JCas aJCas) throws AnalysisEngineProcessException { final CAS cas = aJCas.getCas(); modelProvider.configure(cas); posMappingProvider.configure(cas); TreeTaggerWrapper<Token> treetagger = modelProvider.getResource(); try { List<Token> tokens = new ArrayList<Token>(select(aJCas, Token.class)); final POS[] pos = new POS[tokens.size()]; final Lemma[] lemma = new Lemma[tokens.size()]; final AtomicInteger count = new AtomicInteger(0); treetagger.setHandler(new TokenHandler<Token>() { @Override public void token(Token aToken, String aPos, String aLemma) { synchronized (cas) { if (writePos && aPos != null) { Type posTag = posMappingProvider.getTagType(aPos); POS posAnno = (POS) cas.createAnnotation(posTag, aToken.getBegin(), aToken.getEnd()); posAnno.setPosValue(aPos.intern()); POSUtils.assignCoarseValue(posAnno); aToken.setPos(posAnno); pos[count.get()] = posAnno; } if (writeLemma && aLemma != null) { Lemma lemmaAnno = new Lemma(aJCas, aToken.getBegin(), aToken.getEnd()); lemmaAnno.setValue(aLemma.intern()); aToken.setLemma(lemmaAnno); lemma[count.get()] = lemmaAnno; } count.getAndIncrement(); } } }); treetagger.process(tokens); for (int i = 0; i < count.get(); i++) { if (pos[i] != null) { pos[i].addToIndexes(); } if (lemma[i] != null) { lemma[i].addToIndexes(); } } } catch (TreeTaggerException e) { throw new AnalysisEngineProcessException(e); } catch (IOException e) { throw new AnalysisEngineProcessException(e); } }  @Override void initialize(UimaContext aContext); @Override void process(final JCas aJCas); static final String PARAM_LANGUAGE; static final String PARAM_VARIANT; static final String PARAM_EXECUTABLE_PATH; static final String PARAM_MODEL_ARTIFACT_URI; static final String PARAM_MODEL_LOCATION; static final String PARAM_MODEL_ENCODING; static final String PARAM_MAPPING_ENABLED; static final String PARAM_POS_MAPPING_LOCATION; static final String PARAM_PRINT_TAGSET; static final String PARAM_PERFORMANCE_MODE; static final String PARAM_WRITE_POS; static final String PARAM_WRITE_LEMMA; }
@Test public void longTokenTest() throws Exception { checkModelsAndBinary("en"); AnalysisEngine engine = createEngine(TreeTaggerPosTagger.class); JCas jcas = engine.newJCas(); try { for (int n = 99990; n < 100000; n ++) { System.out.println(n); jcas.setDocumentLanguage("en"); JCasBuilder builder = new JCasBuilder(jcas); builder.add("Start", Token.class); builder.add("with", Token.class); builder.add("good", Token.class); builder.add("tokens", Token.class); builder.add(".", Token.class); builder.add(StringUtils.repeat("b", n), Token.class); builder.add("End", Token.class); builder.add("with", Token.class); builder.add("some", Token.class); builder.add("good", Token.class); builder.add("tokens", Token.class); builder.add(".", Token.class); builder.close(); engine.process(jcas); jcas.reset(); } } finally { engine.destroy(); } }