TokenMerger extends JCasAnnotator_ImplBase { @Override public void process(JCas aJCas) throws AnalysisEngineProcessException { CAS cas = aJCas.getCas(); if (posValue != null) { mappingProvider.configure(cas); } List<AnnotationFS> covers = new ArrayList<>( CasUtil.select(cas, CasUtil.getAnnotationType(cas, annotationType))); Collection<Annotation> toRemove = new ArrayList<Annotation>(); for (AnnotationFS cover : covers) { List<Token> covered = selectCovered(Token.class, cover); if (covered.size() < 2) { continue; } if (constraint != null) { JXPathContext ctx = JXPathContext.newContext(cover); boolean match = ctx.iterate(constraint).hasNext(); if (!match) { continue; } } Iterator<Token> i = covered.iterator(); Token token = i.next(); token.removeFromIndexes(); token.setEnd(covered.get(covered.size() - 1).getEnd()); token.addToIndexes(); if (posValue != null) { updatePos(token, toRemove); } List<String> lemmata = new ArrayList<String>(); if (token.getLemma() != null) { lemmata.add(token.getLemma().getValue()); } while (i.hasNext()) { Token t = i.next(); Lemma lemma = t.getLemma(); if (lemma != null) { lemmata.add(lemma.getValue()); toRemove.add(lemma); } POS pos = t.getPos(); if (pos != null) { toRemove.add(pos); } toRemove.add(t); } if (lemmaMode == LemmaMode.JOIN) { Lemma lemma = token.getLemma(); if (!lemmata.isEmpty()) { if (lemma == null) { lemma = new Lemma(aJCas); } lemma.setValue(StringUtils.join(lemmata, " ")); } else if (lemma != null) { token.setLemma(null); toRemove.add(lemma); } } else if (lemmaMode == LemmaMode.REMOVE) { Lemma lemma = token.getLemma(); if (lemma != null) { token.setLemma(null); toRemove.add(lemma); } } if (token.getLemma() != null) { Lemma lemma = token.getLemma(); lemma.removeFromIndexes(); lemma.setBegin(token.getBegin()); lemma.setEnd(token.getEnd()); lemma.addToIndexes(); } } for (Annotation t : toRemove) { t.removeFromIndexes(); } }  @Override void initialize(UimaContext aContext); @Override void process(JCas aJCas); static final String PARAM_ANNOTATION_TYPE; static final String PARAM_CONSTRAINT; static final String PARAM_LEMMA_MODE; static final String PARAM_POS_VALUE; static final String PARAM_CPOS_VALUE; static final String PARAM_POS_TYPE; static final String PARAM_LANGUAGE; static final String PARAM_POS_MAPPING_LOCATION; }
@Test public void testSimpleMerge() throws Exception { AnalysisEngine filter = createEngine(TokenMerger.class, TokenMerger.PARAM_ANNOTATION_TYPE, NamedEntity.class); JCas jcas = initCas(); filter.process(jcas); assertEquals(asList("I", "love", "New York", "."), pick(select(jcas, Token.class), "cas:text()")); }
@Test public void testWithConstraintMatch() throws Exception { AnalysisEngine filter = createEngine(TokenMerger.class, TokenMerger.PARAM_ANNOTATION_TYPE, NamedEntity.class, TokenMerger.PARAM_CONSTRAINT, ".[value = 'LOCATION']"); JCas jcas = initCas(); filter.process(jcas); assertEquals(asList("I", "love", "New York", "."), toText(select(jcas, Token.class))); }
@Test public void testWithConstraintNoMatch() throws Exception { AnalysisEngine filter = createEngine(TokenMerger.class, TokenMerger.PARAM_ANNOTATION_TYPE, NamedEntity.class, TokenMerger.PARAM_CONSTRAINT, ".[value = 'PERSON']"); JCas jcas = initCas(); filter.process(jcas); assertEquals(asList("I", "love", "New", "York", "."), toText(select(jcas, Token.class))); }
@Test public void testSimpleMergeLemmaJoin() throws Exception { AnalysisEngine filter = createEngine(TokenMerger.class, TokenMerger.PARAM_ANNOTATION_TYPE, NamedEntity.class, TokenMerger.PARAM_LEMMA_MODE, LemmaMode.JOIN); JCas jcas = initCas(); filter.process(jcas); assertEquals(asList("I", "love", "new york", "."), pick(select(jcas, Token.class), "./lemma/value")); }