CandidateAnnotator extends JCasAnnotator_ImplBase { @Override public void process(JCas jcas) throws AnalysisEngineProcessException { getContext().getLogger().log(Level.FINE, "Entering " + this.getClass().getSimpleName()); List<Candidate> candidates = new ArrayList<Candidate>(); try { for (Entry<AnnotationFS, String> entry: FeaturePathFactory.select(jcas.getCas(), featurePath)) { candidates.add(new Candidate(entry.getValue(), entry.getKey().getBegin(), entry.getKey().getEnd())); } } catch (FeaturePathException e) { throw new AnalysisEngineProcessException(e); } List<Candidate> resolvedCandidates; if (resolveOverlaps) { resolvedCandidates = resolveOverlappingCandidates(candidates); } else { resolvedCandidates = candidates; } AnnotationIndex<Annotation> candidateIndex = jcas.getAnnotationIndex(Keyphrase.type); for (Candidate candidate : resolvedCandidates) { Keyphrase keyphrase = new Keyphrase(jcas); keyphrase.setKeyphrase(candidate.term); keyphrase.setBegin(candidate.begin); keyphrase.setEnd(candidate.end); if (!candidateIndex.contains(keyphrase)) { keyphrase.addToIndexes(jcas); } } }  @Override void process(JCas jcas); static final String PARAM_FEATURE_PATH; static final String PARAM_RESOLVE_OVERLAPS; }
@Test public void KeyphraseCandidateTest1() throws Exception { System.out.println("Only tokens. Default settings"); Set<String> expectedResults = new HashSet<String>(); expectedResults.add("example"); expectedResults.add("sentence"); expectedResults.add("funny"); expectedResults.add("second"); AnalysisEngine analysisEngine = AnalysisEngineFactory.createEngine( CandidateAnnotator.class, CandidateAnnotator.PARAM_FEATURE_PATH, Token.class.getName(), CandidateAnnotator.PARAM_RESOLVE_OVERLAPS, false); String testDocument = "example sentence funny. second example."; JCas jcas = setup(testDocument, analysisEngine); analysisEngine.process(jcas); int i=0; for (Keyphrase kc : JCasUtil.select(jcas, Keyphrase.class)) { System.out.println(kc); assertTrue(expectedResults.contains(kc.getKeyphrase())); i++; } assertEquals(5,i); }
@Test public void KeyphraseCandidateTest2() throws Exception { System.out.println("Tokens and NGrams. NGram covers two tokens."); Set<String> expectedResults = new HashSet<String>(); expectedResults.add("example sentence"); expectedResults.add("funny"); expectedResults.add("second"); expectedResults.add("example"); expectedResults.add("sentence"); AnalysisEngine analysisEngine = AnalysisEngineFactory.createEngine( CandidateAnnotator.class, CandidateAnnotator.PARAM_FEATURE_PATH, Token.class.getName(), CandidateAnnotator.PARAM_RESOLVE_OVERLAPS, true); AnalysisEngine analysisEngine2 = AnalysisEngineFactory.createEngine( CandidateAnnotator.class, CandidateAnnotator.PARAM_FEATURE_PATH, NGram.class.getName(), CandidateAnnotator.PARAM_RESOLVE_OVERLAPS, true); String testDocument = "example sentence funny. second example."; JCas jcas = setup(testDocument, analysisEngine); analysisEngine.process(jcas); analysisEngine2.process(jcas); int i=0; for (Keyphrase kc : JCasUtil.select(jcas, Keyphrase.class)) { System.out.println(kc); assertTrue(expectedResults.contains(kc.getKeyphrase())); i++; } assertEquals(6,i); }
@Test public void KeyphraseCandidateTest3() throws Exception { System.out.println("Tokens and Chunks. Chunks equals token. Should only appear once."); Set<String> expectedResults = new HashSet<String>(); expectedResults.add("example"); expectedResults.add("sentence"); expectedResults.add("funny"); expectedResults.add("second"); AnalysisEngine analysisEngine = AnalysisEngineFactory.createEngine( CandidateAnnotator.class, CandidateAnnotator.PARAM_FEATURE_PATH, Token.class.getName(), CandidateAnnotator.PARAM_RESOLVE_OVERLAPS, false); AnalysisEngine analysisEngine2 = AnalysisEngineFactory.createEngine( CandidateAnnotator.class, CandidateAnnotator.PARAM_FEATURE_PATH, NC.class.getName(), CandidateAnnotator.PARAM_RESOLVE_OVERLAPS, false); String testDocument = "example sentence funny. second example."; JCas jcas = setup(testDocument, analysisEngine); analysisEngine.process(jcas); analysisEngine2.process(jcas); int i=0; for (Keyphrase kc : JCasUtil.select(jcas, Keyphrase.class)) { System.out.println(kc); assertTrue(expectedResults.contains(kc.getKeyphrase())); i++; } assertEquals(5,i); }
@Test public void KeyphraseCandidateTest4() throws Exception { System.out.println("Only ngrams and named entities."); Set<String> expectedResults = new HashSet<String>(); expectedResults.add("example sentence"); expectedResults.add("sentence funny"); AnalysisEngine analysisEngine = AnalysisEngineFactory.createEngine( CandidateAnnotator.class, CandidateAnnotator.PARAM_FEATURE_PATH, NamedEntity.class.getName(), CandidateAnnotator.PARAM_RESOLVE_OVERLAPS, false); AnalysisEngine analysisEngine2 = AnalysisEngineFactory.createEngine( CandidateAnnotator.class, CandidateAnnotator.PARAM_FEATURE_PATH, NGram.class.getName(), CandidateAnnotator.PARAM_RESOLVE_OVERLAPS, false); String testDocument = "example sentence funny. second example."; JCas jcas = setup(testDocument, analysisEngine); analysisEngine.process(jcas); analysisEngine2.process(jcas); int i=0; for (Keyphrase kc : JCasUtil.select(jcas, Keyphrase.class)) { System.out.println(kc); assertTrue(expectedResults.contains(kc.getKeyphrase())); i++; } assertEquals(2,i); }
@Test public void KeyphraseCandidateTest5() throws Exception { System.out.println("Only named entities. They partially overlap. Should give a single large candidate."); Set<String> expectedResults = new HashSet<String>(); expectedResults.add("example sentence funny second example"); AnalysisEngine analysisEngine = AnalysisEngineFactory.createEngine( CandidateAnnotator.class, CandidateAnnotator.PARAM_FEATURE_PATH, NamedEntity.class.getName(), CandidateAnnotator.PARAM_RESOLVE_OVERLAPS, true); String testDocument2 = "example sentence funny second example."; JCas jcas = setup2(testDocument2, analysisEngine); analysisEngine.process(jcas); int i=0; for (Keyphrase kc : JCasUtil.select(jcas, Keyphrase.class)) { System.out.println(kc); assertTrue(expectedResults.contains(kc.getKeyphrase())); i++; } assertEquals(1,i); }
@Test public void KeyphraseCandidateTest6() throws Exception { System.out.println("Only named entities. They partially overlap, but no resolve parameter is set."); Set<String> expectedResults = new HashSet<String>(); expectedResults.add("example sentence funny"); expectedResults.add("funny second example"); AnalysisEngine analysisEngine = AnalysisEngineFactory.createEngine( CandidateAnnotator.class, CandidateAnnotator.PARAM_FEATURE_PATH, NamedEntity.class.getName(), CandidateAnnotator.PARAM_RESOLVE_OVERLAPS, false); String testDocument2 = "example sentence funny second example."; JCas jcas = setup2(testDocument2, analysisEngine); analysisEngine.process(jcas); int i=0; for (Keyphrase kc : JCasUtil.select(jcas, Keyphrase.class)) { System.out.println(kc); assertTrue(expectedResults.contains(kc.getKeyphrase())); i++; } assertEquals(2,i); }
@Test public void KeyphraseCandidateTest7() throws Exception { AnalysisEngine engine = AnalysisEngineFactory.createEngine(AnalysisEngineFactory. createEngineDescription( AnalysisEngineFactory.createEngineDescription(BreakIteratorSegmenter.class), AnalysisEngineFactory.createEngineDescription(StanfordPosTagger.class), AnalysisEngineFactory.createEngineDescription(StanfordLemmatizer.class), CandidateAnnotatorFactory. getKeyphraseCandidateAnnotator_lemma(false))); JCas jcas = engine.newJCas(); jcas.setDocumentLanguage("en"); jcas.setDocumentText("This is a sentence."); engine.process(jcas); List<String> expectedResults = new ArrayList<String>(); expectedResults.add("this"); expectedResults.add("be"); expectedResults.add("a"); expectedResults.add("sentence"); expectedResults.add("."); int i=0; for (Keyphrase kc : JCasUtil.select(jcas, Keyphrase.class)) { System.out.println(kc); assertTrue(expectedResults.contains(kc.getKeyphrase())); i++; } assertEquals(5,i); }
@Test public void KeyphraseCandidateTest9() throws Exception { AnalysisEngine engine = AnalysisEngineFactory.createEngine(AnalysisEngineFactory. createEngineDescription( AnalysisEngineFactory.createEngineDescription(BreakIteratorSegmenter.class), AnalysisEngineFactory.createEngineDescription(StanfordPosTagger.class), AnalysisEngineFactory.createEngineDescription(StanfordLemmatizer.class), CandidateAnnotatorFactory.getKeyphraseCandidateAnnotator(N.class.getName(), false))); JCas jcas = engine.newJCas(); jcas.setDocumentLanguage("en"); jcas.setDocumentText("A noun phrase."); engine.process(jcas); List<String> expectedResults = new ArrayList<String>(); expectedResults.add("noun"); expectedResults.add("phrase"); int i=0; for (Keyphrase kc : JCasUtil.select(jcas, Keyphrase.class)) { System.out.println(kc); assertTrue(expectedResults.contains(kc.getKeyphrase())); i++; } assertEquals(2,i); }