@Test public void testProcessWithNormalizationAndApproximateMatching() throws Exception { ExternalResourceDescription extDesc = ExternalResourceFactory.createExternalResourceDescription( ChunkerProviderImplAlt.class, new File("src/test/resources/normalizegazetteer.properties")); TypeSystemDescription tsDesc = TypeSystemDescriptionFactory .createTypeSystemDescription("de.julielab.jcore.types.jcore-semantics-mention-types"); AnalysisEngine gazetteerAnnotator = AnalysisEngineFactory.createPrimitive(GazetteerAnnotator.class, tsDesc, GazetteerAnnotator.PARAM_CHECK_ACRONYMS, false, GazetteerAnnotator.PARAM_OUTPUT_TYPE, "de.julielab.jcore.types.OntClassMention", GazetteerAnnotator.CHUNKER_RESOURCE_NAME, extDesc); JCas myCAS = gazetteerAnnotator.newJCas(); myCAS.reset(); myCAS.setDocumentText(readFile2String(new File(TEXT_PLAIN))); setAbbreviations(myCAS); gazetteerAnnotator.process(myCAS); JFSIndexRepository indexes = myCAS.getJFSIndexRepository(); FSIterator<org.apache.uima.jcas.tcas.Annotation> entityIter = indexes.getAnnotationIndex(OntClassMention.type) .iterator(); int entCount = 0; LOGGER.debug("\n\n+++ OUTPUTTING ENTITIES +++ OUTPUTTING ENTITIES +++ OUTPUTTING ENTITIES +++\n"); Set<String> comparisonSet = new HashSet<>(); while (entityIter.hasNext()) { OntClassMention e = (OntClassMention) entityIter.next(); comparisonSet.add(e.getCoveredText()); LOGGER.debug("entity: " + e.getCoveredText() + "\n" + e); entCount++; } assertEquals(8, entCount); }
public void process(JCas aJCas) throws AnalysisEngineProcessException { if (gazetteer == null) throw new IllegalStateException("The actual gazetteer object is null. Check previous log messages pointing to the error (most probably the dictionary file could not be found)."); String docText = aJCas.getDocumentText(); if (docText == null || docText.length() == 0) return; if (provider.getUseApproximateMatching() && !provider.getTransliterate() && !provider.getCaseSensitive()) docText = docText.toLowerCase(); NormalizedString normalizedDocText = null; if (provider.getNormalize()) { normalizedDocText = StringNormalizerForChunking.normalizeString(docText, normalizationTokenFactory, transliterator); } IndexTermGenerator<Long> longOffsetTermGenerator = TermGenerators.longOffsetTermGenerator(); JCoReHashMapAnnotationIndex<Long, ConceptMention> conceptMentionIndex = new JCoReHashMapAnnotationIndex<>( longOffsetTermGenerator, longOffsetTermGenerator, aJCas, ConceptMention.type); JCoReHashMapAnnotationIndex<Long, Abbreviation> abbreviationIndex = new JCoReHashMapAnnotationIndex<>( longOffsetTermGenerator, longOffsetTermGenerator, aJCas, Abbreviation.type); LOGGER.debug("Performing actual Gazetteer annotation..."); Chunking chunking; if (provider.getNormalize()) chunking = gazetteer.chunk(normalizedDocText.string); else chunking = gazetteer.chunk(docText); LOGGER.debug("Gazetteer annotation done."); if (provider.getUseApproximateMatching()) { List<Chunk> chunkList = filterChunking(chunking); List<OverlappingChunk> overlappingChunks = groupOverlappingChunks(chunkList, chunking.charSequence().toString()); LOGGER.debug("all overlapping chunks:\n"); for (OverlappingChunk overlappingChunk : overlappingChunks) { LOGGER.debug(overlappingChunk.toStringAll()); List<Chunk> bestChunks = overlappingChunk.getBestChunks(); LOGGER.debug("Found {} best chunks.", bestChunks.size()); for (int i = 0; i < bestChunks.size(); i++) { Chunk bestChunk = bestChunks.get(i); LOGGER.debug("Nr. " + i + " best chunk: " + bestChunk.start() + " - " + bestChunk.end() + ": " + bestChunk.score() + " ; type: " + bestChunk.type()); add2Cas(aJCas, bestChunk, normalizedDocText, conceptMentionIndex, abbreviationIndex); } } } else { for (Chunk chunk : chunking.chunkSet()) { add2Cas(aJCas, chunk, normalizedDocText, conceptMentionIndex, abbreviationIndex); } } if (checkAcronyms && !mantraMode) { LOGGER.debug("process() - checking acronyms"); annotateAcronymsWithFullFormEntity(aJCas, conceptMentionIndex); } }
GazetteerAnnotator extends JCasAnnotator_ImplBase { public void process(JCas aJCas) throws AnalysisEngineProcessException { if (gazetteer == null) throw new IllegalStateException("The actual gazetteer object is null. Check previous log messages pointing to the error (most probably the dictionary file could not be found)."); String docText = aJCas.getDocumentText(); if (docText == null || docText.length() == 0) return; if (provider.getUseApproximateMatching() && !provider.getTransliterate() && !provider.getCaseSensitive()) docText = docText.toLowerCase(); NormalizedString normalizedDocText = null; if (provider.getNormalize()) { normalizedDocText = StringNormalizerForChunking.normalizeString(docText, normalizationTokenFactory, transliterator); } IndexTermGenerator<Long> longOffsetTermGenerator = TermGenerators.longOffsetTermGenerator(); JCoReHashMapAnnotationIndex<Long, ConceptMention> conceptMentionIndex = new JCoReHashMapAnnotationIndex<>( longOffsetTermGenerator, longOffsetTermGenerator, aJCas, ConceptMention.type); JCoReHashMapAnnotationIndex<Long, Abbreviation> abbreviationIndex = new JCoReHashMapAnnotationIndex<>( longOffsetTermGenerator, longOffsetTermGenerator, aJCas, Abbreviation.type); LOGGER.debug("Performing actual Gazetteer annotation..."); Chunking chunking; if (provider.getNormalize()) chunking = gazetteer.chunk(normalizedDocText.string); else chunking = gazetteer.chunk(docText); LOGGER.debug("Gazetteer annotation done."); if (provider.getUseApproximateMatching()) { List<Chunk> chunkList = filterChunking(chunking); List<OverlappingChunk> overlappingChunks = groupOverlappingChunks(chunkList, chunking.charSequence().toString()); LOGGER.debug("all overlapping chunks:\n"); for (OverlappingChunk overlappingChunk : overlappingChunks) { LOGGER.debug(overlappingChunk.toStringAll()); List<Chunk> bestChunks = overlappingChunk.getBestChunks(); LOGGER.debug("Found {} best chunks.", bestChunks.size()); for (int i = 0; i < bestChunks.size(); i++) { Chunk bestChunk = bestChunks.get(i); LOGGER.debug("Nr. " + i + " best chunk: " + bestChunk.start() + " - " + bestChunk.end() + ": " + bestChunk.score() + " ; type: " + bestChunk.type()); add2Cas(aJCas, bestChunk, normalizedDocText, conceptMentionIndex, abbreviationIndex); } } } else { for (Chunk chunk : chunking.chunkSet()) { add2Cas(aJCas, chunk, normalizedDocText, conceptMentionIndex, abbreviationIndex); } } if (checkAcronyms && !mantraMode) { LOGGER.debug("process() - checking acronyms"); annotateAcronymsWithFullFormEntity(aJCas, conceptMentionIndex); } } }
GazetteerAnnotator extends JCasAnnotator_ImplBase { public void process(JCas aJCas) throws AnalysisEngineProcessException { if (gazetteer == null) throw new IllegalStateException("The actual gazetteer object is null. Check previous log messages pointing to the error (most probably the dictionary file could not be found)."); String docText = aJCas.getDocumentText(); if (docText == null || docText.length() == 0) return; if (provider.getUseApproximateMatching() && !provider.getTransliterate() && !provider.getCaseSensitive()) docText = docText.toLowerCase(); NormalizedString normalizedDocText = null; if (provider.getNormalize()) { normalizedDocText = StringNormalizerForChunking.normalizeString(docText, normalizationTokenFactory, transliterator); } IndexTermGenerator<Long> longOffsetTermGenerator = TermGenerators.longOffsetTermGenerator(); JCoReHashMapAnnotationIndex<Long, ConceptMention> conceptMentionIndex = new JCoReHashMapAnnotationIndex<>( longOffsetTermGenerator, longOffsetTermGenerator, aJCas, ConceptMention.type); JCoReHashMapAnnotationIndex<Long, Abbreviation> abbreviationIndex = new JCoReHashMapAnnotationIndex<>( longOffsetTermGenerator, longOffsetTermGenerator, aJCas, Abbreviation.type); LOGGER.debug("Performing actual Gazetteer annotation..."); Chunking chunking; if (provider.getNormalize()) chunking = gazetteer.chunk(normalizedDocText.string); else chunking = gazetteer.chunk(docText); LOGGER.debug("Gazetteer annotation done."); if (provider.getUseApproximateMatching()) { List<Chunk> chunkList = filterChunking(chunking); List<OverlappingChunk> overlappingChunks = groupOverlappingChunks(chunkList, chunking.charSequence().toString()); LOGGER.debug("all overlapping chunks:\n"); for (OverlappingChunk overlappingChunk : overlappingChunks) { LOGGER.debug(overlappingChunk.toStringAll()); List<Chunk> bestChunks = overlappingChunk.getBestChunks(); LOGGER.debug("Found {} best chunks.", bestChunks.size()); for (int i = 0; i < bestChunks.size(); i++) { Chunk bestChunk = bestChunks.get(i); LOGGER.debug("Nr. " + i + " best chunk: " + bestChunk.start() + " - " + bestChunk.end() + ": " + bestChunk.score() + " ; type: " + bestChunk.type()); add2Cas(aJCas, bestChunk, normalizedDocText, conceptMentionIndex, abbreviationIndex); } } } else { for (Chunk chunk : chunking.chunkSet()) { add2Cas(aJCas, chunk, normalizedDocText, conceptMentionIndex, abbreviationIndex); } } if (checkAcronyms && !mantraMode) { LOGGER.debug("process() - checking acronyms"); annotateAcronymsWithFullFormEntity(aJCas, conceptMentionIndex); } }  }
GazetteerAnnotator extends JCasAnnotator_ImplBase { public void process(JCas aJCas) throws AnalysisEngineProcessException { if (gazetteer == null) throw new IllegalStateException("The actual gazetteer object is null. Check previous log messages pointing to the error (most probably the dictionary file could not be found)."); String docText = aJCas.getDocumentText(); if (docText == null || docText.length() == 0) return; if (provider.getUseApproximateMatching() && !provider.getTransliterate() && !provider.getCaseSensitive()) docText = docText.toLowerCase(); NormalizedString normalizedDocText = null; if (provider.getNormalize()) { normalizedDocText = StringNormalizerForChunking.normalizeString(docText, normalizationTokenFactory, transliterator); } IndexTermGenerator<Long> longOffsetTermGenerator = TermGenerators.longOffsetTermGenerator(); JCoReHashMapAnnotationIndex<Long, ConceptMention> conceptMentionIndex = new JCoReHashMapAnnotationIndex<>( longOffsetTermGenerator, longOffsetTermGenerator, aJCas, ConceptMention.type); JCoReHashMapAnnotationIndex<Long, Abbreviation> abbreviationIndex = new JCoReHashMapAnnotationIndex<>( longOffsetTermGenerator, longOffsetTermGenerator, aJCas, Abbreviation.type); LOGGER.debug("Performing actual Gazetteer annotation..."); Chunking chunking; if (provider.getNormalize()) chunking = gazetteer.chunk(normalizedDocText.string); else chunking = gazetteer.chunk(docText); LOGGER.debug("Gazetteer annotation done."); if (provider.getUseApproximateMatching()) { List<Chunk> chunkList = filterChunking(chunking); List<OverlappingChunk> overlappingChunks = groupOverlappingChunks(chunkList, chunking.charSequence().toString()); LOGGER.debug("all overlapping chunks:\n"); for (OverlappingChunk overlappingChunk : overlappingChunks) { LOGGER.debug(overlappingChunk.toStringAll()); List<Chunk> bestChunks = overlappingChunk.getBestChunks(); LOGGER.debug("Found {} best chunks.", bestChunks.size()); for (int i = 0; i < bestChunks.size(); i++) { Chunk bestChunk = bestChunks.get(i); LOGGER.debug("Nr. " + i + " best chunk: " + bestChunk.start() + " - " + bestChunk.end() + ": " + bestChunk.score() + " ; type: " + bestChunk.type()); add2Cas(aJCas, bestChunk, normalizedDocText, conceptMentionIndex, abbreviationIndex); } } } else { for (Chunk chunk : chunking.chunkSet()) { add2Cas(aJCas, chunk, normalizedDocText, conceptMentionIndex, abbreviationIndex); } } if (checkAcronyms && !mantraMode) { LOGGER.debug("process() - checking acronyms"); annotateAcronymsWithFullFormEntity(aJCas, conceptMentionIndex); } }  void initialize(UimaContext aContext); void process(JCas aJCas); }
GazetteerAnnotator extends JCasAnnotator_ImplBase { public void process(JCas aJCas) throws AnalysisEngineProcessException { if (gazetteer == null) throw new IllegalStateException("The actual gazetteer object is null. Check previous log messages pointing to the error (most probably the dictionary file could not be found)."); String docText = aJCas.getDocumentText(); if (docText == null || docText.length() == 0) return; if (provider.getUseApproximateMatching() && !provider.getTransliterate() && !provider.getCaseSensitive()) docText = docText.toLowerCase(); NormalizedString normalizedDocText = null; if (provider.getNormalize()) { normalizedDocText = StringNormalizerForChunking.normalizeString(docText, normalizationTokenFactory, transliterator); } IndexTermGenerator<Long> longOffsetTermGenerator = TermGenerators.longOffsetTermGenerator(); JCoReHashMapAnnotationIndex<Long, ConceptMention> conceptMentionIndex = new JCoReHashMapAnnotationIndex<>( longOffsetTermGenerator, longOffsetTermGenerator, aJCas, ConceptMention.type); JCoReHashMapAnnotationIndex<Long, Abbreviation> abbreviationIndex = new JCoReHashMapAnnotationIndex<>( longOffsetTermGenerator, longOffsetTermGenerator, aJCas, Abbreviation.type); LOGGER.debug("Performing actual Gazetteer annotation..."); Chunking chunking; if (provider.getNormalize()) chunking = gazetteer.chunk(normalizedDocText.string); else chunking = gazetteer.chunk(docText); LOGGER.debug("Gazetteer annotation done."); if (provider.getUseApproximateMatching()) { List<Chunk> chunkList = filterChunking(chunking); List<OverlappingChunk> overlappingChunks = groupOverlappingChunks(chunkList, chunking.charSequence().toString()); LOGGER.debug("all overlapping chunks:\n"); for (OverlappingChunk overlappingChunk : overlappingChunks) { LOGGER.debug(overlappingChunk.toStringAll()); List<Chunk> bestChunks = overlappingChunk.getBestChunks(); LOGGER.debug("Found {} best chunks.", bestChunks.size()); for (int i = 0; i < bestChunks.size(); i++) { Chunk bestChunk = bestChunks.get(i); LOGGER.debug("Nr. " + i + " best chunk: " + bestChunk.start() + " - " + bestChunk.end() + ": " + bestChunk.score() + " ; type: " + bestChunk.type()); add2Cas(aJCas, bestChunk, normalizedDocText, conceptMentionIndex, abbreviationIndex); } } } else { for (Chunk chunk : chunking.chunkSet()) { add2Cas(aJCas, chunk, normalizedDocText, conceptMentionIndex, abbreviationIndex); } } if (checkAcronyms && !mantraMode) { LOGGER.debug("process() - checking acronyms"); annotateAcronymsWithFullFormEntity(aJCas, conceptMentionIndex); } }  void initialize(UimaContext aContext); void process(JCas aJCas); static final String CHUNKER_RESOURCE_NAME; final static String PARAM_CHECK_ACRONYMS; final static String PARAM_OUTPUT_TYPE; }