@Test public void testAnnotatorWithTextNormalization() throws ResourceInitializationException, AnalysisEngineProcessException { ExternalResourceDescription extDesc = ExternalResourceFactory.createExternalResourceDescription( ChunkerProviderImplAlt.class, new File("src/test/resources/normalizegazetteer.properties")); TypeSystemDescription tsDesc = TypeSystemDescriptionFactory .createTypeSystemDescription("de.julielab.jcore.types.jcore-semantics-mention-types"); AnalysisEngine annotator = AnalysisEngineFactory.createPrimitive(GazetteerAnnotator.class, tsDesc, GazetteerAnnotator.PARAM_OUTPUT_TYPE, "de.julielab.jcore.types.EntityMention", GazetteerAnnotator.CHUNKER_RESOURCE_NAME, extDesc); JCas jCas = annotator.newJCas(); jCas.setDocumentText("SHP-1 and killer cell lectin like receptor G2 are evil."); annotator.process(jCas); FSIterator<org.apache.uima.jcas.tcas.Annotation> it = jCas.getAnnotationIndex(EntityMention.type).iterator(); assertTrue("There are no entity annotations in the CAS.", it.hasNext()); EntityMention em = (EntityMention) it.next(); assertEquals("Start wrong: ", new Integer(0), new Integer(em.getBegin())); assertEquals("End wrong: ", new Integer(5), new Integer(em.getEnd())); assertEquals("Wrong type: ", "SHP-1", em.getSpecificType()); assertTrue("The secnond entity annotations is missing.", it.hasNext()); em = (EntityMention) it.next(); assertEquals("Start wrong: ", new Integer(10), new Integer(em.getBegin())); assertEquals("End wrong: ", new Integer(45), new Integer(em.getEnd())); assertEquals("Wrong type: ", "KLRG2", em.getSpecificType()); assertFalse("There are too many annotations.", it.hasNext()); jCas.reset(); jCas.setDocumentText( "The chemical was ((2-n-butyl-6,7-dichloro-2-cyclopentyl-2,3-dihydro-1-oxo-1H-inden-5-yl)oxy)acetic acid."); annotator.process(jCas); it = jCas.getAnnotationIndex(EntityMention.type).iterator(); assertTrue("There are no entity annotations in the CAS.", it.hasNext()); em = (EntityMention) it.next(); assertEquals("Start wrong: ", new Integer(17), new Integer(em.getBegin())); assertEquals("End wrong: ", new Integer(103), new Integer(em.getEnd())); assertEquals("Wrong type: ", "CHEM", em.getSpecificType()); assertFalse("There are too many annotations.", it.hasNext()); jCas.reset(); jCas.setDocumentText( "The chemical was ((2-n butyl-6,7-dichloro-2 cyclopentyl-2,3-dihydro 1-oxo-1H-inden-5 yl)oxy)acetic acid."); annotator.process(jCas); it = jCas.getAnnotationIndex(EntityMention.type).iterator(); assertTrue("There are no entity annotations in the CAS.", it.hasNext()); em = (EntityMention) it.next(); assertEquals("Start wrong: ", new Integer(17), new Integer(em.getBegin())); assertEquals("End wrong: ", new Integer(103), new Integer(em.getEnd())); assertEquals("Wrong type: ", "CHEM", em.getSpecificType()); assertFalse("There are too many annotations.", it.hasNext()); jCas.reset(); jCas.setDocumentText( "The chemical was ((3-n-butyl-6,7-dichloro-2-cyclopentyl-2,3-dihydro-1-oxo-1H-inden-5-yl)oxy)acetic acid."); annotator.process(jCas); it = jCas.getAnnotationIndex(EntityMention.type).iterator(); assertFalse("There is an annotation in CAS although there shouldnt be.", it.hasNext()); jCas.reset(); jCas.setDocumentText("Test-dosing unit KLRg1 killer cell lectin like receptor G2 Parkinson's Disease"); annotator.process(jCas); it = jCas.getAnnotationIndex(EntityMention.type).iterator(); Integer counter = 0; while (it.hasNext()) { System.out.println(it.next().getCoveredText()); counter++; } assertEquals("Wrong entity count: ", new Integer(4), counter); }
public void process(JCas aJCas) throws AnalysisEngineProcessException { if (gazetteer == null) throw new IllegalStateException("The actual gazetteer object is null. Check previous log messages pointing to the error (most probably the dictionary file could not be found)."); String docText = aJCas.getDocumentText(); if (docText == null || docText.length() == 0) return; if (provider.getUseApproximateMatching() && !provider.getTransliterate() && !provider.getCaseSensitive()) docText = docText.toLowerCase(); NormalizedString normalizedDocText = null; if (provider.getNormalize()) { normalizedDocText = StringNormalizerForChunking.normalizeString(docText, normalizationTokenFactory, transliterator); } IndexTermGenerator<Long> longOffsetTermGenerator = TermGenerators.longOffsetTermGenerator(); JCoReHashMapAnnotationIndex<Long, ConceptMention> conceptMentionIndex = new JCoReHashMapAnnotationIndex<>( longOffsetTermGenerator, longOffsetTermGenerator, aJCas, ConceptMention.type); JCoReHashMapAnnotationIndex<Long, Abbreviation> abbreviationIndex = new JCoReHashMapAnnotationIndex<>( longOffsetTermGenerator, longOffsetTermGenerator, aJCas, Abbreviation.type); LOGGER.debug("Performing actual Gazetteer annotation..."); Chunking chunking; if (provider.getNormalize()) chunking = gazetteer.chunk(normalizedDocText.string); else chunking = gazetteer.chunk(docText); LOGGER.debug("Gazetteer annotation done."); if (provider.getUseApproximateMatching()) { List<Chunk> chunkList = filterChunking(chunking); List<OverlappingChunk> overlappingChunks = groupOverlappingChunks(chunkList, chunking.charSequence().toString()); LOGGER.debug("all overlapping chunks:\n"); for (OverlappingChunk overlappingChunk : overlappingChunks) { LOGGER.debug(overlappingChunk.toStringAll()); List<Chunk> bestChunks = overlappingChunk.getBestChunks(); LOGGER.debug("Found {} best chunks.", bestChunks.size()); for (int i = 0; i < bestChunks.size(); i++) { Chunk bestChunk = bestChunks.get(i); LOGGER.debug("Nr. " + i + " best chunk: " + bestChunk.start() + " - " + bestChunk.end() + ": " + bestChunk.score() + " ; type: " + bestChunk.type()); add2Cas(aJCas, bestChunk, normalizedDocText, conceptMentionIndex, abbreviationIndex); } } } else { for (Chunk chunk : chunking.chunkSet()) { add2Cas(aJCas, chunk, normalizedDocText, conceptMentionIndex, abbreviationIndex); } } if (checkAcronyms && !mantraMode) { LOGGER.debug("process() - checking acronyms"); annotateAcronymsWithFullFormEntity(aJCas, conceptMentionIndex); } }
GazetteerAnnotator extends JCasAnnotator_ImplBase { public void process(JCas aJCas) throws AnalysisEngineProcessException { if (gazetteer == null) throw new IllegalStateException("The actual gazetteer object is null. Check previous log messages pointing to the error (most probably the dictionary file could not be found)."); String docText = aJCas.getDocumentText(); if (docText == null || docText.length() == 0) return; if (provider.getUseApproximateMatching() && !provider.getTransliterate() && !provider.getCaseSensitive()) docText = docText.toLowerCase(); NormalizedString normalizedDocText = null; if (provider.getNormalize()) { normalizedDocText = StringNormalizerForChunking.normalizeString(docText, normalizationTokenFactory, transliterator); } IndexTermGenerator<Long> longOffsetTermGenerator = TermGenerators.longOffsetTermGenerator(); JCoReHashMapAnnotationIndex<Long, ConceptMention> conceptMentionIndex = new JCoReHashMapAnnotationIndex<>( longOffsetTermGenerator, longOffsetTermGenerator, aJCas, ConceptMention.type); JCoReHashMapAnnotationIndex<Long, Abbreviation> abbreviationIndex = new JCoReHashMapAnnotationIndex<>( longOffsetTermGenerator, longOffsetTermGenerator, aJCas, Abbreviation.type); LOGGER.debug("Performing actual Gazetteer annotation..."); Chunking chunking; if (provider.getNormalize()) chunking = gazetteer.chunk(normalizedDocText.string); else chunking = gazetteer.chunk(docText); LOGGER.debug("Gazetteer annotation done."); if (provider.getUseApproximateMatching()) { List<Chunk> chunkList = filterChunking(chunking); List<OverlappingChunk> overlappingChunks = groupOverlappingChunks(chunkList, chunking.charSequence().toString()); LOGGER.debug("all overlapping chunks:\n"); for (OverlappingChunk overlappingChunk : overlappingChunks) { LOGGER.debug(overlappingChunk.toStringAll()); List<Chunk> bestChunks = overlappingChunk.getBestChunks(); LOGGER.debug("Found {} best chunks.", bestChunks.size()); for (int i = 0; i < bestChunks.size(); i++) { Chunk bestChunk = bestChunks.get(i); LOGGER.debug("Nr. " + i + " best chunk: " + bestChunk.start() + " - " + bestChunk.end() + ": " + bestChunk.score() + " ; type: " + bestChunk.type()); add2Cas(aJCas, bestChunk, normalizedDocText, conceptMentionIndex, abbreviationIndex); } } } else { for (Chunk chunk : chunking.chunkSet()) { add2Cas(aJCas, chunk, normalizedDocText, conceptMentionIndex, abbreviationIndex); } } if (checkAcronyms && !mantraMode) { LOGGER.debug("process() - checking acronyms"); annotateAcronymsWithFullFormEntity(aJCas, conceptMentionIndex); } } }
GazetteerAnnotator extends JCasAnnotator_ImplBase { public void process(JCas aJCas) throws AnalysisEngineProcessException { if (gazetteer == null) throw new IllegalStateException("The actual gazetteer object is null. Check previous log messages pointing to the error (most probably the dictionary file could not be found)."); String docText = aJCas.getDocumentText(); if (docText == null || docText.length() == 0) return; if (provider.getUseApproximateMatching() && !provider.getTransliterate() && !provider.getCaseSensitive()) docText = docText.toLowerCase(); NormalizedString normalizedDocText = null; if (provider.getNormalize()) { normalizedDocText = StringNormalizerForChunking.normalizeString(docText, normalizationTokenFactory, transliterator); } IndexTermGenerator<Long> longOffsetTermGenerator = TermGenerators.longOffsetTermGenerator(); JCoReHashMapAnnotationIndex<Long, ConceptMention> conceptMentionIndex = new JCoReHashMapAnnotationIndex<>( longOffsetTermGenerator, longOffsetTermGenerator, aJCas, ConceptMention.type); JCoReHashMapAnnotationIndex<Long, Abbreviation> abbreviationIndex = new JCoReHashMapAnnotationIndex<>( longOffsetTermGenerator, longOffsetTermGenerator, aJCas, Abbreviation.type); LOGGER.debug("Performing actual Gazetteer annotation..."); Chunking chunking; if (provider.getNormalize()) chunking = gazetteer.chunk(normalizedDocText.string); else chunking = gazetteer.chunk(docText); LOGGER.debug("Gazetteer annotation done."); if (provider.getUseApproximateMatching()) { List<Chunk> chunkList = filterChunking(chunking); List<OverlappingChunk> overlappingChunks = groupOverlappingChunks(chunkList, chunking.charSequence().toString()); LOGGER.debug("all overlapping chunks:\n"); for (OverlappingChunk overlappingChunk : overlappingChunks) { LOGGER.debug(overlappingChunk.toStringAll()); List<Chunk> bestChunks = overlappingChunk.getBestChunks(); LOGGER.debug("Found {} best chunks.", bestChunks.size()); for (int i = 0; i < bestChunks.size(); i++) { Chunk bestChunk = bestChunks.get(i); LOGGER.debug("Nr. " + i + " best chunk: " + bestChunk.start() + " - " + bestChunk.end() + ": " + bestChunk.score() + " ; type: " + bestChunk.type()); add2Cas(aJCas, bestChunk, normalizedDocText, conceptMentionIndex, abbreviationIndex); } } } else { for (Chunk chunk : chunking.chunkSet()) { add2Cas(aJCas, chunk, normalizedDocText, conceptMentionIndex, abbreviationIndex); } } if (checkAcronyms && !mantraMode) { LOGGER.debug("process() - checking acronyms"); annotateAcronymsWithFullFormEntity(aJCas, conceptMentionIndex); } }  }
GazetteerAnnotator extends JCasAnnotator_ImplBase { public void process(JCas aJCas) throws AnalysisEngineProcessException { if (gazetteer == null) throw new IllegalStateException("The actual gazetteer object is null. Check previous log messages pointing to the error (most probably the dictionary file could not be found)."); String docText = aJCas.getDocumentText(); if (docText == null || docText.length() == 0) return; if (provider.getUseApproximateMatching() && !provider.getTransliterate() && !provider.getCaseSensitive()) docText = docText.toLowerCase(); NormalizedString normalizedDocText = null; if (provider.getNormalize()) { normalizedDocText = StringNormalizerForChunking.normalizeString(docText, normalizationTokenFactory, transliterator); } IndexTermGenerator<Long> longOffsetTermGenerator = TermGenerators.longOffsetTermGenerator(); JCoReHashMapAnnotationIndex<Long, ConceptMention> conceptMentionIndex = new JCoReHashMapAnnotationIndex<>( longOffsetTermGenerator, longOffsetTermGenerator, aJCas, ConceptMention.type); JCoReHashMapAnnotationIndex<Long, Abbreviation> abbreviationIndex = new JCoReHashMapAnnotationIndex<>( longOffsetTermGenerator, longOffsetTermGenerator, aJCas, Abbreviation.type); LOGGER.debug("Performing actual Gazetteer annotation..."); Chunking chunking; if (provider.getNormalize()) chunking = gazetteer.chunk(normalizedDocText.string); else chunking = gazetteer.chunk(docText); LOGGER.debug("Gazetteer annotation done."); if (provider.getUseApproximateMatching()) { List<Chunk> chunkList = filterChunking(chunking); List<OverlappingChunk> overlappingChunks = groupOverlappingChunks(chunkList, chunking.charSequence().toString()); LOGGER.debug("all overlapping chunks:\n"); for (OverlappingChunk overlappingChunk : overlappingChunks) { LOGGER.debug(overlappingChunk.toStringAll()); List<Chunk> bestChunks = overlappingChunk.getBestChunks(); LOGGER.debug("Found {} best chunks.", bestChunks.size()); for (int i = 0; i < bestChunks.size(); i++) { Chunk bestChunk = bestChunks.get(i); LOGGER.debug("Nr. " + i + " best chunk: " + bestChunk.start() + " - " + bestChunk.end() + ": " + bestChunk.score() + " ; type: " + bestChunk.type()); add2Cas(aJCas, bestChunk, normalizedDocText, conceptMentionIndex, abbreviationIndex); } } } else { for (Chunk chunk : chunking.chunkSet()) { add2Cas(aJCas, chunk, normalizedDocText, conceptMentionIndex, abbreviationIndex); } } if (checkAcronyms && !mantraMode) { LOGGER.debug("process() - checking acronyms"); annotateAcronymsWithFullFormEntity(aJCas, conceptMentionIndex); } }  void initialize(UimaContext aContext); void process(JCas aJCas); }
GazetteerAnnotator extends JCasAnnotator_ImplBase { public void process(JCas aJCas) throws AnalysisEngineProcessException { if (gazetteer == null) throw new IllegalStateException("The actual gazetteer object is null. Check previous log messages pointing to the error (most probably the dictionary file could not be found)."); String docText = aJCas.getDocumentText(); if (docText == null || docText.length() == 0) return; if (provider.getUseApproximateMatching() && !provider.getTransliterate() && !provider.getCaseSensitive()) docText = docText.toLowerCase(); NormalizedString normalizedDocText = null; if (provider.getNormalize()) { normalizedDocText = StringNormalizerForChunking.normalizeString(docText, normalizationTokenFactory, transliterator); } IndexTermGenerator<Long> longOffsetTermGenerator = TermGenerators.longOffsetTermGenerator(); JCoReHashMapAnnotationIndex<Long, ConceptMention> conceptMentionIndex = new JCoReHashMapAnnotationIndex<>( longOffsetTermGenerator, longOffsetTermGenerator, aJCas, ConceptMention.type); JCoReHashMapAnnotationIndex<Long, Abbreviation> abbreviationIndex = new JCoReHashMapAnnotationIndex<>( longOffsetTermGenerator, longOffsetTermGenerator, aJCas, Abbreviation.type); LOGGER.debug("Performing actual Gazetteer annotation..."); Chunking chunking; if (provider.getNormalize()) chunking = gazetteer.chunk(normalizedDocText.string); else chunking = gazetteer.chunk(docText); LOGGER.debug("Gazetteer annotation done."); if (provider.getUseApproximateMatching()) { List<Chunk> chunkList = filterChunking(chunking); List<OverlappingChunk> overlappingChunks = groupOverlappingChunks(chunkList, chunking.charSequence().toString()); LOGGER.debug("all overlapping chunks:\n"); for (OverlappingChunk overlappingChunk : overlappingChunks) { LOGGER.debug(overlappingChunk.toStringAll()); List<Chunk> bestChunks = overlappingChunk.getBestChunks(); LOGGER.debug("Found {} best chunks.", bestChunks.size()); for (int i = 0; i < bestChunks.size(); i++) { Chunk bestChunk = bestChunks.get(i); LOGGER.debug("Nr. " + i + " best chunk: " + bestChunk.start() + " - " + bestChunk.end() + ": " + bestChunk.score() + " ; type: " + bestChunk.type()); add2Cas(aJCas, bestChunk, normalizedDocText, conceptMentionIndex, abbreviationIndex); } } } else { for (Chunk chunk : chunking.chunkSet()) { add2Cas(aJCas, chunk, normalizedDocText, conceptMentionIndex, abbreviationIndex); } } if (checkAcronyms && !mantraMode) { LOGGER.debug("process() - checking acronyms"); annotateAcronymsWithFullFormEntity(aJCas, conceptMentionIndex); } }  void initialize(UimaContext aContext); void process(JCas aJCas); static final String CHUNKER_RESOURCE_NAME; final static String PARAM_CHECK_ACRONYMS; final static String PARAM_OUTPUT_TYPE; }