@Test public void testAnnotatorWithTextNormalizationMuh() throws ResourceInitializationException, AnalysisEngineProcessException { ExternalResourceDescription extDesc = ExternalResourceFactory.createExternalResourceDescription( ChunkerProviderImplAlt.class, new File("src/test/resources/normalizegazetteer.properties")); TypeSystemDescription tsDesc = TypeSystemDescriptionFactory .createTypeSystemDescription("de.julielab.jcore.types.jcore-semantics-mention-types"); AnalysisEngine annotator = AnalysisEngineFactory.createPrimitive(GazetteerAnnotator.class, tsDesc, GazetteerAnnotator.PARAM_OUTPUT_TYPE, "de.julielab.jcore.types.EntityMention", GazetteerAnnotator.CHUNKER_RESOURCE_NAME, extDesc); JCas jCas = annotator.newJCas(); jCas.setDocumentText("We shall now describe our system setup followed by our proposed solution, which is a fully distributed and absolute localization solution specifically designed for both one-hop and multi-hop WSNs. Our considered WSN consists of Ns number of sensors randomly placed onto a map of predefined size with Nb number of beacons. Let 𝕊 and 𝔹 be the sets describing all sensors and beacons respectively, where each sensor is noted as Sensori, i ∈ 𝕊 and each beacon is noted as Beaconj, j ∈ 𝔹. Each node either a sensor or a beacon is noted as Nodep, p ∈ 𝕊 ∪ 𝔹, and vector V⃗p is used to represent the coordinate of Nodep. Beacons are placed onto the map with fixed coordinates V⃗j, where j ∈ 𝔹. We assume that each beacon is aware of its own absolute location. Whereas each sensor is unaware of its own location, and is configured with an initial guess of location unrelated to its actual deployed location. The two-dimensional (2-D) localization problem is the estimation of Ns unknown-location coordinates V⃗i, where i ∈ 𝕊.\n"); annotator.process(jCas); FSIterator<org.apache.uima.jcas.tcas.Annotation> it = jCas.getAnnotationIndex(EntityMention.type).iterator(); while (it.hasNext()) { Annotation annotation = (Annotation) it.next(); System.out.println(annotation.getCoveredText()); } }
public void process(JCas aJCas) throws AnalysisEngineProcessException { if (gazetteer == null) throw new IllegalStateException("The actual gazetteer object is null. Check previous log messages pointing to the error (most probably the dictionary file could not be found)."); String docText = aJCas.getDocumentText(); if (docText == null || docText.length() == 0) return; if (provider.getUseApproximateMatching() && !provider.getTransliterate() && !provider.getCaseSensitive()) docText = docText.toLowerCase(); NormalizedString normalizedDocText = null; if (provider.getNormalize()) { normalizedDocText = StringNormalizerForChunking.normalizeString(docText, normalizationTokenFactory, transliterator); } IndexTermGenerator<Long> longOffsetTermGenerator = TermGenerators.longOffsetTermGenerator(); JCoReHashMapAnnotationIndex<Long, ConceptMention> conceptMentionIndex = new JCoReHashMapAnnotationIndex<>( longOffsetTermGenerator, longOffsetTermGenerator, aJCas, ConceptMention.type); JCoReHashMapAnnotationIndex<Long, Abbreviation> abbreviationIndex = new JCoReHashMapAnnotationIndex<>( longOffsetTermGenerator, longOffsetTermGenerator, aJCas, Abbreviation.type); LOGGER.debug("Performing actual Gazetteer annotation..."); Chunking chunking; if (provider.getNormalize()) chunking = gazetteer.chunk(normalizedDocText.string); else chunking = gazetteer.chunk(docText); LOGGER.debug("Gazetteer annotation done."); if (provider.getUseApproximateMatching()) { List<Chunk> chunkList = filterChunking(chunking); List<OverlappingChunk> overlappingChunks = groupOverlappingChunks(chunkList, chunking.charSequence().toString()); LOGGER.debug("all overlapping chunks:\n"); for (OverlappingChunk overlappingChunk : overlappingChunks) { LOGGER.debug(overlappingChunk.toStringAll()); List<Chunk> bestChunks = overlappingChunk.getBestChunks(); LOGGER.debug("Found {} best chunks.", bestChunks.size()); for (int i = 0; i < bestChunks.size(); i++) { Chunk bestChunk = bestChunks.get(i); LOGGER.debug("Nr. " + i + " best chunk: " + bestChunk.start() + " - " + bestChunk.end() + ": " + bestChunk.score() + " ; type: " + bestChunk.type()); add2Cas(aJCas, bestChunk, normalizedDocText, conceptMentionIndex, abbreviationIndex); } } } else { for (Chunk chunk : chunking.chunkSet()) { add2Cas(aJCas, chunk, normalizedDocText, conceptMentionIndex, abbreviationIndex); } } if (checkAcronyms && !mantraMode) { LOGGER.debug("process() - checking acronyms"); annotateAcronymsWithFullFormEntity(aJCas, conceptMentionIndex); } }
GazetteerAnnotator extends JCasAnnotator_ImplBase { public void process(JCas aJCas) throws AnalysisEngineProcessException { if (gazetteer == null) throw new IllegalStateException("The actual gazetteer object is null. Check previous log messages pointing to the error (most probably the dictionary file could not be found)."); String docText = aJCas.getDocumentText(); if (docText == null || docText.length() == 0) return; if (provider.getUseApproximateMatching() && !provider.getTransliterate() && !provider.getCaseSensitive()) docText = docText.toLowerCase(); NormalizedString normalizedDocText = null; if (provider.getNormalize()) { normalizedDocText = StringNormalizerForChunking.normalizeString(docText, normalizationTokenFactory, transliterator); } IndexTermGenerator<Long> longOffsetTermGenerator = TermGenerators.longOffsetTermGenerator(); JCoReHashMapAnnotationIndex<Long, ConceptMention> conceptMentionIndex = new JCoReHashMapAnnotationIndex<>( longOffsetTermGenerator, longOffsetTermGenerator, aJCas, ConceptMention.type); JCoReHashMapAnnotationIndex<Long, Abbreviation> abbreviationIndex = new JCoReHashMapAnnotationIndex<>( longOffsetTermGenerator, longOffsetTermGenerator, aJCas, Abbreviation.type); LOGGER.debug("Performing actual Gazetteer annotation..."); Chunking chunking; if (provider.getNormalize()) chunking = gazetteer.chunk(normalizedDocText.string); else chunking = gazetteer.chunk(docText); LOGGER.debug("Gazetteer annotation done."); if (provider.getUseApproximateMatching()) { List<Chunk> chunkList = filterChunking(chunking); List<OverlappingChunk> overlappingChunks = groupOverlappingChunks(chunkList, chunking.charSequence().toString()); LOGGER.debug("all overlapping chunks:\n"); for (OverlappingChunk overlappingChunk : overlappingChunks) { LOGGER.debug(overlappingChunk.toStringAll()); List<Chunk> bestChunks = overlappingChunk.getBestChunks(); LOGGER.debug("Found {} best chunks.", bestChunks.size()); for (int i = 0; i < bestChunks.size(); i++) { Chunk bestChunk = bestChunks.get(i); LOGGER.debug("Nr. " + i + " best chunk: " + bestChunk.start() + " - " + bestChunk.end() + ": " + bestChunk.score() + " ; type: " + bestChunk.type()); add2Cas(aJCas, bestChunk, normalizedDocText, conceptMentionIndex, abbreviationIndex); } } } else { for (Chunk chunk : chunking.chunkSet()) { add2Cas(aJCas, chunk, normalizedDocText, conceptMentionIndex, abbreviationIndex); } } if (checkAcronyms && !mantraMode) { LOGGER.debug("process() - checking acronyms"); annotateAcronymsWithFullFormEntity(aJCas, conceptMentionIndex); } } }
GazetteerAnnotator extends JCasAnnotator_ImplBase { public void process(JCas aJCas) throws AnalysisEngineProcessException { if (gazetteer == null) throw new IllegalStateException("The actual gazetteer object is null. Check previous log messages pointing to the error (most probably the dictionary file could not be found)."); String docText = aJCas.getDocumentText(); if (docText == null || docText.length() == 0) return; if (provider.getUseApproximateMatching() && !provider.getTransliterate() && !provider.getCaseSensitive()) docText = docText.toLowerCase(); NormalizedString normalizedDocText = null; if (provider.getNormalize()) { normalizedDocText = StringNormalizerForChunking.normalizeString(docText, normalizationTokenFactory, transliterator); } IndexTermGenerator<Long> longOffsetTermGenerator = TermGenerators.longOffsetTermGenerator(); JCoReHashMapAnnotationIndex<Long, ConceptMention> conceptMentionIndex = new JCoReHashMapAnnotationIndex<>( longOffsetTermGenerator, longOffsetTermGenerator, aJCas, ConceptMention.type); JCoReHashMapAnnotationIndex<Long, Abbreviation> abbreviationIndex = new JCoReHashMapAnnotationIndex<>( longOffsetTermGenerator, longOffsetTermGenerator, aJCas, Abbreviation.type); LOGGER.debug("Performing actual Gazetteer annotation..."); Chunking chunking; if (provider.getNormalize()) chunking = gazetteer.chunk(normalizedDocText.string); else chunking = gazetteer.chunk(docText); LOGGER.debug("Gazetteer annotation done."); if (provider.getUseApproximateMatching()) { List<Chunk> chunkList = filterChunking(chunking); List<OverlappingChunk> overlappingChunks = groupOverlappingChunks(chunkList, chunking.charSequence().toString()); LOGGER.debug("all overlapping chunks:\n"); for (OverlappingChunk overlappingChunk : overlappingChunks) { LOGGER.debug(overlappingChunk.toStringAll()); List<Chunk> bestChunks = overlappingChunk.getBestChunks(); LOGGER.debug("Found {} best chunks.", bestChunks.size()); for (int i = 0; i < bestChunks.size(); i++) { Chunk bestChunk = bestChunks.get(i); LOGGER.debug("Nr. " + i + " best chunk: " + bestChunk.start() + " - " + bestChunk.end() + ": " + bestChunk.score() + " ; type: " + bestChunk.type()); add2Cas(aJCas, bestChunk, normalizedDocText, conceptMentionIndex, abbreviationIndex); } } } else { for (Chunk chunk : chunking.chunkSet()) { add2Cas(aJCas, chunk, normalizedDocText, conceptMentionIndex, abbreviationIndex); } } if (checkAcronyms && !mantraMode) { LOGGER.debug("process() - checking acronyms"); annotateAcronymsWithFullFormEntity(aJCas, conceptMentionIndex); } }  }
GazetteerAnnotator extends JCasAnnotator_ImplBase { public void process(JCas aJCas) throws AnalysisEngineProcessException { if (gazetteer == null) throw new IllegalStateException("The actual gazetteer object is null. Check previous log messages pointing to the error (most probably the dictionary file could not be found)."); String docText = aJCas.getDocumentText(); if (docText == null || docText.length() == 0) return; if (provider.getUseApproximateMatching() && !provider.getTransliterate() && !provider.getCaseSensitive()) docText = docText.toLowerCase(); NormalizedString normalizedDocText = null; if (provider.getNormalize()) { normalizedDocText = StringNormalizerForChunking.normalizeString(docText, normalizationTokenFactory, transliterator); } IndexTermGenerator<Long> longOffsetTermGenerator = TermGenerators.longOffsetTermGenerator(); JCoReHashMapAnnotationIndex<Long, ConceptMention> conceptMentionIndex = new JCoReHashMapAnnotationIndex<>( longOffsetTermGenerator, longOffsetTermGenerator, aJCas, ConceptMention.type); JCoReHashMapAnnotationIndex<Long, Abbreviation> abbreviationIndex = new JCoReHashMapAnnotationIndex<>( longOffsetTermGenerator, longOffsetTermGenerator, aJCas, Abbreviation.type); LOGGER.debug("Performing actual Gazetteer annotation..."); Chunking chunking; if (provider.getNormalize()) chunking = gazetteer.chunk(normalizedDocText.string); else chunking = gazetteer.chunk(docText); LOGGER.debug("Gazetteer annotation done."); if (provider.getUseApproximateMatching()) { List<Chunk> chunkList = filterChunking(chunking); List<OverlappingChunk> overlappingChunks = groupOverlappingChunks(chunkList, chunking.charSequence().toString()); LOGGER.debug("all overlapping chunks:\n"); for (OverlappingChunk overlappingChunk : overlappingChunks) { LOGGER.debug(overlappingChunk.toStringAll()); List<Chunk> bestChunks = overlappingChunk.getBestChunks(); LOGGER.debug("Found {} best chunks.", bestChunks.size()); for (int i = 0; i < bestChunks.size(); i++) { Chunk bestChunk = bestChunks.get(i); LOGGER.debug("Nr. " + i + " best chunk: " + bestChunk.start() + " - " + bestChunk.end() + ": " + bestChunk.score() + " ; type: " + bestChunk.type()); add2Cas(aJCas, bestChunk, normalizedDocText, conceptMentionIndex, abbreviationIndex); } } } else { for (Chunk chunk : chunking.chunkSet()) { add2Cas(aJCas, chunk, normalizedDocText, conceptMentionIndex, abbreviationIndex); } } if (checkAcronyms && !mantraMode) { LOGGER.debug("process() - checking acronyms"); annotateAcronymsWithFullFormEntity(aJCas, conceptMentionIndex); } }  void initialize(UimaContext aContext); void process(JCas aJCas); }
GazetteerAnnotator extends JCasAnnotator_ImplBase { public void process(JCas aJCas) throws AnalysisEngineProcessException { if (gazetteer == null) throw new IllegalStateException("The actual gazetteer object is null. Check previous log messages pointing to the error (most probably the dictionary file could not be found)."); String docText = aJCas.getDocumentText(); if (docText == null || docText.length() == 0) return; if (provider.getUseApproximateMatching() && !provider.getTransliterate() && !provider.getCaseSensitive()) docText = docText.toLowerCase(); NormalizedString normalizedDocText = null; if (provider.getNormalize()) { normalizedDocText = StringNormalizerForChunking.normalizeString(docText, normalizationTokenFactory, transliterator); } IndexTermGenerator<Long> longOffsetTermGenerator = TermGenerators.longOffsetTermGenerator(); JCoReHashMapAnnotationIndex<Long, ConceptMention> conceptMentionIndex = new JCoReHashMapAnnotationIndex<>( longOffsetTermGenerator, longOffsetTermGenerator, aJCas, ConceptMention.type); JCoReHashMapAnnotationIndex<Long, Abbreviation> abbreviationIndex = new JCoReHashMapAnnotationIndex<>( longOffsetTermGenerator, longOffsetTermGenerator, aJCas, Abbreviation.type); LOGGER.debug("Performing actual Gazetteer annotation..."); Chunking chunking; if (provider.getNormalize()) chunking = gazetteer.chunk(normalizedDocText.string); else chunking = gazetteer.chunk(docText); LOGGER.debug("Gazetteer annotation done."); if (provider.getUseApproximateMatching()) { List<Chunk> chunkList = filterChunking(chunking); List<OverlappingChunk> overlappingChunks = groupOverlappingChunks(chunkList, chunking.charSequence().toString()); LOGGER.debug("all overlapping chunks:\n"); for (OverlappingChunk overlappingChunk : overlappingChunks) { LOGGER.debug(overlappingChunk.toStringAll()); List<Chunk> bestChunks = overlappingChunk.getBestChunks(); LOGGER.debug("Found {} best chunks.", bestChunks.size()); for (int i = 0; i < bestChunks.size(); i++) { Chunk bestChunk = bestChunks.get(i); LOGGER.debug("Nr. " + i + " best chunk: " + bestChunk.start() + " - " + bestChunk.end() + ": " + bestChunk.score() + " ; type: " + bestChunk.type()); add2Cas(aJCas, bestChunk, normalizedDocText, conceptMentionIndex, abbreviationIndex); } } } else { for (Chunk chunk : chunking.chunkSet()) { add2Cas(aJCas, chunk, normalizedDocText, conceptMentionIndex, abbreviationIndex); } } if (checkAcronyms && !mantraMode) { LOGGER.debug("process() - checking acronyms"); annotateAcronymsWithFullFormEntity(aJCas, conceptMentionIndex); } }  void initialize(UimaContext aContext); void process(JCas aJCas); static final String CHUNKER_RESOURCE_NAME; final static String PARAM_CHECK_ACRONYMS; final static String PARAM_OUTPUT_TYPE; }