SentenceAnnotator extends JCasAnnotator_ImplBase { public void process(JCas aJCas) throws AnalysisEngineProcessException { if (StringUtils.isBlank(aJCas.getDocumentText())) { final String docId = JCoReTools.getDocId(aJCas); LOGGER.warn("The document text of document {} is empty.", docId); return; } JCoReCondensedDocumentText documentText; try { documentText = new JCoReCondensedDocumentText(aJCas, cutAwayTypes); } catch (ClassNotFoundException e1) { throw new AnalysisEngineProcessException(e1); } if (sentenceDelimiterTypes != null) { try { JCoReAnnotationIndexMerger indexMerger = new JCoReAnnotationIndexMerger(sentenceDelimiterTypes, false, null, aJCas); List<Integer> borders = new ArrayList<>(); borders.add(0); borders.add(aJCas.getDocumentText().length()); while (indexMerger.incrementAnnotation()) { Annotation a = (Annotation) indexMerger.getAnnotation(); borders.add(documentText.getCondensedOffsetForOriginalOffset(a.getBegin())); borders.add(documentText.getCondensedOffsetForOriginalOffset(a.getEnd())); } borders.sort(null); for (int i = 1; i < borders.size(); ++i) { int start = borders.get(i - 1); int end = borders.get(i); while (start < end && Character.isWhitespace(aJCas.getDocumentText().charAt(start))) ++start; String textSpan = documentText.getCodensedText().substring(start, end); if (!StringUtils.isBlank(textSpan)) doSegmentation(documentText, textSpan, start); } } catch (ClassNotFoundException e) { throw new AnalysisEngineProcessException(e); } } else { if (aJCas.getDocumentText() != null && aJCas.getDocumentText().length() > 0) { doSegmentation(documentText, documentText.getCodensedText(), 0); } else { if (numEmptyCases.get() < 10) { LOGGER.debug("document text empty. Skipping this document."); numEmptyCases.incrementAndGet(); } else if (numEmptyCases.get() == 10) { LOGGER.warn("Encountered 10 documents with an empty text body. This message will not appear again " + "to avoid scrolling in cases where this is expected."); } } } }  void initialize(UimaContext aContext); void process(JCas aJCas); static final String PARAM_MODEL_FILE; static final String PARAM_POSTPROCESSING; static final String PARAM_SENTENCE_DELIMITER_TYPES; static final String PARAM_CUT_AWAY_TYPES; static final String PARAM_MAX_SENTENCE_LENGTH; static final String PARAM_ALWAYS_SPLIT_NEWLINE; }
@Test public void testSplitAtNewlines() throws Exception { JCas jCas = JCasFactory.createJCas("de.julielab.jcore.types.jcore-morpho-syntax-types", "de.julielab.jcore.types.jcore-document-structure-types"); String ls = System.getProperty("line.separator"); jCas.setDocumentText("line1"+ls+"line2"+ls+"line3"); AnalysisEngine jsbd = AnalysisEngineFactory.createEngine(SentenceAnnotator.class, SentenceAnnotator.PARAM_MODEL_FILE, "de/julielab/jcore/ae/jsbd/model/test-model.gz", SentenceAnnotator.PARAM_ALWAYS_SPLIT_NEWLINE, true); jsbd.process(jCas.getCas()); Collection<String> sentences = JCasUtil.select(jCas, Sentence.class).stream().map(Annotation::getCoveredText).collect(Collectors.toList()); assertThat(sentences).containsExactly("line1", "line2", "line3"); }
@Test public void testProcess() { boolean annotationsOK = true; XMLInputSource sentenceXML = null; ResourceSpecifier sentenceSpec = null; AnalysisEngine sentenceAnnotator = null; try { sentenceXML = new XMLInputSource(DESCRIPTOR); sentenceSpec = UIMAFramework.getXMLParser().parseResourceSpecifier(sentenceXML); sentenceAnnotator = UIMAFramework.produceAnalysisEngine(sentenceSpec); } catch (Exception e) { LOGGER.error("testProcess()", e); } for (int i = 0; i < TEST_TEXT.length; i++) { JCas jcas = null; try { jcas = sentenceAnnotator.newJCas(); } catch (ResourceInitializationException e) { LOGGER.error("testProcess()", e); } if (LOGGER.isDebugEnabled()) { LOGGER.debug("testProcess() - testing text: " + TEST_TEXT[i]); } jcas.setDocumentText(TEST_TEXT[i]); TestScope scope1 = new TestScope(jcas, 0, endOffsets[i]); scope1.addToIndexes(); try { sentenceAnnotator.process(jcas, null); } catch (Exception e) { LOGGER.error("testProcess()", e); } JFSIndexRepository indexes = jcas.getJFSIndexRepository(); Iterator sentIter = indexes.getAnnotationIndex(Sentence.type).iterator(); String predictedOffsets = getPredictedOffsets(i, sentIter); if (!predictedOffsets.equals(TEST_TEXT_OFFSETS[i])) { annotationsOK = false; continue; } } assertTrue(annotationsOK); }
@Test public void testUimaFitIntegration() throws UIMAException, IOException { AnalysisEngine sentenceAE = AnalysisEngineFactory.createEngine(SentenceAnnotator.class, SentenceAnnotator.PARAM_MODEL_FILE, "de/julielab/jcore/ae/jsbd/model/test-model.gz", SentenceAnnotator.PARAM_POSTPROCESSING, "biomed"); JCas cas = JCasFactory.createJCas("de.julielab.jcore.types.jcore-morpho-syntax-types"); String abstractText = FileUtils.readFileToString(new File("src/test/resources/test-abstract.txt"), "UTF-8"); cas.setDocumentText(abstractText); sentenceAE.process(cas); Collection<Sentence> sentences = JCasUtil.select(cas, Sentence.class); for (Sentence sentence : sentences) { System.out.println(sentence.getCoveredText()); } assertEquals(14, sentences.size()); }
@Test public void testModelClassPathResource() throws Exception { AnalysisEngine sentenceAE = AnalysisEngineFactory.createEngine(SentenceAnnotator.class, SentenceAnnotator.PARAM_MODEL_FILE, "de/julielab/jcore/ae/jsbd/model/test-model.gz", SentenceAnnotator.PARAM_POSTPROCESSING, "biomed"); JCas cas = JCasFactory.createJCas("de.julielab.jcore.types.jcore-morpho-syntax-types"); String abstractText = FileUtils.readFileToString(new File("src/test/resources/test-abstract.txt"), "UTF-8"); cas.setDocumentText(abstractText); sentenceAE.process(cas); Collection<Sentence> sentences = JCasUtil.select(cas, Sentence.class); System.out.println(sentences.size()); for (Sentence sentence : sentences) { System.out.println(sentence.getCoveredText()); } assertEquals(14, sentences.size()); }
@Test public void testSentenceDelimiterTypes() throws Exception { JCas jCas = JCasFactory.createJCas("de.julielab.jcore.types.jcore-morpho-syntax-types", "de.julielab.jcore.types.jcore-document-structure-types"); jCas.setDocumentText("Introduction " + "We here show good results. This is a figure caption " + "And this is a paragraph without a fullstop for some reason " + "Conclusion " + "We are the greatest."); Title t1 = new Title(jCas, 0, 12); Caption c = new Caption(jCas, 40, 64); Paragraph p = new Paragraph(jCas, 65, 123); Title t2 = new Title(jCas, 124, 134); t1.addToIndexes(); c.addToIndexes(); p.addToIndexes(); t2.addToIndexes(); assertEquals("Introduction", t1.getCoveredText()); assertEquals("This is a figure caption", c.getCoveredText()); assertEquals("And this is a paragraph without a fullstop for some reason", p.getCoveredText()); assertEquals("Conclusion", t2.getCoveredText()); AnalysisEngine jsbd = AnalysisEngineFactory.createEngine(SentenceAnnotator.class, SentenceAnnotator.PARAM_MODEL_FILE, "de/julielab/jcore/ae/jsbd/model/test-model.gz", SentenceAnnotator.PARAM_SENTENCE_DELIMITER_TYPES, new LinkedHashSet<Object>( Arrays.asList(Title.class.getName(), Caption.class.getName(), Paragraph.class.getName()))); jsbd.process(jCas.getCas()); Set<Range<Integer>> expectedSpans = new HashSet<>(); expectedSpans.add(Range.between(0, 12)); expectedSpans.add(Range.between(13, 39)); expectedSpans.add(Range.between(40, 64)); expectedSpans.add(Range.between(65, 123)); expectedSpans.add(Range.between(124, 134)); expectedSpans.add(Range.between(135, 155)); FSIterator<Annotation> it = jCas.getAnnotationIndex(Sentence.type).iterator(); assertTrue(it.hasNext()); while (it.hasNext()) { Annotation sentence = it.next(); Range<Integer> sentenceRange = Range.between(sentence.getBegin(), sentence.getEnd()); assertTrue("Range " + sentenceRange + " was not expected", expectedSpans.remove(sentenceRange)); } assertTrue(expectedSpans.isEmpty()); }
@Test public void testSentenceWhitespaces() throws Exception { JCas jCas = JCasFactory.createJCas("de.julielab.jcore.types.jcore-morpho-syntax-types", "de.julielab.jcore.types.jcore-document-structure-types"); jCas.setDocumentText(" : We present a theoretical study of the electronic subband structure and collective electronic excitation associated with plasmon and surface plasmon modes in metal-based hollow nanosphere. The dependence of the electronic subband energy on the sample parameters of the hollow nanosphere is examined."); AnalysisEngine jsbd = AnalysisEngineFactory.createEngine(SentenceAnnotator.class, SentenceAnnotator.PARAM_MODEL_FILE, "de/julielab/jcore/ae/jsbd/model/test-model.gz"); jsbd.process(jCas.getCas()); Sentence sentence = JCasUtil.select(jCas, Sentence.class).iterator().next(); assertFalse(sentence.getCoveredText().startsWith(" ")); }
@Test public void testTrailingNewline() throws Exception { JCas jCas = JCasFactory.createJCas("de.julielab.jcore.types.jcore-morpho-syntax-types", "de.julielab.jcore.types.jcore-document-structure-types"); jCas.setDocumentText("In1 the next step, we plan to use higher level QM/MM methods to calculate the energy barrier of the reaction catalyzed by endonuclease APE1, in compliance with the mechanism proposed, and to screen for effective inhibitors with the use of the constructed mechanistic full-atomic model of the enzyme. \u2029"); new InternalReference(jCas, 2, 3).addToIndexes(); AnalysisEngine jsbd = AnalysisEngineFactory.createEngine(SentenceAnnotator.class, SentenceAnnotator.PARAM_MODEL_FILE, "de/julielab/jcore/ae/jsbd/model/test-model.gz", SentenceAnnotator.PARAM_CUT_AWAY_TYPES, new String[]{InternalReference.class.getCanonicalName()}); jsbd.process(jCas.getCas()); Sentence sentence = JCasUtil.select(jCas, Sentence.class).iterator().next(); assertFalse(sentence.getCoveredText().endsWith("\u2029")); }