@Test public void testSentenceDelimiterTypes() throws Exception { JCas jCas = JCasFactory.createJCas("de.julielab.jcore.types.jcore-morpho-syntax-types", "de.julielab.jcore.types.jcore-document-structure-types"); jCas.setDocumentText("Introduction " + "We here show good results. This is a figure caption " + "And this is a paragraph without a fullstop for some reason " + "Conclusion " + "We are the greatest."); Title t1 = new Title(jCas, 0, 12); Caption c = new Caption(jCas, 40, 64); Paragraph p = new Paragraph(jCas, 65, 123); Title t2 = new Title(jCas, 124, 134); t1.addToIndexes(); c.addToIndexes(); p.addToIndexes(); t2.addToIndexes(); assertEquals("Introduction", t1.getCoveredText()); assertEquals("This is a figure caption", c.getCoveredText()); assertEquals("And this is a paragraph without a fullstop for some reason", p.getCoveredText()); assertEquals("Conclusion", t2.getCoveredText()); AnalysisEngine jsbd = AnalysisEngineFactory.createEngine(SentenceAnnotator.class, SentenceAnnotator.PARAM_MODEL_FILE, "de/julielab/jcore/ae/jsbd/model/test-model.gz", SentenceAnnotator.PARAM_SENTENCE_DELIMITER_TYPES, new LinkedHashSet<Object>( Arrays.asList(Title.class.getName(), Caption.class.getName(), Paragraph.class.getName()))); jsbd.process(jCas.getCas()); Set<Range<Integer>> expectedSpans = new HashSet<>(); expectedSpans.add(Range.between(0, 12)); expectedSpans.add(Range.between(13, 39)); expectedSpans.add(Range.between(40, 64)); expectedSpans.add(Range.between(65, 123)); expectedSpans.add(Range.between(124, 134)); expectedSpans.add(Range.between(135, 155)); FSIterator<Annotation> it = jCas.getAnnotationIndex(Sentence.type).iterator(); assertTrue(it.hasNext()); while (it.hasNext()) { Annotation sentence = it.next(); Range<Integer> sentenceRange = Range.between(sentence.getBegin(), sentence.getEnd()); assertTrue("Range " + sentenceRange + " was not expected", expectedSpans.remove(sentenceRange)); } assertTrue(expectedSpans.isEmpty()); }
public void process(JCas aJCas) throws AnalysisEngineProcessException { if (StringUtils.isBlank(aJCas.getDocumentText())) { final String docId = JCoReTools.getDocId(aJCas); LOGGER.warn("The document text of document {} is empty.", docId); return; } JCoReCondensedDocumentText documentText; try { documentText = new JCoReCondensedDocumentText(aJCas, cutAwayTypes); } catch (ClassNotFoundException e1) { throw new AnalysisEngineProcessException(e1); } if (sentenceDelimiterTypes != null) { try { JCoReAnnotationIndexMerger indexMerger = new JCoReAnnotationIndexMerger(sentenceDelimiterTypes, false, null, aJCas); List<Integer> borders = new ArrayList<>(); borders.add(0); borders.add(aJCas.getDocumentText().length()); while (indexMerger.incrementAnnotation()) { Annotation a = (Annotation) indexMerger.getAnnotation(); borders.add(documentText.getCondensedOffsetForOriginalOffset(a.getBegin())); borders.add(documentText.getCondensedOffsetForOriginalOffset(a.getEnd())); } borders.sort(null); for (int i = 1; i < borders.size(); ++i) { int start = borders.get(i - 1); int end = borders.get(i); while (start < end && Character.isWhitespace(aJCas.getDocumentText().charAt(start))) ++start; String textSpan = documentText.getCodensedText().substring(start, end); if (!StringUtils.isBlank(textSpan)) doSegmentation(documentText, textSpan, start); } } catch (ClassNotFoundException e) { throw new AnalysisEngineProcessException(e); } } else { if (aJCas.getDocumentText() != null && aJCas.getDocumentText().length() > 0) { doSegmentation(documentText, documentText.getCodensedText(), 0); } else { if (numEmptyCases.get() < 10) { LOGGER.debug("document text empty. Skipping this document."); numEmptyCases.incrementAndGet(); } else if (numEmptyCases.get() == 10) { LOGGER.warn("Encountered 10 documents with an empty text body. This message will not appear again " + "to avoid scrolling in cases where this is expected."); } } } }
SentenceAnnotator extends JCasAnnotator_ImplBase { public void process(JCas aJCas) throws AnalysisEngineProcessException { if (StringUtils.isBlank(aJCas.getDocumentText())) { final String docId = JCoReTools.getDocId(aJCas); LOGGER.warn("The document text of document {} is empty.", docId); return; } JCoReCondensedDocumentText documentText; try { documentText = new JCoReCondensedDocumentText(aJCas, cutAwayTypes); } catch (ClassNotFoundException e1) { throw new AnalysisEngineProcessException(e1); } if (sentenceDelimiterTypes != null) { try { JCoReAnnotationIndexMerger indexMerger = new JCoReAnnotationIndexMerger(sentenceDelimiterTypes, false, null, aJCas); List<Integer> borders = new ArrayList<>(); borders.add(0); borders.add(aJCas.getDocumentText().length()); while (indexMerger.incrementAnnotation()) { Annotation a = (Annotation) indexMerger.getAnnotation(); borders.add(documentText.getCondensedOffsetForOriginalOffset(a.getBegin())); borders.add(documentText.getCondensedOffsetForOriginalOffset(a.getEnd())); } borders.sort(null); for (int i = 1; i < borders.size(); ++i) { int start = borders.get(i - 1); int end = borders.get(i); while (start < end && Character.isWhitespace(aJCas.getDocumentText().charAt(start))) ++start; String textSpan = documentText.getCodensedText().substring(start, end); if (!StringUtils.isBlank(textSpan)) doSegmentation(documentText, textSpan, start); } } catch (ClassNotFoundException e) { throw new AnalysisEngineProcessException(e); } } else { if (aJCas.getDocumentText() != null && aJCas.getDocumentText().length() > 0) { doSegmentation(documentText, documentText.getCodensedText(), 0); } else { if (numEmptyCases.get() < 10) { LOGGER.debug("document text empty. Skipping this document."); numEmptyCases.incrementAndGet(); } else if (numEmptyCases.get() == 10) { LOGGER.warn("Encountered 10 documents with an empty text body. This message will not appear again " + "to avoid scrolling in cases where this is expected."); } } } } }
SentenceAnnotator extends JCasAnnotator_ImplBase { public void process(JCas aJCas) throws AnalysisEngineProcessException { if (StringUtils.isBlank(aJCas.getDocumentText())) { final String docId = JCoReTools.getDocId(aJCas); LOGGER.warn("The document text of document {} is empty.", docId); return; } JCoReCondensedDocumentText documentText; try { documentText = new JCoReCondensedDocumentText(aJCas, cutAwayTypes); } catch (ClassNotFoundException e1) { throw new AnalysisEngineProcessException(e1); } if (sentenceDelimiterTypes != null) { try { JCoReAnnotationIndexMerger indexMerger = new JCoReAnnotationIndexMerger(sentenceDelimiterTypes, false, null, aJCas); List<Integer> borders = new ArrayList<>(); borders.add(0); borders.add(aJCas.getDocumentText().length()); while (indexMerger.incrementAnnotation()) { Annotation a = (Annotation) indexMerger.getAnnotation(); borders.add(documentText.getCondensedOffsetForOriginalOffset(a.getBegin())); borders.add(documentText.getCondensedOffsetForOriginalOffset(a.getEnd())); } borders.sort(null); for (int i = 1; i < borders.size(); ++i) { int start = borders.get(i - 1); int end = borders.get(i); while (start < end && Character.isWhitespace(aJCas.getDocumentText().charAt(start))) ++start; String textSpan = documentText.getCodensedText().substring(start, end); if (!StringUtils.isBlank(textSpan)) doSegmentation(documentText, textSpan, start); } } catch (ClassNotFoundException e) { throw new AnalysisEngineProcessException(e); } } else { if (aJCas.getDocumentText() != null && aJCas.getDocumentText().length() > 0) { doSegmentation(documentText, documentText.getCodensedText(), 0); } else { if (numEmptyCases.get() < 10) { LOGGER.debug("document text empty. Skipping this document."); numEmptyCases.incrementAndGet(); } else if (numEmptyCases.get() == 10) { LOGGER.warn("Encountered 10 documents with an empty text body. This message will not appear again " + "to avoid scrolling in cases where this is expected."); } } } }  }
SentenceAnnotator extends JCasAnnotator_ImplBase { public void process(JCas aJCas) throws AnalysisEngineProcessException { if (StringUtils.isBlank(aJCas.getDocumentText())) { final String docId = JCoReTools.getDocId(aJCas); LOGGER.warn("The document text of document {} is empty.", docId); return; } JCoReCondensedDocumentText documentText; try { documentText = new JCoReCondensedDocumentText(aJCas, cutAwayTypes); } catch (ClassNotFoundException e1) { throw new AnalysisEngineProcessException(e1); } if (sentenceDelimiterTypes != null) { try { JCoReAnnotationIndexMerger indexMerger = new JCoReAnnotationIndexMerger(sentenceDelimiterTypes, false, null, aJCas); List<Integer> borders = new ArrayList<>(); borders.add(0); borders.add(aJCas.getDocumentText().length()); while (indexMerger.incrementAnnotation()) { Annotation a = (Annotation) indexMerger.getAnnotation(); borders.add(documentText.getCondensedOffsetForOriginalOffset(a.getBegin())); borders.add(documentText.getCondensedOffsetForOriginalOffset(a.getEnd())); } borders.sort(null); for (int i = 1; i < borders.size(); ++i) { int start = borders.get(i - 1); int end = borders.get(i); while (start < end && Character.isWhitespace(aJCas.getDocumentText().charAt(start))) ++start; String textSpan = documentText.getCodensedText().substring(start, end); if (!StringUtils.isBlank(textSpan)) doSegmentation(documentText, textSpan, start); } } catch (ClassNotFoundException e) { throw new AnalysisEngineProcessException(e); } } else { if (aJCas.getDocumentText() != null && aJCas.getDocumentText().length() > 0) { doSegmentation(documentText, documentText.getCodensedText(), 0); } else { if (numEmptyCases.get() < 10) { LOGGER.debug("document text empty. Skipping this document."); numEmptyCases.incrementAndGet(); } else if (numEmptyCases.get() == 10) { LOGGER.warn("Encountered 10 documents with an empty text body. This message will not appear again " + "to avoid scrolling in cases where this is expected."); } } } }  void initialize(UimaContext aContext); void process(JCas aJCas); }
SentenceAnnotator extends JCasAnnotator_ImplBase { public void process(JCas aJCas) throws AnalysisEngineProcessException { if (StringUtils.isBlank(aJCas.getDocumentText())) { final String docId = JCoReTools.getDocId(aJCas); LOGGER.warn("The document text of document {} is empty.", docId); return; } JCoReCondensedDocumentText documentText; try { documentText = new JCoReCondensedDocumentText(aJCas, cutAwayTypes); } catch (ClassNotFoundException e1) { throw new AnalysisEngineProcessException(e1); } if (sentenceDelimiterTypes != null) { try { JCoReAnnotationIndexMerger indexMerger = new JCoReAnnotationIndexMerger(sentenceDelimiterTypes, false, null, aJCas); List<Integer> borders = new ArrayList<>(); borders.add(0); borders.add(aJCas.getDocumentText().length()); while (indexMerger.incrementAnnotation()) { Annotation a = (Annotation) indexMerger.getAnnotation(); borders.add(documentText.getCondensedOffsetForOriginalOffset(a.getBegin())); borders.add(documentText.getCondensedOffsetForOriginalOffset(a.getEnd())); } borders.sort(null); for (int i = 1; i < borders.size(); ++i) { int start = borders.get(i - 1); int end = borders.get(i); while (start < end && Character.isWhitespace(aJCas.getDocumentText().charAt(start))) ++start; String textSpan = documentText.getCodensedText().substring(start, end); if (!StringUtils.isBlank(textSpan)) doSegmentation(documentText, textSpan, start); } } catch (ClassNotFoundException e) { throw new AnalysisEngineProcessException(e); } } else { if (aJCas.getDocumentText() != null && aJCas.getDocumentText().length() > 0) { doSegmentation(documentText, documentText.getCodensedText(), 0); } else { if (numEmptyCases.get() < 10) { LOGGER.debug("document text empty. Skipping this document."); numEmptyCases.incrementAndGet(); } else if (numEmptyCases.get() == 10) { LOGGER.warn("Encountered 10 documents with an empty text body. This message will not appear again " + "to avoid scrolling in cases where this is expected."); } } } }  void initialize(UimaContext aContext); void process(JCas aJCas); static final String PARAM_MODEL_FILE; static final String PARAM_POSTPROCESSING; static final String PARAM_SENTENCE_DELIMITER_TYPES; static final String PARAM_CUT_AWAY_TYPES; static final String PARAM_MAX_SENTENCE_LENGTH; static final String PARAM_ALWAYS_SPLIT_NEWLINE; }