@Test public void testTrailingNewline() throws Exception { JCas jCas = JCasFactory.createJCas("de.julielab.jcore.types.jcore-morpho-syntax-types", "de.julielab.jcore.types.jcore-document-structure-types"); jCas.setDocumentText("In1 the next step, we plan to use higher level QM/MM methods to calculate the energy barrier of the reaction catalyzed by endonuclease APE1, in compliance with the mechanism proposed, and to screen for effective inhibitors with the use of the constructed mechanistic full-atomic model of the enzyme. \u2029"); new InternalReference(jCas, 2, 3).addToIndexes(); AnalysisEngine jsbd = AnalysisEngineFactory.createEngine(SentenceAnnotator.class, SentenceAnnotator.PARAM_MODEL_FILE, "de/julielab/jcore/ae/jsbd/model/test-model.gz", SentenceAnnotator.PARAM_CUT_AWAY_TYPES, new String[]{InternalReference.class.getCanonicalName()}); jsbd.process(jCas.getCas()); Sentence sentence = JCasUtil.select(jCas, Sentence.class).iterator().next(); assertFalse(sentence.getCoveredText().endsWith("\u2029")); }
public void process(JCas aJCas) throws AnalysisEngineProcessException { if (StringUtils.isBlank(aJCas.getDocumentText())) { final String docId = JCoReTools.getDocId(aJCas); LOGGER.warn("The document text of document {} is empty.", docId); return; } JCoReCondensedDocumentText documentText; try { documentText = new JCoReCondensedDocumentText(aJCas, cutAwayTypes); } catch (ClassNotFoundException e1) { throw new AnalysisEngineProcessException(e1); } if (sentenceDelimiterTypes != null) { try { JCoReAnnotationIndexMerger indexMerger = new JCoReAnnotationIndexMerger(sentenceDelimiterTypes, false, null, aJCas); List<Integer> borders = new ArrayList<>(); borders.add(0); borders.add(aJCas.getDocumentText().length()); while (indexMerger.incrementAnnotation()) { Annotation a = (Annotation) indexMerger.getAnnotation(); borders.add(documentText.getCondensedOffsetForOriginalOffset(a.getBegin())); borders.add(documentText.getCondensedOffsetForOriginalOffset(a.getEnd())); } borders.sort(null); for (int i = 1; i < borders.size(); ++i) { int start = borders.get(i - 1); int end = borders.get(i); while (start < end && Character.isWhitespace(aJCas.getDocumentText().charAt(start))) ++start; String textSpan = documentText.getCodensedText().substring(start, end); if (!StringUtils.isBlank(textSpan)) doSegmentation(documentText, textSpan, start); } } catch (ClassNotFoundException e) { throw new AnalysisEngineProcessException(e); } } else { if (aJCas.getDocumentText() != null && aJCas.getDocumentText().length() > 0) { doSegmentation(documentText, documentText.getCodensedText(), 0); } else { if (numEmptyCases.get() < 10) { LOGGER.debug("document text empty. Skipping this document."); numEmptyCases.incrementAndGet(); } else if (numEmptyCases.get() == 10) { LOGGER.warn("Encountered 10 documents with an empty text body. This message will not appear again " + "to avoid scrolling in cases where this is expected."); } } } }
SentenceAnnotator extends JCasAnnotator_ImplBase { public void process(JCas aJCas) throws AnalysisEngineProcessException { if (StringUtils.isBlank(aJCas.getDocumentText())) { final String docId = JCoReTools.getDocId(aJCas); LOGGER.warn("The document text of document {} is empty.", docId); return; } JCoReCondensedDocumentText documentText; try { documentText = new JCoReCondensedDocumentText(aJCas, cutAwayTypes); } catch (ClassNotFoundException e1) { throw new AnalysisEngineProcessException(e1); } if (sentenceDelimiterTypes != null) { try { JCoReAnnotationIndexMerger indexMerger = new JCoReAnnotationIndexMerger(sentenceDelimiterTypes, false, null, aJCas); List<Integer> borders = new ArrayList<>(); borders.add(0); borders.add(aJCas.getDocumentText().length()); while (indexMerger.incrementAnnotation()) { Annotation a = (Annotation) indexMerger.getAnnotation(); borders.add(documentText.getCondensedOffsetForOriginalOffset(a.getBegin())); borders.add(documentText.getCondensedOffsetForOriginalOffset(a.getEnd())); } borders.sort(null); for (int i = 1; i < borders.size(); ++i) { int start = borders.get(i - 1); int end = borders.get(i); while (start < end && Character.isWhitespace(aJCas.getDocumentText().charAt(start))) ++start; String textSpan = documentText.getCodensedText().substring(start, end); if (!StringUtils.isBlank(textSpan)) doSegmentation(documentText, textSpan, start); } } catch (ClassNotFoundException e) { throw new AnalysisEngineProcessException(e); } } else { if (aJCas.getDocumentText() != null && aJCas.getDocumentText().length() > 0) { doSegmentation(documentText, documentText.getCodensedText(), 0); } else { if (numEmptyCases.get() < 10) { LOGGER.debug("document text empty. Skipping this document."); numEmptyCases.incrementAndGet(); } else if (numEmptyCases.get() == 10) { LOGGER.warn("Encountered 10 documents with an empty text body. This message will not appear again " + "to avoid scrolling in cases where this is expected."); } } } } }
SentenceAnnotator extends JCasAnnotator_ImplBase { public void process(JCas aJCas) throws AnalysisEngineProcessException { if (StringUtils.isBlank(aJCas.getDocumentText())) { final String docId = JCoReTools.getDocId(aJCas); LOGGER.warn("The document text of document {} is empty.", docId); return; } JCoReCondensedDocumentText documentText; try { documentText = new JCoReCondensedDocumentText(aJCas, cutAwayTypes); } catch (ClassNotFoundException e1) { throw new AnalysisEngineProcessException(e1); } if (sentenceDelimiterTypes != null) { try { JCoReAnnotationIndexMerger indexMerger = new JCoReAnnotationIndexMerger(sentenceDelimiterTypes, false, null, aJCas); List<Integer> borders = new ArrayList<>(); borders.add(0); borders.add(aJCas.getDocumentText().length()); while (indexMerger.incrementAnnotation()) { Annotation a = (Annotation) indexMerger.getAnnotation(); borders.add(documentText.getCondensedOffsetForOriginalOffset(a.getBegin())); borders.add(documentText.getCondensedOffsetForOriginalOffset(a.getEnd())); } borders.sort(null); for (int i = 1; i < borders.size(); ++i) { int start = borders.get(i - 1); int end = borders.get(i); while (start < end && Character.isWhitespace(aJCas.getDocumentText().charAt(start))) ++start; String textSpan = documentText.getCodensedText().substring(start, end); if (!StringUtils.isBlank(textSpan)) doSegmentation(documentText, textSpan, start); } } catch (ClassNotFoundException e) { throw new AnalysisEngineProcessException(e); } } else { if (aJCas.getDocumentText() != null && aJCas.getDocumentText().length() > 0) { doSegmentation(documentText, documentText.getCodensedText(), 0); } else { if (numEmptyCases.get() < 10) { LOGGER.debug("document text empty. Skipping this document."); numEmptyCases.incrementAndGet(); } else if (numEmptyCases.get() == 10) { LOGGER.warn("Encountered 10 documents with an empty text body. This message will not appear again " + "to avoid scrolling in cases where this is expected."); } } } }  }
SentenceAnnotator extends JCasAnnotator_ImplBase { public void process(JCas aJCas) throws AnalysisEngineProcessException { if (StringUtils.isBlank(aJCas.getDocumentText())) { final String docId = JCoReTools.getDocId(aJCas); LOGGER.warn("The document text of document {} is empty.", docId); return; } JCoReCondensedDocumentText documentText; try { documentText = new JCoReCondensedDocumentText(aJCas, cutAwayTypes); } catch (ClassNotFoundException e1) { throw new AnalysisEngineProcessException(e1); } if (sentenceDelimiterTypes != null) { try { JCoReAnnotationIndexMerger indexMerger = new JCoReAnnotationIndexMerger(sentenceDelimiterTypes, false, null, aJCas); List<Integer> borders = new ArrayList<>(); borders.add(0); borders.add(aJCas.getDocumentText().length()); while (indexMerger.incrementAnnotation()) { Annotation a = (Annotation) indexMerger.getAnnotation(); borders.add(documentText.getCondensedOffsetForOriginalOffset(a.getBegin())); borders.add(documentText.getCondensedOffsetForOriginalOffset(a.getEnd())); } borders.sort(null); for (int i = 1; i < borders.size(); ++i) { int start = borders.get(i - 1); int end = borders.get(i); while (start < end && Character.isWhitespace(aJCas.getDocumentText().charAt(start))) ++start; String textSpan = documentText.getCodensedText().substring(start, end); if (!StringUtils.isBlank(textSpan)) doSegmentation(documentText, textSpan, start); } } catch (ClassNotFoundException e) { throw new AnalysisEngineProcessException(e); } } else { if (aJCas.getDocumentText() != null && aJCas.getDocumentText().length() > 0) { doSegmentation(documentText, documentText.getCodensedText(), 0); } else { if (numEmptyCases.get() < 10) { LOGGER.debug("document text empty. Skipping this document."); numEmptyCases.incrementAndGet(); } else if (numEmptyCases.get() == 10) { LOGGER.warn("Encountered 10 documents with an empty text body. This message will not appear again " + "to avoid scrolling in cases where this is expected."); } } } }  void initialize(UimaContext aContext); void process(JCas aJCas); }
SentenceAnnotator extends JCasAnnotator_ImplBase { public void process(JCas aJCas) throws AnalysisEngineProcessException { if (StringUtils.isBlank(aJCas.getDocumentText())) { final String docId = JCoReTools.getDocId(aJCas); LOGGER.warn("The document text of document {} is empty.", docId); return; } JCoReCondensedDocumentText documentText; try { documentText = new JCoReCondensedDocumentText(aJCas, cutAwayTypes); } catch (ClassNotFoundException e1) { throw new AnalysisEngineProcessException(e1); } if (sentenceDelimiterTypes != null) { try { JCoReAnnotationIndexMerger indexMerger = new JCoReAnnotationIndexMerger(sentenceDelimiterTypes, false, null, aJCas); List<Integer> borders = new ArrayList<>(); borders.add(0); borders.add(aJCas.getDocumentText().length()); while (indexMerger.incrementAnnotation()) { Annotation a = (Annotation) indexMerger.getAnnotation(); borders.add(documentText.getCondensedOffsetForOriginalOffset(a.getBegin())); borders.add(documentText.getCondensedOffsetForOriginalOffset(a.getEnd())); } borders.sort(null); for (int i = 1; i < borders.size(); ++i) { int start = borders.get(i - 1); int end = borders.get(i); while (start < end && Character.isWhitespace(aJCas.getDocumentText().charAt(start))) ++start; String textSpan = documentText.getCodensedText().substring(start, end); if (!StringUtils.isBlank(textSpan)) doSegmentation(documentText, textSpan, start); } } catch (ClassNotFoundException e) { throw new AnalysisEngineProcessException(e); } } else { if (aJCas.getDocumentText() != null && aJCas.getDocumentText().length() > 0) { doSegmentation(documentText, documentText.getCodensedText(), 0); } else { if (numEmptyCases.get() < 10) { LOGGER.debug("document text empty. Skipping this document."); numEmptyCases.incrementAndGet(); } else if (numEmptyCases.get() == 10) { LOGGER.warn("Encountered 10 documents with an empty text body. This message will not appear again " + "to avoid scrolling in cases where this is expected."); } } } }  void initialize(UimaContext aContext); void process(JCas aJCas); static final String PARAM_MODEL_FILE; static final String PARAM_POSTPROCESSING; static final String PARAM_SENTENCE_DELIMITER_TYPES; static final String PARAM_CUT_AWAY_TYPES; static final String PARAM_MAX_SENTENCE_LENGTH; static final String PARAM_ALWAYS_SPLIT_NEWLINE; }