@Test public void testSentenceWhitespaces() throws Exception { JCas jCas = JCasFactory.createJCas("de.julielab.jcore.types.jcore-morpho-syntax-types", "de.julielab.jcore.types.jcore-document-structure-types"); jCas.setDocumentText(" : We present a theoretical study of the electronic subband structure and collective electronic excitation associated with plasmon and surface plasmon modes in metal-based hollow nanosphere. The dependence of the electronic subband energy on the sample parameters of the hollow nanosphere is examined."); AnalysisEngine jsbd = AnalysisEngineFactory.createEngine(SentenceAnnotator.class, SentenceAnnotator.PARAM_MODEL_FILE, "de/julielab/jcore/ae/jsbd/model/test-model.gz"); jsbd.process(jCas.getCas()); Sentence sentence = JCasUtil.select(jCas, Sentence.class).iterator().next(); assertFalse(sentence.getCoveredText().startsWith(" ")); }
public void process(JCas aJCas) throws AnalysisEngineProcessException { if (StringUtils.isBlank(aJCas.getDocumentText())) { final String docId = JCoReTools.getDocId(aJCas); LOGGER.warn("The document text of document {} is empty.", docId); return; } JCoReCondensedDocumentText documentText; try { documentText = new JCoReCondensedDocumentText(aJCas, cutAwayTypes); } catch (ClassNotFoundException e1) { throw new AnalysisEngineProcessException(e1); } if (sentenceDelimiterTypes != null) { try { JCoReAnnotationIndexMerger indexMerger = new JCoReAnnotationIndexMerger(sentenceDelimiterTypes, false, null, aJCas); List<Integer> borders = new ArrayList<>(); borders.add(0); borders.add(aJCas.getDocumentText().length()); while (indexMerger.incrementAnnotation()) { Annotation a = (Annotation) indexMerger.getAnnotation(); borders.add(documentText.getCondensedOffsetForOriginalOffset(a.getBegin())); borders.add(documentText.getCondensedOffsetForOriginalOffset(a.getEnd())); } borders.sort(null); for (int i = 1; i < borders.size(); ++i) { int start = borders.get(i - 1); int end = borders.get(i); while (start < end && Character.isWhitespace(aJCas.getDocumentText().charAt(start))) ++start; String textSpan = documentText.getCodensedText().substring(start, end); if (!StringUtils.isBlank(textSpan)) doSegmentation(documentText, textSpan, start); } } catch (ClassNotFoundException e) { throw new AnalysisEngineProcessException(e); } } else { if (aJCas.getDocumentText() != null && aJCas.getDocumentText().length() > 0) { doSegmentation(documentText, documentText.getCodensedText(), 0); } else { if (numEmptyCases.get() < 10) { LOGGER.debug("document text empty. Skipping this document."); numEmptyCases.incrementAndGet(); } else if (numEmptyCases.get() == 10) { LOGGER.warn("Encountered 10 documents with an empty text body. This message will not appear again " + "to avoid scrolling in cases where this is expected."); } } } }
SentenceAnnotator extends JCasAnnotator_ImplBase { public void process(JCas aJCas) throws AnalysisEngineProcessException { if (StringUtils.isBlank(aJCas.getDocumentText())) { final String docId = JCoReTools.getDocId(aJCas); LOGGER.warn("The document text of document {} is empty.", docId); return; } JCoReCondensedDocumentText documentText; try { documentText = new JCoReCondensedDocumentText(aJCas, cutAwayTypes); } catch (ClassNotFoundException e1) { throw new AnalysisEngineProcessException(e1); } if (sentenceDelimiterTypes != null) { try { JCoReAnnotationIndexMerger indexMerger = new JCoReAnnotationIndexMerger(sentenceDelimiterTypes, false, null, aJCas); List<Integer> borders = new ArrayList<>(); borders.add(0); borders.add(aJCas.getDocumentText().length()); while (indexMerger.incrementAnnotation()) { Annotation a = (Annotation) indexMerger.getAnnotation(); borders.add(documentText.getCondensedOffsetForOriginalOffset(a.getBegin())); borders.add(documentText.getCondensedOffsetForOriginalOffset(a.getEnd())); } borders.sort(null); for (int i = 1; i < borders.size(); ++i) { int start = borders.get(i - 1); int end = borders.get(i); while (start < end && Character.isWhitespace(aJCas.getDocumentText().charAt(start))) ++start; String textSpan = documentText.getCodensedText().substring(start, end); if (!StringUtils.isBlank(textSpan)) doSegmentation(documentText, textSpan, start); } } catch (ClassNotFoundException e) { throw new AnalysisEngineProcessException(e); } } else { if (aJCas.getDocumentText() != null && aJCas.getDocumentText().length() > 0) { doSegmentation(documentText, documentText.getCodensedText(), 0); } else { if (numEmptyCases.get() < 10) { LOGGER.debug("document text empty. Skipping this document."); numEmptyCases.incrementAndGet(); } else if (numEmptyCases.get() == 10) { LOGGER.warn("Encountered 10 documents with an empty text body. This message will not appear again " + "to avoid scrolling in cases where this is expected."); } } } } }
SentenceAnnotator extends JCasAnnotator_ImplBase { public void process(JCas aJCas) throws AnalysisEngineProcessException { if (StringUtils.isBlank(aJCas.getDocumentText())) { final String docId = JCoReTools.getDocId(aJCas); LOGGER.warn("The document text of document {} is empty.", docId); return; } JCoReCondensedDocumentText documentText; try { documentText = new JCoReCondensedDocumentText(aJCas, cutAwayTypes); } catch (ClassNotFoundException e1) { throw new AnalysisEngineProcessException(e1); } if (sentenceDelimiterTypes != null) { try { JCoReAnnotationIndexMerger indexMerger = new JCoReAnnotationIndexMerger(sentenceDelimiterTypes, false, null, aJCas); List<Integer> borders = new ArrayList<>(); borders.add(0); borders.add(aJCas.getDocumentText().length()); while (indexMerger.incrementAnnotation()) { Annotation a = (Annotation) indexMerger.getAnnotation(); borders.add(documentText.getCondensedOffsetForOriginalOffset(a.getBegin())); borders.add(documentText.getCondensedOffsetForOriginalOffset(a.getEnd())); } borders.sort(null); for (int i = 1; i < borders.size(); ++i) { int start = borders.get(i - 1); int end = borders.get(i); while (start < end && Character.isWhitespace(aJCas.getDocumentText().charAt(start))) ++start; String textSpan = documentText.getCodensedText().substring(start, end); if (!StringUtils.isBlank(textSpan)) doSegmentation(documentText, textSpan, start); } } catch (ClassNotFoundException e) { throw new AnalysisEngineProcessException(e); } } else { if (aJCas.getDocumentText() != null && aJCas.getDocumentText().length() > 0) { doSegmentation(documentText, documentText.getCodensedText(), 0); } else { if (numEmptyCases.get() < 10) { LOGGER.debug("document text empty. Skipping this document."); numEmptyCases.incrementAndGet(); } else if (numEmptyCases.get() == 10) { LOGGER.warn("Encountered 10 documents with an empty text body. This message will not appear again " + "to avoid scrolling in cases where this is expected."); } } } }  }
SentenceAnnotator extends JCasAnnotator_ImplBase { public void process(JCas aJCas) throws AnalysisEngineProcessException { if (StringUtils.isBlank(aJCas.getDocumentText())) { final String docId = JCoReTools.getDocId(aJCas); LOGGER.warn("The document text of document {} is empty.", docId); return; } JCoReCondensedDocumentText documentText; try { documentText = new JCoReCondensedDocumentText(aJCas, cutAwayTypes); } catch (ClassNotFoundException e1) { throw new AnalysisEngineProcessException(e1); } if (sentenceDelimiterTypes != null) { try { JCoReAnnotationIndexMerger indexMerger = new JCoReAnnotationIndexMerger(sentenceDelimiterTypes, false, null, aJCas); List<Integer> borders = new ArrayList<>(); borders.add(0); borders.add(aJCas.getDocumentText().length()); while (indexMerger.incrementAnnotation()) { Annotation a = (Annotation) indexMerger.getAnnotation(); borders.add(documentText.getCondensedOffsetForOriginalOffset(a.getBegin())); borders.add(documentText.getCondensedOffsetForOriginalOffset(a.getEnd())); } borders.sort(null); for (int i = 1; i < borders.size(); ++i) { int start = borders.get(i - 1); int end = borders.get(i); while (start < end && Character.isWhitespace(aJCas.getDocumentText().charAt(start))) ++start; String textSpan = documentText.getCodensedText().substring(start, end); if (!StringUtils.isBlank(textSpan)) doSegmentation(documentText, textSpan, start); } } catch (ClassNotFoundException e) { throw new AnalysisEngineProcessException(e); } } else { if (aJCas.getDocumentText() != null && aJCas.getDocumentText().length() > 0) { doSegmentation(documentText, documentText.getCodensedText(), 0); } else { if (numEmptyCases.get() < 10) { LOGGER.debug("document text empty. Skipping this document."); numEmptyCases.incrementAndGet(); } else if (numEmptyCases.get() == 10) { LOGGER.warn("Encountered 10 documents with an empty text body. This message will not appear again " + "to avoid scrolling in cases where this is expected."); } } } }  void initialize(UimaContext aContext); void process(JCas aJCas); }
SentenceAnnotator extends JCasAnnotator_ImplBase { public void process(JCas aJCas) throws AnalysisEngineProcessException { if (StringUtils.isBlank(aJCas.getDocumentText())) { final String docId = JCoReTools.getDocId(aJCas); LOGGER.warn("The document text of document {} is empty.", docId); return; } JCoReCondensedDocumentText documentText; try { documentText = new JCoReCondensedDocumentText(aJCas, cutAwayTypes); } catch (ClassNotFoundException e1) { throw new AnalysisEngineProcessException(e1); } if (sentenceDelimiterTypes != null) { try { JCoReAnnotationIndexMerger indexMerger = new JCoReAnnotationIndexMerger(sentenceDelimiterTypes, false, null, aJCas); List<Integer> borders = new ArrayList<>(); borders.add(0); borders.add(aJCas.getDocumentText().length()); while (indexMerger.incrementAnnotation()) { Annotation a = (Annotation) indexMerger.getAnnotation(); borders.add(documentText.getCondensedOffsetForOriginalOffset(a.getBegin())); borders.add(documentText.getCondensedOffsetForOriginalOffset(a.getEnd())); } borders.sort(null); for (int i = 1; i < borders.size(); ++i) { int start = borders.get(i - 1); int end = borders.get(i); while (start < end && Character.isWhitespace(aJCas.getDocumentText().charAt(start))) ++start; String textSpan = documentText.getCodensedText().substring(start, end); if (!StringUtils.isBlank(textSpan)) doSegmentation(documentText, textSpan, start); } } catch (ClassNotFoundException e) { throw new AnalysisEngineProcessException(e); } } else { if (aJCas.getDocumentText() != null && aJCas.getDocumentText().length() > 0) { doSegmentation(documentText, documentText.getCodensedText(), 0); } else { if (numEmptyCases.get() < 10) { LOGGER.debug("document text empty. Skipping this document."); numEmptyCases.incrementAndGet(); } else if (numEmptyCases.get() == 10) { LOGGER.warn("Encountered 10 documents with an empty text body. This message will not appear again " + "to avoid scrolling in cases where this is expected."); } } } }  void initialize(UimaContext aContext); void process(JCas aJCas); static final String PARAM_MODEL_FILE; static final String PARAM_POSTPROCESSING; static final String PARAM_SENTENCE_DELIMITER_TYPES; static final String PARAM_CUT_AWAY_TYPES; static final String PARAM_MAX_SENTENCE_LENGTH; static final String PARAM_ALWAYS_SPLIT_NEWLINE; }