SentenceHolder { public List<Sentence> getDetectedSentences(Sentence leftover) { if (LOG.isTraceEnabled()) { LOG.trace("getDetectedSentences at " + System.identityHashCode(this) + ". leftover=" + leftover); LOG.trace("processedText: " + processedText); LOG.trace("file: " + (file == null ? "null" : file.getPath())); } List<Sentence> sentences = new ArrayList<Sentence>(); int currentIndex = 0; int lastOriginalTextInsertionProcessed = -1; List<Integer> allBoundaries = new ArrayList<Integer>(); for (int sentenceBoundary : sentenceBoundaries) { if (sentenceBoundary <= this.processedText.length()) allBoundaries.add(sentenceBoundary); } boolean haveLeftOvers = this.processedText.length() > 0; if (allBoundaries.size() > 0) { haveLeftOvers = false; int lastSentenceBoundary = allBoundaries.get(allBoundaries.size() - 1); if (lastSentenceBoundary < this.processedText.length()) { haveLeftOvers = true; } if (LOG.isTraceEnabled()) { LOG.trace("haveLeftOvers? " + lastSentenceBoundary + " < " + (this.processedText.length()) + " = " + haveLeftOvers); } } if (haveLeftOvers) allBoundaries.add(this.processedText.length()); for (int sentenceBoundary : allBoundaries) { boolean isLeftover = (!endOfBlock) && (haveLeftOvers && sentenceBoundary == this.processedText.length()); String text = this.processedText.subSequence(currentIndex, sentenceBoundary).toString(); if (!isLeftover && (whiteSpacePattern.matcher(text).matches() || text.length() == 0)) { continue; } int leftOverTextLength = 0; List<Integer> originalIndexes = new ArrayList<>(); TreeMap<Integer, String> originalTextSegments = new TreeMap<>(); String leftoverOriginalText = ""; if (leftover != null) { leftOverTextLength = leftover.getText().length(); text = leftover.getText() + text; originalIndexes.addAll(leftover.getOriginalIndexes()); originalTextSegments.putAll(leftover.getOriginalTextSegments()); leftoverOriginalText = leftover.getLeftoverOriginalText(); leftover = null; } Matcher matcherOpeningWhiteSpace = openingWhiteSpacePattern.matcher(text); int openingWhiteSpaceEnd = 0; if (matcherOpeningWhiteSpace.find()) { openingWhiteSpaceEnd = matcherOpeningWhiteSpace.end(1); } int closingWhiteSpaceStart = text.length(); if (!isLeftover) { Matcher matcherClosingWhiteSpace = closingWhiteSpacePattern.matcher(text); if (matcherClosingWhiteSpace.find()) { closingWhiteSpaceStart = matcherClosingWhiteSpace.start(1); } } Matcher matcherDuplicateWhiteSpace = duplicateWhiteSpacePattern.matcher(text); Set<Integer> duplicateWhiteSpace = new HashSet<Integer>(); while (matcherDuplicateWhiteSpace.find()) { for (int i = matcherDuplicateWhiteSpace.start() + 1; i < matcherDuplicateWhiteSpace.end(); i++) { duplicateWhiteSpace.add(i); } } StringBuilder sb = new StringBuilder(); int i = currentIndex; for (int j = 0; j < text.length(); j++) { boolean appendLetter = false; if (j < openingWhiteSpaceEnd) { } else if (j >= closingWhiteSpaceStart) { } else if (duplicateWhiteSpace.contains(j)) { } else { appendLetter = true; } if (j >= leftOverTextLength) { if (appendLetter) originalIndexes.add(this.originalIndexes.get(i)); if (this.originalTextSegments.containsKey(i)) { originalTextSegments.put(sb.length(), this.originalTextSegments.get(i)); lastOriginalTextInsertionProcessed = i; } i++; } if (appendLetter) sb.append(text.charAt(j)); } Sentence sentence = new Sentence(sb.toString(), originalTextSegments, originalIndexes, !isLeftover, this.newlines, file, sessionId); sentence.setLeftoverOriginalText(leftoverOriginalText); if (LOG.isTraceEnabled()) { LOG.trace("sentence.setText |" + sentence.getText() + "| complete? " + sentence.isComplete() + "|file: " + sentence.getFileName()); } sentences.add(sentence); currentIndex = sentenceBoundary; } TreeMap<Integer, String> leftoverOriginalTextSegments = new TreeMap<Integer, String>(); for (int i : this.originalTextSegments.keySet()) { if (i > lastOriginalTextInsertionProcessed) { leftoverOriginalTextSegments.put(i, this.originalTextSegments.get(i)); } } this.originalTextSegments = leftoverOriginalTextSegments; return sentences; } SentenceHolder(int originalStartIndex, boolean endOfBlock, String sessionId); Set<Integer> getSentenceBoundaries(); void addSentenceBoundary(int boundary); List<Sentence> getDetectedSentences(Sentence leftover); String getProcessedText(); void setProcessedText(String processedText); File getFile(); void setFile(File file); TreeMap<Integer, String> getOriginalTextSegments(); TreeMap<Integer, Integer> getNewlines(); void setSentenceBoundaries(TreeSet<Integer> sentenceBoundaries); void addOriginalTextSegment(int index, String segment); void addNewline(int originalIndex, int lineNumber); List<Integer> getOriginalIndexes(); void addOriginalIndex(int originalIndex); int getOriginalIndex(int index); int getIndex(int originalIndex); int getLineNumber(int originalIndex); int getColumnNumber(int originalIndex); int getOriginalStartIndex();  }
@Test public void testGetDetectedSentences() throws Exception { System.setProperty("config.file", "src/test/resources/test.conf"); ConfigFactory.invalidateCaches(); final Config config = ConfigFactory.load(); final String sessionId = "test"; SentenceHolder holder = new SentenceHolder(0, false, sessionId); String originalText = "Hello <b>World</b>. <o>Output this</o>How are you? Fine<o>Output</o>, "; holder.setProcessedText("Hello World. How are you? Fine, "); for (int i = 0; i < holder.getProcessedText().length(); i++) { if (i < "Hello ".length()) holder.addOriginalIndex(i); else if (i < "Hello World".length()) holder.addOriginalIndex(i + "<b>".length()); else if (i < "Hello World. ".length()) holder.addOriginalIndex(i + "<b></b>".length()); else if (i < "Hello World. How are you? Fine".length()) holder.addOriginalIndex(i + "<b></b><o>Output this</o>".length()); else holder.addOriginalIndex(i + "<b></b><o>Output this</o><o>Output</o>".length()); } holder.getOriginalTextSegments().put("Hello World. ".length() - 1, "<o>Output this</o>"); holder.getOriginalTextSegments().put("Hello World. How are you? Fine,".length() - 1, "<o>Output</o>"); holder.addSentenceBoundary("Hello World.".length()); holder.addSentenceBoundary("Hello World. How are you?".length()); List<Sentence> sentences = holder.getDetectedSentences(null); for (Sentence sentence : sentences) { System.out.println(sentence.getText().toString()); } assertEquals(3, sentences.size()); Sentence sentence1 = sentences.get(0); assertEquals("Hello World.", sentence1.getText()); assertEquals("Hello <b>W".length(), sentence1.getOriginalIndex("Hello W".length())); assertEquals("Hello <b>World</b>.".length() - 1, sentence1.getOriginalIndex("Hello World.".length() - 1)); Sentence sentence2 = sentences.get(1); assertEquals("How are you?", sentence2.getText()); assertEquals("Hello <b>World</b>. <o>Output this</o>H".length(), sentence2.getOriginalIndex("H".length())); for (Entry<Integer, String> originalSegment : sentence2.getOriginalTextSegments().entrySet()) { System.out.println(originalSegment.getKey() + ": \"" + originalSegment.getValue() + "\""); } assertEquals("<o>Output this</o>", sentence2.getOriginalTextSegments().get(0)); Sentence leftover = sentences.get(2); assertFalse(leftover.isComplete()); assertEquals("Fine, ", leftover.getText()); assertEquals("Hello <b>World</b>. <o>Output this</o>How are you? F".length(), leftover.getOriginalIndex("F".length())); for (Entry<Integer, String> originalSegment : leftover.getOriginalTextSegments().entrySet()) { System.out.println(originalSegment.getKey() + ": \"" + originalSegment.getValue() + "\""); } assertEquals("<o>Output</o>", leftover.getOriginalTextSegments().get(4)); SentenceHolder holder2 = new SentenceHolder(0, false, sessionId); String originalText2 = "thanks, and you"; holder2.setProcessedText("thanks, and you"); for (int i = 0; i < holder2.getProcessedText().length(); i++) { holder2.addOriginalIndex(originalText.length() + i); } sentences = holder2.getDetectedSentences(leftover); assertEquals(1, sentences.size()); leftover = sentences.get(0); assertEquals("Fine, thanks, and you", leftover.getText()); assertEquals("Hello <b>World</b>. <o>Output this</o>How are you? F".length(), leftover.getOriginalIndex("F".length())); for (Entry<Integer, String> originalSegment : leftover.getOriginalTextSegments().entrySet()) { System.out.println(originalSegment.getKey() + ": \"" + originalSegment.getValue() + "\""); } assertEquals("<o>Output</o>", leftover.getOriginalTextSegments().get(4)); assertFalse(leftover.isComplete()); SentenceHolder holder3 = new SentenceHolder(0, false, sessionId); String originalText3 = "? Grand."; holder3.setProcessedText(originalText3); for (int i = 0; i < holder2.getProcessedText().length(); i++) { holder3.addOriginalIndex(originalText.length() + originalText2.length() + i); } holder3.addSentenceBoundary("?".length()); holder3.addSentenceBoundary("? Grand.".length()); sentences = holder3.getDetectedSentences(leftover); System.out.println(sentences.toString()); assertEquals(2, sentences.size()); sentence1 = sentences.get(0); assertEquals("Fine, thanks, and you?", sentence1.getText()); assertEquals("Hello <b>World</b>. <o>Output this</o>How are you? F".length(), sentence1.getOriginalIndex("F".length())); for (Entry<Integer, String> originalSegment : sentence1.getOriginalTextSegments().entrySet()) { System.out.println(originalSegment.getKey() + ": \"" + originalSegment.getValue() + "\""); } assertEquals("<o>Output</o>", sentence1.getOriginalTextSegments().get(4)); assertTrue(sentence1.isComplete()); sentence2 = sentences.get(1); assertEquals("Grand.", sentence2.getText()); assertEquals("Hello <b>World</b>. <o>Output this</o>How are you? Fine<o>Output</o>, thanks, and you? G".length(), sentence2.getOriginalIndex("G".length())); assertTrue(sentence2.isComplete()); }