SentenceDetector implements Annotator<AnnotatedText> { public List<Integer> detectSentences(AnnotatedText text, String... labels) throws TalismaneException { LOG.debug("detectSentences"); List<Annotation<RawTextNoSentenceBreakMarker>> noSentenceBreakMarkers = text.getAnnotations(RawTextNoSentenceBreakMarker.class); Matcher matcher = SentenceDetector.POSSIBLE_BOUNDARIES.matcher(text.getText()); List<Integer> possibleBoundaries = new ArrayList<>(); while (matcher.find()) { if (matcher.start() >= text.getAnalysisStart() && matcher.start() < text.getAnalysisEnd()) { boolean noSentences = false; int position = matcher.start(); for (Annotation<RawTextNoSentenceBreakMarker> noSentenceBreakMarker : noSentenceBreakMarkers) { if (noSentenceBreakMarker.getStart() <= position && position < noSentenceBreakMarker.getEnd()) { noSentences = true; break; } } if (!noSentences) possibleBoundaries.add(position); } } List<Annotation<RawTextSentenceBreakMarker>> sentenceBreakMarkers = text.getAnnotations(RawTextSentenceBreakMarker.class); Set<Integer> guessedBoundaries = new TreeSet<>( sentenceBreakMarkers.stream().filter(f -> f.getEnd() >= text.getAnalysisStart()).map(f -> f.getEnd()).collect(Collectors.toList())); Sentence sentence = new Sentence(text.getText(), sessionId); TokenSequence tokenSequence = new TokenSequence(sentence, sessionId); List<PossibleSentenceBoundary> boundaries = new ArrayList<>(); for (int possibleBoundary : possibleBoundaries) { PossibleSentenceBoundary boundary = new PossibleSentenceBoundary(tokenSequence, possibleBoundary); if (LOG.isTraceEnabled()) { LOG.trace("Testing boundary: " + boundary); LOG.trace(" at position: " + possibleBoundary); } List<FeatureResult<?>> featureResults = new ArrayList<>(); for (SentenceDetectorFeature<?> feature : features) { RuntimeEnvironment env = new RuntimeEnvironment(); FeatureResult<?> featureResult = feature.check(boundary, env); if (featureResult != null) featureResults.add(featureResult); } if (LOG.isTraceEnabled()) { SortedSet<String> featureResultSet = featureResults.stream().map(f -> f.toString()).collect(Collectors.toCollection(() -> new TreeSet<String>())); for (String featureResultString : featureResultSet) { LOG.trace(featureResultString); } } List<Decision> decisions = this.decisionMaker.decide(featureResults); if (LOG.isTraceEnabled()) { for (Decision decision : decisions) { LOG.trace(decision.getOutcome() + ": " + decision.getProbability()); } } if (decisions.get(0).getOutcome().equals(SentenceDetectorOutcome.IS_BOUNDARY.name())) { if (LOG.isTraceEnabled()) { LOG.trace("Adding boundary: " + possibleBoundary + 1); } guessedBoundaries.add(possibleBoundary + 1); boundaries.add(boundary); } } if (LOG.isTraceEnabled()) { LOG.trace("context: " + text.getText().toString().replace('\n', '¶').replace('\r', '¶')); for (PossibleSentenceBoundary boundary : boundaries) LOG.trace("boundary: " + boundary.toString()); } if (LOG.isDebugEnabled()) LOG.debug("guessedBoundaries : " + guessedBoundaries.toString()); List<Annotation<SentenceBoundary>> newBoundaries = new ArrayList<>(); int lastBoundary = 0; List<Annotation<SentenceBoundary>> existingBoundaries = text.getAnnotations(SentenceBoundary.class); if (existingBoundaries.size() > 0) { lastBoundary = existingBoundaries.get(existingBoundaries.size() - 1).getEnd(); } while (lastBoundary < text.getAnalysisEnd() && Character.isWhitespace(text.getText().charAt(lastBoundary))) { lastBoundary++; } for (int guessedBoundary : guessedBoundaries) { if (guessedBoundary > lastBoundary) { Annotation<SentenceBoundary> sentenceBoundary = new Annotation<>(lastBoundary, guessedBoundary, new SentenceBoundary(), labels); newBoundaries.add(sentenceBoundary); if (LOG.isTraceEnabled()) { LOG.trace("Added boundary: " + sentenceBoundary); } lastBoundary = guessedBoundary; } } if (text.getAnalysisEnd() == text.getText().length()) { if (text.getAnalysisEnd() > lastBoundary) { Annotation<SentenceBoundary> sentenceBoundary = new Annotation<>(lastBoundary, text.getAnalysisEnd(), new SentenceBoundary(), labels); newBoundaries.add(sentenceBoundary); if (LOG.isTraceEnabled()) { LOG.trace("Added final boundary: " + sentenceBoundary); } } } text.addAnnotations(newBoundaries); return new ArrayList<>(guessedBoundaries); } SentenceDetector(DecisionMaker decisionMaker, Set<SentenceDetectorFeature<?>> features, String sessionId);  SentenceDetector(ClassificationModel sentenceModel, String sessionId);   SentenceDetector(SentenceDetector sentenceDetector); static SentenceDetector getInstance(String sessionId); @Override void annotate(AnnotatedText annotatedText, String... labels); List<Integer> detectSentences(AnnotatedText text, String... labels); DecisionMaker getDecisionMaker(); Set<SentenceDetectorFeature<?>> getFeatures(); SentenceDetector cloneSentenceDetector(); static final Pattern POSSIBLE_BOUNDARIES; }
@Test public void testDetectSentences2() throws Exception { System.setProperty("config.file", "src/test/resources/test.conf"); ConfigFactory.invalidateCaches(); final Config config = ConfigFactory.load(); final String sessionId = "test"; DecisionMaker decisionMaker = new DecisionMaker() { @Override public ScoringStrategy<ClassificationSolution> getDefaultScoringStrategy() { return new GeometricMeanScoringStrategy(); } @Override public List<Decision> decide(List<FeatureResult<?>> featureResults) { List<Decision> decisions = new ArrayList<>(); Decision decision = new Decision(SentenceDetectorOutcome.IS_BOUNDARY.name(), 1.0); decisions.add(decision); return decisions; } }; String[] labels = new String[0]; Set<SentenceDetectorFeature<?>> features = new HashSet<>(); SentenceDetector sentenceDetector = new SentenceDetector(decisionMaker, features, sessionId); String text = "Before analysis. Hello Mr. Jones\nHow are you, Mr. Jones? After"; AnnotatedText annotatedText = new AnnotatedText(text, "Before analysis. ".length(), text.length()); List<Annotation<RawTextNoSentenceBreakMarker>> noSentenceBreakMarkers = new ArrayList<>(); noSentenceBreakMarkers .add(new Annotation<>("Before analysis. Hello ".length(), "Before analysis. Hello Mr.".length(), new RawTextNoSentenceBreakMarker("me"), labels)); noSentenceBreakMarkers.add(new Annotation<>("Before analysis. Hello Mr. Jones\nHow are you, ".length(), "Before analysis. Hello Mr. Jones\nHow are you, Mr.".length(), new RawTextNoSentenceBreakMarker("me"), labels)); annotatedText.addAnnotations(noSentenceBreakMarkers); List<Annotation<SentenceBoundary>> existingBoundaries = new ArrayList<>(); existingBoundaries.add(new Annotation<>("".length(), "Before analysis.".length(), new SentenceBoundary(), labels)); annotatedText.addAnnotations(existingBoundaries); List<Annotation<RawTextSentenceBreakMarker>> sentenceBreaks = new ArrayList<>(); sentenceBreaks.add(new Annotation<>("Before analysis. Hello Mr. Jones".length(), "Before analysis. Hello Mr. Jones\n".length(), new RawTextSentenceBreakMarker("me"), labels)); annotatedText.addAnnotations(sentenceBreaks); List<Integer> guessedBoundaries = sentenceDetector.detectSentences(annotatedText); assertEquals(2, guessedBoundaries.size()); assertEquals("Before analysis. Hello Mr. Jones\n".length(), guessedBoundaries.get(0).intValue()); assertEquals("Before analysis. Hello Mr. Jones\nHow are you, Mr. Jones?".length(), guessedBoundaries.get(1).intValue()); List<Annotation<SentenceBoundary>> sentenceBoundaries = annotatedText.getAnnotations(SentenceBoundary.class); System.out.println(sentenceBoundaries.toString()); assertEquals(4, sentenceBoundaries.size()); assertEquals("".length(), sentenceBoundaries.get(0).getStart()); assertEquals("Before analysis.".length(), sentenceBoundaries.get(0).getEnd()); assertEquals("Before analysis. ".length(), sentenceBoundaries.get(1).getStart()); assertEquals("Before analysis. Hello Mr. Jones\n".length(), sentenceBoundaries.get(1).getEnd()); assertEquals("Before analysis. Hello Mr. Jones\n".length(), sentenceBoundaries.get(2).getStart()); assertEquals("Before analysis. Hello Mr. Jones\nHow are you, Mr. Jones?".length(), sentenceBoundaries.get(2).getEnd()); assertEquals("Before analysis. Hello Mr. Jones\nHow are you, Mr. Jones?".length(), sentenceBoundaries.get(3).getStart()); assertEquals("Before analysis. Hello Mr. Jones\nHow are you, Mr. Jones? After".length(), sentenceBoundaries.get(3).getEnd()); }
@Test public void testDetectSentences() throws Exception { System.setProperty("config.file", "src/test/resources/test.conf"); ConfigFactory.invalidateCaches(); final Config config = ConfigFactory.load(); final String sessionId = "test"; DecisionMaker decisionMaker = new DecisionMaker() { @Override public ScoringStrategy<ClassificationSolution> getDefaultScoringStrategy() { return new GeometricMeanScoringStrategy(); } @Override public List<Decision> decide(List<FeatureResult<?>> featureResults) { List<Decision> decisions = new ArrayList<>(); Decision decision = new Decision(SentenceDetectorOutcome.IS_BOUNDARY.name(), 1.0); decisions.add(decision); return decisions; } }; String[] labels = new String[0]; Set<SentenceDetectorFeature<?>> features = new HashSet<>(); SentenceDetector sentenceDetector = new SentenceDetector(decisionMaker, features, sessionId); String text = "Before analysis. Hello Mr. Jones. How are you, Mr. Jones? After analysis."; AnnotatedText annotatedText = new AnnotatedText(text, "Before analysis. ".length(), "Before analysis. Hello Mr. Jones. How are you, Mr. Jones?".length()); List<Annotation<RawTextNoSentenceBreakMarker>> noSentenceBreakMarkers = new ArrayList<>(); noSentenceBreakMarkers .add(new Annotation<>("Before analysis. Hello ".length(), "Before analysis. Hello Mr.".length(), new RawTextNoSentenceBreakMarker("me"), labels)); noSentenceBreakMarkers.add(new Annotation<>("Before analysis. Hello Mr. Jones. How are you, ".length(), "Before analysis. Hello Mr. Jones. How are you, Mr.".length(), new RawTextNoSentenceBreakMarker("me"), labels)); annotatedText.addAnnotations(noSentenceBreakMarkers); List<Integer> sentenceBreaks = sentenceDetector.detectSentences(annotatedText); assertEquals(2, sentenceBreaks.size()); assertEquals("Before analysis. Hello Mr. Jones.".length(), sentenceBreaks.get(0).intValue()); assertEquals("Before analysis. Hello Mr. Jones. How are you, Mr. Jones?".length(), sentenceBreaks.get(1).intValue()); List<Annotation<SentenceBoundary>> sentenceBoundaries = annotatedText.getAnnotations(SentenceBoundary.class); assertEquals(2, sentenceBoundaries.size()); assertEquals("".length(), sentenceBoundaries.get(0).getStart()); assertEquals("Before analysis. Hello Mr. Jones.".length(), sentenceBoundaries.get(0).getEnd()); assertEquals("Before analysis. Hello Mr. Jones.".length(), sentenceBoundaries.get(1).getStart()); assertEquals("Before analysis. Hello Mr. Jones. How are you, Mr. Jones?".length(), sentenceBoundaries.get(1).getEnd()); }