StanfordSegmenter extends SegmenterBase { @Override protected void process(JCas aJCas, String aText, int aZoneBegin) throws AnalysisEngineProcessException { List<Token> casTokens = null; String language = getLanguage(aJCas); if (isWriteToken()) { casTokens = new ArrayList<Token>(); final Tokenizer<?> tokenizer = getTokenizer(language, aText); List<?> tokens = tokenizer.tokenize(); for (int i = 0; i < tokens.size(); i++) { final Object token = tokens.get(i); CoreLabel l = (CoreLabel) token; String t = l.word(); int begin = l.get(CharacterOffsetBeginAnnotation.class); int end = l.get(CharacterOffsetEndAnnotation.class); casTokens.add(createToken(aJCas, t, aZoneBegin + begin, aZoneBegin + end)); } } if (isWriteSentence()) { if (casTokens == null) { casTokens = selectCovered(aJCas, Token.class, aZoneBegin, aZoneBegin + aText.length()); } List<CoreLabel> tokensInDocument = new ArrayList<CoreLabel>(); Pattern nlPattern = Pattern.compile(".*(\r\n|\n|\r).*"); Matcher nlMatcher = nlPattern.matcher(""); int lastTokenEnd = 0; for (Token token : casTokens) { if (!NewlineIsSentenceBreak.NEVER.equals(newlineIsSentenceBreak)) { nlMatcher.reset( aJCas.getDocumentText().subSequence(lastTokenEnd, token.getBegin())); if (nlMatcher.matches()) { CoreLabel l = new CoreLabel(); l.set(CharacterOffsetBeginAnnotation.class, lastTokenEnd + nlMatcher.start(1)); l.set(CharacterOffsetEndAnnotation.class, lastTokenEnd + nlMatcher.end(1)); l.setWord("\n"); tokensInDocument.add(l); } } lastTokenEnd = token.getEnd(); CoreLabel l = new CoreLabel(); l.set(CharacterOffsetBeginAnnotation.class, token.getBegin()); l.set(CharacterOffsetEndAnnotation.class, token.getEnd()); l.setWord(token.getText()); tokensInDocument.add(l); } PTBEscapingProcessor escaper = new PTBEscapingProcessor(); escaper.apply(tokensInDocument); WordToSentenceProcessor<CoreLabel> proc = new WordToSentenceProcessor<CoreLabel>( boundaryTokenRegex, boundaryFollowersRegex, boundariesToDiscard, xmlBreakElementsToDiscard, regionElementRegex, newlineIsSentenceBreak, null, tokenRegexesToDiscard, isOneSentence, allowEmptySentences); List<List<CoreLabel>> sentencesInDocument = proc.process(tokensInDocument); for (List<CoreLabel> sentence : sentencesInDocument) { int begin = sentence.get(0).get(CharacterOffsetBeginAnnotation.class); int end = sentence.get(sentence.size() - 1).get(CharacterOffsetEndAnnotation.class); createSentence(aJCas, begin, end); } } }   static final String PARAM_LANGUAGE_FALLBACK; static final String PARAM_BOUNDARY_TOKEN_REGEX; static final String PARAM_BOUNDARY_FOLLOWERS_REGEX; static final String PARAM_XML_BREAK_ELEMENTS_TO_DISCARD; static final String PARAM_BOUNDARIES_TO_DISCARD; static final String PARAM_REGION_ELEMENT_REGEX; static final String PARAM_NEWLINE_IS_SENTENCE_BREAK; static final String PARAM_TOKEN_REGEXES_TO_DISCARD; static final String PARAM_IS_ONE_SENTENCE; static final String PARAM_ALLOW_EMPTY_SENTENCES; }
@Test public void testEnglishSpeech() throws Exception { JCas jcas = JCasFactory.createJCas(); jcas.setDocumentLanguage("en"); jcas.setDocumentText("'Let's go! I want to see the Don', he said."); AnalysisEngine aed = createEngine(StanfordSegmenter.class); aed.process(jcas); String[] tokens = { "'", "Let", "'s", "go", "!", "I", "want", "to", "see", "the", "Don", "'", ",", "he", "said", "." }; AssertAnnotations.assertToken(tokens, select(jcas, Token.class)); }
@Test public void testFrench() throws Exception { JCas jcas = JCasFactory.createJCas(); jcas.setDocumentLanguage("fr"); jcas.setDocumentText("Tim a dit Jamie pour la 100e fois de quitter la salle ."); AnalysisEngine aed = createEngine(StanfordSegmenter.class); aed.process(jcas); String[] tokens = { "Tim", "a", "dit", "Jamie", "pour", "la", "100e", "fois", "de", "quitter", "la", "salle", "." }; AssertAnnotations.assertToken(tokens, select(jcas, Token.class)); }
@Test public void testSpanish() throws Exception { JCas jcas = JCasFactory.createJCas(); jcas.setDocumentLanguage("es"); jcas.setDocumentText("Tim dijo a Jamie para la 100ª vez que abandone la sala."); AnalysisEngine aed = createEngine(StanfordSegmenter.class); aed.process(jcas); String[] tokens = { "Tim", "dijo", "a", "Jamie", "para", "la", "100ª", "vez", "que", "abandone", "la", "sala", "." }; AssertAnnotations.assertToken(tokens, select(jcas, Token.class)); }
@Test public void testUnwrapped() throws Exception { String text = "\"Hey you!\", John said."; String[] expectedSentences = { "0 10 \"Hey you!\"", "10 22 , John said." }; String[] expectedTokens = { "0 1 `` \"", "1 4 Hey Hey", "5 8 you you", "8 9 ! !", "9 10 '' \"", "10 11 , ,", "12 16 John John", "17 21 said said", "21 22 . ." }; List<String> sentences = new ArrayList<String>(); List<String> tokens = new ArrayList<String>(); Properties props = new Properties(); props.setProperty("annotators", "tokenize, ssplit"); StanfordCoreNLP pipeline = new StanfordCoreNLP(props); Annotation annotation = pipeline.process(text); for (CoreMap sentence : annotation.get(CoreAnnotations.SentencesAnnotation.class)) { sentences.add(String.format("%d %d %s", sentence.get(CoreAnnotations.CharacterOffsetBeginAnnotation.class), sentence.get(CoreAnnotations.CharacterOffsetEndAnnotation.class), sentence.get(CoreAnnotations.TextAnnotation.class))); for (CoreLabel token : sentence.get(CoreAnnotations.TokensAnnotation.class)) { tokens.add(String.format("%d %d %s %s", token.get(CoreAnnotations.CharacterOffsetBeginAnnotation.class), token.get(CoreAnnotations.CharacterOffsetEndAnnotation.class), token.get(CoreAnnotations.TextAnnotation.class), token.get(CoreAnnotations.OriginalTextAnnotation.class))); } } assertEquals(asList(expectedSentences), sentences); assertEquals(asList(expectedTokens), tokens); }