CoreNlpSegmenter extends SegmenterBase { @Override protected void process(JCas aJCas, String aText, int aZoneBegin) throws AnalysisEngineProcessException { Annotation document = new Annotation(aText); if (isWriteToken()) { tokenizerAnnotator.configure(aJCas.getCas()); tokenizerAnnotator.getResource().annotate(document); for (CoreLabel token : document.get(CoreAnnotations.TokensAnnotation.class)) { if (useCoreLabelWord) { createToken(aJCas, token.word(), token.get(CharacterOffsetBeginAnnotation.class) + aZoneBegin, token.get(CharacterOffsetEndAnnotation.class) + aZoneBegin); } else { createToken(aJCas, token.get(CharacterOffsetBeginAnnotation.class) + aZoneBegin, token.get(CharacterOffsetEndAnnotation.class) + aZoneBegin); } } } if (isWriteSentence()) { sentenceAnnotator.configure(aJCas.getCas()); sentenceAnnotator.getResource().annotate(document); for (CoreMap sentence : document.get(CoreAnnotations.SentencesAnnotation.class)) { createSentence(aJCas, sentence.get(CharacterOffsetBeginAnnotation.class) + aZoneBegin, sentence.get(CharacterOffsetEndAnnotation.class) + aZoneBegin); } } }  @Override void initialize(UimaContext aContext); static final String PARAM_BOUNDARY_TOKEN_REGEX; static final String PARAM_BOUNDARY_MULTI_TOKEN_REGEX; static final String PARAM_HTML_ELEMENTS_TO_DISCARD; static final String PARAM_BOUNDARIES_TO_DISCARD; static final String PARAM_NEWLINE_IS_SENTENCE_BREAK; static final String PARAM_TOKEN_REGEXES_TO_DISCARD; static final String PARAM_VARIANT; static final String PARAM_MODEL_LOCATION; static final String PARAM_TOKENIZATION_OPTIONS; }
@Test public void testEnglishSpeech() throws Exception { JCas jcas = JCasFactory.createJCas(); jcas.setDocumentLanguage("en"); jcas.setDocumentText("'Let's go! I want to see the Don', he said."); AnalysisEngine aed = createEngine(CoreNlpSegmenter.class); aed.process(jcas); String[] tokens = { "'", "Let", "'s", "go", "!", "I", "want", "to", "see", "the", "Don", "'", ",", "he", "said", "." }; AssertAnnotations.assertToken(tokens, select(jcas, Token.class)); }
@Test public void testSpanish() throws Exception { JCas jcas = JCasFactory.createJCas(); jcas.setDocumentLanguage("es"); jcas.setDocumentText("Tim dijo a Jamie para la 100ª vez que abandone la sala."); AnalysisEngine aed = createEngine(CoreNlpSegmenter.class); aed.process(jcas); String[] tokens = { "Tim", "dijo", "a", "Jamie", "para", "la", "100ª", "vez", "que", "abandone", "la", "sala", "." }; AssertAnnotations.assertToken(tokens, select(jcas, Token.class)); }
@Test public void testSpanishClitics() throws Exception { JCas jcas = JCasFactory.createJCas(); jcas.setDocumentLanguage("es"); jcas.setDocumentText("Al entregarles los libros del maestro los abrieron sin inmutarse\n" + "Estaban contentos."); AnalysisEngine aed = createEngine(CoreNlpSegmenter.class, CoreNlpSegmenter.PARAM_NEWLINE_IS_SENTENCE_BREAK, "always", CoreNlpSegmenter.PARAM_TOKENIZATION_OPTIONS, "splitAll=true,ptb3Escaping=false"); aed.process(jcas); String[] sentences = {"Al entregarles los libros del maestro los abrieron sin inmutarse", "Estaban contentos."}; String[] expectedTokens = { "A", "el", "entregarles", "los", "libros", "de", "el", "maestro", "los", "abrieron", "sin", "inmutarse", "Estaban", "contentos", "."}; AssertAnnotations.assertSentence(sentences, select(jcas, Sentence.class)); List<String> tokens = new ArrayList<String>(); for (Token token : select(jcas, Token.class)) { tokens.add(token.getText()); } System.out.printf("%-20s - Expected: %s%n", "Tokens", Arrays.asList(expectedTokens)); System.out.printf("%-20s - Actual : %s%n", "Tokens", tokens); Assert.assertEquals(Arrays.asList(expectedTokens), tokens); }
@Test public void testArabic() throws Exception { AssumeResource.assumeResource(CoreNlpParser.class, "de/tudarmstadt/ukp/dkpro/core/corenlp", "tokenizer", "ar", "atb-bn-arztrain"); JCas jcas = JCasFactory.createJCas(); jcas.setDocumentLanguage("ar"); jcas.setDocumentText("هل من المهم مراقبة وزن الرضيع خلال السنة الاولى من عمره؟\n" + " هل يجب وزن و قياس الطفل خلال السنة الاولى من عمره ؟\n"); AnalysisEngine aed = createEngine(CoreNlpSegmenter.class); aed.process(jcas); String[] sentences = { "هل من المهم مراقبة وزن الرضيع خلال السنة الاولى من عمره؟", "هل يجب وزن و قياس الطفل خلال السنة الاولى من عمره ؟"}; String[] tokens = { "هل", "من", "المهم", "مراقبة", "وزن", "الرضيع", "خلال", "السنة", "الاولى", "من", "عمر", "ه", "؟", "هل", "يجب", "وزن", "و", "قياس", "الطفل", "خلال", "السنة", "الاولى", "من", "عمر", "ه", "؟"}; AssertAnnotations.assertSentence(sentences, select(jcas, Sentence.class)); AssertAnnotations.assertToken(tokens, select(jcas, Token.class)); }