PatternBasedTokenSegmenter extends JCasAnnotator_ImplBase { @Override public void process(JCas aJCas) throws AnalysisEngineProcessException { buf = new StringBuilder(); List<Token> toAdd = new ArrayList<Token>(); List<Token> toRemove = new ArrayList<Token>(); for (Token t : select(aJCas, Token.class)) { String text = t.getCoveredText(); int offset = t.getBegin(); int start = 0; SplitPattern lastPattern = getPattern(text.charAt(0), null); Token firstToken = null; for (int i = 1; i < text.length(); i++) { SplitPattern pattern = getPattern(text.charAt(i), lastPattern); if (pattern != lastPattern) { if (lastPattern == null || lastPattern.includeInOutput) { Token nt = addToken(aJCas, offset, text, start, i, toAdd); firstToken = (firstToken == null) ? nt : firstToken; } start = i; } lastPattern = pattern; } if (start == 0) { if (lastPattern != null && !lastPattern.includeInOutput) { toRemove.add(t); } continue; } if (deleteCover) { toRemove.add(t); } if (lastPattern == null || lastPattern.includeInOutput) { addToken(aJCas, offset, text, start, text.length(), toAdd); } } for (Token t : toAdd) { t.addToIndexes(); } for (Token t : toRemove) { t.removeFromIndexes(); } }  @Override void initialize(UimaContext aContext); @Override void process(JCas aJCas); static final String INCLUDE_PREFIX; static final String EXCLUDE_PREFIX; static final String PARAM_DELETE_COVER; static final String PARAM_PATTERNS; }
@Test public void testProcess() throws Exception { AnalysisEngine seg = createEngine(PatternBasedTokenSegmenter.class, PatternBasedTokenSegmenter.PARAM_PATTERNS, PATTERNS); String content = "This_Is_A_Camel Case_Text"; JCas cas = seg.newJCas(); cas.setDocumentText(content); new Token(cas, 0, 15).addToIndexes(); new Token(cas, 16, 25).addToIndexes(); seg.process(cas); List<String> ref = asList("This", "Is", "A", "Camel", "Case", "Text"); List<String> tokens = toText(select(cas, Token.class)); System.out.println(tokens); assertEquals(ref, tokens); }
@Test public void testProcess2() throws Exception { AnalysisEngine seg = createEngine(PatternBasedTokenSegmenter.class, PatternBasedTokenSegmenter.PARAM_PATTERNS, PATTERNS); String content = "This_Is.A_Camel_ _Case_Text"; JCas cas = seg.newJCas(); cas.setDocumentText(content); new Token(cas, 0, 16).addToIndexes(); new Token(cas, 17, 27).addToIndexes(); seg.process(cas); List<String> ref = asList("This", "Is", "A", "Camel", "Case", "Text"); List<String> tokens = toText(select(cas, Token.class)); System.out.println(tokens); assertEquals(ref, tokens); }
@Test public void testProcess3() throws Exception { AnalysisEngine seg = createEngine(PatternBasedTokenSegmenter.class, PatternBasedTokenSegmenter.PARAM_PATTERNS, PATTERNS); String content = "This_"; JCas cas = seg.newJCas(); cas.setDocumentText(content); new Token(cas, 0, 5).addToIndexes(); seg.process(cas); List<String> ref = asList("This"); List<String> tokens = toText(select(cas, Token.class)); System.out.println(tokens); assertEquals(ref, tokens); }
@Test public void testProcess4() throws Exception { AnalysisEngine seg = createEngine(PatternBasedTokenSegmenter.class, PatternBasedTokenSegmenter.PARAM_PATTERNS, PATTERNS); String content = "rent25to29point9"; JCas cas = seg.newJCas(); cas.setDocumentText(content); new Token(cas, 0, 16).addToIndexes(); seg.process(cas); List<String> ref = asList("rent", "25", "to", "29", "point", "9"); List<String> tokens = toText(select(cas, Token.class)); System.out.println(tokens); assertEquals(ref, tokens); }
@Test public void testProcess5() throws Exception { AnalysisEngine seg = createEngine(PatternBasedTokenSegmenter.class, PatternBasedTokenSegmenter.PARAM_PATTERNS, PATTERNS); String content = "_This"; JCas cas = seg.newJCas(); cas.setDocumentText(content); new Token(cas, 0, 5).addToIndexes(); seg.process(cas); List<String> ref = asList("This"); List<String> tokens = toText(select(cas, Token.class)); System.out.println(tokens); assertEquals(ref, tokens); }
@Test public void testProcess6() throws Exception { AnalysisEngine seg = createEngine(PatternBasedTokenSegmenter.class, PatternBasedTokenSegmenter.PARAM_PATTERNS, PATTERNS); String content = "_This"; JCas cas = seg.newJCas(); cas.setDocumentText(content); new Token(cas, 0, 1).addToIndexes(); new Token(cas, 1, 5).addToIndexes(); seg.process(cas); List<String> ref = asList("This"); List<String> tokens = toText(select(cas, Token.class)); System.out.println(tokens); assertEquals(ref, tokens); }