LayeredPatternTokenizer { public String[] tokenize(String text) throws SequenceException { if (text == null) { text = ""; } int lastMatch = 0; LinkedList<String> splitted = new LinkedList<String>(); Matcher m = tokenPattern.matcher(text); while (m.find()) { String candidate = text.substring(lastMatch, m.start()); if (!candidate.matches("^\\s*$")) { addCharsOfString(text, candidate, splitted); } splitted.add(m.group()); lastMatch = m.end(); } String candidate = text.substring(lastMatch); if (!candidate.matches("^\\s*$")) { addCharsOfString(text, text.substring(lastMatch), splitted); } return splitted.toArray(new String[splitted.size()]); } LayeredPatternTokenizer(); String[] tokenize(String text); boolean isSymbolLayerName(String text); String[] getSymbolLayerName(String text);  }
@Test public void testTokenize() throws SequenceException { String pattern = "Hello_token"; String[] tokens = tokenizer.tokenize(pattern); assertArrayEquals(new String[]{"Hello_token"}, tokens); pattern = " Hello_token "; tokens = tokenizer.tokenize(pattern); assertArrayEquals(new String[]{"Hello_token"}, tokens); pattern = "Foo_0*"; tokens = tokenizer.tokenize(pattern); assertArrayEquals(new String[]{"Foo_0", "*"}, tokens); pattern = "( Foo_0 * Bar_1)+"; tokens = tokenizer.tokenize(pattern); assertArrayEquals(new String[]{"(", "Foo_0", "*", "Bar_1", ")", "+"}, tokens); pattern = "(Foo_bar baz)"; try{ tokens = tokenizer.tokenize(pattern); fail(); } catch (SequenceException e) { assertTrue(true); } pattern ="Hello_word (? world_word)"; tokens = tokenizer.tokenize(pattern); assertArrayEquals(new String[]{"Hello_word", "(", "?", "world_word", ")"}, tokens); pattern = "( B-NP_np I-NP_np* (? B-NP_np I-NP_np*))"; tokens = tokenizer.tokenize(pattern); assertArrayEquals(new String[] { "(", "B-NP_np", "I-NP_np", "*", "(", "?", "B-NP_np", "I-NP_np", "*", ")", ")" }, tokens); }