TextTokenizer { public List<String> tokenize(String string){ Pattern pattern = getPattern(); if(("").equals(string)){ return Collections.emptyList(); } String[] tokens = pattern.split(string, -1); int count = 0; for(int i = 0, max = tokens.length; i < max; i++){ String token = tokens[i]; if(token.length() > 0){ token = TermUtil.trimPunctuation(token); if(token.length() > 0){ tokens[count] = token; count++; } } } if(count < tokens.length){ String[] tmpTokens = new String[count]; System.arraycopy(tokens, 0, tmpTokens, 0, count); tokens = tmpTokens; } return ImmutableList.copyOf(tokens); } TextTokenizer(Pattern pattern); List<String> tokenize(String string); Pattern getPattern();  }
@Test public void tokenize(){ Pattern pattern = Pattern.compile("\\s+"); TextTokenizer tokenizer = new TextTokenizer(pattern); assertEquals(Collections.emptyList(), tokenizer.tokenize("")); assertEquals(Collections.emptyList(), tokenizer.tokenize(" ")); assertEquals(Collections.emptyList(), tokenizer.tokenize("\t\t\t")); assertEquals(Collections.emptyList(), tokenizer.tokenize(",")); assertEquals(Collections.emptyList(), tokenizer.tokenize(",,")); assertEquals(Collections.emptyList(), tokenizer.tokenize(", ,")); assertEquals(Collections.emptyList(), tokenizer.tokenize(" , , ")); assertEquals(Arrays.asList("one", "two", "three"), tokenizer.tokenize("one two three")); assertEquals(Arrays.asList("one", "two", "three"), tokenizer.tokenize("one!, \u00BFtwo?, three.")); }