SimpleKeywordTokenizer implements KeywordTokenizer { @Override public List<String> tokenize(String keyword) { keyword = _normalizeWhitespace(keyword); List<String> tokens = new ArrayList<>(); int start = keyword.indexOf(CharPool.QUOTE); int end = keyword.indexOf(CharPool.QUOTE, start + 1); tokenize(keyword, tokens, start, end); return tokens; }  @Override boolean requiresTokenization(String keyword); @Override List<String> tokenize(String keyword);  }
@Test public void testTokenize() { assertTokenize( "This is a simple token test", "[This, is, a, simple, token, test]"); }
@Test(expected = NullPointerException.class) public void testTokenizeNull() { simpleKeywordTokenizer.tokenize(null); }