LuceneStandardTokenizerFilter extends AbstractFilter { @Override public List<String> filter(String input) { newOutput(); try (Tokenizer t = new StandardTokenizer(AttributeFactory.DEFAULT_ATTRIBUTE_FACTORY)) { t.setReader(new StringReader(input)); t.reset(); CharTermAttribute att = t.addAttribute(CharTermAttribute.class); while (t.incrementToken()) { output.add(att.toString()); } return output; } catch (IOException e) { e.printStackTrace(); } return null; }  @Override List<String> filter(String input); @Override Filter copy();  }
@Test public void testTokenizer() { LuceneStandardTokenizerFilter filter = new LuceneStandardTokenizerFilter(); List<String> tokens = filter.filter("A horse is-a animal."); assertThat(tokens).containsExactly("A", "horse", "is", "a", "animal"); }