BLDutchTokenFilter extends TokenFilter { @Override final public boolean incrementToken() throws IOException { while (input.incrementToken()) { String t = new String(termAtt.buffer(), 0, termAtt.length()); t = process(t); if (ANY_LETTER_PATTERN.matcher(t).find()) { termAtt.copyBuffer(t.toCharArray(), 0, t.length()); return true; } } return false; } BLDutchTokenFilter(TokenStream input); static String process(String input); @Override final boolean incrementToken(); @Override int hashCode(); @Override boolean equals(Object obj);  }
@Test public void testBasics() throws IOException { TokenStream ts = new StubTokenStream( new String[] { "hond", "a.u.b.", "bel(len)", "[pre]cursor", "zo'n", "'Hij", "zij'", "ex-man", "-" }); try { ts = new BLDutchTokenFilter(ts); ts.reset(); CharTermAttribute ta = ts.addAttribute(CharTermAttribute.class); Assert.assertTrue(ts.incrementToken()); Assert.assertEquals("hond", new String(ta.buffer(), 0, ta.length())); Assert.assertTrue(ts.incrementToken()); Assert.assertEquals("aub", new String(ta.buffer(), 0, ta.length())); Assert.assertTrue(ts.incrementToken()); Assert.assertEquals("bellen", new String(ta.buffer(), 0, ta.length())); Assert.assertTrue(ts.incrementToken()); Assert.assertEquals("precursor", new String(ta.buffer(), 0, ta.length())); Assert.assertTrue(ts.incrementToken()); Assert.assertEquals("zo'n", new String(ta.buffer(), 0, ta.length())); Assert.assertTrue(ts.incrementToken()); Assert.assertEquals("Hij", new String(ta.buffer(), 0, ta.length())); Assert.assertTrue(ts.incrementToken()); Assert.assertEquals("zij", new String(ta.buffer(), 0, ta.length())); Assert.assertTrue(ts.incrementToken()); Assert.assertEquals("ex-man", new String(ta.buffer(), 0, ta.length())); Assert.assertFalse(ts.incrementToken()); } finally { ts.close(); } }