HashTokenizer extends BaseTokenizer { public int next() { if (!hasNext()) { throw new IllegalStateException("No next ngram hash to return"); } int hash = calcHash(_normalized, _normalizedPos, _curNGramLength); nextNGram(); return hash; } HashTokenizer(int maxNGramLength);  HashTokenizer(String buffer, int maxNGramLength);  HashTokenizer(char[] buffer, int offset, int length, int maxNGramLength); int next();  }
@Test public void testManyIncrementalAdds() throws Exception { final int maxNGramLength = 3; HashTokenizer tokenizer = new HashTokenizer(maxNGramLength); final int numBigrams = 1000; int numNGrams = 0; for (int i = 0; i < numBigrams; i++) { tokenizer.addText("ab"); while (tokenizer.hasNext()) { tokenizer.next(); numNGrams += 1; } } tokenizer.complete(); while (tokenizer.hasNext()) { tokenizer.next(); numNGrams += 1; } final int totalChars = (numBigrams * 2) + 2; final int exectedNGrams = maxNGramLength * (totalChars - 1); assertEquals(exectedNGrams, numNGrams); }