TextTokenizer extends BaseTokenizer { public String next() { if (!hasNext()) { throw new IllegalStateException("No next ngram to return"); } CharSequence result = CharBuffer.wrap(_normalized, _normalizedPos, _curNGramLength); nextNGram(); return result.toString(); } TextTokenizer(String text, int maxNGramLength);  TextTokenizer(char[] buffer, int offset, int length, int maxNGramLength); String next();  }
@Test public void testMyNGramIteratorTrigrams() throws Exception { TextTokenizer tokenizer = new TextTokenizer("abc", 3); Set<String> ngrams = new HashSet<String>(); while (tokenizer.hasNext()) { String ngram = tokenizer.next(); System.out.println(ngram); ngrams.add(ngram); } assertEquals(12 - 1, ngrams.size()); }
@Test public void testNormalizationAndSpaceCollapse() throws Exception { TextTokenizer tokenizer = new TextTokenizer("A ,!", 3); Set<String> ngrams = new HashSet<String>(); while (tokenizer.hasNext()) { String ngram = tokenizer.next(); ngrams.add(ngram.toString()); } assertEquals(5, ngrams.size()); assertTrue(ngrams.contains(" ")); assertTrue(ngrams.contains(" a")); assertTrue(ngrams.contains(" a ")); assertTrue(ngrams.contains("a")); assertTrue(ngrams.contains("a ")); }
@Test public void testReturnNormalization() throws Exception { TextTokenizer tokenizer = new TextTokenizer("\r\t\n", 1); assertTrue(tokenizer.hasNext()); assertEquals(" ", tokenizer.next()); assertFalse(tokenizer.hasNext()); tokenizer.complete(); assertFalse(tokenizer.hasNext()); }
@Test public void testNormalizedBufferReset() throws Exception { int numChars = 2000; StringBuilder sb = new StringBuilder(); for (int i = 0; i < numChars; i++) { sb.append((char)((int)'a' + (i % 10))); } int numNGrams = 0; TextTokenizer tokenizer = new TextTokenizer(sb.toString(), 3); while (tokenizer.hasNext()) { tokenizer.next(); numNGrams += 1; } assertEquals(3 * ((numChars + 2) - 1), numNGrams); }