@Test public void testNext() throws Exception{ Collection<Token> tokens = new ArrayList<Token>(); tokens.add(newToken("token1 token2 token3", 0, 6)); tokens.add(newToken("token4 token5 token6", 7, 13)); TokenStream tokenStream = new CollectionTokenStream(tokens); TokenFilter filter = new SplitterFilter(tokenStream, " "); Token nextToken = new Token(); filter.next(nextToken); assertNotNull(nextToken); assertEquals("token1", new String(nextToken.termBuffer(), 0, nextToken.termLength())); assertEquals(0, nextToken.startOffset()); assertEquals(6, nextToken.endOffset()); nextToken = filter.next(nextToken); assertNotNull(nextToken); assertEquals("token2", new String(nextToken.termBuffer(), 0, nextToken.termLength())); assertEquals(0, nextToken.startOffset()); assertEquals(6, nextToken.endOffset()); nextToken = filter.next(nextToken); assertNotNull(nextToken); assertEquals("token3", new String(nextToken.termBuffer(), 0, nextToken.termLength())); assertEquals(0, nextToken.startOffset()); assertEquals(6, nextToken.endOffset()); nextToken = filter.next(nextToken); assertNotNull(nextToken); assertEquals("token4", new String(nextToken.termBuffer(), 0, nextToken.termLength())); assertEquals(7, nextToken.startOffset()); assertEquals(13, nextToken.endOffset()); nextToken = filter.next(nextToken); assertNotNull(nextToken); assertEquals("token5", new String(nextToken.termBuffer(), 0, nextToken.termLength())); assertEquals(7, nextToken.startOffset()); assertEquals(13, nextToken.endOffset()); nextToken = filter.next(nextToken); assertNotNull(nextToken); assertEquals("token6", new String(nextToken.termBuffer(), 0, nextToken.termLength())); assertEquals(7, nextToken.startOffset()); assertEquals(13, nextToken.endOffset()); }
@Override public Token next(Token token) throws IOException { if (!splitIterator.hasNext()) { currentToken = input.next(token); if (currentToken == null) return null; String tokenText = new String(currentToken.termBuffer(), 0, currentToken.termLength()); String[] splitts = tokenText.split(splitString); splitIterator = Iterators.forArray(splitts, 0, splitts.length); } if (!splitIterator.hasNext()) return null; token.setStartOffset(currentToken.startOffset()); token.setEndOffset(currentToken.endOffset()); char[] termBuffer = splitIterator.next().toCharArray(); token.setTermBuffer(termBuffer, 0, termBuffer.length); return token; }
SplitterFilter extends TokenFilter { @Override public Token next(Token token) throws IOException { if (!splitIterator.hasNext()) { currentToken = input.next(token); if (currentToken == null) return null; String tokenText = new String(currentToken.termBuffer(), 0, currentToken.termLength()); String[] splitts = tokenText.split(splitString); splitIterator = Iterators.forArray(splitts, 0, splitts.length); } if (!splitIterator.hasNext()) return null; token.setStartOffset(currentToken.startOffset()); token.setEndOffset(currentToken.endOffset()); char[] termBuffer = splitIterator.next().toCharArray(); token.setTermBuffer(termBuffer, 0, termBuffer.length); return token; } }
SplitterFilter extends TokenFilter { @Override public Token next(Token token) throws IOException { if (!splitIterator.hasNext()) { currentToken = input.next(token); if (currentToken == null) return null; String tokenText = new String(currentToken.termBuffer(), 0, currentToken.termLength()); String[] splitts = tokenText.split(splitString); splitIterator = Iterators.forArray(splitts, 0, splitts.length); } if (!splitIterator.hasNext()) return null; token.setStartOffset(currentToken.startOffset()); token.setEndOffset(currentToken.endOffset()); char[] termBuffer = splitIterator.next().toCharArray(); token.setTermBuffer(termBuffer, 0, termBuffer.length); return token; } SplitterFilter(TokenStream input, String splitString); }
SplitterFilter extends TokenFilter { @Override public Token next(Token token) throws IOException { if (!splitIterator.hasNext()) { currentToken = input.next(token); if (currentToken == null) return null; String tokenText = new String(currentToken.termBuffer(), 0, currentToken.termLength()); String[] splitts = tokenText.split(splitString); splitIterator = Iterators.forArray(splitts, 0, splitts.length); } if (!splitIterator.hasNext()) return null; token.setStartOffset(currentToken.startOffset()); token.setEndOffset(currentToken.endOffset()); char[] termBuffer = splitIterator.next().toCharArray(); token.setTermBuffer(termBuffer, 0, termBuffer.length); return token; } SplitterFilter(TokenStream input, String splitString); @Override Token next(Token token); @Override Token next(); String getSplitString(); }
SplitterFilter extends TokenFilter { @Override public Token next(Token token) throws IOException { if (!splitIterator.hasNext()) { currentToken = input.next(token); if (currentToken == null) return null; String tokenText = new String(currentToken.termBuffer(), 0, currentToken.termLength()); String[] splitts = tokenText.split(splitString); splitIterator = Iterators.forArray(splitts, 0, splitts.length); } if (!splitIterator.hasNext()) return null; token.setStartOffset(currentToken.startOffset()); token.setEndOffset(currentToken.endOffset()); char[] termBuffer = splitIterator.next().toCharArray(); token.setTermBuffer(termBuffer, 0, termBuffer.length); return token; } SplitterFilter(TokenStream input, String splitString); @Override Token next(Token token); @Override Token next(); String getSplitString();  }