@Test public void testNextWithEmptyThenValidToken() throws IOException { List<String> extractedKeywords = new ArrayList<String>(); final Token token1 = new Token("", 0, 0); final Token token2 = new Token("ABCD", 0, "ABCD".length()); List<Token> tokens = new ArrayList<Token>(); tokens.add(token1); tokens.add(token2); tokenStream = new TokenStreamTestHelper(tokens); PrefixedTokenRemoverAndExtractorTokenizer sut = new PrefixedTokenRemoverAndExtractorTokenizer(tokenStream, "FOO", "#", extractedKeywords); assertSame(token2, sut.next(reusableToken)); assertEquals("ABCD", token2.term()); assertEquals(0, token2.startOffset()); assertEquals("ABCD".length(), token2.endOffset()); }
@Override public final Token next(final Token reusableToken) throws IOException { Token nextToken = null; assert reusableToken != null; do { nextToken = input.next(reusableToken); if (nextToken == null) { return null; } String termText = nextToken.term(); if (termText.contains(oldPrefix)) { String keyword = termText.replace(oldPrefix, newPrefix); nextToken.reinit(keyword, 0, keyword.length()); if (keyword.startsWith(newPrefix)) { if (!extractedKeywords.contains(keyword)) { if (keyword.replace(newPrefix, "").length() > 0) { extractedKeywords.add(keyword); } } } } } while (nextToken.termLength() == 0 || nextToken.term().startsWith(newPrefix)); return nextToken; }
PrefixedTokenRemoverAndExtractorTokenizer extends TokenFilter { @Override public final Token next(final Token reusableToken) throws IOException { Token nextToken = null; assert reusableToken != null; do { nextToken = input.next(reusableToken); if (nextToken == null) { return null; } String termText = nextToken.term(); if (termText.contains(oldPrefix)) { String keyword = termText.replace(oldPrefix, newPrefix); nextToken.reinit(keyword, 0, keyword.length()); if (keyword.startsWith(newPrefix)) { if (!extractedKeywords.contains(keyword)) { if (keyword.replace(newPrefix, "").length() > 0) { extractedKeywords.add(keyword); } } } } } while (nextToken.termLength() == 0 || nextToken.term().startsWith(newPrefix)); return nextToken; } }
PrefixedTokenRemoverAndExtractorTokenizer extends TokenFilter { @Override public final Token next(final Token reusableToken) throws IOException { Token nextToken = null; assert reusableToken != null; do { nextToken = input.next(reusableToken); if (nextToken == null) { return null; } String termText = nextToken.term(); if (termText.contains(oldPrefix)) { String keyword = termText.replace(oldPrefix, newPrefix); nextToken.reinit(keyword, 0, keyword.length()); if (keyword.startsWith(newPrefix)) { if (!extractedKeywords.contains(keyword)) { if (keyword.replace(newPrefix, "").length() > 0) { extractedKeywords.add(keyword); } } } } } while (nextToken.termLength() == 0 || nextToken.term().startsWith(newPrefix)); return nextToken; } PrefixedTokenRemoverAndExtractorTokenizer(final TokenStream inInput, final String inOldPrefix,
            final String inNewPrefix, final List<String> inExtractedKeywords); }
PrefixedTokenRemoverAndExtractorTokenizer extends TokenFilter { @Override public final Token next(final Token reusableToken) throws IOException { Token nextToken = null; assert reusableToken != null; do { nextToken = input.next(reusableToken); if (nextToken == null) { return null; } String termText = nextToken.term(); if (termText.contains(oldPrefix)) { String keyword = termText.replace(oldPrefix, newPrefix); nextToken.reinit(keyword, 0, keyword.length()); if (keyword.startsWith(newPrefix)) { if (!extractedKeywords.contains(keyword)) { if (keyword.replace(newPrefix, "").length() > 0) { extractedKeywords.add(keyword); } } } } } while (nextToken.termLength() == 0 || nextToken.term().startsWith(newPrefix)); return nextToken; } PrefixedTokenRemoverAndExtractorTokenizer(final TokenStream inInput, final String inOldPrefix,
            final String inNewPrefix, final List<String> inExtractedKeywords); @Override final Token next(final Token reusableToken); }
PrefixedTokenRemoverAndExtractorTokenizer extends TokenFilter { @Override public final Token next(final Token reusableToken) throws IOException { Token nextToken = null; assert reusableToken != null; do { nextToken = input.next(reusableToken); if (nextToken == null) { return null; } String termText = nextToken.term(); if (termText.contains(oldPrefix)) { String keyword = termText.replace(oldPrefix, newPrefix); nextToken.reinit(keyword, 0, keyword.length()); if (keyword.startsWith(newPrefix)) { if (!extractedKeywords.contains(keyword)) { if (keyword.replace(newPrefix, "").length() > 0) { extractedKeywords.add(keyword); } } } } } while (nextToken.termLength() == 0 || nextToken.term().startsWith(newPrefix)); return nextToken; } PrefixedTokenRemoverAndExtractorTokenizer(final TokenStream inInput, final String inOldPrefix,
            final String inNewPrefix, final List<String> inExtractedKeywords); @Override final Token next(final Token reusableToken);  }