@Test public void testNextWithEmptyThenValidToken() throws IOException { final Token token1 = new Token("", 0, 0); final Token token2 = new Token(HashTagTokenizer.HASHTAG_TEMPORARY_REPLACEMENT + "snutsBoo", 0, (HashTagTokenizer.HASHTAG_TEMPORARY_REPLACEMENT + "snutsBoo").length()); List<Token> tokens = new ArrayList<Token>(); tokens.add(token1); tokens.add(token2); tokenStream = new TokenStreamTestHelper(tokens); List<String> extractedHashtags = new ArrayList<String>(); List<String> extractedNonHashtags = new ArrayList<String>(); HashTagTokenizer sut = new HashTagTokenizer(tokenStream, extractedHashtags, extractedNonHashtags, false); assertNull(sut.next(reusableToken)); assertEquals(1, extractedNonHashtags.size()); assertEquals("snutsBoo", extractedNonHashtags.get(0)); assertEquals("", token2.term()); assertEquals(1, extractedHashtags.size()); assertEquals("#snutsBoo", extractedHashtags.get(0)); }
@Override public final Token next(final Token reusableToken) throws IOException { Token nextToken = null; assert reusableToken != null; do { nextToken = input.next(reusableToken); if (nextToken == null) { return null; } String termText = nextToken.term(); termText = termText.replaceAll(UNDERSCORE_TEMPORARY_REPLACEMENT, "_"); termText = termText.replaceAll(HASHTAG_TEMPORARY_REPLACEMENT, "#"); String newTokenText = ""; if (literalMode) { if (termText.contains("#")) { if (!extractedHashtags.contains(termText)) { extractedHashtags.add(termText); } } else { newTokenText = termText; } } else { Substring hashTag = hashTagExtractor.extract(termText, 0); if (hashTag != null) { String hashTagText = hashTag.getContent(); if (!extractedHashtags.contains(hashTagText)) { extractedHashtags.add(hashTagText); } } if (termText.contains("#") || termText.contains("_")) { if (!extractedHashtags.contains(termText)) { extractedHashtags.add(termText); } if (termText.contains("_")) { String temp = termText; while (temp.length() > 1 && temp.startsWith("#")) { temp = temp.substring(1); } if (!extractedHashtags.contains(temp)) { extractedHashtags.add(temp); } if (!extractedNonHashTags.contains(temp)) { extractedNonHashTags.add(temp); } } String[] parts = termText.split("[#_]"); for (int i = 0; i < parts.length; i++) { if (parts[i].length() > 0 && !extractedNonHashTags.contains(parts[i])) { extractedNonHashTags.add(parts[i]); } } } else { newTokenText = termText; } } nextToken.reinit(newTokenText, 0, newTokenText.length()); } while (nextToken.termLength() == 0); return nextToken; }
HashTagTokenizer extends TokenFilter { @Override public final Token next(final Token reusableToken) throws IOException { Token nextToken = null; assert reusableToken != null; do { nextToken = input.next(reusableToken); if (nextToken == null) { return null; } String termText = nextToken.term(); termText = termText.replaceAll(UNDERSCORE_TEMPORARY_REPLACEMENT, "_"); termText = termText.replaceAll(HASHTAG_TEMPORARY_REPLACEMENT, "#"); String newTokenText = ""; if (literalMode) { if (termText.contains("#")) { if (!extractedHashtags.contains(termText)) { extractedHashtags.add(termText); } } else { newTokenText = termText; } } else { Substring hashTag = hashTagExtractor.extract(termText, 0); if (hashTag != null) { String hashTagText = hashTag.getContent(); if (!extractedHashtags.contains(hashTagText)) { extractedHashtags.add(hashTagText); } } if (termText.contains("#") || termText.contains("_")) { if (!extractedHashtags.contains(termText)) { extractedHashtags.add(termText); } if (termText.contains("_")) { String temp = termText; while (temp.length() > 1 && temp.startsWith("#")) { temp = temp.substring(1); } if (!extractedHashtags.contains(temp)) { extractedHashtags.add(temp); } if (!extractedNonHashTags.contains(temp)) { extractedNonHashTags.add(temp); } } String[] parts = termText.split("[#_]"); for (int i = 0; i < parts.length; i++) { if (parts[i].length() > 0 && !extractedNonHashTags.contains(parts[i])) { extractedNonHashTags.add(parts[i]); } } } else { newTokenText = termText; } } nextToken.reinit(newTokenText, 0, newTokenText.length()); } while (nextToken.termLength() == 0); return nextToken; } }
HashTagTokenizer extends TokenFilter { @Override public final Token next(final Token reusableToken) throws IOException { Token nextToken = null; assert reusableToken != null; do { nextToken = input.next(reusableToken); if (nextToken == null) { return null; } String termText = nextToken.term(); termText = termText.replaceAll(UNDERSCORE_TEMPORARY_REPLACEMENT, "_"); termText = termText.replaceAll(HASHTAG_TEMPORARY_REPLACEMENT, "#"); String newTokenText = ""; if (literalMode) { if (termText.contains("#")) { if (!extractedHashtags.contains(termText)) { extractedHashtags.add(termText); } } else { newTokenText = termText; } } else { Substring hashTag = hashTagExtractor.extract(termText, 0); if (hashTag != null) { String hashTagText = hashTag.getContent(); if (!extractedHashtags.contains(hashTagText)) { extractedHashtags.add(hashTagText); } } if (termText.contains("#") || termText.contains("_")) { if (!extractedHashtags.contains(termText)) { extractedHashtags.add(termText); } if (termText.contains("_")) { String temp = termText; while (temp.length() > 1 && temp.startsWith("#")) { temp = temp.substring(1); } if (!extractedHashtags.contains(temp)) { extractedHashtags.add(temp); } if (!extractedNonHashTags.contains(temp)) { extractedNonHashTags.add(temp); } } String[] parts = termText.split("[#_]"); for (int i = 0; i < parts.length; i++) { if (parts[i].length() > 0 && !extractedNonHashTags.contains(parts[i])) { extractedNonHashTags.add(parts[i]); } } } else { newTokenText = termText; } } nextToken.reinit(newTokenText, 0, newTokenText.length()); } while (nextToken.termLength() == 0); return nextToken; } HashTagTokenizer(final TokenStream inInput, final List<String> inExtractedHashtags,
            final List<String> inExtractedNonHashTags, final boolean inLiteralMode); }
HashTagTokenizer extends TokenFilter { @Override public final Token next(final Token reusableToken) throws IOException { Token nextToken = null; assert reusableToken != null; do { nextToken = input.next(reusableToken); if (nextToken == null) { return null; } String termText = nextToken.term(); termText = termText.replaceAll(UNDERSCORE_TEMPORARY_REPLACEMENT, "_"); termText = termText.replaceAll(HASHTAG_TEMPORARY_REPLACEMENT, "#"); String newTokenText = ""; if (literalMode) { if (termText.contains("#")) { if (!extractedHashtags.contains(termText)) { extractedHashtags.add(termText); } } else { newTokenText = termText; } } else { Substring hashTag = hashTagExtractor.extract(termText, 0); if (hashTag != null) { String hashTagText = hashTag.getContent(); if (!extractedHashtags.contains(hashTagText)) { extractedHashtags.add(hashTagText); } } if (termText.contains("#") || termText.contains("_")) { if (!extractedHashtags.contains(termText)) { extractedHashtags.add(termText); } if (termText.contains("_")) { String temp = termText; while (temp.length() > 1 && temp.startsWith("#")) { temp = temp.substring(1); } if (!extractedHashtags.contains(temp)) { extractedHashtags.add(temp); } if (!extractedNonHashTags.contains(temp)) { extractedNonHashTags.add(temp); } } String[] parts = termText.split("[#_]"); for (int i = 0; i < parts.length; i++) { if (parts[i].length() > 0 && !extractedNonHashTags.contains(parts[i])) { extractedNonHashTags.add(parts[i]); } } } else { newTokenText = termText; } } nextToken.reinit(newTokenText, 0, newTokenText.length()); } while (nextToken.termLength() == 0); return nextToken; } HashTagTokenizer(final TokenStream inInput, final List<String> inExtractedHashtags,
            final List<String> inExtractedNonHashTags, final boolean inLiteralMode); @Override final Token next(final Token reusableToken); }
HashTagTokenizer extends TokenFilter { @Override public final Token next(final Token reusableToken) throws IOException { Token nextToken = null; assert reusableToken != null; do { nextToken = input.next(reusableToken); if (nextToken == null) { return null; } String termText = nextToken.term(); termText = termText.replaceAll(UNDERSCORE_TEMPORARY_REPLACEMENT, "_"); termText = termText.replaceAll(HASHTAG_TEMPORARY_REPLACEMENT, "#"); String newTokenText = ""; if (literalMode) { if (termText.contains("#")) { if (!extractedHashtags.contains(termText)) { extractedHashtags.add(termText); } } else { newTokenText = termText; } } else { Substring hashTag = hashTagExtractor.extract(termText, 0); if (hashTag != null) { String hashTagText = hashTag.getContent(); if (!extractedHashtags.contains(hashTagText)) { extractedHashtags.add(hashTagText); } } if (termText.contains("#") || termText.contains("_")) { if (!extractedHashtags.contains(termText)) { extractedHashtags.add(termText); } if (termText.contains("_")) { String temp = termText; while (temp.length() > 1 && temp.startsWith("#")) { temp = temp.substring(1); } if (!extractedHashtags.contains(temp)) { extractedHashtags.add(temp); } if (!extractedNonHashTags.contains(temp)) { extractedNonHashTags.add(temp); } } String[] parts = termText.split("[#_]"); for (int i = 0; i < parts.length; i++) { if (parts[i].length() > 0 && !extractedNonHashTags.contains(parts[i])) { extractedNonHashTags.add(parts[i]); } } } else { newTokenText = termText; } } nextToken.reinit(newTokenText, 0, newTokenText.length()); } while (nextToken.termLength() == 0); return nextToken; } HashTagTokenizer(final TokenStream inInput, final List<String> inExtractedHashtags,
            final List<String> inExtractedNonHashTags, final boolean inLiteralMode); @Override final Token next(final Token reusableToken); static final String HASHTAG_TEMPORARY_REPLACEMENT; static final String UNDERSCORE_TEMPORARY_REPLACEMENT; }