TwitterTokenizer extends EnglishTokenizer { @Override public List<String> tokenize(String text) { text = this.preprocess(text); return super.tokenize(text); } TwitterTokenizer();  TwitterTokenizer(int minNGram, int maxNGram); @Override List<String> tokenize(String text);  }
@Test public void testRemoveUsername() { String tweet = "@PrincessSuperC Hey Cici"; TwitterTokenizer tokenizer = new TwitterTokenizer(); List<String> actualTokens = tokenizer.tokenize(tweet); List<String> expectedTokens = Arrays.asList("hei", "cici"); assertEquals(expectedTokens, actualTokens); }
@Test public void testRemoveHashTagIndicator() { String tweet = "arg #word!"; TwitterTokenizer tokenizer = new TwitterTokenizer(); List<String> actualTokens = tokenizer.tokenize(tweet); List<String> expectedTokens = Arrays.asList("arg", "word"); assertEquals(expectedTokens, actualTokens); }
@Test public void testRemoveCharacterRepetitions() { String tweet = "so huuunggryy!"; TwitterTokenizer tokenizer = new TwitterTokenizer(); List<String> actualTokens = tokenizer.tokenize(tweet); List<String> expectedTokens = Arrays.asList("so", "hungri"); assertEquals(expectedTokens, actualTokens); }
@Test public void testRemoveWordsStartingNumber() { String tweet = "it's 15PM we're in the 20th century"; TwitterTokenizer tokenizer = new TwitterTokenizer(); List<String> actualTokens = tokenizer.tokenize(tweet); List<String> expectedTokens = Arrays.asList("we'r", "centuri"); assertEquals(expectedTokens, actualTokens); }
@Test public void testRemoveHTMLMarkum() { String tweet = "fb &gt; tw"; TwitterTokenizer tokenizer = new TwitterTokenizer(); List<String> actualTokens = tokenizer.tokenize(tweet); List<String> expectedTokens = Arrays.asList("fb", "tw"); assertEquals(expectedTokens, actualTokens); }
@Test public void testWithNGram() { String tweet = "it's not bad movie"; TwitterTokenizer tokenizer = new TwitterTokenizer(2, 2); List<String> actualTokens = tokenizer.tokenize(tweet); List<String> expectedTokens = Arrays.asList("_ bad", "bad", "bad movi", "movi"); assertEquals(expectedTokens, actualTokens); }