TextInstanceCreator extends BaseFunction { protected List<String> extractTokens(String text) { List<String> tokens = this.textAnalyser.tokenize(text); return tokens; } TextInstanceCreator();  TextInstanceCreator(boolean withLabel);  TextInstanceCreator(boolean withLabel, TextTokenizer textAnalyser); @Override void execute(TridentTuple tuple, TridentCollector collector); boolean isWithLabel(); void setWithLabel(boolean withLabel); TextTokenizer getTextAnalyser(); void setTextAnalyser(TextTokenizer textAnalyser);  }
@Test public void testExtractTokens() { TextInstanceCreator<Boolean> instanceCreator = new TextInstanceCreator<Boolean>(false); List<String> actualWords = instanceCreator.extractTokens("I can't argue with some arguments on argus with argues"); List<String> expectedWords = Arrays.asList("i", "can't", "argu", "some", "argument", "argu", "argu"); assertEquals(expectedWords, actualWords); }