EnglishTokenizer implements Serializable, TextTokenizer { public List<String> tokenize(String text) { List<String> words = new ArrayList<String>(); if (text != null && !text.isEmpty()) { TokenStream tokenStream = this.createTokenStream(text); CharTermAttribute charTermAttribute = tokenStream.addAttribute(CharTermAttribute.class); try { while (tokenStream.incrementToken()) { String term = charTermAttribute.toString(); words.add(term); } } catch (IOException ioe) { LOGGER.error("Unable to analyze text. Cause : " + ioe.getMessage(), ioe); } finally { try { tokenStream.end(); tokenStream.close(); } catch (IOException e) { LOGGER.error("Unable to close token stream : " + e.getMessage()); } } } return words; } EnglishTokenizer();  EnglishTokenizer(int minNGram, int maxNGram);  EnglishTokenizer(List<String> stopWords);  EnglishTokenizer(List<String> stopWords, Set<String> stemExclusionsSet); List<String> tokenize(String text); void setNGram(int minNGram, int maxNGram); boolean isnGram(); void setnGram(boolean nGram); int getMinNGram(); void setMinNGram(int minNGram); int getMaxNGram(); void setMaxNGram(int maxNGram); List<String> getStopWords(); void setStopWords(List<String> stopWords); void setStemExclusionsSet(Set<String> stemExclusionsSet); Set<String> getStemExclusionsSet();  }
@Test public void testTokenize() { EnglishTokenizer tokenizer = new EnglishTokenizer(); String text = "I can't argue with some arguments on argus with argues"; List<String> actualTokens = tokenizer.tokenize(text); List<String> expectedTokens = Arrays.asList("i", "can't", "argu", "some", "argument", "argu", "argu"); assertEquals(expectedTokens, actualTokens); }