MailArchivesClusteringAnalyzer extends Analyzer { @Override public TokenStream tokenStream(String fieldName, java.io.Reader reader) { @SuppressWarnings("deprecation") TokenStream result = new StandardTokenizer(Version.LUCENE_CURRENT, reader); result = new StandardFilter(result); result = new LowerCaseFilter(result); result = new ASCIIFoldingFilter(result); result = new AlphaNumericMaxLengthFilter(result); result = new StopFilter(false, result, stopSet); return new PorterStemFilter(result); } MailArchivesClusteringAnalyzer();  MailArchivesClusteringAnalyzer(CharArraySet stopSet); @Override TokenStream tokenStream(String fieldName, java.io.Reader reader);  }
@Test public void testAnalysis() throws Exception { MailArchivesClusteringAnalyzer analyzer = new MailArchivesClusteringAnalyzer(); String text = "A test message\n"; text += "atokenthatistoolongtobeusefulforclustertextanalysis\n"; text += "Mahout is a scalable, machine-learning LIBRARY\n"; text += "we've added some additional stopwords such as html, mailto, regards\t"; text += "apache_hadoop provides the foundation for scalability\n"; text += "www.nabble.com general-help@incubator.apache.org\n"; text += "public void int protected package"; StringReader reader = new StringReader(text); String[] expectedTokens = { "test", "mahout", "scalabl", "machin", "learn", "librari", "weve", "ad", "stopword", "apach", "hadoop", "provid", "foundat", "scalabl" }; TokenStream tokenStream = analyzer.tokenStream("test", reader); assertNotNull(tokenStream); TermAttribute termAtt = tokenStream.addAttribute(TermAttribute.class); int e = -1; while (tokenStream.incrementToken()) { assertEquals(expectedTokens[++e], termAtt.term()); } }