WordCounter extends StandardStep<Document, WordCount> { @Override public void process(Document document) throws IOException { List<String> tokens = document.terms; ArrayList<WordCount> wordCounts = new ArrayList(); for (String t : tokens) { if (t != null) { if ((filterWords == null) || (!filterWords.contains(t))) { wordCounts.add(new WordCount(ByteUtil.fromString(t), 1, 1, 1)); } } } Collections.sort(wordCounts, new WordCount.WordOrder().lessThan()); WordCount last = null; for (WordCount wc : wordCounts) { if (last == null) { last = wc; } else if (CmpUtil.equals(wc.word, last.word)) { last.collectionFrequency += wc.collectionFrequency; last.maxDocumentFrequency += wc.maxDocumentFrequency; } else { processor.process(last); last = wc; } } if (last != null) { processor.process(last); } } WordCounter(TupleFlowParameters parameters); @Override void process(Document document);  }
@Test public void testCountUnigrams() throws IOException, IncompatibleProcessorException { WordCounter counter = new WordCounter(new FakeParameters(Parameters.create())); Document document = new Document(); PostStep post = new PostStep(); counter.setProcessor(post); document.terms = new ArrayList<>(); document.terms.add("one"); document.terms.add("two"); document.terms.add("one"); counter.process(document); assertEquals(2, post.results.size()); for (int i = 0; i < post.results.size(); ++i) { WordCount wc = post.results.get(i); if (CmpUtil.equals(wc.word, ByteUtil.fromString("one"))) { assertEquals(2, wc.collectionFrequency); } else if (CmpUtil.equals(wc.word, ByteUtil.fromString("one"))) { assertEquals(1, wc.collectionFrequency); } } }
@Test public void testCountReducer() throws IOException, IncompatibleProcessorException { Parameters p = Parameters.create(); WordCounter counter = new WordCounter(new FakeParameters(p)); Sorter sorter = new Sorter<WordCount>(new WordCount.WordOrder()); WordCountReducer reducer = new WordCountReducer(); PostStep post = new PostStep(); counter.setProcessor(sorter); sorter.setProcessor(reducer); reducer.setProcessor(post); Document document = new Document(); document.terms = new ArrayList<String>(); document.terms.add("one"); document.terms.add("two"); document.terms.add("one"); counter.process(document); document.terms = new ArrayList<String>(); document.terms.add("two"); document.terms.add("two"); document.terms.add("three"); counter.process(document); document.terms = new ArrayList<String>(); document.terms.add("one"); document.terms.add("three"); document.terms.add("four"); counter.process(document); counter.close(); assertEquals(4, post.results.size()); for (int i = 0; i < post.results.size(); ++i) { WordCount wc = post.results.get(i); if (CmpUtil.compare(wc.word, ByteUtil.fromString("one")) == 0) { assertEquals(3, wc.collectionFrequency); } else if (CmpUtil.compare(wc.word, ByteUtil.fromString("two")) == 0) { assertEquals(3, wc.collectionFrequency); } else if (CmpUtil.compare(wc.word, ByteUtil.fromString("three")) == 0) { assertEquals(2, wc.collectionFrequency); } else if (CmpUtil.compare(wc.word, ByteUtil.fromString("four")) == 0) { assertEquals(1, wc.collectionFrequency); } } }
@Test public void testCountFilter() throws IOException, IncompatibleProcessorException { Parameters p = Parameters.create(); p.set("minThreshold", 2); WordCounter counter = new WordCounter(new FakeParameters(p)); Sorter sorter = new Sorter<WordCount>(new WordCount.WordOrder()); WordCountReducer reducer = new WordCountReducer(); WordCountFilter filter = new WordCountFilter(new FakeParameters(p)); PostStep post = new PostStep(); counter.setProcessor(sorter); sorter.setProcessor(reducer); reducer.setProcessor(filter); filter.setProcessor(post); Document document = new Document(); document.terms = new ArrayList<String>(); document.terms.add("one"); document.terms.add("two"); document.terms.add("one"); counter.process(document); document.terms = new ArrayList<String>(); document.terms.add("two"); document.terms.add("two"); document.terms.add("three"); counter.process(document); document.terms = new ArrayList<String>(); document.terms.add("one"); document.terms.add("three"); document.terms.add("four"); counter.process(document); counter.close(); assertEquals(3, post.results.size()); for (int i = 0; i < post.results.size(); ++i) { WordCount wc = post.results.get(i); if (CmpUtil.compare(wc.word, ByteUtil.fromString("one")) == 0) { assertEquals(3, wc.collectionFrequency); } else if (CmpUtil.compare(wc.word, ByteUtil.fromString("two")) == 0) { assertEquals(3, wc.collectionFrequency); } else if (CmpUtil.compare(wc.word, ByteUtil.fromString("three")) == 0) { assertEquals(2, wc.collectionFrequency); } } }