STCClusteringAlgorithm extends AttrComposite implements ClusteringAlgorithm { @Override public <T extends Document> List<Cluster<T>> cluster( Stream<? extends T> docStream, LanguageComponents languageComponents) { List<T> documents = docStream.collect(Collectors.toList()); List<Cluster<T>> clusters = new ArrayList<>(); context = preprocessing.preprocess(documents.stream(), queryHint.get(), languageComponents); labelFormatter = context.languageComponents.get(LabelFormatter.class); sb = new GeneralizedSuffixTree.SequenceBuilder(); final int[] tokenIndex = context.allTokens.wordIndex; final short[] tokenType = context.allTokens.type; for (int i = 0; i < tokenIndex.length; i++) { if (tokenIndex[i] == -1) { if ((tokenType[i] & (Tokenizer.TF_SEPARATOR_DOCUMENT | Tokenizer.TF_TERMINATOR)) != 0) { sb.endDocument(); } continue; } final int s = i; while (tokenIndex[i + 1] != -1) i++; final int phraseLength = 1 + i - s; if (phraseLength >= 1) { sb.addPhrase(tokenIndex, s, phraseLength); } } sb.buildSuffixTree(); List<ClusterCandidate> baseClusters = createBaseClusters(sb); List<ClusterCandidate> mergedClusters = createMergedClusters(baseClusters); postProcessing(documents, mergedClusters, clusters); return SharedInfrastructure.reorderByWeightedScoreAndSize(clusters, this.scoreWeight.get()); }  @Override Set<Class<?>> requiredLanguageComponents(); @Override List<Cluster<T>> cluster(
      Stream<? extends T> docStream, LanguageComponents languageComponents); static final String NAME; final AttrString queryHint; public AttrDouble ignoreWordIfInHigherDocsPercent; public AttrDouble minBaseClusterScore; public AttrInteger minBaseClusterSize; public AttrInteger maxBaseClusters; public AttrInteger maxClusters; public AttrDouble mergeThreshold; public AttrDouble maxPhraseOverlap; public AttrDouble mostGeneralPhraseCoverage; public AttrInteger maxWordsPerLabel; public AttrInteger maxPhrasesPerLabel; public AttrDouble singleTermBoost; public AttrInteger optimalPhraseLength; public AttrDouble optimalPhraseLengthDev; public AttrDouble documentCountBoost; public AttrDouble scoreWeight; public AttrBoolean mergeStemEquivalentBaseClusters; public BasicPreprocessingPipeline preprocessing; }
@Test public void testClusteringWithDfThreshold() { STCClusteringAlgorithm algorithm = algorithm(); algorithm.preprocessing.wordDfThreshold.set(20); List<Cluster<Document>> clusters = algorithm.cluster( SampleDocumentData.DOCUMENTS_DATA_MINING.stream(), CachedLangComponents.loadCached("English")); Assertions.assertThat(clusters.size()).isEqualTo(0); }
@Test public void testMaxClusters() { STCClusteringAlgorithm algorithm = algorithm(); algorithm.maxClusters.set(9); List<Cluster<Document>> clusters = algorithm.cluster( SampleDocumentData.DOCUMENTS_DATA_MINING.stream(), CachedLangComponents.loadCached("English")); Assertions.assertThat(clusters.size()).isEqualTo(9); }
@Test public void testMergingBaseClustersWithStemEquivalentPhrases() { List<String> titles = Arrays.asList( "good programs . foo1", "foo2 good programs . foo2", "good programs taste good", "good programs are good", "good programming . foo3", "foo4 good programming . foo4", "good programming makes you feel better"); STCClusteringAlgorithm algorithm = algorithm(); algorithm.minBaseClusterScore.set(0d); Stream<TestDocument> documentStream = titles.stream().map(title -> new TestDocument(title)); List<Cluster<Document>> clusters = algorithm.cluster(documentStream, CachedLangComponents.loadCached("English")); Assertions.assertThat(clusters.stream().flatMap(cluster -> cluster.getLabels().stream())) .contains("Good Programs") .doesNotContain("Good Programming"); }
@Test public void testCarrot1008() throws Exception { DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance(); DocumentBuilder db = dbf.newDocumentBuilder(); Element root; try (InputStream is = getClass().getResourceAsStream("CARROT-1008.xml")) { root = db.parse(is).getDocumentElement(); } Stream<TestDocument> docStream = elementStream(root, "document") .map( docElement -> { String title = elementStream(docElement, "title") .map(e -> e.getTextContent()) .collect(Collectors.joining(" . ")); String snippet = elementStream(docElement, "snippet") .map(e -> e.getTextContent()) .collect(Collectors.joining(" . ")); return new TestDocument(title, snippet); }); STCClusteringAlgorithm algorithm = algorithm(); algorithm.maxClusters.set(30); List<Cluster<Document>> clusters = algorithm.cluster(docStream, CachedLangComponents.loadCached("English")); List<String> collect = clusters.stream().flatMap(c -> c.getLabels().stream()).collect(Collectors.toList()); Assertions.assertThat(collect.contains("Guns") && collect.contains("Gun")).isFalse(); }