@Test public void testClusteringWithDfThreshold() { STCClusteringAlgorithm algorithm = algorithm(); algorithm.preprocessing.wordDfThreshold.set(20); List<Cluster<Document>> clusters = algorithm.cluster( SampleDocumentData.DOCUMENTS_DATA_MINING.stream(), CachedLangComponents.loadCached("English")); Assertions.assertThat(clusters.size()).isEqualTo(0); }
@Override public <T extends Document> List<Cluster<T>> cluster( Stream<? extends T> docStream, LanguageComponents languageComponents) { List<T> documents = docStream.collect(Collectors.toList()); List<Cluster<T>> clusters = new ArrayList<>(); context = preprocessing.preprocess(documents.stream(), queryHint.get(), languageComponents); labelFormatter = context.languageComponents.get(LabelFormatter.class); sb = new GeneralizedSuffixTree.SequenceBuilder(); final int[] tokenIndex = context.allTokens.wordIndex; final short[] tokenType = context.allTokens.type; for (int i = 0; i < tokenIndex.length; i++) { if (tokenIndex[i] == -1) { if ((tokenType[i] & (Tokenizer.TF_SEPARATOR_DOCUMENT | Tokenizer.TF_TERMINATOR)) != 0) { sb.endDocument(); } continue; } final int s = i; while (tokenIndex[i + 1] != -1) i++; final int phraseLength = 1 + i - s; if (phraseLength >= 1) { sb.addPhrase(tokenIndex, s, phraseLength); } } sb.buildSuffixTree(); List<ClusterCandidate> baseClusters = createBaseClusters(sb); List<ClusterCandidate> mergedClusters = createMergedClusters(baseClusters); postProcessing(documents, mergedClusters, clusters); return SharedInfrastructure.reorderByWeightedScoreAndSize(clusters, this.scoreWeight.get()); }
STCClusteringAlgorithm extends AttrComposite implements ClusteringAlgorithm { @Override public <T extends Document> List<Cluster<T>> cluster( Stream<? extends T> docStream, LanguageComponents languageComponents) { List<T> documents = docStream.collect(Collectors.toList()); List<Cluster<T>> clusters = new ArrayList<>(); context = preprocessing.preprocess(documents.stream(), queryHint.get(), languageComponents); labelFormatter = context.languageComponents.get(LabelFormatter.class); sb = new GeneralizedSuffixTree.SequenceBuilder(); final int[] tokenIndex = context.allTokens.wordIndex; final short[] tokenType = context.allTokens.type; for (int i = 0; i < tokenIndex.length; i++) { if (tokenIndex[i] == -1) { if ((tokenType[i] & (Tokenizer.TF_SEPARATOR_DOCUMENT | Tokenizer.TF_TERMINATOR)) != 0) { sb.endDocument(); } continue; } final int s = i; while (tokenIndex[i + 1] != -1) i++; final int phraseLength = 1 + i - s; if (phraseLength >= 1) { sb.addPhrase(tokenIndex, s, phraseLength); } } sb.buildSuffixTree(); List<ClusterCandidate> baseClusters = createBaseClusters(sb); List<ClusterCandidate> mergedClusters = createMergedClusters(baseClusters); postProcessing(documents, mergedClusters, clusters); return SharedInfrastructure.reorderByWeightedScoreAndSize(clusters, this.scoreWeight.get()); } }
STCClusteringAlgorithm extends AttrComposite implements ClusteringAlgorithm { @Override public <T extends Document> List<Cluster<T>> cluster( Stream<? extends T> docStream, LanguageComponents languageComponents) { List<T> documents = docStream.collect(Collectors.toList()); List<Cluster<T>> clusters = new ArrayList<>(); context = preprocessing.preprocess(documents.stream(), queryHint.get(), languageComponents); labelFormatter = context.languageComponents.get(LabelFormatter.class); sb = new GeneralizedSuffixTree.SequenceBuilder(); final int[] tokenIndex = context.allTokens.wordIndex; final short[] tokenType = context.allTokens.type; for (int i = 0; i < tokenIndex.length; i++) { if (tokenIndex[i] == -1) { if ((tokenType[i] & (Tokenizer.TF_SEPARATOR_DOCUMENT | Tokenizer.TF_TERMINATOR)) != 0) { sb.endDocument(); } continue; } final int s = i; while (tokenIndex[i + 1] != -1) i++; final int phraseLength = 1 + i - s; if (phraseLength >= 1) { sb.addPhrase(tokenIndex, s, phraseLength); } } sb.buildSuffixTree(); List<ClusterCandidate> baseClusters = createBaseClusters(sb); List<ClusterCandidate> mergedClusters = createMergedClusters(baseClusters); postProcessing(documents, mergedClusters, clusters); return SharedInfrastructure.reorderByWeightedScoreAndSize(clusters, this.scoreWeight.get()); }  }
STCClusteringAlgorithm extends AttrComposite implements ClusteringAlgorithm { @Override public <T extends Document> List<Cluster<T>> cluster( Stream<? extends T> docStream, LanguageComponents languageComponents) { List<T> documents = docStream.collect(Collectors.toList()); List<Cluster<T>> clusters = new ArrayList<>(); context = preprocessing.preprocess(documents.stream(), queryHint.get(), languageComponents); labelFormatter = context.languageComponents.get(LabelFormatter.class); sb = new GeneralizedSuffixTree.SequenceBuilder(); final int[] tokenIndex = context.allTokens.wordIndex; final short[] tokenType = context.allTokens.type; for (int i = 0; i < tokenIndex.length; i++) { if (tokenIndex[i] == -1) { if ((tokenType[i] & (Tokenizer.TF_SEPARATOR_DOCUMENT | Tokenizer.TF_TERMINATOR)) != 0) { sb.endDocument(); } continue; } final int s = i; while (tokenIndex[i + 1] != -1) i++; final int phraseLength = 1 + i - s; if (phraseLength >= 1) { sb.addPhrase(tokenIndex, s, phraseLength); } } sb.buildSuffixTree(); List<ClusterCandidate> baseClusters = createBaseClusters(sb); List<ClusterCandidate> mergedClusters = createMergedClusters(baseClusters); postProcessing(documents, mergedClusters, clusters); return SharedInfrastructure.reorderByWeightedScoreAndSize(clusters, this.scoreWeight.get()); }  @Override Set<Class<?>> requiredLanguageComponents(); @Override List<Cluster<T>> cluster(
      Stream<? extends T> docStream, LanguageComponents languageComponents); }
STCClusteringAlgorithm extends AttrComposite implements ClusteringAlgorithm { @Override public <T extends Document> List<Cluster<T>> cluster( Stream<? extends T> docStream, LanguageComponents languageComponents) { List<T> documents = docStream.collect(Collectors.toList()); List<Cluster<T>> clusters = new ArrayList<>(); context = preprocessing.preprocess(documents.stream(), queryHint.get(), languageComponents); labelFormatter = context.languageComponents.get(LabelFormatter.class); sb = new GeneralizedSuffixTree.SequenceBuilder(); final int[] tokenIndex = context.allTokens.wordIndex; final short[] tokenType = context.allTokens.type; for (int i = 0; i < tokenIndex.length; i++) { if (tokenIndex[i] == -1) { if ((tokenType[i] & (Tokenizer.TF_SEPARATOR_DOCUMENT | Tokenizer.TF_TERMINATOR)) != 0) { sb.endDocument(); } continue; } final int s = i; while (tokenIndex[i + 1] != -1) i++; final int phraseLength = 1 + i - s; if (phraseLength >= 1) { sb.addPhrase(tokenIndex, s, phraseLength); } } sb.buildSuffixTree(); List<ClusterCandidate> baseClusters = createBaseClusters(sb); List<ClusterCandidate> mergedClusters = createMergedClusters(baseClusters); postProcessing(documents, mergedClusters, clusters); return SharedInfrastructure.reorderByWeightedScoreAndSize(clusters, this.scoreWeight.get()); }  @Override Set<Class<?>> requiredLanguageComponents(); @Override List<Cluster<T>> cluster(
      Stream<? extends T> docStream, LanguageComponents languageComponents); static final String NAME; final AttrString queryHint; public AttrDouble ignoreWordIfInHigherDocsPercent; public AttrDouble minBaseClusterScore; public AttrInteger minBaseClusterSize; public AttrInteger maxBaseClusters; public AttrInteger maxClusters; public AttrDouble mergeThreshold; public AttrDouble maxPhraseOverlap; public AttrDouble mostGeneralPhraseCoverage; public AttrInteger maxWordsPerLabel; public AttrInteger maxPhrasesPerLabel; public AttrDouble singleTermBoost; public AttrInteger optimalPhraseLength; public AttrDouble optimalPhraseLengthDev; public AttrDouble documentCountBoost; public AttrDouble scoreWeight; public AttrBoolean mergeStemEquivalentBaseClusters; public BasicPreprocessingPipeline preprocessing; }