LingoClusteringAlgorithm extends AttrComposite implements ClusteringAlgorithm { @Override public <T extends Document> List<Cluster<T>> cluster( Stream<? extends T> docStream, LanguageComponents languageComponents) { List<T> documents = docStream.collect(Collectors.toList()); final PreprocessingContext context = preprocessing.preprocess(documents.stream(), queryHint.get(), languageComponents); List<Cluster<T>> clusters = new ArrayList<>(); if (context.hasLabels()) { final VectorSpaceModelContext vsmContext = new VectorSpaceModelContext(context); final ReducedVectorSpaceModelContext reducedVsmContext = new ReducedVectorSpaceModelContext(vsmContext); LingoProcessingContext lingoContext = new LingoProcessingContext(reducedVsmContext); TermDocumentMatrixBuilder matrixBuilder = this.matrixBuilder; matrixBuilder.buildTermDocumentMatrix(vsmContext); matrixBuilder.buildTermPhraseMatrix(vsmContext); matrixReducer.reduce( reducedVsmContext, computeClusterCount(desiredClusterCount.get(), documents.size())); clusterBuilder.buildLabels(lingoContext, matrixBuilder.termWeighting); clusterBuilder.assignDocuments(lingoContext); clusterBuilder.merge(lingoContext); final LabelFormatter labelFormatter = lingoContext.preprocessingContext.languageComponents.get(LabelFormatter.class); final int[] clusterLabelIndex = lingoContext.clusterLabelFeatureIndex; final BitSet[] clusterDocuments = lingoContext.clusterDocuments; final double[] clusterLabelScore = lingoContext.clusterLabelScore; for (int i = 0; i < clusterLabelIndex.length; i++) { final Cluster<T> cluster = new Cluster<>(); final int labelFeature = clusterLabelIndex[i]; if (labelFeature < 0) { continue; } cluster.addLabel(context.format(labelFormatter, labelFeature)); cluster.setScore(clusterLabelScore[i]); final BitSet bs = clusterDocuments[i]; for (int bit = bs.nextSetBit(0); bit >= 0; bit = bs.nextSetBit(bit + 1)) { cluster.addDocument(documents.get(bit)); } clusters.add(cluster); } } clusters = SharedInfrastructure.reorderByWeightedScoreAndSize(clusters, this.scoreWeight.get()); return clusters; }  @Override Set<Class<?>> requiredLanguageComponents(); @Override List<Cluster<T>> cluster(
      Stream<? extends T> docStream, LanguageComponents languageComponents); static final String NAME; public AttrDouble scoreWeight; public AttrInteger desiredClusterCount; public CompletePreprocessingPipeline preprocessing; public TermDocumentMatrixBuilder matrixBuilder; public TermDocumentMatrixReducer matrixReducer; public ClusterBuilder clusterBuilder; final AttrString queryHint; }
@Test public void testClusteringWithDfThreshold() { LingoClusteringAlgorithm algorithm = algorithm(); algorithm.preprocessing.wordDfThreshold.set(100); List<Cluster<Document>> clusters = algorithm.cluster( SampleDocumentData.DOCUMENTS_DATA_MINING.stream(), CachedLangComponents.loadCached("English")); Assertions.assertThat(clusters).isEmpty(); }
@Test public void testNoLabelCandidates() { LingoClusteringAlgorithm algorithm = algorithm(); algorithm.queryHint.set("test"); Stream<Document> documents = Stream.of(new TestDocument("test"), new TestDocument("test"), new TestDocument("test")); List<Cluster<Document>> clusters = algorithm.cluster(documents, CachedLangComponents.loadCached("English")); Assertions.assertThat(clusters).isEmpty(); }
@Test public void testStemmingUsedWithDefaultAttributes() { LingoClusteringAlgorithm algorithm = algorithm(); algorithm.queryHint.set("test"); Stream<Document> documents = Stream.of( "program", "programs", "programming", "program", "programs", "programming", "other") .map(title -> new TestDocument(title)); List<Cluster<Document>> clusters = algorithm.cluster(documents, CachedLangComponents.loadCached("English")); Assertions.assertThat(clusters).isNotEmpty(); Assertions.assertThat( clusters.stream() .flatMap(c -> c.getLabels().stream()) .map(label -> label.toLowerCase(Locale.ROOT))) .containsOnly("program"); }