@Test public void calSilhouetteCoefficientTest(){ Row[] rows0 = new Row[] { Row.of(0, "0,0,0"), Row.of(0, "0.1,0.1,0.1"), Row.of(0, "0.2,0.2,0.2") }; Row[] rows1 = new Row[] { Row.of(1, "9 9 9"), Row.of(1, "9.1 9.1 9.1"), Row.of(1, "9.2 9.2 9.2") }; ClusterMetricsSummary clusterMetricsSummary1 = ClusterEvaluationUtil.getClusterStatistics(Arrays.asList(rows0), new EuclideanDistance()); ClusterMetricsSummary clusterMetricsSummary2 = ClusterEvaluationUtil.getClusterStatistics(Arrays.asList(rows1), new EuclideanDistance()); ClusterMetricsSummary clusterMetricsSummary = clusterMetricsSummary1.merge(clusterMetricsSummary2); for(Row row : rows1){ Assert.assertEquals(0.99, ClusterEvaluationUtil.calSilhouetteCoefficient(row, clusterMetricsSummary).f0, 0.01); } }
public static Tuple1<Double> calSilhouetteCoefficient(Row row, ClusterMetricsSummary clusterMetricsSummary) { if (!EvaluationUtil.checkRowFieldNotNull(row)) { return Tuple1.of(0.); } String clusterId = row.getField(0).toString(); Vector vec = VectorUtil.getVector(row.getField(1)); double currentClusterDissimilarity = 0.0; double neighboringClusterDissimilarity = Double.MAX_VALUE; if (clusterMetricsSummary.distance instanceof EuclideanDistance) { double normSquare = vec.normL2Square(); for (int i = 0; i < clusterMetricsSummary.k; i++) { double dissimilarity = clusterMetricsSummary.clusterCnt.get(i) * normSquare - 2 * clusterMetricsSummary.clusterCnt.get(i) * MatVecOp.dot(vec, clusterMetricsSummary.meanVector.get(i)) + clusterMetricsSummary.vectorNormL2Sum.get(i); if (clusterId.equals(clusterMetricsSummary.clusterId.get(i))) { if (clusterMetricsSummary.clusterCnt.get(i) > 1) { currentClusterDissimilarity = dissimilarity / (clusterMetricsSummary.clusterCnt.get(i) - 1); } } else { neighboringClusterDissimilarity = Math.min(neighboringClusterDissimilarity, dissimilarity / clusterMetricsSummary.clusterCnt.get(i)); } } } else { for (int i = 0; i < clusterMetricsSummary.k; i++) { double dissimilarity = 1.0 - MatVecOp.dot(vec, clusterMetricsSummary.meanVector.get(i)); if (clusterId.equals(clusterMetricsSummary.clusterId.get(i))) { if (clusterMetricsSummary.clusterCnt.get(i) > 1) { currentClusterDissimilarity = dissimilarity * clusterMetricsSummary.clusterCnt.get(i) / (clusterMetricsSummary.clusterCnt.get(i) - 1); } } else { neighboringClusterDissimilarity = Math.min(neighboringClusterDissimilarity, dissimilarity); } } } return Tuple1.of(currentClusterDissimilarity < neighboringClusterDissimilarity ? 1 - (currentClusterDissimilarity / neighboringClusterDissimilarity) : (neighboringClusterDissimilarity / currentClusterDissimilarity) - 1); }
ClusterEvaluationUtil implements AllWindowFunction<Row, Row, TimeWindow> { public static Tuple1<Double> calSilhouetteCoefficient(Row row, ClusterMetricsSummary clusterMetricsSummary) { if (!EvaluationUtil.checkRowFieldNotNull(row)) { return Tuple1.of(0.); } String clusterId = row.getField(0).toString(); Vector vec = VectorUtil.getVector(row.getField(1)); double currentClusterDissimilarity = 0.0; double neighboringClusterDissimilarity = Double.MAX_VALUE; if (clusterMetricsSummary.distance instanceof EuclideanDistance) { double normSquare = vec.normL2Square(); for (int i = 0; i < clusterMetricsSummary.k; i++) { double dissimilarity = clusterMetricsSummary.clusterCnt.get(i) * normSquare - 2 * clusterMetricsSummary.clusterCnt.get(i) * MatVecOp.dot(vec, clusterMetricsSummary.meanVector.get(i)) + clusterMetricsSummary.vectorNormL2Sum.get(i); if (clusterId.equals(clusterMetricsSummary.clusterId.get(i))) { if (clusterMetricsSummary.clusterCnt.get(i) > 1) { currentClusterDissimilarity = dissimilarity / (clusterMetricsSummary.clusterCnt.get(i) - 1); } } else { neighboringClusterDissimilarity = Math.min(neighboringClusterDissimilarity, dissimilarity / clusterMetricsSummary.clusterCnt.get(i)); } } } else { for (int i = 0; i < clusterMetricsSummary.k; i++) { double dissimilarity = 1.0 - MatVecOp.dot(vec, clusterMetricsSummary.meanVector.get(i)); if (clusterId.equals(clusterMetricsSummary.clusterId.get(i))) { if (clusterMetricsSummary.clusterCnt.get(i) > 1) { currentClusterDissimilarity = dissimilarity * clusterMetricsSummary.clusterCnt.get(i) / (clusterMetricsSummary.clusterCnt.get(i) - 1); } } else { neighboringClusterDissimilarity = Math.min(neighboringClusterDissimilarity, dissimilarity); } } } return Tuple1.of(currentClusterDissimilarity < neighboringClusterDissimilarity ? 1 - (currentClusterDissimilarity / neighboringClusterDissimilarity) : (neighboringClusterDissimilarity / currentClusterDissimilarity) - 1); } }
ClusterEvaluationUtil implements AllWindowFunction<Row, Row, TimeWindow> { public static Tuple1<Double> calSilhouetteCoefficient(Row row, ClusterMetricsSummary clusterMetricsSummary) { if (!EvaluationUtil.checkRowFieldNotNull(row)) { return Tuple1.of(0.); } String clusterId = row.getField(0).toString(); Vector vec = VectorUtil.getVector(row.getField(1)); double currentClusterDissimilarity = 0.0; double neighboringClusterDissimilarity = Double.MAX_VALUE; if (clusterMetricsSummary.distance instanceof EuclideanDistance) { double normSquare = vec.normL2Square(); for (int i = 0; i < clusterMetricsSummary.k; i++) { double dissimilarity = clusterMetricsSummary.clusterCnt.get(i) * normSquare - 2 * clusterMetricsSummary.clusterCnt.get(i) * MatVecOp.dot(vec, clusterMetricsSummary.meanVector.get(i)) + clusterMetricsSummary.vectorNormL2Sum.get(i); if (clusterId.equals(clusterMetricsSummary.clusterId.get(i))) { if (clusterMetricsSummary.clusterCnt.get(i) > 1) { currentClusterDissimilarity = dissimilarity / (clusterMetricsSummary.clusterCnt.get(i) - 1); } } else { neighboringClusterDissimilarity = Math.min(neighboringClusterDissimilarity, dissimilarity / clusterMetricsSummary.clusterCnt.get(i)); } } } else { for (int i = 0; i < clusterMetricsSummary.k; i++) { double dissimilarity = 1.0 - MatVecOp.dot(vec, clusterMetricsSummary.meanVector.get(i)); if (clusterId.equals(clusterMetricsSummary.clusterId.get(i))) { if (clusterMetricsSummary.clusterCnt.get(i) > 1) { currentClusterDissimilarity = dissimilarity * clusterMetricsSummary.clusterCnt.get(i) / (clusterMetricsSummary.clusterCnt.get(i) - 1); } } else { neighboringClusterDissimilarity = Math.min(neighboringClusterDissimilarity, dissimilarity); } } } return Tuple1.of(currentClusterDissimilarity < neighboringClusterDissimilarity ? 1 - (currentClusterDissimilarity / neighboringClusterDissimilarity) : (neighboringClusterDissimilarity / currentClusterDissimilarity) - 1); } ClusterEvaluationUtil(ContinuousDistance distance); }
ClusterEvaluationUtil implements AllWindowFunction<Row, Row, TimeWindow> { public static Tuple1<Double> calSilhouetteCoefficient(Row row, ClusterMetricsSummary clusterMetricsSummary) { if (!EvaluationUtil.checkRowFieldNotNull(row)) { return Tuple1.of(0.); } String clusterId = row.getField(0).toString(); Vector vec = VectorUtil.getVector(row.getField(1)); double currentClusterDissimilarity = 0.0; double neighboringClusterDissimilarity = Double.MAX_VALUE; if (clusterMetricsSummary.distance instanceof EuclideanDistance) { double normSquare = vec.normL2Square(); for (int i = 0; i < clusterMetricsSummary.k; i++) { double dissimilarity = clusterMetricsSummary.clusterCnt.get(i) * normSquare - 2 * clusterMetricsSummary.clusterCnt.get(i) * MatVecOp.dot(vec, clusterMetricsSummary.meanVector.get(i)) + clusterMetricsSummary.vectorNormL2Sum.get(i); if (clusterId.equals(clusterMetricsSummary.clusterId.get(i))) { if (clusterMetricsSummary.clusterCnt.get(i) > 1) { currentClusterDissimilarity = dissimilarity / (clusterMetricsSummary.clusterCnt.get(i) - 1); } } else { neighboringClusterDissimilarity = Math.min(neighboringClusterDissimilarity, dissimilarity / clusterMetricsSummary.clusterCnt.get(i)); } } } else { for (int i = 0; i < clusterMetricsSummary.k; i++) { double dissimilarity = 1.0 - MatVecOp.dot(vec, clusterMetricsSummary.meanVector.get(i)); if (clusterId.equals(clusterMetricsSummary.clusterId.get(i))) { if (clusterMetricsSummary.clusterCnt.get(i) > 1) { currentClusterDissimilarity = dissimilarity * clusterMetricsSummary.clusterCnt.get(i) / (clusterMetricsSummary.clusterCnt.get(i) - 1); } } else { neighboringClusterDissimilarity = Math.min(neighboringClusterDissimilarity, dissimilarity); } } } return Tuple1.of(currentClusterDissimilarity < neighboringClusterDissimilarity ? 1 - (currentClusterDissimilarity / neighboringClusterDissimilarity) : (neighboringClusterDissimilarity / currentClusterDissimilarity) - 1); } ClusterEvaluationUtil(ContinuousDistance distance); static Params extractParamsFromConfusionMatrix(LongMatrix longMatrix); static Tuple1<Double> calSilhouetteCoefficient(Row row, ClusterMetricsSummary clusterMetricsSummary); static Params getBasicClusterStatistics(Iterable<Row> rows); static ClusterMetricsSummary getClusterStatistics(Iterable<Row> rows, ContinuousDistance distance); @Override void apply(TimeWindow timeWindow, Iterable<Row> rows, Collector<Row> collector); }
ClusterEvaluationUtil implements AllWindowFunction<Row, Row, TimeWindow> { public static Tuple1<Double> calSilhouetteCoefficient(Row row, ClusterMetricsSummary clusterMetricsSummary) { if (!EvaluationUtil.checkRowFieldNotNull(row)) { return Tuple1.of(0.); } String clusterId = row.getField(0).toString(); Vector vec = VectorUtil.getVector(row.getField(1)); double currentClusterDissimilarity = 0.0; double neighboringClusterDissimilarity = Double.MAX_VALUE; if (clusterMetricsSummary.distance instanceof EuclideanDistance) { double normSquare = vec.normL2Square(); for (int i = 0; i < clusterMetricsSummary.k; i++) { double dissimilarity = clusterMetricsSummary.clusterCnt.get(i) * normSquare - 2 * clusterMetricsSummary.clusterCnt.get(i) * MatVecOp.dot(vec, clusterMetricsSummary.meanVector.get(i)) + clusterMetricsSummary.vectorNormL2Sum.get(i); if (clusterId.equals(clusterMetricsSummary.clusterId.get(i))) { if (clusterMetricsSummary.clusterCnt.get(i) > 1) { currentClusterDissimilarity = dissimilarity / (clusterMetricsSummary.clusterCnt.get(i) - 1); } } else { neighboringClusterDissimilarity = Math.min(neighboringClusterDissimilarity, dissimilarity / clusterMetricsSummary.clusterCnt.get(i)); } } } else { for (int i = 0; i < clusterMetricsSummary.k; i++) { double dissimilarity = 1.0 - MatVecOp.dot(vec, clusterMetricsSummary.meanVector.get(i)); if (clusterId.equals(clusterMetricsSummary.clusterId.get(i))) { if (clusterMetricsSummary.clusterCnt.get(i) > 1) { currentClusterDissimilarity = dissimilarity * clusterMetricsSummary.clusterCnt.get(i) / (clusterMetricsSummary.clusterCnt.get(i) - 1); } } else { neighboringClusterDissimilarity = Math.min(neighboringClusterDissimilarity, dissimilarity); } } } return Tuple1.of(currentClusterDissimilarity < neighboringClusterDissimilarity ? 1 - (currentClusterDissimilarity / neighboringClusterDissimilarity) : (neighboringClusterDissimilarity / currentClusterDissimilarity) - 1); } ClusterEvaluationUtil(ContinuousDistance distance); static Params extractParamsFromConfusionMatrix(LongMatrix longMatrix); static Tuple1<Double> calSilhouetteCoefficient(Row row, ClusterMetricsSummary clusterMetricsSummary); static Params getBasicClusterStatistics(Iterable<Row> rows); static ClusterMetricsSummary getClusterStatistics(Iterable<Row> rows, ContinuousDistance distance); @Override void apply(TimeWindow timeWindow, Iterable<Row> rows, Collector<Row> collector); static int COUNT; static int MEAN; static int SUM_2; }