FiniteGMM extends VarGMM { public double[] getPriorAdjustedClusterProportions() { double[] mixingCoeffs = mixingComponents.getCoeffs(); double sum = - mixingComponents.getPrior(); for (double coeff : mixingCoeffs) { sum += coeff; } double[] proportions = new double[K]; for (int i=0; i<K; i++) { proportions[i] = (mixingCoeffs[i] - mixingComponents.getPrior()) / sum; } return proportions; } FiniteGMM(MacroBaseConf conf); @Override void trainTest(List<Datum> trainData, List<Datum> testData); @Override double[] getClusterProportions(); double[] getPriorAdjustedClusterProportions(); @Override double[] getClusterProbabilities(Datum d);  }
@Test public void bivariateOkSeparatedNormalTest() throws Exception { MacroBaseConf conf = new MacroBaseConf() .set(MacroBaseConf.RANDOM_SEED, 4) .set(MacroBaseConf.TRANSFORM_TYPE, "MEAN_FIELD_GMM") .set(GMMConf.NUM_MIXTURES, 3) .set(MacroBaseConf.DATA_LOADER_TYPE, "CSV_LOADER") .set(MacroBaseConf.CSV_COMPRESSION, CSVIngester.Compression.GZIP) .set(MacroBaseConf.CSV_INPUT_FILE, "src/test/resources/data/3gaussians-7000points.csv.gz") .set(MacroBaseConf.METRICS, "XX, YY") .set(MacroBaseConf.ATTRIBUTES, ""); List<Datum> data = Drainer.drainIngest(conf); assertEquals(7000, data.size()); double[][] clusterMeans = { {1.5, 2}, {2, 0}, {4.5, 1}, }; List<RealVector> vectorClusterMeans = new ArrayList<>(3); for (int k = 0; k < 3; k++) { vectorClusterMeans.add(new ArrayRealVector(clusterMeans[k])); } double[][][] clusterCovariances = { {{0.5, 0.4}, {0.4, 0.5}}, {{0.3, 0}, {0, 0.6}}, {{0.9, 0.2}, {0.2, 0.3}}, }; FiniteGMM finiteGMM = new FiniteGMM(conf); finiteGMM.train(data); List<RealVector> calculatedMeans = finiteGMM.getClusterCenters(); List<RealMatrix> calculatedCovariances = finiteGMM.getClusterCovariances(); for (int i = 0; i < 3; i++) { boolean identified = false; for (int j = 0; j < 3; j++) { if (calculatedMeans.get(i).getDistance(vectorClusterMeans.get(j)) < 0.1) { for (int p = 0; p < 2; p++) { for (int q = 0; q < 2; q++) { assertEquals(clusterCovariances[j][p][q], calculatedCovariances.get(i).getEntry(p, q), 0.1); } } identified = true; break; } } assertEquals("a cluster was not identified", true, identified); } double[][] boundaries = { {0, 6.01}, {-2, 4.01}, }; List<Datum> scoredData = DiagnosticsUtils.createGridFixedIncrement(boundaries, 0.05); JsonUtils.dumpAsJson(finiteGMM.getClusterCovariances(), "FiniteGMMTest-bivariateOkSeparatedNormalTest-covariances.json"); JsonUtils.dumpAsJson(finiteGMM.getClusterCenters(), "FiniteGMMTest-bivariateOkSeparatedNormalTest-means.json"); JsonUtils.dumpAsJson(finiteGMM.getPriorAdjustedClusterProportions(), "FiniteGMMTest-bivariateOkSeparatedNormalTest-weights.json"); conf.set(GMMConf.SCORE_DUMP_FILE_CONFIG_PARAM, "3gaussians-7k-grid.json"); ScoreDumper dumper = new ScoreDumper(conf); dumper.dumpScores(finiteGMM, scoredData); conf.set(GMMConf.SCORE_DUMP_FILE_CONFIG_PARAM, "3gaussians-7k-data.json"); dumper = new ScoreDumper(conf); dumper.dumpScores(finiteGMM, data); }
@Test public void unusedOrOverlappingClusterTest() throws Exception { MacroBaseConf conf = new MacroBaseConf() .set(MacroBaseConf.RANDOM_SEED, 4) .set(MacroBaseConf.TRANSFORM_TYPE, "MEAN_FIELD_GMM") .set(GMMConf.NUM_MIXTURES, 4) .set(MacroBaseConf.DATA_LOADER_TYPE, "CSV_LOADER") .set(MacroBaseConf.CSV_COMPRESSION, CSVIngester.Compression.GZIP) .set(MacroBaseConf.CSV_INPUT_FILE, "src/test/resources/data/2gaussians-500points.csv.gz") .set(MacroBaseConf.METRICS, "XX, YY") .set(MacroBaseConf.ATTRIBUTES, ""); List<Datum> data = Drainer.drainIngest(conf); assertEquals(500, data.size()); FiniteGMM finiteGMM = new FiniteGMM(conf); List<RealVector> calculatedMeans = null; double[] calculatedWeights = null; int numClustersIdentified = 0; while (numClustersIdentified < 2) { finiteGMM.train(data); calculatedMeans = finiteGMM.getClusterCenters(); calculatedWeights = finiteGMM.getPriorAdjustedClusterProportions(); numClustersIdentified = 0; for (double weight : calculatedWeights) { if (weight > 0.1) { numClustersIdentified += 1; } } } double[][] clusterMeans = { {1, 1}, {10, 3}, }; List<RealVector> vectorClusterMeans = new ArrayList<>(2); for (int k = 0; k < 2; k++) { vectorClusterMeans.add(new ArrayRealVector(clusterMeans[k])); } double[] clusterWeights = { 200, 300, }; double[] identifiedWeights = {0, 0}; for (int i = 0; i < 4; i++) { for (int j = 0; j < 2; j++) { if (calculatedMeans.get(i).getDistance(vectorClusterMeans.get(j)) < 1) { identifiedWeights[j] += calculatedWeights[i]; log.debug("adding {} to cluster {}", calculatedWeights[i], vectorClusterMeans.get(j)); break; } } } assertEquals(identifiedWeights[0] * 500, clusterWeights[0], 1); assertEquals(identifiedWeights[1] * 500, clusterWeights[1], 1); }