@Test public void updateSumMatrixTest() { int sampleWeight = 1; KMeansPredictModelData predictModelData = new KMeansModelDataConverter().load(modelRows); List<FastDistanceVectorData> samples = new ArrayList<>(); List<Vector> vectorList = new ArrayList<>(); for (int i = 0; i < length; i++) { vectorList.add(DenseVector.ones(predictModelData.params.vectorSize).scale(i)); } vectorList.forEach(vec -> samples .add((predictModelData.params.distanceType.getFastDistance()).prepareVectorData(Tuple2.of(vec, null)))); DenseMatrix distanceMatrix = new DenseMatrix(predictModelData.params.k, 1); double[] sumMatrix = new double[predictModelData.params.k * (predictModelData.params.vectorSize + 1)]; for (FastDistanceVectorData sample : samples) { KMeansUtil.updateSumMatrix( sample, sampleWeight, predictModelData.centroids, predictModelData.params.vectorSize, sumMatrix, predictModelData.params.k, predictModelData.params.distanceType.getFastDistance(), distanceMatrix); } Arrays.equals(sumMatrix, new double[] {35.0, 35.0, 35.0, 5.0, 10.0, 10.0, 10.0, 5.0}); }
public static int updateSumMatrix(FastDistanceVectorData sample, long sampleWeight, FastDistanceMatrixData centroids, int vectorSize, double[] sumMatrix, int k, FastDistance fastDistance, DenseMatrix distanceMatrix) { Preconditions.checkNotNull(sumMatrix); Preconditions.checkNotNull(distanceMatrix); Preconditions.checkArgument(distanceMatrix.numRows() == centroids.getVectors().numCols() && distanceMatrix.numCols() == 1, "Memory not preallocated!"); fastDistance.calc(sample, centroids, distanceMatrix); int clusterIndex = getClosestClusterIndex(sample, centroids, k, fastDistance, distanceMatrix).f0; int startIndex = clusterIndex * (vectorSize + 1); Vector vec = sample.getVector(); if (vec instanceof DenseVector) { BLAS.axpy(vectorSize, sampleWeight, ((DenseVector)vec).getData(), 0, sumMatrix, startIndex); } else { SparseVector sparseVector = (SparseVector)vec; sparseVector.forEach((index, value) -> sumMatrix[startIndex + index] += sampleWeight * value); } sumMatrix[startIndex + vectorSize] += sampleWeight; return clusterIndex; }
KMeansUtil implements Serializable { public static int updateSumMatrix(FastDistanceVectorData sample, long sampleWeight, FastDistanceMatrixData centroids, int vectorSize, double[] sumMatrix, int k, FastDistance fastDistance, DenseMatrix distanceMatrix) { Preconditions.checkNotNull(sumMatrix); Preconditions.checkNotNull(distanceMatrix); Preconditions.checkArgument(distanceMatrix.numRows() == centroids.getVectors().numCols() && distanceMatrix.numCols() == 1, "Memory not preallocated!"); fastDistance.calc(sample, centroids, distanceMatrix); int clusterIndex = getClosestClusterIndex(sample, centroids, k, fastDistance, distanceMatrix).f0; int startIndex = clusterIndex * (vectorSize + 1); Vector vec = sample.getVector(); if (vec instanceof DenseVector) { BLAS.axpy(vectorSize, sampleWeight, ((DenseVector)vec).getData(), 0, sumMatrix, startIndex); } else { SparseVector sparseVector = (SparseVector)vec; sparseVector.forEach((index, value) -> sumMatrix[startIndex + index] += sampleWeight * value); } sumMatrix[startIndex + vectorSize] += sampleWeight; return clusterIndex; } }
KMeansUtil implements Serializable { public static int updateSumMatrix(FastDistanceVectorData sample, long sampleWeight, FastDistanceMatrixData centroids, int vectorSize, double[] sumMatrix, int k, FastDistance fastDistance, DenseMatrix distanceMatrix) { Preconditions.checkNotNull(sumMatrix); Preconditions.checkNotNull(distanceMatrix); Preconditions.checkArgument(distanceMatrix.numRows() == centroids.getVectors().numCols() && distanceMatrix.numCols() == 1, "Memory not preallocated!"); fastDistance.calc(sample, centroids, distanceMatrix); int clusterIndex = getClosestClusterIndex(sample, centroids, k, fastDistance, distanceMatrix).f0; int startIndex = clusterIndex * (vectorSize + 1); Vector vec = sample.getVector(); if (vec instanceof DenseVector) { BLAS.axpy(vectorSize, sampleWeight, ((DenseVector)vec).getData(), 0, sumMatrix, startIndex); } else { SparseVector sparseVector = (SparseVector)vec; sparseVector.forEach((index, value) -> sumMatrix[startIndex + index] += sampleWeight * value); } sumMatrix[startIndex + vectorSize] += sampleWeight; return clusterIndex; }  }
KMeansUtil implements Serializable { public static int updateSumMatrix(FastDistanceVectorData sample, long sampleWeight, FastDistanceMatrixData centroids, int vectorSize, double[] sumMatrix, int k, FastDistance fastDistance, DenseMatrix distanceMatrix) { Preconditions.checkNotNull(sumMatrix); Preconditions.checkNotNull(distanceMatrix); Preconditions.checkArgument(distanceMatrix.numRows() == centroids.getVectors().numCols() && distanceMatrix.numCols() == 1, "Memory not preallocated!"); fastDistance.calc(sample, centroids, distanceMatrix); int clusterIndex = getClosestClusterIndex(sample, centroids, k, fastDistance, distanceMatrix).f0; int startIndex = clusterIndex * (vectorSize + 1); Vector vec = sample.getVector(); if (vec instanceof DenseVector) { BLAS.axpy(vectorSize, sampleWeight, ((DenseVector)vec).getData(), 0, sumMatrix, startIndex); } else { SparseVector sparseVector = (SparseVector)vec; sparseVector.forEach((index, value) -> sumMatrix[startIndex + index] += sampleWeight * value); } sumMatrix[startIndex + vectorSize] += sampleWeight; return clusterIndex; }  static FastDistanceMatrixData buildCentroidsMatrix(List<FastDistanceVectorData> vectors,
                                                              FastDistance distance,
                                                              int vectorSize); static int updateSumMatrix(FastDistanceVectorData sample,
                                      long sampleWeight,
                                      FastDistanceMatrixData centroids,
                                      int vectorSize,
                                      double[] sumMatrix,
                                      int k,
                                      FastDistance fastDistance,
                                      DenseMatrix distanceMatrix); static Tuple2<Integer, Double> getClosestClusterIndex(FastDistanceVectorData sample,
                                                                 FastDistanceMatrixData centroids,
                                                                 int k,
                                                                 FastDistance distance,
                                                                 DenseMatrix distanceMatrix); static double[] getClusterDistances(FastDistanceVectorData sample,
                                               FastDistanceMatrixData centroids,
                                               FastDistance distance,
                                               DenseMatrix distanceMatrix); static Tuple2<Integer, Double> getClosestClusterIndex(KMeansTrainModelData trainModelData,
                                                                 Vector sample,
                                                                 ContinuousDistance distance); static double[] getClusterDistances(KMeansTrainModelData trainModelData,
                                               Vector sample,
                                               ContinuousDistance distance); static int getMinPointIndex(double[] data, int endIndex); static int[] getKmeansPredictColIdxs(KMeansTrainModelData.ParamSummary params, String[] dataCols); static Vector getKMeansPredictVector(int[] colIdxs, Row row); static KMeansTrainModelData transformPredictDataToTrainData(KMeansPredictModelData predictModelData); static KMeansPredictModelData transformTrainDataToPredictData(KMeansTrainModelData trainModelData); static double[] getProbArrayFromDistanceArray(double[] distances); static KMeansTrainModelData loadModelForTrain(Params params, Iterable<String> data); }
KMeansUtil implements Serializable { public static int updateSumMatrix(FastDistanceVectorData sample, long sampleWeight, FastDistanceMatrixData centroids, int vectorSize, double[] sumMatrix, int k, FastDistance fastDistance, DenseMatrix distanceMatrix) { Preconditions.checkNotNull(sumMatrix); Preconditions.checkNotNull(distanceMatrix); Preconditions.checkArgument(distanceMatrix.numRows() == centroids.getVectors().numCols() && distanceMatrix.numCols() == 1, "Memory not preallocated!"); fastDistance.calc(sample, centroids, distanceMatrix); int clusterIndex = getClosestClusterIndex(sample, centroids, k, fastDistance, distanceMatrix).f0; int startIndex = clusterIndex * (vectorSize + 1); Vector vec = sample.getVector(); if (vec instanceof DenseVector) { BLAS.axpy(vectorSize, sampleWeight, ((DenseVector)vec).getData(), 0, sumMatrix, startIndex); } else { SparseVector sparseVector = (SparseVector)vec; sparseVector.forEach((index, value) -> sumMatrix[startIndex + index] += sampleWeight * value); } sumMatrix[startIndex + vectorSize] += sampleWeight; return clusterIndex; }  static FastDistanceMatrixData buildCentroidsMatrix(List<FastDistanceVectorData> vectors,
                                                              FastDistance distance,
                                                              int vectorSize); static int updateSumMatrix(FastDistanceVectorData sample,
                                      long sampleWeight,
                                      FastDistanceMatrixData centroids,
                                      int vectorSize,
                                      double[] sumMatrix,
                                      int k,
                                      FastDistance fastDistance,
                                      DenseMatrix distanceMatrix); static Tuple2<Integer, Double> getClosestClusterIndex(FastDistanceVectorData sample,
                                                                 FastDistanceMatrixData centroids,
                                                                 int k,
                                                                 FastDistance distance,
                                                                 DenseMatrix distanceMatrix); static double[] getClusterDistances(FastDistanceVectorData sample,
                                               FastDistanceMatrixData centroids,
                                               FastDistance distance,
                                               DenseMatrix distanceMatrix); static Tuple2<Integer, Double> getClosestClusterIndex(KMeansTrainModelData trainModelData,
                                                                 Vector sample,
                                                                 ContinuousDistance distance); static double[] getClusterDistances(KMeansTrainModelData trainModelData,
                                               Vector sample,
                                               ContinuousDistance distance); static int getMinPointIndex(double[] data, int endIndex); static int[] getKmeansPredictColIdxs(KMeansTrainModelData.ParamSummary params, String[] dataCols); static Vector getKMeansPredictVector(int[] colIdxs, Row row); static KMeansTrainModelData transformPredictDataToTrainData(KMeansPredictModelData predictModelData); static KMeansPredictModelData transformTrainDataToPredictData(KMeansTrainModelData trainModelData); static double[] getProbArrayFromDistanceArray(double[] distances); static KMeansTrainModelData loadModelForTrain(Params params, Iterable<String> data);  }