@Test public void testMinSimilarity() { int len = 200, size = 10; Row[] left = new Row[len]; Random random = new Random(); for(int i = 0; i < left.length; i++){ left[i] = Row.of(i, new SparseVector(size, new int[]{random.nextInt(size), random.nextInt(size)}, new double[]{random.nextDouble(), random.nextDouble()})); } MemSourceBatchOp leftData = new MemSourceBatchOp(Arrays.asList(left), new String[] {"leftId", "leftVec"}); BatchOperator op = new ApproxVectorSimilarityTopNLSHBatchOp() .setLeftCol("leftVec") .setRightCol("leftVec") .setLeftIdCol("leftId") .setRightIdCol("leftId") .setDistanceType("JACCARD") .setOutputCol("distance"); List<Row> res = op.linkFrom(leftData, leftData).collect(); Assert.assertTrue(res.size() >= 200 && res.size() <= 1000); }
@Override public ApproxVectorSimilarityTopNLSHBatchOp linkFrom(BatchOperator<?>... inputs) { checkOpSize(2, inputs); checkOpSize(2, inputs); String leftIdCol = this.getLeftIdCol(); String rightIdCol = this.getRightIdCol(); DataSet<BaseLSH> lsh = LocalitySensitiveHashApproxFunctions.buildLSH(inputs[0], inputs[1], this.getParams()); DataSet<Row> res = LocalitySensitiveHashApproxFunctions.approxNearestNeighbors( inputs[1].select(new String[] {rightIdCol, getRightCol()}).getDataSet(), inputs[0].select(new String[] {leftIdCol, getLeftCol()}).getDataSet(), this.getTopN(), lsh); this.setOutput(res, getTopNOutputSchema(inputs, leftIdCol, rightIdCol)); return this; }
ApproxVectorSimilarityTopNLSHBatchOp extends BatchOperator<ApproxVectorSimilarityTopNLSHBatchOp> implements ApproxVectorTopNLSHParams<ApproxVectorSimilarityTopNLSHBatchOp> { @Override public ApproxVectorSimilarityTopNLSHBatchOp linkFrom(BatchOperator<?>... inputs) { checkOpSize(2, inputs); checkOpSize(2, inputs); String leftIdCol = this.getLeftIdCol(); String rightIdCol = this.getRightIdCol(); DataSet<BaseLSH> lsh = LocalitySensitiveHashApproxFunctions.buildLSH(inputs[0], inputs[1], this.getParams()); DataSet<Row> res = LocalitySensitiveHashApproxFunctions.approxNearestNeighbors( inputs[1].select(new String[] {rightIdCol, getRightCol()}).getDataSet(), inputs[0].select(new String[] {leftIdCol, getLeftCol()}).getDataSet(), this.getTopN(), lsh); this.setOutput(res, getTopNOutputSchema(inputs, leftIdCol, rightIdCol)); return this; } }
ApproxVectorSimilarityTopNLSHBatchOp extends BatchOperator<ApproxVectorSimilarityTopNLSHBatchOp> implements ApproxVectorTopNLSHParams<ApproxVectorSimilarityTopNLSHBatchOp> { @Override public ApproxVectorSimilarityTopNLSHBatchOp linkFrom(BatchOperator<?>... inputs) { checkOpSize(2, inputs); checkOpSize(2, inputs); String leftIdCol = this.getLeftIdCol(); String rightIdCol = this.getRightIdCol(); DataSet<BaseLSH> lsh = LocalitySensitiveHashApproxFunctions.buildLSH(inputs[0], inputs[1], this.getParams()); DataSet<Row> res = LocalitySensitiveHashApproxFunctions.approxNearestNeighbors( inputs[1].select(new String[] {rightIdCol, getRightCol()}).getDataSet(), inputs[0].select(new String[] {leftIdCol, getLeftCol()}).getDataSet(), this.getTopN(), lsh); this.setOutput(res, getTopNOutputSchema(inputs, leftIdCol, rightIdCol)); return this; } ApproxVectorSimilarityTopNLSHBatchOp();  ApproxVectorSimilarityTopNLSHBatchOp(Params params); }
ApproxVectorSimilarityTopNLSHBatchOp extends BatchOperator<ApproxVectorSimilarityTopNLSHBatchOp> implements ApproxVectorTopNLSHParams<ApproxVectorSimilarityTopNLSHBatchOp> { @Override public ApproxVectorSimilarityTopNLSHBatchOp linkFrom(BatchOperator<?>... inputs) { checkOpSize(2, inputs); checkOpSize(2, inputs); String leftIdCol = this.getLeftIdCol(); String rightIdCol = this.getRightIdCol(); DataSet<BaseLSH> lsh = LocalitySensitiveHashApproxFunctions.buildLSH(inputs[0], inputs[1], this.getParams()); DataSet<Row> res = LocalitySensitiveHashApproxFunctions.approxNearestNeighbors( inputs[1].select(new String[] {rightIdCol, getRightCol()}).getDataSet(), inputs[0].select(new String[] {leftIdCol, getLeftCol()}).getDataSet(), this.getTopN(), lsh); this.setOutput(res, getTopNOutputSchema(inputs, leftIdCol, rightIdCol)); return this; } ApproxVectorSimilarityTopNLSHBatchOp();  ApproxVectorSimilarityTopNLSHBatchOp(Params params); static TableSchema getTopNOutputSchema(BatchOperator[] inputs, String leftIdCol, String rightIdCol); @Override ApproxVectorSimilarityTopNLSHBatchOp linkFrom(BatchOperator<?>... inputs); }
ApproxVectorSimilarityTopNLSHBatchOp extends BatchOperator<ApproxVectorSimilarityTopNLSHBatchOp> implements ApproxVectorTopNLSHParams<ApproxVectorSimilarityTopNLSHBatchOp> { @Override public ApproxVectorSimilarityTopNLSHBatchOp linkFrom(BatchOperator<?>... inputs) { checkOpSize(2, inputs); checkOpSize(2, inputs); String leftIdCol = this.getLeftIdCol(); String rightIdCol = this.getRightIdCol(); DataSet<BaseLSH> lsh = LocalitySensitiveHashApproxFunctions.buildLSH(inputs[0], inputs[1], this.getParams()); DataSet<Row> res = LocalitySensitiveHashApproxFunctions.approxNearestNeighbors( inputs[1].select(new String[] {rightIdCol, getRightCol()}).getDataSet(), inputs[0].select(new String[] {leftIdCol, getLeftCol()}).getDataSet(), this.getTopN(), lsh); this.setOutput(res, getTopNOutputSchema(inputs, leftIdCol, rightIdCol)); return this; } ApproxVectorSimilarityTopNLSHBatchOp();  ApproxVectorSimilarityTopNLSHBatchOp(Params params); static TableSchema getTopNOutputSchema(BatchOperator[] inputs, String leftIdCol, String rightIdCol); @Override ApproxVectorSimilarityTopNLSHBatchOp linkFrom(BatchOperator<?>... inputs);  }