@Test public void testMinSimilarity() { int len = 200, size = 10; Random random = new Random(); Row[] left = new Row[len]; Row[] right = new Row[len]; for(int i = 0; i < left.length; i++){ left[i] = Row.of(i, new SparseVector(size, new int[]{random.nextInt(size), random.nextInt(size)}, new double[]{random.nextDouble(), random.nextDouble()})); right[i] = Row.of(i, new SparseVector(size, new int[]{random.nextInt(size), random.nextInt(size)}, new double[]{random.nextDouble(), random.nextDouble()})); } MemSourceBatchOp leftData = new MemSourceBatchOp(Arrays.asList(left), new String[] {"leftId", "leftVec"}); MemSourceBatchOp rightData = new MemSourceBatchOp(Arrays.asList(right), new String[] {"rightId", "rightVec"}); BatchOperator op = new ApproxVectorSimilarityJoinLSHBatchOp() .setLeftCol("leftVec") .setRightCol("rightVec") .setLeftIdCol("leftId") .setRightIdCol("rightId") .setDistanceType("JACCARD") .setOutputCol("distance") .setDistanceThreshold(0.6); List<Row> res = op.linkFrom(leftData, rightData).collect(); for(Row row : res){ Assert.assertTrue((double)row.getField(2) < 0.6); } }
@Override public ApproxVectorSimilarityJoinLSHBatchOp linkFrom(BatchOperator<?>... inputs) { checkOpSize(2, inputs); String leftIdCol = this.getLeftIdCol(); String rightIdCol = this.getRightIdCol(); DataSet<BaseLSH> lsh = LocalitySensitiveHashApproxFunctions.buildLSH(inputs[0], inputs[1], this.getParams()); DataSet<Row> res = LocalitySensitiveHashApproxFunctions.approxSimilarityJoin( inputs[0].select(new String[] {leftIdCol, getLeftCol()}).getDataSet(), inputs[1].select(new String[] {rightIdCol, getRightCol()}).getDataSet(), this.getDistanceThreshold(), lsh); this.setOutput(res, getJoinOutputSchema(inputs, leftIdCol, rightIdCol)); return this; }
ApproxVectorSimilarityJoinLSHBatchOp extends BatchOperator<ApproxVectorSimilarityJoinLSHBatchOp> implements ApproxVectorJoinLSHParams<ApproxVectorSimilarityJoinLSHBatchOp> { @Override public ApproxVectorSimilarityJoinLSHBatchOp linkFrom(BatchOperator<?>... inputs) { checkOpSize(2, inputs); String leftIdCol = this.getLeftIdCol(); String rightIdCol = this.getRightIdCol(); DataSet<BaseLSH> lsh = LocalitySensitiveHashApproxFunctions.buildLSH(inputs[0], inputs[1], this.getParams()); DataSet<Row> res = LocalitySensitiveHashApproxFunctions.approxSimilarityJoin( inputs[0].select(new String[] {leftIdCol, getLeftCol()}).getDataSet(), inputs[1].select(new String[] {rightIdCol, getRightCol()}).getDataSet(), this.getDistanceThreshold(), lsh); this.setOutput(res, getJoinOutputSchema(inputs, leftIdCol, rightIdCol)); return this; } }
ApproxVectorSimilarityJoinLSHBatchOp extends BatchOperator<ApproxVectorSimilarityJoinLSHBatchOp> implements ApproxVectorJoinLSHParams<ApproxVectorSimilarityJoinLSHBatchOp> { @Override public ApproxVectorSimilarityJoinLSHBatchOp linkFrom(BatchOperator<?>... inputs) { checkOpSize(2, inputs); String leftIdCol = this.getLeftIdCol(); String rightIdCol = this.getRightIdCol(); DataSet<BaseLSH> lsh = LocalitySensitiveHashApproxFunctions.buildLSH(inputs[0], inputs[1], this.getParams()); DataSet<Row> res = LocalitySensitiveHashApproxFunctions.approxSimilarityJoin( inputs[0].select(new String[] {leftIdCol, getLeftCol()}).getDataSet(), inputs[1].select(new String[] {rightIdCol, getRightCol()}).getDataSet(), this.getDistanceThreshold(), lsh); this.setOutput(res, getJoinOutputSchema(inputs, leftIdCol, rightIdCol)); return this; } ApproxVectorSimilarityJoinLSHBatchOp();  ApproxVectorSimilarityJoinLSHBatchOp(Params params); }
ApproxVectorSimilarityJoinLSHBatchOp extends BatchOperator<ApproxVectorSimilarityJoinLSHBatchOp> implements ApproxVectorJoinLSHParams<ApproxVectorSimilarityJoinLSHBatchOp> { @Override public ApproxVectorSimilarityJoinLSHBatchOp linkFrom(BatchOperator<?>... inputs) { checkOpSize(2, inputs); String leftIdCol = this.getLeftIdCol(); String rightIdCol = this.getRightIdCol(); DataSet<BaseLSH> lsh = LocalitySensitiveHashApproxFunctions.buildLSH(inputs[0], inputs[1], this.getParams()); DataSet<Row> res = LocalitySensitiveHashApproxFunctions.approxSimilarityJoin( inputs[0].select(new String[] {leftIdCol, getLeftCol()}).getDataSet(), inputs[1].select(new String[] {rightIdCol, getRightCol()}).getDataSet(), this.getDistanceThreshold(), lsh); this.setOutput(res, getJoinOutputSchema(inputs, leftIdCol, rightIdCol)); return this; } ApproxVectorSimilarityJoinLSHBatchOp();  ApproxVectorSimilarityJoinLSHBatchOp(Params params); @Override ApproxVectorSimilarityJoinLSHBatchOp linkFrom(BatchOperator<?>... inputs); }
ApproxVectorSimilarityJoinLSHBatchOp extends BatchOperator<ApproxVectorSimilarityJoinLSHBatchOp> implements ApproxVectorJoinLSHParams<ApproxVectorSimilarityJoinLSHBatchOp> { @Override public ApproxVectorSimilarityJoinLSHBatchOp linkFrom(BatchOperator<?>... inputs) { checkOpSize(2, inputs); String leftIdCol = this.getLeftIdCol(); String rightIdCol = this.getRightIdCol(); DataSet<BaseLSH> lsh = LocalitySensitiveHashApproxFunctions.buildLSH(inputs[0], inputs[1], this.getParams()); DataSet<Row> res = LocalitySensitiveHashApproxFunctions.approxSimilarityJoin( inputs[0].select(new String[] {leftIdCol, getLeftCol()}).getDataSet(), inputs[1].select(new String[] {rightIdCol, getRightCol()}).getDataSet(), this.getDistanceThreshold(), lsh); this.setOutput(res, getJoinOutputSchema(inputs, leftIdCol, rightIdCol)); return this; } ApproxVectorSimilarityJoinLSHBatchOp();  ApproxVectorSimilarityJoinLSHBatchOp(Params params); @Override ApproxVectorSimilarityJoinLSHBatchOp linkFrom(BatchOperator<?>... inputs);  }