RangeComparator { public RangeStats compare(Iterator<PartitionKey> sourceKeys, Iterator<PartitionKey> targetKeys, Function<PartitionKey, PartitionComparator> partitionTaskProvider) { final RangeStats rangeStats = RangeStats.newStats(); if (context.startToken.equals(context.endToken)) return rangeStats; Phaser phaser = new Phaser(1); AtomicLong partitionCount = new AtomicLong(0); AtomicReference<BigInteger> highestTokenSeen = new AtomicReference<>(context.startToken); logger.info("Comparing range [{},{}]", context.startToken, context.endToken); try { PartitionKey sourceKey = nextKey(sourceKeys); PartitionKey targetKey = nextKey(targetKeys); if ((sourceKey == null) != (targetKey == null)) { if (sourceKey == null) { logger.info("First in range, source iter is empty {}", context); onlyInTarget(rangeStats, targetKey); targetKeys.forEachRemaining(key -> onlyInTarget(rangeStats, key)); } else { logger.info("First in range, target iter is empty {}", context); onlyInSource(rangeStats, sourceKey); sourceKeys.forEachRemaining(key -> onlyInSource(rangeStats, key)); } return rangeStats; } while (sourceKey != null && targetKey != null) { int ret = sourceKey.compareTo(targetKey); if (ret > 0) { onlyInTarget(rangeStats, targetKey); targetKey = nextKey(targetKeys); } else if (ret < 0) { onlyInSource(rangeStats, sourceKey); sourceKey = nextKey(sourceKeys); } else { Verify.verify(sourceKey.equals(targetKey), "Can only compare partitions with identical keys: (%s, %s)", sourceKey, targetKey); if (!context.isTokenAllowed(sourceKey.getTokenAsBigInteger())) { logger.debug("Skipping disallowed token {}", sourceKey.getTokenAsBigInteger()); rangeStats.skipPartition(); sourceKey = nextKey(sourceKeys); targetKey = nextKey(targetKeys); continue; } BigInteger token = sourceKey.getTokenAsBigInteger(); try { PartitionComparator comparisonTask = partitionTaskProvider.apply(sourceKey); comparisonExecutor.submit(comparisonTask, onSuccess(rangeStats, partitionCount, token, highestTokenSeen, mismatchReporter, journal), onError(rangeStats, token, errorReporter), phaser); } catch (Throwable t) { recordError(rangeStats, token, errorReporter, t); } finally { sourceKey = nextKey(sourceKeys); targetKey = nextKey(targetKeys); } } } if (sourceKey != null) onlyInSource(rangeStats, sourceKey); else if (targetKey != null) onlyInTarget(rangeStats, targetKey); drain(sourceKeys, targetKeys, rangeStats); } catch (Exception e) { logger.debug("Waiting for {} in flight tasks before propagating error", phaser.getUnarrivedParties()); phaser.arriveAndAwaitAdvance(); throw new RuntimeException(String.format("Error encountered during range comparison for [%s:%s]", context.startToken, context.endToken), e); } logger.debug("Waiting for {} in flight tasks before returning", phaser.getUnarrivedParties()); phaser.arriveAndAwaitAdvance(); if (!rangeStats.allMatches()) logger.info("Segment [{}:{}] stats - ({})", context.startToken, context.endToken, rangeStats); return rangeStats; } RangeComparator(DiffContext context,
                           BiConsumer<Throwable, BigInteger> errorReporter,
                           BiConsumer<MismatchType,BigInteger> mismatchReporter,
                           BiConsumer<RangeStats, BigInteger> journal,
                           ComparisonExecutor comparisonExecutor); RangeStats compare(Iterator<PartitionKey> sourceKeys,
                              Iterator<PartitionKey> targetKeys,
                              Function<PartitionKey, PartitionComparator> partitionTaskProvider);  }
@Test public void skipComparisonOfDisallowedTokens() { RangeComparator comparator = comparator(context(0L, 100L, 2, 3, 4)); RangeStats stats = comparator.compare(keys(0, 1, 2, 3, 4, 5, 6), keys(0, 1, 2, 3, 4, 5, 6), this::alwaysMatch); assertFalse(stats.isEmpty()); assertEquals(0, stats.getOnlyInSource()); assertEquals(0, stats.getOnlyInTarget()); assertEquals(4, stats.getMatchedPartitions()); assertEquals(3, stats.getSkippedPartitions()); assertNothingReported(errors, mismatches, journal); assertCompared(0, 1, 5 , 6); }
@Test public void checkpointEveryTenPartitions() { RangeComparator comparator = comparator(context(0L, 100L)); comparator.compare(keys(LongStream.range(0, 25).toArray()), keys(LongStream.range(0, 25).toArray()), this::alwaysMatch); assertReported(9, RangeStats.withValues(10, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L), journal); assertReported(19, RangeStats.withValues(20, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L), journal); assertEquals(2, journal.keySet().size()); }
@Test public void recordHighestSeenPartitionWhenTasksCompleteOutOfOrder() { RangeComparator comparator = comparator(context(0L, 100L)); long[] tokens = new long[] {2, 8, 1, 4, 100, 3, 5, 7, 6, 9}; comparator.compare(keys(tokens), keys(tokens), this::alwaysMatch); assertReported(100, RangeStats.withValues(10, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L), journal); assertEquals(1, journal.keySet().size()); assertNothingReported(errors, mismatches); assertCompared(tokens); }
@Test public void recordHighestSeenPartitionWhenTasksCompleteOutOfOrderWithErrors() { RuntimeException toThrow = new RuntimeException("Test"); RangeComparator comparator = comparator(context(0L, 100L)); long[] tokens = new long[] {2, 8, 1, 11, 4, 100, 3, 5, 7, 6, 9, 0}; comparator.compare(keys(tokens), keys(tokens), throwDuringExecution(toThrow, 3, 2)); assertReported(100, RangeStats.withValues(10, 0L, 2L, 0L, 0L, 0L, 0L, 0L, 0L), journal); assertEquals(1, journal.keySet().size()); assertReported(2, toThrow, errors); assertReported(3, toThrow, errors); assertEquals(2, errors.keySet().size()); assertNothingReported(mismatches); assertCompared(8, 1, 11, 4, 100, 5, 7, 6, 9, 0); }
@Test public void rowLevelMismatchIncrementsPartitionMismatches() { RangeComparator comparator = comparator(context(0L, 100L)); RangeStats stats = comparator.compare(keys(0, 1, 2), keys(0, 1 ,2), this::rowMismatch); assertEquals(0, stats.getMatchedPartitions()); assertEquals(3, stats.getMismatchedPartitions()); assertEquals(0, stats.getMatchedValues()); assertEquals(0, stats.getMismatchedValues()); assertNothingReported(errors, journal); assertReported(0, MismatchType.PARTITION_MISMATCH, mismatches); assertReported(1, MismatchType.PARTITION_MISMATCH, mismatches); assertReported(2, MismatchType.PARTITION_MISMATCH, mismatches); assertCompared(0, 1, 2); }
@Test public void valueMismatchIncrementsPartitionMismatches() { RangeComparator comparator = comparator(context(0L, 100L)); RangeStats stats = comparator.compare(keys(0, 1, 2), keys(0, 1 ,2), this::valuesMismatch); assertEquals(0, stats.getMatchedPartitions()); assertEquals(3, stats.getMismatchedPartitions()); assertEquals(0, stats.getMatchedValues()); assertEquals(30, stats.getMismatchedValues()); assertNothingReported(errors, journal); assertReported(0, MismatchType.PARTITION_MISMATCH, mismatches); assertReported(1, MismatchType.PARTITION_MISMATCH, mismatches); assertReported(2, MismatchType.PARTITION_MISMATCH, mismatches); assertCompared(0, 1, 2); }
@Test public void matchingPartitionIncrementsCount() { RangeComparator comparator = comparator(context(0L, 100L)); RangeStats stats = comparator.compare(keys(0, 1, 2), keys(0, 1 ,2), this::alwaysMatch); assertEquals(3, stats.getMatchedPartitions()); assertEquals(0, stats.getMismatchedPartitions()); assertEquals(0, stats.getMatchedValues()); assertEquals(0, stats.getMismatchedValues()); assertNothingReported(errors, mismatches, journal); assertCompared(0, 1, 2); }
@Test public void waitForAllInFlightTasksToComplete() throws InterruptedException { CountDownLatch taskSubmissions = new CountDownLatch(2); List<CountDownLatch> taskGates = Lists.newArrayList(new CountDownLatch(1), new CountDownLatch(1)); RangeComparator comparator = comparator(context(0L, 100L)); Thread t = new Thread(() -> comparator.compare(keys(0, 1), keys(0, 1), waitUntilNotified(taskSubmissions, taskGates)), "CallingThread"); t.setDaemon(true); t.start(); taskSubmissions.await(); assertThreadWaits(t); taskGates.get(0).countDown(); assertThreadWaits(t); taskGates.get(1).countDown(); t.join(); assertNothingReported(errors, mismatches, journal); assertCompared(0, 1); }
@Test public void emptyRange() { RangeComparator comparator = comparator(context(100L, 100L)); RangeStats stats = comparator.compare(keys(), keys(), this::alwaysMatch); assertTrue(stats.isEmpty()); assertNothingReported(errors, mismatches, journal); assertCompared(); }
@Test public void sourceAndTargetKeysAreEmpty() { RangeComparator comparator = comparator(context(0L, 100L)); RangeStats stats = comparator.compare(keys(), keys(), this::alwaysMatch); assertTrue(stats.isEmpty()); assertNothingReported(errors, mismatches, journal); assertCompared(); }
@Test public void partitionPresentOnlyInSource() { RangeComparator comparator = comparator(context(0L, 100L)); RangeStats stats = comparator.compare(keys(0, 1, 2), keys(0, 1), this::alwaysMatch); assertFalse(stats.isEmpty()); assertEquals(1, stats.getOnlyInSource()); assertEquals(0, stats.getOnlyInTarget()); assertEquals(2, stats.getMatchedPartitions()); assertReported(2, MismatchType.ONLY_IN_SOURCE, mismatches); assertNothingReported(errors, journal); assertCompared(0, 1); }
@Test public void partitionPresentOnlyInTarget() { RangeComparator comparator = comparator(context(0L, 100L)); RangeStats stats = comparator.compare(keys(7, 8), keys(7, 8, 9), this::alwaysMatch); assertFalse(stats.isEmpty()); assertEquals(0, stats.getOnlyInSource()); assertEquals(1, stats.getOnlyInTarget()); assertEquals(2, stats.getMatchedPartitions()); assertReported(9, MismatchType.ONLY_IN_TARGET, mismatches); assertNothingReported(errors, journal); assertCompared(7, 8); }
@Test public void multipleOnlyInSource() { RangeComparator comparator = comparator(context(0L, 100L)); RangeStats stats = comparator.compare(keys(4, 5, 6, 7, 8), keys(4, 5), this::alwaysMatch); assertFalse(stats.isEmpty()); assertEquals(3, stats.getOnlyInSource()); assertEquals(0, stats.getOnlyInTarget()); assertEquals(2, stats.getMatchedPartitions()); assertReported(6, MismatchType.ONLY_IN_SOURCE, mismatches); assertReported(7, MismatchType.ONLY_IN_SOURCE, mismatches); assertReported(8, MismatchType.ONLY_IN_SOURCE, mismatches); assertNothingReported(errors, journal); assertCompared(4, 5); }
@Test public void multipleOnlyInTarget() { RangeComparator comparator = comparator(context(0L, 100L)); RangeStats stats = comparator.compare(keys(4, 5), keys(4, 5, 6, 7, 8), this::alwaysMatch); assertFalse(stats.isEmpty()); assertEquals(0, stats.getOnlyInSource()); assertEquals(3, stats.getOnlyInTarget()); assertEquals(2, stats.getMatchedPartitions()); assertReported(6, MismatchType.ONLY_IN_TARGET, mismatches); assertReported(7, MismatchType.ONLY_IN_TARGET, mismatches); assertReported(8, MismatchType.ONLY_IN_TARGET, mismatches); assertNothingReported(errors, journal); assertCompared(4, 5); }
@Test public void multipleOnlyInBoth() { RangeComparator comparator = comparator(context(0L, 100L)); RangeStats stats = comparator.compare(keys(0, 1, 3, 5, 7, 9), keys(0, 2, 4, 6, 8, 9), this::alwaysMatch); assertFalse(stats.isEmpty()); assertEquals(4, stats.getOnlyInSource()); assertEquals(4, stats.getOnlyInTarget()); assertEquals(2, stats.getMatchedPartitions()); assertReported(1, MismatchType.ONLY_IN_SOURCE, mismatches); assertReported(3, MismatchType.ONLY_IN_SOURCE, mismatches); assertReported(5, MismatchType.ONLY_IN_SOURCE, mismatches); assertReported(7, MismatchType.ONLY_IN_SOURCE, mismatches); assertReported(2, MismatchType.ONLY_IN_TARGET, mismatches); assertReported(4, MismatchType.ONLY_IN_TARGET, mismatches); assertReported(6, MismatchType.ONLY_IN_TARGET, mismatches); assertReported(8, MismatchType.ONLY_IN_TARGET, mismatches); assertNothingReported(errors, journal); assertCompared(0, 9); }
@Test public void sourceKeysIsEmpty() { RangeComparator comparator = comparator(context(0L, 100L)); RangeStats stats = comparator.compare(keys(), keys(4, 5), this::alwaysMatch); assertFalse(stats.isEmpty()); assertEquals(0, stats.getOnlyInSource()); assertEquals(2, stats.getOnlyInTarget()); assertEquals(0, stats.getMatchedPartitions()); assertReported(4, MismatchType.ONLY_IN_TARGET, mismatches); assertReported(5, MismatchType.ONLY_IN_TARGET, mismatches); assertNothingReported(errors, journal); assertCompared(); }
@Test public void targetKeysIsEmpty() { RangeComparator comparator = comparator(context(0L, 100L)); RangeStats stats = comparator.compare(keys(4, 5), keys(), this::alwaysMatch); assertFalse(stats.isEmpty()); assertEquals(2, stats.getOnlyInSource()); assertEquals(0, stats.getOnlyInTarget()); assertEquals(0, stats.getMatchedPartitions()); assertReported(4, MismatchType.ONLY_IN_SOURCE, mismatches); assertReported(5, MismatchType.ONLY_IN_SOURCE, mismatches); assertNothingReported(errors, journal); assertCompared(); }