@Test public void testGetCompactionDetailsTest() throws StoreException, InterruptedException { blobStore.logSegmentsNotInJournal = null; CompactionPolicyTest.verifyCompactionDetails(null, blobStore, compactionPolicy); long maxLogSegmentCapacity = blobStore.segmentCapacity - blobStore.segmentHeaderSize - DEFAULT_MAX_BLOB_SIZE; long logSegmentCount = blobStore.capacityInBytes / blobStore.segmentCapacity; blobStore.logSegmentsNotInJournal = CompactionPolicyTest.generateRandomLogSegmentName((int) logSegmentCount); for (int k = 0; k < 5; k++) { for (int i = 0; i < logSegmentCount; i++) { for (int j = i; j < logSegmentCount; j++) { List<String> bestCandidates = blobStore.logSegmentsNotInJournal.subList(i, j + 1); long bestCost = maxLogSegmentCapacity / bestCandidates.size(); NavigableMap<String, Long> validDataSize = CompactionPolicyTest.generateValidDataSize(blobStore.logSegmentsNotInJournal, bestCandidates, bestCost, maxLogSegmentCapacity); mockBlobStoreStats.validDataSizeByLogSegments = validDataSize; if ((j - i) >= config.storeMinLogSegmentCountToReclaimToTriggerCompaction) { CompactionPolicyTest.verifyCompactionDetails( new CompactionDetails(time.milliseconds() - messageRetentionTimeInMs, bestCandidates), blobStore, compactionPolicy); Pair<Integer, Integer> bestCandidateIndexes = new Pair<>(i, j); Pair<Integer, Integer> secondBestCandidateIndexes = findNonOverLappingCandidate(bestCandidateIndexes, (int) logSegmentCount - 1); if (secondBestCandidateIndexes != null) { CompactionPolicyTest.updateValidDataSize(validDataSize, blobStore.logSegmentsNotInJournal.subList(secondBestCandidateIndexes.getFirst(), secondBestCandidateIndexes.getSecond() + 1), bestCost + 1); List<String> secondBestCandidate = blobStore.logSegmentsNotInJournal.subList(secondBestCandidateIndexes.getFirst(), secondBestCandidateIndexes.getSecond() + 1); mockBlobStoreStats.validDataSizeByLogSegments = validDataSize; CompactionPolicyTest.verifyCompactionDetails( new CompactionDetails(time.milliseconds() - messageRetentionTimeInMs, bestCandidates), blobStore, compactionPolicy); CompactionPolicyTest.updateValidDataSize(validDataSize, blobStore.logSegmentsNotInJournal.subList(secondBestCandidateIndexes.getFirst(), secondBestCandidateIndexes.getSecond() + 1), bestCost - 1); mockBlobStoreStats.validDataSizeByLogSegments = validDataSize; CompactionPolicyTest.verifyCompactionDetails( new CompactionDetails(time.milliseconds() - messageRetentionTimeInMs, secondBestCandidate), blobStore, compactionPolicy); } } else { CompactionPolicyTest.verifyCompactionDetails(null, blobStore, compactionPolicy); } } } } }
@Override public CompactionDetails getCompactionDetails(long totalCapacity, long usedCapacity, long segmentCapacity, long segmentHeaderSize, List<String> logSegmentsNotInJournal, BlobStoreStats blobStoreStats, String dataDir) throws StoreException { CompactionDetails details = null; logger.trace("UsedCapacity {} vs TotalCapacity {}", usedCapacity, totalCapacity); if (usedCapacity >= (storeConfig.storeMinUsedCapacityToTriggerCompactionInPercentage / 100.0) * totalCapacity) { if (logSegmentsNotInJournal != null) { Pair<Long, NavigableMap<String, Long>> validDataSizeByLogSegment = blobStoreStats.getValidDataSizeByLogSegment( new TimeRange(time.milliseconds() - messageRetentionTimeInMs - ERROR_MARGIN_MS, ERROR_MARGIN_MS)); logger.info("Valid data size from BlobStoreStats {} ", validDataSizeByLogSegment); NavigableMap<String, Long> potentialLogSegmentValidSizeMap = validDataSizeByLogSegment.getSecond() .subMap(logSegmentsNotInJournal.get(0), true, logSegmentsNotInJournal.get(logSegmentsNotInJournal.size() - 1), true); CostBenefitInfo bestCandidateToCompact = getBestCandidateToCompact(potentialLogSegmentValidSizeMap, segmentCapacity, segmentHeaderSize, blobStoreStats.getMaxBlobSize()); if (bestCandidateToCompact != null) { details = new CompactionDetails(validDataSizeByLogSegment.getFirst(), bestCandidateToCompact.getSegmentsToCompact()); logger.info("Best candidate to compact {}", bestCandidateToCompact); } else { logger.trace("No best candidate found"); } } } return details; }
StatsBasedCompactionPolicy implements CompactionPolicy { @Override public CompactionDetails getCompactionDetails(long totalCapacity, long usedCapacity, long segmentCapacity, long segmentHeaderSize, List<String> logSegmentsNotInJournal, BlobStoreStats blobStoreStats, String dataDir) throws StoreException { CompactionDetails details = null; logger.trace("UsedCapacity {} vs TotalCapacity {}", usedCapacity, totalCapacity); if (usedCapacity >= (storeConfig.storeMinUsedCapacityToTriggerCompactionInPercentage / 100.0) * totalCapacity) { if (logSegmentsNotInJournal != null) { Pair<Long, NavigableMap<String, Long>> validDataSizeByLogSegment = blobStoreStats.getValidDataSizeByLogSegment( new TimeRange(time.milliseconds() - messageRetentionTimeInMs - ERROR_MARGIN_MS, ERROR_MARGIN_MS)); logger.info("Valid data size from BlobStoreStats {} ", validDataSizeByLogSegment); NavigableMap<String, Long> potentialLogSegmentValidSizeMap = validDataSizeByLogSegment.getSecond() .subMap(logSegmentsNotInJournal.get(0), true, logSegmentsNotInJournal.get(logSegmentsNotInJournal.size() - 1), true); CostBenefitInfo bestCandidateToCompact = getBestCandidateToCompact(potentialLogSegmentValidSizeMap, segmentCapacity, segmentHeaderSize, blobStoreStats.getMaxBlobSize()); if (bestCandidateToCompact != null) { details = new CompactionDetails(validDataSizeByLogSegment.getFirst(), bestCandidateToCompact.getSegmentsToCompact()); logger.info("Best candidate to compact {}", bestCandidateToCompact); } else { logger.trace("No best candidate found"); } } } return details; } }
StatsBasedCompactionPolicy implements CompactionPolicy { @Override public CompactionDetails getCompactionDetails(long totalCapacity, long usedCapacity, long segmentCapacity, long segmentHeaderSize, List<String> logSegmentsNotInJournal, BlobStoreStats blobStoreStats, String dataDir) throws StoreException { CompactionDetails details = null; logger.trace("UsedCapacity {} vs TotalCapacity {}", usedCapacity, totalCapacity); if (usedCapacity >= (storeConfig.storeMinUsedCapacityToTriggerCompactionInPercentage / 100.0) * totalCapacity) { if (logSegmentsNotInJournal != null) { Pair<Long, NavigableMap<String, Long>> validDataSizeByLogSegment = blobStoreStats.getValidDataSizeByLogSegment( new TimeRange(time.milliseconds() - messageRetentionTimeInMs - ERROR_MARGIN_MS, ERROR_MARGIN_MS)); logger.info("Valid data size from BlobStoreStats {} ", validDataSizeByLogSegment); NavigableMap<String, Long> potentialLogSegmentValidSizeMap = validDataSizeByLogSegment.getSecond() .subMap(logSegmentsNotInJournal.get(0), true, logSegmentsNotInJournal.get(logSegmentsNotInJournal.size() - 1), true); CostBenefitInfo bestCandidateToCompact = getBestCandidateToCompact(potentialLogSegmentValidSizeMap, segmentCapacity, segmentHeaderSize, blobStoreStats.getMaxBlobSize()); if (bestCandidateToCompact != null) { details = new CompactionDetails(validDataSizeByLogSegment.getFirst(), bestCandidateToCompact.getSegmentsToCompact()); logger.info("Best candidate to compact {}", bestCandidateToCompact); } else { logger.trace("No best candidate found"); } } } return details; } StatsBasedCompactionPolicy(StoreConfig storeConfig, Time time); }
StatsBasedCompactionPolicy implements CompactionPolicy { @Override public CompactionDetails getCompactionDetails(long totalCapacity, long usedCapacity, long segmentCapacity, long segmentHeaderSize, List<String> logSegmentsNotInJournal, BlobStoreStats blobStoreStats, String dataDir) throws StoreException { CompactionDetails details = null; logger.trace("UsedCapacity {} vs TotalCapacity {}", usedCapacity, totalCapacity); if (usedCapacity >= (storeConfig.storeMinUsedCapacityToTriggerCompactionInPercentage / 100.0) * totalCapacity) { if (logSegmentsNotInJournal != null) { Pair<Long, NavigableMap<String, Long>> validDataSizeByLogSegment = blobStoreStats.getValidDataSizeByLogSegment( new TimeRange(time.milliseconds() - messageRetentionTimeInMs - ERROR_MARGIN_MS, ERROR_MARGIN_MS)); logger.info("Valid data size from BlobStoreStats {} ", validDataSizeByLogSegment); NavigableMap<String, Long> potentialLogSegmentValidSizeMap = validDataSizeByLogSegment.getSecond() .subMap(logSegmentsNotInJournal.get(0), true, logSegmentsNotInJournal.get(logSegmentsNotInJournal.size() - 1), true); CostBenefitInfo bestCandidateToCompact = getBestCandidateToCompact(potentialLogSegmentValidSizeMap, segmentCapacity, segmentHeaderSize, blobStoreStats.getMaxBlobSize()); if (bestCandidateToCompact != null) { details = new CompactionDetails(validDataSizeByLogSegment.getFirst(), bestCandidateToCompact.getSegmentsToCompact()); logger.info("Best candidate to compact {}", bestCandidateToCompact); } else { logger.trace("No best candidate found"); } } } return details; } StatsBasedCompactionPolicy(StoreConfig storeConfig, Time time); @Override CompactionDetails getCompactionDetails(long totalCapacity, long usedCapacity, long segmentCapacity,
      long segmentHeaderSize, List<String> logSegmentsNotInJournal, BlobStoreStats blobStoreStats, String dataDir); }
StatsBasedCompactionPolicy implements CompactionPolicy { @Override public CompactionDetails getCompactionDetails(long totalCapacity, long usedCapacity, long segmentCapacity, long segmentHeaderSize, List<String> logSegmentsNotInJournal, BlobStoreStats blobStoreStats, String dataDir) throws StoreException { CompactionDetails details = null; logger.trace("UsedCapacity {} vs TotalCapacity {}", usedCapacity, totalCapacity); if (usedCapacity >= (storeConfig.storeMinUsedCapacityToTriggerCompactionInPercentage / 100.0) * totalCapacity) { if (logSegmentsNotInJournal != null) { Pair<Long, NavigableMap<String, Long>> validDataSizeByLogSegment = blobStoreStats.getValidDataSizeByLogSegment( new TimeRange(time.milliseconds() - messageRetentionTimeInMs - ERROR_MARGIN_MS, ERROR_MARGIN_MS)); logger.info("Valid data size from BlobStoreStats {} ", validDataSizeByLogSegment); NavigableMap<String, Long> potentialLogSegmentValidSizeMap = validDataSizeByLogSegment.getSecond() .subMap(logSegmentsNotInJournal.get(0), true, logSegmentsNotInJournal.get(logSegmentsNotInJournal.size() - 1), true); CostBenefitInfo bestCandidateToCompact = getBestCandidateToCompact(potentialLogSegmentValidSizeMap, segmentCapacity, segmentHeaderSize, blobStoreStats.getMaxBlobSize()); if (bestCandidateToCompact != null) { details = new CompactionDetails(validDataSizeByLogSegment.getFirst(), bestCandidateToCompact.getSegmentsToCompact()); logger.info("Best candidate to compact {}", bestCandidateToCompact); } else { logger.trace("No best candidate found"); } } } return details; } StatsBasedCompactionPolicy(StoreConfig storeConfig, Time time); @Override CompactionDetails getCompactionDetails(long totalCapacity, long usedCapacity, long segmentCapacity,
      long segmentHeaderSize, List<String> logSegmentsNotInJournal, BlobStoreStats blobStoreStats, String dataDir);  }