@SuppressWarnings("unchecked") @Test public void testMergeExpiredFiles() throws Exception { ManualEnvironmentEdge edge = new ManualEnvironmentEdge(); long now = defaultTtl + 2; edge.setValue(now); EnvironmentEdgeManager.injectEdge(edge); try { StoreFile expiredFile = createFile(), notExpiredFile = createFile(); when(expiredFile.getReader().getMaxTimestamp()).thenReturn(now - defaultTtl - 1); when(notExpiredFile.getReader().getMaxTimestamp()).thenReturn(now - defaultTtl + 1); List<StoreFile> expired = Lists.newArrayList(expiredFile, expiredFile); List<StoreFile> notExpired = Lists.newArrayList(notExpiredFile, notExpiredFile); List<StoreFile> mixed = Lists.newArrayList(expiredFile, notExpiredFile); StripeCompactionPolicy policy = createPolicy(HBaseConfiguration.create(), defaultSplitSize, defaultSplitCount, defaultInitialCount, true); StripeCompactionPolicy.StripeInformationProvider si = createStripesWithFiles(expired, expired, expired); verifyWholeStripesCompaction(policy, si, 0, 2, null, 1, Long.MAX_VALUE, false); si = createStripesWithFiles(notExpired, notExpired, notExpired); assertNull(policy.selectCompaction(si, al(), false)); si = createStripesWithFiles(notExpired, expired, notExpired); verifyWholeStripesCompaction(policy, si, 1, 2, null, 1, Long.MAX_VALUE, false); si = createStripesWithFiles(notExpired, expired, notExpired, expired, expired, notExpired); verifyWholeStripesCompaction(policy, si, 3, 4, null, 1, Long.MAX_VALUE, false); si = createStripesWithFiles(expired, expired, notExpired, expired, mixed); verifyWholeStripesCompaction(policy, si, 0, 1, null, 1, Long.MAX_VALUE, false); } finally { EnvironmentEdgeManager.reset(); } }
public StripeCompactionRequest selectCompaction(StripeInformationProvider si, List<StoreFile> filesCompacting, boolean isOffpeak) throws IOException { if (!filesCompacting.isEmpty()) { LOG.debug("Not selecting compaction: " + filesCompacting.size() + " files compacting"); return null; } Collection<StoreFile> allFiles = si.getStorefiles(); if (StoreUtils.hasReferences(allFiles)) { LOG.debug("There are references in the store; compacting all files"); long targetKvs = estimateTargetKvs(allFiles, config.getInitialCount()).getFirst(); SplitStripeCompactionRequest request = new SplitStripeCompactionRequest( allFiles, OPEN_KEY, OPEN_KEY, targetKvs); request.setMajorRangeFull(); return request; } int stripeCount = si.getStripeCount(); List<StoreFile> l0Files = si.getLevel0Files(); boolean shouldCompactL0 = (this.config.getLevel0MinFiles() <= l0Files.size()); if (stripeCount == 0) { if (!shouldCompactL0) return null; return selectNewStripesCompaction(si); } boolean canDropDeletesNoL0 = l0Files.size() == 0; if (shouldCompactL0) { if (!canDropDeletesNoL0) { StripeCompactionRequest result = selectSingleStripeCompaction( si, true, canDropDeletesNoL0, isOffpeak); if (result != null) return result; } LOG.debug("Selecting L0 compaction with " + l0Files.size() + " files"); return new BoundaryStripeCompactionRequest(l0Files, si.getStripeBoundaries()); } StripeCompactionRequest result = selectExpiredMergeCompaction(si, canDropDeletesNoL0); if (result != null) return result; return selectSingleStripeCompaction(si, false, canDropDeletesNoL0, isOffpeak); }
StripeCompactionPolicy extends CompactionPolicy { public StripeCompactionRequest selectCompaction(StripeInformationProvider si, List<StoreFile> filesCompacting, boolean isOffpeak) throws IOException { if (!filesCompacting.isEmpty()) { LOG.debug("Not selecting compaction: " + filesCompacting.size() + " files compacting"); return null; } Collection<StoreFile> allFiles = si.getStorefiles(); if (StoreUtils.hasReferences(allFiles)) { LOG.debug("There are references in the store; compacting all files"); long targetKvs = estimateTargetKvs(allFiles, config.getInitialCount()).getFirst(); SplitStripeCompactionRequest request = new SplitStripeCompactionRequest( allFiles, OPEN_KEY, OPEN_KEY, targetKvs); request.setMajorRangeFull(); return request; } int stripeCount = si.getStripeCount(); List<StoreFile> l0Files = si.getLevel0Files(); boolean shouldCompactL0 = (this.config.getLevel0MinFiles() <= l0Files.size()); if (stripeCount == 0) { if (!shouldCompactL0) return null; return selectNewStripesCompaction(si); } boolean canDropDeletesNoL0 = l0Files.size() == 0; if (shouldCompactL0) { if (!canDropDeletesNoL0) { StripeCompactionRequest result = selectSingleStripeCompaction( si, true, canDropDeletesNoL0, isOffpeak); if (result != null) return result; } LOG.debug("Selecting L0 compaction with " + l0Files.size() + " files"); return new BoundaryStripeCompactionRequest(l0Files, si.getStripeBoundaries()); } StripeCompactionRequest result = selectExpiredMergeCompaction(si, canDropDeletesNoL0); if (result != null) return result; return selectSingleStripeCompaction(si, false, canDropDeletesNoL0, isOffpeak); } }
StripeCompactionPolicy extends CompactionPolicy { public StripeCompactionRequest selectCompaction(StripeInformationProvider si, List<StoreFile> filesCompacting, boolean isOffpeak) throws IOException { if (!filesCompacting.isEmpty()) { LOG.debug("Not selecting compaction: " + filesCompacting.size() + " files compacting"); return null; } Collection<StoreFile> allFiles = si.getStorefiles(); if (StoreUtils.hasReferences(allFiles)) { LOG.debug("There are references in the store; compacting all files"); long targetKvs = estimateTargetKvs(allFiles, config.getInitialCount()).getFirst(); SplitStripeCompactionRequest request = new SplitStripeCompactionRequest( allFiles, OPEN_KEY, OPEN_KEY, targetKvs); request.setMajorRangeFull(); return request; } int stripeCount = si.getStripeCount(); List<StoreFile> l0Files = si.getLevel0Files(); boolean shouldCompactL0 = (this.config.getLevel0MinFiles() <= l0Files.size()); if (stripeCount == 0) { if (!shouldCompactL0) return null; return selectNewStripesCompaction(si); } boolean canDropDeletesNoL0 = l0Files.size() == 0; if (shouldCompactL0) { if (!canDropDeletesNoL0) { StripeCompactionRequest result = selectSingleStripeCompaction( si, true, canDropDeletesNoL0, isOffpeak); if (result != null) return result; } LOG.debug("Selecting L0 compaction with " + l0Files.size() + " files"); return new BoundaryStripeCompactionRequest(l0Files, si.getStripeBoundaries()); } StripeCompactionRequest result = selectExpiredMergeCompaction(si, canDropDeletesNoL0); if (result != null) return result; return selectSingleStripeCompaction(si, false, canDropDeletesNoL0, isOffpeak); } StripeCompactionPolicy(
      Configuration conf, StoreConfigInformation storeConfigInfo, StripeStoreConfig config); }
StripeCompactionPolicy extends CompactionPolicy { public StripeCompactionRequest selectCompaction(StripeInformationProvider si, List<StoreFile> filesCompacting, boolean isOffpeak) throws IOException { if (!filesCompacting.isEmpty()) { LOG.debug("Not selecting compaction: " + filesCompacting.size() + " files compacting"); return null; } Collection<StoreFile> allFiles = si.getStorefiles(); if (StoreUtils.hasReferences(allFiles)) { LOG.debug("There are references in the store; compacting all files"); long targetKvs = estimateTargetKvs(allFiles, config.getInitialCount()).getFirst(); SplitStripeCompactionRequest request = new SplitStripeCompactionRequest( allFiles, OPEN_KEY, OPEN_KEY, targetKvs); request.setMajorRangeFull(); return request; } int stripeCount = si.getStripeCount(); List<StoreFile> l0Files = si.getLevel0Files(); boolean shouldCompactL0 = (this.config.getLevel0MinFiles() <= l0Files.size()); if (stripeCount == 0) { if (!shouldCompactL0) return null; return selectNewStripesCompaction(si); } boolean canDropDeletesNoL0 = l0Files.size() == 0; if (shouldCompactL0) { if (!canDropDeletesNoL0) { StripeCompactionRequest result = selectSingleStripeCompaction( si, true, canDropDeletesNoL0, isOffpeak); if (result != null) return result; } LOG.debug("Selecting L0 compaction with " + l0Files.size() + " files"); return new BoundaryStripeCompactionRequest(l0Files, si.getStripeBoundaries()); } StripeCompactionRequest result = selectExpiredMergeCompaction(si, canDropDeletesNoL0); if (result != null) return result; return selectSingleStripeCompaction(si, false, canDropDeletesNoL0, isOffpeak); } StripeCompactionPolicy(
      Configuration conf, StoreConfigInformation storeConfigInfo, StripeStoreConfig config); List<StoreFile> preSelectFilesForCoprocessor(StripeInformationProvider si,
      List<StoreFile> filesCompacting); StripeCompactionRequest createEmptyRequest(
      StripeInformationProvider si, CompactionRequest request); StripeStoreFlusher.StripeFlushRequest selectFlush(
      StripeInformationProvider si, int kvCount); StripeCompactionRequest selectCompaction(StripeInformationProvider si,
      List<StoreFile> filesCompacting, boolean isOffpeak); boolean needsCompactions(StripeInformationProvider si, List<StoreFile> filesCompacting); @Override boolean isMajorCompaction(Collection<StoreFile> filesToCompact); @Override boolean throttleCompaction(long compactionSize); static long getTotalFileSize(final Collection<StoreFile> candidates); }
StripeCompactionPolicy extends CompactionPolicy { public StripeCompactionRequest selectCompaction(StripeInformationProvider si, List<StoreFile> filesCompacting, boolean isOffpeak) throws IOException { if (!filesCompacting.isEmpty()) { LOG.debug("Not selecting compaction: " + filesCompacting.size() + " files compacting"); return null; } Collection<StoreFile> allFiles = si.getStorefiles(); if (StoreUtils.hasReferences(allFiles)) { LOG.debug("There are references in the store; compacting all files"); long targetKvs = estimateTargetKvs(allFiles, config.getInitialCount()).getFirst(); SplitStripeCompactionRequest request = new SplitStripeCompactionRequest( allFiles, OPEN_KEY, OPEN_KEY, targetKvs); request.setMajorRangeFull(); return request; } int stripeCount = si.getStripeCount(); List<StoreFile> l0Files = si.getLevel0Files(); boolean shouldCompactL0 = (this.config.getLevel0MinFiles() <= l0Files.size()); if (stripeCount == 0) { if (!shouldCompactL0) return null; return selectNewStripesCompaction(si); } boolean canDropDeletesNoL0 = l0Files.size() == 0; if (shouldCompactL0) { if (!canDropDeletesNoL0) { StripeCompactionRequest result = selectSingleStripeCompaction( si, true, canDropDeletesNoL0, isOffpeak); if (result != null) return result; } LOG.debug("Selecting L0 compaction with " + l0Files.size() + " files"); return new BoundaryStripeCompactionRequest(l0Files, si.getStripeBoundaries()); } StripeCompactionRequest result = selectExpiredMergeCompaction(si, canDropDeletesNoL0); if (result != null) return result; return selectSingleStripeCompaction(si, false, canDropDeletesNoL0, isOffpeak); } StripeCompactionPolicy(
      Configuration conf, StoreConfigInformation storeConfigInfo, StripeStoreConfig config); List<StoreFile> preSelectFilesForCoprocessor(StripeInformationProvider si,
      List<StoreFile> filesCompacting); StripeCompactionRequest createEmptyRequest(
      StripeInformationProvider si, CompactionRequest request); StripeStoreFlusher.StripeFlushRequest selectFlush(
      StripeInformationProvider si, int kvCount); StripeCompactionRequest selectCompaction(StripeInformationProvider si,
      List<StoreFile> filesCompacting, boolean isOffpeak); boolean needsCompactions(StripeInformationProvider si, List<StoreFile> filesCompacting); @Override boolean isMajorCompaction(Collection<StoreFile> filesToCompact); @Override boolean throttleCompaction(long compactionSize); static long getTotalFileSize(final Collection<StoreFile> candidates);  }