@Test public void testHardDelete() { try { List<MockId> keys = setupIndexForTest(); boolean recordContentChecked = false; for (int i = 0; i < 3; i++) { assertEquals("There should have been no progress reported for hard deletes", 0, index.getHardDeleteProgress()); StoreFindToken token = (StoreFindToken) index.hardDeleter.getStartTokenSafeToPersistTo(); assertEquals("Token type mismatch", token.getType(), FindTokenType.Uninitialized); Set<StoreKey> hardDeletedKeys = new HashSet<>(); boolean tokenMovedForward = index.hardDelete(); Assert.assertTrue(tokenMovedForward); if (!recordContentChecked) { checkRecordHardDeleted(keys, hardDeletedKeys); } assertEquals("Recovery range size mismatch", 0, index.hardDeleter.getHardDeleteRecoveryRange().getSize()); index.persistAndAdvanceStartTokenSafeToPersist(); token = (StoreFindToken) index.hardDeleter.getStartTokenSafeToPersistTo(); assertEquals("Token type mismatch", token.getType(), FindTokenType.IndexBased); assertEquals("Token key mismatch", token.getStoreKey(), keys.get(1)); assertEquals("Token offset mismatch", token.getOffset().getOffset(), 0); tokenMovedForward = index.hardDelete(); Assert.assertTrue(tokenMovedForward); if (!recordContentChecked) { checkRecordHardDeleted(keys, hardDeletedKeys); } assertEquals("Recovery range size mismatch", 0, index.hardDeleter.getHardDeleteRecoveryRange().getSize()); token = (StoreFindToken) index.hardDeleter.getStartTokenSafeToPersistTo(); assertEquals("Token type mismatch", token.getType(), FindTokenType.IndexBased); assertEquals("Token key mismatch", token.getStoreKey(), keys.get(1)); assertEquals("Token offset mismatch", token.getOffset().getOffset(), 0); tokenMovedForward = index.hardDelete(); Assert.assertTrue(tokenMovedForward); if (!recordContentChecked) { checkRecordHardDeleted(keys, hardDeletedKeys); } assertEquals("Recovery range size mismatch", 0, index.hardDeleter.getHardDeleteRecoveryRange().getSize()); index.persistAndAdvanceStartTokenSafeToPersist(); token = (StoreFindToken) index.hardDeleter.getStartTokenSafeToPersistTo(); assertEquals("Token type mismatch", token.getType(), FindTokenType.IndexBased); assertEquals("Token key mismatch", token.getStoreKey(), keys.get(4)); assertEquals("Token offset mismatch", token.getOffset().getOffset(), 2 * 2 * helper.sizeOfEntry); tokenMovedForward = index.hardDelete(); Assert.assertTrue(tokenMovedForward); if (!recordContentChecked) { checkRecordHardDeleted(keys, hardDeletedKeys); } assertEquals("Recovery range size mismatch", 0, index.hardDeleter.getHardDeleteRecoveryRange().getSize()); tokenMovedForward = index.hardDelete(); Assert.assertTrue(tokenMovedForward); hardDeletedKeys.add(keys.get(1)); hardDeletedKeys.add(keys.get(5)); if (!recordContentChecked) { checkRecordHardDeleted(keys, hardDeletedKeys); } assertEquals("Recovery range size mismatch", 2, index.hardDeleter.getHardDeleteRecoveryRange().getSize()); tokenMovedForward = index.hardDelete(); Assert.assertTrue(tokenMovedForward); if (!recordContentChecked) { checkRecordHardDeleted(keys, hardDeletedKeys); } assertEquals("Recovery range size mismatch", 2, index.hardDeleter.getHardDeleteRecoveryRange().getSize()); index.persistAndAdvanceStartTokenSafeToPersist(); token = (StoreFindToken) index.hardDeleter.getStartTokenSafeToPersistTo(); assertEquals("Token type mismatch", token.getType(), FindTokenType.IndexBased); assertEquals("Token key mismatch", token.getStoreKey(), keys.get(8)); assertEquals("Token offset mismatch", token.getOffset().getOffset(), 5 * 2 * helper.sizeOfEntry); tokenMovedForward = index.hardDelete(); Assert.assertTrue(tokenMovedForward); if (!recordContentChecked) { checkRecordHardDeleted(keys, hardDeletedKeys); } assertEquals("Recovery range size mismatch", 0, index.hardDeleter.getHardDeleteRecoveryRange().getSize()); tokenMovedForward = index.hardDelete(); Assert.assertTrue(tokenMovedForward); if (!recordContentChecked) { checkRecordHardDeleted(keys, hardDeletedKeys); } tokenMovedForward = index.hardDelete(); Assert.assertTrue(tokenMovedForward); hardDeletedKeys.add(keys.get(9)); hardDeletedKeys.add(keys.get(7)); if (!recordContentChecked) { checkRecordHardDeleted(keys, hardDeletedKeys); } assertEquals("Recovery range size mismatch", 2, index.hardDeleter.getHardDeleteRecoveryRange().getSize()); token = (StoreFindToken) index.hardDeleter.getStartTokenSafeToPersistTo(); index.hardDeleter.getHardDeleteRecoveryRange().pruneTill(token); assertEquals("Recovery range size mismatch", 2, index.hardDeleter.getHardDeleteRecoveryRange().getSize()); tokenMovedForward = index.hardDelete(); Assert.assertFalse(tokenMovedForward); index.persistAndAdvanceStartTokenSafeToPersist(); token = (StoreFindToken) index.hardDeleter.getStartTokenSafeToPersistTo(); assertEquals("Token type mismatch", token.getType(), FindTokenType.JournalBased); assertEquals("Token offset mismatch", token.getOffset().getOffset(), 17 * helper.sizeOfEntry); assertEquals("Recovery range size mismatch", 0, index.hardDeleter.getHardDeleteRecoveryRange().getSize()); index.pruneHardDeleteRecoveryRange(); index.performHardDeleteRecovery(); index.resetHardDeleterTokens(); recordContentChecked = true; } index.close(false); } catch (Exception e) { e.printStackTrace(); fail("Unexpected exception"); } }
boolean hardDelete() throws StoreException { if (index.getCurrentEndOffset().compareTo(index.getStartOffset()) > 0) { final Timer.Context context = metrics.hardDeleteTime.time(); try { FindInfo info = index.findDeletedEntriesSince(startToken, scanSizeInBytes, time.seconds() - messageRetentionSeconds); endToken = info.getFindToken(); logger.trace("New range for hard deletes : startToken {}, endToken {} for {}", startToken, info.getFindToken(), dataDir); if (!endToken.equals(startToken)) { if (!info.getMessageEntries().isEmpty()) { if (injectedBeforeHardDeleteSupplier != null) { injectedBeforeHardDeleteSupplier.get(); } performHardDeletes(info.getMessageEntries()); } startToken = endToken; return true; } } catch (StoreException e) { if (e.getErrorCode() != StoreErrorCodes.Store_Shutting_Down) { metrics.hardDeleteExceptionsCount.inc(); } throw e; } finally { context.stop(); } } return false; }
HardDeleter implements Runnable { boolean hardDelete() throws StoreException { if (index.getCurrentEndOffset().compareTo(index.getStartOffset()) > 0) { final Timer.Context context = metrics.hardDeleteTime.time(); try { FindInfo info = index.findDeletedEntriesSince(startToken, scanSizeInBytes, time.seconds() - messageRetentionSeconds); endToken = info.getFindToken(); logger.trace("New range for hard deletes : startToken {}, endToken {} for {}", startToken, info.getFindToken(), dataDir); if (!endToken.equals(startToken)) { if (!info.getMessageEntries().isEmpty()) { if (injectedBeforeHardDeleteSupplier != null) { injectedBeforeHardDeleteSupplier.get(); } performHardDeletes(info.getMessageEntries()); } startToken = endToken; return true; } } catch (StoreException e) { if (e.getErrorCode() != StoreErrorCodes.Store_Shutting_Down) { metrics.hardDeleteExceptionsCount.inc(); } throw e; } finally { context.stop(); } } return false; } }
HardDeleter implements Runnable { boolean hardDelete() throws StoreException { if (index.getCurrentEndOffset().compareTo(index.getStartOffset()) > 0) { final Timer.Context context = metrics.hardDeleteTime.time(); try { FindInfo info = index.findDeletedEntriesSince(startToken, scanSizeInBytes, time.seconds() - messageRetentionSeconds); endToken = info.getFindToken(); logger.trace("New range for hard deletes : startToken {}, endToken {} for {}", startToken, info.getFindToken(), dataDir); if (!endToken.equals(startToken)) { if (!info.getMessageEntries().isEmpty()) { if (injectedBeforeHardDeleteSupplier != null) { injectedBeforeHardDeleteSupplier.get(); } performHardDeletes(info.getMessageEntries()); } startToken = endToken; return true; } } catch (StoreException e) { if (e.getErrorCode() != StoreErrorCodes.Store_Shutting_Down) { metrics.hardDeleteExceptionsCount.inc(); } throw e; } finally { context.stop(); } } return false; } HardDeleter(StoreConfig config, StoreMetrics metrics, String dataDir, Log log, PersistentIndex index,
      MessageStoreHardDelete hardDelete, StoreKeyFactory factory, DiskIOScheduler diskIOScheduler, Time time); }
HardDeleter implements Runnable { boolean hardDelete() throws StoreException { if (index.getCurrentEndOffset().compareTo(index.getStartOffset()) > 0) { final Timer.Context context = metrics.hardDeleteTime.time(); try { FindInfo info = index.findDeletedEntriesSince(startToken, scanSizeInBytes, time.seconds() - messageRetentionSeconds); endToken = info.getFindToken(); logger.trace("New range for hard deletes : startToken {}, endToken {} for {}", startToken, info.getFindToken(), dataDir); if (!endToken.equals(startToken)) { if (!info.getMessageEntries().isEmpty()) { if (injectedBeforeHardDeleteSupplier != null) { injectedBeforeHardDeleteSupplier.get(); } performHardDeletes(info.getMessageEntries()); } startToken = endToken; return true; } } catch (StoreException e) { if (e.getErrorCode() != StoreErrorCodes.Store_Shutting_Down) { metrics.hardDeleteExceptionsCount.inc(); } throw e; } finally { context.stop(); } } return false; } HardDeleter(StoreConfig config, StoreMetrics metrics, String dataDir, Log log, PersistentIndex index,
      MessageStoreHardDelete hardDelete, StoreKeyFactory factory, DiskIOScheduler diskIOScheduler, Time time); @Override void run(); }
HardDeleter implements Runnable { boolean hardDelete() throws StoreException { if (index.getCurrentEndOffset().compareTo(index.getStartOffset()) > 0) { final Timer.Context context = metrics.hardDeleteTime.time(); try { FindInfo info = index.findDeletedEntriesSince(startToken, scanSizeInBytes, time.seconds() - messageRetentionSeconds); endToken = info.getFindToken(); logger.trace("New range for hard deletes : startToken {}, endToken {} for {}", startToken, info.getFindToken(), dataDir); if (!endToken.equals(startToken)) { if (!info.getMessageEntries().isEmpty()) { if (injectedBeforeHardDeleteSupplier != null) { injectedBeforeHardDeleteSupplier.get(); } performHardDeletes(info.getMessageEntries()); } startToken = endToken; return true; } } catch (StoreException e) { if (e.getErrorCode() != StoreErrorCodes.Store_Shutting_Down) { metrics.hardDeleteExceptionsCount.inc(); } throw e; } finally { context.stop(); } } return false; } HardDeleter(StoreConfig config, StoreMetrics metrics, String dataDir, Log log, PersistentIndex index,
      MessageStoreHardDelete hardDelete, StoreKeyFactory factory, DiskIOScheduler diskIOScheduler, Time time); @Override void run(); static final short Cleanup_Token_Version_V0; static final short Cleanup_Token_Version_V1; }