@Test public void testTTLNext() throws Exception { int daysForSnapshot = configuration.getBackupRetentionDays() + 1; prepTest(daysForSnapshot); backupTTLService.execute(); List<String> remoteFiles = getAllFiles(); Assert.assertEquals(6, remoteFiles.size()); Assert.assertTrue(remoteFiles.contains(allFilesMap.get("mc-4-Data.db"))); Assert.assertTrue(remoteFiles.contains(allFilesMap.get("mc-6-Data.db"))); Assert.assertTrue(remoteFiles.contains(allFilesMap.get("mc-7-Data.db"))); Assert.assertTrue(remoteFiles.contains(allFilesMap.get("META2"))); Assert.assertTrue(remoteFiles.contains(allFilesMap.get("mc-3-Data.db"))); Assert.assertTrue(remoteFiles.contains(allFilesMap.get("mc-5-Data.db"))); Assert.assertFalse(remoteFiles.contains(allFilesMap.get("mc-1-Data.db"))); Assert.assertFalse(remoteFiles.contains(allFilesMap.get("mc-2-Data.db"))); Assert.assertFalse(remoteFiles.contains(allFilesMap.get("META0"))); Assert.assertFalse(remoteFiles.contains(allFilesMap.get("META1"))); }
@Override public void execute() throws Exception { if (instanceState.getRestoreStatus() != null && instanceState.getRestoreStatus().getStatus() != null && instanceState.getRestoreStatus().getStatus() == Status.STARTED) { logger.info("Not executing the TTL Task for backups as Priam is in restore mode."); return; } if (!lock.tryLock()) { logger.warn("{} is already running! Try again later.", JOBNAME); throw new Exception(JOBNAME + " already running"); } try { filesInMeta.clear(); filesToDelete.clear(); Instant dateToTtl = DateUtil.getInstant().minus(config.getBackupRetentionDays(), ChronoUnit.DAYS); List<AbstractBackupPath> metas = metaProxy.findMetaFiles( new DateUtil.DateRange(dateToTtl, DateUtil.getInstant())); if (metas.size() == 0) { logger.info("No meta file found and thus cannot run TTL Service"); return; } AbstractBackupPath metaFile = metas.get(metas.size() - 1); Path localFile = metaProxy.downloadMetaFile(metaFile); new MetaFileWalker().readMeta(localFile); logger.info("No. of component files loaded from meta file: {}", filesInMeta.size()); FileUtils.deleteQuietly(localFile.toFile()); if (filesInMeta.isEmpty()) { logger.warn("Meta file was empty. This should not happen. Getting out!!"); return; } metas = metaProxy.findMetaFiles( new DateUtil.DateRange( start_of_feature, dateToTtl.minus(1, ChronoUnit.HOURS))); if (metas != null && metas.size() != 0) { logger.info( "Will delete(TTL) {} META files starting from: [{}]", metas.size(), metas.get(metas.size() - 1).getLastModified()); for (AbstractBackupPath meta : metas) { deleteFile(meta, false); } } Iterator<String> remoteFileLocations = fileSystem.listFileSystem(getSSTPrefix(), null, null); dateToTtl = dateToTtl.minus(config.getGracePeriodDaysForCompaction(), ChronoUnit.DAYS); logger.info( "Will delete(TTL) SST_V2 files which are before this time: {}. Input: [TTL: {} days, Grace Period: {} days]", dateToTtl, config.getBackupRetentionDays(), config.getGracePeriodDaysForCompaction()); while (remoteFileLocations.hasNext()) { AbstractBackupPath abstractBackupPath = abstractBackupPathProvider.get(); abstractBackupPath.parseRemote(remoteFileLocations.next()); if (abstractBackupPath.getLastModified().isAfter(dateToTtl)) { logger.info( "Breaking from TTL. Got a key which is after the TTL time: {}", abstractBackupPath.getRemotePath()); break; } if (!filesInMeta.containsKey(abstractBackupPath.getRemotePath())) { deleteFile(abstractBackupPath, false); } else { if (logger.isDebugEnabled()) logger.debug( "Not deleting this key as it is referenced in backups: {}", abstractBackupPath.getRemotePath()); } } deleteFile(null, true); logger.info("Finished processing files for TTL service"); } finally { lock.unlock(); } }
BackupTTLTask extends Task { @Override public void execute() throws Exception { if (instanceState.getRestoreStatus() != null && instanceState.getRestoreStatus().getStatus() != null && instanceState.getRestoreStatus().getStatus() == Status.STARTED) { logger.info("Not executing the TTL Task for backups as Priam is in restore mode."); return; } if (!lock.tryLock()) { logger.warn("{} is already running! Try again later.", JOBNAME); throw new Exception(JOBNAME + " already running"); } try { filesInMeta.clear(); filesToDelete.clear(); Instant dateToTtl = DateUtil.getInstant().minus(config.getBackupRetentionDays(), ChronoUnit.DAYS); List<AbstractBackupPath> metas = metaProxy.findMetaFiles( new DateUtil.DateRange(dateToTtl, DateUtil.getInstant())); if (metas.size() == 0) { logger.info("No meta file found and thus cannot run TTL Service"); return; } AbstractBackupPath metaFile = metas.get(metas.size() - 1); Path localFile = metaProxy.downloadMetaFile(metaFile); new MetaFileWalker().readMeta(localFile); logger.info("No. of component files loaded from meta file: {}", filesInMeta.size()); FileUtils.deleteQuietly(localFile.toFile()); if (filesInMeta.isEmpty()) { logger.warn("Meta file was empty. This should not happen. Getting out!!"); return; } metas = metaProxy.findMetaFiles( new DateUtil.DateRange( start_of_feature, dateToTtl.minus(1, ChronoUnit.HOURS))); if (metas != null && metas.size() != 0) { logger.info( "Will delete(TTL) {} META files starting from: [{}]", metas.size(), metas.get(metas.size() - 1).getLastModified()); for (AbstractBackupPath meta : metas) { deleteFile(meta, false); } } Iterator<String> remoteFileLocations = fileSystem.listFileSystem(getSSTPrefix(), null, null); dateToTtl = dateToTtl.minus(config.getGracePeriodDaysForCompaction(), ChronoUnit.DAYS); logger.info( "Will delete(TTL) SST_V2 files which are before this time: {}. Input: [TTL: {} days, Grace Period: {} days]", dateToTtl, config.getBackupRetentionDays(), config.getGracePeriodDaysForCompaction()); while (remoteFileLocations.hasNext()) { AbstractBackupPath abstractBackupPath = abstractBackupPathProvider.get(); abstractBackupPath.parseRemote(remoteFileLocations.next()); if (abstractBackupPath.getLastModified().isAfter(dateToTtl)) { logger.info( "Breaking from TTL. Got a key which is after the TTL time: {}", abstractBackupPath.getRemotePath()); break; } if (!filesInMeta.containsKey(abstractBackupPath.getRemotePath())) { deleteFile(abstractBackupPath, false); } else { if (logger.isDebugEnabled()) logger.debug( "Not deleting this key as it is referenced in backups: {}", abstractBackupPath.getRemotePath()); } } deleteFile(null, true); logger.info("Finished processing files for TTL service"); } finally { lock.unlock(); } } }
BackupTTLTask extends Task { @Override public void execute() throws Exception { if (instanceState.getRestoreStatus() != null && instanceState.getRestoreStatus().getStatus() != null && instanceState.getRestoreStatus().getStatus() == Status.STARTED) { logger.info("Not executing the TTL Task for backups as Priam is in restore mode."); return; } if (!lock.tryLock()) { logger.warn("{} is already running! Try again later.", JOBNAME); throw new Exception(JOBNAME + " already running"); } try { filesInMeta.clear(); filesToDelete.clear(); Instant dateToTtl = DateUtil.getInstant().minus(config.getBackupRetentionDays(), ChronoUnit.DAYS); List<AbstractBackupPath> metas = metaProxy.findMetaFiles( new DateUtil.DateRange(dateToTtl, DateUtil.getInstant())); if (metas.size() == 0) { logger.info("No meta file found and thus cannot run TTL Service"); return; } AbstractBackupPath metaFile = metas.get(metas.size() - 1); Path localFile = metaProxy.downloadMetaFile(metaFile); new MetaFileWalker().readMeta(localFile); logger.info("No. of component files loaded from meta file: {}", filesInMeta.size()); FileUtils.deleteQuietly(localFile.toFile()); if (filesInMeta.isEmpty()) { logger.warn("Meta file was empty. This should not happen. Getting out!!"); return; } metas = metaProxy.findMetaFiles( new DateUtil.DateRange( start_of_feature, dateToTtl.minus(1, ChronoUnit.HOURS))); if (metas != null && metas.size() != 0) { logger.info( "Will delete(TTL) {} META files starting from: [{}]", metas.size(), metas.get(metas.size() - 1).getLastModified()); for (AbstractBackupPath meta : metas) { deleteFile(meta, false); } } Iterator<String> remoteFileLocations = fileSystem.listFileSystem(getSSTPrefix(), null, null); dateToTtl = dateToTtl.minus(config.getGracePeriodDaysForCompaction(), ChronoUnit.DAYS); logger.info( "Will delete(TTL) SST_V2 files which are before this time: {}. Input: [TTL: {} days, Grace Period: {} days]", dateToTtl, config.getBackupRetentionDays(), config.getGracePeriodDaysForCompaction()); while (remoteFileLocations.hasNext()) { AbstractBackupPath abstractBackupPath = abstractBackupPathProvider.get(); abstractBackupPath.parseRemote(remoteFileLocations.next()); if (abstractBackupPath.getLastModified().isAfter(dateToTtl)) { logger.info( "Breaking from TTL. Got a key which is after the TTL time: {}", abstractBackupPath.getRemotePath()); break; } if (!filesInMeta.containsKey(abstractBackupPath.getRemotePath())) { deleteFile(abstractBackupPath, false); } else { if (logger.isDebugEnabled()) logger.debug( "Not deleting this key as it is referenced in backups: {}", abstractBackupPath.getRemotePath()); } } deleteFile(null, true); logger.info("Finished processing files for TTL service"); } finally { lock.unlock(); } } @Inject  BackupTTLTask(
            IConfiguration configuration,
            IBackupRestoreConfig backupRestoreConfig,
            @Named("v2") IMetaProxy metaProxy,
            IFileSystemContext backupFileSystemCtx,
            Provider<AbstractBackupPath> abstractBackupPathProvider,
            InstanceState instanceState); }
BackupTTLTask extends Task { @Override public void execute() throws Exception { if (instanceState.getRestoreStatus() != null && instanceState.getRestoreStatus().getStatus() != null && instanceState.getRestoreStatus().getStatus() == Status.STARTED) { logger.info("Not executing the TTL Task for backups as Priam is in restore mode."); return; } if (!lock.tryLock()) { logger.warn("{} is already running! Try again later.", JOBNAME); throw new Exception(JOBNAME + " already running"); } try { filesInMeta.clear(); filesToDelete.clear(); Instant dateToTtl = DateUtil.getInstant().minus(config.getBackupRetentionDays(), ChronoUnit.DAYS); List<AbstractBackupPath> metas = metaProxy.findMetaFiles( new DateUtil.DateRange(dateToTtl, DateUtil.getInstant())); if (metas.size() == 0) { logger.info("No meta file found and thus cannot run TTL Service"); return; } AbstractBackupPath metaFile = metas.get(metas.size() - 1); Path localFile = metaProxy.downloadMetaFile(metaFile); new MetaFileWalker().readMeta(localFile); logger.info("No. of component files loaded from meta file: {}", filesInMeta.size()); FileUtils.deleteQuietly(localFile.toFile()); if (filesInMeta.isEmpty()) { logger.warn("Meta file was empty. This should not happen. Getting out!!"); return; } metas = metaProxy.findMetaFiles( new DateUtil.DateRange( start_of_feature, dateToTtl.minus(1, ChronoUnit.HOURS))); if (metas != null && metas.size() != 0) { logger.info( "Will delete(TTL) {} META files starting from: [{}]", metas.size(), metas.get(metas.size() - 1).getLastModified()); for (AbstractBackupPath meta : metas) { deleteFile(meta, false); } } Iterator<String> remoteFileLocations = fileSystem.listFileSystem(getSSTPrefix(), null, null); dateToTtl = dateToTtl.minus(config.getGracePeriodDaysForCompaction(), ChronoUnit.DAYS); logger.info( "Will delete(TTL) SST_V2 files which are before this time: {}. Input: [TTL: {} days, Grace Period: {} days]", dateToTtl, config.getBackupRetentionDays(), config.getGracePeriodDaysForCompaction()); while (remoteFileLocations.hasNext()) { AbstractBackupPath abstractBackupPath = abstractBackupPathProvider.get(); abstractBackupPath.parseRemote(remoteFileLocations.next()); if (abstractBackupPath.getLastModified().isAfter(dateToTtl)) { logger.info( "Breaking from TTL. Got a key which is after the TTL time: {}", abstractBackupPath.getRemotePath()); break; } if (!filesInMeta.containsKey(abstractBackupPath.getRemotePath())) { deleteFile(abstractBackupPath, false); } else { if (logger.isDebugEnabled()) logger.debug( "Not deleting this key as it is referenced in backups: {}", abstractBackupPath.getRemotePath()); } } deleteFile(null, true); logger.info("Finished processing files for TTL service"); } finally { lock.unlock(); } } @Inject  BackupTTLTask(
            IConfiguration configuration,
            IBackupRestoreConfig backupRestoreConfig,
            @Named("v2") IMetaProxy metaProxy,
            IFileSystemContext backupFileSystemCtx,
            Provider<AbstractBackupPath> abstractBackupPathProvider,
            InstanceState instanceState); @Override void execute(); @Override String getName(); static TaskTimer getTimer(IBackupRestoreConfig backupRestoreConfig); }
BackupTTLTask extends Task { @Override public void execute() throws Exception { if (instanceState.getRestoreStatus() != null && instanceState.getRestoreStatus().getStatus() != null && instanceState.getRestoreStatus().getStatus() == Status.STARTED) { logger.info("Not executing the TTL Task for backups as Priam is in restore mode."); return; } if (!lock.tryLock()) { logger.warn("{} is already running! Try again later.", JOBNAME); throw new Exception(JOBNAME + " already running"); } try { filesInMeta.clear(); filesToDelete.clear(); Instant dateToTtl = DateUtil.getInstant().minus(config.getBackupRetentionDays(), ChronoUnit.DAYS); List<AbstractBackupPath> metas = metaProxy.findMetaFiles( new DateUtil.DateRange(dateToTtl, DateUtil.getInstant())); if (metas.size() == 0) { logger.info("No meta file found and thus cannot run TTL Service"); return; } AbstractBackupPath metaFile = metas.get(metas.size() - 1); Path localFile = metaProxy.downloadMetaFile(metaFile); new MetaFileWalker().readMeta(localFile); logger.info("No. of component files loaded from meta file: {}", filesInMeta.size()); FileUtils.deleteQuietly(localFile.toFile()); if (filesInMeta.isEmpty()) { logger.warn("Meta file was empty. This should not happen. Getting out!!"); return; } metas = metaProxy.findMetaFiles( new DateUtil.DateRange( start_of_feature, dateToTtl.minus(1, ChronoUnit.HOURS))); if (metas != null && metas.size() != 0) { logger.info( "Will delete(TTL) {} META files starting from: [{}]", metas.size(), metas.get(metas.size() - 1).getLastModified()); for (AbstractBackupPath meta : metas) { deleteFile(meta, false); } } Iterator<String> remoteFileLocations = fileSystem.listFileSystem(getSSTPrefix(), null, null); dateToTtl = dateToTtl.minus(config.getGracePeriodDaysForCompaction(), ChronoUnit.DAYS); logger.info( "Will delete(TTL) SST_V2 files which are before this time: {}. Input: [TTL: {} days, Grace Period: {} days]", dateToTtl, config.getBackupRetentionDays(), config.getGracePeriodDaysForCompaction()); while (remoteFileLocations.hasNext()) { AbstractBackupPath abstractBackupPath = abstractBackupPathProvider.get(); abstractBackupPath.parseRemote(remoteFileLocations.next()); if (abstractBackupPath.getLastModified().isAfter(dateToTtl)) { logger.info( "Breaking from TTL. Got a key which is after the TTL time: {}", abstractBackupPath.getRemotePath()); break; } if (!filesInMeta.containsKey(abstractBackupPath.getRemotePath())) { deleteFile(abstractBackupPath, false); } else { if (logger.isDebugEnabled()) logger.debug( "Not deleting this key as it is referenced in backups: {}", abstractBackupPath.getRemotePath()); } } deleteFile(null, true); logger.info("Finished processing files for TTL service"); } finally { lock.unlock(); } } @Inject  BackupTTLTask(
            IConfiguration configuration,
            IBackupRestoreConfig backupRestoreConfig,
            @Named("v2") IMetaProxy metaProxy,
            IFileSystemContext backupFileSystemCtx,
            Provider<AbstractBackupPath> abstractBackupPathProvider,
            InstanceState instanceState); @Override void execute(); @Override String getName(); static TaskTimer getTimer(IBackupRestoreConfig backupRestoreConfig); static final String JOBNAME; }