DataUpdateManager { public void expireSegments(File collectionDir, DateTime minTime, DateTime maxTime, String schedule) throws IOException { final String dataSchedule = schedule; File[] higherDataDirs = collectionDir.listFiles(new FilenameFilter() { @Override public boolean accept(File dir, String name) { return name.startsWith(StorageUtils.getDataDirPrefix()) && StorageUtils.getSchedule(name).equals(dataSchedule) && StarTreeConstants.Schedule.valueOf(dataSchedule).getLowerSchedule() != null; } }); for (File higherDataDir : higherDataDirs) { String lowerSchedule = StarTreeConstants.Schedule.valueOf(StorageUtils.getSchedule(higherDataDir.getName())).getLowerSchedule(); DateTime maxDateTime = StorageUtils.getMaxTime(higherDataDir.getName()); DateTime startDateTime = StorageUtils.getMinTime(higherDataDir.getName()); while (startDateTime.compareTo(maxDateTime) < 0) { DateTime endDateTime = StarTreeConstants.Schedule.valueOf(lowerSchedule).getEndDateTime(startDateTime); final String lowerDir = StorageUtils.getDataDirPrefix(lowerSchedule, startDateTime, endDateTime); File[] lowerDataDir = collectionDir.listFiles(new FileFilter() { @Override public boolean accept(File pathname) { return StorageUtils.isExpirable(pathname, lowerDir); } }); for (File expireDataDir : lowerDataDir) { LOGGER.info("Deleting segment {}", expireDataDir); FileUtils.deleteDirectory(expireDataDir); } startDateTime = endDateTime; } } } DataUpdateManager(File rootDir, boolean autoExpire); void deleteCollection(String collection); void deleteData(String collection,
                         String schedule,
                         DateTime minTime,
                         DateTime maxTime); void deleteData(String collection, String schedule, DateTime maxTime); void updateData(String collection,
                         String schedule,
                         DateTime minTime,
                         DateTime maxTime,
                         byte[] data); void updateData(String collection,
                         String schedule,
                         DateTime minTime,
                         DateTime maxTime,
                         InputStream data); void expireSegments(File collectionDir, DateTime minTime, DateTime maxTime, String schedule); void persistTree(String collection,
                          String schedule,
                          DateTime minTime,
                          DateTime maxTime,
                          final StarTree starTree); StarTree mergeTrees(Collection<StarTree> starTrees);  }
@Test public void testExpireSegments() throws Exception { dataUpdateManager = new DataUpdateManager(rootDir, true); File collectionDir = new File(rootDir, collection); FileUtils.forceMkdir(collectionDir); DateTime[][] dataDirs = new DateTime[6][2]; dataDirs[0] = new DateTime[]{StarTreeConstants.DATE_TIME_FORMATTER.parseDateTime("2015-06-01-000000"), StarTreeConstants.DATE_TIME_FORMATTER.parseDateTime("2015-06-01-010000")}; dataDirs[1] = new DateTime[]{StarTreeConstants.DATE_TIME_FORMATTER.parseDateTime("2015-05-31-000000"), StarTreeConstants.DATE_TIME_FORMATTER.parseDateTime("2015-05-31-010000")}; dataDirs[2] = new DateTime[]{StarTreeConstants.DATE_TIME_FORMATTER.parseDateTime("2015-05-31-230000"), StarTreeConstants.DATE_TIME_FORMATTER.parseDateTime("2015-05-31-000000")}; dataDirs[3] = new DateTime[]{StarTreeConstants.DATE_TIME_FORMATTER.parseDateTime("2015-05-31-000000"), StarTreeConstants.DATE_TIME_FORMATTER.parseDateTime("2015-06-01-000000")}; dataDirs[4] = new DateTime[]{StarTreeConstants.DATE_TIME_FORMATTER.parseDateTime("2015-06-01-000000"), StarTreeConstants.DATE_TIME_FORMATTER.parseDateTime("2015-06-02-000000")}; dataDirs[5] = new DateTime[]{StarTreeConstants.DATE_TIME_FORMATTER.parseDateTime("2015-05-01-000000"), StarTreeConstants.DATE_TIME_FORMATTER.parseDateTime("2015-06-01-000000")}; File dataDirHourly1 = new File(collectionDir, StorageUtils.getDataDirName(treeId, StarTreeConstants.Schedule.HOURLY.name(), dataDirs[0][0], dataDirs[0][1])); FileUtils.forceMkdir(dataDirHourly1); File dataDirHourly2 = new File(collectionDir, StorageUtils.getDataDirName(treeId, StarTreeConstants.Schedule.HOURLY.name(), dataDirs[1][0], dataDirs[1][1])); FileUtils.forceMkdir(dataDirHourly2); File dataDirHourly3 = new File(collectionDir, StorageUtils.getDataDirName(treeId, StarTreeConstants.Schedule.HOURLY.name(), dataDirs[2][0], dataDirs[2][1])); FileUtils.forceMkdir(dataDirHourly3); Assert.assertTrue(dataDirHourly1.exists()); Assert.assertTrue(dataDirHourly2.exists()); Assert.assertTrue(dataDirHourly3.exists()); File dataDirDaily1 = new File(collectionDir, StorageUtils.getDataDirName(treeId, StarTreeConstants.Schedule.DAILY.name(), dataDirs[3][0], dataDirs[3][1])); FileUtils.forceMkdir(dataDirDaily1); dataUpdateManager.expireSegments(collectionDir, dataDirs[3][0], dataDirs[3][1], StarTreeConstants.Schedule.DAILY.name()); Assert.assertTrue(dataDirHourly1.exists()); Assert.assertTrue(dataDirHourly2.exists()); Assert.assertTrue(dataDirHourly3.exists()); Assert.assertTrue(dataDirDaily1.exists()); dataDirHourly1.setLastModified(dataDirHourly1.lastModified() - TimeUnit.MILLISECONDS.convert(6, TimeUnit.MINUTES)); dataDirHourly2.setLastModified(dataDirHourly2.lastModified() - TimeUnit.MILLISECONDS.convert(6, TimeUnit.MINUTES)); dataUpdateManager.expireSegments(collectionDir, dataDirs[3][0], dataDirs[3][1], StarTreeConstants.Schedule.DAILY.name()); Assert.assertTrue(dataDirHourly1.exists()); Assert.assertFalse(dataDirHourly2.exists()); Assert.assertTrue(dataDirHourly3.exists()); Assert.assertTrue(dataDirDaily1.exists()); File dataDirDaily2 = new File(collectionDir, StorageUtils.getDataDirName(treeId, StarTreeConstants.Schedule.DAILY.name(), dataDirs[4][0], dataDirs[4][1])); FileUtils.forceMkdir(dataDirDaily2); File dataDirMonthly = new File(collectionDir, StorageUtils.getDataDirName(treeId, StarTreeConstants.Schedule.MONTHLY.name(), dataDirs[5][0], dataDirs[5][1])); FileUtils.forceMkdir(dataDirMonthly); dataUpdateManager.expireSegments(collectionDir, dataDirs[5][0], dataDirs[5][1], StarTreeConstants.Schedule.MONTHLY.name()); Assert.assertTrue(dataDirHourly1.exists()); Assert.assertTrue(dataDirHourly3.exists()); Assert.assertTrue(dataDirDaily1.exists()); Assert.assertTrue(dataDirDaily2.exists()); Assert.assertTrue(dataDirMonthly.exists()); dataDirDaily1.setLastModified(dataDirDaily1.lastModified() - TimeUnit.MILLISECONDS.convert(6, TimeUnit.MINUTES)); dataDirDaily2.setLastModified(dataDirDaily2.lastModified() - TimeUnit.MILLISECONDS.convert(6, TimeUnit.MINUTES)); dataUpdateManager.expireSegments(collectionDir, dataDirs[5][0], dataDirs[5][1], StarTreeConstants.Schedule.MONTHLY.name()); Assert.assertTrue(dataDirHourly1.exists()); Assert.assertTrue(dataDirHourly3.exists()); Assert.assertFalse(dataDirDaily1.exists()); Assert.assertTrue(dataDirDaily2.exists()); Assert.assertTrue(dataDirMonthly.exists()); FileUtils.forceDelete(dataDirHourly1); FileUtils.forceDelete(dataDirHourly3); FileUtils.forceDelete(dataDirDaily2); FileUtils.forceDelete(dataDirMonthly); }