JobHistoryUtils { public static List<FileStatus> getHistoryDirsForCleaning(FileContext fc, Path root, long cutoff) throws IOException { List<FileStatus> fsList = new ArrayList<FileStatus>(); Calendar cCal = Calendar.getInstance(); cCal.setTimeInMillis(cutoff); int cYear = cCal.get(Calendar.YEAR); int cMonth = cCal.get(Calendar.MONTH) + 1; int cDate = cCal.get(Calendar.DATE); RemoteIterator<FileStatus> yearDirIt = fc.listStatus(root); while (yearDirIt.hasNext()) { FileStatus yearDir = yearDirIt.next(); try { int year = Integer.parseInt(yearDir.getPath().getName()); if (year <= cYear) { RemoteIterator<FileStatus> monthDirIt = fc.listStatus(yearDir.getPath()); while (monthDirIt.hasNext()) { FileStatus monthDir = monthDirIt.next(); try { int month = Integer.parseInt(monthDir.getPath().getName()); if (year < cYear || month <= cMonth) { RemoteIterator<FileStatus> dateDirIt = fc.listStatus(monthDir.getPath()); while (dateDirIt.hasNext()) { FileStatus dateDir = dateDirIt.next(); try { int date = Integer.parseInt(dateDir.getPath().getName()); if (year < cYear || month < cMonth || date <= cDate) { fsList.addAll(remoteIterToList( fc.listStatus(dateDir.getPath()))); } } catch (NumberFormatException nfe) { } } } } catch (NumberFormatException nfe) { } } } } catch (NumberFormatException nfe) { } } return fsList; }  static boolean isValidJobHistoryFileName(String pathString); static JobID getJobIDFromHistoryFilePath(String pathString); static PathFilter getConfFileFilter(); static PathFilter getHistoryFileFilter(); static String getConfiguredHistoryStagingDirPrefix(Configuration conf, String jobId); static String getConfiguredHistoryIntermediateDoneDirPrefix(
      Configuration conf); static FsPermission getConfiguredHistoryIntermediateUserDoneDirPermissions(
            Configuration conf); static String getConfiguredHistoryServerDoneDirPrefix(
      Configuration conf); static String getHistoryIntermediateDoneDirForUser(Configuration conf); static boolean shouldCreateNonUserDirectory(Configuration conf); static Path getStagingJobHistoryFile(Path dir, JobId jobId, int attempt); static Path getStagingJobHistoryFile(Path dir, String jobId, int attempt); static String getIntermediateConfFileName(JobId jobId); static String getIntermediateSummaryFileName(JobId jobId); static Path getStagingConfFile(Path logDir, JobId jobId, int attempt); static String serialNumberDirectoryComponent(JobId id, String serialNumberFormat); static String getTimestampPartFromPath(String path); static String historyLogSubdirectory(JobId id, String timestampComponent, String serialNumberFormat); static String timestampDirectoryComponent(long millisecondTime); static String doneSubdirsBeforeSerialTail(); static int jobSerialNumber(JobId id); static List<FileStatus> localGlobber(FileContext fc, Path root, String tail); static List<FileStatus> localGlobber(FileContext fc, Path root, String tail,
      PathFilter filter); static List<FileStatus> localGlobber(FileContext fc, Path root, String tail,
      PathFilter filter, AtomicBoolean hasFlatFiles); static Path getPreviousJobHistoryPath(
      Configuration conf, ApplicationAttemptId applicationAttemptId); static List<FileStatus> getHistoryDirsForCleaning(FileContext fc,
      Path root, long cutoff); static final FsPermission HISTORY_STAGING_DIR_PERMISSIONS; static final FsPermission HISTORY_STAGING_USER_DIR_PERMISSIONS; static final FsPermission HISTORY_DONE_DIR_PERMISSION; static final FsPermission HISTORY_DONE_FILE_PERMISSION; static final FsPermission HISTORY_DONE_DIR_UMASK; static final FsPermission HISTORY_INTERMEDIATE_DONE_DIR_PERMISSIONS; static final FsPermission HISTORY_INTERMEDIATE_FILE_PERMISSIONS; static final String CONF_FILE_NAME_SUFFIX; static final String SUMMARY_FILE_NAME_SUFFIX; static final String JOB_HISTORY_FILE_EXTENSION; static final int VERSION; static final int SERIAL_NUMBER_DIRECTORY_DIGITS; static final String TIMESTAMP_DIR_REGEX; static final Pattern TIMESTAMP_DIR_PATTERN; }
@Test @SuppressWarnings("unchecked") public void testGetHistoryDirsForCleaning() throws IOException { Path pRoot = new Path(TEST_DIR, "org.apache.hadoop.mapreduce.v2.jobhistory." + "TestJobHistoryUtils.testGetHistoryDirsForCleaning"); FileContext fc = FileContext.getFileContext(); Calendar cCal = Calendar.getInstance(); int year = 2013; int month = 7; int day = 21; cCal.set(year, month - 1, day, 1, 0); long cutoff = cCal.getTimeInMillis(); clearDir(fc, pRoot); Path pId00 = createPath(fc, pRoot, year, month, day, "000000"); Path pId01 = createPath(fc, pRoot, year, month, day+1, "000001"); Path pId02 = createPath(fc, pRoot, year, month, day-1, "000002"); Path pId03 = createPath(fc, pRoot, year, month+1, day, "000003"); Path pId04 = createPath(fc, pRoot, year, month+1, day+1, "000004"); Path pId05 = createPath(fc, pRoot, year, month+1, day-1, "000005"); Path pId06 = createPath(fc, pRoot, year, month-1, day, "000006"); Path pId07 = createPath(fc, pRoot, year, month-1, day+1, "000007"); Path pId08 = createPath(fc, pRoot, year, month-1, day-1, "000008"); Path pId09 = createPath(fc, pRoot, year+1, month, day, "000009"); Path pId10 = createPath(fc, pRoot, year+1, month, day+1, "000010"); Path pId11 = createPath(fc, pRoot, year+1, month, day-1, "000011"); Path pId12 = createPath(fc, pRoot, year+1, month+1, day, "000012"); Path pId13 = createPath(fc, pRoot, year+1, month+1, day+1, "000013"); Path pId14 = createPath(fc, pRoot, year+1, month+1, day-1, "000014"); Path pId15 = createPath(fc, pRoot, year+1, month-1, day, "000015"); Path pId16 = createPath(fc, pRoot, year+1, month-1, day+1, "000016"); Path pId17 = createPath(fc, pRoot, year+1, month-1, day-1, "000017"); Path pId18 = createPath(fc, pRoot, year-1, month, day, "000018"); Path pId19 = createPath(fc, pRoot, year-1, month, day+1, "000019"); Path pId20 = createPath(fc, pRoot, year-1, month, day-1, "000020"); Path pId21 = createPath(fc, pRoot, year-1, month+1, day, "000021"); Path pId22 = createPath(fc, pRoot, year-1, month+1, day+1, "000022"); Path pId23 = createPath(fc, pRoot, year-1, month+1, day-1, "000023"); Path pId24 = createPath(fc, pRoot, year-1, month-1, day, "000024"); Path pId25 = createPath(fc, pRoot, year-1, month-1, day+1, "000025"); Path pId26 = createPath(fc, pRoot, year-1, month-1, day-1, "000026"); Path pId27 = createPath(fc, pRoot, "foo", "" + month, "" + day, "000027"); Path pId28 = createPath(fc, pRoot, "" + year, "foo", "" + day, "000028"); Path pId29 = createPath(fc, pRoot, "" + year, "" + month, "foo", "000029"); List<FileStatus> dirs = JobHistoryUtils .getHistoryDirsForCleaning(fc, pRoot, cutoff); Collections.sort(dirs); Assert.assertEquals(14, dirs.size()); Assert.assertEquals(pId26.toUri().getPath(), dirs.get(0).getPath().toUri().getPath()); Assert.assertEquals(pId24.toUri().getPath(), dirs.get(1).getPath().toUri().getPath()); Assert.assertEquals(pId25.toUri().getPath(), dirs.get(2).getPath().toUri().getPath()); Assert.assertEquals(pId20.toUri().getPath(), dirs.get(3).getPath().toUri().getPath()); Assert.assertEquals(pId18.toUri().getPath(), dirs.get(4).getPath().toUri().getPath()); Assert.assertEquals(pId19.toUri().getPath(), dirs.get(5).getPath().toUri().getPath()); Assert.assertEquals(pId23.toUri().getPath(), dirs.get(6).getPath().toUri().getPath()); Assert.assertEquals(pId21.toUri().getPath(), dirs.get(7).getPath().toUri().getPath()); Assert.assertEquals(pId22.toUri().getPath(), dirs.get(8).getPath().toUri().getPath()); Assert.assertEquals(pId08.toUri().getPath(), dirs.get(9).getPath().toUri().getPath()); Assert.assertEquals(pId06.toUri().getPath(), dirs.get(10).getPath().toUri().getPath()); Assert.assertEquals(pId07.toUri().getPath(), dirs.get(11).getPath().toUri().getPath()); Assert.assertEquals(pId02.toUri().getPath(), dirs.get(12).getPath().toUri().getPath()); Assert.assertEquals(pId00.toUri().getPath(), dirs.get(13).getPath().toUri().getPath()); }