@Test public void testPruneFileListBySize() throws IOException { long maxFileSize = 20L; FileStatus[] origList = new FileStatus[2]; FileSystem hdfs = FileSystem.get(UTIL.getConfiguration()); Path inputPath = new Path("/inputdir_filesize"); boolean os = hdfs.mkdirs(inputPath); assertTrue(os); assertTrue(hdfs.exists(inputPath)); final String JOB_HISTORY_FILE_NAME = "src/test/resources/job_1329348432655_0001-1329348443227-user-Sleep+job-1329348468601-10-1-SUCCEEDED-default.jhist"; File jobHistoryfile = new File(JOB_HISTORY_FILE_NAME); Path srcPath = new Path(jobHistoryfile.toURI()); hdfs.copyFromLocalFile(srcPath, inputPath); Path expPath = new Path(inputPath.toUri() + "/" + srcPath.getName()); assertTrue(hdfs.exists(expPath)); origList[0] = hdfs.getFileStatus(expPath); final String JOB_CONF_FILE_NAME = "src/test/resources/job_1329348432655_0001_conf.xml"; File jobConfFile = new File(JOB_CONF_FILE_NAME); srcPath = new Path(jobConfFile.toURI()); hdfs.copyFromLocalFile(srcPath, inputPath); expPath = new Path(inputPath.toUri() + "/" + srcPath.getName()); assertTrue(hdfs.exists(expPath)); origList[1] = hdfs.getFileStatus(expPath); FileStatus [] prunedList = FileLister.pruneFileListBySize(maxFileSize, origList, hdfs, inputPath); assertNotNull(prunedList); assertTrue(prunedList.length == 0); Path emptyFile = new Path(inputPath.toUri() + "/" + "job_1329341111111_0101-1329111113227-user2-Sleep.jhist"); os = hdfs.createNewFile(emptyFile); assertTrue(os); assertTrue(hdfs.exists(emptyFile)); origList[0] = hdfs.getFileStatus(emptyFile); Path emptyConfFile = new Path(inputPath.toUri() + "/" + "job_1329341111111_0101_conf.xml"); os = hdfs.createNewFile(emptyConfFile); assertTrue(os); assertTrue(hdfs.exists(emptyConfFile)); origList[1] = hdfs.getFileStatus(emptyConfFile); prunedList = FileLister.pruneFileListBySize(maxFileSize, origList, hdfs, inputPath); assertNotNull(prunedList); assertTrue(prunedList.length == 2); }
static FileStatus[] pruneFileListBySize(long maxFileSize, FileStatus[] origList, FileSystem hdfs, Path inputPath) { LOG.info("Pruning orig list of size " + origList.length + " for source" + inputPath.toUri()); long fileSize = 0L; List<FileStatus> prunedFileList = new ArrayList<FileStatus>(); Set<String> toBeRemovedJobId = new HashSet<String>(); for (int i = 0; i < origList.length; i++) { fileSize = origList[i].getLen(); if (fileSize <= maxFileSize) { prunedFileList.add(origList[i]); } else { Path hugeFile = origList[i].getPath(); LOG.info("In getListFilesToProcess filesize " + fileSize + " has exceeded maxFileSize " + maxFileSize + " for " + hugeFile.toUri()); toBeRemovedJobId.add(getJobIdFromPath(hugeFile)); } } if (prunedFileList.size() == 0) { LOG.info("Found no files worth processing. Returning 0 sized array"); return new FileStatus[0]; } String jobId = null; ListIterator<FileStatus> it = prunedFileList.listIterator(); while (it.hasNext()) { if (toBeRemovedJobId.size() == 0) { break; } Path curFile = it.next().getPath(); jobId = getJobIdFromPath(curFile); if (toBeRemovedJobId.contains(jobId)) { LOG.info("Removing from prunedList " + curFile.toUri()); it.remove(); toBeRemovedJobId.remove(jobId); } } return prunedFileList.toArray(new FileStatus[prunedFileList.size()]); }
FileLister { static FileStatus[] pruneFileListBySize(long maxFileSize, FileStatus[] origList, FileSystem hdfs, Path inputPath) { LOG.info("Pruning orig list of size " + origList.length + " for source" + inputPath.toUri()); long fileSize = 0L; List<FileStatus> prunedFileList = new ArrayList<FileStatus>(); Set<String> toBeRemovedJobId = new HashSet<String>(); for (int i = 0; i < origList.length; i++) { fileSize = origList[i].getLen(); if (fileSize <= maxFileSize) { prunedFileList.add(origList[i]); } else { Path hugeFile = origList[i].getPath(); LOG.info("In getListFilesToProcess filesize " + fileSize + " has exceeded maxFileSize " + maxFileSize + " for " + hugeFile.toUri()); toBeRemovedJobId.add(getJobIdFromPath(hugeFile)); } } if (prunedFileList.size() == 0) { LOG.info("Found no files worth processing. Returning 0 sized array"); return new FileStatus[0]; } String jobId = null; ListIterator<FileStatus> it = prunedFileList.listIterator(); while (it.hasNext()) { if (toBeRemovedJobId.size() == 0) { break; } Path curFile = it.next().getPath(); jobId = getJobIdFromPath(curFile); if (toBeRemovedJobId.contains(jobId)) { LOG.info("Removing from prunedList " + curFile.toUri()); it.remove(); toBeRemovedJobId.remove(jobId); } } return prunedFileList.toArray(new FileStatus[prunedFileList.size()]); } }
FileLister { static FileStatus[] pruneFileListBySize(long maxFileSize, FileStatus[] origList, FileSystem hdfs, Path inputPath) { LOG.info("Pruning orig list of size " + origList.length + " for source" + inputPath.toUri()); long fileSize = 0L; List<FileStatus> prunedFileList = new ArrayList<FileStatus>(); Set<String> toBeRemovedJobId = new HashSet<String>(); for (int i = 0; i < origList.length; i++) { fileSize = origList[i].getLen(); if (fileSize <= maxFileSize) { prunedFileList.add(origList[i]); } else { Path hugeFile = origList[i].getPath(); LOG.info("In getListFilesToProcess filesize " + fileSize + " has exceeded maxFileSize " + maxFileSize + " for " + hugeFile.toUri()); toBeRemovedJobId.add(getJobIdFromPath(hugeFile)); } } if (prunedFileList.size() == 0) { LOG.info("Found no files worth processing. Returning 0 sized array"); return new FileStatus[0]; } String jobId = null; ListIterator<FileStatus> it = prunedFileList.listIterator(); while (it.hasNext()) { if (toBeRemovedJobId.size() == 0) { break; } Path curFile = it.next().getPath(); jobId = getJobIdFromPath(curFile); if (toBeRemovedJobId.contains(jobId)) { LOG.info("Removing from prunedList " + curFile.toUri()); it.remove(); toBeRemovedJobId.remove(jobId); } } return prunedFileList.toArray(new FileStatus[prunedFileList.size()]); } FileLister(); }
FileLister { static FileStatus[] pruneFileListBySize(long maxFileSize, FileStatus[] origList, FileSystem hdfs, Path inputPath) { LOG.info("Pruning orig list of size " + origList.length + " for source" + inputPath.toUri()); long fileSize = 0L; List<FileStatus> prunedFileList = new ArrayList<FileStatus>(); Set<String> toBeRemovedJobId = new HashSet<String>(); for (int i = 0; i < origList.length; i++) { fileSize = origList[i].getLen(); if (fileSize <= maxFileSize) { prunedFileList.add(origList[i]); } else { Path hugeFile = origList[i].getPath(); LOG.info("In getListFilesToProcess filesize " + fileSize + " has exceeded maxFileSize " + maxFileSize + " for " + hugeFile.toUri()); toBeRemovedJobId.add(getJobIdFromPath(hugeFile)); } } if (prunedFileList.size() == 0) { LOG.info("Found no files worth processing. Returning 0 sized array"); return new FileStatus[0]; } String jobId = null; ListIterator<FileStatus> it = prunedFileList.listIterator(); while (it.hasNext()) { if (toBeRemovedJobId.size() == 0) { break; } Path curFile = it.next().getPath(); jobId = getJobIdFromPath(curFile); if (toBeRemovedJobId.contains(jobId)) { LOG.info("Removing from prunedList " + curFile.toUri()); it.remove(); toBeRemovedJobId.remove(jobId); } } return prunedFileList.toArray(new FileStatus[prunedFileList.size()]); } FileLister(); static FileStatus[] listFiles(boolean recurse, FileSystem hdfs, Path inputPath,
      JobFileModifiedRangePathFilter jobFileModifiedRangePathFilter); static FileStatus[] getListFilesToProcess(long maxFileSize, boolean recurse,
      FileSystem hdfs, Path inputPath, JobFileModifiedRangePathFilter pathFilter); }
FileLister { static FileStatus[] pruneFileListBySize(long maxFileSize, FileStatus[] origList, FileSystem hdfs, Path inputPath) { LOG.info("Pruning orig list of size " + origList.length + " for source" + inputPath.toUri()); long fileSize = 0L; List<FileStatus> prunedFileList = new ArrayList<FileStatus>(); Set<String> toBeRemovedJobId = new HashSet<String>(); for (int i = 0; i < origList.length; i++) { fileSize = origList[i].getLen(); if (fileSize <= maxFileSize) { prunedFileList.add(origList[i]); } else { Path hugeFile = origList[i].getPath(); LOG.info("In getListFilesToProcess filesize " + fileSize + " has exceeded maxFileSize " + maxFileSize + " for " + hugeFile.toUri()); toBeRemovedJobId.add(getJobIdFromPath(hugeFile)); } } if (prunedFileList.size() == 0) { LOG.info("Found no files worth processing. Returning 0 sized array"); return new FileStatus[0]; } String jobId = null; ListIterator<FileStatus> it = prunedFileList.listIterator(); while (it.hasNext()) { if (toBeRemovedJobId.size() == 0) { break; } Path curFile = it.next().getPath(); jobId = getJobIdFromPath(curFile); if (toBeRemovedJobId.contains(jobId)) { LOG.info("Removing from prunedList " + curFile.toUri()); it.remove(); toBeRemovedJobId.remove(jobId); } } return prunedFileList.toArray(new FileStatus[prunedFileList.size()]); } FileLister(); static FileStatus[] listFiles(boolean recurse, FileSystem hdfs, Path inputPath,
      JobFileModifiedRangePathFilter jobFileModifiedRangePathFilter); static FileStatus[] getListFilesToProcess(long maxFileSize, boolean recurse,
      FileSystem hdfs, Path inputPath, JobFileModifiedRangePathFilter pathFilter);  }