@SuppressWarnings("deprecation") @Test (timeout = 30000) public void testSetupDistributedCache() throws Exception { Configuration conf = new Configuration(); conf.setClass("fs.mockfs.impl", MockFileSystem.class, FileSystem.class); URI mockUri = URI.create("mockfs: FileSystem mockFs = ((FilterFileSystem)FileSystem.get(mockUri, conf)) .getRawFileSystem(); URI archive = new URI("mockfs: Path archivePath = new Path(archive); URI file = new URI("mockfs: Path filePath = new Path(file); when(mockFs.resolvePath(archivePath)).thenReturn(archivePath); when(mockFs.resolvePath(filePath)).thenReturn(filePath); DistributedCache.addCacheArchive(archive, conf); conf.set(MRJobConfig.CACHE_ARCHIVES_TIMESTAMPS, "10"); conf.set(MRJobConfig.CACHE_ARCHIVES_SIZES, "10"); conf.set(MRJobConfig.CACHE_ARCHIVES_VISIBILITIES, "true"); DistributedCache.addCacheFile(file, conf); conf.set(MRJobConfig.CACHE_FILE_TIMESTAMPS, "11"); conf.set(MRJobConfig.CACHE_FILES_SIZES, "11"); conf.set(MRJobConfig.CACHE_FILE_VISIBILITIES, "true"); Map<String, LocalResource> localResources = new HashMap<String, LocalResource>(); MRApps.setupDistributedCache(conf, localResources); assertEquals(2, localResources.size()); LocalResource lr = localResources.get("something.zip"); assertNotNull(lr); assertEquals(10l, lr.getSize()); assertEquals(10l, lr.getTimestamp()); assertEquals(LocalResourceType.ARCHIVE, lr.getType()); lr = localResources.get("something"); assertNotNull(lr); assertEquals(11l, lr.getSize()); assertEquals(11l, lr.getTimestamp()); assertEquals(LocalResourceType.FILE, lr.getType()); }
@SuppressWarnings("deprecation") public static void setupDistributedCache( Configuration conf, Map<String, LocalResource> localResources) throws IOException { parseDistributedCacheArtifacts(conf, localResources, LocalResourceType.ARCHIVE, DistributedCache.getCacheArchives(conf), DistributedCache.getArchiveTimestamps(conf), getFileSizes(conf, MRJobConfig.CACHE_ARCHIVES_SIZES), DistributedCache.getArchiveVisibilities(conf)); parseDistributedCacheArtifacts(conf, localResources, LocalResourceType.FILE, DistributedCache.getCacheFiles(conf), DistributedCache.getFileTimestamps(conf), getFileSizes(conf, MRJobConfig.CACHE_FILES_SIZES), DistributedCache.getFileVisibilities(conf)); }
MRApps extends Apps { @SuppressWarnings("deprecation") public static void setupDistributedCache( Configuration conf, Map<String, LocalResource> localResources) throws IOException { parseDistributedCacheArtifacts(conf, localResources, LocalResourceType.ARCHIVE, DistributedCache.getCacheArchives(conf), DistributedCache.getArchiveTimestamps(conf), getFileSizes(conf, MRJobConfig.CACHE_ARCHIVES_SIZES), DistributedCache.getArchiveVisibilities(conf)); parseDistributedCacheArtifacts(conf, localResources, LocalResourceType.FILE, DistributedCache.getCacheFiles(conf), DistributedCache.getFileTimestamps(conf), getFileSizes(conf, MRJobConfig.CACHE_FILES_SIZES), DistributedCache.getFileVisibilities(conf)); } }
MRApps extends Apps { @SuppressWarnings("deprecation") public static void setupDistributedCache( Configuration conf, Map<String, LocalResource> localResources) throws IOException { parseDistributedCacheArtifacts(conf, localResources, LocalResourceType.ARCHIVE, DistributedCache.getCacheArchives(conf), DistributedCache.getArchiveTimestamps(conf), getFileSizes(conf, MRJobConfig.CACHE_ARCHIVES_SIZES), DistributedCache.getArchiveVisibilities(conf)); parseDistributedCacheArtifacts(conf, localResources, LocalResourceType.FILE, DistributedCache.getCacheFiles(conf), DistributedCache.getFileTimestamps(conf), getFileSizes(conf, MRJobConfig.CACHE_FILES_SIZES), DistributedCache.getFileVisibilities(conf)); }  }
MRApps extends Apps { @SuppressWarnings("deprecation") public static void setupDistributedCache( Configuration conf, Map<String, LocalResource> localResources) throws IOException { parseDistributedCacheArtifacts(conf, localResources, LocalResourceType.ARCHIVE, DistributedCache.getCacheArchives(conf), DistributedCache.getArchiveTimestamps(conf), getFileSizes(conf, MRJobConfig.CACHE_ARCHIVES_SIZES), DistributedCache.getArchiveVisibilities(conf)); parseDistributedCacheArtifacts(conf, localResources, LocalResourceType.FILE, DistributedCache.getCacheFiles(conf), DistributedCache.getFileTimestamps(conf), getFileSizes(conf, MRJobConfig.CACHE_FILES_SIZES), DistributedCache.getFileVisibilities(conf)); }  static String toString(JobId jid); static JobId toJobID(String jid); static String toString(TaskId tid); static TaskId toTaskID(String tid); static String toString(TaskAttemptId taid); static TaskAttemptId toTaskAttemptID(String taid); static String taskSymbol(TaskType type); static TaskType taskType(String symbol); static TaskAttemptStateUI taskAttemptState(String attemptStateStr); static TaskStateUI taskState(String taskStateStr); @SuppressWarnings("deprecation") static void setClasspath(Map<String, String> environment,
      Configuration conf); @SuppressWarnings("deprecation") static void addClasspathToEnv(Map<String, String> environment,
      String classpathEnvVar, Configuration conf); static void setJobClassLoader(Configuration conf); static ClassLoader createJobClassLoader(Configuration conf); static void setClassLoader(ClassLoader classLoader,
      Configuration conf); static Path getStagingAreaDir(Configuration conf, String user); static String getJobFile(Configuration conf, String user, 
      org.apache.hadoop.mapreduce.JobID jobId); static Path getEndJobCommitSuccessFile(Configuration conf, String user,
      JobId jobId); static Path getEndJobCommitFailureFile(Configuration conf, String user,
      JobId jobId); static Path getStartJobCommitFile(Configuration conf, String user,
      JobId jobId); @SuppressWarnings("deprecation") static void setupDistributedCache( 
      Configuration conf, 
      Map<String, LocalResource> localResources); @SuppressWarnings("deprecation") static void setupDistributedCacheLocal(Configuration conf); static void addLog4jSystemProperties(
      String logLevel, long logSize, int numBackups, List<String> vargs, 
      Configuration conf); static void setEnvFromInputString(Map<String, String> env,
      String envString, Configuration conf); @Public @Unstable static void addToEnvironment(Map<String, String> environment,
      String variable, String value, Configuration conf); static String crossPlatformifyMREnv(Configuration conf, Environment env); }
MRApps extends Apps { @SuppressWarnings("deprecation") public static void setupDistributedCache( Configuration conf, Map<String, LocalResource> localResources) throws IOException { parseDistributedCacheArtifacts(conf, localResources, LocalResourceType.ARCHIVE, DistributedCache.getCacheArchives(conf), DistributedCache.getArchiveTimestamps(conf), getFileSizes(conf, MRJobConfig.CACHE_ARCHIVES_SIZES), DistributedCache.getArchiveVisibilities(conf)); parseDistributedCacheArtifacts(conf, localResources, LocalResourceType.FILE, DistributedCache.getCacheFiles(conf), DistributedCache.getFileTimestamps(conf), getFileSizes(conf, MRJobConfig.CACHE_FILES_SIZES), DistributedCache.getFileVisibilities(conf)); }  static String toString(JobId jid); static JobId toJobID(String jid); static String toString(TaskId tid); static TaskId toTaskID(String tid); static String toString(TaskAttemptId taid); static TaskAttemptId toTaskAttemptID(String taid); static String taskSymbol(TaskType type); static TaskType taskType(String symbol); static TaskAttemptStateUI taskAttemptState(String attemptStateStr); static TaskStateUI taskState(String taskStateStr); @SuppressWarnings("deprecation") static void setClasspath(Map<String, String> environment,
      Configuration conf); @SuppressWarnings("deprecation") static void addClasspathToEnv(Map<String, String> environment,
      String classpathEnvVar, Configuration conf); static void setJobClassLoader(Configuration conf); static ClassLoader createJobClassLoader(Configuration conf); static void setClassLoader(ClassLoader classLoader,
      Configuration conf); static Path getStagingAreaDir(Configuration conf, String user); static String getJobFile(Configuration conf, String user, 
      org.apache.hadoop.mapreduce.JobID jobId); static Path getEndJobCommitSuccessFile(Configuration conf, String user,
      JobId jobId); static Path getEndJobCommitFailureFile(Configuration conf, String user,
      JobId jobId); static Path getStartJobCommitFile(Configuration conf, String user,
      JobId jobId); @SuppressWarnings("deprecation") static void setupDistributedCache( 
      Configuration conf, 
      Map<String, LocalResource> localResources); @SuppressWarnings("deprecation") static void setupDistributedCacheLocal(Configuration conf); static void addLog4jSystemProperties(
      String logLevel, long logSize, int numBackups, List<String> vargs, 
      Configuration conf); static void setEnvFromInputString(Map<String, String> env,
      String envString, Configuration conf); @Public @Unstable static void addToEnvironment(Map<String, String> environment,
      String variable, String value, Configuration conf); static String crossPlatformifyMREnv(Configuration conf, Environment env); static final Log LOG; }