MRApps extends Apps { @SuppressWarnings("deprecation") public static void setClasspath(Map<String, String> environment, Configuration conf) throws IOException { boolean userClassesTakesPrecedence = conf.getBoolean(MRJobConfig.MAPREDUCE_JOB_USER_CLASSPATH_FIRST, false); String classpathEnvVar = conf.getBoolean(MRJobConfig.MAPREDUCE_JOB_CLASSLOADER, false) ? Environment.APP_CLASSPATH.name() : Environment.CLASSPATH.name(); MRApps.addToEnvironment(environment, classpathEnvVar, crossPlatformifyMREnv(conf, Environment.PWD), conf); if (!userClassesTakesPrecedence) { MRApps.setMRFrameworkClasspath(environment, conf); } MRApps.addToEnvironment( environment, classpathEnvVar, MRJobConfig.JOB_JAR + Path.SEPARATOR + MRJobConfig.JOB_JAR, conf); MRApps.addToEnvironment( environment, classpathEnvVar, MRJobConfig.JOB_JAR + Path.SEPARATOR + "classes" + Path.SEPARATOR, conf); MRApps.addToEnvironment( environment, classpathEnvVar, MRJobConfig.JOB_JAR + Path.SEPARATOR + "lib" + Path.SEPARATOR + "*", conf); MRApps.addToEnvironment( environment, classpathEnvVar, crossPlatformifyMREnv(conf, Environment.PWD) + Path.SEPARATOR + "*", conf); addToClasspathIfNotJar(DistributedCache.getFileClassPaths(conf), DistributedCache.getCacheFiles(conf), conf, environment, classpathEnvVar); addToClasspathIfNotJar(DistributedCache.getArchiveClassPaths(conf), DistributedCache.getCacheArchives(conf), conf, environment, classpathEnvVar); if (userClassesTakesPrecedence) { MRApps.setMRFrameworkClasspath(environment, conf); } }  static String toString(JobId jid); static JobId toJobID(String jid); static String toString(TaskId tid); static TaskId toTaskID(String tid); static String toString(TaskAttemptId taid); static TaskAttemptId toTaskAttemptID(String taid); static String taskSymbol(TaskType type); static TaskType taskType(String symbol); static TaskAttemptStateUI taskAttemptState(String attemptStateStr); static TaskStateUI taskState(String taskStateStr); @SuppressWarnings("deprecation") static void setClasspath(Map<String, String> environment,
      Configuration conf); static void setJobClassLoader(Configuration conf); static Path getStagingAreaDir(Configuration conf, String user); static String getJobFile(Configuration conf, String user, 
      org.apache.hadoop.mapreduce.JobID jobId); static Path getEndJobCommitSuccessFile(Configuration conf, String user,
      JobId jobId); static Path getEndJobCommitFailureFile(Configuration conf, String user,
      JobId jobId); static Path getStartJobCommitFile(Configuration conf, String user,
      JobId jobId); static void setupDistributedCache( 
      Configuration conf, 
      Map<String, LocalResource> localResources); static void addLog4jSystemProperties(
      String logLevel, long logSize, int numBackups, List<String> vargs); static void setEnvFromInputString(Map<String, String> env,
      String envString, Configuration conf); @Public @Unstable static void addToEnvironment(Map<String, String> environment,
      String variable, String value, Configuration conf); static String crossPlatformifyMREnv(Configuration conf, Environment env); static final Log LOG; }
@Test (timeout = 120000) public void testSetClasspath() throws IOException { Configuration conf = new Configuration(); conf.setBoolean(MRConfig.MAPREDUCE_APP_SUBMISSION_CROSS_PLATFORM, true); Job job = Job.getInstance(conf); Map<String, String> environment = new HashMap<String, String>(); MRApps.setClasspath(environment, job.getConfiguration()); assertTrue(environment.get("CLASSPATH").startsWith( ApplicationConstants.Environment.PWD.$$() + ApplicationConstants.CLASS_PATH_SEPARATOR)); String yarnAppClasspath = job.getConfiguration().get( YarnConfiguration.YARN_APPLICATION_CLASSPATH, StringUtils.join(",", YarnConfiguration.DEFAULT_YARN_CROSS_PLATFORM_APPLICATION_CLASSPATH)); if (yarnAppClasspath != null) { yarnAppClasspath = yarnAppClasspath.replaceAll(",\\s*", ApplicationConstants.CLASS_PATH_SEPARATOR).trim(); } assertTrue(environment.get("CLASSPATH").contains(yarnAppClasspath)); String mrAppClasspath = job.getConfiguration().get( MRJobConfig.MAPREDUCE_APPLICATION_CLASSPATH, MRJobConfig.DEFAULT_MAPREDUCE_CROSS_PLATFORM_APPLICATION_CLASSPATH); if (mrAppClasspath != null) { mrAppClasspath = mrAppClasspath.replaceAll(",\\s*", ApplicationConstants.CLASS_PATH_SEPARATOR).trim(); } assertTrue(environment.get("CLASSPATH").contains(mrAppClasspath)); }
@Test (timeout = 120000) public void testSetClasspathWithUserPrecendence() { Configuration conf = new Configuration(); conf.setBoolean(MRConfig.MAPREDUCE_APP_SUBMISSION_CROSS_PLATFORM, true); conf.setBoolean(MRJobConfig.MAPREDUCE_JOB_USER_CLASSPATH_FIRST, true); Map<String, String> env = new HashMap<String, String>(); try { MRApps.setClasspath(env, conf); } catch (Exception e) { fail("Got exception while setting classpath"); } String env_str = env.get("CLASSPATH"); String expectedClasspath = StringUtils.join(ApplicationConstants.CLASS_PATH_SEPARATOR, Arrays.asList(ApplicationConstants.Environment.PWD.$$(), "job.jar/job.jar", "job.jar/classes/", "job.jar/lib/*", ApplicationConstants.Environment.PWD.$$() + "/*")); assertTrue("MAPREDUCE_JOB_USER_CLASSPATH_FIRST set, but not taking effect!", env_str.startsWith(expectedClasspath)); }
@Test (timeout = 120000) public void testSetClasspathWithNoUserPrecendence() { Configuration conf = new Configuration(); conf.setBoolean(MRConfig.MAPREDUCE_APP_SUBMISSION_CROSS_PLATFORM, true); conf.setBoolean(MRJobConfig.MAPREDUCE_JOB_USER_CLASSPATH_FIRST, false); Map<String, String> env = new HashMap<String, String>(); try { MRApps.setClasspath(env, conf); } catch (Exception e) { fail("Got exception while setting classpath"); } String env_str = env.get("CLASSPATH"); String expectedClasspath = StringUtils.join(ApplicationConstants.CLASS_PATH_SEPARATOR, Arrays.asList("job.jar/job.jar", "job.jar/classes/", "job.jar/lib/*", ApplicationConstants.Environment.PWD.$$() + "/*")); assertTrue("MAPREDUCE_JOB_USER_CLASSPATH_FIRST false, and job.jar is not in" + " the classpath!", env_str.contains(expectedClasspath)); assertFalse("MAPREDUCE_JOB_USER_CLASSPATH_FIRST false, but taking effect!", env_str.startsWith(expectedClasspath)); }
@Test (timeout = 120000) public void testSetClasspathWithJobClassloader() throws IOException { Configuration conf = new Configuration(); conf.setBoolean(MRConfig.MAPREDUCE_APP_SUBMISSION_CROSS_PLATFORM, true); conf.setBoolean(MRJobConfig.MAPREDUCE_JOB_CLASSLOADER, true); Map<String, String> env = new HashMap<String, String>(); MRApps.setClasspath(env, conf); String cp = env.get("CLASSPATH"); String appCp = env.get("APP_CLASSPATH"); assertFalse("MAPREDUCE_JOB_CLASSLOADER true, but job.jar is in the" + " classpath!", cp.contains("jar" + ApplicationConstants.CLASS_PATH_SEPARATOR + "job")); assertFalse("MAPREDUCE_JOB_CLASSLOADER true, but PWD is in the classpath!", cp.contains("PWD")); String expectedAppClasspath = StringUtils.join(ApplicationConstants.CLASS_PATH_SEPARATOR, Arrays.asList(ApplicationConstants.Environment.PWD.$$(), "job.jar/job.jar", "job.jar/classes/", "job.jar/lib/*", ApplicationConstants.Environment.PWD.$$() + "/*")); assertEquals("MAPREDUCE_JOB_CLASSLOADER true, but job.jar is not in the app" + " classpath!", expectedAppClasspath, appCp); }
@Test (timeout = 3000000) public void testSetClasspathWithFramework() throws IOException { final String FRAMEWORK_NAME = "some-framework-name"; final String FRAMEWORK_PATH = "some-framework-path#" + FRAMEWORK_NAME; Configuration conf = new Configuration(); conf.setBoolean(MRConfig.MAPREDUCE_APP_SUBMISSION_CROSS_PLATFORM, true); conf.set(MRJobConfig.MAPREDUCE_APPLICATION_FRAMEWORK_PATH, FRAMEWORK_PATH); Map<String, String> env = new HashMap<String, String>(); try { MRApps.setClasspath(env, conf); fail("Failed to catch framework path set without classpath change"); } catch (IllegalArgumentException e) { assertTrue("Unexpected IllegalArgumentException", e.getMessage().contains("Could not locate MapReduce framework name '" + FRAMEWORK_NAME + "'")); } env.clear(); final String FRAMEWORK_CLASSPATH = FRAMEWORK_NAME + "/*.jar"; conf.set(MRJobConfig.MAPREDUCE_APPLICATION_CLASSPATH, FRAMEWORK_CLASSPATH); MRApps.setClasspath(env, conf); final String stdClasspath = StringUtils.join(ApplicationConstants.CLASS_PATH_SEPARATOR, Arrays.asList("job.jar/job.jar", "job.jar/classes/", "job.jar/lib/*", ApplicationConstants.Environment.PWD.$$() + "/*")); String expectedClasspath = StringUtils.join(ApplicationConstants.CLASS_PATH_SEPARATOR, Arrays.asList(ApplicationConstants.Environment.PWD.$$(), FRAMEWORK_CLASSPATH, stdClasspath)); assertEquals("Incorrect classpath with framework and no user precedence", expectedClasspath, env.get("CLASSPATH")); env.clear(); conf.setBoolean(MRJobConfig.MAPREDUCE_JOB_USER_CLASSPATH_FIRST, true); MRApps.setClasspath(env, conf); expectedClasspath = StringUtils.join(ApplicationConstants.CLASS_PATH_SEPARATOR, Arrays.asList(ApplicationConstants.Environment.PWD.$$(), stdClasspath, FRAMEWORK_CLASSPATH)); assertEquals("Incorrect classpath with framework and user precedence", expectedClasspath, env.get("CLASSPATH")); }