DistributedCacheJars { public static void addJarsToDistributedCache(Job job, String jarDirectory) throws IOException { addJarsToDistributedCache(job, qualifiedPathFromString(jarDirectory, job.getConfiguration())); } private  DistributedCacheJars(); static void addJarsToDistributedCache(Job job, String jarDirectory); static void addJarsToDistributedCache(Job job, File jarDirectory); static void addJarsToDistributedCache(Job job, Path jarDirectory); static void addJarsToDistributedCache(Job job, Collection<Path> jarFiles); static List<Path> getJarsFromConfiguration(Configuration conf); static Collection<Path> listJarFilesFromDirectory(Configuration conf, Path jarDirectory); static List<Path> deDuplicateFilenames(Iterable<Path> jarFiles);  }
@Test public void testJarsDeDupe() throws IOException { final File tempDir = getLocalTempDir(); Set<String> dedupedJarNames = new HashSet<String>(4); dedupedJarNames.add("myjar_a.jar"); dedupedJarNames.add("myjar_b.jar"); dedupedJarNames.add("myjar_0.jar"); dedupedJarNames.add("myjar_1.jar"); Job job = new Job(); List<String> someJars = new ArrayList<String>(); someJars.add("/somepath/myjar_a.jar"); someJars.add("/another/path/myjar_b.jar"); someJars.add("/myjar_0.jar"); someJars.add("/another/path/myjar_b.jar"); someJars.add("/yet/another/path/myjar_b.jar"); job.getConfiguration().set(CONF_TMPJARS, StringUtils.join(someJars, ",")); assertEquals(0, tempDir.list().length); createTestJars(tempDir, "myjar_0.jar", "myjar_1.jar"); assertEquals(2, tempDir.list().length); DistributedCacheJars.addJarsToDistributedCache(job, tempDir); String listedJars = job.getConfiguration().get(CONF_TMPJARS); String[] jars = listedJars.split(","); for (String jar : jars) { Path p = new Path(jar); assertTrue(dedupedJarNames.contains(p.getName())); dedupedJarNames.remove(p.getName()); } assertEquals(0, dedupedJarNames.size()); }