ThinArchiveUtils { public static String replaceLocalPathsWithStoragePaths(final File projectFolder, String localJarSpec, final Props jobProps, final Logger log) { File startupDependenciesFile = getStartupDependenciesFile(projectFolder); String baseDependencyPath = jobProps.get(DEPENDENCY_STORAGE_ROOT_PATH_PROP); if (!startupDependenciesFile.exists() || baseDependencyPath == null) { log.debug("Skipping replacing dependency paths with common HDFS startup dependencies."); return localJarSpec; } String[] localDependencies = localJarSpec.split(","); try { Set<Dependency> startupDeps = parseStartupDependencies(startupDependenciesFile); Map<String, Dependency> pathToDep = new HashMap<>(); for (Dependency dep : startupDeps) { pathToDep.put(getDependencyFile(projectFolder, dep).getFile().getCanonicalPath(), dep); } List<String> finalDependencies = new ArrayList<>(); for (String localDepPath : localDependencies) { final String localDepCanonicalPath = new File(projectFolder, localDepPath).getCanonicalPath(); if (pathToDep.containsKey(localDepCanonicalPath)) { if (baseDependencyPath.endsWith("/")) { baseDependencyPath = baseDependencyPath.substring(0, baseDependencyPath.length() - 1); } String pathToDependencyInStorage = baseDependencyPath + "/" + convertIvyCoordinateToPath(pathToDep.get(localDepCanonicalPath)); finalDependencies.add(pathToDependencyInStorage); } else { finalDependencies.add(localDepPath); } } String localAndRemoteJarSpec = String.join(",", finalDependencies); log.debug("replaceLocalPathsWithStoragePaths: localAndRemoteJarSpec: " + localAndRemoteJarSpec); return localAndRemoteJarSpec; } catch (IOException | InvalidHashException e) { log.warn("Error while opening and parsing startup dependencies file " + startupDependenciesFile.getAbsolutePath()); return localJarSpec; } }  static File getStartupDependenciesFile(final File projectFolder); static DependencyFile getDependencyFile(final File projectFolder, final Dependency d); static Set<Dependency> parseStartupDependencies(final String rawJson); static Set<Dependency> parseStartupDependencies(final File f); static void writeStartupDependencies(final File f,
      final Set<Dependency> dependencies); static String convertIvyCoordinateToPath(final Dependency dep); static String replaceLocalPathsWithStoragePaths(final File projectFolder,
      String localJarSpec, final Props jobProps, final Logger log); static void validateDependencyHash(final DependencyFile f); static void validateDependencyHash(final File f, final Dependency d); static Set<Dependency> filterNullFromDeps(Set<Dependency> dependencies); static final String DEPENDENCY_STORAGE_ROOT_PATH_PROP; }
@Test public void testReplaceLocalPathsWithStoragePathsFAT() throws Exception { Dependency depA = ThinArchiveTestUtils.getDepA(); File projectDir = TEMP_DIR.newFolder("sample_proj"); Props props = new Props(); props.put(DEPENDENCY_STORAGE_ROOT_PATH_PROP, HDFS_DEP_ROOT_PATH); File depAFile = new File(projectDir, depA.getDestination() + File.separator + depA.getFileName()); FileUtils.writeStringToFile(depAFile, ThinArchiveTestUtils.getDepAContent()); List<String> jarPaths = new ArrayList<>(); jarPaths.add(depAFile.getCanonicalPath()); String jarPathsStr = String.join(",", jarPaths); List<String> expectedJarPaths = new ArrayList<>(); expectedJarPaths.add(jarPaths.get(0)); String expectedJarPathsStr = String.join(",", expectedJarPaths); String resultingJarPathsStr = ThinArchiveUtils.replaceLocalPathsWithStoragePaths(projectDir, jarPathsStr, props, log); assertEquals(expectedJarPathsStr, resultingJarPathsStr); }