@Test void testJobLoadingOrder() throws Exception { Properties sysProps = new Properties(); FileSystem fs = FileSystem.getLocal(new Configuration()); File tmpDir = Files.createTempDir(); tmpDir.deleteOnExit(); Path localBasePath = new Path(tmpDir.getAbsolutePath(), "PullFileLoaderTestDir"); fs.mkdirs(localBasePath); for (int i=5; i>0; i--) { String job = localBasePath.toString() + "/job" + i + ".conf"; PrintWriter writer = new PrintWriter(job, "UTF-8"); writer.println("key=job" + i + "_val"); writer.close(); Thread.sleep(1000); } List<Config> configs = loader.loadPullFilesRecursively(localBasePath, ConfigUtils.propertiesToConfig(sysProps), false); int i = 5; for (Config config : configs) { Assert.assertEquals(config.getString("key"), "job" + i + "_val"); i--; } }
public List<Config> loadPullFilesRecursively(Path path, Config sysProps, boolean loadGlobalProperties) { return Lists.transform(this.fetchJobFilesRecursively(path), new Function<Path, Config>() { @Nullable @Override public Config apply(@Nullable Path jobFile) { if (jobFile == null) { return null; } try { return PullFileLoader.this.loadPullFile(jobFile, sysProps, loadGlobalProperties); } catch (IOException e) { log.error("Cannot load job from {} due to {}", jobFile, ExceptionUtils.getFullStackTrace(e)); return null; } } }); }
PullFileLoader { public List<Config> loadPullFilesRecursively(Path path, Config sysProps, boolean loadGlobalProperties) { return Lists.transform(this.fetchJobFilesRecursively(path), new Function<Path, Config>() { @Nullable @Override public Config apply(@Nullable Path jobFile) { if (jobFile == null) { return null; } try { return PullFileLoader.this.loadPullFile(jobFile, sysProps, loadGlobalProperties); } catch (IOException e) { log.error("Cannot load job from {} due to {}", jobFile, ExceptionUtils.getFullStackTrace(e)); return null; } } }); } }
PullFileLoader { public List<Config> loadPullFilesRecursively(Path path, Config sysProps, boolean loadGlobalProperties) { return Lists.transform(this.fetchJobFilesRecursively(path), new Function<Path, Config>() { @Nullable @Override public Config apply(@Nullable Path jobFile) { if (jobFile == null) { return null; } try { return PullFileLoader.this.loadPullFile(jobFile, sysProps, loadGlobalProperties); } catch (IOException e) { log.error("Cannot load job from {} due to {}", jobFile, ExceptionUtils.getFullStackTrace(e)); return null; } } }); } PullFileLoader(Path rootDirectory, FileSystem fs, Collection<String> javaPropsPullFileExtensions,
      Collection<String> hoconPullFileExtensions); }
PullFileLoader { public List<Config> loadPullFilesRecursively(Path path, Config sysProps, boolean loadGlobalProperties) { return Lists.transform(this.fetchJobFilesRecursively(path), new Function<Path, Config>() { @Nullable @Override public Config apply(@Nullable Path jobFile) { if (jobFile == null) { return null; } try { return PullFileLoader.this.loadPullFile(jobFile, sysProps, loadGlobalProperties); } catch (IOException e) { log.error("Cannot load job from {} due to {}", jobFile, ExceptionUtils.getFullStackTrace(e)); return null; } } }); } PullFileLoader(Path rootDirectory, FileSystem fs, Collection<String> javaPropsPullFileExtensions,
      Collection<String> hoconPullFileExtensions); Config loadPullFile(Path path, Config sysProps, boolean loadGlobalProperties, boolean resolve); Config loadPullFile(Path path, Config sysProps, boolean loadGlobalProperties); List<Config> loadPullFilesRecursively(Path path, Config sysProps, boolean loadGlobalProperties); List<Path> fetchJobFilesRecursively(Path path); }
PullFileLoader { public List<Config> loadPullFilesRecursively(Path path, Config sysProps, boolean loadGlobalProperties) { return Lists.transform(this.fetchJobFilesRecursively(path), new Function<Path, Config>() { @Nullable @Override public Config apply(@Nullable Path jobFile) { if (jobFile == null) { return null; } try { return PullFileLoader.this.loadPullFile(jobFile, sysProps, loadGlobalProperties); } catch (IOException e) { log.error("Cannot load job from {} due to {}", jobFile, ExceptionUtils.getFullStackTrace(e)); return null; } } }); } PullFileLoader(Path rootDirectory, FileSystem fs, Collection<String> javaPropsPullFileExtensions,
      Collection<String> hoconPullFileExtensions); Config loadPullFile(Path path, Config sysProps, boolean loadGlobalProperties, boolean resolve); Config loadPullFile(Path path, Config sysProps, boolean loadGlobalProperties); List<Config> loadPullFilesRecursively(Path path, Config sysProps, boolean loadGlobalProperties); List<Path> fetchJobFilesRecursively(Path path); static final String GLOBAL_PROPS_EXTENSION; static final PathFilter GLOBAL_PROPS_PATH_FILTER; static final String GLOBAL_HOCON_EXTENSION; static final PathFilter GLOBAL_HOCON_PATH_FILTER; static final PathFilter GLOBAL_PATH_FILTER; static final Set<String> DEFAULT_JAVA_PROPS_PULL_FILE_EXTENSIONS; static final Set<String> DEFAULT_HOCON_PULL_FILE_EXTENSIONS; static final String PROPERTY_DELIMITER_PARSING_ENABLED_KEY; static final boolean DEFAULT_PROPERTY_DELIMITER_PARSING_ENABLED_KEY; }