@Test public void testRecursiveJobLoading() throws Exception { Path path; Config pullFile; Properties sysProps = new Properties(); sysProps.put("key1", "sysProps1"); Collection<Config> configs = loader.loadPullFilesRecursively(this.basePath, ConfigUtils.propertiesToConfig(sysProps), false); path = new Path(this.basePath, "ajob.pull"); pullFile = pullFileFromPath(configs, path); Assert.assertEquals(pullFile.getString("key1"), "sysProps1"); Assert.assertEquals(pullFile.getString("key2"), "aValue"); Assert.assertEquals(pullFile.getString("key10"), "aValue"); Assert.assertEquals(pullFile.getString(ConfigurationKeys.JOB_CONFIG_FILE_PATH_KEY), path.toString()); Assert.assertEquals(pullFile.entrySet().size(), 4); path = new Path(this.basePath, "dir1/job.pull"); pullFile = pullFileFromPath(configs, path); Assert.assertEquals(pullFile.getString("key1"), "jobValue1,jobValue2,jobValue3"); Assert.assertEquals(pullFile.getString("key2"), "jobValue2"); Assert.assertEquals(pullFile.getString(ConfigurationKeys.JOB_CONFIG_FILE_PATH_KEY), path.toString()); Assert.assertEquals(pullFile.entrySet().size(), 3); path = new Path(this.basePath, "dir1/job.conf"); pullFile = pullFileFromPath(configs, path); Assert.assertEquals(pullFile.getString("key1"), "jobValue1,jobValue2,jobValue3"); Assert.assertEquals(pullFile.getString("key2"), "jobValue2"); Assert.assertEquals(pullFile.getString("key10"), "jobValue2"); Assert.assertEquals(pullFile.getString(ConfigurationKeys.JOB_CONFIG_FILE_PATH_KEY), path.toString()); Assert.assertEquals(pullFile.entrySet().size(), 4); }
public List<Config> loadPullFilesRecursively(Path path, Config sysProps, boolean loadGlobalProperties) { return Lists.transform(this.fetchJobFilesRecursively(path), new Function<Path, Config>() { @Nullable @Override public Config apply(@Nullable Path jobFile) { if (jobFile == null) { return null; } try { return PullFileLoader.this.loadPullFile(jobFile, sysProps, loadGlobalProperties); } catch (IOException e) { log.error("Cannot load job from {} due to {}", jobFile, ExceptionUtils.getFullStackTrace(e)); return null; } } }); }
PullFileLoader { public List<Config> loadPullFilesRecursively(Path path, Config sysProps, boolean loadGlobalProperties) { return Lists.transform(this.fetchJobFilesRecursively(path), new Function<Path, Config>() { @Nullable @Override public Config apply(@Nullable Path jobFile) { if (jobFile == null) { return null; } try { return PullFileLoader.this.loadPullFile(jobFile, sysProps, loadGlobalProperties); } catch (IOException e) { log.error("Cannot load job from {} due to {}", jobFile, ExceptionUtils.getFullStackTrace(e)); return null; } } }); } }
PullFileLoader { public List<Config> loadPullFilesRecursively(Path path, Config sysProps, boolean loadGlobalProperties) { return Lists.transform(this.fetchJobFilesRecursively(path), new Function<Path, Config>() { @Nullable @Override public Config apply(@Nullable Path jobFile) { if (jobFile == null) { return null; } try { return PullFileLoader.this.loadPullFile(jobFile, sysProps, loadGlobalProperties); } catch (IOException e) { log.error("Cannot load job from {} due to {}", jobFile, ExceptionUtils.getFullStackTrace(e)); return null; } } }); } PullFileLoader(Path rootDirectory, FileSystem fs, Collection<String> javaPropsPullFileExtensions,
      Collection<String> hoconPullFileExtensions); }
PullFileLoader { public List<Config> loadPullFilesRecursively(Path path, Config sysProps, boolean loadGlobalProperties) { return Lists.transform(this.fetchJobFilesRecursively(path), new Function<Path, Config>() { @Nullable @Override public Config apply(@Nullable Path jobFile) { if (jobFile == null) { return null; } try { return PullFileLoader.this.loadPullFile(jobFile, sysProps, loadGlobalProperties); } catch (IOException e) { log.error("Cannot load job from {} due to {}", jobFile, ExceptionUtils.getFullStackTrace(e)); return null; } } }); } PullFileLoader(Path rootDirectory, FileSystem fs, Collection<String> javaPropsPullFileExtensions,
      Collection<String> hoconPullFileExtensions); Config loadPullFile(Path path, Config sysProps, boolean loadGlobalProperties, boolean resolve); Config loadPullFile(Path path, Config sysProps, boolean loadGlobalProperties); List<Config> loadPullFilesRecursively(Path path, Config sysProps, boolean loadGlobalProperties); List<Path> fetchJobFilesRecursively(Path path); }
PullFileLoader { public List<Config> loadPullFilesRecursively(Path path, Config sysProps, boolean loadGlobalProperties) { return Lists.transform(this.fetchJobFilesRecursively(path), new Function<Path, Config>() { @Nullable @Override public Config apply(@Nullable Path jobFile) { if (jobFile == null) { return null; } try { return PullFileLoader.this.loadPullFile(jobFile, sysProps, loadGlobalProperties); } catch (IOException e) { log.error("Cannot load job from {} due to {}", jobFile, ExceptionUtils.getFullStackTrace(e)); return null; } } }); } PullFileLoader(Path rootDirectory, FileSystem fs, Collection<String> javaPropsPullFileExtensions,
      Collection<String> hoconPullFileExtensions); Config loadPullFile(Path path, Config sysProps, boolean loadGlobalProperties, boolean resolve); Config loadPullFile(Path path, Config sysProps, boolean loadGlobalProperties); List<Config> loadPullFilesRecursively(Path path, Config sysProps, boolean loadGlobalProperties); List<Path> fetchJobFilesRecursively(Path path); static final String GLOBAL_PROPS_EXTENSION; static final PathFilter GLOBAL_PROPS_PATH_FILTER; static final String GLOBAL_HOCON_EXTENSION; static final PathFilter GLOBAL_HOCON_PATH_FILTER; static final PathFilter GLOBAL_PATH_FILTER; static final Set<String> DEFAULT_JAVA_PROPS_PULL_FILE_EXTENSIONS; static final Set<String> DEFAULT_HOCON_PULL_FILE_EXTENSIONS; static final String PROPERTY_DELIMITER_PARSING_ENABLED_KEY; static final boolean DEFAULT_PROPERTY_DELIMITER_PARSING_ENABLED_KEY; }