@Test public void testloadGenericJobConfigs() throws ConfigurationException, IOException { Properties properties = new Properties(); properties.setProperty(ConfigurationKeys.JOB_CONFIG_FILE_GENERAL_PATH_KEY, this.jobConfigDir.getAbsolutePath()); List<Properties> jobConfigs = SchedulerUtils.loadGenericJobConfigs(properties, JobSpecResolver.mock()); Assert.assertEquals(jobConfigs.size(), 4); Properties jobProps1 = getJobConfigForFile(jobConfigs, "test111.pull"); Assert.assertEquals(jobProps1.stringPropertyNames().size(), 7); Assert.assertTrue(jobProps1.containsKey(ConfigurationKeys.JOB_CONFIG_FILE_DIR_KEY) || jobProps1.containsKey( ConfigurationKeys.JOB_CONFIG_FILE_GENERAL_PATH_KEY)); Assert.assertTrue(jobProps1.containsKey(ConfigurationKeys.JOB_CONFIG_FILE_PATH_KEY)); Assert.assertEquals(jobProps1.getProperty("k1"), "d1"); Assert.assertEquals(jobProps1.getProperty("k2"), "a2"); Assert.assertEquals(jobProps1.getProperty("k3"), "a3"); Assert.assertEquals(jobProps1.getProperty("k8"), "a8"); Assert.assertEquals(jobProps1.getProperty("k9"), "a8"); Properties jobProps2 = getJobConfigForFile(jobConfigs, "test11.pull"); Assert.assertEquals(jobProps2.stringPropertyNames().size(), 6); Assert.assertTrue(jobProps2.containsKey(ConfigurationKeys.JOB_CONFIG_FILE_DIR_KEY) || jobProps1.containsKey( ConfigurationKeys.JOB_CONFIG_FILE_GENERAL_PATH_KEY)); Assert.assertTrue(jobProps2.containsKey(ConfigurationKeys.JOB_CONFIG_FILE_PATH_KEY)); Assert.assertEquals(jobProps2.getProperty("k1"), "c1"); Assert.assertEquals(jobProps2.getProperty("k2"), "a2"); Assert.assertEquals(jobProps2.getProperty("k3"), "b3"); Assert.assertEquals(jobProps2.getProperty("k6"), "a6"); Properties jobProps3 = getJobConfigForFile(jobConfigs, "test12.PULL"); Assert.assertEquals(jobProps3.stringPropertyNames().size(), 6); Assert.assertTrue(jobProps3.containsKey(ConfigurationKeys.JOB_CONFIG_FILE_DIR_KEY) || jobProps1.containsKey( ConfigurationKeys.JOB_CONFIG_FILE_GENERAL_PATH_KEY)); Assert.assertTrue(jobProps3.containsKey(ConfigurationKeys.JOB_CONFIG_FILE_PATH_KEY)); Assert.assertEquals(jobProps3.getProperty("k1"), "b1"); Assert.assertEquals(jobProps3.getProperty("k2"), "a2"); Assert.assertEquals(jobProps3.getProperty("k3"), "a3"); Assert.assertEquals(jobProps3.getProperty("k7"), "a7"); Properties jobProps4 = getJobConfigForFile(jobConfigs, "test21.PULL"); Assert.assertEquals(jobProps4.stringPropertyNames().size(), 5); Assert.assertTrue(jobProps4.containsKey(ConfigurationKeys.JOB_CONFIG_FILE_DIR_KEY) || jobProps1.containsKey( ConfigurationKeys.JOB_CONFIG_FILE_GENERAL_PATH_KEY)); Assert.assertTrue(jobProps4.containsKey(ConfigurationKeys.JOB_CONFIG_FILE_PATH_KEY)); Assert.assertEquals(jobProps4.getProperty("k1"), "a1"); Assert.assertEquals(jobProps4.getProperty("k2"), "b2"); Assert.assertEquals(jobProps4.getProperty("k5"), "b5"); }
public static List<Properties> loadGenericJobConfigs(Properties sysProps, JobSpecResolver resolver) throws ConfigurationException, IOException { Path rootPath = new Path(sysProps.getProperty(ConfigurationKeys.JOB_CONFIG_FILE_GENERAL_PATH_KEY)); PullFileLoader loader = new PullFileLoader(rootPath, rootPath.getFileSystem(new Configuration()), getJobConfigurationFileExtensions(sysProps), PullFileLoader.DEFAULT_HOCON_PULL_FILE_EXTENSIONS); Config sysConfig = ConfigUtils.propertiesToConfig(sysProps); Collection<Config> configs = loader.loadPullFilesRecursively(rootPath, sysConfig, true); List<Properties> jobConfigs = Lists.newArrayList(); for (Config config : configs) { try { jobConfigs.add(resolveTemplate(ConfigUtils.configToProperties(config), resolver)); } catch (IOException ioe) { LOGGER.error("Could not parse job config at " + ConfigUtils.getString(config, ConfigurationKeys.JOB_CONFIG_FILE_PATH_KEY, "Unknown path"), ioe); } } return jobConfigs; }
SchedulerUtils { public static List<Properties> loadGenericJobConfigs(Properties sysProps, JobSpecResolver resolver) throws ConfigurationException, IOException { Path rootPath = new Path(sysProps.getProperty(ConfigurationKeys.JOB_CONFIG_FILE_GENERAL_PATH_KEY)); PullFileLoader loader = new PullFileLoader(rootPath, rootPath.getFileSystem(new Configuration()), getJobConfigurationFileExtensions(sysProps), PullFileLoader.DEFAULT_HOCON_PULL_FILE_EXTENSIONS); Config sysConfig = ConfigUtils.propertiesToConfig(sysProps); Collection<Config> configs = loader.loadPullFilesRecursively(rootPath, sysConfig, true); List<Properties> jobConfigs = Lists.newArrayList(); for (Config config : configs) { try { jobConfigs.add(resolveTemplate(ConfigUtils.configToProperties(config), resolver)); } catch (IOException ioe) { LOGGER.error("Could not parse job config at " + ConfigUtils.getString(config, ConfigurationKeys.JOB_CONFIG_FILE_PATH_KEY, "Unknown path"), ioe); } } return jobConfigs; } }
SchedulerUtils { public static List<Properties> loadGenericJobConfigs(Properties sysProps, JobSpecResolver resolver) throws ConfigurationException, IOException { Path rootPath = new Path(sysProps.getProperty(ConfigurationKeys.JOB_CONFIG_FILE_GENERAL_PATH_KEY)); PullFileLoader loader = new PullFileLoader(rootPath, rootPath.getFileSystem(new Configuration()), getJobConfigurationFileExtensions(sysProps), PullFileLoader.DEFAULT_HOCON_PULL_FILE_EXTENSIONS); Config sysConfig = ConfigUtils.propertiesToConfig(sysProps); Collection<Config> configs = loader.loadPullFilesRecursively(rootPath, sysConfig, true); List<Properties> jobConfigs = Lists.newArrayList(); for (Config config : configs) { try { jobConfigs.add(resolveTemplate(ConfigUtils.configToProperties(config), resolver)); } catch (IOException ioe) { LOGGER.error("Could not parse job config at " + ConfigUtils.getString(config, ConfigurationKeys.JOB_CONFIG_FILE_PATH_KEY, "Unknown path"), ioe); } } return jobConfigs; }  }
SchedulerUtils { public static List<Properties> loadGenericJobConfigs(Properties sysProps, JobSpecResolver resolver) throws ConfigurationException, IOException { Path rootPath = new Path(sysProps.getProperty(ConfigurationKeys.JOB_CONFIG_FILE_GENERAL_PATH_KEY)); PullFileLoader loader = new PullFileLoader(rootPath, rootPath.getFileSystem(new Configuration()), getJobConfigurationFileExtensions(sysProps), PullFileLoader.DEFAULT_HOCON_PULL_FILE_EXTENSIONS); Config sysConfig = ConfigUtils.propertiesToConfig(sysProps); Collection<Config> configs = loader.loadPullFilesRecursively(rootPath, sysConfig, true); List<Properties> jobConfigs = Lists.newArrayList(); for (Config config : configs) { try { jobConfigs.add(resolveTemplate(ConfigUtils.configToProperties(config), resolver)); } catch (IOException ioe) { LOGGER.error("Could not parse job config at " + ConfigUtils.getString(config, ConfigurationKeys.JOB_CONFIG_FILE_PATH_KEY, "Unknown path"), ioe); } } return jobConfigs; }  static List<Properties> loadGenericJobConfigs(Properties sysProps, JobSpecResolver resolver); static List<Properties> loadGenericJobConfigs(Properties sysProps, Path commonPropsPath,
      Path jobConfigPathDir, JobSpecResolver resolver); static Properties loadGenericJobConfig(Properties sysProps, Path jobConfigPath, Path jobConfigPathDir,
      JobSpecResolver resolver); static void addPathAlterationObserver(PathAlterationObserverScheduler monitor, PathAlterationListener listener,
      Path rootDirPath); }
SchedulerUtils { public static List<Properties> loadGenericJobConfigs(Properties sysProps, JobSpecResolver resolver) throws ConfigurationException, IOException { Path rootPath = new Path(sysProps.getProperty(ConfigurationKeys.JOB_CONFIG_FILE_GENERAL_PATH_KEY)); PullFileLoader loader = new PullFileLoader(rootPath, rootPath.getFileSystem(new Configuration()), getJobConfigurationFileExtensions(sysProps), PullFileLoader.DEFAULT_HOCON_PULL_FILE_EXTENSIONS); Config sysConfig = ConfigUtils.propertiesToConfig(sysProps); Collection<Config> configs = loader.loadPullFilesRecursively(rootPath, sysConfig, true); List<Properties> jobConfigs = Lists.newArrayList(); for (Config config : configs) { try { jobConfigs.add(resolveTemplate(ConfigUtils.configToProperties(config), resolver)); } catch (IOException ioe) { LOGGER.error("Could not parse job config at " + ConfigUtils.getString(config, ConfigurationKeys.JOB_CONFIG_FILE_PATH_KEY, "Unknown path"), ioe); } } return jobConfigs; }  static List<Properties> loadGenericJobConfigs(Properties sysProps, JobSpecResolver resolver); static List<Properties> loadGenericJobConfigs(Properties sysProps, Path commonPropsPath,
      Path jobConfigPathDir, JobSpecResolver resolver); static Properties loadGenericJobConfig(Properties sysProps, Path jobConfigPath, Path jobConfigPathDir,
      JobSpecResolver resolver); static void addPathAlterationObserver(PathAlterationObserverScheduler monitor, PathAlterationListener listener,
      Path rootDirPath); static final String JOB_PROPS_FILE_EXTENSION; }