BlueprintConfigurationProcessor { public void doUpdateForBlueprintExport() { if (clusterTopology.isNameNodeHAEnabled()) { doNameNodeHAUpdate(); } if (clusterTopology.isYarnResourceManagerHAEnabled()) { doYarnResourceManagerHAUpdate(); } if (isOozieServerHAEnabled(clusterTopology.getConfiguration().getFullProperties())) { doOozieServerHAUpdate(); } Collection<Configuration> allConfigs = new ArrayList<Configuration>(); allConfigs.add(clusterTopology.getConfiguration()); for (HostGroupInfo groupInfo : clusterTopology.getHostGroupInfo().values()) { Configuration hgConfiguration = groupInfo.getConfiguration(); if (! hgConfiguration.getFullProperties(1).isEmpty()) { allConfigs.add(new Configuration(hgConfiguration.getProperties(), null, new Configuration(hgConfiguration.getParentConfiguration().getProperties(), null))); } } for (Configuration configuration : allConfigs) { doSingleHostExportUpdate(singleHostTopologyUpdaters, configuration); doSingleHostExportUpdate(dbHostTopologyUpdaters, configuration); doMultiHostExportUpdate(multiHostTopologyUpdaters, configuration); doNonTopologyUpdate(nonTopologyUpdaters, configuration); doRemovePropertyExport(removePropertyUpdaters, configuration); doFilterPriorToExport(configuration); } } BlueprintConfigurationProcessor(ClusterTopology clusterTopology); Map<String, Map<String, PropertyUpdater>> getRemovePropertyUpdaters(); void initRemovePropertyUpdaters(); Collection<String> getRequiredHostGroups(); Set<String> doUpdateForClusterCreate(); void doUpdateForBlueprintExport(); void doNameNodeHAUpdate(); void doYarnResourceManagerHAUpdate(); void doOozieServerHAUpdate();  }
@Test public void testDoUpdateForBlueprintExport_SingleHostProperty() throws Exception { Map<String, Map<String, String>> properties = new HashMap<String, Map<String, String>>(); Map<String, String> typeProps = new HashMap<String, String>(); typeProps.put("yarn.resourcemanager.hostname", "testhost"); properties.put("yarn-site", typeProps); Configuration clusterConfig = new Configuration(properties, Collections.<String, Map<String, Map<String, String>>>emptyMap()); Collection<String> hgComponents = new HashSet<String>(); hgComponents.add("NAMENODE"); hgComponents.add("SECONDARY_NAMENODE"); hgComponents.add("RESOURCEMANAGER"); TestHostGroup group1 = new TestHostGroup("group1", hgComponents, Collections.singleton("testhost")); Collection<String> hgComponents2 = new HashSet<String>(); hgComponents2.add("DATANODE"); hgComponents2.add("HDFS_CLIENT"); TestHostGroup group2 = new TestHostGroup("group2", hgComponents2, Collections.singleton("testhost2")); Collection<TestHostGroup> hostGroups = new HashSet<TestHostGroup>(); hostGroups.add(group1); hostGroups.add(group2); ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); BlueprintConfigurationProcessor configProcessor = new BlueprintConfigurationProcessor(topology); configProcessor.doUpdateForBlueprintExport(); String updatedVal = properties.get("yarn-site").get("yarn.resourcemanager.hostname"); assertEquals("%HOSTGROUP::group1%", updatedVal); }
@Test public void testDoUpdateForBlueprintExport_FilterProperties() throws Exception { Map<String, Map<String, String>> properties = new HashMap<String, Map<String, String>>(); Map<String, String> kerberosEnvProps = new HashMap<String, String>(); kerberosEnvProps.put("admin_server_host", "test"); kerberosEnvProps.put("kdc_hosts", "test"); kerberosEnvProps.put("realm", "test"); kerberosEnvProps.put("kdc_type", "test"); kerberosEnvProps.put("ldap-url", "test"); kerberosEnvProps.put("container_dn", "test"); properties.put("kerberos-env", kerberosEnvProps); Map<String, String> krb5ConfProps = new HashMap<String, String>(); krb5ConfProps.put("domains", "test"); properties.put("krb5-conf", krb5ConfProps); Map<String, String> tezSiteConfProps = new HashMap<String, String>(); tezSiteConfProps.put("tez.tez-ui.history-url.base", "test"); properties.put("tez-site", tezSiteConfProps); Configuration clusterConfig = new Configuration(properties, Collections.<String, Map<String, Map<String, String>>>emptyMap()); Collection<String> hgComponents = new HashSet<String>(); hgComponents.add("NAMENODE"); hgComponents.add("SECONDARY_NAMENODE"); hgComponents.add("RESOURCEMANAGER"); TestHostGroup group1 = new TestHostGroup("group1", hgComponents, Collections.singleton("testhost")); Collection<String> hgComponents2 = new HashSet<String>(); hgComponents2.add("DATANODE"); hgComponents2.add("HDFS_CLIENT"); TestHostGroup group2 = new TestHostGroup("group2", hgComponents2, Collections.singleton("testhost2")); Collection<TestHostGroup> hostGroups = new HashSet<TestHostGroup>(); hostGroups.add(group1); hostGroups.add(group2); ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); BlueprintConfigurationProcessor configProcessor = new BlueprintConfigurationProcessor(topology); configProcessor.doUpdateForBlueprintExport(); assertEquals(properties.size(), 3); assertEquals(((Map) properties.get("kerberos-env")).size(), 0); assertEquals(((Map) properties.get("krb5-conf")).size(), 0); assertEquals(((Map) properties.get("tez-site")).size(), 0); }
@Test public void testDoUpdateForBlueprintExportRangerHAPolicyMgrExternalUrlProperty() throws Exception { Map<String, String> rangerAdminProperties = Maps.newHashMap(); rangerAdminProperties.put("DB_FLAVOR", "test_db_flavor"); rangerAdminProperties.put("policymgr_external_url", "test_policymgr_external_url"); Map<String, Map<String, String>> properties = ImmutableMap.of("admin-properties", rangerAdminProperties); Configuration clusterConfig = new Configuration(properties, ImmutableMap.<String, Map<String,Map<String,String>>>of()); Collection<String> hostGroup1Components = ImmutableSet.of("RANGER_ADMIN"); TestHostGroup group1 = new TestHostGroup("group1", hostGroup1Components, Collections.singleton("testhost1")); Collection<String> hostGroup2Components = ImmutableSet.of("RANGER_ADMIN"); TestHostGroup group2 = new TestHostGroup("group2", hostGroup2Components, Collections.singleton("testhost2")); Collection<TestHostGroup> hostGroups = ImmutableSet.of(group1, group2); ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); BlueprintConfigurationProcessor configProcessor = new BlueprintConfigurationProcessor(topology); configProcessor.doUpdateForBlueprintExport(); assertEquals("policymgr_external_url property's original value should be exported when Ranger Admin is deployed to multiple hosts.", "test_policymgr_external_url", properties.get("admin-properties").get("policymgr_external_url")); }
@Test public void testDoUpdateForBlueprintExport_SingleHostProperty_specifiedInParentConfig() throws Exception { Map<String, Map<String, String>> properties = new HashMap<String, Map<String, String>>(); Map<String, String> yarnSiteProps = new HashMap<String, String>(); yarnSiteProps.put("yarn.resourcemanager.hostname", "testhost"); properties.put("yarn-site", yarnSiteProps); Map<String, Map<String, String>> parentProperties = new HashMap<String, Map<String, String>>(); Map<String, String> parentYarnSiteProps = new HashMap<String, String>(); parentYarnSiteProps.put("yarn.resourcemanager.resource-tracker.address", "testhost"); parentProperties.put("yarn-site", parentYarnSiteProps); Configuration parentClusterConfig = new Configuration(parentProperties, Collections.<String, Map<String, Map<String, String>>>emptyMap()); Configuration clusterConfig = new Configuration(properties, Collections.<String, Map<String, Map<String, String>>>emptyMap(), parentClusterConfig); Collection<String> hgComponents = new HashSet<String>(); hgComponents.add("NAMENODE"); hgComponents.add("SECONDARY_NAMENODE"); hgComponents.add("RESOURCEMANAGER"); TestHostGroup group1 = new TestHostGroup("group1", hgComponents, Collections.singleton("testhost")); Collection<String> hgComponents2 = new HashSet<String>(); hgComponents2.add("DATANODE"); hgComponents2.add("HDFS_CLIENT"); TestHostGroup group2 = new TestHostGroup("group2", hgComponents2, Collections.singleton("testhost2")); Collection<TestHostGroup> hostGroups = new HashSet<TestHostGroup>(); hostGroups.add(group1); hostGroups.add(group2); ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); BlueprintConfigurationProcessor configProcessor = new BlueprintConfigurationProcessor(topology); configProcessor.doUpdateForBlueprintExport(); assertEquals("%HOSTGROUP::group1%", clusterConfig.getPropertyValue("yarn-site", "yarn.resourcemanager.hostname")); assertEquals("%HOSTGROUP::group1%", clusterConfig.getPropertyValue("yarn-site", "yarn.resourcemanager.resource-tracker.address")); }
@Test public void testDoUpdateForBlueprintExport_SingleHostProperty_hostGroupConfiguration() throws Exception { Map<String, Map<String, String>> properties = new HashMap<String, Map<String, String>>(); Map<String, String> typeProps = new HashMap<String, String>(); typeProps.put("yarn.resourcemanager.hostname", "testhost"); properties.put("yarn-site", typeProps); Configuration clusterConfig = new Configuration(properties, Collections.<String, Map<String, Map<String, String>>>emptyMap()); Collection<String> hgComponents = new HashSet<String>(); hgComponents.add("NAMENODE"); hgComponents.add("SECONDARY_NAMENODE"); hgComponents.add("RESOURCEMANAGER"); TestHostGroup group1 = new TestHostGroup("group1", hgComponents, Collections.singleton("testhost")); Collection<String> hgComponents2 = new HashSet<String>(); hgComponents2.add("DATANODE"); hgComponents2.add("HDFS_CLIENT"); Map<String, Map<String, String>> group2Properties = new HashMap<String, Map<String, String>>(); Map<String, String> group2YarnSiteProps = new HashMap<String, String>(); group2YarnSiteProps.put("yarn.resourcemanager.resource-tracker.address", "testhost"); group2Properties.put("yarn-site", group2YarnSiteProps); Configuration group2BPConfiguration = new Configuration(Collections.<String, Map<String, String>>emptyMap(), Collections.<String, Map<String, Map<String, String>>>emptyMap(), clusterConfig); Configuration group2Configuration = new Configuration(group2Properties, Collections.<String, Map<String, Map<String, String>>>emptyMap(), group2BPConfiguration); TestHostGroup group2 = new TestHostGroup("group2", hgComponents2, Collections.singleton("testhost2"), group2Configuration); Collection<TestHostGroup> hostGroups = new HashSet<TestHostGroup>(); hostGroups.add(group1); hostGroups.add(group2); ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); BlueprintConfigurationProcessor configProcessor = new BlueprintConfigurationProcessor(topology); configProcessor.doUpdateForBlueprintExport(); assertEquals("%HOSTGROUP::group1%", properties.get("yarn-site").get("yarn.resourcemanager.hostname")); assertEquals("%HOSTGROUP::group1%", group2Configuration.getPropertyValue("yarn-site", "yarn.resourcemanager.resource-tracker.address")); }
@Test public void testDoUpdateForBlueprintExport_SingleHostProperty__withPort() throws Exception { Map<String, Map<String, String>> properties = new HashMap<String, Map<String, String>>(); Map<String, String> typeProps = new HashMap<String, String>(); typeProps.put("fs.defaultFS", "testhost:8020"); properties.put("core-site", typeProps); Configuration clusterConfig = new Configuration(properties, Collections.<String, Map<String, Map<String, String>>>emptyMap()); Collection<String> hgComponents = new HashSet<String>(); hgComponents.add("NAMENODE"); hgComponents.add("SECONDARY_NAMENODE"); TestHostGroup group1 = new TestHostGroup("group1", hgComponents, Collections.singleton("testhost")); Collection<String> hgComponents2 = new HashSet<String>(); hgComponents2.add("DATANODE"); hgComponents2.add("HDFS_CLIENT"); TestHostGroup group2 = new TestHostGroup("group2", hgComponents2, Collections.singleton("testhost2")); Collection<TestHostGroup> hostGroups = new HashSet<TestHostGroup>(); hostGroups.add(group1); hostGroups.add(group2); ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); BlueprintConfigurationProcessor configProcessor = new BlueprintConfigurationProcessor(topology); configProcessor.doUpdateForBlueprintExport(); String updatedVal = properties.get("core-site").get("fs.defaultFS"); assertEquals("%HOSTGROUP::group1%:8020", updatedVal); }
@Test public void testDoUpdateForBlueprintExport_SingleHostProperty__ExternalReference() throws Exception { Map<String, Map<String, String>> properties = new HashMap<String, Map<String, String>>(); Map<String, String> typeProps = new HashMap<String, String>(); typeProps.put("yarn.resourcemanager.hostname", "external-host"); properties.put("yarn-site", typeProps); Configuration clusterConfig = new Configuration(properties, Collections.<String, Map<String, Map<String, String>>>emptyMap()); Collection<String> hgComponents = new HashSet<String>(); hgComponents.add("NAMENODE"); hgComponents.add("SECONDARY_NAMENODE"); hgComponents.add("RESOURCEMANAGER"); TestHostGroup group1 = new TestHostGroup("group1", hgComponents, Collections.singleton("testhost")); Collection<String> hgComponents2 = new HashSet<String>(); hgComponents2.add("DATANODE"); hgComponents2.add("HDFS_CLIENT"); TestHostGroup group2 = new TestHostGroup("group2", hgComponents2, Collections.singleton("testhost2")); Collection<TestHostGroup> hostGroups = new HashSet<TestHostGroup>(); hostGroups.add(group1); hostGroups.add(group2); ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); BlueprintConfigurationProcessor configProcessor = new BlueprintConfigurationProcessor(topology); configProcessor.doUpdateForBlueprintExport(); assertFalse(properties.get("yarn-site").containsKey("yarn.resourcemanager.hostname")); }
@Test public void testDoUpdateForBlueprintExport_MultiHostProperty() throws Exception { Map<String, Map<String, String>> properties = new HashMap<String, Map<String, String>>(); Map<String, String> typeProps = new HashMap<String, String>(); typeProps.put("hbase.zookeeper.quorum", "testhost,testhost2,testhost2a,testhost2b"); properties.put("hbase-site", typeProps); Configuration clusterConfig = new Configuration(properties, Collections.<String, Map<String, Map<String, String>>>emptyMap()); Collection<String> hgComponents = new HashSet<String>(); hgComponents.add("NAMENODE"); hgComponents.add("SECONDARY_NAMENODE"); hgComponents.add("ZOOKEEPER_SERVER"); TestHostGroup group1 = new TestHostGroup("group1", hgComponents, Collections.singleton("testhost")); Collection<String> hgComponents2 = new HashSet<String>(); hgComponents2.add("DATANODE"); hgComponents2.add("HDFS_CLIENT"); hgComponents2.add("ZOOKEEPER_SERVER"); Set<String> hosts2 = new HashSet<String>(); hosts2.add("testhost2"); hosts2.add("testhost2a"); hosts2.add("testhost2b"); TestHostGroup group2 = new TestHostGroup("group2", hgComponents2, hosts2); Collection<String> hgComponents3 = new HashSet<String>(); hgComponents2.add("HDFS_CLIENT"); hgComponents2.add("ZOOKEEPER_CLIENT"); Set<String> hosts3 = new HashSet<String>(); hosts3.add("testhost3"); hosts3.add("testhost3a"); TestHostGroup group3 = new TestHostGroup("group3", hgComponents3, hosts3); Collection<TestHostGroup> hostGroups = new HashSet<TestHostGroup>(); hostGroups.add(group1); hostGroups.add(group2); hostGroups.add(group3); ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); BlueprintConfigurationProcessor configProcessor = new BlueprintConfigurationProcessor(topology); configProcessor.doUpdateForBlueprintExport(); String updatedVal = properties.get("hbase-site").get("hbase.zookeeper.quorum"); assertEquals("%HOSTGROUP::group1%,%HOSTGROUP::group2%", updatedVal); }
@Test public void testDoUpdateForBlueprintExport_MultiHostProperty__WithPorts() throws Exception { Map<String, Map<String, String>> properties = new HashMap<String, Map<String, String>>(); Map<String, String> typeProps = new HashMap<String, String>(); typeProps.put("templeton.zookeeper.hosts", "testhost:5050,testhost2:9090,testhost2a:9090,testhost2b:9090"); properties.put("webhcat-site", typeProps); Configuration clusterConfig = new Configuration(properties, Collections.<String, Map<String, Map<String, String>>>emptyMap()); Collection<String> hgComponents = new HashSet<String>(); hgComponents.add("NAMENODE"); hgComponents.add("SECONDARY_NAMENODE"); hgComponents.add("ZOOKEEPER_SERVER"); TestHostGroup group1 = new TestHostGroup("group1", hgComponents, Collections.singleton("testhost")); Collection<String> hgComponents2 = new HashSet<String>(); hgComponents2.add("DATANODE"); hgComponents2.add("HDFS_CLIENT"); hgComponents2.add("ZOOKEEPER_SERVER"); Set<String> hosts2 = new HashSet<String>(); hosts2.add("testhost2"); hosts2.add("testhost2a"); hosts2.add("testhost2b"); TestHostGroup group2 = new TestHostGroup("group2", hgComponents2, hosts2); Collection<String> hgComponents3 = new HashSet<String>(); hgComponents2.add("HDFS_CLIENT"); hgComponents2.add("ZOOKEEPER_CLIENT"); Set<String> hosts3 = new HashSet<String>(); hosts3.add("testhost3"); hosts3.add("testhost3a"); TestHostGroup group3 = new TestHostGroup("group3", hgComponents3, hosts3); Collection<TestHostGroup> hostGroups = new HashSet<TestHostGroup>(); hostGroups.add(group1); hostGroups.add(group2); hostGroups.add(group3); ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); BlueprintConfigurationProcessor configProcessor = new BlueprintConfigurationProcessor(topology); configProcessor.doUpdateForBlueprintExport(); String updatedVal = properties.get("webhcat-site").get("templeton.zookeeper.hosts"); assertEquals("%HOSTGROUP::group1%:5050,%HOSTGROUP::group2%:9090", updatedVal); }
@Test public void testDoUpdateForBlueprintExport_MultiHostProperty__YAML() throws Exception { Map<String, Map<String, String>> properties = new HashMap<String, Map<String, String>>(); Map<String, String> typeProps = new HashMap<String, String>(); typeProps.put("storm.zookeeper.servers", "['testhost:5050','testhost2:9090','testhost2a:9090','testhost2b:9090']"); typeProps.put("drpc_server_host", "['testhost:5050']"); typeProps.put("storm_ui_server_host", "['testhost:5050']"); typeProps.put("supervisor_hosts", "['testhost:5050','testhost2:9090']"); properties.put("storm-site", typeProps); Configuration clusterConfig = new Configuration(properties, Collections.<String, Map<String, Map<String, String>>>emptyMap()); Collection<String> hgComponents = new HashSet<String>(); hgComponents.add("NAMENODE"); hgComponents.add("SECONDARY_NAMENODE"); hgComponents.add("ZOOKEEPER_SERVER"); hgComponents.add("DRPC_SERVER"); hgComponents.add("STORM_UI_SERVER"); hgComponents.add("SUPERVISOR"); TestHostGroup group1 = new TestHostGroup("group1", hgComponents, Collections.singleton("testhost")); Collection<String> hgComponents2 = new HashSet<String>(); hgComponents2.add("DATANODE"); hgComponents2.add("HDFS_CLIENT"); hgComponents2.add("ZOOKEEPER_SERVER"); hgComponents2.add("SUPERVISOR"); Set<String> hosts2 = new HashSet<String>(); hosts2.add("testhost2"); hosts2.add("testhost2a"); hosts2.add("testhost2b"); TestHostGroup group2 = new TestHostGroup("group2", hgComponents2, hosts2); Collection<String> hgComponents3 = new HashSet<String>(); hgComponents2.add("HDFS_CLIENT"); hgComponents2.add("ZOOKEEPER_CLIENT"); Set<String> hosts3 = new HashSet<String>(); hosts3.add("testhost3"); hosts3.add("testhost3a"); TestHostGroup group3 = new TestHostGroup("group3", hgComponents3, hosts3); Collection<TestHostGroup> hostGroups = new HashSet<TestHostGroup>(); hostGroups.add(group1); hostGroups.add(group2); hostGroups.add(group3); ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); BlueprintConfigurationProcessor configProcessor = new BlueprintConfigurationProcessor(topology); configProcessor.doUpdateForBlueprintExport(); String updatedVal = properties.get("storm-site").get("storm.zookeeper.servers"); assertEquals("['%HOSTGROUP::group1%:5050','%HOSTGROUP::group2%:9090']", updatedVal); String updatedVa2 = properties.get("storm-site").get("drpc_server_host"); assertEquals("['%HOSTGROUP::group1%:5050']", updatedVa2); String updatedVa3 = properties.get("storm-site").get("storm_ui_server_host"); assertEquals("['%HOSTGROUP::group1%:5050']", updatedVa3); String updatedVa4 = properties.get("storm-site").get("supervisor_hosts"); assertEquals("['%HOSTGROUP::group1%:5050','%HOSTGROUP::group2%:9090']", updatedVa4); }
@Test public void testDoUpdateForBlueprintExport_DBHostProperty() throws Exception { Map<String, Map<String, String>> properties = new HashMap<String, Map<String, String>>(); Map<String, String> hiveSiteProps = new HashMap<String, String>(); hiveSiteProps.put("javax.jdo.option.ConnectionURL", "jdbc:mysql: properties.put("hive-site", hiveSiteProps); Configuration clusterConfig = new Configuration(properties, Collections.<String, Map<String, Map<String, String>>>emptyMap()); Collection<String> hgComponents = new HashSet<String>(); hgComponents.add("NAMENODE"); hgComponents.add("SECONDARY_NAMENODE"); hgComponents.add("RESOURCEMANAGER"); hgComponents.add("MYSQL_SERVER"); TestHostGroup group1 = new TestHostGroup("group1", hgComponents, Collections.singleton("testhost")); Collection<String> hgComponents2 = new HashSet<String>(); hgComponents2.add("DATANODE"); hgComponents2.add("HDFS_CLIENT"); TestHostGroup group2 = new TestHostGroup("group2", hgComponents2, Collections.singleton("testhost2")); Collection<TestHostGroup> hostGroups = new HashSet<TestHostGroup>(); hostGroups.add(group1); hostGroups.add(group2); ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); BlueprintConfigurationProcessor configProcessor = new BlueprintConfigurationProcessor(topology); configProcessor.doUpdateForBlueprintExport(); String updatedVal = properties.get("hive-site").get("javax.jdo.option.ConnectionURL"); assertEquals("jdbc:mysql: }
@Test public void testDoUpdateForBlueprintExport_DBHostProperty__External() throws Exception { Map<String, Map<String, String>> properties = new HashMap<String, Map<String, String>>(); Map<String, String> typeProps = new HashMap<String, String>(); typeProps.put("javax.jdo.option.ConnectionURL", "jdbc:mysql: properties.put("hive-site", typeProps); Configuration clusterConfig = new Configuration(properties, Collections.<String, Map<String, Map<String, String>>>emptyMap()); Collection<String> hgComponents = new HashSet<String>(); hgComponents.add("NAMENODE"); hgComponents.add("SECONDARY_NAMENODE"); hgComponents.add("RESOURCEMANAGER"); TestHostGroup group1 = new TestHostGroup("group1", hgComponents, Collections.singleton("testhost")); Collection<String> hgComponents2 = new HashSet<String>(); hgComponents2.add("DATANODE"); hgComponents2.add("HDFS_CLIENT"); TestHostGroup group2 = new TestHostGroup("group2", hgComponents2, Collections.singleton("testhost2")); Collection<TestHostGroup> hostGroups = new HashSet<TestHostGroup>(); hostGroups.add(group1); hostGroups.add(group2); ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); BlueprintConfigurationProcessor configProcessor = new BlueprintConfigurationProcessor(topology); configProcessor.doUpdateForBlueprintExport(); assertFalse(properties.get("hive-site").containsKey("javax.jdo.option.ConnectionURL")); }
@Test public void testDoUpdateForBlueprintExport_PasswordFilterApplied() throws Exception { Map<String, Map<String, String>> properties = new HashMap<String, Map<String, String>>(); Map<String, String> typeProps = new HashMap<String, String>(); typeProps.put("REPOSITORY_CONFIG_PASSWORD", "test-password-one"); typeProps.put("SSL_KEYSTORE_PASSWORD", "test-password-two"); typeProps.put("SSL_TRUSTSTORE_PASSWORD", "test-password-three"); typeProps.put("XAAUDIT.DB.PASSWORD", "test-password-four"); typeProps.put("test.ssl.password", "test-password-five"); typeProps.put("test.password.should.be.included", "test-another-pwd"); Map<String, String> secretProps = new HashMap<String, String>(); secretProps.put("knox_master_secret", "test-secret-one"); secretProps.put("test.secret.should.be.included", "test-another-secret"); Map<String, String> customProps = new HashMap<String, String>(); customProps.put("my_test_PASSWORD", "should be excluded"); customProps.put("PASSWORD_mytest", "should be included"); customProps.put("my_test_SECRET", "should be excluded"); customProps.put("SECRET_mytest", "should be included"); properties.put("ranger-yarn-plugin-properties", typeProps); properties.put("custom-test-properties", customProps); properties.put("secret-test-properties", secretProps); Configuration clusterConfig = new Configuration(properties, Collections.<String, Map<String, Map<String, String>>>emptyMap()); Collection<String> hgComponents = new HashSet<String>(); hgComponents.add("NAMENODE"); hgComponents.add("SECONDARY_NAMENODE"); hgComponents.add("RESOURCEMANAGER"); TestHostGroup group1 = new TestHostGroup("group1", hgComponents, Collections.singleton("testhost")); Collection<String> hgComponents2 = new HashSet<String>(); hgComponents2.add("DATANODE"); hgComponents2.add("HDFS_CLIENT"); TestHostGroup group2 = new TestHostGroup("group2", hgComponents2, Collections.singleton("testhost2")); Collection<TestHostGroup> hostGroups = new HashSet<TestHostGroup>(); hostGroups.add(group1); hostGroups.add(group2); ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); BlueprintConfigurationProcessor configProcessor = new BlueprintConfigurationProcessor(topology); configProcessor.doUpdateForBlueprintExport(); assertEquals("Exported properties map was not of the expected size", 2, properties.get("custom-test-properties").size()); assertEquals("ranger-yarn-plugin-properties config type was not properly exported", 1, properties.get("ranger-yarn-plugin-properties").size()); assertEquals("Exported secret properties map was not of the expected size", 1, properties.get("secret-test-properties").size()); assertFalse("Password property should have been excluded", properties.get("ranger-yarn-plugin-properties").containsKey("REPOSITORY_CONFIG_PASSWORD")); assertFalse("Password property should have been excluded", properties.get("ranger-yarn-plugin-properties").containsKey("SSL_KEYSTORE_PASSWORD")); assertFalse("Password property should have been excluded", properties.get("ranger-yarn-plugin-properties").containsKey("SSL_TRUSTSTORE_PASSWORD")); assertFalse("Password property should have been excluded", properties.get("ranger-yarn-plugin-properties").containsKey("XAAUDIT.DB.PASSWORD")); assertFalse("Password property should have been excluded", properties.get("ranger-yarn-plugin-properties").containsKey("test.ssl.password")); assertTrue("Expected password property not found", properties.get("ranger-yarn-plugin-properties").containsKey("test.password.should.be.included")); assertFalse("Secret property should have been excluded", properties.get("secret-test-properties").containsKey("knox_master_secret")); assertTrue("Expected secret property not found", properties.get("secret-test-properties").containsKey("test.secret.should.be.included")); assertEquals("custom-test-properties type was not properly exported", 2, properties.get("custom-test-properties").size()); assertFalse("Password property should have been excluded", properties.get("custom-test-properties").containsKey("my_test_PASSWORD")); assertTrue("Expected password property not found", properties.get("custom-test-properties").containsKey("PASSWORD_mytest")); assertEquals("Expected password property should not have been modified", "should be included", properties.get("custom-test-properties").get("PASSWORD_mytest")); }
@Test public void testFalconConfigExport() throws Exception { final String expectedHostName = "c6401.apache.ambari.org"; final String expectedPortNum = "808080"; final String expectedHostGroupName = "host_group_1"; Map<String, Map<String, String>> configProperties = new HashMap<String, Map<String, String>>(); Map<String, String> falconStartupProperties = new HashMap<String, String>(); configProperties.put("falcon-startup.properties", falconStartupProperties); falconStartupProperties.put("*.broker.url", expectedHostName + ":" + expectedPortNum); falconStartupProperties.put("*.falcon.service.authentication.kerberos.principal", "falcon/" + expectedHostName + "@EXAMPLE.COM"); falconStartupProperties.put("*.falcon.http.authentication.kerberos.principal", "HTTP/" + expectedHostName + "@EXAMPLE.COM"); Configuration clusterConfig = new Configuration(configProperties, Collections.<String, Map<String, Map<String, String>>>emptyMap()); Collection<String> groupComponents = new HashSet<String>(); groupComponents.add("FALCON_SERVER"); Collection<String> hosts = new ArrayList<String>(); hosts.add(expectedHostName); hosts.add("serverTwo"); TestHostGroup group = new TestHostGroup(expectedHostGroupName, groupComponents, hosts); Collection<TestHostGroup> hostGroups = new HashSet<TestHostGroup>(); hostGroups.add(group); ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); BlueprintConfigurationProcessor configProcessor = new BlueprintConfigurationProcessor(topology); configProcessor.doUpdateForBlueprintExport(); assertEquals("Falcon Broker URL property not properly exported", createExportedAddress(expectedPortNum, expectedHostGroupName), falconStartupProperties.get("*.broker.url")); assertEquals("Falcon Kerberos Principal property not properly exported", "falcon/" + "%HOSTGROUP::" + expectedHostGroupName + "%" + "@EXAMPLE.COM", falconStartupProperties.get("*.falcon.service.authentication.kerberos.principal")); assertEquals("Falcon Kerberos HTTP Principal property not properly exported", "HTTP/" + "%HOSTGROUP::" + expectedHostGroupName + "%" + "@EXAMPLE.COM", falconStartupProperties.get("*.falcon.http.authentication.kerberos.principal")); }
@Test public void testTezConfigExport() throws Exception { final String expectedHostName = "c6401.apache.ambari.org"; final String expectedHostGroupName = "host_group_1"; Map<String, Map<String, String>> configProperties = new HashMap<String, Map<String, String>>(); Map<String, String> tezSiteProperties = new HashMap<String, String>(); configProperties.put("tez-site", tezSiteProperties); tezSiteProperties.put("tez.tez-ui.history-url.base", "http: Configuration clusterConfig = new Configuration(configProperties, Collections.<String, Map<String, Map<String, String>>>emptyMap()); Collection<String> groupComponents = new HashSet<String>(); groupComponents.add("TEZ_CLIENT"); Collection<String> hosts = new ArrayList<String>(); hosts.add(expectedHostName); hosts.add("serverTwo"); TestHostGroup group = new TestHostGroup(expectedHostGroupName, groupComponents, hosts); Collection<TestHostGroup> hostGroups = new HashSet<TestHostGroup>(); hostGroups.add(group); ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); BlueprintConfigurationProcessor configProcessor = new BlueprintConfigurationProcessor(topology); configProcessor.doUpdateForBlueprintExport(); assertFalse("tez.tez-ui.history-url.base should not be present in exported blueprint in tez-site", tezSiteProperties.containsKey("tez.tez-ui.history-url.base")); }
@Test public void testKerberosConfigExport() throws Exception { final String expectedHostName = "c6401.apache.ambari.org"; final String expectedHostGroupName = "host_group_1"; Map<String, Map<String, String>> configProperties = new HashMap<String, Map<String, String>>(); Map<String, String> kerberosEnvProperties = new HashMap<String, String>(); Map<String, String> coreSiteProperties = new HashMap<String, String>(); configProperties.put("kerberos-env", kerberosEnvProperties); configProperties.put("core-site", coreSiteProperties); kerberosEnvProperties.put("admin_server_host", expectedHostName); kerberosEnvProperties.put("kdc_hosts", expectedHostName); coreSiteProperties.put("hadoop.proxyuser.yarn.hosts", expectedHostName); Configuration clusterConfig = new Configuration(configProperties, Collections.<String, Map<String, Map<String, String>>>emptyMap()); Collection<String> groupComponents = new HashSet<String>(); groupComponents.add("TEZ_CLIENT"); groupComponents.add("RESOURCEMANAGER"); Collection<String> hosts = new ArrayList<String>(); hosts.add(expectedHostName); hosts.add("serverTwo"); TestHostGroup group = new TestHostGroup(expectedHostGroupName, groupComponents, hosts); Collection<TestHostGroup> hostGroups = new HashSet<TestHostGroup>(); hostGroups.add(group); ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); BlueprintConfigurationProcessor configProcessor = new BlueprintConfigurationProcessor(topology); configProcessor.doUpdateForBlueprintExport(); assertFalse("admin_server_host should not be present in exported blueprint in kerberos-env", kerberosEnvProperties.containsKey("admin_server_host")); assertFalse("kdc_hosts should not be present in exported blueprint in kerberos-env", kerberosEnvProperties.containsKey("kdc_hosts")); assertEquals("hadoop.proxyuser.yarn.hosts was not exported correctly", createExportedHostName("host_group_1"), coreSiteProperties.get("hadoop.proxyuser.yarn.hosts")); }
@Test public void testDoNameNodeHighAvailabilityExportWithHAEnabled() throws Exception { final String expectedNameService = "mynameservice"; final String expectedHostName = "c6401.apache.ambari.org"; final String expectedPortNum = "808080"; final String expectedNodeOne = "nn1"; final String expectedNodeTwo = "nn2"; final String expectedHostGroupName = "host_group_1"; Map<String, Map<String, String>> configProperties = new HashMap<String, Map<String, String>>(); Map<String, String> hdfsSiteProperties = new HashMap<String, String>(); Map<String, String> coreSiteProperties = new HashMap<String, String>(); Map<String, String> hbaseSiteProperties = new HashMap<String, String>(); configProperties.put("hdfs-site", hdfsSiteProperties); configProperties.put("core-site", coreSiteProperties); configProperties.put("hbase-site", hbaseSiteProperties); hdfsSiteProperties.put("dfs.internal.nameservices", expectedNameService); hdfsSiteProperties.put("dfs.nameservices", expectedNameService); hdfsSiteProperties.put("dfs.ha.namenodes.mynameservice", expectedNodeOne + ", " + expectedNodeTwo); hdfsSiteProperties.put("dfs.namenode.https-address." + expectedNameService + "." + expectedNodeOne, expectedHostName + ":" + expectedPortNum); hdfsSiteProperties.put("dfs.namenode.https-address." + expectedNameService + "." + expectedNodeTwo, expectedHostName + ":" + expectedPortNum); hdfsSiteProperties.put("dfs.namenode.http-address." + expectedNameService + "." + expectedNodeOne, expectedHostName + ":" + expectedPortNum); hdfsSiteProperties.put("dfs.namenode.http-address." + expectedNameService + "." + expectedNodeTwo, expectedHostName + ":" + expectedPortNum); hdfsSiteProperties.put("dfs.namenode.rpc-address." + expectedNameService + "." + expectedNodeOne, expectedHostName + ":" + expectedPortNum); hdfsSiteProperties.put("dfs.namenode.rpc-address." + expectedNameService + "." + expectedNodeTwo, expectedHostName + ":" + expectedPortNum); Configuration clusterConfig = new Configuration(configProperties, Collections.<String, Map<String, Map<String, String>>>emptyMap()); Collection<String> groupComponents = new HashSet<String>(); groupComponents.add("NAMENODE"); Collection<String> hosts = new ArrayList<String>(); hosts.add(expectedHostName); hosts.add("serverTwo"); TestHostGroup group = new TestHostGroup(expectedHostGroupName, groupComponents, hosts); Collection<TestHostGroup> hostGroups = new HashSet<TestHostGroup>(); hostGroups.add(group); ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); BlueprintConfigurationProcessor configProcessor = new BlueprintConfigurationProcessor(topology); configProcessor.doUpdateForBlueprintExport(); assertEquals("HTTPS address HA property not properly exported", createExportedAddress(expectedPortNum, expectedHostGroupName), hdfsSiteProperties.get("dfs.namenode.https-address." + expectedNameService + "." + expectedNodeOne)); assertEquals("HTTPS address HA property not properly exported", createExportedAddress(expectedPortNum, expectedHostGroupName), hdfsSiteProperties.get("dfs.namenode.https-address." + expectedNameService + "." + expectedNodeTwo)); assertEquals("HTTPS address HA property not properly exported", createExportedAddress(expectedPortNum, expectedHostGroupName), hdfsSiteProperties.get("dfs.namenode.http-address." + expectedNameService + "." + expectedNodeOne)); assertEquals("HTTPS address HA property not properly exported", createExportedAddress(expectedPortNum, expectedHostGroupName), hdfsSiteProperties.get("dfs.namenode.http-address." + expectedNameService + "." + expectedNodeTwo)); assertEquals("HTTPS address HA property not properly exported", createExportedAddress(expectedPortNum, expectedHostGroupName), hdfsSiteProperties.get("dfs.namenode.rpc-address." + expectedNameService + "." + expectedNodeOne)); assertEquals("HTTPS address HA property not properly exported", createExportedAddress(expectedPortNum, expectedHostGroupName), hdfsSiteProperties.get("dfs.namenode.rpc-address." + expectedNameService + "." + expectedNodeTwo)); }
@Test public void testDoNameNodeHighAvailabilityExportWithHAEnabledPrimaryNamePreferenceNotExported() throws Exception { final String expectedNameService = "mynameservice"; final String expectedHostName = "c6401.apache.ambari.org"; final String expectedPortNum = "808080"; final String expectedNodeOne = "nn1"; final String expectedNodeTwo = "nn2"; final String expectedHostGroupName = "host_group_1"; Map<String, Map<String, String>> configProperties = new HashMap<>(); Map<String, String> hdfsSiteProperties = new HashMap<>(); Map<String, String> coreSiteProperties = new HashMap<>(); Map<String, String> hbaseSiteProperties = new HashMap<>(); Map<String, String> hadoopEnvProperties = new HashMap<>(); configProperties.put("hdfs-site", hdfsSiteProperties); configProperties.put("core-site", coreSiteProperties); configProperties.put("hbase-site", hbaseSiteProperties); configProperties.put("hadoop-env", hadoopEnvProperties); hdfsSiteProperties.put("dfs.internal.nameservices", expectedNameService); hdfsSiteProperties.put("dfs.nameservices", expectedNameService); hdfsSiteProperties.put("dfs.ha.namenodes.mynameservice", expectedNodeOne + ", " + expectedNodeTwo); hdfsSiteProperties.put("dfs.namenode.https-address." + expectedNameService + "." + expectedNodeOne, expectedHostName + ":" + expectedPortNum); hdfsSiteProperties.put("dfs.namenode.https-address." + expectedNameService + "." + expectedNodeTwo, expectedHostName + ":" + expectedPortNum); hdfsSiteProperties.put("dfs.namenode.http-address." + expectedNameService + "." + expectedNodeOne, expectedHostName + ":" + expectedPortNum); hdfsSiteProperties.put("dfs.namenode.http-address." + expectedNameService + "." + expectedNodeTwo, expectedHostName + ":" + expectedPortNum); hdfsSiteProperties.put("dfs.namenode.rpc-address." + expectedNameService + "." + expectedNodeOne, expectedHostName + ":" + expectedPortNum); hdfsSiteProperties.put("dfs.namenode.rpc-address." + expectedNameService + "." + expectedNodeTwo, expectedHostName + ":" + expectedPortNum); hadoopEnvProperties.put("dfs_ha_initial_namenode_active", expectedHostName); hadoopEnvProperties.put("dfs_ha_initial_namenode_standby", expectedHostName); Configuration clusterConfig = new Configuration(configProperties, Collections.<String, Map<String, Map<String, String>>>emptyMap()); Collection<String> groupComponents = new HashSet<String>(); groupComponents.add("NAMENODE"); Collection<String> hosts = new ArrayList<String>(); hosts.add(expectedHostName); hosts.add("serverTwo"); TestHostGroup group = new TestHostGroup(expectedHostGroupName, groupComponents, hosts); Collection<TestHostGroup> hostGroups = new HashSet<TestHostGroup>(); hostGroups.add(group); ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); BlueprintConfigurationProcessor configProcessor = new BlueprintConfigurationProcessor(topology); configProcessor.doUpdateForBlueprintExport(); assertEquals("HTTPS address HA property not properly exported", createExportedAddress(expectedPortNum, expectedHostGroupName), hdfsSiteProperties.get("dfs.namenode.https-address." + expectedNameService + "." + expectedNodeOne)); assertEquals("HTTPS address HA property not properly exported", createExportedAddress(expectedPortNum, expectedHostGroupName), hdfsSiteProperties.get("dfs.namenode.https-address." + expectedNameService + "." + expectedNodeTwo)); assertEquals("HTTPS address HA property not properly exported", createExportedAddress(expectedPortNum, expectedHostGroupName), hdfsSiteProperties.get("dfs.namenode.http-address." + expectedNameService + "." + expectedNodeOne)); assertEquals("HTTPS address HA property not properly exported", createExportedAddress(expectedPortNum, expectedHostGroupName), hdfsSiteProperties.get("dfs.namenode.http-address." + expectedNameService + "." + expectedNodeTwo)); assertEquals("HTTPS address HA property not properly exported", createExportedAddress(expectedPortNum, expectedHostGroupName), hdfsSiteProperties.get("dfs.namenode.rpc-address." + expectedNameService + "." + expectedNodeOne)); assertEquals("HTTPS address HA property not properly exported", createExportedAddress(expectedPortNum, expectedHostGroupName), hdfsSiteProperties.get("dfs.namenode.rpc-address." + expectedNameService + "." + expectedNodeTwo)); assertNull("Initial NameNode HA property exported although should not have", hadoopEnvProperties.get("dfs_ha_initial_namenode_active")); assertNull("Initial NameNode HA property exported although should not have", hadoopEnvProperties.get("dfs_ha_initial_namenode_standby")); }
@Test public void testDoNameNodeHighAvailabilityExportWithHAEnabledNameServicePropertiesIncluded() throws Exception { final String expectedNameService = "mynameservice"; final String expectedHostName = "c6401.apache.ambari.org"; Map<String, Map<String, String>> configProperties = new HashMap<String, Map<String, String>>(); Map<String, String> coreSiteProperties = new HashMap<String, String>(); Map<String, String> hbaseSiteProperties = new HashMap<String, String>(); Map<String, String> accumuloSiteProperties = new HashMap<String, String>(); configProperties.put("core-site", coreSiteProperties); configProperties.put("hbase-site", hbaseSiteProperties); configProperties.put("accumulo-site", accumuloSiteProperties); coreSiteProperties.put("fs.defaultFS", "hdfs: hbaseSiteProperties.put("hbase.rootdir", "hdfs: accumuloSiteProperties.put("instance.volumes", "hdfs: Configuration clusterConfig = new Configuration(configProperties, Collections.<String, Map<String, Map<String, String>>>emptyMap()); Collection<String> groupComponents = new HashSet<String>(); groupComponents.add("RESOURCEMANAGER"); Collection<String> hosts = new ArrayList<String>(); hosts.add(expectedHostName); hosts.add("serverTwo"); TestHostGroup group = new TestHostGroup("group1", groupComponents, hosts); Collection<TestHostGroup> hostGroups = new HashSet<TestHostGroup>(); hostGroups.add(group); ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); BlueprintConfigurationProcessor configProcessor = new BlueprintConfigurationProcessor(topology); configProcessor.doUpdateForBlueprintExport(); assertEquals("Property containing an HA nameservice (fs.defaultFS), was not correctly exported by the processor", "hdfs: assertEquals("Property containing an HA nameservice (hbase.rootdir), was not correctly exported by the processor", "hdfs: assertEquals("Property containing an HA nameservice (instance.volumes), was not correctly exported by the processor", "hdfs: }
@Test public void testDoNameNodeHighAvailabilityExportWithHANotEnabled() throws Exception { Map<String, Map<String, String>> configProperties = new HashMap<String, Map<String, String>>(); Map<String, String> hdfsSiteProperties = new HashMap<String, String>(); configProperties.put("hdfs-site", hdfsSiteProperties); assertEquals("Incorrect initial state for hdfs-site config", 0, hdfsSiteProperties.size()); Configuration clusterConfig = new Configuration(configProperties, Collections.<String, Map<String, Map<String, String>>>emptyMap()); Collection<String> groupComponents = new HashSet<String>(); groupComponents.add("NAMENODE"); TestHostGroup group = new TestHostGroup("group1", groupComponents, Collections.singleton("host1")); Collection<TestHostGroup> hostGroups = new HashSet<TestHostGroup>(); hostGroups.add(group); ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); BlueprintConfigurationProcessor configProcessor = new BlueprintConfigurationProcessor(topology); configProcessor.doUpdateForBlueprintExport(); assertEquals("Incorrect state for hdfs-site config after HA call in non-HA environment, should be zero", 0, hdfsSiteProperties.size()); }
@Test public void testDoNameNodeHighAvailabilityExportWithHAEnabledMultipleServices() throws Exception { final String expectedNameServiceOne = "mynameserviceOne"; final String expectedNameServiceTwo = "mynameserviceTwo"; final String expectedHostNameOne = "c6401.apache.ambari.org"; final String expectedHostNameTwo = "c6402.apache.ambari.org"; final String expectedPortNum = "808080"; final String expectedNodeOne = "nn1"; final String expectedNodeTwo = "nn2"; final String expectedHostGroupName = "host_group_1"; Map<String, Map<String, String>> configProperties = new HashMap<String, Map<String, String>>(); Map<String, String> hdfsSiteProperties = new HashMap<String, String>(); configProperties.put("hdfs-site", hdfsSiteProperties); hdfsSiteProperties.put("dfs.internal.nameservices", expectedNameServiceOne + "," + expectedNameServiceTwo); hdfsSiteProperties.put("dfs.nameservices", expectedNameServiceOne + "," + expectedNameServiceTwo); hdfsSiteProperties.put("dfs.ha.namenodes." + expectedNameServiceOne, expectedNodeOne + ", " + expectedNodeTwo); hdfsSiteProperties.put("dfs.ha.namenodes." + expectedNameServiceTwo, expectedNodeOne + ", " + expectedNodeTwo); hdfsSiteProperties.put("dfs.namenode.https-address." + expectedNameServiceOne + "." + expectedNodeOne, expectedHostNameOne + ":" + expectedPortNum); hdfsSiteProperties.put("dfs.namenode.https-address." + expectedNameServiceOne + "." + expectedNodeTwo, expectedHostNameOne + ":" + expectedPortNum); hdfsSiteProperties.put("dfs.namenode.http-address." + expectedNameServiceOne + "." + expectedNodeOne, expectedHostNameOne + ":" + expectedPortNum); hdfsSiteProperties.put("dfs.namenode.http-address." + expectedNameServiceOne + "." + expectedNodeTwo, expectedHostNameOne + ":" + expectedPortNum); hdfsSiteProperties.put("dfs.namenode.rpc-address." + expectedNameServiceOne + "." + expectedNodeOne, expectedHostNameOne + ":" + expectedPortNum); hdfsSiteProperties.put("dfs.namenode.rpc-address." + expectedNameServiceOne + "." + expectedNodeTwo, expectedHostNameOne + ":" + expectedPortNum); hdfsSiteProperties.put("dfs.namenode.https-address." + expectedNameServiceTwo + "." + expectedNodeOne, expectedHostNameTwo + ":" + expectedPortNum); hdfsSiteProperties.put("dfs.namenode.https-address." + expectedNameServiceTwo + "." + expectedNodeTwo, expectedHostNameTwo + ":" + expectedPortNum); hdfsSiteProperties.put("dfs.namenode.http-address." + expectedNameServiceTwo + "." + expectedNodeOne, expectedHostNameTwo + ":" + expectedPortNum); hdfsSiteProperties.put("dfs.namenode.http-address." + expectedNameServiceTwo + "." + expectedNodeTwo, expectedHostNameTwo + ":" + expectedPortNum); hdfsSiteProperties.put("dfs.namenode.rpc-address." + expectedNameServiceTwo + "." + expectedNodeOne, expectedHostNameTwo + ":" + expectedPortNum); hdfsSiteProperties.put("dfs.namenode.rpc-address." + expectedNameServiceTwo + "." + expectedNodeTwo, expectedHostNameTwo + ":" + expectedPortNum); Configuration clusterConfig = new Configuration(configProperties, Collections.<String, Map<String, Map<String, String>>>emptyMap()); Collection<String> groupComponents = new HashSet<String>(); groupComponents.add("RESOURCEMANAGER"); Collection<String> hosts = new ArrayList<String>(); hosts.add(expectedHostNameOne); hosts.add(expectedHostNameTwo); hosts.add("serverTwo"); TestHostGroup group = new TestHostGroup(expectedHostGroupName, groupComponents, hosts); Collection<TestHostGroup> hostGroups = new HashSet<TestHostGroup>(); hostGroups.add(group); ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); BlueprintConfigurationProcessor configProcessor = new BlueprintConfigurationProcessor(topology); configProcessor.doUpdateForBlueprintExport(); assertEquals("HTTPS address HA property not properly exported", createExportedAddress(expectedPortNum, expectedHostGroupName), hdfsSiteProperties.get("dfs.namenode.https-address." + expectedNameServiceOne + "." + expectedNodeOne)); assertEquals("HTTPS address HA property not properly exported", createExportedAddress(expectedPortNum, expectedHostGroupName), hdfsSiteProperties.get("dfs.namenode.https-address." + expectedNameServiceOne + "." + expectedNodeTwo)); assertEquals("HTTPS address HA property not properly exported", createExportedAddress(expectedPortNum, expectedHostGroupName), hdfsSiteProperties.get("dfs.namenode.http-address." + expectedNameServiceOne + "." + expectedNodeOne)); assertEquals("HTTPS address HA property not properly exported", createExportedAddress(expectedPortNum, expectedHostGroupName), hdfsSiteProperties.get("dfs.namenode.http-address." + expectedNameServiceOne + "." + expectedNodeTwo)); assertEquals("HTTPS address HA property not properly exported", createExportedAddress(expectedPortNum, expectedHostGroupName), hdfsSiteProperties.get("dfs.namenode.rpc-address." + expectedNameServiceOne + "." + expectedNodeOne)); assertEquals("HTTPS address HA property not properly exported", createExportedAddress(expectedPortNum, expectedHostGroupName), hdfsSiteProperties.get("dfs.namenode.rpc-address." + expectedNameServiceOne + "." + expectedNodeTwo)); assertEquals("HTTPS address HA property not properly exported", createExportedAddress(expectedPortNum, expectedHostGroupName), hdfsSiteProperties.get("dfs.namenode.https-address." + expectedNameServiceTwo + "." + expectedNodeOne)); assertEquals("HTTPS address HA property not properly exported", createExportedAddress(expectedPortNum, expectedHostGroupName), hdfsSiteProperties.get("dfs.namenode.https-address." + expectedNameServiceTwo + "." + expectedNodeTwo)); assertEquals("HTTPS address HA property not properly exported", createExportedAddress(expectedPortNum, expectedHostGroupName), hdfsSiteProperties.get("dfs.namenode.http-address." + expectedNameServiceTwo + "." + expectedNodeOne)); assertEquals("HTTPS address HA property not properly exported", createExportedAddress(expectedPortNum, expectedHostGroupName), hdfsSiteProperties.get("dfs.namenode.http-address." + expectedNameServiceTwo + "." + expectedNodeTwo)); assertEquals("HTTPS address HA property not properly exported", createExportedAddress(expectedPortNum, expectedHostGroupName), hdfsSiteProperties.get("dfs.namenode.rpc-address." + expectedNameServiceTwo + "." + expectedNodeOne)); assertEquals("HTTPS address HA property not properly exported", createExportedAddress(expectedPortNum, expectedHostGroupName), hdfsSiteProperties.get("dfs.namenode.rpc-address." + expectedNameServiceTwo + "." + expectedNodeTwo)); }
@Test public void testYarnConfigExported() throws Exception { final String expectedHostName = "c6401.apache.ambari.org"; final String expectedPortNum = "808080"; final String expectedHostGroupName = "host_group_1"; Map<String, Map<String, String>> configProperties = new HashMap<String, Map<String, String>>(); Map<String, String> yarnSiteProperties = new HashMap<String, String>(); configProperties.put("yarn-site", yarnSiteProperties); yarnSiteProperties.put("yarn.log.server.url", "http: yarnSiteProperties.put("yarn.resourcemanager.hostname", expectedHostName); yarnSiteProperties.put("yarn.resourcemanager.resource-tracker.address", expectedHostName + ":" + expectedPortNum); yarnSiteProperties.put("yarn.resourcemanager.webapp.address", expectedHostName + ":" + expectedPortNum); yarnSiteProperties.put("yarn.resourcemanager.scheduler.address", expectedHostName + ":" + expectedPortNum); yarnSiteProperties.put("yarn.resourcemanager.address", expectedHostName + ":" + expectedPortNum); yarnSiteProperties.put("yarn.resourcemanager.admin.address", expectedHostName + ":" + expectedPortNum); yarnSiteProperties.put("yarn.timeline-service.address", expectedHostName + ":" + expectedPortNum); yarnSiteProperties.put("yarn.timeline-service.webapp.address", expectedHostName + ":" + expectedPortNum); yarnSiteProperties.put("yarn.timeline-service.webapp.https.address", expectedHostName + ":" + expectedPortNum); yarnSiteProperties.put("yarn.log.server.web-service.url", expectedHostName + ":" + expectedPortNum); Configuration clusterConfig = new Configuration(configProperties, Collections.<String, Map<String, Map<String, String>>>emptyMap()); Collection<String> groupComponents = new HashSet<String>(); groupComponents.add("RESOURCEMANAGER"); Collection<String> hosts = new ArrayList<String>(); hosts.add(expectedHostName); hosts.add("serverTwo"); TestHostGroup group = new TestHostGroup(expectedHostGroupName, groupComponents, hosts); Collection<TestHostGroup> hostGroups = new HashSet<TestHostGroup>(); hostGroups.add(group); ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); BlueprintConfigurationProcessor configProcessor = new BlueprintConfigurationProcessor(topology); configProcessor.doUpdateForBlueprintExport(); assertEquals("Yarn Log Server URL was incorrectly exported", "http: assertEquals("Yarn ResourceManager hostname was incorrectly exported", createExportedHostName(expectedHostGroupName), yarnSiteProperties.get("yarn.resourcemanager.hostname")); assertEquals("Yarn ResourceManager tracker address was incorrectly exported", createExportedAddress(expectedPortNum, expectedHostGroupName), yarnSiteProperties.get("yarn.resourcemanager.resource-tracker.address")); assertEquals("Yarn ResourceManager webapp address was incorrectly exported", createExportedAddress(expectedPortNum, expectedHostGroupName), yarnSiteProperties.get("yarn.resourcemanager.webapp.address")); assertEquals("Yarn ResourceManager scheduler address was incorrectly exported", createExportedAddress(expectedPortNum, expectedHostGroupName), yarnSiteProperties.get("yarn.resourcemanager.scheduler.address")); assertEquals("Yarn ResourceManager address was incorrectly exported", createExportedAddress(expectedPortNum, expectedHostGroupName), yarnSiteProperties.get("yarn.resourcemanager.address")); assertEquals("Yarn ResourceManager admin address was incorrectly exported", createExportedAddress(expectedPortNum, expectedHostGroupName), yarnSiteProperties.get("yarn.resourcemanager.admin.address")); assertEquals("Yarn ResourceManager timeline-service address was incorrectly exported", createExportedAddress(expectedPortNum, expectedHostGroupName), yarnSiteProperties.get("yarn.timeline-service.address")); assertEquals("Yarn ResourceManager timeline webapp address was incorrectly exported", createExportedAddress(expectedPortNum, expectedHostGroupName), yarnSiteProperties.get("yarn.timeline-service.webapp.address")); assertEquals("Yarn ResourceManager timeline webapp HTTPS address was incorrectly exported", createExportedAddress(expectedPortNum, expectedHostGroupName), yarnSiteProperties.get("yarn.timeline-service.webapp.https.address")); assertEquals("Yarn ResourceManager timeline web service url was incorrectly exported", createExportedAddress(expectedPortNum, expectedHostGroupName), yarnSiteProperties.get("yarn.log.server.web-service.url")); }
@Test public void testYarnConfigExportedWithDefaultZeroHostAddress() throws Exception { final String expectedHostName = "c6401.apache.ambari.org"; final String expectedPortNum = "808080"; final String expectedHostGroupName = "host_group_1"; Map<String, Map<String, String>> configProperties = new HashMap<String, Map<String, String>>(); Map<String, String> yarnSiteProperties = new HashMap<String, String>(); configProperties.put("yarn-site", yarnSiteProperties); yarnSiteProperties.put("yarn.log.server.url", "http: yarnSiteProperties.put("yarn.resourcemanager.hostname", expectedHostName); yarnSiteProperties.put("yarn.resourcemanager.resource-tracker.address", expectedHostName + ":" + expectedPortNum); yarnSiteProperties.put("yarn.resourcemanager.webapp.address", expectedHostName + ":" + expectedPortNum); yarnSiteProperties.put("yarn.resourcemanager.scheduler.address", expectedHostName + ":" + expectedPortNum); yarnSiteProperties.put("yarn.resourcemanager.address", expectedHostName + ":" + expectedPortNum); yarnSiteProperties.put("yarn.resourcemanager.admin.address", expectedHostName + ":" + expectedPortNum); yarnSiteProperties.put("yarn.timeline-service.address", "0.0.0.0" + ":" + expectedPortNum); yarnSiteProperties.put("yarn.timeline-service.webapp.address", "0.0.0.0" + ":" + expectedPortNum); yarnSiteProperties.put("yarn.timeline-service.webapp.https.address", "0.0.0.0" + ":" + expectedPortNum); Configuration clusterConfig = new Configuration(configProperties, Collections.<String, Map<String, Map<String, String>>>emptyMap()); Collection<String> groupComponents = new HashSet<String>(); groupComponents.add("RESOURCEMANAGER"); Collection<String> hosts = new ArrayList<String>(); hosts.add(expectedHostName); hosts.add("serverTwo"); TestHostGroup group = new TestHostGroup(expectedHostGroupName, groupComponents, hosts); Collection<TestHostGroup> hostGroups = new HashSet<TestHostGroup>(); hostGroups.add(group); ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); BlueprintConfigurationProcessor configProcessor = new BlueprintConfigurationProcessor(topology); configProcessor.doUpdateForBlueprintExport(); assertEquals("Yarn Log Server URL was incorrectly exported", "http: assertEquals("Yarn ResourceManager hostname was incorrectly exported", createExportedHostName(expectedHostGroupName), yarnSiteProperties.get("yarn.resourcemanager.hostname")); assertEquals("Yarn ResourceManager tracker address was incorrectly exported", createExportedAddress(expectedPortNum, expectedHostGroupName), yarnSiteProperties.get("yarn.resourcemanager.resource-tracker.address")); assertEquals("Yarn ResourceManager webapp address was incorrectly exported", createExportedAddress(expectedPortNum, expectedHostGroupName), yarnSiteProperties.get("yarn.resourcemanager.webapp.address")); assertEquals("Yarn ResourceManager scheduler address was incorrectly exported", createExportedAddress(expectedPortNum, expectedHostGroupName), yarnSiteProperties.get("yarn.resourcemanager.scheduler.address")); assertEquals("Yarn ResourceManager address was incorrectly exported", createExportedAddress(expectedPortNum, expectedHostGroupName), yarnSiteProperties.get("yarn.resourcemanager.address")); assertEquals("Yarn ResourceManager admin address was incorrectly exported", createExportedAddress(expectedPortNum, expectedHostGroupName), yarnSiteProperties.get("yarn.resourcemanager.admin.address")); assertEquals("Yarn ResourceManager timeline-service address was incorrectly exported", "0.0.0.0" + ":" + expectedPortNum, yarnSiteProperties.get("yarn.timeline-service.address")); assertEquals("Yarn ResourceManager timeline webapp address was incorrectly exported", "0.0.0.0" + ":" + expectedPortNum, yarnSiteProperties.get("yarn.timeline-service.webapp.address")); assertEquals("Yarn ResourceManager timeline webapp HTTPS address was incorrectly exported", "0.0.0.0" + ":" + expectedPortNum, yarnSiteProperties.get("yarn.timeline-service.webapp.https.address")); }
@Test public void testHDFSConfigExported() throws Exception { final String expectedHostName = "c6401.apache.ambari.org"; final String expectedPortNum = "808080"; final String expectedHostGroupName = "host_group_1"; Map<String, Map<String, String>> configProperties = new HashMap<String, Map<String, String>>(); Map<String, String> hdfsSiteProperties = new HashMap<String, String>(); Map<String, String> coreSiteProperties = new HashMap<String, String>(); Map<String, String> hbaseSiteProperties = new HashMap<String, String>(); Map<String, String> accumuloSiteProperties = new HashMap<String, String>(); configProperties.put("hdfs-site", hdfsSiteProperties); configProperties.put("core-site", coreSiteProperties); configProperties.put("hbase-site", hbaseSiteProperties); configProperties.put("accumulo-site", accumuloSiteProperties); hdfsSiteProperties.put("dfs.http.address", expectedHostName + ":" + expectedPortNum); hdfsSiteProperties.put("dfs.https.address", expectedHostName + ":" + expectedPortNum); hdfsSiteProperties.put("dfs.namenode.http-address", expectedHostName + ":" + expectedPortNum); hdfsSiteProperties.put("dfs.namenode.https-address", expectedHostName + ":" + expectedPortNum); hdfsSiteProperties.put("dfs.secondary.http.address", expectedHostName + ":" + expectedPortNum); hdfsSiteProperties.put("dfs.namenode.secondary.http-address", expectedHostName + ":" + expectedPortNum); hdfsSiteProperties.put("dfs.namenode.shared.edits.dir", expectedHostName + ":" + expectedPortNum); coreSiteProperties.put("fs.default.name", expectedHostName + ":" + expectedPortNum); coreSiteProperties.put("fs.defaultFS", "hdfs: hbaseSiteProperties.put("hbase.rootdir", "hdfs: accumuloSiteProperties.put("instance.volumes", "hdfs: Configuration clusterConfig = new Configuration(configProperties, Collections.<String, Map<String, Map<String, String>>>emptyMap()); Collection<String> groupComponents = new HashSet<String>(); groupComponents.add("NAMENODE"); groupComponents.add("SECONDARY_NAMENODE"); Collection<String> hosts = new ArrayList<String>(); hosts.add(expectedHostName); hosts.add("serverTwo"); TestHostGroup group = new TestHostGroup(expectedHostGroupName, groupComponents, hosts); Collection<TestHostGroup> hostGroups = new HashSet<TestHostGroup>(); hostGroups.add(group); ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); BlueprintConfigurationProcessor configProcessor = new BlueprintConfigurationProcessor(topology); configProcessor.doUpdateForBlueprintExport(); assertEquals("hdfs config property not exported properly", createExportedAddress(expectedPortNum, expectedHostGroupName), hdfsSiteProperties.get("dfs.http.address")); assertEquals("hdfs config property not exported properly", createExportedAddress(expectedPortNum, expectedHostGroupName), hdfsSiteProperties.get("dfs.https.address")); assertEquals("hdfs config property not exported properly", createExportedAddress(expectedPortNum, expectedHostGroupName), hdfsSiteProperties.get("dfs.namenode.http-address")); assertEquals("hdfs config property not exported properly", createExportedAddress(expectedPortNum, expectedHostGroupName), hdfsSiteProperties.get("dfs.namenode.https-address")); assertEquals("hdfs config property not exported properly", createExportedAddress(expectedPortNum, expectedHostGroupName), hdfsSiteProperties.get("dfs.secondary.http.address")); assertEquals("hdfs config property not exported properly", createExportedAddress(expectedPortNum, expectedHostGroupName), hdfsSiteProperties.get("dfs.namenode.secondary.http-address")); assertEquals("hdfs config property not exported properly", createExportedAddress(expectedPortNum, expectedHostGroupName), hdfsSiteProperties.get("dfs.namenode.shared.edits.dir")); assertEquals("hdfs config in core-site not exported properly", createExportedAddress(expectedPortNum, expectedHostGroupName), coreSiteProperties.get("fs.default.name")); assertEquals("hdfs config in core-site not exported properly", "hdfs: assertEquals("hdfs config in hbase-site not exported properly", "hdfs: assertEquals("hdfs config in accumulo-site not exported properly", "hdfs: }
@Test public void testHiveConfigExported() throws Exception { final String expectedHostName = "c6401.apache.ambari.org"; final String expectedHostNameTwo = "c6402.ambari.apache.org"; final String expectedPortNum = "808080"; final String expectedHostGroupName = "host_group_1"; final String expectedHostGroupNameTwo = "host_group_2"; Map<String, Map<String, String>> configProperties = new HashMap<String, Map<String, String>>(); Map<String, String> hiveSiteProperties = new HashMap<String, String>(); Map<String, String> hiveEnvProperties = new HashMap<String, String>(); Map<String, String> webHCatSiteProperties = new HashMap<String, String>(); Map<String, String> coreSiteProperties = new HashMap<String, String>(); configProperties.put("hive-site", hiveSiteProperties); configProperties.put("hive-env", hiveEnvProperties); configProperties.put("webhcat-site", webHCatSiteProperties); configProperties.put("core-site", coreSiteProperties); hiveSiteProperties.put("hive.metastore.uris", "thrift: hiveSiteProperties.put("javax.jdo.option.ConnectionURL", expectedHostName + ":" + expectedPortNum); hiveSiteProperties.put("hive.zookeeper.quorum", expectedHostName + ":" + expectedPortNum + "," + expectedHostNameTwo + ":" + expectedPortNum); hiveSiteProperties.put("hive.cluster.delegation.token.store.zookeeper.connectString", expectedHostName + ":" + expectedPortNum + "," + expectedHostNameTwo + ":" + expectedPortNum); webHCatSiteProperties.put("templeton.hive.properties", expectedHostName + "," + expectedHostNameTwo); webHCatSiteProperties.put("templeton.kerberos.principal", expectedHostName); coreSiteProperties.put("hadoop.proxyuser.hive.hosts", expectedHostName + "," + expectedHostNameTwo); coreSiteProperties.put("hadoop.proxyuser.HTTP.hosts", expectedHostName + "," + expectedHostNameTwo); coreSiteProperties.put("hadoop.proxyuser.hcat.hosts", expectedHostName + "," + expectedHostNameTwo); Configuration clusterConfig = new Configuration(configProperties, Collections.<String, Map<String, Map<String, String>>>emptyMap()); Collection<String> groupComponents = new HashSet<String>(); groupComponents.add("HIVE_SERVER"); Collection<String> hosts = new ArrayList<String>(); hosts.add(expectedHostName); hosts.add("serverTwo"); TestHostGroup group = new TestHostGroup(expectedHostGroupName, groupComponents, hosts); Collection<String> groupComponents2 = new HashSet<String>(); groupComponents2.add("HIVE_CLIENT"); Collection<String> hosts2 = new ArrayList<String>(); hosts2.add(expectedHostNameTwo); hosts2.add("serverFour"); TestHostGroup group2 = new TestHostGroup(expectedHostGroupNameTwo, groupComponents2, hosts2); Collection<TestHostGroup> hostGroups = new HashSet<TestHostGroup>(); hostGroups.add(group); hostGroups.add(group2); ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); BlueprintConfigurationProcessor configProcessor = new BlueprintConfigurationProcessor(topology); configProcessor.doUpdateForBlueprintExport(); assertEquals("hive property not properly exported", "thrift: assertEquals("hive property not properly exported", createExportedAddress(expectedPortNum, expectedHostGroupName), hiveSiteProperties.get("javax.jdo.option.ConnectionURL")); assertEquals("hive property not properly exported", createExportedHostName(expectedHostGroupName) + "," + createExportedHostName(expectedHostGroupNameTwo), webHCatSiteProperties.get("templeton.hive.properties")); assertEquals("hive property not properly exported", createExportedHostName(expectedHostGroupName), webHCatSiteProperties.get("templeton.kerberos.principal")); assertEquals("hive property not properly exported", createExportedHostName(expectedHostGroupName) + "," + createExportedHostName(expectedHostGroupNameTwo), coreSiteProperties.get("hadoop.proxyuser.hive.hosts")); assertEquals("hive property not properly exported", createExportedHostName(expectedHostGroupName) + "," + createExportedHostName(expectedHostGroupNameTwo), coreSiteProperties.get("hadoop.proxyuser.HTTP.hosts")); assertEquals("hive property not properly exported", createExportedHostName(expectedHostGroupName) + "," + createExportedHostName(expectedHostGroupNameTwo), coreSiteProperties.get("hadoop.proxyuser.hcat.hosts")); assertEquals("hive zookeeper quorum property not properly exported", createExportedAddress(expectedPortNum, expectedHostGroupName) + "," + createExportedAddress(expectedPortNum, expectedHostGroupNameTwo), hiveSiteProperties.get("hive.zookeeper.quorum")); assertEquals("hive zookeeper connectString property not properly exported", createExportedAddress(expectedPortNum, expectedHostGroupName) + "," + createExportedAddress(expectedPortNum, expectedHostGroupNameTwo), hiveSiteProperties.get("hive.cluster.delegation.token.store.zookeeper.connectString")); }
@Test public void testHiveConfigExportedMultipleHiveMetaStoreServers() throws Exception { final String expectedHostName = "c6401.apache.ambari.org"; final String expectedHostNameTwo = "c6402.ambari.apache.org"; final String expectedPortNum = "808080"; final String expectedHostGroupName = "host_group_1"; final String expectedHostGroupNameTwo = "host_group_2"; Map<String, Map<String, String>> configProperties = new HashMap<String, Map<String, String>>(); Map<String, String> hiveSiteProperties = new HashMap<String, String>(); Map<String, String> hiveEnvProperties = new HashMap<String, String>(); Map<String, String> webHCatSiteProperties = new HashMap<String, String>(); Map<String, String> coreSiteProperties = new HashMap<String, String>(); configProperties.put("hive-site", hiveSiteProperties); configProperties.put("hive-env", hiveEnvProperties); configProperties.put("webhcat-site", webHCatSiteProperties); configProperties.put("core-site", coreSiteProperties); hiveSiteProperties.put("hive.metastore.uris", "thrift: hiveSiteProperties.put("hive.server2.authentication.ldap.url", "ldap: hiveSiteProperties.put("javax.jdo.option.ConnectionURL", expectedHostName + ":" + expectedPortNum); hiveSiteProperties.put("hive.zookeeper.quorum", expectedHostName + ":" + expectedPortNum + "," + expectedHostNameTwo + ":" + expectedPortNum); hiveSiteProperties.put("hive.cluster.delegation.token.store.zookeeper.connectString", expectedHostName + ":" + expectedPortNum + "," + expectedHostNameTwo + ":" + expectedPortNum); webHCatSiteProperties.put("templeton.hive.properties", expectedHostName + "," + expectedHostNameTwo); webHCatSiteProperties.put("templeton.kerberos.principal", expectedHostName); coreSiteProperties.put("hadoop.proxyuser.hive.hosts", expectedHostName + "," + expectedHostNameTwo); coreSiteProperties.put("hadoop.proxyuser.HTTP.hosts", expectedHostName + "," + expectedHostNameTwo); coreSiteProperties.put("hadoop.proxyuser.hcat.hosts", expectedHostName + "," + expectedHostNameTwo); Configuration clusterConfig = new Configuration(configProperties, Collections.<String, Map<String, Map<String, String>>>emptyMap()); Collection<String> groupComponents = new HashSet<String>(); groupComponents.add("NAMENODE"); Collection<String> hosts = new ArrayList<String>(); hosts.add(expectedHostName); hosts.add("serverTwo"); TestHostGroup group = new TestHostGroup(expectedHostGroupName, groupComponents, hosts); Collection<String> groupComponents2 = new HashSet<String>(); groupComponents2.add("DATANODE"); Collection<String> hosts2 = new ArrayList<String>(); hosts2.add(expectedHostNameTwo); hosts2.add("serverThree"); TestHostGroup group2 = new TestHostGroup(expectedHostGroupNameTwo, groupComponents2, hosts2); Collection<TestHostGroup> hostGroups = new HashSet<TestHostGroup>(); hostGroups.add(group); hostGroups.add(group2); ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); BlueprintConfigurationProcessor configProcessor = new BlueprintConfigurationProcessor(topology); configProcessor.doUpdateForBlueprintExport(); System.out.println("RWN: exported value of hive.metastore.uris = " + hiveSiteProperties.get("hive.metastore.uris")); assertEquals("hive property not properly exported", "thrift: assertEquals("hive property not properly exported", createExportedAddress(expectedPortNum, expectedHostGroupName), hiveSiteProperties.get("javax.jdo.option.ConnectionURL")); assertEquals("hive property not properly exported", createExportedHostName(expectedHostGroupName) + "," + createExportedHostName(expectedHostGroupNameTwo), webHCatSiteProperties.get("templeton.hive.properties")); assertEquals("hive property not properly exported", createExportedHostName(expectedHostGroupName), webHCatSiteProperties.get("templeton.kerberos.principal")); assertEquals("hive property not properly exported", createExportedHostName(expectedHostGroupName) + "," + createExportedHostName(expectedHostGroupNameTwo), coreSiteProperties.get("hadoop.proxyuser.hive.hosts")); assertFalse("hive.server2.authentication.ldap.url should not have been present in the exported configuration", hiveSiteProperties.containsKey("hive.server2.authentication.ldap.url")); assertEquals("hive property not properly exported", createExportedHostName(expectedHostGroupName) + "," + createExportedHostName(expectedHostGroupNameTwo), coreSiteProperties.get("hadoop.proxyuser.HTTP.hosts")); assertEquals("hive property not properly exported", createExportedHostName(expectedHostGroupName) + "," + createExportedHostName(expectedHostGroupNameTwo), coreSiteProperties.get("hadoop.proxyuser.hcat.hosts")); assertEquals("hive zookeeper quorum property not properly exported", createExportedAddress(expectedPortNum, expectedHostGroupName) + "," + createExportedAddress(expectedPortNum, expectedHostGroupNameTwo), hiveSiteProperties.get("hive.zookeeper.quorum")); assertEquals("hive zookeeper connectString property not properly exported", createExportedAddress(expectedPortNum, expectedHostGroupName) + "," + createExportedAddress(expectedPortNum, expectedHostGroupNameTwo), hiveSiteProperties.get("hive.cluster.delegation.token.store.zookeeper.connectString")); }
@Test public void testZookeeperConfigExported() throws Exception { final String expectedHostName = "c6401.apache.ambari.org"; final String expectedHostNameTwo = "c6402.ambari.apache.org"; final String expectedHostGroupName = "host_group_1"; final String expectedHostGroupNameTwo = "host_group_2"; final String expectedPortNumberOne = "2112"; final String expectedPortNumberTwo = "1221"; Map<String, Map<String, String>> configProperties = new HashMap<String, Map<String, String>>(); Map<String, String> coreSiteProperties = new HashMap<String, String>(); Map<String, String> hbaseSiteProperties = new HashMap<String, String>(); Map<String, String> webHCatSiteProperties = new HashMap<String, String>(); Map<String, String> sliderClientProperties = new HashMap<String, String>(); Map<String, String> yarnSiteProperties = new HashMap<String, String>(); Map<String, String> kafkaBrokerProperties = new HashMap<String, String>(); Map<String, String> accumuloSiteProperties = new HashMap<String, String>(); configProperties.put("core-site", coreSiteProperties); configProperties.put("hbase-site", hbaseSiteProperties); configProperties.put("webhcat-site", webHCatSiteProperties); configProperties.put("slider-client", sliderClientProperties); configProperties.put("yarn-site", yarnSiteProperties); configProperties.put("kafka-broker", kafkaBrokerProperties); configProperties.put("accumulo-site", accumuloSiteProperties); coreSiteProperties.put("ha.zookeeper.quorum", expectedHostName + "," + expectedHostNameTwo); hbaseSiteProperties.put("hbase.zookeeper.quorum", expectedHostName + "," + expectedHostNameTwo); webHCatSiteProperties.put("templeton.zookeeper.hosts", expectedHostName + "," + expectedHostNameTwo); yarnSiteProperties.put("hadoop.registry.zk.quorum", createHostAddress(expectedHostName, expectedPortNumberOne) + "," + createHostAddress(expectedHostNameTwo, expectedPortNumberTwo)); sliderClientProperties.put("slider.zookeeper.quorum", createHostAddress(expectedHostName, expectedPortNumberOne) + "," + createHostAddress(expectedHostNameTwo, expectedPortNumberTwo)); kafkaBrokerProperties.put("zookeeper.connect", createHostAddress(expectedHostName, expectedPortNumberOne) + "," + createHostAddress(expectedHostNameTwo, expectedPortNumberTwo)); accumuloSiteProperties.put("instance.zookeeper.host", createHostAddress(expectedHostName, expectedPortNumberOne) + "," + createHostAddress(expectedHostNameTwo, expectedPortNumberTwo)); Configuration clusterConfig = new Configuration(configProperties, Collections.<String, Map<String, Map<String, String>>>emptyMap()); Collection<String> groupComponents = new HashSet<String>(); groupComponents.add("ZOOKEEPER_SERVER"); Collection<String> hosts = new ArrayList<String>(); hosts.add(expectedHostName); hosts.add("serverTwo"); TestHostGroup group = new TestHostGroup(expectedHostGroupName, groupComponents, hosts); Collection<String> groupComponents2 = new HashSet<String>(); groupComponents2.add("ZOOKEEPER_SERVER"); Collection<String> hosts2 = new ArrayList<String>(); hosts2.add(expectedHostNameTwo); hosts2.add("serverFour"); TestHostGroup group2 = new TestHostGroup(expectedHostGroupNameTwo, groupComponents2, hosts2); Collection<TestHostGroup> hostGroups = new HashSet<TestHostGroup>(); hostGroups.add(group); hostGroups.add(group2); ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); BlueprintConfigurationProcessor configProcessor = new BlueprintConfigurationProcessor(topology); configProcessor.doUpdateForBlueprintExport(); assertEquals("zookeeper config not properly exported", createExportedHostName(expectedHostGroupName) + "," + createExportedHostName(expectedHostGroupNameTwo), coreSiteProperties.get("ha.zookeeper.quorum")); assertEquals("zookeeper config not properly exported", createExportedHostName(expectedHostGroupName) + "," + createExportedHostName(expectedHostGroupNameTwo), hbaseSiteProperties.get("hbase.zookeeper.quorum")); assertEquals("zookeeper config not properly exported", createExportedHostName(expectedHostGroupName) + "," + createExportedHostName(expectedHostGroupNameTwo), webHCatSiteProperties.get("templeton.zookeeper.hosts")); assertEquals("yarn-site zookeeper config not properly exported", createExportedHostName(expectedHostGroupName, expectedPortNumberOne) + "," + createExportedHostName(expectedHostGroupNameTwo, expectedPortNumberTwo), yarnSiteProperties.get("hadoop.registry.zk.quorum")); assertEquals("slider-client zookeeper config not properly exported", createExportedHostName(expectedHostGroupName, expectedPortNumberOne) + "," + createExportedHostName(expectedHostGroupNameTwo, expectedPortNumberTwo), sliderClientProperties.get("slider.zookeeper.quorum")); assertEquals("kafka zookeeper config not properly exported", createExportedHostName(expectedHostGroupName, expectedPortNumberOne) + "," + createExportedHostName(expectedHostGroupNameTwo, expectedPortNumberTwo), kafkaBrokerProperties.get("zookeeper.connect")); assertEquals("accumulo-site zookeeper config not properly exported", createExportedHostName(expectedHostGroupName, expectedPortNumberOne) + "," + createExportedHostName(expectedHostGroupNameTwo, expectedPortNumberTwo), accumuloSiteProperties.get("instance.zookeeper.host")); }
@Test public void testKnoxSecurityConfigExported() throws Exception { final String expectedHostName = "c6401.apache.ambari.org"; final String expectedHostNameTwo = "c6402.ambari.apache.org"; final String expectedHostGroupName = "host_group_1"; final String expectedHostGroupNameTwo = "host_group_2"; Map<String, Map<String, String>> configProperties = new HashMap<String, Map<String, String>>(); Map<String, String> coreSiteProperties = new HashMap<String, String>(); Map<String, String> webHCatSiteProperties = new HashMap<String, String>(); Map<String, String> oozieSiteProperties = new HashMap<String, String>(); configProperties.put("core-site", coreSiteProperties); configProperties.put("webhcat-site", webHCatSiteProperties); configProperties.put("oozie-site", oozieSiteProperties); coreSiteProperties.put("hadoop.proxyuser.knox.hosts", expectedHostName + "," + expectedHostNameTwo); webHCatSiteProperties.put("webhcat.proxyuser.knox.hosts", expectedHostName + "," + expectedHostNameTwo); oozieSiteProperties.put("hadoop.proxyuser.knox.hosts", expectedHostName + "," + expectedHostNameTwo); oozieSiteProperties.put("oozie.service.ProxyUserService.proxyuser.knox.hosts", expectedHostName + "," + expectedHostNameTwo); Configuration clusterConfig = new Configuration(configProperties, Collections.<String, Map<String, Map<String, String>>>emptyMap()); Collection<String> groupComponents = new HashSet<String>(); groupComponents.add("KNOX_GATEWAY"); Collection<String> hosts = new ArrayList<String>(); hosts.add(expectedHostName); hosts.add("serverTwo"); TestHostGroup group = new TestHostGroup(expectedHostGroupName, groupComponents, hosts); Collection<String> groupComponents2 = new HashSet<String>(); groupComponents2.add("KNOX_GATEWAY"); Collection<String> hosts2 = new ArrayList<String>(); hosts2.add(expectedHostNameTwo); hosts2.add("serverFour"); TestHostGroup group2 = new TestHostGroup(expectedHostGroupNameTwo, groupComponents2, hosts2); Collection<TestHostGroup> hostGroups = new HashSet<TestHostGroup>(); hostGroups.add(group); hostGroups.add(group2); ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); BlueprintConfigurationProcessor configProcessor = new BlueprintConfigurationProcessor(topology); configProcessor.doUpdateForBlueprintExport(); assertEquals("Knox for core-site config not properly exported", createExportedHostName(expectedHostGroupName) + "," + createExportedHostName(expectedHostGroupNameTwo), coreSiteProperties.get("hadoop.proxyuser.knox.hosts")); assertEquals("Knox config for WebHCat not properly exported", createExportedHostName(expectedHostGroupName) + "," + createExportedHostName(expectedHostGroupNameTwo), webHCatSiteProperties.get("webhcat.proxyuser.knox.hosts")); assertEquals("Knox config for Oozie not properly exported", createExportedHostName(expectedHostGroupName) + "," + createExportedHostName(expectedHostGroupNameTwo), oozieSiteProperties.get("hadoop.proxyuser.knox.hosts")); assertEquals("Knox config for Oozie not properly exported", createExportedHostName(expectedHostGroupName) + "," + createExportedHostName(expectedHostGroupNameTwo), oozieSiteProperties.get("oozie.service.ProxyUserService.proxyuser.knox.hosts")); }
@Test public void testKafkaConfigExported() throws Exception { final String expectedHostName = "c6401.apache.ambari.org"; final String expectedHostGroupName = "host_group_1"; final String expectedPortNumberOne = "2112"; Map<String, Map<String, String>> configProperties = new HashMap<String, Map<String, String>>(); Map<String, String> kafkaBrokerProperties = new HashMap<String, String>(); configProperties.put("kafka-broker", kafkaBrokerProperties); kafkaBrokerProperties.put("kafka.ganglia.metrics.host", createHostAddress(expectedHostName, expectedPortNumberOne)); Configuration clusterConfig = new Configuration(configProperties, Collections.<String, Map<String, Map<String, String>>>emptyMap()); Collection<String> groupComponents = new HashSet<String>(); groupComponents.add("KAFKA_BROKER"); Collection<String> hosts = new ArrayList<String>(); hosts.add(expectedHostName); hosts.add("serverTwo"); TestHostGroup group = new TestHostGroup(expectedHostGroupName, groupComponents, hosts); Collection<String> groupComponents2 = new HashSet<String>(); groupComponents2.add("NAMENODE"); TestHostGroup group2 = new TestHostGroup("group2", groupComponents2, Collections.singleton("group2Host")); Collection<TestHostGroup> hostGroups = new HashSet<TestHostGroup>(); hostGroups.add(group); hostGroups.add(group2); ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); BlueprintConfigurationProcessor configProcessor = new BlueprintConfigurationProcessor(topology); configProcessor.doUpdateForBlueprintExport(); assertEquals("kafka Ganglia config not properly exported", createExportedHostName(expectedHostGroupName, expectedPortNumberOne), kafkaBrokerProperties.get("kafka.ganglia.metrics.host")); }
@Test public void testPropertyWithUndefinedHostisExported() throws Exception { final String expectedHostName = "c6401.apache.ambari.org"; final String expectedHostGroupName = "host_group_1"; Map<String, Map<String, String>> configProperties = new HashMap<String, Map<String, String>>(); Map<String, String> properties = new HashMap<String, String>(); configProperties.put("storm-site", properties); properties.put("storm.zookeeper.servers", expectedHostName); properties.put("nimbus.childopts", "undefined"); properties.put("worker.childopts", "some other info, undefined, more info"); Configuration clusterConfig = new Configuration(configProperties, Collections.<String, Map<String, Map<String, String>>>emptyMap()); Collection<String> groupComponents = new HashSet<String>(); groupComponents.add("ZOOKEEPER_SERVER"); Collection<String> hosts = new ArrayList<String>(); hosts.add(expectedHostName); hosts.add("serverTwo"); TestHostGroup group = new TestHostGroup(expectedHostGroupName, groupComponents, hosts); Collection<TestHostGroup> hostGroups = new HashSet<TestHostGroup>(); hostGroups.add(group); ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); BlueprintConfigurationProcessor configProcessor = new BlueprintConfigurationProcessor(topology); configProcessor.doUpdateForBlueprintExport(); assertEquals("Property was incorrectly exported", "%HOSTGROUP::" + expectedHostGroupName + "%", properties.get("storm.zookeeper.servers")); assertEquals("Property with undefined host was incorrectly exported", "undefined", properties.get("nimbus.childopts")); assertEquals("Property with undefined host was incorrectly exported", "some other info, undefined, more info", properties.get("worker.childopts")); }
@Test public void testHiveInteractiveLlapZookeeperConfigExported() throws Exception { final String expectedHostName = "c6401.apache.ambari.org"; final String expectedHostNameTwo = "c6402.ambari.apache.org"; final String expectedHostGroupName = "host_group_1"; final String expectedHostGroupNameTwo = "host_group_2"; final String llapZkProperty = "hive.llap.zk.sm.connectionString"; Map<String, Map<String, String>> configProperties = new HashMap<String, Map<String, String>>(); Map<String, String> hiveInteractiveSiteProperties = new HashMap<String, String>(); configProperties.put("hive-interactive-site", hiveInteractiveSiteProperties); hiveInteractiveSiteProperties.put(llapZkProperty, createHostAddress(expectedHostName, "2181") + "," + createHostAddress(expectedHostNameTwo, "2181")); Configuration clusterConfig = new Configuration(configProperties, Collections.<String, Map<String, Map<String, String>>>emptyMap()); Collection<String> hgComponents = new HashSet<String>(); hgComponents.add("ZOOKEEPER_SERVER"); TestHostGroup group1 = new TestHostGroup(expectedHostGroupName, hgComponents, Collections.singleton(expectedHostName)); Collection<String> hgComponents2 = new HashSet<String>(); hgComponents2.add("ZOOKEEPER_SERVER"); TestHostGroup group2 = new TestHostGroup(expectedHostGroupNameTwo, hgComponents2, Collections.singleton(expectedHostNameTwo)); Collection<TestHostGroup> hostGroups = new HashSet<TestHostGroup>(); hostGroups.add(group1); hostGroups.add(group2); ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); BlueprintConfigurationProcessor updater = new BlueprintConfigurationProcessor(topology); updater.doUpdateForBlueprintExport(); final String expectedPropertyValue = createExportedAddress("2181", expectedHostGroupName) + "," + createExportedAddress("2181", expectedHostGroupNameTwo); assertEquals("hive.llap.zk.sm.connectionString property not updated correctly", expectedPropertyValue, hiveInteractiveSiteProperties.get(llapZkProperty)); }
@Test public void testOozieHAEnabledExport() throws Exception { final String expectedHostName = "c6401.apache.ambari.org"; final String expectedHostNameTwo = "c6402.ambari.apache.org"; final String expectedExternalHost = "c6408.ambari.apache.org"; final String expectedHostGroupName = "host_group_1"; final String expectedHostGroupNameTwo = "host_group_2"; final String expectedPortNum = "80000"; Map<String, Map<String, String>> configProperties = new HashMap<String, Map<String, String>>(); Map<String, String> oozieSiteProperties = new HashMap<String, String>(); Map<String, String> oozieEnvProperties = new HashMap<String, String>(); Map<String, String> coreSiteProperties = new HashMap<String, String>(); configProperties.put("oozie-site", oozieSiteProperties); configProperties.put("oozie-env", oozieEnvProperties); configProperties.put("hive-env", oozieEnvProperties); configProperties.put("core-site", coreSiteProperties); oozieSiteProperties.put("oozie.base.url", expectedHostName + ":" + expectedPortNum); oozieSiteProperties.put("oozie.authentication.kerberos.principal", expectedHostName); oozieSiteProperties.put("oozie.service.HadoopAccessorService.kerberos.principal", expectedHostName); oozieSiteProperties.put("oozie.service.JPAService.jdbc.url", "jdbc:mysql: oozieSiteProperties.put("oozie.services.ext", "org.apache.oozie.service.ZKLocksService,org.apache.oozie.service.ZKXLogStreamingService,org.apache.oozie.service.ZKJobsConcurrencyService,org.apache.oozie.service.ZKUUIDService"); oozieSiteProperties.put("oozie.zookeeper.connection.string", createHostAddress(expectedHostName, "2181") + "," + createHostAddress(expectedHostNameTwo, "2181")); oozieEnvProperties.put("oozie_existing_mysql_host", expectedExternalHost); coreSiteProperties.put("hadoop.proxyuser.oozie.hosts", expectedHostName + "," + expectedHostNameTwo); Configuration clusterConfig = new Configuration(configProperties, Collections.<String, Map<String, Map<String, String>>>emptyMap()); Collection<String> hgComponents = new HashSet<String>(); hgComponents.add("OOZIE_SERVER"); hgComponents.add("ZOOKEEPER_SERVER"); TestHostGroup group1 = new TestHostGroup(expectedHostGroupName, hgComponents, Collections.singleton(expectedHostName)); Collection<String> hgComponents2 = new HashSet<String>(); hgComponents2.add("OOZIE_SERVER"); hgComponents2.add("ZOOKEEPER_SERVER"); TestHostGroup group2 = new TestHostGroup(expectedHostGroupNameTwo, hgComponents2, Collections.singleton(expectedHostNameTwo)); Collection<TestHostGroup> hostGroups = new HashSet<TestHostGroup>(); hostGroups.add(group1); hostGroups.add(group2); expect(stack.getCardinality("OOZIE_SERVER")).andReturn(new Cardinality("1+")).anyTimes(); ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); BlueprintConfigurationProcessor updater = new BlueprintConfigurationProcessor(topology); updater.doUpdateForBlueprintExport(); assertEquals("oozie property not updated correctly", createExportedHostName(expectedHostGroupName, expectedPortNum), oozieSiteProperties.get("oozie.base.url")); assertEquals("oozie property not updated correctly", createExportedHostName(expectedHostGroupName), oozieSiteProperties.get("oozie.authentication.kerberos.principal")); assertEquals("oozie property not updated correctly", createExportedHostName(expectedHostGroupName), oozieSiteProperties.get("oozie.service.HadoopAccessorService.kerberos.principal")); assertEquals("oozie property not updated correctly", createExportedHostName(expectedHostGroupName) + "," + createExportedHostName(expectedHostGroupNameTwo), coreSiteProperties.get("hadoop.proxyuser.oozie.hosts")); assertEquals("oozie property not updated correctly", createExportedAddress("2181", expectedHostGroupName) + "," + createExportedAddress("2181", expectedHostGroupNameTwo), oozieSiteProperties.get("oozie.zookeeper.connection.string")); }
@Test public void testYarnHighAvailabilityExport() throws Exception { final String expectedHostName = "c6401.apache.ambari.org"; final String expectedHostNameTwo = "c6402.apache.ambari.org"; final String expectedPortNum = "808080"; final String expectedHostGroupName = "host_group_1"; final String expectedHostGroupNameTwo = "host_group_2"; Map<String, Map<String, String>> configProperties = new HashMap<String, Map<String, String>>(); Map<String, String> yarnSiteProperties = new HashMap<String, String>(); configProperties.put("yarn-site", yarnSiteProperties); yarnSiteProperties.put("yarn.log.server.url", "http: yarnSiteProperties.put("yarn.resourcemanager.hostname", expectedHostName); yarnSiteProperties.put("yarn.resourcemanager.resource-tracker.address", expectedHostName + ":" + expectedPortNum); yarnSiteProperties.put("yarn.resourcemanager.webapp.address", expectedHostName + ":" + expectedPortNum); yarnSiteProperties.put("yarn.resourcemanager.scheduler.address", expectedHostName + ":" + expectedPortNum); yarnSiteProperties.put("yarn.resourcemanager.address", expectedHostName + ":" + expectedPortNum); yarnSiteProperties.put("yarn.resourcemanager.admin.address", expectedHostName + ":" + expectedPortNum); yarnSiteProperties.put("yarn.timeline-service.address", expectedHostName + ":" + expectedPortNum); yarnSiteProperties.put("yarn.timeline-service.webapp.address", expectedHostName + ":" + expectedPortNum); yarnSiteProperties.put("yarn.timeline-service.webapp.https.address", expectedHostName + ":" + expectedPortNum); yarnSiteProperties.put("yarn.resourcemanager.ha.enabled", "true"); yarnSiteProperties.put("yarn.resourcemanager.ha.rm-ids", "rm1, rm2"); yarnSiteProperties.put("yarn.resourcemanager.hostname.rm1", expectedHostName); yarnSiteProperties.put("yarn.resourcemanager.hostname.rm2", expectedHostNameTwo); yarnSiteProperties.put("yarn.resourcemanager.webapp.address.rm1", expectedHostName + ":" + expectedPortNum); yarnSiteProperties.put("yarn.resourcemanager.webapp.address.rm2", expectedHostNameTwo + ":" + expectedPortNum); yarnSiteProperties.put("yarn.resourcemanager.webapp.https.address.rm1", expectedHostName + ":" + expectedPortNum); yarnSiteProperties.put("yarn.resourcemanager.webapp.https.address.rm2", expectedHostNameTwo + ":" + expectedPortNum); yarnSiteProperties.put("yarn.resourcemanager.zk-address", expectedHostName + ":" + "2181" + "," + expectedHostNameTwo + ":" + "2181"); yarnSiteProperties.put("yarn.resourcemanager.webapp.https.address", expectedHostName + ":" + "8080"); Configuration clusterConfig = new Configuration(configProperties, Collections.<String, Map<String, Map<String, String>>>emptyMap()); Collection<String> hgComponents = new HashSet<String>(); hgComponents.add("RESOURCEMANAGER"); hgComponents.add("APP_TIMELINE_SERVER"); hgComponents.add("HISTORYSERVER"); TestHostGroup group1 = new TestHostGroup(expectedHostGroupName, hgComponents, Collections.singleton(expectedHostName)); Collection<String> hgComponents2 = new HashSet<String>(); hgComponents2.add("RESOURCEMANAGER"); TestHostGroup group2 = new TestHostGroup(expectedHostGroupNameTwo, hgComponents2, Collections.singleton(expectedHostNameTwo)); Collection<TestHostGroup> hostGroups = new HashSet<TestHostGroup>(); hostGroups.add(group1); hostGroups.add(group2); expect(stack.getCardinality("RESOURCEMANAGER")).andReturn(new Cardinality("1-2")).anyTimes(); ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); BlueprintConfigurationProcessor updater = new BlueprintConfigurationProcessor(topology); updater.doUpdateForBlueprintExport(); assertEquals("Yarn Log Server URL was incorrectly updated", "http: assertEquals("Yarn ResourceManager hostname was incorrectly updated", createExportedHostName(expectedHostGroupName), yarnSiteProperties.get("yarn.resourcemanager.hostname")); assertEquals("Yarn ResourceManager tracker address was incorrectly updated", createExportedHostName(expectedHostGroupName, expectedPortNum), yarnSiteProperties.get("yarn.resourcemanager.resource-tracker.address")); assertEquals("Yarn ResourceManager webapp address was incorrectly updated", createExportedHostName(expectedHostGroupName, expectedPortNum), yarnSiteProperties.get("yarn.resourcemanager.webapp.address")); assertEquals("Yarn ResourceManager scheduler address was incorrectly updated", createExportedHostName(expectedHostGroupName, expectedPortNum), yarnSiteProperties.get("yarn.resourcemanager.scheduler.address")); assertEquals("Yarn ResourceManager address was incorrectly updated", createExportedHostName(expectedHostGroupName, expectedPortNum), yarnSiteProperties.get("yarn.resourcemanager.address")); assertEquals("Yarn ResourceManager admin address was incorrectly updated", createExportedHostName(expectedHostGroupName, expectedPortNum), yarnSiteProperties.get("yarn.resourcemanager.admin.address")); assertEquals("Yarn ResourceManager timeline-service address was incorrectly updated", createExportedHostName(expectedHostGroupName, expectedPortNum), yarnSiteProperties.get("yarn.timeline-service.address")); assertEquals("Yarn ResourceManager timeline webapp address was incorrectly updated", createExportedHostName(expectedHostGroupName, expectedPortNum), yarnSiteProperties.get("yarn.timeline-service.webapp.address")); assertEquals("Yarn ResourceManager timeline webapp HTTPS address was incorrectly updated", createExportedHostName(expectedHostGroupName, expectedPortNum), yarnSiteProperties.get("yarn.timeline-service.webapp.https.address")); assertEquals("Yarn ResourceManager rm1 hostname not exported properly", createExportedHostName(expectedHostGroupName), yarnSiteProperties.get("yarn.resourcemanager.hostname.rm1")); assertEquals("Yarn ResourceManager rm2 hostname not exported properly", createExportedHostName(expectedHostGroupNameTwo), yarnSiteProperties.get("yarn.resourcemanager.hostname.rm2")); assertEquals("Yarn ResourceManager rm1 web address not exported properly", createExportedHostName(expectedHostGroupName, expectedPortNum), yarnSiteProperties.get("yarn.resourcemanager.webapp.address.rm1")); assertEquals("Yarn ResourceManager rm2 web address not exported properly", createExportedHostName(expectedHostGroupNameTwo, expectedPortNum), yarnSiteProperties.get("yarn.resourcemanager.webapp.address.rm2")); assertEquals("Yarn ResourceManager rm1 HTTPS address not exported properly", createExportedHostName(expectedHostGroupName, expectedPortNum), yarnSiteProperties.get("yarn.resourcemanager.webapp.https.address.rm1")); assertEquals("Yarn ResourceManager rm2 HTTPS address not exported properly", createExportedHostName(expectedHostGroupNameTwo, expectedPortNum), yarnSiteProperties.get("yarn.resourcemanager.webapp.https.address.rm2")); assertEquals("Yarn Zookeeper address property not exported properly", createExportedHostName(expectedHostGroupName, "2181") + "," + createExportedHostName(expectedHostGroupNameTwo, "2181"), yarnSiteProperties.get("yarn.resourcemanager.zk-address")); assertEquals("Yarn RM webapp address not exported properly", createExportedHostName(expectedHostGroupName, "8080"), yarnSiteProperties.get("yarn.resourcemanager.webapp.https.address")); }
@Test public void testRangerEnvBlueprintExport() throws Exception { List<String> configTypesWithRangerHdfsAuditDir = ImmutableList.of( "ranger-env", "ranger-yarn-audit", "ranger-hdfs-audit", "ranger-hbase-audit", "ranger-hive-audit", "ranger-knox-audit", "ranger-kafka-audit", "ranger-storm-audit", "ranger-atlas-audit" ); Map<String, Map<String, String>> clusterConfigProperties = new HashMap<>(); for (String configType: configTypesWithRangerHdfsAuditDir) { Map<String, String> configProperties = new HashMap<>(); configProperties.put("xasecure.audit.destination.hdfs.dir", "hdfs: clusterConfigProperties.put(configType, configProperties); } Map<String, Map<String, String>> parentProperties = new HashMap<>(); Configuration parentClusterConfig = new Configuration(parentProperties, new HashMap<String, Map<String, Map<String, String>>>()); Configuration clusterConfig = new Configuration(clusterConfigProperties, new HashMap<String, Map<String, Map<String, String>>>(), parentClusterConfig); Collection<String> rangerComponents = new HashSet<>(); rangerComponents.add("RANGER_ADMIN"); rangerComponents.add("RANGER_USERSYNC"); Collection<String> hdfsComponents = new HashSet<String>(); hdfsComponents.add("NAMENODE"); hdfsComponents.add("DATANODE"); TestHostGroup group1 = new TestHostGroup("group1", rangerComponents, Collections.singleton("host1")); group1.components.add("DATANODE"); TestHostGroup group2 = new TestHostGroup("group2", hdfsComponents, Collections.singleton("nn_host")); Collection<TestHostGroup> hostGroups = Lists.newArrayList(group1, group2); ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); BlueprintConfigurationProcessor configProcessor = new BlueprintConfigurationProcessor(topology); configProcessor.doUpdateForBlueprintExport(); String expectedAuditHdfsDir = "hdfs: assertEquals(expectedAuditHdfsDir, clusterConfig.getPropertyValue("ranger-env", "xasecure.audit.destination.hdfs.dir")); assertEquals(expectedAuditHdfsDir, clusterConfig.getPropertyValue("ranger-yarn-audit", "xasecure.audit.destination.hdfs.dir")); assertEquals(expectedAuditHdfsDir, clusterConfig.getPropertyValue("ranger-hdfs-audit", "xasecure.audit.destination.hdfs.dir")); assertEquals(expectedAuditHdfsDir, clusterConfig.getPropertyValue("ranger-hbase-audit", "xasecure.audit.destination.hdfs.dir")); assertEquals(expectedAuditHdfsDir, clusterConfig.getPropertyValue("ranger-hive-audit", "xasecure.audit.destination.hdfs.dir")); assertEquals(expectedAuditHdfsDir, clusterConfig.getPropertyValue("ranger-knox-audit", "xasecure.audit.destination.hdfs.dir")); assertEquals(expectedAuditHdfsDir, clusterConfig.getPropertyValue("ranger-kafka-audit", "xasecure.audit.destination.hdfs.dir")); assertEquals(expectedAuditHdfsDir, clusterConfig.getPropertyValue("ranger-storm-audit", "xasecure.audit.destination.hdfs.dir")); assertEquals(expectedAuditHdfsDir, clusterConfig.getPropertyValue("ranger-atlas-audit", "xasecure.audit.destination.hdfs.dir")); }
@Test public void testRangerEnvExportBlueprintWithHdfsHA() throws Exception { List<String> configTypesWithRangerHdfsAuditDir = ImmutableList.of( "ranger-env", "ranger-yarn-audit", "ranger-hdfs-audit", "ranger-hbase-audit", "ranger-hive-audit", "ranger-knox-audit", "ranger-kafka-audit", "ranger-storm-audit", "ranger-atlas-audit" ); Map<String, Map<String, String>> clusterConfigProperties = new HashMap<>(); for (String configType: configTypesWithRangerHdfsAuditDir) { Map<String, String> configProperties = new HashMap<>(); configProperties.put("xasecure.audit.destination.hdfs.dir", "hdfs: clusterConfigProperties.put(configType, configProperties); } final String hdfsSiteConfigType = "hdfs-site"; Map<String, String> hdfsSiteProperties = new HashMap<>(); clusterConfigProperties.put(hdfsSiteConfigType, hdfsSiteProperties); hdfsSiteProperties.put("dfs.nameservices", "my_name_service"); hdfsSiteProperties.put("dfs.ha.namenodes.my_name_service", "nn1,nn2"); Map<String, Map<String, String>> parentProperties = new HashMap<>(); Configuration parentClusterConfig = new Configuration(parentProperties, new HashMap<String, Map<String, Map<String, String>>>()); Configuration clusterConfig = new Configuration(clusterConfigProperties, new HashMap<String, Map<String, Map<String, String>>>(), parentClusterConfig); Collection<String> rangerComponents = new HashSet<>(); rangerComponents.add("RANGER_ADMIN"); rangerComponents.add("RANGER_USERSYNC"); Collection<String> hdfsComponents = new HashSet<String>(); hdfsComponents.add("NAMENODE"); hdfsComponents.add("DATANODE"); TestHostGroup group1 = new TestHostGroup("group1", rangerComponents, Collections.singleton("host1")); group1.components.addAll(hdfsComponents); TestHostGroup group2 = new TestHostGroup("group2", hdfsComponents, Collections.singleton("host2")); Collection<TestHostGroup> hostGroups = Lists.newArrayList(group1, group2); ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); BlueprintConfigurationProcessor configProcessor = new BlueprintConfigurationProcessor(topology); configProcessor.doUpdateForBlueprintExport(); String expectedAuditHdfsDir = "hdfs: assertEquals(expectedAuditHdfsDir, clusterConfig.getPropertyValue("ranger-env", "xasecure.audit.destination.hdfs.dir")); assertEquals(expectedAuditHdfsDir, clusterConfig.getPropertyValue("ranger-yarn-audit", "xasecure.audit.destination.hdfs.dir")); assertEquals(expectedAuditHdfsDir, clusterConfig.getPropertyValue("ranger-hdfs-audit", "xasecure.audit.destination.hdfs.dir")); assertEquals(expectedAuditHdfsDir, clusterConfig.getPropertyValue("ranger-hbase-audit", "xasecure.audit.destination.hdfs.dir")); assertEquals(expectedAuditHdfsDir, clusterConfig.getPropertyValue("ranger-hive-audit", "xasecure.audit.destination.hdfs.dir")); assertEquals(expectedAuditHdfsDir, clusterConfig.getPropertyValue("ranger-knox-audit", "xasecure.audit.destination.hdfs.dir")); assertEquals(expectedAuditHdfsDir, clusterConfig.getPropertyValue("ranger-kafka-audit", "xasecure.audit.destination.hdfs.dir")); assertEquals(expectedAuditHdfsDir, clusterConfig.getPropertyValue("ranger-storm-audit", "xasecure.audit.destination.hdfs.dir")); assertEquals(expectedAuditHdfsDir, clusterConfig.getPropertyValue("ranger-atlas-audit", "xasecure.audit.destination.hdfs.dir")); }
@Test public void testDoUpdateForBlueprintExport_NonTopologyProperty__AtlasClusterName() throws Exception { Map<String, Map<String, String>> properties = new HashMap<String, Map<String, String>>(); Configuration clusterConfig = new Configuration(properties, Collections.<String, Map<String, Map<String, String>>>emptyMap()); Collection<String> hgComponents = new HashSet<String>(); hgComponents.add("ATLAS_SERVER"); TestHostGroup group1 = new TestHostGroup("group1", hgComponents, Collections.singleton("testhost")); Collection<TestHostGroup> hostGroups = new HashSet<TestHostGroup>(); hostGroups.add(group1); ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); Long clusterId = topology.getClusterId(); Map<String, String> typeProps = new HashMap<String, String>(); typeProps.put("atlas.cluster.name", String.valueOf(clusterId)); properties.put("hive-site", typeProps); BlueprintConfigurationProcessor configProcessor = new BlueprintConfigurationProcessor(topology); configProcessor.doUpdateForBlueprintExport(); String updatedVal = properties.get("hive-site").get("atlas.cluster.name"); assertEquals("primary", updatedVal); }
@Test public void testDoUpdateForBlueprintExport_NonTopologyProperty() throws Exception { String someString = "String.To.Represent.A.String.Value"; Map<String, Map<String, String>> properties = new HashMap<String, Map<String, String>>(); Configuration clusterConfig = new Configuration(properties, Collections.<String, Map<String, Map<String, String>>>emptyMap()); Collection<String> hgComponents = new HashSet<String>(); hgComponents.add("ATLAS_SERVER"); hgComponents.add("HIVE_SERVER"); hgComponents.add("KAFKA_BROKER"); hgComponents.add("NIMBUS"); TestHostGroup group1 = new TestHostGroup("group1", hgComponents, Collections.singleton("testhost")); Collection<TestHostGroup> hostGroups = new HashSet<TestHostGroup>(); hostGroups.add(group1); ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); Long clusterId = topology.getClusterId(); Map<String, String> hiveSiteProps = new HashMap<String, String>(); hiveSiteProps.put("hive.exec.post.hooks", someString); properties.put("hive-site", hiveSiteProps); Map<String, String> kafkaBrokerProps = new HashMap<String, String>(); kafkaBrokerProps.put("kafka.metrics.reporters", someString); properties.put("kafka-broker", kafkaBrokerProps); Map<String, String> stormSiteProps = new HashMap<String, String>(); stormSiteProps.put("metrics.reporter.register", someString); properties.put("storm-site", stormSiteProps); BlueprintConfigurationProcessor configProcessor = new BlueprintConfigurationProcessor(topology); configProcessor.doUpdateForBlueprintExport(); String hiveExecPostHooks = properties.get("hive-site").get("hive.exec.post.hooks"); String kafkaMetricsReporters = properties.get("kafka-broker").get("kafka.metrics.reporters"); String metricsReporterRegister = properties.get("storm-site").get("metrics.reporter.register"); assertEquals(someString, hiveExecPostHooks); assertEquals(someString, kafkaMetricsReporters); assertEquals(someString, metricsReporterRegister); }
@Test public void testStackPasswordPropertyFilter() throws Exception{ Map<String, Map<String, String>> properties = new HashMap<String, Map<String, String>>(); Map<String, String> rangerAdminSiteProps = new HashMap<String, String>(); rangerAdminSiteProps.put("ranger.service.https.attrib.keystore.pass", "SECRET:admin-prp:1:ranger.service.pass"); properties.put("ranger-admin-site", rangerAdminSiteProps); Map<String, Map<String, String>> parentProperties = new HashMap<String, Map<String, String>>(); Configuration parentClusterConfig = new Configuration(parentProperties, Collections.<String, Map<String, Map<String, String>>>emptyMap()); Configuration clusterConfig = new Configuration(properties, Collections.<String, Map<String, Map<String, String>>>emptyMap(), parentClusterConfig); Collection<String> hgComponents = new HashSet<String>(); hgComponents.add("NAMENODE"); hgComponents.add("SECONDARY_NAMENODE"); hgComponents.add("RESOURCEMANAGER"); TestHostGroup group1 = new TestHostGroup("group1", hgComponents, Collections.singleton("testhost")); Collection<String> hgComponents2 = new HashSet<String>(); hgComponents2.add("DATANODE"); hgComponents2.add("HDFS_CLIENT"); TestHostGroup group2 = new TestHostGroup("group2", hgComponents2, Collections.singleton("testhost2")); Collection<TestHostGroup> hostGroups = new HashSet<TestHostGroup>(); hostGroups.add(group1); hostGroups.add(group2); expect(stack.isPasswordProperty((String) anyObject(),(String) anyObject(),(String) anyObject())).andReturn(true).once(); ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); BlueprintConfigurationProcessor configProcessor = new BlueprintConfigurationProcessor(topology); configProcessor.doUpdateForBlueprintExport(); assertFalse(properties.get("ranger-admin-site").containsKey("ranger.service.https.attrib.keystore.pass")); }