BlueprintConfigurationProcessor { public Set<String> doUpdateForClusterCreate() throws ConfigurationTopologyException { Set<String> configTypesUpdated = new HashSet<String>(); Configuration clusterConfig = clusterTopology.getConfiguration(); Map<String, HostGroupInfo> groupInfoMap = clusterTopology.getHostGroupInfo(); doRecommendConfigurations(clusterConfig, configTypesUpdated); doFilterPriorToClusterUpdate(clusterConfig, configTypesUpdated); Map<String, Map<String, String>> clusterProps = clusterConfig.getFullProperties(); for (Map<String, Map<String, PropertyUpdater>> updaterMap : createCollectionOfUpdaters()) { for (Map.Entry<String, Map<String, PropertyUpdater>> entry : updaterMap.entrySet()) { String type = entry.getKey(); for (Map.Entry<String, PropertyUpdater> updaterEntry : entry.getValue().entrySet()) { String propertyName = updaterEntry.getKey(); PropertyUpdater updater = updaterEntry.getValue(); Map<String, String> typeMap = clusterProps.get(type); if (typeMap != null && typeMap.containsKey(propertyName) && typeMap.get(propertyName) != null) { final String originalValue = typeMap.get(propertyName); final String updatedValue = updater.updateForClusterCreate(propertyName, originalValue, clusterProps, clusterTopology); if(updatedValue == null ) { continue; } if (!updatedValue.equals(originalValue)) { configTypesUpdated.add(type); } clusterConfig.setProperty(type, propertyName, updatedValue); } for (HostGroupInfo groupInfo : groupInfoMap.values()) { Configuration hgConfig = groupInfo.getConfiguration(); Map<String, Map<String, String>> hgConfigProps = hgConfig.getFullProperties(1); Map<String, String> hgTypeMap = hgConfigProps.get(type); if (hgTypeMap != null && hgTypeMap.containsKey(propertyName)) { final String originalValue = hgTypeMap.get(propertyName); final String updatedValue = updater.updateForClusterCreate(propertyName, originalValue, hgConfigProps, clusterTopology); if (!updatedValue.equals(originalValue)) { configTypesUpdated.add(type); } hgConfig.setProperty(type, propertyName, updatedValue); } } } } } if (clusterTopology.isNameNodeHAEnabled()) { Map<String, String> hdfsSiteConfig = clusterConfig.getFullProperties().get("hdfs-site"); String nameservices = hdfsSiteConfig.get("dfs.nameservices"); String int_nameservices = hdfsSiteConfig.get("dfs.internal.nameservices"); if(int_nameservices == null && nameservices != null) { clusterConfig.setProperty("hdfs-site", "dfs.internal.nameservices", nameservices); } if (! isNameNodeHAInitialActiveNodeSet(clusterProps) && ! isNameNodeHAInitialStandbyNodeSet(clusterProps)) { Collection<String> nnHosts = clusterTopology.getHostAssignmentsForComponent("NAMENODE"); if (nnHosts.size() != 2) { throw new ConfigurationTopologyException("NAMENODE HA requires exactly 2 hosts running NAMENODE but there are: " + nnHosts.size() + " Hosts: " + nnHosts); } Iterator<String> nnHostIterator = nnHosts.iterator(); clusterConfig.setProperty("hadoop-env", "dfs_ha_initial_namenode_active", nnHostIterator.next()); clusterConfig.setProperty("hadoop-env", "dfs_ha_initial_namenode_standby", nnHostIterator.next()); configTypesUpdated.add("hadoop-env"); } } setMissingConfigurations(clusterConfig, configTypesUpdated); trimProperties(clusterConfig, clusterTopology); return configTypesUpdated; } BlueprintConfigurationProcessor(ClusterTopology clusterTopology); Map<String, Map<String, PropertyUpdater>> getRemovePropertyUpdaters(); void initRemovePropertyUpdaters(); Collection<String> getRequiredHostGroups(); Set<String> doUpdateForClusterCreate(); void doUpdateForBlueprintExport(); void doNameNodeHAUpdate(); void doYarnResourceManagerHAUpdate(); void doOozieServerHAUpdate();  }
@Test public void testDoUpdateForClusterCreate_SingleHostProperty__defaultValue() throws Exception { Map<String, Map<String, String>> properties = new HashMap<String, Map<String, String>>(); Map<String, String> typeProps = new HashMap<String, String>(); Map<String, String> typeProps2 = new HashMap<String, String>(); typeProps.put("yarn.resourcemanager.hostname", "localhost"); typeProps2.put("oozie_heapsize", "1024"); typeProps2.put("oozie_permsize", "128"); properties.put("oozie-env", typeProps2); properties.put("yarn-site", typeProps); Configuration clusterConfig = new Configuration(properties, Collections.<String, Map<String, Map<String, String>>>emptyMap()); Collection<String> group1Components = new HashSet<String>(); group1Components.add("NAMENODE"); group1Components.add("SECONDARY_NAMENODE"); group1Components.add("RESOURCEMANAGER"); TestHostGroup group1 = new TestHostGroup("group1", group1Components, Collections.singleton("testhost")); Collection<String> group2Components = new HashSet<String>(); group2Components.add("DATANODE"); group2Components.add("HDFS_CLIENT"); TestHostGroup group2 = new TestHostGroup("group2", group2Components, Collections.singleton("testhost2")); Collection<TestHostGroup> hostGroups = new HashSet<TestHostGroup>(); hostGroups.add(group1); hostGroups.add(group2); ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); BlueprintConfigurationProcessor updater = new BlueprintConfigurationProcessor(topology); Set<String> configTypesUpdated = updater.doUpdateForClusterCreate(); String updatedVal = properties.get("yarn-site").get("yarn.resourcemanager.hostname"); assertEquals("testhost", updatedVal); String updatedVal1 = properties.get("oozie-env").get("oozie_heapsize"); assertEquals("1024m", updatedVal1); String updatedVal2 = properties.get("oozie-env").get("oozie_permsize"); assertEquals("128m", updatedVal2); assertEquals("Incorrect number of config types updated", 3, configTypesUpdated.size()); assertTrue("Expected config type not updated", configTypesUpdated.contains("oozie-env")); assertTrue("Expected config type not updated", configTypesUpdated.contains("yarn-site")); assertTrue("Expected config type not updated", configTypesUpdated.contains("cluster-env")); }
@Test public void testDoUpdateForClusterCreate_SingleHostProperty__defaultValue_providedInParent() throws Exception { Map<String, Map<String, String>> properties = new HashMap<String, Map<String, String>>(); Map<String, String> yarnSiteProps = new HashMap<String, String>(); yarnSiteProps.put("yarn.resourcemanager.hostname", "localhost"); properties.put("yarn-site", yarnSiteProps); Map<String, Map<String, String>> parentProperties = new HashMap<String, Map<String, String>>(); Map<String, String> parentYarnSiteProps = new HashMap<String, String>(); parentYarnSiteProps.put("yarn.resourcemanager.resource-tracker.address", "localhost"); parentProperties.put("yarn-site", parentYarnSiteProps); Configuration parentClusterConfig = new Configuration(parentProperties, Collections.<String, Map<String, Map<String, String>>>emptyMap()); Configuration clusterConfig = new Configuration(properties, Collections.<String, Map<String, Map<String, String>>>emptyMap(), parentClusterConfig); Collection<String> group1Components = new HashSet<String>(); group1Components.add("NAMENODE"); group1Components.add("SECONDARY_NAMENODE"); group1Components.add("RESOURCEMANAGER"); TestHostGroup group1 = new TestHostGroup("group1", group1Components, Collections.singleton("testhost")); Collection<String> group2Components = new HashSet<String>(); group2Components.add("DATANODE"); group2Components.add("HDFS_CLIENT"); TestHostGroup group2 = new TestHostGroup("group2", group2Components, Collections.singleton("testhost2")); Collection<TestHostGroup> hostGroups = new HashSet<TestHostGroup>(); hostGroups.add(group1); hostGroups.add(group2); ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); BlueprintConfigurationProcessor updater = new BlueprintConfigurationProcessor(topology); updater.doUpdateForClusterCreate(); assertEquals("testhost", clusterConfig.getPropertyValue("yarn-site", "yarn.resourcemanager.hostname")); assertEquals("testhost", clusterConfig.getPropertyValue("yarn-site", "yarn.resourcemanager.resource-tracker.address")); }
@Test public void testDoUpdateForClusterCreate_SingleHostProperty__defaultValue_hostGroupConfig() throws Exception { Map<String, Map<String, String>> properties = new HashMap<String, Map<String, String>>(); Map<String, String> yarnSiteProps = new HashMap<String, String>(); yarnSiteProps.put("yarn.resourcemanager.hostname", "localhost"); properties.put("yarn-site", yarnSiteProps); Configuration clusterConfig = new Configuration(properties, Collections.<String, Map<String, Map<String, String>>>emptyMap()); Collection<String> group1Components = new HashSet<String>(); group1Components.add("NAMENODE"); group1Components.add("SECONDARY_NAMENODE"); group1Components.add("RESOURCEMANAGER"); TestHostGroup group1 = new TestHostGroup("group1", group1Components, Collections.singleton("testhost")); Collection<String> group2Components = new HashSet<String>(); group2Components.add("DATANODE"); group2Components.add("HDFS_CLIENT"); Map<String, Map<String, String>> group2Properties = new HashMap<String, Map<String, String>>(); Map<String, String> group2YarnSiteProperties = new HashMap<String, String>(); group2YarnSiteProperties.put("yarn.resourcemanager.resource-tracker.address", "localhost"); group2Properties.put("yarn-site", group2YarnSiteProperties); Configuration group2BPConfig = new Configuration(Collections.<String, Map<String, String>>emptyMap(), Collections.<String, Map<String, Map<String, String>>>emptyMap(), clusterConfig); Configuration group2Config = new Configuration(group2Properties, Collections.<String, Map<String, Map<String, String>>>emptyMap(), group2BPConfig); TestHostGroup group2 = new TestHostGroup("group2", group2Components, Collections.singleton("testhost2"), group2Config); Collection<TestHostGroup> hostGroups = new HashSet<TestHostGroup>(); hostGroups.add(group1); hostGroups.add(group2); ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); BlueprintConfigurationProcessor updater = new BlueprintConfigurationProcessor(topology); updater.doUpdateForClusterCreate(); assertEquals("testhost", clusterConfig.getPropertyValue("yarn-site", "yarn.resourcemanager.hostname")); assertEquals("testhost", group2Config.getProperties().get("yarn-site").get("yarn.resourcemanager.resource-tracker.address")); }
@Test public void testDoUpdateForClusterCreate_SingleHostProperty__defaultValue_BPHostGroupConfig() throws Exception { Map<String, Map<String, String>> properties = new HashMap<String, Map<String, String>>(); Map<String, String> yarnSiteProps = new HashMap<String, String>(); yarnSiteProps.put("yarn.resourcemanager.hostname", "localhost"); properties.put("yarn-site", yarnSiteProps); Configuration clusterConfig = new Configuration(properties, Collections.<String, Map<String, Map<String, String>>>emptyMap()); Collection<String> group1Components = new HashSet<String>(); group1Components.add("NAMENODE"); group1Components.add("SECONDARY_NAMENODE"); group1Components.add("RESOURCEMANAGER"); TestHostGroup group1 = new TestHostGroup("group1", group1Components, Collections.singleton("testhost")); Collection<String> group2Components = new HashSet<String>(); group2Components.add("DATANODE"); group2Components.add("HDFS_CLIENT"); Map<String, Map<String, String>> group2BPProperties = new HashMap<String, Map<String, String>>(); Map<String, String> group2YarnSiteProperties = new HashMap<String, String>(); group2YarnSiteProperties.put("yarn.resourcemanager.resource-tracker.address", "localhost"); group2BPProperties.put("yarn-site", group2YarnSiteProperties); Configuration group2BPConfig = new Configuration(group2BPProperties, Collections.<String, Map<String, Map<String, String>>>emptyMap(), clusterConfig); Configuration group2Config = new Configuration(new HashMap<String, Map<String, String>>(), Collections.<String, Map<String, Map<String, String>>>emptyMap()); TestHostGroup group2 = new TestHostGroup("group2", group2Components, Collections.singleton("testhost2"), group2Config); Collection<TestHostGroup> hostGroups = new HashSet<TestHostGroup>(); hostGroups.add(group1); hostGroups.add(group2); ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); topology.getHostGroupInfo().get("group2").getConfiguration().setParentConfiguration(group2BPConfig); BlueprintConfigurationProcessor updater = new BlueprintConfigurationProcessor(topology); updater.doUpdateForClusterCreate(); assertEquals("testhost", clusterConfig.getPropertyValue("yarn-site", "yarn.resourcemanager.hostname")); assertEquals("testhost", group2Config.getProperties().get("yarn-site").get("yarn.resourcemanager.resource-tracker.address")); }
@Test public void testDoUpdateForClusterCreate_SingleHostProperty__MissingComponent() throws Exception { Map<String, Map<String, String>> properties = new HashMap<String, Map<String, String>>(); Map<String, String> typeProps = new HashMap<String, String>(); typeProps.put("yarn.resourcemanager.hostname", "localhost"); typeProps.put("yarn.timeline-service.address", "localhost"); properties.put("yarn-site", typeProps); Configuration clusterConfig = new Configuration(properties, Collections.<String, Map<String, Map<String, String>>>emptyMap()); Collection<String> group1Components = new HashSet<String>(); group1Components.add("NAMENODE"); group1Components.add("SECONDARY_NAMENODE"); group1Components.add("RESOURCEMANAGER"); TestHostGroup group1 = new TestHostGroup("group1", group1Components, Collections.singleton("testhost")); Collection<String> group2Components = new HashSet<String>(); group2Components.add("DATANODE"); group2Components.add("HDFS_CLIENT"); TestHostGroup group2 = new TestHostGroup("group2", group2Components, Collections.singleton("testhost2")); Collection<TestHostGroup> hostGroups = new HashSet<TestHostGroup>(); hostGroups.add(group1); hostGroups.add(group2); expect(stack.getCardinality("APP_TIMELINE_SERVER")).andReturn(new Cardinality("1")).anyTimes(); ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); BlueprintConfigurationProcessor updater = new BlueprintConfigurationProcessor(topology); try { updater.doUpdateForClusterCreate(); fail("IllegalArgumentException should have been thrown"); } catch (IllegalArgumentException illegalArgumentException) { } }
@Test public void testDoUpdateForClusterCreate_SingleHostProperty__MultipleMatchingHostGroupsError() throws Exception { Map<String, Map<String, String>> properties = new HashMap<String, Map<String, String>>(); Map<String, String> typeProps = new HashMap<String, String>(); typeProps.put("yarn.resourcemanager.hostname", "localhost"); typeProps.put("yarn.timeline-service.address", "localhost"); properties.put("yarn-site", typeProps); Configuration clusterConfig = new Configuration(properties, Collections.<String, Map<String, Map<String, String>>>emptyMap()); Collection<String> group1Components = new HashSet<String>(); group1Components.add("NAMENODE"); group1Components.add("SECONDARY_NAMENODE"); group1Components.add("RESOURCEMANAGER"); group1Components.add("APP_TIMELINE_SERVER"); TestHostGroup group1 = new TestHostGroup("group1", group1Components, Collections.singleton("testhost")); Collection<String> group2Components = new HashSet<String>(); group2Components.add("DATANODE"); group2Components.add("HDFS_CLIENT"); group2Components.add("APP_TIMELINE_SERVER"); TestHostGroup group2 = new TestHostGroup("group2", group2Components, Collections.singleton("testhost2")); Collection<TestHostGroup> hostGroups = new HashSet<TestHostGroup>(); hostGroups.add(group1); hostGroups.add(group2); expect(stack.getCardinality("APP_TIMELINE_SERVER")).andReturn(new Cardinality("0-1")).anyTimes(); ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); BlueprintConfigurationProcessor updater = new BlueprintConfigurationProcessor(topology); try { updater.doUpdateForClusterCreate(); fail("IllegalArgumentException should have been thrown"); } catch (IllegalArgumentException illegalArgumentException) { } }
@Test public void testDoUpdateForClusterCreate_SingleHostProperty__MultipleAppTimelineServer() throws Exception { Map<String, Map<String, String>> properties = new HashMap<String, Map<String, String>>(); Map<String, String> typeProps = new HashMap<String, String>(); typeProps.put("yarn.timeline-service.address", "testhost:10200"); typeProps.put("yarn.timeline-service.webapp.address", "testhost:8188"); typeProps.put("yarn.timeline-service.webapp.https.address", "testhost:8190"); properties.put("yarn-site", typeProps); Configuration clusterConfig = new Configuration(properties, Collections.<String, Map<String, Map<String, String>>>emptyMap()); Collection<String> group1Components = new HashSet<String>(); group1Components.add("NAMENODE"); group1Components.add("SECONDARY_NAMENODE"); group1Components.add("RESOURCEMANAGER"); group1Components.add("APP_TIMELINE_SERVER"); TestHostGroup group1 = new TestHostGroup("group1", group1Components, Collections.singleton("testhost")); Collection<String> group2Components = new HashSet<String>(); group2Components.add("DATANODE"); group2Components.add("HDFS_CLIENT"); group2Components.add("APP_TIMELINE_SERVER"); TestHostGroup group2 = new TestHostGroup("group2", group2Components, Collections.singleton("testhost2")); Collection<TestHostGroup> hostGroups = new HashSet<TestHostGroup>(); hostGroups.add(group1); hostGroups.add(group2); expect(stack.getCardinality("APP_TIMELINE_SERVER")).andReturn(new Cardinality("0-1")).anyTimes(); ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); BlueprintConfigurationProcessor updater = new BlueprintConfigurationProcessor(topology); updater.doUpdateForClusterCreate(); String updatedVal = topology.getConfiguration().getFullProperties().get("yarn-site").get("yarn.timeline-service.address"); assertEquals("Timeline Server config property should not have been updated", "testhost:10200", updatedVal); }
@Test public void testDoUpdateForClusterCreate_SingleHostProperty__MissingOptionalComponent() throws Exception { final String expectedHostName = "localhost"; Map<String, Map<String, String>> properties = new HashMap<String, Map<String, String>>(); Map<String, String> typeProps = new HashMap<String, String>(); typeProps.put("yarn.timeline-service.address", expectedHostName); properties.put("yarn-site", typeProps); Configuration clusterConfig = new Configuration(properties, Collections.<String, Map<String, Map<String, String>>>emptyMap()); Collection<String> group1Components = new HashSet<String>(); group1Components.add("NAMENODE"); group1Components.add("SECONDARY_NAMENODE"); group1Components.add("RESOURCEMANAGER"); TestHostGroup group1 = new TestHostGroup("group1", group1Components, Collections.singleton("testhost")); Collection<String> group2Components = new HashSet<String>(); group2Components.add("DATANODE"); group2Components.add("HDFS_CLIENT"); TestHostGroup group2 = new TestHostGroup("group2", group2Components, Collections.singleton("testhost2")); Collection<TestHostGroup> hostGroups = new HashSet<TestHostGroup>(); hostGroups.add(group1); hostGroups.add(group2); expect(stack.getCardinality("APP_TIMELINE_SERVER")).andReturn(new Cardinality("0-1")).anyTimes(); ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); BlueprintConfigurationProcessor updater = new BlueprintConfigurationProcessor(topology); updater.doUpdateForClusterCreate(); String updatedVal = topology.getConfiguration().getFullProperties().get("yarn-site").get("yarn.timeline-service.address"); assertEquals("Timeline Server config property should not have been updated", expectedHostName, updatedVal); }
@Test public void testDoUpdateForClusterCreate_SingleHostProperty__defaultValue__WithPort() throws Exception { Map<String, Map<String, String>> properties = new HashMap<String, Map<String, String>>(); Map<String, String> typeProps = new HashMap<String, String>(); typeProps.put("fs.defaultFS", "localhost:5050"); properties.put("core-site", typeProps); Configuration clusterConfig = new Configuration(properties, Collections.<String, Map<String, Map<String, String>>>emptyMap()); Collection<String> hgComponents = new HashSet<String>(); hgComponents.add("NAMENODE"); hgComponents.add("SECONDARY_NAMENODE"); hgComponents.add("RESOURCEMANAGER"); TestHostGroup group1 = new TestHostGroup("group1", hgComponents, Collections.singleton("testhost")); Collection<String> hgComponents2 = new HashSet<String>(); hgComponents2.add("DATANODE"); hgComponents2.add("HDFS_CLIENT"); TestHostGroup group2 = new TestHostGroup("group2", hgComponents2, Collections.singleton("testhost2")); Collection<TestHostGroup> hostGroups = new HashSet<TestHostGroup>(); hostGroups.add(group1); hostGroups.add(group2); ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); BlueprintConfigurationProcessor updater = new BlueprintConfigurationProcessor(topology); updater.doUpdateForClusterCreate(); String updatedVal = topology.getConfiguration().getFullProperties().get("core-site").get("fs.defaultFS"); assertEquals("testhost:5050", updatedVal); }
@Test public void testDoUpdateForClusterCreate_MultiHostProperty__defaultValues() throws Exception { Map<String, Map<String, String>> properties = new HashMap<String, Map<String, String>>(); Map<String, String> typeProps = new HashMap<String, String>(); typeProps.put("hbase.zookeeper.quorum", "localhost"); properties.put("hbase-site", typeProps); Configuration clusterConfig = new Configuration(properties, Collections.<String, Map<String, Map<String, String>>>emptyMap()); Collection<String> hgComponents = new HashSet<String>(); hgComponents.add("NAMENODE"); hgComponents.add("SECONDARY_NAMENODE"); hgComponents.add("ZOOKEEPER_SERVER"); TestHostGroup group1 = new TestHostGroup("group1", hgComponents, Collections.singleton("testhost")); Collection<String> hgComponents2 = new HashSet<String>(); hgComponents2.add("DATANODE"); hgComponents2.add("HDFS_CLIENT"); hgComponents2.add("ZOOKEEPER_SERVER"); Set<String> hosts2 = new HashSet<String>(); hosts2.add("testhost2"); hosts2.add("testhost2a"); hosts2.add("testhost2b"); TestHostGroup group2 = new TestHostGroup("group2", hgComponents2, hosts2); Collection<String> hgComponents3 = new HashSet<String>(); hgComponents2.add("HDFS_CLIENT"); hgComponents2.add("ZOOKEEPER_CLIENT"); Set<String> hosts3 = new HashSet<String>(); hosts3.add("testhost3"); hosts3.add("testhost3a"); TestHostGroup group3 = new TestHostGroup("group3", hgComponents3, hosts3); Collection<TestHostGroup> hostGroups = new HashSet<TestHostGroup>(); hostGroups.add(group1); hostGroups.add(group2); hostGroups.add(group3); ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); BlueprintConfigurationProcessor updater = new BlueprintConfigurationProcessor(topology); updater.doUpdateForClusterCreate(); String updatedVal = topology.getConfiguration().getFullProperties().get("hbase-site").get("hbase.zookeeper.quorum"); String[] hosts = updatedVal.split(","); Collection<String> expectedHosts = new HashSet<String>(); expectedHosts.add("testhost"); expectedHosts.add("testhost2"); expectedHosts.add("testhost2a"); expectedHosts.add("testhost2b"); assertEquals(4, hosts.length); for (String host : hosts) { assertTrue(expectedHosts.contains(host)); expectedHosts.remove(host); } }
@Test public void testDoUpdateForClusterCreate_MultiHostProperty__defaultValues___withPorts() throws Exception { Map<String, Map<String, String>> properties = new HashMap<String, Map<String, String>>(); Map<String, String> typeProps = new HashMap<String, String>(); typeProps.put("templeton.zookeeper.hosts", "localhost:9090"); properties.put("webhcat-site", typeProps); Configuration clusterConfig = new Configuration(properties, Collections.<String, Map<String, Map<String, String>>>emptyMap()); Collection<String> hgComponents = new HashSet<String>(); hgComponents.add("NAMENODE"); hgComponents.add("SECONDARY_NAMENODE"); hgComponents.add("ZOOKEEPER_SERVER"); TestHostGroup group1 = new TestHostGroup("group1", hgComponents, Collections.singleton("testhost")); Collection<String> hgComponents2 = new HashSet<String>(); hgComponents2.add("DATANODE"); hgComponents2.add("HDFS_CLIENT"); hgComponents2.add("ZOOKEEPER_SERVER"); Set<String> hosts2 = new HashSet<String>(); hosts2.add("testhost2"); hosts2.add("testhost2a"); hosts2.add("testhost2b"); TestHostGroup group2 = new TestHostGroup("group2", hgComponents2, hosts2); Collection<String> hgComponents3 = new HashSet<String>(); hgComponents2.add("HDFS_CLIENT"); hgComponents2.add("ZOOKEEPER_CLIENT"); Set<String> hosts3 = new HashSet<String>(); hosts3.add("testhost3"); hosts3.add("testhost3a"); TestHostGroup group3 = new TestHostGroup("group3", hgComponents3, hosts3); Collection<TestHostGroup> hostGroups = new HashSet<TestHostGroup>(); hostGroups.add(group1); hostGroups.add(group2); hostGroups.add(group3); ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); BlueprintConfigurationProcessor updater = new BlueprintConfigurationProcessor(topology); updater.doUpdateForClusterCreate(); String updatedVal = topology.getConfiguration().getFullProperties().get("webhcat-site").get("templeton.zookeeper.hosts"); String[] hosts = updatedVal.split(","); Collection<String> expectedHosts = new HashSet<String>(); expectedHosts.add("testhost:9090"); expectedHosts.add("testhost2:9090"); expectedHosts.add("testhost2a:9090"); expectedHosts.add("testhost2b:9090"); assertEquals(4, hosts.length); for (String host : hosts) { assertTrue(expectedHosts.contains(host)); expectedHosts.remove(host); } }
@Test public void testDoUpdateForClusterVerifyRetrySettingsDefault() throws Exception { Map<String, Map<String, String>> configProperties = new HashMap<String, Map<String, String>>(); HashMap<String, String> clusterEnvProperties = new HashMap<String, String>(); configProperties.put("cluster-env", clusterEnvProperties); Configuration clusterConfig = new Configuration(configProperties, Collections.<String, Map<String, Map<String, String>>>emptyMap()); TestHostGroup testHostGroup = new TestHostGroup("test-host-group-one", Collections.<String>emptySet(), Collections.<String>emptySet()); ClusterTopology topology = createClusterTopology(bp, clusterConfig, Collections.singleton(testHostGroup)); BlueprintConfigurationProcessor updater = new BlueprintConfigurationProcessor(topology); Set<String> updatedConfigTypes = updater.doUpdateForClusterCreate(); assertEquals("Incorrect number of properties added to cluster-env for retry", 3, clusterEnvProperties.size()); assertEquals("command_retry_enabled was not set to the expected default", "true", clusterEnvProperties.get("command_retry_enabled")); assertEquals("commands_to_retry was not set to the expected default", "INSTALL,START", clusterEnvProperties.get("commands_to_retry")); assertEquals("command_retry_max_time_in_sec was not set to the expected default", "600", clusterEnvProperties.get("command_retry_max_time_in_sec")); assertEquals("Incorrect number of config types updated by this operation", 1, updatedConfigTypes.size()); assertTrue("Expected type not included in the updated set", updatedConfigTypes.contains("cluster-env")); }
@Test public void testDoUpdateForClusterVerifyRetrySettingsCustomized() throws Exception { Map<String, Map<String, String>> configProperties = new HashMap<String, Map<String, String>>(); HashMap<String, String> clusterEnvProperties = new HashMap<String, String>(); configProperties.put("cluster-env", clusterEnvProperties); clusterEnvProperties.put("command_retry_enabled", "false"); clusterEnvProperties.put("commands_to_retry", "TEST"); clusterEnvProperties.put("command_retry_max_time_in_sec", "1"); Configuration clusterConfig = new Configuration(configProperties, Collections.<String, Map<String, Map<String, String>>>emptyMap()); TestHostGroup testHostGroup = new TestHostGroup("test-host-group-one", Collections.<String>emptySet(), Collections.<String>emptySet()); ClusterTopology topology = createClusterTopology(bp, clusterConfig, Collections.singleton(testHostGroup)); BlueprintConfigurationProcessor updater = new BlueprintConfigurationProcessor(topology); Set<String> updatedConfigTypes = updater.doUpdateForClusterCreate(); assertEquals("Incorrect number of properties added to cluster-env for retry", 3, clusterEnvProperties.size()); assertEquals("command_retry_enabled was not set to the expected default", "false", clusterEnvProperties.get("command_retry_enabled")); assertEquals("commands_to_retry was not set to the expected default", "TEST", clusterEnvProperties.get("commands_to_retry")); assertEquals("command_retry_max_time_in_sec was not set to the expected default", "1", clusterEnvProperties.get("command_retry_max_time_in_sec")); assertEquals("Incorrect number of config types updated", 0, updatedConfigTypes.size()); }
@Test public void testDoUpdateForClusterWithNameNodeHAEnabledSpecifyingHostNamesDirectly() throws Exception { final String expectedNameService = "mynameservice"; final String expectedHostName = "c6401.apache.ambari.org"; final String expectedHostNameTwo = "server-two"; final String expectedPortNum = "808080"; final String expectedNodeOne = "nn1"; final String expectedNodeTwo = "nn2"; final String expectedHostGroupName = "host_group_1"; Map<String, Map<String, String>> configProperties = new HashMap<String, Map<String, String>>(); Map<String, String> hdfsSiteProperties = new HashMap<String, String>(); Map<String, String> hbaseSiteProperties = new HashMap<String, String>(); Map<String, String> hadoopEnvProperties = new HashMap<String, String>(); Map<String, String> coreSiteProperties = new HashMap<String, String>(); Map<String, String> accumuloSiteProperties = new HashMap<String, String>(); configProperties.put("hdfs-site", hdfsSiteProperties); configProperties.put("hadoop-env", hadoopEnvProperties); configProperties.put("core-site", coreSiteProperties); configProperties.put("hbase-site", hbaseSiteProperties); configProperties.put("accumulo-site", accumuloSiteProperties); hdfsSiteProperties.put("dfs.nameservices", expectedNameService); hdfsSiteProperties.put("dfs.ha.namenodes.mynameservice", expectedNodeOne + ", " + expectedNodeTwo); hdfsSiteProperties.put("dfs.namenode.https-address." + expectedNameService + "." + expectedNodeOne, createHostAddress(expectedHostName, expectedPortNum)); hdfsSiteProperties.put("dfs.namenode.https-address." + expectedNameService + "." + expectedNodeTwo, createHostAddress(expectedHostNameTwo, expectedPortNum)); hdfsSiteProperties.put("dfs.namenode.http-address." + expectedNameService + "." + expectedNodeOne, createHostAddress(expectedHostName, expectedPortNum)); hdfsSiteProperties.put("dfs.namenode.http-address." + expectedNameService + "." + expectedNodeTwo, createHostAddress(expectedHostNameTwo, expectedPortNum)); hdfsSiteProperties.put("dfs.namenode.rpc-address." + expectedNameService + "." + expectedNodeOne, createHostAddress(expectedHostName, expectedPortNum)); hdfsSiteProperties.put("dfs.namenode.rpc-address." + expectedNameService + "." + expectedNodeTwo, createHostAddress(expectedHostNameTwo, expectedPortNum)); hdfsSiteProperties.put("dfs.secondary.http.address", "localhost:8080"); hdfsSiteProperties.put("dfs.namenode.secondary.http-address", "localhost:8080"); coreSiteProperties.put("fs.defaultFS", "hdfs: hbaseSiteProperties.put("hbase.rootdir", "hdfs: accumuloSiteProperties.put("instance.volumes", "hdfs: Configuration clusterConfig = new Configuration(configProperties, Collections.<String, Map<String, Map<String, String>>>emptyMap()); Collection<String> hgComponents = new HashSet<String>(); hgComponents.add("NAMENODE"); TestHostGroup group1 = new TestHostGroup(expectedHostGroupName, hgComponents, Collections.singleton(expectedHostName)); Collection<String> hgComponents2 = new HashSet<String>(); hgComponents2.add("NAMENODE"); TestHostGroup group2 = new TestHostGroup("host-group-2", hgComponents2, Collections.singleton(expectedHostNameTwo)); Collection<TestHostGroup> hostGroups = new HashSet<TestHostGroup>(); hostGroups.add(group1); hostGroups.add(group2); expect(stack.getCardinality("NAMENODE")).andReturn(new Cardinality("1-2")).anyTimes(); expect(stack.getCardinality("SECONDARY_NAMENODE")).andReturn(new Cardinality("1")).anyTimes(); ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); BlueprintConfigurationProcessor updater = new BlueprintConfigurationProcessor(topology); updater.doUpdateForClusterCreate(); assertEquals("dfs.internal.nameservices wasn't added", expectedNameService, hdfsSiteProperties.get("dfs.internal.nameservices")); assertEquals("HTTPS address HA property not properly exported", expectedHostName + ":" + expectedPortNum, hdfsSiteProperties.get("dfs.namenode.https-address." + expectedNameService + "." + expectedNodeOne)); assertEquals("HTTPS address HA property not properly exported", expectedHostNameTwo + ":" + expectedPortNum, hdfsSiteProperties.get("dfs.namenode.https-address." + expectedNameService + "." + expectedNodeTwo)); assertEquals("HTTPS address HA property not properly exported", expectedHostName + ":" + expectedPortNum, hdfsSiteProperties.get("dfs.namenode.http-address." + expectedNameService + "." + expectedNodeOne)); assertEquals("HTTPS address HA property not properly exported", expectedHostNameTwo + ":" + expectedPortNum, hdfsSiteProperties.get("dfs.namenode.http-address." + expectedNameService + "." + expectedNodeTwo)); assertEquals("HTTPS address HA property not properly exported", expectedHostName + ":" + expectedPortNum, hdfsSiteProperties.get("dfs.namenode.rpc-address." + expectedNameService + "." + expectedNodeOne)); assertEquals("HTTPS address HA property not properly exported", expectedHostNameTwo + ":" + expectedPortNum, hdfsSiteProperties.get("dfs.namenode.rpc-address." + expectedNameService + "." + expectedNodeTwo)); String activeHost = hadoopEnvProperties.get("dfs_ha_initial_namenode_active"); if (activeHost.equals(expectedHostName)) { assertEquals("Standby Namenode hostname was not set correctly", expectedHostNameTwo, hadoopEnvProperties.get("dfs_ha_initial_namenode_standby")); } else if (activeHost.equals(expectedHostNameTwo)) { assertEquals("Standby Namenode hostname was not set correctly", expectedHostName, hadoopEnvProperties.get("dfs_ha_initial_namenode_standby")); } else { fail("Active Namenode hostname was not set correctly: " + activeHost); } assertEquals("fs.defaultFS should not be modified by cluster update when NameNode HA is enabled.", "hdfs: assertEquals("hbase.rootdir should not be modified by cluster update when NameNode HA is enabled.", "hdfs: assertEquals("instance.volumes should not be modified by cluster update when NameNode HA is enabled.", "hdfs: }
@Test public void testHiveConfigClusterUpdateCustomValueSpecifyingHostNamesMetaStoreHA() throws Exception { final String expectedHostGroupName = "host_group_1"; final String expectedPropertyValue = "hive.metastore.local=false,hive.metastore.uris=thrift: Map<String, Map<String, String>> configProperties = new HashMap<String, Map<String, String>>(); Map<String, String> webHCatSiteProperties = new HashMap<String, String>(); configProperties.put("webhcat-site", webHCatSiteProperties); webHCatSiteProperties.put("templeton.hive.properties", expectedPropertyValue); Configuration clusterConfig = new Configuration(configProperties, Collections.<String, Map<String, Map<String, String>>>emptyMap()); Collection<String> hgComponents = new HashSet<String>(); hgComponents.add("NAMENODE"); TestHostGroup group1 = new TestHostGroup(expectedHostGroupName, hgComponents, Collections.singleton("some-host")); Collection<String> hgComponents2 = new HashSet<String>(); hgComponents2.add("DATANODE"); TestHostGroup group2 = new TestHostGroup("host_group_2", hgComponents2, Collections.singleton("some-host2")); Collection<TestHostGroup> hostGroups = new HashSet<TestHostGroup>(); hostGroups.add(group1); hostGroups.add(group2); ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); BlueprintConfigurationProcessor updater = new BlueprintConfigurationProcessor(topology); updater.doUpdateForClusterCreate(); assertEquals("Unexpected config update for templeton.hive.properties", expectedPropertyValue, webHCatSiteProperties.get("templeton.hive.properties")); }
@Test public void testHiveConfigClusterUpdateSpecifyingHostNamesHiveServer2HA() throws Exception { final String expectedHostGroupName = "host_group_1"; final String expectedMetaStoreURIs = "thrift: Map<String, Map<String, String>> configProperties = new HashMap<String, Map<String, String>>(); Map<String, String> hiveEnvProperties = new HashMap<String, String>(); Map<String, String> hiveSiteProperties = new HashMap<String, String>(); configProperties.put("hive-env", hiveEnvProperties); configProperties.put("hive-site", hiveSiteProperties); hiveSiteProperties.put("hive.server2.support.dynamic.service.discovery", "true"); hiveSiteProperties.put("hive.metastore.uris", expectedMetaStoreURIs); Configuration clusterConfig = new Configuration(configProperties, Collections.<String, Map<String, Map<String, String>>>emptyMap()); Collection<String> hgComponents = new HashSet<String>(); hgComponents.add("HIVE_SERVER"); TestHostGroup group1 = new TestHostGroup(expectedHostGroupName, hgComponents, Collections.singleton("some-host")); Collection<String> hgComponents2 = new HashSet<String>(); hgComponents2.add("HIVE_SERVER"); TestHostGroup group2 = new TestHostGroup("host_group_2", hgComponents2, Collections.singleton("some-host2")); Collection<TestHostGroup> hostGroups = new HashSet<TestHostGroup>(); hostGroups.add(group1); hostGroups.add(group2); expect(stack.getCardinality("HIVE_SERVER")).andReturn(new Cardinality("1+")).anyTimes(); ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); BlueprintConfigurationProcessor updater = new BlueprintConfigurationProcessor(topology); updater.doUpdateForClusterCreate(); assertEquals("Unexpected config update for hive.metastore.uris", expectedMetaStoreURIs, hiveSiteProperties.get("hive.metastore.uris")); }
@Test public void testHiveConfigClusterUpdateUsingExportedNamesHiveServer2HA() throws Exception { final String expectedHostGroupNameOne = "host_group_1"; final String expectedHostGroupNameTwo = "host_group_2"; final String expectedHostNameOne = "c6401.ambari.apache.org"; final String expectedHostNameTwo = "c6402.ambari.apache.org"; final String inputMetaStoreURIs = "thrift: final String expectedMetaStoreURIs = "thrift: Map<String, Map<String, String>> configProperties = new HashMap<String, Map<String, String>>(); Map<String, String> hiveEnvProperties = new HashMap<String, String>(); Map<String, String> hiveSiteProperties = new HashMap<String, String>(); configProperties.put("hive-env", hiveEnvProperties); configProperties.put("hive-site", hiveSiteProperties); hiveSiteProperties.put("hive.server2.support.dynamic.service.discovery", "true"); hiveSiteProperties.put("hive.metastore.uris", inputMetaStoreURIs); Configuration clusterConfig = new Configuration(configProperties, Collections.<String, Map<String, Map<String, String>>>emptyMap()); Collection<String> hgComponents = new HashSet<String>(); hgComponents.add("HIVE_SERVER"); hgComponents.add("HIVE_METASTORE"); TestHostGroup group1 = new TestHostGroup(expectedHostGroupNameOne, hgComponents, Collections.singleton(expectedHostNameOne)); Collection<String> hgComponents2 = new HashSet<String>(); hgComponents2.add("HIVE_SERVER"); hgComponents2.add("HIVE_METASTORE"); TestHostGroup group2 = new TestHostGroup(expectedHostGroupNameTwo, hgComponents2, Collections.singleton(expectedHostNameTwo)); Collection<TestHostGroup> hostGroups = new HashSet<TestHostGroup>(); hostGroups.add(group1); hostGroups.add(group2); expect(stack.getCardinality("HIVE_SERVER")).andReturn(new Cardinality("1+")).anyTimes(); ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); BlueprintConfigurationProcessor updater = new BlueprintConfigurationProcessor(topology); updater.doUpdateForClusterCreate(); assertEquals("Unexpected config update for hive.metastore.uris", expectedMetaStoreURIs, hiveSiteProperties.get("hive.metastore.uris")); }
@Test public void testOozieConfigClusterUpdateHAEnabledSpecifyingHostNamesDirectly() throws Exception { final String expectedHostName = "c6401.apache.ambari.org"; final String expectedHostNameTwo = "c6402.ambari.apache.org"; final String expectedExternalHost = "c6408.ambari.apache.org"; final String expectedHostGroupName = "host_group_1"; final String expectedHostGroupNameTwo = "host_group_2"; Map<String, Map<String, String>> configProperties = new HashMap<String, Map<String, String>>(); Map<String, String> oozieSiteProperties = new HashMap<String, String>(); Map<String, String> oozieEnvProperties = new HashMap<String, String>(); Map<String, String> coreSiteProperties = new HashMap<String, String>(); configProperties.put("oozie-site", oozieSiteProperties); configProperties.put("oozie-env", oozieEnvProperties); configProperties.put("hive-env", oozieEnvProperties); configProperties.put("core-site", coreSiteProperties); oozieSiteProperties.put("oozie.base.url", expectedHostName); oozieSiteProperties.put("oozie.authentication.kerberos.principal", expectedHostName); oozieSiteProperties.put("oozie.service.HadoopAccessorService.kerberos.principal", expectedHostName); oozieSiteProperties.put("oozie.service.JPAService.jdbc.url", "jdbc:mysql: oozieSiteProperties.put("oozie.services.ext", "org.apache.oozie.service.ZKLocksService,org.apache.oozie.service.ZKXLogStreamingService,org.apache.oozie.service.ZKJobsConcurrencyService,org.apache.oozie.service.ZKUUIDService"); oozieEnvProperties.put("oozie_existing_mysql_host", expectedExternalHost); coreSiteProperties.put("hadoop.proxyuser.oozie.hosts", expectedHostName + "," + expectedHostNameTwo); Configuration clusterConfig = new Configuration(configProperties, Collections.<String, Map<String, Map<String, String>>>emptyMap()); Collection<String> hgComponents = new HashSet<String>(); hgComponents.add("OOZIE_SERVER"); TestHostGroup group1 = new TestHostGroup(expectedHostGroupName, hgComponents, Collections.singleton("host1")); Collection<String> hgComponents2 = new HashSet<String>(); hgComponents2.add("OOZIE_SERVER"); TestHostGroup group2 = new TestHostGroup(expectedHostGroupNameTwo, hgComponents2, Collections.singleton("host2")); Collection<TestHostGroup> hostGroups = new HashSet<TestHostGroup>(); hostGroups.add(group1); hostGroups.add(group2); expect(stack.getCardinality("OOZIE_SERVER")).andReturn(new Cardinality("1+")).anyTimes(); ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); BlueprintConfigurationProcessor updater = new BlueprintConfigurationProcessor(topology); updater.doUpdateForClusterCreate(); assertEquals("oozie property not updated correctly", expectedHostName, oozieSiteProperties.get("oozie.base.url")); assertEquals("oozie property not updated correctly", expectedHostName, oozieSiteProperties.get("oozie.authentication.kerberos.principal")); assertEquals("oozie property not updated correctly", expectedHostName, oozieSiteProperties.get("oozie.service.HadoopAccessorService.kerberos.principal")); assertEquals("oozie property not updated correctly", expectedHostName + "," + expectedHostNameTwo, coreSiteProperties.get("hadoop.proxyuser.oozie.hosts")); }
@Test public void testYarnHighAvailabilityConfigClusterUpdateSpecifyingHostNamesDirectly() throws Exception { final String expectedHostName = "c6401.apache.ambari.org"; final String expectedPortNum = "808080"; final String expectedHostGroupName = "host_group_1"; final String expectedHostGroupNameTwo = "host_group_2"; Map<String, Map<String, String>> configProperties = new HashMap<String, Map<String, String>>(); Map<String, String> yarnSiteProperties = new HashMap<String, String>(); configProperties.put("yarn-site", yarnSiteProperties); yarnSiteProperties.put("yarn.log.server.url", "http: yarnSiteProperties.put("yarn.resourcemanager.hostname", expectedHostName); yarnSiteProperties.put("yarn.resourcemanager.resource-tracker.address", expectedHostName + ":" + expectedPortNum); yarnSiteProperties.put("yarn.resourcemanager.webapp.address", expectedHostName + ":" + expectedPortNum); yarnSiteProperties.put("yarn.resourcemanager.scheduler.address", expectedHostName + ":" + expectedPortNum); yarnSiteProperties.put("yarn.resourcemanager.address", expectedHostName + ":" + expectedPortNum); yarnSiteProperties.put("yarn.resourcemanager.admin.address", expectedHostName + ":" + expectedPortNum); yarnSiteProperties.put("yarn.timeline-service.address", expectedHostName + ":" + expectedPortNum); yarnSiteProperties.put("yarn.timeline-service.webapp.address", expectedHostName + ":" + expectedPortNum); yarnSiteProperties.put("yarn.timeline-service.webapp.https.address", expectedHostName + ":" + expectedPortNum); yarnSiteProperties.put("yarn.resourcemanager.ha.enabled", "true"); yarnSiteProperties.put("yarn.resourcemanager.ha.rm-ids", "rm1, rm2"); Configuration clusterConfig = new Configuration(configProperties, Collections.<String, Map<String, Map<String, String>>>emptyMap()); Collection<String> hgComponents = new HashSet<String>(); hgComponents.add("RESOURCEMANAGER"); hgComponents.add("APP_TIMELINE_SERVER"); hgComponents.add("HISTORYSERVER"); TestHostGroup group1 = new TestHostGroup(expectedHostGroupName, hgComponents, Collections.singleton(expectedHostName)); Collection<String> hgComponents2 = new HashSet<String>(); hgComponents2.add("RESOURCEMANAGER"); TestHostGroup group2 = new TestHostGroup(expectedHostGroupNameTwo, hgComponents2, Collections.singleton("host2")); Collection<TestHostGroup> hostGroups = new HashSet<TestHostGroup>(); hostGroups.add(group1); hostGroups.add(group2); expect(stack.getCardinality("RESOURCEMANAGER")).andReturn(new Cardinality("1-2")).anyTimes(); ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); BlueprintConfigurationProcessor updater = new BlueprintConfigurationProcessor(topology); updater.doUpdateForClusterCreate(); assertEquals("Yarn Log Server URL was incorrectly updated", "http: assertEquals("Yarn ResourceManager hostname was incorrectly exported", expectedHostName, yarnSiteProperties.get("yarn.resourcemanager.hostname")); assertEquals("Yarn ResourceManager tracker address was incorrectly updated", createHostAddress(expectedHostName, expectedPortNum), yarnSiteProperties.get("yarn.resourcemanager.resource-tracker.address")); assertEquals("Yarn ResourceManager webapp address was incorrectly updated", createHostAddress(expectedHostName, expectedPortNum), yarnSiteProperties.get("yarn.resourcemanager.webapp.address")); assertEquals("Yarn ResourceManager scheduler address was incorrectly updated", createHostAddress(expectedHostName, expectedPortNum), yarnSiteProperties.get("yarn.resourcemanager.scheduler.address")); assertEquals("Yarn ResourceManager address was incorrectly updated", createHostAddress(expectedHostName, expectedPortNum), yarnSiteProperties.get("yarn.resourcemanager.address")); assertEquals("Yarn ResourceManager admin address was incorrectly updated", createHostAddress(expectedHostName, expectedPortNum), yarnSiteProperties.get("yarn.resourcemanager.admin.address")); assertEquals("Yarn ResourceManager timeline-service address was incorrectly updated", createHostAddress(expectedHostName, expectedPortNum), yarnSiteProperties.get("yarn.timeline-service.address")); assertEquals("Yarn ResourceManager timeline webapp address was incorrectly updated", createHostAddress(expectedHostName, expectedPortNum), yarnSiteProperties.get("yarn.timeline-service.webapp.address")); assertEquals("Yarn ResourceManager timeline webapp HTTPS address was incorrectly updated", createHostAddress(expectedHostName, expectedPortNum), yarnSiteProperties.get("yarn.timeline-service.webapp.https.address")); }
@Test public void testHDFSConfigClusterUpdateQuorumJournalURLSpecifyingHostNamesDirectly() throws Exception { final String expectedHostNameOne = "c6401.apache.ambari.org"; final String expectedHostNameTwo = "c6402.apache.ambari.org"; final String expectedPortNum = "808080"; final String expectedHostGroupName = "host_group_1"; final String expectedHostGroupNameTwo = "host_group_2"; final String expectedQuorumJournalURL = "qjournal: createHostAddress(expectedHostNameTwo, expectedPortNum) + "/mycluster"; Map<String, Map<String, String>> configProperties = new HashMap<String, Map<String, String>>(); Map<String, String> hdfsSiteProperties = new HashMap<String, String>(); configProperties.put("hdfs-site", hdfsSiteProperties); hdfsSiteProperties.put("dfs.namenode.shared.edits.dir", expectedQuorumJournalURL); Configuration clusterConfig = new Configuration(configProperties, Collections.<String, Map<String, Map<String, String>>>emptyMap()); Collection<String> hgComponents = new HashSet<String>(); hgComponents.add("NAMENODE"); TestHostGroup group1 = new TestHostGroup(expectedHostGroupName, hgComponents, Collections.singleton("host1")); Collection<String> hgComponents2 = new HashSet<String>(); hgComponents2.add("DATANODE"); TestHostGroup group2 = new TestHostGroup(expectedHostGroupNameTwo, hgComponents2, Collections.singleton("host2")); Collection<TestHostGroup> hostGroups = new HashSet<TestHostGroup>(); hostGroups.add(group1); hostGroups.add(group2); ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); BlueprintConfigurationProcessor updater = new BlueprintConfigurationProcessor(topology); updater.doUpdateForClusterCreate(); assertEquals("HDFS HA shared edits directory property should not have been modified, since FQDNs were specified.", expectedQuorumJournalURL, hdfsSiteProperties.get("dfs.namenode.shared.edits.dir")); }
@Test public void testDoUpdateForClusterCreate_MultiHostProperty__defaultValues___YAML() throws Exception { Map<String, Map<String, String>> properties = new HashMap<String, Map<String, String>>(); Map<String, String> typeProps = new HashMap<String, String>(); typeProps.put("storm.zookeeper.servers", "['localhost']"); properties.put("storm-site", typeProps); Configuration clusterConfig = new Configuration(properties, Collections.<String, Map<String, Map<String, String>>>emptyMap()); Collection<String> hgComponents = new HashSet<String>(); hgComponents.add("NAMENODE"); hgComponents.add("SECONDARY_NAMENODE"); hgComponents.add("ZOOKEEPER_SERVER"); TestHostGroup group1 = new TestHostGroup("group1", hgComponents, Collections.singleton("testhost")); Collection<String> hgComponents2 = new HashSet<String>(); hgComponents2.add("DATANODE"); hgComponents2.add("HDFS_CLIENT"); hgComponents2.add("ZOOKEEPER_SERVER"); Set<String> hosts2 = new HashSet<String>(); hosts2.add("testhost2"); hosts2.add("testhost2a"); hosts2.add("testhost2b"); TestHostGroup group2 = new TestHostGroup("group2", hgComponents2, hosts2); Collection<String> hgComponents3 = new HashSet<String>(); hgComponents2.add("HDFS_CLIENT"); hgComponents2.add("ZOOKEEPER_CLIENT"); Set<String> hosts3 = new HashSet<String>(); hosts3.add("testhost3"); hosts3.add("testhost3a"); TestHostGroup group3 = new TestHostGroup("group3", hgComponents3, hosts3); Collection<TestHostGroup> hostGroups = new HashSet<TestHostGroup>(); hostGroups.add(group1); hostGroups.add(group2); hostGroups.add(group3); ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); BlueprintConfigurationProcessor updater = new BlueprintConfigurationProcessor(topology); updater.doUpdateForClusterCreate(); String updatedVal = topology.getConfiguration().getFullProperties().get("storm-site").get("storm.zookeeper.servers"); assertTrue(updatedVal.startsWith("[")); assertTrue(updatedVal.endsWith("]")); updatedVal = updatedVal.replaceAll("[\\[\\]]", ""); String[] hosts = updatedVal.split(","); Collection<String> expectedHosts = new HashSet<String>(); expectedHosts.add("'testhost'"); expectedHosts.add("'testhost2'"); expectedHosts.add("'testhost2a'"); expectedHosts.add("'testhost2b'"); assertEquals(4, hosts.length); for (String host : hosts) { assertTrue(expectedHosts.contains(host)); expectedHosts.remove(host); } }
@Test public void testDoUpdateForClusterCreate_Storm_Nimbus_HA_Enabled__defaultValues_YAML() throws Exception { Map<String, Map<String, String>> properties = new HashMap<String, Map<String, String>>(); Map<String, String> typeProps = new HashMap<String, String>(); typeProps.put("nimbus.seeds", "localhost"); properties.put("storm-site", typeProps); Configuration clusterConfig = new Configuration(properties, Collections.<String, Map<String, Map<String, String>>>emptyMap()); Collection<String> hgComponents = new HashSet<String>(); hgComponents.add("NIMBUS"); TestHostGroup group1 = new TestHostGroup("group1", hgComponents, Collections.singleton("testhost")); Collection<String> hgComponents2 = new HashSet<String>(); hgComponents2.add("NIMBUS"); Set<String> hosts2 = new HashSet<String>(); hosts2.add("testhost2"); TestHostGroup group2 = new TestHostGroup("group2", hgComponents2, hosts2); Collection<TestHostGroup> hostGroups = new HashSet<TestHostGroup>(); hostGroups.add(group1); hostGroups.add(group2); ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); BlueprintConfigurationProcessor updater = new BlueprintConfigurationProcessor(topology); updater.doUpdateForClusterCreate(); String updatedVal = topology.getConfiguration().getFullProperties().get("storm-site").get("nimbus.seeds"); assertTrue("Updated YAML value should start with bracket", updatedVal.startsWith("[")); assertTrue("Updated YAML value should end with bracket", updatedVal.endsWith("]")); updatedVal = updatedVal.replaceAll("[\\[\\]]", ""); String[] hosts = updatedVal.split(","); Collection<String> expectedHosts = new HashSet<String>(); expectedHosts.add("testhost"); expectedHosts.add("testhost2"); assertEquals("Incorrect number of hosts found in updated Nimbus config property", 2, hosts.length); for (String host : hosts) { assertTrue("Expected host name = " + host + " not found in updated Nimbus config property", expectedHosts.contains(host)); expectedHosts.remove(host); } }
@Test public void testDoUpdateForClusterCreate_Storm_Nimbus_HA_Enabled__FQDN_ValuesSpecified_YAML() throws Exception { final String expectedValue = "[c6401.ambari.apache.org, c6402.ambari.apache.org]"; Map<String, Map<String, String>> properties = new HashMap<String, Map<String, String>>(); Map<String, String> typeProps = new HashMap<String, String>(); typeProps.put("nimbus.seeds", expectedValue); properties.put("storm-site", typeProps); Configuration clusterConfig = new Configuration(properties, Collections.<String, Map<String, Map<String, String>>>emptyMap()); Collection<String> hgComponents = new HashSet<String>(); hgComponents.add("NIMBUS"); TestHostGroup group1 = new TestHostGroup("group1", hgComponents, Collections.singleton("testhost")); Collection<String> hgComponents2 = new HashSet<String>(); hgComponents2.add("NIMBUS"); Set<String> hosts2 = new HashSet<String>(); hosts2.add("testhost2"); TestHostGroup group2 = new TestHostGroup("group2", hgComponents2, hosts2); Collection<TestHostGroup> hostGroups = new HashSet<TestHostGroup>(); hostGroups.add(group1); hostGroups.add(group2); ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); BlueprintConfigurationProcessor updater = new BlueprintConfigurationProcessor(topology); updater.doUpdateForClusterCreate(); String updatedVal = topology.getConfiguration().getFullProperties().get("storm-site").get("nimbus.seeds"); assertEquals("nimbus.seeds property should not be updated when FQDNs are specified in configuration", expectedValue, updatedVal); }
@Test public void testDoUpdateForClusterCreate_MProperty__defaultValues() throws Exception { Map<String, Map<String, String>> properties = new HashMap<String, Map<String, String>>(); Map<String, String> typeProps = new HashMap<String, String>(); typeProps.put("hbase_master_heapsize", "512m"); properties.put("hbase-env", typeProps); Configuration clusterConfig = new Configuration(properties, Collections.<String, Map<String, Map<String, String>>>emptyMap()); Collection<String> hgComponents = new HashSet<String>(); hgComponents.add("NAMENODE"); hgComponents.add("SECONDARY_NAMENODE"); hgComponents.add("RESOURCEMANAGER"); TestHostGroup group1 = new TestHostGroup("group1", hgComponents, Collections.singleton("testhost")); Collection<String> hgComponents2 = new HashSet<String>(); hgComponents2.add("DATANODE"); hgComponents2.add("HDFS_CLIENT"); TestHostGroup group2 = new TestHostGroup("group2", hgComponents2, Collections.singleton("testhost2")); Collection<TestHostGroup> hostGroups = new HashSet<TestHostGroup>(); hostGroups.add(group1); hostGroups.add(group2); ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); BlueprintConfigurationProcessor updater = new BlueprintConfigurationProcessor(topology); updater.doUpdateForClusterCreate(); String updatedVal = topology.getConfiguration().getFullProperties().get("hbase-env").get("hbase_master_heapsize"); assertEquals("512m", updatedVal); }
@Test public void testDoUpdateForClusterCreate_MProperty__missingM() throws Exception { Map<String, Map<String, String>> properties = new HashMap<String, Map<String, String>>(); Map<String, String> typeProps = new HashMap<String, String>(); typeProps.put("hbase_master_heapsize", "512"); properties.put("hbase-env", typeProps); Configuration clusterConfig = new Configuration(properties, Collections.<String, Map<String, Map<String, String>>>emptyMap()); Collection<String> hgComponents = new HashSet<String>(); hgComponents.add("NAMENODE"); hgComponents.add("SECONDARY_NAMENODE"); hgComponents.add("RESOURCEMANAGER"); TestHostGroup group1 = new TestHostGroup("group1", hgComponents, Collections.singleton("testhost")); Collection<String> hgComponents2 = new HashSet<String>(); hgComponents2.add("DATANODE"); hgComponents2.add("HDFS_CLIENT"); TestHostGroup group2 = new TestHostGroup("group2", hgComponents2, Collections.singleton("testhost2")); Collection<TestHostGroup> hostGroups = new HashSet<TestHostGroup>(); hostGroups.add(group1); hostGroups.add(group2); ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); BlueprintConfigurationProcessor updater = new BlueprintConfigurationProcessor(topology); updater.doUpdateForClusterCreate(); String updatedVal = topology.getConfiguration().getFullProperties().get("hbase-env").get("hbase_master_heapsize"); assertEquals("512m", updatedVal); }
@Test public void testDoUpdateForClusterCreate_SingleHostProperty__exportedValue() throws Exception { Map<String, Map<String, String>> properties = new HashMap<String, Map<String, String>>(); Map<String, String> typeProps = new HashMap<String, String>(); typeProps.put("yarn.resourcemanager.hostname", "%HOSTGROUP::group1%"); properties.put("yarn-site", typeProps); Configuration clusterConfig = new Configuration(properties, Collections.<String, Map<String, Map<String, String>>>emptyMap()); Collection<String> hgComponents = new HashSet<String>(); hgComponents.add("NAMENODE"); hgComponents.add("SECONDARY_NAMENODE"); hgComponents.add("RESOURCEMANAGER"); TestHostGroup group1 = new TestHostGroup("group1", hgComponents, Collections.singleton("testhost")); Collection<String> hgComponents2 = new HashSet<String>(); hgComponents2.add("DATANODE"); hgComponents2.add("HDFS_CLIENT"); TestHostGroup group2 = new TestHostGroup("group2", hgComponents2, Collections.singleton("testhost2")); Collection<TestHostGroup> hostGroups = new HashSet<TestHostGroup>(); hostGroups.add(group1); hostGroups.add(group2); ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); BlueprintConfigurationProcessor updater = new BlueprintConfigurationProcessor(topology); updater.doUpdateForClusterCreate(); String updatedVal = topology.getConfiguration().getFullProperties().get("yarn-site").get("yarn.resourcemanager.hostname"); assertEquals("testhost", updatedVal); }
@Test public void testDoUpdateForClusterCreate_SingleHostProperty__exportedValue_UsingMinusSymbolInHostGroupName() throws Exception { Map<String, Map<String, String>> properties = new HashMap<String, Map<String, String>>(); Map<String, String> typeProps = new HashMap<String, String>(); typeProps.put("yarn.resourcemanager.hostname", "%HOSTGROUP::os-amb-r6-secha-1427972156-hbaseha-3-6%"); properties.put("yarn-site", typeProps); Configuration clusterConfig = new Configuration(properties, Collections.<String, Map<String, Map<String, String>>>emptyMap()); Collection<String> hgComponents = new HashSet<String>(); hgComponents.add("NAMENODE"); hgComponents.add("SECONDARY_NAMENODE"); hgComponents.add("RESOURCEMANAGER"); TestHostGroup group1 = new TestHostGroup("os-amb-r6-secha-1427972156-hbaseha-3-6", hgComponents, Collections.singleton("testhost")); Collection<String> hgComponents2 = new HashSet<String>(); hgComponents2.add("DATANODE"); hgComponents2.add("HDFS_CLIENT"); TestHostGroup group2 = new TestHostGroup("group2", hgComponents2, Collections.singleton("testhost2")); Collection<TestHostGroup> hostGroups = new HashSet<TestHostGroup>(); hostGroups.add(group1); hostGroups.add(group2); ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); BlueprintConfigurationProcessor updater = new BlueprintConfigurationProcessor(topology); updater.doUpdateForClusterCreate(); String updatedVal = topology.getConfiguration().getFullProperties().get("yarn-site").get("yarn.resourcemanager.hostname"); assertEquals("testhost", updatedVal); }
@Test public void testDoUpdateForClusterCreate_SingleHostProperty__exportedValue_WithPort_UsingMinusSymbolInHostGroupName() throws Exception { Map<String, Map<String, String>> properties = new HashMap<String, Map<String, String>>(); Map<String, String> typeProps = new HashMap<String, String>(); typeProps.put("yarn.resourcemanager.hostname", "%HOSTGROUP::os-amb-r6-secha-1427972156-hbaseha-3-6%:2180"); properties.put("yarn-site", typeProps); Configuration clusterConfig = new Configuration(properties, Collections.<String, Map<String, Map<String, String>>>emptyMap()); Collection<String> hgComponents = new HashSet<String>(); hgComponents.add("NAMENODE"); hgComponents.add("SECONDARY_NAMENODE"); hgComponents.add("RESOURCEMANAGER"); TestHostGroup group1 = new TestHostGroup("os-amb-r6-secha-1427972156-hbaseha-3-6", hgComponents, Collections.singleton("testhost")); Collection<String> hgComponents2 = new HashSet<String>(); hgComponents2.add("DATANODE"); hgComponents2.add("HDFS_CLIENT"); TestHostGroup group2 = new TestHostGroup("group2", hgComponents2, Collections.singleton("testhost2")); Collection<TestHostGroup> hostGroups = new HashSet<TestHostGroup>(); hostGroups.add(group1); hostGroups.add(group2); ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); BlueprintConfigurationProcessor updater = new BlueprintConfigurationProcessor(topology); updater.doUpdateForClusterCreate(); String updatedVal = topology.getConfiguration().getFullProperties().get("yarn-site").get("yarn.resourcemanager.hostname"); assertEquals("testhost:2180", updatedVal); }
@Test public void testDoUpdateForClusterCreate_SingleHostProperty__exportedValue__WithPort() throws Exception { Map<String, Map<String, String>> properties = new HashMap<String, Map<String, String>>(); Map<String, String> typeProps = new HashMap<String, String>(); typeProps.put("fs.defaultFS", "%HOSTGROUP::group1%:5050"); properties.put("core-site", typeProps); Configuration clusterConfig = new Configuration(properties, Collections.<String, Map<String, Map<String, String>>>emptyMap()); Collection<String> hgComponents = new HashSet<String>(); hgComponents.add("NAMENODE"); hgComponents.add("SECONDARY_NAMENODE"); hgComponents.add("RESOURCEMANAGER"); TestHostGroup group1 = new TestHostGroup("group1", hgComponents, Collections.singleton("testhost")); Collection<String> hgComponents2 = new HashSet<String>(); hgComponents2.add("DATANODE"); hgComponents2.add("HDFS_CLIENT"); TestHostGroup group2 = new TestHostGroup("group2", hgComponents2, Collections.singleton("testhost2")); Collection<TestHostGroup> hostGroups = new HashSet<TestHostGroup>(); hostGroups.add(group1); hostGroups.add(group2); ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); BlueprintConfigurationProcessor updater = new BlueprintConfigurationProcessor(topology); updater.doUpdateForClusterCreate(); String updatedVal = topology.getConfiguration().getFullProperties().get("core-site").get("fs.defaultFS"); assertEquals("testhost:5050", updatedVal); }
@Test public void testDoUpdateForClusterCreate_MultiHostProperty__exportedValues() throws Exception { Map<String, Map<String, String>> properties = new HashMap<String, Map<String, String>>(); Map<String, String> typeProps = new HashMap<String, String>(); typeProps.put("hbase.zookeeper.quorum", "%HOSTGROUP::group1%,%HOSTGROUP::group2%"); properties.put("hbase-site", typeProps); Configuration clusterConfig = new Configuration(properties, Collections.<String, Map<String, Map<String, String>>>emptyMap()); Collection<String> hgComponents = new HashSet<String>(); hgComponents.add("NAMENODE"); hgComponents.add("SECONDARY_NAMENODE"); hgComponents.add("ZOOKEEPER_SERVER"); TestHostGroup group1 = new TestHostGroup("group1", hgComponents, Collections.singleton("testhost")); Collection<String> hgComponents2 = new HashSet<String>(); hgComponents2.add("DATANODE"); hgComponents2.add("HDFS_CLIENT"); hgComponents2.add("ZOOKEEPER_SERVER"); Set<String> hosts2 = new HashSet<String>(); hosts2.add("testhost2"); hosts2.add("testhost2a"); hosts2.add("testhost2b"); TestHostGroup group2 = new TestHostGroup("group2", hgComponents2, hosts2); Collection<String> hgComponents3 = new HashSet<String>(); hgComponents2.add("HDFS_CLIENT"); hgComponents2.add("ZOOKEEPER_CLIENT"); Set<String> hosts3 = new HashSet<String>(); hosts3.add("testhost3"); hosts3.add("testhost3a"); TestHostGroup group3 = new TestHostGroup("group3", hgComponents3, hosts3); Collection<TestHostGroup> hostGroups = new HashSet<TestHostGroup>(); hostGroups.add(group1); hostGroups.add(group2); hostGroups.add(group3); ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); BlueprintConfigurationProcessor updater = new BlueprintConfigurationProcessor(topology); updater.doUpdateForClusterCreate(); String updatedVal = topology.getConfiguration().getFullProperties().get("hbase-site").get("hbase.zookeeper.quorum"); String[] hosts = updatedVal.split(","); Collection<String> expectedHosts = new HashSet<String>(); expectedHosts.add("testhost"); expectedHosts.add("testhost2"); expectedHosts.add("testhost2a"); expectedHosts.add("testhost2b"); assertEquals(4, hosts.length); for (String host : hosts) { assertTrue(expectedHosts.contains(host)); expectedHosts.remove(host); } }
@Test public void testDoUpdateForClusterCreate_MultiHostProperty__exportedValues___withPorts() throws Exception { Map<String, Map<String, String>> properties = new HashMap<String, Map<String, String>>(); Map<String, String> typeProps = new HashMap<String, String>(); typeProps.put("templeton.zookeeper.hosts", "%HOSTGROUP::group1%:9090,%HOSTGROUP::group2%:9091"); properties.put("webhcat-site", typeProps); Configuration clusterConfig = new Configuration(properties, Collections.<String, Map<String, Map<String, String>>>emptyMap()); Collection<String> hgComponents = new HashSet<String>(); hgComponents.add("NAMENODE"); hgComponents.add("SECONDARY_NAMENODE"); hgComponents.add("ZOOKEEPER_SERVER"); TestHostGroup group1 = new TestHostGroup("group1", hgComponents, Collections.singleton("testhost")); Collection<String> hgComponents2 = new HashSet<String>(); hgComponents2.add("DATANODE"); hgComponents2.add("HDFS_CLIENT"); hgComponents2.add("ZOOKEEPER_SERVER"); Set<String> hosts2 = new HashSet<String>(); hosts2.add("testhost2"); hosts2.add("testhost2a"); hosts2.add("testhost2b"); TestHostGroup group2 = new TestHostGroup("group2", hgComponents2, hosts2); Collection<String> hgComponents3 = new HashSet<String>(); hgComponents2.add("HDFS_CLIENT"); hgComponents2.add("ZOOKEEPER_CLIENT"); Set<String> hosts3 = new HashSet<String>(); hosts3.add("testhost3"); hosts3.add("testhost3a"); TestHostGroup group3 = new TestHostGroup("group3", hgComponents3, hosts3); Collection<TestHostGroup> hostGroups = new HashSet<TestHostGroup>(); hostGroups.add(group1); hostGroups.add(group2); hostGroups.add(group3); ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); BlueprintConfigurationProcessor updater = new BlueprintConfigurationProcessor(topology); updater.doUpdateForClusterCreate(); String updatedVal = topology.getConfiguration().getFullProperties().get("webhcat-site").get("templeton.zookeeper.hosts"); String[] hosts = updatedVal.split(","); Collection<String> expectedHosts = new HashSet<String>(); expectedHosts.add("testhost:9090"); expectedHosts.add("testhost2:9091"); expectedHosts.add("testhost2a:9091"); expectedHosts.add("testhost2b:9091"); assertEquals(4, hosts.length); for (String host : hosts) { assertTrue(expectedHosts.contains(host)); expectedHosts.remove(host); } }
@Test public void testDoUpdateForClusterCreate_MultiHostProperty__exportedValues___withPorts_UsingMinusSymbolInHostGroupName() throws Exception { Map<String, Map<String, String>> properties = new HashMap<String, Map<String, String>>(); Map<String, String> typeProps = new HashMap<String, String>(); typeProps.put("ha.zookeeper.quorum", "%HOSTGROUP::os-amb-r6-secha-1427972156-hbaseha-3-6%:2181,%HOSTGROUP::os-amb-r6-secha-1427972156-hbaseha-3-5%:2181,%HOSTGROUP::os-amb-r6-secha-1427972156-hbaseha-3-7%:2181"); properties.put("core-site", typeProps); Configuration clusterConfig = new Configuration(properties, Collections.<String, Map<String, Map<String, String>>>emptyMap()); Collection<String> hgComponents = new HashSet<String>(); hgComponents.add("NAMENODE"); hgComponents.add("SECONDARY_NAMENODE"); hgComponents.add("ZOOKEEPER_SERVER"); TestHostGroup group1 = new TestHostGroup("os-amb-r6-secha-1427972156-hbaseha-3-6", hgComponents, Collections.singleton("testhost")); Collection<String> hgComponents2 = new HashSet<String>(); hgComponents2.add("DATANODE"); hgComponents2.add("HDFS_CLIENT"); hgComponents2.add("ZOOKEEPER_SERVER"); Set<String> hosts2 = new HashSet<String>(); hosts2.add("testhost2"); hosts2.add("testhost2a"); hosts2.add("testhost2b"); TestHostGroup group2 = new TestHostGroup("os-amb-r6-secha-1427972156-hbaseha-3-5", hgComponents2, hosts2); Collection<String> hgComponents3 = new HashSet<String>(); hgComponents2.add("HDFS_CLIENT"); hgComponents2.add("ZOOKEEPER_CLIENT"); Set<String> hosts3 = new HashSet<String>(); hosts3.add("testhost3"); hosts3.add("testhost3a"); TestHostGroup group3 = new TestHostGroup("os-amb-r6-secha-1427972156-hbaseha-3-7", hgComponents3, hosts3); Collection<TestHostGroup> hostGroups = new HashSet<TestHostGroup>(); hostGroups.add(group1); hostGroups.add(group2); hostGroups.add(group3); ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); BlueprintConfigurationProcessor updater = new BlueprintConfigurationProcessor(topology); updater.doUpdateForClusterCreate(); String updatedVal = topology.getConfiguration().getFullProperties().get("core-site").get("ha.zookeeper.quorum"); String[] hosts = updatedVal.split(","); Collection<String> expectedHosts = new HashSet<String>(); expectedHosts.add("testhost:2181"); expectedHosts.add("testhost2:2181"); expectedHosts.add("testhost2a:2181"); expectedHosts.add("testhost2b:2181"); expectedHosts.add("testhost3:2181"); expectedHosts.add("testhost3a:2181"); assertEquals(6, hosts.length); for (String host : hosts) { assertTrue("Expected host :" + host + "was not included in the multi-server list in this property.", expectedHosts.contains(host)); expectedHosts.remove(host); } }
@Test public void testDoUpdateForClusterCreate_MultiHostProperty_exportedValues_withPorts_singleHostValue() throws Exception { Map<String, Map<String, String>> properties = new HashMap<String, Map<String, String>>(); Map<String, String> yarnSiteConfig = new HashMap<String, String>(); yarnSiteConfig.put("hadoop.registry.zk.quorum", "%HOSTGROUP::host_group_1%:2181"); properties.put("yarn-site", yarnSiteConfig); Configuration clusterConfig = new Configuration(properties, Collections.<String, Map<String, Map<String, String>>>emptyMap()); Collection<String> hgComponents = new HashSet<String>(); hgComponents.add("NAMENODE"); hgComponents.add("SECONDARY_NAMENODE"); hgComponents.add("ZOOKEEPER_SERVER"); TestHostGroup group1 = new TestHostGroup("host_group_1", hgComponents, Collections.singleton("testhost")); Collection<TestHostGroup> hostGroups = new HashSet<TestHostGroup>(); hostGroups.add(group1); ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); BlueprintConfigurationProcessor updater = new BlueprintConfigurationProcessor(topology); updater.doUpdateForClusterCreate(); assertEquals("Multi-host property with single host value was not correctly updated for cluster create.", "testhost:2181", topology.getConfiguration().getFullProperties().get("yarn-site").get("hadoop.registry.zk.quorum")); }
@Test public void testDoUpdateForClusterCreate_MultiHostProperty__exportedValues___YAML() throws Exception { Map<String, Map<String, String>> properties = new HashMap<String, Map<String, String>>(); Map<String, String> typeProps = new HashMap<String, String>(); typeProps.put("storm.zookeeper.servers", "['%HOSTGROUP::group1%:9090','%HOSTGROUP::group2%:9091']"); typeProps.put("nimbus.seeds", "[%HOSTGROUP::group1%, %HOSTGROUP::group4%]"); properties.put("storm-site", typeProps); Configuration clusterConfig = new Configuration(properties, Collections.<String, Map<String, Map<String, String>>>emptyMap()); Collection<String> hgComponents = new HashSet<String>(); hgComponents.add("NAMENODE"); hgComponents.add("SECONDARY_NAMENODE"); hgComponents.add("ZOOKEEPER_SERVER"); hgComponents.add("NIMBUS"); TestHostGroup group1 = new TestHostGroup("group1", hgComponents, Collections.singleton("testhost")); Collection<String> hgComponents2 = new HashSet<String>(); hgComponents2.add("DATANODE"); hgComponents2.add("HDFS_CLIENT"); hgComponents2.add("ZOOKEEPER_SERVER"); hgComponents2.add("NIMBUS"); Set<String> hosts2 = new HashSet<String>(); hosts2.add("testhost2"); hosts2.add("testhost2a"); hosts2.add("testhost2b"); TestHostGroup group2 = new TestHostGroup("group2", hgComponents2, hosts2); Collection<String> hgComponents3 = new HashSet<String>(); hgComponents2.add("HDFS_CLIENT"); hgComponents2.add("ZOOKEEPER_CLIENT"); Set<String> hosts3 = new HashSet<String>(); hosts3.add("testhost3"); hosts3.add("testhost3a"); TestHostGroup group3 = new TestHostGroup("group3", hgComponents3, hosts3); Collection<String> hgComponents4 = new HashSet<String>(); hgComponents4.add("NIMBUS"); Set<String> hosts4 = new HashSet<String>(); hosts4.add("testhost4"); TestHostGroup group4 = new TestHostGroup("group4", hgComponents4, hosts4); Collection<TestHostGroup> hostGroups = new HashSet<TestHostGroup>(); hostGroups.add(group1); hostGroups.add(group2); hostGroups.add(group3); hostGroups.add(group4); ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); BlueprintConfigurationProcessor updater = new BlueprintConfigurationProcessor(topology); updater.doUpdateForClusterCreate(); String updatedVal = topology.getConfiguration().getFullProperties().get("storm-site").get("storm.zookeeper.servers"); assertTrue(updatedVal.startsWith("[")); assertTrue(updatedVal.endsWith("]")); updatedVal = updatedVal.replaceAll("[\\[\\]]", ""); String[] hosts = updatedVal.split(","); Collection<String> expectedHosts = new HashSet<String>(); expectedHosts.add("'testhost:9090'"); expectedHosts.add("'testhost2:9091'"); expectedHosts.add("'testhost2a:9091'"); expectedHosts.add("'testhost2b:9091'"); assertEquals(4, hosts.length); for (String host : hosts) { assertTrue(expectedHosts.contains(host)); expectedHosts.remove(host); } String updatedNimbusSeedsVal = topology.getConfiguration().getFullProperties().get("storm-site").get("nimbus.seeds"); assertTrue("Updated YAML value should start with bracket", updatedNimbusSeedsVal.startsWith("[")); assertTrue("Updated YAML value should end with bracket", updatedNimbusSeedsVal.endsWith("]")); updatedNimbusSeedsVal = updatedNimbusSeedsVal.replaceAll("[\\[\\]]", ""); String[] nimbusHosts = updatedNimbusSeedsVal.split(","); Collection<String> expectedNimbusHosts = new HashSet<String>(); expectedNimbusHosts.add("testhost"); expectedNimbusHosts.add("testhost4"); assertEquals("Incorrect number of hosts found in updated Nimbus config property", 2, nimbusHosts.length); for (String host : nimbusHosts) { assertTrue("Expected Nimbus host = " + host + " not found in nimbus.seeds property value", expectedNimbusHosts.contains(host)); expectedHosts.remove(host); } }
@Test public void testDoUpdateForClusterCreate_DBHostProperty__defaultValue() throws Exception { Map<String, Map<String, String>> properties = new HashMap<String, Map<String, String>>(); Map<String, String> hiveSiteProps = new HashMap<String, String>(); hiveSiteProps.put("javax.jdo.option.ConnectionURL", "jdbc:mysql: Map<String, String> hiveEnvProps = new HashMap<String, String>(); hiveEnvProps.put("hive_database", "New MySQL Database"); properties.put("hive-site", hiveSiteProps); properties.put("hive-env", hiveEnvProps); Configuration clusterConfig = new Configuration(properties, Collections.<String, Map<String, Map<String, String>>>emptyMap()); Collection<String> hgComponents = new HashSet<String>(); hgComponents.add("NAMENODE"); hgComponents.add("SECONDARY_NAMENODE"); hgComponents.add("RESOURCEMANAGER"); hgComponents.add("MYSQL_SERVER"); TestHostGroup group1 = new TestHostGroup("group1", hgComponents, Collections.singleton("testhost")); Collection<String> hgComponents2 = new HashSet<String>(); hgComponents2.add("DATANODE"); hgComponents2.add("HDFS_CLIENT"); TestHostGroup group2 = new TestHostGroup("group2", hgComponents2, Collections.singleton("testhost2")); Collection<TestHostGroup> hostGroups = new HashSet<TestHostGroup>(); hostGroups.add(group1); hostGroups.add(group2); ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); BlueprintConfigurationProcessor updater = new BlueprintConfigurationProcessor(topology); updater.doUpdateForClusterCreate(); String updatedVal = topology.getConfiguration().getFullProperties().get("hive-site").get("javax.jdo.option.ConnectionURL"); assertEquals("jdbc:mysql: }
@Test public void testDoUpdateForClusterCreate_DBHostProperty__exportedValue() throws Exception { Map<String, Map<String, String>> properties = new HashMap<String, Map<String, String>>(); Map<String, String> hiveSiteProps = new HashMap<String, String>(); hiveSiteProps.put("javax.jdo.option.ConnectionURL", "jdbc:mysql: Map<String, String> hiveEnvProps = new HashMap<String, String>(); hiveEnvProps.put("hive_database", "New MySQL Database"); properties.put("hive-site", hiveSiteProps); properties.put("hive-env", hiveEnvProps); Configuration clusterConfig = new Configuration(properties, Collections.<String, Map<String, Map<String, String>>>emptyMap()); Collection<String> hgComponents = new HashSet<String>(); hgComponents.add("NAMENODE"); hgComponents.add("SECONDARY_NAMENODE"); hgComponents.add("RESOURCEMANAGER"); hgComponents.add("MYSQL_SERVER"); TestHostGroup group1 = new TestHostGroup("group1", hgComponents, Collections.singleton("testhost")); Collection<String> hgComponents2 = new HashSet<String>(); hgComponents2.add("DATANODE"); hgComponents2.add("HDFS_CLIENT"); TestHostGroup group2 = new TestHostGroup("group2", hgComponents2, Collections.singleton("testhost2")); Collection<TestHostGroup> hostGroups = new HashSet<TestHostGroup>(); hostGroups.add(group1); hostGroups.add(group2); ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); BlueprintConfigurationProcessor updater = new BlueprintConfigurationProcessor(topology); updater.doUpdateForClusterCreate(); String updatedVal = topology.getConfiguration().getFullProperties().get("hive-site").get("javax.jdo.option.ConnectionURL"); assertEquals("jdbc:mysql: }
@Test public void testDoUpdateForClusterCreate_DBHostProperty__external() throws Exception { Map<String, Map<String, String>> properties = new HashMap<String, Map<String, String>>(); Map<String, String> typeProps = new HashMap<String, String>(); typeProps.put("javax.jdo.option.ConnectionURL", "jdbc:mysql: typeProps.put("hive_database", "Existing MySQL Database"); properties.put("hive-env", typeProps); Configuration clusterConfig = new Configuration(properties, Collections.<String, Map<String, Map<String, String>>>emptyMap()); Collection<String> hgComponents = new HashSet<String>(); hgComponents.add("NAMENODE"); hgComponents.add("SECONDARY_NAMENODE"); hgComponents.add("RESOURCEMANAGER"); TestHostGroup group1 = new TestHostGroup("group1", hgComponents, Collections.singleton("testhost")); Collection<String> hgComponents2 = new HashSet<String>(); hgComponents2.add("DATANODE"); hgComponents2.add("HDFS_CLIENT"); TestHostGroup group2 = new TestHostGroup("group2", hgComponents2, Collections.singleton("testhost2")); Collection<TestHostGroup> hostGroups = new HashSet<TestHostGroup>(); hostGroups.add(group1); hostGroups.add(group2); ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); BlueprintConfigurationProcessor updater = new BlueprintConfigurationProcessor(topology); updater.doUpdateForClusterCreate(); String updatedVal = topology.getConfiguration().getFullProperties().get("hive-env").get("javax.jdo.option.ConnectionURL"); assertEquals("jdbc:mysql: }
@Test public void testExcludedPropertiesShouldBeAddedWhenServiceIsInBlueprint() throws Exception { reset(stack); expect(stack.getName()).andReturn("testStack").anyTimes(); expect(stack.getVersion()).andReturn("1").anyTimes(); expect(stack.isMasterComponent((String) anyObject())).andReturn(false).anyTimes(); expect(stack.getExcludedConfigurationTypes("FALCON")).andReturn(Collections.singleton("oozie-site")); expect(stack.getExcludedConfigurationTypes("OOZIE")).andReturn(Collections.<String>emptySet()); expect(stack.getConfigurationProperties("FALCON", "oozie-site")).andReturn(Collections.singletonMap("oozie.service.ELService.ext.functions.coord-job-submit-instances", "testValue")).anyTimes(); expect(stack.getServiceForConfigType("oozie-site")).andReturn("OOZIE").anyTimes(); Map<String, Map<String, String>> properties = new HashMap<String, Map<String, String>>(); Configuration clusterConfig = new Configuration(properties, Collections.<String, Map<String, Map<String, String>>>emptyMap()); Collection<String> hgComponents = new HashSet<String>(); hgComponents.add("FALCON_SERVER"); hgComponents.add("FALCON_CLIENT"); hgComponents.add("OOZIE_SERVER"); hgComponents.add("OOZIE_CLIENT"); List<String> hosts = new ArrayList<String>(); hosts.add("c6401.apache.ambari.org"); hosts.add("serverTwo"); TestHostGroup group1 = new TestHostGroup("host_group_1", hgComponents, hosts); Collection<TestHostGroup> hostGroups = new HashSet<TestHostGroup>(); hostGroups.add(group1); ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); BlueprintConfigurationProcessor updater = new BlueprintConfigurationProcessor(topology); updater.doUpdateForClusterCreate(); assertEquals("Falcon Broker URL property not properly exported", "testValue", clusterConfig.getPropertyValue("oozie-site", "oozie.service.ELService.ext.functions.coord-job-submit-instances")); }
@Test public void testExcludedPropertiesShouldBeIgnoredWhenServiceIsNotInBlueprint() throws Exception { reset(stack); expect(stack.getName()).andReturn("testStack").anyTimes(); expect(stack.getVersion()).andReturn("1").anyTimes(); expect(stack.isMasterComponent((String) anyObject())).andReturn(false).anyTimes(); expect(stack.getExcludedConfigurationTypes("FALCON")).andReturn(Collections.singleton("oozie-site")).anyTimes(); expect(stack.getConfigurationProperties("FALCON", "oozie-site")).andReturn(Collections.singletonMap("oozie.service.ELService.ext.functions.coord-job-submit-instances", "testValue")).anyTimes(); expect(stack.getServiceForConfigType("oozie-site")).andReturn("OOZIE").anyTimes(); Map<String, Map<String, String>> properties = new HashMap<String, Map<String, String>>(); Configuration clusterConfig = new Configuration(properties, Collections.<String, Map<String, Map<String, String>>>emptyMap()); Collection<String> hgComponents = new HashSet<String>(); hgComponents.add("FALCON_SERVER"); hgComponents.add("FALCON_CLIENT"); List<String> hosts = new ArrayList<String>(); hosts.add("c6401.apache.ambari.org"); hosts.add("serverTwo"); TestHostGroup group1 = new TestHostGroup("host_group_1", hgComponents, hosts); Collection<TestHostGroup> hostGroups = new HashSet<TestHostGroup>(); hostGroups.add(group1); ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); BlueprintConfigurationProcessor updater = new BlueprintConfigurationProcessor(topology); updater.doUpdateForClusterCreate(); Assert.assertNull("Excluded properties shouldn't be added in this setup!", clusterConfig.getPropertyValue("oozie-site", "oozie.service.ELService.ext.functions.coord-job-submit-instances")); }
@Test public void testAddExcludedPropertiesAreOverwrittenByBlueprintConfigs() throws Exception { reset(stack); expect(stack.getName()).andReturn("testStack").anyTimes(); expect(stack.getVersion()).andReturn("1").anyTimes(); expect(stack.isMasterComponent((String) anyObject())).andReturn(false).anyTimes(); expect(stack.getConfigurationPropertiesWithMetadata(anyObject(String.class), anyObject(String.class))).andReturn(Collections.<String, Stack.ConfigProperty>emptyMap()).anyTimes(); expect(stack.getExcludedConfigurationTypes("FALCON")).andReturn(Collections.singleton("oozie-site")).anyTimes(); expect(stack.getConfigurationProperties("FALCON", "oozie-site")).andReturn(Collections.singletonMap("oozie.service.ELService.ext.functions.coord-job-submit-instances", "testValue")).anyTimes(); Map<String, Map<String, String>> properties = new HashMap<String, Map<String, String>>(); Map<String, String> typeProps = new HashMap<String, String>(); typeProps.put("oozie.service.ELService.ext.functions.coord-job-submit-instances", "overridedValue"); properties.put("oozie-site", typeProps); Configuration clusterConfig = new Configuration(properties, Collections.<String, Map<String, Map<String, String>>>emptyMap()); Collection<String> hgComponents = new HashSet<String>(); hgComponents.add("FALCON_SERVER"); hgComponents.add("FALCON_CLIENT"); List<String> hosts = new ArrayList<String>(); hosts.add("c6401.apache.ambari.org"); hosts.add("serverTwo"); TestHostGroup group1 = new TestHostGroup("host_group_1", hgComponents, hosts); Collection<TestHostGroup> hostGroups = new HashSet<TestHostGroup>(); hostGroups.add(group1); ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); BlueprintConfigurationProcessor updater = new BlueprintConfigurationProcessor(topology); updater.doUpdateForClusterCreate(); assertEquals("Falcon Broker URL property not properly exported", "overridedValue", clusterConfig.getPropertyValue("oozie-site", "oozie.service.ELService.ext.functions.coord-job-submit-instances")); }
@Test public void testExcludedPropertiesHandlingWhenExcludedConfigServiceIsNotFoundInStack() throws Exception { reset(stack); expect(stack.getName()).andReturn("testStack").anyTimes(); expect(stack.getVersion()).andReturn("1").anyTimes(); expect(stack.isMasterComponent((String) anyObject())).andReturn(false).anyTimes(); Set<String> excludedConfigTypes = new HashSet<String>(); excludedConfigTypes.add("oozie-site"); excludedConfigTypes.add("storm-site"); expect(stack.getExcludedConfigurationTypes("FALCON")).andReturn(excludedConfigTypes); expect(stack.getExcludedConfigurationTypes("OOZIE")).andReturn(Collections.<String>emptySet()); expect(stack.getConfigurationProperties("FALCON", "oozie-site")).andReturn(Collections.singletonMap("oozie.service.ELService.ext.functions.coord-job-submit-instances", "testValue")).anyTimes(); expect(stack.getServiceForConfigType("oozie-site")).andReturn("OOZIE").anyTimes(); expect(stack.getServiceForConfigType("storm-site")).andThrow(new IllegalArgumentException("TEST: Configuration not found in stack definitions!")); Map<String, Map<String, String>> properties = new HashMap<String, Map<String, String>>(); Configuration clusterConfig = new Configuration(properties, Collections.<String, Map<String, Map<String, String>>>emptyMap()); Collection<String> hgComponents = new HashSet<String>(); hgComponents.add("FALCON_SERVER"); hgComponents.add("FALCON_CLIENT"); hgComponents.add("OOZIE_SERVER"); hgComponents.add("OOZIE_CLIENT"); List<String> hosts = new ArrayList<String>(); hosts.add("c6401.apache.ambari.org"); hosts.add("serverTwo"); TestHostGroup group1 = new TestHostGroup("host_group_1", hgComponents, hosts); Collection<TestHostGroup> hostGroups = new HashSet<TestHostGroup>(); hostGroups.add(group1); ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); BlueprintConfigurationProcessor updater = new BlueprintConfigurationProcessor(topology); updater.doUpdateForClusterCreate(); assertEquals("Falcon Broker URL property not properly exported", "testValue", clusterConfig.getPropertyValue("oozie-site", "oozie.service.ELService.ext.functions.coord-job-submit-instances")); }
@Test public void testFalconConfigClusterUpdate() throws Exception { final String expectedHostName = "c6401.apache.ambari.org"; final String expectedPortNum = "808080"; final String expectedHostGroupName = "host_group_1"; Map<String, Map<String, String>> properties = new HashMap<String, Map<String, String>>(); Map<String, String> falconStartupProperties = new HashMap<String, String>(); properties.put("falcon-startup.properties", falconStartupProperties); falconStartupProperties.put("*.broker.url", createExportedAddress(expectedPortNum, expectedHostGroupName)); falconStartupProperties.put("*.falcon.service.authentication.kerberos.principal", "falcon/" + createExportedHostName(expectedHostGroupName) + "@EXAMPLE.COM"); falconStartupProperties.put("*.falcon.http.authentication.kerberos.principal", "HTTP/" + createExportedHostName(expectedHostGroupName) + "@EXAMPLE.COM"); Configuration clusterConfig = new Configuration(properties, Collections.<String, Map<String, Map<String, String>>>emptyMap()); Collection<String> hgComponents = new HashSet<String>(); hgComponents.add("FALCON_SERVER"); hgComponents.add("FALCON_CLIENT"); List<String> hosts = new ArrayList<String>(); hosts.add("c6401.apache.ambari.org"); hosts.add("server-two"); TestHostGroup group1 = new TestHostGroup("host_group_1", hgComponents, hosts); Collection<TestHostGroup> hostGroups = new HashSet<TestHostGroup>(); hostGroups.add(group1); ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); BlueprintConfigurationProcessor updater = new BlueprintConfigurationProcessor(topology); updater.doUpdateForClusterCreate(); assertEquals("Falcon Broker URL property not properly exported", expectedHostName + ":" + expectedPortNum, falconStartupProperties.get("*.broker.url")); assertEquals("Falcon Kerberos Principal property not properly exported", "falcon/" + expectedHostName + "@EXAMPLE.COM", falconStartupProperties.get("*.falcon.service.authentication.kerberos.principal")); assertEquals("Falcon Kerberos HTTP Principal property not properly exported", "HTTP/" + expectedHostName + "@EXAMPLE.COM", falconStartupProperties.get("*.falcon.http.authentication.kerberos.principal")); }
@Test public void testFalconConfigClusterUpdateDefaultConfig() throws Exception { final String expectedHostName = "c6401.apache.ambari.org"; final String expectedPortNum = "808080"; final String expectedHostGroupName = "host_group_1"; Map<String, Map<String, String>> properties = new HashMap<String, Map<String, String>>(); Map<String, String> falconStartupProperties = new HashMap<String, String>(); properties.put("falcon-startup.properties", falconStartupProperties); falconStartupProperties.put("*.broker.url", "localhost:" + expectedPortNum); falconStartupProperties.put("*.falcon.service.authentication.kerberos.principal", "falcon/" + "localhost" + "@EXAMPLE.COM"); falconStartupProperties.put("*.falcon.http.authentication.kerberos.principal", "HTTP/" + "localhost" + "@EXAMPLE.COM"); Configuration clusterConfig = new Configuration(properties, Collections.<String, Map<String, Map<String, String>>>emptyMap()); Collection<String> hgComponents = new HashSet<String>(); hgComponents.add("FALCON_SERVER"); hgComponents.add("FALCON_CLIENT"); List<String> hosts = new ArrayList<String>(); hosts.add(expectedHostName); hosts.add("server-two"); TestHostGroup group1 = new TestHostGroup(expectedHostGroupName, hgComponents, hosts); Collection<TestHostGroup> hostGroups = new HashSet<TestHostGroup>(); hostGroups.add(group1); ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); BlueprintConfigurationProcessor updater = new BlueprintConfigurationProcessor(topology); updater.doUpdateForClusterCreate(); assertEquals("Falcon Broker URL property not properly exported", expectedHostName + ":" + expectedPortNum, falconStartupProperties.get("*.broker.url")); assertEquals("Falcon Kerberos Principal property not properly exported", "falcon/" + expectedHostName + "@EXAMPLE.COM", falconStartupProperties.get("*.falcon.service.authentication.kerberos.principal")); assertEquals("Falcon Kerberos HTTP Principal property not properly exported", "HTTP/" + expectedHostName + "@EXAMPLE.COM", falconStartupProperties.get("*.falcon.http.authentication.kerberos.principal")); }
@Test public void testHiveConfigClusterUpdateCustomValue() throws Exception { final String expectedHostGroupName = "host_group_1"; final String expectedPropertyValue = "hive.metastore.local=false,hive.metastore.uris=thrift: Map<String, Map<String, String>> properties = new HashMap<String, Map<String, String>>(); Map<String, String> webHCatSiteProperties = new HashMap<String, String>(); properties.put("webhcat-site", webHCatSiteProperties); webHCatSiteProperties.put("templeton.hive.properties", expectedPropertyValue); Configuration clusterConfig = new Configuration(properties, Collections.<String, Map<String, Map<String, String>>>emptyMap()); Collection<String> hgComponents = new HashSet<String>(); hgComponents.add("NAMENODE"); List<String> hosts = new ArrayList<String>(); hosts.add("some-hose"); TestHostGroup group1 = new TestHostGroup(expectedHostGroupName, hgComponents, hosts); Collection<TestHostGroup> hostGroups = new HashSet<TestHostGroup>(); hostGroups.add(group1); ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); BlueprintConfigurationProcessor updater = new BlueprintConfigurationProcessor(topology); updater.doUpdateForClusterCreate(); assertEquals("Unexpected config update for templeton.hive.properties", expectedPropertyValue, webHCatSiteProperties.get("templeton.hive.properties")); }
@Test public void testHiveConfigClusterUpdatePropertiesFilterAuthenticationOff() throws Exception { reset(stack); final String expectedHostGroupName = "host_group_1"; Map<String, Map<String, String>> properties = new HashMap<String, Map<String, String>>(); Map<String, String> hiveSiteProperties = new HashMap<String, String>(); properties.put("hive-site", hiveSiteProperties); hiveSiteProperties.put("hive.server2.authentication", "NONE"); hiveSiteProperties.put("hive.server2.authentication.kerberos.keytab", " "); hiveSiteProperties.put("hive.server2.authentication.kerberos.principal", " "); Map<String, Stack.ConfigProperty> mapOfMetadata = new HashMap<String, Stack.ConfigProperty>(); Stack.ConfigProperty configProperty1 = new Stack.ConfigProperty("hive-site", "hive.server2.authentication.kerberos.keytab", " ") { @Override Set<PropertyDependencyInfo> getDependsOnProperties() { PropertyDependencyInfo dependencyInfo = new PropertyDependencyInfo("hive-site", "hive.server2.authentication"); return Collections.singleton(dependencyInfo); } }; Stack.ConfigProperty configProperty2 = new Stack.ConfigProperty("hive-site", "hive.server2.authentication.kerberos.principal", " ") { @Override Set<PropertyDependencyInfo> getDependsOnProperties() { PropertyDependencyInfo dependencyInfo = new PropertyDependencyInfo("hive-site", "hive.server2.authentication"); return Collections.singleton(dependencyInfo); } }; mapOfMetadata.put("hive.server2.authentication.kerberos.keytab", configProperty1); mapOfMetadata.put("hive.server2.authentication.kerberos.principal", configProperty2); expect(stack.getName()).andReturn("testStack").anyTimes(); expect(stack.getVersion()).andReturn("1").anyTimes(); expect(stack.isMasterComponent((String) anyObject())).andReturn(false).anyTimes(); Set<String> emptySet = Collections.emptySet(); expect(stack.getExcludedConfigurationTypes(anyObject(String.class))).andReturn(emptySet).anyTimes(); expect(stack.getServiceForConfigType("hive-site")).andReturn("HIVE").atLeastOnce(); expect(stack.getConfigurationPropertiesWithMetadata("HIVE", "hive-site")).andReturn(mapOfMetadata).atLeastOnce(); Configuration clusterConfig = new Configuration(properties, Collections.<String, Map<String, Map<String, String>>>emptyMap()); Collection<String> hgComponents = new HashSet<String>(); hgComponents.add("NAMENODE"); List<String> hosts = new ArrayList<String>(); hosts.add("some-hose"); TestHostGroup group1 = new TestHostGroup(expectedHostGroupName, hgComponents, hosts); Collection<TestHostGroup> hostGroups = new HashSet<TestHostGroup>(); hostGroups.add(group1); ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); BlueprintConfigurationProcessor updater = new BlueprintConfigurationProcessor(topology); updater.doUpdateForClusterCreate(); assertFalse("hive.server2.authentication.kerberos.keytab should have been filtered out of configuration", hiveSiteProperties.containsKey("hive.server2.authentication.kerberos.keytab")); assertFalse("hive.server2.authentication.kerberos.principal should have been filtered out of configuration", hiveSiteProperties.containsKey("hive.server2.authentication.kerberos.principal")); }
@Test public void testHiveConfigClusterUpdatePropertiesFilterAuthenticationOffFilterThrowsError() throws Exception { reset(stack); final String expectedHostGroupName = "host_group_1"; Map<String, Map<String, String>> properties = new HashMap<String, Map<String, String>>(); Map<String, String> hiveSiteProperties = new HashMap<String, String>(); properties.put("hive-site", hiveSiteProperties); hiveSiteProperties.put("hive.server2.authentication", "NONE"); hiveSiteProperties.put("hive.server2.authentication.kerberos.keytab", " "); hiveSiteProperties.put("hive.server2.authentication.kerberos.principal", " "); Map<String, Stack.ConfigProperty> mapOfMetadata = new HashMap<String, Stack.ConfigProperty>(); Stack.ConfigProperty configProperty1 = new Stack.ConfigProperty("hive-site", "hive.server2.authentication.kerberos.keytab", " ") { @Override Set<PropertyDependencyInfo> getDependsOnProperties() { PropertyDependencyInfo dependencyInfo = new PropertyDependencyInfo("hive-site", "hive.server2.authentication"); return Collections.singleton(dependencyInfo); } }; Stack.ConfigProperty configProperty2 = new Stack.ConfigProperty("hive-site", "hive.server2.authentication.kerberos.principal", " ") { @Override Set<PropertyDependencyInfo> getDependsOnProperties() { PropertyDependencyInfo dependencyInfo = new PropertyDependencyInfo("hive-site", "hive.server2.authentication"); return Collections.singleton(dependencyInfo); } }; mapOfMetadata.put("hive.server2.authentication.kerberos.keytab", configProperty1); mapOfMetadata.put("hive.server2.authentication.kerberos.principal", configProperty2); expect(stack.getName()).andReturn("testStack").anyTimes(); expect(stack.getVersion()).andReturn("1").anyTimes(); expect(stack.isMasterComponent((String) anyObject())).andReturn(false).anyTimes(); Set<String> emptySet = Collections.emptySet(); expect(stack.getExcludedConfigurationTypes(anyObject(String.class))).andReturn(emptySet).anyTimes(); expect(stack.getServiceForConfigType("hive-site")).andThrow(new RuntimeException("Expected Test Error")).once(); expect(stack.getConfigurationPropertiesWithMetadata("HIVE", "hive-site")).andReturn(mapOfMetadata).atLeastOnce(); Configuration clusterConfig = new Configuration(properties, Collections.<String, Map<String, Map<String, String>>>emptyMap()); Collection<String> hgComponents = new HashSet<String>(); hgComponents.add("NAMENODE"); List<String> hosts = new ArrayList<String>(); hosts.add("some-hose"); TestHostGroup group1 = new TestHostGroup(expectedHostGroupName, hgComponents, hosts); Collection<TestHostGroup> hostGroups = new HashSet<TestHostGroup>(); hostGroups.add(group1); ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); BlueprintConfigurationProcessor updater = new BlueprintConfigurationProcessor(topology); updater.doUpdateForClusterCreate(); assertTrue("hive.server2.authentication.kerberos.keytab should not have been filtered, due to error condition", hiveSiteProperties.containsKey("hive.server2.authentication.kerberos.keytab")); assertTrue("hive.server2.authentication.kerberos.principal should not have been filtered, due to error condition", hiveSiteProperties.containsKey("hive.server2.authentication.kerberos.principal")); }
@Test public void testHiveConfigClusterUpdatePropertiesFilterAuthenticationOn() throws Exception { reset(stack); final String expectedHostGroupName = "host_group_1"; Map<String, Map<String, String>> properties = new HashMap<String, Map<String, String>>(); Map<String, String> hiveSiteProperties = new HashMap<String, String>(); properties.put("hive-site", hiveSiteProperties); hiveSiteProperties.put("hive.server2.authentication", "KERBEROS"); hiveSiteProperties.put("hive.server2.authentication.kerberos.keytab", " "); hiveSiteProperties.put("hive.server2.authentication.kerberos.principal", " "); Map<String, Stack.ConfigProperty> mapOfMetadata = new HashMap<String, Stack.ConfigProperty>(); Stack.ConfigProperty configProperty1 = new Stack.ConfigProperty("hive-site", "hive.server2.authentication.kerberos.keytab", " ") { @Override Set<PropertyDependencyInfo> getDependsOnProperties() { PropertyDependencyInfo dependencyInfo = new PropertyDependencyInfo("hive-site", "hive.server2.authentication"); return Collections.singleton(dependencyInfo); } }; Stack.ConfigProperty configProperty2 = new Stack.ConfigProperty("hive-site", "hive.server2.authentication.kerberos.principal", " ") { @Override Set<PropertyDependencyInfo> getDependsOnProperties() { PropertyDependencyInfo dependencyInfo = new PropertyDependencyInfo("hive-site", "hive.server2.authentication"); return Collections.singleton(dependencyInfo); } }; mapOfMetadata.put("hive.server2.authentication.kerberos.keytab", configProperty1); mapOfMetadata.put("hive.server2.authentication.kerberos.principal", configProperty2); expect(stack.getName()).andReturn("testStack").anyTimes(); expect(stack.getVersion()).andReturn("1").anyTimes(); expect(stack.isMasterComponent((String) anyObject())).andReturn(false).anyTimes(); Set<String> emptySet = Collections.emptySet(); expect(stack.getExcludedConfigurationTypes(anyObject(String.class))).andReturn(emptySet).anyTimes(); expect(stack.getServiceForConfigType("hive-site")).andReturn("HIVE").atLeastOnce(); expect(stack.getConfigurationPropertiesWithMetadata("HIVE", "hive-site")).andReturn(mapOfMetadata).atLeastOnce(); Configuration clusterConfig = new Configuration(properties, Collections.<String, Map<String, Map<String, String>>>emptyMap()); Collection<String> hgComponents = new HashSet<String>(); hgComponents.add("NAMENODE"); List<String> hosts = new ArrayList<String>(); hosts.add("some-hose"); TestHostGroup group1 = new TestHostGroup(expectedHostGroupName, hgComponents, hosts); Collection<TestHostGroup> hostGroups = new HashSet<TestHostGroup>(); hostGroups.add(group1); ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); BlueprintConfigurationProcessor updater = new BlueprintConfigurationProcessor(topology); updater.doUpdateForClusterCreate(); assertTrue("hive.server2.authentication.kerberos.keytab should have been included in configuration", hiveSiteProperties.containsKey("hive.server2.authentication.kerberos.keytab")); assertTrue("hive.server2.authentication.kerberos.principal should have been included in configuration", hiveSiteProperties.containsKey("hive.server2.authentication.kerberos.principal")); }
@Test public void testHBaseConfigClusterUpdatePropertiesFilterAuthorizationOff() throws Exception { reset(stack); final String expectedHostGroupName = "host_group_1"; Map<String, Map<String, String>> properties = new HashMap<String, Map<String, String>>(); Map<String, String> hbaseSiteProperties = new HashMap<String, String>(); properties.put("hbase-site", hbaseSiteProperties); hbaseSiteProperties.put("hbase.security.authorization", "false"); hbaseSiteProperties.put("hbase.coprocessor.regionserver.classes", " "); hbaseSiteProperties.put("hbase.coprocessor.master.classes", ""); Map<String, Stack.ConfigProperty> mapOfMetadata = new HashMap<String, Stack.ConfigProperty>(); Stack.ConfigProperty configProperty1 = new Stack.ConfigProperty("hbase-site", "hbase.coprocessor.regionserver.classes", " ") { @Override Set<PropertyDependencyInfo> getDependsOnProperties() { PropertyDependencyInfo dependencyInfo = new PropertyDependencyInfo("hbase-site", "hbase.security.authorization"); return Collections.singleton(dependencyInfo); } }; Stack.ConfigProperty configProperty2 = new Stack.ConfigProperty("hbase-site", "hbase.coprocessor.master.classes", "") { @Override Set<PropertyDependencyInfo> getDependsOnProperties() { PropertyDependencyInfo dependencyInfo = new PropertyDependencyInfo("hbase-site", "hbase.security.authorization"); return Collections.singleton(dependencyInfo); } }; Stack.ConfigProperty configProperty3 = new Stack.ConfigProperty("hbase-site", "hbase.coprocessor.region.classes", "") { @Override Set<PropertyDependencyInfo> getDependsOnProperties() { PropertyDependencyInfo dependencyInfo = new PropertyDependencyInfo("hbase-site", "hbase.security.authorization"); return Collections.singleton(dependencyInfo); } }; mapOfMetadata.put("hbase.coprocessor.regionserver.classes", configProperty1); mapOfMetadata.put("hbase.coprocessor.master.classes", configProperty2); mapOfMetadata.put("hbase.coprocessor.region.classes", configProperty3); expect(stack.getName()).andReturn("testStack").anyTimes(); expect(stack.getVersion()).andReturn("1").anyTimes(); expect(stack.isMasterComponent((String) anyObject())).andReturn(false).anyTimes(); Set<String> emptySet = Collections.emptySet(); expect(stack.getExcludedConfigurationTypes(anyObject(String.class))).andReturn(emptySet).anyTimes(); expect(stack.getServiceForConfigType("hbase-site")).andReturn("HBASE").atLeastOnce(); expect(stack.getConfigurationPropertiesWithMetadata("HBASE", "hbase-site")).andReturn(mapOfMetadata).atLeastOnce(); Configuration clusterConfig = new Configuration(properties, Collections.<String, Map<String, Map<String, String>>>emptyMap()); Collection<String> hgComponents = new HashSet<String>(); hgComponents.add("NAMENODE"); List<String> hosts = new ArrayList<String>(); hosts.add("some-hose"); TestHostGroup group1 = new TestHostGroup(expectedHostGroupName, hgComponents, hosts); Collection<TestHostGroup> hostGroups = new HashSet<TestHostGroup>(); hostGroups.add(group1); ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); BlueprintConfigurationProcessor updater = new BlueprintConfigurationProcessor(topology); updater.doUpdateForClusterCreate(); assertFalse("hbase.coprocessor.regionserver.classes should have been filtered out of configuration", hbaseSiteProperties.containsKey("hbase.coprocessor.regionserver.classes")); assertTrue("hbase.coprocessor.master.classes should not have been filtered out of configuration", hbaseSiteProperties.containsKey("hbase.coprocessor.master.classes")); assertTrue("hbase.coprocessor.region.classes should not have been filtered out of configuration", hbaseSiteProperties.containsKey("hbase.coprocessor.master.classes")); }
@Test public void testHBaseConfigClusterUpdatePropertiesFilterAuthorizationOn() throws Exception { reset(stack); final String expectedHostGroupName = "host_group_1"; Map<String, Map<String, String>> properties = new HashMap<String, Map<String, String>>(); Map<String, String> hbaseSiteProperties = new HashMap<String, String>(); properties.put("hbase-site", hbaseSiteProperties); hbaseSiteProperties.put("hbase.security.authorization", "true"); hbaseSiteProperties.put("hbase.coprocessor.regionserver.classes", " "); Map<String, Stack.ConfigProperty> mapOfMetadata = new HashMap<String, Stack.ConfigProperty>(); Stack.ConfigProperty configProperty1 = new Stack.ConfigProperty("hbase-site", "hbase.coprocessor.regionserver.classes", " ") { @Override Set<PropertyDependencyInfo> getDependsOnProperties() { PropertyDependencyInfo dependencyInfo = new PropertyDependencyInfo("hbase-site", "hbase.security.authorization"); return Collections.singleton(dependencyInfo); } }; mapOfMetadata.put("hbase.coprocessor.regionserver.classes", configProperty1); expect(stack.getName()).andReturn("testStack").anyTimes(); expect(stack.getVersion()).andReturn("1").anyTimes(); expect(stack.isMasterComponent((String) anyObject())).andReturn(false).anyTimes(); Set<String> emptySet = Collections.emptySet(); expect(stack.getExcludedConfigurationTypes(anyObject(String.class))).andReturn(emptySet).anyTimes(); expect(stack.getServiceForConfigType("hbase-site")).andReturn("HBASE").atLeastOnce(); expect(stack.getConfigurationPropertiesWithMetadata("HBASE", "hbase-site")).andReturn(mapOfMetadata).atLeastOnce(); Configuration clusterConfig = new Configuration(properties, Collections.<String, Map<String, Map<String, String>>>emptyMap()); Collection<String> hgComponents = new HashSet<String>(); hgComponents.add("NAMENODE"); List<String> hosts = new ArrayList<String>(); hosts.add("some-hose"); TestHostGroup group1 = new TestHostGroup(expectedHostGroupName, hgComponents, hosts); Collection<TestHostGroup> hostGroups = new HashSet<TestHostGroup>(); hostGroups.add(group1); ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); BlueprintConfigurationProcessor updater = new BlueprintConfigurationProcessor(topology); updater.doUpdateForClusterCreate(); assertTrue("hbase.coprocessor.regionserver.classes should have been included in configuration", hbaseSiteProperties.containsKey("hbase.coprocessor.regionserver.classes")); }
@Test public void testHiveConfigClusterUpdateDefaultValue() throws Exception { final String expectedHostGroupName = "host_group_1"; final String expectedHostName = "c6401.ambari.apache.org"; final String expectedPropertyValue = "hive.metastore.local=false,hive.metastore.uris=thrift: Map<String, Map<String, String>> properties = new HashMap<String, Map<String, String>>(); Map<String, String> webHCatSiteProperties = new HashMap<String, String>(); properties.put("webhcat-site", webHCatSiteProperties); webHCatSiteProperties.put("templeton.hive.properties", expectedPropertyValue); Configuration clusterConfig = new Configuration(properties, Collections.<String, Map<String, Map<String, String>>>emptyMap()); Collection<String> hgComponents = new HashSet<String>(); hgComponents.add("HIVE_METASTORE"); List<String> hosts = new ArrayList<String>(); hosts.add(expectedHostName); TestHostGroup group1 = new TestHostGroup(expectedHostGroupName, hgComponents, hosts); Collection<TestHostGroup> hostGroups = new HashSet<TestHostGroup>(); hostGroups.add(group1); ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); BlueprintConfigurationProcessor updater = new BlueprintConfigurationProcessor(topology); updater.doUpdateForClusterCreate(); assertEquals("Unexpected config update for templeton.hive.properties", "hive.metastore.local=false,hive.metastore.uris=thrift: webHCatSiteProperties.get("templeton.hive.properties")); }
@Test public void testAtlas() throws Exception { final String expectedHostGroupName = "host_group_1"; final String host1 = "c6401.ambari.apache.org"; final String host2 = "c6402.ambari.apache.org"; final String host3 = "c6403.ambari.apache.org"; Map<String, Map<String, String>> properties = new HashMap<String, Map<String, String>>(); Map<String, String> atlasProperties = new HashMap<String, String>(); properties.put("application-properties", atlasProperties); atlasProperties.put("atlas.kafka.bootstrap.servers", "localhost:6667"); atlasProperties.put("atlas.kafka.zookeeper.connect", "localhost:2181"); atlasProperties.put("atlas.graph.index.search.solr.zookeeper-url", "localhost:2181/ambari-solr"); atlasProperties.put("atlas.graph.storage.hostname", "localhost"); atlasProperties.put("atlas.audit.hbase.zookeeper.quorum", "localhost"); Configuration clusterConfig = new Configuration(properties, Collections.<String, Map<String, Map<String, String>>>emptyMap()); Collection<String> hgComponents = new HashSet<String>(); hgComponents.add("KAFKA_BROKER"); hgComponents.add("ZOOKEEPER_SERVER"); hgComponents.add("HBASE_MASTER"); List<String> hosts = new ArrayList<String>(); hosts.add(host1); hosts.add(host2); hosts.add(host3); TestHostGroup group1 = new TestHostGroup(expectedHostGroupName, hgComponents, hosts); Collection<TestHostGroup> hostGroups = new HashSet<TestHostGroup>(); hostGroups.add(group1); ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); BlueprintConfigurationProcessor updater = new BlueprintConfigurationProcessor(topology); updater.doUpdateForClusterCreate(); List<String> hostArray = Arrays.asList(atlasProperties.get("atlas.kafka.bootstrap.servers").split(",")); List<String> expected = Arrays.asList("c6401.ambari.apache.org:6667","c6402.ambari.apache.org:6667", "c6403.ambari.apache.org:6667"); Assert.assertTrue(hostArray.containsAll(expected) && expected.containsAll(hostArray)); hostArray = Arrays.asList(atlasProperties.get("atlas.kafka.zookeeper.connect").split(",")); expected = Arrays.asList("c6401.ambari.apache.org:2181","c6402.ambari.apache.org:2181", "c6403.ambari.apache.org:2181"); Assert.assertTrue(hostArray.containsAll(expected) && expected.containsAll(hostArray)); hostArray = Arrays.asList(atlasProperties.get("atlas.graph.index.search.solr.zookeeper-url").split(",")); expected = Arrays.asList("c6401.ambari.apache.org:2181/ambari-solr","c6402.ambari.apache.org:2181/ambari-solr", "c6403.ambari.apache.org:2181/ambari-solr"); Assert.assertTrue(hostArray.containsAll(expected) && expected.containsAll(hostArray)); hostArray = Arrays.asList(atlasProperties.get("atlas.graph.storage.hostname").split(",")); expected = Arrays.asList("c6401.ambari.apache.org","c6402.ambari.apache.org", "c6403.ambari.apache.org"); Assert.assertTrue(hostArray.containsAll(expected) && expected.containsAll(hostArray)); hostArray = Arrays.asList(atlasProperties.get("atlas.audit.hbase.zookeeper.quorum").split(",")); expected = Arrays.asList("c6401.ambari.apache.org","c6402.ambari.apache.org", "c6403.ambari.apache.org"); Assert.assertTrue(hostArray.containsAll(expected) && expected.containsAll(hostArray)); }
@Test public void testHiveConfigClusterUpdateExportedHostGroupValue() throws Exception { final String expectedHostGroupName = "host_group_1"; final String expectedHostName = "c6401.ambari.apache.org"; final String expectedPropertyValue = "hive.metastore.local=false,hive.metastore.uris=thrift: Map<String, Map<String, String>> properties = new HashMap<String, Map<String, String>>(); Map<String, String> webHCatSiteProperties = new HashMap<String, String>(); properties.put("webhcat-site", webHCatSiteProperties); webHCatSiteProperties.put("templeton.hive.properties", expectedPropertyValue); Configuration clusterConfig = new Configuration(properties, Collections.<String, Map<String, Map<String, String>>>emptyMap()); Collection<String> hgComponents = new HashSet<String>(); hgComponents.add("HIVE_METASTORE"); List<String> hosts = new ArrayList<String>(); hosts.add(expectedHostName); TestHostGroup group1 = new TestHostGroup(expectedHostGroupName, hgComponents, hosts); Collection<TestHostGroup> hostGroups = new HashSet<TestHostGroup>(); hostGroups.add(group1); ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); BlueprintConfigurationProcessor updater = new BlueprintConfigurationProcessor(topology); updater.doUpdateForClusterCreate(); assertEquals("Unexpected config update for templeton.hive.properties", "hive.metastore.local=false,hive.metastore.uris=thrift: webHCatSiteProperties.get("templeton.hive.properties")); }
@Test public void testStormAndKafkaConfigClusterUpdateWithoutGangliaServer() throws Exception { final String expectedHostGroupName = "host_group_1"; Map<String, Map<String, String>> properties = new HashMap<String, Map<String, String>>(); Map<String, String> stormSiteProperties = new HashMap<String, String>(); Map<String, String> kafkaBrokerProperties = new HashMap<String, String>(); properties.put("storm-site", stormSiteProperties); properties.put("kafka-broker", kafkaBrokerProperties); stormSiteProperties.put("worker.childopts", "localhost"); stormSiteProperties.put("supervisor.childopts", "localhost"); stormSiteProperties.put("nimbus.childopts", "localhost"); kafkaBrokerProperties.put("kafka.ganglia.metrics.host", "localhost"); Configuration clusterConfig = new Configuration(properties, Collections.<String, Map<String, Map<String, String>>>emptyMap()); Collection<String> hgComponents = new HashSet<String>(); hgComponents.add("HIVE_METASTORE"); TestHostGroup group1 = new TestHostGroup(expectedHostGroupName, hgComponents, Collections.singleton("testserver")); Collection<TestHostGroup> hostGroups = new HashSet<TestHostGroup>(); hostGroups.add(group1); expect(stack.getCardinality("GANGLIA_SERVER")).andReturn(new Cardinality("1")).anyTimes(); ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); BlueprintConfigurationProcessor updater = new BlueprintConfigurationProcessor(topology); updater.doUpdateForClusterCreate(); assertEquals("worker startup settings not properly handled by cluster create", "localhost", stormSiteProperties.get("worker.childopts")); assertEquals("supervisor startup settings not properly handled by cluster create", "localhost", stormSiteProperties.get("supervisor.childopts")); assertEquals("nimbus startup settings not properly handled by cluster create", "localhost", stormSiteProperties.get("nimbus.childopts")); assertEquals("Kafka ganglia host property not properly handled by cluster create", "localhost", kafkaBrokerProperties.get("kafka.ganglia.metrics.host")); }
@Test public void testStormandKafkaConfigClusterUpdateWithGangliaServer() throws Exception { final String expectedHostName = "c6401.apache.ambari.org"; final String expectedHostGroupName = "host_group_1"; Map<String, Map<String, String>> properties = new HashMap<String, Map<String, String>>(); Map<String, String> stormSiteProperties = new HashMap<String, String>(); Map<String, String> kafkaBrokerProperties = new HashMap<String, String>(); properties.put("storm-site", stormSiteProperties); properties.put("kafka-broker", kafkaBrokerProperties); stormSiteProperties.put("worker.childopts", "localhost"); stormSiteProperties.put("supervisor.childopts", "localhost"); stormSiteProperties.put("nimbus.childopts", "localhost"); kafkaBrokerProperties.put("kafka.ganglia.metrics.host", "localhost"); Configuration clusterConfig = new Configuration(properties, Collections.<String, Map<String, Map<String, String>>>emptyMap()); Collection<String> hgComponents = new HashSet<String>(); hgComponents.add("GANGLIA_SERVER"); TestHostGroup group1 = new TestHostGroup(expectedHostGroupName, hgComponents, Collections.singleton(expectedHostName)); Collection<TestHostGroup> hostGroups = new HashSet<TestHostGroup>(); hostGroups.add(group1); ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); BlueprintConfigurationProcessor updater = new BlueprintConfigurationProcessor(topology); updater.doUpdateForClusterCreate(); assertEquals("worker startup settings not properly handled by cluster create", expectedHostName, stormSiteProperties.get("worker.childopts")); assertEquals("supervisor startup settings not properly handled by cluster create", expectedHostName, stormSiteProperties.get("supervisor.childopts")); assertEquals("nimbus startup settings not properly handled by cluster create", expectedHostName, stormSiteProperties.get("nimbus.childopts")); assertEquals("Kafka ganglia host property not properly handled by cluster create", expectedHostName, kafkaBrokerProperties.get("kafka.ganglia.metrics.host")); }
@Test public void testDoUpdateForClusterWithNameNodeHAEnabled() throws Exception { final String expectedNameService = "mynameservice"; final String expectedHostName = "c6401.apache.ambari.org"; final String expectedHostNameTwo = "server-two"; final String expectedPortNum = "808080"; final String expectedNodeOne = "nn1"; final String expectedNodeTwo = "nn2"; final String expectedHostGroupName = "host_group_1"; Map<String, Map<String, String>> properties = new HashMap<String, Map<String, String>>(); Map<String, String> hdfsSiteProperties = new HashMap<String, String>(); Map<String, String> hbaseSiteProperties = new HashMap<String, String>(); Map<String, String> hadoopEnvProperties = new HashMap<String, String>(); Map<String, String> coreSiteProperties = new HashMap<String, String>(); Map<String, String> accumuloSiteProperties =new HashMap<String, String>(); properties.put("hdfs-site", hdfsSiteProperties); properties.put("hadoop-env", hadoopEnvProperties); properties.put("core-site", coreSiteProperties); properties.put("hbase-site", hbaseSiteProperties); properties.put("accumulo-site", accumuloSiteProperties); hdfsSiteProperties.put("dfs.nameservices", expectedNameService); hdfsSiteProperties.put("dfs.ha.namenodes.mynameservice", expectedNodeOne + ", " + expectedNodeTwo); hdfsSiteProperties.put("dfs.namenode.https-address." + expectedNameService + "." + expectedNodeOne, createExportedAddress(expectedPortNum, expectedHostGroupName)); hdfsSiteProperties.put("dfs.namenode.https-address." + expectedNameService + "." + expectedNodeTwo, createExportedAddress(expectedPortNum, expectedHostGroupName)); hdfsSiteProperties.put("dfs.namenode.http-address." + expectedNameService + "." + expectedNodeOne, createExportedAddress(expectedPortNum, expectedHostGroupName)); hdfsSiteProperties.put("dfs.namenode.http-address." + expectedNameService + "." + expectedNodeTwo, createExportedAddress(expectedPortNum, expectedHostGroupName)); hdfsSiteProperties.put("dfs.namenode.rpc-address." + expectedNameService + "." + expectedNodeOne, createExportedAddress(expectedPortNum, expectedHostGroupName)); hdfsSiteProperties.put("dfs.namenode.rpc-address." + expectedNameService + "." + expectedNodeTwo, createExportedAddress(expectedPortNum, expectedHostGroupName)); hdfsSiteProperties.put("dfs.secondary.http.address", "localhost:8080"); hdfsSiteProperties.put("dfs.namenode.secondary.http-address", "localhost:8080"); hdfsSiteProperties.put("dfs.namenode.http-address", "localhost:8080"); hdfsSiteProperties.put("dfs.namenode.https-address", "localhost:8081"); hdfsSiteProperties.put("dfs.namenode.rpc-address", "localhost:8082"); coreSiteProperties.put("fs.defaultFS", "hdfs: hbaseSiteProperties.put("hbase.rootdir", "hdfs: accumuloSiteProperties.put("instance.volumes", "hdfs: Configuration clusterConfig = new Configuration(properties, Collections.<String, Map<String, Map<String, String>>>emptyMap()); Collection<String> hgComponents = new HashSet<String>(); hgComponents.add("NAMENODE"); TestHostGroup group1 = new TestHostGroup(expectedHostGroupName, hgComponents, Collections.singleton(expectedHostName)); Collection<String> hgComponents2 = new HashSet<String>(); hgComponents2.add("NAMENODE"); TestHostGroup group2 = new TestHostGroup("host-group-2", hgComponents2, Collections.singleton(expectedHostNameTwo)); Collection<TestHostGroup> hostGroups = new ArrayList<TestHostGroup>(); hostGroups.add(group1); hostGroups.add(group2); expect(stack.getCardinality("NAMENODE")).andReturn(new Cardinality("1-2")).anyTimes(); expect(stack.getCardinality("SECONDARY_NAMENODE")).andReturn(new Cardinality("1")).anyTimes(); ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); BlueprintConfigurationProcessor updater = new BlueprintConfigurationProcessor(topology); Set<String> updatedConfigTypes = updater.doUpdateForClusterCreate(); assertEquals("dfs.internal.nameservices wasn't added", expectedNameService, hdfsSiteProperties.get("dfs.internal.nameservices")); assertEquals("HTTPS address HA property not properly exported", expectedHostName + ":" + expectedPortNum, hdfsSiteProperties.get("dfs.namenode.https-address." + expectedNameService + "." + expectedNodeOne)); assertEquals("HTTPS address HA property not properly exported", expectedHostName + ":" + expectedPortNum, hdfsSiteProperties.get("dfs.namenode.https-address." + expectedNameService + "." + expectedNodeTwo)); assertEquals("HTTPS address HA property not properly exported", expectedHostName + ":" + expectedPortNum, hdfsSiteProperties.get("dfs.namenode.http-address." + expectedNameService + "." + expectedNodeOne)); assertEquals("HTTPS address HA property not properly exported", expectedHostName + ":" + expectedPortNum, hdfsSiteProperties.get("dfs.namenode.http-address." + expectedNameService + "." + expectedNodeTwo)); assertEquals("HTTPS address HA property not properly exported", expectedHostName + ":" + expectedPortNum, hdfsSiteProperties.get("dfs.namenode.rpc-address." + expectedNameService + "." + expectedNodeOne)); assertEquals("HTTPS address HA property not properly exported", expectedHostName + ":" + expectedPortNum, hdfsSiteProperties.get("dfs.namenode.rpc-address." + expectedNameService + "." + expectedNodeTwo)); String initialActiveHost = hadoopEnvProperties.get("dfs_ha_initial_namenode_active"); String expectedStandbyHost = null; if (initialActiveHost.equals(expectedHostName)) { expectedStandbyHost = expectedHostNameTwo; } else if (initialActiveHost.equals(expectedHostNameTwo)) { expectedStandbyHost = expectedHostName; } else { fail("Active Namenode hostname was not set correctly"); } assertEquals("Standby Namenode hostname was not set correctly", expectedStandbyHost, hadoopEnvProperties.get("dfs_ha_initial_namenode_standby")); assertEquals("fs.defaultFS should not be modified by cluster update when NameNode HA is enabled.", "hdfs: assertEquals("hbase.rootdir should not be modified by cluster update when NameNode HA is enabled.", "hdfs: assertEquals("instance.volumes should not be modified by cluster update when NameNode HA is enabled.", "hdfs: assertFalse("dfs.namenode.http-address should have been filtered out of this HA configuration", hdfsSiteProperties.containsKey("dfs.namenode.http-address")); assertFalse("dfs.namenode.https-address should have been filtered out of this HA configuration", hdfsSiteProperties.containsKey("dfs.namenode.https-address")); assertFalse("dfs.namenode.rpc-address should have been filtered out of this HA configuration", hdfsSiteProperties.containsKey("dfs.namenode.rpc-address")); assertEquals("Incorrect number of updated config types returned, set = " + updatedConfigTypes, 3, updatedConfigTypes.size()); assertTrue("Expected config type not found in updated set", updatedConfigTypes.contains("cluster-env")); assertTrue("Expected config type not found in updated set", updatedConfigTypes.contains("hdfs-site")); assertTrue("Expected config type not found in updated set", updatedConfigTypes.contains("hadoop-env")); }
@Test public void testDoUpdateForClusterWithNameNodeHANotEnabled() throws Exception { final String expectedHostName = "c6401.apache.ambari.org"; final String expectedHostNameTwo = "serverTwo"; final String expectedHostGroupName = "host_group_1"; Map<String, Map<String, String>> properties = new HashMap<String, Map<String, String>>(); Map<String, String> hdfsSiteProperties = new HashMap<String, String>(); Map<String, String> hbaseSiteProperties = new HashMap<String, String>(); Map<String, String> hadoopEnvProperties = new HashMap<String, String>(); Map<String, String> coreSiteProperties = new HashMap<String, String>(); Map<String, String> accumuloSiteProperties =new HashMap<String, String>(); properties.put("hdfs-site", hdfsSiteProperties); properties.put("hadoop-env", hadoopEnvProperties); properties.put("core-site", coreSiteProperties); properties.put("hbase-site", hbaseSiteProperties); properties.put("accumulo-site", accumuloSiteProperties); hdfsSiteProperties.put("dfs.secondary.http.address", "localhost:8080"); hdfsSiteProperties.put("dfs.namenode.secondary.http-address", "localhost:8080"); hdfsSiteProperties.put("dfs.namenode.http-address", "localhost:8080"); hdfsSiteProperties.put("dfs.namenode.https-address", "localhost:8081"); hdfsSiteProperties.put("dfs.namenode.rpc-address", "localhost:8082"); Configuration clusterConfig = new Configuration(properties, Collections.<String, Map<String, Map<String, String>>>emptyMap()); Collection<String> hgComponents = new HashSet<String>(); hgComponents.add("NAMENODE"); hgComponents.add("SECONDARY_NAMENODE"); TestHostGroup group1 = new TestHostGroup(expectedHostGroupName, hgComponents, Collections.singleton(expectedHostName)); Collection<String> hgComponents2 = new HashSet<String>(); TestHostGroup group2 = new TestHostGroup("host-group-2", hgComponents2, Collections.singleton(expectedHostNameTwo)); Collection<TestHostGroup> hostGroups = new ArrayList<TestHostGroup>(); hostGroups.add(group1); hostGroups.add(group2); expect(stack.getCardinality("NAMENODE")).andReturn(new Cardinality("1-2")).anyTimes(); expect(stack.getCardinality("SECONDARY_NAMENODE")).andReturn(new Cardinality("1")).anyTimes(); ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); BlueprintConfigurationProcessor updater = new BlueprintConfigurationProcessor(topology); Set<String> updatedConfigTypes = updater.doUpdateForClusterCreate(); assertTrue("dfs.namenode.http-address should have been included in this HA configuration", hdfsSiteProperties.containsKey("dfs.namenode.http-address")); assertTrue("dfs.namenode.https-address should have been included in this HA configuration", hdfsSiteProperties.containsKey("dfs.namenode.https-address")); assertTrue("dfs.namenode.rpc-address should have been included in this HA configuration", hdfsSiteProperties.containsKey("dfs.namenode.rpc-address")); assertEquals("Incorrect number of updated config types returned, set = " + updatedConfigTypes, 2, updatedConfigTypes.size()); assertTrue("Expected config type 'cluster-env' not found in updated set", updatedConfigTypes.contains("cluster-env")); assertTrue("Expected config type 'hdfs-site' not found in updated set", updatedConfigTypes.contains("hdfs-site")); }
@Test public void testDoUpdateForClusterWithNameNodeHAEnabledAndActiveNodeSet() throws Exception { final String expectedNameService = "mynameservice"; final String expectedHostName = "server-three"; final String expectedHostNameTwo = "server-four"; final String expectedPortNum = "808080"; final String expectedNodeOne = "nn1"; final String expectedNodeTwo = "nn2"; final String expectedHostGroupName = "host_group_1"; Map<String, Map<String, String>> properties = new HashMap<String, Map<String, String>>(); Map<String, String> hdfsSiteProperties = new HashMap<String, String>(); Map<String, String> hadoopEnvProperties = new HashMap<String, String>(); properties.put("hdfs-site", hdfsSiteProperties); properties.put("hadoop-env", hadoopEnvProperties); hdfsSiteProperties.put("dfs.nameservices", expectedNameService); hdfsSiteProperties.put("dfs.ha.namenodes.mynameservice", expectedNodeOne + ", " + expectedNodeTwo); hdfsSiteProperties.put("dfs.namenode.https-address." + expectedNameService + "." + expectedNodeOne, createExportedAddress(expectedPortNum, expectedHostGroupName)); hdfsSiteProperties.put("dfs.namenode.https-address." + expectedNameService + "." + expectedNodeTwo, createExportedAddress(expectedPortNum, expectedHostGroupName)); hdfsSiteProperties.put("dfs.namenode.http-address." + expectedNameService + "." + expectedNodeOne, createExportedAddress(expectedPortNum, expectedHostGroupName)); hdfsSiteProperties.put("dfs.namenode.http-address." + expectedNameService + "." + expectedNodeTwo, createExportedAddress(expectedPortNum, expectedHostGroupName)); hdfsSiteProperties.put("dfs.namenode.rpc-address." + expectedNameService + "." + expectedNodeOne, createExportedAddress(expectedPortNum, expectedHostGroupName)); hdfsSiteProperties.put("dfs.namenode.rpc-address." + expectedNameService + "." + expectedNodeTwo, createExportedAddress(expectedPortNum, expectedHostGroupName)); hadoopEnvProperties.put("dfs_ha_initial_namenode_active", expectedHostName); hadoopEnvProperties.put("dfs_ha_initial_namenode_standby", expectedHostNameTwo); Configuration clusterConfig = new Configuration(properties, Collections.<String, Map<String, Map<String, String>>>emptyMap()); Collection<String> hgComponents = new HashSet<String>(); hgComponents.add("NAMENODE"); Collection<String> hosts = new ArrayList<String>(); hosts.add(expectedHostName); hosts.add(expectedHostNameTwo); TestHostGroup group = new TestHostGroup(expectedHostGroupName, hgComponents, hosts); Collection<TestHostGroup> hostGroups = new ArrayList<TestHostGroup>(); hostGroups.add(group); ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); BlueprintConfigurationProcessor updater = new BlueprintConfigurationProcessor(topology); updater.doUpdateForClusterCreate(); String expectedPropertyValue = hdfsSiteProperties.get("dfs.namenode.https-address." + expectedNameService + "." + expectedNodeOne); if (! expectedPropertyValue.equals(expectedHostName + ":" + expectedPortNum) && ! expectedPropertyValue.equals(expectedHostNameTwo + ":" + expectedPortNum)) { fail("HTTPS address HA property not properly exported"); } assertEquals("HTTPS address HA property not properly exported", expectedPropertyValue, hdfsSiteProperties.get("dfs.namenode.https-address." + expectedNameService + "." + expectedNodeTwo)); assertEquals("HTTPS address HA property not properly exported", expectedPropertyValue, hdfsSiteProperties.get("dfs.namenode.http-address." + expectedNameService + "." + expectedNodeOne)); assertEquals("HTTPS address HA property not properly exported", expectedPropertyValue, hdfsSiteProperties.get("dfs.namenode.http-address." + expectedNameService + "." + expectedNodeTwo)); assertEquals("HTTPS address HA property not properly exported", expectedPropertyValue, hdfsSiteProperties.get("dfs.namenode.rpc-address." + expectedNameService + "." + expectedNodeOne)); assertEquals("HTTPS address HA property not properly exported", expectedPropertyValue, hdfsSiteProperties.get("dfs.namenode.rpc-address." + expectedNameService + "." + expectedNodeTwo)); assertEquals("Active Namenode hostname was not set correctly", expectedHostName, hadoopEnvProperties.get("dfs_ha_initial_namenode_active")); assertEquals("Standby Namenode hostname was not set correctly", expectedHostNameTwo, hadoopEnvProperties.get("dfs_ha_initial_namenode_standby")); }
@Test public void testHDFSConfigClusterUpdateQuorumJournalURL() throws Exception { final String expectedHostNameOne = "c6401.apache.ambari.org"; final String expectedHostNameTwo = "c6402.apache.ambari.org"; final String expectedPortNum = "808080"; final String expectedHostGroupName = "host_group_1"; final String expectedHostGroupNameTwo = "host_group_2"; Map<String, Map<String, String>> properties = new HashMap<String, Map<String, String>>(); Map<String, String> hdfsSiteProperties = new HashMap<String, String>(); properties.put("hdfs-site", hdfsSiteProperties); hdfsSiteProperties.put("dfs.namenode.shared.edits.dir", "qjournal: Configuration clusterConfig = new Configuration(properties, Collections.<String, Map<String, Map<String, String>>>emptyMap()); Collection<String> hgComponents1 = new HashSet<String>(); hgComponents1.add("NAMENODE"); TestHostGroup group1 = new TestHostGroup(expectedHostGroupName, hgComponents1, Collections.singleton(expectedHostNameOne)); Collection<String> hgComponents2 = new HashSet<String>(); hgComponents2.add("NAMENODE"); TestHostGroup group2 = new TestHostGroup(expectedHostGroupNameTwo, hgComponents2, Collections.singleton(expectedHostNameTwo)); Collection<TestHostGroup> hostGroups = new ArrayList<TestHostGroup>(); hostGroups.add(group1); hostGroups.add(group2); ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); BlueprintConfigurationProcessor updater = new BlueprintConfigurationProcessor(topology); updater.doUpdateForClusterCreate(); assertEquals("HDFS HA shared edits directory property not properly updated for cluster create.", "qjournal: hdfsSiteProperties.get("dfs.namenode.shared.edits.dir")); }
@Test public void testHDFSConfigClusterUpdateQuorumJournalURL_UsingMinusSymbolInHostName() throws Exception { final String expectedHostNameOne = "c6401.apache.ambari.org"; final String expectedHostNameTwo = "c6402.apache.ambari.org"; final String expectedPortNum = "808080"; final String expectedHostGroupName = "host-group-1"; final String expectedHostGroupNameTwo = "host-group-2"; Map<String, Map<String, String>> properties = new HashMap<String, Map<String, String>>(); Map<String, String> hdfsSiteProperties = new HashMap<String, String>(); properties.put("hdfs-site", hdfsSiteProperties); hdfsSiteProperties.put("dfs.namenode.shared.edits.dir", "qjournal: Configuration clusterConfig = new Configuration(properties, Collections.<String, Map<String, Map<String, String>>>emptyMap()); Collection<String> hgComponents1 = new HashSet<String>(); hgComponents1.add("NAMENODE"); TestHostGroup group1 = new TestHostGroup(expectedHostGroupName, hgComponents1, Collections.singleton(expectedHostNameOne)); Collection<String> hgComponents2 = new HashSet<String>(); hgComponents2.add("NAMENODE"); TestHostGroup group2 = new TestHostGroup(expectedHostGroupNameTwo, hgComponents2, Collections.singleton(expectedHostNameTwo)); Collection<TestHostGroup> hostGroups = new ArrayList<TestHostGroup>(); hostGroups.add(group1); hostGroups.add(group2); ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); BlueprintConfigurationProcessor updater = new BlueprintConfigurationProcessor(topology); updater.doUpdateForClusterCreate(); assertEquals("HDFS HA shared edits directory property not properly updated for cluster create.", "qjournal: hdfsSiteProperties.get("dfs.namenode.shared.edits.dir")); }
@Test public void testHadoopHaNameNode() throws Exception { final String configType = "hadoop-env"; Map<String, Map<String, String>> properties = new HashMap<>(); Map<String, String> hdfsSite = new HashMap<>(); hdfsSite.put("dfs.nameservices", "mycluster"); hdfsSite.put("dfs.ha.namenodes.mycluster", "nn1,nn2"); hdfsSite.put("dfs.namenode.http-address", "%HOSTGROUP::master_1%:50070"); hdfsSite.put("dfs.namenode.http-address.mycluster.nn1", "%HOSTGROUP::master_1%:50070"); hdfsSite.put("dfs.namenode.http-address.mycluster.nn2", "%HOSTGROUP::master_2%:50070"); hdfsSite.put("dfs.namenode.https-address", "%HOSTGROUP::master_1%:50470"); hdfsSite.put("dfs.namenode.https-address.mycluster.nn1", "%HOSTGROUP::master_1%:50470"); hdfsSite.put("dfs.namenode.https-address.mycluster.nn2", "%HOSTGROUP::master_2%:50470"); hdfsSite.put("dfs.namenode.rpc-address.mycluster.nn1", "%HOSTGROUP::master_1%:8020"); hdfsSite.put("dfs.namenode.rpc-address.mycluster.nn2", "%HOSTGROUP::master_2%:8020"); hdfsSite.put("dfs.namenode.shared.edits.dir", "qjournal: hdfsSite.put("dfs.ha.automatic-failover.enabled", "true"); hdfsSite.put("dfs.ha.fencing.methods", "shell(/bin/true)"); properties.put("hdfs-site", hdfsSite); Map<String, String> hadoopEnv = new HashMap<>(); hadoopEnv.put("dfs_ha_initial_namenode_active", "%HOSTGROUP::master_1%"); hadoopEnv.put("dfs_ha_initial_namenode_standby", "%HOSTGROUP::master_2%"); properties.put("hadoop-env", hadoopEnv); Map<String, Map<String, String>> parentProperties = new HashMap<>(); Configuration parentClusterConfig = new Configuration(parentProperties, Collections.<String, Map<String, Map<String, String>>>emptyMap()); Configuration clusterConfig = new Configuration(properties, Collections.<String, Map<String, Map<String, String>>>emptyMap(), parentClusterConfig); TestHostGroup group1 = new TestHostGroup("master_1", ImmutableSet.of("DATANODE", "NAMENODE"), Collections.singleton("node_1")); TestHostGroup group2 = new TestHostGroup("master_2", ImmutableSet.of("DATANODE", "NAMENODE"), Collections.singleton("node_2")); Collection<TestHostGroup> hostGroups = Lists.newArrayList(group1, group2); ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); BlueprintConfigurationProcessor configProcessor = new BlueprintConfigurationProcessor(topology); configProcessor.doUpdateForClusterCreate(); assertEquals("node_1", clusterConfig.getPropertyValue(configType, "dfs_ha_initial_namenode_active")); assertEquals("node_2", clusterConfig.getPropertyValue(configType, "dfs_ha_initial_namenode_standby")); }
@Test public void testAllDefaultUserAndGroupProxyPropertiesSet() throws Exception { Map<String, Map<String, String>> properties = new HashMap<String, Map<String, String>>(); Map<String, String> oozieEnvProperties = new HashMap<String, String>(); Map<String, String> hiveEnvProperties = new HashMap<String, String>(); Map<String, String> hbaseEnvProperties = new HashMap<String, String>(); Map<String, String> falconEnvProperties = new HashMap<String, String>(); properties.put("oozie-env", oozieEnvProperties); properties.put("hive-env", hiveEnvProperties); properties.put("hbase-env", hbaseEnvProperties); properties.put("falcon-env", falconEnvProperties); oozieEnvProperties.put("oozie_user", "test-oozie-user"); hiveEnvProperties.put("hive_user", "test-hive-user"); hiveEnvProperties.put("webhcat_user", "test-hcat-user"); hbaseEnvProperties.put("hbase_user", "test-hbase-user"); falconEnvProperties.put("falcon_user", "test-falcon-user"); Configuration clusterConfig = new Configuration(properties, Collections.<String, Map<String, Map<String, String>>>emptyMap()); Collection<String> hgComponents1 = new HashSet<String>(); hgComponents1.add("OOZIE_SERVER"); hgComponents1.add("HIVE_SERVER"); hgComponents1.add("HBASE_MASTER"); hgComponents1.add("FALCON_SERVER"); TestHostGroup group1 = new TestHostGroup("group1", hgComponents1, Collections.singleton("host1")); Collection<TestHostGroup> hostGroups = Collections.singletonList(group1); ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); BlueprintConfigurationProcessor configProcessor = new BlueprintConfigurationProcessor(topology); configProcessor.doUpdateForClusterCreate(); assertEquals("*", properties.get("core-site").get("hadoop.proxyuser.test-oozie-user.hosts")); assertEquals("*", properties.get("core-site").get("hadoop.proxyuser.test-oozie-user.groups")); assertEquals("*", properties.get("core-site").get("hadoop.proxyuser.test-hive-user.hosts")); assertEquals("*", properties.get("core-site").get("hadoop.proxyuser.test-hive-user.groups")); assertEquals("*", properties.get("core-site").get("hadoop.proxyuser.test-hcat-user.hosts")); assertEquals("*", properties.get("core-site").get("hadoop.proxyuser.test-hcat-user.groups")); assertEquals("*", properties.get("core-site").get("hadoop.proxyuser.test-hbase-user.hosts")); assertEquals("*", properties.get("core-site").get("hadoop.proxyuser.test-hbase-user.groups")); assertEquals("*", properties.get("core-site").get("hadoop.proxyuser.test-falcon-user.hosts")); assertEquals("*", properties.get("core-site").get("hadoop.proxyuser.test-falcon-user.groups")); }
@Test public void testRelevantDefaultUserAndGroupProxyPropertiesSet() throws Exception { Map<String, Map<String, String>> properties = new HashMap<String, Map<String, String>>(); Map<String, String> oozieEnvProperties = new HashMap<String, String>(); Map<String, String> falconEnvProperties = new HashMap<String, String>(); properties.put("oozie-env", oozieEnvProperties); properties.put("falcon-env", falconEnvProperties); oozieEnvProperties.put("oozie_user", "test-oozie-user"); falconEnvProperties.put("falcon_user", "test-falcon-user"); Configuration clusterConfig = new Configuration(properties, Collections.<String, Map<String, Map<String, String>>>emptyMap()); Collection<String> hgComponents1 = new HashSet<String>(); hgComponents1.add("OOZIE_SERVER"); hgComponents1.add("FALCON_SERVER"); TestHostGroup group1 = new TestHostGroup("group1", hgComponents1, Collections.singleton("host1")); Collection<TestHostGroup> hostGroups = Collections.singletonList(group1); ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); BlueprintConfigurationProcessor configProcessor = new BlueprintConfigurationProcessor(topology); configProcessor.doUpdateForClusterCreate(); Map<String, String> coreSiteProperties = properties.get("core-site"); assertEquals(4, coreSiteProperties.size()); assertEquals("*", coreSiteProperties.get("hadoop.proxyuser.test-oozie-user.hosts")); assertEquals("*", coreSiteProperties.get("hadoop.proxyuser.test-oozie-user.groups")); assertEquals("*", coreSiteProperties.get("hadoop.proxyuser.test-falcon-user.hosts")); assertEquals("*", coreSiteProperties.get("hadoop.proxyuser.test-falcon-user.groups")); }
@Test public void testDefaultUserAndGroupProxyPropertiesSetWhenNotProvided() throws Exception { Map<String, Map<String, String>> properties = new HashMap<String, Map<String, String>>(); Map<String, String> coreSiteProperties = new HashMap<String, String>(); Map<String, String> oozieEnvProperties = new HashMap<String, String>(); Map<String, String> falconEnvProperties = new HashMap<String, String>(); properties.put("core-site", coreSiteProperties); properties.put("oozie-env", oozieEnvProperties); properties.put("falcon-env", falconEnvProperties); coreSiteProperties.put("hadoop.proxyuser.test-oozie-user.hosts", "testOozieHostsVal"); coreSiteProperties.put("hadoop.proxyuser.test-oozie-user.groups", "testOozieGroupsVal"); oozieEnvProperties.put("oozie_user", "test-oozie-user"); falconEnvProperties.put("falcon_user", "test-falcon-user"); Configuration clusterConfig = new Configuration(properties, Collections.<String, Map<String, Map<String, String>>>emptyMap()); Collection<String> hgComponents1 = new HashSet<String>(); hgComponents1.add("OOZIE_SERVER"); hgComponents1.add("FALCON_SERVER"); TestHostGroup group1 = new TestHostGroup("group1", hgComponents1, Collections.singleton("host1")); Collection<TestHostGroup> hostGroups = Collections.singletonList(group1); ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); BlueprintConfigurationProcessor configProcessor = new BlueprintConfigurationProcessor(topology); configProcessor.doUpdateForClusterCreate(); assertEquals(4, coreSiteProperties.size()); assertEquals("testOozieHostsVal", coreSiteProperties.get("hadoop.proxyuser.test-oozie-user.hosts")); assertEquals("testOozieGroupsVal", coreSiteProperties.get("hadoop.proxyuser.test-oozie-user.groups")); assertEquals("*", coreSiteProperties.get("hadoop.proxyuser.test-falcon-user.hosts")); assertEquals("*", coreSiteProperties.get("hadoop.proxyuser.test-falcon-user.groups")); }
@Test public void testDefaultUserAndGroupProxyPropertiesSetWhenNotProvided2() throws Exception { Map<String, Map<String, String>> properties = new HashMap<String, Map<String, String>>(); Map<String, String> falconEnvProperties = new HashMap<String, String>(); properties.put("falcon-env", falconEnvProperties); falconEnvProperties.put("falcon_user", "test-falcon-user"); Map<String, Map<String, String>> parentProperties = new HashMap<String, Map<String, String>>(); Map<String, String> oozieEnvProperties = new HashMap<String, String>(); parentProperties.put("oozie-env", oozieEnvProperties); oozieEnvProperties.put("oozie_user", "test-oozie-user"); Map<String, String> coreSiteProperties = new HashMap<String, String>(); parentProperties.put("core-site", coreSiteProperties); coreSiteProperties.put("hadoop.proxyuser.test-oozie-user.hosts", "testOozieHostsVal"); Configuration parentClusterConfig = new Configuration(parentProperties, Collections.<String, Map<String, Map<String, String>>>emptyMap()); Configuration clusterConfig = new Configuration(properties, Collections.<String, Map<String, Map<String, String>>>emptyMap(), parentClusterConfig); Collection<String> hgComponents1 = new HashSet<String>(); hgComponents1.add("OOZIE_SERVER"); hgComponents1.add("FALCON_SERVER"); TestHostGroup group1 = new TestHostGroup("group1", hgComponents1, Collections.singleton("host1")); Collection<TestHostGroup> hostGroups = Collections.singletonList(group1); ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); BlueprintConfigurationProcessor configProcessor = new BlueprintConfigurationProcessor(topology); configProcessor.doUpdateForClusterCreate(); Map<String, String> leafConfigCoreSiteProps = properties.get("core-site"); assertEquals(3, leafConfigCoreSiteProps.size()); assertEquals("testOozieHostsVal", clusterConfig.getPropertyValue("core-site", "hadoop.proxyuser.test-oozie-user.hosts")); assertEquals("*", leafConfigCoreSiteProps.get("hadoop.proxyuser.test-oozie-user.groups")); assertEquals("*", leafConfigCoreSiteProps.get("hadoop.proxyuser.test-falcon-user.hosts")); assertEquals("*", leafConfigCoreSiteProps.get("hadoop.proxyuser.test-falcon-user.groups")); }
@Test public void testHiveWithoutAtlas() throws Exception { Map<String, Map<String, String>> properties = new HashMap<String, Map<String, String>>(); Map<String, String> hiveProperties = new HashMap<String, String>(); hiveProperties.put("hive.exec.post.hooks", ""); properties.put("hive-site", hiveProperties); Map<String, String> hiveEnv = new HashMap<String, String>(); hiveEnv.put("hive.atlas.hook", "false"); properties.put("hive-env", hiveEnv); Map<String, Map<String, String>> parentProperties = new HashMap<String, Map<String, String>>(); Configuration parentClusterConfig = new Configuration(parentProperties, Collections.<String, Map<String, Map<String, String>>>emptyMap()); Configuration clusterConfig = new Configuration(properties, Collections.<String, Map<String, Map<String, String>>>emptyMap(), parentClusterConfig); Collection<String> hgComponents1 = new HashSet<String>(); hgComponents1.add("HIVE_SERVER"); TestHostGroup group1 = new TestHostGroup("group1", hgComponents1, Collections.singleton("host1")); Collection<TestHostGroup> hostGroups = Collections.singletonList(group1); ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); BlueprintConfigurationProcessor configProcessor = new BlueprintConfigurationProcessor(topology); configProcessor.doUpdateForClusterCreate(); assertEquals(null, clusterConfig.getPropertyValue("hive-site", "atlas.cluster.name")); assertEquals(null, clusterConfig.getPropertyValue("hive-site", "atlas.rest.address")); }
@Test public void testAtlasHivePropertiesWithHTTPS() throws Exception { Map<String, Map<String, String>> properties = new HashMap<String, Map<String, String>>(); Map<String, String> atlasProperties = new HashMap<String, String>(); properties.put("application-properties", atlasProperties); atlasProperties.put("atlas.enableTLS", "true"); atlasProperties.put("atlas.server.bind.address", "localhost"); atlasProperties.put("atlas.server.https.port", "99999"); Map<String, String> atlasEnv = new HashMap<String, String>(); properties.put("atlas-env", atlasEnv); Map<String, String> hiveProperties = new HashMap<String, String>(); hiveProperties.put("hive.exec.post.hooks", "foo"); properties.put("hive-site", hiveProperties); Map<String, String> hiveEnv = new HashMap<String, String>(); hiveEnv.put("hive.atlas.hook", "false"); properties.put("hive-env", hiveEnv); Map<String, Map<String, String>> parentProperties = new HashMap<String, Map<String, String>>(); Configuration parentClusterConfig = new Configuration(parentProperties, Collections.<String, Map<String, Map<String, String>>>emptyMap()); Configuration clusterConfig = new Configuration(properties, Collections.<String, Map<String, Map<String, String>>>emptyMap(), parentClusterConfig); Collection<String> hgComponents1 = new HashSet<String>(); hgComponents1.add("ATLAS_SERVER"); hgComponents1.add("HIVE_SERVER"); TestHostGroup group1 = new TestHostGroup("group1", hgComponents1, Collections.singleton("host1")); Collection<TestHostGroup> hostGroups = Collections.singletonList(group1); ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); BlueprintConfigurationProcessor configProcessor = new BlueprintConfigurationProcessor(topology); configProcessor.doUpdateForClusterCreate(); assertEquals("foo,org.apache.atlas.hive.hook.HiveHook", clusterConfig.getPropertyValue("hive-site", "hive.exec.post.hooks")); assertEquals(null, clusterConfig.getPropertyValue("hive-site", "atlas.cluster.name")); assertEquals(null, clusterConfig.getPropertyValue("hive-site", "atlas.rest.address")); }
@Test public void testStormAmsPropertiesDefault() throws Exception { Map<String, Map<String, String>> properties = new HashMap<String, Map<String, String>>(); Map<String, String> stormSite = new HashMap<String, String>(); stormSite.put("metrics.reporter.register", ""); properties.put("storm-site", stormSite); Map<String, Map<String, String>> parentProperties = new HashMap<String, Map<String, String>>(); Configuration parentClusterConfig = new Configuration(parentProperties, Collections.<String, Map<String, Map<String, String>>>emptyMap()); Configuration clusterConfig = new Configuration(properties, Collections.<String, Map<String, Map<String, String>>>emptyMap(), parentClusterConfig); Collection<String> hgComponents1 = new HashSet<String>(); hgComponents1.add("METRICS_COLLECTOR"); hgComponents1.add("NIMBUS"); TestHostGroup group1 = new TestHostGroup("group1", hgComponents1, Collections.singleton("host1")); Collection<TestHostGroup> hostGroups = Collections.singletonList(group1); ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); BlueprintConfigurationProcessor configProcessor = new BlueprintConfigurationProcessor(topology); configProcessor.doUpdateForClusterCreate(); assertEquals("org.apache.hadoop.metrics2.sink.storm.StormTimelineMetricsReporter", clusterConfig.getPropertyValue("storm-site", "metrics.reporter.register")); }
@Test public void testStormAmsPropertiesUserDefinedReporter() throws Exception { Map<String, Map<String, String>> properties = new HashMap<String, Map<String, String>>(); Map<String, String> stormSite = new HashMap<String, String>(); stormSite.put("metrics.reporter.register", "user.Reporter"); properties.put("storm-site", stormSite); Map<String, Map<String, String>> parentProperties = new HashMap<String, Map<String, String>>(); Configuration parentClusterConfig = new Configuration(parentProperties, Collections.<String, Map<String, Map<String, String>>>emptyMap()); Configuration clusterConfig = new Configuration(properties, Collections.<String, Map<String, Map<String, String>>>emptyMap(), parentClusterConfig); Collection<String> hgComponents1 = new HashSet<String>(); hgComponents1.add("METRICS_COLLECTOR"); hgComponents1.add("NIMBUS"); TestHostGroup group1 = new TestHostGroup("group1", hgComponents1, Collections.singleton("host1")); Collection<TestHostGroup> hostGroups = Collections.singletonList(group1); ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); BlueprintConfigurationProcessor configProcessor = new BlueprintConfigurationProcessor(topology); configProcessor.doUpdateForClusterCreate(); assertEquals("user.Reporter", clusterConfig.getPropertyValue("storm-site", "metrics.reporter.register")); }
@Test public void testKafkaAmsProperties() throws Exception { Map<String, Map<String, String>> properties = new HashMap<String, Map<String, String>>(); Map<String, String> stormSite = new HashMap<String, String>(); stormSite.put("kafka.metrics.reporters", ""); properties.put("kafka-broker", stormSite); Map<String, Map<String, String>> parentProperties = new HashMap<String, Map<String, String>>(); Configuration parentClusterConfig = new Configuration(parentProperties, Collections.<String, Map<String, Map<String, String>>>emptyMap()); Configuration clusterConfig = new Configuration(properties, Collections.<String, Map<String, Map<String, String>>>emptyMap(), parentClusterConfig); Collection<String> hgComponents1 = new HashSet<String>(); hgComponents1.add("METRICS_COLLECTOR"); hgComponents1.add("KAFKA_BROKER"); TestHostGroup group1 = new TestHostGroup("group1", hgComponents1, Collections.singleton("host1")); Collection<TestHostGroup> hostGroups = Collections.singletonList(group1); ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); BlueprintConfigurationProcessor configProcessor = new BlueprintConfigurationProcessor(topology); configProcessor.doUpdateForClusterCreate(); assertEquals("org.apache.hadoop.metrics2.sink.kafka.KafkaTimelineMetricsReporter", clusterConfig.getPropertyValue("kafka-broker", "kafka.metrics.reporters")); }
@Test public void testKafkaAmsPropertiesMultipleReporters() throws Exception { Map<String, Map<String, String>> properties = new HashMap<String, Map<String, String>>(); Map<String, String> stormSite = new HashMap<String, String>(); stormSite.put("kafka.metrics.reporters", "user.Reporter"); properties.put("kafka-broker", stormSite); Map<String, Map<String, String>> parentProperties = new HashMap<String, Map<String, String>>(); Configuration parentClusterConfig = new Configuration(parentProperties, Collections.<String, Map<String, Map<String, String>>>emptyMap()); Configuration clusterConfig = new Configuration(properties, Collections.<String, Map<String, Map<String, String>>>emptyMap(), parentClusterConfig); Collection<String> hgComponents1 = new HashSet<String>(); hgComponents1.add("METRICS_COLLECTOR"); hgComponents1.add("KAFKA_BROKER"); TestHostGroup group1 = new TestHostGroup("group1", hgComponents1, Collections.singleton("host1")); Collection<TestHostGroup> hostGroups = Collections.singletonList(group1); ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); BlueprintConfigurationProcessor configProcessor = new BlueprintConfigurationProcessor(topology); configProcessor.doUpdateForClusterCreate(); assertEquals("user.Reporter,org.apache.hadoop.metrics2.sink.kafka.KafkaTimelineMetricsReporter", clusterConfig.getPropertyValue("kafka-broker", "kafka.metrics.reporters")); }
@Test public void testRecommendConfiguration_applyStackDefaultsOnly() throws Exception { final String expectedHostName = "c6401.apache.ambari.org"; final String expectedPortNum = "808080"; final String expectedHostGroupName = "host_group_1"; Map<String, Map<String, String>> properties = new HashMap<String, Map<String, String>>(); Map<String, String> coreSiteMap = new HashMap<String, String>(); properties.put("core-site", coreSiteMap); coreSiteMap.put("fs.default.name", expectedHostName + ":" + expectedPortNum); coreSiteMap.put("fs.defaultFS", "hdfs: coreSiteMap.put("fs.stackDefault.key2", "dummyValue"); Map<String, String> dummySiteMap = new HashMap<String, String>(); properties.put("dummy-site", dummySiteMap); dummySiteMap.put("dummy.prop", "dummyValue2"); Map<String, Map<String, String>> parentProperties = new HashMap<String, Map<String, String>>(); Collection<String> hgComponents = new HashSet<String>(); hgComponents.add("NAMENODE"); hgComponents.add("SECONDARY_NAMENODE"); hgComponents.add("RESOURCEMANAGER"); TestHostGroup group1 = new TestHostGroup("group1", hgComponents, Collections.singleton("testhost")); Collection<String> hgComponents2 = new HashSet<String>(); hgComponents2.add("DATANODE"); hgComponents2.add("HDFS_CLIENT"); TestHostGroup group2 = new TestHostGroup("group2", hgComponents2, Collections.singleton(expectedHostGroupName)); Collection<TestHostGroup> hostGroups = new HashSet<TestHostGroup>(); hostGroups.add(group1); hostGroups.add(group2); Configuration parentConfig = new Configuration(parentProperties, Collections.<String, Map<String, Map<String, String>>>emptyMap(), createStackDefaults()); Configuration clusterConfig = new Configuration(properties, Collections.<String, Map<String, Map<String, String>>>emptyMap(), parentConfig); ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); topology.getAdvisedConfigurations().putAll(createAdvisedConfigMap()); topology.setConfigRecommendationStrategy(ConfigRecommendationStrategy.ONLY_STACK_DEFAULTS_APPLY); BlueprintConfigurationProcessor configProcessor = new BlueprintConfigurationProcessor(topology); reset(stack); expect(stack.getConfiguration(bp.getServices())).andReturn(createStackDefaults()).anyTimes(); Set<String> emptySet = Collections.emptySet(); expect(stack.getExcludedConfigurationTypes(anyObject(String.class))).andReturn(emptySet).anyTimes(); replay(stack); Set<String> configTypeUpdated = configProcessor.doUpdateForClusterCreate(); assertEquals(expectedHostName + ":" + expectedPortNum, clusterConfig.getPropertyValue("core-site", "fs.default.name")); assertEquals("stackDefaultUpgraded", clusterConfig.getPropertyValue("core-site", "fs.stackDefault.key1")); assertNull(clusterConfig.getPropertyValue("core-site", "fs.stackDefault.key2")); assertNull(clusterConfig.getPropertyValue("core-site", "fs.notStackDefault")); assertTrue(configTypeUpdated.contains("dummy-site")); }
@Test public void testRecommendConfiguration_EmptyConfiguration_applyStackDefaultsOnly() throws Exception { final String expectedHostGroupName = "host_group_1"; Map<String, Map<String, String>> properties = new HashMap<String, Map<String, String>>(); Collection<String> hgComponents = new HashSet<String>(); hgComponents.add("NAMENODE"); hgComponents.add("SECONDARY_NAMENODE"); hgComponents.add("RESOURCEMANAGER"); TestHostGroup group1 = new TestHostGroup("group1", hgComponents, Collections.singleton("testhost")); Collection<String> hgComponents2 = new HashSet<String>(); hgComponents2.add("DATANODE"); hgComponents2.add("HDFS_CLIENT"); TestHostGroup group2 = new TestHostGroup("group2", hgComponents2, Collections.singleton(expectedHostGroupName)); Collection<TestHostGroup> hostGroups = new HashSet<TestHostGroup>(); hostGroups.add(group1); hostGroups.add(group2); Map<String, Map<String, String>> parentProperties = new HashMap<String, Map<String, String>>(); Configuration parentConfig = new Configuration(parentProperties, Collections.<String, Map<String, Map<String, String>>>emptyMap(), createStackDefaults()); Configuration clusterConfig = new Configuration(properties, Collections.<String, Map<String, Map<String, String>>>emptyMap(), parentConfig); ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); topology.getAdvisedConfigurations().putAll(createAdvisedConfigMap()); topology.setConfigRecommendationStrategy(ConfigRecommendationStrategy.ONLY_STACK_DEFAULTS_APPLY); BlueprintConfigurationProcessor configProcessor = new BlueprintConfigurationProcessor(topology); reset(stack); expect(stack.getConfiguration(bp.getServices())).andReturn(createStackDefaults()).anyTimes(); Set<String> emptySet = Collections.emptySet(); expect(stack.getExcludedConfigurationTypes(anyObject(String.class))).andReturn(emptySet).anyTimes(); replay(stack); configProcessor.doUpdateForClusterCreate(); assertEquals("stackDefaultUpgraded", clusterConfig.getPropertyValue("core-site", "fs.stackDefault.key1")); assertNull(clusterConfig.getPropertyValue("core-site", "fs.stackDefault.key2")); assertNull(clusterConfig.getPropertyValue("core-site", "fs.notStackDefault")); }
@Test public void testRecommendConfiguration_applyAlways() throws Exception { final String expectedHostName = "c6401.apache.ambari.org"; final String expectedPortNum = "808080"; final String expectedHostGroupName = "host_group_1"; Map<String, Map<String, String>> properties = new HashMap<String, Map<String, String>>(); Map<String, String> coreSiteMap = new HashMap<String, String>(); properties.put("core-site", coreSiteMap); coreSiteMap.put("fs.default.name", expectedHostName + ":" + expectedPortNum); coreSiteMap.put("fs.defaultFS", "hdfs: coreSiteMap.put("fs.stackDefault.key2", "dummyValue"); Map<String, String> dummySiteMap = new HashMap<String, String>(); properties.put("dummy-site", dummySiteMap); dummySiteMap.put("dummy.prop", "dummyValue"); Map<String, String> dummy2SiteMap = new HashMap<String, String>(); properties.put("dummy2-site", dummy2SiteMap); Map<String, Map<String, String>> parentProperties = new HashMap<String, Map<String, String>>(); Collection<String> hgComponents = new HashSet<String>(); hgComponents.add("NAMENODE"); hgComponents.add("SECONDARY_NAMENODE"); hgComponents.add("RESOURCEMANAGER"); TestHostGroup group1 = new TestHostGroup("group1", hgComponents, Collections.singleton("testhost")); Collection<String> hgComponents2 = new HashSet<String>(); hgComponents2.add("DATANODE"); hgComponents2.add("HDFS_CLIENT"); TestHostGroup group2 = new TestHostGroup("group2", hgComponents2, Collections.singleton(expectedHostGroupName)); Collection<TestHostGroup> hostGroups = new HashSet<TestHostGroup>(); hostGroups.add(group1); hostGroups.add(group2); Configuration parentClusterConfig = new Configuration(parentProperties, Collections.<String, Map<String, Map<String, String>>>emptyMap(), createStackDefaults()); Configuration clusterConfig = new Configuration(properties, Collections.<String, Map<String, Map<String, String>>>emptyMap(), parentClusterConfig); ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); topology.getAdvisedConfigurations().putAll(createAdvisedConfigMap()); topology.setConfigRecommendationStrategy(ConfigRecommendationStrategy.ALWAYS_APPLY); BlueprintConfigurationProcessor configProcessor = new BlueprintConfigurationProcessor(topology); Set<String> configTypes = configProcessor.doUpdateForClusterCreate(); assertEquals(expectedHostName + ":" + expectedPortNum, clusterConfig.getPropertyValue("core-site","fs.default.name")); assertEquals("stackDefaultUpgraded", clusterConfig.getPropertyValue("core-site", "fs.stackDefault.key1")); assertNull(clusterConfig.getPropertyValue("core-site", "fs.stackDefault.key2")); assertNotNull(clusterConfig.getPropertyValue("core-site", "fs.notStackDefault")); assertEquals(3, topology.getAdvisedConfigurations().size()); assertFalse(configTypes.contains("dummy-site")); assertFalse(configTypes.contains("dummy2-site")); }
@Test public void testRecommendConfiguration_neverApply() throws Exception { final String expectedHostName = "c6401.apache.ambari.org"; final String expectedPortNum = "808080"; final String expectedHostGroupName = "host_group_1"; Map<String, Map<String, String>> properties = new HashMap<String, Map<String, String>>(); Map<String, String> coreSiteMap = new HashMap<String, String>(); properties.put("core-site", coreSiteMap); coreSiteMap.put("fs.default.name", expectedHostName + ":" + expectedPortNum); coreSiteMap.put("fs.defaultFS", "hdfs: coreSiteMap.put("fs.stackDefault.key2", "dummyValue"); Map<String, Map<String, String>> parentProperties = new HashMap<String, Map<String, String>>(); Collection<String> hgComponents = new HashSet<String>(); hgComponents.add("NAMENODE"); hgComponents.add("SECONDARY_NAMENODE"); hgComponents.add("RESOURCEMANAGER"); TestHostGroup group1 = new TestHostGroup("group1", hgComponents, Collections.singleton("testhost")); Collection<String> hgComponents2 = new HashSet<String>(); hgComponents2.add("DATANODE"); hgComponents2.add("HDFS_CLIENT"); TestHostGroup group2 = new TestHostGroup("group2", hgComponents2, Collections.singleton(expectedHostGroupName)); Collection<TestHostGroup> hostGroups = new HashSet<TestHostGroup>(); hostGroups.add(group1); hostGroups.add(group2); Configuration parentClusterConfig = new Configuration(parentProperties, Collections.<String, Map<String, Map<String, String>>>emptyMap(), createStackDefaults()); Configuration clusterConfig = new Configuration(properties, Collections.<String, Map<String, Map<String, String>>>emptyMap(), parentClusterConfig); ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); topology.getAdvisedConfigurations().putAll(createAdvisedConfigMap()); topology.setConfigRecommendationStrategy(ConfigRecommendationStrategy.NEVER_APPLY); BlueprintConfigurationProcessor configProcessor = new BlueprintConfigurationProcessor(topology); configProcessor.doUpdateForClusterCreate(); assertEquals(expectedHostName + ":" + expectedPortNum, clusterConfig.getPropertyValue("core-site", "fs.default.name")); assertNull(clusterConfig.getPropertyValue("core-site", "fs.notStackDefault")); assertEquals("stackDefaultValue1", clusterConfig.getPropertyValue("core-site", "fs.stackDefault.key1")); assertNotNull(clusterConfig.getPropertyValue("core-site", "fs.stackDefault.key2")); }
@Test public void testRangerAdminProperties() throws Exception { final String rangerAdminConfigType = "admin-properties"; Map<String, Map<String, String>> properties = new HashMap<String, Map<String, String>>(); Map<String, String> rangerAdminProperties = new HashMap<String, String>(); properties.put(rangerAdminConfigType, rangerAdminProperties); rangerAdminProperties.put("policymgr_external_url", "http: Map<String, Map<String, String>> parentProperties = new HashMap<String, Map<String, String>>(); Configuration parentClusterConfig = new Configuration(parentProperties, Collections.<String, Map<String, Map<String, String>>>emptyMap()); Configuration clusterConfig = new Configuration(properties, Collections.<String, Map<String, Map<String, String>>>emptyMap(), parentClusterConfig); Collection<String> rangerComponents = new HashSet<String>(); rangerComponents.add("RANGER_ADMIN"); rangerComponents.add("RANGER_USERSYNC"); TestHostGroup group1 = new TestHostGroup("group1", rangerComponents, Collections.singleton("host1")); Collection<TestHostGroup> hostGroups = Collections.singletonList(group1); ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); BlueprintConfigurationProcessor configProcessor = new BlueprintConfigurationProcessor(topology); configProcessor.doUpdateForClusterCreate(); assertEquals("http: }
@Test public void testRangerAdminProperties_defaults() throws Exception { final String rangerAdminConfigType = "admin-properties"; Map<String, Map<String, String>> properties = new HashMap<String, Map<String, String>>(); Map<String, String> rangerAdminProperties = new HashMap<String, String>(); properties.put(rangerAdminConfigType, rangerAdminProperties); rangerAdminProperties.put("policymgr_external_url", "http: Map<String, Map<String, String>> parentProperties = new HashMap<String, Map<String, String>>(); Configuration parentClusterConfig = new Configuration(parentProperties, Collections.<String, Map<String, Map<String, String>>>emptyMap()); Configuration clusterConfig = new Configuration(properties, Collections.<String, Map<String, Map<String, String>>>emptyMap(), parentClusterConfig); Collection<String> rangerComponents = new HashSet<String>(); rangerComponents.add("RANGER_ADMIN"); rangerComponents.add("RANGER_USERSYNC"); TestHostGroup group1 = new TestHostGroup("group1", rangerComponents, Collections.singleton("host1")); Collection<TestHostGroup> hostGroups = Collections.singletonList(group1); ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); BlueprintConfigurationProcessor configProcessor = new BlueprintConfigurationProcessor(topology); configProcessor.doUpdateForClusterCreate(); assertEquals("http: }
@Test public void testRangerAdminProperties_HA() throws Exception { final String rangerAdminConfigType = "admin-properties"; Map<String, Map<String, String>> properties = new HashMap<String, Map<String, String>>(); Map<String, String> rangerAdminProperties = new HashMap<String, String>(); properties.put(rangerAdminConfigType, rangerAdminProperties); rangerAdminProperties.put("policymgr_external_url", "http: Map<String, Map<String, String>> parentProperties = new HashMap<String, Map<String, String>>(); Configuration parentClusterConfig = new Configuration(parentProperties, Collections.<String, Map<String, Map<String, String>>>emptyMap()); Configuration clusterConfig = new Configuration(properties, Collections.<String, Map<String, Map<String, String>>>emptyMap(), parentClusterConfig); Collection<String> rangerComponents = new HashSet<String>(); rangerComponents.add("RANGER_ADMIN"); rangerComponents.add("RANGER_USERSYNC"); TestHostGroup group1 = new TestHostGroup("group1", rangerComponents, Collections.singleton("host1")); TestHostGroup group2 = new TestHostGroup("group2", rangerComponents, Collections.singleton("host2")); Collection<TestHostGroup> hostGroups = Lists.newArrayList(group1, group2); ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); BlueprintConfigurationProcessor configProcessor = new BlueprintConfigurationProcessor(topology); configProcessor.doUpdateForClusterCreate(); assertEquals("http: }
@Test public void testRangerEnv_defaults() throws Exception { List<String> configTypesWithRangerHdfsAuditDir = ImmutableList.of( "ranger-env", "ranger-yarn-audit", "ranger-hdfs-audit", "ranger-hbase-audit", "ranger-hive-audit", "ranger-knox-audit", "ranger-kafka-audit", "ranger-storm-audit", "ranger-atlas-audit" ); Map<String, Map<String, String>> clusterConfigProperties = new HashMap<>(); for (String configType: configTypesWithRangerHdfsAuditDir) { Map<String, String> configProperties = new HashMap<>(); configProperties.put("xasecure.audit.destination.hdfs.dir", "hdfs: clusterConfigProperties.put(configType, configProperties); } Map<String, Map<String, String>> parentProperties = new HashMap<>(); Configuration parentClusterConfig = new Configuration(parentProperties, new HashMap<String, Map<String, Map<String, String>>>()); Configuration clusterConfig = new Configuration(clusterConfigProperties, new HashMap<String, Map<String, Map<String, String>>>(), parentClusterConfig); Collection<String> rangerComponents = new HashSet<>(); rangerComponents.add("RANGER_ADMIN"); rangerComponents.add("RANGER_USERSYNC"); Collection<String> hdfsComponents = new HashSet<String>(); hdfsComponents.add("NAMENODE"); hdfsComponents.add("DATANODE"); TestHostGroup group1 = new TestHostGroup("group1", rangerComponents, Collections.singleton("host1")); group1.components.add("DATANODE"); TestHostGroup group2 = new TestHostGroup("group2", hdfsComponents, Collections.singleton("nn_host")); Collection<TestHostGroup> hostGroups = Lists.newArrayList(group1, group2); ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); BlueprintConfigurationProcessor configProcessor = new BlueprintConfigurationProcessor(topology); configProcessor.doUpdateForClusterCreate(); String expectedAuditHdfsDir = "hdfs: assertEquals(expectedAuditHdfsDir, clusterConfig.getPropertyValue("ranger-env", "xasecure.audit.destination.hdfs.dir")); assertEquals(expectedAuditHdfsDir, clusterConfig.getPropertyValue("ranger-yarn-audit", "xasecure.audit.destination.hdfs.dir")); assertEquals(expectedAuditHdfsDir, clusterConfig.getPropertyValue("ranger-hdfs-audit", "xasecure.audit.destination.hdfs.dir")); assertEquals(expectedAuditHdfsDir, clusterConfig.getPropertyValue("ranger-hbase-audit", "xasecure.audit.destination.hdfs.dir")); assertEquals(expectedAuditHdfsDir, clusterConfig.getPropertyValue("ranger-hive-audit", "xasecure.audit.destination.hdfs.dir")); assertEquals(expectedAuditHdfsDir, clusterConfig.getPropertyValue("ranger-knox-audit", "xasecure.audit.destination.hdfs.dir")); assertEquals(expectedAuditHdfsDir, clusterConfig.getPropertyValue("ranger-kafka-audit", "xasecure.audit.destination.hdfs.dir")); assertEquals(expectedAuditHdfsDir, clusterConfig.getPropertyValue("ranger-storm-audit", "xasecure.audit.destination.hdfs.dir")); assertEquals(expectedAuditHdfsDir, clusterConfig.getPropertyValue("ranger-atlas-audit", "xasecure.audit.destination.hdfs.dir")); }
@Test public void testRangerEnv_defaults_NO_HDFS() throws Exception { List<String> configTypesWithRangerHdfsAuditDir = ImmutableList.of( "ranger-env", "ranger-yarn-audit", "ranger-hdfs-audit", "ranger-hbase-audit", "ranger-hive-audit", "ranger-knox-audit", "ranger-kafka-audit", "ranger-storm-audit", "ranger-atlas-audit" ); Map<String, Map<String, String>> clusterConfigProperties = new HashMap<>(); for (String configType: configTypesWithRangerHdfsAuditDir) { Map<String, String> configProperties = new HashMap<>(); configProperties.put("xasecure.audit.destination.hdfs.dir", "hdfs: clusterConfigProperties.put(configType, configProperties); } Map<String, Map<String, String>> parentProperties = new HashMap<>(); Configuration parentClusterConfig = new Configuration(parentProperties, new HashMap<String, Map<String, Map<String, String>>>()); Configuration clusterConfig = new Configuration(clusterConfigProperties, new HashMap<String, Map<String, Map<String, String>>>(), parentClusterConfig); Collection<String> rangerComponents = new HashSet<>(); rangerComponents.add("RANGER_ADMIN"); rangerComponents.add("RANGER_USERSYNC"); TestHostGroup group1 = new TestHostGroup("group1", rangerComponents, Collections.singleton("host1")); group1.components.add("OOZIE_SERVER"); expect(stack.getCardinality("NAMENODE")).andReturn(new Cardinality("1+")).anyTimes(); Collection<TestHostGroup> hostGroups = Lists.newArrayList(group1); ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); BlueprintConfigurationProcessor configProcessor = new BlueprintConfigurationProcessor(topology); configProcessor.doUpdateForClusterCreate(); String expectedAuditHdfsDir = "hdfs: assertEquals(expectedAuditHdfsDir, clusterConfig.getPropertyValue("ranger-env", "xasecure.audit.destination.hdfs.dir")); assertEquals(expectedAuditHdfsDir, clusterConfig.getPropertyValue("ranger-yarn-audit", "xasecure.audit.destination.hdfs.dir")); assertEquals(expectedAuditHdfsDir, clusterConfig.getPropertyValue("ranger-hdfs-audit", "xasecure.audit.destination.hdfs.dir")); assertEquals(expectedAuditHdfsDir, clusterConfig.getPropertyValue("ranger-hbase-audit", "xasecure.audit.destination.hdfs.dir")); assertEquals(expectedAuditHdfsDir, clusterConfig.getPropertyValue("ranger-hive-audit", "xasecure.audit.destination.hdfs.dir")); assertEquals(expectedAuditHdfsDir, clusterConfig.getPropertyValue("ranger-knox-audit", "xasecure.audit.destination.hdfs.dir")); assertEquals(expectedAuditHdfsDir, clusterConfig.getPropertyValue("ranger-kafka-audit", "xasecure.audit.destination.hdfs.dir")); assertEquals(expectedAuditHdfsDir, clusterConfig.getPropertyValue("ranger-storm-audit", "xasecure.audit.destination.hdfs.dir")); assertEquals(expectedAuditHdfsDir, clusterConfig.getPropertyValue("ranger-atlas-audit", "xasecure.audit.destination.hdfs.dir")); }
@Test public void testRangerEnv() throws Exception { List<String> configTypesWithRangerHdfsAuditDir = ImmutableList.of( "ranger-env", "ranger-yarn-audit", "ranger-hdfs-audit", "ranger-hbase-audit", "ranger-hive-audit", "ranger-knox-audit", "ranger-kafka-audit", "ranger-storm-audit", "ranger-atlas-audit" ); Map<String, Map<String, String>> clusterConfigProperties = new HashMap<>(); for (String configType: configTypesWithRangerHdfsAuditDir) { Map<String, String> configProperties = new HashMap<>(); configProperties.put("xasecure.audit.destination.hdfs.dir", "hdfs: clusterConfigProperties.put(configType, configProperties); } Map<String, Map<String, String>> parentProperties = new HashMap<>(); Configuration parentClusterConfig = new Configuration(parentProperties, new HashMap<String, Map<String, Map<String, String>>>()); Configuration clusterConfig = new Configuration(clusterConfigProperties, new HashMap<String, Map<String, Map<String, String>>>(), parentClusterConfig); Collection<String> rangerComponents = new HashSet<>(); rangerComponents.add("RANGER_ADMIN"); rangerComponents.add("RANGER_USERSYNC"); Collection<String> hdfsComponents = new HashSet<String>(); hdfsComponents.add("NAMENODE"); hdfsComponents.add("DATANODE"); TestHostGroup group1 = new TestHostGroup("group1", rangerComponents, Collections.singleton("host1")); group1.components.add("DATANODE"); TestHostGroup group2 = new TestHostGroup("group2", hdfsComponents, Collections.singleton("nn_host")); Collection<TestHostGroup> hostGroups = Lists.newArrayList(group1, group2); ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); BlueprintConfigurationProcessor configProcessor = new BlueprintConfigurationProcessor(topology); configProcessor.doUpdateForClusterCreate(); String expectedAuditHdfsDir = "hdfs: assertEquals(expectedAuditHdfsDir, clusterConfig.getPropertyValue("ranger-env", "xasecure.audit.destination.hdfs.dir")); assertEquals(expectedAuditHdfsDir, clusterConfig.getPropertyValue("ranger-yarn-audit", "xasecure.audit.destination.hdfs.dir")); assertEquals(expectedAuditHdfsDir, clusterConfig.getPropertyValue("ranger-hdfs-audit", "xasecure.audit.destination.hdfs.dir")); assertEquals(expectedAuditHdfsDir, clusterConfig.getPropertyValue("ranger-hbase-audit", "xasecure.audit.destination.hdfs.dir")); assertEquals(expectedAuditHdfsDir, clusterConfig.getPropertyValue("ranger-hive-audit", "xasecure.audit.destination.hdfs.dir")); assertEquals(expectedAuditHdfsDir, clusterConfig.getPropertyValue("ranger-knox-audit", "xasecure.audit.destination.hdfs.dir")); assertEquals(expectedAuditHdfsDir, clusterConfig.getPropertyValue("ranger-kafka-audit", "xasecure.audit.destination.hdfs.dir")); assertEquals(expectedAuditHdfsDir, clusterConfig.getPropertyValue("ranger-storm-audit", "xasecure.audit.destination.hdfs.dir")); assertEquals(expectedAuditHdfsDir, clusterConfig.getPropertyValue("ranger-atlas-audit", "xasecure.audit.destination.hdfs.dir")); }
@Test public void testRangerEnvWithHdfsHA() throws Exception { List<String> configTypesWithRangerHdfsAuditDir = ImmutableList.of( "ranger-env", "ranger-yarn-audit", "ranger-hdfs-audit", "ranger-hbase-audit", "ranger-hive-audit", "ranger-knox-audit", "ranger-kafka-audit", "ranger-storm-audit", "ranger-atlas-audit" ); Map<String, Map<String, String>> clusterConfigProperties = new HashMap<>(); for (String configType: configTypesWithRangerHdfsAuditDir) { Map<String, String> configProperties = new HashMap<>(); configProperties.put("xasecure.audit.destination.hdfs.dir", "hdfs: clusterConfigProperties.put(configType, configProperties); } final String hdfsSiteConfigType = "hdfs-site"; Map<String, String> hdfsSiteProperties = new HashMap<>(); clusterConfigProperties.put(hdfsSiteConfigType, hdfsSiteProperties); hdfsSiteProperties.put("dfs.nameservices", "my_name_service"); hdfsSiteProperties.put("dfs.ha.namenodes.my_name_service", "nn1,nn2"); Map<String, Map<String, String>> parentProperties = new HashMap<>(); Configuration parentClusterConfig = new Configuration(parentProperties, new HashMap<String, Map<String, Map<String, String>>>()); Configuration clusterConfig = new Configuration(clusterConfigProperties, new HashMap<String, Map<String, Map<String, String>>>(), parentClusterConfig); Collection<String> rangerComponents = new HashSet<>(); rangerComponents.add("RANGER_ADMIN"); rangerComponents.add("RANGER_USERSYNC"); Collection<String> hdfsComponents = new HashSet<String>(); hdfsComponents.add("NAMENODE"); hdfsComponents.add("DATANODE"); TestHostGroup group1 = new TestHostGroup("group1", rangerComponents, Collections.singleton("host1")); group1.components.addAll(hdfsComponents); TestHostGroup group2 = new TestHostGroup("group2", hdfsComponents, Collections.singleton("host2")); Collection<TestHostGroup> hostGroups = Lists.newArrayList(group1, group2); ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); BlueprintConfigurationProcessor configProcessor = new BlueprintConfigurationProcessor(topology); configProcessor.doUpdateForClusterCreate(); String expectedAuditHdfsDir = "hdfs: assertEquals(expectedAuditHdfsDir, clusterConfig.getPropertyValue("ranger-env", "xasecure.audit.destination.hdfs.dir")); assertEquals(expectedAuditHdfsDir, clusterConfig.getPropertyValue("ranger-yarn-audit", "xasecure.audit.destination.hdfs.dir")); assertEquals(expectedAuditHdfsDir, clusterConfig.getPropertyValue("ranger-hdfs-audit", "xasecure.audit.destination.hdfs.dir")); assertEquals(expectedAuditHdfsDir, clusterConfig.getPropertyValue("ranger-hbase-audit", "xasecure.audit.destination.hdfs.dir")); assertEquals(expectedAuditHdfsDir, clusterConfig.getPropertyValue("ranger-hive-audit", "xasecure.audit.destination.hdfs.dir")); assertEquals(expectedAuditHdfsDir, clusterConfig.getPropertyValue("ranger-knox-audit", "xasecure.audit.destination.hdfs.dir")); assertEquals(expectedAuditHdfsDir, clusterConfig.getPropertyValue("ranger-kafka-audit", "xasecure.audit.destination.hdfs.dir")); assertEquals(expectedAuditHdfsDir, clusterConfig.getPropertyValue("ranger-storm-audit", "xasecure.audit.destination.hdfs.dir")); assertEquals(expectedAuditHdfsDir, clusterConfig.getPropertyValue("ranger-atlas-audit", "xasecure.audit.destination.hdfs.dir")); }
@Test public void testRangerKmsServerProperties() throws Exception { final String kmsSiteConfigType = "kms-site"; Map<String, Map<String, String>> properties = new HashMap<String, Map<String, String>>(); Map<String, String> kmsSiteProperties = new HashMap<String, String>(); properties.put(kmsSiteConfigType, kmsSiteProperties); kmsSiteProperties.put("hadoop.kms.authentication.signer.secret.provider.zookeeper.connection.string", createHostAddress("%HOSTGROUP::group1%", "2181") + "," + createHostAddress("%HOSTGROUP::group2%", "2181")); kmsSiteProperties.put("hadoop.kms.key.provider.uri", "dbks: Map<String, Map<String, String>> parentProperties = new HashMap<String, Map<String, String>>(); Configuration parentClusterConfig = new Configuration(parentProperties, Collections.<String, Map<String, Map<String, String>>>emptyMap()); Configuration clusterConfig = new Configuration(properties, Collections.<String, Map<String, Map<String, String>>>emptyMap(), parentClusterConfig); Collection<String> kmsServerComponents = new HashSet<String>(); kmsServerComponents.add("RANGER_KMS_SERVER"); TestHostGroup group1 = new TestHostGroup("group1", kmsServerComponents, Collections.singleton("host1")); TestHostGroup group2 = new TestHostGroup("group2", kmsServerComponents, Collections.singleton("host2")); Collection<TestHostGroup> hostGroups = new HashSet<TestHostGroup>(); hostGroups.add(group1); hostGroups.add(group2); ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); BlueprintConfigurationProcessor configProcessor = new BlueprintConfigurationProcessor(topology); configProcessor.doUpdateForClusterCreate(); assertEquals("host1:2181,host2:2181", clusterConfig.getPropertyValue(kmsSiteConfigType, "hadoop.kms.authentication.signer.secret.provider.zookeeper.connection.string")); assertEquals("dbks: }
@Test public void testRangerKmsServerProperties_default() throws Exception { final String kmsSiteConfigType = "kms-site"; Map<String, Map<String, String>> properties = new HashMap<String, Map<String, String>>(); Map<String, String> kmsSiteProperties = new HashMap<String, String>(); properties.put(kmsSiteConfigType, kmsSiteProperties); kmsSiteProperties.put("hadoop.kms.authentication.signer.secret.provider.zookeeper.connection.string", createHostAddress("%HOSTGROUP::group1%", "2181")); Map<String, Map<String, String>> parentProperties = new HashMap<String, Map<String, String>>(); Configuration parentClusterConfig = new Configuration(parentProperties, Collections.<String, Map<String, Map<String, String>>>emptyMap()); Configuration clusterConfig = new Configuration(properties, Collections.<String, Map<String, Map<String, String>>>emptyMap(), parentClusterConfig); Collection<String> kmsServerComponents = new HashSet<String>(); kmsServerComponents.add("RANGER_KMS_SERVER"); TestHostGroup group1 = new TestHostGroup("group1", kmsServerComponents, Collections.singleton("host1")); Collection<TestHostGroup> hostGroups = Collections.singleton(group1); ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); BlueprintConfigurationProcessor configProcessor = new BlueprintConfigurationProcessor(topology); configProcessor.doUpdateForClusterCreate(); assertEquals("host1:2181", clusterConfig.getPropertyValue(kmsSiteConfigType, "hadoop.kms.authentication.signer.secret.provider.zookeeper.connection.string")); }
@Test public void testHdfsWithRangerKmsServer() throws Exception { final String configType = "hdfs-site"; Map<String, Map<String, String>> properties = new HashMap<String, Map<String, String>>(); Map<String, String> configProperties = new HashMap<String, String>(); properties.put(configType, configProperties); configProperties.put("dfs.encryption.key.provider.uri", "kms: Map<String, Map<String, String>> parentProperties = new HashMap<String, Map<String, String>>(); Configuration parentClusterConfig = new Configuration(parentProperties, Collections.<String, Map<String, Map<String, String>>>emptyMap()); Configuration clusterConfig = new Configuration(properties, Collections.<String, Map<String, Map<String, String>>>emptyMap(), parentClusterConfig); Collection<String> kmsServerComponents = new HashSet<String>(); kmsServerComponents.add("RANGER_KMS_SERVER"); Collection<String> hdfsComponents = new HashSet<String>(); hdfsComponents.add("NAMENODE"); hdfsComponents.add("DATANODE"); TestHostGroup group1 = new TestHostGroup("group1", kmsServerComponents, Collections.singleton("host1")); group1.components.add("DATANODE"); TestHostGroup group2 = new TestHostGroup("group2", hdfsComponents, Collections.singleton("host2")); Collection<TestHostGroup> hostGroups = Lists.newArrayList(group1, group2); ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); BlueprintConfigurationProcessor configProcessor = new BlueprintConfigurationProcessor(topology); configProcessor.doUpdateForClusterCreate(); String updatedVal = clusterConfig.getPropertyValue(configType, "dfs.encryption.key.provider.uri"); Assert.assertTrue(updatedVal.startsWith("kms: Assert.assertTrue(updatedVal.endsWith(":9292/kms")); String hostsString = updatedVal.substring(11,updatedVal.length()-9); List<String> hostArray = Arrays.asList(hostsString.split(";")); List<String> expected = Arrays.asList("host1","host2"); Assert.assertTrue(hostArray.containsAll(expected) && expected.containsAll(hostArray)); }
@Test public void testHdfsWithNoRangerKmsServer() throws Exception { expect(stack.getCardinality("NAMENODE")).andReturn(new Cardinality("1-2")).anyTimes(); expect(stack.getCardinality("DATANODE")).andReturn(new Cardinality("1+")).anyTimes(); expect(stack.getCardinality("RANGER_KMS_SERVER")).andReturn(new Cardinality("1+")).anyTimes(); final String configType = "hdfs-site"; Map<String, Map<String, String>> properties = new HashMap<String, Map<String, String>>(); Map<String, String> configProperties = new HashMap<String, String>(); properties.put(configType, configProperties); configProperties.put("dfs.encryption.key.provider.uri", "leave_untouched"); Map<String, Map<String, String>> parentProperties = new HashMap<String, Map<String, String>>(); Configuration parentClusterConfig = new Configuration(parentProperties, Collections.<String, Map<String, Map<String, String>>>emptyMap()); Configuration clusterConfig = new Configuration(properties, Collections.<String, Map<String, Map<String, String>>>emptyMap(), parentClusterConfig); Collection<String> hdfsComponents = new HashSet<String>(); hdfsComponents.add("NAMENODE"); hdfsComponents.add("DATANODE"); TestHostGroup group1 = new TestHostGroup("group1", Collections.singletonList("DATANODE"), Collections.singleton("host1")); TestHostGroup group2 = new TestHostGroup("group2", hdfsComponents, Collections.singleton("host2")); Collection<TestHostGroup> hostGroups = Lists.newArrayList(group1, group2); ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); BlueprintConfigurationProcessor configProcessor = new BlueprintConfigurationProcessor(topology); configProcessor.doUpdateForClusterCreate(); assertEquals("leave_untouched", clusterConfig.getPropertyValue(configType, "dfs.encryption.key.provider.uri")); }
@Test public void testHdfsWithRangerKmsServer_default() throws Exception { final String configType = "hdfs-site"; Map<String, Map<String, String>> properties = new HashMap<String, Map<String, String>>(); Map<String, String> configProperties = new HashMap<String, String>(); properties.put(configType, configProperties); configProperties.put("dfs.encryption.key.provider.uri", "kms: Map<String, Map<String, String>> parentProperties = new HashMap<String, Map<String, String>>(); Configuration parentClusterConfig = new Configuration(parentProperties, Collections.<String, Map<String, Map<String, String>>>emptyMap()); Configuration clusterConfig = new Configuration(properties, Collections.<String, Map<String, Map<String, String>>>emptyMap(), parentClusterConfig); Collection<String> kmsServerComponents = new HashSet<String>(); kmsServerComponents.add("RANGER_KMS_SERVER"); Collection<String> hdfsComponents = new HashSet<String>(); hdfsComponents.add("NAMENODE"); hdfsComponents.add("DATANODE"); TestHostGroup group1 = new TestHostGroup("group1", kmsServerComponents, Collections.singleton("host1")); group1.components.add("DATANODE"); TestHostGroup group2 = new TestHostGroup("group2", hdfsComponents, Collections.singleton("host2")); Collection<TestHostGroup> hostGroups = Lists.newArrayList(group1, group2); ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); BlueprintConfigurationProcessor configProcessor = new BlueprintConfigurationProcessor(topology); configProcessor.doUpdateForClusterCreate(); assertEquals("kms: }
@Test public void testHdfsWithRangerKmsServer__multiple_hosts__localhost() throws Exception { final String configType = "hdfs-site"; Map<String, Map<String, String>> properties = new HashMap<String, Map<String, String>>(); Map<String, String> configProperties = new HashMap<String, String>(); properties.put(configType, configProperties); configProperties.put("dfs.encryption.key.provider.uri", "kms: Map<String, Map<String, String>> parentProperties = new HashMap<String, Map<String, String>>(); Configuration parentClusterConfig = new Configuration(parentProperties, Collections.<String, Map<String, Map<String, String>>>emptyMap()); Configuration clusterConfig = new Configuration(properties, Collections.<String, Map<String, Map<String, String>>>emptyMap(), parentClusterConfig); Collection<String> kmsServerComponents = new HashSet<String>(); kmsServerComponents.add("RANGER_KMS_SERVER"); Collection<String> hdfsComponents = new HashSet<String>(); hdfsComponents.add("NAMENODE"); hdfsComponents.add("DATANODE"); Collection<String> hosts = new HashSet<String>(); hosts.add("host1"); hosts.add("host2"); TestHostGroup group1 = new TestHostGroup("group1", kmsServerComponents, hosts); group1.components.add("DATANODE"); TestHostGroup group2 = new TestHostGroup("group2", hdfsComponents, Collections.singleton("host3")); Collection<TestHostGroup> hostGroups = Lists.newArrayList(group1, group2); ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); BlueprintConfigurationProcessor configProcessor = new BlueprintConfigurationProcessor(topology); configProcessor.doUpdateForClusterCreate(); String updatedVal = clusterConfig.getPropertyValue(configType, "dfs.encryption.key.provider.uri"); Assert.assertTrue(updatedVal.startsWith("kms: Assert.assertTrue(updatedVal.endsWith(":9292/kms")); String hostsString = updatedVal.substring(11,updatedVal.length()-9); List<String> hostArray = Arrays.asList(hostsString.split(";")); List<String> expected = Arrays.asList("host1","host2"); Assert.assertTrue(hostArray.containsAll(expected) && expected.containsAll(hostArray)); }
@Test public void testHdfsWithRangerKmsServer__multiple_hosts__hostgroup() throws Exception { final String configType = "hdfs-site"; Map<String, Map<String, String>> properties = new HashMap<String, Map<String, String>>(); Map<String, String> configProperties = new HashMap<String, String>(); properties.put(configType, configProperties); configProperties.put("dfs.encryption.key.provider.uri", "kms: Map<String, Map<String, String>> parentProperties = new HashMap<String, Map<String, String>>(); Configuration parentClusterConfig = new Configuration(parentProperties, Collections.<String, Map<String, Map<String, String>>>emptyMap()); Configuration clusterConfig = new Configuration(properties, Collections.<String, Map<String, Map<String, String>>>emptyMap(), parentClusterConfig); Collection<String> kmsServerComponents = new HashSet<String>(); kmsServerComponents.add("RANGER_KMS_SERVER"); Collection<String> hdfsComponents = new HashSet<String>(); hdfsComponents.add("NAMENODE"); hdfsComponents.add("DATANODE"); Collection<String> hosts = new HashSet<String>(); hosts.add("host1"); hosts.add("host2"); TestHostGroup group1 = new TestHostGroup("group1", kmsServerComponents, hosts); group1.components.add("DATANODE"); TestHostGroup group2 = new TestHostGroup("group2", hdfsComponents, Collections.singleton("host3")); Collection<TestHostGroup> hostGroups = Lists.newArrayList(group1, group2); ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); BlueprintConfigurationProcessor configProcessor = new BlueprintConfigurationProcessor(topology); configProcessor.doUpdateForClusterCreate(); String updatedVal = clusterConfig.getPropertyValue(configType, "dfs.encryption.key.provider.uri"); Assert.assertTrue(updatedVal.startsWith("kms: Assert.assertTrue(updatedVal.endsWith(":9292/kms")); String hostsString = updatedVal.substring(11,updatedVal.length()-9); List<String> hostArray = Arrays.asList(hostsString.split(";")); List<String> expected = Arrays.asList("host1","host2"); Assert.assertTrue(hostArray.containsAll(expected) && expected.containsAll(hostArray)); }
@Test public void testResolutionOfDRPCServerAndNN() throws Exception { final String stormConfigType = "storm-site"; final String mrConfigType = "mapred-site"; Map<String, Map<String, String>> properties = new HashMap<String, Map<String, String>>(); Map<String, String> stormConfigProperties = new HashMap<String, String>(); Map<String, String> mrConfigProperties = new HashMap<String, String>(); properties.put(stormConfigType, stormConfigProperties); properties.put(mrConfigType, mrConfigProperties); stormConfigProperties.put("drpc.servers", "['%HOSTGROUP::group1%']"); mrConfigProperties.put("mapreduce.job.hdfs-servers", "['%HOSTGROUP::group2%']"); Map<String, Map<String, String>> parentProperties = new HashMap<String, Map<String, String>>(); Configuration parentClusterConfig = new Configuration(parentProperties, Collections.<String, Map<String, Map<String, String>>>emptyMap()); Configuration clusterConfig = new Configuration(properties, Collections.<String, Map<String, Map<String, String>>>emptyMap(), parentClusterConfig); Collection<String> stormComponents = new HashSet<String>(); stormComponents.add("NIMBUS"); stormComponents.add("DRPC_SERVER"); Collection<String> hdfsComponents = new HashSet<String>(); hdfsComponents.add("NAMENODE"); TestHostGroup group1 = new TestHostGroup("group1", stormComponents, Collections.singleton("host1")); group1.components.add("DATANODE"); TestHostGroup group2 = new TestHostGroup("group2", hdfsComponents, Collections.singleton("host2")); Collection<TestHostGroup> hostGroups = Lists.newArrayList(group1, group2); ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); BlueprintConfigurationProcessor configProcessor = new BlueprintConfigurationProcessor(topology); configProcessor.doUpdateForClusterCreate(); assertEquals("['host1']", clusterConfig.getPropertyValue(stormConfigType, "drpc.servers")); assertEquals("['host2']", clusterConfig.getPropertyValue(mrConfigType, "mapreduce.job.hdfs-servers")); }
@Test public void testHadoopWithRangerKmsServer() throws Exception { final String configType = "core-site"; Map<String, Map<String, String>> properties = new HashMap<String, Map<String, String>>(); Map<String, String> configProperties = new HashMap<String, String>(); properties.put(configType, configProperties); configProperties.put("hadoop.security.key.provider.path", "kms: Map<String, Map<String, String>> parentProperties = new HashMap<String, Map<String, String>>(); Configuration parentClusterConfig = new Configuration(parentProperties, Collections.<String, Map<String, Map<String, String>>>emptyMap()); Configuration clusterConfig = new Configuration(properties, Collections.<String, Map<String, Map<String, String>>>emptyMap(), parentClusterConfig); Collection<String> kmsServerComponents = new HashSet<String>(); kmsServerComponents.add("RANGER_KMS_SERVER"); Collection<String> hdfsComponents = new HashSet<String>(); hdfsComponents.add("NAMENODE"); hdfsComponents.add("DATANODE"); TestHostGroup group1 = new TestHostGroup("group1", kmsServerComponents, Collections.singleton("host1")); group1.components.add("DATANODE"); TestHostGroup group2 = new TestHostGroup("group2", hdfsComponents, Collections.singleton("host2")); Collection<TestHostGroup> hostGroups = Lists.newArrayList(group1, group2); ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); BlueprintConfigurationProcessor configProcessor = new BlueprintConfigurationProcessor(topology); configProcessor.doUpdateForClusterCreate(); assertEquals("kms: }
@Test public void testHadoopWithNoRangerKmsServer() throws Exception { expect(stack.getCardinality("NAMENODE")).andReturn(new Cardinality("1-2")).anyTimes(); expect(stack.getCardinality("DATANODE")).andReturn(new Cardinality("1+")).anyTimes(); expect(stack.getCardinality("RANGER_KMS_SERVER")).andReturn(new Cardinality("1+")).anyTimes(); final String configType = "core-site"; Map<String, Map<String, String>> properties = new HashMap<String, Map<String, String>>(); Map<String, String> configProperties = new HashMap<String, String>(); properties.put(configType, configProperties); configProperties.put("hadoop.security.key.provider.path", "leave_untouched"); Map<String, Map<String, String>> parentProperties = new HashMap<String, Map<String, String>>(); Configuration parentClusterConfig = new Configuration(parentProperties, Collections.<String, Map<String, Map<String, String>>>emptyMap()); Configuration clusterConfig = new Configuration(properties, Collections.<String, Map<String, Map<String, String>>>emptyMap(), parentClusterConfig); Collection<String> hdfsComponents = new HashSet<String>(); hdfsComponents.add("NAMENODE"); hdfsComponents.add("DATANODE"); TestHostGroup group1 = new TestHostGroup("group1", Collections.singletonList("DATANODE"), Collections.singleton("host1")); TestHostGroup group2 = new TestHostGroup("group2", hdfsComponents, Collections.singleton("host2")); Collection<TestHostGroup> hostGroups = Lists.newArrayList(group1, group2); ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); BlueprintConfigurationProcessor configProcessor = new BlueprintConfigurationProcessor(topology); configProcessor.doUpdateForClusterCreate(); assertEquals("leave_untouched", clusterConfig.getPropertyValue(configType, "hadoop.security.key.provider.path")); }
@Test public void testHadoopWithRangerKmsServer_default() throws Exception { final String configType = "core-site"; Map<String, Map<String, String>> properties = new HashMap<String, Map<String, String>>(); Map<String, String> configProperties = new HashMap<String, String>(); properties.put(configType, configProperties); configProperties.put("hadoop.security.key.provider.path", "kms: Map<String, Map<String, String>> parentProperties = new HashMap<String, Map<String, String>>(); Configuration parentClusterConfig = new Configuration(parentProperties, Collections.<String, Map<String, Map<String, String>>>emptyMap()); Configuration clusterConfig = new Configuration(properties, Collections.<String, Map<String, Map<String, String>>>emptyMap(), parentClusterConfig); Collection<String> kmsServerComponents = new HashSet<String>(); kmsServerComponents.add("RANGER_KMS_SERVER"); Collection<String> hdfsComponents = new HashSet<String>(); hdfsComponents.add("NAMENODE"); hdfsComponents.add("DATANODE"); TestHostGroup group1 = new TestHostGroup("group1", kmsServerComponents, Collections.singleton("host1")); group1.components.add("DATANODE"); TestHostGroup group2 = new TestHostGroup("group2", hdfsComponents, Collections.singleton("host2")); Collection<TestHostGroup> hostGroups = Lists.newArrayList(group1, group2); ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); BlueprintConfigurationProcessor configProcessor = new BlueprintConfigurationProcessor(topology); configProcessor.doUpdateForClusterCreate(); assertEquals("kms: }
@Test public void testHawqConfigClusterUpdate() throws Exception { final String expectedHostNameHawqMaster = "c6401.apache.ambari.org"; final String expectedHostNameHawqStandby = "c6402.apache.ambari.org"; final String expectedHostNameNamenode = "c6403.apache.ambari.org"; final String expectedPortNamenode = "8020"; Map<String, Map<String, String>> properties = new HashMap<String, Map<String, String>>(); Map<String, String> hawqSite = new HashMap<String, String>(); properties.put("hawq-site", hawqSite); hawqSite.put("hawq_master_address_host", "localhost"); hawqSite.put("hawq_standby_address_host", "localhost"); hawqSite.put("hawq_dfs_url", createHostAddress("localhost", expectedPortNamenode) + "/hawq_data"); Configuration clusterConfig = new Configuration(properties, Collections.<String, Map<String, Map<String, String>>>emptyMap()); Collection<String> hgComponents = new HashSet<String>(); hgComponents.add("NAMENODE"); TestHostGroup group1 = new TestHostGroup("group1", hgComponents, Collections.singleton(expectedHostNameNamenode)); Collection<String> hgComponents2 = new HashSet<String>(); hgComponents2.add("HAWQMASTER"); TestHostGroup group2 = new TestHostGroup("group2", hgComponents2, Collections.singleton(expectedHostNameHawqMaster)); Collection<String> hgComponents3 = new HashSet<String>(); hgComponents3.add("HAWQSTANDBY"); TestHostGroup group3 = new TestHostGroup("group3", hgComponents3, Collections.singleton(expectedHostNameHawqStandby)); Collection<TestHostGroup> hostGroups = new HashSet<TestHostGroup>(); hostGroups.add(group1); hostGroups.add(group2); hostGroups.add(group3); ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); BlueprintConfigurationProcessor updater = new BlueprintConfigurationProcessor(topology); updater.doUpdateForClusterCreate(); assertEquals(expectedHostNameHawqMaster, hawqSite.get("hawq_master_address_host")); assertEquals(expectedHostNameHawqStandby, hawqSite.get("hawq_standby_address_host")); assertEquals(createHostAddress(expectedHostNameNamenode, expectedPortNamenode) + "/hawq_data", hawqSite.get("hawq_dfs_url")); }
@Test public void testHawqNonHaConfigClusterUpdate() throws Exception { final String expectedHostNameHawqMaster = "c6401.apache.ambari.org"; Map<String, Map<String, String>> properties = new HashMap<String, Map<String, String>>(); Map<String, String> hawqSite = new HashMap<String, String>(); properties.put("hawq-site", hawqSite); hawqSite.put("hawq_master_address_host", "localhost"); hawqSite.put("hawq_standby_address_host", "localhost"); Configuration clusterConfig = new Configuration(properties, Collections.<String, Map<String, Map<String, String>>>emptyMap()); Collection<String> hgComponents1 = new HashSet<String>(); hgComponents1.add("HAWQMASTER"); TestHostGroup group1 = new TestHostGroup("group1", hgComponents1, Collections.singleton(expectedHostNameHawqMaster)); Collection<TestHostGroup> hostGroups = new HashSet<TestHostGroup>(); hostGroups.add(group1); ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); BlueprintConfigurationProcessor updater = new BlueprintConfigurationProcessor(topology); updater.doUpdateForClusterCreate(); assertEquals(expectedHostNameHawqMaster, hawqSite.get("hawq_master_address_host")); assertFalse("hawq_standby_address_host should have been filtered out of this non-HAWQ HA configuration", hawqSite.containsKey("hawq_standby_address_host")); }
@Test public void testAmsPropertiesDefault() throws Exception { Map<String, Map<String, String>> properties = new HashMap<String, Map<String, String>>(); Map<String, String> amsSite = new HashMap<String, String>(); amsSite.put("timeline.metrics.service.webapp.address", "localhost:6188"); properties.put("ams-site", amsSite); Map<String, Map<String, String>> parentProperties = new HashMap<String, Map<String, String>>(); Configuration parentClusterConfig = new Configuration(parentProperties, Collections.<String, Map<String, Map<String, String>>>emptyMap()); Configuration clusterConfig = new Configuration(properties, Collections.<String, Map<String, Map<String, String>>>emptyMap(), parentClusterConfig); Collection<String> hgComponents1 = new HashSet<String>(); hgComponents1.add("METRICS_COLLECTOR"); TestHostGroup group1 = new TestHostGroup("group1", hgComponents1, Collections.singleton("host1")); Collection<TestHostGroup> hostGroups = Collections.singletonList(group1); ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); BlueprintConfigurationProcessor configProcessor = new BlueprintConfigurationProcessor(topology); configProcessor.doUpdateForClusterCreate(); assertEquals("0.0.0.0:6188", clusterConfig.getPropertyValue("ams-site", "timeline.metrics.service.webapp.address")); }
@Test public void testAmsPropertiesSpecialAddress() throws Exception { Map<String, Map<String, String>> properties = new HashMap<String, Map<String, String>>(); Map<String, String> amsSite = new HashMap<String, String>(); amsSite.put("timeline.metrics.service.webapp.address", "0.0.0.0:6188"); properties.put("ams-site", amsSite); Map<String, Map<String, String>> parentProperties = new HashMap<String, Map<String, String>>(); Configuration parentClusterConfig = new Configuration(parentProperties, Collections.<String, Map<String, Map<String, String>>>emptyMap()); Configuration clusterConfig = new Configuration(properties, Collections.<String, Map<String, Map<String, String>>>emptyMap(), parentClusterConfig); Collection<String> hgComponents1 = new HashSet<String>(); hgComponents1.add("METRICS_COLLECTOR"); TestHostGroup group1 = new TestHostGroup("group1", hgComponents1, Collections.singleton("host1")); Collection<TestHostGroup> hostGroups = Collections.singletonList(group1); ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); BlueprintConfigurationProcessor configProcessor = new BlueprintConfigurationProcessor(topology); configProcessor.doUpdateForClusterCreate(); assertEquals("0.0.0.0:6188", clusterConfig.getPropertyValue("ams-site", "timeline.metrics.service.webapp.address")); }
@Test public void testAmsPropertiesSpecialAddressMultipleCollectors() throws Exception { Map<String, Map<String, String>> properties = new HashMap<String, Map<String, String>>(); Map<String, String> amsSite = new HashMap<String, String>(); amsSite.put("timeline.metrics.service.webapp.address", "0.0.0.0:6188"); properties.put("ams-site", amsSite); Map<String, Map<String, String>> parentProperties = new HashMap<String, Map<String, String>>(); Configuration parentClusterConfig = new Configuration(parentProperties, Collections.<String, Map<String, Map<String, String>>>emptyMap()); Configuration clusterConfig = new Configuration(properties, Collections.<String, Map<String, Map<String, String>>>emptyMap(), parentClusterConfig); Collection<String> hgComponents1 = new HashSet<String>(); Collection<String> hgComponents2 = new HashSet<String>(); hgComponents1.add("METRICS_COLLECTOR"); hgComponents2.add("METRICS_COLLECTOR"); TestHostGroup group1 = new TestHostGroup("group1", hgComponents1, Collections.singleton("host1")); TestHostGroup group2 = new TestHostGroup("group2", hgComponents1, Collections.singleton("host2")); Collection<TestHostGroup> hostGroups = new LinkedList<>(); hostGroups.add(group1); hostGroups.add(group2); ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); BlueprintConfigurationProcessor configProcessor = new BlueprintConfigurationProcessor(topology); configProcessor.doUpdateForClusterCreate(); assertEquals("0.0.0.0:6188", clusterConfig.getPropertyValue("ams-site", "timeline.metrics.service.webapp.address")); }
@Test public void testValuesTrimming() throws Exception { reset(stack); Map<String, Map<String, String>> properties = new HashMap<String, Map<String, String>>(); Map<String, String> hdfsSite = new HashMap<String, String>(); hdfsSite.put("test.spaces", " spaces at the end should be deleted "); hdfsSite.put("test.directories", " /all/spaces , should/be , deleted "); hdfsSite.put("test.password", " stays, same "); hdfsSite.put("test.single.space", " "); hdfsSite.put("test.host", " https: properties.put("hdfs-site", hdfsSite); Map<String, Stack.ConfigProperty> propertyConfigs = new HashMap<>(); ValueAttributesInfo valueAttributesInfoDirs = new ValueAttributesInfo(); valueAttributesInfoDirs.setType("directories"); ValueAttributesInfo valueAttributesInfoHost = new ValueAttributesInfo(); valueAttributesInfoHost.setType("host"); propertyConfigs.put("test.directories", new Stack.ConfigProperty( new StackConfigurationResponse(null,null,null,null,"hdfs-site",null,null,null,valueAttributesInfoDirs,null))); propertyConfigs.put("test.password", new Stack.ConfigProperty( new StackConfigurationResponse(null,null,null,null,"hdfs-site",null,Collections.singleton(PropertyInfo.PropertyType.PASSWORD),null,null,null))); propertyConfigs.put("test.host", new Stack.ConfigProperty( new StackConfigurationResponse(null,null,null,null,"hdfs-site",null,null,null,valueAttributesInfoHost,null))); expect(stack.getServiceForConfigType("hdfs-site")).andReturn("HDFS").anyTimes(); expect(stack.getConfigurationPropertiesWithMetadata("HDFS", "hdfs-site")).andReturn(propertyConfigs).anyTimes(); Map<String, Map<String, String>> parentProperties = new HashMap<String, Map<String, String>>(); Configuration parentClusterConfig = new Configuration(parentProperties, Collections.<String, Map<String, Map<String, String>>>emptyMap()); Configuration clusterConfig = new Configuration(properties, Collections.<String, Map<String, Map<String, String>>>emptyMap(), parentClusterConfig); Collection<String> hgComponents1 = new HashSet<String>(); TestHostGroup group1 = new TestHostGroup("group1", hgComponents1, Collections.singleton("host1")); Collection<TestHostGroup> hostGroups = Collections.singletonList(group1); ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); BlueprintConfigurationProcessor configProcessor = new BlueprintConfigurationProcessor(topology); configProcessor.doUpdateForClusterCreate(); assertEquals(" spaces at the end should be deleted", clusterConfig.getPropertyValue("hdfs-site", "test.spaces")); assertEquals("/all/spaces,should/be,deleted", clusterConfig.getPropertyValue("hdfs-site", "test.directories")); assertEquals( " stays, same ", clusterConfig.getPropertyValue("hdfs-site", "test.password")); assertEquals(" https: clusterConfig.getPropertyValue("hdfs-site", "test.host")); assertEquals(" ", clusterConfig.getPropertyValue("hdfs-site", "test.single.space")); }