@Test(expected=InvalidTopologyException.class) public void testValidateRequiredProperties_SqlaInOozieStackHdp22() throws Exception { Map<String, String> hiveEnvConfig = new HashMap<String, String>(); hiveEnvConfig.put("oozie_database","Existing SQL Anywhere Database"); configProperties.put("oozie-env", hiveEnvConfig); group1Components.add("OOZIE_SERVER"); services.addAll(Arrays.asList("OOZIE")); Configuration config = new Configuration(new HashMap<String, Map<String, String>>(), new HashMap<String, Map<String, Map<String, String>>>()); expect(group1.getConfiguration()).andReturn(config).anyTimes(); expect(stack.getComponents("OOZIE")).andReturn(Collections.singleton("OOZIE_SERVER")).anyTimes(); expect(stack.getVersion()).andReturn("2.2").once(); expect(stack.getName()).andReturn("HDP").once(); expect(blueprint.getHostGroupsForComponent("OOZIE_SERVER")).andReturn(Collections.singleton(group1)).anyTimes(); replay(blueprint, stack, group1, group2, dependency1); BlueprintValidator validator = new BlueprintValidatorImpl(blueprint); validator.validateRequiredProperties(); }
@Override public void validateRequiredProperties() throws InvalidTopologyException { Map<String, Map<String, Collection<String>>> missingProperties = new HashMap<String, Map<String, Collection<String>>>(); Map<String, Map<String, String>> clusterConfigurations = blueprint.getConfiguration().getProperties(); if(clusterConfigurations != null) { StringBuilder errorMessage = new StringBuilder(); boolean containsSecretReferences = false; for (Map.Entry<String, Map<String, String>> configEntry : clusterConfigurations.entrySet()) { String configType = configEntry.getKey(); if (configEntry.getValue() != null) { for (Map.Entry<String, String> propertyEntry : configEntry.getValue().entrySet()) { String propertyName = propertyEntry.getKey(); String propertyValue = propertyEntry.getValue(); if (propertyValue != null) { if (SecretReference.isSecret(propertyValue)) { errorMessage.append(" Config:" + configType + " Property:" + propertyName+"\n"); containsSecretReferences = true; } } } } } if(containsSecretReferences) { throw new InvalidTopologyException("Secret references are not allowed in blueprints, " + "replace following properties with real passwords:\n"+errorMessage.toString()); } } for (HostGroup hostGroup : blueprint.getHostGroups().values()) { Collection<String> processedServices = new HashSet<String>(); Map<String, Collection<String>> allRequiredProperties = new HashMap<String, Collection<String>>(); Map<String, Map<String, String>> operationalConfiguration = new HashMap<String, Map<String, String>>(clusterConfigurations); operationalConfiguration.putAll(hostGroup.getConfiguration().getProperties()); for (String component : hostGroup.getComponentNames()) { if (component.equals("MYSQL_SERVER")) { Map<String, String> hiveEnvConfig = clusterConfigurations.get("hive-env"); if (hiveEnvConfig != null && !hiveEnvConfig.isEmpty() && hiveEnvConfig.get("hive_database") != null && hiveEnvConfig.get("hive_database").startsWith("Existing")) { throw new InvalidTopologyException("Incorrect configuration: MYSQL_SERVER component is available but hive" + " using existing db!"); } } if (ClusterTopologyImpl.isNameNodeHAEnabled(clusterConfigurations) && component.equals("NAMENODE")) { Map<String, String> hadoopEnvConfig = clusterConfigurations.get("hadoop-env"); if(hadoopEnvConfig != null && !hadoopEnvConfig.isEmpty() && hadoopEnvConfig.containsKey("dfs_ha_initial_namenode_active") && hadoopEnvConfig.containsKey("dfs_ha_initial_namenode_standby")) { ArrayList<HostGroup> hostGroupsForComponent = new ArrayList<HostGroup>( blueprint.getHostGroupsForComponent(component)); Set<String> givenHostGroups = new HashSet<String>(); givenHostGroups.add(hadoopEnvConfig.get("dfs_ha_initial_namenode_active")); givenHostGroups.add(hadoopEnvConfig.get("dfs_ha_initial_namenode_standby")); if(givenHostGroups.size() != hostGroupsForComponent.size()) { throw new IllegalArgumentException("NAMENODE HA host groups mapped incorrectly for properties 'dfs_ha_initial_namenode_active' and 'dfs_ha_initial_namenode_standby'. Expected Host groups are :" + hostGroupsForComponent); } if(HostGroup.HOSTGROUP_REGEX.matcher(hadoopEnvConfig.get("dfs_ha_initial_namenode_active")).matches() && HostGroup.HOSTGROUP_REGEX.matcher(hadoopEnvConfig.get("dfs_ha_initial_namenode_standby")).matches()){ for (HostGroup hostGroupForComponent : hostGroupsForComponent) { Iterator<String> itr = givenHostGroups.iterator(); while(itr.hasNext()){ if(itr.next().contains(hostGroupForComponent.getName())){ itr.remove(); } } } if(!givenHostGroups.isEmpty()){ throw new IllegalArgumentException("NAMENODE HA host groups mapped incorrectly for properties 'dfs_ha_initial_namenode_active' and 'dfs_ha_initial_namenode_standby'. Expected Host groups are :" + hostGroupsForComponent); } } } } if (component.equals("HIVE_METASTORE")) { Map<String, String> hiveEnvConfig = clusterConfigurations.get("hive-env"); if (hiveEnvConfig != null && !hiveEnvConfig.isEmpty() && hiveEnvConfig.get("hive_database") !=null && hiveEnvConfig.get("hive_database").equals("Existing SQL Anywhere Database") && VersionUtils.compareVersions(stack.getVersion(), "2.3.0.0") < 0 && stack.getName().equalsIgnoreCase("HDP")) { throw new InvalidTopologyException("Incorrect configuration: SQL Anywhere db is available only for stack HDP-2.3+ " + "and repo version 2.3.2+!"); } } if (component.equals("OOZIE_SERVER")) { Map<String, String> oozieEnvConfig = clusterConfigurations.get("oozie-env"); if (oozieEnvConfig != null && !oozieEnvConfig.isEmpty() && oozieEnvConfig.get("oozie_database") !=null && oozieEnvConfig.get("oozie_database").equals("Existing SQL Anywhere Database") && VersionUtils.compareVersions(stack.getVersion(), "2.3.0.0") < 0 && stack.getName().equalsIgnoreCase("HDP")) { throw new InvalidTopologyException("Incorrect configuration: SQL Anywhere db is available only for stack HDP-2.3+ " + "and repo version 2.3.2+!"); } } if (! component.equals("AMBARI_SERVER")) { String serviceName = stack.getServiceForComponent(component); if (processedServices.add(serviceName)) { Collection<Stack.ConfigProperty> requiredServiceConfigs = stack.getRequiredConfigurationProperties(serviceName); for (Stack.ConfigProperty requiredConfig : requiredServiceConfigs) { String configCategory = requiredConfig.getType(); String propertyName = requiredConfig.getName(); if (! stack.isPasswordProperty(serviceName, configCategory, propertyName)) { Collection<String> typeRequirements = allRequiredProperties.get(configCategory); if (typeRequirements == null) { typeRequirements = new HashSet<String>(); allRequiredProperties.put(configCategory, typeRequirements); } typeRequirements.add(propertyName); } } } } } for (Map.Entry<String, Collection<String>> requiredTypeProperties : allRequiredProperties.entrySet()) { String requiredCategory = requiredTypeProperties.getKey(); Collection<String> requiredProperties = requiredTypeProperties.getValue(); Collection<String> operationalTypeProps = operationalConfiguration.containsKey(requiredCategory) ? operationalConfiguration.get(requiredCategory).keySet() : Collections.<String>emptyList(); requiredProperties.removeAll(operationalTypeProps); if (! requiredProperties.isEmpty()) { String hostGroupName = hostGroup.getName(); Map<String, Collection<String>> hostGroupMissingProps = missingProperties.get(hostGroupName); if (hostGroupMissingProps == null) { hostGroupMissingProps = new HashMap<String, Collection<String>>(); missingProperties.put(hostGroupName, hostGroupMissingProps); } hostGroupMissingProps.put(requiredCategory, requiredProperties); } } } if (! missingProperties.isEmpty()) { throw new InvalidTopologyException("Missing required properties. Specify a value for these " + "properties in the blueprint configuration. " + missingProperties); } }
BlueprintValidatorImpl implements BlueprintValidator { @Override public void validateRequiredProperties() throws InvalidTopologyException { Map<String, Map<String, Collection<String>>> missingProperties = new HashMap<String, Map<String, Collection<String>>>(); Map<String, Map<String, String>> clusterConfigurations = blueprint.getConfiguration().getProperties(); if(clusterConfigurations != null) { StringBuilder errorMessage = new StringBuilder(); boolean containsSecretReferences = false; for (Map.Entry<String, Map<String, String>> configEntry : clusterConfigurations.entrySet()) { String configType = configEntry.getKey(); if (configEntry.getValue() != null) { for (Map.Entry<String, String> propertyEntry : configEntry.getValue().entrySet()) { String propertyName = propertyEntry.getKey(); String propertyValue = propertyEntry.getValue(); if (propertyValue != null) { if (SecretReference.isSecret(propertyValue)) { errorMessage.append(" Config:" + configType + " Property:" + propertyName+"\n"); containsSecretReferences = true; } } } } } if(containsSecretReferences) { throw new InvalidTopologyException("Secret references are not allowed in blueprints, " + "replace following properties with real passwords:\n"+errorMessage.toString()); } } for (HostGroup hostGroup : blueprint.getHostGroups().values()) { Collection<String> processedServices = new HashSet<String>(); Map<String, Collection<String>> allRequiredProperties = new HashMap<String, Collection<String>>(); Map<String, Map<String, String>> operationalConfiguration = new HashMap<String, Map<String, String>>(clusterConfigurations); operationalConfiguration.putAll(hostGroup.getConfiguration().getProperties()); for (String component : hostGroup.getComponentNames()) { if (component.equals("MYSQL_SERVER")) { Map<String, String> hiveEnvConfig = clusterConfigurations.get("hive-env"); if (hiveEnvConfig != null && !hiveEnvConfig.isEmpty() && hiveEnvConfig.get("hive_database") != null && hiveEnvConfig.get("hive_database").startsWith("Existing")) { throw new InvalidTopologyException("Incorrect configuration: MYSQL_SERVER component is available but hive" + " using existing db!"); } } if (ClusterTopologyImpl.isNameNodeHAEnabled(clusterConfigurations) && component.equals("NAMENODE")) { Map<String, String> hadoopEnvConfig = clusterConfigurations.get("hadoop-env"); if(hadoopEnvConfig != null && !hadoopEnvConfig.isEmpty() && hadoopEnvConfig.containsKey("dfs_ha_initial_namenode_active") && hadoopEnvConfig.containsKey("dfs_ha_initial_namenode_standby")) { ArrayList<HostGroup> hostGroupsForComponent = new ArrayList<HostGroup>( blueprint.getHostGroupsForComponent(component)); Set<String> givenHostGroups = new HashSet<String>(); givenHostGroups.add(hadoopEnvConfig.get("dfs_ha_initial_namenode_active")); givenHostGroups.add(hadoopEnvConfig.get("dfs_ha_initial_namenode_standby")); if(givenHostGroups.size() != hostGroupsForComponent.size()) { throw new IllegalArgumentException("NAMENODE HA host groups mapped incorrectly for properties 'dfs_ha_initial_namenode_active' and 'dfs_ha_initial_namenode_standby'. Expected Host groups are :" + hostGroupsForComponent); } if(HostGroup.HOSTGROUP_REGEX.matcher(hadoopEnvConfig.get("dfs_ha_initial_namenode_active")).matches() && HostGroup.HOSTGROUP_REGEX.matcher(hadoopEnvConfig.get("dfs_ha_initial_namenode_standby")).matches()){ for (HostGroup hostGroupForComponent : hostGroupsForComponent) { Iterator<String> itr = givenHostGroups.iterator(); while(itr.hasNext()){ if(itr.next().contains(hostGroupForComponent.getName())){ itr.remove(); } } } if(!givenHostGroups.isEmpty()){ throw new IllegalArgumentException("NAMENODE HA host groups mapped incorrectly for properties 'dfs_ha_initial_namenode_active' and 'dfs_ha_initial_namenode_standby'. Expected Host groups are :" + hostGroupsForComponent); } } } } if (component.equals("HIVE_METASTORE")) { Map<String, String> hiveEnvConfig = clusterConfigurations.get("hive-env"); if (hiveEnvConfig != null && !hiveEnvConfig.isEmpty() && hiveEnvConfig.get("hive_database") !=null && hiveEnvConfig.get("hive_database").equals("Existing SQL Anywhere Database") && VersionUtils.compareVersions(stack.getVersion(), "2.3.0.0") < 0 && stack.getName().equalsIgnoreCase("HDP")) { throw new InvalidTopologyException("Incorrect configuration: SQL Anywhere db is available only for stack HDP-2.3+ " + "and repo version 2.3.2+!"); } } if (component.equals("OOZIE_SERVER")) { Map<String, String> oozieEnvConfig = clusterConfigurations.get("oozie-env"); if (oozieEnvConfig != null && !oozieEnvConfig.isEmpty() && oozieEnvConfig.get("oozie_database") !=null && oozieEnvConfig.get("oozie_database").equals("Existing SQL Anywhere Database") && VersionUtils.compareVersions(stack.getVersion(), "2.3.0.0") < 0 && stack.getName().equalsIgnoreCase("HDP")) { throw new InvalidTopologyException("Incorrect configuration: SQL Anywhere db is available only for stack HDP-2.3+ " + "and repo version 2.3.2+!"); } } if (! component.equals("AMBARI_SERVER")) { String serviceName = stack.getServiceForComponent(component); if (processedServices.add(serviceName)) { Collection<Stack.ConfigProperty> requiredServiceConfigs = stack.getRequiredConfigurationProperties(serviceName); for (Stack.ConfigProperty requiredConfig : requiredServiceConfigs) { String configCategory = requiredConfig.getType(); String propertyName = requiredConfig.getName(); if (! stack.isPasswordProperty(serviceName, configCategory, propertyName)) { Collection<String> typeRequirements = allRequiredProperties.get(configCategory); if (typeRequirements == null) { typeRequirements = new HashSet<String>(); allRequiredProperties.put(configCategory, typeRequirements); } typeRequirements.add(propertyName); } } } } } for (Map.Entry<String, Collection<String>> requiredTypeProperties : allRequiredProperties.entrySet()) { String requiredCategory = requiredTypeProperties.getKey(); Collection<String> requiredProperties = requiredTypeProperties.getValue(); Collection<String> operationalTypeProps = operationalConfiguration.containsKey(requiredCategory) ? operationalConfiguration.get(requiredCategory).keySet() : Collections.<String>emptyList(); requiredProperties.removeAll(operationalTypeProps); if (! requiredProperties.isEmpty()) { String hostGroupName = hostGroup.getName(); Map<String, Collection<String>> hostGroupMissingProps = missingProperties.get(hostGroupName); if (hostGroupMissingProps == null) { hostGroupMissingProps = new HashMap<String, Collection<String>>(); missingProperties.put(hostGroupName, hostGroupMissingProps); } hostGroupMissingProps.put(requiredCategory, requiredProperties); } } } if (! missingProperties.isEmpty()) { throw new InvalidTopologyException("Missing required properties. Specify a value for these " + "properties in the blueprint configuration. " + missingProperties); } } }
BlueprintValidatorImpl implements BlueprintValidator { @Override public void validateRequiredProperties() throws InvalidTopologyException { Map<String, Map<String, Collection<String>>> missingProperties = new HashMap<String, Map<String, Collection<String>>>(); Map<String, Map<String, String>> clusterConfigurations = blueprint.getConfiguration().getProperties(); if(clusterConfigurations != null) { StringBuilder errorMessage = new StringBuilder(); boolean containsSecretReferences = false; for (Map.Entry<String, Map<String, String>> configEntry : clusterConfigurations.entrySet()) { String configType = configEntry.getKey(); if (configEntry.getValue() != null) { for (Map.Entry<String, String> propertyEntry : configEntry.getValue().entrySet()) { String propertyName = propertyEntry.getKey(); String propertyValue = propertyEntry.getValue(); if (propertyValue != null) { if (SecretReference.isSecret(propertyValue)) { errorMessage.append(" Config:" + configType + " Property:" + propertyName+"\n"); containsSecretReferences = true; } } } } } if(containsSecretReferences) { throw new InvalidTopologyException("Secret references are not allowed in blueprints, " + "replace following properties with real passwords:\n"+errorMessage.toString()); } } for (HostGroup hostGroup : blueprint.getHostGroups().values()) { Collection<String> processedServices = new HashSet<String>(); Map<String, Collection<String>> allRequiredProperties = new HashMap<String, Collection<String>>(); Map<String, Map<String, String>> operationalConfiguration = new HashMap<String, Map<String, String>>(clusterConfigurations); operationalConfiguration.putAll(hostGroup.getConfiguration().getProperties()); for (String component : hostGroup.getComponentNames()) { if (component.equals("MYSQL_SERVER")) { Map<String, String> hiveEnvConfig = clusterConfigurations.get("hive-env"); if (hiveEnvConfig != null && !hiveEnvConfig.isEmpty() && hiveEnvConfig.get("hive_database") != null && hiveEnvConfig.get("hive_database").startsWith("Existing")) { throw new InvalidTopologyException("Incorrect configuration: MYSQL_SERVER component is available but hive" + " using existing db!"); } } if (ClusterTopologyImpl.isNameNodeHAEnabled(clusterConfigurations) && component.equals("NAMENODE")) { Map<String, String> hadoopEnvConfig = clusterConfigurations.get("hadoop-env"); if(hadoopEnvConfig != null && !hadoopEnvConfig.isEmpty() && hadoopEnvConfig.containsKey("dfs_ha_initial_namenode_active") && hadoopEnvConfig.containsKey("dfs_ha_initial_namenode_standby")) { ArrayList<HostGroup> hostGroupsForComponent = new ArrayList<HostGroup>( blueprint.getHostGroupsForComponent(component)); Set<String> givenHostGroups = new HashSet<String>(); givenHostGroups.add(hadoopEnvConfig.get("dfs_ha_initial_namenode_active")); givenHostGroups.add(hadoopEnvConfig.get("dfs_ha_initial_namenode_standby")); if(givenHostGroups.size() != hostGroupsForComponent.size()) { throw new IllegalArgumentException("NAMENODE HA host groups mapped incorrectly for properties 'dfs_ha_initial_namenode_active' and 'dfs_ha_initial_namenode_standby'. Expected Host groups are :" + hostGroupsForComponent); } if(HostGroup.HOSTGROUP_REGEX.matcher(hadoopEnvConfig.get("dfs_ha_initial_namenode_active")).matches() && HostGroup.HOSTGROUP_REGEX.matcher(hadoopEnvConfig.get("dfs_ha_initial_namenode_standby")).matches()){ for (HostGroup hostGroupForComponent : hostGroupsForComponent) { Iterator<String> itr = givenHostGroups.iterator(); while(itr.hasNext()){ if(itr.next().contains(hostGroupForComponent.getName())){ itr.remove(); } } } if(!givenHostGroups.isEmpty()){ throw new IllegalArgumentException("NAMENODE HA host groups mapped incorrectly for properties 'dfs_ha_initial_namenode_active' and 'dfs_ha_initial_namenode_standby'. Expected Host groups are :" + hostGroupsForComponent); } } } } if (component.equals("HIVE_METASTORE")) { Map<String, String> hiveEnvConfig = clusterConfigurations.get("hive-env"); if (hiveEnvConfig != null && !hiveEnvConfig.isEmpty() && hiveEnvConfig.get("hive_database") !=null && hiveEnvConfig.get("hive_database").equals("Existing SQL Anywhere Database") && VersionUtils.compareVersions(stack.getVersion(), "2.3.0.0") < 0 && stack.getName().equalsIgnoreCase("HDP")) { throw new InvalidTopologyException("Incorrect configuration: SQL Anywhere db is available only for stack HDP-2.3+ " + "and repo version 2.3.2+!"); } } if (component.equals("OOZIE_SERVER")) { Map<String, String> oozieEnvConfig = clusterConfigurations.get("oozie-env"); if (oozieEnvConfig != null && !oozieEnvConfig.isEmpty() && oozieEnvConfig.get("oozie_database") !=null && oozieEnvConfig.get("oozie_database").equals("Existing SQL Anywhere Database") && VersionUtils.compareVersions(stack.getVersion(), "2.3.0.0") < 0 && stack.getName().equalsIgnoreCase("HDP")) { throw new InvalidTopologyException("Incorrect configuration: SQL Anywhere db is available only for stack HDP-2.3+ " + "and repo version 2.3.2+!"); } } if (! component.equals("AMBARI_SERVER")) { String serviceName = stack.getServiceForComponent(component); if (processedServices.add(serviceName)) { Collection<Stack.ConfigProperty> requiredServiceConfigs = stack.getRequiredConfigurationProperties(serviceName); for (Stack.ConfigProperty requiredConfig : requiredServiceConfigs) { String configCategory = requiredConfig.getType(); String propertyName = requiredConfig.getName(); if (! stack.isPasswordProperty(serviceName, configCategory, propertyName)) { Collection<String> typeRequirements = allRequiredProperties.get(configCategory); if (typeRequirements == null) { typeRequirements = new HashSet<String>(); allRequiredProperties.put(configCategory, typeRequirements); } typeRequirements.add(propertyName); } } } } } for (Map.Entry<String, Collection<String>> requiredTypeProperties : allRequiredProperties.entrySet()) { String requiredCategory = requiredTypeProperties.getKey(); Collection<String> requiredProperties = requiredTypeProperties.getValue(); Collection<String> operationalTypeProps = operationalConfiguration.containsKey(requiredCategory) ? operationalConfiguration.get(requiredCategory).keySet() : Collections.<String>emptyList(); requiredProperties.removeAll(operationalTypeProps); if (! requiredProperties.isEmpty()) { String hostGroupName = hostGroup.getName(); Map<String, Collection<String>> hostGroupMissingProps = missingProperties.get(hostGroupName); if (hostGroupMissingProps == null) { hostGroupMissingProps = new HashMap<String, Collection<String>>(); missingProperties.put(hostGroupName, hostGroupMissingProps); } hostGroupMissingProps.put(requiredCategory, requiredProperties); } } } if (! missingProperties.isEmpty()) { throw new InvalidTopologyException("Missing required properties. Specify a value for these " + "properties in the blueprint configuration. " + missingProperties); } } BlueprintValidatorImpl(Blueprint blueprint); }
BlueprintValidatorImpl implements BlueprintValidator { @Override public void validateRequiredProperties() throws InvalidTopologyException { Map<String, Map<String, Collection<String>>> missingProperties = new HashMap<String, Map<String, Collection<String>>>(); Map<String, Map<String, String>> clusterConfigurations = blueprint.getConfiguration().getProperties(); if(clusterConfigurations != null) { StringBuilder errorMessage = new StringBuilder(); boolean containsSecretReferences = false; for (Map.Entry<String, Map<String, String>> configEntry : clusterConfigurations.entrySet()) { String configType = configEntry.getKey(); if (configEntry.getValue() != null) { for (Map.Entry<String, String> propertyEntry : configEntry.getValue().entrySet()) { String propertyName = propertyEntry.getKey(); String propertyValue = propertyEntry.getValue(); if (propertyValue != null) { if (SecretReference.isSecret(propertyValue)) { errorMessage.append(" Config:" + configType + " Property:" + propertyName+"\n"); containsSecretReferences = true; } } } } } if(containsSecretReferences) { throw new InvalidTopologyException("Secret references are not allowed in blueprints, " + "replace following properties with real passwords:\n"+errorMessage.toString()); } } for (HostGroup hostGroup : blueprint.getHostGroups().values()) { Collection<String> processedServices = new HashSet<String>(); Map<String, Collection<String>> allRequiredProperties = new HashMap<String, Collection<String>>(); Map<String, Map<String, String>> operationalConfiguration = new HashMap<String, Map<String, String>>(clusterConfigurations); operationalConfiguration.putAll(hostGroup.getConfiguration().getProperties()); for (String component : hostGroup.getComponentNames()) { if (component.equals("MYSQL_SERVER")) { Map<String, String> hiveEnvConfig = clusterConfigurations.get("hive-env"); if (hiveEnvConfig != null && !hiveEnvConfig.isEmpty() && hiveEnvConfig.get("hive_database") != null && hiveEnvConfig.get("hive_database").startsWith("Existing")) { throw new InvalidTopologyException("Incorrect configuration: MYSQL_SERVER component is available but hive" + " using existing db!"); } } if (ClusterTopologyImpl.isNameNodeHAEnabled(clusterConfigurations) && component.equals("NAMENODE")) { Map<String, String> hadoopEnvConfig = clusterConfigurations.get("hadoop-env"); if(hadoopEnvConfig != null && !hadoopEnvConfig.isEmpty() && hadoopEnvConfig.containsKey("dfs_ha_initial_namenode_active") && hadoopEnvConfig.containsKey("dfs_ha_initial_namenode_standby")) { ArrayList<HostGroup> hostGroupsForComponent = new ArrayList<HostGroup>( blueprint.getHostGroupsForComponent(component)); Set<String> givenHostGroups = new HashSet<String>(); givenHostGroups.add(hadoopEnvConfig.get("dfs_ha_initial_namenode_active")); givenHostGroups.add(hadoopEnvConfig.get("dfs_ha_initial_namenode_standby")); if(givenHostGroups.size() != hostGroupsForComponent.size()) { throw new IllegalArgumentException("NAMENODE HA host groups mapped incorrectly for properties 'dfs_ha_initial_namenode_active' and 'dfs_ha_initial_namenode_standby'. Expected Host groups are :" + hostGroupsForComponent); } if(HostGroup.HOSTGROUP_REGEX.matcher(hadoopEnvConfig.get("dfs_ha_initial_namenode_active")).matches() && HostGroup.HOSTGROUP_REGEX.matcher(hadoopEnvConfig.get("dfs_ha_initial_namenode_standby")).matches()){ for (HostGroup hostGroupForComponent : hostGroupsForComponent) { Iterator<String> itr = givenHostGroups.iterator(); while(itr.hasNext()){ if(itr.next().contains(hostGroupForComponent.getName())){ itr.remove(); } } } if(!givenHostGroups.isEmpty()){ throw new IllegalArgumentException("NAMENODE HA host groups mapped incorrectly for properties 'dfs_ha_initial_namenode_active' and 'dfs_ha_initial_namenode_standby'. Expected Host groups are :" + hostGroupsForComponent); } } } } if (component.equals("HIVE_METASTORE")) { Map<String, String> hiveEnvConfig = clusterConfigurations.get("hive-env"); if (hiveEnvConfig != null && !hiveEnvConfig.isEmpty() && hiveEnvConfig.get("hive_database") !=null && hiveEnvConfig.get("hive_database").equals("Existing SQL Anywhere Database") && VersionUtils.compareVersions(stack.getVersion(), "2.3.0.0") < 0 && stack.getName().equalsIgnoreCase("HDP")) { throw new InvalidTopologyException("Incorrect configuration: SQL Anywhere db is available only for stack HDP-2.3+ " + "and repo version 2.3.2+!"); } } if (component.equals("OOZIE_SERVER")) { Map<String, String> oozieEnvConfig = clusterConfigurations.get("oozie-env"); if (oozieEnvConfig != null && !oozieEnvConfig.isEmpty() && oozieEnvConfig.get("oozie_database") !=null && oozieEnvConfig.get("oozie_database").equals("Existing SQL Anywhere Database") && VersionUtils.compareVersions(stack.getVersion(), "2.3.0.0") < 0 && stack.getName().equalsIgnoreCase("HDP")) { throw new InvalidTopologyException("Incorrect configuration: SQL Anywhere db is available only for stack HDP-2.3+ " + "and repo version 2.3.2+!"); } } if (! component.equals("AMBARI_SERVER")) { String serviceName = stack.getServiceForComponent(component); if (processedServices.add(serviceName)) { Collection<Stack.ConfigProperty> requiredServiceConfigs = stack.getRequiredConfigurationProperties(serviceName); for (Stack.ConfigProperty requiredConfig : requiredServiceConfigs) { String configCategory = requiredConfig.getType(); String propertyName = requiredConfig.getName(); if (! stack.isPasswordProperty(serviceName, configCategory, propertyName)) { Collection<String> typeRequirements = allRequiredProperties.get(configCategory); if (typeRequirements == null) { typeRequirements = new HashSet<String>(); allRequiredProperties.put(configCategory, typeRequirements); } typeRequirements.add(propertyName); } } } } } for (Map.Entry<String, Collection<String>> requiredTypeProperties : allRequiredProperties.entrySet()) { String requiredCategory = requiredTypeProperties.getKey(); Collection<String> requiredProperties = requiredTypeProperties.getValue(); Collection<String> operationalTypeProps = operationalConfiguration.containsKey(requiredCategory) ? operationalConfiguration.get(requiredCategory).keySet() : Collections.<String>emptyList(); requiredProperties.removeAll(operationalTypeProps); if (! requiredProperties.isEmpty()) { String hostGroupName = hostGroup.getName(); Map<String, Collection<String>> hostGroupMissingProps = missingProperties.get(hostGroupName); if (hostGroupMissingProps == null) { hostGroupMissingProps = new HashMap<String, Collection<String>>(); missingProperties.put(hostGroupName, hostGroupMissingProps); } hostGroupMissingProps.put(requiredCategory, requiredProperties); } } } if (! missingProperties.isEmpty()) { throw new InvalidTopologyException("Missing required properties. Specify a value for these " + "properties in the blueprint configuration. " + missingProperties); } } BlueprintValidatorImpl(Blueprint blueprint); @Override void validateTopology(); @Override void validateRequiredProperties(); Collection<String> verifyComponentCardinalityCount(String component,
                                                            Cardinality cardinality,
                                                            AutoDeployInfo autoDeploy); }
BlueprintValidatorImpl implements BlueprintValidator { @Override public void validateRequiredProperties() throws InvalidTopologyException { Map<String, Map<String, Collection<String>>> missingProperties = new HashMap<String, Map<String, Collection<String>>>(); Map<String, Map<String, String>> clusterConfigurations = blueprint.getConfiguration().getProperties(); if(clusterConfigurations != null) { StringBuilder errorMessage = new StringBuilder(); boolean containsSecretReferences = false; for (Map.Entry<String, Map<String, String>> configEntry : clusterConfigurations.entrySet()) { String configType = configEntry.getKey(); if (configEntry.getValue() != null) { for (Map.Entry<String, String> propertyEntry : configEntry.getValue().entrySet()) { String propertyName = propertyEntry.getKey(); String propertyValue = propertyEntry.getValue(); if (propertyValue != null) { if (SecretReference.isSecret(propertyValue)) { errorMessage.append(" Config:" + configType + " Property:" + propertyName+"\n"); containsSecretReferences = true; } } } } } if(containsSecretReferences) { throw new InvalidTopologyException("Secret references are not allowed in blueprints, " + "replace following properties with real passwords:\n"+errorMessage.toString()); } } for (HostGroup hostGroup : blueprint.getHostGroups().values()) { Collection<String> processedServices = new HashSet<String>(); Map<String, Collection<String>> allRequiredProperties = new HashMap<String, Collection<String>>(); Map<String, Map<String, String>> operationalConfiguration = new HashMap<String, Map<String, String>>(clusterConfigurations); operationalConfiguration.putAll(hostGroup.getConfiguration().getProperties()); for (String component : hostGroup.getComponentNames()) { if (component.equals("MYSQL_SERVER")) { Map<String, String> hiveEnvConfig = clusterConfigurations.get("hive-env"); if (hiveEnvConfig != null && !hiveEnvConfig.isEmpty() && hiveEnvConfig.get("hive_database") != null && hiveEnvConfig.get("hive_database").startsWith("Existing")) { throw new InvalidTopologyException("Incorrect configuration: MYSQL_SERVER component is available but hive" + " using existing db!"); } } if (ClusterTopologyImpl.isNameNodeHAEnabled(clusterConfigurations) && component.equals("NAMENODE")) { Map<String, String> hadoopEnvConfig = clusterConfigurations.get("hadoop-env"); if(hadoopEnvConfig != null && !hadoopEnvConfig.isEmpty() && hadoopEnvConfig.containsKey("dfs_ha_initial_namenode_active") && hadoopEnvConfig.containsKey("dfs_ha_initial_namenode_standby")) { ArrayList<HostGroup> hostGroupsForComponent = new ArrayList<HostGroup>( blueprint.getHostGroupsForComponent(component)); Set<String> givenHostGroups = new HashSet<String>(); givenHostGroups.add(hadoopEnvConfig.get("dfs_ha_initial_namenode_active")); givenHostGroups.add(hadoopEnvConfig.get("dfs_ha_initial_namenode_standby")); if(givenHostGroups.size() != hostGroupsForComponent.size()) { throw new IllegalArgumentException("NAMENODE HA host groups mapped incorrectly for properties 'dfs_ha_initial_namenode_active' and 'dfs_ha_initial_namenode_standby'. Expected Host groups are :" + hostGroupsForComponent); } if(HostGroup.HOSTGROUP_REGEX.matcher(hadoopEnvConfig.get("dfs_ha_initial_namenode_active")).matches() && HostGroup.HOSTGROUP_REGEX.matcher(hadoopEnvConfig.get("dfs_ha_initial_namenode_standby")).matches()){ for (HostGroup hostGroupForComponent : hostGroupsForComponent) { Iterator<String> itr = givenHostGroups.iterator(); while(itr.hasNext()){ if(itr.next().contains(hostGroupForComponent.getName())){ itr.remove(); } } } if(!givenHostGroups.isEmpty()){ throw new IllegalArgumentException("NAMENODE HA host groups mapped incorrectly for properties 'dfs_ha_initial_namenode_active' and 'dfs_ha_initial_namenode_standby'. Expected Host groups are :" + hostGroupsForComponent); } } } } if (component.equals("HIVE_METASTORE")) { Map<String, String> hiveEnvConfig = clusterConfigurations.get("hive-env"); if (hiveEnvConfig != null && !hiveEnvConfig.isEmpty() && hiveEnvConfig.get("hive_database") !=null && hiveEnvConfig.get("hive_database").equals("Existing SQL Anywhere Database") && VersionUtils.compareVersions(stack.getVersion(), "2.3.0.0") < 0 && stack.getName().equalsIgnoreCase("HDP")) { throw new InvalidTopologyException("Incorrect configuration: SQL Anywhere db is available only for stack HDP-2.3+ " + "and repo version 2.3.2+!"); } } if (component.equals("OOZIE_SERVER")) { Map<String, String> oozieEnvConfig = clusterConfigurations.get("oozie-env"); if (oozieEnvConfig != null && !oozieEnvConfig.isEmpty() && oozieEnvConfig.get("oozie_database") !=null && oozieEnvConfig.get("oozie_database").equals("Existing SQL Anywhere Database") && VersionUtils.compareVersions(stack.getVersion(), "2.3.0.0") < 0 && stack.getName().equalsIgnoreCase("HDP")) { throw new InvalidTopologyException("Incorrect configuration: SQL Anywhere db is available only for stack HDP-2.3+ " + "and repo version 2.3.2+!"); } } if (! component.equals("AMBARI_SERVER")) { String serviceName = stack.getServiceForComponent(component); if (processedServices.add(serviceName)) { Collection<Stack.ConfigProperty> requiredServiceConfigs = stack.getRequiredConfigurationProperties(serviceName); for (Stack.ConfigProperty requiredConfig : requiredServiceConfigs) { String configCategory = requiredConfig.getType(); String propertyName = requiredConfig.getName(); if (! stack.isPasswordProperty(serviceName, configCategory, propertyName)) { Collection<String> typeRequirements = allRequiredProperties.get(configCategory); if (typeRequirements == null) { typeRequirements = new HashSet<String>(); allRequiredProperties.put(configCategory, typeRequirements); } typeRequirements.add(propertyName); } } } } } for (Map.Entry<String, Collection<String>> requiredTypeProperties : allRequiredProperties.entrySet()) { String requiredCategory = requiredTypeProperties.getKey(); Collection<String> requiredProperties = requiredTypeProperties.getValue(); Collection<String> operationalTypeProps = operationalConfiguration.containsKey(requiredCategory) ? operationalConfiguration.get(requiredCategory).keySet() : Collections.<String>emptyList(); requiredProperties.removeAll(operationalTypeProps); if (! requiredProperties.isEmpty()) { String hostGroupName = hostGroup.getName(); Map<String, Collection<String>> hostGroupMissingProps = missingProperties.get(hostGroupName); if (hostGroupMissingProps == null) { hostGroupMissingProps = new HashMap<String, Collection<String>>(); missingProperties.put(hostGroupName, hostGroupMissingProps); } hostGroupMissingProps.put(requiredCategory, requiredProperties); } } } if (! missingProperties.isEmpty()) { throw new InvalidTopologyException("Missing required properties. Specify a value for these " + "properties in the blueprint configuration. " + missingProperties); } } BlueprintValidatorImpl(Blueprint blueprint); @Override void validateTopology(); @Override void validateRequiredProperties(); Collection<String> verifyComponentCardinalityCount(String component,
                                                            Cardinality cardinality,
                                                            AutoDeployInfo autoDeploy);  }