CmClusterValidator { public boolean validateBlueprint(ClusterBlueprint blueprint) { logger.info("Start to validate blueprint for cluster " + blueprint.getName()); String distro = blueprint.getHadoopStack().getDistro(); String distroVersion = CmUtils.distroVersionOfHadoopStack(blueprint.getHadoopStack()); try { List<String> unRecogConfigTypes = new ArrayList<String>(); List<String> unRecogConfigKeys = new ArrayList<>(); validateConfigs(blueprint.getConfiguration(), unRecogConfigTypes, unRecogConfigKeys, distroVersion); Set<String> availableRoles = AvailableServiceRoleContainer.allRoles(distroVersion); Set<String> definedServices = new HashSet<String>(); Map<String, Integer> definedRoles = new HashMap<String, Integer>(); List<String> unRecogRoles = null; Set<String> invalidRacks = null; if (blueprint.getNodeGroups() == null || blueprint.getNodeGroups().isEmpty()) { return false; } int nnGroupsNum = 0; for (NodeGroupInfo group : blueprint.getNodeGroups()) { validateConfigs(group.getConfiguration(), unRecogConfigTypes, unRecogConfigKeys, distroVersion); if (group.getRoles().contains("HDFS_NAMENODE")) { nnGroupsNum++; } for (String roleName: group.getRoles()) { if (!availableRoles.contains(roleName)) { if (unRecogRoles == null) { unRecogRoles = new ArrayList<String>(); } unRecogRoles.add(roleName); } else { if (!definedRoles.containsKey(roleName)) { definedRoles.put(roleName, group.getInstanceNum()); } else { Integer instanceNum = definedRoles.get(roleName) + group.getInstanceNum(); definedRoles.put(roleName, instanceNum); } definedServices.add(AvailableServiceRoleContainer.load(roleName).getParent().getDisplayName()); } } } if (nnGroupsNum > 1) { errorMsgList.add("Namenode federation is not supported currently"); } if (unRecogRoles != null && !unRecogRoles.isEmpty()) { errorMsgList.add("Roles " + unRecogRoles.toString() + " are not available by distro " + distro); } if (!unRecogConfigTypes.isEmpty()) { warningMsgList.add("Configurations for " + unRecogConfigTypes.toString() + " are not available by distro " + distro); } if (!unRecogConfigKeys.isEmpty()) { errorMsgList.add("Configuration items " + unRecogConfigKeys.toString() + " are invalid"); } if (invalidRacks != null && !invalidRacks.isEmpty()) { errorMsgList.add("Racks " + invalidRacks.toString() + " are invalid," + " rack names must be slash-separated, like Unix paths. For example, \"/rack1\" and \"/cabinet3/rack4\""); } for (String serviceName : definedServices) { for (AvailableServiceRole.Dependency dependency : AvailableServiceRoleContainer.load(serviceName).getDependencies()) { if (!dependency.isRequired()) { continue; } if (dependency.getServices().size() == 1 && !definedServices.contains(dependency.getServices().get(0))) { if (serviceName.equals("YARN") && isComputeOnly(definedServices)) { continue; } warningMsgList.add(serviceName + " depends on " + dependency.getServices().get(0) + " service"); } else { boolean found = false; for (String dependService : dependency.getServices()) { if (definedServices.contains(dependService)) { found = true; } } if (!found) { warningMsgList.add(serviceName + " depends on one service of " + dependency.getServices().toString()); } } } Set<String> requiredRoles = new HashSet<String>(); switch (serviceName) { case "HDFS": if (!isComputeOnly(definedServices)) { requiredRoles.add("HDFS_NAMENODE"); requiredRoles.add("HDFS_DATANODE"); } if (checkRequiredRoles(serviceName, requiredRoles, definedRoles.keySet(), errorMsgList)) { if (nnGroupsNum == 1) { if (definedRoles.get("HDFS_NAMENODE") < 2 && !definedRoles.containsKey("HDFS_SECONDARY_NAMENODE")) { errorMsgList.add("HDFS service not configured for High Availability must have a SecondaryNameNode"); } if (definedRoles.get("HDFS_NAMENODE") >= 2 && !definedRoles.containsKey("HDFS_JOURNALNODE")) { errorMsgList.add("HDFS service configured for High Availability must have journal nodes"); } } } if (definedRoles.containsKey("HDFS_JOURNALNODE")) { if (definedRoles.get("HDFS_JOURNALNODE") > 1 && definedRoles.get("HDFS_JOURNALNODE") < 3) { errorMsgList.add(Constants.WRONG_NUM_OF_JOURNALNODE); } else if (definedRoles.get("HDFS_JOURNALNODE") % 2 == 0) { warningMsgList.add(Constants.ODD_NUM_OF_JOURNALNODE); } } break; case "YARN": requiredRoles.add("YARN_RESOURCE_MANAGER"); requiredRoles.add("YARN_NODE_MANAGER"); requiredRoles.add("YARN_JOB_HISTORY"); if (checkRequiredRoles(serviceName, requiredRoles, definedRoles.keySet(), errorMsgList)) { if (definedRoles.get("YARN_RESOURCE_MANAGER") > 1) { errorMsgList.add(Constants.WRONG_NUM_OF_RESOURCEMANAGER); } } break; case "MAPREDUCE": requiredRoles.add("MAPREDUCE_JOBTRACKER"); requiredRoles.add("MAPREDUCE_TASKTRACKER"); if (checkRequiredRoles(serviceName, requiredRoles, definedRoles.keySet(), errorMsgList)) { if (definedRoles.get("MAPREDUCE_JOBTRACKER") > 1) { errorMsgList.add(Constants.WRONG_NUM_OF_JOBTRACKER); } } break; case "HBASE": requiredRoles.add("HBASE_MASTER"); requiredRoles.add("HBASE_REGION_SERVER"); checkRequiredRoles(serviceName, requiredRoles, definedRoles.keySet(), errorMsgList); break; case "ZOOKEEPER": requiredRoles.add("ZOOKEEPER_SERVER"); if (checkRequiredRoles(serviceName, requiredRoles, definedRoles.keySet(), errorMsgList)) { if (definedRoles.get("ZOOKEEPER_SERVER") > 0 && definedRoles.get("ZOOKEEPER_SERVER") < 3) { errorMsgList.add(Constants.WRONG_NUM_OF_ZOOKEEPER); } else if (definedRoles.get("ZOOKEEPER_SERVER") % 2 == 0) { warningMsgList.add(Constants.ODD_NUM_OF_ZOOKEEPER); } } break; case "HIVE": requiredRoles.add("HIVE_METASTORE"); requiredRoles.add("HIVE_SERVER2"); checkRequiredRoles(serviceName, requiredRoles, definedRoles.keySet(), errorMsgList); String[] requiredConfigs = { "hive_metastore_database_host", "hive_metastore_database_name", "hive_metastore_database_password", "hive_metastore_database_port", "hive_metastore_database_type", "hive_metastore_database_user" }; boolean configured = true; if (blueprint.getConfiguration().containsKey("HIVE")) { Map<String, String> configuredItems = (Map<String, String>) blueprint.getConfiguration().get("HIVE"); for (String item : requiredConfigs) { if (!configuredItems.containsKey(item)) { configured = false; break; } } } else { configured = false; } if (!configured) { errorMsgList.add("HIVE service depends on an external database, please setup one and provide configuration properties [" + StringUtils.join(requiredConfigs, ",") + "] for HIVE service"); } break; case "OOZIE": if (definedRoles.get("OOZIE_SERVER") > 1) { errorMsgList.add("only one OOZIE_SERVER is allowed for OOZIE service"); } break; case "SENTRY": if (definedRoles.get("SENTRY_SERVER") > 1) { errorMsgList.add("only one SENTRY_SERVER is allowed for SENTRY service"); } break; case "SQOOP": if (definedRoles.get("SQOOP_SERVER") > 1) { errorMsgList.add("only one SQOOP_SERVER is allowed for SQOOP service"); } break; case "ISILON": requiredRoles.add("YARN_RESOURCE_MANAGER"); requiredRoles.add("YARN_JOB_HISTORY"); requiredRoles.add("YARN_NODE_MANAGER"); requiredRoles.add("GATEWAY"); break; default: break; } } } catch (IOException e) { } if (!warningMsgList.isEmpty() || !errorMsgList.isEmpty()) { throw ValidationException.VALIDATION_FAIL("Blueprint", errorMsgList, warningMsgList); } return true; } CmClusterValidator(); boolean validateBlueprint(ClusterBlueprint blueprint); @Override String toString();  }
@Test(groups = {"TestCmClusterValidator"}) public void testSuccess() { try { ClusterBlueprint blueprint = generateBlueprint(); CmClusterValidator validator = new CmClusterValidator(); Assert.assertTrue(validator.validateBlueprint(blueprint)); } catch (ValidationException e) { System.out.println("warning_msg_list: " + e.getWarningMsgList()); System.out.println("error_msg_list: " + e.getFailedMsgList()); } }
@Test(groups = {"TestCmClusterValidator"}) public void testUnrecogRolesOfDistro() { try { ClusterBlueprint blueprint = generateBlueprint(); blueprint.getHadoopStack().setDistro("CDH-4.0.0"); CmClusterValidator validator = new CmClusterValidator(); Assert.assertFalse(validator.validateBlueprint(blueprint)); } catch (ValidationException e) { System.out.println("warning_msg_list: " + e.getWarningMsgList()); System.out.println("error_msg_list: " + e.getFailedMsgList()); } }
@Test(groups = {"TestCmClusterValidator"}) public void testUnrecogRoles() { try { ClusterBlueprint blueprint = generateBlueprint(); blueprint.getNodeGroups().get(0).getRoles().add("FAKE_ROLE01"); blueprint.getNodeGroups().get(1).getRoles().add("FAKE_ROLE02"); CmClusterValidator validator = new CmClusterValidator(); Assert.assertFalse(validator.validateBlueprint(blueprint)); } catch (ValidationException e) { System.out.println("warning_msg_list: " + e.getWarningMsgList()); System.out.println("error_msg_list: " + e.getFailedMsgList()); } }
@Test(groups = {"TestCmClusterValidator"}) public void testUnrecogConfigTypes() { try { ClusterBlueprint blueprint = generateBlueprint(); blueprint.getConfiguration().put("FAKE_CONFIG", new HashMap<String, String>()); CmClusterValidator validator = new CmClusterValidator(); Assert.assertTrue(validator.validateBlueprint(blueprint)); } catch (ValidationException e) { System.out.println("warning_msg_list: " + e.getWarningMsgList()); System.out.println("error_msg_list: " + e.getFailedMsgList()); } }
@Test(groups = {"TestCmClusterValidator"}) public void testBadConfigItems() { try { ClusterBlueprint blueprint = generateBlueprint(); Map<String, String> hdfsConfig = (Map<String, String>) blueprint.getConfiguration().get("HDFS"); hdfsConfig.put("fake_config_01", "value"); Map<String, String> nnConfig = (Map<String, String>) blueprint.getNodeGroups().get(0).getConfiguration().get("HDFS_NAMENODE"); nnConfig.put("fake_config_02", "value"); CmClusterValidator validator = new CmClusterValidator(); Assert.assertFalse(validator.validateBlueprint(blueprint)); } catch (ValidationException e) { System.out.println("warning_msg_list: " + e.getWarningMsgList()); System.out.println("error_msg_list: " + e.getFailedMsgList()); } }
@Test(groups = {"TestCmClusterValidator"}) public void testMissedRoles() { try { ClusterBlueprint blueprint = generateBlueprint(); blueprint.getNodeGroups().get(1).getRoles().remove("HDFS_DATANODE"); blueprint.getNodeGroups().get(0).getRoles().remove("YARN_RESOURCE_MANAGER"); CmClusterValidator validator = new CmClusterValidator(); Assert.assertFalse(validator.validateBlueprint(blueprint)); } catch (ValidationException e) { System.out.println("warning_msg_list: " + e.getWarningMsgList()); System.out.println("error_msg_list: " + e.getFailedMsgList()); } }
@Test(groups = {"TestCmClusterValidator"}) public void testNnHA() { try { ClusterBlueprint blueprint = generateBlueprint(); blueprint.getNodeGroups().get(0).getRoles().remove("HDFS_SECONDARY_NAMENODE"); CmClusterValidator validator = new CmClusterValidator(); Assert.assertFalse(validator.validateBlueprint(blueprint)); } catch (ValidationException e) { System.out.println("warning_msg_list: " + e.getWarningMsgList()); System.out.println("error_msg_list: " + e.getFailedMsgList()); } }
@Test(groups = {"TestCmClusterValidator"}) public void testNnFederation() { try { ClusterBlueprint blueprint = generateBlueprint(); blueprint.getNodeGroups().get(1).getRoles().add("HDFS_NAMENODE"); CmClusterValidator validator = new CmClusterValidator(); Assert.assertFalse(validator.validateBlueprint(blueprint)); } catch (ValidationException e) { System.out.println("warning_msg_list: " + e.getWarningMsgList()); System.out.println("error_msg_list: " + e.getFailedMsgList()); } }
@Test(groups = {"TestCmClusterValidator"}) public void testServiceDependency01() { try { ClusterBlueprint blueprint = generateBlueprint(); blueprint.getNodeGroups().get(0).getRoles().remove("HDFS_NAMENODE"); blueprint.getNodeGroups().get(0).getRoles().remove("HDFS_SECONDARY_NAMENODE"); blueprint.getNodeGroups().get(1).getRoles().remove("HDFS_DATANODE"); CmClusterValidator validator = new CmClusterValidator(); Assert.assertFalse(validator.validateBlueprint(blueprint)); } catch (ValidationException e) { System.out.println("warning_msg_list: " + e.getWarningMsgList()); System.out.println("error_msg_list: " + e.getFailedMsgList()); } }
@Test(groups = {"TestCmClusterValidator"}) public void testServiceDependency02() { try { ClusterBlueprint blueprint = generateBlueprint(); blueprint.getNodeGroups().get(0).getRoles().add("HIVE_METASTORE"); blueprint.getNodeGroups().get(0).getRoles().remove("YARN_RESOURCE_MANAGER"); blueprint.getNodeGroups().get(0).getRoles().remove("YARN_JOB_HISTORY"); blueprint.getNodeGroups().get(1).getRoles().add("HIVE_SERVER2"); blueprint.getNodeGroups().get(1).getRoles().remove("YARN_NODE_MANAGER"); CmClusterValidator validator = new CmClusterValidator(); Assert.assertFalse(validator.validateBlueprint(blueprint)); } catch (ValidationException e) { System.out.println("warning_msg_list: " + e.getWarningMsgList()); System.out.println("error_msg_list: " + e.getFailedMsgList()); } }
@Test(groups = {"TestCmClusterValidator"}) public void testBadInstanceNum01() { try { ClusterBlueprint blueprint = generateBlueprint(); blueprint.getNodeGroups().get(0).getRoles().add("ZOOKEEPER_SERVER"); blueprint.getNodeGroups().get(0).getRoles().add("HDFS_JOURNALNODE"); CmClusterValidator validator = new CmClusterValidator(); Assert.assertFalse(validator.validateBlueprint(blueprint)); } catch (ValidationException e) { System.out.println("warning_msg_list: " + e.getWarningMsgList()); System.out.println("error_msg_list: " + e.getFailedMsgList()); } }
@Test(groups = {"TestCmClusterValidator"}) public void testBadInstanceNum02() { try { ClusterBlueprint blueprint = generateBlueprint(); blueprint.getNodeGroups().get(0).getRoles().add("ZOOKEEPER_SERVER"); blueprint.getNodeGroups().get(1).getRoles().add("ZOOKEEPER_SERVER"); blueprint.getNodeGroups().get(0).getRoles().add("HDFS_JOURNALNODE"); blueprint.getNodeGroups().get(1).getRoles().add("HDFS_JOURNALNODE"); blueprint.getNodeGroups().get(1).getRoles().add("YARN_RESOURCE_MANAGER"); CmClusterValidator validator = new CmClusterValidator(); Assert.assertFalse(validator.validateBlueprint(blueprint)); } catch (ValidationException e) { System.out.println("warning_msg_list: " + e.getWarningMsgList()); System.out.println("error_msg_list: " + e.getFailedMsgList()); } }
@Test(groups = {"TestCmClusterValidator"}) public void testComputeOnly() { ClusterBlueprint blueprint = generateBlueprint(); CmClusterValidator validator = new CmClusterValidator(); Assert.assertTrue(validator.validateBlueprint(blueprint)); }