UpgradeHelper { public List<UpgradeGroupHolder> createSequence(UpgradePack upgradePack, UpgradeContext context) throws AmbariException { context.setAmbariMetaInfo(m_ambariMetaInfo.get()); Cluster cluster = context.getCluster(); MasterHostResolver mhr = context.getResolver(); Map<String, Map<String, ProcessingComponent>> allTasks = upgradePack.getTasks(); List<UpgradeGroupHolder> groups = new ArrayList<>(); for (Grouping group : upgradePack.getGroups(context.getDirection())) { if (!context.isScoped(group.scope)) { continue; } if (null != group.condition && !group.condition.isSatisfied(context)) { LOG.info("Skipping {} while building upgrade orchestration due to {}", group, group.condition ); continue; } UpgradeGroupHolder groupHolder = new UpgradeGroupHolder(); groupHolder.name = group.name; groupHolder.title = group.title; groupHolder.groupClass = group.getClass(); groupHolder.skippable = group.skippable; groupHolder.supportsAutoSkipOnFailure = group.supportsAutoSkipOnFailure; groupHolder.allowRetry = group.allowRetry; if (context.getDirection().isDowngrade()) { groupHolder.skippable = true; } Task.Type functionName = null; if (group instanceof UpgradeFunction) { functionName = ((UpgradeFunction) group).getFunction(); } if (upgradePack.getType() == UpgradeType.NON_ROLLING) { group.performServiceCheck = false; } StageWrapperBuilder builder = group.getBuilder(); List<UpgradePack.OrderService> services = group.services; if (upgradePack.getType() == UpgradeType.ROLLING) { if (context.getDirection().isDowngrade() && !services.isEmpty()) { List<UpgradePack.OrderService> reverse = new ArrayList<>(services); Collections.reverse(reverse); services = reverse; } } for (UpgradePack.OrderService service : services) { if (!context.isServiceSupported(service.serviceName)) { continue; } if (upgradePack.getType() == UpgradeType.ROLLING && !allTasks.containsKey(service.serviceName)) { continue; } for (String component : service.components) { if (upgradePack.getType() == UpgradeType.ROLLING && !allTasks.get(service.serviceName).containsKey(component)) { continue; } HostsType hostsType = mhr.getMasterAndHosts(service.serviceName, component); if (null == hostsType) { continue; } if (!hostsType.unhealthy.isEmpty()) { context.addUnhealthy(hostsType.unhealthy); } Service svc = cluster.getService(service.serviceName); ProcessingComponent pc = null; if (null == functionName) { pc = allTasks.get(service.serviceName).get(component); } else { if (functionName == Type.STOP) { pc = new ProcessingComponent(); pc.name = component; pc.tasks = new ArrayList<>(); pc.tasks.add(new StopTask()); } else { if (allTasks.containsKey(service.serviceName) && allTasks.get(service.serviceName).containsKey(component)) { pc = allTasks.get(service.serviceName).get(component); } else { pc = new ProcessingComponent(); pc.name = component; pc.tasks = new ArrayList<>(); if (functionName == Type.START) { pc.tasks.add(new StartTask()); } if (functionName == Type.RESTART) { pc.tasks.add(new RestartTask()); } } } } if (pc == null) { LOG.error(MessageFormat.format("Couldn't create a processing component for service {0} and component {1}.", service.serviceName, component)); continue; } setDisplayNames(context, service.serviceName, component); if (service.serviceName.equalsIgnoreCase("HDFS") && component.equalsIgnoreCase("NAMENODE")) { switch (upgradePack.getType()) { case ROLLING: if (!hostsType.hosts.isEmpty() && hostsType.master != null && hostsType.secondary != null) { LinkedHashSet<String> order = new LinkedHashSet<String>(); order.add(hostsType.secondary); order.add(hostsType.master); hostsType.hosts = order; builder.add(context, hostsType, service.serviceName, svc.isClientOnlyService(), pc, null); } else { LOG.warn("Could not orchestrate NameNode. Hosts could not be resolved: hosts={}, active={}, standby={}", StringUtils.join(hostsType.hosts, ','), hostsType.master, hostsType.secondary); } break; case NON_ROLLING: boolean isNameNodeHA = mhr.isNameNodeHA(); if (isNameNodeHA && hostsType.master != null && hostsType.secondary != null) { HostsType ht1 = new HostsType(); LinkedHashSet<String> h1Hosts = new LinkedHashSet<String>(); h1Hosts.add(hostsType.master); ht1.hosts = h1Hosts; Map<String, String> h1Params = new HashMap<String, String>(); h1Params.put("desired_namenode_role", "active"); HostsType ht2 = new HostsType(); LinkedHashSet<String> h2Hosts = new LinkedHashSet<String>(); h2Hosts.add(hostsType.secondary); ht2.hosts = h2Hosts; Map<String, String> h2Params = new HashMap<String, String>(); h2Params.put("desired_namenode_role", "standby"); builder.add(context, ht1, service.serviceName, svc.isClientOnlyService(), pc, h1Params); builder.add(context, ht2, service.serviceName, svc.isClientOnlyService(), pc, h2Params); } else { builder.add(context, hostsType, service.serviceName, svc.isClientOnlyService(), pc, null); } break; } } else { builder.add(context, hostsType, service.serviceName, svc.isClientOnlyService(), pc, null); } } } List<StageWrapper> proxies = builder.build(context); if (CollectionUtils.isNotEmpty(proxies)) { groupHolder.items = proxies; postProcess(context, groupHolder); groups.add(groupHolder); } } if (LOG.isDebugEnabled()) { for (UpgradeGroupHolder group : groups) { LOG.debug(group.name); int i = 0; for (StageWrapper proxy : group.items) { LOG.debug(" Stage {}", Integer.valueOf(i++)); int j = 0; for (TaskWrapper task : proxy.getTasks()) { LOG.debug(" Task {} {}", Integer.valueOf(j++), task); } } } } return groups; }  UpgradePack suggestUpgradePack(String clusterName, String upgradeFromVersion, String upgradeToVersion,
    Direction direction, UpgradeType upgradeType, String preferredUpgradePackName); List<UpgradeGroupHolder> createSequence(UpgradePack upgradePack,
      UpgradeContext context); Set<Resource> getStageResources(String clusterName, Long requestId, List<Long> stageIds); Resource getTaskResource(String clusterName, Long requestId, Long stageId, Long taskId); @Transactional void putComponentsToUpgradingState(String version,
      Map<Service, Set<ServiceComponent>> targetServices, StackId targetStack);  }
@Test public void testUpgradeOrchestration() throws Exception { Map<String, UpgradePack> upgrades = ambariMetaInfo.getUpgradePacks("foo", "bar"); assertTrue(upgrades.isEmpty()); upgrades = ambariMetaInfo.getUpgradePacks("HDP", "2.1.1"); ServiceInfo si = ambariMetaInfo.getService("HDP", "2.1.1", "ZOOKEEPER"); si.setDisplayName("Zk"); ComponentInfo ci = si.getComponentByName("ZOOKEEPER_SERVER"); ci.setDisplayName("ZooKeeper1 Server2"); assertTrue(upgrades.containsKey("upgrade_test")); UpgradePack upgrade = upgrades.get("upgrade_test"); assertNotNull(upgrade); Cluster cluster = makeCluster(); UpgradeContext context = m_upgradeContextFactory.create(cluster, UpgradeType.ROLLING, Direction.UPGRADE, UPGRADE_VERSION, new HashMap<String, Object>()); context.setResolver(m_masterHostResolver); List<UpgradeGroupHolder> groups = m_upgradeHelper.createSequence(upgrade, context); assertEquals(7, groups.size()); assertEquals("PRE_CLUSTER", groups.get(0).name); assertEquals("ZOOKEEPER", groups.get(1).name); assertEquals("CORE_MASTER", groups.get(2).name); assertEquals("CORE_SLAVES", groups.get(3).name); assertEquals("HIVE", groups.get(4).name); assertEquals("OOZIE", groups.get(5).name); UpgradeGroupHolder holder = groups.get(2); boolean found = false; for (StageWrapper sw : holder.items) { if (sw.getTasksJson().contains("Upgrading your database")) { found = true; } } assertTrue("Expected to find replaced text for Upgrading", found); UpgradeGroupHolder group = groups.get(1); assertTrue(group.items.get(1).getText().contains("ZooKeeper1 Server2")); assertEquals(group.items.get(5).getText(), "Service Check Zk"); group = groups.get(3); assertEquals(8, group.items.size()); StageWrapper sw = group.items.get(3); assertEquals("Validate Partial Upgrade", sw.getText()); assertEquals(1, sw.getTasks().size()); assertEquals(1, sw.getTasks().get(0).getTasks().size()); Task t = sw.getTasks().get(0).getTasks().get(0); assertEquals(ManualTask.class, t.getClass()); ManualTask mt = (ManualTask) t; assertTrue(mt.messages.get(0).contains("DataNode and NodeManager")); assertNotNull(mt.structuredOut); assertTrue(mt.structuredOut.contains("DATANODE")); assertTrue(mt.structuredOut.contains("NODEMANAGER")); UpgradeGroupHolder postGroup = groups.get(6); assertEquals("POST_CLUSTER", postGroup.name); assertEquals("Finalize Upgrade", postGroup.title); assertEquals(3, postGroup.items.size()); assertEquals("Confirm Finalize", postGroup.items.get(0).getText()); assertEquals("Execute HDFS Finalize", postGroup.items.get(1).getText()); assertEquals("Save Cluster State", postGroup.items.get(2).getText()); assertEquals(StageWrapper.Type.SERVER_SIDE_ACTION, postGroup.items.get(2).getType()); assertEquals(4, groups.get(0).items.size()); assertEquals(6, groups.get(1).items.size()); assertEquals(9, groups.get(2).items.size()); assertEquals(8, groups.get(3).items.size()); stackManagerMock.invalidateCurrentPaths(); ambariMetaInfo.init(); }
@Test public void testPartialUpgradeOrchestration() throws Exception { Map<String, UpgradePack> upgrades = ambariMetaInfo.getUpgradePacks("foo", "bar"); assertTrue(upgrades.isEmpty()); upgrades = ambariMetaInfo.getUpgradePacks("HDP", "2.1.1"); ServiceInfo si = ambariMetaInfo.getService("HDP", "2.1.1", "ZOOKEEPER"); si.setDisplayName("Zk"); ComponentInfo ci = si.getComponentByName("ZOOKEEPER_SERVER"); ci.setDisplayName("ZooKeeper1 Server2"); assertTrue(upgrades.containsKey("upgrade_test_partial")); UpgradePack upgrade = upgrades.get("upgrade_test_partial"); assertNotNull(upgrade); Cluster cluster = makeCluster(); UpgradeContext context = m_upgradeContextFactory.create(cluster, UpgradeType.ROLLING, Direction.UPGRADE, UPGRADE_VERSION, new HashMap<String, Object>()); context.setResolver(m_masterHostResolver); context.setSupportedServices(Collections.singleton("ZOOKEEPER")); context.setScope(UpgradeScope.PARTIAL); List<Grouping> groupings = upgrade.getGroups(Direction.UPGRADE); assertEquals(8, groupings.size()); assertEquals(UpgradeScope.COMPLETE, groupings.get(6).scope); List<UpgradeGroupHolder> groups = m_upgradeHelper.createSequence(upgrade, context); assertEquals(3, groups.size()); assertEquals("PRE_CLUSTER", groups.get(0).name); assertEquals("ZOOKEEPER", groups.get(1).name); assertEquals("POST_CLUSTER", groups.get(2).name); UpgradeGroupHolder group = groups.get(1); assertTrue(group.items.get(1).getText().contains("ZooKeeper1 Server2")); assertEquals("Service Check Zk", group.items.get(6).getText()); UpgradeGroupHolder postGroup = groups.get(2); assertEquals("POST_CLUSTER", postGroup.name); assertEquals("Finalize Upgrade", postGroup.title); assertEquals(2, postGroup.items.size()); assertEquals("Confirm Finalize", postGroup.items.get(0).getText()); assertEquals("Save Cluster State", postGroup.items.get(1).getText()); assertEquals(StageWrapper.Type.SERVER_SIDE_ACTION, postGroup.items.get(1).getType()); assertEquals(3, groups.get(0).items.size()); assertEquals(7, groups.get(1).items.size()); assertEquals(2, groups.get(2).items.size()); stackManagerMock.invalidateCurrentPaths(); ambariMetaInfo.init(); }
@Test public void testCompleteUpgradeOrchestration() throws Exception { Map<String, UpgradePack> upgrades = ambariMetaInfo.getUpgradePacks("foo", "bar"); assertTrue(upgrades.isEmpty()); upgrades = ambariMetaInfo.getUpgradePacks("HDP", "2.1.1"); ServiceInfo si = ambariMetaInfo.getService("HDP", "2.1.1", "ZOOKEEPER"); si.setDisplayName("Zk"); ComponentInfo ci = si.getComponentByName("ZOOKEEPER_SERVER"); ci.setDisplayName("ZooKeeper1 Server2"); assertTrue(upgrades.containsKey("upgrade_test_partial")); UpgradePack upgrade = upgrades.get("upgrade_test_partial"); assertNotNull(upgrade); Cluster cluster = makeCluster(); UpgradeContext context = m_upgradeContextFactory.create(cluster, UpgradeType.ROLLING, Direction.UPGRADE, UPGRADE_VERSION, new HashMap<String, Object>()); context.setResolver(m_masterHostResolver); context.setSupportedServices(Collections.singleton("ZOOKEEPER")); context.setScope(UpgradeScope.COMPLETE); List<Grouping> groupings = upgrade.getGroups(Direction.UPGRADE); assertEquals(8, groupings.size()); assertEquals(UpgradeScope.COMPLETE, groupings.get(6).scope); List<UpgradeGroupHolder> groups = m_upgradeHelper.createSequence(upgrade, context); assertEquals(4, groups.size()); assertEquals("PRE_CLUSTER", groups.get(0).name); assertEquals("ZOOKEEPER", groups.get(1).name); assertEquals("ALL_HOSTS", groups.get(2).name); assertEquals("POST_CLUSTER", groups.get(3).name); UpgradeGroupHolder group = groups.get(1); assertTrue(group.items.get(1).getText().contains("ZooKeeper1 Server2")); assertEquals("Service Check Zk", group.items.get(5).getText()); UpgradeGroupHolder postGroup = groups.get(3); assertEquals("POST_CLUSTER", postGroup.name); assertEquals("Finalize Upgrade", postGroup.title); assertEquals(2, postGroup.items.size()); assertEquals("Confirm Finalize", postGroup.items.get(0).getText()); assertEquals("Save Cluster State", postGroup.items.get(1).getText()); assertEquals(StageWrapper.Type.SERVER_SIDE_ACTION, postGroup.items.get(1).getType()); assertEquals(3, groups.get(0).items.size()); assertEquals(6, groups.get(1).items.size()); assertEquals(1, groups.get(2).items.size()); stackManagerMock.invalidateCurrentPaths(); ambariMetaInfo.init(); }
@Test public void testUpgradeServerActionOrchestration() throws Exception { Map<String, UpgradePack> upgrades = ambariMetaInfo.getUpgradePacks("HDP", "2.1.1"); ServiceInfo si = ambariMetaInfo.getService("HDP", "2.1.1", "ZOOKEEPER"); si.setDisplayName("Zk"); ComponentInfo ci = si.getComponentByName("ZOOKEEPER_SERVER"); ci.setDisplayName("ZooKeeper1 Server2"); assertTrue(upgrades.containsKey("upgrade_server_action_test")); UpgradePack upgrade = upgrades.get("upgrade_server_action_test"); assertNotNull(upgrade); Cluster cluster = makeCluster(); UpgradeContext context = m_upgradeContextFactory.create(cluster, UpgradeType.ROLLING, Direction.UPGRADE, UPGRADE_VERSION, new HashMap<String, Object>()); context.setResolver(m_masterHostResolver); List<UpgradeGroupHolder> groups = m_upgradeHelper.createSequence(upgrade, context); assertEquals(1, groups.size()); UpgradeGroupHolder group = groups.get(0); assertEquals("CLUSTER_SERVER_ACTIONS", group.name); List<StageWrapper> stageWrappers = group.items; assertEquals(6, stageWrappers.size()); assertEquals("Pre Upgrade", stageWrappers.get(0).getText()); assertEquals("Pre Upgrade Zookeeper", stageWrappers.get(1).getText()); assertEquals("Configuring", stageWrappers.get(2).getText()); assertEquals("Configuring HDFS", stageWrappers.get(3).getText()); assertEquals("Calculating Properties", stageWrappers.get(4).getText()); assertEquals("Calculating HDFS Properties", stageWrappers.get(5).getText()); stackManagerMock.invalidateCurrentPaths(); ambariMetaInfo.init(); }
@Test public void testUpgradeOrchestrationWithHostsInMM() throws Exception { Map<String, UpgradePack> upgrades = ambariMetaInfo.getUpgradePacks("foo", "bar"); assertTrue(upgrades.isEmpty()); upgrades = ambariMetaInfo.getUpgradePacks("HDP", "2.1.1"); ServiceInfo si = ambariMetaInfo.getService("HDP", "2.1.1", "ZOOKEEPER"); si.setDisplayName("Zk"); ComponentInfo ci = si.getComponentByName("ZOOKEEPER_SERVER"); ci.setDisplayName("ZooKeeper1 Server2"); assertTrue(upgrades.containsKey("upgrade_test")); UpgradePack upgrade = upgrades.get("upgrade_test"); assertNotNull(upgrade); Cluster cluster = makeCluster(); Host hostInMaintenanceMode = cluster.getHosts().iterator().next(); hostInMaintenanceMode.setMaintenanceState(cluster.getClusterId(), MaintenanceState.ON); MasterHostResolver masterHostResolver = new MasterHostResolver(null, cluster, ""); UpgradeContext context = m_upgradeContextFactory.create(cluster, UpgradeType.ROLLING, Direction.UPGRADE, UPGRADE_VERSION, new HashMap<String, Object>()); context.setResolver(masterHostResolver); List<UpgradeGroupHolder> groups = m_upgradeHelper.createSequence(upgrade, context); assertEquals(7, groups.size()); for (UpgradeGroupHolder group : groups) { for (StageWrapper stageWrapper : group.items) { Set<String> hosts = stageWrapper.getHosts(); assertFalse(hosts.contains(hostInMaintenanceMode.getHostName())); } } stackManagerMock.invalidateCurrentPaths(); ambariMetaInfo.init(); }
@Test public void testNamenodeOrder() throws Exception { Map<String, UpgradePack> upgrades = ambariMetaInfo.getUpgradePacks("HDP", "2.1.1"); assertTrue(upgrades.containsKey("upgrade_test")); UpgradePack upgrade = upgrades.get("upgrade_test"); assertNotNull(upgrade); Cluster cluster = makeCluster(); UpgradeContext context = m_upgradeContextFactory.create(cluster, UpgradeType.ROLLING, Direction.UPGRADE, UPGRADE_VERSION, new HashMap<String, Object>()); context.setResolver(m_masterHostResolver); List<UpgradeGroupHolder> groups = m_upgradeHelper.createSequence(upgrade, context); assertEquals(7, groups.size()); UpgradeGroupHolder mastersGroup = groups.get(2); assertEquals("CORE_MASTER", mastersGroup.name); List<String> orderedNameNodes = new LinkedList<>(); for (StageWrapper sw : mastersGroup.items) { if (sw.getType().equals(StageWrapper.Type.RESTART) && sw.getText().toLowerCase().contains("NameNode".toLowerCase())) { for (TaskWrapper tw : sw.getTasks()) { for (String hostName : tw.getHosts()) { orderedNameNodes.add(hostName); } } } } assertEquals(2, orderedNameNodes.size()); assertEquals("h2", orderedNameNodes.get(0)); assertEquals("h1", orderedNameNodes.get(1)); }
@Test public void testUpgradeOrchestrationWithNoHeartbeat() throws Exception { Map<String, UpgradePack> upgrades = ambariMetaInfo.getUpgradePacks("foo", "bar"); assertTrue(upgrades.isEmpty()); upgrades = ambariMetaInfo.getUpgradePacks("HDP", "2.1.1"); assertTrue(upgrades.containsKey("upgrade_test")); UpgradePack upgrade = upgrades.get("upgrade_test"); assertNotNull(upgrade); Cluster cluster = makeCluster(false); Clusters clusters = injector.getInstance(Clusters.class); Host h4 = clusters.getHost("h4"); h4.setState(HostState.HEARTBEAT_LOST); List<ServiceComponentHost> schs = cluster.getServiceComponentHosts("h4"); assertEquals(1, schs.size()); assertEquals(HostState.HEARTBEAT_LOST, schs.get(0).getHostState()); UpgradeContext context = m_upgradeContextFactory.create(cluster, UpgradeType.ROLLING, Direction.UPGRADE, UPGRADE_VERSION, new HashMap<String, Object>()); context.setResolver(m_masterHostResolver); List<UpgradeGroupHolder> groups = m_upgradeHelper.createSequence(upgrade, context); assertEquals(7, groups.size()); assertEquals("PRE_CLUSTER", groups.get(0).name); assertEquals("ZOOKEEPER", groups.get(1).name); assertEquals("CORE_MASTER", groups.get(2).name); assertEquals("CORE_SLAVES", groups.get(3).name); assertEquals("HIVE", groups.get(4).name); assertEquals("OOZIE", groups.get(5).name); UpgradeGroupHolder postGroup = groups.get(6); assertEquals("POST_CLUSTER", postGroup.name); assertEquals("Finalize Upgrade", postGroup.title); assertEquals(3, postGroup.items.size()); assertEquals("Confirm Finalize", postGroup.items.get(0).getText()); assertEquals("Execute HDFS Finalize", postGroup.items.get(1).getText()); assertEquals("Save Cluster State", postGroup.items.get(2).getText()); assertEquals(StageWrapper.Type.SERVER_SIDE_ACTION, postGroup.items.get(2).getType()); assertEquals(6, groups.get(1).items.size()); assertEquals(9, groups.get(2).items.size()); assertEquals(7, groups.get(3).items.size()); }
@Test public void testDowngradeOrchestration() throws Exception { Map<String, UpgradePack> upgrades = ambariMetaInfo.getUpgradePacks("HDP", "2.1.1"); assertTrue(upgrades.containsKey("upgrade_test")); UpgradePack upgrade = upgrades.get("upgrade_test"); assertNotNull(upgrade); Cluster cluster = makeCluster(); UpgradeContext context = m_upgradeContextFactory.create(cluster, UpgradeType.ROLLING, Direction.DOWNGRADE, DOWNGRADE_VERSION, new HashMap<String, Object>()); context.setResolver(m_masterHostResolver); List<UpgradeGroupHolder> groups = m_upgradeHelper.createSequence(upgrade, context); assertEquals(7, groups.size()); assertEquals("PRE_CLUSTER", groups.get(0).name); assertEquals("OOZIE", groups.get(1).name); assertEquals("HIVE", groups.get(2).name); assertEquals("CORE_SLAVES", groups.get(3).name); assertEquals("CORE_MASTER", groups.get(4).name); assertEquals("ZOOKEEPER", groups.get(5).name); UpgradeGroupHolder postGroup = groups.get(6); assertEquals("POST_CLUSTER", postGroup.name); assertEquals("Finalize Downgrade", postGroup.title); assertEquals(3, postGroup.items.size()); assertEquals("Confirm Finalize", postGroup.items.get(0).getText()); assertEquals("Execute HDFS Finalize", postGroup.items.get(1).getText()); assertEquals("Save Cluster State", postGroup.items.get(2).getText()); assertEquals(StageWrapper.Type.SERVER_SIDE_ACTION, postGroup.items.get(2).getType()); assertEquals(4, groups.get(0).items.size()); assertEquals(8, groups.get(1).items.size()); assertEquals(5, groups.get(2).items.size()); assertEquals(8, groups.get(3).items.size()); assertEquals(8, groups.get(4).items.size()); }
@Test public void testBuckets() throws Exception { Map<String, UpgradePack> upgrades = ambariMetaInfo.getUpgradePacks("HDP", "2.1.1"); assertTrue(upgrades.containsKey("upgrade_bucket_test")); UpgradePack upgrade = upgrades.get("upgrade_bucket_test"); assertNotNull(upgrade); Cluster cluster = makeCluster(); UpgradeContext context = m_upgradeContextFactory.create(cluster, UpgradeType.ROLLING, Direction.UPGRADE, UPGRADE_VERSION, new HashMap<String, Object>()); context.setResolver(m_masterHostResolver); List<UpgradeGroupHolder> groups = m_upgradeHelper.createSequence(upgrade, context); assertEquals(1, groups.size()); UpgradeGroupHolder group = groups.iterator().next(); assertEquals(22, group.items.size()); }
@Test public void testManualTaskPostProcessing() throws Exception { Map<String, UpgradePack> upgrades = ambariMetaInfo.getUpgradePacks("HDP", "2.1.1"); assertTrue(upgrades.containsKey("upgrade_test")); UpgradePack upgrade = upgrades.get("upgrade_test"); assertNotNull(upgrade); Cluster cluster = makeCluster(); UpgradeContext context = m_upgradeContextFactory.create(cluster, UpgradeType.ROLLING, Direction.UPGRADE, UPGRADE_VERSION, new HashMap<String, Object>()); context.setResolver(m_masterHostResolver); List<UpgradeGroupHolder> groups = m_upgradeHelper.createSequence(upgrade, context); assertEquals(7, groups.size()); UpgradeGroupHolder zookeeperGroup = groups.get(1); assertEquals("ZOOKEEPER", zookeeperGroup.name); ManualTask manualTask = (ManualTask) zookeeperGroup.items.get(0).getTasks().get( 0).getTasks().get(0); assertEquals(1, manualTask.messages.size()); assertEquals("This is a manual task with a placeholder of placeholder-rendered-properly", manualTask.messages.get(0)); }
@Test public void testConditionalDeleteTask() throws Exception { Map<String, UpgradePack> upgrades = ambariMetaInfo.getUpgradePacks("HDP", "2.1.1"); assertTrue(upgrades.containsKey("upgrade_test")); UpgradePack upgrade = upgrades.get("upgrade_test"); ConfigUpgradePack cup = ambariMetaInfo.getConfigUpgradePack("HDP", "2.1.1"); assertNotNull(upgrade); Cluster cluster = makeCluster(); UpgradeContext context = m_upgradeContextFactory.create(cluster, UpgradeType.ROLLING, Direction.UPGRADE, UPGRADE_VERSION, new HashMap<String, Object>()); context.setResolver(m_masterHostResolver); List<UpgradeGroupHolder> groups = m_upgradeHelper.createSequence(upgrade, context); assertEquals(7, groups.size()); UpgradeGroupHolder hiveGroup = groups.get(4); assertEquals("HIVE", hiveGroup.name); ConfigureTask configureTask = (ConfigureTask) hiveGroup.items.get(1).getTasks().get(0).getTasks().get(0); Map<String, String> hiveConfigs = new HashMap<>(); hiveConfigs.put("hive.server2.transport.mode", "http"); hiveConfigs.put("hive.server2.thrift.port", "10001"); hiveConfigs.put("condition", "1"); ConfigurationRequest configurationRequest = new ConfigurationRequest(); configurationRequest.setClusterName(cluster.getClusterName()); configurationRequest.setType("hive-site"); configurationRequest.setVersionTag("version2"); configurationRequest.setProperties(hiveConfigs); final ClusterRequest clusterRequest = new ClusterRequest( cluster.getClusterId(), cluster.getClusterName(), cluster.getDesiredStackVersion().getStackVersion(), null); clusterRequest.setDesiredConfig(Collections.singletonList(configurationRequest)); m_managementController.updateClusters(new HashSet<ClusterRequest>() { { add(clusterRequest); } }, null); Map<String, String> configProperties = configureTask.getConfigurationChanges(cluster, cup); assertFalse(configProperties.isEmpty()); assertEquals(configProperties.get(ConfigureTask.PARAMETER_CONFIG_TYPE), "hive-site"); String configurationJson = configProperties.get(ConfigureTask.PARAMETER_TRANSFERS); assertNotNull(configurationJson); List<ConfigUpgradeChangeDefinition.Transfer> transfers = m_gson.fromJson(configurationJson, new TypeToken<List<ConfigUpgradeChangeDefinition.Transfer>>() { }.getType()); System.out.println(">> transfers"+transfers); assertEquals(6, transfers.size()); assertEquals("copy-key", transfers.get(0).fromKey); assertEquals("copy-key-to", transfers.get(0).toKey); assertEquals("move-key", transfers.get(1).fromKey); assertEquals("move-key-to", transfers.get(1).toKey); assertEquals("delete-key", transfers.get(2).deleteKey); assertEquals("delete-http-1", transfers.get(3).deleteKey); assertEquals("delete-http-2", transfers.get(4).deleteKey); assertEquals("delete-http-3", transfers.get(5).deleteKey); }
@Test public void testConfigTaskConditionMet() throws Exception { Map<String, UpgradePack> upgrades = ambariMetaInfo.getUpgradePacks("HDP", "2.1.1"); assertTrue(upgrades.containsKey("upgrade_test")); UpgradePack upgrade = upgrades.get("upgrade_test"); ConfigUpgradePack cup = ambariMetaInfo.getConfigUpgradePack("HDP", "2.1.1"); assertNotNull(upgrade); Cluster cluster = makeCluster(); UpgradeContext context = m_upgradeContextFactory.create(cluster, UpgradeType.ROLLING, Direction.UPGRADE, UPGRADE_VERSION, new HashMap<String, Object>()); context.setResolver(m_masterHostResolver); List<UpgradeGroupHolder> groups = m_upgradeHelper.createSequence(upgrade, context); assertEquals(7, groups.size()); UpgradeGroupHolder hiveGroup = groups.get(4); assertEquals("HIVE", hiveGroup.name); ConfigureTask configureTask = (ConfigureTask) hiveGroup.items.get(2).getTasks().get( 0).getTasks().get(0); Map<String, String> configProperties = configureTask.getConfigurationChanges(cluster, cup); assertFalse(configProperties.isEmpty()); assertEquals(configProperties.get(ConfigureTask.PARAMETER_CONFIG_TYPE), "hive-site"); assertTrue(configProperties.containsKey(ConfigureTask.PARAMETER_KEY_VALUE_PAIRS)); assertTrue(configProperties.containsKey(ConfigureTask.PARAMETER_REPLACEMENTS)); assertTrue(configProperties.containsKey(ConfigureTask.PARAMETER_TRANSFERS)); String configurationJson = configProperties.get(ConfigureTask.PARAMETER_KEY_VALUE_PAIRS); String transferJson = configProperties.get(ConfigureTask.PARAMETER_TRANSFERS); String replacementJson = configProperties.get(ConfigureTask.PARAMETER_REPLACEMENTS); assertNotNull(configurationJson); assertNotNull(transferJson); assertNotNull(replacementJson); List<ConfigUpgradeChangeDefinition.ConfigurationKeyValue> keyValuePairs = m_gson.fromJson(configurationJson, new TypeToken<List<ConfigUpgradeChangeDefinition.ConfigurationKeyValue>>() { }.getType()); assertEquals("setKeyOne", keyValuePairs.get(0).key); assertEquals("1", keyValuePairs.get(0).value); assertEquals("setKeyTwo", keyValuePairs.get(1).key); assertEquals("2", keyValuePairs.get(1).value); assertEquals("setKeyThree", keyValuePairs.get(2).key); assertEquals("3", keyValuePairs.get(2).value); assertEquals("setKeyFour", keyValuePairs.get(3).key); assertEquals("4", keyValuePairs.get(3).value); List<ConfigUpgradeChangeDefinition.Transfer> transfers = m_gson.fromJson(transferJson, new TypeToken<List<ConfigUpgradeChangeDefinition.Transfer>>() { }.getType()); System.out.println(" testConfigTaskConditionMet >> transfer"+transfers); assertEquals("copy-key-one", transfers.get(0).fromKey); assertEquals("copy-to-key-one", transfers.get(0).toKey); assertEquals("copy-key-two", transfers.get(1).fromKey); assertEquals("copy-to-key-two", transfers.get(1).toKey); assertEquals("copy-key-three", transfers.get(2).fromKey); assertEquals("copy-to-key-three", transfers.get(2).toKey); assertEquals("copy-key-four", transfers.get(3).fromKey); assertEquals("copy-to-key-four", transfers.get(3).toKey); assertEquals("move-key-one", transfers.get(4).fromKey); assertEquals("move-to-key-one", transfers.get(4).toKey); assertEquals("move-key-two", transfers.get(5).fromKey); assertEquals("move-to-key-two", transfers.get(5).toKey); assertEquals("move-key-three", transfers.get(6).fromKey); assertEquals("move-to-key-three", transfers.get(6).toKey); assertEquals("move-key-four", transfers.get(7).fromKey); assertEquals("move-to-key-four", transfers.get(7).toKey); assertEquals("delete-key-one", transfers.get(8).deleteKey); assertEquals("delete-key-two", transfers.get(9).deleteKey); assertEquals("delete-key-three", transfers.get(10).deleteKey); assertEquals("delete-key-four", transfers.get(11).deleteKey); List<ConfigUpgradeChangeDefinition.Replace> replacements = m_gson.fromJson(replacementJson, new TypeToken<List<ConfigUpgradeChangeDefinition.Replace>>() { }.getType()); assertEquals("replace-key-one", replacements.get(0).key); assertEquals("abc", replacements.get(0).find); assertEquals("abc-replaced", replacements.get(0).replaceWith); assertEquals("replace-key-two", replacements.get(1).key); assertEquals("efg", replacements.get(1).find); assertEquals("efg-replaced", replacements.get(1).replaceWith); assertEquals("replace-key-three", replacements.get(2).key); assertEquals("ijk", replacements.get(2).find); assertEquals("ijk-replaced", replacements.get(2).replaceWith); assertEquals("replace-key-four", replacements.get(3).key); assertEquals("lmn", replacements.get(3).find); assertEquals("lmn-replaced", replacements.get(3).replaceWith); }
@Test public void testConfigTaskConditionSkipped() throws Exception { Map<String, UpgradePack> upgrades = ambariMetaInfo.getUpgradePacks("HDP", "2.1.1"); assertTrue(upgrades.containsKey("upgrade_test")); UpgradePack upgrade = upgrades.get("upgrade_test"); ConfigUpgradePack cup = ambariMetaInfo.getConfigUpgradePack("HDP", "2.1.1"); assertNotNull(upgrade); Cluster cluster = makeCluster(); UpgradeContext context = m_upgradeContextFactory.create(cluster, UpgradeType.ROLLING, Direction.UPGRADE, UPGRADE_VERSION, new HashMap<String, Object>()); context.setResolver(m_masterHostResolver); List<UpgradeGroupHolder> groups = m_upgradeHelper.createSequence(upgrade, context); assertEquals(7, groups.size()); UpgradeGroupHolder hiveGroup = groups.get(4); assertEquals("HIVE", hiveGroup.name); ConfigureTask configureTask = (ConfigureTask) hiveGroup.items.get(3).getTasks().get(0).getTasks().get(0); Map<String, String> configProperties = configureTask.getConfigurationChanges(cluster, cup); assertFalse(configProperties.isEmpty()); assertEquals(configProperties.get(ConfigureTask.PARAMETER_CONFIG_TYPE), "hive-site"); assertTrue(configProperties.containsKey(ConfigureTask.PARAMETER_KEY_VALUE_PAIRS)); assertTrue(configProperties.containsKey(ConfigureTask.PARAMETER_REPLACEMENTS)); assertTrue(configProperties.containsKey(ConfigureTask.PARAMETER_TRANSFERS)); String configurationJson = configProperties.get(ConfigureTask.PARAMETER_KEY_VALUE_PAIRS); String transferJson = configProperties.get(ConfigureTask.PARAMETER_TRANSFERS); String replacementJson = configProperties.get(ConfigureTask.PARAMETER_REPLACEMENTS); assertNotNull(configurationJson); assertNotNull(transferJson); assertNotNull(replacementJson); List<ConfigUpgradeChangeDefinition.ConfigurationKeyValue> keyValuePairs = m_gson.fromJson(configurationJson, new TypeToken<List<ConfigUpgradeChangeDefinition.ConfigurationKeyValue>>() { }.getType()); assertTrue(keyValuePairs.isEmpty()); List<ConfigUpgradeChangeDefinition.Replace> replacements = m_gson.fromJson(replacementJson, new TypeToken<List<ConfigUpgradeChangeDefinition.Replace>>() { }.getType()); assertTrue(replacements.isEmpty()); List<ConfigUpgradeChangeDefinition.Transfer> transfers = m_gson.fromJson(transferJson, new TypeToken<List<ConfigUpgradeChangeDefinition.Transfer>>() { }.getType()); assertTrue(transfers.isEmpty()); }
@Test public void testConfigureTask() throws Exception { Map<String, UpgradePack> upgrades = ambariMetaInfo.getUpgradePacks("HDP", "2.1.1"); assertTrue(upgrades.containsKey("upgrade_test")); UpgradePack upgrade = upgrades.get("upgrade_test"); ConfigUpgradePack cup = ambariMetaInfo.getConfigUpgradePack("HDP", "2.1.1"); assertNotNull(upgrade); Cluster cluster = makeCluster(); UpgradeContext context = m_upgradeContextFactory.create(cluster, UpgradeType.ROLLING, Direction.UPGRADE, UPGRADE_VERSION, new HashMap<String, Object>()); context.setResolver(m_masterHostResolver); List<UpgradeGroupHolder> groups = m_upgradeHelper.createSequence(upgrade, context); assertEquals(7, groups.size()); UpgradeGroupHolder hiveGroup = groups.get(4); assertEquals("HIVE", hiveGroup.name); ConfigureTask configureTask = (ConfigureTask) hiveGroup.items.get(1).getTasks().get(0).getTasks().get(0); Map<String, String> configProperties = configureTask.getConfigurationChanges(cluster, cup); assertFalse(configProperties.isEmpty()); assertEquals(configProperties.get(ConfigureTask.PARAMETER_CONFIG_TYPE), "hive-site"); Map<String, String> hiveConfigs = new HashMap<>(); hiveConfigs.put("fooKey", "THIS-BETTER-CHANGE"); hiveConfigs.put("ifFooKey", "ifFooValue"); ConfigurationRequest configurationRequest = new ConfigurationRequest(); configurationRequest.setClusterName(cluster.getClusterName()); configurationRequest.setType("hive-site"); configurationRequest.setVersionTag("version2"); configurationRequest.setProperties(hiveConfigs); final ClusterRequest clusterRequest = new ClusterRequest( cluster.getClusterId(), cluster.getClusterName(), cluster.getDesiredStackVersion().getStackVersion(), null); clusterRequest.setDesiredConfig(Collections.singletonList(configurationRequest)); m_managementController.updateClusters(new HashSet<ClusterRequest>() { { add(clusterRequest); } }, null); configProperties = configureTask.getConfigurationChanges(cluster, cup); assertFalse(configProperties.isEmpty()); assertEquals( configProperties.get(ConfigureTask.PARAMETER_CONFIG_TYPE), "hive-site"); String configurationJson = configProperties.get(ConfigureTask.PARAMETER_KEY_VALUE_PAIRS); assertNotNull(configurationJson); List<ConfigUpgradeChangeDefinition.ConfigurationKeyValue> keyValuePairs = m_gson.fromJson( configurationJson, new TypeToken<List<ConfigUpgradeChangeDefinition.ConfigurationKeyValue>>() { }.getType()); assertEquals("fooKey", keyValuePairs.get(0).key); assertEquals("fooValue", keyValuePairs.get(0).value); }
@Test public void testConfigureTaskWithMultipleConfigurations() throws Exception { Map<String, UpgradePack> upgrades = ambariMetaInfo.getUpgradePacks("HDP", "2.1.1"); assertTrue(upgrades.containsKey("upgrade_test")); UpgradePack upgrade = upgrades.get("upgrade_test"); ConfigUpgradePack cup = ambariMetaInfo.getConfigUpgradePack("HDP", "2.1.1"); assertNotNull(upgrade); Cluster cluster = makeCluster(); UpgradeContext context = m_upgradeContextFactory.create(cluster, UpgradeType.ROLLING, Direction.UPGRADE, UPGRADE_VERSION, new HashMap<String, Object>()); context.setResolver(m_masterHostResolver); List<UpgradeGroupHolder> groups = m_upgradeHelper.createSequence(upgrade, context); assertEquals(7, groups.size()); UpgradeGroupHolder hiveGroup = groups.get(4); assertEquals("HIVE", hiveGroup.name); ConfigureTask configureTask = (ConfigureTask) hiveGroup.items.get(1).getTasks().get(0).getTasks().get(0); Map<String, String> configProperties = configureTask.getConfigurationChanges(cluster, cup); assertFalse(configProperties.isEmpty()); assertEquals(configProperties.get(ConfigureTask.PARAMETER_CONFIG_TYPE), "hive-site"); String configurationJson = configProperties.get(ConfigureTask.PARAMETER_KEY_VALUE_PAIRS); String transferJson = configProperties.get(ConfigureTask.PARAMETER_TRANSFERS); assertNotNull(configurationJson); assertNotNull(transferJson); List<ConfigUpgradeChangeDefinition.ConfigurationKeyValue> keyValuePairs = m_gson.fromJson(configurationJson, new TypeToken<List<ConfigUpgradeChangeDefinition.ConfigurationKeyValue>>() { }.getType()); List<ConfigUpgradeChangeDefinition.Transfer> transfers = m_gson.fromJson(transferJson, new TypeToken<List<ConfigUpgradeChangeDefinition.Transfer>>() { }.getType()); assertEquals("fooKey", keyValuePairs.get(0).key); assertEquals("fooValue", keyValuePairs.get(0).value); assertEquals("fooKey2", keyValuePairs.get(1).key); assertEquals("fooValue2", keyValuePairs.get(1).value); assertEquals("fooKey3", keyValuePairs.get(2).key); assertEquals("fooValue3", keyValuePairs.get(2).value); assertEquals("copy-key", transfers.get(0).fromKey); assertEquals("copy-key-to", transfers.get(0).toKey); assertEquals("move-key", transfers.get(1).fromKey); assertEquals("move-key-to", transfers.get(1).toKey); }
@Test public void testServiceCheckUpgradeStages() throws Exception { Map<String, UpgradePack> upgrades = ambariMetaInfo.getUpgradePacks("HDP", "2.2.0"); assertTrue(upgrades.containsKey("upgrade_test_checks")); UpgradePack upgrade = upgrades.get("upgrade_test_checks"); assertNotNull(upgrade); Cluster c = makeCluster(); Set<String> additionalServices = new HashSet<String>() {{ add("HBASE"); add("PIG"); add("TEZ"); add("AMBARI_METRICS"); }}; for(String service : additionalServices) { c.addService(service); } int numServiceChecksExpected = 0; Collection<Service> services = c.getServices().values(); for(Service service : services) { ServiceInfo si = ambariMetaInfo.getService(c.getCurrentStackVersion().getStackName(), c.getCurrentStackVersion().getStackVersion(), service.getName()); if (null == si.getCommandScript()) { continue; } if (service.getName().equalsIgnoreCase("TEZ")) { assertTrue("Expect Tez to not have any service checks", false); } if (service.getName().equalsIgnoreCase("AMBARI_METRICS")) { continue; } numServiceChecksExpected++; } UpgradeContext context = m_upgradeContextFactory.create(c, UpgradeType.ROLLING, Direction.UPGRADE, UPGRADE_VERSION, new HashMap<String, Object>()); context.setResolver(m_masterHostResolver); List<UpgradeGroupHolder> groups = m_upgradeHelper.createSequence(upgrade, context); assertEquals(7, groups.size()); UpgradeGroupHolder holder = groups.get(3); assertEquals(holder.name, "SERVICE_CHECK_1"); assertEquals(7, holder.items.size()); int numServiceChecksActual = 0; for (StageWrapper sw : holder.items) { for(Service service : services) { Pattern p = Pattern.compile(".*" + service.getName(), Pattern.CASE_INSENSITIVE); Matcher matcher = p.matcher(sw.getText()); if (matcher.matches()) { numServiceChecksActual++; continue; } } } assertEquals(numServiceChecksActual, numServiceChecksExpected); UpgradeGroupHolder zookeeperGroup = groups.get(1); assertEquals("ZOOKEEPER", zookeeperGroup.name); ManualTask manualTask = (ManualTask) zookeeperGroup.items.get(0).getTasks().get( 0).getTasks().get(0); assertEquals(1, manualTask.messages.size()); assertEquals( "This is a manual task with a placeholder of placeholder-rendered-properly", manualTask.messages.get(0)); }
@Test public void testServiceCheckDowngradeStages() throws Exception { Map<String, UpgradePack> upgrades = ambariMetaInfo.getUpgradePacks("HDP", "2.1.1"); assertTrue(upgrades.containsKey("upgrade_test_checks")); UpgradePack upgrade = upgrades.get("upgrade_test_checks"); assertNotNull(upgrade); Cluster cluster = makeCluster(); UpgradeContext context = m_upgradeContextFactory.create(cluster, UpgradeType.ROLLING, Direction.DOWNGRADE, DOWNGRADE_VERSION, new HashMap<String, Object>()); context.setResolver(m_masterHostResolver); List<UpgradeGroupHolder> groups = m_upgradeHelper.createSequence(upgrade, context); assertEquals(5, groups.size()); UpgradeGroupHolder zookeeperGroup = groups.get(3); assertEquals("ZOOKEEPER", zookeeperGroup.name); ManualTask manualTask = (ManualTask) zookeeperGroup.items.get(0).getTasks().get( 0).getTasks().get(0); assertEquals(1, manualTask.messages.size()); assertEquals( "This is a manual task with a placeholder of placeholder-rendered-properly", manualTask.messages.get(0)); }
@Test public void testUpgradeOrchestrationFullTask() throws Exception { Map<String, UpgradePack> upgrades = ambariMetaInfo.getUpgradePacks("HDP", "2.1.1"); ServiceInfo si = ambariMetaInfo.getService("HDP", "2.1.1", "ZOOKEEPER"); si.setDisplayName("Zk"); ComponentInfo ci = si.getComponentByName("ZOOKEEPER_SERVER"); ci.setDisplayName("ZooKeeper1 Server2"); assertTrue(upgrades.containsKey("upgrade_to_new_stack")); UpgradePack upgrade = upgrades.get("upgrade_to_new_stack"); assertNotNull(upgrade); Cluster cluster = makeCluster(); UpgradeContext context = m_upgradeContextFactory.create(cluster, UpgradeType.ROLLING, Direction.UPGRADE, UPGRADE_VERSION, new HashMap<String, Object>()); context.setResolver(m_masterHostResolver); List<UpgradeGroupHolder> groups = m_upgradeHelper.createSequence(upgrade, context); assertEquals(6, groups.size()); assertEquals("PRE_CLUSTER", groups.get(0).name); assertEquals("ZOOKEEPER", groups.get(1).name); assertEquals("CORE_MASTER", groups.get(2).name); assertEquals("CORE_SLAVES", groups.get(3).name); assertEquals("HIVE", groups.get(4).name); UpgradeGroupHolder holder = groups.get(2); boolean found = false; for (StageWrapper sw : holder.items) { if (sw.getTasksJson().contains("Upgrading your database")) { found = true; } } assertTrue("Expected to find replaced text for Upgrading", found); UpgradeGroupHolder group = groups.get(1); assertTrue(group.items.get(1).getText().contains("ZooKeeper1 Server2")); assertEquals(group.items.get(4).getText(), "Service Check Zk"); group = groups.get(3); assertEquals(8, group.items.size()); StageWrapper sw = group.items.get(3); assertEquals("Validate Partial Upgrade", sw.getText()); assertEquals(1, sw.getTasks().size()); assertEquals(1, sw.getTasks().get(0).getTasks().size()); Task t = sw.getTasks().get(0).getTasks().get(0); assertEquals(ManualTask.class, t.getClass()); ManualTask mt = (ManualTask) t; assertTrue(mt.messages.get(0).contains("DataNode and NodeManager")); assertNotNull(mt.structuredOut); assertTrue(mt.structuredOut.contains("DATANODE")); assertTrue(mt.structuredOut.contains("NODEMANAGER")); UpgradeGroupHolder postGroup = groups.get(5); assertEquals(postGroup.name, "POST_CLUSTER"); assertEquals(postGroup.title, "Finalize Upgrade"); assertEquals(4, postGroup.items.size()); assertEquals("Confirm Finalize", postGroup.items.get(0).getText()); assertEquals("Execute HDFS Finalize", postGroup.items.get(1).getText()); assertEquals("Save Cluster State", postGroup.items.get(2).getText()); assertEquals(StageWrapper.Type.SERVER_SIDE_ACTION, postGroup.items.get(2).getType()); assertEquals("Run On All 2.2.1.0-1234", postGroup.items.get(3).getText()); assertEquals(1, postGroup.items.get(3).getTasks().size()); Set<String> hosts = postGroup.items.get(3).getTasks().get(0).getHosts(); assertNotNull(hosts); assertEquals(4, hosts.size()); assertEquals(4, groups.get(0).items.size()); assertEquals(5, groups.get(1).items.size()); assertEquals(9, groups.get(2).items.size()); assertEquals(8, groups.get(3).items.size()); stackManagerMock.invalidateCurrentPaths(); ambariMetaInfo.init(); }
@Test public void testUpgradeWithMultipleTasksInOwnStage() throws Exception { Map<String, UpgradePack> upgrades = ambariMetaInfo.getUpgradePacks("HDP", "2.1.1"); assertTrue(upgrades.containsKey("upgrade_test")); UpgradePack upgrade = upgrades.get("upgrade_test"); assertNotNull(upgrade); assertTrue(upgrade.getType() == UpgradeType.ROLLING); List<Grouping> upgradePackGroups = upgrade.getGroups(Direction.UPGRADE); boolean foundService = false; for (Grouping group : upgradePackGroups) { if (group.title.equals("Oozie")) { foundService = true; } } assertTrue(foundService); Cluster cluster = makeCluster(); UpgradeContext context = m_upgradeContextFactory.create(cluster, UpgradeType.ROLLING, Direction.UPGRADE, UPGRADE_VERSION, new HashMap<String, Object>()); context.setResolver(m_masterHostResolver); List<UpgradeGroupHolder> groups = m_upgradeHelper.createSequence(upgrade, context); int numPrepareStages = 0; for (UpgradeGroupHolder group : groups) { if (group.name.equals("OOZIE")) { assertTrue(group.items.size() > 0); for (StageWrapper sw : group.items) { if (sw.getText().equalsIgnoreCase("Preparing Oozie Server on h2 (Batch 1 of 2)") || sw.getText().equalsIgnoreCase("Preparing Oozie Server on h3 (Batch 2 of 2)")) { numPrepareStages++; List<TaskWrapper> taskWrappers = sw.getTasks(); assertEquals(1, taskWrappers.size()); List<Task> tasks = taskWrappers.get(0).getTasks(); assertEquals(1, taskWrappers.get(0).getHosts().size()); assertEquals(1, tasks.size()); ExecuteTask task = (ExecuteTask) tasks.get(0); assertTrue("scripts/oozie_server.py".equalsIgnoreCase(task.script)); assertTrue("stop".equalsIgnoreCase(task.function)); } if (sw.getText().equalsIgnoreCase("Preparing Oozie Server on h2")) { numPrepareStages++; List<TaskWrapper> taskWrappers = sw.getTasks(); assertEquals(1, taskWrappers.size()); List<Task> tasks = taskWrappers.get(0).getTasks(); assertEquals(1, taskWrappers.get(0).getHosts().size()); assertEquals(1, tasks.size()); ExecuteTask task = (ExecuteTask) tasks.get(0); assertTrue("scripts/oozie_server_upgrade.py".equalsIgnoreCase(task.script)); assertTrue("upgrade_oozie_database_and_sharelib".equalsIgnoreCase(task.function)); } } } } assertEquals(3, numPrepareStages); }
@Test public void testDowngradeAfterPartialUpgrade() throws Exception { Clusters clusters = injector.getInstance(Clusters.class); ServiceFactory serviceFactory = injector.getInstance(ServiceFactory.class); String clusterName = "c1"; StackId stackId = new StackId("HDP-2.1.1"); clusters.addCluster(clusterName, stackId); Cluster c = clusters.getCluster(clusterName); helper.getOrCreateRepositoryVersion(stackId, c.getDesiredStackVersion().getStackVersion()); c.createClusterVersion(stackId, c.getDesiredStackVersion().getStackVersion(), "admin", RepositoryVersionState.INSTALLING); for (int i = 0; i < 2; i++) { String hostName = "h" + (i+1); clusters.addHost(hostName); Host host = clusters.getHost(hostName); Map<String, String> hostAttributes = new HashMap<>(); hostAttributes.put("os_family", "redhat"); hostAttributes.put("os_release_version", "6"); host.setHostAttributes(hostAttributes); clusters.mapHostToCluster(hostName, clusterName); } c.addService(serviceFactory.createNew(c, "HDFS")); Service s = c.getService("HDFS"); ServiceComponent sc = s.addServiceComponent("NAMENODE"); sc.addServiceComponentHost("h1"); sc.addServiceComponentHost("h2"); List<ServiceComponentHost> schs = c.getServiceComponentHosts("HDFS", "NAMENODE"); assertEquals(2, schs.size()); HostsType type = new HostsType(); type.master = "h1"; type.secondary = "h2"; expect(m_masterHostResolver.getMasterAndHosts("ZOOKEEPER", "ZOOKEEPER_SERVER")).andReturn(null).anyTimes(); expect(m_masterHostResolver.getMasterAndHosts("HDFS", "NAMENODE")).andReturn(type).anyTimes(); expect(m_masterHostResolver.getCluster()).andReturn(c).anyTimes(); replay(m_masterHostResolver); UpgradeContext context = m_upgradeContextFactory.create(c, UpgradeType.ROLLING, Direction.DOWNGRADE, DOWNGRADE_VERSION, new HashMap<String, Object>()); context.setResolver(m_masterHostResolver); Map<String, UpgradePack> upgrades = ambariMetaInfo.getUpgradePacks("HDP", "2.1.1"); assertTrue(upgrades.containsKey("upgrade_direction")); UpgradePack upgrade = upgrades.get("upgrade_direction"); assertNotNull(upgrade); List<UpgradeGroupHolder> groups = m_upgradeHelper.createSequence(upgrade, context); assertEquals(2, groups.size()); UpgradeGroupHolder group = groups.get(0); assertEquals(1, group.items.size()); assertEquals("PRE_POST_CLUSTER", group.name); group = groups.get(1); assertEquals("POST_CLUSTER", group.name); assertEquals(3, group.items.size()); StageWrapper stage = group.items.get(1); assertEquals("NameNode Finalize", stage.getText()); assertEquals(1, stage.getTasks().size()); TaskWrapper task = stage.getTasks().get(0); assertEquals(1, task.getHosts().size()); }
@Test public void testOrchestrationNoServerSideOnDowngrade() throws Exception { Clusters clusters = injector.getInstance(Clusters.class); ServiceFactory serviceFactory = injector.getInstance(ServiceFactory.class); String clusterName = "c1"; StackId stackId = new StackId("HDP-2.1.1"); StackId stackId2 = new StackId("HDP-2.2.0"); clusters.addCluster(clusterName, stackId); Cluster c = clusters.getCluster(clusterName); helper.getOrCreateRepositoryVersion(stackId, c.getDesiredStackVersion().getStackVersion()); helper.getOrCreateRepositoryVersion(stackId2,"2.2.0"); c.createClusterVersion(stackId, c.getDesiredStackVersion().getStackVersion(), "admin", RepositoryVersionState.INSTALLING); for (int i = 0; i < 2; i++) { String hostName = "h" + (i+1); clusters.addHost(hostName); Host host = clusters.getHost(hostName); Map<String, String> hostAttributes = new HashMap<>(); hostAttributes.put("os_family", "redhat"); hostAttributes.put("os_release_version", "6"); host.setHostAttributes(hostAttributes); clusters.mapHostToCluster(hostName, clusterName); } c.addService(serviceFactory.createNew(c, "STORM")); Service s = c.getService("STORM"); ServiceComponent sc = s.addServiceComponent("NIMBUS"); ServiceComponentHost sch1 = sc.addServiceComponentHost("h1"); ServiceComponentHost sch2 = sc.addServiceComponentHost("h2"); UpgradePack upgradePack = new UpgradePack() { @Override public List<Grouping> getGroups(Direction direction) { Grouping g = new Grouping(); OrderService orderService = new OrderService(); orderService.serviceName = "STORM"; orderService.components = Collections.singletonList("NIMBUS"); g.name = "GROUP1"; g.title = "Nimbus Group"; g.services.add(orderService); return Lists.newArrayList(g); } @Override public Map<String, Map<String, ProcessingComponent>> getTasks() { ManualTask mt = new ManualTask(); mt.messages = Lists.newArrayList("My New Message"); ProcessingComponent pc = new ProcessingComponent(); pc.name = "NIMBUS_MESSAGE"; pc.preTasks = Lists.<Task>newArrayList(mt); return Collections.singletonMap("STORM", Collections.singletonMap("NIMBUS", pc)); } }; MasterHostResolver resolver = new MasterHostResolver(m_configHelper, c); UpgradeContext context = m_upgradeContextFactory.create(c, UpgradeType.NON_ROLLING, Direction.UPGRADE, "2.2.0", new HashMap<String, Object>()); context.setResolver(resolver); List<UpgradeGroupHolder> groups = m_upgradeHelper.createSequence(upgradePack, context); assertEquals(1, groups.size()); sch1.setVersion("2.1.1"); sch2.setVersion("2.1.1"); resolver = new MasterHostResolver(m_configHelper, c, "2.1.1"); context = m_upgradeContextFactory.create(c, UpgradeType.NON_ROLLING, Direction.DOWNGRADE, "2.1.1", new HashMap<String, Object>()); context.setResolver(resolver); groups = m_upgradeHelper.createSequence(upgradePack, context); assertTrue(groups.isEmpty()); }
@Test public void testHostGroupingOrchestration() throws Exception { Clusters clusters = injector.getInstance(Clusters.class); ServiceFactory serviceFactory = injector.getInstance(ServiceFactory.class); String clusterName = "c1"; StackId stackId = new StackId("HDP-2.1.1"); StackId stackId2 = new StackId("HDP-2.2.0"); clusters.addCluster(clusterName, stackId); Cluster c = clusters.getCluster(clusterName); helper.getOrCreateRepositoryVersion(stackId, c.getDesiredStackVersion().getStackVersion()); helper.getOrCreateRepositoryVersion(stackId2, "2.2.0"); c.createClusterVersion(stackId, c.getDesiredStackVersion().getStackVersion(), "admin", RepositoryVersionState.INSTALLING); for (int i = 0; i < 2; i++) { String hostName = "h" + (i+1); clusters.addHost(hostName); Host host = clusters.getHost(hostName); Map<String, String> hostAttributes = new HashMap<>(); hostAttributes.put("os_family", "redhat"); hostAttributes.put("os_release_version", "6"); host.setHostAttributes(hostAttributes); clusters.mapHostToCluster(hostName, clusterName); } c.addService(serviceFactory.createNew(c, "ZOOKEEPER")); c.addService(serviceFactory.createNew(c, "HBASE")); Service zookeeper = c.getService("ZOOKEEPER"); Service hbase = c.getService("HBASE"); ServiceComponent zookeeperServer = zookeeper.addServiceComponent("ZOOKEEPER_SERVER"); ServiceComponentHost zookeeperServer1 = zookeeperServer.addServiceComponentHost("h1"); ServiceComponentHost zookeeperServer2 = zookeeperServer.addServiceComponentHost("h2"); ServiceComponent hbaseMaster = hbase.addServiceComponent("HBASE_MASTER"); ServiceComponentHost hbaseMaster1 = hbaseMaster.addServiceComponentHost("h1"); HostOrderItem hostItem = new HostOrderItem(HostOrderActionType.HOST_UPGRADE, Lists.newArrayList("h1", "h2")); HostOrderItem checkItem = new HostOrderItem(HostOrderActionType.SERVICE_CHECK, Lists.newArrayList("ZOOKEEPER", "HBASE")); Grouping g = new HostOrderGrouping(); ((HostOrderGrouping) g).setHostOrderItems(Lists.newArrayList(hostItem, checkItem)); g.title = "Some Title"; UpgradePack upgradePack = new UpgradePack(); Field field = UpgradePack.class.getDeclaredField("groups"); field.setAccessible(true); field.set(upgradePack, Lists.newArrayList(g)); field = UpgradePack.class.getDeclaredField("type" ); field.setAccessible(true); field.set(upgradePack, UpgradeType.HOST_ORDERED); MasterHostResolver resolver = new MasterHostResolver(m_configHelper, c); UpgradeContext context = m_upgradeContextFactory.create(c, UpgradeType.HOST_ORDERED, Direction.UPGRADE, "2.2.0", new HashMap<String, Object>()); context.setResolver(resolver); List<UpgradeGroupHolder> groups = m_upgradeHelper.createSequence(upgradePack, context); assertEquals(1, groups.size()); UpgradeGroupHolder holder = groups.get(0); assertEquals(9, holder.items.size()); for (int i = 0; i < 7; i++) { StageWrapper w = holder.items.get(i); if (i == 0 || i == 4) { assertEquals(StageWrapper.Type.STOP, w.getType()); } else if (i == 1 || i == 5) { assertEquals(StageWrapper.Type.SERVER_SIDE_ACTION, w.getType()); assertEquals(1, w.getTasks().size()); assertEquals(1, w.getTasks().get(0).getTasks().size()); Task t = w.getTasks().get(0).getTasks().get(0); assertEquals(ManualTask.class, t.getClass()); ManualTask mt = (ManualTask) t; assertNotNull(mt.structuredOut); assertTrue(mt.structuredOut.contains("type")); assertTrue(mt.structuredOut.contains(HostOrderItem.HostOrderActionType.HOST_UPGRADE.toString())); assertTrue(mt.structuredOut.contains("host")); assertTrue(mt.structuredOut.contains(i == 1 ? "h1" : "h2")); } else { assertEquals(StageWrapper.Type.RESTART, w.getType()); } } assertEquals(StageWrapper.Type.SERVICE_CHECK, holder.items.get(7).getType()); assertEquals(StageWrapper.Type.SERVICE_CHECK, holder.items.get(8).getType()); zookeeperServer1.setVersion("2.1.1"); zookeeperServer2.setVersion("2.1.1"); hbaseMaster1.setVersion("2.1.1"); resolver = new MasterHostResolver(m_configHelper, c, "2.1.1"); m_upgradeContextFactory.create(c, UpgradeType.HOST_ORDERED, Direction.DOWNGRADE, "2.1.1", new HashMap<String, Object>()); context.setResolver(resolver); groups = m_upgradeHelper.createSequence(upgradePack, context); assertEquals(1, groups.size()); assertEquals(2, groups.get(0).items.size()); zookeeperServer1.setVersion("2.1.1"); zookeeperServer2.setVersion("2.2.0"); hbaseMaster1.setVersion("2.1.1"); resolver = new MasterHostResolver(m_configHelper, c, "2.1.1"); m_upgradeContextFactory.create(c, UpgradeType.HOST_ORDERED, Direction.DOWNGRADE, "2.1.1", new HashMap<String, Object>()); context.setResolver(resolver); groups = m_upgradeHelper.createSequence(upgradePack, context); assertEquals(1, groups.size()); assertEquals(5, groups.get(0).items.size()); }
@Test public void testUpgradeConditions() throws Exception { Map<String, UpgradePack> upgrades = ambariMetaInfo.getUpgradePacks("HDP", "2.2.0"); assertTrue(upgrades.containsKey("upgrade_test_conditions")); UpgradePack upgrade = upgrades.get("upgrade_test_conditions"); assertNotNull(upgrade); Cluster cluster = makeCluster(); UpgradeContext context = m_upgradeContextFactory.create(cluster, UpgradeType.ROLLING, Direction.UPGRADE, UPGRADE_VERSION, new HashMap<String, Object>()); context.setResolver(m_masterHostResolver); List<UpgradeGroupHolder> groups = m_upgradeHelper.createSequence(upgrade, context); assertEquals(0, groups.size()); Map<String, String> fooConfigs = new HashMap<>(); fooConfigs.put("foo-property", "foo-value"); ConfigurationRequest configurationRequest = new ConfigurationRequest(); configurationRequest.setClusterName(cluster.getClusterName()); configurationRequest.setType("foo-site"); configurationRequest.setVersionTag("version1"); configurationRequest.setProperties(fooConfigs); final ClusterRequest clusterRequest = new ClusterRequest(cluster.getClusterId(), cluster.getClusterName(), cluster.getDesiredStackVersion().getStackVersion(), null); clusterRequest.setDesiredConfig(Collections.singletonList(configurationRequest)); m_managementController.updateClusters(Sets.newHashSet(clusterRequest), null); groups = m_upgradeHelper.createSequence(upgrade, context); assertEquals(1, groups.size()); assertEquals("ZOOKEEPER_CONFIG_CONDITION_TEST", groups.get(0).name); cluster.setSecurityType(SecurityType.KERBEROS); groups = m_upgradeHelper.createSequence(upgrade, context); assertEquals(3, groups.size()); }