UpgradeResourceProvider extends AbstractControllerResourceProvider { public void applyStackAndProcessConfigurations(String stackName, Cluster cluster, String version, Direction direction, UpgradePack upgradePack, String userName) throws AmbariException { RepositoryVersionEntity targetRve = s_repoVersionDAO.findByStackNameAndVersion(stackName, version); if (null == targetRve) { LOG.info("Could not find version entity for {}; not setting new configs", version); return; } if (null == userName) { userName = getManagementController().getAuthName(); } StackEntity targetStack = targetRve.getStack(); StackId currentStackId = cluster.getCurrentStackVersion(); StackId desiredStackId = cluster.getDesiredStackVersion(); StackId targetStackId = new StackId(targetStack); switch (direction) { case UPGRADE: if (currentStackId.equals(targetStackId)) { return; } break; case DOWNGRADE: if (desiredStackId.equals(targetStackId)) { return; } break; } Map<String, Map<String, String>> newConfigurationsByType = null; ConfigHelper configHelper = getManagementController().getConfigHelper(); if (direction == Direction.UPGRADE) { Map<String, Map<String, String>> oldStackDefaultConfigurationsByType = configHelper.getDefaultProperties( currentStackId, cluster, true); newConfigurationsByType = configHelper.getDefaultProperties(targetStackId, cluster, true); Set<String> skipConfigTypes = new HashSet<>(); Set<String> upgradePackServices = new HashSet<>(); Set<String> upgradePackConfigTypes = new HashSet<>(); AmbariMetaInfo ambariMetaInfo = s_metaProvider.get(); Map<String, ServiceInfo> stackServicesMap = ambariMetaInfo.getServices( targetStack.getStackName(), targetStack.getStackVersion()); for (Grouping group : upgradePack.getGroups(direction)) { for (UpgradePack.OrderService service : group.services) { if (service.serviceName == null || upgradePackServices.contains(service.serviceName)) { continue; } upgradePackServices.add(service.serviceName); ServiceInfo serviceInfo = stackServicesMap.get(service.serviceName); if (serviceInfo == null) { continue; } Set<String> serviceConfigTypes = serviceInfo.getConfigTypeAttributes().keySet(); for (String serviceConfigType : serviceConfigTypes) { if (!upgradePackConfigTypes.contains(serviceConfigType)) { upgradePackConfigTypes.add(serviceConfigType); } } } } Set<String> servicesNotInUpgradePack = new HashSet<>(stackServicesMap.keySet()); servicesNotInUpgradePack.removeAll(upgradePackServices); for (String serviceNotInUpgradePack : servicesNotInUpgradePack) { ServiceInfo serviceInfo = stackServicesMap.get(serviceNotInUpgradePack); Set<String> configTypesOfServiceNotInUpgradePack = serviceInfo.getConfigTypeAttributes().keySet(); for (String configType : configTypesOfServiceNotInUpgradePack) { if (!upgradePackConfigTypes.contains(configType) && !skipConfigTypes.contains(configType)) { skipConfigTypes.add(configType); } } } Iterator<String> iterator = newConfigurationsByType.keySet().iterator(); while (iterator.hasNext()) { String configType = iterator.next(); if (skipConfigTypes.contains(configType)) { LOG.info("Stack Upgrade: Removing configs for config-type {}", configType); iterator.remove(); } } Map<String, DesiredConfig> existingDesiredConfigurationsByType = cluster.getDesiredConfigs(); for (Map.Entry<String, DesiredConfig> existingEntry : existingDesiredConfigurationsByType.entrySet()) { String configurationType = existingEntry.getKey(); if(skipConfigTypes.contains(configurationType)) { LOG.info("Stack Upgrade: Skipping config-type {} as upgrade-pack contains no updates to its service", configurationType); continue; } Config currentClusterConfig = cluster.getDesiredConfigByType(configurationType); if (null == currentClusterConfig) { continue; } Map<String, String> configurationTypeDefaultConfigurations = oldStackDefaultConfigurationsByType.get( configurationType); if (null == configurationTypeDefaultConfigurations) { configurationTypeDefaultConfigurations = Collections.emptyMap(); } Map<String, String> existingConfigurations = currentClusterConfig.getProperties(); Map<String, String> newDefaultConfigurations = newConfigurationsByType.get( configurationType); if (null == newDefaultConfigurations) { newConfigurationsByType.put(configurationType, existingConfigurations); continue; } else { Iterator<Map.Entry<String, String>> iter = newDefaultConfigurations.entrySet().iterator(); while (iter.hasNext()) { Map.Entry<String, String> entry = iter.next(); if (entry.getValue() == null) { iter.remove(); } } } for (Map.Entry<String, String> existingConfigurationEntry : existingConfigurations.entrySet()) { String existingConfigurationKey = existingConfigurationEntry.getKey(); String existingConfigurationValue = existingConfigurationEntry.getValue(); if (newDefaultConfigurations.containsKey(existingConfigurationKey)) { String newDefaultConfigurationValue = newDefaultConfigurations.get( existingConfigurationKey); if (!StringUtils.equals(existingConfigurationValue, newDefaultConfigurationValue)) { String oldDefaultValue = configurationTypeDefaultConfigurations.get( existingConfigurationKey); if (!StringUtils.equals(existingConfigurationValue, oldDefaultValue)) { newDefaultConfigurations.put(existingConfigurationKey, existingConfigurationValue); } } } else { newDefaultConfigurations.put(existingConfigurationKey, existingConfigurationValue); } } Iterator<Map.Entry<String, String>> newDefaultConfigurationsIterator = newDefaultConfigurations.entrySet().iterator(); while( newDefaultConfigurationsIterator.hasNext() ){ Map.Entry<String, String> newConfigurationEntry = newDefaultConfigurationsIterator.next(); String newConfigurationPropertyName = newConfigurationEntry.getKey(); if (configurationTypeDefaultConfigurations.containsKey(newConfigurationPropertyName) && !existingConfigurations.containsKey(newConfigurationPropertyName)) { LOG.info( "The property {}/{} exists in both {} and {} but is not part of the current set of configurations and will therefore not be included in the configuration merge", configurationType, newConfigurationPropertyName, currentStackId, targetStackId); newDefaultConfigurationsIterator.remove(); } } } } else { cluster.applyLatestConfigurations(cluster.getCurrentStackVersion()); } cluster.setDesiredStackVersion( new StackId(targetStack.getStackName(), targetStack.getStackVersion()), true); if (null != newConfigurationsByType) { configHelper.createConfigTypes(cluster, getManagementController(), newConfigurationsByType, userName, "Configuration created for Upgrade"); } } @Inject  UpgradeResourceProvider(@Assisted AmbariManagementController controller); @Override RequestStatus createResources(final Request request); @Override Set<Resource> getResources(Request request, Predicate predicate); @Override RequestStatus updateResources(final Request request, Predicate predicate); @Override RequestStatus deleteResources(Request request, Predicate predicate); void applyStackAndProcessConfigurations(String stackName, Cluster cluster, String version, Direction direction, UpgradePack upgradePack, String userName); static final String UPGRADE_CLUSTER_NAME; static final String UPGRADE_VERSION; static final String UPGRADE_TYPE; static final String UPGRADE_PACK; static final String UPGRADE_REQUEST_ID; static final String UPGRADE_FROM_VERSION; static final String UPGRADE_TO_VERSION; static final String UPGRADE_DIRECTION; static final String UPGRADE_DOWNGRADE_ALLOWED; static final String UPGRADE_REQUEST_STATUS; static final String UPGRADE_SUSPENDED; static final String UPGRADE_ABORT_REASON; static final String UPGRADE_SKIP_PREREQUISITE_CHECKS; static final String UPGRADE_FAIL_ON_CHECK_WARNINGS; static final String CONST_UPGRADE_GROUP_NAME; static final String CONST_UPGRADE_ITEM_TEXT; static final String CONST_CUSTOM_COMMAND_NAME; }
@Test public void testMergeConfigurations() throws Exception { StackId stack211 = new StackId("HDP-2.1.1"); StackId stack220 = new StackId("HDP-2.2.0"); Map<String, Map<String, String>> stack211Configs = new HashMap<>(); Map<String, String> stack211FooType = new HashMap<>(); Map<String, String> stack211BarType = new HashMap<>(); Map<String, String> stack211BazType = new HashMap<>(); stack211Configs.put("foo-site", stack211FooType); stack211Configs.put("bar-site", stack211BarType); stack211Configs.put("baz-site", stack211BazType); stack211FooType.put("1", "one"); stack211FooType.put("11", "one-one"); stack211BarType.put("2", "two"); stack211BazType.put("3", "three"); Map<String, Map<String, String>> stack220Configs = new HashMap<>(); Map<String, String> stack220FooType = new HashMap<>(); Map<String, String> stack220BazType = new HashMap<>(); Map<String, String> stack220FlumeEnvType = new HashMap<>(); stack220Configs.put("foo-site", stack220FooType); stack220Configs.put("baz-site", stack220BazType); stack220Configs.put("flume-env", stack220FlumeEnvType); stack220FooType.put("1", "one-new"); stack220FooType.put("111", "one-one-one"); stack220BazType.put("3", "three-new"); stack220FlumeEnvType.put("flume_env_key", "flume-env-value"); Map<String, String> clusterFooType = new HashMap<>(); Map<String, String> clusterBarType = new HashMap<>(); Map<String, String> clusterBazType = new HashMap<>(); Config fooConfig = EasyMock.createNiceMock(Config.class); Config barConfig = EasyMock.createNiceMock(Config.class); Config bazConfig = EasyMock.createNiceMock(Config.class); clusterFooType.put("1", "one"); clusterFooType.put("11", "one-one"); clusterBarType.put("2", "two"); clusterBazType.put("3", "three-changed"); expect(fooConfig.getProperties()).andReturn(clusterFooType); expect(barConfig.getProperties()).andReturn(clusterBarType); expect(bazConfig.getProperties()).andReturn(clusterBazType); Map<String, DesiredConfig> desiredConfigurations = new HashMap<>(); desiredConfigurations.put("foo-site", null); desiredConfigurations.put("bar-site", null); desiredConfigurations.put("baz-site", null); Cluster cluster = EasyMock.createNiceMock(Cluster.class); expect(cluster.getCurrentStackVersion()).andReturn(stack211); expect(cluster.getDesiredStackVersion()).andReturn(stack220); expect(cluster.getDesiredConfigs()).andReturn(desiredConfigurations); expect(cluster.getDesiredConfigByType("foo-site")).andReturn(fooConfig); expect(cluster.getDesiredConfigByType("bar-site")).andReturn(barConfig); expect(cluster.getDesiredConfigByType("baz-site")).andReturn(bazConfig); EasyMock.reset(configHelper); expect( configHelper.getDefaultProperties(EasyMock.eq(stack211), EasyMock.anyObject(Cluster.class), EasyMock.anyBoolean())).andReturn( stack211Configs).anyTimes(); expect( configHelper.getDefaultProperties(EasyMock.eq(stack220), EasyMock.anyObject(Cluster.class), EasyMock.anyBoolean())).andReturn( stack220Configs).anyTimes(); Capture<Map<String, Map<String, String>>> expectedConfigurationsCapture = EasyMock.newCapture(); configHelper.createConfigTypes(EasyMock.anyObject(Cluster.class), EasyMock.anyObject(AmbariManagementController.class), EasyMock.capture(expectedConfigurationsCapture), EasyMock.anyObject(String.class), EasyMock.anyObject(String.class)); EasyMock.expectLastCall().once(); EasyMock.replay(configHelper, cluster, fooConfig, barConfig, bazConfig); UpgradeResourceProvider upgradeResourceProvider = createProvider(amc); Map<String, UpgradePack> upgradePacks = ambariMetaInfo.getUpgradePacks("HDP", "2.1.1"); UpgradePack upgrade = upgradePacks.get("upgrade_to_new_stack"); upgradeResourceProvider.applyStackAndProcessConfigurations(stack211.getStackName(), cluster, "2.2.0.0", Direction.UPGRADE, upgrade, "admin"); Map<String, Map<String, String>> expectedConfigurations = expectedConfigurationsCapture.getValue(); Map<String, String> expectedFooType = expectedConfigurations.get("foo-site"); Map<String, String> expectedBarType = expectedConfigurations.get("bar-site"); Map<String, String> expectedBazType = expectedConfigurations.get("baz-site"); assertFalse(expectedConfigurations.containsKey("flume-env")); assertEquals("one-new", expectedFooType.get("1")); assertEquals("one-one", expectedFooType.get("11")); assertEquals("two", expectedBarType.get("2")); assertEquals("three-changed", expectedBazType.get("3")); }