@Test public void testGetResources() throws Exception { Resource.Type type = Resource.Type.ClientConfig; AmbariManagementController managementController = createNiceMock(AmbariManagementController.class); Clusters clusters = createNiceMock(Clusters.class); Cluster cluster = createNiceMock(Cluster.class); AmbariMetaInfo ambariMetaInfo = createNiceMock(AmbariMetaInfo.class); StackId stackId = createNiceMock(StackId.class); ComponentInfo componentInfo = createNiceMock(ComponentInfo.class); ServiceInfo serviceInfo = createNiceMock(ServiceInfo.class); CommandScriptDefinition commandScriptDefinition = createNiceMock(CommandScriptDefinition.class); Config clusterConfig = createNiceMock(Config.class); DesiredConfig desiredConfig = createNiceMock(DesiredConfig.class); Host host = createNiceMock(Host.class); Service service = createNiceMock(Service.class); ServiceComponent serviceComponent = createNiceMock(ServiceComponent.class); ServiceComponentHost serviceComponentHost = createNiceMock(ServiceComponentHost.class); ServiceOsSpecific serviceOsSpecific = createNiceMock(ServiceOsSpecific.class); ConfigHelper configHelper = createNiceMock(ConfigHelper.class); Configuration configuration = PowerMock.createStrictMockAndExpectNew(Configuration.class); Map<String, String> configMap = createNiceMock(Map.class); File newFile = File.createTempFile("config",".json",new File("/tmp/")); newFile.deleteOnExit(); Runtime runtime = createMock(Runtime.class); Process process = createNiceMock(Process.class); Map<String, DesiredConfig> desiredConfigMap = new HashMap<String, DesiredConfig>(); desiredConfigMap.put("hive-site", desiredConfig); Map<String, Map<String, String>> allConfigTags = new HashMap<String, Map<String, String>>(); Map<String, Map<String, String>> properties = new HashMap<String, Map<String, String>>(); Map<String, Map<String, String>> configTags = new HashMap<String, Map<String, String>>(); Map<String, Map<String, Map<String, String>>> attributes = new HashMap<String, Map<String, Map<String, String>>>(); ClientConfigFileDefinition clientConfigFileDefinition = new ClientConfigFileDefinition(); clientConfigFileDefinition.setDictionaryName("pig-env"); clientConfigFileDefinition.setFileName("pig-env.sh"); clientConfigFileDefinition.setType("env"); List <ClientConfigFileDefinition> clientConfigFileDefinitionList = new LinkedList<ClientConfigFileDefinition>(); clientConfigFileDefinitionList.add(clientConfigFileDefinition); ResourceProvider provider = AbstractControllerResourceProvider.getResourceProvider( type, PropertyHelper.getPropertyIds(type), PropertyHelper.getKeyPropertyIds(type), managementController); String clusterName = "C1"; String serviceName = "PIG"; String componentName = "PIG"; String displayName = "Pig Client"; String hostName = "Host100"; String publicHostname = "Host100"; String desiredState = "INSTALLED"; String stackName = "S1"; String stackVersion = "V1"; String stackRoot="/tmp/stacks/S1/V1"; String packageFolder="PIG/package"; if (System.getProperty("os.name").contains("Windows")) { stackRoot = "C:\\tmp\\stacks\\S1\\V1"; packageFolder = "PIG\\package"; } HashMap<String, Host> hosts = new HashMap<String, Host>(); hosts.put(hostName, host); HashMap<String, Service> services = new HashMap<String, Service>(); services.put(serviceName,service); HashMap<String, ServiceComponent> serviceComponentMap = new HashMap<String, ServiceComponent>(); serviceComponentMap.put(componentName,serviceComponent); HashMap<String, ServiceComponentHost> serviceComponentHosts = new HashMap<String, ServiceComponentHost>(); serviceComponentHosts.put(componentName, serviceComponentHost); HashMap<String, ServiceOsSpecific> serviceOsSpecificHashMap = new HashMap<String, ServiceOsSpecific>(); serviceOsSpecificHashMap.put("key",serviceOsSpecific); ServiceComponentHostResponse shr1 = new ServiceComponentHostResponse(clusterName, serviceName, componentName, displayName, hostName, publicHostname,desiredState, "", null, null, null); Set<ServiceComponentHostResponse> responses = new LinkedHashSet<ServiceComponentHostResponse>(); responses.add(shr1); Map<String, String> returnConfigMap = new HashMap<String, String>(); returnConfigMap.put(Configuration.SERVER_TMP_DIR.getKey(), Configuration.SERVER_TMP_DIR.getDefaultValue()); returnConfigMap.put(Configuration.AMBARI_PYTHON_WRAP.getKey(), Configuration.AMBARI_PYTHON_WRAP.getDefaultValue()); expect(managementController.getConfigHelper()).andReturn(configHelper); expect(managementController.getAmbariMetaInfo()).andReturn(ambariMetaInfo).anyTimes(); expect(managementController.getClusters()).andReturn(clusters).anyTimes(); expect(clusters.getCluster(clusterName)).andReturn(cluster).anyTimes(); expect(configHelper.getEffectiveConfigProperties(cluster, configTags)).andReturn(properties); expect(configHelper.getEffectiveConfigAttributes(cluster, configTags)).andReturn(attributes); expect(configMap.get(Configuration.SERVER_TMP_DIR.getKey())).andReturn(Configuration.SERVER_TMP_DIR.getDefaultValue()); expect(configMap.get(Configuration.AMBARI_PYTHON_WRAP.getKey())).andReturn(Configuration.AMBARI_PYTHON_WRAP.getDefaultValue()); expect(configuration.getConfigsMap()).andReturn(returnConfigMap); expect(configuration.getResourceDirPath()).andReturn(stackRoot); expect(configuration.getJavaVersion()).andReturn(8); expect(configuration.areHostsSysPrepped()).andReturn("false"); expect(configuration.isAgentStackRetryOnInstallEnabled()).andReturn("false"); expect(configuration.getAgentStackRetryOnInstallCount()).andReturn("5"); expect(configuration.getExternalScriptThreadPoolSize()).andReturn(Configuration.THREAD_POOL_SIZE_FOR_EXTERNAL_SCRIPT.getDefaultValue()); expect(configuration.getExternalScriptTimeout()).andReturn(Configuration.EXTERNAL_SCRIPT_TIMEOUT.getDefaultValue()); Map<String,String> props = new HashMap<String, String>(); props.put("key","value"); expect(clusterConfig.getProperties()).andReturn(props); expect(configHelper.getEffectiveDesiredTags(cluster, null)).andReturn(allConfigTags); expect(cluster.getClusterName()).andReturn(clusterName); expect(managementController.getHostComponents(EasyMock.<Set<ServiceComponentHostRequest>>anyObject())).andReturn(responses).anyTimes(); expect(cluster.getCurrentStackVersion()).andReturn(stackId); PowerMock.mockStaticPartial(StageUtils.class, "getClusterHostInfo"); Map<String, Set<String>> clusterHostInfo = new HashMap<String, Set<String>>(); Set<String> all_hosts = new HashSet<String>(Arrays.asList("Host100","Host101","Host102")); Set<String> some_hosts = new HashSet<String>(Arrays.asList("0-1","2")); Set<String> ohter_hosts = new HashSet<String>(Arrays.asList("0,1")); Set<String> clusterHostTypes = new HashSet<String>(Arrays.asList("nm_hosts", "hs_host", "namenode_host", "rm_host", "snamenode_host", "slave_hosts", "zookeeper_hosts")); for (String hostTypes: clusterHostTypes) { if (hostTypes.equals("slave_hosts")) { clusterHostInfo.put(hostTypes, ohter_hosts); } else { clusterHostInfo.put(hostTypes, some_hosts); } } Map<String, Host> stringHostMap = new HashMap<String, Host>(); stringHostMap.put(hostName, host); clusterHostInfo.put("all_hosts",all_hosts); expect(StageUtils.getClusterHostInfo(cluster)).andReturn(clusterHostInfo); expect(stackId.getStackName()).andReturn(stackName).anyTimes(); expect(stackId.getStackVersion()).andReturn(stackVersion).anyTimes(); expect(ambariMetaInfo.getComponent(stackName, stackVersion, serviceName, componentName)).andReturn(componentInfo); expect(ambariMetaInfo.getService(stackName, stackVersion, serviceName)).andReturn(serviceInfo); expect(serviceInfo.getServicePackageFolder()).andReturn(packageFolder); expect(ambariMetaInfo.getComponent((String) anyObject(), (String) anyObject(), (String) anyObject(), (String) anyObject())).andReturn(componentInfo).anyTimes(); expect(componentInfo.getCommandScript()).andReturn(commandScriptDefinition); expect(componentInfo.getClientConfigFiles()).andReturn(clientConfigFileDefinitionList); expect(cluster.getConfig("hive-site", null)).andReturn(clusterConfig); expect(clusterConfig.getType()).andReturn("hive-site").anyTimes(); expect(cluster.getDesiredConfigs()).andReturn(desiredConfigMap); expect(clusters.getHost(hostName)).andReturn(host); HashMap<String, String> rcaParams = new HashMap<String, String>(); rcaParams.put("key","value"); expect(managementController.getRcaParameters()).andReturn(rcaParams).anyTimes(); expect(ambariMetaInfo.getService(stackName, stackVersion, serviceName)).andReturn(serviceInfo); expect(serviceInfo.getOsSpecifics()).andReturn(new HashMap<String, ServiceOsSpecific>()).anyTimes(); Set<String> userSet = new HashSet<String>(); userSet.add("hdfs"); expect(configHelper.getPropertyValuesWithPropertyType( stackId, PropertyInfo.PropertyType.USER, cluster, desiredConfigMap)).andReturn(userSet); Map<PropertyInfo, String> userProperties = new HashMap<>(); Map<PropertyInfo, String> groupProperties = new HashMap<>(); PropertyInfo userProperty = new PropertyInfo(); userProperty.setFilename("hadoop-env.xml"); userProperty.setName("hdfs-user"); userProperty.setValue("hdfsUser"); PropertyInfo groupProperty = new PropertyInfo(); groupProperty.setFilename("hadoop-env.xml"); groupProperty.setName("hdfs-group"); groupProperty.setValue("hdfsGroup"); ValueAttributesInfo valueAttributesInfo = new ValueAttributesInfo(); valueAttributesInfo.setType("user"); Set<UserGroupInfo> userGroupEntries = new HashSet<>(); UserGroupInfo userGroupInfo = new UserGroupInfo(); userGroupInfo.setType("hadoop-env"); userGroupInfo.setName("hdfs-group"); userGroupEntries.add(userGroupInfo); valueAttributesInfo.setUserGroupEntries(userGroupEntries); userProperty.setPropertyValueAttributes(valueAttributesInfo); userProperties.put(userProperty, "hdfsUser"); groupProperties.put(groupProperty, "hdfsGroup"); Map<String, Set<String>> userGroupsMap = new HashMap<>(); userGroupsMap.put("hdfsUser", new HashSet<String>(Arrays.asList("hdfsGroup"))); expect(configHelper.getPropertiesWithPropertyType( stackId, PropertyInfo.PropertyType.USER, cluster, desiredConfigMap)).andReturn(userProperties).anyTimes(); expect(configHelper.getPropertiesWithPropertyType( stackId, PropertyInfo.PropertyType.GROUP, cluster, desiredConfigMap)).andReturn(groupProperties).anyTimes(); expect(configHelper.createUserGroupsMap(stackId, cluster, desiredConfigMap)).andReturn(userGroupsMap).anyTimes(); PowerMock.expectNew(File.class, new Class<?>[]{String.class}, anyObject(String.class)).andReturn(newFile).anyTimes(); String commandLine = "ambari-python-wrap /tmp/stacks/S1/V1/PIG/package/null generate_configs "+newFile + " /tmp/stacks/S1/V1/PIG/package /var/lib/ambari-server/tmp/structured-out.json " + "INFO /var/lib/ambari-server/tmp"; if (System.getProperty("os.name").contains("Windows")) { commandLine = "ambari-python-wrap " + stackRoot + "\\PIG\\package\\null generate_configs null " + stackRoot + "\\PIG\\package /var/lib/ambari-server/tmp\\structured-out.json " + "INFO /var/lib/ambari-server/tmp"; } ProcessBuilder processBuilder = PowerMock.createNiceMock(ProcessBuilder.class); PowerMock.expectNew(ProcessBuilder.class,Arrays.asList(commandLine.split("\\s+"))).andReturn(processBuilder).once(); expect(processBuilder.start()).andReturn(process).once(); InputStream inputStream = new ByteArrayInputStream("some logging info".getBytes()); expect(process.getInputStream()).andReturn(inputStream); ClientConfigResourceProvider.TarUtils tarUtilMock = PowerMockito.mock(ClientConfigResourceProvider.TarUtils.class); whenNew(ClientConfigResourceProvider.TarUtils.class).withAnyArguments().thenReturn(tarUtilMock); tarUtilMock.tarConfigFiles(); expectLastCall().once(); Request request = PropertyHelper.getReadRequest(ClientConfigResourceProvider.COMPONENT_CLUSTER_NAME_PROPERTY_ID, "c1", ClientConfigResourceProvider.COMPONENT_COMPONENT_NAME_PROPERTY_ID, ClientConfigResourceProvider.COMPONENT_SERVICE_NAME_PROPERTY_ID); Predicate predicate = new PredicateBuilder().property(ClientConfigResourceProvider.COMPONENT_CLUSTER_NAME_PROPERTY_ID). equals("c1").and().property(ClientConfigResourceProvider.COMPONENT_SERVICE_NAME_PROPERTY_ID).equals("PIG").toPredicate(); replay(managementController, clusters, cluster, ambariMetaInfo, stackId, componentInfo, commandScriptDefinition, clusterConfig, host, service, serviceComponent, serviceComponentHost, serviceInfo, configHelper, runtime, process, configMap); PowerMock.replayAll(); Set<Resource> resources = provider.getResources(request, predicate); assertFalse(resources.isEmpty()); String str = FileUtils.readFileToString(newFile); assertTrue(str.contains("\"user_groups\":\"{\\\"hdfsUser\\\":[\\\"hdfsGroup\\\"]}")); verify(managementController, clusters, cluster, ambariMetaInfo, stackId, componentInfo,commandScriptDefinition, clusterConfig, host, service, serviceComponent, serviceComponentHost, serviceInfo, configHelper, runtime, process); PowerMock.verifyAll(); }
@Override @Transactional public Set<Resource> getResources(Request request, Predicate predicate) throws SystemException, UnsupportedPropertyException, NoSuchResourceException, NoSuchParentResourceException { Set<Resource> resources = new HashSet<Resource>(); final Set<ServiceComponentHostRequest> requests = new HashSet<ServiceComponentHostRequest>(); for (Map<String, Object> propertyMap : getPropertyMaps(predicate)) { requests.add(getRequest(propertyMap)); } Set<ServiceComponentHostResponse> responses = null; try { responses = getResources(new Command<Set<ServiceComponentHostResponse>>() { @Override public Set<ServiceComponentHostResponse> invoke() throws AmbariException { return getManagementController().getHostComponents(requests); } }); } catch (Exception e) { throw new SystemException("Failed to get components ", e); } Map<String,ServiceComponentHostResponse> componentMap = new HashMap<>(); for (ServiceComponentHostResponse resp: responses) { String componentName = resp.getComponentName(); if (!componentMap.containsKey(componentName)) { componentMap.put(resp.getComponentName(),resp); } } ServiceComponentHostRequest schRequest = requests.iterator().next(); String requestComponentName = schRequest.getComponentName(); String requestServiceName = schRequest.getServiceName(); String requestHostName = schRequest.getHostname(); Map<String,List<ServiceComponentHostResponse>> serviceToComponentMap = new HashMap<String,List<ServiceComponentHostResponse>>(); List <ServiceComponentHostResponse> schWithConfigFiles = new ArrayList<>(); Configuration configs = new Configuration(); Map<String, String> configMap = configs.getConfigsMap(); String TMP_PATH = configMap.get(Configuration.SERVER_TMP_DIR.getKey()); String pythonCmd = configMap.get(Configuration.AMBARI_PYTHON_WRAP.getKey()); List<String> pythonCompressFilesCmds = new ArrayList<>(); for (ServiceComponentHostResponse response : componentMap.values()){ AmbariManagementController managementController = getManagementController(); ConfigHelper configHelper = managementController.getConfigHelper(); Cluster cluster = null; Clusters clusters = managementController.getClusters(); try { cluster = clusters.getCluster(response.getClusterName()); StackId stackId = cluster.getCurrentStackVersion(); String serviceName = response.getServiceName(); String componentName = response.getComponentName(); String hostName = response.getHostname(); ComponentInfo componentInfo = null; String packageFolder = null; componentInfo = managementController.getAmbariMetaInfo(). getComponent(stackId.getStackName(), stackId.getStackVersion(), serviceName, componentName); packageFolder = managementController.getAmbariMetaInfo(). getService(stackId.getStackName(), stackId.getStackVersion(), serviceName).getServicePackageFolder(); String commandScript = componentInfo.getCommandScript().getScript(); List<ClientConfigFileDefinition> clientConfigFiles = componentInfo.getClientConfigFiles(); if (clientConfigFiles == null) { if (componentMap.size() == 1) { throw new SystemException("No configuration files defined for the component " + componentInfo.getName()); } else { LOG.debug(String.format("No configuration files defined for the component %s",componentInfo.getName())); continue; } } schWithConfigFiles.add(response); if (serviceToComponentMap.containsKey(response.getServiceName())) { List <ServiceComponentHostResponse> schResponseList = serviceToComponentMap.get(serviceName); schResponseList.add(response); } else { List <ServiceComponentHostResponse> schResponseList = new ArrayList<>(); schResponseList.add(response); serviceToComponentMap.put(serviceName,schResponseList); } String resourceDirPath = configs.getResourceDirPath(); String packageFolderAbsolute = resourceDirPath + File.separator + packageFolder; String commandScriptAbsolute = packageFolderAbsolute + File.separator + commandScript; Map<String, Map<String, String>> configurations = new TreeMap<String, Map<String, String>>(); Map<String, Long> configVersions = new TreeMap<String, Long>(); Map<String, Map<PropertyType, Set<String>>> configPropertiesTypes = new TreeMap<>(); Map<String, Map<String, Map<String, String>>> configurationAttributes = new TreeMap<String, Map<String, Map<String, String>>>(); Map<String, DesiredConfig> desiredClusterConfigs = cluster.getDesiredConfigs(); for (Map.Entry<String, DesiredConfig> desiredConfigEntry : desiredClusterConfigs.entrySet()) { String configType = desiredConfigEntry.getKey(); DesiredConfig desiredConfig = desiredConfigEntry.getValue(); Config clusterConfig = cluster.getConfig(configType, desiredConfig.getTag()); if (clusterConfig != null) { Map<String, String> props = new HashMap<String, String>(clusterConfig.getProperties()); Map<String, Map<String, String>> allConfigTags = null; allConfigTags = configHelper .getEffectiveDesiredTags(cluster, schRequest.getHostname()); Map<String, Map<String, String>> configTags = new HashMap<String, Map<String, String>>(); for (Map.Entry<String, Map<String, String>> entry : allConfigTags.entrySet()) { if (entry.getKey().equals(clusterConfig.getType())) { configTags.put(clusterConfig.getType(), entry.getValue()); } } Map<String, Map<String, String>> properties = configHelper .getEffectiveConfigProperties(cluster, configTags); if (!properties.isEmpty()) { for (Map<String, String> propertyMap : properties.values()) { props.putAll(propertyMap); } } configurations.put(clusterConfig.getType(), props); configVersions.put(clusterConfig.getType(), clusterConfig.getVersion()); configPropertiesTypes.put(clusterConfig.getType(), clusterConfig.getPropertiesTypes()); Map<String, Map<String, String>> attrs = new TreeMap<String, Map<String, String>>(); configHelper.cloneAttributesMap(clusterConfig.getPropertiesAttributes(), attrs); Map<String, Map<String, Map<String, String>>> attributes = configHelper .getEffectiveConfigAttributes(cluster, configTags); for (Map<String, Map<String, String>> attributesMap : attributes.values()) { configHelper.cloneAttributesMap(attributesMap, attrs); } configurationAttributes.put(clusterConfig.getType(), attrs); } } ConfigHelper.processHiddenAttribute(configurations, configurationAttributes, componentName, true); for (Map.Entry<String, Map<String, Map<String, String>>> configurationAttributesEntry : configurationAttributes.entrySet()) { Map<String, Map<String, String>> attrs = configurationAttributesEntry.getValue(); attrs.remove("hidden"); } for (Map.Entry<String, Map<String, String>> configEntry : configurations.entrySet()) { String configType = configEntry.getKey(); Map<String, String> configProperties = configEntry.getValue(); Long configVersion = configVersions.get(configType); Map<PropertyType, Set<String>> propertiesTypes = configPropertiesTypes.get(configType); SecretReference.replacePasswordsWithReferences(propertiesTypes, configProperties, configType, configVersion); } Map<String, Set<String>> clusterHostInfo = null; ServiceInfo serviceInfo = null; String osFamily = null; clusterHostInfo = StageUtils.getClusterHostInfo(cluster); serviceInfo = managementController.getAmbariMetaInfo().getService(stackId.getStackName(), stackId.getStackVersion(), serviceName); try { clusterHostInfo = StageUtils.substituteHostIndexes(clusterHostInfo); } catch (AmbariException e) { throw new SystemException(e.getMessage(), e); } osFamily = clusters.getHost(hostName).getOsFamily(); TreeMap<String, String> hostLevelParams = new TreeMap<String, String>(); hostLevelParams.put(JDK_LOCATION, managementController.getJdkResourceUrl()); hostLevelParams.put(JAVA_HOME, managementController.getJavaHome()); hostLevelParams.put(JAVA_VERSION, String.valueOf(configs.getJavaVersion())); hostLevelParams.put(JDK_NAME, managementController.getJDKName()); hostLevelParams.put(JCE_NAME, managementController.getJCEName()); hostLevelParams.put(STACK_NAME, stackId.getStackName()); hostLevelParams.put(STACK_VERSION, stackId.getStackVersion()); hostLevelParams.put(DB_NAME, managementController.getServerDB()); hostLevelParams.put(MYSQL_JDBC_URL, managementController.getMysqljdbcUrl()); hostLevelParams.put(ORACLE_JDBC_URL, managementController.getOjdbcUrl()); hostLevelParams.put(HOST_SYS_PREPPED, configs.areHostsSysPrepped()); hostLevelParams.putAll(managementController.getRcaParameters()); hostLevelParams.put(AGENT_STACK_RETRY_ON_UNAVAILABILITY, configs.isAgentStackRetryOnInstallEnabled()); hostLevelParams.put(AGENT_STACK_RETRY_COUNT, configs.getAgentStackRetryOnInstallCount()); ServiceOsSpecific anyOs = null; if (serviceInfo.getOsSpecifics().containsKey(AmbariMetaInfo.ANY_OS)) { anyOs = serviceInfo.getOsSpecifics().get(AmbariMetaInfo.ANY_OS); } ServiceOsSpecific hostOs = populateServicePackagesInfo(serviceInfo, hostLevelParams, osFamily); List<ServiceOsSpecific.Package> packages = new ArrayList<ServiceOsSpecific.Package>(); if (anyOs != null) { packages.addAll(anyOs.getPackages()); } if (hostOs != null) { packages.addAll(hostOs.getPackages()); } String packageList = gson.toJson(packages); hostLevelParams.put(PACKAGE_LIST, packageList); Set<String> userSet = configHelper.getPropertyValuesWithPropertyType(stackId, PropertyType.USER, cluster, desiredClusterConfigs); String userList = gson.toJson(userSet); hostLevelParams.put(USER_LIST, userList); Map<String, Set<String>> userGroupsMap = configHelper.createUserGroupsMap( stackId, cluster, desiredClusterConfigs); String userGroups = gson.toJson(userGroupsMap); hostLevelParams.put(USER_GROUPS,userGroups); Set<String> groupSet = configHelper.getPropertyValuesWithPropertyType(stackId, PropertyType.GROUP, cluster, desiredClusterConfigs); String groupList = gson.toJson(groupSet); hostLevelParams.put(GROUP_LIST, groupList); Set<String> notManagedHdfsPathSet = configHelper.getPropertyValuesWithPropertyType(stackId, PropertyType.NOT_MANAGED_HDFS_PATH, cluster, desiredClusterConfigs); String notManagedHdfsPathList = gson.toJson(notManagedHdfsPathSet); hostLevelParams.put(NOT_MANAGED_HDFS_PATH_LIST, notManagedHdfsPathList); String jsonConfigurations = null; Map<String, Object> commandParams = new HashMap<String, Object>(); List<Map<String, String>> xmlConfigs = new LinkedList<Map<String, String>>(); List<Map<String, String>> envConfigs = new LinkedList<Map<String, String>>(); List<Map<String, String>> propertiesConfigs = new LinkedList<Map<String, String>>(); for (ClientConfigFileDefinition clientConfigFile : clientConfigFiles) { Map<String, String> fileDict = new HashMap<String, String>(); fileDict.put(clientConfigFile.getFileName(), clientConfigFile.getDictionaryName()); if (clientConfigFile.getType().equals("xml")) { xmlConfigs.add(fileDict); } else if (clientConfigFile.getType().equals("env")) { envConfigs.add(fileDict); } else if (clientConfigFile.getType().equals("properties")) { propertiesConfigs.add(fileDict); } } commandParams.put("xml_configs_list", xmlConfigs); commandParams.put("env_configs_list", envConfigs); commandParams.put("properties_configs_list", propertiesConfigs); commandParams.put("output_file", componentName + "-configs" + Configuration.DEF_ARCHIVE_EXTENSION); Map<String, Object> jsonContent = new TreeMap<String, Object>(); jsonContent.put("configurations", configurations); jsonContent.put("configuration_attributes", configurationAttributes); jsonContent.put("commandParams", commandParams); jsonContent.put("clusterHostInfo", clusterHostInfo); jsonContent.put("hostLevelParams", hostLevelParams); jsonContent.put("hostname", hostName); jsonContent.put("clusterName", cluster.getClusterName()); jsonConfigurations = gson.toJson(jsonContent); File jsonFileName = new File(TMP_PATH + File.separator + componentName + "-configuration.json"); File tmpDirectory = new File(jsonFileName.getParent()); if (!tmpDirectory.exists()) { try { tmpDirectory.mkdirs(); tmpDirectory.setWritable(true, true); tmpDirectory.setReadable(true, true); } catch (SecurityException se) { throw new SystemException("Failed to get temporary directory to store configurations", se); } } PrintWriter printWriter = null; try { printWriter = new PrintWriter(jsonFileName.getAbsolutePath()); printWriter.print(jsonConfigurations); printWriter.close(); } catch (FileNotFoundException e) { throw new SystemException("Failed to write configurations to json file ", e); } String cmd = pythonCmd + " " + commandScriptAbsolute + " generate_configs " + jsonFileName.getAbsolutePath() + " " + packageFolderAbsolute + " " + TMP_PATH + File.separator + "structured-out.json" + " INFO " + TMP_PATH; pythonCompressFilesCmds.add(cmd); } catch (AmbariException e) { throw new SystemException("Controller error ", e); } } if (schWithConfigFiles.isEmpty()) { throw new SystemException("No configuration files defined for any component" ); } Integer totalCommands = pythonCompressFilesCmds.size() * 2; Integer threadPoolSize = Math.min(totalCommands,configs.getExternalScriptThreadPoolSize()); ExecutorService processExecutor = Executors.newFixedThreadPool(threadPoolSize); List<CommandLineThreadWrapper> pythonCmdThreads = executeCommands(processExecutor, pythonCompressFilesCmds); Integer timeout = configs.getExternalScriptTimeout(); waitForAllThreadsToJoin(processExecutor, pythonCmdThreads, timeout); if (StringUtils.isEmpty(requestComponentName)) { TarUtils tarUtils; String fileName; List <ServiceComponentHostResponse> schToTarConfigFiles = schWithConfigFiles; if (StringUtils.isNotEmpty(requestHostName)) { fileName = requestHostName + "(" + Resource.InternalType.Host.toString().toUpperCase()+")"; } else if (StringUtils.isNotEmpty(requestServiceName)) { fileName = requestServiceName + "(" + Resource.InternalType.Service.toString().toUpperCase()+")"; schToTarConfigFiles = serviceToComponentMap.get(requestServiceName); } else { fileName = schRequest.getClusterName() + "(" + Resource.InternalType.Cluster.toString().toUpperCase()+")"; } tarUtils = new TarUtils(TMP_PATH, fileName, schToTarConfigFiles); tarUtils.tarConfigFiles(); } Resource resource = new ResourceImpl(Resource.Type.ClientConfig); resources.add(resource); return resources; }
ClientConfigResourceProvider extends AbstractControllerResourceProvider { @Override @Transactional public Set<Resource> getResources(Request request, Predicate predicate) throws SystemException, UnsupportedPropertyException, NoSuchResourceException, NoSuchParentResourceException { Set<Resource> resources = new HashSet<Resource>(); final Set<ServiceComponentHostRequest> requests = new HashSet<ServiceComponentHostRequest>(); for (Map<String, Object> propertyMap : getPropertyMaps(predicate)) { requests.add(getRequest(propertyMap)); } Set<ServiceComponentHostResponse> responses = null; try { responses = getResources(new Command<Set<ServiceComponentHostResponse>>() { @Override public Set<ServiceComponentHostResponse> invoke() throws AmbariException { return getManagementController().getHostComponents(requests); } }); } catch (Exception e) { throw new SystemException("Failed to get components ", e); } Map<String,ServiceComponentHostResponse> componentMap = new HashMap<>(); for (ServiceComponentHostResponse resp: responses) { String componentName = resp.getComponentName(); if (!componentMap.containsKey(componentName)) { componentMap.put(resp.getComponentName(),resp); } } ServiceComponentHostRequest schRequest = requests.iterator().next(); String requestComponentName = schRequest.getComponentName(); String requestServiceName = schRequest.getServiceName(); String requestHostName = schRequest.getHostname(); Map<String,List<ServiceComponentHostResponse>> serviceToComponentMap = new HashMap<String,List<ServiceComponentHostResponse>>(); List <ServiceComponentHostResponse> schWithConfigFiles = new ArrayList<>(); Configuration configs = new Configuration(); Map<String, String> configMap = configs.getConfigsMap(); String TMP_PATH = configMap.get(Configuration.SERVER_TMP_DIR.getKey()); String pythonCmd = configMap.get(Configuration.AMBARI_PYTHON_WRAP.getKey()); List<String> pythonCompressFilesCmds = new ArrayList<>(); for (ServiceComponentHostResponse response : componentMap.values()){ AmbariManagementController managementController = getManagementController(); ConfigHelper configHelper = managementController.getConfigHelper(); Cluster cluster = null; Clusters clusters = managementController.getClusters(); try { cluster = clusters.getCluster(response.getClusterName()); StackId stackId = cluster.getCurrentStackVersion(); String serviceName = response.getServiceName(); String componentName = response.getComponentName(); String hostName = response.getHostname(); ComponentInfo componentInfo = null; String packageFolder = null; componentInfo = managementController.getAmbariMetaInfo(). getComponent(stackId.getStackName(), stackId.getStackVersion(), serviceName, componentName); packageFolder = managementController.getAmbariMetaInfo(). getService(stackId.getStackName(), stackId.getStackVersion(), serviceName).getServicePackageFolder(); String commandScript = componentInfo.getCommandScript().getScript(); List<ClientConfigFileDefinition> clientConfigFiles = componentInfo.getClientConfigFiles(); if (clientConfigFiles == null) { if (componentMap.size() == 1) { throw new SystemException("No configuration files defined for the component " + componentInfo.getName()); } else { LOG.debug(String.format("No configuration files defined for the component %s",componentInfo.getName())); continue; } } schWithConfigFiles.add(response); if (serviceToComponentMap.containsKey(response.getServiceName())) { List <ServiceComponentHostResponse> schResponseList = serviceToComponentMap.get(serviceName); schResponseList.add(response); } else { List <ServiceComponentHostResponse> schResponseList = new ArrayList<>(); schResponseList.add(response); serviceToComponentMap.put(serviceName,schResponseList); } String resourceDirPath = configs.getResourceDirPath(); String packageFolderAbsolute = resourceDirPath + File.separator + packageFolder; String commandScriptAbsolute = packageFolderAbsolute + File.separator + commandScript; Map<String, Map<String, String>> configurations = new TreeMap<String, Map<String, String>>(); Map<String, Long> configVersions = new TreeMap<String, Long>(); Map<String, Map<PropertyType, Set<String>>> configPropertiesTypes = new TreeMap<>(); Map<String, Map<String, Map<String, String>>> configurationAttributes = new TreeMap<String, Map<String, Map<String, String>>>(); Map<String, DesiredConfig> desiredClusterConfigs = cluster.getDesiredConfigs(); for (Map.Entry<String, DesiredConfig> desiredConfigEntry : desiredClusterConfigs.entrySet()) { String configType = desiredConfigEntry.getKey(); DesiredConfig desiredConfig = desiredConfigEntry.getValue(); Config clusterConfig = cluster.getConfig(configType, desiredConfig.getTag()); if (clusterConfig != null) { Map<String, String> props = new HashMap<String, String>(clusterConfig.getProperties()); Map<String, Map<String, String>> allConfigTags = null; allConfigTags = configHelper .getEffectiveDesiredTags(cluster, schRequest.getHostname()); Map<String, Map<String, String>> configTags = new HashMap<String, Map<String, String>>(); for (Map.Entry<String, Map<String, String>> entry : allConfigTags.entrySet()) { if (entry.getKey().equals(clusterConfig.getType())) { configTags.put(clusterConfig.getType(), entry.getValue()); } } Map<String, Map<String, String>> properties = configHelper .getEffectiveConfigProperties(cluster, configTags); if (!properties.isEmpty()) { for (Map<String, String> propertyMap : properties.values()) { props.putAll(propertyMap); } } configurations.put(clusterConfig.getType(), props); configVersions.put(clusterConfig.getType(), clusterConfig.getVersion()); configPropertiesTypes.put(clusterConfig.getType(), clusterConfig.getPropertiesTypes()); Map<String, Map<String, String>> attrs = new TreeMap<String, Map<String, String>>(); configHelper.cloneAttributesMap(clusterConfig.getPropertiesAttributes(), attrs); Map<String, Map<String, Map<String, String>>> attributes = configHelper .getEffectiveConfigAttributes(cluster, configTags); for (Map<String, Map<String, String>> attributesMap : attributes.values()) { configHelper.cloneAttributesMap(attributesMap, attrs); } configurationAttributes.put(clusterConfig.getType(), attrs); } } ConfigHelper.processHiddenAttribute(configurations, configurationAttributes, componentName, true); for (Map.Entry<String, Map<String, Map<String, String>>> configurationAttributesEntry : configurationAttributes.entrySet()) { Map<String, Map<String, String>> attrs = configurationAttributesEntry.getValue(); attrs.remove("hidden"); } for (Map.Entry<String, Map<String, String>> configEntry : configurations.entrySet()) { String configType = configEntry.getKey(); Map<String, String> configProperties = configEntry.getValue(); Long configVersion = configVersions.get(configType); Map<PropertyType, Set<String>> propertiesTypes = configPropertiesTypes.get(configType); SecretReference.replacePasswordsWithReferences(propertiesTypes, configProperties, configType, configVersion); } Map<String, Set<String>> clusterHostInfo = null; ServiceInfo serviceInfo = null; String osFamily = null; clusterHostInfo = StageUtils.getClusterHostInfo(cluster); serviceInfo = managementController.getAmbariMetaInfo().getService(stackId.getStackName(), stackId.getStackVersion(), serviceName); try { clusterHostInfo = StageUtils.substituteHostIndexes(clusterHostInfo); } catch (AmbariException e) { throw new SystemException(e.getMessage(), e); } osFamily = clusters.getHost(hostName).getOsFamily(); TreeMap<String, String> hostLevelParams = new TreeMap<String, String>(); hostLevelParams.put(JDK_LOCATION, managementController.getJdkResourceUrl()); hostLevelParams.put(JAVA_HOME, managementController.getJavaHome()); hostLevelParams.put(JAVA_VERSION, String.valueOf(configs.getJavaVersion())); hostLevelParams.put(JDK_NAME, managementController.getJDKName()); hostLevelParams.put(JCE_NAME, managementController.getJCEName()); hostLevelParams.put(STACK_NAME, stackId.getStackName()); hostLevelParams.put(STACK_VERSION, stackId.getStackVersion()); hostLevelParams.put(DB_NAME, managementController.getServerDB()); hostLevelParams.put(MYSQL_JDBC_URL, managementController.getMysqljdbcUrl()); hostLevelParams.put(ORACLE_JDBC_URL, managementController.getOjdbcUrl()); hostLevelParams.put(HOST_SYS_PREPPED, configs.areHostsSysPrepped()); hostLevelParams.putAll(managementController.getRcaParameters()); hostLevelParams.put(AGENT_STACK_RETRY_ON_UNAVAILABILITY, configs.isAgentStackRetryOnInstallEnabled()); hostLevelParams.put(AGENT_STACK_RETRY_COUNT, configs.getAgentStackRetryOnInstallCount()); ServiceOsSpecific anyOs = null; if (serviceInfo.getOsSpecifics().containsKey(AmbariMetaInfo.ANY_OS)) { anyOs = serviceInfo.getOsSpecifics().get(AmbariMetaInfo.ANY_OS); } ServiceOsSpecific hostOs = populateServicePackagesInfo(serviceInfo, hostLevelParams, osFamily); List<ServiceOsSpecific.Package> packages = new ArrayList<ServiceOsSpecific.Package>(); if (anyOs != null) { packages.addAll(anyOs.getPackages()); } if (hostOs != null) { packages.addAll(hostOs.getPackages()); } String packageList = gson.toJson(packages); hostLevelParams.put(PACKAGE_LIST, packageList); Set<String> userSet = configHelper.getPropertyValuesWithPropertyType(stackId, PropertyType.USER, cluster, desiredClusterConfigs); String userList = gson.toJson(userSet); hostLevelParams.put(USER_LIST, userList); Map<String, Set<String>> userGroupsMap = configHelper.createUserGroupsMap( stackId, cluster, desiredClusterConfigs); String userGroups = gson.toJson(userGroupsMap); hostLevelParams.put(USER_GROUPS,userGroups); Set<String> groupSet = configHelper.getPropertyValuesWithPropertyType(stackId, PropertyType.GROUP, cluster, desiredClusterConfigs); String groupList = gson.toJson(groupSet); hostLevelParams.put(GROUP_LIST, groupList); Set<String> notManagedHdfsPathSet = configHelper.getPropertyValuesWithPropertyType(stackId, PropertyType.NOT_MANAGED_HDFS_PATH, cluster, desiredClusterConfigs); String notManagedHdfsPathList = gson.toJson(notManagedHdfsPathSet); hostLevelParams.put(NOT_MANAGED_HDFS_PATH_LIST, notManagedHdfsPathList); String jsonConfigurations = null; Map<String, Object> commandParams = new HashMap<String, Object>(); List<Map<String, String>> xmlConfigs = new LinkedList<Map<String, String>>(); List<Map<String, String>> envConfigs = new LinkedList<Map<String, String>>(); List<Map<String, String>> propertiesConfigs = new LinkedList<Map<String, String>>(); for (ClientConfigFileDefinition clientConfigFile : clientConfigFiles) { Map<String, String> fileDict = new HashMap<String, String>(); fileDict.put(clientConfigFile.getFileName(), clientConfigFile.getDictionaryName()); if (clientConfigFile.getType().equals("xml")) { xmlConfigs.add(fileDict); } else if (clientConfigFile.getType().equals("env")) { envConfigs.add(fileDict); } else if (clientConfigFile.getType().equals("properties")) { propertiesConfigs.add(fileDict); } } commandParams.put("xml_configs_list", xmlConfigs); commandParams.put("env_configs_list", envConfigs); commandParams.put("properties_configs_list", propertiesConfigs); commandParams.put("output_file", componentName + "-configs" + Configuration.DEF_ARCHIVE_EXTENSION); Map<String, Object> jsonContent = new TreeMap<String, Object>(); jsonContent.put("configurations", configurations); jsonContent.put("configuration_attributes", configurationAttributes); jsonContent.put("commandParams", commandParams); jsonContent.put("clusterHostInfo", clusterHostInfo); jsonContent.put("hostLevelParams", hostLevelParams); jsonContent.put("hostname", hostName); jsonContent.put("clusterName", cluster.getClusterName()); jsonConfigurations = gson.toJson(jsonContent); File jsonFileName = new File(TMP_PATH + File.separator + componentName + "-configuration.json"); File tmpDirectory = new File(jsonFileName.getParent()); if (!tmpDirectory.exists()) { try { tmpDirectory.mkdirs(); tmpDirectory.setWritable(true, true); tmpDirectory.setReadable(true, true); } catch (SecurityException se) { throw new SystemException("Failed to get temporary directory to store configurations", se); } } PrintWriter printWriter = null; try { printWriter = new PrintWriter(jsonFileName.getAbsolutePath()); printWriter.print(jsonConfigurations); printWriter.close(); } catch (FileNotFoundException e) { throw new SystemException("Failed to write configurations to json file ", e); } String cmd = pythonCmd + " " + commandScriptAbsolute + " generate_configs " + jsonFileName.getAbsolutePath() + " " + packageFolderAbsolute + " " + TMP_PATH + File.separator + "structured-out.json" + " INFO " + TMP_PATH; pythonCompressFilesCmds.add(cmd); } catch (AmbariException e) { throw new SystemException("Controller error ", e); } } if (schWithConfigFiles.isEmpty()) { throw new SystemException("No configuration files defined for any component" ); } Integer totalCommands = pythonCompressFilesCmds.size() * 2; Integer threadPoolSize = Math.min(totalCommands,configs.getExternalScriptThreadPoolSize()); ExecutorService processExecutor = Executors.newFixedThreadPool(threadPoolSize); List<CommandLineThreadWrapper> pythonCmdThreads = executeCommands(processExecutor, pythonCompressFilesCmds); Integer timeout = configs.getExternalScriptTimeout(); waitForAllThreadsToJoin(processExecutor, pythonCmdThreads, timeout); if (StringUtils.isEmpty(requestComponentName)) { TarUtils tarUtils; String fileName; List <ServiceComponentHostResponse> schToTarConfigFiles = schWithConfigFiles; if (StringUtils.isNotEmpty(requestHostName)) { fileName = requestHostName + "(" + Resource.InternalType.Host.toString().toUpperCase()+")"; } else if (StringUtils.isNotEmpty(requestServiceName)) { fileName = requestServiceName + "(" + Resource.InternalType.Service.toString().toUpperCase()+")"; schToTarConfigFiles = serviceToComponentMap.get(requestServiceName); } else { fileName = schRequest.getClusterName() + "(" + Resource.InternalType.Cluster.toString().toUpperCase()+")"; } tarUtils = new TarUtils(TMP_PATH, fileName, schToTarConfigFiles); tarUtils.tarConfigFiles(); } Resource resource = new ResourceImpl(Resource.Type.ClientConfig); resources.add(resource); return resources; } }
ClientConfigResourceProvider extends AbstractControllerResourceProvider { @Override @Transactional public Set<Resource> getResources(Request request, Predicate predicate) throws SystemException, UnsupportedPropertyException, NoSuchResourceException, NoSuchParentResourceException { Set<Resource> resources = new HashSet<Resource>(); final Set<ServiceComponentHostRequest> requests = new HashSet<ServiceComponentHostRequest>(); for (Map<String, Object> propertyMap : getPropertyMaps(predicate)) { requests.add(getRequest(propertyMap)); } Set<ServiceComponentHostResponse> responses = null; try { responses = getResources(new Command<Set<ServiceComponentHostResponse>>() { @Override public Set<ServiceComponentHostResponse> invoke() throws AmbariException { return getManagementController().getHostComponents(requests); } }); } catch (Exception e) { throw new SystemException("Failed to get components ", e); } Map<String,ServiceComponentHostResponse> componentMap = new HashMap<>(); for (ServiceComponentHostResponse resp: responses) { String componentName = resp.getComponentName(); if (!componentMap.containsKey(componentName)) { componentMap.put(resp.getComponentName(),resp); } } ServiceComponentHostRequest schRequest = requests.iterator().next(); String requestComponentName = schRequest.getComponentName(); String requestServiceName = schRequest.getServiceName(); String requestHostName = schRequest.getHostname(); Map<String,List<ServiceComponentHostResponse>> serviceToComponentMap = new HashMap<String,List<ServiceComponentHostResponse>>(); List <ServiceComponentHostResponse> schWithConfigFiles = new ArrayList<>(); Configuration configs = new Configuration(); Map<String, String> configMap = configs.getConfigsMap(); String TMP_PATH = configMap.get(Configuration.SERVER_TMP_DIR.getKey()); String pythonCmd = configMap.get(Configuration.AMBARI_PYTHON_WRAP.getKey()); List<String> pythonCompressFilesCmds = new ArrayList<>(); for (ServiceComponentHostResponse response : componentMap.values()){ AmbariManagementController managementController = getManagementController(); ConfigHelper configHelper = managementController.getConfigHelper(); Cluster cluster = null; Clusters clusters = managementController.getClusters(); try { cluster = clusters.getCluster(response.getClusterName()); StackId stackId = cluster.getCurrentStackVersion(); String serviceName = response.getServiceName(); String componentName = response.getComponentName(); String hostName = response.getHostname(); ComponentInfo componentInfo = null; String packageFolder = null; componentInfo = managementController.getAmbariMetaInfo(). getComponent(stackId.getStackName(), stackId.getStackVersion(), serviceName, componentName); packageFolder = managementController.getAmbariMetaInfo(). getService(stackId.getStackName(), stackId.getStackVersion(), serviceName).getServicePackageFolder(); String commandScript = componentInfo.getCommandScript().getScript(); List<ClientConfigFileDefinition> clientConfigFiles = componentInfo.getClientConfigFiles(); if (clientConfigFiles == null) { if (componentMap.size() == 1) { throw new SystemException("No configuration files defined for the component " + componentInfo.getName()); } else { LOG.debug(String.format("No configuration files defined for the component %s",componentInfo.getName())); continue; } } schWithConfigFiles.add(response); if (serviceToComponentMap.containsKey(response.getServiceName())) { List <ServiceComponentHostResponse> schResponseList = serviceToComponentMap.get(serviceName); schResponseList.add(response); } else { List <ServiceComponentHostResponse> schResponseList = new ArrayList<>(); schResponseList.add(response); serviceToComponentMap.put(serviceName,schResponseList); } String resourceDirPath = configs.getResourceDirPath(); String packageFolderAbsolute = resourceDirPath + File.separator + packageFolder; String commandScriptAbsolute = packageFolderAbsolute + File.separator + commandScript; Map<String, Map<String, String>> configurations = new TreeMap<String, Map<String, String>>(); Map<String, Long> configVersions = new TreeMap<String, Long>(); Map<String, Map<PropertyType, Set<String>>> configPropertiesTypes = new TreeMap<>(); Map<String, Map<String, Map<String, String>>> configurationAttributes = new TreeMap<String, Map<String, Map<String, String>>>(); Map<String, DesiredConfig> desiredClusterConfigs = cluster.getDesiredConfigs(); for (Map.Entry<String, DesiredConfig> desiredConfigEntry : desiredClusterConfigs.entrySet()) { String configType = desiredConfigEntry.getKey(); DesiredConfig desiredConfig = desiredConfigEntry.getValue(); Config clusterConfig = cluster.getConfig(configType, desiredConfig.getTag()); if (clusterConfig != null) { Map<String, String> props = new HashMap<String, String>(clusterConfig.getProperties()); Map<String, Map<String, String>> allConfigTags = null; allConfigTags = configHelper .getEffectiveDesiredTags(cluster, schRequest.getHostname()); Map<String, Map<String, String>> configTags = new HashMap<String, Map<String, String>>(); for (Map.Entry<String, Map<String, String>> entry : allConfigTags.entrySet()) { if (entry.getKey().equals(clusterConfig.getType())) { configTags.put(clusterConfig.getType(), entry.getValue()); } } Map<String, Map<String, String>> properties = configHelper .getEffectiveConfigProperties(cluster, configTags); if (!properties.isEmpty()) { for (Map<String, String> propertyMap : properties.values()) { props.putAll(propertyMap); } } configurations.put(clusterConfig.getType(), props); configVersions.put(clusterConfig.getType(), clusterConfig.getVersion()); configPropertiesTypes.put(clusterConfig.getType(), clusterConfig.getPropertiesTypes()); Map<String, Map<String, String>> attrs = new TreeMap<String, Map<String, String>>(); configHelper.cloneAttributesMap(clusterConfig.getPropertiesAttributes(), attrs); Map<String, Map<String, Map<String, String>>> attributes = configHelper .getEffectiveConfigAttributes(cluster, configTags); for (Map<String, Map<String, String>> attributesMap : attributes.values()) { configHelper.cloneAttributesMap(attributesMap, attrs); } configurationAttributes.put(clusterConfig.getType(), attrs); } } ConfigHelper.processHiddenAttribute(configurations, configurationAttributes, componentName, true); for (Map.Entry<String, Map<String, Map<String, String>>> configurationAttributesEntry : configurationAttributes.entrySet()) { Map<String, Map<String, String>> attrs = configurationAttributesEntry.getValue(); attrs.remove("hidden"); } for (Map.Entry<String, Map<String, String>> configEntry : configurations.entrySet()) { String configType = configEntry.getKey(); Map<String, String> configProperties = configEntry.getValue(); Long configVersion = configVersions.get(configType); Map<PropertyType, Set<String>> propertiesTypes = configPropertiesTypes.get(configType); SecretReference.replacePasswordsWithReferences(propertiesTypes, configProperties, configType, configVersion); } Map<String, Set<String>> clusterHostInfo = null; ServiceInfo serviceInfo = null; String osFamily = null; clusterHostInfo = StageUtils.getClusterHostInfo(cluster); serviceInfo = managementController.getAmbariMetaInfo().getService(stackId.getStackName(), stackId.getStackVersion(), serviceName); try { clusterHostInfo = StageUtils.substituteHostIndexes(clusterHostInfo); } catch (AmbariException e) { throw new SystemException(e.getMessage(), e); } osFamily = clusters.getHost(hostName).getOsFamily(); TreeMap<String, String> hostLevelParams = new TreeMap<String, String>(); hostLevelParams.put(JDK_LOCATION, managementController.getJdkResourceUrl()); hostLevelParams.put(JAVA_HOME, managementController.getJavaHome()); hostLevelParams.put(JAVA_VERSION, String.valueOf(configs.getJavaVersion())); hostLevelParams.put(JDK_NAME, managementController.getJDKName()); hostLevelParams.put(JCE_NAME, managementController.getJCEName()); hostLevelParams.put(STACK_NAME, stackId.getStackName()); hostLevelParams.put(STACK_VERSION, stackId.getStackVersion()); hostLevelParams.put(DB_NAME, managementController.getServerDB()); hostLevelParams.put(MYSQL_JDBC_URL, managementController.getMysqljdbcUrl()); hostLevelParams.put(ORACLE_JDBC_URL, managementController.getOjdbcUrl()); hostLevelParams.put(HOST_SYS_PREPPED, configs.areHostsSysPrepped()); hostLevelParams.putAll(managementController.getRcaParameters()); hostLevelParams.put(AGENT_STACK_RETRY_ON_UNAVAILABILITY, configs.isAgentStackRetryOnInstallEnabled()); hostLevelParams.put(AGENT_STACK_RETRY_COUNT, configs.getAgentStackRetryOnInstallCount()); ServiceOsSpecific anyOs = null; if (serviceInfo.getOsSpecifics().containsKey(AmbariMetaInfo.ANY_OS)) { anyOs = serviceInfo.getOsSpecifics().get(AmbariMetaInfo.ANY_OS); } ServiceOsSpecific hostOs = populateServicePackagesInfo(serviceInfo, hostLevelParams, osFamily); List<ServiceOsSpecific.Package> packages = new ArrayList<ServiceOsSpecific.Package>(); if (anyOs != null) { packages.addAll(anyOs.getPackages()); } if (hostOs != null) { packages.addAll(hostOs.getPackages()); } String packageList = gson.toJson(packages); hostLevelParams.put(PACKAGE_LIST, packageList); Set<String> userSet = configHelper.getPropertyValuesWithPropertyType(stackId, PropertyType.USER, cluster, desiredClusterConfigs); String userList = gson.toJson(userSet); hostLevelParams.put(USER_LIST, userList); Map<String, Set<String>> userGroupsMap = configHelper.createUserGroupsMap( stackId, cluster, desiredClusterConfigs); String userGroups = gson.toJson(userGroupsMap); hostLevelParams.put(USER_GROUPS,userGroups); Set<String> groupSet = configHelper.getPropertyValuesWithPropertyType(stackId, PropertyType.GROUP, cluster, desiredClusterConfigs); String groupList = gson.toJson(groupSet); hostLevelParams.put(GROUP_LIST, groupList); Set<String> notManagedHdfsPathSet = configHelper.getPropertyValuesWithPropertyType(stackId, PropertyType.NOT_MANAGED_HDFS_PATH, cluster, desiredClusterConfigs); String notManagedHdfsPathList = gson.toJson(notManagedHdfsPathSet); hostLevelParams.put(NOT_MANAGED_HDFS_PATH_LIST, notManagedHdfsPathList); String jsonConfigurations = null; Map<String, Object> commandParams = new HashMap<String, Object>(); List<Map<String, String>> xmlConfigs = new LinkedList<Map<String, String>>(); List<Map<String, String>> envConfigs = new LinkedList<Map<String, String>>(); List<Map<String, String>> propertiesConfigs = new LinkedList<Map<String, String>>(); for (ClientConfigFileDefinition clientConfigFile : clientConfigFiles) { Map<String, String> fileDict = new HashMap<String, String>(); fileDict.put(clientConfigFile.getFileName(), clientConfigFile.getDictionaryName()); if (clientConfigFile.getType().equals("xml")) { xmlConfigs.add(fileDict); } else if (clientConfigFile.getType().equals("env")) { envConfigs.add(fileDict); } else if (clientConfigFile.getType().equals("properties")) { propertiesConfigs.add(fileDict); } } commandParams.put("xml_configs_list", xmlConfigs); commandParams.put("env_configs_list", envConfigs); commandParams.put("properties_configs_list", propertiesConfigs); commandParams.put("output_file", componentName + "-configs" + Configuration.DEF_ARCHIVE_EXTENSION); Map<String, Object> jsonContent = new TreeMap<String, Object>(); jsonContent.put("configurations", configurations); jsonContent.put("configuration_attributes", configurationAttributes); jsonContent.put("commandParams", commandParams); jsonContent.put("clusterHostInfo", clusterHostInfo); jsonContent.put("hostLevelParams", hostLevelParams); jsonContent.put("hostname", hostName); jsonContent.put("clusterName", cluster.getClusterName()); jsonConfigurations = gson.toJson(jsonContent); File jsonFileName = new File(TMP_PATH + File.separator + componentName + "-configuration.json"); File tmpDirectory = new File(jsonFileName.getParent()); if (!tmpDirectory.exists()) { try { tmpDirectory.mkdirs(); tmpDirectory.setWritable(true, true); tmpDirectory.setReadable(true, true); } catch (SecurityException se) { throw new SystemException("Failed to get temporary directory to store configurations", se); } } PrintWriter printWriter = null; try { printWriter = new PrintWriter(jsonFileName.getAbsolutePath()); printWriter.print(jsonConfigurations); printWriter.close(); } catch (FileNotFoundException e) { throw new SystemException("Failed to write configurations to json file ", e); } String cmd = pythonCmd + " " + commandScriptAbsolute + " generate_configs " + jsonFileName.getAbsolutePath() + " " + packageFolderAbsolute + " " + TMP_PATH + File.separator + "structured-out.json" + " INFO " + TMP_PATH; pythonCompressFilesCmds.add(cmd); } catch (AmbariException e) { throw new SystemException("Controller error ", e); } } if (schWithConfigFiles.isEmpty()) { throw new SystemException("No configuration files defined for any component" ); } Integer totalCommands = pythonCompressFilesCmds.size() * 2; Integer threadPoolSize = Math.min(totalCommands,configs.getExternalScriptThreadPoolSize()); ExecutorService processExecutor = Executors.newFixedThreadPool(threadPoolSize); List<CommandLineThreadWrapper> pythonCmdThreads = executeCommands(processExecutor, pythonCompressFilesCmds); Integer timeout = configs.getExternalScriptTimeout(); waitForAllThreadsToJoin(processExecutor, pythonCmdThreads, timeout); if (StringUtils.isEmpty(requestComponentName)) { TarUtils tarUtils; String fileName; List <ServiceComponentHostResponse> schToTarConfigFiles = schWithConfigFiles; if (StringUtils.isNotEmpty(requestHostName)) { fileName = requestHostName + "(" + Resource.InternalType.Host.toString().toUpperCase()+")"; } else if (StringUtils.isNotEmpty(requestServiceName)) { fileName = requestServiceName + "(" + Resource.InternalType.Service.toString().toUpperCase()+")"; schToTarConfigFiles = serviceToComponentMap.get(requestServiceName); } else { fileName = schRequest.getClusterName() + "(" + Resource.InternalType.Cluster.toString().toUpperCase()+")"; } tarUtils = new TarUtils(TMP_PATH, fileName, schToTarConfigFiles); tarUtils.tarConfigFiles(); } Resource resource = new ResourceImpl(Resource.Type.ClientConfig); resources.add(resource); return resources; } @AssistedInject  ClientConfigResourceProvider(@Assisted Set<String> propertyIds,
                               @Assisted Map<Resource.Type, String> keyPropertyIds,
                               @Assisted AmbariManagementController managementController); }
ClientConfigResourceProvider extends AbstractControllerResourceProvider { @Override @Transactional public Set<Resource> getResources(Request request, Predicate predicate) throws SystemException, UnsupportedPropertyException, NoSuchResourceException, NoSuchParentResourceException { Set<Resource> resources = new HashSet<Resource>(); final Set<ServiceComponentHostRequest> requests = new HashSet<ServiceComponentHostRequest>(); for (Map<String, Object> propertyMap : getPropertyMaps(predicate)) { requests.add(getRequest(propertyMap)); } Set<ServiceComponentHostResponse> responses = null; try { responses = getResources(new Command<Set<ServiceComponentHostResponse>>() { @Override public Set<ServiceComponentHostResponse> invoke() throws AmbariException { return getManagementController().getHostComponents(requests); } }); } catch (Exception e) { throw new SystemException("Failed to get components ", e); } Map<String,ServiceComponentHostResponse> componentMap = new HashMap<>(); for (ServiceComponentHostResponse resp: responses) { String componentName = resp.getComponentName(); if (!componentMap.containsKey(componentName)) { componentMap.put(resp.getComponentName(),resp); } } ServiceComponentHostRequest schRequest = requests.iterator().next(); String requestComponentName = schRequest.getComponentName(); String requestServiceName = schRequest.getServiceName(); String requestHostName = schRequest.getHostname(); Map<String,List<ServiceComponentHostResponse>> serviceToComponentMap = new HashMap<String,List<ServiceComponentHostResponse>>(); List <ServiceComponentHostResponse> schWithConfigFiles = new ArrayList<>(); Configuration configs = new Configuration(); Map<String, String> configMap = configs.getConfigsMap(); String TMP_PATH = configMap.get(Configuration.SERVER_TMP_DIR.getKey()); String pythonCmd = configMap.get(Configuration.AMBARI_PYTHON_WRAP.getKey()); List<String> pythonCompressFilesCmds = new ArrayList<>(); for (ServiceComponentHostResponse response : componentMap.values()){ AmbariManagementController managementController = getManagementController(); ConfigHelper configHelper = managementController.getConfigHelper(); Cluster cluster = null; Clusters clusters = managementController.getClusters(); try { cluster = clusters.getCluster(response.getClusterName()); StackId stackId = cluster.getCurrentStackVersion(); String serviceName = response.getServiceName(); String componentName = response.getComponentName(); String hostName = response.getHostname(); ComponentInfo componentInfo = null; String packageFolder = null; componentInfo = managementController.getAmbariMetaInfo(). getComponent(stackId.getStackName(), stackId.getStackVersion(), serviceName, componentName); packageFolder = managementController.getAmbariMetaInfo(). getService(stackId.getStackName(), stackId.getStackVersion(), serviceName).getServicePackageFolder(); String commandScript = componentInfo.getCommandScript().getScript(); List<ClientConfigFileDefinition> clientConfigFiles = componentInfo.getClientConfigFiles(); if (clientConfigFiles == null) { if (componentMap.size() == 1) { throw new SystemException("No configuration files defined for the component " + componentInfo.getName()); } else { LOG.debug(String.format("No configuration files defined for the component %s",componentInfo.getName())); continue; } } schWithConfigFiles.add(response); if (serviceToComponentMap.containsKey(response.getServiceName())) { List <ServiceComponentHostResponse> schResponseList = serviceToComponentMap.get(serviceName); schResponseList.add(response); } else { List <ServiceComponentHostResponse> schResponseList = new ArrayList<>(); schResponseList.add(response); serviceToComponentMap.put(serviceName,schResponseList); } String resourceDirPath = configs.getResourceDirPath(); String packageFolderAbsolute = resourceDirPath + File.separator + packageFolder; String commandScriptAbsolute = packageFolderAbsolute + File.separator + commandScript; Map<String, Map<String, String>> configurations = new TreeMap<String, Map<String, String>>(); Map<String, Long> configVersions = new TreeMap<String, Long>(); Map<String, Map<PropertyType, Set<String>>> configPropertiesTypes = new TreeMap<>(); Map<String, Map<String, Map<String, String>>> configurationAttributes = new TreeMap<String, Map<String, Map<String, String>>>(); Map<String, DesiredConfig> desiredClusterConfigs = cluster.getDesiredConfigs(); for (Map.Entry<String, DesiredConfig> desiredConfigEntry : desiredClusterConfigs.entrySet()) { String configType = desiredConfigEntry.getKey(); DesiredConfig desiredConfig = desiredConfigEntry.getValue(); Config clusterConfig = cluster.getConfig(configType, desiredConfig.getTag()); if (clusterConfig != null) { Map<String, String> props = new HashMap<String, String>(clusterConfig.getProperties()); Map<String, Map<String, String>> allConfigTags = null; allConfigTags = configHelper .getEffectiveDesiredTags(cluster, schRequest.getHostname()); Map<String, Map<String, String>> configTags = new HashMap<String, Map<String, String>>(); for (Map.Entry<String, Map<String, String>> entry : allConfigTags.entrySet()) { if (entry.getKey().equals(clusterConfig.getType())) { configTags.put(clusterConfig.getType(), entry.getValue()); } } Map<String, Map<String, String>> properties = configHelper .getEffectiveConfigProperties(cluster, configTags); if (!properties.isEmpty()) { for (Map<String, String> propertyMap : properties.values()) { props.putAll(propertyMap); } } configurations.put(clusterConfig.getType(), props); configVersions.put(clusterConfig.getType(), clusterConfig.getVersion()); configPropertiesTypes.put(clusterConfig.getType(), clusterConfig.getPropertiesTypes()); Map<String, Map<String, String>> attrs = new TreeMap<String, Map<String, String>>(); configHelper.cloneAttributesMap(clusterConfig.getPropertiesAttributes(), attrs); Map<String, Map<String, Map<String, String>>> attributes = configHelper .getEffectiveConfigAttributes(cluster, configTags); for (Map<String, Map<String, String>> attributesMap : attributes.values()) { configHelper.cloneAttributesMap(attributesMap, attrs); } configurationAttributes.put(clusterConfig.getType(), attrs); } } ConfigHelper.processHiddenAttribute(configurations, configurationAttributes, componentName, true); for (Map.Entry<String, Map<String, Map<String, String>>> configurationAttributesEntry : configurationAttributes.entrySet()) { Map<String, Map<String, String>> attrs = configurationAttributesEntry.getValue(); attrs.remove("hidden"); } for (Map.Entry<String, Map<String, String>> configEntry : configurations.entrySet()) { String configType = configEntry.getKey(); Map<String, String> configProperties = configEntry.getValue(); Long configVersion = configVersions.get(configType); Map<PropertyType, Set<String>> propertiesTypes = configPropertiesTypes.get(configType); SecretReference.replacePasswordsWithReferences(propertiesTypes, configProperties, configType, configVersion); } Map<String, Set<String>> clusterHostInfo = null; ServiceInfo serviceInfo = null; String osFamily = null; clusterHostInfo = StageUtils.getClusterHostInfo(cluster); serviceInfo = managementController.getAmbariMetaInfo().getService(stackId.getStackName(), stackId.getStackVersion(), serviceName); try { clusterHostInfo = StageUtils.substituteHostIndexes(clusterHostInfo); } catch (AmbariException e) { throw new SystemException(e.getMessage(), e); } osFamily = clusters.getHost(hostName).getOsFamily(); TreeMap<String, String> hostLevelParams = new TreeMap<String, String>(); hostLevelParams.put(JDK_LOCATION, managementController.getJdkResourceUrl()); hostLevelParams.put(JAVA_HOME, managementController.getJavaHome()); hostLevelParams.put(JAVA_VERSION, String.valueOf(configs.getJavaVersion())); hostLevelParams.put(JDK_NAME, managementController.getJDKName()); hostLevelParams.put(JCE_NAME, managementController.getJCEName()); hostLevelParams.put(STACK_NAME, stackId.getStackName()); hostLevelParams.put(STACK_VERSION, stackId.getStackVersion()); hostLevelParams.put(DB_NAME, managementController.getServerDB()); hostLevelParams.put(MYSQL_JDBC_URL, managementController.getMysqljdbcUrl()); hostLevelParams.put(ORACLE_JDBC_URL, managementController.getOjdbcUrl()); hostLevelParams.put(HOST_SYS_PREPPED, configs.areHostsSysPrepped()); hostLevelParams.putAll(managementController.getRcaParameters()); hostLevelParams.put(AGENT_STACK_RETRY_ON_UNAVAILABILITY, configs.isAgentStackRetryOnInstallEnabled()); hostLevelParams.put(AGENT_STACK_RETRY_COUNT, configs.getAgentStackRetryOnInstallCount()); ServiceOsSpecific anyOs = null; if (serviceInfo.getOsSpecifics().containsKey(AmbariMetaInfo.ANY_OS)) { anyOs = serviceInfo.getOsSpecifics().get(AmbariMetaInfo.ANY_OS); } ServiceOsSpecific hostOs = populateServicePackagesInfo(serviceInfo, hostLevelParams, osFamily); List<ServiceOsSpecific.Package> packages = new ArrayList<ServiceOsSpecific.Package>(); if (anyOs != null) { packages.addAll(anyOs.getPackages()); } if (hostOs != null) { packages.addAll(hostOs.getPackages()); } String packageList = gson.toJson(packages); hostLevelParams.put(PACKAGE_LIST, packageList); Set<String> userSet = configHelper.getPropertyValuesWithPropertyType(stackId, PropertyType.USER, cluster, desiredClusterConfigs); String userList = gson.toJson(userSet); hostLevelParams.put(USER_LIST, userList); Map<String, Set<String>> userGroupsMap = configHelper.createUserGroupsMap( stackId, cluster, desiredClusterConfigs); String userGroups = gson.toJson(userGroupsMap); hostLevelParams.put(USER_GROUPS,userGroups); Set<String> groupSet = configHelper.getPropertyValuesWithPropertyType(stackId, PropertyType.GROUP, cluster, desiredClusterConfigs); String groupList = gson.toJson(groupSet); hostLevelParams.put(GROUP_LIST, groupList); Set<String> notManagedHdfsPathSet = configHelper.getPropertyValuesWithPropertyType(stackId, PropertyType.NOT_MANAGED_HDFS_PATH, cluster, desiredClusterConfigs); String notManagedHdfsPathList = gson.toJson(notManagedHdfsPathSet); hostLevelParams.put(NOT_MANAGED_HDFS_PATH_LIST, notManagedHdfsPathList); String jsonConfigurations = null; Map<String, Object> commandParams = new HashMap<String, Object>(); List<Map<String, String>> xmlConfigs = new LinkedList<Map<String, String>>(); List<Map<String, String>> envConfigs = new LinkedList<Map<String, String>>(); List<Map<String, String>> propertiesConfigs = new LinkedList<Map<String, String>>(); for (ClientConfigFileDefinition clientConfigFile : clientConfigFiles) { Map<String, String> fileDict = new HashMap<String, String>(); fileDict.put(clientConfigFile.getFileName(), clientConfigFile.getDictionaryName()); if (clientConfigFile.getType().equals("xml")) { xmlConfigs.add(fileDict); } else if (clientConfigFile.getType().equals("env")) { envConfigs.add(fileDict); } else if (clientConfigFile.getType().equals("properties")) { propertiesConfigs.add(fileDict); } } commandParams.put("xml_configs_list", xmlConfigs); commandParams.put("env_configs_list", envConfigs); commandParams.put("properties_configs_list", propertiesConfigs); commandParams.put("output_file", componentName + "-configs" + Configuration.DEF_ARCHIVE_EXTENSION); Map<String, Object> jsonContent = new TreeMap<String, Object>(); jsonContent.put("configurations", configurations); jsonContent.put("configuration_attributes", configurationAttributes); jsonContent.put("commandParams", commandParams); jsonContent.put("clusterHostInfo", clusterHostInfo); jsonContent.put("hostLevelParams", hostLevelParams); jsonContent.put("hostname", hostName); jsonContent.put("clusterName", cluster.getClusterName()); jsonConfigurations = gson.toJson(jsonContent); File jsonFileName = new File(TMP_PATH + File.separator + componentName + "-configuration.json"); File tmpDirectory = new File(jsonFileName.getParent()); if (!tmpDirectory.exists()) { try { tmpDirectory.mkdirs(); tmpDirectory.setWritable(true, true); tmpDirectory.setReadable(true, true); } catch (SecurityException se) { throw new SystemException("Failed to get temporary directory to store configurations", se); } } PrintWriter printWriter = null; try { printWriter = new PrintWriter(jsonFileName.getAbsolutePath()); printWriter.print(jsonConfigurations); printWriter.close(); } catch (FileNotFoundException e) { throw new SystemException("Failed to write configurations to json file ", e); } String cmd = pythonCmd + " " + commandScriptAbsolute + " generate_configs " + jsonFileName.getAbsolutePath() + " " + packageFolderAbsolute + " " + TMP_PATH + File.separator + "structured-out.json" + " INFO " + TMP_PATH; pythonCompressFilesCmds.add(cmd); } catch (AmbariException e) { throw new SystemException("Controller error ", e); } } if (schWithConfigFiles.isEmpty()) { throw new SystemException("No configuration files defined for any component" ); } Integer totalCommands = pythonCompressFilesCmds.size() * 2; Integer threadPoolSize = Math.min(totalCommands,configs.getExternalScriptThreadPoolSize()); ExecutorService processExecutor = Executors.newFixedThreadPool(threadPoolSize); List<CommandLineThreadWrapper> pythonCmdThreads = executeCommands(processExecutor, pythonCompressFilesCmds); Integer timeout = configs.getExternalScriptTimeout(); waitForAllThreadsToJoin(processExecutor, pythonCmdThreads, timeout); if (StringUtils.isEmpty(requestComponentName)) { TarUtils tarUtils; String fileName; List <ServiceComponentHostResponse> schToTarConfigFiles = schWithConfigFiles; if (StringUtils.isNotEmpty(requestHostName)) { fileName = requestHostName + "(" + Resource.InternalType.Host.toString().toUpperCase()+")"; } else if (StringUtils.isNotEmpty(requestServiceName)) { fileName = requestServiceName + "(" + Resource.InternalType.Service.toString().toUpperCase()+")"; schToTarConfigFiles = serviceToComponentMap.get(requestServiceName); } else { fileName = schRequest.getClusterName() + "(" + Resource.InternalType.Cluster.toString().toUpperCase()+")"; } tarUtils = new TarUtils(TMP_PATH, fileName, schToTarConfigFiles); tarUtils.tarConfigFiles(); } Resource resource = new ResourceImpl(Resource.Type.ClientConfig); resources.add(resource); return resources; } @AssistedInject  ClientConfigResourceProvider(@Assisted Set<String> propertyIds,
                               @Assisted Map<Resource.Type, String> keyPropertyIds,
                               @Assisted AmbariManagementController managementController); @Override RequestStatus createResources(Request request); @Override @Transactional Set<Resource> getResources(Request request, Predicate predicate); @Override RequestStatus updateResources(final Request request, Predicate predicate); @Override RequestStatus deleteResources(Request request, Predicate predicate); }
ClientConfigResourceProvider extends AbstractControllerResourceProvider { @Override @Transactional public Set<Resource> getResources(Request request, Predicate predicate) throws SystemException, UnsupportedPropertyException, NoSuchResourceException, NoSuchParentResourceException { Set<Resource> resources = new HashSet<Resource>(); final Set<ServiceComponentHostRequest> requests = new HashSet<ServiceComponentHostRequest>(); for (Map<String, Object> propertyMap : getPropertyMaps(predicate)) { requests.add(getRequest(propertyMap)); } Set<ServiceComponentHostResponse> responses = null; try { responses = getResources(new Command<Set<ServiceComponentHostResponse>>() { @Override public Set<ServiceComponentHostResponse> invoke() throws AmbariException { return getManagementController().getHostComponents(requests); } }); } catch (Exception e) { throw new SystemException("Failed to get components ", e); } Map<String,ServiceComponentHostResponse> componentMap = new HashMap<>(); for (ServiceComponentHostResponse resp: responses) { String componentName = resp.getComponentName(); if (!componentMap.containsKey(componentName)) { componentMap.put(resp.getComponentName(),resp); } } ServiceComponentHostRequest schRequest = requests.iterator().next(); String requestComponentName = schRequest.getComponentName(); String requestServiceName = schRequest.getServiceName(); String requestHostName = schRequest.getHostname(); Map<String,List<ServiceComponentHostResponse>> serviceToComponentMap = new HashMap<String,List<ServiceComponentHostResponse>>(); List <ServiceComponentHostResponse> schWithConfigFiles = new ArrayList<>(); Configuration configs = new Configuration(); Map<String, String> configMap = configs.getConfigsMap(); String TMP_PATH = configMap.get(Configuration.SERVER_TMP_DIR.getKey()); String pythonCmd = configMap.get(Configuration.AMBARI_PYTHON_WRAP.getKey()); List<String> pythonCompressFilesCmds = new ArrayList<>(); for (ServiceComponentHostResponse response : componentMap.values()){ AmbariManagementController managementController = getManagementController(); ConfigHelper configHelper = managementController.getConfigHelper(); Cluster cluster = null; Clusters clusters = managementController.getClusters(); try { cluster = clusters.getCluster(response.getClusterName()); StackId stackId = cluster.getCurrentStackVersion(); String serviceName = response.getServiceName(); String componentName = response.getComponentName(); String hostName = response.getHostname(); ComponentInfo componentInfo = null; String packageFolder = null; componentInfo = managementController.getAmbariMetaInfo(). getComponent(stackId.getStackName(), stackId.getStackVersion(), serviceName, componentName); packageFolder = managementController.getAmbariMetaInfo(). getService(stackId.getStackName(), stackId.getStackVersion(), serviceName).getServicePackageFolder(); String commandScript = componentInfo.getCommandScript().getScript(); List<ClientConfigFileDefinition> clientConfigFiles = componentInfo.getClientConfigFiles(); if (clientConfigFiles == null) { if (componentMap.size() == 1) { throw new SystemException("No configuration files defined for the component " + componentInfo.getName()); } else { LOG.debug(String.format("No configuration files defined for the component %s",componentInfo.getName())); continue; } } schWithConfigFiles.add(response); if (serviceToComponentMap.containsKey(response.getServiceName())) { List <ServiceComponentHostResponse> schResponseList = serviceToComponentMap.get(serviceName); schResponseList.add(response); } else { List <ServiceComponentHostResponse> schResponseList = new ArrayList<>(); schResponseList.add(response); serviceToComponentMap.put(serviceName,schResponseList); } String resourceDirPath = configs.getResourceDirPath(); String packageFolderAbsolute = resourceDirPath + File.separator + packageFolder; String commandScriptAbsolute = packageFolderAbsolute + File.separator + commandScript; Map<String, Map<String, String>> configurations = new TreeMap<String, Map<String, String>>(); Map<String, Long> configVersions = new TreeMap<String, Long>(); Map<String, Map<PropertyType, Set<String>>> configPropertiesTypes = new TreeMap<>(); Map<String, Map<String, Map<String, String>>> configurationAttributes = new TreeMap<String, Map<String, Map<String, String>>>(); Map<String, DesiredConfig> desiredClusterConfigs = cluster.getDesiredConfigs(); for (Map.Entry<String, DesiredConfig> desiredConfigEntry : desiredClusterConfigs.entrySet()) { String configType = desiredConfigEntry.getKey(); DesiredConfig desiredConfig = desiredConfigEntry.getValue(); Config clusterConfig = cluster.getConfig(configType, desiredConfig.getTag()); if (clusterConfig != null) { Map<String, String> props = new HashMap<String, String>(clusterConfig.getProperties()); Map<String, Map<String, String>> allConfigTags = null; allConfigTags = configHelper .getEffectiveDesiredTags(cluster, schRequest.getHostname()); Map<String, Map<String, String>> configTags = new HashMap<String, Map<String, String>>(); for (Map.Entry<String, Map<String, String>> entry : allConfigTags.entrySet()) { if (entry.getKey().equals(clusterConfig.getType())) { configTags.put(clusterConfig.getType(), entry.getValue()); } } Map<String, Map<String, String>> properties = configHelper .getEffectiveConfigProperties(cluster, configTags); if (!properties.isEmpty()) { for (Map<String, String> propertyMap : properties.values()) { props.putAll(propertyMap); } } configurations.put(clusterConfig.getType(), props); configVersions.put(clusterConfig.getType(), clusterConfig.getVersion()); configPropertiesTypes.put(clusterConfig.getType(), clusterConfig.getPropertiesTypes()); Map<String, Map<String, String>> attrs = new TreeMap<String, Map<String, String>>(); configHelper.cloneAttributesMap(clusterConfig.getPropertiesAttributes(), attrs); Map<String, Map<String, Map<String, String>>> attributes = configHelper .getEffectiveConfigAttributes(cluster, configTags); for (Map<String, Map<String, String>> attributesMap : attributes.values()) { configHelper.cloneAttributesMap(attributesMap, attrs); } configurationAttributes.put(clusterConfig.getType(), attrs); } } ConfigHelper.processHiddenAttribute(configurations, configurationAttributes, componentName, true); for (Map.Entry<String, Map<String, Map<String, String>>> configurationAttributesEntry : configurationAttributes.entrySet()) { Map<String, Map<String, String>> attrs = configurationAttributesEntry.getValue(); attrs.remove("hidden"); } for (Map.Entry<String, Map<String, String>> configEntry : configurations.entrySet()) { String configType = configEntry.getKey(); Map<String, String> configProperties = configEntry.getValue(); Long configVersion = configVersions.get(configType); Map<PropertyType, Set<String>> propertiesTypes = configPropertiesTypes.get(configType); SecretReference.replacePasswordsWithReferences(propertiesTypes, configProperties, configType, configVersion); } Map<String, Set<String>> clusterHostInfo = null; ServiceInfo serviceInfo = null; String osFamily = null; clusterHostInfo = StageUtils.getClusterHostInfo(cluster); serviceInfo = managementController.getAmbariMetaInfo().getService(stackId.getStackName(), stackId.getStackVersion(), serviceName); try { clusterHostInfo = StageUtils.substituteHostIndexes(clusterHostInfo); } catch (AmbariException e) { throw new SystemException(e.getMessage(), e); } osFamily = clusters.getHost(hostName).getOsFamily(); TreeMap<String, String> hostLevelParams = new TreeMap<String, String>(); hostLevelParams.put(JDK_LOCATION, managementController.getJdkResourceUrl()); hostLevelParams.put(JAVA_HOME, managementController.getJavaHome()); hostLevelParams.put(JAVA_VERSION, String.valueOf(configs.getJavaVersion())); hostLevelParams.put(JDK_NAME, managementController.getJDKName()); hostLevelParams.put(JCE_NAME, managementController.getJCEName()); hostLevelParams.put(STACK_NAME, stackId.getStackName()); hostLevelParams.put(STACK_VERSION, stackId.getStackVersion()); hostLevelParams.put(DB_NAME, managementController.getServerDB()); hostLevelParams.put(MYSQL_JDBC_URL, managementController.getMysqljdbcUrl()); hostLevelParams.put(ORACLE_JDBC_URL, managementController.getOjdbcUrl()); hostLevelParams.put(HOST_SYS_PREPPED, configs.areHostsSysPrepped()); hostLevelParams.putAll(managementController.getRcaParameters()); hostLevelParams.put(AGENT_STACK_RETRY_ON_UNAVAILABILITY, configs.isAgentStackRetryOnInstallEnabled()); hostLevelParams.put(AGENT_STACK_RETRY_COUNT, configs.getAgentStackRetryOnInstallCount()); ServiceOsSpecific anyOs = null; if (serviceInfo.getOsSpecifics().containsKey(AmbariMetaInfo.ANY_OS)) { anyOs = serviceInfo.getOsSpecifics().get(AmbariMetaInfo.ANY_OS); } ServiceOsSpecific hostOs = populateServicePackagesInfo(serviceInfo, hostLevelParams, osFamily); List<ServiceOsSpecific.Package> packages = new ArrayList<ServiceOsSpecific.Package>(); if (anyOs != null) { packages.addAll(anyOs.getPackages()); } if (hostOs != null) { packages.addAll(hostOs.getPackages()); } String packageList = gson.toJson(packages); hostLevelParams.put(PACKAGE_LIST, packageList); Set<String> userSet = configHelper.getPropertyValuesWithPropertyType(stackId, PropertyType.USER, cluster, desiredClusterConfigs); String userList = gson.toJson(userSet); hostLevelParams.put(USER_LIST, userList); Map<String, Set<String>> userGroupsMap = configHelper.createUserGroupsMap( stackId, cluster, desiredClusterConfigs); String userGroups = gson.toJson(userGroupsMap); hostLevelParams.put(USER_GROUPS,userGroups); Set<String> groupSet = configHelper.getPropertyValuesWithPropertyType(stackId, PropertyType.GROUP, cluster, desiredClusterConfigs); String groupList = gson.toJson(groupSet); hostLevelParams.put(GROUP_LIST, groupList); Set<String> notManagedHdfsPathSet = configHelper.getPropertyValuesWithPropertyType(stackId, PropertyType.NOT_MANAGED_HDFS_PATH, cluster, desiredClusterConfigs); String notManagedHdfsPathList = gson.toJson(notManagedHdfsPathSet); hostLevelParams.put(NOT_MANAGED_HDFS_PATH_LIST, notManagedHdfsPathList); String jsonConfigurations = null; Map<String, Object> commandParams = new HashMap<String, Object>(); List<Map<String, String>> xmlConfigs = new LinkedList<Map<String, String>>(); List<Map<String, String>> envConfigs = new LinkedList<Map<String, String>>(); List<Map<String, String>> propertiesConfigs = new LinkedList<Map<String, String>>(); for (ClientConfigFileDefinition clientConfigFile : clientConfigFiles) { Map<String, String> fileDict = new HashMap<String, String>(); fileDict.put(clientConfigFile.getFileName(), clientConfigFile.getDictionaryName()); if (clientConfigFile.getType().equals("xml")) { xmlConfigs.add(fileDict); } else if (clientConfigFile.getType().equals("env")) { envConfigs.add(fileDict); } else if (clientConfigFile.getType().equals("properties")) { propertiesConfigs.add(fileDict); } } commandParams.put("xml_configs_list", xmlConfigs); commandParams.put("env_configs_list", envConfigs); commandParams.put("properties_configs_list", propertiesConfigs); commandParams.put("output_file", componentName + "-configs" + Configuration.DEF_ARCHIVE_EXTENSION); Map<String, Object> jsonContent = new TreeMap<String, Object>(); jsonContent.put("configurations", configurations); jsonContent.put("configuration_attributes", configurationAttributes); jsonContent.put("commandParams", commandParams); jsonContent.put("clusterHostInfo", clusterHostInfo); jsonContent.put("hostLevelParams", hostLevelParams); jsonContent.put("hostname", hostName); jsonContent.put("clusterName", cluster.getClusterName()); jsonConfigurations = gson.toJson(jsonContent); File jsonFileName = new File(TMP_PATH + File.separator + componentName + "-configuration.json"); File tmpDirectory = new File(jsonFileName.getParent()); if (!tmpDirectory.exists()) { try { tmpDirectory.mkdirs(); tmpDirectory.setWritable(true, true); tmpDirectory.setReadable(true, true); } catch (SecurityException se) { throw new SystemException("Failed to get temporary directory to store configurations", se); } } PrintWriter printWriter = null; try { printWriter = new PrintWriter(jsonFileName.getAbsolutePath()); printWriter.print(jsonConfigurations); printWriter.close(); } catch (FileNotFoundException e) { throw new SystemException("Failed to write configurations to json file ", e); } String cmd = pythonCmd + " " + commandScriptAbsolute + " generate_configs " + jsonFileName.getAbsolutePath() + " " + packageFolderAbsolute + " " + TMP_PATH + File.separator + "structured-out.json" + " INFO " + TMP_PATH; pythonCompressFilesCmds.add(cmd); } catch (AmbariException e) { throw new SystemException("Controller error ", e); } } if (schWithConfigFiles.isEmpty()) { throw new SystemException("No configuration files defined for any component" ); } Integer totalCommands = pythonCompressFilesCmds.size() * 2; Integer threadPoolSize = Math.min(totalCommands,configs.getExternalScriptThreadPoolSize()); ExecutorService processExecutor = Executors.newFixedThreadPool(threadPoolSize); List<CommandLineThreadWrapper> pythonCmdThreads = executeCommands(processExecutor, pythonCompressFilesCmds); Integer timeout = configs.getExternalScriptTimeout(); waitForAllThreadsToJoin(processExecutor, pythonCmdThreads, timeout); if (StringUtils.isEmpty(requestComponentName)) { TarUtils tarUtils; String fileName; List <ServiceComponentHostResponse> schToTarConfigFiles = schWithConfigFiles; if (StringUtils.isNotEmpty(requestHostName)) { fileName = requestHostName + "(" + Resource.InternalType.Host.toString().toUpperCase()+")"; } else if (StringUtils.isNotEmpty(requestServiceName)) { fileName = requestServiceName + "(" + Resource.InternalType.Service.toString().toUpperCase()+")"; schToTarConfigFiles = serviceToComponentMap.get(requestServiceName); } else { fileName = schRequest.getClusterName() + "(" + Resource.InternalType.Cluster.toString().toUpperCase()+")"; } tarUtils = new TarUtils(TMP_PATH, fileName, schToTarConfigFiles); tarUtils.tarConfigFiles(); } Resource resource = new ResourceImpl(Resource.Type.ClientConfig); resources.add(resource); return resources; } @AssistedInject  ClientConfigResourceProvider(@Assisted Set<String> propertyIds,
                               @Assisted Map<Resource.Type, String> keyPropertyIds,
                               @Assisted AmbariManagementController managementController); @Override RequestStatus createResources(Request request); @Override @Transactional Set<Resource> getResources(Request request, Predicate predicate); @Override RequestStatus updateResources(final Request request, Predicate predicate); @Override RequestStatus deleteResources(Request request, Predicate predicate);  }