DatabaseConsistencyCheckHelper { static void checkServiceConfigs() { LOG.info("Checking services and their configs"); String GET_SERVICES_WITHOUT_CONFIGS_QUERY = "select c.cluster_name, service_name from clusterservices cs " + "join clusters c on cs.cluster_id=c.cluster_id " + "where service_name not in (select service_name from serviceconfig sc where sc.cluster_id=cs.cluster_id and sc.service_name=cs.service_name and sc.group_id is null)"; String GET_SERVICE_CONFIG_WITHOUT_MAPPING_QUERY = "select c.cluster_name, sc.service_name, sc.version from serviceconfig sc " + "join clusters c on sc.cluster_id=c.cluster_id " + "where service_config_id not in (select service_config_id from serviceconfigmapping) and group_id is null"; String GET_STACK_NAME_VERSION_QUERY = "select c.cluster_name, s.stack_name, s.stack_version from clusters c " + "join stack s on c.desired_stack_id = s.stack_id"; String GET_SERVICES_WITH_CONFIGS_QUERY = "select c.cluster_name, cs.service_name, cc.type_name, sc.version from clusterservices cs " + "join serviceconfig sc on cs.service_name=sc.service_name and cs.cluster_id=sc.cluster_id " + "join serviceconfigmapping scm on sc.service_config_id=scm.service_config_id " + "join clusterconfig cc on scm.config_id=cc.config_id and sc.cluster_id=cc.cluster_id " + "join clusters c on cc.cluster_id=c.cluster_id and sc.stack_id=c.desired_stack_id " + "where sc.group_id is null and sc.service_config_id=(select max(service_config_id) from serviceconfig sc2 where sc2.service_name=sc.service_name and sc2.cluster_id=sc.cluster_id) " + "group by c.cluster_name, cs.service_name, cc.type_name, sc.version"; String GET_NOT_SELECTED_SERVICE_CONFIGS_QUERY = "select c.cluster_name, cs.service_name, cc.type_name from clusterservices cs " + "join serviceconfig sc on cs.service_name=sc.service_name and cs.cluster_id=sc.cluster_id " + "join serviceconfigmapping scm on sc.service_config_id=scm.service_config_id " + "join clusterconfig cc on scm.config_id=cc.config_id and cc.cluster_id=sc.cluster_id " + "join clusters c on cc.cluster_id=c.cluster_id " + "where sc.group_id is null and sc.service_config_id = (select max(service_config_id) from serviceconfig sc2 where sc2.service_name=sc.service_name and sc2.cluster_id=sc.cluster_id) " + "group by c.cluster_name, cs.service_name, cc.type_name " + "having sum(cc.selected) < 1"; Multimap<String, String> clusterServiceMap = HashMultimap.create(); Map<String, Map<String, String>> clusterStackInfo = new HashMap<>(); Map<String, Multimap<String, String>> clusterServiceVersionMap = new HashMap<>(); Map<String, Multimap<String, String>> clusterServiceConfigType = new HashMap<>(); ResultSet rs = null; Statement statement = null; ensureConnection(); LOG.info("Getting ambari metainfo instance"); if (ambariMetaInfo == null) { ambariMetaInfo = injector.getInstance(AmbariMetaInfo.class); } try { LOG.info("Executing query 'GET_SERVICES_WITHOUT_CONFIGS'"); statement = connection.createStatement(ResultSet.TYPE_SCROLL_SENSITIVE, ResultSet.CONCUR_UPDATABLE); rs = statement.executeQuery(GET_SERVICES_WITHOUT_CONFIGS_QUERY); if (rs != null) { while (rs.next()) { clusterServiceMap.put(rs.getString("cluster_name"), rs.getString("service_name")); } for (String clusterName : clusterServiceMap.keySet()) { warning("Service(s): {}, from cluster {} has no config(s) in serviceconfig table!", StringUtils.join(clusterServiceMap.get(clusterName), ","), clusterName); } } LOG.info("Executing query 'GET_SERVICE_CONFIG_WITHOUT_MAPPING'"); rs = statement.executeQuery(GET_SERVICE_CONFIG_WITHOUT_MAPPING_QUERY); if (rs != null) { String serviceName = null, version = null, clusterName = null; while (rs.next()) { serviceName = rs.getString("service_name"); clusterName = rs.getString("cluster_name"); version = rs.getString("version"); if (clusterServiceVersionMap.get(clusterName) != null) { Multimap<String, String> serviceVersion = clusterServiceVersionMap.get(clusterName); serviceVersion.put(serviceName, version); } else { Multimap<String, String> serviceVersion = HashMultimap.create();; serviceVersion.put(serviceName, version); clusterServiceVersionMap.put(clusterName, serviceVersion); } } for (String clName : clusterServiceVersionMap.keySet()) { Multimap<String, String> serviceVersion = clusterServiceVersionMap.get(clName); for (String servName : serviceVersion.keySet()) { error("In cluster {}, service config mapping is unavailable (in table serviceconfigmapping) for service {} with version(s) {}! ", clName, servName, StringUtils.join(serviceVersion.get(servName), ",")); } } } LOG.info("Getting stack info from database"); rs = statement.executeQuery(GET_STACK_NAME_VERSION_QUERY); if (rs != null) { while (rs.next()) { Map<String, String> stackInfoMap = new HashMap<>(); stackInfoMap.put(rs.getString("stack_name"), rs.getString("stack_version")); clusterStackInfo.put(rs.getString("cluster_name"), stackInfoMap); } } Set<String> serviceNames = new HashSet<>(); Map<String, Map<Integer, Multimap<String, String>>> dbClusterServiceVersionConfigs = new HashMap<>(); Multimap<String, String> stackServiceConfigs = HashMultimap.create(); LOG.info("Executing query 'GET_SERVICES_WITH_CONFIGS'"); rs = statement.executeQuery(GET_SERVICES_WITH_CONFIGS_QUERY); if (rs != null) { String serviceName = null, configType = null, clusterName = null; Integer serviceVersion = null; while (rs.next()) { clusterName = rs.getString("cluster_name"); serviceName = rs.getString("service_name"); configType = rs.getString("type_name"); serviceVersion = rs.getInt("version"); serviceNames.add(serviceName); if (dbClusterServiceVersionConfigs.get(clusterName) != null) { Map<Integer, Multimap<String, String>> dbServiceVersionConfigs = dbClusterServiceVersionConfigs.get(clusterName); if (dbServiceVersionConfigs.get(serviceVersion) != null) { dbServiceVersionConfigs.get(serviceVersion).put(serviceName, configType); } else { Multimap<String, String> dbServiceConfigs = HashMultimap.create(); dbServiceConfigs.put(serviceName, configType); dbServiceVersionConfigs.put(serviceVersion, dbServiceConfigs); } } else { Map<Integer, Multimap<String, String>> dbServiceVersionConfigs = new HashMap<>(); Multimap<String, String> dbServiceConfigs = HashMultimap.create(); dbServiceConfigs.put(serviceName, configType); dbServiceVersionConfigs.put(serviceVersion, dbServiceConfigs); dbClusterServiceVersionConfigs.put(clusterName, dbServiceVersionConfigs); } } } LOG.info("Comparing service configs from stack with configs that we got from db"); for (Map.Entry<String, Map<String, String>> clusterStackInfoEntry : clusterStackInfo.entrySet()) { String clusterName = clusterStackInfoEntry.getKey(); Map<String, String> stackInfo = clusterStackInfoEntry.getValue(); String stackName = stackInfo.keySet().iterator().next(); String stackVersion = stackInfo.get(stackName); LOG.info("Getting services from metainfo"); Map<String, ServiceInfo> serviceInfoMap = ambariMetaInfo.getServices(stackName, stackVersion); for (String serviceName : serviceNames) { LOG.info("Processing {}-{} / {}", stackName, stackVersion, serviceName); ServiceInfo serviceInfo = serviceInfoMap.get(serviceName); if (serviceInfo != null) { Set<String> configTypes = serviceInfo.getConfigTypeAttributes().keySet(); for (String configType : configTypes) { stackServiceConfigs.put(serviceName, configType); } } else { warning("Service {} is not available for stack {} in cluster {}", serviceName, stackName + "-" + stackVersion, clusterName); } } LOG.info("Comparing required service configs from stack with mapped service configs from db"); Map<Integer, Multimap<String, String>> dbServiceVersionConfigs = dbClusterServiceVersionConfigs.get(clusterName); if (dbServiceVersionConfigs != null) { for (Integer serviceVersion : dbServiceVersionConfigs.keySet()) { Multimap<String, String> dbServiceConfigs = dbServiceVersionConfigs.get(serviceVersion); if (dbServiceConfigs != null) { for (String serviceName : dbServiceConfigs.keySet()) { Collection<String> serviceConfigsFromStack = stackServiceConfigs.get(serviceName); Collection<String> serviceConfigsFromDB = dbServiceConfigs.get(serviceName); if (serviceConfigsFromDB != null && serviceConfigsFromStack != null) { serviceConfigsFromStack.removeAll(serviceConfigsFromDB); if(!dbServiceConfigs.containsKey("RANGER")) { removeStringsByRegexp(serviceConfigsFromStack, "^ranger-"+ serviceName.toLowerCase() + "-" + "*"); } if (!serviceConfigsFromStack.isEmpty()) { error("Required config(s): {} is(are) not available for service {} with service config version {} in cluster {}", StringUtils.join(serviceConfigsFromStack, ","), serviceName, Integer.toString(serviceVersion), clusterName); } } } } } } } LOG.info("Getting services which has mapped configs which are not selected in clusterconfig"); rs = statement.executeQuery(GET_NOT_SELECTED_SERVICE_CONFIGS_QUERY); if (rs != null) { String serviceName = null, configType = null, clusterName = null; while (rs.next()) { clusterName = rs.getString("cluster_name"); serviceName = rs.getString("service_name"); configType = rs.getString("type_name"); if (clusterServiceConfigType.get(clusterName) != null) { Multimap<String, String> serviceConfigs = clusterServiceConfigType.get(clusterName); serviceConfigs.put(serviceName, configType); } else { Multimap<String, String> serviceConfigs = HashMultimap.create(); serviceConfigs.put(serviceName, configType); clusterServiceConfigType.put(clusterName, serviceConfigs); } } } for (String clusterName : clusterServiceConfigType.keySet()) { Multimap<String, String> serviceConfig = clusterServiceConfigType.get(clusterName); for (String serviceName : serviceConfig.keySet()) { error("You have non selected configs: {} for service {} from cluster {}!", StringUtils.join(serviceConfig.get(serviceName), ","), serviceName, clusterName); } } } catch (SQLException e) { LOG.error("Exception occurred during complex service check procedure: ", e); } catch (AmbariException e) { LOG.error("Exception occurred during complex service check procedure: ", e); } finally { if (rs != null) { try { rs.close(); } catch (SQLException e) { LOG.error("Exception occurred during result set closing procedure: ", e); } } if (statement != null) { try { statement.close(); } catch (SQLException e) { LOG.error("Exception occurred during statement closing procedure: ", e); } } } }  static DatabaseConsistencyCheckResult getLastCheckResult(); static void resetCheckResult(); static void setInjector(Injector injector); static void setConnection(Connection connection); static void closeConnection(); static DatabaseConsistencyCheckResult runAllDBChecks(boolean fixIssues); static void checkDBVersionCompatible();  }
@Test public void testCheckServiceConfigs() throws Exception { EasyMockSupport easyMockSupport = new EasyMockSupport(); final AmbariMetaInfo mockAmbariMetainfo = easyMockSupport.createNiceMock(AmbariMetaInfo.class); final DBAccessor mockDBDbAccessor = easyMockSupport.createNiceMock(DBAccessor.class); final Connection mockConnection = easyMockSupport.createNiceMock(Connection.class); final ResultSet mockResultSet = easyMockSupport.createNiceMock(ResultSet.class); final ResultSet stackResultSet = easyMockSupport.createNiceMock(ResultSet.class); final ResultSet serviceConfigResultSet = easyMockSupport.createNiceMock(ResultSet.class); final Statement mockStatement = easyMockSupport.createNiceMock(Statement.class); final ServiceInfo mockHDFSServiceInfo = easyMockSupport.createNiceMock(ServiceInfo.class); final StackManagerFactory mockStackManagerFactory = easyMockSupport.createNiceMock(StackManagerFactory.class); final EntityManager mockEntityManager = easyMockSupport.createNiceMock(EntityManager.class); final Clusters mockClusters = easyMockSupport.createNiceMock(Clusters.class); final OsFamily mockOSFamily = easyMockSupport.createNiceMock(OsFamily.class); final Injector mockInjector = Guice.createInjector(new AbstractModule() { @Override protected void configure() { bind(AmbariMetaInfo.class).toInstance(mockAmbariMetainfo); bind(StackManagerFactory.class).toInstance(mockStackManagerFactory); bind(EntityManager.class).toInstance(mockEntityManager); bind(DBAccessor.class).toInstance(mockDBDbAccessor); bind(Clusters.class).toInstance(mockClusters); bind(OsFamily.class).toInstance(mockOSFamily); } }); Map<String, ServiceInfo> services = new HashMap<>(); services.put("HDFS", mockHDFSServiceInfo); Map<String, Map<String, Map<String, String>>> configAttributes = new HashMap<>(); configAttributes.put("core-site", new HashMap<String, Map<String, String>>()); expect(mockHDFSServiceInfo.getConfigTypeAttributes()).andReturn(configAttributes); expect(mockAmbariMetainfo.getServices("HDP", "2.2")).andReturn(services); expect(serviceConfigResultSet.next()).andReturn(true).times(2); expect(serviceConfigResultSet.getString("service_name")).andReturn("HDFS").andReturn("HBASE"); expect(serviceConfigResultSet.getString("type_name")).andReturn("core-site").andReturn("hbase-env"); expect(stackResultSet.next()).andReturn(true); expect(stackResultSet.getString("stack_name")).andReturn("HDP"); expect(stackResultSet.getString("stack_version")).andReturn("2.2"); expect(mockConnection.createStatement(ResultSet.TYPE_SCROLL_SENSITIVE, ResultSet.CONCUR_UPDATABLE)).andReturn(mockStatement); expect(mockStatement.executeQuery("select c.cluster_name, service_name from clusterservices cs " + "join clusters c on cs.cluster_id=c.cluster_id " + "where service_name not in (select service_name from serviceconfig sc where sc.cluster_id=cs.cluster_id and sc.service_name=cs.service_name and sc.group_id is null)")).andReturn(mockResultSet); expect(mockStatement.executeQuery("select c.cluster_name, sc.service_name, sc.version from serviceconfig sc " + "join clusters c on sc.cluster_id=c.cluster_id " + "where service_config_id not in (select service_config_id from serviceconfigmapping) and group_id is null")).andReturn(mockResultSet); expect(mockStatement.executeQuery("select c.cluster_name, s.stack_name, s.stack_version from clusters c " + "join stack s on c.desired_stack_id = s.stack_id")).andReturn(stackResultSet); expect(mockStatement.executeQuery("select c.cluster_name, cs.service_name, cc.type_name, sc.version from clusterservices cs " + "join serviceconfig sc on cs.service_name=sc.service_name and cs.cluster_id=sc.cluster_id " + "join serviceconfigmapping scm on sc.service_config_id=scm.service_config_id " + "join clusterconfig cc on scm.config_id=cc.config_id and sc.cluster_id=cc.cluster_id " + "join clusters c on cc.cluster_id=c.cluster_id and sc.stack_id=c.desired_stack_id " + "where sc.group_id is null and sc.service_config_id=(select max(service_config_id) from serviceconfig sc2 where sc2.service_name=sc.service_name and sc2.cluster_id=sc.cluster_id) " + "group by c.cluster_name, cs.service_name, cc.type_name, sc.version")).andReturn(serviceConfigResultSet); expect(mockStatement.executeQuery("select c.cluster_name, cs.service_name, cc.type_name from clusterservices cs " + "join serviceconfig sc on cs.service_name=sc.service_name and cs.cluster_id=sc.cluster_id " + "join serviceconfigmapping scm on sc.service_config_id=scm.service_config_id " + "join clusterconfig cc on scm.config_id=cc.config_id and cc.cluster_id=sc.cluster_id " + "join clusters c on cc.cluster_id=c.cluster_id " + "where sc.group_id is null and sc.service_config_id = (select max(service_config_id) from serviceconfig sc2 where sc2.service_name=sc.service_name and sc2.cluster_id=sc.cluster_id) " + "group by c.cluster_name, cs.service_name, cc.type_name " + "having sum(cc.selected) < 1")).andReturn(mockResultSet); DatabaseConsistencyCheckHelper.setInjector(mockInjector); DatabaseConsistencyCheckHelper.setConnection(mockConnection); easyMockSupport.replayAll(); mockAmbariMetainfo.init(); DatabaseConsistencyCheckHelper.checkServiceConfigs(); easyMockSupport.verifyAll(); }