UpgradeCatalog240 extends AbstractUpgradeCatalog { protected void updateAlerts() { final Map<String, String> hdfsVisibilityMap = new HashMap<String, String>(){{ put("mergeHaMetrics", "HIDDEN"); put("appId", "HIDDEN"); put("metricName", "HIDDEN"); }}; final Map<String, String> defaultKeytabVisibilityMap = new HashMap<String, String>(){{ put("default.smoke.principal", "HIDDEN"); put("default.smoke.keytab", "HIDDEN"); }}; final Map<String, String> percentParameterMap = new HashMap<String, String>(){{ put("units", "%"); put("type", "PERCENT"); }}; Map<String, Map<String, String>> visibilityMap = new HashMap<String, Map<String, String>>(){{ put("hive_webhcat_server_status", new HashMap<String, String>(){{ put("default.smoke.user", "HIDDEN"); }}); put("hive_metastore_process", defaultKeytabVisibilityMap); put("hive_server_process", defaultKeytabVisibilityMap); put("zookeeper_server_process", new HashMap<String, String>(){{ put("socket.command", "HIDDEN"); put("socket.command.response", "HIDDEN"); }}); }}; Map<String, Map<String, String>> reportingPercentMap = new HashMap<String, Map<String, String>>(){{ put("hawq_segment_process_percent", percentParameterMap); put("mapreduce_history_server_cpu", percentParameterMap); put("yarn_nodemanager_webui_percent", percentParameterMap); put("yarn_resourcemanager_cpu", percentParameterMap); put("datanode_process_percent", percentParameterMap); put("datanode_storage_percent", percentParameterMap); put("journalnode_process_percent", percentParameterMap); put("namenode_cpu", percentParameterMap); put("namenode_hdfs_capacity_utilization", percentParameterMap); put("datanode_storage", percentParameterMap); put("datanode_heap_usage", percentParameterMap); put("storm_supervisor_process_percent", percentParameterMap); put("hbase_regionserver_process_percent", percentParameterMap); put("hbase_master_cpu", percentParameterMap); put("zookeeper_server_process_percent", percentParameterMap); put("metrics_monitor_process_percent", percentParameterMap); put("ams_metrics_collector_hbase_master_cpu", percentParameterMap); }}; Map<String, Map<String, Integer>> reportingMultiplierMap = new HashMap<String, Map<String, Integer>>(){{ put("hawq_segment_process_percent", new HashMap<String, Integer>() {{ put("warning", 100); put("critical", 100); }}); put("yarn_nodemanager_webui_percent", new HashMap<String, Integer>() {{ put("warning", 100); put("critical", 100); }}); put("datanode_process_percent", new HashMap<String, Integer>() {{ put("warning", 100); put("critical", 100); }}); put("datanode_storage_percent", new HashMap<String, Integer>() {{ put("warning", 100); put("critical", 100); }}); put("journalnode_process_percent", new HashMap<String, Integer>() {{ put("warning", 100); put("critical", 100); }}); put("storm_supervisor_process_percent", new HashMap<String, Integer>() {{ put("warning", 100); put("critical", 100); }}); put("hbase_regionserver_process_percent", new HashMap<String, Integer>() {{ put("warning", 100); put("critical", 100); }}); put("zookeeper_server_process_percent", new HashMap<String, Integer>() {{ put("warning", 100); put("critical", 100); }}); put("metrics_monitor_process_percent", new HashMap<String, Integer>() {{ put("warning", 100); put("critical", 100); }}); }}; Map<String, Map<String, Integer>> scriptAlertMultiplierMap = new HashMap<String, Map<String, Integer>>(){{ put("ambari_agent_disk_usage", new HashMap<String, Integer>() {{ put("percent.used.space.warning.threshold", 100); put("percent.free.space.critical.threshold", 100); }}); put("namenode_last_checkpoint", new HashMap<String, Integer>() {{ put("checkpoint.time.warning.threshold", 100); put("checkpoint.time.critical.threshold", 100); }}); }}; String newNameservicePropertyValue = "{{hdfs-site/dfs.internal.nameservices}}"; final Set<String> alertNamesForNameserviceUpdate = new HashSet<String>() {{ add("namenode_webui"); add("namenode_hdfs_blocks_health"); add("namenode_hdfs_pending_deletion_blocks"); add("namenode_rpc_latency"); add("namenode_directory_status"); add("datanode_health_summary"); add("namenode_cpu"); add("namenode_hdfs_capacity_utilization"); }}; Set<String> alertNamesForPropertyUpdates = new HashSet<String>() {{ add("hawq_segment_process_percent"); add("mapreduce_history_server_cpu"); add("yarn_nodemanager_webui_percent"); add("yarn_resourcemanager_cpu"); add("datanode_process_percent"); add("datanode_storage_percent"); add("journalnode_process_percent"); add("namenode_cpu"); add("namenode_hdfs_capacity_utilization"); add("datanode_storage"); add("datanode_heap_usage"); add("storm_supervisor_process_percent"); add("hbase_regionserver_process_percent"); add("hbase_master_cpu"); add("zookeeper_server_process_percent"); add("metrics_monitor_process_percent"); add("ams_metrics_collector_hbase_master_cpu"); add("ambari_agent_disk_usage"); add("namenode_last_checkpoint"); addAll(alertNamesForNameserviceUpdate); }}; Set<String> alertForRemoval = new HashSet<String>() {{ add("storm_rest_api"); add("mapreduce_history_server_process"); }}; LOG.info("Updating alert definitions."); AmbariManagementController ambariManagementController = injector.getInstance(AmbariManagementController.class); AlertDefinitionDAO alertDefinitionDAO = injector.getInstance(AlertDefinitionDAO.class); Clusters clusters = ambariManagementController.getClusters(); Map<String, Cluster> clusterMap = getCheckedClusterMap(clusters); for (final Cluster cluster : clusterMap.values()) { long clusterID = cluster.getClusterId(); final AlertDefinitionEntity namenodeLastCheckpointAlertDefinitionEntity = alertDefinitionDAO.findByName( clusterID, "namenode_last_checkpoint"); final AlertDefinitionEntity namenodeHAHealthAlertDefinitionEntity = alertDefinitionDAO.findByName( clusterID, "namenode_ha_health"); final AlertDefinitionEntity nodemanagerHealthAlertDefinitionEntity = alertDefinitionDAO.findByName( clusterID, "yarn_nodemanager_health"); final AlertDefinitionEntity nodemanagerHealthSummaryAlertDefinitionEntity = alertDefinitionDAO.findByName( clusterID, "nodemanager_health_summary"); final AlertDefinitionEntity hiveMetastoreProcessAlertDefinitionEntity = alertDefinitionDAO.findByName( clusterID, "hive_metastore_process"); final AlertDefinitionEntity hiveServerProcessAlertDefinitionEntity = alertDefinitionDAO.findByName( clusterID, "hive_server_process"); final AlertDefinitionEntity hiveWebhcatServerStatusAlertDefinitionEntity = alertDefinitionDAO.findByName( clusterID, "hive_webhcat_server_status"); final AlertDefinitionEntity flumeAgentStatusAlertDefinitionEntity = alertDefinitionDAO.findByName( clusterID, "flume_agent_status"); final AlertDefinitionEntity zookeeperServerProcessAlertDefinitionEntity = alertDefinitionDAO.findByName( clusterID, "zookeeper_server_process"); Map<AlertDefinitionEntity, List<String>> alertDefinitionParams = new HashMap<>(); checkedPutToMap(alertDefinitionParams, namenodeLastCheckpointAlertDefinitionEntity, Lists.newArrayList("connection.timeout", "checkpoint.time.warning.threshold", "checkpoint.time.critical.threshold", "checkpoint.txns.multiplier.warning.threshold", "checkpoint.txns.multiplier.critical.threshold")); checkedPutToMap(alertDefinitionParams, namenodeHAHealthAlertDefinitionEntity, Lists.newArrayList("connection.timeout")); checkedPutToMap(alertDefinitionParams, nodemanagerHealthAlertDefinitionEntity, Lists.newArrayList("connection.timeout")); checkedPutToMap(alertDefinitionParams, nodemanagerHealthSummaryAlertDefinitionEntity, Lists.newArrayList("connection.timeout")); checkedPutToMap(alertDefinitionParams, hiveMetastoreProcessAlertDefinitionEntity, Lists.newArrayList("default.smoke.user", "default.smoke.principal", "default.smoke.keytab")); checkedPutToMap(alertDefinitionParams, hiveServerProcessAlertDefinitionEntity, Lists.newArrayList("default.smoke.user", "default.smoke.principal", "default.smoke.keytab")); checkedPutToMap(alertDefinitionParams, hiveWebhcatServerStatusAlertDefinitionEntity, Lists.newArrayList("default.smoke.user", "connection.timeout")); checkedPutToMap(alertDefinitionParams, flumeAgentStatusAlertDefinitionEntity, Lists.newArrayList("run.directory")); checkedPutToMap(alertDefinitionParams, zookeeperServerProcessAlertDefinitionEntity, Lists.newArrayList("socket.command", "socket.command.response")); Map<Long, AlertDefinitionEntity> definitionsForPropertyUpdates = new HashMap<>(); for (Map.Entry<AlertDefinitionEntity, List<String>> entry : alertDefinitionParams.entrySet()){ AlertDefinitionEntity alertDefinition = entry.getKey(); String source = alertDefinition.getSource(); alertDefinition.setSource(addParam(source, entry.getValue())); definitionsForPropertyUpdates.put(alertDefinition.getDefinitionId(), alertDefinition); } for (String name : alertNamesForPropertyUpdates) { AlertDefinitionEntity alertDefinition = alertDefinitionDAO.findByName(clusterID, name); if (alertDefinition != null && !definitionsForPropertyUpdates.containsKey(alertDefinition.getDefinitionId())) { definitionsForPropertyUpdates.put(alertDefinition.getDefinitionId(), alertDefinition); } } for (AlertDefinitionEntity alertDefinition : definitionsForPropertyUpdates.values()) { if (visibilityMap.containsKey(alertDefinition.getDefinitionName())) { for (Map.Entry<String, String> entry : visibilityMap.get(alertDefinition.getDefinitionName()).entrySet()){ String paramName = entry.getKey(); String visibilityValue = entry.getValue(); String source = alertDefinition.getSource(); alertDefinition.setSource(addParamOption(source, paramName, "visibility", visibilityValue)); } } if (scriptAlertMultiplierMap.containsKey(alertDefinition.getDefinitionName())) { for (Map.Entry<String, Integer> entry : scriptAlertMultiplierMap.get(alertDefinition.getDefinitionName()).entrySet()){ String paramName = entry.getKey(); Integer multiplier = entry.getValue(); String source = alertDefinition.getSource(); Float oldValue = getParamFloatValue(source, paramName); if (oldValue == null) { alertDefinition.setSource(addParam(source, Arrays.asList(paramName))); } else { Integer newValue = Math.round(oldValue * multiplier); alertDefinition.setSource(setParamIntegerValue(source, paramName, newValue)); } } } if (reportingMultiplierMap.containsKey(alertDefinition.getDefinitionName())) { for (Map.Entry<String, Integer> entry : reportingMultiplierMap.get(alertDefinition.getDefinitionName()).entrySet()){ String reportingName = entry.getKey(); Integer multiplier = entry.getValue(); String source = alertDefinition.getSource(); Float oldValue = getReportingFloatValue(source, reportingName); Integer newValue = Math.round(oldValue * multiplier); alertDefinition.setSource(setReportingIntegerValue(source, reportingName, newValue)); } } if (reportingPercentMap.containsKey(alertDefinition.getDefinitionName())) { for (Map.Entry<String, String> entry : reportingPercentMap.get(alertDefinition.getDefinitionName()).entrySet()){ String paramName = entry.getKey(); String paramValue = entry.getValue(); String source = alertDefinition.getSource(); alertDefinition.setSource(addReportingOption(source, paramName, paramValue)); } } if (alertNamesForNameserviceUpdate.contains(alertDefinition.getDefinitionName())) { String source = alertDefinition.getSource(); alertDefinition.setSource(setNameservice(source, newNameservicePropertyValue)); } alertDefinition.setHash(UUID.randomUUID().toString()); alertDefinitionDAO.merge(alertDefinition); } final AlertDefinitionEntity atlasMetadataServerWebUI = alertDefinitionDAO.findByName( clusterID, "metadata_server_webui"); if (atlasMetadataServerWebUI != null) { String source = atlasMetadataServerWebUI.getSource(); JsonObject sourceJson = new JsonParser().parse(source).getAsJsonObject(); JsonObject uriJson = sourceJson.get("uri").getAsJsonObject(); uriJson.remove("kerberos_keytab"); uriJson.remove("kerberos_principal"); uriJson.addProperty("kerberos_keytab", "{{cluster-env/smokeuser_keytab}}"); uriJson.addProperty("kerberos_principal", "{{cluster-env/smokeuser_principal_name}}"); atlasMetadataServerWebUI.setSource(sourceJson.toString()); atlasMetadataServerWebUI.setHash(UUID.randomUUID().toString()); alertDefinitionDAO.merge(atlasMetadataServerWebUI); } for (String alertName: alertForRemoval) { AlertDefinitionEntity alertDefinition = alertDefinitionDAO.findByName(clusterID, alertName); if (alertDefinition != null) { LOG.info("Removing alert : " + alertName); alertDefinitionDAO.remove(alertDefinition); } } } } @Inject  UpgradeCatalog240(Injector injector); @Override String getTargetVersion(); @Override String getSourceVersion(); static final String DESIRED_VERSION_COLUMN_NAME; static final String BLUEPRINT_SETTING_TABLE; static final String BLUEPRINT_NAME_COL; static final String SETTING_NAME_COL; static final String SETTING_DATA_COL; static final String ID; static final String BLUEPRINT_TABLE; static final String VIEWINSTANCE_TABLE; static final String SHORT_URL_COLUMN; static final String CLUSTER_HANDLE_COLUMN; static final String REQUESTSCHEDULE_TABLE; static final String AUTHENTICATED_USER_ID_COLUMN; static final String TIMELINE_METRICS_SINK_COLLECTION_PERIOD; static final String ONE_DIR_PER_PARITION_PROPERTY; static final String VIEWURL_TABLE; static final String URL_ID_COLUMN; static final String PRECISION_TABLE_TTL_PROPERTY; static final String CLUSTER_SECOND_TABLE_TTL_PROPERTY; static final String CLUSTER_TYPE_COLUMN; static final String REMOTE_AMBARI_CLUSTER_TABLE; static final String REMOTE_AMBARI_CLUSTER_SERVICE_TABLE; static final String CLUSTER_ID; static final String SERVICE_NAME; static final String CLUSTER_NAME; }
@Test public void testNameserviceAlertsUpdate() { EasyMockSupport easyMockSupport = new EasyMockSupport(); long clusterId = 1; final AmbariManagementController mockAmbariManagementController = easyMockSupport.createNiceMock(AmbariManagementController.class); final AlertDefinitionDAO mockAlertDefinitionDAO = easyMockSupport.createNiceMock(AlertDefinitionDAO.class); final Clusters mockClusters = easyMockSupport.createStrictMock(Clusters.class); final Cluster mockClusterExpected = easyMockSupport.createNiceMock(Cluster.class); final AlertDefinitionEntity namenodeWebUiAlertDefinitionEntity = new AlertDefinitionEntity(); final AlertDefinitionEntity mockStormRestApiAlertDefinitionEntity = easyMockSupport.createNiceMock(AlertDefinitionEntity.class); namenodeWebUiAlertDefinitionEntity.setDefinitionName("namenode_webui"); namenodeWebUiAlertDefinitionEntity.setSource("{\"uri\": {\"high_availability\": {\"nameservice\": \"{{hdfs-site/dfs.nameservices}}\",\"alias_key\" : \"{{hdfs-site/dfs.ha.namenodes.{{ha-nameservice}}}}\",\"http_pattern\" : \"{{hdfs-site/dfs.namenode.http-address.{{ha-nameservice}}.{{alias}}}}\",\"https_pattern\" : \"{{hdfs-site/dfs.namenode.https-address.{{ha-nameservice}}.{{alias}}}}\"}}}"); final Injector mockInjector = Guice.createInjector(new AbstractModule() { @Override protected void configure() { bind(AmbariManagementController.class).toInstance(mockAmbariManagementController); bind(Clusters.class).toInstance(mockClusters); bind(EntityManager.class).toInstance(entityManager); bind(AlertDefinitionDAO.class).toInstance(mockAlertDefinitionDAO); bind(DBAccessor.class).toInstance(createNiceMock(DBAccessor.class)); bind(OsFamily.class).toInstance(createNiceMock(OsFamily.class)); bind(PasswordEncoder.class).toInstance(createNiceMock(PasswordEncoder.class)); bind(HookContextFactory.class).toInstance(createMock(HookContextFactory.class)); bind(HookService.class).toInstance(createMock(HookService.class)); } }); expect(mockAmbariManagementController.getClusters()).andReturn(mockClusters).once(); expect(mockClusters.getClusters()).andReturn(new HashMap<String, Cluster>() {{ put("normal", mockClusterExpected); }}).atLeastOnce(); expect(mockClusterExpected.getClusterId()).andReturn(clusterId).anyTimes(); expect(mockAlertDefinitionDAO.findByName(eq(clusterId), eq("namenode_webui"))) .andReturn(namenodeWebUiAlertDefinitionEntity).atLeastOnce(); expect(mockAlertDefinitionDAO.findByName(eq(clusterId), eq("mapreduce_history_server_process"))) .andReturn(mockStormRestApiAlertDefinitionEntity).atLeastOnce(); expect(mockAlertDefinitionDAO.findByName(eq(clusterId), eq("storm_rest_api"))) .andReturn(mockStormRestApiAlertDefinitionEntity).atLeastOnce(); mockAlertDefinitionDAO.remove(mockStormRestApiAlertDefinitionEntity); expectLastCall().once(); easyMockSupport.replayAll(); mockInjector.getInstance(UpgradeCatalog240.class).updateAlerts(); assertTrue(namenodeWebUiAlertDefinitionEntity.getSource().contains("{{hdfs-site/dfs.internal.nameservices}}")); easyMockSupport.verifyAll(); }
@Test public void testAtlasMetastoreAlertHashUpdate() { EasyMockSupport easyMockSupport = new EasyMockSupport(); long clusterId = 1; final AmbariManagementController mockAmbariManagementController = easyMockSupport.createNiceMock(AmbariManagementController.class); final AlertDefinitionDAO mockAlertDefinitionDAO = easyMockSupport.createNiceMock(AlertDefinitionDAO.class); final Clusters mockClusters = easyMockSupport.createStrictMock(Clusters.class); final Cluster mockClusterExpected = easyMockSupport.createNiceMock(Cluster.class); final AlertDefinitionEntity atlasMetastoreAlertDefinitionEntity = new AlertDefinitionEntity(); atlasMetastoreAlertDefinitionEntity.setDefinitionName("metadata_server_webui"); atlasMetastoreAlertDefinitionEntity.setHash("initial_hash"); atlasMetastoreAlertDefinitionEntity.setSource("{\"uri\": {\n" + " \"http\": \"{{hostname}}:{{application-properties/atlas.server.http.port}}\",\n" + " \"https\": \"{{hostname}}:{{application-properties/atlas.server.https.port}}\",\n" + " \"https_property\": \"{{application-properties/atlas.enableTLS}}\",\n" + " \"https_property_value\": \"true\",\n" + " \"default_port\": 21000,\n" + " \"kerberos_keytab\": \"{{application-properties/atlas.http.authentication.kerberos.keytab}}\",\n" + " \"kerberos_principal\": \"{{application-properties/atlas.http.authentication.kerberos.principal}}\",\n" + " \"connection_timeout\": 5.0\n" + " }}"); final Injector mockInjector = Guice.createInjector(new AbstractModule() { @Override protected void configure() { bind(AmbariManagementController.class).toInstance(mockAmbariManagementController); bind(Clusters.class).toInstance(mockClusters); bind(EntityManager.class).toInstance(entityManager); bind(AlertDefinitionDAO.class).toInstance(mockAlertDefinitionDAO); bind(DBAccessor.class).toInstance(createNiceMock(DBAccessor.class)); bind(OsFamily.class).toInstance(createNiceMock(OsFamily.class)); bind(PasswordEncoder.class).toInstance(createNiceMock(PasswordEncoder.class)); bind(HookContextFactory.class).toInstance(createMock(HookContextFactory.class)); bind(HookService.class).toInstance(createMock(HookService.class)); } }); expect(mockAmbariManagementController.getClusters()).andReturn(mockClusters).once(); expect(mockClusters.getClusters()).andReturn(new HashMap<String, Cluster>() {{ put("normal", mockClusterExpected); }}).atLeastOnce(); expect(mockClusterExpected.getClusterId()).andReturn(clusterId).anyTimes(); expect(mockAlertDefinitionDAO.findByName(eq(clusterId), eq("metadata_server_webui"))) .andReturn(atlasMetastoreAlertDefinitionEntity).atLeastOnce(); expect(mockAlertDefinitionDAO.merge(atlasMetastoreAlertDefinitionEntity)).andReturn(atlasMetastoreAlertDefinitionEntity); easyMockSupport.replayAll(); mockInjector.getInstance(UpgradeCatalog240.class).updateAlerts(); assertFalse(atlasMetastoreAlertDefinitionEntity.getHash().equals("initial_hash")); easyMockSupport.verifyAll(); }