ThresholdMonitoringReporter extends ScheduledMetrics2Reporter { private void checkMetrics(Map<String, Map<MetricName, Metric>> metricsGroupedByName, Check check) { List<CheckResult> checkResults = new LinkedList<CheckResult>(); Map<MetricName, Metric> metricsOfName = metricsGroupedByName.get(check.getTarget().getName()); if (metricsOfName == null) { metricsOfName = Collections.emptyMap(); } for (Map.Entry<MetricName, Metric> entry : metricsOfName.entrySet()) { if (entry.getKey().matches(check.getTarget())) { try { checkResults.addAll(check.check(entry.getKey(), asMap(entry.getValue()))); } catch (RuntimeException e) { logger.warn(e.getMessage(), e); } } } try { addIncident(check, checkResults); } catch (RuntimeException e) { logger.warn(e.getMessage(), e); } } ThresholdMonitoringReporter(ThresholdMonitoringReporterBuilder builder); static ThresholdMonitoringReporterBuilder forRegistry(Metric2Registry registry); @Override void reportMetrics(Map<MetricName, Gauge> gauges, Map<MetricName, Counter> counters, Map<MetricName, Histogram> histograms, Map<MetricName, Meter> meters, Map<MetricName, Timer> timers);  }
@Test public void testAlerting() throws Exception { Check check = createCheckCheckingMean(1, 5); when(alertingPlugin.getChecks()).thenReturn(Collections.singletonMap(check.getId(), check)); checkMetrics(); ArgumentCaptor<Alerter.AlertArguments> alertArguments = ArgumentCaptor.forClass(Alerter.AlertArguments.class); verify(alerter).alert(alertArguments.capture()); final Incident incident = alertArguments.getValue().getIncident(); assertEquals(check.getId(), incident.getCheckId()); assertEquals("Test Timer", incident.getCheckName()); assertEquals(CheckResult.Status.OK, incident.getOldStatus()); assertEquals(CheckResult.Status.WARN, incident.getNewStatus()); assertEquals(1, incident.getCheckResults().size()); List<CheckResult> checkResults = incident.getCheckResults().iterator().next().getResults(); assertEquals(2, checkResults.size()); CheckResult result = checkResults.get(0); assertEquals("test_timer,signature=timer3 mean < 5.0 is false", result.getFailingExpression()); assertEquals(6.0, result.getCurrentValue(), 0); assertEquals(CheckResult.Status.WARN, result.getStatus()); }
@Test public void testAlertAfter2Failures() throws Exception { Check check = createCheckCheckingMean(2, 6); when(alertingPlugin.getChecks()).thenReturn(Collections.singletonMap(check.getId(), check)); checkMetrics(); verify(alerter, times(0)).alert(any(Alerter.AlertArguments.class)); verify(incidentRepository).createIncident(any(Incident.class)); checkMetrics(); verify(alerter).alert(any(Alerter.AlertArguments.class)); verify(incidentRepository).updateIncident(any(Incident.class)); }
@Test public void testNoAlertWhenFailureRecovers() throws Exception { Check check = createCheckCheckingMean(2, 6); when(alertingPlugin.getChecks()).thenReturn(Collections.singletonMap(check.getId(), check)); checkMetrics(7, 0, 0); verify(alerter, times(0)).alert(any(Alerter.AlertArguments.class)); final Incident incident = incidentRepository.getIncidentByCheckId(check.getId()); assertNotNull(incident); assertEquals(CheckResult.Status.OK, incident.getOldStatus()); assertEquals(CheckResult.Status.WARN, incident.getNewStatus()); assertNotNull(incident.getFirstFailureAt()); assertNull(incident.getResolvedAt()); assertEquals(1, incident.getConsecutiveFailures()); System.out.println(incident); checkMetrics(1, 0, 0); verify(alerter, times(0)).alert(any(Alerter.AlertArguments.class)); assertNull(incidentRepository.getIncidentByCheckId(check.getId())); }
@Test public void testAlertWhenBackToOk() throws Exception { Check check = createCheckCheckingMean(1, 6); when(alertingPlugin.getChecks()).thenReturn(Collections.singletonMap(check.getId(), check)); checkMetrics(7, 0, 0); verify(alerter, times(1)).alert(any(Alerter.AlertArguments.class)); Incident incident = incidentRepository.getIncidentByCheckId(check.getId()); assertNotNull(incident); assertEquals(CheckResult.Status.OK, incident.getOldStatus()); assertEquals(CheckResult.Status.WARN, incident.getNewStatus()); assertNotNull(incident.getFirstFailureAt()); assertNull(incident.getResolvedAt()); assertEquals(1, incident.getConsecutiveFailures()); System.out.println(incident); checkMetrics(1, 0, 0); ArgumentCaptor<Alerter.AlertArguments> alertArguments = ArgumentCaptor.forClass(Alerter.AlertArguments.class); verify(alerter, times(2)).alert(alertArguments.capture()); incident = alertArguments.getValue().getIncident(); assertNotNull(incident); assertEquals(CheckResult.Status.WARN, incident.getOldStatus()); assertEquals(CheckResult.Status.OK, incident.getNewStatus()); assertNotNull(incident.getFirstFailureAt()); assertNotNull(incident.getResolvedAt()); }
@Test public void testDontDeleteIncidentIfThereAreNonOkResultsFromOtherInstances() { Check check = createCheckCheckingMean(2, 6); when(alertingPlugin.getChecks()).thenReturn(Collections.singletonMap(check.getId(), check)); incidentRepository.createIncident( new Incident(check, new MeasurementSession("testApp", "testHost2", "testInstance"), Arrays.asList(new CheckResult("test", 10, CheckResult.Status.CRITICAL)))); checkMetrics(7, 0, 0); verify(alerter, times(0)).alert(any(Alerter.AlertArguments.class)); verify(incidentRepository).updateIncident(any(Incident.class)); Incident storedIncident = incidentRepository.getIncidentByCheckId(check.getId()); assertEquals(CheckResult.Status.CRITICAL, storedIncident.getOldStatus()); assertEquals(CheckResult.Status.CRITICAL, storedIncident.getNewStatus()); assertEquals(2, storedIncident.getCheckResults().size()); assertEquals(2, storedIncident.getVersion()); boolean containsTestHost = false; boolean containsTestHost2 = false; for (CheckResults checkResults : storedIncident.getCheckResults()) { if (checkResults.getMeasurementSession().getHostName().equals("testHost2")) { containsTestHost2 = true; } else if (checkResults.getMeasurementSession().getHostName().equals("testHost")) { containsTestHost = true; } } assertTrue(containsTestHost); assertTrue(containsTestHost2); System.out.println(storedIncident); System.out.println(JsonUtils.toJson(storedIncident)); checkMetrics(1, 0, 0); verify(alerter, times(0)).alert(any(Alerter.AlertArguments.class)); verify(incidentRepository, times(0)).deleteIncident(any(Incident.class)); verify(incidentRepository, times(2)).updateIncident(any(Incident.class)); storedIncident = incidentRepository.getIncidentByCheckId(check.getId()); assertEquals(CheckResult.Status.CRITICAL, storedIncident.getOldStatus()); assertEquals(CheckResult.Status.CRITICAL, storedIncident.getNewStatus()); assertEquals(1, storedIncident.getCheckResults().size()); assertEquals(3, storedIncident.getVersion()); }