MetricsReporter extends ControllerMaintainer { @Override public boolean maintain() { reportDeploymentMetrics(); reportRemainingRotations(); reportQueuedNameServiceRequests(); reportInfrastructureUpgradeMetrics(); reportAuditLog(); return true; } MetricsReporter(Controller controller, Metric metric); @Override boolean maintain(); static final String DEPLOYMENT_FAIL_METRIC; static final String DEPLOYMENT_AVERAGE_DURATION; static final String DEPLOYMENT_FAILING_UPGRADES; static final String DEPLOYMENT_BUILD_AGE_SECONDS; static final String DEPLOYMENT_WARNINGS; static final String OS_CHANGE_DURATION; static final String PLATFORM_CHANGE_DURATION; static final String OS_NODE_COUNT; static final String PLATFORM_NODE_COUNT; static final String REMAINING_ROTATIONS; static final String NAME_SERVICE_REQUESTS_QUEUED; static final String OPERATION_PREFIX; }
@Test public void audit_log_metric() { var tester = new ControllerTester(); MetricsReporter metricsReporter = createReporter(tester.controller()); HttpRequest req1 = HttpRequest.createTestRequest( "http: com.yahoo.jdisc.http.HttpRequest.Method.PUT ); req1.getJDiscRequest().setUserPrincipal(() -> "user.janedoe"); tester.controller().auditLogger().log((req1)); HttpRequest req2 = HttpRequest.createTestRequest( "http: com.yahoo.jdisc.http.HttpRequest.Method.POST ); req2.getJDiscRequest().setUserPrincipal(() -> "user.johndoe"); tester.controller().auditLogger().log((req2)); metricsReporter.maintain(); assertEquals(1, getMetric(MetricsReporter.OPERATION_PREFIX + "zone", "user.janedoe")); assertEquals(1, getMetric(MetricsReporter.OPERATION_PREFIX + "routing", "user.johndoe")); HttpRequest req3 = HttpRequest.createTestRequest( "http: com.yahoo.jdisc.http.HttpRequest.Method.PUT ); req3.getJDiscRequest().setUserPrincipal(() -> "user.janedoe"); tester.controller().auditLogger().log((req3)); HttpRequest req4 = HttpRequest.createTestRequest( "http: com.yahoo.jdisc.http.HttpRequest.Method.POST ); req4.getJDiscRequest().setUserPrincipal(() -> "user.johndoe"); tester.controller().auditLogger().log((req4)); HttpRequest req5 = HttpRequest.createTestRequest( "http: com.yahoo.jdisc.http.HttpRequest.Method.PUT ); req5.getJDiscRequest().setUserPrincipal(() -> "user.johndoe"); tester.controller().auditLogger().log((req5)); HttpRequest req6 = HttpRequest.createTestRequest( "http: com.yahoo.jdisc.http.HttpRequest.Method.POST ); req6.getJDiscRequest().setUserPrincipal(() -> "user.janedoe"); tester.controller().auditLogger().log((req6)); metricsReporter.maintain(); assertEquals(2, getMetric(MetricsReporter.OPERATION_PREFIX + "zone", "user.janedoe")); assertEquals(2, getMetric(MetricsReporter.OPERATION_PREFIX + "routing", "user.johndoe")); assertEquals(1, getMetric(MetricsReporter.OPERATION_PREFIX + "zone", "user.johndoe")); assertEquals(1, getMetric(MetricsReporter.OPERATION_PREFIX + "routing", "user.janedoe")); }
@Test public void deployment_fail_ratio() { var tester = new DeploymentTester(); ApplicationPackage applicationPackage = new ApplicationPackageBuilder() .environment(Environment.prod) .region("us-west-1") .build(); MetricsReporter metricsReporter = createReporter(tester.controller()); metricsReporter.maintain(); assertEquals(0.0, metrics.getMetric(MetricsReporter.DEPLOYMENT_FAIL_METRIC)); var context1 = tester.newDeploymentContext("app1", "tenant1", "default"); var context2 = tester.newDeploymentContext("app2", "tenant1", "default"); var context3 = tester.newDeploymentContext("app3", "tenant1", "default"); var context4 = tester.newDeploymentContext("app4", "tenant1", "default"); context1.submit(applicationPackage).deploy(); context2.submit(applicationPackage).deploy(); context3.submit(applicationPackage).deploy(); context4.submit(applicationPackage).deploy(); metricsReporter.maintain(); assertEquals(0.0, metrics.getMetric(MetricsReporter.DEPLOYMENT_FAIL_METRIC)); context1.submit(applicationPackage) .triggerJobs() .failDeployment(systemTest); metricsReporter.maintain(); assertEquals(25.0, metrics.getMetric(MetricsReporter.DEPLOYMENT_FAIL_METRIC)); }
@Test public void deployment_average_duration() { var tester = new DeploymentTester(); ApplicationPackage applicationPackage = new ApplicationPackageBuilder() .environment(Environment.prod) .region("us-west-1") .build(); MetricsReporter reporter = createReporter(tester.controller()); var context = tester.newDeploymentContext() .submit(applicationPackage) .deploy(); reporter.maintain(); assertEquals(Duration.ZERO, getAverageDeploymentDuration(context.instanceId())); context.submit(applicationPackage); tester.triggerJobs(); tester.clock().advance(Duration.ofHours(1)); context.runJob(systemTest); tester.clock().advance(Duration.ofMinutes(30)); context.runJob(stagingTest); tester.triggerJobs(); tester.clock().advance(Duration.ofMinutes(90)); context.runJob(productionUsWest1); reporter.maintain(); assertEquals(Duration.ofMinutes(80), getAverageDeploymentDuration(context.instanceId())); context.submit(applicationPackage) .triggerJobs(); tester.clock().advance(Duration.ofHours(12)); reporter.maintain(); assertEquals(Duration.ofHours(12) .plus(Duration.ofHours(12)) .plus(Duration.ofMinutes(90)) .dividedBy(3), getAverageDeploymentDuration(context.instanceId())); }
@Test public void deployments_failing_upgrade() { var tester = new DeploymentTester(); ApplicationPackage applicationPackage = new ApplicationPackageBuilder() .environment(Environment.prod) .region("us-west-1") .build(); MetricsReporter reporter = createReporter(tester.controller()); var context = tester.newDeploymentContext(); context.submit(applicationPackage).deploy(); reporter.maintain(); assertEquals(0, getDeploymentsFailingUpgrade(context.instanceId())); context.submit(applicationPackage) .triggerJobs() .failDeployment(systemTest); reporter.maintain(); assertEquals(0, getDeploymentsFailingUpgrade(context.instanceId())); context.deploy(); assertFalse("Change deployed", context.instance().change().hasTargets()); Version version = Version.fromString("7.1"); tester.controllerTester().upgradeSystem(version); tester.upgrader().maintain(); context.failDeployment(systemTest) .failDeployment(stagingTest); reporter.maintain(); assertEquals(2, getDeploymentsFailingUpgrade(context.instanceId())); context.runJob(systemTest) .runJob(stagingTest) .failDeployment(productionUsWest1); reporter.maintain(); assertEquals(1, getDeploymentsFailingUpgrade(context.instanceId())); context.runJob(productionUsWest1); assertFalse("Upgrade deployed", context.instance().change().hasTargets()); reporter.maintain(); assertEquals(0, getDeploymentsFailingUpgrade(context.instanceId())); }
@Test public void deployment_warnings_metric() { var tester = new DeploymentTester(); ApplicationPackage applicationPackage = new ApplicationPackageBuilder() .environment(Environment.prod) .region("us-west-1") .region("us-east-3") .build(); MetricsReporter reporter = createReporter(tester.controller()); var context = tester.newDeploymentContext(); tester.configServer().generateWarnings(context.deploymentIdIn(ZoneId.from("prod", "us-west-1")), 3); tester.configServer().generateWarnings(context.deploymentIdIn(ZoneId.from("prod", "us-west-1")), 4); context.submit(applicationPackage).deploy(); reporter.maintain(); assertEquals(4, getDeploymentWarnings(context.instanceId())); }
@Test public void build_time_reporting() { var tester = new DeploymentTester(); var applicationPackage = new ApplicationPackageBuilder().region("us-west-1").build(); var context = tester.newDeploymentContext() .submit(applicationPackage) .deploy(); assertEquals(1000, context.lastSubmission().get().buildTime().get().toEpochMilli()); MetricsReporter reporter = createReporter(tester.controller()); reporter.maintain(); assertEquals(tester.clock().instant().getEpochSecond() - 1, getMetric(MetricsReporter.DEPLOYMENT_BUILD_AGE_SECONDS, context.instanceId())); }
@Test public void name_service_queue_size_metric() { var tester = new DeploymentTester(); ApplicationPackage applicationPackage = new ApplicationPackageBuilder() .environment(Environment.prod) .globalServiceId("default") .region("us-west-1") .region("us-east-3") .build(); MetricsReporter reporter = createReporter(tester.controller()); var context = tester.newDeploymentContext() .deferDnsUpdates(); reporter.maintain(); assertEquals("Queue is empty initially", 0, metrics.getMetric(MetricsReporter.NAME_SERVICE_REQUESTS_QUEUED).intValue()); context.submit(applicationPackage).deploy(); reporter.maintain(); assertEquals("Deployment queues name services requests", 15, metrics.getMetric(MetricsReporter.NAME_SERVICE_REQUESTS_QUEUED).intValue()); context.flushDnsUpdates(); reporter.maintain(); assertEquals("Queue consumed", 0, metrics.getMetric(MetricsReporter.NAME_SERVICE_REQUESTS_QUEUED).intValue()); }
@Test public void platform_change_duration() { var tester = new ControllerTester(); var reporter = createReporter(tester.controller()); var zone = ZoneId.from("prod.eu-west-1"); tester.zoneRegistry().setUpgradePolicy(UpgradePolicy.create().upgrade(ZoneApiMock.from(zone))); var systemUpgrader = new SystemUpgrader(tester.controller(), Duration.ofDays(1) ); tester.configServer().bootstrap(List.of(zone), SystemApplication.configServer); var version0 = Version.fromString("7.0"); tester.upgradeSystem(version0); reporter.maintain(); var hosts = tester.configServer().nodeRepository().list(zone, SystemApplication.configServer.id()); assertPlatformChangeDuration(Duration.ZERO, hosts); var targets = List.of(Version.fromString("7.1"), Version.fromString("7.2")); for (int i = 0; i < targets.size(); i++) { var currentVersion = i == 0 ? version0 : targets.get(i - 1); var version = targets.get(i); tester.upgradeController(version); reporter.maintain(); assertPlatformChangeDuration(Duration.ZERO, hosts); systemUpgrader.maintain(); tester.clock().advance(Duration.ofMinutes(30)); runAll(tester::computeVersionStatus, reporter); assertPlatformChangeDuration(Duration.ZERO, hosts); assertEquals("Wanted version is raised for all nodes", version, getNodes(zone, hosts, tester).stream() .map(Node::wantedVersion) .min(Comparator.naturalOrder()) .get()); suspend(hosts, zone, tester); var firstHost = hosts.get(0); upgradeTo(version, List.of(firstHost), zone, tester); tester.clock().advance(Duration.ofHours(1)); runAll(tester::computeVersionStatus, reporter); assertPlatformChangeDuration(Duration.ZERO, List.of(firstHost)); assertPlatformChangeDuration(Duration.ofHours(1), hosts.subList(1, hosts.size())); upgradeTo(version, hosts.subList(1, hosts.size()), zone, tester); runAll(tester::computeVersionStatus, reporter); assertPlatformChangeDuration(Duration.ZERO, hosts); assertEquals(version, tester.controller().systemVersion()); assertPlatformNodeCount(hosts.size(), version); } }
@Test public void os_change_duration() { var tester = new ControllerTester(); var reporter = createReporter(tester.controller()); var zone = ZoneId.from("prod.eu-west-1"); var cloud = CloudName.defaultName(); tester.zoneRegistry().setOsUpgradePolicy(cloud, UpgradePolicy.create().upgrade(ZoneApiMock.from(zone))); var osUpgrader = new OsUpgrader(tester.controller(), Duration.ofDays(1), CloudName.defaultName()); var statusUpdater = new OsVersionStatusUpdater(tester.controller(), Duration.ofDays(1) ); tester.configServer().bootstrap(List.of(zone), SystemApplication.configServerHost, SystemApplication.tenantHost); var version0 = Version.fromString("8.0"); tester.controller().upgradeOsIn(cloud, version0, Optional.empty(), false); osUpgrader.maintain(); tester.configServer().setOsVersion(version0, SystemApplication.tenantHost.id(), zone); tester.configServer().setOsVersion(version0, SystemApplication.configServerHost.id(), zone); runAll(statusUpdater, reporter); List<Node> hosts = tester.configServer().nodeRepository().list(zone); assertOsChangeDuration(Duration.ZERO, hosts); var targets = List.of(Version.fromString("8.1"), Version.fromString("8.2")); var allVersions = Stream.concat(Stream.of(version0), targets.stream()).collect(Collectors.toSet()); for (int i = 0; i < targets.size(); i++) { var currentVersion = i == 0 ? version0 : targets.get(i - 1); var nextVersion = targets.get(i); tester.controller().upgradeOsIn(cloud, nextVersion, Optional.empty(), false); runAll(osUpgrader, statusUpdater, reporter); assertOsChangeDuration(Duration.ZERO, hosts); assertOsNodeCount(hosts.size(), currentVersion); tester.clock().advance(Duration.ofMinutes(30).plus(Duration.ofSeconds(1))); runAll(statusUpdater, reporter); assertOsChangeDuration(Duration.ZERO, hosts); assertEquals("Wanted OS version is raised for all nodes", nextVersion, tester.configServer().nodeRepository().list(zone, SystemApplication.tenantHost.id()).stream() .map(Node::wantedOsVersion).min(Comparator.naturalOrder()).get()); assertTrue("No nodes are suspended", tester.controller().serviceRegistry().configServer() .nodeRepository().list(zone).stream() .noneMatch(node -> node.serviceState() == Node.ServiceState.allowedDown)); tester.clock().advance(Duration.ofMinutes(30)); runAll(statusUpdater, reporter); assertOsChangeDuration(Duration.ZERO, hosts); var suspendedHosts = hosts.subList(0, 3); suspend(suspendedHosts, zone, tester); runAll(statusUpdater, reporter); assertOsChangeDuration(Duration.ZERO, hosts); var hostsUpgraded = suspendedHosts.subList(0, 2); tester.clock().advance(Duration.ofMinutes(20)); runAll(statusUpdater, reporter); assertOsChangeDuration(Duration.ofMinutes(20), hostsUpgraded); upgradeOsTo(nextVersion, hostsUpgraded, zone, tester); runAll(statusUpdater, reporter); assertOsChangeDuration(Duration.ZERO, hostsUpgraded); assertOsNodeCount(hostsUpgraded.size(), nextVersion); var brokenHost = suspendedHosts.get(2); tester.clock().advance(Duration.ofMinutes(15)); runAll(statusUpdater, reporter); assertOsChangeDuration(Duration.ofMinutes(35), List.of(brokenHost)); upgradeOsTo(nextVersion, List.of(brokenHost), zone, tester); runAll(statusUpdater, reporter); assertOsChangeDuration(Duration.ZERO, List.of(brokenHost)); assertOsNodeCount(hostsUpgraded.size() + 1, nextVersion); var remainingHosts = hosts.subList(3, hosts.size()); suspend(remainingHosts, zone, tester); upgradeOsTo(nextVersion, remainingHosts, zone, tester); runAll(statusUpdater, reporter); assertOsChangeDuration(Duration.ZERO, hosts); assertOsNodeCount(hosts.size(), nextVersion); assertOsNodeCount(0, currentVersion); Set<Version> versionDimensions = metrics.getMetrics((dimensions) -> true) .entrySet() .stream() .filter(kv -> kv.getValue().containsKey(MetricsReporter.OS_NODE_COUNT)) .map(kv -> kv.getKey().getDimensions()) .map(dimensions -> dimensions.get("currentVersion")) .map(Version::fromString) .collect(Collectors.toSet()); assertTrue("Reports only OS versions", allVersions.containsAll(versionDimensions)); } }