@Test public void os_change_duration() { var tester = new ControllerTester(); var reporter = createReporter(tester.controller()); var zone = ZoneId.from("prod.eu-west-1"); var cloud = CloudName.defaultName(); tester.zoneRegistry().setOsUpgradePolicy(cloud, UpgradePolicy.create().upgrade(ZoneApiMock.from(zone))); var osUpgrader = new OsUpgrader(tester.controller(), Duration.ofDays(1), CloudName.defaultName()); var statusUpdater = new OsVersionStatusUpdater(tester.controller(), Duration.ofDays(1) ); tester.configServer().bootstrap(List.of(zone), SystemApplication.configServerHost, SystemApplication.tenantHost); var version0 = Version.fromString("8.0"); tester.controller().upgradeOsIn(cloud, version0, Optional.empty(), false); osUpgrader.maintain(); tester.configServer().setOsVersion(version0, SystemApplication.tenantHost.id(), zone); tester.configServer().setOsVersion(version0, SystemApplication.configServerHost.id(), zone); runAll(statusUpdater, reporter); List<Node> hosts = tester.configServer().nodeRepository().list(zone); assertOsChangeDuration(Duration.ZERO, hosts); var targets = List.of(Version.fromString("8.1"), Version.fromString("8.2")); var allVersions = Stream.concat(Stream.of(version0), targets.stream()).collect(Collectors.toSet()); for (int i = 0; i < targets.size(); i++) { var currentVersion = i == 0 ? version0 : targets.get(i - 1); var nextVersion = targets.get(i); tester.controller().upgradeOsIn(cloud, nextVersion, Optional.empty(), false); runAll(osUpgrader, statusUpdater, reporter); assertOsChangeDuration(Duration.ZERO, hosts); assertOsNodeCount(hosts.size(), currentVersion); tester.clock().advance(Duration.ofMinutes(30).plus(Duration.ofSeconds(1))); runAll(statusUpdater, reporter); assertOsChangeDuration(Duration.ZERO, hosts); assertEquals("Wanted OS version is raised for all nodes", nextVersion, tester.configServer().nodeRepository().list(zone, SystemApplication.tenantHost.id()).stream() .map(Node::wantedOsVersion).min(Comparator.naturalOrder()).get()); assertTrue("No nodes are suspended", tester.controller().serviceRegistry().configServer() .nodeRepository().list(zone).stream() .noneMatch(node -> node.serviceState() == Node.ServiceState.allowedDown)); tester.clock().advance(Duration.ofMinutes(30)); runAll(statusUpdater, reporter); assertOsChangeDuration(Duration.ZERO, hosts); var suspendedHosts = hosts.subList(0, 3); suspend(suspendedHosts, zone, tester); runAll(statusUpdater, reporter); assertOsChangeDuration(Duration.ZERO, hosts); var hostsUpgraded = suspendedHosts.subList(0, 2); tester.clock().advance(Duration.ofMinutes(20)); runAll(statusUpdater, reporter); assertOsChangeDuration(Duration.ofMinutes(20), hostsUpgraded); upgradeOsTo(nextVersion, hostsUpgraded, zone, tester); runAll(statusUpdater, reporter); assertOsChangeDuration(Duration.ZERO, hostsUpgraded); assertOsNodeCount(hostsUpgraded.size(), nextVersion); var brokenHost = suspendedHosts.get(2); tester.clock().advance(Duration.ofMinutes(15)); runAll(statusUpdater, reporter); assertOsChangeDuration(Duration.ofMinutes(35), List.of(brokenHost)); upgradeOsTo(nextVersion, List.of(brokenHost), zone, tester); runAll(statusUpdater, reporter); assertOsChangeDuration(Duration.ZERO, List.of(brokenHost)); assertOsNodeCount(hostsUpgraded.size() + 1, nextVersion); var remainingHosts = hosts.subList(3, hosts.size()); suspend(remainingHosts, zone, tester); upgradeOsTo(nextVersion, remainingHosts, zone, tester); runAll(statusUpdater, reporter); assertOsChangeDuration(Duration.ZERO, hosts); assertOsNodeCount(hosts.size(), nextVersion); assertOsNodeCount(0, currentVersion); Set<Version> versionDimensions = metrics.getMetrics((dimensions) -> true) .entrySet() .stream() .filter(kv -> kv.getValue().containsKey(MetricsReporter.OS_NODE_COUNT)) .map(kv -> kv.getKey().getDimensions()) .map(dimensions -> dimensions.get("currentVersion")) .map(Version::fromString) .collect(Collectors.toSet()); assertTrue("Reports only OS versions", allVersions.containsAll(versionDimensions)); } }
@Override public boolean maintain() { reportDeploymentMetrics(); reportRemainingRotations(); reportQueuedNameServiceRequests(); reportInfrastructureUpgradeMetrics(); reportAuditLog(); return true; }
MetricsReporter extends ControllerMaintainer { @Override public boolean maintain() { reportDeploymentMetrics(); reportRemainingRotations(); reportQueuedNameServiceRequests(); reportInfrastructureUpgradeMetrics(); reportAuditLog(); return true; } }
MetricsReporter extends ControllerMaintainer { @Override public boolean maintain() { reportDeploymentMetrics(); reportRemainingRotations(); reportQueuedNameServiceRequests(); reportInfrastructureUpgradeMetrics(); reportAuditLog(); return true; } MetricsReporter(Controller controller, Metric metric); }
MetricsReporter extends ControllerMaintainer { @Override public boolean maintain() { reportDeploymentMetrics(); reportRemainingRotations(); reportQueuedNameServiceRequests(); reportInfrastructureUpgradeMetrics(); reportAuditLog(); return true; } MetricsReporter(Controller controller, Metric metric); @Override boolean maintain(); }
MetricsReporter extends ControllerMaintainer { @Override public boolean maintain() { reportDeploymentMetrics(); reportRemainingRotations(); reportQueuedNameServiceRequests(); reportInfrastructureUpgradeMetrics(); reportAuditLog(); return true; } MetricsReporter(Controller controller, Metric metric); @Override boolean maintain(); static final String DEPLOYMENT_FAIL_METRIC; static final String DEPLOYMENT_AVERAGE_DURATION; static final String DEPLOYMENT_FAILING_UPGRADES; static final String DEPLOYMENT_BUILD_AGE_SECONDS; static final String DEPLOYMENT_WARNINGS; static final String OS_CHANGE_DURATION; static final String PLATFORM_CHANGE_DURATION; static final String OS_NODE_COUNT; static final String PLATFORM_NODE_COUNT; static final String REMAINING_ROTATIONS; static final String NAME_SERVICE_REQUESTS_QUEUED; static final String OPERATION_PREFIX; }