DynamicProvisioningMaintainer extends NodeRepositoryMaintainer { @Override protected boolean maintain() { try (Mutex lock = nodeRepository().lockUnallocated()) { NodeList nodes = nodeRepository().list(); resumeProvisioning(nodes, lock); convergeToCapacity(nodes); } return true; } DynamicProvisioningMaintainer(NodeRepository nodeRepository,
                                  Duration interval,
                                  HostProvisioner hostProvisioner,
                                  FlagSource flagSource,
                                  Metric metric);   }
@Test public void delegates_to_host_provisioner_and_writes_back_result() { var tester = new DynamicProvisioningTester().addInitialNodes(); tester.hostProvisioner.with(Behaviour.failDeprovisioning); Node host3 = tester.nodeRepository.getNode("host3").orElseThrow(); Node host4 = tester.nodeRepository.getNode("host4").orElseThrow(); Node host41 = tester.nodeRepository.getNode("host4-1").orElseThrow(); assertTrue("No IP addresses assigned", Stream.of(host3, host4, host41).map(node -> node.ipConfig().primary()).allMatch(Set::isEmpty)); Node host3new = host3.with(host3.ipConfig().with(Set.of("::3:0"))); Node host4new = host4.with(host4.ipConfig().with(Set.of("::4:0"))); Node host41new = host41.with(host41.ipConfig().with(Set.of("::4:1", "::4:2"))); tester.maintainer.maintain(); assertEquals(host3new, tester.nodeRepository.getNode("host3").get()); assertEquals(host4new, tester.nodeRepository.getNode("host4").get()); assertEquals(host41new, tester.nodeRepository.getNode("host4-1").get()); }
@Test public void correctly_fails_if_irrecoverable_failure() { var tester = new DynamicProvisioningTester(); tester.hostProvisioner.with(Behaviour.failProvisioning); Node host4 = tester.addNode("host4", Optional.empty(), NodeType.host, Node.State.provisioned); Node host41 = tester.addNode("host4-1", Optional.of("host4"), NodeType.tenant, Node.State.reserved, DynamicProvisioningTester.tenantApp); assertTrue("No IP addresses assigned", Stream.of(host4, host41).map(node -> node.ipConfig().primary()).allMatch(Set::isEmpty)); tester.maintainer.maintain(); assertEquals(Set.of("host4", "host4-1"), tester.nodeRepository.getNodes(Node.State.failed).stream().map(Node::hostname).collect(Collectors.toSet())); }
@Test public void finds_nodes_that_need_deprovisioning_without_pre_provisioning() { var tester = new DynamicProvisioningTester().addInitialNodes(); assertTrue(tester.nodeRepository.getNode("host2").isPresent()); assertTrue(tester.nodeRepository.getNode("host3").isPresent()); tester.maintainer.maintain(); assertTrue(tester.nodeRepository.getNode("host2").isEmpty()); assertTrue(tester.nodeRepository.getNode("host3").isEmpty()); }
@Test public void does_not_deprovision_when_preprovisioning_enabled() { var tester = new DynamicProvisioningTester().addInitialNodes(); tester.flagSource.withListFlag(Flags.TARGET_CAPACITY.id(), List.of(new HostCapacity(1, 3, 2, 1)), HostCapacity.class); Optional<Node> failedHost = tester.nodeRepository.getNode("host2"); assertTrue(failedHost.isPresent()); tester.maintainer.maintain(); assertTrue("Failed host is deprovisioned", tester.nodeRepository.getNode(failedHost.get().hostname()).isEmpty()); assertEquals(1, tester.hostProvisioner.deprovisionedHosts); }
@Test public void provision_deficit_and_deprovision_excess() { var tester = new DynamicProvisioningTester().addInitialNodes(); tester.flagSource.withListFlag(Flags.TARGET_CAPACITY.id(), List.of(new HostCapacity(24, 64, 100, 2), new HostCapacity(16, 24, 100, 1)), HostCapacity.class); assertTrue(tester.nodeRepository.getNode("host2").isPresent()); assertEquals(0 ,tester.hostProvisioner.provisionedHosts.size()); Optional<Node> failedHost = tester.nodeRepository.getNode("host2"); assertTrue(failedHost.isPresent()); tester.maintainer.maintain(); assertTrue("Failed host is deprovisioned", tester.nodeRepository.getNode(failedHost.get().hostname()).isEmpty()); assertTrue("Host with matching resources is kept", tester.nodeRepository.getNode("host3").isPresent()); NodeResources resources = new NodeResources(24, 64, 100, 1); assertEquals(2, tester.provisionedHostsMatching(resources)); }
@Test public void does_not_remove_if_host_provisioner_failed() { var tester = new DynamicProvisioningTester(); Node host2 = tester.addNode("host2", Optional.empty(), NodeType.host, Node.State.failed, DynamicProvisioningTester.tenantApp); tester.hostProvisioner.with(Behaviour.failDeprovisioning); tester.maintainer.maintain(); assertTrue(tester.nodeRepository.getNode(host2.hostname()).isPresent()); }
@Test public void provision_exact_capacity() { var tester = new DynamicProvisioningTester(Cloud.builder().dynamicProvisioning(false).build()); NodeResources resources1 = new NodeResources(24, 64, 100, 1); NodeResources resources2 = new NodeResources(16, 24, 100, 1); tester.flagSource.withListFlag(Flags.TARGET_CAPACITY.id(), List.of(new HostCapacity(resources1.vcpu(), resources1.memoryGb(), resources1.diskGb(), 1), new HostCapacity(resources2.vcpu(), resources2.memoryGb(), resources2.diskGb(), 2)), HostCapacity.class); tester.maintainer.maintain(); assertEquals(1, tester.provisionedHostsMatching(resources1)); assertEquals(2, tester.provisionedHostsMatching(resources2)); tester.assertNodesUnchanged(); NodeResources resources3 = new NodeResources(48, 128, 1000, 1); tester.flagSource.withListFlag(Flags.TARGET_CAPACITY.id(), List.of(new HostCapacity(resources1.vcpu(), resources1.memoryGb(), resources1.diskGb(), 1), new HostCapacity(resources3.vcpu(), resources3.memoryGb(), resources3.diskGb(), 1)), HostCapacity.class); tester.maintainer.maintain(); assertEquals(1, tester.provisionedHostsMatching(resources1)); assertEquals(0, tester.provisionedHostsMatching(resources2)); assertEquals(1, tester.provisionedHostsMatching(resources3)); assertEquals(2, tester.nodeRepository.getNodes(Node.State.deprovisioned).size()); tester.maintainer.maintain(); List<Node> provisioned = tester.nodeRepository.list().state(Node.State.provisioned).asList(); tester.nodeRepository.setReady(provisioned, Agent.system, this.getClass().getSimpleName()); tester.provisioningTester.activateTenantHosts(); ApplicationId application = ProvisioningTester.makeApplicationId(); tester.provisioningTester.deploy(application, Capacity.from(new ClusterResources(2, 1, new NodeResources(4, 8, 50, 0.1)))); assertEquals(2, tester.nodeRepository.list().owner(application).size()); tester.assertNodesUnchanged(); tester.flagSource.withListFlag(Flags.TARGET_CAPACITY.id(), List.of(), HostCapacity.class); tester.assertNodesUnchanged(); tester.flagSource.withListFlag(Flags.TARGET_CAPACITY.id(), List.of(new HostCapacity(resources1.vcpu(), resources1.memoryGb(), resources1.diskGb(), 1)), HostCapacity.class); tester.assertNodesUnchanged(); }