SpareCapacityMaintainer extends NodeRepositoryMaintainer { @Override protected boolean maintain() { boolean success = true; if ( ! nodeRepository().zone().getCloud().allowHostSharing()) return success; NodeList allNodes = nodeRepository().list(); CapacityChecker capacityChecker = new CapacityChecker(allNodes); List<Node> overcommittedHosts = capacityChecker.findOvercommittedHosts(); metric.set("overcommittedHosts", overcommittedHosts.size(), null); retireOvercommitedHosts(allNodes, overcommittedHosts); Optional<CapacityChecker.HostFailurePath> failurePath = capacityChecker.worstCaseHostLossLeadingToFailure(); if (failurePath.isPresent()) { int spareHostCapacity = failurePath.get().hostsCausingFailure.size() - 1; if (spareHostCapacity == 0) { List<Move> mitigation = findMitigation(failurePath.get()); if (execute(mitigation)) { spareHostCapacity++; } else { success = false; } } metric.set("spareHostCapacity", spareHostCapacity, null); } return success; } SpareCapacityMaintainer(Deployer deployer,
                                   NodeRepository nodeRepository,
                                   Metric metric,
                                   Duration interval);  SpareCapacityMaintainer(Deployer deployer,
                                   NodeRepository nodeRepository,
                                   Metric metric,
                                   Duration interval,
                                   int maxIterations);   }
@Test public void testEmpty() { var tester = new SpareCapacityMaintainerTester(); tester.maintainer.maintain(); assertEquals(0, tester.deployer.redeployments); assertEquals(0, tester.nodeRepository.list().retired().size()); }
@Test public void testOneSpare() { var tester = new SpareCapacityMaintainerTester(); tester.addHosts(2, new NodeResources(10, 100, 1000, 1)); tester.addNodes(0, 1, new NodeResources(10, 100, 1000, 1), 0); tester.maintainer.maintain(); assertEquals(0, tester.deployer.redeployments); assertEquals(0, tester.nodeRepository.list().retired().size()); assertEquals(1, tester.metric.values.get("spareHostCapacity")); }
@Test public void testTwoSpares() { var tester = new SpareCapacityMaintainerTester(); tester.addHosts(3, new NodeResources(10, 100, 1000, 1)); tester.addNodes(0, 1, new NodeResources(10, 100, 1000, 1), 0); tester.maintainer.maintain(); assertEquals(0, tester.deployer.redeployments); assertEquals(0, tester.nodeRepository.list().retired().size()); assertEquals(2, tester.metric.values.get("spareHostCapacity")); }
@Test public void testNoSpares() { var tester = new SpareCapacityMaintainerTester(); tester.addHosts(2, new NodeResources(10, 100, 1000, 1)); tester.addNodes(0, 2, new NodeResources(10, 100, 1000, 1), 0); tester.maintainer.maintain(); assertEquals(0, tester.deployer.redeployments); assertEquals(0, tester.nodeRepository.list().retired().size()); assertEquals(0, tester.metric.values.get("spareHostCapacity")); }
@Test public void testAllWorksAsSpares() { var tester = new SpareCapacityMaintainerTester(); tester.addHosts(4, new NodeResources(10, 100, 1000, 1)); tester.addNodes(0, 2, new NodeResources(5, 50, 500, 0.5), 0); tester.addNodes(1, 2, new NodeResources(5, 50, 500, 0.5), 2); tester.maintainer.maintain(); assertEquals(0, tester.deployer.redeployments); assertEquals(0, tester.nodeRepository.list().retired().size()); assertEquals(2, tester.metric.values.get("spareHostCapacity")); }
@Test public void testMoveIsNeeded() { var tester = new SpareCapacityMaintainerTester(); tester.addHosts(6, new NodeResources(10, 100, 1000, 1)); tester.addNodes(0, 2, new NodeResources(10, 100, 1000, 1), 0); tester.addNodes(1, 2, new NodeResources(5, 50, 500, 0.5), 2); tester.addNodes(2, 2, new NodeResources(5, 50, 500, 0.5), 4); tester.maintainer.maintain(); assertEquals(1, tester.deployer.redeployments); assertEquals(1, tester.nodeRepository.list().retired().size()); assertEquals(1, tester.metric.values.get("spareHostCapacity")); tester.maintainer.maintain(); assertEquals(1, tester.deployer.redeployments); assertEquals(1, tester.nodeRepository.list().retired().size()); assertEquals(1, tester.metric.values.get("spareHostCapacity")); }
@Test public void testMultipleMovesAreNeeded() { var tester = new SpareCapacityMaintainerTester(); tester.addHosts(4, new NodeResources(12, 120, 1200, 1.2)); tester.addHosts(4, new NodeResources(10, 100, 1000, 1)); tester.addNodes(0, 2, new NodeResources(10, 100, 1000, 1.0), 0); tester.addNodes(1, 2, new NodeResources(12, 120, 1200, 1.2), 2); tester.addNodes(2, 2, new NodeResources(5, 50, 500, 0.5), 4); tester.addNodes(3, 2, new NodeResources(5, 50, 500, 0.5), 6); tester.maintainer.maintain(); assertEquals(1, tester.deployer.redeployments); assertEquals(1, tester.nodeRepository.list().retired().size()); assertEquals(1, tester.metric.values.get("spareHostCapacity")); }
@Test public void testMultipleNodesMustMoveFromOneHost() { var tester = new SpareCapacityMaintainerTester(); setupMultipleHosts(tester, 4); tester.maintainer.maintain(); assertEquals(1, tester.deployer.redeployments); assertEquals(1, tester.nodeRepository.list().retired().size()); assertEquals(1, tester.metric.values.get("spareHostCapacity")); }
@Test public void testMultipleNodesMustMoveFromOneHostButInsufficientCapacity() { var tester = new SpareCapacityMaintainerTester(); setupMultipleHosts(tester, 3); tester.maintainer.maintain(); assertEquals(0, tester.deployer.redeployments); assertEquals(0, tester.nodeRepository.list().retired().size()); assertEquals(0, tester.metric.values.get("spareHostCapacity")); }
@Test public void testTooManyIterationsAreNeeded() { var tester = new SpareCapacityMaintainerTester(5); tester.addHosts(2, new NodeResources(10, 100, 1000, 1)); tester.addHosts(1, new NodeResources(9, 90, 900, 0.9)); tester.addHosts(1, new NodeResources(8, 80, 800, 0.8)); tester.addHosts(1, new NodeResources(7, 70, 700, 0.7)); tester.addHosts(1, new NodeResources(6, 60, 600, 0.6)); tester.addHosts(1, new NodeResources(5, 50, 500, 0.5)); tester.addHosts(1, new NodeResources(4, 40, 400, 0.4)); tester.addNodes(0, 1, new NodeResources(10, 100, 1000, 1.0), 0); tester.addNodes(1, 1, new NodeResources( 9, 90, 900, 0.9), 1); tester.addNodes(2, 1, new NodeResources( 8, 80, 800, 0.8), 2); tester.addNodes(3, 1, new NodeResources( 7, 70, 700, 0.7), 3); tester.addNodes(4, 1, new NodeResources( 6, 60, 600, 0.6), 4); tester.addNodes(5, 1, new NodeResources( 5, 50, 500, 0.5), 5); tester.addNodes(6, 1, new NodeResources( 4, 40, 400, 0.4), 6); tester.maintainer.maintain(); assertEquals(0, tester.deployer.redeployments); assertEquals(0, tester.nodeRepository.list().retired().size()); assertEquals(0, tester.metric.values.get("spareHostCapacity")); }
@Test public void retireFromOvercommitedHosts() { var tester = new SpareCapacityMaintainerTester(5); tester.addHosts(7, new NodeResources(10, 100, 1000, 1)); tester.addNodes(0, 5, new NodeResources( 7, 70, 700, 0.7), 0); tester.addNodes(1, 4, new NodeResources( 2, 20, 200, 0.2), 0); tester.addNodes(2, 2, new NodeResources( 1.1, 10, 100, 0.1), 1); tester.maintainer.maintain(); assertEquals(2, tester.metric.values.get("overcommittedHosts")); assertEquals(1, tester.deployer.redeployments); assertEquals(List.of(new NodeResources( 1.1, 10, 100, 0.1)), tester.nodeRepository.list().retired().mapToList(Node::resources)); }
@Test @Ignore public void testLargeNodeRepo() { int hosts = 200; var tester = new SpareCapacityMaintainerTester(); tester.addHosts(hosts, new NodeResources(100, 1000, 10000, 10)); int hostOffset = 0; for (int i = 0; i < 200; i++) { int applicationSize = 10; int resourceSize = 10; tester.addNodes(i, applicationSize, new NodeResources(resourceSize, resourceSize * 10, resourceSize * 100, 0.1), hostOffset); hostOffset = (hostOffset + applicationSize) % hosts; } long startTime = System.currentTimeMillis(); tester.maintainer.maintain(); long totalTime = System.currentTimeMillis() - startTime; System.out.println("Complete in " + ( totalTime / 1000) + " seconds"); assertEquals(0, tester.deployer.redeployments); assertEquals(0, tester.nodeRepository.list().retired().size()); assertEquals(0, tester.metric.values.get("spareHostCapacity")); }