NodeFailer extends NodeRepositoryMaintainer { private Optional<Node> getNode(String hostname, ApplicationId application, @SuppressWarnings("unused") Mutex applicationLock) { return nodeRepository().getNode(hostname, Node.State.active) .filter(node -> node.allocation().isPresent()) .filter(node -> node.allocation().get().owner().equals(application)); } NodeFailer(Deployer deployer, HostLivenessTracker hostLivenessTracker,
                      ServiceMonitor serviceMonitor, NodeRepository nodeRepository,
                      Duration downTimeLimit, Duration interval, Clock clock, Orchestrator orchestrator,
                      ThrottlePolicy throttlePolicy, Metric metric); static List<String> reasonsToFailParentHost(Node hostNode);  }
@Test public void hw_fail_only_if_whole_host_is_suspended() { NodeFailTester tester = NodeFailTester.withTwoApplicationsOnDocker(6); String hostWithFailureReports = selectFirstParentHostWithNActiveNodesExcept(tester.nodeRepository, 2); assertEquals(Node.State.active, tester.nodeRepository.getNode(hostWithFailureReports).get().state()); Map<Node.State, List<String>> hostnamesByState = tester.nodeRepository.list().childrenOf(hostWithFailureReports).asList().stream() .collect(Collectors.groupingBy(Node::state, Collectors.mapping(Node::hostname, Collectors.toList()))); assertEquals(2, hostnamesByState.get(Node.State.active).size()); String activeChild1 = hostnamesByState.get(Node.State.active).get(0); String activeChild2 = hostnamesByState.get(Node.State.active).get(1); assertEquals(1, hostnamesByState.get(Node.State.ready).size()); String readyChild = hostnamesByState.get(Node.State.ready).get(0); Report badTotalMemorySizeReport = Report.basicReport("badTotalMemorySize", HARD_FAIL, Instant.now(), "too low"); tester.nodeRepository.getNodes().stream() .filter(node -> node.hostname().equals(hostWithFailureReports)) .forEach(node -> { Node updatedNode = node.with(node.reports().withReport(badTotalMemorySizeReport)); tester.nodeRepository.write(updatedNode, () -> {}); }); tester.allNodesMakeAConfigRequestExcept(); tester.failer.run(); assertEquals(Node.State.failed, tester.nodeRepository.getNode(readyChild).get().state()); assertEquals(Node.State.active, tester.nodeRepository.getNode(hostWithFailureReports).get().state()); assertEquals(Node.State.active, tester.nodeRepository.getNode(activeChild1).get().state()); assertEquals(Node.State.active, tester.nodeRepository.getNode(activeChild2).get().state()); tester.suspend(hostWithFailureReports); tester.clock.advance(Duration.ofHours(25)); tester.allNodesMakeAConfigRequestExcept(); tester.failer.run(); assertEquals(Node.State.failed, tester.nodeRepository.getNode(readyChild).get().state()); assertEquals(Node.State.active, tester.nodeRepository.getNode(hostWithFailureReports).get().state()); assertEquals(Node.State.active, tester.nodeRepository.getNode(activeChild1).get().state()); assertEquals(Node.State.active, tester.nodeRepository.getNode(activeChild2).get().state()); tester.suspend(activeChild1); tester.clock.advance(Duration.ofHours(25)); tester.allNodesMakeAConfigRequestExcept(); tester.failer.run(); assertEquals(Node.State.failed, tester.nodeRepository.getNode(readyChild).get().state()); assertEquals(Node.State.active, tester.nodeRepository.getNode(hostWithFailureReports).get().state()); assertEquals(Node.State.failed, tester.nodeRepository.getNode(activeChild1).get().state()); assertEquals(Node.State.active, tester.nodeRepository.getNode(activeChild2).get().state()); tester.suspend(activeChild2); tester.clock.advance(Duration.ofHours(25)); tester.allNodesMakeAConfigRequestExcept(); tester.failer.run(); assertEquals(Node.State.failed, tester.nodeRepository.getNode(readyChild).get().state()); assertEquals(Node.State.failed, tester.nodeRepository.getNode(hostWithFailureReports).get().state()); assertEquals(Node.State.failed, tester.nodeRepository.getNode(activeChild1).get().state()); assertEquals(Node.State.failed, tester.nodeRepository.getNode(activeChild2).get().state()); }
@Test public void nodes_for_suspended_applications_are_not_failed() { NodeFailTester tester = NodeFailTester.withTwoApplications(); tester.suspend(NodeFailTester.app1); String host_from_suspended_app = tester.nodeRepository.getNodes(NodeFailTester.app1, Node.State.active).get(1).hostname(); String host_from_normal_app = tester.nodeRepository.getNodes(NodeFailTester.app2, Node.State.active).get(3).hostname(); tester.serviceMonitor.setHostDown(host_from_suspended_app); tester.serviceMonitor.setHostDown(host_from_normal_app); tester.failer.run(); tester.clock.advance(Duration.ofMinutes(65)); tester.failer.run(); assertEquals(Node.State.failed, tester.nodeRepository.getNode(host_from_normal_app).get().state()); assertEquals(Node.State.active, tester.nodeRepository.getNode(host_from_suspended_app).get().state()); }
@Test public void node_failing() { NodeFailTester tester = NodeFailTester.withTwoApplications(); for (int minutes = 0; minutes < 24 * 60; minutes +=5 ) { tester.failer.run(); tester.clock.advance(Duration.ofMinutes(5)); tester.allNodesMakeAConfigRequestExcept(); assertEquals( 0, tester.deployer.redeployments); assertEquals(12, tester.nodeRepository.getNodes(NodeType.tenant, Node.State.active).size()); assertEquals( 0, tester.nodeRepository.getNodes(NodeType.tenant, Node.State.failed).size()); assertEquals( 4, tester.nodeRepository.getNodes(NodeType.tenant, Node.State.ready).size()); } Node readyFail1 = tester.nodeRepository.getNodes(NodeType.tenant, Node.State.ready).get(2); Node readyFail2 = tester.nodeRepository.getNodes(NodeType.tenant, Node.State.ready).get(3); tester.nodeRepository.write(readyFail1.with(new Reports().withReport(badTotalMemorySizeReport)), () -> {}); tester.nodeRepository.write(readyFail2.with(new Reports().withReport(badTotalMemorySizeReport)), () -> {}); assertEquals(4, tester.nodeRepository.getNodes(NodeType.tenant, Node.State.ready).size()); tester.failer.run(); assertEquals(2, tester.nodeRepository.getNodes(NodeType.tenant, Node.State.ready).size()); assertEquals(Node.State.failed, tester.nodeRepository.getNode(readyFail1.hostname()).get().state()); assertEquals(Node.State.failed, tester.nodeRepository.getNode(readyFail2.hostname()).get().state()); String downHost1 = tester.nodeRepository.getNodes(NodeFailTester.app1, Node.State.active).get(1).hostname(); String downHost2 = tester.nodeRepository.getNodes(NodeFailTester.app2, Node.State.active).get(3).hostname(); tester.serviceMonitor.setHostDown(downHost1); tester.serviceMonitor.setHostDown(downHost2); for (int minutes = 0; minutes < 45; minutes +=5 ) { tester.failer.run(); tester.clock.advance(Duration.ofMinutes(5)); tester.allNodesMakeAConfigRequestExcept(); assertEquals( 0, tester.deployer.redeployments); assertEquals(12, tester.nodeRepository.getNodes(NodeType.tenant, Node.State.active).size()); assertEquals( 2, tester.nodeRepository.getNodes(NodeType.tenant, Node.State.failed).size()); assertEquals( 2, tester.nodeRepository.getNodes(NodeType.tenant, Node.State.ready).size()); } tester.serviceMonitor.setHostUp(downHost1); tester.clock.advance(Duration.ofDays(1)); tester.allNodesMakeAConfigRequestExcept(); tester.failer.run(); assertEquals( 1, tester.deployer.redeployments); assertEquals(12, tester.nodeRepository.getNodes(NodeType.tenant, Node.State.active).size()); assertEquals( 3, tester.nodeRepository.getNodes(NodeType.tenant, Node.State.failed).size()); assertEquals( 1, tester.nodeRepository.getNodes(NodeType.tenant, Node.State.ready).size()); assertEquals(downHost2, tester.nodeRepository.getNodes(NodeType.tenant, Node.State.failed).get(0).hostname()); tester.serviceMonitor.setHostDown(downHost1); tester.failer.run(); tester.clock.advance(Duration.ofMinutes(5)); tester.allNodesMakeAConfigRequestExcept(); tester.clock.advance(Duration.ofMinutes(120)); tester.failer = tester.createFailer(); tester.failer.run(); tester.clock.advance(Duration.ofMinutes(5)); tester.allNodesMakeAConfigRequestExcept(); tester.failer.run(); assertEquals( 2, tester.deployer.redeployments); assertEquals(12, tester.nodeRepository.getNodes(NodeType.tenant, Node.State.active).size()); assertEquals( 4, tester.nodeRepository.getNodes(NodeType.tenant, Node.State.failed).size()); assertEquals( 0, tester.nodeRepository.getNodes(NodeType.tenant, Node.State.ready).size()); Node lastNode = tester.highestIndex(tester.nodeRepository.getNodes(NodeFailTester.app1, Node.State.active)); tester.serviceMonitor.setHostDown(lastNode.hostname()); for (int minutes = 0; minutes < 75; minutes +=5 ) { tester.failer.run(); tester.clock.advance(Duration.ofMinutes(5)); tester.allNodesMakeAConfigRequestExcept(); assertEquals( 2, tester.deployer.redeployments); assertEquals(12, tester.nodeRepository.getNodes(NodeType.tenant, Node.State.active).size()); assertEquals( 4, tester.nodeRepository.getNodes(NodeType.tenant, Node.State.failed).size()); assertEquals( 0, tester.nodeRepository.getNodes(NodeType.tenant, Node.State.ready).size()); } tester.createReadyNodes(1, 16, NodeFailTester.nodeResources); tester.clock.advance(Duration.ofDays(1)); tester.allNodesMakeAConfigRequestExcept(); tester.failer.run(); assertEquals( 3, tester.deployer.redeployments); assertEquals(12, tester.nodeRepository.getNodes(NodeType.tenant, Node.State.active).size()); assertEquals( 5, tester.nodeRepository.getNodes(NodeType.tenant, Node.State.failed).size()); assertEquals( 0, tester.nodeRepository.getNodes(NodeType.tenant, Node.State.ready).size()); assertTrue("The index of the last failed node is not reused", tester.highestIndex(tester.nodeRepository.getNodes(NodeFailTester.app1, Node.State.active)).allocation().get().membership().index() > lastNode.allocation().get().membership().index()); }