@Test public void hw_fail_only_if_whole_host_is_suspended() { NodeFailTester tester = NodeFailTester.withTwoApplicationsOnDocker(6); String hostWithFailureReports = selectFirstParentHostWithNActiveNodesExcept(tester.nodeRepository, 2); assertEquals(Node.State.active, tester.nodeRepository.getNode(hostWithFailureReports).get().state()); Map<Node.State, List<String>> hostnamesByState = tester.nodeRepository.list().childrenOf(hostWithFailureReports).asList().stream() .collect(Collectors.groupingBy(Node::state, Collectors.mapping(Node::hostname, Collectors.toList()))); assertEquals(2, hostnamesByState.get(Node.State.active).size()); String activeChild1 = hostnamesByState.get(Node.State.active).get(0); String activeChild2 = hostnamesByState.get(Node.State.active).get(1); assertEquals(1, hostnamesByState.get(Node.State.ready).size()); String readyChild = hostnamesByState.get(Node.State.ready).get(0); Report badTotalMemorySizeReport = Report.basicReport("badTotalMemorySize", HARD_FAIL, Instant.now(), "too low"); tester.nodeRepository.getNodes().stream() .filter(node -> node.hostname().equals(hostWithFailureReports)) .forEach(node -> { Node updatedNode = node.with(node.reports().withReport(badTotalMemorySizeReport)); tester.nodeRepository.write(updatedNode, () -> {}); }); tester.allNodesMakeAConfigRequestExcept(); tester.failer.run(); assertEquals(Node.State.failed, tester.nodeRepository.getNode(readyChild).get().state()); assertEquals(Node.State.active, tester.nodeRepository.getNode(hostWithFailureReports).get().state()); assertEquals(Node.State.active, tester.nodeRepository.getNode(activeChild1).get().state()); assertEquals(Node.State.active, tester.nodeRepository.getNode(activeChild2).get().state()); tester.suspend(hostWithFailureReports); tester.clock.advance(Duration.ofHours(25)); tester.allNodesMakeAConfigRequestExcept(); tester.failer.run(); assertEquals(Node.State.failed, tester.nodeRepository.getNode(readyChild).get().state()); assertEquals(Node.State.active, tester.nodeRepository.getNode(hostWithFailureReports).get().state()); assertEquals(Node.State.active, tester.nodeRepository.getNode(activeChild1).get().state()); assertEquals(Node.State.active, tester.nodeRepository.getNode(activeChild2).get().state()); tester.suspend(activeChild1); tester.clock.advance(Duration.ofHours(25)); tester.allNodesMakeAConfigRequestExcept(); tester.failer.run(); assertEquals(Node.State.failed, tester.nodeRepository.getNode(readyChild).get().state()); assertEquals(Node.State.active, tester.nodeRepository.getNode(hostWithFailureReports).get().state()); assertEquals(Node.State.failed, tester.nodeRepository.getNode(activeChild1).get().state()); assertEquals(Node.State.active, tester.nodeRepository.getNode(activeChild2).get().state()); tester.suspend(activeChild2); tester.clock.advance(Duration.ofHours(25)); tester.allNodesMakeAConfigRequestExcept(); tester.failer.run(); assertEquals(Node.State.failed, tester.nodeRepository.getNode(readyChild).get().state()); assertEquals(Node.State.failed, tester.nodeRepository.getNode(hostWithFailureReports).get().state()); assertEquals(Node.State.failed, tester.nodeRepository.getNode(activeChild1).get().state()); assertEquals(Node.State.failed, tester.nodeRepository.getNode(activeChild2).get().state()); }
private Optional<Node> getNode(String hostname, ApplicationId application, @SuppressWarnings("unused") Mutex applicationLock) { return nodeRepository().getNode(hostname, Node.State.active) .filter(node -> node.allocation().isPresent()) .filter(node -> node.allocation().get().owner().equals(application)); }
NodeFailer extends NodeRepositoryMaintainer { private Optional<Node> getNode(String hostname, ApplicationId application, @SuppressWarnings("unused") Mutex applicationLock) { return nodeRepository().getNode(hostname, Node.State.active) .filter(node -> node.allocation().isPresent()) .filter(node -> node.allocation().get().owner().equals(application)); } }
NodeFailer extends NodeRepositoryMaintainer { private Optional<Node> getNode(String hostname, ApplicationId application, @SuppressWarnings("unused") Mutex applicationLock) { return nodeRepository().getNode(hostname, Node.State.active) .filter(node -> node.allocation().isPresent()) .filter(node -> node.allocation().get().owner().equals(application)); } NodeFailer(Deployer deployer, HostLivenessTracker hostLivenessTracker,
                      ServiceMonitor serviceMonitor, NodeRepository nodeRepository,
                      Duration downTimeLimit, Duration interval, Clock clock, Orchestrator orchestrator,
                      ThrottlePolicy throttlePolicy, Metric metric); }
NodeFailer extends NodeRepositoryMaintainer { private Optional<Node> getNode(String hostname, ApplicationId application, @SuppressWarnings("unused") Mutex applicationLock) { return nodeRepository().getNode(hostname, Node.State.active) .filter(node -> node.allocation().isPresent()) .filter(node -> node.allocation().get().owner().equals(application)); } NodeFailer(Deployer deployer, HostLivenessTracker hostLivenessTracker,
                      ServiceMonitor serviceMonitor, NodeRepository nodeRepository,
                      Duration downTimeLimit, Duration interval, Clock clock, Orchestrator orchestrator,
                      ThrottlePolicy throttlePolicy, Metric metric); static List<String> reasonsToFailParentHost(Node hostNode); }
NodeFailer extends NodeRepositoryMaintainer { private Optional<Node> getNode(String hostname, ApplicationId application, @SuppressWarnings("unused") Mutex applicationLock) { return nodeRepository().getNode(hostname, Node.State.active) .filter(node -> node.allocation().isPresent()) .filter(node -> node.allocation().get().owner().equals(application)); } NodeFailer(Deployer deployer, HostLivenessTracker hostLivenessTracker,
                      ServiceMonitor serviceMonitor, NodeRepository nodeRepository,
                      Duration downTimeLimit, Duration interval, Clock clock, Orchestrator orchestrator,
                      ThrottlePolicy throttlePolicy, Metric metric); static List<String> reasonsToFailParentHost(Node hostNode);  }