@Test public void clusterBootstrapErrorHandlerWhenNodeCountLessThanOneAfterTheRollbackThenClusterProvisionFailed() throws CloudbreakOrchestratorFailedException { Stack stack = TestUtil.stack(); when(orchestrator.getAvailableNodes(any(GatewayConfig.class), anySet())).thenReturn(new ArrayList<>()); when(instanceMetaDataService.findNotTerminatedByPrivateAddress(anyLong(), anyString())).thenAnswer((Answer<Optional<InstanceMetaData>>) invocation -> { Object[] args = invocation.getArguments(); String ip = (String) args[1]; for (InstanceMetaData instanceMetaData : stack.getNotDeletedInstanceMetaDataSet()) { if (instanceMetaData.getPrivateIp().equals(ip)) { return Optional.of(instanceMetaData); } } return Optional.empty(); }); when(instanceGroupService.findOneWithInstanceMetadataByGroupNameInStack(anyLong(), anyString())) .thenAnswer((Answer<Optional<InstanceGroup>>) invocation -> { Object[] args = invocation.getArguments(); String name = (String) args[1]; for (InstanceMetaData instanceMetaData : stack.getNotDeletedInstanceMetaDataSet()) { if (instanceMetaData.getInstanceGroup().getGroupName().equals(name)) { InstanceGroup instanceGroup = instanceMetaData.getInstanceGroup(); instanceGroup.getInstanceMetaDataSet().forEach(im -> im.setInstanceStatus(InstanceStatus.TERMINATED)); return Optional.of(instanceGroup); } } return Optional.empty(); }); when(cloudbreakMessagesService.getMessage(eq(CLUSTER_BOOTSTRAPPER_ERROR_INVALID_NODECOUNT.getMessage()), any())).thenReturn("invalide.nodecount"); thrown.expect(CloudbreakOrchestratorFailedException.class); thrown.expectMessage("invalide.nodecount"); underTest.terminateFailedNodes(null, orchestrator, TestUtil.stack(), new GatewayConfig("10.0.0.1", "198.0.0.1", "10.0.0.1", 443, "instanceId", false), prepareNodes(stack)); }
public void terminateFailedNodes(HostOrchestrator hostOrchestrator, ContainerOrchestrator containerOrchestrator, Stack stack, GatewayConfig gatewayConfig, Set<Node> nodes) throws CloudbreakOrchestratorFailedException { List<String> allAvailableNode; allAvailableNode = hostOrchestrator != null ? hostOrchestrator.getAvailableNodes(gatewayConfig, nodes) : containerOrchestrator.getAvailableNodes(gatewayConfig, nodes); List<Node> missingNodes = selectMissingNodes(nodes, allAvailableNode); if (!missingNodes.isEmpty()) { eventService.fireCloudbreakEvent(stack.getId(), Status.UPDATE_IN_PROGRESS.name(), CLUSTER_BOOTSTRAPPER_ERROR_BOOTSTRAP_FAILED_ON_NODES, Arrays.asList(String.valueOf(missingNodes.size()))); for (Node missingNode : missingNodes) { InstanceMetaData instanceMetaData = instanceMetaDataService.findNotTerminatedByPrivateAddress(stack.getId(), missingNode.getPrivateIp()) .orElseThrow(NotFoundException.notFound("instanceMetaData", missingNode.getPrivateIp())); InstanceGroup ig = instanceGroupService.findOneWithInstanceMetadataByGroupNameInStack(stack.getId(), instanceMetaData.getInstanceGroup().getGroupName()) .orElseThrow(NotFoundException.notFound("instanceGroup", instanceMetaData.getInstanceGroup().getGroupName())); if (ig.getNodeCount() < 1) { throw new CloudbreakOrchestratorFailedException(cloudbreakMessagesService.getMessage( CLUSTER_BOOTSTRAPPER_ERROR_INVALID_NODECOUNT.getMessage(), Collections.singletonList(ig.getGroupName()))); } instanceGroupService.save(ig); eventService.fireCloudbreakEvent(stack.getId(), Status.UPDATE_IN_PROGRESS.name(), CLUSTER_BOOTSTRAPPER_ERROR_DELETING_NODE, Arrays.asList(instanceMetaData.getInstanceId(), ig.getGroupName())); deleteResourceAndDependencies(stack, instanceMetaData); deleteInstanceResourceFromDatabase(stack, instanceMetaData); instanceMetaData.setTerminationDate(clock.getCurrentTimeMillis()); instanceMetaData.setInstanceStatus(InstanceStatus.TERMINATED); instanceMetaDataService.save(instanceMetaData); LOGGER.debug("InstanceMetadata [name: {}, id: {}] status set to {}.", instanceMetaData.getId(), instanceMetaData.getInstanceId(), instanceMetaData.getInstanceStatus()); } } }
ClusterBootstrapperErrorHandler { public void terminateFailedNodes(HostOrchestrator hostOrchestrator, ContainerOrchestrator containerOrchestrator, Stack stack, GatewayConfig gatewayConfig, Set<Node> nodes) throws CloudbreakOrchestratorFailedException { List<String> allAvailableNode; allAvailableNode = hostOrchestrator != null ? hostOrchestrator.getAvailableNodes(gatewayConfig, nodes) : containerOrchestrator.getAvailableNodes(gatewayConfig, nodes); List<Node> missingNodes = selectMissingNodes(nodes, allAvailableNode); if (!missingNodes.isEmpty()) { eventService.fireCloudbreakEvent(stack.getId(), Status.UPDATE_IN_PROGRESS.name(), CLUSTER_BOOTSTRAPPER_ERROR_BOOTSTRAP_FAILED_ON_NODES, Arrays.asList(String.valueOf(missingNodes.size()))); for (Node missingNode : missingNodes) { InstanceMetaData instanceMetaData = instanceMetaDataService.findNotTerminatedByPrivateAddress(stack.getId(), missingNode.getPrivateIp()) .orElseThrow(NotFoundException.notFound("instanceMetaData", missingNode.getPrivateIp())); InstanceGroup ig = instanceGroupService.findOneWithInstanceMetadataByGroupNameInStack(stack.getId(), instanceMetaData.getInstanceGroup().getGroupName()) .orElseThrow(NotFoundException.notFound("instanceGroup", instanceMetaData.getInstanceGroup().getGroupName())); if (ig.getNodeCount() < 1) { throw new CloudbreakOrchestratorFailedException(cloudbreakMessagesService.getMessage( CLUSTER_BOOTSTRAPPER_ERROR_INVALID_NODECOUNT.getMessage(), Collections.singletonList(ig.getGroupName()))); } instanceGroupService.save(ig); eventService.fireCloudbreakEvent(stack.getId(), Status.UPDATE_IN_PROGRESS.name(), CLUSTER_BOOTSTRAPPER_ERROR_DELETING_NODE, Arrays.asList(instanceMetaData.getInstanceId(), ig.getGroupName())); deleteResourceAndDependencies(stack, instanceMetaData); deleteInstanceResourceFromDatabase(stack, instanceMetaData); instanceMetaData.setTerminationDate(clock.getCurrentTimeMillis()); instanceMetaData.setInstanceStatus(InstanceStatus.TERMINATED); instanceMetaDataService.save(instanceMetaData); LOGGER.debug("InstanceMetadata [name: {}, id: {}] status set to {}.", instanceMetaData.getId(), instanceMetaData.getInstanceId(), instanceMetaData.getInstanceStatus()); } } } }
ClusterBootstrapperErrorHandler { public void terminateFailedNodes(HostOrchestrator hostOrchestrator, ContainerOrchestrator containerOrchestrator, Stack stack, GatewayConfig gatewayConfig, Set<Node> nodes) throws CloudbreakOrchestratorFailedException { List<String> allAvailableNode; allAvailableNode = hostOrchestrator != null ? hostOrchestrator.getAvailableNodes(gatewayConfig, nodes) : containerOrchestrator.getAvailableNodes(gatewayConfig, nodes); List<Node> missingNodes = selectMissingNodes(nodes, allAvailableNode); if (!missingNodes.isEmpty()) { eventService.fireCloudbreakEvent(stack.getId(), Status.UPDATE_IN_PROGRESS.name(), CLUSTER_BOOTSTRAPPER_ERROR_BOOTSTRAP_FAILED_ON_NODES, Arrays.asList(String.valueOf(missingNodes.size()))); for (Node missingNode : missingNodes) { InstanceMetaData instanceMetaData = instanceMetaDataService.findNotTerminatedByPrivateAddress(stack.getId(), missingNode.getPrivateIp()) .orElseThrow(NotFoundException.notFound("instanceMetaData", missingNode.getPrivateIp())); InstanceGroup ig = instanceGroupService.findOneWithInstanceMetadataByGroupNameInStack(stack.getId(), instanceMetaData.getInstanceGroup().getGroupName()) .orElseThrow(NotFoundException.notFound("instanceGroup", instanceMetaData.getInstanceGroup().getGroupName())); if (ig.getNodeCount() < 1) { throw new CloudbreakOrchestratorFailedException(cloudbreakMessagesService.getMessage( CLUSTER_BOOTSTRAPPER_ERROR_INVALID_NODECOUNT.getMessage(), Collections.singletonList(ig.getGroupName()))); } instanceGroupService.save(ig); eventService.fireCloudbreakEvent(stack.getId(), Status.UPDATE_IN_PROGRESS.name(), CLUSTER_BOOTSTRAPPER_ERROR_DELETING_NODE, Arrays.asList(instanceMetaData.getInstanceId(), ig.getGroupName())); deleteResourceAndDependencies(stack, instanceMetaData); deleteInstanceResourceFromDatabase(stack, instanceMetaData); instanceMetaData.setTerminationDate(clock.getCurrentTimeMillis()); instanceMetaData.setInstanceStatus(InstanceStatus.TERMINATED); instanceMetaDataService.save(instanceMetaData); LOGGER.debug("InstanceMetadata [name: {}, id: {}] status set to {}.", instanceMetaData.getId(), instanceMetaData.getInstanceId(), instanceMetaData.getInstanceStatus()); } } }  }
ClusterBootstrapperErrorHandler { public void terminateFailedNodes(HostOrchestrator hostOrchestrator, ContainerOrchestrator containerOrchestrator, Stack stack, GatewayConfig gatewayConfig, Set<Node> nodes) throws CloudbreakOrchestratorFailedException { List<String> allAvailableNode; allAvailableNode = hostOrchestrator != null ? hostOrchestrator.getAvailableNodes(gatewayConfig, nodes) : containerOrchestrator.getAvailableNodes(gatewayConfig, nodes); List<Node> missingNodes = selectMissingNodes(nodes, allAvailableNode); if (!missingNodes.isEmpty()) { eventService.fireCloudbreakEvent(stack.getId(), Status.UPDATE_IN_PROGRESS.name(), CLUSTER_BOOTSTRAPPER_ERROR_BOOTSTRAP_FAILED_ON_NODES, Arrays.asList(String.valueOf(missingNodes.size()))); for (Node missingNode : missingNodes) { InstanceMetaData instanceMetaData = instanceMetaDataService.findNotTerminatedByPrivateAddress(stack.getId(), missingNode.getPrivateIp()) .orElseThrow(NotFoundException.notFound("instanceMetaData", missingNode.getPrivateIp())); InstanceGroup ig = instanceGroupService.findOneWithInstanceMetadataByGroupNameInStack(stack.getId(), instanceMetaData.getInstanceGroup().getGroupName()) .orElseThrow(NotFoundException.notFound("instanceGroup", instanceMetaData.getInstanceGroup().getGroupName())); if (ig.getNodeCount() < 1) { throw new CloudbreakOrchestratorFailedException(cloudbreakMessagesService.getMessage( CLUSTER_BOOTSTRAPPER_ERROR_INVALID_NODECOUNT.getMessage(), Collections.singletonList(ig.getGroupName()))); } instanceGroupService.save(ig); eventService.fireCloudbreakEvent(stack.getId(), Status.UPDATE_IN_PROGRESS.name(), CLUSTER_BOOTSTRAPPER_ERROR_DELETING_NODE, Arrays.asList(instanceMetaData.getInstanceId(), ig.getGroupName())); deleteResourceAndDependencies(stack, instanceMetaData); deleteInstanceResourceFromDatabase(stack, instanceMetaData); instanceMetaData.setTerminationDate(clock.getCurrentTimeMillis()); instanceMetaData.setInstanceStatus(InstanceStatus.TERMINATED); instanceMetaDataService.save(instanceMetaData); LOGGER.debug("InstanceMetadata [name: {}, id: {}] status set to {}.", instanceMetaData.getId(), instanceMetaData.getInstanceId(), instanceMetaData.getInstanceStatus()); } } }  void terminateFailedNodes(HostOrchestrator hostOrchestrator, ContainerOrchestrator containerOrchestrator,
            Stack stack, GatewayConfig gatewayConfig, Set<Node> nodes); }
ClusterBootstrapperErrorHandler { public void terminateFailedNodes(HostOrchestrator hostOrchestrator, ContainerOrchestrator containerOrchestrator, Stack stack, GatewayConfig gatewayConfig, Set<Node> nodes) throws CloudbreakOrchestratorFailedException { List<String> allAvailableNode; allAvailableNode = hostOrchestrator != null ? hostOrchestrator.getAvailableNodes(gatewayConfig, nodes) : containerOrchestrator.getAvailableNodes(gatewayConfig, nodes); List<Node> missingNodes = selectMissingNodes(nodes, allAvailableNode); if (!missingNodes.isEmpty()) { eventService.fireCloudbreakEvent(stack.getId(), Status.UPDATE_IN_PROGRESS.name(), CLUSTER_BOOTSTRAPPER_ERROR_BOOTSTRAP_FAILED_ON_NODES, Arrays.asList(String.valueOf(missingNodes.size()))); for (Node missingNode : missingNodes) { InstanceMetaData instanceMetaData = instanceMetaDataService.findNotTerminatedByPrivateAddress(stack.getId(), missingNode.getPrivateIp()) .orElseThrow(NotFoundException.notFound("instanceMetaData", missingNode.getPrivateIp())); InstanceGroup ig = instanceGroupService.findOneWithInstanceMetadataByGroupNameInStack(stack.getId(), instanceMetaData.getInstanceGroup().getGroupName()) .orElseThrow(NotFoundException.notFound("instanceGroup", instanceMetaData.getInstanceGroup().getGroupName())); if (ig.getNodeCount() < 1) { throw new CloudbreakOrchestratorFailedException(cloudbreakMessagesService.getMessage( CLUSTER_BOOTSTRAPPER_ERROR_INVALID_NODECOUNT.getMessage(), Collections.singletonList(ig.getGroupName()))); } instanceGroupService.save(ig); eventService.fireCloudbreakEvent(stack.getId(), Status.UPDATE_IN_PROGRESS.name(), CLUSTER_BOOTSTRAPPER_ERROR_DELETING_NODE, Arrays.asList(instanceMetaData.getInstanceId(), ig.getGroupName())); deleteResourceAndDependencies(stack, instanceMetaData); deleteInstanceResourceFromDatabase(stack, instanceMetaData); instanceMetaData.setTerminationDate(clock.getCurrentTimeMillis()); instanceMetaData.setInstanceStatus(InstanceStatus.TERMINATED); instanceMetaDataService.save(instanceMetaData); LOGGER.debug("InstanceMetadata [name: {}, id: {}] status set to {}.", instanceMetaData.getId(), instanceMetaData.getInstanceId(), instanceMetaData.getInstanceStatus()); } } }  void terminateFailedNodes(HostOrchestrator hostOrchestrator, ContainerOrchestrator containerOrchestrator,
            Stack stack, GatewayConfig gatewayConfig, Set<Node> nodes);  }