@Test public void shouldTriggerRecoveryInAutoRecoveryHostsAndUpdateHostMetaData() { Blueprint blueprint = new Blueprint(); blueprint.setBlueprintText("{ \"Blueprints\": { \"blueprint_name\": \"MyBlueprint\" } }"); cluster.setBlueprint(blueprint); when(stackService.findByCrn(STACK_CRN)).thenReturn(stack); when(cloudbreakMessagesService.getMessage(any(), anyCollection())).thenReturn("auto recovery").thenReturn("failed node"); InstanceMetaData host1 = getHost("host1", "master", InstanceStatus.SERVICES_HEALTHY, InstanceGroupType.GATEWAY); when(instanceMetaDataService.findHostInStack(eq(stack.getId()), eq("host1"))).thenReturn(Optional.of(host1)); InstanceMetaData host2 = getHost("host2", "group2", InstanceStatus.SERVICES_HEALTHY, InstanceGroupType.GATEWAY); when(instanceMetaDataService.findHostInStack(eq(stack.getId()), eq("host2"))).thenReturn(Optional.of(host2)); HostGroupRepairView hostGroup = getHostGroup(host1, RecoveryMode.AUTO); when(hostGroupService.getRepairViewByClusterIdAndName(stack.getCluster().getId(), host1.getInstanceGroup().getGroupName())) .thenReturn(Optional.of(hostGroup)); when(hostGroupService.getRepairViewByClusterIdAndName(stack.getCluster().getId(), host2.getInstanceGroup().getGroupName())) .thenReturn(Optional.of(getHostGroup(host2, RecoveryMode.MANUAL))); when(instanceMetaDataService.findNotTerminatedForStack(eq(stack.getId()))).thenReturn(new HashSet<>(Arrays.asList(host1, host2))); underTest.reportHealthChange(STACK_CRN, Set.of("host1", "host2"), Set.of()); Map<String, List<String>> autoRecoveredNodes = Map.of("master", List.of("host1")); verify(flowManager).triggerClusterRepairFlow(STACK_ID, autoRecoveredNodes, false, false); verify(cloudbreakMessagesService, times(1)).getMessage(CLUSTER_AUTORECOVERY_REQUESTED.getMessage(), Set.of("host1")); verify(cloudbreakEventService, times(1)).fireCloudbreakEvent(STACK_ID, "RECOVERY", CLUSTER_AUTORECOVERY_REQUESTED, List.of("host1")); verify(cloudbreakMessagesService, times(1)).getMessage(CLUSTER_FAILED_NODES_REPORTED.getMessage(), Set.of("host2")); verify(cloudbreakEventService, times(1)).fireCloudbreakEvent(STACK_ID, "RECOVERY", CLUSTER_FAILED_NODES_REPORTED, List.of("host2")); verify(updateHostsValidator, times(1)).validateComponentsCategory(stack, hostGroup.getName()); }
public void reportHealthChange(String crn, Set<String> failedNodes, Set<String> newHealthyNodes) { if (!Sets.intersection(failedNodes, newHealthyNodes).isEmpty()) { throw new BadRequestException("Failed nodes " + failedNodes + " and healthy nodes " + newHealthyNodes + " should not have common items."); } if (failedNodes.isEmpty() && newHealthyNodes.isEmpty()) { return; } try { transactionService.required(() -> { Stack stack = stackService.findByCrn(crn); if (stack != null && !stack.getStatus().isInProgress()) { handleHealthChange(failedNodes, newHealthyNodes, stack); } else { LOGGER.debug("Stack [{}] status is {}, thus we do not handle failure report.", stack.getName(), stack.getStatus()); } return null; }); } catch (TransactionExecutionException e) { throw new TransactionRuntimeExecutionException(e); } }
ClusterOperationService { public void reportHealthChange(String crn, Set<String> failedNodes, Set<String> newHealthyNodes) { if (!Sets.intersection(failedNodes, newHealthyNodes).isEmpty()) { throw new BadRequestException("Failed nodes " + failedNodes + " and healthy nodes " + newHealthyNodes + " should not have common items."); } if (failedNodes.isEmpty() && newHealthyNodes.isEmpty()) { return; } try { transactionService.required(() -> { Stack stack = stackService.findByCrn(crn); if (stack != null && !stack.getStatus().isInProgress()) { handleHealthChange(failedNodes, newHealthyNodes, stack); } else { LOGGER.debug("Stack [{}] status is {}, thus we do not handle failure report.", stack.getName(), stack.getStatus()); } return null; }); } catch (TransactionExecutionException e) { throw new TransactionRuntimeExecutionException(e); } } }
ClusterOperationService { public void reportHealthChange(String crn, Set<String> failedNodes, Set<String> newHealthyNodes) { if (!Sets.intersection(failedNodes, newHealthyNodes).isEmpty()) { throw new BadRequestException("Failed nodes " + failedNodes + " and healthy nodes " + newHealthyNodes + " should not have common items."); } if (failedNodes.isEmpty() && newHealthyNodes.isEmpty()) { return; } try { transactionService.required(() -> { Stack stack = stackService.findByCrn(crn); if (stack != null && !stack.getStatus().isInProgress()) { handleHealthChange(failedNodes, newHealthyNodes, stack); } else { LOGGER.debug("Stack [{}] status is {}, thus we do not handle failure report.", stack.getName(), stack.getStatus()); } return null; }); } catch (TransactionExecutionException e) { throw new TransactionRuntimeExecutionException(e); } }  }
ClusterOperationService { public void reportHealthChange(String crn, Set<String> failedNodes, Set<String> newHealthyNodes) { if (!Sets.intersection(failedNodes, newHealthyNodes).isEmpty()) { throw new BadRequestException("Failed nodes " + failedNodes + " and healthy nodes " + newHealthyNodes + " should not have common items."); } if (failedNodes.isEmpty() && newHealthyNodes.isEmpty()) { return; } try { transactionService.required(() -> { Stack stack = stackService.findByCrn(crn); if (stack != null && !stack.getStatus().isInProgress()) { handleHealthChange(failedNodes, newHealthyNodes, stack); } else { LOGGER.debug("Stack [{}] status is {}, thus we do not handle failure report.", stack.getName(), stack.getStatus()); } return null; }); } catch (TransactionExecutionException e) { throw new TransactionRuntimeExecutionException(e); } }  @Measure(ClusterOperationService.class) Cluster create(Stack stack, Cluster cluster, List<ClusterComponent> components, User user); void delete(Long stackId, boolean forced); FlowIdentifier updateHosts(Long stackId, HostGroupAdjustmentV4Request hostGroupAdjustment); FlowIdentifier updateUserNamePassword(Long stackId, UserNamePasswordV4Request userNamePasswordJson); void reportHealthChange(String crn, Set<String> failedNodes, Set<String> newHealthyNodes); FlowIdentifier updateStatus(Long stackId, StatusRequest statusRequest); FlowIdentifier updateStatus(Stack stack, StatusRequest statusRequest); FlowIdentifier recreate(Stack stack, String blueprintName, Set<HostGroup> hostGroups, boolean validateBlueprint); FlowIdentifier triggerMaintenanceModeValidation(Stack stack); FlowIdentifier updateSalt(Stack stack); FlowIdentifier updatePillarConfiguration(Stack stack); FlowIdentifier rotateClusterCertificates(Stack stack, CertificatesRotationV4Request certificatesRotationV4Request); }
ClusterOperationService { public void reportHealthChange(String crn, Set<String> failedNodes, Set<String> newHealthyNodes) { if (!Sets.intersection(failedNodes, newHealthyNodes).isEmpty()) { throw new BadRequestException("Failed nodes " + failedNodes + " and healthy nodes " + newHealthyNodes + " should not have common items."); } if (failedNodes.isEmpty() && newHealthyNodes.isEmpty()) { return; } try { transactionService.required(() -> { Stack stack = stackService.findByCrn(crn); if (stack != null && !stack.getStatus().isInProgress()) { handleHealthChange(failedNodes, newHealthyNodes, stack); } else { LOGGER.debug("Stack [{}] status is {}, thus we do not handle failure report.", stack.getName(), stack.getStatus()); } return null; }); } catch (TransactionExecutionException e) { throw new TransactionRuntimeExecutionException(e); } }  @Measure(ClusterOperationService.class) Cluster create(Stack stack, Cluster cluster, List<ClusterComponent> components, User user); void delete(Long stackId, boolean forced); FlowIdentifier updateHosts(Long stackId, HostGroupAdjustmentV4Request hostGroupAdjustment); FlowIdentifier updateUserNamePassword(Long stackId, UserNamePasswordV4Request userNamePasswordJson); void reportHealthChange(String crn, Set<String> failedNodes, Set<String> newHealthyNodes); FlowIdentifier updateStatus(Long stackId, StatusRequest statusRequest); FlowIdentifier updateStatus(Stack stack, StatusRequest statusRequest); FlowIdentifier recreate(Stack stack, String blueprintName, Set<HostGroup> hostGroups, boolean validateBlueprint); FlowIdentifier triggerMaintenanceModeValidation(Stack stack); FlowIdentifier updateSalt(Stack stack); FlowIdentifier updatePillarConfiguration(Stack stack); FlowIdentifier rotateClusterCertificates(Stack stack, CertificatesRotationV4Request certificatesRotationV4Request);  }