@Test void upscaleAwsASGroupFail() throws AmazonAutoscalingFailed { AmazonAutoScalingRetryClient amazonAutoScalingRetryClient = mock(AmazonAutoScalingRetryClient.class); AmazonCloudFormationRetryClient amazonCloudFormationRetryClient = mock(AmazonCloudFormationRetryClient.class); DescribeAutoScalingGroupsResult describeAutoScalingGroupsResult = new DescribeAutoScalingGroupsResult(); List<AutoScalingGroup> autoScalingGroups = new ArrayList<>(); AutoScalingGroup masterASGroup = new AutoScalingGroup(); masterASGroup.setAutoScalingGroupName("masterASG"); List<Instance> masterASGInstances = new ArrayList<>(); masterASGInstances.add(new Instance().withInstanceId("i-master1")); masterASGInstances.add(new Instance().withInstanceId("i-master2")); masterASGroup.setInstances(masterASGInstances); AutoScalingGroup workerASGroup = new AutoScalingGroup(); workerASGroup.setAutoScalingGroupName("workerASG"); List<Instance> workerASGInstances = new ArrayList<>(); workerASGInstances.add(new Instance().withInstanceId("i-worker1")); workerASGInstances.add(new Instance().withInstanceId("i-worker2")); workerASGInstances.add(new Instance().withInstanceId("i-worker3")); workerASGroup.setInstances(workerASGInstances); autoScalingGroups.add(masterASGroup); autoScalingGroups.add(workerASGroup); describeAutoScalingGroupsResult.setAutoScalingGroups(autoScalingGroups); when(amazonAutoScalingRetryClient.describeAutoScalingGroups(any(DescribeAutoScalingGroupsRequest.class))) .thenReturn(describeAutoScalingGroupsResult); when(awsClient.createAutoScalingRetryClient(any(AwsCredentialView.class), anyString())).thenReturn(amazonAutoScalingRetryClient); when(awsClient.createCloudFormationRetryClient(any(AwsCredentialView.class), anyString())).thenReturn(amazonCloudFormationRetryClient); when(cfStackUtil.getAutoscalingGroupName(any(AuthenticatedContext.class), any(AmazonCloudFormationRetryClient.class), eq("worker"))) .thenReturn("workerASG"); when(cfStackUtil.getAutoscalingGroupName(any(AuthenticatedContext.class), any(AmazonCloudFormationRetryClient.class), eq("master"))) .thenReturn("masterASG"); AuthenticatedContext authenticatedContext = new AuthenticatedContext(new CloudContext(1L, "teststack", "AWS", "AWS", Location.location(Region.region("eu-west-1"), AvailabilityZone.availabilityZone("eu-west-1a")), "1", "1"), new CloudCredential()); List<CloudResource> allInstances = new ArrayList<>(); allInstances.add(CloudResource.builder().type(ResourceType.AWS_INSTANCE).status(CommonStatus.CREATED) .name("worker1").group("worker").instanceId("i-worker1").build()); allInstances.add(CloudResource.builder().type(ResourceType.AWS_INSTANCE).status(CommonStatus.CREATED) .name("worker2").group("worker").instanceId("i-worker2").build()); allInstances.add(CloudResource.builder().type(ResourceType.AWS_INSTANCE).status(CommonStatus.CREATED) .name("worker3").group("worker").instanceId("i-worker3").build()); CloudResource workerInstance4 = CloudResource.builder().type(ResourceType.AWS_INSTANCE).status(CommonStatus.CREATED) .name("worker4").group("worker").instanceId("i-worker4").build(); allInstances.add(workerInstance4); CloudResource workerInstance5 = CloudResource.builder().type(ResourceType.AWS_INSTANCE).status(CommonStatus.CREATED) .name("worker5").group("worker").instanceId("i-worker5").build(); allInstances.add(workerInstance5); when(cfStackUtil.getInstanceCloudResources(eq(authenticatedContext), eq(amazonCloudFormationRetryClient), eq(amazonAutoScalingRetryClient), anyList())) .thenReturn(allInstances); InstanceAuthentication instanceAuthentication = new InstanceAuthentication("sshkey", "", "cloudbreak"); List<Group> groups = new ArrayList<>(); Group master = getMasterGroup(instanceAuthentication); groups.add(master); Group worker = getWorkerGroup(instanceAuthentication); groups.add(worker); CloudStack cloudStack = new CloudStack(groups, getNetwork(), null, emptyMap(), emptyMap(), null, instanceAuthentication, instanceAuthentication.getLoginUserName(), instanceAuthentication.getPublicKey(), null); List<CloudResource> cloudResourceList = Collections.emptyList(); AutoScalingGroup newWorkerASGroup = new AutoScalingGroup(); newWorkerASGroup.setAutoScalingGroupName("workerASG"); List<Instance> newWorkerASGInstances = new ArrayList<>(); newWorkerASGInstances.add(new Instance().withInstanceId("i-worker1")); newWorkerASGInstances.add(new Instance().withInstanceId("i-worker2")); newWorkerASGInstances.add(new Instance().withInstanceId("i-worker3")); newWorkerASGInstances.add(new Instance().withInstanceId("i-worker4")); newWorkerASGInstances.add(new Instance().withInstanceId("i-worker5")); newWorkerASGroup.setInstances(newWorkerASGInstances); when(awsAutoScalingService.getAutoscalingGroups(eq(amazonAutoScalingRetryClient), any())) .thenReturn(Collections.singletonList(newWorkerASGroup)); doThrow(new AmazonAutoscalingFailed("autoscaling failed")) .when(awsAutoScalingService).scheduleStatusChecks(eq(List.of(worker)), eq(authenticatedContext), eq(amazonCloudFormationRetryClient), any(Date.class)); assertThrows(CloudConnectorException.class, () -> awsUpscaleService.upscale(authenticatedContext, cloudStack, cloudResourceList), "Autoscaling group update failed: 'autoscaling failed' Original autoscaling group state has been recovered."); verify(awsAutoScalingService, times(1)).updateAutoscalingGroup(any(AmazonAutoScalingRetryClient.class), eq("workerASG"), eq(5)); verify(awsAutoScalingService, times(1)).scheduleStatusChecks(eq(List.of(worker)), eq(authenticatedContext), eq(amazonCloudFormationRetryClient), any()); verify(awsComputeResourceService, times(0)).buildComputeResourcesForUpscale(eq(authenticatedContext), eq(cloudStack), anyList(), anyList(), any(), any()); verify(awsAutoScalingService, times(1)).suspendAutoScaling(eq(authenticatedContext), eq(cloudStack)); verify(awsAutoScalingService, times(1)).terminateInstance(eq(amazonAutoScalingRetryClient), eq("i-worker4")); verify(awsAutoScalingService, times(1)).terminateInstance(eq(amazonAutoScalingRetryClient), eq("i-worker5")); Map<String, Integer> desiredGroups = new HashMap<>(); desiredGroups.put("workerASG", 3); desiredGroups.put("masterASG", 2); verify(awsAutoScalingService, times(1)).scheduleStatusChecks(eq(desiredGroups), eq(authenticatedContext), any(Date.class)); }
public List<CloudResourceStatus> upscale(AuthenticatedContext ac, CloudStack stack, List<CloudResource> resources) { AmazonCloudFormationRetryClient cloudFormationClient = getCloudFormationRetryClient(ac); AwsCredentialView credentialView = new AwsCredentialView(ac.getCloudCredential()); String regionName = ac.getCloudContext().getLocation().getRegion().value(); AmazonAutoScalingRetryClient amazonASClient = getAutoScalingRetryClient(ac); AmazonEC2Client amazonEC2Client = getEC2Client(ac); List<Group> scaledGroups = cloudResourceHelper.getScaledGroups(stack); Map<String, Group> desiredAutoscalingGroupsByName = getAutoScaleGroupsByNameFromCloudFormationTemplate(ac, cloudFormationClient, scaledGroups); LOGGER.info("Desired autoscaling groups: {}", desiredAutoscalingGroupsByName); awsAutoScalingService.resumeAutoScaling(amazonASClient, desiredAutoscalingGroupsByName.keySet(), UPSCALE_PROCESSES); Map<String, Integer> originalAutoScalingGroupsBySize = getAutoScalingGroupsBySize(desiredAutoscalingGroupsByName.keySet(), amazonASClient); LOGGER.info("Update autoscaling groups for stack: {}", ac.getCloudContext().getName()); Date timeBeforeASUpdate = new Date(); updateAutoscalingGroups(amazonASClient, desiredAutoscalingGroupsByName, originalAutoScalingGroupsBySize); try { awsAutoScalingService.scheduleStatusChecks(scaledGroups, ac, cloudFormationClient, timeBeforeASUpdate); } catch (AmazonAutoscalingFailed amazonAutoscalingFailed) { return recoverOriginalState(ac, stack, amazonASClient, desiredAutoscalingGroupsByName, originalAutoScalingGroupsBySize, amazonAutoscalingFailed); } awsAutoScalingService.suspendAutoScaling(ac, stack); List<CloudResource> instances = cfStackUtil.getInstanceCloudResources(ac, cloudFormationClient, amazonASClient, scaledGroups); associateElasticIpWithNewInstances(stack, resources, cloudFormationClient, amazonEC2Client, scaledGroups, instances); List<Group> groupsWithNewInstances = getGroupsWithNewInstances(scaledGroups); List<CloudResource> newInstances = getNewInstances(scaledGroups, instances); List<CloudResource> reattachableVolumeSets = getReattachableVolumeSets(scaledGroups, resources); List<CloudResource> networkResources = resources.stream() .filter(cloudResource -> ResourceType.AWS_SUBNET.equals(cloudResource.getType())) .collect(Collectors.toList()); awsComputeResourceService.buildComputeResourcesForUpscale(ac, stack, groupsWithNewInstances, newInstances, reattachableVolumeSets, networkResources); awsTaggingService.tagRootVolumes(ac, amazonEC2Client, instances, stack.getTags()); awsCloudWatchService.addCloudWatchAlarmsForSystemFailures(instances, stack, regionName, credentialView); return singletonList(new CloudResourceStatus(cfStackUtil.getCloudFormationStackResource(resources), ResourceStatus.UPDATED)); }
AwsUpscaleService { public List<CloudResourceStatus> upscale(AuthenticatedContext ac, CloudStack stack, List<CloudResource> resources) { AmazonCloudFormationRetryClient cloudFormationClient = getCloudFormationRetryClient(ac); AwsCredentialView credentialView = new AwsCredentialView(ac.getCloudCredential()); String regionName = ac.getCloudContext().getLocation().getRegion().value(); AmazonAutoScalingRetryClient amazonASClient = getAutoScalingRetryClient(ac); AmazonEC2Client amazonEC2Client = getEC2Client(ac); List<Group> scaledGroups = cloudResourceHelper.getScaledGroups(stack); Map<String, Group> desiredAutoscalingGroupsByName = getAutoScaleGroupsByNameFromCloudFormationTemplate(ac, cloudFormationClient, scaledGroups); LOGGER.info("Desired autoscaling groups: {}", desiredAutoscalingGroupsByName); awsAutoScalingService.resumeAutoScaling(amazonASClient, desiredAutoscalingGroupsByName.keySet(), UPSCALE_PROCESSES); Map<String, Integer> originalAutoScalingGroupsBySize = getAutoScalingGroupsBySize(desiredAutoscalingGroupsByName.keySet(), amazonASClient); LOGGER.info("Update autoscaling groups for stack: {}", ac.getCloudContext().getName()); Date timeBeforeASUpdate = new Date(); updateAutoscalingGroups(amazonASClient, desiredAutoscalingGroupsByName, originalAutoScalingGroupsBySize); try { awsAutoScalingService.scheduleStatusChecks(scaledGroups, ac, cloudFormationClient, timeBeforeASUpdate); } catch (AmazonAutoscalingFailed amazonAutoscalingFailed) { return recoverOriginalState(ac, stack, amazonASClient, desiredAutoscalingGroupsByName, originalAutoScalingGroupsBySize, amazonAutoscalingFailed); } awsAutoScalingService.suspendAutoScaling(ac, stack); List<CloudResource> instances = cfStackUtil.getInstanceCloudResources(ac, cloudFormationClient, amazonASClient, scaledGroups); associateElasticIpWithNewInstances(stack, resources, cloudFormationClient, amazonEC2Client, scaledGroups, instances); List<Group> groupsWithNewInstances = getGroupsWithNewInstances(scaledGroups); List<CloudResource> newInstances = getNewInstances(scaledGroups, instances); List<CloudResource> reattachableVolumeSets = getReattachableVolumeSets(scaledGroups, resources); List<CloudResource> networkResources = resources.stream() .filter(cloudResource -> ResourceType.AWS_SUBNET.equals(cloudResource.getType())) .collect(Collectors.toList()); awsComputeResourceService.buildComputeResourcesForUpscale(ac, stack, groupsWithNewInstances, newInstances, reattachableVolumeSets, networkResources); awsTaggingService.tagRootVolumes(ac, amazonEC2Client, instances, stack.getTags()); awsCloudWatchService.addCloudWatchAlarmsForSystemFailures(instances, stack, regionName, credentialView); return singletonList(new CloudResourceStatus(cfStackUtil.getCloudFormationStackResource(resources), ResourceStatus.UPDATED)); } }
AwsUpscaleService { public List<CloudResourceStatus> upscale(AuthenticatedContext ac, CloudStack stack, List<CloudResource> resources) { AmazonCloudFormationRetryClient cloudFormationClient = getCloudFormationRetryClient(ac); AwsCredentialView credentialView = new AwsCredentialView(ac.getCloudCredential()); String regionName = ac.getCloudContext().getLocation().getRegion().value(); AmazonAutoScalingRetryClient amazonASClient = getAutoScalingRetryClient(ac); AmazonEC2Client amazonEC2Client = getEC2Client(ac); List<Group> scaledGroups = cloudResourceHelper.getScaledGroups(stack); Map<String, Group> desiredAutoscalingGroupsByName = getAutoScaleGroupsByNameFromCloudFormationTemplate(ac, cloudFormationClient, scaledGroups); LOGGER.info("Desired autoscaling groups: {}", desiredAutoscalingGroupsByName); awsAutoScalingService.resumeAutoScaling(amazonASClient, desiredAutoscalingGroupsByName.keySet(), UPSCALE_PROCESSES); Map<String, Integer> originalAutoScalingGroupsBySize = getAutoScalingGroupsBySize(desiredAutoscalingGroupsByName.keySet(), amazonASClient); LOGGER.info("Update autoscaling groups for stack: {}", ac.getCloudContext().getName()); Date timeBeforeASUpdate = new Date(); updateAutoscalingGroups(amazonASClient, desiredAutoscalingGroupsByName, originalAutoScalingGroupsBySize); try { awsAutoScalingService.scheduleStatusChecks(scaledGroups, ac, cloudFormationClient, timeBeforeASUpdate); } catch (AmazonAutoscalingFailed amazonAutoscalingFailed) { return recoverOriginalState(ac, stack, amazonASClient, desiredAutoscalingGroupsByName, originalAutoScalingGroupsBySize, amazonAutoscalingFailed); } awsAutoScalingService.suspendAutoScaling(ac, stack); List<CloudResource> instances = cfStackUtil.getInstanceCloudResources(ac, cloudFormationClient, amazonASClient, scaledGroups); associateElasticIpWithNewInstances(stack, resources, cloudFormationClient, amazonEC2Client, scaledGroups, instances); List<Group> groupsWithNewInstances = getGroupsWithNewInstances(scaledGroups); List<CloudResource> newInstances = getNewInstances(scaledGroups, instances); List<CloudResource> reattachableVolumeSets = getReattachableVolumeSets(scaledGroups, resources); List<CloudResource> networkResources = resources.stream() .filter(cloudResource -> ResourceType.AWS_SUBNET.equals(cloudResource.getType())) .collect(Collectors.toList()); awsComputeResourceService.buildComputeResourcesForUpscale(ac, stack, groupsWithNewInstances, newInstances, reattachableVolumeSets, networkResources); awsTaggingService.tagRootVolumes(ac, amazonEC2Client, instances, stack.getTags()); awsCloudWatchService.addCloudWatchAlarmsForSystemFailures(instances, stack, regionName, credentialView); return singletonList(new CloudResourceStatus(cfStackUtil.getCloudFormationStackResource(resources), ResourceStatus.UPDATED)); }  }
AwsUpscaleService { public List<CloudResourceStatus> upscale(AuthenticatedContext ac, CloudStack stack, List<CloudResource> resources) { AmazonCloudFormationRetryClient cloudFormationClient = getCloudFormationRetryClient(ac); AwsCredentialView credentialView = new AwsCredentialView(ac.getCloudCredential()); String regionName = ac.getCloudContext().getLocation().getRegion().value(); AmazonAutoScalingRetryClient amazonASClient = getAutoScalingRetryClient(ac); AmazonEC2Client amazonEC2Client = getEC2Client(ac); List<Group> scaledGroups = cloudResourceHelper.getScaledGroups(stack); Map<String, Group> desiredAutoscalingGroupsByName = getAutoScaleGroupsByNameFromCloudFormationTemplate(ac, cloudFormationClient, scaledGroups); LOGGER.info("Desired autoscaling groups: {}", desiredAutoscalingGroupsByName); awsAutoScalingService.resumeAutoScaling(amazonASClient, desiredAutoscalingGroupsByName.keySet(), UPSCALE_PROCESSES); Map<String, Integer> originalAutoScalingGroupsBySize = getAutoScalingGroupsBySize(desiredAutoscalingGroupsByName.keySet(), amazonASClient); LOGGER.info("Update autoscaling groups for stack: {}", ac.getCloudContext().getName()); Date timeBeforeASUpdate = new Date(); updateAutoscalingGroups(amazonASClient, desiredAutoscalingGroupsByName, originalAutoScalingGroupsBySize); try { awsAutoScalingService.scheduleStatusChecks(scaledGroups, ac, cloudFormationClient, timeBeforeASUpdate); } catch (AmazonAutoscalingFailed amazonAutoscalingFailed) { return recoverOriginalState(ac, stack, amazonASClient, desiredAutoscalingGroupsByName, originalAutoScalingGroupsBySize, amazonAutoscalingFailed); } awsAutoScalingService.suspendAutoScaling(ac, stack); List<CloudResource> instances = cfStackUtil.getInstanceCloudResources(ac, cloudFormationClient, amazonASClient, scaledGroups); associateElasticIpWithNewInstances(stack, resources, cloudFormationClient, amazonEC2Client, scaledGroups, instances); List<Group> groupsWithNewInstances = getGroupsWithNewInstances(scaledGroups); List<CloudResource> newInstances = getNewInstances(scaledGroups, instances); List<CloudResource> reattachableVolumeSets = getReattachableVolumeSets(scaledGroups, resources); List<CloudResource> networkResources = resources.stream() .filter(cloudResource -> ResourceType.AWS_SUBNET.equals(cloudResource.getType())) .collect(Collectors.toList()); awsComputeResourceService.buildComputeResourcesForUpscale(ac, stack, groupsWithNewInstances, newInstances, reattachableVolumeSets, networkResources); awsTaggingService.tagRootVolumes(ac, amazonEC2Client, instances, stack.getTags()); awsCloudWatchService.addCloudWatchAlarmsForSystemFailures(instances, stack, regionName, credentialView); return singletonList(new CloudResourceStatus(cfStackUtil.getCloudFormationStackResource(resources), ResourceStatus.UPDATED)); }  List<CloudResourceStatus> upscale(AuthenticatedContext ac, CloudStack stack, List<CloudResource> resources); }
AwsUpscaleService { public List<CloudResourceStatus> upscale(AuthenticatedContext ac, CloudStack stack, List<CloudResource> resources) { AmazonCloudFormationRetryClient cloudFormationClient = getCloudFormationRetryClient(ac); AwsCredentialView credentialView = new AwsCredentialView(ac.getCloudCredential()); String regionName = ac.getCloudContext().getLocation().getRegion().value(); AmazonAutoScalingRetryClient amazonASClient = getAutoScalingRetryClient(ac); AmazonEC2Client amazonEC2Client = getEC2Client(ac); List<Group> scaledGroups = cloudResourceHelper.getScaledGroups(stack); Map<String, Group> desiredAutoscalingGroupsByName = getAutoScaleGroupsByNameFromCloudFormationTemplate(ac, cloudFormationClient, scaledGroups); LOGGER.info("Desired autoscaling groups: {}", desiredAutoscalingGroupsByName); awsAutoScalingService.resumeAutoScaling(amazonASClient, desiredAutoscalingGroupsByName.keySet(), UPSCALE_PROCESSES); Map<String, Integer> originalAutoScalingGroupsBySize = getAutoScalingGroupsBySize(desiredAutoscalingGroupsByName.keySet(), amazonASClient); LOGGER.info("Update autoscaling groups for stack: {}", ac.getCloudContext().getName()); Date timeBeforeASUpdate = new Date(); updateAutoscalingGroups(amazonASClient, desiredAutoscalingGroupsByName, originalAutoScalingGroupsBySize); try { awsAutoScalingService.scheduleStatusChecks(scaledGroups, ac, cloudFormationClient, timeBeforeASUpdate); } catch (AmazonAutoscalingFailed amazonAutoscalingFailed) { return recoverOriginalState(ac, stack, amazonASClient, desiredAutoscalingGroupsByName, originalAutoScalingGroupsBySize, amazonAutoscalingFailed); } awsAutoScalingService.suspendAutoScaling(ac, stack); List<CloudResource> instances = cfStackUtil.getInstanceCloudResources(ac, cloudFormationClient, amazonASClient, scaledGroups); associateElasticIpWithNewInstances(stack, resources, cloudFormationClient, amazonEC2Client, scaledGroups, instances); List<Group> groupsWithNewInstances = getGroupsWithNewInstances(scaledGroups); List<CloudResource> newInstances = getNewInstances(scaledGroups, instances); List<CloudResource> reattachableVolumeSets = getReattachableVolumeSets(scaledGroups, resources); List<CloudResource> networkResources = resources.stream() .filter(cloudResource -> ResourceType.AWS_SUBNET.equals(cloudResource.getType())) .collect(Collectors.toList()); awsComputeResourceService.buildComputeResourcesForUpscale(ac, stack, groupsWithNewInstances, newInstances, reattachableVolumeSets, networkResources); awsTaggingService.tagRootVolumes(ac, amazonEC2Client, instances, stack.getTags()); awsCloudWatchService.addCloudWatchAlarmsForSystemFailures(instances, stack, regionName, credentialView); return singletonList(new CloudResourceStatus(cfStackUtil.getCloudFormationStackResource(resources), ResourceStatus.UPDATED)); }  List<CloudResourceStatus> upscale(AuthenticatedContext ac, CloudStack stack, List<CloudResource> resources);  }