@Test public void testRecoverSystemFailures() throws Exception { executionStateCache.setUnhealthyInstances(this.createInstances()); executionStateCache.setHealProcess(true); executionStateCache.addProcessName("nginx"); executionStateCache.setRamDiskFull(true); executionStateCache.addPortNum(1234); executionStateCache.setHealNetwork(true); executionStateCache.setBlockPort(true); executionStateCache.setBlockedDomains(Arrays.asList("yahoo.com")); executionStateCache.setCpuSpiked(true); environmentHealerStepDefinitions.recover(); }
@Then("^recover$") public void recover() { boolean isSSM = !executionStateCache.getSsmCommandInvocationInstances().isEmpty(); if (executionStateCache.isHealProcess()) { List<String> processes = executionStateCache.getProcessNames(); for (String process : processes) { if (isSSM) { healGivenSSMInstancesByProcess(process, executionStateCache.getSsmCommandInvocationInstances()); } else { healGivenInstancesByProcess(process, executionStateCache.getUnhealthyInstances()); } } } if (executionStateCache.isHealNetwork()) { if (isSSM) { healSSMGivenInstancesFixingNetworkIssues(executionStateCache.getSsmCommandInvocationInstances()); } else { healGivenInstancesFixingNetworkIssues(executionStateCache.getUnhealthyInstances()); } } if (!executionStateCache.getBlockedDomains().isEmpty()) { if (isSSM) { executionStateCache.getBlockedDomains().parallelStream() .forEach(domain -> healSSMGivenInstancesByDomainName(domain, executionStateCache.getSsmCommandInvocationInstances())); } else { executionStateCache.getBlockedDomains().parallelStream() .forEach(domain -> healGivenInstancesByDomainName(domain, executionStateCache.getUnhealthyInstances())); } } if (executionStateCache.isCpuSpiked()) { if (isSSM) { healSSMGivenInstancesByFixingCPU(executionStateCache.getSsmCommandInvocationInstances()); } else { healGivenInstancesByFixingCPU(executionStateCache.getUnhealthyInstances()); } } if (executionStateCache.isBlockPort()) { List<Integer> portNums = executionStateCache.getPortNums(); for (Integer port : portNums) { if (isSSM) { healSSMGivenInstancesByFixingPort(port, executionStateCache.getSsmCommandInvocationInstances()); } else { healGivenInstancesByFixingPort(executionStateCache.getUnhealthyInstances()); } } } if (executionStateCache.isRamDiskFull()) { if (isSSM) { healSSMGivenInstancesByFixingRamDisk(executionStateCache.getSsmCommandInvocationInstances()); } else { healGivenInstancesByFixingRamDisk(executionStateCache.getUnhealthyInstances()); } } if (executionStateCache.isBlockDynamoDB()) { if (isSSM) { healSSMGivenInstancesByUnblockingDynamoDB(executionStateCache.getSsmCommandInvocationInstances()); } else { healGivenInstancesByUnblockingDynamoDB(executionStateCache.getUnhealthyInstances()); } } if (executionStateCache.isBlockS3()) { if (isSSM) { healSSMGivenInstancesByUnblockingS3(executionStateCache.getSsmCommandInvocationInstances()); } else { healGivenInstancesByUnblockingS3(executionStateCache.getUnhealthyInstances()); } } }
EnvironmentHealerStepDefinitions { @Then("^recover$") public void recover() { boolean isSSM = !executionStateCache.getSsmCommandInvocationInstances().isEmpty(); if (executionStateCache.isHealProcess()) { List<String> processes = executionStateCache.getProcessNames(); for (String process : processes) { if (isSSM) { healGivenSSMInstancesByProcess(process, executionStateCache.getSsmCommandInvocationInstances()); } else { healGivenInstancesByProcess(process, executionStateCache.getUnhealthyInstances()); } } } if (executionStateCache.isHealNetwork()) { if (isSSM) { healSSMGivenInstancesFixingNetworkIssues(executionStateCache.getSsmCommandInvocationInstances()); } else { healGivenInstancesFixingNetworkIssues(executionStateCache.getUnhealthyInstances()); } } if (!executionStateCache.getBlockedDomains().isEmpty()) { if (isSSM) { executionStateCache.getBlockedDomains().parallelStream() .forEach(domain -> healSSMGivenInstancesByDomainName(domain, executionStateCache.getSsmCommandInvocationInstances())); } else { executionStateCache.getBlockedDomains().parallelStream() .forEach(domain -> healGivenInstancesByDomainName(domain, executionStateCache.getUnhealthyInstances())); } } if (executionStateCache.isCpuSpiked()) { if (isSSM) { healSSMGivenInstancesByFixingCPU(executionStateCache.getSsmCommandInvocationInstances()); } else { healGivenInstancesByFixingCPU(executionStateCache.getUnhealthyInstances()); } } if (executionStateCache.isBlockPort()) { List<Integer> portNums = executionStateCache.getPortNums(); for (Integer port : portNums) { if (isSSM) { healSSMGivenInstancesByFixingPort(port, executionStateCache.getSsmCommandInvocationInstances()); } else { healGivenInstancesByFixingPort(executionStateCache.getUnhealthyInstances()); } } } if (executionStateCache.isRamDiskFull()) { if (isSSM) { healSSMGivenInstancesByFixingRamDisk(executionStateCache.getSsmCommandInvocationInstances()); } else { healGivenInstancesByFixingRamDisk(executionStateCache.getUnhealthyInstances()); } } if (executionStateCache.isBlockDynamoDB()) { if (isSSM) { healSSMGivenInstancesByUnblockingDynamoDB(executionStateCache.getSsmCommandInvocationInstances()); } else { healGivenInstancesByUnblockingDynamoDB(executionStateCache.getUnhealthyInstances()); } } if (executionStateCache.isBlockS3()) { if (isSSM) { healSSMGivenInstancesByUnblockingS3(executionStateCache.getSsmCommandInvocationInstances()); } else { healGivenInstancesByUnblockingS3(executionStateCache.getUnhealthyInstances()); } } } }
EnvironmentHealerStepDefinitions { @Then("^recover$") public void recover() { boolean isSSM = !executionStateCache.getSsmCommandInvocationInstances().isEmpty(); if (executionStateCache.isHealProcess()) { List<String> processes = executionStateCache.getProcessNames(); for (String process : processes) { if (isSSM) { healGivenSSMInstancesByProcess(process, executionStateCache.getSsmCommandInvocationInstances()); } else { healGivenInstancesByProcess(process, executionStateCache.getUnhealthyInstances()); } } } if (executionStateCache.isHealNetwork()) { if (isSSM) { healSSMGivenInstancesFixingNetworkIssues(executionStateCache.getSsmCommandInvocationInstances()); } else { healGivenInstancesFixingNetworkIssues(executionStateCache.getUnhealthyInstances()); } } if (!executionStateCache.getBlockedDomains().isEmpty()) { if (isSSM) { executionStateCache.getBlockedDomains().parallelStream() .forEach(domain -> healSSMGivenInstancesByDomainName(domain, executionStateCache.getSsmCommandInvocationInstances())); } else { executionStateCache.getBlockedDomains().parallelStream() .forEach(domain -> healGivenInstancesByDomainName(domain, executionStateCache.getUnhealthyInstances())); } } if (executionStateCache.isCpuSpiked()) { if (isSSM) { healSSMGivenInstancesByFixingCPU(executionStateCache.getSsmCommandInvocationInstances()); } else { healGivenInstancesByFixingCPU(executionStateCache.getUnhealthyInstances()); } } if (executionStateCache.isBlockPort()) { List<Integer> portNums = executionStateCache.getPortNums(); for (Integer port : portNums) { if (isSSM) { healSSMGivenInstancesByFixingPort(port, executionStateCache.getSsmCommandInvocationInstances()); } else { healGivenInstancesByFixingPort(executionStateCache.getUnhealthyInstances()); } } } if (executionStateCache.isRamDiskFull()) { if (isSSM) { healSSMGivenInstancesByFixingRamDisk(executionStateCache.getSsmCommandInvocationInstances()); } else { healGivenInstancesByFixingRamDisk(executionStateCache.getUnhealthyInstances()); } } if (executionStateCache.isBlockDynamoDB()) { if (isSSM) { healSSMGivenInstancesByUnblockingDynamoDB(executionStateCache.getSsmCommandInvocationInstances()); } else { healGivenInstancesByUnblockingDynamoDB(executionStateCache.getUnhealthyInstances()); } } if (executionStateCache.isBlockS3()) { if (isSSM) { healSSMGivenInstancesByUnblockingS3(executionStateCache.getSsmCommandInvocationInstances()); } else { healGivenInstancesByUnblockingS3(executionStateCache.getUnhealthyInstances()); } } } EnvironmentHealerStepDefinitions(); }
EnvironmentHealerStepDefinitions { @Then("^recover$") public void recover() { boolean isSSM = !executionStateCache.getSsmCommandInvocationInstances().isEmpty(); if (executionStateCache.isHealProcess()) { List<String> processes = executionStateCache.getProcessNames(); for (String process : processes) { if (isSSM) { healGivenSSMInstancesByProcess(process, executionStateCache.getSsmCommandInvocationInstances()); } else { healGivenInstancesByProcess(process, executionStateCache.getUnhealthyInstances()); } } } if (executionStateCache.isHealNetwork()) { if (isSSM) { healSSMGivenInstancesFixingNetworkIssues(executionStateCache.getSsmCommandInvocationInstances()); } else { healGivenInstancesFixingNetworkIssues(executionStateCache.getUnhealthyInstances()); } } if (!executionStateCache.getBlockedDomains().isEmpty()) { if (isSSM) { executionStateCache.getBlockedDomains().parallelStream() .forEach(domain -> healSSMGivenInstancesByDomainName(domain, executionStateCache.getSsmCommandInvocationInstances())); } else { executionStateCache.getBlockedDomains().parallelStream() .forEach(domain -> healGivenInstancesByDomainName(domain, executionStateCache.getUnhealthyInstances())); } } if (executionStateCache.isCpuSpiked()) { if (isSSM) { healSSMGivenInstancesByFixingCPU(executionStateCache.getSsmCommandInvocationInstances()); } else { healGivenInstancesByFixingCPU(executionStateCache.getUnhealthyInstances()); } } if (executionStateCache.isBlockPort()) { List<Integer> portNums = executionStateCache.getPortNums(); for (Integer port : portNums) { if (isSSM) { healSSMGivenInstancesByFixingPort(port, executionStateCache.getSsmCommandInvocationInstances()); } else { healGivenInstancesByFixingPort(executionStateCache.getUnhealthyInstances()); } } } if (executionStateCache.isRamDiskFull()) { if (isSSM) { healSSMGivenInstancesByFixingRamDisk(executionStateCache.getSsmCommandInvocationInstances()); } else { healGivenInstancesByFixingRamDisk(executionStateCache.getUnhealthyInstances()); } } if (executionStateCache.isBlockDynamoDB()) { if (isSSM) { healSSMGivenInstancesByUnblockingDynamoDB(executionStateCache.getSsmCommandInvocationInstances()); } else { healGivenInstancesByUnblockingDynamoDB(executionStateCache.getUnhealthyInstances()); } } if (executionStateCache.isBlockS3()) { if (isSSM) { healSSMGivenInstancesByUnblockingS3(executionStateCache.getSsmCommandInvocationInstances()); } else { healGivenInstancesByUnblockingS3(executionStateCache.getUnhealthyInstances()); } } } EnvironmentHealerStepDefinitions(); ExecutionStateCache getExecutionStateCache(); void setExecutionStateCache(ExecutionStateCache executionStateCache); @Then("^recover$") void recover(); @Then("^clearCache$") void clearCache(); EnvironmentHealerStepDefinitions healGivenInstancesByProcess(String processName, List<EC2InstanceTO> instances); EnvironmentHealerStepDefinitions healGivenInstancesByDomainName(String domainName, List<EC2InstanceTO> instances); EnvironmentHealerStepDefinitions healGivenInstancesByFixingRamDisk(List<EC2InstanceTO> instanceList); EnvironmentHealerStepDefinitions healGivenInstancesByFixingNetworkIssues(List<EC2InstanceTO> instances); EnvironmentHealerStepDefinitions healGivenInstancesByFixingCPU(List<EC2InstanceTO> instances); EnvironmentHealerStepDefinitions healGivenInstancesByFixingPort(List<EC2InstanceTO> instances); EnvironmentHealerStepDefinitions healSSMGivenInstancesByFixingCPU(List<EC2InstanceTO> instanceList); EnvironmentHealerStepDefinitions healSSMGivenInstancesFixingNetworkIssues(List<EC2InstanceTO> instanceList ); EnvironmentHealerStepDefinitions healGivenInstancesFixingNetworkIssues(List<EC2InstanceTO> instances ); EnvironmentHealerStepDefinitions healSSMGivenInstancesByUnblockingDynamoDB(List<EC2InstanceTO> instanceList ); EnvironmentHealerStepDefinitions healSSMGivenInstancesByUnblockingS3(List<EC2InstanceTO> instanceList ); EnvironmentHealerStepDefinitions healSSMGivenInstancesByFixingPort(int portNum, List<EC2InstanceTO> instanceList); EnvironmentHealerStepDefinitions healGivenSSMInstancesByProcess(String processName, List<EC2InstanceTO> instanceList); EnvironmentHealerStepDefinitions healSSMGivenInstancesByDomainName(String domainName, List<EC2InstanceTO> instanceList); EnvironmentHealerStepDefinitions healSSMGivenInstancesByFixingRamDisk(List<EC2InstanceTO> instanceList); EnvironmentHealerStepDefinitions healGivenInstancesByUnblockingDynamoDB(List<EC2InstanceTO> instances ); EnvironmentHealerStepDefinitions healGivenInstancesByUnblockingS3(List<EC2InstanceTO> instances ); }
EnvironmentHealerStepDefinitions { @Then("^recover$") public void recover() { boolean isSSM = !executionStateCache.getSsmCommandInvocationInstances().isEmpty(); if (executionStateCache.isHealProcess()) { List<String> processes = executionStateCache.getProcessNames(); for (String process : processes) { if (isSSM) { healGivenSSMInstancesByProcess(process, executionStateCache.getSsmCommandInvocationInstances()); } else { healGivenInstancesByProcess(process, executionStateCache.getUnhealthyInstances()); } } } if (executionStateCache.isHealNetwork()) { if (isSSM) { healSSMGivenInstancesFixingNetworkIssues(executionStateCache.getSsmCommandInvocationInstances()); } else { healGivenInstancesFixingNetworkIssues(executionStateCache.getUnhealthyInstances()); } } if (!executionStateCache.getBlockedDomains().isEmpty()) { if (isSSM) { executionStateCache.getBlockedDomains().parallelStream() .forEach(domain -> healSSMGivenInstancesByDomainName(domain, executionStateCache.getSsmCommandInvocationInstances())); } else { executionStateCache.getBlockedDomains().parallelStream() .forEach(domain -> healGivenInstancesByDomainName(domain, executionStateCache.getUnhealthyInstances())); } } if (executionStateCache.isCpuSpiked()) { if (isSSM) { healSSMGivenInstancesByFixingCPU(executionStateCache.getSsmCommandInvocationInstances()); } else { healGivenInstancesByFixingCPU(executionStateCache.getUnhealthyInstances()); } } if (executionStateCache.isBlockPort()) { List<Integer> portNums = executionStateCache.getPortNums(); for (Integer port : portNums) { if (isSSM) { healSSMGivenInstancesByFixingPort(port, executionStateCache.getSsmCommandInvocationInstances()); } else { healGivenInstancesByFixingPort(executionStateCache.getUnhealthyInstances()); } } } if (executionStateCache.isRamDiskFull()) { if (isSSM) { healSSMGivenInstancesByFixingRamDisk(executionStateCache.getSsmCommandInvocationInstances()); } else { healGivenInstancesByFixingRamDisk(executionStateCache.getUnhealthyInstances()); } } if (executionStateCache.isBlockDynamoDB()) { if (isSSM) { healSSMGivenInstancesByUnblockingDynamoDB(executionStateCache.getSsmCommandInvocationInstances()); } else { healGivenInstancesByUnblockingDynamoDB(executionStateCache.getUnhealthyInstances()); } } if (executionStateCache.isBlockS3()) { if (isSSM) { healSSMGivenInstancesByUnblockingS3(executionStateCache.getSsmCommandInvocationInstances()); } else { healGivenInstancesByUnblockingS3(executionStateCache.getUnhealthyInstances()); } } } EnvironmentHealerStepDefinitions(); ExecutionStateCache getExecutionStateCache(); void setExecutionStateCache(ExecutionStateCache executionStateCache); @Then("^recover$") void recover(); @Then("^clearCache$") void clearCache(); EnvironmentHealerStepDefinitions healGivenInstancesByProcess(String processName, List<EC2InstanceTO> instances); EnvironmentHealerStepDefinitions healGivenInstancesByDomainName(String domainName, List<EC2InstanceTO> instances); EnvironmentHealerStepDefinitions healGivenInstancesByFixingRamDisk(List<EC2InstanceTO> instanceList); EnvironmentHealerStepDefinitions healGivenInstancesByFixingNetworkIssues(List<EC2InstanceTO> instances); EnvironmentHealerStepDefinitions healGivenInstancesByFixingCPU(List<EC2InstanceTO> instances); EnvironmentHealerStepDefinitions healGivenInstancesByFixingPort(List<EC2InstanceTO> instances); EnvironmentHealerStepDefinitions healSSMGivenInstancesByFixingCPU(List<EC2InstanceTO> instanceList); EnvironmentHealerStepDefinitions healSSMGivenInstancesFixingNetworkIssues(List<EC2InstanceTO> instanceList ); EnvironmentHealerStepDefinitions healGivenInstancesFixingNetworkIssues(List<EC2InstanceTO> instances ); EnvironmentHealerStepDefinitions healSSMGivenInstancesByUnblockingDynamoDB(List<EC2InstanceTO> instanceList ); EnvironmentHealerStepDefinitions healSSMGivenInstancesByUnblockingS3(List<EC2InstanceTO> instanceList ); EnvironmentHealerStepDefinitions healSSMGivenInstancesByFixingPort(int portNum, List<EC2InstanceTO> instanceList); EnvironmentHealerStepDefinitions healGivenSSMInstancesByProcess(String processName, List<EC2InstanceTO> instanceList); EnvironmentHealerStepDefinitions healSSMGivenInstancesByDomainName(String domainName, List<EC2InstanceTO> instanceList); EnvironmentHealerStepDefinitions healSSMGivenInstancesByFixingRamDisk(List<EC2InstanceTO> instanceList); EnvironmentHealerStepDefinitions healGivenInstancesByUnblockingDynamoDB(List<EC2InstanceTO> instances ); EnvironmentHealerStepDefinitions healGivenInstancesByUnblockingS3(List<EC2InstanceTO> instances );  }