@Test public void tearDownFailTest() throws Exception { Map<String, String> privateIpsByFQDN = new HashMap<>(); privateIpsByFQDN.put("10-0-0-1.example.com", "10.0.0.1"); privateIpsByFQDN.put("10-0-0-2.example.com", "10.0.0.2"); privateIpsByFQDN.put("10-0-0-3.example.com", "10.0.0.3"); mockStatic(SaltStates.class); PowerMockito.doThrow(new NullPointerException()).when(SaltStates.class); SaltStates.stopMinions(eq(saltConnector), eq(privateIpsByFQDN)); try { saltOrchestrator.tearDown(Collections.singletonList(gatewayConfig), privateIpsByFQDN, Set.of(), null); fail(); } catch (CloudbreakOrchestratorFailedException e) { assertTrue(e.getCause() instanceof NullPointerException); } }
@Override public void tearDown(List<GatewayConfig> allGatewayConfigs, Map<String, String> removeNodePrivateIPsByFQDN, Set<Node> remainingNodes, ExitCriteriaModel exitModel) throws CloudbreakOrchestratorException { LOGGER.debug("Tear down hosts: {},", removeNodePrivateIPsByFQDN); LOGGER.debug("Gateway config for tear down: {}", allGatewayConfigs); GatewayConfig primaryGateway = saltService.getPrimaryGatewayConfig(allGatewayConfigs); Set<String> gatewayTargetIpAddresses = getGatewayPrivateIps(allGatewayConfigs); try (SaltConnector saltConnector = saltService.createSaltConnector(primaryGateway)) { SaltStates.stopMinions(saltConnector, removeNodePrivateIPsByFQDN); if (!CollectionUtils.isEmpty(remainingNodes)) { OrchestratorBootstrap hostSave = new PillarSave(saltConnector, gatewayTargetIpAddresses, remainingNodes); Callable<Boolean> saltPillarRunner = saltRunner.runner(hostSave, exitCriteria, exitModel); saltPillarRunner.call(); } } catch (Exception e) { LOGGER.info("Error occurred during salt minion tear down", e); throw new CloudbreakOrchestratorFailedException(e); } List<GatewayConfig> liveGateways = allGatewayConfigs.stream() .filter(gw -> !removeNodePrivateIPsByFQDN.containsValue(gw.getPrivateAddress())).collect(Collectors.toList()); for (GatewayConfig gatewayConfig : liveGateways) { try (SaltConnector sc = saltService.createSaltConnector(gatewayConfig)) { sc.wheel("key.delete", removeNodePrivateIPsByFQDN.keySet(), Object.class); removeDeadSaltMinions(gatewayConfig); } catch (Exception e) { LOGGER.info("Error occurred during salt minion tear down", e); throw new CloudbreakOrchestratorFailedException(e); } } }
SaltOrchestrator implements HostOrchestrator { @Override public void tearDown(List<GatewayConfig> allGatewayConfigs, Map<String, String> removeNodePrivateIPsByFQDN, Set<Node> remainingNodes, ExitCriteriaModel exitModel) throws CloudbreakOrchestratorException { LOGGER.debug("Tear down hosts: {},", removeNodePrivateIPsByFQDN); LOGGER.debug("Gateway config for tear down: {}", allGatewayConfigs); GatewayConfig primaryGateway = saltService.getPrimaryGatewayConfig(allGatewayConfigs); Set<String> gatewayTargetIpAddresses = getGatewayPrivateIps(allGatewayConfigs); try (SaltConnector saltConnector = saltService.createSaltConnector(primaryGateway)) { SaltStates.stopMinions(saltConnector, removeNodePrivateIPsByFQDN); if (!CollectionUtils.isEmpty(remainingNodes)) { OrchestratorBootstrap hostSave = new PillarSave(saltConnector, gatewayTargetIpAddresses, remainingNodes); Callable<Boolean> saltPillarRunner = saltRunner.runner(hostSave, exitCriteria, exitModel); saltPillarRunner.call(); } } catch (Exception e) { LOGGER.info("Error occurred during salt minion tear down", e); throw new CloudbreakOrchestratorFailedException(e); } List<GatewayConfig> liveGateways = allGatewayConfigs.stream() .filter(gw -> !removeNodePrivateIPsByFQDN.containsValue(gw.getPrivateAddress())).collect(Collectors.toList()); for (GatewayConfig gatewayConfig : liveGateways) { try (SaltConnector sc = saltService.createSaltConnector(gatewayConfig)) { sc.wheel("key.delete", removeNodePrivateIPsByFQDN.keySet(), Object.class); removeDeadSaltMinions(gatewayConfig); } catch (Exception e) { LOGGER.info("Error occurred during salt minion tear down", e); throw new CloudbreakOrchestratorFailedException(e); } } } }
SaltOrchestrator implements HostOrchestrator { @Override public void tearDown(List<GatewayConfig> allGatewayConfigs, Map<String, String> removeNodePrivateIPsByFQDN, Set<Node> remainingNodes, ExitCriteriaModel exitModel) throws CloudbreakOrchestratorException { LOGGER.debug("Tear down hosts: {},", removeNodePrivateIPsByFQDN); LOGGER.debug("Gateway config for tear down: {}", allGatewayConfigs); GatewayConfig primaryGateway = saltService.getPrimaryGatewayConfig(allGatewayConfigs); Set<String> gatewayTargetIpAddresses = getGatewayPrivateIps(allGatewayConfigs); try (SaltConnector saltConnector = saltService.createSaltConnector(primaryGateway)) { SaltStates.stopMinions(saltConnector, removeNodePrivateIPsByFQDN); if (!CollectionUtils.isEmpty(remainingNodes)) { OrchestratorBootstrap hostSave = new PillarSave(saltConnector, gatewayTargetIpAddresses, remainingNodes); Callable<Boolean> saltPillarRunner = saltRunner.runner(hostSave, exitCriteria, exitModel); saltPillarRunner.call(); } } catch (Exception e) { LOGGER.info("Error occurred during salt minion tear down", e); throw new CloudbreakOrchestratorFailedException(e); } List<GatewayConfig> liveGateways = allGatewayConfigs.stream() .filter(gw -> !removeNodePrivateIPsByFQDN.containsValue(gw.getPrivateAddress())).collect(Collectors.toList()); for (GatewayConfig gatewayConfig : liveGateways) { try (SaltConnector sc = saltService.createSaltConnector(gatewayConfig)) { sc.wheel("key.delete", removeNodePrivateIPsByFQDN.keySet(), Object.class); removeDeadSaltMinions(gatewayConfig); } catch (Exception e) { LOGGER.info("Error occurred during salt minion tear down", e); throw new CloudbreakOrchestratorFailedException(e); } } }  }
SaltOrchestrator implements HostOrchestrator { @Override public void tearDown(List<GatewayConfig> allGatewayConfigs, Map<String, String> removeNodePrivateIPsByFQDN, Set<Node> remainingNodes, ExitCriteriaModel exitModel) throws CloudbreakOrchestratorException { LOGGER.debug("Tear down hosts: {},", removeNodePrivateIPsByFQDN); LOGGER.debug("Gateway config for tear down: {}", allGatewayConfigs); GatewayConfig primaryGateway = saltService.getPrimaryGatewayConfig(allGatewayConfigs); Set<String> gatewayTargetIpAddresses = getGatewayPrivateIps(allGatewayConfigs); try (SaltConnector saltConnector = saltService.createSaltConnector(primaryGateway)) { SaltStates.stopMinions(saltConnector, removeNodePrivateIPsByFQDN); if (!CollectionUtils.isEmpty(remainingNodes)) { OrchestratorBootstrap hostSave = new PillarSave(saltConnector, gatewayTargetIpAddresses, remainingNodes); Callable<Boolean> saltPillarRunner = saltRunner.runner(hostSave, exitCriteria, exitModel); saltPillarRunner.call(); } } catch (Exception e) { LOGGER.info("Error occurred during salt minion tear down", e); throw new CloudbreakOrchestratorFailedException(e); } List<GatewayConfig> liveGateways = allGatewayConfigs.stream() .filter(gw -> !removeNodePrivateIPsByFQDN.containsValue(gw.getPrivateAddress())).collect(Collectors.toList()); for (GatewayConfig gatewayConfig : liveGateways) { try (SaltConnector sc = saltService.createSaltConnector(gatewayConfig)) { sc.wheel("key.delete", removeNodePrivateIPsByFQDN.keySet(), Object.class); removeDeadSaltMinions(gatewayConfig); } catch (Exception e) { LOGGER.info("Error occurred during salt minion tear down", e); throw new CloudbreakOrchestratorFailedException(e); } } }  @Override void bootstrap(List<GatewayConfig> allGatewayConfigs, Set<Node> targets, BootstrapParams params,
            ExitCriteriaModel exitModel); @Override Map<String, Map<String, String>> formatAndMountDisksOnNodes(List<GatewayConfig> allGateway, Set<Node> nodes, Set<Node> allNodes,
            ExitCriteriaModel exitModel, String platformVariant); @Override void bootstrapNewNodes(List<GatewayConfig> allGatewayConfigs, Set<Node> targets, Set<Node> allNodes, byte[] stateConfigZip, BootstrapParams params,
            ExitCriteriaModel exitModel); @SuppressFBWarnings("REC_CATCH_EXCEPTION") @Override void initServiceRun(List<GatewayConfig> allGateway, Set<Node> allNodes, SaltConfig saltConfig, ExitCriteriaModel exitModel); @Override void initSaltConfig(List<GatewayConfig> allGateway, Set<Node> allNodes, SaltConfig saltConfig, ExitCriteriaModel exitModel); @SuppressFBWarnings("REC_CATCH_EXCEPTION") @Override void runService(List<GatewayConfig> allGateway, Set<Node> allNodes, SaltConfig saltConfig, ExitCriteriaModel exitModel); @SuppressFBWarnings("REC_CATCH_EXCEPTION") @Override void changePrimaryGateway(GatewayConfig formerGateway, GatewayConfig newPrimaryGateway, List<GatewayConfig> allGatewayConfigs, Set<Node> allNodes,
            ExitCriteriaModel exitCriteriaModel); @SuppressFBWarnings("REC_CATCH_EXCEPTION") @Override void installFreeIpa(GatewayConfig primaryGateway, List<GatewayConfig> allGatewayConfigs, Set<Node> allNodes,
            ExitCriteriaModel exitCriteriaModel); @Override Optional<String> getFreeIpaMasterHostname(GatewayConfig primaryGateway); @Override void resetClusterManager(GatewayConfig gatewayConfig, Set<String> target, Set<Node> allNodes, ExitCriteriaModel exitCriteriaModel); @Override void stopClusterManagerOnMaster(GatewayConfig gatewayConfig, Set<Node> allNodes, ExitCriteriaModel exitCriteriaModel); @Override void startClusterManagerOnMaster(GatewayConfig gatewayConfig, Set<Node> allNodes, ExitCriteriaModel exitCriteriaModel); @SuppressFBWarnings("REC_CATCH_EXCEPTION") @Override void upgradeClusterManager(GatewayConfig gatewayConfig, Set<String> target, Set<Node> allNodes, SaltConfig pillarConfig,
            ExitCriteriaModel exitCriteriaModel); @Override void tearDown(List<GatewayConfig> allGatewayConfigs, Map<String, String> removeNodePrivateIPsByFQDN,
            Set<Node> remainingNodes, ExitCriteriaModel exitModel); Map<String, Map<String, String>> getPackageVersionsFromAllHosts(GatewayConfig gateway, Map<String, Optional<String>> packages); Map<String, String> runCommandOnAllHosts(GatewayConfig gateway, String command); @Override Map<String, JsonNode> getGrainOnAllHosts(GatewayConfig gateway, String grain); @Override void uploadRecipes(List<GatewayConfig> allGatewayConfigs, Map<String, List<RecipeModel>> recipes, ExitCriteriaModel exitModel); @Override void uploadKeytabs(List<GatewayConfig> allGatewayConfigs, Set<KeytabModel> keytabModels, ExitCriteriaModel exitModel); @Override Map<String, Map<String, String>> formatAndMountDisksOnNodesLegacy(List<GatewayConfig> gatewayConfigs, Set<Node> nodes, Set<Node> allNodes,
            ExitCriteriaModel exitCriteriaModel, String platformVariant); @Override void applyDiagnosticsState(List<GatewayConfig> gatewayConfigs, String state, Map<String, Object> properties,
            ExitCriteriaModel exitCriteriaModel); @Override byte[] getStateConfigZip(); @Override void preClusterManagerStartRecipes(GatewayConfig gatewayConfig, Set<Node> allNodes, ExitCriteriaModel exitCriteriaModel); @Override void postClusterManagerStartRecipes(GatewayConfig gatewayConfig, Set<Node> allNodes, ExitCriteriaModel exitCriteriaModel); @Override void postInstallRecipes(GatewayConfig gatewayConfig, Set<Node> allNodes, ExitCriteriaModel exitCriteriaModel); @Override void preTerminationRecipes(GatewayConfig gatewayConfig, Set<Node> allNodes, ExitCriteriaModel exitCriteriaModel, boolean forced); @Override void stopClusterManagerAgent(GatewayConfig gatewayConfig, Set<Node> nodes, ExitCriteriaModel exitCriteriaModel, boolean adJoinable,
            boolean ipaJoinable, boolean forced); @Override void uploadGatewayPillar(List<GatewayConfig> allGatewayConfigs, Set<Node> allNodes, ExitCriteriaModel exitModel, SaltConfig saltConfig); void leaveDomain(GatewayConfig gatewayConfig, Set<Node> allNodes, String roleToRemove, String roleToAdd, ExitCriteriaModel exitCriteriaModel); @Override List<String> getMissingNodes(GatewayConfig gatewayConfig, Set<Node> nodes); @Override List<String> getAvailableNodes(GatewayConfig gatewayConfig, Set<Node> nodes); @Override boolean isBootstrapApiAvailable(GatewayConfig gatewayConfig); @Override String name(); @Override Map<String, String> getMembers(GatewayConfig gatewayConfig, List<String> privateIps); @Override void backupDatabase(GatewayConfig primaryGateway, Set<String> target, Set<Node> allNodes, SaltConfig saltConfig,
            ExitCriteriaModel exitModel); @Override void restoreDatabase(GatewayConfig primaryGateway, Set<String> target, Set<Node> allNodes, SaltConfig saltConfig,
                                ExitCriteriaModel exitModel); }
SaltOrchestrator implements HostOrchestrator { @Override public void tearDown(List<GatewayConfig> allGatewayConfigs, Map<String, String> removeNodePrivateIPsByFQDN, Set<Node> remainingNodes, ExitCriteriaModel exitModel) throws CloudbreakOrchestratorException { LOGGER.debug("Tear down hosts: {},", removeNodePrivateIPsByFQDN); LOGGER.debug("Gateway config for tear down: {}", allGatewayConfigs); GatewayConfig primaryGateway = saltService.getPrimaryGatewayConfig(allGatewayConfigs); Set<String> gatewayTargetIpAddresses = getGatewayPrivateIps(allGatewayConfigs); try (SaltConnector saltConnector = saltService.createSaltConnector(primaryGateway)) { SaltStates.stopMinions(saltConnector, removeNodePrivateIPsByFQDN); if (!CollectionUtils.isEmpty(remainingNodes)) { OrchestratorBootstrap hostSave = new PillarSave(saltConnector, gatewayTargetIpAddresses, remainingNodes); Callable<Boolean> saltPillarRunner = saltRunner.runner(hostSave, exitCriteria, exitModel); saltPillarRunner.call(); } } catch (Exception e) { LOGGER.info("Error occurred during salt minion tear down", e); throw new CloudbreakOrchestratorFailedException(e); } List<GatewayConfig> liveGateways = allGatewayConfigs.stream() .filter(gw -> !removeNodePrivateIPsByFQDN.containsValue(gw.getPrivateAddress())).collect(Collectors.toList()); for (GatewayConfig gatewayConfig : liveGateways) { try (SaltConnector sc = saltService.createSaltConnector(gatewayConfig)) { sc.wheel("key.delete", removeNodePrivateIPsByFQDN.keySet(), Object.class); removeDeadSaltMinions(gatewayConfig); } catch (Exception e) { LOGGER.info("Error occurred during salt minion tear down", e); throw new CloudbreakOrchestratorFailedException(e); } } }  @Override void bootstrap(List<GatewayConfig> allGatewayConfigs, Set<Node> targets, BootstrapParams params,
            ExitCriteriaModel exitModel); @Override Map<String, Map<String, String>> formatAndMountDisksOnNodes(List<GatewayConfig> allGateway, Set<Node> nodes, Set<Node> allNodes,
            ExitCriteriaModel exitModel, String platformVariant); @Override void bootstrapNewNodes(List<GatewayConfig> allGatewayConfigs, Set<Node> targets, Set<Node> allNodes, byte[] stateConfigZip, BootstrapParams params,
            ExitCriteriaModel exitModel); @SuppressFBWarnings("REC_CATCH_EXCEPTION") @Override void initServiceRun(List<GatewayConfig> allGateway, Set<Node> allNodes, SaltConfig saltConfig, ExitCriteriaModel exitModel); @Override void initSaltConfig(List<GatewayConfig> allGateway, Set<Node> allNodes, SaltConfig saltConfig, ExitCriteriaModel exitModel); @SuppressFBWarnings("REC_CATCH_EXCEPTION") @Override void runService(List<GatewayConfig> allGateway, Set<Node> allNodes, SaltConfig saltConfig, ExitCriteriaModel exitModel); @SuppressFBWarnings("REC_CATCH_EXCEPTION") @Override void changePrimaryGateway(GatewayConfig formerGateway, GatewayConfig newPrimaryGateway, List<GatewayConfig> allGatewayConfigs, Set<Node> allNodes,
            ExitCriteriaModel exitCriteriaModel); @SuppressFBWarnings("REC_CATCH_EXCEPTION") @Override void installFreeIpa(GatewayConfig primaryGateway, List<GatewayConfig> allGatewayConfigs, Set<Node> allNodes,
            ExitCriteriaModel exitCriteriaModel); @Override Optional<String> getFreeIpaMasterHostname(GatewayConfig primaryGateway); @Override void resetClusterManager(GatewayConfig gatewayConfig, Set<String> target, Set<Node> allNodes, ExitCriteriaModel exitCriteriaModel); @Override void stopClusterManagerOnMaster(GatewayConfig gatewayConfig, Set<Node> allNodes, ExitCriteriaModel exitCriteriaModel); @Override void startClusterManagerOnMaster(GatewayConfig gatewayConfig, Set<Node> allNodes, ExitCriteriaModel exitCriteriaModel); @SuppressFBWarnings("REC_CATCH_EXCEPTION") @Override void upgradeClusterManager(GatewayConfig gatewayConfig, Set<String> target, Set<Node> allNodes, SaltConfig pillarConfig,
            ExitCriteriaModel exitCriteriaModel); @Override void tearDown(List<GatewayConfig> allGatewayConfigs, Map<String, String> removeNodePrivateIPsByFQDN,
            Set<Node> remainingNodes, ExitCriteriaModel exitModel); Map<String, Map<String, String>> getPackageVersionsFromAllHosts(GatewayConfig gateway, Map<String, Optional<String>> packages); Map<String, String> runCommandOnAllHosts(GatewayConfig gateway, String command); @Override Map<String, JsonNode> getGrainOnAllHosts(GatewayConfig gateway, String grain); @Override void uploadRecipes(List<GatewayConfig> allGatewayConfigs, Map<String, List<RecipeModel>> recipes, ExitCriteriaModel exitModel); @Override void uploadKeytabs(List<GatewayConfig> allGatewayConfigs, Set<KeytabModel> keytabModels, ExitCriteriaModel exitModel); @Override Map<String, Map<String, String>> formatAndMountDisksOnNodesLegacy(List<GatewayConfig> gatewayConfigs, Set<Node> nodes, Set<Node> allNodes,
            ExitCriteriaModel exitCriteriaModel, String platformVariant); @Override void applyDiagnosticsState(List<GatewayConfig> gatewayConfigs, String state, Map<String, Object> properties,
            ExitCriteriaModel exitCriteriaModel); @Override byte[] getStateConfigZip(); @Override void preClusterManagerStartRecipes(GatewayConfig gatewayConfig, Set<Node> allNodes, ExitCriteriaModel exitCriteriaModel); @Override void postClusterManagerStartRecipes(GatewayConfig gatewayConfig, Set<Node> allNodes, ExitCriteriaModel exitCriteriaModel); @Override void postInstallRecipes(GatewayConfig gatewayConfig, Set<Node> allNodes, ExitCriteriaModel exitCriteriaModel); @Override void preTerminationRecipes(GatewayConfig gatewayConfig, Set<Node> allNodes, ExitCriteriaModel exitCriteriaModel, boolean forced); @Override void stopClusterManagerAgent(GatewayConfig gatewayConfig, Set<Node> nodes, ExitCriteriaModel exitCriteriaModel, boolean adJoinable,
            boolean ipaJoinable, boolean forced); @Override void uploadGatewayPillar(List<GatewayConfig> allGatewayConfigs, Set<Node> allNodes, ExitCriteriaModel exitModel, SaltConfig saltConfig); void leaveDomain(GatewayConfig gatewayConfig, Set<Node> allNodes, String roleToRemove, String roleToAdd, ExitCriteriaModel exitCriteriaModel); @Override List<String> getMissingNodes(GatewayConfig gatewayConfig, Set<Node> nodes); @Override List<String> getAvailableNodes(GatewayConfig gatewayConfig, Set<Node> nodes); @Override boolean isBootstrapApiAvailable(GatewayConfig gatewayConfig); @Override String name(); @Override Map<String, String> getMembers(GatewayConfig gatewayConfig, List<String> privateIps); @Override void backupDatabase(GatewayConfig primaryGateway, Set<String> target, Set<Node> allNodes, SaltConfig saltConfig,
            ExitCriteriaModel exitModel); @Override void restoreDatabase(GatewayConfig primaryGateway, Set<String> target, Set<Node> allNodes, SaltConfig saltConfig,
                                ExitCriteriaModel exitModel);  }