@Test public void testNoReplacementNoSpotAvailable() throws Exception { List<PriamInstance> allInstances = getInstances(1); List<String> racMembership = getRacMembership(1); racMembership.add(instanceInfo.getInstanceId()); new Expectations() { { factory.getAllIds(anyString); result = allInstances; membership.getRacMembership(); result = racMembership; } }; deadTokenRetriever = new DeadTokenRetriever( factory, membership, configuration, new FakeSleeper(), instanceInfo); PriamInstance priamInstance = deadTokenRetriever.get(); Assert.assertNull(priamInstance); new Verifications() { { factory.delete(withAny(priamInstance)); times = 0; } }; }
@Override public PriamInstance get() throws Exception { logger.info("Looking for a token from any dead node"); final List<PriamInstance> allInstancesWithinCluster = factory.getAllIds(config.getAppName()); List<String> asgInstances = membership.getRacMembership(); if (config.isDualAccount()) { asgInstances = getDualAccountRacMembership(asgInstances); } else { logger.info("Single Account cluster"); } sleeper.sleep(new Random().nextInt(5000) + 10000); logger.info( "About to iterate through all instances within cluster " + config.getAppName() + " to find an available token to acquire."); for (PriamInstance priamInstance : allInstancesWithinCluster) { if (!priamInstance.getRac().equals(instanceInfo.getRac()) || asgInstances.contains(priamInstance.getInstanceId()) || super.isInstanceDummy(priamInstance)) continue; logger.info("Found dead instance: {}", priamInstance.toString()); PriamInstance markAsDead = factory.create( priamInstance.getApp() + "-dead", priamInstance.getId(), priamInstance.getInstanceId(), priamInstance.getHostName(), priamInstance.getHostIP(), priamInstance.getRac(), priamInstance.getVolumes(), priamInstance.getToken()); factory.delete(priamInstance); TokenRetrieverUtils.InferredTokenOwnership inferredTokenInformation = TokenRetrieverUtils.inferTokenOwnerFromGossip( allInstancesWithinCluster, priamInstance.getToken(), priamInstance.getDC()); switch (inferredTokenInformation.getTokenInformationStatus()) { case GOOD: if (inferredTokenInformation.getTokenInformation() == null) { logger.error( "If you see this message, it should not have happened. We expect token ownership information if all nodes agree. This is a code bounty issue."); return null; } if (inferredTokenInformation.getTokenInformation().isLive()) { logger.info( "This token is considered alive unanimously! We will not replace this instance."); return null; } else this.replacedIp = inferredTokenInformation.getTokenInformation().getIpAddress(); break; case UNREACHABLE_NODES: this.replacedIp = priamInstance.getHostIP(); logger.warn( "Unable to reach sufficient nodes. Please check security group permissions or there might be a network partition."); logger.info( "Will try to replace token: {} with replacedIp from Token database: {}", priamInstance.getToken(), priamInstance.getHostIP()); break; case MISMATCH: logger.info( "Mismatch in gossip. We will not replace this instance, until gossip settles down."); return null; default: throw new IllegalStateException( "Unexpected value: " + inferredTokenInformation.getTokenInformationStatus()); } PriamInstance result; try { result = factory.create( config.getAppName(), markAsDead.getId(), instanceInfo.getInstanceId(), instanceInfo.getHostname(), instanceInfo.getHostIP(), instanceInfo.getRac(), markAsDead.getVolumes(), markAsDead.getToken()); } catch (Exception ex) { long sleepTime = super.getSleepTime(); logger.warn( "Exception when acquiring dead token: " + priamInstance.getToken() + " , will sleep for " + sleepTime + " millisecs before we retry."); Thread.sleep(sleepTime); throw ex; } logger.info( "Acquired token: " + priamInstance.getToken() + " and we will replace with replacedIp: " + replacedIp); return result; } logger.info("This node was NOT able to acquire any dead token"); return null; }
DeadTokenRetriever extends TokenRetrieverBase implements IDeadTokenRetriever { @Override public PriamInstance get() throws Exception { logger.info("Looking for a token from any dead node"); final List<PriamInstance> allInstancesWithinCluster = factory.getAllIds(config.getAppName()); List<String> asgInstances = membership.getRacMembership(); if (config.isDualAccount()) { asgInstances = getDualAccountRacMembership(asgInstances); } else { logger.info("Single Account cluster"); } sleeper.sleep(new Random().nextInt(5000) + 10000); logger.info( "About to iterate through all instances within cluster " + config.getAppName() + " to find an available token to acquire."); for (PriamInstance priamInstance : allInstancesWithinCluster) { if (!priamInstance.getRac().equals(instanceInfo.getRac()) || asgInstances.contains(priamInstance.getInstanceId()) || super.isInstanceDummy(priamInstance)) continue; logger.info("Found dead instance: {}", priamInstance.toString()); PriamInstance markAsDead = factory.create( priamInstance.getApp() + "-dead", priamInstance.getId(), priamInstance.getInstanceId(), priamInstance.getHostName(), priamInstance.getHostIP(), priamInstance.getRac(), priamInstance.getVolumes(), priamInstance.getToken()); factory.delete(priamInstance); TokenRetrieverUtils.InferredTokenOwnership inferredTokenInformation = TokenRetrieverUtils.inferTokenOwnerFromGossip( allInstancesWithinCluster, priamInstance.getToken(), priamInstance.getDC()); switch (inferredTokenInformation.getTokenInformationStatus()) { case GOOD: if (inferredTokenInformation.getTokenInformation() == null) { logger.error( "If you see this message, it should not have happened. We expect token ownership information if all nodes agree. This is a code bounty issue."); return null; } if (inferredTokenInformation.getTokenInformation().isLive()) { logger.info( "This token is considered alive unanimously! We will not replace this instance."); return null; } else this.replacedIp = inferredTokenInformation.getTokenInformation().getIpAddress(); break; case UNREACHABLE_NODES: this.replacedIp = priamInstance.getHostIP(); logger.warn( "Unable to reach sufficient nodes. Please check security group permissions or there might be a network partition."); logger.info( "Will try to replace token: {} with replacedIp from Token database: {}", priamInstance.getToken(), priamInstance.getHostIP()); break; case MISMATCH: logger.info( "Mismatch in gossip. We will not replace this instance, until gossip settles down."); return null; default: throw new IllegalStateException( "Unexpected value: " + inferredTokenInformation.getTokenInformationStatus()); } PriamInstance result; try { result = factory.create( config.getAppName(), markAsDead.getId(), instanceInfo.getInstanceId(), instanceInfo.getHostname(), instanceInfo.getHostIP(), instanceInfo.getRac(), markAsDead.getVolumes(), markAsDead.getToken()); } catch (Exception ex) { long sleepTime = super.getSleepTime(); logger.warn( "Exception when acquiring dead token: " + priamInstance.getToken() + " , will sleep for " + sleepTime + " millisecs before we retry."); Thread.sleep(sleepTime); throw ex; } logger.info( "Acquired token: " + priamInstance.getToken() + " and we will replace with replacedIp: " + replacedIp); return result; } logger.info("This node was NOT able to acquire any dead token"); return null; } }
DeadTokenRetriever extends TokenRetrieverBase implements IDeadTokenRetriever { @Override public PriamInstance get() throws Exception { logger.info("Looking for a token from any dead node"); final List<PriamInstance> allInstancesWithinCluster = factory.getAllIds(config.getAppName()); List<String> asgInstances = membership.getRacMembership(); if (config.isDualAccount()) { asgInstances = getDualAccountRacMembership(asgInstances); } else { logger.info("Single Account cluster"); } sleeper.sleep(new Random().nextInt(5000) + 10000); logger.info( "About to iterate through all instances within cluster " + config.getAppName() + " to find an available token to acquire."); for (PriamInstance priamInstance : allInstancesWithinCluster) { if (!priamInstance.getRac().equals(instanceInfo.getRac()) || asgInstances.contains(priamInstance.getInstanceId()) || super.isInstanceDummy(priamInstance)) continue; logger.info("Found dead instance: {}", priamInstance.toString()); PriamInstance markAsDead = factory.create( priamInstance.getApp() + "-dead", priamInstance.getId(), priamInstance.getInstanceId(), priamInstance.getHostName(), priamInstance.getHostIP(), priamInstance.getRac(), priamInstance.getVolumes(), priamInstance.getToken()); factory.delete(priamInstance); TokenRetrieverUtils.InferredTokenOwnership inferredTokenInformation = TokenRetrieverUtils.inferTokenOwnerFromGossip( allInstancesWithinCluster, priamInstance.getToken(), priamInstance.getDC()); switch (inferredTokenInformation.getTokenInformationStatus()) { case GOOD: if (inferredTokenInformation.getTokenInformation() == null) { logger.error( "If you see this message, it should not have happened. We expect token ownership information if all nodes agree. This is a code bounty issue."); return null; } if (inferredTokenInformation.getTokenInformation().isLive()) { logger.info( "This token is considered alive unanimously! We will not replace this instance."); return null; } else this.replacedIp = inferredTokenInformation.getTokenInformation().getIpAddress(); break; case UNREACHABLE_NODES: this.replacedIp = priamInstance.getHostIP(); logger.warn( "Unable to reach sufficient nodes. Please check security group permissions or there might be a network partition."); logger.info( "Will try to replace token: {} with replacedIp from Token database: {}", priamInstance.getToken(), priamInstance.getHostIP()); break; case MISMATCH: logger.info( "Mismatch in gossip. We will not replace this instance, until gossip settles down."); return null; default: throw new IllegalStateException( "Unexpected value: " + inferredTokenInformation.getTokenInformationStatus()); } PriamInstance result; try { result = factory.create( config.getAppName(), markAsDead.getId(), instanceInfo.getInstanceId(), instanceInfo.getHostname(), instanceInfo.getHostIP(), instanceInfo.getRac(), markAsDead.getVolumes(), markAsDead.getToken()); } catch (Exception ex) { long sleepTime = super.getSleepTime(); logger.warn( "Exception when acquiring dead token: " + priamInstance.getToken() + " , will sleep for " + sleepTime + " millisecs before we retry."); Thread.sleep(sleepTime); throw ex; } logger.info( "Acquired token: " + priamInstance.getToken() + " and we will replace with replacedIp: " + replacedIp); return result; } logger.info("This node was NOT able to acquire any dead token"); return null; } @Inject  DeadTokenRetriever(
            IPriamInstanceFactory factory,
            IMembership membership,
            IConfiguration config,
            Sleeper sleeper,
            InstanceInfo instanceInfo); }
DeadTokenRetriever extends TokenRetrieverBase implements IDeadTokenRetriever { @Override public PriamInstance get() throws Exception { logger.info("Looking for a token from any dead node"); final List<PriamInstance> allInstancesWithinCluster = factory.getAllIds(config.getAppName()); List<String> asgInstances = membership.getRacMembership(); if (config.isDualAccount()) { asgInstances = getDualAccountRacMembership(asgInstances); } else { logger.info("Single Account cluster"); } sleeper.sleep(new Random().nextInt(5000) + 10000); logger.info( "About to iterate through all instances within cluster " + config.getAppName() + " to find an available token to acquire."); for (PriamInstance priamInstance : allInstancesWithinCluster) { if (!priamInstance.getRac().equals(instanceInfo.getRac()) || asgInstances.contains(priamInstance.getInstanceId()) || super.isInstanceDummy(priamInstance)) continue; logger.info("Found dead instance: {}", priamInstance.toString()); PriamInstance markAsDead = factory.create( priamInstance.getApp() + "-dead", priamInstance.getId(), priamInstance.getInstanceId(), priamInstance.getHostName(), priamInstance.getHostIP(), priamInstance.getRac(), priamInstance.getVolumes(), priamInstance.getToken()); factory.delete(priamInstance); TokenRetrieverUtils.InferredTokenOwnership inferredTokenInformation = TokenRetrieverUtils.inferTokenOwnerFromGossip( allInstancesWithinCluster, priamInstance.getToken(), priamInstance.getDC()); switch (inferredTokenInformation.getTokenInformationStatus()) { case GOOD: if (inferredTokenInformation.getTokenInformation() == null) { logger.error( "If you see this message, it should not have happened. We expect token ownership information if all nodes agree. This is a code bounty issue."); return null; } if (inferredTokenInformation.getTokenInformation().isLive()) { logger.info( "This token is considered alive unanimously! We will not replace this instance."); return null; } else this.replacedIp = inferredTokenInformation.getTokenInformation().getIpAddress(); break; case UNREACHABLE_NODES: this.replacedIp = priamInstance.getHostIP(); logger.warn( "Unable to reach sufficient nodes. Please check security group permissions or there might be a network partition."); logger.info( "Will try to replace token: {} with replacedIp from Token database: {}", priamInstance.getToken(), priamInstance.getHostIP()); break; case MISMATCH: logger.info( "Mismatch in gossip. We will not replace this instance, until gossip settles down."); return null; default: throw new IllegalStateException( "Unexpected value: " + inferredTokenInformation.getTokenInformationStatus()); } PriamInstance result; try { result = factory.create( config.getAppName(), markAsDead.getId(), instanceInfo.getInstanceId(), instanceInfo.getHostname(), instanceInfo.getHostIP(), instanceInfo.getRac(), markAsDead.getVolumes(), markAsDead.getToken()); } catch (Exception ex) { long sleepTime = super.getSleepTime(); logger.warn( "Exception when acquiring dead token: " + priamInstance.getToken() + " , will sleep for " + sleepTime + " millisecs before we retry."); Thread.sleep(sleepTime); throw ex; } logger.info( "Acquired token: " + priamInstance.getToken() + " and we will replace with replacedIp: " + replacedIp); return result; } logger.info("This node was NOT able to acquire any dead token"); return null; } @Inject  DeadTokenRetriever(
            IPriamInstanceFactory factory,
            IMembership membership,
            IConfiguration config,
            Sleeper sleeper,
            InstanceInfo instanceInfo); @Override PriamInstance get(); @Override String getReplaceIp(); @Override void setLocMap(ListMultimap<String, PriamInstance> locMap); }
DeadTokenRetriever extends TokenRetrieverBase implements IDeadTokenRetriever { @Override public PriamInstance get() throws Exception { logger.info("Looking for a token from any dead node"); final List<PriamInstance> allInstancesWithinCluster = factory.getAllIds(config.getAppName()); List<String> asgInstances = membership.getRacMembership(); if (config.isDualAccount()) { asgInstances = getDualAccountRacMembership(asgInstances); } else { logger.info("Single Account cluster"); } sleeper.sleep(new Random().nextInt(5000) + 10000); logger.info( "About to iterate through all instances within cluster " + config.getAppName() + " to find an available token to acquire."); for (PriamInstance priamInstance : allInstancesWithinCluster) { if (!priamInstance.getRac().equals(instanceInfo.getRac()) || asgInstances.contains(priamInstance.getInstanceId()) || super.isInstanceDummy(priamInstance)) continue; logger.info("Found dead instance: {}", priamInstance.toString()); PriamInstance markAsDead = factory.create( priamInstance.getApp() + "-dead", priamInstance.getId(), priamInstance.getInstanceId(), priamInstance.getHostName(), priamInstance.getHostIP(), priamInstance.getRac(), priamInstance.getVolumes(), priamInstance.getToken()); factory.delete(priamInstance); TokenRetrieverUtils.InferredTokenOwnership inferredTokenInformation = TokenRetrieverUtils.inferTokenOwnerFromGossip( allInstancesWithinCluster, priamInstance.getToken(), priamInstance.getDC()); switch (inferredTokenInformation.getTokenInformationStatus()) { case GOOD: if (inferredTokenInformation.getTokenInformation() == null) { logger.error( "If you see this message, it should not have happened. We expect token ownership information if all nodes agree. This is a code bounty issue."); return null; } if (inferredTokenInformation.getTokenInformation().isLive()) { logger.info( "This token is considered alive unanimously! We will not replace this instance."); return null; } else this.replacedIp = inferredTokenInformation.getTokenInformation().getIpAddress(); break; case UNREACHABLE_NODES: this.replacedIp = priamInstance.getHostIP(); logger.warn( "Unable to reach sufficient nodes. Please check security group permissions or there might be a network partition."); logger.info( "Will try to replace token: {} with replacedIp from Token database: {}", priamInstance.getToken(), priamInstance.getHostIP()); break; case MISMATCH: logger.info( "Mismatch in gossip. We will not replace this instance, until gossip settles down."); return null; default: throw new IllegalStateException( "Unexpected value: " + inferredTokenInformation.getTokenInformationStatus()); } PriamInstance result; try { result = factory.create( config.getAppName(), markAsDead.getId(), instanceInfo.getInstanceId(), instanceInfo.getHostname(), instanceInfo.getHostIP(), instanceInfo.getRac(), markAsDead.getVolumes(), markAsDead.getToken()); } catch (Exception ex) { long sleepTime = super.getSleepTime(); logger.warn( "Exception when acquiring dead token: " + priamInstance.getToken() + " , will sleep for " + sleepTime + " millisecs before we retry."); Thread.sleep(sleepTime); throw ex; } logger.info( "Acquired token: " + priamInstance.getToken() + " and we will replace with replacedIp: " + replacedIp); return result; } logger.info("This node was NOT able to acquire any dead token"); return null; } @Inject  DeadTokenRetriever(
            IPriamInstanceFactory factory,
            IMembership membership,
            IConfiguration config,
            Sleeper sleeper,
            InstanceInfo instanceInfo); @Override PriamInstance get(); @Override String getReplaceIp(); @Override void setLocMap(ListMultimap<String, PriamInstance> locMap);  }