@Test public void minionDownTest() throws Exception { String liveHostId = "livehost"; String replicaHostId = "replicahost"; String emptyHostId = "emptyhost"; spawn.hostManager.updateHostState(makeHostState("otherhost", true)); spawn.hostManager.updateHostState(makeHostState(emptyHostId, true)); HostState host1 = makeHostState(liveHostId, true); Job job = makeSingleTaskJob(liveHostId, replicaHostId); JobKey[] jobKeys = new JobKey[]{new JobKey(job.getId(), 0)}; host1.setStopped(jobKeys); spawn.hostManager.updateHostState(host1); HostState host2 = makeHostState("downhost", false); host2.setReplicas(jobKeys); spawn.hostManager.updateHostState(host2); assertTrue("should not be able to fail host with task that is also on a down host", !hostFailWorker.checkHostStatesForFailure(liveHostId)); assertTrue("should be able to fail empty host", hostFailWorker.checkHostStatesForFailure(emptyHostId)); }
protected boolean checkHostStatesForFailure(String failedHostUuid) { Collection<HostState> hostStates = spawn.hostManager.listHostStatus(null); for (HostState hostState : hostStates) { if (!failedHostUuid.equals(hostState.getHostUuid()) && shouldBlockHostFailure(ImmutableSet.of(failedHostUuid), hostState)) { log.warn("Unable to fail host: " + failedHostUuid + " because one of the minions (" + hostState.getHostUuid() + ") on " + hostState.getHost() + " is currently down. Retry when all minions are available"); return false; } } return true; }
HostFailWorker { protected boolean checkHostStatesForFailure(String failedHostUuid) { Collection<HostState> hostStates = spawn.hostManager.listHostStatus(null); for (HostState hostState : hostStates) { if (!failedHostUuid.equals(hostState.getHostUuid()) && shouldBlockHostFailure(ImmutableSet.of(failedHostUuid), hostState)) { log.warn("Unable to fail host: " + failedHostUuid + " because one of the minions (" + hostState.getHostUuid() + ") on " + hostState.getHost() + " is currently down. Retry when all minions are available"); return false; } } return true; } }
HostFailWorker { protected boolean checkHostStatesForFailure(String failedHostUuid) { Collection<HostState> hostStates = spawn.hostManager.listHostStatus(null); for (HostState hostState : hostStates) { if (!failedHostUuid.equals(hostState.getHostUuid()) && shouldBlockHostFailure(ImmutableSet.of(failedHostUuid), hostState)) { log.warn("Unable to fail host: " + failedHostUuid + " because one of the minions (" + hostState.getHostUuid() + ") on " + hostState.getHost() + " is currently down. Retry when all minions are available"); return false; } } return true; } HostFailWorker(Spawn spawn, ScheduledExecutorService executorService); }
HostFailWorker { protected boolean checkHostStatesForFailure(String failedHostUuid) { Collection<HostState> hostStates = spawn.hostManager.listHostStatus(null); for (HostState hostState : hostStates) { if (!failedHostUuid.equals(hostState.getHostUuid()) && shouldBlockHostFailure(ImmutableSet.of(failedHostUuid), hostState)) { log.warn("Unable to fail host: " + failedHostUuid + " because one of the minions (" + hostState.getHostUuid() + ") on " + hostState.getHost() + " is currently down. Retry when all minions are available"); return false; } } return true; } HostFailWorker(Spawn spawn, ScheduledExecutorService executorService); void initFailHostTaskSchedule(); void markHostsToFail(String hostIds, FailState state); Set<String> queuedHosts(); FailState getFailureState(String hostId); boolean shouldKickTasks(String hostId); String getFailureStateString(String hostId, boolean up); void removeHostsForFailure(String hostIds); void setObeyTaskSlots(boolean obey); JSONObject getInfoForHostFailure(String hostsToFail, boolean deadFilesystem); void updateFullMinions(); }
HostFailWorker { protected boolean checkHostStatesForFailure(String failedHostUuid) { Collection<HostState> hostStates = spawn.hostManager.listHostStatus(null); for (HostState hostState : hostStates) { if (!failedHostUuid.equals(hostState.getHostUuid()) && shouldBlockHostFailure(ImmutableSet.of(failedHostUuid), hostState)) { log.warn("Unable to fail host: " + failedHostUuid + " because one of the minions (" + hostState.getHostUuid() + ") on " + hostState.getHost() + " is currently down. Retry when all minions are available"); return false; } } return true; } HostFailWorker(Spawn spawn, ScheduledExecutorService executorService); void initFailHostTaskSchedule(); void markHostsToFail(String hostIds, FailState state); Set<String> queuedHosts(); FailState getFailureState(String hostId); boolean shouldKickTasks(String hostId); String getFailureStateString(String hostId, boolean up); void removeHostsForFailure(String hostIds); void setObeyTaskSlots(boolean obey); JSONObject getInfoForHostFailure(String hostsToFail, boolean deadFilesystem); void updateFullMinions();  }