@Test public void failWarningTest() throws JSONException { spawn.hostManager.updateHostState(makeHostState("a", true, 1000, 2000)); spawn.hostManager.updateHostState(makeHostState("b", true, 500, 2000)); spawn.hostManager.updateHostState(makeHostState("c", true, 1500, 2000)); JSONObject failBMessage = hostFailWorker.getInfoForHostFailure("b", true); assertTrue("should have ~.5 disk usage before fail", failBMessage.get("prefail") != null && areClose(failBMessage.getDouble("prefail"), .5)); assertTrue("should have ~.75 disk usage after fail", failBMessage.get("postfail") != null && areClose(failBMessage.getDouble("postfail"), .75)); spawn.hostManager.updateHostState(makeHostState("d", true, 10000, 11000)); JSONObject failDMessage = hostFailWorker.getInfoForHostFailure("d", true); assertTrue("should get fatal warning after failing too-big host", failDMessage.has("fatal")); }
public JSONObject getInfoForHostFailure(String hostsToFail, boolean deadFilesystem) throws JSONException { if (hostsToFail == null) { return new JSONObject(); } HashSet<String> ids = new HashSet<>(Arrays.asList(hostsToFail.split(","))); long totalClusterAvail = 0, totalClusterUsed = 0, hostAvail = 0; List<String> hostsDown = new ArrayList<>(); for (HostState host : spawn.hostManager.listHostStatus(null)) { if (host.getMax() != null && host.getUsed() != null) { if (getFailureState(host.getHostUuid()) == FailState.ALIVE) { totalClusterAvail += host.getMax().getDisk(); totalClusterUsed += host.getUsed().getDisk(); } if (ids.contains(host.getHostUuid())) { hostAvail += host.getMax().getDisk(); } } if (!ids.contains(host.getHostUuid()) && shouldBlockHostFailure(ids, host)) { hostsDown.add(host.getHostUuid() + " on " + host.getHost()); } } totalClusterAvail = Math.max(1, totalClusterAvail); hostAvail = Math.min(totalClusterAvail - 1, hostAvail); return constructInfoMessage(hostsToFail, deadFilesystem, (double) (totalClusterUsed) / totalClusterAvail, (double) (totalClusterUsed) / (totalClusterAvail - hostAvail), hostsDown); }
HostFailWorker { public JSONObject getInfoForHostFailure(String hostsToFail, boolean deadFilesystem) throws JSONException { if (hostsToFail == null) { return new JSONObject(); } HashSet<String> ids = new HashSet<>(Arrays.asList(hostsToFail.split(","))); long totalClusterAvail = 0, totalClusterUsed = 0, hostAvail = 0; List<String> hostsDown = new ArrayList<>(); for (HostState host : spawn.hostManager.listHostStatus(null)) { if (host.getMax() != null && host.getUsed() != null) { if (getFailureState(host.getHostUuid()) == FailState.ALIVE) { totalClusterAvail += host.getMax().getDisk(); totalClusterUsed += host.getUsed().getDisk(); } if (ids.contains(host.getHostUuid())) { hostAvail += host.getMax().getDisk(); } } if (!ids.contains(host.getHostUuid()) && shouldBlockHostFailure(ids, host)) { hostsDown.add(host.getHostUuid() + " on " + host.getHost()); } } totalClusterAvail = Math.max(1, totalClusterAvail); hostAvail = Math.min(totalClusterAvail - 1, hostAvail); return constructInfoMessage(hostsToFail, deadFilesystem, (double) (totalClusterUsed) / totalClusterAvail, (double) (totalClusterUsed) / (totalClusterAvail - hostAvail), hostsDown); } }
HostFailWorker { public JSONObject getInfoForHostFailure(String hostsToFail, boolean deadFilesystem) throws JSONException { if (hostsToFail == null) { return new JSONObject(); } HashSet<String> ids = new HashSet<>(Arrays.asList(hostsToFail.split(","))); long totalClusterAvail = 0, totalClusterUsed = 0, hostAvail = 0; List<String> hostsDown = new ArrayList<>(); for (HostState host : spawn.hostManager.listHostStatus(null)) { if (host.getMax() != null && host.getUsed() != null) { if (getFailureState(host.getHostUuid()) == FailState.ALIVE) { totalClusterAvail += host.getMax().getDisk(); totalClusterUsed += host.getUsed().getDisk(); } if (ids.contains(host.getHostUuid())) { hostAvail += host.getMax().getDisk(); } } if (!ids.contains(host.getHostUuid()) && shouldBlockHostFailure(ids, host)) { hostsDown.add(host.getHostUuid() + " on " + host.getHost()); } } totalClusterAvail = Math.max(1, totalClusterAvail); hostAvail = Math.min(totalClusterAvail - 1, hostAvail); return constructInfoMessage(hostsToFail, deadFilesystem, (double) (totalClusterUsed) / totalClusterAvail, (double) (totalClusterUsed) / (totalClusterAvail - hostAvail), hostsDown); } HostFailWorker(Spawn spawn, ScheduledExecutorService executorService); }
HostFailWorker { public JSONObject getInfoForHostFailure(String hostsToFail, boolean deadFilesystem) throws JSONException { if (hostsToFail == null) { return new JSONObject(); } HashSet<String> ids = new HashSet<>(Arrays.asList(hostsToFail.split(","))); long totalClusterAvail = 0, totalClusterUsed = 0, hostAvail = 0; List<String> hostsDown = new ArrayList<>(); for (HostState host : spawn.hostManager.listHostStatus(null)) { if (host.getMax() != null && host.getUsed() != null) { if (getFailureState(host.getHostUuid()) == FailState.ALIVE) { totalClusterAvail += host.getMax().getDisk(); totalClusterUsed += host.getUsed().getDisk(); } if (ids.contains(host.getHostUuid())) { hostAvail += host.getMax().getDisk(); } } if (!ids.contains(host.getHostUuid()) && shouldBlockHostFailure(ids, host)) { hostsDown.add(host.getHostUuid() + " on " + host.getHost()); } } totalClusterAvail = Math.max(1, totalClusterAvail); hostAvail = Math.min(totalClusterAvail - 1, hostAvail); return constructInfoMessage(hostsToFail, deadFilesystem, (double) (totalClusterUsed) / totalClusterAvail, (double) (totalClusterUsed) / (totalClusterAvail - hostAvail), hostsDown); } HostFailWorker(Spawn spawn, ScheduledExecutorService executorService); void initFailHostTaskSchedule(); void markHostsToFail(String hostIds, FailState state); Set<String> queuedHosts(); FailState getFailureState(String hostId); boolean shouldKickTasks(String hostId); String getFailureStateString(String hostId, boolean up); void removeHostsForFailure(String hostIds); void setObeyTaskSlots(boolean obey); JSONObject getInfoForHostFailure(String hostsToFail, boolean deadFilesystem); void updateFullMinions(); }
HostFailWorker { public JSONObject getInfoForHostFailure(String hostsToFail, boolean deadFilesystem) throws JSONException { if (hostsToFail == null) { return new JSONObject(); } HashSet<String> ids = new HashSet<>(Arrays.asList(hostsToFail.split(","))); long totalClusterAvail = 0, totalClusterUsed = 0, hostAvail = 0; List<String> hostsDown = new ArrayList<>(); for (HostState host : spawn.hostManager.listHostStatus(null)) { if (host.getMax() != null && host.getUsed() != null) { if (getFailureState(host.getHostUuid()) == FailState.ALIVE) { totalClusterAvail += host.getMax().getDisk(); totalClusterUsed += host.getUsed().getDisk(); } if (ids.contains(host.getHostUuid())) { hostAvail += host.getMax().getDisk(); } } if (!ids.contains(host.getHostUuid()) && shouldBlockHostFailure(ids, host)) { hostsDown.add(host.getHostUuid() + " on " + host.getHost()); } } totalClusterAvail = Math.max(1, totalClusterAvail); hostAvail = Math.min(totalClusterAvail - 1, hostAvail); return constructInfoMessage(hostsToFail, deadFilesystem, (double) (totalClusterUsed) / totalClusterAvail, (double) (totalClusterUsed) / (totalClusterAvail - hostAvail), hostsDown); } HostFailWorker(Spawn spawn, ScheduledExecutorService executorService); void initFailHostTaskSchedule(); void markHostsToFail(String hostIds, FailState state); Set<String> queuedHosts(); FailState getFailureState(String hostId); boolean shouldKickTasks(String hostId); String getFailureStateString(String hostId, boolean up); void removeHostsForFailure(String hostIds); void setObeyTaskSlots(boolean obey); JSONObject getInfoForHostFailure(String hostsToFail, boolean deadFilesystem); void updateFullMinions();  }