@Test public void testRetainAssignment() throws Exception { List<ServerAndLoad> servers = randomServers(10, 10); List<HRegionInfo> regions = randomRegions(100); Map<HRegionInfo, ServerName> existing = new TreeMap<HRegionInfo, ServerName>(); for (int i = 0; i < regions.size(); i++) { ServerName sn = servers.get(i % servers.size()).getServerName(); ServerName snWithOldStartCode = ServerName.valueOf(sn.getHostname(), sn.getPort(), sn.getStartcode() - 10); existing.put(regions.get(i), snWithOldStartCode); } List<ServerName> listOfServerNames = getListOfServerNames(servers); Map<ServerName, List<HRegionInfo>> assignment = loadBalancer.retainAssignment(existing, listOfServerNames); assertRetainedAssignment(existing, listOfServerNames, assignment); List<ServerAndLoad> servers2 = new ArrayList<ServerAndLoad>(servers); servers2.add(randomServer(10)); servers2.add(randomServer(10)); listOfServerNames = getListOfServerNames(servers2); assignment = loadBalancer.retainAssignment(existing, listOfServerNames); assertRetainedAssignment(existing, listOfServerNames, assignment); List<ServerAndLoad> servers3 = new ArrayList<ServerAndLoad>(servers); servers3.remove(0); servers3.remove(0); listOfServerNames = getListOfServerNames(servers3); assignment = loadBalancer.retainAssignment(existing, listOfServerNames); assertRetainedAssignment(existing, listOfServerNames, assignment); }
@Override public Map<ServerName, List<HRegionInfo>> retainAssignment(Map<HRegionInfo, ServerName> regions, List<ServerName> servers) { metricsBalancer.incrMiscInvocations(); if (regions == null || regions.isEmpty()) { return null; } List<ServerName> backupMasters = normalizeServers(servers); int numServers = servers == null ? 0 : servers.size(); int numBackupMasters = backupMasters == null ? 0 : backupMasters.size(); if (numServers == 0 && numBackupMasters == 0) { LOG.warn("Wanted to do retain assignment but no servers to assign to"); return null; } Map<ServerName, List<HRegionInfo>> assignments = new TreeMap<ServerName, List<HRegionInfo>>(); if (numServers + numBackupMasters == 1) { ServerName server = numServers > 0 ? servers.get(0) : backupMasters.get(0); assignments.put(server, new ArrayList<HRegionInfo>(regions.keySet())); return assignments; } ArrayListMultimap<String, ServerName> serversByHostname = ArrayListMultimap.create(); for (ServerName server : servers) { assignments.put(server, new ArrayList<HRegionInfo>()); if (!server.equals(masterServerName)) { serversByHostname.put(server.getHostname(), server); } } if (numBackupMasters > 0) { for (ServerName server : backupMasters) { assignments.put(server, new ArrayList<HRegionInfo>()); } } Set<String> oldHostsNoLongerPresent = Sets.newTreeSet(); boolean masterIncluded = servers.contains(masterServerName); int numRandomAssignments = 0; int numRetainedAssigments = 0; for (Map.Entry<HRegionInfo, ServerName> entry : regions.entrySet()) { HRegionInfo region = entry.getKey(); ServerName oldServerName = entry.getValue(); List<ServerName> localServers = new ArrayList<ServerName>(); if (oldServerName != null) { localServers = serversByHostname.get(oldServerName.getHostname()); } if (masterIncluded && shouldBeOnMaster(region)) { assignments.get(masterServerName).add(region); if (localServers.contains(masterServerName)) { numRetainedAssigments++; } else { numRandomAssignments++; } } else if (localServers.isEmpty()) { ServerName randomServer = randomAssignment(region, servers, backupMasters); assignments.get(randomServer).add(region); numRandomAssignments++; if (oldServerName != null) oldHostsNoLongerPresent.add(oldServerName.getHostname()); } else if (localServers.size() == 1) { assignments.get(localServers.get(0)).add(region); numRetainedAssigments++; } else { int size = localServers.size(); ServerName target = localServers.contains(oldServerName) ? oldServerName : localServers.get(RANDOM .nextInt(size)); assignments.get(target).add(region); numRetainedAssigments++; } } String randomAssignMsg = ""; if (numRandomAssignments > 0) { randomAssignMsg = numRandomAssignments + " regions were assigned " + "to random hosts, since the old hosts for these regions are no " + "longer present in the cluster. These hosts were:\n " + Joiner.on("\n ").join(oldHostsNoLongerPresent); } LOG.info("Reassigned " + regions.size() + " regions. " + numRetainedAssigments + " retained the pre-restart assignment. " + randomAssignMsg); return assignments; }
BaseLoadBalancer implements LoadBalancer { @Override public Map<ServerName, List<HRegionInfo>> retainAssignment(Map<HRegionInfo, ServerName> regions, List<ServerName> servers) { metricsBalancer.incrMiscInvocations(); if (regions == null || regions.isEmpty()) { return null; } List<ServerName> backupMasters = normalizeServers(servers); int numServers = servers == null ? 0 : servers.size(); int numBackupMasters = backupMasters == null ? 0 : backupMasters.size(); if (numServers == 0 && numBackupMasters == 0) { LOG.warn("Wanted to do retain assignment but no servers to assign to"); return null; } Map<ServerName, List<HRegionInfo>> assignments = new TreeMap<ServerName, List<HRegionInfo>>(); if (numServers + numBackupMasters == 1) { ServerName server = numServers > 0 ? servers.get(0) : backupMasters.get(0); assignments.put(server, new ArrayList<HRegionInfo>(regions.keySet())); return assignments; } ArrayListMultimap<String, ServerName> serversByHostname = ArrayListMultimap.create(); for (ServerName server : servers) { assignments.put(server, new ArrayList<HRegionInfo>()); if (!server.equals(masterServerName)) { serversByHostname.put(server.getHostname(), server); } } if (numBackupMasters > 0) { for (ServerName server : backupMasters) { assignments.put(server, new ArrayList<HRegionInfo>()); } } Set<String> oldHostsNoLongerPresent = Sets.newTreeSet(); boolean masterIncluded = servers.contains(masterServerName); int numRandomAssignments = 0; int numRetainedAssigments = 0; for (Map.Entry<HRegionInfo, ServerName> entry : regions.entrySet()) { HRegionInfo region = entry.getKey(); ServerName oldServerName = entry.getValue(); List<ServerName> localServers = new ArrayList<ServerName>(); if (oldServerName != null) { localServers = serversByHostname.get(oldServerName.getHostname()); } if (masterIncluded && shouldBeOnMaster(region)) { assignments.get(masterServerName).add(region); if (localServers.contains(masterServerName)) { numRetainedAssigments++; } else { numRandomAssignments++; } } else if (localServers.isEmpty()) { ServerName randomServer = randomAssignment(region, servers, backupMasters); assignments.get(randomServer).add(region); numRandomAssignments++; if (oldServerName != null) oldHostsNoLongerPresent.add(oldServerName.getHostname()); } else if (localServers.size() == 1) { assignments.get(localServers.get(0)).add(region); numRetainedAssigments++; } else { int size = localServers.size(); ServerName target = localServers.contains(oldServerName) ? oldServerName : localServers.get(RANDOM .nextInt(size)); assignments.get(target).add(region); numRetainedAssigments++; } } String randomAssignMsg = ""; if (numRandomAssignments > 0) { randomAssignMsg = numRandomAssignments + " regions were assigned " + "to random hosts, since the old hosts for these regions are no " + "longer present in the cluster. These hosts were:\n " + Joiner.on("\n ").join(oldHostsNoLongerPresent); } LOG.info("Reassigned " + regions.size() + " regions. " + numRetainedAssigments + " retained the pre-restart assignment. " + randomAssignMsg); return assignments; } }
BaseLoadBalancer implements LoadBalancer { @Override public Map<ServerName, List<HRegionInfo>> retainAssignment(Map<HRegionInfo, ServerName> regions, List<ServerName> servers) { metricsBalancer.incrMiscInvocations(); if (regions == null || regions.isEmpty()) { return null; } List<ServerName> backupMasters = normalizeServers(servers); int numServers = servers == null ? 0 : servers.size(); int numBackupMasters = backupMasters == null ? 0 : backupMasters.size(); if (numServers == 0 && numBackupMasters == 0) { LOG.warn("Wanted to do retain assignment but no servers to assign to"); return null; } Map<ServerName, List<HRegionInfo>> assignments = new TreeMap<ServerName, List<HRegionInfo>>(); if (numServers + numBackupMasters == 1) { ServerName server = numServers > 0 ? servers.get(0) : backupMasters.get(0); assignments.put(server, new ArrayList<HRegionInfo>(regions.keySet())); return assignments; } ArrayListMultimap<String, ServerName> serversByHostname = ArrayListMultimap.create(); for (ServerName server : servers) { assignments.put(server, new ArrayList<HRegionInfo>()); if (!server.equals(masterServerName)) { serversByHostname.put(server.getHostname(), server); } } if (numBackupMasters > 0) { for (ServerName server : backupMasters) { assignments.put(server, new ArrayList<HRegionInfo>()); } } Set<String> oldHostsNoLongerPresent = Sets.newTreeSet(); boolean masterIncluded = servers.contains(masterServerName); int numRandomAssignments = 0; int numRetainedAssigments = 0; for (Map.Entry<HRegionInfo, ServerName> entry : regions.entrySet()) { HRegionInfo region = entry.getKey(); ServerName oldServerName = entry.getValue(); List<ServerName> localServers = new ArrayList<ServerName>(); if (oldServerName != null) { localServers = serversByHostname.get(oldServerName.getHostname()); } if (masterIncluded && shouldBeOnMaster(region)) { assignments.get(masterServerName).add(region); if (localServers.contains(masterServerName)) { numRetainedAssigments++; } else { numRandomAssignments++; } } else if (localServers.isEmpty()) { ServerName randomServer = randomAssignment(region, servers, backupMasters); assignments.get(randomServer).add(region); numRandomAssignments++; if (oldServerName != null) oldHostsNoLongerPresent.add(oldServerName.getHostname()); } else if (localServers.size() == 1) { assignments.get(localServers.get(0)).add(region); numRetainedAssigments++; } else { int size = localServers.size(); ServerName target = localServers.contains(oldServerName) ? oldServerName : localServers.get(RANDOM .nextInt(size)); assignments.get(target).add(region); numRetainedAssigments++; } } String randomAssignMsg = ""; if (numRandomAssignments > 0) { randomAssignMsg = numRandomAssignments + " regions were assigned " + "to random hosts, since the old hosts for these regions are no " + "longer present in the cluster. These hosts were:\n " + Joiner.on("\n ").join(oldHostsNoLongerPresent); } LOG.info("Reassigned " + regions.size() + " regions. " + numRetainedAssigments + " retained the pre-restart assignment. " + randomAssignMsg); return assignments; }  }
BaseLoadBalancer implements LoadBalancer { @Override public Map<ServerName, List<HRegionInfo>> retainAssignment(Map<HRegionInfo, ServerName> regions, List<ServerName> servers) { metricsBalancer.incrMiscInvocations(); if (regions == null || regions.isEmpty()) { return null; } List<ServerName> backupMasters = normalizeServers(servers); int numServers = servers == null ? 0 : servers.size(); int numBackupMasters = backupMasters == null ? 0 : backupMasters.size(); if (numServers == 0 && numBackupMasters == 0) { LOG.warn("Wanted to do retain assignment but no servers to assign to"); return null; } Map<ServerName, List<HRegionInfo>> assignments = new TreeMap<ServerName, List<HRegionInfo>>(); if (numServers + numBackupMasters == 1) { ServerName server = numServers > 0 ? servers.get(0) : backupMasters.get(0); assignments.put(server, new ArrayList<HRegionInfo>(regions.keySet())); return assignments; } ArrayListMultimap<String, ServerName> serversByHostname = ArrayListMultimap.create(); for (ServerName server : servers) { assignments.put(server, new ArrayList<HRegionInfo>()); if (!server.equals(masterServerName)) { serversByHostname.put(server.getHostname(), server); } } if (numBackupMasters > 0) { for (ServerName server : backupMasters) { assignments.put(server, new ArrayList<HRegionInfo>()); } } Set<String> oldHostsNoLongerPresent = Sets.newTreeSet(); boolean masterIncluded = servers.contains(masterServerName); int numRandomAssignments = 0; int numRetainedAssigments = 0; for (Map.Entry<HRegionInfo, ServerName> entry : regions.entrySet()) { HRegionInfo region = entry.getKey(); ServerName oldServerName = entry.getValue(); List<ServerName> localServers = new ArrayList<ServerName>(); if (oldServerName != null) { localServers = serversByHostname.get(oldServerName.getHostname()); } if (masterIncluded && shouldBeOnMaster(region)) { assignments.get(masterServerName).add(region); if (localServers.contains(masterServerName)) { numRetainedAssigments++; } else { numRandomAssignments++; } } else if (localServers.isEmpty()) { ServerName randomServer = randomAssignment(region, servers, backupMasters); assignments.get(randomServer).add(region); numRandomAssignments++; if (oldServerName != null) oldHostsNoLongerPresent.add(oldServerName.getHostname()); } else if (localServers.size() == 1) { assignments.get(localServers.get(0)).add(region); numRetainedAssigments++; } else { int size = localServers.size(); ServerName target = localServers.contains(oldServerName) ? oldServerName : localServers.get(RANDOM .nextInt(size)); assignments.get(target).add(region); numRetainedAssigments++; } } String randomAssignMsg = ""; if (numRandomAssignments > 0) { randomAssignMsg = numRandomAssignments + " regions were assigned " + "to random hosts, since the old hosts for these regions are no " + "longer present in the cluster. These hosts were:\n " + Joiner.on("\n ").join(oldHostsNoLongerPresent); } LOG.info("Reassigned " + regions.size() + " regions. " + numRetainedAssigments + " retained the pre-restart assignment. " + randomAssignMsg); return assignments; }  @Override void setConf(Configuration conf); void excludeServer(ServerName serverName); Set<ServerName> getExcludedServers(); @Override Configuration getConf(); @Override void setClusterStatus(ClusterStatus st); @Override void setMasterServices(MasterServices masterServices); @Override Map<ServerName, List<HRegionInfo>> roundRobinAssignment(List<HRegionInfo> regions,
      List<ServerName> servers); @Override Map<HRegionInfo, ServerName> immediateAssignment(List<HRegionInfo> regions,
      List<ServerName> servers); @Override ServerName randomAssignment(HRegionInfo regionInfo, List<ServerName> servers); @Override Map<ServerName, List<HRegionInfo>> retainAssignment(Map<HRegionInfo, ServerName> regions,
      List<ServerName> servers); @Override void initialize(); @Override void regionOnline(HRegionInfo regionInfo, ServerName sn); @Override void regionOffline(HRegionInfo regionInfo); @Override boolean isStopped(); @Override void stop(String why); }
BaseLoadBalancer implements LoadBalancer { @Override public Map<ServerName, List<HRegionInfo>> retainAssignment(Map<HRegionInfo, ServerName> regions, List<ServerName> servers) { metricsBalancer.incrMiscInvocations(); if (regions == null || regions.isEmpty()) { return null; } List<ServerName> backupMasters = normalizeServers(servers); int numServers = servers == null ? 0 : servers.size(); int numBackupMasters = backupMasters == null ? 0 : backupMasters.size(); if (numServers == 0 && numBackupMasters == 0) { LOG.warn("Wanted to do retain assignment but no servers to assign to"); return null; } Map<ServerName, List<HRegionInfo>> assignments = new TreeMap<ServerName, List<HRegionInfo>>(); if (numServers + numBackupMasters == 1) { ServerName server = numServers > 0 ? servers.get(0) : backupMasters.get(0); assignments.put(server, new ArrayList<HRegionInfo>(regions.keySet())); return assignments; } ArrayListMultimap<String, ServerName> serversByHostname = ArrayListMultimap.create(); for (ServerName server : servers) { assignments.put(server, new ArrayList<HRegionInfo>()); if (!server.equals(masterServerName)) { serversByHostname.put(server.getHostname(), server); } } if (numBackupMasters > 0) { for (ServerName server : backupMasters) { assignments.put(server, new ArrayList<HRegionInfo>()); } } Set<String> oldHostsNoLongerPresent = Sets.newTreeSet(); boolean masterIncluded = servers.contains(masterServerName); int numRandomAssignments = 0; int numRetainedAssigments = 0; for (Map.Entry<HRegionInfo, ServerName> entry : regions.entrySet()) { HRegionInfo region = entry.getKey(); ServerName oldServerName = entry.getValue(); List<ServerName> localServers = new ArrayList<ServerName>(); if (oldServerName != null) { localServers = serversByHostname.get(oldServerName.getHostname()); } if (masterIncluded && shouldBeOnMaster(region)) { assignments.get(masterServerName).add(region); if (localServers.contains(masterServerName)) { numRetainedAssigments++; } else { numRandomAssignments++; } } else if (localServers.isEmpty()) { ServerName randomServer = randomAssignment(region, servers, backupMasters); assignments.get(randomServer).add(region); numRandomAssignments++; if (oldServerName != null) oldHostsNoLongerPresent.add(oldServerName.getHostname()); } else if (localServers.size() == 1) { assignments.get(localServers.get(0)).add(region); numRetainedAssigments++; } else { int size = localServers.size(); ServerName target = localServers.contains(oldServerName) ? oldServerName : localServers.get(RANDOM .nextInt(size)); assignments.get(target).add(region); numRetainedAssigments++; } } String randomAssignMsg = ""; if (numRandomAssignments > 0) { randomAssignMsg = numRandomAssignments + " regions were assigned " + "to random hosts, since the old hosts for these regions are no " + "longer present in the cluster. These hosts were:\n " + Joiner.on("\n ").join(oldHostsNoLongerPresent); } LOG.info("Reassigned " + regions.size() + " regions. " + numRetainedAssigments + " retained the pre-restart assignment. " + randomAssignMsg); return assignments; }  @Override void setConf(Configuration conf); void excludeServer(ServerName serverName); Set<ServerName> getExcludedServers(); @Override Configuration getConf(); @Override void setClusterStatus(ClusterStatus st); @Override void setMasterServices(MasterServices masterServices); @Override Map<ServerName, List<HRegionInfo>> roundRobinAssignment(List<HRegionInfo> regions,
      List<ServerName> servers); @Override Map<HRegionInfo, ServerName> immediateAssignment(List<HRegionInfo> regions,
      List<ServerName> servers); @Override ServerName randomAssignment(HRegionInfo regionInfo, List<ServerName> servers); @Override Map<ServerName, List<HRegionInfo>> retainAssignment(Map<HRegionInfo, ServerName> regions,
      List<ServerName> servers); @Override void initialize(); @Override void regionOnline(HRegionInfo regionInfo, ServerName sn); @Override void regionOffline(HRegionInfo regionInfo); @Override boolean isStopped(); @Override void stop(String why); static final String BACKUP_MASTER_WEIGHT_KEY; static final int DEFAULT_BACKUP_MASTER_WEIGHT; }