@Test public void testBalanceCluster() throws Exception { for (int[] mockCluster : clusterStateMocks) { Map<ServerName, List<HRegionInfo>> servers = mockClusterServers(mockCluster); List<ServerAndLoad> list = convertToList(servers); LOG.info("Mock Cluster : " + printMock(list) + " " + printStats(list)); List<RegionPlan> plans = loadBalancer.balanceCluster(servers); List<ServerAndLoad> balancedCluster = reconcile(list, plans, servers); LOG.info("Mock Balance : " + printMock(balancedCluster)); assertClusterAsBalanced(balancedCluster); List<RegionPlan> secondPlans = loadBalancer.balanceCluster(servers); assertNull(secondPlans); for (Map.Entry<ServerName, List<HRegionInfo>> entry : servers.entrySet()) { returnRegions(entry.getValue()); returnServer(entry.getKey()); } } }
@Override public List<RegionPlan> balanceCluster(Map<ServerName, List<HRegionInfo>> clusterState) { List<RegionPlan> plans = balanceMasterRegions(clusterState); if (plans != null) { return plans; } filterExcludedServers(clusterState); if (!needsBalance(new ClusterLoadState(masterServerName, getBackupMasters(), backupMasterWeight, clusterState))) { return null; } long startTime = EnvironmentEdgeManager.currentTimeMillis(); Cluster cluster = new Cluster(masterServerName, clusterState, loads, regionFinder, getBackupMasters(), tablesOnMaster); double currentCost = computeCost(cluster, Double.MAX_VALUE); double initCost = currentCost; double newCost = currentCost; long computedMaxSteps = Math.min(this.maxSteps, ((long)cluster.numRegions * (long)this.stepsPerRegion * (long)cluster.numServers)); long step; for (step = 0; step < computedMaxSteps; step++) { int pickerIdx = RANDOM.nextInt(pickers.length); RegionPicker p = pickers[pickerIdx]; Pair<Pair<Integer, Integer>, Pair<Integer, Integer>> picks = p.pick(cluster); int leftServer = picks.getFirst().getFirst(); int leftRegion = picks.getFirst().getSecond(); int rightServer = picks.getSecond().getFirst(); int rightRegion = picks.getSecond().getSecond(); if (rightServer < 0 || leftServer < 0) { continue; } if (leftRegion < 0 && rightRegion < 0) { continue; } cluster.moveOrSwapRegion(leftServer, rightServer, leftRegion, rightRegion); newCost = computeCost(cluster, currentCost); if (newCost < currentCost) { currentCost = newCost; } else { cluster.moveOrSwapRegion(leftServer, rightServer, rightRegion, leftRegion); } if (EnvironmentEdgeManager.currentTimeMillis() - startTime > maxRunningTime) { break; } } long endTime = EnvironmentEdgeManager.currentTimeMillis(); metricsBalancer.balanceCluster(endTime - startTime); if (initCost > currentCost) { plans = createRegionPlans(cluster); if (LOG.isDebugEnabled()) { LOG.debug("Finished computing new load balance plan. Computation took " + (endTime - startTime) + "ms to try " + step + " different iterations. Found a solution that moves " + plans.size() + " regions; Going from a computed cost of " + initCost + " to a new cost of " + currentCost); } return plans; } if (LOG.isDebugEnabled()) { LOG.debug("Could not find a better load balance plan. Tried " + step + " different configurations in " + (endTime - startTime) + "ms, and did not find anything with a computed cost less than " + initCost); } return null; }
StochasticLoadBalancer extends BaseLoadBalancer { @Override public List<RegionPlan> balanceCluster(Map<ServerName, List<HRegionInfo>> clusterState) { List<RegionPlan> plans = balanceMasterRegions(clusterState); if (plans != null) { return plans; } filterExcludedServers(clusterState); if (!needsBalance(new ClusterLoadState(masterServerName, getBackupMasters(), backupMasterWeight, clusterState))) { return null; } long startTime = EnvironmentEdgeManager.currentTimeMillis(); Cluster cluster = new Cluster(masterServerName, clusterState, loads, regionFinder, getBackupMasters(), tablesOnMaster); double currentCost = computeCost(cluster, Double.MAX_VALUE); double initCost = currentCost; double newCost = currentCost; long computedMaxSteps = Math.min(this.maxSteps, ((long)cluster.numRegions * (long)this.stepsPerRegion * (long)cluster.numServers)); long step; for (step = 0; step < computedMaxSteps; step++) { int pickerIdx = RANDOM.nextInt(pickers.length); RegionPicker p = pickers[pickerIdx]; Pair<Pair<Integer, Integer>, Pair<Integer, Integer>> picks = p.pick(cluster); int leftServer = picks.getFirst().getFirst(); int leftRegion = picks.getFirst().getSecond(); int rightServer = picks.getSecond().getFirst(); int rightRegion = picks.getSecond().getSecond(); if (rightServer < 0 || leftServer < 0) { continue; } if (leftRegion < 0 && rightRegion < 0) { continue; } cluster.moveOrSwapRegion(leftServer, rightServer, leftRegion, rightRegion); newCost = computeCost(cluster, currentCost); if (newCost < currentCost) { currentCost = newCost; } else { cluster.moveOrSwapRegion(leftServer, rightServer, rightRegion, leftRegion); } if (EnvironmentEdgeManager.currentTimeMillis() - startTime > maxRunningTime) { break; } } long endTime = EnvironmentEdgeManager.currentTimeMillis(); metricsBalancer.balanceCluster(endTime - startTime); if (initCost > currentCost) { plans = createRegionPlans(cluster); if (LOG.isDebugEnabled()) { LOG.debug("Finished computing new load balance plan. Computation took " + (endTime - startTime) + "ms to try " + step + " different iterations. Found a solution that moves " + plans.size() + " regions; Going from a computed cost of " + initCost + " to a new cost of " + currentCost); } return plans; } if (LOG.isDebugEnabled()) { LOG.debug("Could not find a better load balance plan. Tried " + step + " different configurations in " + (endTime - startTime) + "ms, and did not find anything with a computed cost less than " + initCost); } return null; } }
StochasticLoadBalancer extends BaseLoadBalancer { @Override public List<RegionPlan> balanceCluster(Map<ServerName, List<HRegionInfo>> clusterState) { List<RegionPlan> plans = balanceMasterRegions(clusterState); if (plans != null) { return plans; } filterExcludedServers(clusterState); if (!needsBalance(new ClusterLoadState(masterServerName, getBackupMasters(), backupMasterWeight, clusterState))) { return null; } long startTime = EnvironmentEdgeManager.currentTimeMillis(); Cluster cluster = new Cluster(masterServerName, clusterState, loads, regionFinder, getBackupMasters(), tablesOnMaster); double currentCost = computeCost(cluster, Double.MAX_VALUE); double initCost = currentCost; double newCost = currentCost; long computedMaxSteps = Math.min(this.maxSteps, ((long)cluster.numRegions * (long)this.stepsPerRegion * (long)cluster.numServers)); long step; for (step = 0; step < computedMaxSteps; step++) { int pickerIdx = RANDOM.nextInt(pickers.length); RegionPicker p = pickers[pickerIdx]; Pair<Pair<Integer, Integer>, Pair<Integer, Integer>> picks = p.pick(cluster); int leftServer = picks.getFirst().getFirst(); int leftRegion = picks.getFirst().getSecond(); int rightServer = picks.getSecond().getFirst(); int rightRegion = picks.getSecond().getSecond(); if (rightServer < 0 || leftServer < 0) { continue; } if (leftRegion < 0 && rightRegion < 0) { continue; } cluster.moveOrSwapRegion(leftServer, rightServer, leftRegion, rightRegion); newCost = computeCost(cluster, currentCost); if (newCost < currentCost) { currentCost = newCost; } else { cluster.moveOrSwapRegion(leftServer, rightServer, rightRegion, leftRegion); } if (EnvironmentEdgeManager.currentTimeMillis() - startTime > maxRunningTime) { break; } } long endTime = EnvironmentEdgeManager.currentTimeMillis(); metricsBalancer.balanceCluster(endTime - startTime); if (initCost > currentCost) { plans = createRegionPlans(cluster); if (LOG.isDebugEnabled()) { LOG.debug("Finished computing new load balance plan. Computation took " + (endTime - startTime) + "ms to try " + step + " different iterations. Found a solution that moves " + plans.size() + " regions; Going from a computed cost of " + initCost + " to a new cost of " + currentCost); } return plans; } if (LOG.isDebugEnabled()) { LOG.debug("Could not find a better load balance plan. Tried " + step + " different configurations in " + (endTime - startTime) + "ms, and did not find anything with a computed cost less than " + initCost); } return null; }  }
StochasticLoadBalancer extends BaseLoadBalancer { @Override public List<RegionPlan> balanceCluster(Map<ServerName, List<HRegionInfo>> clusterState) { List<RegionPlan> plans = balanceMasterRegions(clusterState); if (plans != null) { return plans; } filterExcludedServers(clusterState); if (!needsBalance(new ClusterLoadState(masterServerName, getBackupMasters(), backupMasterWeight, clusterState))) { return null; } long startTime = EnvironmentEdgeManager.currentTimeMillis(); Cluster cluster = new Cluster(masterServerName, clusterState, loads, regionFinder, getBackupMasters(), tablesOnMaster); double currentCost = computeCost(cluster, Double.MAX_VALUE); double initCost = currentCost; double newCost = currentCost; long computedMaxSteps = Math.min(this.maxSteps, ((long)cluster.numRegions * (long)this.stepsPerRegion * (long)cluster.numServers)); long step; for (step = 0; step < computedMaxSteps; step++) { int pickerIdx = RANDOM.nextInt(pickers.length); RegionPicker p = pickers[pickerIdx]; Pair<Pair<Integer, Integer>, Pair<Integer, Integer>> picks = p.pick(cluster); int leftServer = picks.getFirst().getFirst(); int leftRegion = picks.getFirst().getSecond(); int rightServer = picks.getSecond().getFirst(); int rightRegion = picks.getSecond().getSecond(); if (rightServer < 0 || leftServer < 0) { continue; } if (leftRegion < 0 && rightRegion < 0) { continue; } cluster.moveOrSwapRegion(leftServer, rightServer, leftRegion, rightRegion); newCost = computeCost(cluster, currentCost); if (newCost < currentCost) { currentCost = newCost; } else { cluster.moveOrSwapRegion(leftServer, rightServer, rightRegion, leftRegion); } if (EnvironmentEdgeManager.currentTimeMillis() - startTime > maxRunningTime) { break; } } long endTime = EnvironmentEdgeManager.currentTimeMillis(); metricsBalancer.balanceCluster(endTime - startTime); if (initCost > currentCost) { plans = createRegionPlans(cluster); if (LOG.isDebugEnabled()) { LOG.debug("Finished computing new load balance plan. Computation took " + (endTime - startTime) + "ms to try " + step + " different iterations. Found a solution that moves " + plans.size() + " regions; Going from a computed cost of " + initCost + " to a new cost of " + currentCost); } return plans; } if (LOG.isDebugEnabled()) { LOG.debug("Could not find a better load balance plan. Tried " + step + " different configurations in " + (endTime - startTime) + "ms, and did not find anything with a computed cost less than " + initCost); } return null; }  @Override void setConf(Configuration conf); @Override void setClusterStatus(ClusterStatus st); @Override void setMasterServices(MasterServices masterServices); @Override List<RegionPlan> balanceCluster(Map<ServerName, List<HRegionInfo>> clusterState); }
StochasticLoadBalancer extends BaseLoadBalancer { @Override public List<RegionPlan> balanceCluster(Map<ServerName, List<HRegionInfo>> clusterState) { List<RegionPlan> plans = balanceMasterRegions(clusterState); if (plans != null) { return plans; } filterExcludedServers(clusterState); if (!needsBalance(new ClusterLoadState(masterServerName, getBackupMasters(), backupMasterWeight, clusterState))) { return null; } long startTime = EnvironmentEdgeManager.currentTimeMillis(); Cluster cluster = new Cluster(masterServerName, clusterState, loads, regionFinder, getBackupMasters(), tablesOnMaster); double currentCost = computeCost(cluster, Double.MAX_VALUE); double initCost = currentCost; double newCost = currentCost; long computedMaxSteps = Math.min(this.maxSteps, ((long)cluster.numRegions * (long)this.stepsPerRegion * (long)cluster.numServers)); long step; for (step = 0; step < computedMaxSteps; step++) { int pickerIdx = RANDOM.nextInt(pickers.length); RegionPicker p = pickers[pickerIdx]; Pair<Pair<Integer, Integer>, Pair<Integer, Integer>> picks = p.pick(cluster); int leftServer = picks.getFirst().getFirst(); int leftRegion = picks.getFirst().getSecond(); int rightServer = picks.getSecond().getFirst(); int rightRegion = picks.getSecond().getSecond(); if (rightServer < 0 || leftServer < 0) { continue; } if (leftRegion < 0 && rightRegion < 0) { continue; } cluster.moveOrSwapRegion(leftServer, rightServer, leftRegion, rightRegion); newCost = computeCost(cluster, currentCost); if (newCost < currentCost) { currentCost = newCost; } else { cluster.moveOrSwapRegion(leftServer, rightServer, rightRegion, leftRegion); } if (EnvironmentEdgeManager.currentTimeMillis() - startTime > maxRunningTime) { break; } } long endTime = EnvironmentEdgeManager.currentTimeMillis(); metricsBalancer.balanceCluster(endTime - startTime); if (initCost > currentCost) { plans = createRegionPlans(cluster); if (LOG.isDebugEnabled()) { LOG.debug("Finished computing new load balance plan. Computation took " + (endTime - startTime) + "ms to try " + step + " different iterations. Found a solution that moves " + plans.size() + " regions; Going from a computed cost of " + initCost + " to a new cost of " + currentCost); } return plans; } if (LOG.isDebugEnabled()) { LOG.debug("Could not find a better load balance plan. Tried " + step + " different configurations in " + (endTime - startTime) + "ms, and did not find anything with a computed cost less than " + initCost); } return null; }  @Override void setConf(Configuration conf); @Override void setClusterStatus(ClusterStatus st); @Override void setMasterServices(MasterServices masterServices); @Override List<RegionPlan> balanceCluster(Map<ServerName, List<HRegionInfo>> clusterState);  }