@Test(groups = {"gobblin.yarn", "disabledOnTravis"}, dependsOnMethods = "testScaleDownWithInUseInstances") public void testScaleDown() throws Exception { this.yarnService.requestTargetNumberOfContainers(4, Collections.EMPTY_SET); Assert.assertEquals(this.yarnService.getNumRequestedContainers(), 4); Assert.assertTrue(this.yarnService.waitForContainerCount(4, 60000)); }
public synchronized void requestTargetNumberOfContainers(int numTargetContainers, Set<String> inUseInstances) { LOGGER.debug("Requesting numTargetContainers {} current numRequestedContainers {} in use instances {} map size {}", numTargetContainers, this.numRequestedContainers, inUseInstances, this.containerMap.size()); int numAllocatedContainers = this.containerMap.size(); int numContainers = Math.max(numRequestedContainers, numAllocatedContainers); for (int i = numContainers; i < numTargetContainers; i++) { requestContainer(Optional.<String>absent()); } if (numTargetContainers < numAllocatedContainers) { LOGGER.debug("Shrinking number of containers by {}", (numAllocatedContainers - numTargetContainers)); List<Container> containersToRelease = new ArrayList<>(); int numToShutdown = numContainers - numTargetContainers; for (Map.Entry<ContainerId, Map.Entry<Container, String>> entry : this.containerMap.entrySet()) { if (!inUseInstances.contains(entry.getValue().getValue())) { containersToRelease.add(entry.getValue().getKey()); } if (containersToRelease.size() == numToShutdown) { break; } } LOGGER.debug("Shutting down containers {}", containersToRelease); this.eventBus.post(new ContainerReleaseRequest(containersToRelease)); } this.numRequestedContainers = numTargetContainers; }
YarnService extends AbstractIdleService { public synchronized void requestTargetNumberOfContainers(int numTargetContainers, Set<String> inUseInstances) { LOGGER.debug("Requesting numTargetContainers {} current numRequestedContainers {} in use instances {} map size {}", numTargetContainers, this.numRequestedContainers, inUseInstances, this.containerMap.size()); int numAllocatedContainers = this.containerMap.size(); int numContainers = Math.max(numRequestedContainers, numAllocatedContainers); for (int i = numContainers; i < numTargetContainers; i++) { requestContainer(Optional.<String>absent()); } if (numTargetContainers < numAllocatedContainers) { LOGGER.debug("Shrinking number of containers by {}", (numAllocatedContainers - numTargetContainers)); List<Container> containersToRelease = new ArrayList<>(); int numToShutdown = numContainers - numTargetContainers; for (Map.Entry<ContainerId, Map.Entry<Container, String>> entry : this.containerMap.entrySet()) { if (!inUseInstances.contains(entry.getValue().getValue())) { containersToRelease.add(entry.getValue().getKey()); } if (containersToRelease.size() == numToShutdown) { break; } } LOGGER.debug("Shutting down containers {}", containersToRelease); this.eventBus.post(new ContainerReleaseRequest(containersToRelease)); } this.numRequestedContainers = numTargetContainers; } }
YarnService extends AbstractIdleService { public synchronized void requestTargetNumberOfContainers(int numTargetContainers, Set<String> inUseInstances) { LOGGER.debug("Requesting numTargetContainers {} current numRequestedContainers {} in use instances {} map size {}", numTargetContainers, this.numRequestedContainers, inUseInstances, this.containerMap.size()); int numAllocatedContainers = this.containerMap.size(); int numContainers = Math.max(numRequestedContainers, numAllocatedContainers); for (int i = numContainers; i < numTargetContainers; i++) { requestContainer(Optional.<String>absent()); } if (numTargetContainers < numAllocatedContainers) { LOGGER.debug("Shrinking number of containers by {}", (numAllocatedContainers - numTargetContainers)); List<Container> containersToRelease = new ArrayList<>(); int numToShutdown = numContainers - numTargetContainers; for (Map.Entry<ContainerId, Map.Entry<Container, String>> entry : this.containerMap.entrySet()) { if (!inUseInstances.contains(entry.getValue().getValue())) { containersToRelease.add(entry.getValue().getKey()); } if (containersToRelease.size() == numToShutdown) { break; } } LOGGER.debug("Shutting down containers {}", containersToRelease); this.eventBus.post(new ContainerReleaseRequest(containersToRelease)); } this.numRequestedContainers = numTargetContainers; } YarnService(Config config, String applicationName, String applicationId, YarnConfiguration yarnConfiguration,
      FileSystem fs, EventBus eventBus, HelixManager helixManager); }
YarnService extends AbstractIdleService { public synchronized void requestTargetNumberOfContainers(int numTargetContainers, Set<String> inUseInstances) { LOGGER.debug("Requesting numTargetContainers {} current numRequestedContainers {} in use instances {} map size {}", numTargetContainers, this.numRequestedContainers, inUseInstances, this.containerMap.size()); int numAllocatedContainers = this.containerMap.size(); int numContainers = Math.max(numRequestedContainers, numAllocatedContainers); for (int i = numContainers; i < numTargetContainers; i++) { requestContainer(Optional.<String>absent()); } if (numTargetContainers < numAllocatedContainers) { LOGGER.debug("Shrinking number of containers by {}", (numAllocatedContainers - numTargetContainers)); List<Container> containersToRelease = new ArrayList<>(); int numToShutdown = numContainers - numTargetContainers; for (Map.Entry<ContainerId, Map.Entry<Container, String>> entry : this.containerMap.entrySet()) { if (!inUseInstances.contains(entry.getValue().getValue())) { containersToRelease.add(entry.getValue().getKey()); } if (containersToRelease.size() == numToShutdown) { break; } } LOGGER.debug("Shutting down containers {}", containersToRelease); this.eventBus.post(new ContainerReleaseRequest(containersToRelease)); } this.numRequestedContainers = numTargetContainers; } YarnService(Config config, String applicationName, String applicationId, YarnConfiguration yarnConfiguration,
      FileSystem fs, EventBus eventBus, HelixManager helixManager); @SuppressWarnings("unused") @Subscribe void handleNewContainerRequest(NewContainerRequest newContainerRequest); @SuppressWarnings("unused") @Subscribe void handleContainerShutdownRequest(ContainerShutdownRequest containerShutdownRequest); @Subscribe void handleContainerReleaseRequest(ContainerReleaseRequest containerReleaseRequest); void updateToken(); synchronized void requestTargetNumberOfContainers(int numTargetContainers, Set<String> inUseInstances); }
YarnService extends AbstractIdleService { public synchronized void requestTargetNumberOfContainers(int numTargetContainers, Set<String> inUseInstances) { LOGGER.debug("Requesting numTargetContainers {} current numRequestedContainers {} in use instances {} map size {}", numTargetContainers, this.numRequestedContainers, inUseInstances, this.containerMap.size()); int numAllocatedContainers = this.containerMap.size(); int numContainers = Math.max(numRequestedContainers, numAllocatedContainers); for (int i = numContainers; i < numTargetContainers; i++) { requestContainer(Optional.<String>absent()); } if (numTargetContainers < numAllocatedContainers) { LOGGER.debug("Shrinking number of containers by {}", (numAllocatedContainers - numTargetContainers)); List<Container> containersToRelease = new ArrayList<>(); int numToShutdown = numContainers - numTargetContainers; for (Map.Entry<ContainerId, Map.Entry<Container, String>> entry : this.containerMap.entrySet()) { if (!inUseInstances.contains(entry.getValue().getValue())) { containersToRelease.add(entry.getValue().getKey()); } if (containersToRelease.size() == numToShutdown) { break; } } LOGGER.debug("Shutting down containers {}", containersToRelease); this.eventBus.post(new ContainerReleaseRequest(containersToRelease)); } this.numRequestedContainers = numTargetContainers; } YarnService(Config config, String applicationName, String applicationId, YarnConfiguration yarnConfiguration,
      FileSystem fs, EventBus eventBus, HelixManager helixManager); @SuppressWarnings("unused") @Subscribe void handleNewContainerRequest(NewContainerRequest newContainerRequest); @SuppressWarnings("unused") @Subscribe void handleContainerShutdownRequest(ContainerShutdownRequest containerShutdownRequest); @Subscribe void handleContainerReleaseRequest(ContainerReleaseRequest containerReleaseRequest); void updateToken(); synchronized void requestTargetNumberOfContainers(int numTargetContainers, Set<String> inUseInstances);  }