SingularityScheduler { @Timed public void drainPendingQueue(final SingularitySchedulerStateCache stateCache) { final long start = System.currentTimeMillis(); final ImmutableList<SingularityPendingRequest> pendingRequests = ImmutableList.copyOf(requestManager.getPendingRequests()); if (pendingRequests.isEmpty()) { LOG.trace("Pending queue was empty"); return; } LOG.info("Pending queue had {} requests", pendingRequests.size()); int totalNewScheduledTasks = 0; int heldForScheduledActiveTask = 0; int obsoleteRequests = 0; for (SingularityPendingRequest pendingRequest : pendingRequests) { Optional<SingularityRequestWithState> maybeRequest = requestManager.getRequest(pendingRequest.getRequestId()); if (!isRequestActive(maybeRequest)) { LOG.debug("Pending request {} was obsolete (request {})", pendingRequest, SingularityRequestWithState.getRequestState(maybeRequest)); obsoleteRequests++; continue; } Optional<SingularityRequestDeployState> maybeRequestDeployState = deployManager.getRequestDeployState(pendingRequest.getRequestId()); Optional<SingularityPendingDeploy> maybePendingDeploy = deployManager.getPendingDeploy(maybeRequest.get().getRequest().getId()); if (!shouldScheduleTasks(maybeRequest.get().getRequest(), pendingRequest, maybePendingDeploy, maybeRequestDeployState)) { LOG.debug("Pending request {} was obsolete (request {})", pendingRequest, SingularityRequestWithState.getRequestState(maybeRequest)); obsoleteRequests++; continue; } final List<SingularityTaskId> matchingTaskIds = getMatchingTaskIds(stateCache, maybeRequest.get().getRequest(), pendingRequest); final SingularityDeployStatistics deployStatistics = getDeployStatistics(pendingRequest.getRequestId(), pendingRequest.getDeployId()); final RequestState requestState = checkCooldown(maybeRequest.get(), deployStatistics); int numScheduledTasks = scheduleTasks(stateCache, maybeRequest.get().getRequest(), requestState, deployStatistics, pendingRequest, matchingTaskIds, maybePendingDeploy); if (numScheduledTasks == 0 && !matchingTaskIds.isEmpty() && maybeRequest.get().getRequest().isScheduled() && pendingRequest.getPendingType() == PendingType.NEW_DEPLOY) { LOG.trace("Holding pending request {} because it is scheduled and has an active task", pendingRequest); heldForScheduledActiveTask++; continue; } LOG.debug("Pending request {} resulted in {} new scheduled tasks", pendingRequest, numScheduledTasks); totalNewScheduledTasks += numScheduledTasks; requestManager.deletePendingRequest(pendingRequest); } LOG.info("Scheduled {} new tasks ({} obsolete requests, {} held) in {}", totalNewScheduledTasks, obsoleteRequests, heldForScheduledActiveTask, JavaUtils.duration(start)); } @Inject  SingularityScheduler(TaskRequestManager taskRequestManager, SingularityConfiguration configuration, SingularityCooldown cooldown, DeployManager deployManager,
    TaskManager taskManager, RequestManager requestManager, SlaveManager slaveManager, RackManager rackManager, SingularityMailer mailer); @Timed void checkForDecomissions(SingularitySchedulerStateCache stateCache); @Timed void drainPendingQueue(final SingularitySchedulerStateCache stateCache); @Timed List<SingularityTaskRequest> getDueTasks(); @Timed void handleCompletedTask(Optional<SingularityTask> task, SingularityTaskId taskId, boolean wasActive, long timestamp, ExtendedTaskState state,
    SingularityCreateResult taskHistoryUpdateCreateResult, SingularitySchedulerStateCache stateCache);  }
@Test public void testSchedulerIsolatesPendingTasksBasedOnDeploy() { initRequest(); initFirstDeploy(); initSecondDeploy(); SingularityPendingTask p1 = pendingTask(requestId, firstDeployId, PendingType.ONEOFF); SingularityPendingTask p2 = pendingTask(requestId, firstDeployId, PendingType.TASK_DONE); SingularityPendingTask p3 = pendingTask(requestId, secondDeployId, PendingType.TASK_DONE); taskManager.savePendingTask(p1); taskManager.savePendingTask(p2); taskManager.savePendingTask(p3); requestManager.addToPendingQueue(new SingularityPendingRequest(requestId, secondDeployId, System.currentTimeMillis(), Optional.<String> absent(), PendingType.NEW_DEPLOY, Optional.<Boolean> absent(), Optional.<String> absent())); scheduler.drainPendingQueue(stateCacheProvider.get()); List<SingularityPendingTask> returnedScheduledTasks = taskManager.getPendingTasks(); Assert.assertEquals(3, returnedScheduledTasks.size()); Assert.assertTrue(returnedScheduledTasks.contains(p1)); Assert.assertTrue(returnedScheduledTasks.contains(p2)); Assert.assertTrue(!returnedScheduledTasks.contains(p3)); boolean found = false; for (SingularityPendingTask pendingTask : returnedScheduledTasks) { if (pendingTask.getPendingTaskId().getDeployId().equals(secondDeployId)) { found = true; Assert.assertEquals(PendingType.NEW_DEPLOY, pendingTask.getPendingTaskId().getPendingType()); } } Assert.assertTrue(found); }
@Test public void testDeployAllInstancesAtOnce() { initRequest(); SingularityRequest request = requestResource.getRequest(requestId).getRequest(); requestResource.postRequest(request.toBuilder().setInstances(Optional.of(2)).build()); initFirstDeploy(); launchTask(request, firstDeploy, 1, TaskState.TASK_RUNNING); launchTask(request, firstDeploy, 2, TaskState.TASK_RUNNING); deploy(secondDeployId); deployChecker.checkDeploys(); scheduler.drainPendingQueue(stateCacheProvider.get()); Assert.assertEquals(2, taskManager.getActiveTaskIds().size()); Assert.assertEquals(2, taskManager.getPendingTaskIds().size()); resourceOffers(); Assert.assertEquals(2, taskManager.getActiveTaskIdsForDeploy(requestId, secondDeployId).size()); for (SingularityTaskId taskId : taskManager.getActiveTaskIdsForDeploy(requestId, secondDeployId)) { statusUpdate(taskManager.getTask(taskId).get(), TaskState.TASK_RUNNING); } deployChecker.checkDeploys(); Assert.assertEquals(2, taskManager.getCleanupTaskIds().size()); Assert.assertEquals(DeployState.SUCCEEDED, deployManager.getDeployResult(requestId, secondDeployId).get().getDeployState()); Assert.assertEquals(2, taskManager.getActiveTaskIdsForDeploy(requestId, secondDeployId).size()); }
@Test public void testDeployOneInstanceAtATime() { initRequest(); SingularityRequest request = requestResource.getRequest(requestId).getRequest(); requestResource.postRequest(request.toBuilder().setInstances(Optional.of(2)).build()); initFirstDeploy(); SingularityTask firstTask = launchTask(request, firstDeploy, 1, TaskState.TASK_RUNNING); SingularityTask secondTask = launchTask(request, firstDeploy, 2, TaskState.TASK_RUNNING); deploy(secondDeployId, Optional.<Boolean> absent(), Optional.of(1), Optional.<Boolean> absent(), false); deployChecker.checkDeploys(); scheduler.drainPendingQueue(stateCacheProvider.get()); Assert.assertEquals(1, taskManager.getPendingTaskIds().size()); resourceOffers(); Assert.assertEquals(1, taskManager.getActiveTaskIdsForDeploy(requestId, secondDeployId).size()); SingularityTaskId firstNewTaskId = taskManager.getActiveTaskIdsForDeploy(requestId, secondDeployId).get(0); statusUpdate(taskManager.getTask(firstNewTaskId).get(), TaskState.TASK_RUNNING); deployChecker.checkDeploys(); Assert.assertEquals(1, taskManager.getCleanupTaskIds().size()); Assert.assertTrue(taskManager.getCleanupTaskIds().contains(firstTask.getTaskId())); SingularityDeployProgress deployProgressStepOne = deployManager.getPendingDeploys().get(0).getDeployProgress().get(); Assert.assertTrue(deployProgressStepOne.isStepComplete()); Assert.assertEquals(1, deployProgressStepOne.getTargetActiveInstances()); cleaner.drainCleanupQueue(); statusUpdate(firstTask, TaskState.TASK_KILLED); deployChecker.checkDeploys(); SingularityDeployProgress deployProgressStepTwo = deployManager.getPendingDeploys().get(0).getDeployProgress().get(); Assert.assertFalse(deployProgressStepTwo.isStepComplete()); Assert.assertEquals(2, deployProgressStepTwo.getTargetActiveInstances()); scheduler.drainPendingQueue(stateCacheProvider.get()); Assert.assertEquals(1, taskManager.getPendingTaskIds().size()); resourceOffers(); Assert.assertEquals(2, taskManager.getActiveTaskIdsForDeploy(requestId, secondDeployId).size()); for (SingularityTaskId taskId : taskManager.getActiveTaskIdsForDeploy(requestId, secondDeployId)) { statusUpdate(taskManager.getTask(taskId).get(), TaskState.TASK_RUNNING); } deployChecker.checkDeploys(); Assert.assertEquals(2, taskManager.getActiveTaskIdsForDeploy(requestId, secondDeployId).size()); Assert.assertEquals(DeployState.SUCCEEDED, deployManager.getDeployResult(requestId, secondDeployId).get().getDeployState()); }
@Test public void testDeployWithManualStep() { initRequest(); SingularityRequest request = requestResource.getRequest(requestId).getRequest(); requestResource.postRequest(request.toBuilder().setInstances(Optional.of(2)).build()); initFirstDeploy(); SingularityTask firstTask = launchTask(request, firstDeploy, 1, TaskState.TASK_RUNNING); SingularityTask secondTask = launchTask(request, firstDeploy, 2, TaskState.TASK_RUNNING); deploy(secondDeployId, Optional.<Boolean>absent(), Optional.of(1), Optional.of(false), false); deployChecker.checkDeploys(); scheduler.drainPendingQueue(stateCacheProvider.get()); Assert.assertEquals(1, taskManager.getPendingTaskIds().size()); resourceOffers(); Assert.assertEquals(1, taskManager.getActiveTaskIdsForDeploy(requestId, secondDeployId).size()); SingularityTaskId firstNewTaskId = taskManager.getActiveTaskIdsForDeploy(requestId, secondDeployId).get(0); statusUpdate(taskManager.getTask(firstNewTaskId).get(), TaskState.TASK_RUNNING); deployChecker.checkDeploys(); Assert.assertEquals(1, taskManager.getCleanupTaskIds().size()); Assert.assertTrue(taskManager.getCleanupTaskIds().contains(firstTask.getTaskId())); SingularityDeployProgress deployProgressStepOne = deployManager.getPendingDeploys().get(0).getDeployProgress().get(); Assert.assertTrue(deployProgressStepOne.isStepComplete()); Assert.assertEquals(1, deployProgressStepOne.getTargetActiveInstances()); cleaner.drainCleanupQueue(); statusUpdate(firstTask, TaskState.TASK_KILLED); deployChecker.checkDeploys(); deployProgressStepOne = deployManager.getPendingDeploys().get(0).getDeployProgress().get(); Assert.assertTrue(deployProgressStepOne.isStepComplete()); Assert.assertEquals(1, deployProgressStepOne.getTargetActiveInstances()); deployResource.updatePendingDeploy(new SingularityUpdatePendingDeployRequest(requestId, secondDeployId, 2)); deployChecker.checkDeploys(); SingularityDeployProgress deployProgressStepTwo = deployManager.getPendingDeploys().get(0).getDeployProgress().get(); Assert.assertFalse(deployProgressStepTwo.isStepComplete()); Assert.assertEquals(2, deployProgressStepTwo.getTargetActiveInstances()); scheduler.drainPendingQueue(stateCacheProvider.get()); Assert.assertEquals(1, taskManager.getPendingTaskIds().size()); resourceOffers(); Assert.assertEquals(2, taskManager.getActiveTaskIdsForDeploy(requestId, secondDeployId).size()); for (SingularityTaskId taskId : taskManager.getActiveTaskIdsForDeploy(requestId, secondDeployId)) { statusUpdate(taskManager.getTask(taskId).get(), TaskState.TASK_RUNNING); } deployChecker.checkDeploys(); Assert.assertEquals(2, taskManager.getActiveTaskIdsForDeploy(requestId, secondDeployId).size()); Assert.assertEquals(DeployState.SUCCEEDED, deployManager.getDeployResult(requestId, secondDeployId).get().getDeployState()); }
@Test public void testDeployMultipleInstancesAtOnce() { initRequest(); SingularityRequest request = requestResource.getRequest(requestId).getRequest(); requestResource.postRequest(request.toBuilder().setInstances(Optional.of(4)).build()); initFirstDeploy(); SingularityTask firstTask = launchTask(request, firstDeploy, 1, TaskState.TASK_RUNNING); SingularityTask secondTask = launchTask(request, firstDeploy, 2, TaskState.TASK_RUNNING); SingularityTask thirdTask = launchTask(request, firstDeploy, 3, TaskState.TASK_RUNNING); SingularityTask fourthTask = launchTask(request, firstDeploy, 4, TaskState.TASK_RUNNING); deploy(secondDeployId, Optional.<Boolean>absent(), Optional.of(2), Optional.<Boolean> absent(), false); deployChecker.checkDeploys(); scheduler.drainPendingQueue(stateCacheProvider.get()); Assert.assertEquals(2, taskManager.getPendingTaskIds().size()); resourceOffers(); Assert.assertEquals(2, taskManager.getActiveTaskIdsForDeploy(requestId, secondDeployId).size()); for (SingularityTaskId taskId : taskManager.getActiveTaskIdsForDeploy(requestId, secondDeployId)) { statusUpdate(taskManager.getTask(taskId).get(), TaskState.TASK_RUNNING); } deployChecker.checkDeploys(); Assert.assertEquals(2, taskManager.getCleanupTaskIds().size()); List<SingularityTaskId> cleanupTaskIds = taskManager.getCleanupTaskIds(); Assert.assertTrue(cleanupTaskIds.contains(firstTask.getTaskId()) && cleanupTaskIds.contains(secondTask.getTaskId())); SingularityDeployProgress deployProgressStepOne = deployManager.getPendingDeploys().get(0).getDeployProgress().get(); Assert.assertTrue(deployProgressStepOne.isStepComplete()); Assert.assertEquals(2, deployProgressStepOne.getTargetActiveInstances()); cleaner.drainCleanupQueue(); statusUpdate(firstTask, TaskState.TASK_KILLED); statusUpdate(secondTask, TaskState.TASK_KILLED); deployChecker.checkDeploys(); SingularityDeployProgress deployProgressStepTwo = deployManager.getPendingDeploys().get(0).getDeployProgress().get(); Assert.assertFalse(deployProgressStepTwo.isStepComplete()); Assert.assertEquals(4, deployProgressStepTwo.getTargetActiveInstances()); scheduler.drainPendingQueue(stateCacheProvider.get()); Assert.assertEquals(2, taskManager.getPendingTaskIds().size()); resourceOffers(); Assert.assertEquals(4, taskManager.getActiveTaskIdsForDeploy(requestId, secondDeployId).size()); for (SingularityTaskId taskId : taskManager.getActiveTaskIdsForDeploy(requestId, secondDeployId)) { statusUpdate(taskManager.getTask(taskId).get(), TaskState.TASK_RUNNING); } deployChecker.checkDeploys(); Assert.assertEquals(4, taskManager.getActiveTaskIdsForDeploy(requestId, secondDeployId).size()); Assert.assertEquals(DeployState.SUCCEEDED, deployManager.getDeployResult(requestId, secondDeployId).get().getDeployState()); }
@Test public void testCancelDeploy() { initRequest(); SingularityRequest request = requestResource.getRequest(requestId).getRequest(); initFirstDeploy(); SingularityTask firstTask = launchTask(request, firstDeploy, 1, TaskState.TASK_RUNNING); deploy(secondDeployId, Optional.<Boolean>absent(), Optional.of(1), Optional.of(false), false); deployChecker.checkDeploys(); scheduler.drainPendingQueue(stateCacheProvider.get()); Assert.assertEquals(1, taskManager.getPendingTaskIds().size()); resourceOffers(); Assert.assertEquals(1, taskManager.getActiveTaskIdsForDeploy(requestId, secondDeployId).size()); SingularityTaskId firstNewTaskId = taskManager.getActiveTaskIdsForDeploy(requestId, secondDeployId).get(0); statusUpdate(taskManager.getTask(firstNewTaskId).get(), TaskState.TASK_RUNNING); deployResource.cancelDeploy(requestId, secondDeployId); deployChecker.checkDeploys(); Assert.assertTrue(taskManager.getCleanupTaskIds().contains(firstNewTaskId)); Assert.assertFalse(taskManager.getCleanupTaskIds().contains(firstTask.getTaskId())); Assert.assertEquals(DeployState.CANCELED, deployManager.getDeployResult(requestId, secondDeployId).get().getDeployState()); }
@Test public void testDeployFails() { initRequest(); SingularityRequest request = requestResource.getRequest(requestId).getRequest(); initFirstDeploy(); SingularityTask firstTask = launchTask(request, firstDeploy, 1, TaskState.TASK_RUNNING); deploy(secondDeployId, Optional.<Boolean>absent(), Optional.of(1), Optional.of(false), false); deployChecker.checkDeploys(); scheduler.drainPendingQueue(stateCacheProvider.get()); Assert.assertEquals(1, taskManager.getPendingTaskIds().size()); resourceOffers(); Assert.assertEquals(1, taskManager.getActiveTaskIdsForDeploy(requestId, secondDeployId).size()); SingularityTaskId firstNewTaskId = taskManager.getActiveTaskIdsForDeploy(requestId, secondDeployId).get(0); statusUpdate(taskManager.getTask(firstNewTaskId).get(), TaskState.TASK_FAILED); deployChecker.checkDeploys(); Assert.assertFalse(taskManager.getCleanupTaskIds().contains(firstTask.getTaskId())); Assert.assertEquals(DeployState.FAILED, deployManager.getDeployResult(requestId, secondDeployId).get().getDeployState()); }
@Test public void testLbUpdatesAfterEachDeployStep() { initLoadBalancedRequest(); SingularityRequest request = requestResource.getRequest(requestId).getRequest(); requestResource.postRequest(request.toBuilder().setInstances(Optional.of(2)).build()); initFirstDeploy(); SingularityTask firstTask = launchTask(request, firstDeploy, 1, TaskState.TASK_RUNNING); SingularityTask secondTask = launchTask(request, firstDeploy, 2, TaskState.TASK_RUNNING); deploy(secondDeployId, Optional.<Boolean>absent(), Optional.of(1), Optional.<Boolean> absent(), true); deployChecker.checkDeploys(); scheduler.drainPendingQueue(stateCacheProvider.get()); Assert.assertEquals(1, taskManager.getPendingTaskIds().size()); resourceOffers(); Assert.assertEquals(1, taskManager.getActiveTaskIdsForDeploy(requestId, secondDeployId).size()); SingularityTaskId firstNewTaskId = taskManager.getActiveTaskIdsForDeploy(requestId, secondDeployId).get(0); statusUpdate(taskManager.getTask(firstNewTaskId).get(), TaskState.TASK_RUNNING); deployChecker.checkDeploys(); SingularityPendingDeploy pendingDeploy = deployManager.getPendingDeploy(requestId).get(); Assert.assertEquals(DeployState.WAITING, pendingDeploy.getCurrentDeployState()); testingLbClient.setNextBaragonRequestState(BaragonRequestState.WAITING); deployChecker.checkDeploys(); pendingDeploy = deployManager.getPendingDeploy(requestId).get(); Assert.assertEquals(DeployState.WAITING, pendingDeploy.getCurrentDeployState()); testingLbClient.setNextBaragonRequestState(BaragonRequestState.SUCCESS); deployChecker.checkDeploys(); Assert.assertTrue(taskManager.getCleanupTaskIds().contains(firstTask.getTaskId())); pendingDeploy = deployManager.getPendingDeploy(requestId).get(); SingularityDeployProgress deployProgressStepOne = pendingDeploy.getDeployProgress().get(); Assert.assertTrue(deployProgressStepOne.isStepComplete()); Assert.assertEquals(1, deployProgressStepOne.getTargetActiveInstances()); cleaner.drainCleanupQueue(); statusUpdate(firstTask, TaskState.TASK_KILLED); deployChecker.checkDeploys(); pendingDeploy = deployManager.getPendingDeploy(requestId).get(); Assert.assertFalse(pendingDeploy.getDeployProgress().get().isStepComplete()); Assert.assertEquals(2, pendingDeploy.getDeployProgress().get().getTargetActiveInstances()); scheduler.drainPendingQueue(stateCacheProvider.get()); Assert.assertEquals(1, taskManager.getPendingTaskIds().size()); resourceOffers(); Assert.assertEquals(2, taskManager.getActiveTaskIdsForDeploy(requestId, secondDeployId).size()); SingularityTaskId secondNewTaskId = null; for (SingularityTaskId taskId : taskManager.getActiveTaskIdsForDeploy(requestId, secondDeployId)) { if (taskId.getInstanceNo() == 2) { secondNewTaskId = taskId; } } statusUpdate(taskManager.getTask(secondNewTaskId).get(), TaskState.TASK_RUNNING); testingLbClient.setNextBaragonRequestState(BaragonRequestState.WAITING); deployChecker.checkDeploys(); pendingDeploy = deployManager.getPendingDeploy(requestId).get(); Assert.assertEquals(DeployState.WAITING, pendingDeploy.getCurrentDeployState()); testingLbClient.setNextBaragonRequestState(BaragonRequestState.SUCCESS); deployChecker.checkDeploys(); Assert.assertEquals(2, taskManager.getActiveTaskIdsForDeploy(requestId, secondDeployId).size()); Assert.assertEquals(DeployState.SUCCEEDED, deployManager.getDeployResult(requestId, secondDeployId).get().getDeployState()); }
@Test public void testCanceledDeployTasksStayActiveUntilReplaced() { initRequest(); SingularityRequest request = requestResource.getRequest(requestId).getRequest(); requestResource.postRequest(request.toBuilder().setInstances(Optional.of(2)).build()); initFirstDeploy(); SingularityTask firstTask = launchTask(request, firstDeploy, 1, TaskState.TASK_RUNNING); SingularityTask secondTask = launchTask(request, firstDeploy, 2, TaskState.TASK_RUNNING); deploy(secondDeployId, Optional.<Boolean> absent(), Optional.of(1), Optional.<Boolean> absent(), false); deployChecker.checkDeploys(); scheduler.drainPendingQueue(stateCacheProvider.get()); Assert.assertEquals(1, taskManager.getPendingTaskIds().size()); resourceOffers(); Assert.assertEquals(1, taskManager.getActiveTaskIdsForDeploy(requestId, secondDeployId).size()); SingularityTaskId firstNewTaskId = taskManager.getActiveTaskIdsForDeploy(requestId, secondDeployId).get(0); statusUpdate(taskManager.getTask(firstNewTaskId).get(), TaskState.TASK_RUNNING); deployChecker.checkDeploys(); Assert.assertEquals(1, taskManager.getCleanupTaskIds().size()); Assert.assertTrue(taskManager.getCleanupTaskIds().contains(firstTask.getTaskId())); SingularityDeployProgress deployProgressStepOne = deployManager.getPendingDeploys().get(0).getDeployProgress().get(); Assert.assertTrue(deployProgressStepOne.isStepComplete()); Assert.assertEquals(1, deployProgressStepOne.getTargetActiveInstances()); cleaner.drainCleanupQueue(); statusUpdate(firstTask, TaskState.TASK_KILLED); deployChecker.checkDeploys(); deployResource.cancelDeploy(requestId, secondDeployId); deployChecker.checkDeploys(); scheduler.drainPendingQueue(stateCacheProvider.get()); List<SingularityPendingTaskId> pendingTaskIds = taskManager.getPendingTaskIds(); Assert.assertEquals(1, pendingTaskIds.size()); Assert.assertEquals(firstDeployId, pendingTaskIds.get(0).getDeployId()); cleaner.drainCleanupQueue(); List<SingularityTaskId> cleanupTaskIds = taskManager.getCleanupTaskIds(); Assert.assertEquals(1, cleanupTaskIds.size()); Assert.assertEquals(secondDeployId, cleanupTaskIds.get(0).getDeployId()); resourceOffers(); for (SingularityTaskId taskId : taskManager.getActiveTaskIdsForDeploy(requestId, firstDeployId)) { statusUpdate(taskManager.getTask(taskId).get(), TaskState.TASK_RUNNING); } cleaner.drainCleanupQueue(); Assert.assertEquals(0, taskManager.getCleanupTaskIds().size()); Assert.assertEquals(2, taskManager.getActiveTaskIdsForDeploy(requestId, firstDeployId).size()); }
@Test public void testAfterDeployWaitsForScheduledTaskToFinish() { initScheduledRequest(); initFirstDeploy(); SingularityTask firstTask = launchTask(request, firstDeploy, 1, TaskState.TASK_RUNNING); Assert.assertTrue(taskManager.getPendingTasks().isEmpty()); Assert.assertTrue(taskManager.getActiveTaskIds().contains(firstTask.getTaskId())); Assert.assertEquals(1, taskManager.getActiveTaskIds().size()); Assert.assertTrue(taskManager.getCleanupTaskIds().isEmpty()); deploy("nextDeployId"); deployChecker.checkDeploys(); scheduler.drainPendingQueue(stateCacheProvider.get()); Assert.assertTrue(taskManager.getPendingTasks().isEmpty()); Assert.assertTrue(taskManager.getActiveTaskIds().contains(firstTask.getTaskId())); Assert.assertEquals(1, taskManager.getActiveTaskIds().size()); Assert.assertTrue(!taskManager.getCleanupTaskIds().isEmpty()); statusUpdate(firstTask, TaskState.TASK_FINISHED); scheduler.drainPendingQueue(stateCacheProvider.get()); cleaner.drainCleanupQueue(); Assert.assertTrue(!taskManager.getPendingTasks().isEmpty()); Assert.assertTrue(taskManager.getActiveTaskIds().isEmpty()); Assert.assertTrue(taskManager.getCleanupTaskIds().isEmpty()); SingularityPendingTaskId pendingTaskId = taskManager.getPendingTaskIds().get(0); Assert.assertEquals("nextDeployId", pendingTaskId.getDeployId()); Assert.assertEquals(requestId, pendingTaskId.getRequestId()); }
@Test public void testSchedulerCanBatchOnOffers() { initRequest(); initFirstDeploy(); requestResource.postRequest(request.toBuilder().setInstances(Optional.of(3)).build()); scheduler.drainPendingQueue(stateCacheProvider.get()); List<Offer> oneOffer = Arrays.asList(createOffer(12, 1024)); sms.resourceOffers(driver, oneOffer); Assert.assertTrue(taskManager.getActiveTasks().size() == 3); Assert.assertTrue(taskManager.getPendingTaskIds().isEmpty()); Assert.assertTrue(requestManager.getPendingRequests().isEmpty()); }
@Test public void testSchedulerExhaustsOffers() { initRequest(); initFirstDeploy(); requestResource.postRequest(request.toBuilder().setInstances(Optional.of(10)).build()); scheduler.drainPendingQueue(stateCacheProvider.get()); sms.resourceOffers(driver, Arrays.asList(createOffer(2, 1024), createOffer(1, 1024))); Assert.assertTrue(taskManager.getActiveTaskIds().size() == 3); Assert.assertTrue(taskManager.getPendingTaskIds().size() == 7); }
@Test public void testSchedulerRandomizesOffers() { initRequest(); initFirstDeploy(); requestResource.postRequest(request.toBuilder().setInstances(Optional.of(15)).build()); scheduler.drainPendingQueue(stateCacheProvider.get()); sms.resourceOffers(driver, Arrays.asList(createOffer(20, 1024), createOffer(20, 1024))); Assert.assertTrue(taskManager.getActiveTaskIds().size() == 15); Set<String> offerIds = Sets.newHashSet(); for (SingularityTask activeTask : taskManager.getActiveTasks()) { offerIds.add(activeTask.getOffer().getId().getValue()); } Assert.assertTrue(offerIds.size() == 2); }
@Test public void testSchedulerHandlesFinishedTasks() { initScheduledRequest(); initFirstDeploy(); schedule = "*/1 * * * * ? 1995"; requestResource.postRequest(request.toBuilder().setQuartzSchedule(Optional.of(schedule)).build()); scheduler.drainPendingQueue(stateCacheProvider.get()); Assert.assertTrue(requestResource.getActiveRequests().isEmpty()); Assert.assertTrue(requestManager.getRequest(requestId).get().getState() == RequestState.FINISHED); Assert.assertTrue(taskManager.getPendingTaskIds().isEmpty()); schedule = "*/1 * * * * ?"; requestResource.postRequest(request.toBuilder().setQuartzSchedule(Optional.of(schedule)).build()); scheduler.drainPendingQueue(stateCacheProvider.get()); Assert.assertTrue(!requestResource.getActiveRequests().isEmpty()); Assert.assertTrue(requestManager.getRequest(requestId).get().getState() == RequestState.ACTIVE); Assert.assertTrue(!taskManager.getPendingTaskIds().isEmpty()); }
@Test public void testScheduledJobLivesThroughDeploy() { initScheduledRequest(); initFirstDeploy(); createAndSchedulePendingTask(firstDeployId); Assert.assertTrue(!taskManager.getPendingTaskIds().isEmpty()); deploy("d2"); scheduler.drainPendingQueue(stateCacheProvider.get()); deployChecker.checkDeploys(); scheduler.drainPendingQueue(stateCacheProvider.get()); Assert.assertTrue(!taskManager.getPendingTaskIds().isEmpty()); }
@Test public void testRunOnceRunOnlyOnce() { SingularityRequestBuilder bldr = new SingularityRequestBuilder(requestId, RequestType.RUN_ONCE); request = bldr.build(); saveRequest(request); deployResource.deploy(new SingularityDeployRequest(new SingularityDeployBuilder(requestId, "d1").setCommand(Optional.of("cmd")).build(), Optional.<Boolean> absent(), Optional.<String> absent())); scheduler.drainPendingQueue(stateCacheProvider.get()); deployChecker.checkDeploys(); resourceOffers(); Assert.assertTrue(deployManager.getRequestDeployState(requestId).get().getActiveDeploy().isPresent()); Assert.assertTrue(!deployManager.getRequestDeployState(requestId).get().getPendingDeploy().isPresent()); Assert.assertEquals(1, taskManager.getActiveTaskIds().size()); statusUpdate(taskManager.getActiveTasks().get(0), TaskState.TASK_LOST); resourceOffers(); Assert.assertTrue(taskManager.getActiveTaskIds().isEmpty()); deployResource.deploy(new SingularityDeployRequest(new SingularityDeployBuilder(requestId, "d2").setCommand(Optional.of("cmd")).build(), Optional.<Boolean>absent(), Optional.<String> absent())); scheduler.drainPendingQueue(stateCacheProvider.get()); deployChecker.checkDeploys(); resourceOffers(); Assert.assertTrue(deployManager.getRequestDeployState(requestId).get().getActiveDeploy().isPresent()); Assert.assertTrue(!deployManager.getRequestDeployState(requestId).get().getPendingDeploy().isPresent()); Assert.assertEquals(1, taskManager.getActiveTaskIds().size()); statusUpdate(taskManager.getActiveTasks().get(0), TaskState.TASK_FINISHED); resourceOffers(); Assert.assertTrue(taskManager.getActiveTaskIds().isEmpty()); }
@Test public void testRunOnceDontMoveDuringDecomission() { SingularityRequestBuilder bldr = new SingularityRequestBuilder(requestId, RequestType.RUN_ONCE); request = bldr.build(); saveRequest(request); deployResource.deploy(new SingularityDeployRequest(new SingularityDeployBuilder(requestId, "d1").setCommand(Optional.of("cmd")).build(), Optional.<Boolean> absent(), Optional.<String> absent())); scheduler.drainPendingQueue(stateCacheProvider.get()); deployChecker.checkDeploys(); validateTaskDoesntMoveDuringDecommission(); }
@Test public void testRetries() { SingularityRequestBuilder bldr = new SingularityRequestBuilder(requestId, RequestType.RUN_ONCE); request = bldr.setNumRetriesOnFailure(Optional.of(2)).build(); saveRequest(request); deployResource.deploy(new SingularityDeployRequest(new SingularityDeployBuilder(requestId, "d1").setCommand(Optional.of("cmd")).build(), Optional.<Boolean> absent(), Optional.<String> absent())); scheduler.drainPendingQueue(stateCacheProvider.get()); deployChecker.checkDeploys(); resourceOffers(); Assert.assertEquals(1, taskManager.getActiveTaskIds().size()); statusUpdate(taskManager.getActiveTasks().get(0), TaskState.TASK_LOST); resourceOffers(); Assert.assertEquals(1, taskManager.getActiveTaskIds().size()); statusUpdate(taskManager.getActiveTasks().get(0), TaskState.TASK_LOST); resourceOffers(); Assert.assertEquals(1, taskManager.getActiveTaskIds().size()); statusUpdate(taskManager.getActiveTasks().get(0), TaskState.TASK_LOST); resourceOffers(); Assert.assertTrue(taskManager.getActiveTaskIds().isEmpty()); }
@Test public void testUnpauseOnDeploy() { initRequest(); initFirstDeploy(); requestManager.pause(request, System.currentTimeMillis(), Optional.<String>absent(), Optional.<String>absent()); boolean exception = false; try { deploy("d2"); } catch (Exception e) { exception = true; } Assert.assertTrue(exception); deploy("d3", Optional.of(true)); Assert.assertTrue(requestManager.getRequest(requestId).get().getState() == RequestState.DEPLOYING_TO_UNPAUSE); scheduler.drainPendingQueue(stateCacheProvider.get()); sms.resourceOffers(driver, Arrays.asList(createOffer(20, 20000, "slave1", "host1"))); statusUpdate(taskManager.getActiveTasks().get(0), TaskState.TASK_FAILED); deployChecker.checkDeploys(); Assert.assertTrue(requestManager.getRequest(requestId).get().getState() == RequestState.PAUSED); Assert.assertTrue(taskManager.getActiveTaskIds().isEmpty()); Assert.assertTrue(taskManager.getPendingTaskIds().isEmpty()); Assert.assertTrue(requestManager.getPendingRequests().isEmpty()); deploy("d4", Optional.of(true)); Assert.assertTrue(requestManager.getRequest(requestId).get().getState() == RequestState.DEPLOYING_TO_UNPAUSE); scheduler.drainPendingQueue(stateCacheProvider.get()); sms.resourceOffers(driver, Arrays.asList(createOffer(20, 20000, "slave1", "host1"))); statusUpdate(taskManager.getActiveTasks().get(0), TaskState.TASK_RUNNING); deployChecker.checkDeploys(); RequestState requestState = requestManager.getRequest(requestId).get().getState(); Assert.assertTrue(requestState == RequestState.ACTIVE); }
@Test public void testOnDemandTasksPersist() { SingularityRequestBuilder bldr = new SingularityRequestBuilder(requestId, RequestType.ON_DEMAND); requestResource.postRequest(bldr.build()); deploy("d2"); deployChecker.checkDeploys(); requestResource.scheduleImmediately(requestId); resourceOffers(); requestResource.scheduleImmediately(requestId); resourceOffers(); Assert.assertEquals(2, taskManager.getActiveTaskIds().size()); requestResource.scheduleImmediately(requestId); scheduler.drainPendingQueue(stateCacheProvider.get()); requestResource.scheduleImmediately(requestId); scheduler.drainPendingQueue(stateCacheProvider.get()); Assert.assertEquals(2, taskManager.getPendingTaskIds().size()); resourceOffers(); Assert.assertEquals(4, taskManager.getActiveTaskIds().size()); }
@Test public void testScaleDownTakesHighestInstances() { initRequest(); initFirstDeploy(); saveAndSchedule(request.toBuilder().setInstances(Optional.of(5))); resourceOffers(); Assert.assertEquals(5, taskManager.getActiveTaskIds().size()); requestResource.scale(requestId, new SingularityScaleRequest(Optional.of(2), Optional.<Long> absent(), Optional.<Boolean> absent(), Optional.<String> absent(), Optional.<String>absent())); resourceOffers(); cleaner.drainCleanupQueue(); Assert.assertEquals(3, taskManager.getKilledTaskIdRecords().size()); for (SingularityKilledTaskIdRecord taskId : taskManager.getKilledTaskIdRecords()) { Assert.assertTrue(taskId.getTaskId().getInstanceNo() > 2); scheduler.drainPendingQueue(stateCacheProvider.get()); } }
@Test public void testMaxTasksPerOffer() { configuration.setMaxTasksPerOffer(3); initRequest(); initFirstDeploy(); requestResource.postRequest(request.toBuilder().setInstances(Optional.of(20)).build()); scheduler.drainPendingQueue(stateCacheProvider.get()); sms.resourceOffers(driver, Arrays.asList(createOffer(36, 12024))); Assert.assertTrue(taskManager.getActiveTasks().size() == 3); sms.resourceOffers(driver, Arrays.asList(createOffer(20, 20000, "slave1", "host1"), createOffer(20, 20000, "slave2", "host2"))); Assert.assertTrue(taskManager.getActiveTasks().size() == 9); configuration.setMaxTasksPerOffer(0); resourceOffers(); Assert.assertTrue(taskManager.getActiveTasks().size() == 20); }
@Test public void testRequestedPorts() { final SingularityDeployBuilder deployBuilder = dockerDeployWithPorts(3); initRequest(); initAndFinishDeploy(request, deployBuilder); requestResource.postRequest(request.toBuilder().setInstances(Optional.of(2)).build()); scheduler.drainPendingQueue(stateCacheProvider.get()); String[] portRangeWithNoRequestedPorts = {"65:70"}; sms.resourceOffers(driver, Arrays.asList(createOffer(20, 20000, "slave1", "host1", Optional.<String> absent(), Collections.<String, String>emptyMap(), portRangeWithNoRequestedPorts))); Assert.assertEquals(0, taskManager.getActiveTasks().size()); String[] portRangeWithSomeRequestedPorts = {"80:82"}; sms.resourceOffers(driver, Arrays.asList(createOffer(20, 20000, "slave1", "host1", Optional.<String> absent(), Collections.<String, String>emptyMap(), portRangeWithSomeRequestedPorts))); Assert.assertEquals(0, taskManager.getActiveTasks().size()); String[] portRangeWithRequestedButNotEnoughPorts = {"80:80", "8080:8080"}; sms.resourceOffers(driver, Arrays.asList(createOffer(20, 20000, "slave1", "host1", Optional.<String> absent(), Collections.<String, String>emptyMap(), portRangeWithRequestedButNotEnoughPorts))); Assert.assertEquals(0, taskManager.getActiveTasks().size()); String[] portRangeWithNeededPorts = {"80:83", "8080:8080"}; sms.resourceOffers(driver, Arrays.asList(createOffer(20, 20000, "slave1", "host1", Optional.<String> absent(), Collections.<String, String>emptyMap(), portRangeWithNeededPorts))); Assert.assertEquals(1, taskManager.getActiveTaskIds().size()); }
@Test public void testPortIndices() { configuration.setNewTaskCheckerBaseDelaySeconds(0); configuration.setHealthcheckIntervalSeconds(0); configuration.setDeployHealthyBySeconds(0); configuration.setKillAfterTasksDoNotRunDefaultSeconds(1); configuration.setHealthcheckMaxRetries(Optional.of(0)); initRequest(); firstDeploy = initAndFinishDeploy(request, new SingularityDeployBuilder(request.getId(), firstDeployId) .setCommand(Optional.of("sleep 100")) .setHealthcheckUri(Optional.of("http: .setResources(Optional.of(new Resources(1, 64, 3))) .setHealthcheckPortIndex(Optional.of(1))); requestResource.postRequest(request.toBuilder().setInstances(Optional.of(2)).build()); scheduler.drainPendingQueue(stateCacheProvider.get()); String[] portRange = {"80:82"}; sms.resourceOffers(driver, Arrays.asList(createOffer(20, 20000, "slave1", "host1", Optional.<String> absent(), Collections.<String, String>emptyMap(), portRange))); SingularityTaskId firstTaskId = taskManager.getActiveTaskIdsForRequest(requestId).get(0); SingularityTask firstTask = taskManager.getTask(firstTaskId).get(); statusUpdate(firstTask, TaskState.TASK_RUNNING); newTaskChecker.enqueueNewTaskCheck(firstTask, requestManager.getRequest(requestId), healthchecker); finishNewTaskChecks(); finishHealthchecks(); finishNewTaskChecksAndCleanup(); Assert.assertTrue(taskManager.getLastHealthcheck(firstTask.getTaskId()).get().toString().contains("host1:81")); }