SchedulerUtils { public static void normalizeAndValidateRequest(ResourceRequest resReq, Resource maximumResource, String queueName, YarnScheduler scheduler, boolean isRecovery, RMContext rmContext) throws InvalidResourceRequestException { normalizeAndValidateRequest(resReq, maximumResource, queueName, scheduler, isRecovery, rmContext, null); }  static ContainerStatus createAbnormalContainerStatus(
      ContainerId containerId, String diagnostics); static ContainerStatus createPreemptedContainerStatus(
      ContainerId containerId, String diagnostics); static void normalizeRequests(
    List<ResourceRequest> asks,
    ResourceCalculator resourceCalculator,
    Resource clusterResource,
    Resource minimumResource,
    Resource maximumResource); static void normalizeRequest(
    ResourceRequest ask,
    ResourceCalculator resourceCalculator,
    Resource clusterResource,
    Resource minimumResource,
    Resource maximumResource); static void normalizeRequests(
      List<ResourceRequest> asks,
      ResourceCalculator resourceCalculator, 
      Resource clusterResource,
      Resource minimumResource,
      Resource maximumResource,
      Resource incrementResource); static void normalizeRequest(
      ResourceRequest ask, 
      ResourceCalculator resourceCalculator, 
      Resource clusterResource,
      Resource minimumResource,
      Resource maximumResource,
      Resource incrementResource); static void normalizeAndValidateRequest(ResourceRequest resReq,
      Resource maximumResource, String queueName, YarnScheduler scheduler,
      boolean isRecovery, RMContext rmContext); static void normalizeAndValidateRequest(ResourceRequest resReq,
      Resource maximumResource, String queueName, YarnScheduler scheduler,
      boolean isRecovery, RMContext rmContext, QueueInfo queueInfo); static void normalizeAndvalidateRequest(ResourceRequest resReq,
      Resource maximumResource, String queueName, YarnScheduler scheduler,
      RMContext rmContext); static void normalizeAndvalidateRequest(ResourceRequest resReq,
      Resource maximumResource, String queueName, YarnScheduler scheduler,
      RMContext rmContext, QueueInfo queueInfo); static boolean checkQueueLabelExpression(Set<String> queueLabels,
      String labelExpression, RMContext rmContext); static AccessType toAccessType(QueueACL acl); static boolean checkResourceRequestMatchingNodePartition(
      String requestedPartition, String nodePartition,
      SchedulingMode schedulingMode); @Private static boolean hasPendingResourceRequest(ResourceCalculator rc,
      ResourceUsage usage, String nodePartition, Resource cluster,
      SchedulingMode schedulingMode); static final String RELEASED_CONTAINER; static final String LOST_CONTAINER; static final String PREEMPTED_CONTAINER; static final String COMPLETED_APPLICATION; static final String EXPIRED_CONTAINER; static final String UNRESERVED_CONTAINER; }
@Test (timeout = 30000) public void testValidateResourceRequestWithErrorLabelsPermission() throws IOException { YarnScheduler scheduler = mock(YarnScheduler.class); Set<String> queueAccessibleNodeLabels = Sets.newHashSet(); QueueInfo queueInfo = mock(QueueInfo.class); when(queueInfo.getQueueName()).thenReturn("queue"); when(queueInfo.getAccessibleNodeLabels()).thenReturn(queueAccessibleNodeLabels); when(scheduler.getQueueInfo(any(String.class), anyBoolean(), anyBoolean())) .thenReturn(queueInfo); Resource maxResource = Resources.createResource( YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB, YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES); try { queueAccessibleNodeLabels.clear(); queueAccessibleNodeLabels.addAll(Arrays.asList("x", "y")); rmContext.getNodeLabelManager().addToCluserNodeLabels( ImmutableSet.of(NodeLabel.newInstance("x"), NodeLabel.newInstance("y"))); Resource resource = Resources.createResource( 0, YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES); ResourceRequest resReq = BuilderUtils.newResourceRequest( mock(Priority.class), ResourceRequest.ANY, resource, 1); resReq.setNodeLabelExpression("x"); SchedulerUtils.normalizeAndvalidateRequest(resReq, maxResource, "queue", scheduler, rmContext); resReq.setNodeLabelExpression("y"); SchedulerUtils.normalizeAndvalidateRequest(resReq, maxResource, "queue", scheduler, rmContext); resReq.setNodeLabelExpression(""); SchedulerUtils.normalizeAndvalidateRequest(resReq, maxResource, "queue", scheduler, rmContext); resReq.setNodeLabelExpression(" "); SchedulerUtils.normalizeAndvalidateRequest(resReq, maxResource, "queue", scheduler, rmContext); } catch (InvalidResourceRequestException e) { e.printStackTrace(); fail("Should be valid when request labels is a subset of queue labels"); } finally { rmContext.getNodeLabelManager().removeFromClusterNodeLabels( Arrays.asList("x", "y")); } try { queueAccessibleNodeLabels.clear(); queueAccessibleNodeLabels.addAll(Arrays.asList("x", "y")); Resource resource = Resources.createResource( 0, YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES); ResourceRequest resReq = BuilderUtils.newResourceRequest( mock(Priority.class), ResourceRequest.ANY, resource, 1); resReq.setNodeLabelExpression("x"); SchedulerUtils.normalizeAndvalidateRequest(resReq, maxResource, "queue", scheduler, rmContext); fail("Should fail"); } catch (InvalidResourceRequestException e) { } try { queueAccessibleNodeLabels.clear(); queueAccessibleNodeLabels.addAll(Arrays.asList("x", "y")); rmContext.getNodeLabelManager().addToCluserNodeLabels( ImmutableSet.of(NodeLabel.newInstance("x"), NodeLabel.newInstance("y"))); Resource resource = Resources.createResource( 0, YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES); ResourceRequest resReq = BuilderUtils.newResourceRequest( mock(Priority.class), ResourceRequest.ANY, resource, 1); resReq.setNodeLabelExpression("z"); SchedulerUtils.normalizeAndvalidateRequest(resReq, maxResource, "queue", scheduler, rmContext); fail("Should fail"); } catch (InvalidResourceRequestException e) { } finally { rmContext.getNodeLabelManager().removeFromClusterNodeLabels( Arrays.asList("x", "y")); } try { queueAccessibleNodeLabels.clear(); queueAccessibleNodeLabels.addAll(Arrays.asList("x", "y")); rmContext.getNodeLabelManager().addToCluserNodeLabels( ImmutableSet.of(NodeLabel.newInstance("x"), NodeLabel.newInstance("y"))); Resource resource = Resources.createResource( 0, YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES); ResourceRequest resReq = BuilderUtils.newResourceRequest( mock(Priority.class), ResourceRequest.ANY, resource, 1); resReq.setNodeLabelExpression("x && y"); SchedulerUtils.normalizeAndvalidateRequest(resReq, maxResource, "queue", scheduler, rmContext); fail("Should fail"); } catch (InvalidResourceRequestException e) { } finally { rmContext.getNodeLabelManager().removeFromClusterNodeLabels( Arrays.asList("x", "y")); } queueAccessibleNodeLabels.clear(); try { queueAccessibleNodeLabels.clear(); Resource resource = Resources.createResource( 0, YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES); ResourceRequest resReq = BuilderUtils.newResourceRequest( mock(Priority.class), ResourceRequest.ANY, resource, 1); SchedulerUtils.normalizeAndvalidateRequest(resReq, maxResource, "queue", scheduler, rmContext); resReq.setNodeLabelExpression(""); SchedulerUtils.normalizeAndvalidateRequest(resReq, maxResource, "queue", scheduler, rmContext); resReq.setNodeLabelExpression(" "); SchedulerUtils.normalizeAndvalidateRequest(resReq, maxResource, "queue", scheduler, rmContext); } catch (InvalidResourceRequestException e) { e.printStackTrace(); fail("Should be valid when request labels is empty"); } try { queueAccessibleNodeLabels.clear(); rmContext.getNodeLabelManager().addToCluserNodeLabels( ImmutableSet.of(NodeLabel.newInstance("x"))); Resource resource = Resources.createResource( 0, YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES); ResourceRequest resReq = BuilderUtils.newResourceRequest( mock(Priority.class), ResourceRequest.ANY, resource, 1); resReq.setNodeLabelExpression("x"); SchedulerUtils.normalizeAndvalidateRequest(resReq, maxResource, "queue", scheduler, rmContext); fail("Should fail"); } catch (InvalidResourceRequestException e) { } finally { rmContext.getNodeLabelManager().removeFromClusterNodeLabels( Arrays.asList("x")); } try { queueAccessibleNodeLabels.clear(); queueAccessibleNodeLabels.add(RMNodeLabelsManager.ANY); rmContext.getNodeLabelManager().addToCluserNodeLabels( ImmutableSet.of(NodeLabel.newInstance("x"), NodeLabel.newInstance("y"), NodeLabel.newInstance("z"))); Resource resource = Resources.createResource( 0, YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES); ResourceRequest resReq = BuilderUtils.newResourceRequest( mock(Priority.class), ResourceRequest.ANY, resource, 1); resReq.setNodeLabelExpression("x"); SchedulerUtils.normalizeAndvalidateRequest(resReq, maxResource, "queue", scheduler, rmContext); resReq.setNodeLabelExpression("y"); SchedulerUtils.normalizeAndvalidateRequest(resReq, maxResource, "queue", scheduler, rmContext); resReq.setNodeLabelExpression("z"); SchedulerUtils.normalizeAndvalidateRequest(resReq, maxResource, "queue", scheduler, rmContext); } catch (InvalidResourceRequestException e) { e.printStackTrace(); fail("Should be valid when queue can access any labels"); } finally { rmContext.getNodeLabelManager().removeFromClusterNodeLabels( Arrays.asList("x", "y", "z")); } try { queueAccessibleNodeLabels.clear(); queueAccessibleNodeLabels.add(RMNodeLabelsManager.ANY); Resource resource = Resources.createResource( 0, YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES); ResourceRequest resReq = BuilderUtils.newResourceRequest( mock(Priority.class), ResourceRequest.ANY, resource, 1); resReq.setNodeLabelExpression("x"); SchedulerUtils.normalizeAndvalidateRequest(resReq, maxResource, "queue", scheduler, rmContext); fail("Should fail"); } catch (InvalidResourceRequestException e) { } try { queueAccessibleNodeLabels.clear(); queueAccessibleNodeLabels.addAll(Arrays.asList("x", "y")); rmContext.getNodeLabelManager().addToCluserNodeLabels( ImmutableSet.of(NodeLabel.newInstance("x"), NodeLabel.newInstance("y"))); Resource resource = Resources.createResource( 0, YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES); ResourceRequest resReq = BuilderUtils.newResourceRequest( mock(Priority.class), "rack", resource, 1); resReq.setNodeLabelExpression("x"); SchedulerUtils.normalizeAndvalidateRequest(resReq, maxResource, "queue", scheduler, rmContext); fail("Should fail"); } catch (InvalidResourceRequestException e) { } finally { rmContext.getNodeLabelManager().removeFromClusterNodeLabels( Arrays.asList("x", "y")); } try { queueAccessibleNodeLabels.clear(); queueAccessibleNodeLabels.addAll(Arrays .asList(CommonNodeLabelsManager.ANY)); rmContext.getNodeLabelManager().addToCluserNodeLabels( ImmutableSet.of(NodeLabel.newInstance("x"))); Resource resource = Resources.createResource( 0, YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES); ResourceRequest resReq = BuilderUtils.newResourceRequest( mock(Priority.class), "rack", resource, 1); resReq.setNodeLabelExpression("x"); SchedulerUtils.normalizeAndvalidateRequest(resReq, maxResource, "queue", scheduler, rmContext); fail("Should fail"); } catch (InvalidResourceRequestException e) { } finally { rmContext.getNodeLabelManager().removeFromClusterNodeLabels( Arrays.asList("x")); } }
@Test (timeout = 30000) public void testNormalizeNodeLabelExpression() throws IOException { YarnScheduler scheduler = mock(YarnScheduler.class); Set<String> queueAccessibleNodeLabels = Sets.newHashSet(); QueueInfo queueInfo = mock(QueueInfo.class); when(queueInfo.getQueueName()).thenReturn("queue"); when(queueInfo.getAccessibleNodeLabels()).thenReturn(queueAccessibleNodeLabels); when(queueInfo.getDefaultNodeLabelExpression()).thenReturn(" x "); when(scheduler.getQueueInfo(any(String.class), anyBoolean(), anyBoolean())) .thenReturn(queueInfo); Resource maxResource = Resources.createResource( YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB, YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES); try { queueAccessibleNodeLabels.clear(); queueAccessibleNodeLabels.addAll(Arrays.asList("x", "y")); rmContext.getNodeLabelManager().addToCluserNodeLabels( ImmutableSet.of(NodeLabel.newInstance("x"), NodeLabel.newInstance("y"))); Resource resource = Resources.createResource( 0, YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES); ResourceRequest resReq = BuilderUtils.newResourceRequest( mock(Priority.class), ResourceRequest.ANY, resource, 1); SchedulerUtils.normalizeAndvalidateRequest(resReq, maxResource, "queue", scheduler, rmContext); Assert.assertTrue(resReq.getNodeLabelExpression().equals("x")); resReq.setNodeLabelExpression(" y "); SchedulerUtils.normalizeAndvalidateRequest(resReq, maxResource, "queue", scheduler, rmContext); Assert.assertTrue(resReq.getNodeLabelExpression().equals("y")); } catch (InvalidResourceRequestException e) { e.printStackTrace(); fail("Should be valid when request labels is a subset of queue labels"); } finally { rmContext.getNodeLabelManager().removeFromClusterNodeLabels( Arrays.asList("x", "y")); } }