FSQueueConverter { public void convertQueueHierarchy(FSQueue queue) { List<FSQueue> children = queue.getChildQueues(); final String queueName = queue.getName(); emitChildQueues(queueName, children); emitMaxAMShare(queueName, queue); emitMaxParallelApps(queueName, queue); emitMaxAllocations(queueName, queue); emitPreemptionDisabled(queueName, queue); emitChildCapacity(queue); emitMaximumCapacity(queueName, queue); emitSizeBasedWeight(queueName); emitOrderingPolicy(queueName, queue); checkMaxChildCapacitySetting(queue); for (FSQueue childQueue : children) { convertQueueHierarchy(childQueue); } } FSQueueConverter(FSQueueConverterBuilder builder); void convertQueueHierarchy(FSQueue queue); static final float QUEUE_MAX_AM_SHARE_DISABLED; }
@Test public void testConvertQueueHierarchy() { converter = builder.build(); converter.convertQueueHierarchy(rootQueue); assertEquals("root children", "default,admins,users", csConfig.get(PREFIX + "root.queues")); assertEquals("root.admins children", "bob,alice", csConfig.get(PREFIX + "root.admins.queues")); assertNull("root.default children", csConfig.get(PREFIX + "root.default" + ".queues")); assertEquals("root.users children", "john,joe", csConfig.get(PREFIX + "root.users.queues")); Set<String> leafs = Sets.difference(ALL_QUEUES, Sets.newHashSet("root", "root.default", "root.admins", "root.users")); assertNoValueForQueues(leafs, ".queues", csConfig); }
@Test public void testQueueMaxAMShare() { converter = builder.build(); converter.convertQueueHierarchy(rootQueue); assertEquals("root.admins.bob AM share", "1.0", csConfig.get(PREFIX + "root.admins.bob.maximum-am-resource-percent")); assertEquals("root.admins.alice AM share", "0.15", csConfig.get(PREFIX + "root.admins.alice.maximum-am-resource-percent")); Set<String> remaining = Sets.difference(ALL_QUEUES, Sets.newHashSet("root.admins.bob", "root.admins.alice")); assertNoValueForQueues(remaining, ".maximum-am-resource-percent", csConfig); }
@Test public void testQueueMaxParallelApps() { converter = builder.build(); converter.convertQueueHierarchy(rootQueue); assertEquals("root.admins.alice max apps", 2, csConfig.getInt(PREFIX + "root.admins.alice.max-parallel-apps", -1)); Set<String> remaining = Sets.difference(ALL_QUEUES, Sets.newHashSet("root.admins.alice")); assertNoValueForQueues(remaining, ".max-parallel-apps", csConfig); }
@Test public void testQueueMaxAllocations() { converter = builder.build(); converter.convertQueueHierarchy(rootQueue); assertEquals("root.admins max vcores", 3, csConfig.getInt(PREFIX + "root.admins.maximum-allocation-vcores", -1)); assertEquals("root.admins max memory", 4096, csConfig.getInt(PREFIX + "root.admins.maximum-allocation-mb", -1)); assertEquals("root.users.john max vcores", 2, csConfig.getInt(PREFIX + "root.users.john.maximum-allocation-vcores", -1)); assertEquals("root.users.john max memory", 8192, csConfig.getInt(PREFIX + "root.users.john.maximum-allocation-mb", -1)); Set<String> remaining = Sets.difference(ALL_QUEUES, Sets.newHashSet("root.admins", "root.users.john")); assertNoValueForQueues(remaining, ".maximum-allocation-vcores", csConfig); assertNoValueForQueues(remaining, ".maximum-allocation-mb", csConfig); }
@Test public void testQueuePreemptionDisabled() { converter = builder.withPreemptionEnabled(true).build(); converter.convertQueueHierarchy(rootQueue); assertTrue("root.admins.alice preemption setting", csConfig.getBoolean(PREFIX + "root.admins.alice.disable_preemption", false)); assertTrue("root.users.joe preemption setting", csConfig.getBoolean(PREFIX + "root.users.joe.disable_preemption", false)); Set<String> remaining = Sets.difference(ALL_QUEUES, Sets.newHashSet("root.admins.alice", "root.users.joe")); assertNoValueForQueues(remaining, ".disable_preemption", csConfig); }
@Test public void testQueuePreemptionDisabledWhenGlobalPreemptionDisabled() { converter = builder.build(); converter.convertQueueHierarchy(rootQueue); assertNoValueForQueues(ALL_QUEUES, ".disable_preemption", csConfig); }
@Test public void testChildCapacity() { converter = builder.build(); converter.convertQueueHierarchy(rootQueue); assertEquals("root.default capacity", "33.333", csConfig.get(PREFIX + "root.default.capacity")); assertEquals("root.admins capacity", "33.333", csConfig.get(PREFIX + "root.admins.capacity")); assertEquals("root.users capacity", "33.334", csConfig.get(PREFIX + "root.users.capacity")); assertEquals("root.users.john capacity", "25.000", csConfig.get(PREFIX + "root.users.john.capacity")); assertEquals("root.users.joe capacity", "75.000", csConfig.get(PREFIX + "root.users.joe.capacity")); assertEquals("root.admins.alice capacity", "75.000", csConfig.get(PREFIX + "root.admins.alice.capacity")); assertEquals("root.admins.bob capacity", "25.000", csConfig.get(PREFIX + "root.admins.bob.capacity")); }
@Test public void testQueueMaximumCapacity() { converter = builder.build(); converter.convertQueueHierarchy(rootQueue); assertValueForQueues(ALL_QUEUES, ".maximum-capacity", csConfig, "100"); verify(ruleHandler, times(3)).handleMaxResources(); }
@Test public void testQueueMinimumCapacity() { converter = builder.build(); converter.convertQueueHierarchy(rootQueue); verify(ruleHandler, times(2)).handleMinResources(); }
@Test public void testQueueWithNoAutoCreateChildQueue() { converter = builder .withCapacitySchedulerConfig(csConfig) .build(); converter.convertQueueHierarchy(rootQueue); assertNoValueForQueues(ALL_QUEUES, ".auto-create-child-queue.enabled", csConfig); }
@Test public void testQueueSizeBasedWeightEnabled() { converter = builder.withSizeBasedWeight(true).build(); converter.convertQueueHierarchy(rootQueue); assertTrueForQueues(ALL_QUEUES, ".ordering-policy.fair.enable-size-based-weight", csConfig); }
@Test public void testQueueSizeBasedWeightDisabled() { converter = builder.build(); converter.convertQueueHierarchy(rootQueue); assertNoValueForQueues(ALL_QUEUES, ".ordering-policy.fair.enable-size-based-weight", csConfig); }
@Test public void testQueueOrderingPolicy() throws Exception { converter = builder.build(); String absolutePath = new File("src/test/resources/fair-scheduler-orderingpolicy.xml") .getAbsolutePath(); yarnConfig.set(FairSchedulerConfiguration.ALLOCATION_FILE, FILE_PREFIX + absolutePath); fs.close(); fs = createFairScheduler(); rootQueue = fs.getQueueManager().getRootQueue(); converter.convertQueueHierarchy(rootQueue); assertEquals("root ordering policy", null, csConfig.get(PREFIX + "root.ordering-policy")); assertEquals("root.default ordering policy", "fair", csConfig.get(PREFIX + "root.default.ordering-policy")); assertEquals("root.admins ordering policy", null, csConfig.get(PREFIX + "root.admins.ordering-policy")); assertEquals("root.users ordering policy", null, csConfig.get(PREFIX + "root.users.ordering-policy")); assertEquals("root.users.joe ordering policy", "fair", csConfig.get(PREFIX + "root.users.joe.ordering-policy")); assertEquals("root.users.john ordering policy", "fifo", csConfig.get(PREFIX + "root.users.john.ordering-policy")); assertEquals("root.admins.alice ordering policy", "fifo", csConfig.get(PREFIX + "root.admins.alice.ordering-policy")); assertEquals("root.admins.bob ordering policy", "fair", csConfig.get(PREFIX + "root.admins.bob.ordering-policy")); }
@Test public void testQueueUnsupportedMixedOrderingPolicy() throws IOException { converter = builder.withDrfUsed(true).build(); String absolutePath = new File("src/test/resources/fair-scheduler-orderingpolicy-mixed.xml") .getAbsolutePath(); yarnConfig.set(FairSchedulerConfiguration.ALLOCATION_FILE, FILE_PREFIX + absolutePath); fs.close(); fs = createFairScheduler(); rootQueue = fs.getQueueManager().getRootQueue(); converter.convertQueueHierarchy(rootQueue); verify(ruleHandler, times(5)).handleFairAsDrf(anyString()); }
@Test public void testQueueMaxChildCapacityNotSupported() { converter = builder.build(); expectedException.expect(UnsupportedPropertyException.class); expectedException.expectMessage("test"); Mockito.doThrow(new UnsupportedPropertyException("test")) .when(ruleHandler).handleMaxChildCapacity(); converter.convertQueueHierarchy(rootQueue); }
@Test public void testReservationSystemNotSupported() { converter = builder.build(); expectedException.expect(UnsupportedPropertyException.class); expectedException.expectMessage("maxCapacity"); Mockito.doThrow(new UnsupportedPropertyException("maxCapacity")) .when(ruleHandler).handleMaxChildCapacity(); yarnConfig.setBoolean(YarnConfiguration.RM_RESERVATION_SYSTEM_ENABLE, true); converter.convertQueueHierarchy(rootQueue); }