Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

YARN-7769. FS QueueManager should not create default queue at init. Contributed by Benjamin Teke

This commit is contained in:
Szilard Nemeth 2021-05-22 14:55:01 +02:00
parent c70ee2d548
commit 1e44bdb84c
10 changed files with 60 additions and 39 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueueManager.java
View File

@ -107,10 +107,6 @@ public class QueueManager {
rootQueue.setDynamic(false);
queues.put(rootQueue.getName(), rootQueue);
// Create the default queue
FSLeafQueue defaultQueue =
getLeafQueue(YarnConfiguration.DEFAULT_QUEUE_NAME, true);
defaultQueue.setDynamic(false);
// Recursively reinitialize to propagate queue properties
rootQueue.reinit(true);
}

10
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMRestart.java
View File

@ -20,6 +20,7 @@ package org.apache.hadoop.yarn.server.resourcemanager.applicationsmanager;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
@ -1313,8 +1314,13 @@ public class TestAMRestart extends ParameterizedSchedulerTestBase {
Assert.assertEquals(0,
queue.getQueueResourceUsage().getUsed().getVirtualCores());
} else if (getSchedulerType() == SchedulerType.FAIR) {
FSLeafQueue queue = ((FairScheduler) scheduler).getQueueManager()
.getLeafQueue("root.default", false);
// The default queue is not auto created after YARN-7769 so
// user-as-default-queue option is used
Collection<FSLeafQueue> queues = ((FairScheduler) scheduler)
.getQueueManager().getLeafQueues();
Assert.assertEquals(1, queues.size());
FSLeafQueue queue = queues.iterator().next();
Assert.assertEquals(0, queue.getResourceUsage().getMemorySize());
Assert.assertEquals(0, queue.getResourceUsage().getVirtualCores());
}

6
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestAppRunnability.java
View File

@ -125,9 +125,8 @@ public class TestAppRunnability extends FairSchedulerTestBase {
new AppAttemptAddedSchedulerEvent(createAppAttemptId(1, 1), false);
scheduler.handle(attemptAddedEvent);
// Scheduler should have two queues (the default and the one created for
// user1)
assertEquals(2, scheduler.getQueueManager().getLeafQueues().size());
// Scheduler should have one queue (the one created for user1)
assertEquals(1, scheduler.getQueueManager().getLeafQueues().size());
// That queue should have one app
assertEquals(1, scheduler.getQueueManager().getLeafQueue("user1", true)
@ -174,6 +173,7 @@ public class TestAppRunnability extends FairSchedulerTestBase {
AllocationFileWriter.create()
.addQueue(new AllocationFileQueue.Builder("jerry").build())
.addQueue(new AllocationFileQueue.Builder("default").build())
.writeToFile(ALLOC_FILE);
// Restarting resource manager since the file location and content is

2
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSLeafQueue.java
View File

@ -147,7 +147,7 @@ public class TestFSLeafQueue extends FairSchedulerTestBase {
createSchedulingRequest(1 * 1024, "queueB", "user1");
scheduler.update();
Collection<FSLeafQueue> queues = scheduler.getQueueManager().getLeafQueues();
assertEquals(3, queues.size());
assertEquals(2, queues.size());
}
@Test

58
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
View File

@ -1074,22 +1074,22 @@ public class TestFairScheduler extends FairSchedulerTestBase {
QueueManager queueManager = scheduler.getQueueManager();
FSLeafQueue leafQueue = queueManager.getLeafQueue("parent.child", true);
Assert.assertEquals(2, queueManager.getLeafQueues().size());
Assert.assertEquals(1, queueManager.getLeafQueues().size());
Assert.assertNotNull(leafQueue);
Assert.assertEquals("root.parent.child", leafQueue.getName());
FSLeafQueue leafQueue2 = queueManager.getLeafQueue("parent", true);
Assert.assertNull(leafQueue2);
Assert.assertEquals(2, queueManager.getLeafQueues().size());
Assert.assertEquals(1, queueManager.getLeafQueues().size());
FSLeafQueue leafQueue3 = queueManager.getLeafQueue("parent.child.grandchild", true);
Assert.assertNull(leafQueue3);
Assert.assertEquals(2, queueManager.getLeafQueues().size());
Assert.assertEquals(1, queueManager.getLeafQueues().size());
FSLeafQueue leafQueue4 = queueManager.getLeafQueue("parent.sister", true);
Assert.assertNotNull(leafQueue4);
Assert.assertEquals("root.parent.sister", leafQueue4.getName());
Assert.assertEquals(3, queueManager.getLeafQueues().size());
Assert.assertEquals(2, queueManager.getLeafQueues().size());
}
@Test
@ -1771,8 +1771,8 @@ public class TestFairScheduler extends FairSchedulerTestBase {
scheduler.start();
scheduler.reinitialize(conf, resourceManager.getRMContext());
// only default queue
assertEquals(1, scheduler.getQueueManager().getLeafQueues().size());
// no queue by default
assertEquals(0, scheduler.getQueueManager().getLeafQueues().size());
// Submit app with empty queue
// Submit fails before we reach the placement check.
@ -1783,7 +1783,7 @@ public class TestFairScheduler extends FairSchedulerTestBase {
scheduler.handle(appAddedEvent);
// submission rejected
assertEquals(1, scheduler.getQueueManager().getLeafQueues().size());
assertEquals(0, scheduler.getQueueManager().getLeafQueues().size());
assertNull(scheduler.getSchedulerApp(appAttemptId));
assertEquals(0, resourceManager.getRMContext().getRMApps().size());
}
@ -1794,8 +1794,8 @@ public class TestFairScheduler extends FairSchedulerTestBase {
scheduler.start();
scheduler.reinitialize(conf, resourceManager.getRMContext());
// only default queue
assertEquals(1, scheduler.getQueueManager().getLeafQueues().size());
// no queue by default
assertEquals(0, scheduler.getQueueManager().getLeafQueues().size());
// Submit app with queue name (.A)
// Submit fails before we reach the placement check.
@ -1805,7 +1805,7 @@ public class TestFairScheduler extends FairSchedulerTestBase {
"user1");
scheduler.handle(appAddedEvent1);
// submission rejected
assertEquals(1, scheduler.getQueueManager().getLeafQueues().size());
assertEquals(0, scheduler.getQueueManager().getLeafQueues().size());
assertNull(scheduler.getSchedulerApp(appAttemptId1));
assertEquals(0, resourceManager.getRMContext().getRMApps().size());
@ -1817,7 +1817,7 @@ public class TestFairScheduler extends FairSchedulerTestBase {
"user1");
scheduler.handle(appAddedEvent2);
// submission rejected
assertEquals(1, scheduler.getQueueManager().getLeafQueues().size());
assertEquals(0, scheduler.getQueueManager().getLeafQueues().size());
assertNull(scheduler.getSchedulerApp(appAttemptId2));
assertEquals(0, resourceManager.getRMContext().getRMApps().size());
@ -1829,7 +1829,7 @@ public class TestFairScheduler extends FairSchedulerTestBase {
"user1", new ApplicationPlacementContext("A.B"));
scheduler.handle(appAddedEvent3);
// submission accepted
assertEquals(2, scheduler.getQueueManager().getLeafQueues().size());
assertEquals(1, scheduler.getQueueManager().getLeafQueues().size());
assertNull(scheduler.getSchedulerApp(appAttemptId3));
assertEquals(0, resourceManager.getRMContext().getRMApps().size());
}
@ -1863,7 +1863,7 @@ public class TestFairScheduler extends FairSchedulerTestBase {
scheduler.update();
Collection<FSLeafQueue> queues = scheduler.getQueueManager().getLeafQueues();
assertEquals(3, queues.size());
assertEquals(2, queues.size());
for (FSLeafQueue p : queues) {
if (p.getName().equals("root.queueA")) {
@ -1944,6 +1944,8 @@ public class TestFairScheduler extends FairSchedulerTestBase {
.fairDefaultQueueSchedulingPolicy()
.addQueue(new AllocationFileQueue.Builder("root")
.schedulingPolicy("drf")
.subQueue(new AllocationFileQueue.Builder("default")
.weight(1).build())
.subQueue(new AllocationFileQueue.Builder("child1")
.weight(1).build())
.subQueue(new AllocationFileQueue.Builder("child2")
@ -1981,6 +1983,8 @@ public class TestFairScheduler extends FairSchedulerTestBase {
.fairDefaultQueueSchedulingPolicy()
.addQueue(new AllocationFileQueue.Builder("root")
.schedulingPolicy("drf")
.subQueue(new AllocationFileQueue.Builder("default")
.weight(1).build())
.subQueue(new AllocationFileQueue.Builder("child1")
.weight(1).build())
.subQueue(new AllocationFileQueue.Builder("child2")
@ -2015,6 +2019,17 @@ public class TestFairScheduler extends FairSchedulerTestBase {
conf.setClass(CommonConfigurationKeys.HADOOP_SECURITY_GROUP_MAPPING,
SimpleGroupsMapping.class, GroupMappingServiceProvider.class);
conf.set(FairSchedulerConfiguration.USER_AS_DEFAULT_QUEUE, "true");
conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE);
AllocationFileWriter.create()
.fairDefaultQueueSchedulingPolicy()
.addQueue(new AllocationFileQueue.Builder("root")
.schedulingPolicy("drf")
.subQueue(new AllocationFileQueue.Builder("default")
.weight(1).build())
.build())
.writeToFile(ALLOC_FILE);
scheduler.init(conf);
scheduler.start();
scheduler.reinitialize(conf, resourceManager.getRMContext());
@ -2129,13 +2144,12 @@ public class TestFairScheduler extends FairSchedulerTestBase {
QueueManager queueManager = scheduler.getQueueManager();
Collection<FSLeafQueue> leafQueues = queueManager.getLeafQueues();
Assert.assertEquals(4, leafQueues.size());
Assert.assertEquals(3, leafQueues.size());
Assert.assertNotNull(queueManager.getLeafQueue("queueA", false));
Assert.assertNotNull(queueManager.getLeafQueue("queueB.queueC", false));
Assert.assertNotNull(queueManager.getLeafQueue("queueB.queueD", false));
Assert.assertNotNull(queueManager.getLeafQueue("default", false));
// Make sure querying for queues didn't create any new ones:
Assert.assertEquals(4, leafQueues.size());
Assert.assertEquals(3, leafQueues.size());
}
@Test
@ -4736,8 +4750,8 @@ public class TestFairScheduler extends FairSchedulerTestBase {
scheduler.start();
scheduler.reinitialize(conf, resourceManager.getRMContext());
// only default queue
assertEquals(1, scheduler.getQueueManager().getLeafQueues().size());
// no queue by default
assertEquals(0, scheduler.getQueueManager().getLeafQueues().size());
// submit app with queue name "A"
ApplicationAttemptId appAttemptId1 = createAppAttemptId(1, 1);
@ -4747,7 +4761,7 @@ public class TestFairScheduler extends FairSchedulerTestBase {
appAttemptId1.getApplicationId(), "A", "user1", apc);
scheduler.handle(appAddedEvent1);
// submission accepted
assertEquals(2, scheduler.getQueueManager().getLeafQueues().size());
assertEquals(1, scheduler.getQueueManager().getLeafQueues().size());
assertNotNull(scheduler.getSchedulerApplications().get(appAttemptId1.
getApplicationId()));
@ -4771,7 +4785,7 @@ public class TestFairScheduler extends FairSchedulerTestBase {
// expected ignore: rules should have filtered this out
}
// submission rejected
assertEquals(2, scheduler.getQueueManager().getLeafQueues().size());
assertEquals(1, scheduler.getQueueManager().getLeafQueues().size());
assertNull(scheduler.getSchedulerApplications().get(appAttemptId2.
getApplicationId()));
assertNull(scheduler.getSchedulerApp(appAttemptId2));
@ -4783,7 +4797,7 @@ public class TestFairScheduler extends FairSchedulerTestBase {
appAttemptId3.getApplicationId(), "B.C", "user1", apc);
scheduler.handle(appAddedEvent3);
// submission accepted
assertEquals(3, scheduler.getQueueManager().getLeafQueues().size());
assertEquals(2, scheduler.getQueueManager().getLeafQueues().size());
assertNotNull(scheduler.getSchedulerApplications().get(appAttemptId3.
getApplicationId()));
@ -4806,7 +4820,7 @@ public class TestFairScheduler extends FairSchedulerTestBase {
// expected ignore: rules should have filtered this out
}
// submission rejected
assertEquals(3, scheduler.getQueueManager().getLeafQueues().size());
assertEquals(2, scheduler.getQueueManager().getLeafQueues().size());
assertNull(scheduler.getSchedulerApplications().get(appAttemptId4.
getApplicationId()));
assertNull(scheduler.getSchedulerApp(appAttemptId4));

3
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerFairShare.java
View File

@ -68,6 +68,9 @@ public class TestFairSchedulerFairShare extends FairSchedulerTestBase {
String policy) {
AllocationFileWriter allocationFileWriter = AllocationFileWriter.create()
.addQueue(new AllocationFileQueue.Builder("root")
.subQueue(new AllocationFileQueue.Builder("default")
.weight(1)
.build())
.subQueue(new AllocationFileQueue.Builder("parentA")
.weight(8)
.subQueue(new AllocationFileQueue.Builder("childA1").build())

7
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestQueueManager.java
View File

@ -116,10 +116,11 @@ public class TestQueueManager {
assertNotNull(queueManager.getLeafQueue("queue1.queue2", false));
assertNull(queueManager.getLeafQueue("queue1", false));
// Should never to be able to create a queue under the default queue
// Since YARN-7769 FS doesn't create the default queue during init, so
// it should be possible to create a queue under the root.default queue
updateConfiguredLeafQueues(queueManager, "default.queue3");
assertNull(queueManager.getLeafQueue("default.queue3", false));
assertNotNull(queueManager.getLeafQueue("default", false));
assertNotNull(queueManager.getLeafQueue("default.queue3", false));
assertNull(queueManager.getLeafQueue("default", false));
}
/**

2
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestFSQueueConverter.java
View File

@ -151,7 +151,7 @@ public class TestFSQueueConverter {
converter.convertQueueHierarchy(rootQueue);
// root children
assertEquals("root children", "default,admins,users,misc",
assertEquals("root children", "admins,users,misc,default",
csConfig.get(PREFIX + "root.queues"));
// root.admins children

2
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/fairscheduler/TestRMWebServicesFairScheduler.java
View File

@ -132,7 +132,7 @@ public class TestRMWebServicesFairScheduler extends JerseyTestBase {
JSONObject json = response.getEntity(JSONObject.class);
JSONArray subQueueInfo = json.getJSONObject("scheduler")
.getJSONObject("schedulerInfo").getJSONObject("rootQueue")
.getJSONObject("childQueues").getJSONArray("queue").getJSONObject(1)
.getJSONObject("childQueues").getJSONArray("queue").getJSONObject(0)
.getJSONObject("childQueues").getJSONArray("queue");
// subQueueInfo is consist of subqueue1 and subqueue2 info
assertEquals(2, subQueueInfo.length());

5
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/fairscheduler/TestRMWebServicesFairSchedulerCustomResourceTypes.java
View File

@ -220,8 +220,9 @@ public class TestRMWebServicesFairSchedulerCustomResourceTypes
.getJSONObject("schedulerInfo").getJSONObject("rootQueue")
.getJSONObject("childQueues").getJSONArray("queue");
// childQueueInfo consists of subqueue1 and subqueue2 info
assertEquals(2, queues.length());
assertEquals(1, queues.length());
// firstChildQueue info contains subqueue1 and subqueue2 info
JSONObject firstChildQueue = queues.getJSONObject(0);
new FairSchedulerJsonVerifications(customResourceTypes)
.verify(firstChildQueue);