Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

YARN-2336. Fair scheduler's REST API returns a missing '[' bracket JSON for deep queue tree. Contributed by Kenji Kikushima and Akira Ajisaka.

This commit is contained in:
Tsuyoshi Ozawa 2015-05-26 19:07:40 +09:00
parent 56996a685e
commit 9a3d617b63
7 changed files with 250 additions and 131 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
hadoop-yarn-project/CHANGES.txt
View File

@ -52,6 +52,9 @@ Release 2.8.0 - UNRELEASED
INCOMPATIBLE CHANGES
YARN-2336. Fair scheduler's REST API returns a missing '[' bracket JSON for
deep queue tree. (Kenji Kikushima and Akira Ajisaka via ozawa)
NEW FEATURES
YARN-3345. Add non-exclusive node label API. (Wangda Tan via jianhe)

3
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/JAXBContextResolver.java
View File

@ -53,7 +53,8 @@ public class JAXBContextResolver implements ContextResolver<JAXBContext> {
NodesInfo.class, RemoteExceptionData.class,
CapacitySchedulerQueueInfoList.class, ResourceInfo.class,
UsersInfo.class, UserInfo.class, ApplicationStatisticsInfo.class,
StatisticsItemInfo.class, CapacitySchedulerHealthInfo.class };
StatisticsItemInfo.class, CapacitySchedulerHealthInfo.class,
FairSchedulerQueueInfoList.class};
// these dao classes need root unwrapping
final Class[] rootUnwrappedTypes =
{ NewApplication.class, ApplicationSubmissionContextInfo.class,

25
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/FairSchedulerQueueInfo.java
View File

@ -19,7 +19,6 @@
package org.apache.hadoop.yarn.server.resourcemanager.webapp.dao;
import java.util.ArrayList;
import java.util.Collection;
import javax.xml.bind.annotation.XmlAccessType;
@ -61,7 +60,7 @@ public class FairSchedulerQueueInfo {
private String queueName;
private String schedulingPolicy;
private Collection<FairSchedulerQueueInfo> childQueues;
private FairSchedulerQueueInfoList childQueues;
public FairSchedulerQueueInfo() {
}
@ -95,20 +94,34 @@ public class FairSchedulerQueueInfo {
maxApps = allocConf.getQueueMaxApps(queueName);
childQueues = new ArrayList<FairSchedulerQueueInfo>();
if (allocConf.isReservable(queueName) &&
!allocConf.getShowReservationAsQueues(queueName)) {
return;
}
childQueues = getChildQueues(queue, scheduler);
}
protected FairSchedulerQueueInfoList getChildQueues(FSQueue queue,
FairScheduler scheduler) {
// Return null to omit 'childQueues' field from the return value of
// REST API if it is empty. We omit the field to keep the consistency
// with CapacitySchedulerQueueInfo, which omits 'queues' field if empty.
Collection<FSQueue> children = queue.getChildQueues();
if (children.isEmpty()) {
return null;
}
FairSchedulerQueueInfoList list = new FairSchedulerQueueInfoList();
for (FSQueue child : children) {
if (child instanceof FSLeafQueue) {
childQueues.add(new FairSchedulerLeafQueueInfo((FSLeafQueue)child, scheduler));
list.addToQueueInfoList(
new FairSchedulerLeafQueueInfo((FSLeafQueue) child, scheduler));
} else {
childQueues.add(new FairSchedulerQueueInfo(child, scheduler));
list.addToQueueInfoList(
new FairSchedulerQueueInfo(child, scheduler));
}
}
return list;
}
/**
@ -191,6 +204,6 @@ public class FairSchedulerQueueInfo {
}
public Collection<FairSchedulerQueueInfo> getChildQueues() {
return childQueues;
return childQueues.getQueueInfoList();
}
}

49
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/FairSchedulerQueueInfoList.java Normal file
View File

@ -0,0 +1,49 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.yarn.server.resourcemanager.webapp.dao;
import java.util.ArrayList;
import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType;
import javax.xml.bind.annotation.XmlRootElement;
/**
* FairScheduler QueueInfo list used for mapping to XML or JSON.
*/
@XmlRootElement
@XmlAccessorType(XmlAccessType.FIELD)
public class FairSchedulerQueueInfoList {
private ArrayList<FairSchedulerQueueInfo> queue;
public FairSchedulerQueueInfoList() {
queue = new ArrayList<>();
}
public ArrayList<FairSchedulerQueueInfo> getQueueInfoList() {
return this.queue;
}
public boolean addToQueueInfoList(FairSchedulerQueueInfo e) {
return this.queue.add(e);
}
public FairSchedulerQueueInfo getQueueInfo(int i) {
return this.queue.get(i);
}
}

10
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java
View File

@ -21,6 +21,7 @@ package org.apache.hadoop.yarn.server.resourcemanager.webapp;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
import java.io.StringReader;
@ -575,6 +576,15 @@ public class TestRMWebServicesCapacitySched extends JerseyTestBase {
user.getInt("numPendingApplications");
checkResourcesUsed(user);
}
// Verify 'queues' field is omitted from CapacitySchedulerLeafQueueInfo.
try {
b1.getJSONObject("queues");
fail("CapacitySchedulerQueueInfo should omit field 'queues'" +
"if child queue is empty.");
} catch (JSONException je) {
assertEquals("JSONObject[\"queues\"] not found.", je.getMessage());
}
} finally {
rm.stop();
}

35
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesFairScheduler.java
View File

@ -19,6 +19,7 @@
package org.apache.hadoop.yarn.server.resourcemanager.webapp;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.fail;
import javax.ws.rs.core.MediaType;
@ -27,8 +28,10 @@ import org.apache.hadoop.yarn.server.resourcemanager.MockRM;
import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.QueueManager;
import org.apache.hadoop.yarn.webapp.GenericExceptionHandler;
import org.apache.hadoop.yarn.webapp.JerseyTestBase;
import org.codehaus.jettison.json.JSONArray;
import org.codehaus.jettison.json.JSONException;
import org.codehaus.jettison.json.JSONObject;
import org.junit.Test;
@ -99,6 +102,38 @@ public class TestRMWebServicesFairScheduler extends JerseyTestBase {
verifyClusterScheduler(json);
}
@Test
public void testClusterSchedulerWithSubQueues() throws JSONException,
Exception {
FairScheduler scheduler = (FairScheduler)rm.getResourceScheduler();
QueueManager queueManager = scheduler.getQueueManager();
// create LeafQueue
queueManager.getLeafQueue("root.q.subqueue1", true);
queueManager.getLeafQueue("root.q.subqueue2", true);
WebResource r = resource();
ClientResponse response = r.path("ws").path("v1").path("cluster")
.path("scheduler").accept(MediaType.APPLICATION_JSON)
.get(ClientResponse.class);
assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
JSONObject json = response.getEntity(JSONObject.class);
JSONArray subQueueInfo = json.getJSONObject("scheduler")
.getJSONObject("schedulerInfo").getJSONObject("rootQueue")
.getJSONObject("childQueues").getJSONArray("queue")
.getJSONObject(1).getJSONObject("childQueues").getJSONArray("queue");
// subQueueInfo is consist of subqueue1 and subqueue2 info
assertEquals(2, subQueueInfo.length());
// Verify 'childQueues' field is omitted from FairSchedulerLeafQueueInfo.
try {
subQueueInfo.getJSONObject(1).getJSONObject("childQueues");
fail("FairSchedulerQueueInfo should omit field 'childQueues'" +
"if child queue is empty.");
} catch (JSONException je) {
assertEquals("JSONObject[\"childQueues\"] not found.", je.getMessage());
}
}
private void verifyClusterScheduler(JSONObject json) throws JSONException,
Exception {
assertEquals("incorrect number of elements", 1, json.length());

32
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/ResourceManagerRest.md
View File

@ -321,10 +321,10 @@ The capacity scheduler supports hierarchical queues. This one request will print
| usedResources | string | A string describing the current resources used by the queue |
| queueName | string | The name of the queue |
| state | string of QueueState | The state of the queue |
| queues | array of queues(JSON)/zero or more queue objects(XML) | A collection of sub-queue information |
| queues | array of queues(JSON)/zero or more queue objects(XML) | A collection of sub-queue information. Omitted if the queue has no sub-queues. |
| resourcesUsed | A single resource object | The total amount of resources used by this queue |
### Elements of the queues object for a Leaf queue - contains all elements in parent plus the following:
### Elements of the queues object for a Leaf queue - contains all the elements in parent except 'queues' plus the following:
| Item | Data Type | Description |
|:---- |:---- |:---- |
@ -1005,9 +1005,9 @@ Response Body:
| clusterResources | A single resource object | The capacity of the cluster |
| queueName | string | The name of the queue |
| schedulingPolicy | string | The name of the scheduling policy used by the queue |
| childQueues | array of queues(JSON)/queue objects(XML) | A collection of sub-queue information |
| childQueues | array of queues(JSON)/queue objects(XML) | A collection of sub-queue information. Omitted if the queue has no childQueues. |
### Elements of the queues object for a Leaf queue - contains all elements in parent plus the following
### Elements of the queues object for a Leaf queue - contains all the elements in parent except 'childQueues' plus the following
| Item | Data Type | Description |
|:---- |:---- |:---- |
@ -1044,7 +1044,8 @@ Response Body:
"scheduler": {
"schedulerInfo": {
"rootQueue": {
"childQueues": [
"childQueues": {
"queue": [
{
"clusterResources": {
"memory": 8192,
@ -1075,6 +1076,8 @@ Response Body:
},
{
"childQueues": {
"queue": [
{
"clusterResources": {
"memory": 8192,
"vCores": 8
@ -1096,13 +1099,13 @@ Response Body:
"numPendingApps": 0,
"queueName": "root.sample_queue.sample_sub_queue",
"schedulingPolicy": "fair",
"type": [
"fairSchedulerLeafQueueInfo"
],
"type": "fairSchedulerLeafQueueInfo",
"usedResources": {
"memory": 0,
"vCores": 0
}
}
]
},
"clusterResources": {
"memory": 8192,
@ -1129,6 +1132,7 @@ Response Body:
}
}
],
},
"clusterResources": {
"memory": 8192,
"vCores": 8
@ -1203,7 +1207,8 @@ Response Body:
</clusterResources>
<queueName>root</queueName>
<schedulingPolicy>fair</schedulingPolicy>
<childQueues xsi:type="fairSchedulerLeafQueueInfo">
<childQueues>
<queue xsi:type="fairSchedulerLeafQueueInfo">
<maxApps>2147483647</maxApps>
<minResources>
<memory>0</memory>
@ -1229,8 +1234,8 @@ Response Body:
<schedulingPolicy>fair</schedulingPolicy>
<numPendingApps>0</numPendingApps>
<numActiveApps>0</numActiveApps>
</childQueues>
<childQueues>
</queue>
<queue>
<maxApps>50</maxApps>
<minResources>
<memory>10000</memory>
@ -1254,7 +1259,8 @@ Response Body:
</clusterResources>
<queueName>root.sample_queue</queueName>
<schedulingPolicy>fair</schedulingPolicy>
<childQueues xsi:type="fairSchedulerLeafQueueInfo">
<childQueues>
<queue xsi:type="fairSchedulerLeafQueueInfo">
<maxApps>2147483647</maxApps>
<minResources>
<memory>5000</memory>
@ -1280,7 +1286,9 @@ Response Body:
<schedulingPolicy>fair</schedulingPolicy>
<numPendingApps>0</numPendingApps>
<numActiveApps>0</numActiveApps>
</queue>
</childQueues>
</queue>
</childQueues>
</rootQueue>
</schedulerInfo>