YARN-2336. Fair scheduler's REST API returns a missing '[' bracket JSON for deep queue tree. Contributed by Kenji Kikushima and Akira Ajisaka.
This commit is contained in:
Tsuyoshi Ozawa
parent
56996a685e
commit
9a3d617b63
|
|
@ -52,6 +52,9 @@ Release 2.8.0 - UNRELEASED
|
|||
|
||||
INCOMPATIBLE CHANGES
|
||||
|
||||
YARN-2336. Fair scheduler's REST API returns a missing '[' bracket JSON for
|
||||
deep queue tree. (Kenji Kikushima and Akira Ajisaka via ozawa)
|
||||
|
||||
NEW FEATURES
|
||||
|
||||
YARN-3345. Add non-exclusive node label API. (Wangda Tan via jianhe)
|
||||
|
|
|
|||
|
|
@ -53,7 +53,8 @@ public JAXBContextResolver() throws Exception {
|
|||
NodesInfo.class, RemoteExceptionData.class,
|
||||
CapacitySchedulerQueueInfoList.class, ResourceInfo.class,
|
||||
UsersInfo.class, UserInfo.class, ApplicationStatisticsInfo.class,
|
||||
StatisticsItemInfo.class, CapacitySchedulerHealthInfo.class };
|
||||
StatisticsItemInfo.class, CapacitySchedulerHealthInfo.class,
|
||||
FairSchedulerQueueInfoList.class};
|
||||
// these dao classes need root unwrapping
|
||||
final Class[] rootUnwrappedTypes =
|
||||
{ NewApplication.class, ApplicationSubmissionContextInfo.class,
|
||||
|
|
|
|||
|
|
@ -19,7 +19,6 @@
|
|||
package org.apache.hadoop.yarn.server.resourcemanager.webapp.dao;
|
||||
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collection;
|
||||
|
||||
import javax.xml.bind.annotation.XmlAccessType;
|
||||
|
|
@ -60,9 +59,9 @@ public class FairSchedulerQueueInfo {
|
|||
|
||||
private String queueName;
|
||||
private String schedulingPolicy;
|
||||
|
||||
private Collection<FairSchedulerQueueInfo> childQueues;
|
||||
|
||||
|
||||
private FairSchedulerQueueInfoList childQueues;
|
||||
|
||||
public FairSchedulerQueueInfo() {
|
||||
}
|
||||
|
||||
|
|
@ -95,20 +94,34 @@ public FairSchedulerQueueInfo(FSQueue queue, FairScheduler scheduler) {
|
|||
|
||||
maxApps = allocConf.getQueueMaxApps(queueName);
|
||||
|
||||
childQueues = new ArrayList<FairSchedulerQueueInfo>();
|
||||
if (allocConf.isReservable(queueName) &&
|
||||
!allocConf.getShowReservationAsQueues(queueName)) {
|
||||
return;
|
||||
}
|
||||
|
||||
childQueues = getChildQueues(queue, scheduler);
|
||||
}
|
||||
|
||||
protected FairSchedulerQueueInfoList getChildQueues(FSQueue queue,
|
||||
FairScheduler scheduler) {
|
||||
// Return null to omit 'childQueues' field from the return value of
|
||||
// REST API if it is empty. We omit the field to keep the consistency
|
||||
// with CapacitySchedulerQueueInfo, which omits 'queues' field if empty.
|
||||
Collection<FSQueue> children = queue.getChildQueues();
|
||||
if (children.isEmpty()) {
|
||||
return null;
|
||||
}
|
||||
FairSchedulerQueueInfoList list = new FairSchedulerQueueInfoList();
|
||||
for (FSQueue child : children) {
|
||||
if (child instanceof FSLeafQueue) {
|
||||
childQueues.add(new FairSchedulerLeafQueueInfo((FSLeafQueue)child, scheduler));
|
||||
list.addToQueueInfoList(
|
||||
new FairSchedulerLeafQueueInfo((FSLeafQueue) child, scheduler));
|
||||
} else {
|
||||
childQueues.add(new FairSchedulerQueueInfo(child, scheduler));
|
||||
list.addToQueueInfoList(
|
||||
new FairSchedulerQueueInfo(child, scheduler));
|
||||
}
|
||||
}
|
||||
return list;
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -189,8 +202,8 @@ public float getMaxResourcesFraction() {
|
|||
public String getSchedulingPolicy() {
|
||||
return schedulingPolicy;
|
||||
}
|
||||
|
||||
|
||||
public Collection<FairSchedulerQueueInfo> getChildQueues() {
|
||||
return childQueues;
|
||||
return childQueues.getQueueInfoList();
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,49 @@
|
|||
/**
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.apache.hadoop.yarn.server.resourcemanager.webapp.dao;
|
||||
|
||||
import java.util.ArrayList;
|
||||
|
||||
import javax.xml.bind.annotation.XmlAccessType;
|
||||
import javax.xml.bind.annotation.XmlAccessorType;
|
||||
import javax.xml.bind.annotation.XmlRootElement;
|
||||
|
||||
/**
|
||||
* FairScheduler QueueInfo list used for mapping to XML or JSON.
|
||||
*/
|
||||
@XmlRootElement
|
||||
@XmlAccessorType(XmlAccessType.FIELD)
|
||||
public class FairSchedulerQueueInfoList {
|
||||
private ArrayList<FairSchedulerQueueInfo> queue;
|
||||
|
||||
public FairSchedulerQueueInfoList() {
|
||||
queue = new ArrayList<>();
|
||||
}
|
||||
|
||||
public ArrayList<FairSchedulerQueueInfo> getQueueInfoList() {
|
||||
return this.queue;
|
||||
}
|
||||
|
||||
public boolean addToQueueInfoList(FairSchedulerQueueInfo e) {
|
||||
return this.queue.add(e);
|
||||
}
|
||||
|
||||
public FairSchedulerQueueInfo getQueueInfo(int i) {
|
||||
return this.queue.get(i);
|
||||
}
|
||||
}
|
||||
|
|
@ -21,6 +21,7 @@
|
|||
import static org.junit.Assert.assertEquals;
|
||||
import static org.junit.Assert.assertNotNull;
|
||||
import static org.junit.Assert.assertTrue;
|
||||
import static org.junit.Assert.fail;
|
||||
|
||||
import java.io.StringReader;
|
||||
|
||||
|
|
@ -575,6 +576,15 @@ public void testPerUserResourcesJSON() throws Exception {
|
|||
user.getInt("numPendingApplications");
|
||||
checkResourcesUsed(user);
|
||||
}
|
||||
|
||||
// Verify 'queues' field is omitted from CapacitySchedulerLeafQueueInfo.
|
||||
try {
|
||||
b1.getJSONObject("queues");
|
||||
fail("CapacitySchedulerQueueInfo should omit field 'queues'" +
|
||||
"if child queue is empty.");
|
||||
} catch (JSONException je) {
|
||||
assertEquals("JSONObject[\"queues\"] not found.", je.getMessage());
|
||||
}
|
||||
} finally {
|
||||
rm.stop();
|
||||
}
|
||||
|
|
|
|||
|
|
@ -19,6 +19,7 @@
|
|||
package org.apache.hadoop.yarn.server.resourcemanager.webapp;
|
||||
|
||||
import static org.junit.Assert.assertEquals;
|
||||
import static org.junit.Assert.fail;
|
||||
|
||||
import javax.ws.rs.core.MediaType;
|
||||
|
||||
|
|
@ -27,8 +28,10 @@
|
|||
import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.QueueManager;
|
||||
import org.apache.hadoop.yarn.webapp.GenericExceptionHandler;
|
||||
import org.apache.hadoop.yarn.webapp.JerseyTestBase;
|
||||
import org.codehaus.jettison.json.JSONArray;
|
||||
import org.codehaus.jettison.json.JSONException;
|
||||
import org.codehaus.jettison.json.JSONObject;
|
||||
import org.junit.Test;
|
||||
|
|
@ -99,6 +102,38 @@ public void testClusterSchedulerSlash() throws JSONException, Exception {
|
|||
verifyClusterScheduler(json);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testClusterSchedulerWithSubQueues() throws JSONException,
|
||||
Exception {
|
||||
FairScheduler scheduler = (FairScheduler)rm.getResourceScheduler();
|
||||
QueueManager queueManager = scheduler.getQueueManager();
|
||||
// create LeafQueue
|
||||
queueManager.getLeafQueue("root.q.subqueue1", true);
|
||||
queueManager.getLeafQueue("root.q.subqueue2", true);
|
||||
|
||||
WebResource r = resource();
|
||||
ClientResponse response = r.path("ws").path("v1").path("cluster")
|
||||
.path("scheduler").accept(MediaType.APPLICATION_JSON)
|
||||
.get(ClientResponse.class);
|
||||
assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
|
||||
JSONObject json = response.getEntity(JSONObject.class);
|
||||
JSONArray subQueueInfo = json.getJSONObject("scheduler")
|
||||
.getJSONObject("schedulerInfo").getJSONObject("rootQueue")
|
||||
.getJSONObject("childQueues").getJSONArray("queue")
|
||||
.getJSONObject(1).getJSONObject("childQueues").getJSONArray("queue");
|
||||
// subQueueInfo is consist of subqueue1 and subqueue2 info
|
||||
assertEquals(2, subQueueInfo.length());
|
||||
|
||||
// Verify 'childQueues' field is omitted from FairSchedulerLeafQueueInfo.
|
||||
try {
|
||||
subQueueInfo.getJSONObject(1).getJSONObject("childQueues");
|
||||
fail("FairSchedulerQueueInfo should omit field 'childQueues'" +
|
||||
"if child queue is empty.");
|
||||
} catch (JSONException je) {
|
||||
assertEquals("JSONObject[\"childQueues\"] not found.", je.getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
private void verifyClusterScheduler(JSONObject json) throws JSONException,
|
||||
Exception {
|
||||
assertEquals("incorrect number of elements", 1, json.length());
|
||||
|
|
|
|||
|
|
@ -321,10 +321,10 @@ The capacity scheduler supports hierarchical queues. This one request will print
|
|||
| usedResources | string | A string describing the current resources used by the queue |
|
||||
| queueName | string | The name of the queue |
|
||||
| state | string of QueueState | The state of the queue |
|
||||
| queues | array of queues(JSON)/zero or more queue objects(XML) | A collection of sub-queue information |
|
||||
| queues | array of queues(JSON)/zero or more queue objects(XML) | A collection of sub-queue information. Omitted if the queue has no sub-queues. |
|
||||
| resourcesUsed | A single resource object | The total amount of resources used by this queue |
|
||||
|
||||
### Elements of the queues object for a Leaf queue - contains all elements in parent plus the following:
|
||||
### Elements of the queues object for a Leaf queue - contains all the elements in parent except 'queues' plus the following:
|
||||
|
||||
| Item | Data Type | Description |
|
||||
|:---- |:---- |:---- |
|
||||
|
|
@ -1005,9 +1005,9 @@ Response Body:
|
|||
| clusterResources | A single resource object | The capacity of the cluster |
|
||||
| queueName | string | The name of the queue |
|
||||
| schedulingPolicy | string | The name of the scheduling policy used by the queue |
|
||||
| childQueues | array of queues(JSON)/queue objects(XML) | A collection of sub-queue information |
|
||||
| childQueues | array of queues(JSON)/queue objects(XML) | A collection of sub-queue information. Omitted if the queue has no childQueues. |
|
||||
|
||||
### Elements of the queues object for a Leaf queue - contains all elements in parent plus the following
|
||||
### Elements of the queues object for a Leaf queue - contains all the elements in parent except 'childQueues' plus the following
|
||||
|
||||
| Item | Data Type | Description |
|
||||
|:---- |:---- |:---- |
|
||||
|
|
@ -1044,43 +1044,15 @@ Response Body:
|
|||
"scheduler": {
|
||||
"schedulerInfo": {
|
||||
"rootQueue": {
|
||||
"childQueues": [
|
||||
{
|
||||
"clusterResources": {
|
||||
"memory": 8192,
|
||||
"vCores": 8
|
||||
},
|
||||
"fairResources": {
|
||||
"memory": 0,
|
||||
"vCores": 0
|
||||
},
|
||||
"maxApps": 2147483647,
|
||||
"maxResources": {
|
||||
"memory": 8192,
|
||||
"vCores": 8
|
||||
},
|
||||
"minResources": {
|
||||
"memory": 0,
|
||||
"vCores": 0
|
||||
},
|
||||
"numActiveApps": 0,
|
||||
"numPendingApps": 0,
|
||||
"queueName": "root.default",
|
||||
"schedulingPolicy": "fair",
|
||||
"type": "fairSchedulerLeafQueueInfo",
|
||||
"usedResources": {
|
||||
"memory": 0,
|
||||
"vCores": 0
|
||||
}
|
||||
},
|
||||
{
|
||||
"childQueues": {
|
||||
"childQueues": {
|
||||
"queue": [
|
||||
{
|
||||
"clusterResources": {
|
||||
"memory": 8192,
|
||||
"vCores": 8
|
||||
},
|
||||
"fairResources": {
|
||||
"memory": 10000,
|
||||
"memory": 0,
|
||||
"vCores": 0
|
||||
},
|
||||
"maxApps": 2147483647,
|
||||
|
|
@ -1089,46 +1061,78 @@ Response Body:
|
|||
"vCores": 8
|
||||
},
|
||||
"minResources": {
|
||||
"memory": 5000,
|
||||
"memory": 0,
|
||||
"vCores": 0
|
||||
},
|
||||
"numActiveApps": 0,
|
||||
"numPendingApps": 0,
|
||||
"queueName": "root.sample_queue.sample_sub_queue",
|
||||
"queueName": "root.default",
|
||||
"schedulingPolicy": "fair",
|
||||
"type": [
|
||||
"fairSchedulerLeafQueueInfo"
|
||||
],
|
||||
"type": "fairSchedulerLeafQueueInfo",
|
||||
"usedResources": {
|
||||
"memory": 0,
|
||||
"vCores": 0
|
||||
}
|
||||
},
|
||||
"clusterResources": {
|
||||
"memory": 8192,
|
||||
"vCores": 8
|
||||
},
|
||||
"fairResources": {
|
||||
"memory": 10000,
|
||||
"vCores": 0
|
||||
},
|
||||
"maxApps": 50,
|
||||
"maxResources": {
|
||||
"memory": 8192,
|
||||
"vCores": 0
|
||||
},
|
||||
"minResources": {
|
||||
"memory": 10000,
|
||||
"vCores": 0
|
||||
},
|
||||
"queueName": "root.sample_queue",
|
||||
"schedulingPolicy": "fair",
|
||||
"usedResources": {
|
||||
"memory": 0,
|
||||
"vCores": 0
|
||||
{
|
||||
"childQueues": {
|
||||
"queue": [
|
||||
{
|
||||
"clusterResources": {
|
||||
"memory": 8192,
|
||||
"vCores": 8
|
||||
},
|
||||
"fairResources": {
|
||||
"memory": 10000,
|
||||
"vCores": 0
|
||||
},
|
||||
"maxApps": 2147483647,
|
||||
"maxResources": {
|
||||
"memory": 8192,
|
||||
"vCores": 8
|
||||
},
|
||||
"minResources": {
|
||||
"memory": 5000,
|
||||
"vCores": 0
|
||||
},
|
||||
"numActiveApps": 0,
|
||||
"numPendingApps": 0,
|
||||
"queueName": "root.sample_queue.sample_sub_queue",
|
||||
"schedulingPolicy": "fair",
|
||||
"type": "fairSchedulerLeafQueueInfo",
|
||||
"usedResources": {
|
||||
"memory": 0,
|
||||
"vCores": 0
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
"clusterResources": {
|
||||
"memory": 8192,
|
||||
"vCores": 8
|
||||
},
|
||||
"fairResources": {
|
||||
"memory": 10000,
|
||||
"vCores": 0
|
||||
},
|
||||
"maxApps": 50,
|
||||
"maxResources": {
|
||||
"memory": 8192,
|
||||
"vCores": 0
|
||||
},
|
||||
"minResources": {
|
||||
"memory": 10000,
|
||||
"vCores": 0
|
||||
},
|
||||
"queueName": "root.sample_queue",
|
||||
"schedulingPolicy": "fair",
|
||||
"usedResources": {
|
||||
"memory": 0,
|
||||
"vCores": 0
|
||||
}
|
||||
}
|
||||
}
|
||||
],
|
||||
],
|
||||
},
|
||||
"clusterResources": {
|
||||
"memory": 8192,
|
||||
"vCores": 8
|
||||
|
|
@ -1203,61 +1207,11 @@ Response Body:
|
|||
</clusterResources>
|
||||
<queueName>root</queueName>
|
||||
<schedulingPolicy>fair</schedulingPolicy>
|
||||
<childQueues xsi:type="fairSchedulerLeafQueueInfo">
|
||||
<maxApps>2147483647</maxApps>
|
||||
<minResources>
|
||||
<memory>0</memory>
|
||||
<vCores>0</vCores>
|
||||
</minResources>
|
||||
<maxResources>
|
||||
<memory>8192</memory>
|
||||
<vCores>8</vCores>
|
||||
</maxResources>
|
||||
<usedResources>
|
||||
<memory>0</memory>
|
||||
<vCores>0</vCores>
|
||||
</usedResources>
|
||||
<fairResources>
|
||||
<memory>0</memory>
|
||||
<vCores>0</vCores>
|
||||
</fairResources>
|
||||
<clusterResources>
|
||||
<memory>8192</memory>
|
||||
<vCores>8</vCores>
|
||||
</clusterResources>
|
||||
<queueName>root.default</queueName>
|
||||
<schedulingPolicy>fair</schedulingPolicy>
|
||||
<numPendingApps>0</numPendingApps>
|
||||
<numActiveApps>0</numActiveApps>
|
||||
</childQueues>
|
||||
<childQueues>
|
||||
<maxApps>50</maxApps>
|
||||
<minResources>
|
||||
<memory>10000</memory>
|
||||
<vCores>0</vCores>
|
||||
</minResources>
|
||||
<maxResources>
|
||||
<memory>8192</memory>
|
||||
<vCores>0</vCores>
|
||||
</maxResources>
|
||||
<usedResources>
|
||||
<memory>0</memory>
|
||||
<vCores>0</vCores>
|
||||
</usedResources>
|
||||
<fairResources>
|
||||
<memory>10000</memory>
|
||||
<vCores>0</vCores>
|
||||
</fairResources>
|
||||
<clusterResources>
|
||||
<memory>8192</memory>
|
||||
<vCores>8</vCores>
|
||||
</clusterResources>
|
||||
<queueName>root.sample_queue</queueName>
|
||||
<schedulingPolicy>fair</schedulingPolicy>
|
||||
<childQueues xsi:type="fairSchedulerLeafQueueInfo">
|
||||
<queue xsi:type="fairSchedulerLeafQueueInfo">
|
||||
<maxApps>2147483647</maxApps>
|
||||
<minResources>
|
||||
<memory>5000</memory>
|
||||
<memory>0</memory>
|
||||
<vCores>0</vCores>
|
||||
</minResources>
|
||||
<maxResources>
|
||||
|
|
@ -1268,6 +1222,33 @@ Response Body:
|
|||
<memory>0</memory>
|
||||
<vCores>0</vCores>
|
||||
</usedResources>
|
||||
<fairResources>
|
||||
<memory>0</memory>
|
||||
<vCores>0</vCores>
|
||||
</fairResources>
|
||||
<clusterResources>
|
||||
<memory>8192</memory>
|
||||
<vCores>8</vCores>
|
||||
</clusterResources>
|
||||
<queueName>root.default</queueName>
|
||||
<schedulingPolicy>fair</schedulingPolicy>
|
||||
<numPendingApps>0</numPendingApps>
|
||||
<numActiveApps>0</numActiveApps>
|
||||
</queue>
|
||||
<queue>
|
||||
<maxApps>50</maxApps>
|
||||
<minResources>
|
||||
<memory>10000</memory>
|
||||
<vCores>0</vCores>
|
||||
</minResources>
|
||||
<maxResources>
|
||||
<memory>8192</memory>
|
||||
<vCores>0</vCores>
|
||||
</maxResources>
|
||||
<usedResources>
|
||||
<memory>0</memory>
|
||||
<vCores>0</vCores>
|
||||
</usedResources>
|
||||
<fairResources>
|
||||
<memory>10000</memory>
|
||||
<vCores>0</vCores>
|
||||
|
|
@ -1276,11 +1257,38 @@ Response Body:
|
|||
<memory>8192</memory>
|
||||
<vCores>8</vCores>
|
||||
</clusterResources>
|
||||
<queueName>root.sample_queue.sample_sub_queue</queueName>
|
||||
<queueName>root.sample_queue</queueName>
|
||||
<schedulingPolicy>fair</schedulingPolicy>
|
||||
<numPendingApps>0</numPendingApps>
|
||||
<numActiveApps>0</numActiveApps>
|
||||
</childQueues>
|
||||
<childQueues>
|
||||
<queue xsi:type="fairSchedulerLeafQueueInfo">
|
||||
<maxApps>2147483647</maxApps>
|
||||
<minResources>
|
||||
<memory>5000</memory>
|
||||
<vCores>0</vCores>
|
||||
</minResources>
|
||||
<maxResources>
|
||||
<memory>8192</memory>
|
||||
<vCores>8</vCores>
|
||||
</maxResources>
|
||||
<usedResources>
|
||||
<memory>0</memory>
|
||||
<vCores>0</vCores>
|
||||
</usedResources>
|
||||
<fairResources>
|
||||
<memory>10000</memory>
|
||||
<vCores>0</vCores>
|
||||
</fairResources>
|
||||
<clusterResources>
|
||||
<memory>8192</memory>
|
||||
<vCores>8</vCores>
|
||||
</clusterResources>
|
||||
<queueName>root.sample_queue.sample_sub_queue</queueName>
|
||||
<schedulingPolicy>fair</schedulingPolicy>
|
||||
<numPendingApps>0</numPendingApps>
|
||||
<numActiveApps>0</numActiveApps>
|
||||
</queue>
|
||||
</childQueues>
|
||||
</queue>
|
||||
</childQueues>
|
||||
</rootQueue>
|
||||
</schedulerInfo>
|
||||
|
|
|
|||
Loading…
Reference in New Issue