Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

YARN-5203.Return ResourceRequest JAXB object in ResourceManager Cluster Applications REST API. Contributed by Ellen Hui.

This commit is contained in:
Subru Krishnan 2016-07-28 16:03:24 -07:00
parent a1890c32c5
commit 4e756d7271
8 changed files with 330 additions and 19 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppAttemptBlock.java
View File

@ -40,6 +40,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptMetrics;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppInfo;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ResourceRequestInfo;
import org.apache.hadoop.yarn.server.webapp.AppAttemptBlock;
import org.apache.hadoop.yarn.server.webapp.dao.AppAttemptInfo;
import org.apache.hadoop.yarn.util.Times;
@ -71,7 +72,7 @@ public class RMAppAttemptBlock extends AppAttemptBlock{
.get(this.appAttemptId.getApplicationId()), true,
WebAppUtils.getHttpSchemePrefix(conf));
List<ResourceRequest> resourceRequests = app.getResourceRequests();
List<ResourceRequestInfo> resourceRequests = app.getResourceRequests();
if (resourceRequests == null || resourceRequests.isEmpty()) {
return;
}
@ -88,7 +89,7 @@ public class RMAppAttemptBlock extends AppAttemptBlock{
.th(".labelexpression", "NodeLabelExpression")._()._().tbody();
StringBuilder resourceRequestTableData = new StringBuilder("[\n");
for (ResourceRequest resourceRequest : resourceRequests) {
for (ResourceRequestInfo resourceRequest : resourceRequests) {
if (resourceRequest.getNumContainers() == 0) {
continue;
}
@ -118,19 +119,19 @@ public class RMAppAttemptBlock extends AppAttemptBlock{
div._();
}
private Resource getTotalResource(List<ResourceRequest> requests) {
private Resource getTotalResource(List<ResourceRequestInfo> requests) {
Resource totalResource = Resource.newInstance(0, 0);
if (requests == null) {
return totalResource;
}
for (ResourceRequest request : requests) {
for (ResourceRequestInfo request : requests) {
if (request.getNumContainers() == 0) {
continue;
}
if (request.getResourceName().equals(ResourceRequest.ANY)) {
Resources.addTo(
totalResource,
Resources.multiply(request.getCapability(),
totalResource,
Resources.multiply(request.getCapability().getResource(),
request.getNumContainers()));
}
}

19
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java
View File

@ -17,10 +17,12 @@
*/
package org.apache.hadoop.yarn.server.resourcemanager.webapp.dao;
import java.util.ArrayList;
import java.util.List;
import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType;
import javax.xml.bind.annotation.XmlElement;
import javax.xml.bind.annotation.XmlRootElement;
import javax.xml.bind.annotation.XmlTransient;
@ -97,7 +99,10 @@ public class AppInfo {
protected int numNonAMContainerPreempted;
protected int numAMContainerPreempted;
protected List<ResourceRequest> resourceRequests;
// list of resource requests
@XmlElement(name = "resourceRequests")
private List<ResourceRequestInfo> resourceRequests =
new ArrayList<ResourceRequestInfo>();
protected LogAggregationStatus logAggregationStatus;
protected boolean unmanagedApplication;
@ -182,8 +187,16 @@ public class AppInfo {
queueUsagePercentage = resourceReport.getQueueUsagePercentage();
clusterUsagePercentage = resourceReport.getClusterUsagePercentage();
}
resourceRequests = rm.getRMContext().getScheduler()
List<ResourceRequest> resourceRequestsRaw = rm.getRMContext()
.getScheduler()
.getPendingResourceRequestsForAttempt(attempt.getAppAttemptId());
if (resourceRequestsRaw != null) {
for (ResourceRequest req : resourceRequestsRaw) {
resourceRequests.add(new ResourceRequestInfo(req));
}
}
}
}
@ -352,7 +365,7 @@ public class AppInfo {
return vcoreSeconds;
}
public List<ResourceRequest> getResourceRequests() {
public List<ResourceRequestInfo> getResourceRequests() {
return this.resourceRequests;
}

63
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ExecutionTypeRequestInfo.java Normal file
View File

@ -0,0 +1,63 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.yarn.server.resourcemanager.webapp.dao;
import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType;
import javax.xml.bind.annotation.XmlElement;
import javax.xml.bind.annotation.XmlRootElement;
import org.apache.hadoop.yarn.api.records.ExecutionType;
import org.apache.hadoop.yarn.api.records.ExecutionTypeRequest;
/**
* Simple class representing an execution type request.
*/
@XmlRootElement(name = "ExecutionTypeRequest")
@XmlAccessorType(XmlAccessType.FIELD)
public class ExecutionTypeRequestInfo {
@XmlElement(name = "executionType")
private String executionType;
@XmlElement(name = "enforceExecutionType")
private boolean enforceExecutionType;
public ExecutionTypeRequestInfo() {
}
public ExecutionTypeRequestInfo(ExecutionTypeRequest executionTypeRequest) {
executionType = executionTypeRequest.getExecutionType().name();
enforceExecutionType = executionTypeRequest.getEnforceExecutionType();
}
public ExecutionType getExecutionType() {
return ExecutionType.valueOf(executionType);
}
public void setExecutionType(ExecutionType executionType) {
this.executionType = executionType.name();
}
public boolean getEnforceExecutionType() {
return enforceExecutionType;
}
public void setEnforceExecutionType(boolean enforceExecutionType) {
this.enforceExecutionType = enforceExecutionType;
}
}

4
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ResourceInfo.java
View File

@ -58,4 +58,8 @@ public class ResourceInfo {
public void setvCores(int vCores) {
this.vCores = vCores;
}
public Resource getResource() {
return Resource.newInstance(memory, vCores);
}
}

131
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ResourceRequestInfo.java Normal file
View File

@ -0,0 +1,131 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.yarn.server.resourcemanager.webapp.dao;
import org.apache.hadoop.yarn.api.records.ExecutionTypeRequest;
import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.api.records.ResourceRequest;
import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType;
import javax.xml.bind.annotation.XmlElement;
import javax.xml.bind.annotation.XmlRootElement;
/**
* Simple class representing a resource request.
*/
@XmlRootElement(name = "resourceRequests")
@XmlAccessorType(XmlAccessType.FIELD)
public class ResourceRequestInfo {
@XmlElement(name = "priority")
private int priority;
@XmlElement(name = "resourceName")
private String resourceName;
@XmlElement(name = "capability")
private ResourceInfo capability;
@XmlElement(name = "numContainers")
private int numContainers;
@XmlElement(name = "relaxLocality")
private boolean relaxLocality;
@XmlElement(name = "nodeLabelExpression")
private String nodeLabelExpression;
@XmlElement(name = "executionTypeRequest")
private ExecutionTypeRequestInfo executionTypeRequest;
@XmlElement(name = "executionType")
private String executionType;
@XmlElement(name = "enforceExecutionType")
private boolean enforceExecutionType;
public ResourceRequestInfo() {
}
public ResourceRequestInfo(ResourceRequest request) {
priority = request.getPriority().getPriority();
resourceName = request.getResourceName();
capability = new ResourceInfo(request.getCapability());
numContainers = request.getNumContainers();
relaxLocality = request.getRelaxLocality();
nodeLabelExpression = request.getNodeLabelExpression();
if (request.getExecutionTypeRequest() != null) {
executionTypeRequest =
new ExecutionTypeRequestInfo(request.getExecutionTypeRequest());
}
}
public Priority getPriority() {
return Priority.newInstance(priority);
}
public void setPriority(Priority priority) {
this.priority = priority.getPriority();
}
public String getResourceName() {
return resourceName;
}
public void setResourceName(String resourceName) {
this.resourceName = resourceName;
}
public ResourceInfo getCapability() {
return capability;
}
public void setCapability(ResourceInfo capability) {
this.capability = capability;
}
public int getNumContainers() {
return numContainers;
}
public void setNumContainers(int numContainers) {
this.numContainers = numContainers;
}
public boolean getRelaxLocality() {
return relaxLocality;
}
public void setRelaxLocality(boolean relaxLocality) {
this.relaxLocality = relaxLocality;
}
public String getNodeLabelExpression() {
return nodeLabelExpression;
}
public void setNodeLabelExpression(String nodeLabelExpression) {
this.nodeLabelExpression = nodeLabelExpression;
}
public void setExecutionTypeRequest(
ExecutionTypeRequest executionTypeRequest) {
this.executionTypeRequest =
new ExecutionTypeRequestInfo(executionTypeRequest);
}
public ExecutionTypeRequestInfo getExecutionTypeRequest() {
return executionTypeRequest;
}
}

16
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockAM.java
View File

@ -39,6 +39,7 @@ import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.ResourceRequest;
import org.apache.hadoop.yarn.api.records.ExecutionTypeRequest;
import org.apache.hadoop.yarn.security.AMRMTokenIdentifier;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt;
@ -197,8 +198,15 @@ public class MockAM {
return createResourceReq(resource, memory, priority, containers, null);
}
public ResourceRequest createResourceReq(String resource, int memory, int priority,
int containers, String labelExpression) throws Exception {
public ResourceRequest createResourceReq(String resource, int memory,
int priority, int containers, String labelExpression) throws Exception {
return createResourceReq(resource, memory, priority, containers,
labelExpression, ExecutionTypeRequest.newInstance());
}
public ResourceRequest createResourceReq(String resource, int memory,
int priority, int containers, String labelExpression,
ExecutionTypeRequest executionTypeRequest) throws Exception {
ResourceRequest req = Records.newRecord(ResourceRequest.class);
req.setResourceName(resource);
req.setNumContainers(containers);
@ -209,9 +217,11 @@ public class MockAM {
capability.setMemorySize(memory);
req.setCapability(capability);
if (labelExpression != null) {
req.setNodeLabelExpression(labelExpression);
req.setNodeLabelExpression(labelExpression);
}
req.setExecutionTypeRequest(executionTypeRequest);
return req;
}
public AllocateResponse allocate(

6
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java
View File

@ -613,12 +613,12 @@ public class MockRM extends ResourceManager {
sub.setLogAggregationContext(logAggregationContext);
}
sub.setCancelTokensWhenComplete(cancelTokensWhenComplete);
ResourceRequest amResourceRequest = ResourceRequest.newInstance(
Priority.newInstance(0), ResourceRequest.ANY, capability, 1);
if (amLabel != null && !amLabel.isEmpty()) {
ResourceRequest amResourceRequest = ResourceRequest.newInstance(
Priority.newInstance(0), ResourceRequest.ANY, capability, 1);
amResourceRequest.setNodeLabelExpression(amLabel.trim());
sub.setAMContainerResourceRequest(amResourceRequest);
}
sub.setAMContainerResourceRequest(amResourceRequest);
req.setApplicationSubmissionContext(sub);
UserGroupInformation fakeUser =
UserGroupInformation.createUserForTesting(user, new String[] {"someGroup"});

97
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java
View File

@ -34,6 +34,7 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.yarn.api.records.ContainerState;
import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
import org.apache.hadoop.yarn.api.records.ResourceRequest;
import org.apache.hadoop.yarn.api.records.YarnApplicationState;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.server.resourcemanager.MockAM;
@ -1152,6 +1153,30 @@ public class TestRMWebServicesApps extends JerseyTestBase {
rm.stop();
}
@Test
public void testUnmarshalAppInfo() throws JSONException, Exception {
rm.start();
MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048);
RMApp app1 = rm.submitApp(CONTAINER_MB, "testwordcount", "user1");
amNodeManager.nodeHeartbeat(true);
WebResource r = resource();
ClientResponse response = r.path("ws").path("v1").path("cluster")
.path("apps").path(app1.getApplicationId().toString())
.accept(MediaType.APPLICATION_XML).get(ClientResponse.class);
AppInfo appInfo = response.getEntity(AppInfo.class);
// Check only a few values; all are validated in testSingleApp.
assertEquals(app1.getApplicationId().toString(), appInfo.getAppId());
assertEquals(app1.getName(), appInfo.getName());
assertEquals(app1.createApplicationState(), appInfo.getState());
assertEquals(app1.getAMResourceRequest().getCapability().getMemorySize(),
appInfo.getAllocatedMB());
rm.stop();
}
@Test
public void testSingleAppsSlash() throws JSONException, Exception {
rm.start();
@ -1284,10 +1309,10 @@ public class TestRMWebServicesApps extends JerseyTestBase {
rm.stop();
}
public void verifyAppsXML(NodeList nodes, RMApp app) throws JSONException,
Exception {
public void verifyAppsXML(NodeList nodes, RMApp app)
throws JSONException, Exception {
for (int i = 0; i < nodes.getLength(); i++) {
for (int i = 0; i < nodes.getLength(); i++) {
Element element = (Element) nodes.item(i);
verifyAppInfoGeneric(app,
@ -1322,6 +1347,26 @@ public class TestRMWebServicesApps extends JerseyTestBase {
WebServicesTestUtils.getXmlString(element, "appNodeLabelExpression"),
WebServicesTestUtils.getXmlString(element, "amNodeLabelExpression"),
WebServicesTestUtils.getXmlString(element, "amRPCAddress"));
assertEquals(element.getElementsByTagName("resourceRequests").getLength(),
1);
Element resourceRequests =
(Element) element.getElementsByTagName("resourceRequests").item(0);
Element capability =
(Element) resourceRequests.getElementsByTagName("capability").item(0);
verifyResourceRequestsGeneric(app,
WebServicesTestUtils.getXmlString(resourceRequests,
"nodeLabelExpression"),
WebServicesTestUtils.getXmlInt(resourceRequests, "numContainers"),
WebServicesTestUtils.getXmlBoolean(resourceRequests, "relaxLocality"),
WebServicesTestUtils.getXmlInt(resourceRequests, "priority"),
WebServicesTestUtils.getXmlString(resourceRequests, "resourceName"),
WebServicesTestUtils.getXmlLong(capability, "memory"),
WebServicesTestUtils.getXmlLong(capability, "vCores"),
WebServicesTestUtils.getXmlString(resourceRequests, "executionType"),
WebServicesTestUtils.getXmlBoolean(resourceRequests,
"enforceExecutionType"));
}
}
@ -1348,7 +1393,6 @@ public class TestRMWebServicesApps extends JerseyTestBase {
}
assertEquals("incorrect number of elements", expectedNumberOfElements,
info.length());
verifyAppInfoGeneric(app, info.getString("id"), info.getString("user"),
info.getString("name"), info.getString("applicationType"),
info.getString("queue"), info.getInt("priority"),
@ -1370,6 +1414,8 @@ public class TestRMWebServicesApps extends JerseyTestBase {
appNodeLabelExpression,
amNodeLabelExpression,
amRPCAddress);
verifyResourceRequests(info.getJSONArray("resourceRequests"), app);
}
public void verifyAppInfoGeneric(RMApp app, String id, String user,
@ -1449,6 +1495,49 @@ public class TestRMWebServicesApps extends JerseyTestBase {
amRPCAddress);
}
public void verifyResourceRequests(JSONArray resourceRequest, RMApp app)
throws JSONException {
JSONObject requestInfo = resourceRequest.getJSONObject(0);
verifyResourceRequestsGeneric(app,
requestInfo.getString("nodeLabelExpression"),
requestInfo.getInt("numContainers"),
requestInfo.getBoolean("relaxLocality"), requestInfo.getInt("priority"),
requestInfo.getString("resourceName"),
requestInfo.getJSONObject("capability").getLong("memory"),
requestInfo.getJSONObject("capability").getLong("vCores"),
requestInfo.getJSONObject("executionTypeRequest")
.getString("executionType"),
requestInfo.getJSONObject("executionTypeRequest")
.getBoolean("enforceExecutionType"));
}
public void verifyResourceRequestsGeneric(RMApp app,
String nodeLabelExpression, int numContainers, boolean relaxLocality,
int priority, String resourceName, long memory, long vCores,
String executionType, boolean enforceExecutionType) {
ResourceRequest request = app.getAMResourceRequest();
assertEquals("nodeLabelExpression doesn't match",
request.getNodeLabelExpression(), nodeLabelExpression);
assertEquals("numContainers doesn't match", request.getNumContainers(),
numContainers);
assertEquals("relaxLocality doesn't match", request.getRelaxLocality(),
relaxLocality);
assertEquals("priority does not match", request.getPriority().getPriority(),
priority);
assertEquals("resourceName does not match", request.getResourceName(),
resourceName);
assertEquals("memory does not match",
request.getCapability().getMemorySize(), memory);
assertEquals("vCores does not match",
request.getCapability().getVirtualCores(), vCores);
assertEquals("executionType does not match",
request.getExecutionTypeRequest().getExecutionType().name(),
executionType);
assertEquals("enforceExecutionType does not match",
request.getExecutionTypeRequest().getEnforceExecutionType(),
enforceExecutionType);
}
@Test
public void testAppAttempts() throws JSONException, Exception {
rm.start();