YARN-7817. Add Resource reference to RM's NodeInfo object so REST API can get non memory/vcore resource usages. (Sunil G via wangda)

Change-Id: Ia7ceeabd82046645ddeaf487c763288f36cfbdee
This commit is contained in:
Wangda Tan 2018-01-26 15:43:27 +08:00
parent 2e5865606b
commit e0cfb0a31a
8 changed files with 114 additions and 10 deletions

View File

@ -55,7 +55,7 @@ public JAXBContextResolver() throws Exception {
UsersInfo.class, UserInfo.class, ApplicationStatisticsInfo.class, UsersInfo.class, UserInfo.class, ApplicationStatisticsInfo.class,
StatisticsItemInfo.class, CapacitySchedulerHealthInfo.class, StatisticsItemInfo.class, CapacitySchedulerHealthInfo.class,
FairSchedulerQueueInfoList.class, AppTimeoutsInfo.class, FairSchedulerQueueInfoList.class, AppTimeoutsInfo.class,
AppTimeoutInfo.class }; AppTimeoutInfo.class, ResourceInformationsInfo.class };
// these dao classes need root unwrapping // these dao classes need root unwrapping
final Class[] rootUnwrappedTypes = final Class[] rootUnwrappedTypes =
{ NewApplication.class, ApplicationSubmissionContextInfo.class, { NewApplication.class, ApplicationSubmissionContextInfo.class,

View File

@ -61,6 +61,12 @@ public class ClusterMetricsInfo {
private int activeNodes; private int activeNodes;
private int shutdownNodes; private int shutdownNodes;
// Total used resource of the cluster, including all partitions
private ResourceInfo totalUsedResourcesAcrossPartition;
// Total registered resources of the cluster, including all partitions
private ResourceInfo totalClusterResourcesAcrossPartition;
public ClusterMetricsInfo() { public ClusterMetricsInfo() {
} // JAXB needs this } // JAXB needs this
@ -92,9 +98,20 @@ public ClusterMetricsInfo(final ResourceScheduler rs) {
this.containersReserved = metrics.getReservedContainers(); this.containersReserved = metrics.getReservedContainers();
if (rs instanceof CapacityScheduler) { if (rs instanceof CapacityScheduler) {
CapacityScheduler cs = (CapacityScheduler) rs;
this.totalMB = availableMB + allocatedMB + reservedMB; this.totalMB = availableMB + allocatedMB + reservedMB;
this.totalVirtualCores = this.totalVirtualCores =
availableVirtualCores + allocatedVirtualCores + containersReserved; availableVirtualCores + allocatedVirtualCores + containersReserved;
// TODO, add support of other schedulers to get total used resources
// across partition.
if (cs.getRootQueue() != null
&& cs.getRootQueue().getQueueResourceUsage() != null
&& cs.getRootQueue().getQueueResourceUsage().getAllUsed() != null) {
totalUsedResourcesAcrossPartition = new ResourceInfo(
cs.getRootQueue().getQueueResourceUsage().getAllUsed());
totalClusterResourcesAcrossPartition = new ResourceInfo(
cs.getClusterResource());
}
} else { } else {
this.totalMB = availableMB + allocatedMB; this.totalMB = availableMB + allocatedMB;
this.totalVirtualCores = availableVirtualCores + allocatedVirtualCores; this.totalVirtualCores = availableVirtualCores + allocatedVirtualCores;
@ -310,4 +327,11 @@ public void setShutdownNodes(int shutdownNodes) {
this.shutdownNodes = shutdownNodes; this.shutdownNodes = shutdownNodes;
} }
public ResourceInfo getTotalUsedResourcesAcrossPartition() {
return totalUsedResourcesAcrossPartition;
}
public ResourceInfo getTotalClusterResourcesAcrossPartition() {
return totalClusterResourcesAcrossPartition;
}
} }

View File

@ -58,6 +58,8 @@ public class NodeInfo {
private int numQueuedContainers; private int numQueuedContainers;
protected ArrayList<String> nodeLabels = new ArrayList<String>(); protected ArrayList<String> nodeLabels = new ArrayList<String>();
protected ResourceUtilizationInfo resourceUtilization; protected ResourceUtilizationInfo resourceUtilization;
protected ResourceInfo usedResource;
protected ResourceInfo availableResource;
public NodeInfo() { public NodeInfo() {
} // JAXB needs this } // JAXB needs this
@ -75,6 +77,8 @@ public NodeInfo(RMNode ni, ResourceScheduler sched) {
this.usedVirtualCores = report.getUsedResource().getVirtualCores(); this.usedVirtualCores = report.getUsedResource().getVirtualCores();
this.availableVirtualCores = this.availableVirtualCores =
report.getAvailableResource().getVirtualCores(); report.getAvailableResource().getVirtualCores();
this.usedResource = new ResourceInfo(report.getUsedResource());
this.availableResource = new ResourceInfo(report.getAvailableResource());
} }
this.id = id.toString(); this.id = id.toString();
this.rack = ni.getRackName(); this.rack = ni.getRackName();
@ -183,6 +187,22 @@ public ArrayList<String> getNodeLabels() {
return this.nodeLabels; return this.nodeLabels;
} }
public ResourceInfo getUsedResource() {
return usedResource;
}
public void setUsedResource(ResourceInfo used) {
this.usedResource = used;
}
public ResourceInfo getAvailableResource() {
return availableResource;
}
public void setAvailableResource(ResourceInfo avail) {
this.availableResource = avail;
}
public ResourceUtilizationInfo getResourceUtilization() { public ResourceUtilizationInfo getResourceUtilization() {
return this.resourceUtilization; return this.resourceUtilization;
} }

View File

@ -26,6 +26,7 @@
import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.util.resource.Resources; import org.apache.hadoop.yarn.util.resource.Resources;
@XmlRootElement @XmlRootElement
@XmlAccessorType(XmlAccessType.NONE) @XmlAccessorType(XmlAccessType.NONE)
public class ResourceInfo { public class ResourceInfo {
@ -34,6 +35,9 @@ public class ResourceInfo {
long memory; long memory;
@XmlElement @XmlElement
int vCores; int vCores;
@XmlElement
ResourceInformationsInfo resourceInformations =
new ResourceInformationsInfo();
private Resource resources; private Resource resources;
@ -41,9 +45,13 @@ public ResourceInfo() {
} }
public ResourceInfo(Resource res) { public ResourceInfo(Resource res) {
memory = res.getMemorySize(); // Make sure no NPE.
vCores = res.getVirtualCores(); if (res != null) {
resources = Resources.clone(res); memory = res.getMemorySize();
vCores = res.getVirtualCores();
resources = Resources.clone(res);
resourceInformations.addAll(res.getAllResourcesListCopy());
}
} }
public long getMemorySize() { public long getMemorySize() {
@ -84,4 +92,8 @@ public void setvCores(int vCores) {
public Resource getResource() { public Resource getResource() {
return Resource.newInstance(resources); return Resource.newInstance(resources);
} }
public ResourceInformationsInfo getResourcesInformations() {
return resourceInformations;
}
} }

View File

@ -0,0 +1,48 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.yarn.server.resourcemanager.webapp.dao;
import java.util.ArrayList;
import java.util.List;
import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType;
import javax.xml.bind.annotation.XmlElement;
import javax.xml.bind.annotation.XmlRootElement;
import org.apache.hadoop.yarn.api.records.ResourceInformation;
@XmlRootElement(name = "resourceInformations")
@XmlAccessorType(XmlAccessType.FIELD)
public class ResourceInformationsInfo {
@XmlElement(name = "resourceInformation")
protected ArrayList<ResourceInformation> resourceInformation =
new ArrayList<ResourceInformation>();
public ResourceInformationsInfo() {
} // JAXB needs this
public ArrayList<ResourceInformation> getApps() {
return resourceInformation;
}
public void addAll(List<ResourceInformation> resourcesInformationsInfo) {
resourceInformation.addAll(resourcesInformationsInfo);
}
}

View File

@ -169,8 +169,8 @@ export default DS.Model.extend({
&& this.get("totalUsedResourcesAcrossPartition")) { && this.get("totalUsedResourcesAcrossPartition")) {
var usages = []; var usages = [];
var clusterResourceInformations = this.get("totalClusterResourcesAcrossPartition").resourcesInformations; var clusterResourceInformations = this.get("totalClusterResourcesAcrossPartition").resourceInformations.resourceInformation;
var usedResourceInformations = this.get("totalUsedResourcesAcrossPartition").resourcesInformations; var usedResourceInformations = this.get("totalUsedResourcesAcrossPartition").resourceInformations.resourceInformation;
clusterResourceInformations.forEach(function(cluster) { clusterResourceInformations.forEach(function(cluster) {
var perResourceTypeUsage = { var perResourceTypeUsage = {

View File

@ -99,7 +99,7 @@ export default DS.Model.extend({
const usedResource = this.get("usedResource"); const usedResource = this.get("usedResource");
const availableResource = this.get("availableResource"); const availableResource = this.get("availableResource");
var resourceInformations = usedResource ? usedResource.resourcesInformations : []; var resourceInformations = usedResource ? usedResource.resourceInformations.resourceInformation : [];
for (var i = 0; i < resourceInformations.length; i++) { for (var i = 0; i < resourceInformations.length; i++) {
ri = resourceInformations[i]; ri = resourceInformations[i];
if (ri.name === "yarn.io/gpu") { if (ri.name === "yarn.io/gpu") {
@ -108,7 +108,7 @@ export default DS.Model.extend({
} }
var available = 0; var available = 0;
resourceInformations = availableResource ? availableResource.resourcesInformations : []; resourceInformations = availableResource ? availableResource.resourceInformations.resourceInformation : [];
for (i = 0; i < resourceInformations.length; i++) { for (i = 0; i < resourceInformations.length; i++) {
ri = resourceInformations[i]; ri = resourceInformations[i];
if (ri.name === "yarn.io/gpu") { if (ri.name === "yarn.io/gpu") {

View File

@ -42,8 +42,8 @@ export default DS.JSONAPISerializer.extend({
availableVirtualCores: payload.availableVirtualCores, availableVirtualCores: payload.availableVirtualCores,
version: payload.version, version: payload.version,
nodeLabels: payload.nodeLabels, nodeLabels: payload.nodeLabels,
usedResource: payload.used, usedResource: payload.usedResource,
availableResource: payload.avail availableResource: payload.availableResource
} }
}; };
return fixedPayload; return fixedPayload;