YARN-7817. Add Resource reference to RM's NodeInfo object so REST API can get non memory/vcore resource usages. (Sunil G via wangda)
Change-Id: Ia7ceeabd82046645ddeaf487c763288f36cfbdee
This commit is contained in:
Wangda Tan
parent
2e5865606b
commit
e0cfb0a31a
|
|
@ -55,7 +55,7 @@ public JAXBContextResolver() throws Exception {
|
|||
UsersInfo.class, UserInfo.class, ApplicationStatisticsInfo.class,
|
||||
StatisticsItemInfo.class, CapacitySchedulerHealthInfo.class,
|
||||
FairSchedulerQueueInfoList.class, AppTimeoutsInfo.class,
|
||||
AppTimeoutInfo.class };
|
||||
AppTimeoutInfo.class, ResourceInformationsInfo.class };
|
||||
// these dao classes need root unwrapping
|
||||
final Class[] rootUnwrappedTypes =
|
||||
{ NewApplication.class, ApplicationSubmissionContextInfo.class,
|
||||
|
|
|
|||
|
|
@ -61,6 +61,12 @@ public class ClusterMetricsInfo {
|
|||
private int activeNodes;
|
||||
private int shutdownNodes;
|
||||
|
||||
// Total used resource of the cluster, including all partitions
|
||||
private ResourceInfo totalUsedResourcesAcrossPartition;
|
||||
|
||||
// Total registered resources of the cluster, including all partitions
|
||||
private ResourceInfo totalClusterResourcesAcrossPartition;
|
||||
|
||||
public ClusterMetricsInfo() {
|
||||
} // JAXB needs this
|
||||
|
||||
|
|
@ -92,9 +98,20 @@ public ClusterMetricsInfo(final ResourceScheduler rs) {
|
|||
this.containersReserved = metrics.getReservedContainers();
|
||||
|
||||
if (rs instanceof CapacityScheduler) {
|
||||
CapacityScheduler cs = (CapacityScheduler) rs;
|
||||
this.totalMB = availableMB + allocatedMB + reservedMB;
|
||||
this.totalVirtualCores =
|
||||
availableVirtualCores + allocatedVirtualCores + containersReserved;
|
||||
// TODO, add support of other schedulers to get total used resources
|
||||
// across partition.
|
||||
if (cs.getRootQueue() != null
|
||||
&& cs.getRootQueue().getQueueResourceUsage() != null
|
||||
&& cs.getRootQueue().getQueueResourceUsage().getAllUsed() != null) {
|
||||
totalUsedResourcesAcrossPartition = new ResourceInfo(
|
||||
cs.getRootQueue().getQueueResourceUsage().getAllUsed());
|
||||
totalClusterResourcesAcrossPartition = new ResourceInfo(
|
||||
cs.getClusterResource());
|
||||
}
|
||||
} else {
|
||||
this.totalMB = availableMB + allocatedMB;
|
||||
this.totalVirtualCores = availableVirtualCores + allocatedVirtualCores;
|
||||
|
|
@ -310,4 +327,11 @@ public void setShutdownNodes(int shutdownNodes) {
|
|||
this.shutdownNodes = shutdownNodes;
|
||||
}
|
||||
|
||||
public ResourceInfo getTotalUsedResourcesAcrossPartition() {
|
||||
return totalUsedResourcesAcrossPartition;
|
||||
}
|
||||
|
||||
public ResourceInfo getTotalClusterResourcesAcrossPartition() {
|
||||
return totalClusterResourcesAcrossPartition;
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -58,6 +58,8 @@ public class NodeInfo {
|
|||
private int numQueuedContainers;
|
||||
protected ArrayList<String> nodeLabels = new ArrayList<String>();
|
||||
protected ResourceUtilizationInfo resourceUtilization;
|
||||
protected ResourceInfo usedResource;
|
||||
protected ResourceInfo availableResource;
|
||||
|
||||
public NodeInfo() {
|
||||
} // JAXB needs this
|
||||
|
|
@ -75,6 +77,8 @@ public NodeInfo(RMNode ni, ResourceScheduler sched) {
|
|||
this.usedVirtualCores = report.getUsedResource().getVirtualCores();
|
||||
this.availableVirtualCores =
|
||||
report.getAvailableResource().getVirtualCores();
|
||||
this.usedResource = new ResourceInfo(report.getUsedResource());
|
||||
this.availableResource = new ResourceInfo(report.getAvailableResource());
|
||||
}
|
||||
this.id = id.toString();
|
||||
this.rack = ni.getRackName();
|
||||
|
|
@ -183,6 +187,22 @@ public ArrayList<String> getNodeLabels() {
|
|||
return this.nodeLabels;
|
||||
}
|
||||
|
||||
public ResourceInfo getUsedResource() {
|
||||
return usedResource;
|
||||
}
|
||||
|
||||
public void setUsedResource(ResourceInfo used) {
|
||||
this.usedResource = used;
|
||||
}
|
||||
|
||||
public ResourceInfo getAvailableResource() {
|
||||
return availableResource;
|
||||
}
|
||||
|
||||
public void setAvailableResource(ResourceInfo avail) {
|
||||
this.availableResource = avail;
|
||||
}
|
||||
|
||||
public ResourceUtilizationInfo getResourceUtilization() {
|
||||
return this.resourceUtilization;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -26,6 +26,7 @@
|
|||
import org.apache.hadoop.yarn.api.records.Resource;
|
||||
import org.apache.hadoop.yarn.util.resource.Resources;
|
||||
|
||||
|
||||
@XmlRootElement
|
||||
@XmlAccessorType(XmlAccessType.NONE)
|
||||
public class ResourceInfo {
|
||||
|
|
@ -34,6 +35,9 @@ public class ResourceInfo {
|
|||
long memory;
|
||||
@XmlElement
|
||||
int vCores;
|
||||
@XmlElement
|
||||
ResourceInformationsInfo resourceInformations =
|
||||
new ResourceInformationsInfo();
|
||||
|
||||
private Resource resources;
|
||||
|
||||
|
|
@ -41,9 +45,13 @@ public ResourceInfo() {
|
|||
}
|
||||
|
||||
public ResourceInfo(Resource res) {
|
||||
// Make sure no NPE.
|
||||
if (res != null) {
|
||||
memory = res.getMemorySize();
|
||||
vCores = res.getVirtualCores();
|
||||
resources = Resources.clone(res);
|
||||
resourceInformations.addAll(res.getAllResourcesListCopy());
|
||||
}
|
||||
}
|
||||
|
||||
public long getMemorySize() {
|
||||
|
|
@ -84,4 +92,8 @@ public void setvCores(int vCores) {
|
|||
public Resource getResource() {
|
||||
return Resource.newInstance(resources);
|
||||
}
|
||||
|
||||
public ResourceInformationsInfo getResourcesInformations() {
|
||||
return resourceInformations;
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,48 @@
|
|||
/**
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.apache.hadoop.yarn.server.resourcemanager.webapp.dao;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
import javax.xml.bind.annotation.XmlAccessType;
|
||||
import javax.xml.bind.annotation.XmlAccessorType;
|
||||
import javax.xml.bind.annotation.XmlElement;
|
||||
import javax.xml.bind.annotation.XmlRootElement;
|
||||
|
||||
import org.apache.hadoop.yarn.api.records.ResourceInformation;
|
||||
|
||||
@XmlRootElement(name = "resourceInformations")
|
||||
@XmlAccessorType(XmlAccessType.FIELD)
|
||||
public class ResourceInformationsInfo {
|
||||
|
||||
@XmlElement(name = "resourceInformation")
|
||||
protected ArrayList<ResourceInformation> resourceInformation =
|
||||
new ArrayList<ResourceInformation>();
|
||||
|
||||
public ResourceInformationsInfo() {
|
||||
} // JAXB needs this
|
||||
|
||||
public ArrayList<ResourceInformation> getApps() {
|
||||
return resourceInformation;
|
||||
}
|
||||
|
||||
public void addAll(List<ResourceInformation> resourcesInformationsInfo) {
|
||||
resourceInformation.addAll(resourcesInformationsInfo);
|
||||
}
|
||||
}
|
||||
|
|
@ -169,8 +169,8 @@ export default DS.Model.extend({
|
|||
&& this.get("totalUsedResourcesAcrossPartition")) {
|
||||
var usages = [];
|
||||
|
||||
var clusterResourceInformations = this.get("totalClusterResourcesAcrossPartition").resourcesInformations;
|
||||
var usedResourceInformations = this.get("totalUsedResourcesAcrossPartition").resourcesInformations;
|
||||
var clusterResourceInformations = this.get("totalClusterResourcesAcrossPartition").resourceInformations.resourceInformation;
|
||||
var usedResourceInformations = this.get("totalUsedResourcesAcrossPartition").resourceInformations.resourceInformation;
|
||||
|
||||
clusterResourceInformations.forEach(function(cluster) {
|
||||
var perResourceTypeUsage = {
|
||||
|
|
|
|||
|
|
@ -99,7 +99,7 @@ export default DS.Model.extend({
|
|||
|
||||
const usedResource = this.get("usedResource");
|
||||
const availableResource = this.get("availableResource");
|
||||
var resourceInformations = usedResource ? usedResource.resourcesInformations : [];
|
||||
var resourceInformations = usedResource ? usedResource.resourceInformations.resourceInformation : [];
|
||||
for (var i = 0; i < resourceInformations.length; i++) {
|
||||
ri = resourceInformations[i];
|
||||
if (ri.name === "yarn.io/gpu") {
|
||||
|
|
@ -108,7 +108,7 @@ export default DS.Model.extend({
|
|||
}
|
||||
|
||||
var available = 0;
|
||||
resourceInformations = availableResource ? availableResource.resourcesInformations : [];
|
||||
resourceInformations = availableResource ? availableResource.resourceInformations.resourceInformation : [];
|
||||
for (i = 0; i < resourceInformations.length; i++) {
|
||||
ri = resourceInformations[i];
|
||||
if (ri.name === "yarn.io/gpu") {
|
||||
|
|
|
|||
|
|
@ -42,8 +42,8 @@ export default DS.JSONAPISerializer.extend({
|
|||
availableVirtualCores: payload.availableVirtualCores,
|
||||
version: payload.version,
|
||||
nodeLabels: payload.nodeLabels,
|
||||
usedResource: payload.used,
|
||||
availableResource: payload.avail
|
||||
usedResource: payload.usedResource,
|
||||
availableResource: payload.availableResource
|
||||
}
|
||||
};
|
||||
return fixedPayload;
|
||||
|
|
|
|||
Loading…
Reference in New Issue