YARN-7817. Add Resource reference to RM's NodeInfo object so REST API can get non memory/vcore resource usages. (Sunil G via wangda)
Change-Id: Ia7ceeabd82046645ddeaf487c763288f36cfbdee
This commit is contained in:
parent
2e5865606b
commit
e0cfb0a31a
|
@ -55,7 +55,7 @@ public class JAXBContextResolver implements ContextResolver<JAXBContext> {
|
||||||
UsersInfo.class, UserInfo.class, ApplicationStatisticsInfo.class,
|
UsersInfo.class, UserInfo.class, ApplicationStatisticsInfo.class,
|
||||||
StatisticsItemInfo.class, CapacitySchedulerHealthInfo.class,
|
StatisticsItemInfo.class, CapacitySchedulerHealthInfo.class,
|
||||||
FairSchedulerQueueInfoList.class, AppTimeoutsInfo.class,
|
FairSchedulerQueueInfoList.class, AppTimeoutsInfo.class,
|
||||||
AppTimeoutInfo.class };
|
AppTimeoutInfo.class, ResourceInformationsInfo.class };
|
||||||
// these dao classes need root unwrapping
|
// these dao classes need root unwrapping
|
||||||
final Class[] rootUnwrappedTypes =
|
final Class[] rootUnwrappedTypes =
|
||||||
{ NewApplication.class, ApplicationSubmissionContextInfo.class,
|
{ NewApplication.class, ApplicationSubmissionContextInfo.class,
|
||||||
|
|
|
@ -61,6 +61,12 @@ public class ClusterMetricsInfo {
|
||||||
private int activeNodes;
|
private int activeNodes;
|
||||||
private int shutdownNodes;
|
private int shutdownNodes;
|
||||||
|
|
||||||
|
// Total used resource of the cluster, including all partitions
|
||||||
|
private ResourceInfo totalUsedResourcesAcrossPartition;
|
||||||
|
|
||||||
|
// Total registered resources of the cluster, including all partitions
|
||||||
|
private ResourceInfo totalClusterResourcesAcrossPartition;
|
||||||
|
|
||||||
public ClusterMetricsInfo() {
|
public ClusterMetricsInfo() {
|
||||||
} // JAXB needs this
|
} // JAXB needs this
|
||||||
|
|
||||||
|
@ -92,9 +98,20 @@ public class ClusterMetricsInfo {
|
||||||
this.containersReserved = metrics.getReservedContainers();
|
this.containersReserved = metrics.getReservedContainers();
|
||||||
|
|
||||||
if (rs instanceof CapacityScheduler) {
|
if (rs instanceof CapacityScheduler) {
|
||||||
|
CapacityScheduler cs = (CapacityScheduler) rs;
|
||||||
this.totalMB = availableMB + allocatedMB + reservedMB;
|
this.totalMB = availableMB + allocatedMB + reservedMB;
|
||||||
this.totalVirtualCores =
|
this.totalVirtualCores =
|
||||||
availableVirtualCores + allocatedVirtualCores + containersReserved;
|
availableVirtualCores + allocatedVirtualCores + containersReserved;
|
||||||
|
// TODO, add support of other schedulers to get total used resources
|
||||||
|
// across partition.
|
||||||
|
if (cs.getRootQueue() != null
|
||||||
|
&& cs.getRootQueue().getQueueResourceUsage() != null
|
||||||
|
&& cs.getRootQueue().getQueueResourceUsage().getAllUsed() != null) {
|
||||||
|
totalUsedResourcesAcrossPartition = new ResourceInfo(
|
||||||
|
cs.getRootQueue().getQueueResourceUsage().getAllUsed());
|
||||||
|
totalClusterResourcesAcrossPartition = new ResourceInfo(
|
||||||
|
cs.getClusterResource());
|
||||||
|
}
|
||||||
} else {
|
} else {
|
||||||
this.totalMB = availableMB + allocatedMB;
|
this.totalMB = availableMB + allocatedMB;
|
||||||
this.totalVirtualCores = availableVirtualCores + allocatedVirtualCores;
|
this.totalVirtualCores = availableVirtualCores + allocatedVirtualCores;
|
||||||
|
@ -310,4 +327,11 @@ public class ClusterMetricsInfo {
|
||||||
this.shutdownNodes = shutdownNodes;
|
this.shutdownNodes = shutdownNodes;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public ResourceInfo getTotalUsedResourcesAcrossPartition() {
|
||||||
|
return totalUsedResourcesAcrossPartition;
|
||||||
|
}
|
||||||
|
|
||||||
|
public ResourceInfo getTotalClusterResourcesAcrossPartition() {
|
||||||
|
return totalClusterResourcesAcrossPartition;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -58,6 +58,8 @@ public class NodeInfo {
|
||||||
private int numQueuedContainers;
|
private int numQueuedContainers;
|
||||||
protected ArrayList<String> nodeLabels = new ArrayList<String>();
|
protected ArrayList<String> nodeLabels = new ArrayList<String>();
|
||||||
protected ResourceUtilizationInfo resourceUtilization;
|
protected ResourceUtilizationInfo resourceUtilization;
|
||||||
|
protected ResourceInfo usedResource;
|
||||||
|
protected ResourceInfo availableResource;
|
||||||
|
|
||||||
public NodeInfo() {
|
public NodeInfo() {
|
||||||
} // JAXB needs this
|
} // JAXB needs this
|
||||||
|
@ -75,6 +77,8 @@ public class NodeInfo {
|
||||||
this.usedVirtualCores = report.getUsedResource().getVirtualCores();
|
this.usedVirtualCores = report.getUsedResource().getVirtualCores();
|
||||||
this.availableVirtualCores =
|
this.availableVirtualCores =
|
||||||
report.getAvailableResource().getVirtualCores();
|
report.getAvailableResource().getVirtualCores();
|
||||||
|
this.usedResource = new ResourceInfo(report.getUsedResource());
|
||||||
|
this.availableResource = new ResourceInfo(report.getAvailableResource());
|
||||||
}
|
}
|
||||||
this.id = id.toString();
|
this.id = id.toString();
|
||||||
this.rack = ni.getRackName();
|
this.rack = ni.getRackName();
|
||||||
|
@ -183,6 +187,22 @@ public class NodeInfo {
|
||||||
return this.nodeLabels;
|
return this.nodeLabels;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public ResourceInfo getUsedResource() {
|
||||||
|
return usedResource;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setUsedResource(ResourceInfo used) {
|
||||||
|
this.usedResource = used;
|
||||||
|
}
|
||||||
|
|
||||||
|
public ResourceInfo getAvailableResource() {
|
||||||
|
return availableResource;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setAvailableResource(ResourceInfo avail) {
|
||||||
|
this.availableResource = avail;
|
||||||
|
}
|
||||||
|
|
||||||
public ResourceUtilizationInfo getResourceUtilization() {
|
public ResourceUtilizationInfo getResourceUtilization() {
|
||||||
return this.resourceUtilization;
|
return this.resourceUtilization;
|
||||||
}
|
}
|
||||||
|
|
|
@ -26,6 +26,7 @@ import javax.xml.bind.annotation.XmlRootElement;
|
||||||
import org.apache.hadoop.yarn.api.records.Resource;
|
import org.apache.hadoop.yarn.api.records.Resource;
|
||||||
import org.apache.hadoop.yarn.util.resource.Resources;
|
import org.apache.hadoop.yarn.util.resource.Resources;
|
||||||
|
|
||||||
|
|
||||||
@XmlRootElement
|
@XmlRootElement
|
||||||
@XmlAccessorType(XmlAccessType.NONE)
|
@XmlAccessorType(XmlAccessType.NONE)
|
||||||
public class ResourceInfo {
|
public class ResourceInfo {
|
||||||
|
@ -34,6 +35,9 @@ public class ResourceInfo {
|
||||||
long memory;
|
long memory;
|
||||||
@XmlElement
|
@XmlElement
|
||||||
int vCores;
|
int vCores;
|
||||||
|
@XmlElement
|
||||||
|
ResourceInformationsInfo resourceInformations =
|
||||||
|
new ResourceInformationsInfo();
|
||||||
|
|
||||||
private Resource resources;
|
private Resource resources;
|
||||||
|
|
||||||
|
@ -41,9 +45,13 @@ public class ResourceInfo {
|
||||||
}
|
}
|
||||||
|
|
||||||
public ResourceInfo(Resource res) {
|
public ResourceInfo(Resource res) {
|
||||||
|
// Make sure no NPE.
|
||||||
|
if (res != null) {
|
||||||
memory = res.getMemorySize();
|
memory = res.getMemorySize();
|
||||||
vCores = res.getVirtualCores();
|
vCores = res.getVirtualCores();
|
||||||
resources = Resources.clone(res);
|
resources = Resources.clone(res);
|
||||||
|
resourceInformations.addAll(res.getAllResourcesListCopy());
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
public long getMemorySize() {
|
public long getMemorySize() {
|
||||||
|
@ -84,4 +92,8 @@ public class ResourceInfo {
|
||||||
public Resource getResource() {
|
public Resource getResource() {
|
||||||
return Resource.newInstance(resources);
|
return Resource.newInstance(resources);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public ResourceInformationsInfo getResourcesInformations() {
|
||||||
|
return resourceInformations;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -0,0 +1,48 @@
|
||||||
|
/**
|
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one
|
||||||
|
* or more contributor license agreements. See the NOTICE file
|
||||||
|
* distributed with this work for additional information
|
||||||
|
* regarding copyright ownership. The ASF licenses this file
|
||||||
|
* to you under the Apache License, Version 2.0 (the
|
||||||
|
* "License"); you may not use this file except in compliance
|
||||||
|
* with the License. You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
|
package org.apache.hadoop.yarn.server.resourcemanager.webapp.dao;
|
||||||
|
|
||||||
|
import java.util.ArrayList;
|
||||||
|
import java.util.List;
|
||||||
|
|
||||||
|
import javax.xml.bind.annotation.XmlAccessType;
|
||||||
|
import javax.xml.bind.annotation.XmlAccessorType;
|
||||||
|
import javax.xml.bind.annotation.XmlElement;
|
||||||
|
import javax.xml.bind.annotation.XmlRootElement;
|
||||||
|
|
||||||
|
import org.apache.hadoop.yarn.api.records.ResourceInformation;
|
||||||
|
|
||||||
|
@XmlRootElement(name = "resourceInformations")
|
||||||
|
@XmlAccessorType(XmlAccessType.FIELD)
|
||||||
|
public class ResourceInformationsInfo {
|
||||||
|
|
||||||
|
@XmlElement(name = "resourceInformation")
|
||||||
|
protected ArrayList<ResourceInformation> resourceInformation =
|
||||||
|
new ArrayList<ResourceInformation>();
|
||||||
|
|
||||||
|
public ResourceInformationsInfo() {
|
||||||
|
} // JAXB needs this
|
||||||
|
|
||||||
|
public ArrayList<ResourceInformation> getApps() {
|
||||||
|
return resourceInformation;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void addAll(List<ResourceInformation> resourcesInformationsInfo) {
|
||||||
|
resourceInformation.addAll(resourcesInformationsInfo);
|
||||||
|
}
|
||||||
|
}
|
|
@ -169,8 +169,8 @@ export default DS.Model.extend({
|
||||||
&& this.get("totalUsedResourcesAcrossPartition")) {
|
&& this.get("totalUsedResourcesAcrossPartition")) {
|
||||||
var usages = [];
|
var usages = [];
|
||||||
|
|
||||||
var clusterResourceInformations = this.get("totalClusterResourcesAcrossPartition").resourcesInformations;
|
var clusterResourceInformations = this.get("totalClusterResourcesAcrossPartition").resourceInformations.resourceInformation;
|
||||||
var usedResourceInformations = this.get("totalUsedResourcesAcrossPartition").resourcesInformations;
|
var usedResourceInformations = this.get("totalUsedResourcesAcrossPartition").resourceInformations.resourceInformation;
|
||||||
|
|
||||||
clusterResourceInformations.forEach(function(cluster) {
|
clusterResourceInformations.forEach(function(cluster) {
|
||||||
var perResourceTypeUsage = {
|
var perResourceTypeUsage = {
|
||||||
|
|
|
@ -99,7 +99,7 @@ export default DS.Model.extend({
|
||||||
|
|
||||||
const usedResource = this.get("usedResource");
|
const usedResource = this.get("usedResource");
|
||||||
const availableResource = this.get("availableResource");
|
const availableResource = this.get("availableResource");
|
||||||
var resourceInformations = usedResource ? usedResource.resourcesInformations : [];
|
var resourceInformations = usedResource ? usedResource.resourceInformations.resourceInformation : [];
|
||||||
for (var i = 0; i < resourceInformations.length; i++) {
|
for (var i = 0; i < resourceInformations.length; i++) {
|
||||||
ri = resourceInformations[i];
|
ri = resourceInformations[i];
|
||||||
if (ri.name === "yarn.io/gpu") {
|
if (ri.name === "yarn.io/gpu") {
|
||||||
|
@ -108,7 +108,7 @@ export default DS.Model.extend({
|
||||||
}
|
}
|
||||||
|
|
||||||
var available = 0;
|
var available = 0;
|
||||||
resourceInformations = availableResource ? availableResource.resourcesInformations : [];
|
resourceInformations = availableResource ? availableResource.resourceInformations.resourceInformation : [];
|
||||||
for (i = 0; i < resourceInformations.length; i++) {
|
for (i = 0; i < resourceInformations.length; i++) {
|
||||||
ri = resourceInformations[i];
|
ri = resourceInformations[i];
|
||||||
if (ri.name === "yarn.io/gpu") {
|
if (ri.name === "yarn.io/gpu") {
|
||||||
|
|
|
@ -42,8 +42,8 @@ export default DS.JSONAPISerializer.extend({
|
||||||
availableVirtualCores: payload.availableVirtualCores,
|
availableVirtualCores: payload.availableVirtualCores,
|
||||||
version: payload.version,
|
version: payload.version,
|
||||||
nodeLabels: payload.nodeLabels,
|
nodeLabels: payload.nodeLabels,
|
||||||
usedResource: payload.used,
|
usedResource: payload.usedResource,
|
||||||
availableResource: payload.avail
|
availableResource: payload.availableResource
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
return fixedPayload;
|
return fixedPayload;
|
||||||
|
|
Loading…
Reference in New Issue