From e0cfb0a31a1c6ad0f9dcf0705b44229593a0915f Mon Sep 17 00:00:00 2001 From: Wangda Tan Date: Fri, 26 Jan 2018 15:43:27 +0800 Subject: [PATCH] YARN-7817. Add Resource reference to RM's NodeInfo object so REST API can get non memory/vcore resource usages. (Sunil G via wangda) Change-Id: Ia7ceeabd82046645ddeaf487c763288f36cfbdee --- .../webapp/JAXBContextResolver.java | 2 +- .../webapp/dao/ClusterMetricsInfo.java | 24 ++++++++++ .../resourcemanager/webapp/dao/NodeInfo.java | 20 ++++++++ .../webapp/dao/ResourceInfo.java | 18 +++++-- .../webapp/dao/ResourceInformationsInfo.java | 48 +++++++++++++++++++ .../main/webapp/app/models/cluster-metric.js | 4 +- .../main/webapp/app/models/yarn-rm-node.js | 4 +- .../webapp/app/serializers/yarn-rm-node.js | 4 +- 8 files changed, 114 insertions(+), 10 deletions(-) create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ResourceInformationsInfo.java diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/JAXBContextResolver.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/JAXBContextResolver.java index 2f50a24eac3..2e4204e08af 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/JAXBContextResolver.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/JAXBContextResolver.java @@ -55,7 +55,7 @@ public class JAXBContextResolver implements ContextResolver { UsersInfo.class, UserInfo.class, ApplicationStatisticsInfo.class, StatisticsItemInfo.class, CapacitySchedulerHealthInfo.class, FairSchedulerQueueInfoList.class, AppTimeoutsInfo.class, - AppTimeoutInfo.class }; + AppTimeoutInfo.class, ResourceInformationsInfo.class }; // these dao classes need root unwrapping final Class[] rootUnwrappedTypes = { NewApplication.class, ApplicationSubmissionContextInfo.class, diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ClusterMetricsInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ClusterMetricsInfo.java index 3214cb9f85e..84f70d9f2b6 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ClusterMetricsInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ClusterMetricsInfo.java @@ -61,6 +61,12 @@ public class ClusterMetricsInfo { private int activeNodes; private int shutdownNodes; + // Total used resource of the cluster, including all partitions + private ResourceInfo totalUsedResourcesAcrossPartition; + + // Total registered resources of the cluster, including all partitions + private ResourceInfo totalClusterResourcesAcrossPartition; + public ClusterMetricsInfo() { } // JAXB needs this @@ -92,9 +98,20 @@ public class ClusterMetricsInfo { this.containersReserved = metrics.getReservedContainers(); if (rs instanceof CapacityScheduler) { + CapacityScheduler cs = (CapacityScheduler) rs; this.totalMB = availableMB + allocatedMB + reservedMB; this.totalVirtualCores = availableVirtualCores + allocatedVirtualCores + containersReserved; + // TODO, add support of other schedulers to get total used resources + // across partition. + if (cs.getRootQueue() != null + && cs.getRootQueue().getQueueResourceUsage() != null + && cs.getRootQueue().getQueueResourceUsage().getAllUsed() != null) { + totalUsedResourcesAcrossPartition = new ResourceInfo( + cs.getRootQueue().getQueueResourceUsage().getAllUsed()); + totalClusterResourcesAcrossPartition = new ResourceInfo( + cs.getClusterResource()); + } } else { this.totalMB = availableMB + allocatedMB; this.totalVirtualCores = availableVirtualCores + allocatedVirtualCores; @@ -310,4 +327,11 @@ public class ClusterMetricsInfo { this.shutdownNodes = shutdownNodes; } + public ResourceInfo getTotalUsedResourcesAcrossPartition() { + return totalUsedResourcesAcrossPartition; + } + + public ResourceInfo getTotalClusterResourcesAcrossPartition() { + return totalClusterResourcesAcrossPartition; + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/NodeInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/NodeInfo.java index 2530c8ea117..3cec215f0e2 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/NodeInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/NodeInfo.java @@ -58,6 +58,8 @@ public class NodeInfo { private int numQueuedContainers; protected ArrayList nodeLabels = new ArrayList(); protected ResourceUtilizationInfo resourceUtilization; + protected ResourceInfo usedResource; + protected ResourceInfo availableResource; public NodeInfo() { } // JAXB needs this @@ -75,6 +77,8 @@ public class NodeInfo { this.usedVirtualCores = report.getUsedResource().getVirtualCores(); this.availableVirtualCores = report.getAvailableResource().getVirtualCores(); + this.usedResource = new ResourceInfo(report.getUsedResource()); + this.availableResource = new ResourceInfo(report.getAvailableResource()); } this.id = id.toString(); this.rack = ni.getRackName(); @@ -183,6 +187,22 @@ public class NodeInfo { return this.nodeLabels; } + public ResourceInfo getUsedResource() { + return usedResource; + } + + public void setUsedResource(ResourceInfo used) { + this.usedResource = used; + } + + public ResourceInfo getAvailableResource() { + return availableResource; + } + + public void setAvailableResource(ResourceInfo avail) { + this.availableResource = avail; + } + public ResourceUtilizationInfo getResourceUtilization() { return this.resourceUtilization; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ResourceInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ResourceInfo.java index e13980afc39..5bed936d402 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ResourceInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ResourceInfo.java @@ -26,6 +26,7 @@ import javax.xml.bind.annotation.XmlRootElement; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.util.resource.Resources; + @XmlRootElement @XmlAccessorType(XmlAccessType.NONE) public class ResourceInfo { @@ -34,6 +35,9 @@ public class ResourceInfo { long memory; @XmlElement int vCores; + @XmlElement + ResourceInformationsInfo resourceInformations = + new ResourceInformationsInfo(); private Resource resources; @@ -41,9 +45,13 @@ public class ResourceInfo { } public ResourceInfo(Resource res) { - memory = res.getMemorySize(); - vCores = res.getVirtualCores(); - resources = Resources.clone(res); + // Make sure no NPE. + if (res != null) { + memory = res.getMemorySize(); + vCores = res.getVirtualCores(); + resources = Resources.clone(res); + resourceInformations.addAll(res.getAllResourcesListCopy()); + } } public long getMemorySize() { @@ -84,4 +92,8 @@ public class ResourceInfo { public Resource getResource() { return Resource.newInstance(resources); } + + public ResourceInformationsInfo getResourcesInformations() { + return resourceInformations; + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ResourceInformationsInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ResourceInformationsInfo.java new file mode 100644 index 00000000000..651d7dfef88 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ResourceInformationsInfo.java @@ -0,0 +1,48 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.server.resourcemanager.webapp.dao; + +import java.util.ArrayList; +import java.util.List; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlElement; +import javax.xml.bind.annotation.XmlRootElement; + +import org.apache.hadoop.yarn.api.records.ResourceInformation; + +@XmlRootElement(name = "resourceInformations") +@XmlAccessorType(XmlAccessType.FIELD) +public class ResourceInformationsInfo { + + @XmlElement(name = "resourceInformation") + protected ArrayList resourceInformation = + new ArrayList(); + + public ResourceInformationsInfo() { + } // JAXB needs this + + public ArrayList getApps() { + return resourceInformation; + } + + public void addAll(List resourcesInformationsInfo) { + resourceInformation.addAll(resourcesInformationsInfo); + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/models/cluster-metric.js b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/models/cluster-metric.js index bbc6f084402..764723a2018 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/models/cluster-metric.js +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/models/cluster-metric.js @@ -169,8 +169,8 @@ export default DS.Model.extend({ && this.get("totalUsedResourcesAcrossPartition")) { var usages = []; - var clusterResourceInformations = this.get("totalClusterResourcesAcrossPartition").resourcesInformations; - var usedResourceInformations = this.get("totalUsedResourcesAcrossPartition").resourcesInformations; + var clusterResourceInformations = this.get("totalClusterResourcesAcrossPartition").resourceInformations.resourceInformation; + var usedResourceInformations = this.get("totalUsedResourcesAcrossPartition").resourceInformations.resourceInformation; clusterResourceInformations.forEach(function(cluster) { var perResourceTypeUsage = { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/models/yarn-rm-node.js b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/models/yarn-rm-node.js index 3b2385fcd5c..64fe7b86914 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/models/yarn-rm-node.js +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/models/yarn-rm-node.js @@ -99,7 +99,7 @@ export default DS.Model.extend({ const usedResource = this.get("usedResource"); const availableResource = this.get("availableResource"); - var resourceInformations = usedResource ? usedResource.resourcesInformations : []; + var resourceInformations = usedResource ? usedResource.resourceInformations.resourceInformation : []; for (var i = 0; i < resourceInformations.length; i++) { ri = resourceInformations[i]; if (ri.name === "yarn.io/gpu") { @@ -108,7 +108,7 @@ export default DS.Model.extend({ } var available = 0; - resourceInformations = availableResource ? availableResource.resourcesInformations : []; + resourceInformations = availableResource ? availableResource.resourceInformations.resourceInformation : []; for (i = 0; i < resourceInformations.length; i++) { ri = resourceInformations[i]; if (ri.name === "yarn.io/gpu") { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/serializers/yarn-rm-node.js b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/serializers/yarn-rm-node.js index a3a1d59168f..c27fd924905 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/serializers/yarn-rm-node.js +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/serializers/yarn-rm-node.js @@ -42,8 +42,8 @@ export default DS.JSONAPISerializer.extend({ availableVirtualCores: payload.availableVirtualCores, version: payload.version, nodeLabels: payload.nodeLabels, - usedResource: payload.used, - availableResource: payload.avail + usedResource: payload.usedResource, + availableResource: payload.availableResource } }; return fixedPayload;