YARN-7779. Display allocation tags in RM web UI and expose same through REST API. Contributed by Weiwei Yang.
This commit is contained in:
Sunil G
Arun Suresh
parent
adbe87abf8
commit
9b81cb0537
|
|
@ -20,6 +20,7 @@ package org.apache.hadoop.yarn.sls.nodemanager;
|
|||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
|
||||
import org.apache.hadoop.classification.InterfaceAudience.Private;
|
||||
|
|
@ -212,6 +213,11 @@ public class NodeInfo {
|
|||
return null;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Map<String, Long> getAllocationTagsWithCount() {
|
||||
return null;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Resource getPhysicalResource() {
|
||||
return null;
|
||||
|
|
|
|||
|
|
@ -37,6 +37,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmnode
|
|||
|
||||
import java.util.Collections;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
|
||||
@Private
|
||||
|
|
@ -202,6 +203,11 @@ public class RMNodeWrapper implements RMNode {
|
|||
return null;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Map<String, Long> getAllocationTagsWithCount() {
|
||||
return node.getAllocationTagsWithCount();
|
||||
}
|
||||
|
||||
@Override
|
||||
public Resource getPhysicalResource() {
|
||||
return null;
|
||||
|
|
|
|||
|
|
@ -20,6 +20,7 @@ package org.apache.hadoop.yarn.server.resourcemanager.rmnode;
|
|||
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
|
||||
import org.apache.hadoop.net.Node;
|
||||
|
|
@ -182,4 +183,10 @@ public interface RMNode {
|
|||
* @return the decommissioning timeout in second.
|
||||
*/
|
||||
Integer getDecommissioningTimeout();
|
||||
|
||||
/**
|
||||
* Get the allocation tags and their counts associated with this node.
|
||||
* @return a map of each allocation tag and its count.
|
||||
*/
|
||||
Map<String, Long> getAllocationTagsWithCount();
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1529,4 +1529,10 @@ public class RMNodeImpl implements RMNode, EventHandler<RMNodeEvent> {
|
|||
public Integer getDecommissioningTimeout() {
|
||||
return decommissioningTimeout;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Map<String, Long> getAllocationTagsWithCount() {
|
||||
return context.getAllocationTagsManager()
|
||||
.getAllocationTagsWithCount(getNodeID());
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -548,4 +548,15 @@ public class AllocationTagsManager {
|
|||
readLock.unlock();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns a map whose key is the allocation tag and value is the
|
||||
* count of allocations with this tag.
|
||||
*
|
||||
* @param nodeId
|
||||
* @return allocation tag to count mapping
|
||||
*/
|
||||
public Map<String, Long> getAllocationTagsWithCount(NodeId nodeId) {
|
||||
return globalNodeMapping.getTypeToTagsWithCount().get(nodeId);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -81,12 +81,14 @@ class NodesPage extends RmView {
|
|||
|
||||
if (!this.opportunisticContainersEnabled) {
|
||||
trbody.th(".containers", "Containers")
|
||||
.th(".allocationTags", "Allocation Tags")
|
||||
.th(".mem", "Mem Used")
|
||||
.th(".mem", "Mem Avail")
|
||||
.th(".vcores", "VCores Used")
|
||||
.th(".vcores", "VCores Avail");
|
||||
} else {
|
||||
trbody.th(".containers", "Running Containers (G)")
|
||||
.th(".allocationTags", "Allocation Tags")
|
||||
.th(".mem", "Mem Used (G)")
|
||||
.th(".mem", "Mem Avail (G)")
|
||||
.th(".vcores", "VCores Used (G)")
|
||||
|
|
@ -167,6 +169,7 @@ class NodesPage extends RmView {
|
|||
.append(Times.format(info.getLastHealthUpdate())).append("\",\"")
|
||||
.append(info.getHealthReport()).append("\",\"")
|
||||
.append(String.valueOf(info.getNumContainers())).append("\",\"")
|
||||
.append(info.getAllocationTagsSummary()).append("\",\"")
|
||||
.append("<br title='").append(String.valueOf(usedMemory))
|
||||
.append("'>").append(StringUtils.byteDesc(usedMemory * BYTES_IN_MB))
|
||||
.append("\",\"").append("<br title='")
|
||||
|
|
|
|||
|
|
@ -0,0 +1,56 @@
|
|||
/**
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.hadoop.yarn.server.resourcemanager.webapp.dao;
|
||||
|
||||
import javax.xml.bind.annotation.XmlAccessType;
|
||||
import javax.xml.bind.annotation.XmlAccessorType;
|
||||
import javax.xml.bind.annotation.XmlRootElement;
|
||||
|
||||
/**
|
||||
* DAO object to display node allocation tag.
|
||||
*/
|
||||
@XmlRootElement(name = "allocationTagInfo")
|
||||
@XmlAccessorType(XmlAccessType.FIELD)
|
||||
public class AllocationTagInfo {
|
||||
|
||||
private String allocationTag;
|
||||
private long allocationsCount;
|
||||
|
||||
public AllocationTagInfo() {
|
||||
// JAXB needs this
|
||||
}
|
||||
|
||||
public AllocationTagInfo(String tag, long count) {
|
||||
this.allocationTag = tag;
|
||||
this.allocationsCount = count;
|
||||
}
|
||||
|
||||
public String getAllocationTag() {
|
||||
return this.allocationTag;
|
||||
}
|
||||
|
||||
public long getAllocationsCount() {
|
||||
return this.allocationsCount;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return allocationTag + "(" + allocationsCount + ")";
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,59 @@
|
|||
/**
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.hadoop.yarn.server.resourcemanager.webapp.dao;
|
||||
|
||||
import org.apache.hadoop.util.StringUtils;
|
||||
|
||||
import javax.xml.bind.annotation.XmlAccessType;
|
||||
import javax.xml.bind.annotation.XmlAccessorType;
|
||||
import javax.xml.bind.annotation.XmlRootElement;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Iterator;
|
||||
|
||||
/**
|
||||
* DAO object to display node allocation tags.
|
||||
*/
|
||||
@XmlRootElement(name = "allocationTagsInfo")
|
||||
@XmlAccessorType(XmlAccessType.FIELD)
|
||||
public class AllocationTagsInfo {
|
||||
|
||||
private ArrayList<AllocationTagInfo> allocationTagInfo;
|
||||
|
||||
public AllocationTagsInfo() {
|
||||
allocationTagInfo = new ArrayList<>();
|
||||
}
|
||||
|
||||
public void addAllocationTag(AllocationTagInfo info) {
|
||||
allocationTagInfo.add(info);
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
StringBuffer sb = new StringBuffer();
|
||||
Iterator<AllocationTagInfo> it = allocationTagInfo.iterator();
|
||||
while (it.hasNext()) {
|
||||
AllocationTagInfo current = it.next();
|
||||
sb.append(current.toString());
|
||||
if (it.hasNext()) {
|
||||
sb.append(StringUtils.COMMA);
|
||||
}
|
||||
}
|
||||
return sb.toString();
|
||||
}
|
||||
}
|
||||
|
|
@ -20,6 +20,7 @@ package org.apache.hadoop.yarn.server.resourcemanager.webapp.dao;
|
|||
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collections;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
|
||||
import javax.xml.bind.annotation.XmlAccessType;
|
||||
|
|
@ -57,6 +58,7 @@ public class NodeInfo {
|
|||
private long usedVirtualCoresOpport;
|
||||
private int numQueuedContainers;
|
||||
protected ArrayList<String> nodeLabels = new ArrayList<String>();
|
||||
private AllocationTagsInfo allocationTags;
|
||||
protected ResourceUtilizationInfo resourceUtilization;
|
||||
protected ResourceInfo usedResource;
|
||||
protected ResourceInfo availableResource;
|
||||
|
|
@ -111,6 +113,14 @@ public class NodeInfo {
|
|||
Collections.sort(nodeLabels);
|
||||
}
|
||||
|
||||
// add allocation tags
|
||||
allocationTags = new AllocationTagsInfo();
|
||||
Map<String, Long> allocationTagsInfo = ni.getAllocationTagsWithCount();
|
||||
if (allocationTagsInfo != null) {
|
||||
allocationTagsInfo.forEach((tag, count) ->
|
||||
allocationTags.addAllocationTag(new AllocationTagInfo(tag, count)));
|
||||
}
|
||||
|
||||
// update node and containers resource utilization
|
||||
this.resourceUtilization = new ResourceUtilizationInfo(ni);
|
||||
}
|
||||
|
|
@ -207,6 +217,11 @@ public class NodeInfo {
|
|||
return this.resourceUtilization;
|
||||
}
|
||||
|
||||
public String getAllocationTagsSummary() {
|
||||
return this.allocationTags == null ? "" :
|
||||
this.allocationTags.toString();
|
||||
}
|
||||
|
||||
@VisibleForTesting
|
||||
public void setId(String id) {
|
||||
this.id = id;
|
||||
|
|
|
|||
|
|
@ -22,6 +22,7 @@ import java.util.ArrayList;
|
|||
import java.util.Collections;
|
||||
import java.util.List;
|
||||
import java.util.Set;
|
||||
import java.util.Map;
|
||||
|
||||
import org.apache.hadoop.net.Node;
|
||||
import org.apache.hadoop.yarn.api.records.ApplicationId;
|
||||
|
|
@ -279,6 +280,11 @@ public class MockNodes {
|
|||
return null;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Map<String, Long> getAllocationTagsWithCount() {
|
||||
return null;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Resource getPhysicalResource() {
|
||||
return this.physicalResource;
|
||||
|
|
|
|||
|
|
@ -48,8 +48,8 @@ public class TestNodesPage {
|
|||
|
||||
// Number of Actual Table Headers for NodesPage.NodesBlock might change in
|
||||
// future. In that case this value should be adjusted to the new value.
|
||||
final int numberOfThInMetricsTable = 23;
|
||||
final int numberOfActualTableHeaders = 13;
|
||||
private final int numberOfThInMetricsTable = 23;
|
||||
private final int numberOfActualTableHeaders = 14;
|
||||
private final int numberOfThForOpportunisticContainers = 4;
|
||||
|
||||
private Injector injector;
|
||||
|
|
|
|||
|
|
@ -22,10 +22,15 @@ import static org.apache.hadoop.yarn.webapp.WebServicesTestUtils.assertResponseS
|
|||
import static org.junit.Assert.assertEquals;
|
||||
import static org.junit.Assert.assertTrue;
|
||||
import static org.junit.Assert.fail;
|
||||
import static org.mockito.Mockito.mock;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
import java.io.StringReader;
|
||||
import java.util.ArrayList;
|
||||
import java.util.EnumSet;
|
||||
import java.util.Map;
|
||||
import java.util.TreeMap;
|
||||
import java.util.Iterator;
|
||||
|
||||
import javax.ws.rs.core.MediaType;
|
||||
import javax.xml.parsers.DocumentBuilder;
|
||||
|
|
@ -51,6 +56,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeStartedEvent;
|
|||
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeStatusEvent;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerNodeReport;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.constraint.AllocationTagsManager;
|
||||
import org.apache.hadoop.yarn.util.RackResolver;
|
||||
import org.apache.hadoop.yarn.util.YarnVersionInfo;
|
||||
import org.apache.hadoop.yarn.webapp.GenericExceptionHandler;
|
||||
|
|
@ -734,7 +740,7 @@ public class TestRMWebServicesNodes extends JerseyTestBase {
|
|||
|
||||
public void verifyNodeInfo(JSONObject nodeInfo, RMNode nm)
|
||||
throws JSONException, Exception {
|
||||
assertEquals("incorrect number of elements", 18, nodeInfo.length());
|
||||
assertEquals("incorrect number of elements", 19, nodeInfo.length());
|
||||
|
||||
JSONObject resourceInfo = nodeInfo.getJSONObject("resourceUtilization");
|
||||
verifyNodeInfoGeneric(nm, nodeInfo.getString("state"),
|
||||
|
|
@ -837,4 +843,73 @@ public class TestRMWebServicesNodes extends JerseyTestBase {
|
|||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testNodesAllocationTags() throws Exception {
|
||||
NodeId nm1 = NodeId.newInstance("host1", 1234);
|
||||
NodeId nm2 = NodeId.newInstance("host2", 2345);
|
||||
AllocationTagsManager atm = mock(AllocationTagsManager.class);
|
||||
|
||||
Map<String, Map<String, Long>> expectedAllocationTags = new TreeMap<>();
|
||||
Map<String, Long> nm1Tags = new TreeMap<>();
|
||||
nm1Tags.put("A", 1L);
|
||||
nm1Tags.put("B", 2L);
|
||||
Map<String, Long> nm2Tags = new TreeMap<>();
|
||||
nm2Tags.put("C", 1L);
|
||||
nm2Tags.put("D", 2L);
|
||||
expectedAllocationTags.put(nm1.toString(), nm1Tags);
|
||||
expectedAllocationTags.put(nm2.toString(), nm2Tags);
|
||||
|
||||
when(atm.getAllocationTagsWithCount(nm1)).thenReturn(nm1Tags);
|
||||
when(atm.getAllocationTagsWithCount(nm2)).thenReturn(nm2Tags);
|
||||
rm.getRMContext().setAllocationTagsManager(atm);
|
||||
|
||||
rm.start();
|
||||
|
||||
rm.registerNode(nm1.toString(), 1024);
|
||||
rm.registerNode(nm2.toString(), 1024);
|
||||
|
||||
WebResource r = resource();
|
||||
ClientResponse response = r.path("ws").path("v1").path("cluster")
|
||||
.path("nodes").accept("application/json").get(ClientResponse.class);
|
||||
assertEquals(MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8,
|
||||
response.getType().toString());
|
||||
JSONObject nodesInfoJson = response.getEntity(JSONObject.class);
|
||||
verifyNodeAllocationTag(nodesInfoJson, expectedAllocationTags);
|
||||
|
||||
rm.stop();
|
||||
}
|
||||
|
||||
private void verifyNodeAllocationTag(JSONObject json,
|
||||
Map<String, Map<String, Long>> expectedAllocationTags)
|
||||
throws JSONException {
|
||||
JSONArray nodes = json.getJSONObject("nodes").getJSONArray("node");
|
||||
assertEquals(expectedAllocationTags.size(), nodes.length());
|
||||
for (int i=0; i<nodes.length(); i++) {
|
||||
JSONObject nodeJson = nodes.getJSONObject(i);
|
||||
String nodeId = nodeJson.getString("id");
|
||||
|
||||
// Ensure the response contains all nodes info
|
||||
assertTrue("Nodes info should have expected node IDs",
|
||||
expectedAllocationTags.containsKey(nodeId));
|
||||
|
||||
Map<String, Long> expectedTags = expectedAllocationTags.get(nodeId);
|
||||
JSONArray tagsInfo = nodeJson.getJSONObject("allocationTags")
|
||||
.getJSONArray("allocationTagInfo");
|
||||
|
||||
// Ensure number of tags are expected.
|
||||
assertEquals(expectedTags.size(), tagsInfo.length());
|
||||
|
||||
// Iterate expected tags and make sure the actual
|
||||
// tags/counts are matched.
|
||||
Iterator<String> it = expectedTags.keySet().iterator();
|
||||
for (int j=0; j<tagsInfo.length(); j++) {
|
||||
JSONObject tagInfo = tagsInfo.getJSONObject(j);
|
||||
String expectedTag = it.next();
|
||||
assertEquals(tagInfo.getString("allocationTag"), expectedTag);
|
||||
assertEquals(tagInfo.getLong("allocationsCount"),
|
||||
expectedTags.get(expectedTag).longValue());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in New Issue