Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

YARN-6232. Update resource usage and preempted resource calculations to take into account all resource types. Contributed by Varun Vasudev.

This commit is contained in:
Sunil G 2017-03-06 11:34:20 +05:30 committed by Wangda Tan
parent 7805deed48
commit dae65f3bef
41 changed files with 868 additions and 253 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

58
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationResourceUsageReport.java
View File

@ -24,6 +24,9 @@ import org.apache.hadoop.classification.InterfaceStability.Stable;
import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.yarn.util.Records;
import java.util.HashMap;
import java.util.Map;
/**
* Contains various scheduling metrics to be reported by UI and CLI.
*/
@ -35,9 +38,9 @@ public abstract class ApplicationResourceUsageReport {
@Unstable
public static ApplicationResourceUsageReport newInstance(
int numUsedContainers, int numReservedContainers, Resource usedResources,
Resource reservedResources, Resource neededResources, long memorySeconds,
long vcoreSeconds, float queueUsagePerc, float clusterUsagePerc,
long preemptedMemorySeconds, long preemptedVcoresSeconds) {
Resource reservedResources, Resource neededResources,
Map<String, Long> resourceSecondsMap, float queueUsagePerc,
float clusterUsagePerc, Map<String, Long> preemtedResourceSecondsMap) {
ApplicationResourceUsageReport report =
Records.newRecord(ApplicationResourceUsageReport.class);
report.setNumUsedContainers(numUsedContainers);
@ -45,12 +48,10 @@ public abstract class ApplicationResourceUsageReport {
report.setUsedResources(usedResources);
report.setReservedResources(reservedResources);
report.setNeededResources(neededResources);
report.setMemorySeconds(memorySeconds);
report.setVcoreSeconds(vcoreSeconds);
report.setResourceSecondsMap(resourceSecondsMap);
report.setQueueUsagePercentage(queueUsagePerc);
report.setClusterUsagePercentage(clusterUsagePerc);
report.setPreemptedMemorySeconds(preemptedMemorySeconds);
report.setPreemptedVcoreSeconds(preemptedVcoresSeconds);
report.setPreemptedResourceSecondsMap(preemtedResourceSecondsMap);
return report;
}
@ -229,4 +230,47 @@ public abstract class ApplicationResourceUsageReport {
@Public
@Unstable
public abstract long getPreemptedVcoreSeconds();
/**
* Get the aggregated number of resources that the application has
* allocated times the number of seconds the application has been running.
* @return map containing the resource name and aggregated resource-seconds
*/
@Public
@Unstable
public abstract Map<String, Long> getResourceSecondsMap();
/**
* Set the aggregated number of resources that the application has
* allocated times the number of seconds the application has been running.
* @param resourceSecondsMap map containing the resource name and aggregated
* resource-seconds
*/
@Private
@Unstable
public abstract void setResourceSecondsMap(
Map<String, Long> resourceSecondsMap);
/**
* Get the aggregated number of resources preempted that the application has
* allocated times the number of seconds the application has been running.
* @return map containing the resource name and aggregated preempted
* resource-seconds
*/
@Public
@Unstable
public abstract Map<String, Long> getPreemptedResourceSecondsMap();
/**
* Set the aggregated number of resources preempted that the application has
* allocated times the number of seconds the application has been running.
* @param preemptedResourceSecondsMap map containing the resource name and
* aggregated preempted resource-seconds
*/
@Private
@Unstable
public abstract void setPreemptedResourceSecondsMap(
Map<String, Long> preemptedResourceSecondsMap);
}

7
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
View File

@ -195,6 +195,11 @@ message LocalResourceProto {
optional bool should_be_uploaded_to_shared_cache = 7;
}
message StringLongMapProto {
required string key = 1;
required int64 value = 2;
}
message ApplicationResourceUsageReportProto {
optional int32 num_used_containers = 1;
optional int32 num_reserved_containers = 2;
@ -207,6 +212,8 @@ message ApplicationResourceUsageReportProto {
optional float cluster_usage_percentage = 9;
optional int64 preempted_memory_seconds = 10;
optional int64 preempted_vcore_seconds = 11;
repeated StringLongMapProto application_resource_usage_map = 12;
repeated StringLongMapProto application_preempted_resource_usage_map = 13;
}
message ApplicationReportProto {

35
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java
View File

@ -62,6 +62,8 @@ import org.apache.hadoop.yarn.util.Times;
import com.google.common.annotations.VisibleForTesting;
import static org.apache.hadoop.yarn.util.StringHelper.getResourceSecondsString;
@Private
@Unstable
public class ApplicationCLI extends YarnCLI {
@ -711,24 +713,9 @@ public class ApplicationCLI extends YarnCLI {
appReportStr.println(appReport.getRpcPort());
appReportStr.print("\tAM Host : ");
appReportStr.println(appReport.getHost());
appReportStr.print("\tAggregate Resource Allocation : ");
ApplicationResourceUsageReport usageReport =
appReport.getApplicationResourceUsageReport();
if (usageReport != null) {
//completed app report in the timeline server doesn't have usage report
appReportStr.print(usageReport.getMemorySeconds() + " MB-seconds, ");
appReportStr.println(usageReport.getVcoreSeconds() + " vcore-seconds");
appReportStr.print("\tAggregate Resource Preempted : ");
appReportStr.print(usageReport.getPreemptedMemorySeconds() +
" MB-seconds, ");
appReportStr.println(usageReport.getPreemptedVcoreSeconds() +
" vcore-seconds");
} else {
appReportStr.println("N/A");
appReportStr.print("\tAggregate Resource Preempted : ");
appReportStr.println("N/A");
}
printResourceUsage(appReportStr, usageReport);
appReportStr.print("\tLog Aggregation Status : ");
appReportStr.println(appReport.getLogAggregationStatus() == null ? "N/A"
: appReport.getLogAggregationStatus());
@ -759,6 +746,22 @@ public class ApplicationCLI extends YarnCLI {
return 0;
}
private void printResourceUsage(PrintWriter appReportStr,
ApplicationResourceUsageReport usageReport) {
appReportStr.print("\tAggregate Resource Allocation : ");
if (usageReport != null) {
appReportStr.println(
getResourceSecondsString(usageReport.getResourceSecondsMap()));
appReportStr.print("\tAggregate Resource Preempted : ");
appReportStr.println(getResourceSecondsString(
usageReport.getPreemptedResourceSecondsMap()));
} else {
appReportStr.println("N/A");
appReportStr.print("\tAggregate Resource Preempted : ");
appReportStr.println("N/A");
}
}
private String getAllValidApplicationStates() {
StringBuilder sb = new StringBuilder();
sb.append("The valid application state can be" + " one of the following: ");

16
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java
View File

@ -39,8 +39,10 @@ import java.util.ArrayList;
import java.util.Collections;
import java.util.Date;
import java.util.EnumSet;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.regex.Pattern;
@ -69,6 +71,7 @@ import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.api.records.QueueInfo;
import org.apache.hadoop.yarn.api.records.QueueState;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.ResourceInformation;
import org.apache.hadoop.yarn.api.records.ResourceUtilization;
import org.apache.hadoop.yarn.api.records.SignalContainerCommand;
import org.apache.hadoop.yarn.api.records.YarnApplicationAttemptState;
@ -118,9 +121,18 @@ public class TestYarnCLI {
for (int i = 0; i < 2; ++i) {
ApplicationCLI cli = createAndGetAppCLI();
ApplicationId applicationId = ApplicationId.newInstance(1234, 5);
Map<String, Long> resourceSecondsMap = new HashMap<>();
Map<String, Long> preemptedResoureSecondsMap = new HashMap<>();
resourceSecondsMap.put(ResourceInformation.MEMORY_MB.getName(), 123456L);
resourceSecondsMap.put(ResourceInformation.VCORES.getName(), 4567L);
preemptedResoureSecondsMap
.put(ResourceInformation.MEMORY_MB.getName(), 1111L);
preemptedResoureSecondsMap
.put(ResourceInformation.VCORES.getName(), 2222L);
ApplicationResourceUsageReport usageReport = i == 0 ? null :
ApplicationResourceUsageReport.newInstance(
2, 0, null, null, null, 123456, 4567, 0, 0, 1111, 2222);
ApplicationResourceUsageReport
.newInstance(2, 0, null, null, null, resourceSecondsMap, 0, 0,
preemptedResoureSecondsMap);
ApplicationReport newApplicationReport = ApplicationReport.newInstance(
applicationId, ApplicationAttemptId.newInstance(applicationId, 1),
"user", "queue", "appname", "host", 124, null,

149
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationResourceUsageReportPBImpl.java
View File

@ -22,12 +22,16 @@ import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.yarn.api.records.ApplicationResourceUsageReport;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.ResourceInformation;
import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationResourceUsageReportProto;
import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationResourceUsageReportProtoOrBuilder;
import org.apache.hadoop.yarn.proto.YarnProtos.ResourceProto;
import com.google.protobuf.TextFormat;
import java.util.HashMap;
import java.util.Map;
@Private
@Unstable
public class ApplicationResourceUsageReportPBImpl
@ -41,6 +45,9 @@ extends ApplicationResourceUsageReport {
Resource reservedResources;
Resource neededResources;
private Map<String, Long> resourceSecondsMap;
private Map<String, Long> preemptedResourceSecondsMap;
public ApplicationResourceUsageReportPBImpl() {
builder = ApplicationResourceUsageReportProto.newBuilder();
}
@ -49,6 +56,8 @@ extends ApplicationResourceUsageReport {
ApplicationResourceUsageReportProto proto) {
this.proto = proto;
viaProto = true;
getResourceSecondsMap();
getPreemptedResourceSecondsMap();
}
public synchronized ApplicationResourceUsageReportProto getProto() {
@ -89,6 +98,23 @@ extends ApplicationResourceUsageReport {
if (this.neededResources != null) {
builder.setNeededResources(convertToProtoFormat(this.neededResources));
}
builder.clearApplicationResourceUsageMap();
builder.clearApplicationPreemptedResourceUsageMap();
if (preemptedResourceSecondsMap != null && !preemptedResourceSecondsMap
.isEmpty()) {
builder.addAllApplicationPreemptedResourceUsageMap(ProtoUtils
.convertMapToStringLongMapProtoList(preemptedResourceSecondsMap));
}
if (resourceSecondsMap != null && !resourceSecondsMap.isEmpty()) {
builder.addAllApplicationResourceUsageMap(
ProtoUtils.convertMapToStringLongMapProtoList(resourceSecondsMap));
}
builder.setMemorySeconds(this.getMemorySeconds());
builder.setVcoreSeconds(this.getVcoreSeconds());
builder.setPreemptedMemorySeconds(this.getPreemptedMemorySeconds());
builder.setPreemptedVcoreSeconds(this.getPreemptedVcoreSeconds());
}
private void mergeLocalToProto() {
@ -196,54 +222,64 @@ extends ApplicationResourceUsageReport {
@Override
public synchronized void setMemorySeconds(long memory_seconds) {
maybeInitBuilder();
builder.setMemorySeconds(memory_seconds);
getResourceSecondsMap()
.put(ResourceInformation.MEMORY_MB.getName(), memory_seconds);
}
@Override
public synchronized long getMemorySeconds() {
ApplicationResourceUsageReportProtoOrBuilder p = viaProto ? proto : builder;
return p.getMemorySeconds();
Map<String, Long> tmp = getResourceSecondsMap();
if (tmp.containsKey(ResourceInformation.MEMORY_MB.getName())) {
return tmp.get(ResourceInformation.MEMORY_MB.getName());
}
return 0;
}
@Override
public synchronized void setVcoreSeconds(long vcore_seconds) {
maybeInitBuilder();
builder.setVcoreSeconds(vcore_seconds);
getResourceSecondsMap()
.put(ResourceInformation.VCORES.getName(), vcore_seconds);
}
@Override
public synchronized long getVcoreSeconds() {
ApplicationResourceUsageReportProtoOrBuilder p = viaProto ? proto : builder;
return (p.getVcoreSeconds());
Map<String, Long> tmp = getResourceSecondsMap();
if (tmp.containsKey(ResourceInformation.VCORES.getName())) {
return tmp.get(ResourceInformation.VCORES.getName());
}
return 0;
}
@Override
public synchronized void setPreemptedMemorySeconds(
long preemptedMemorySeconds) {
maybeInitBuilder();
builder.setPreemptedMemorySeconds(preemptedMemorySeconds);
getPreemptedResourceSecondsMap()
.put(ResourceInformation.MEMORY_MB.getName(), preemptedMemorySeconds);
}
@Override
public synchronized long getPreemptedMemorySeconds() {
ApplicationResourceUsageReportProtoOrBuilder p =
viaProto ? proto : builder;
return p.getPreemptedMemorySeconds();
Map<String, Long> tmp = getPreemptedResourceSecondsMap();
if (tmp.containsKey(ResourceInformation.MEMORY_MB.getName())) {
return tmp.get(ResourceInformation.MEMORY_MB.getName());
}
return 0;
}
@Override
public synchronized void setPreemptedVcoreSeconds(
long vcoreSeconds) {
maybeInitBuilder();
builder.setPreemptedVcoreSeconds(vcoreSeconds);
getPreemptedResourceSecondsMap()
.put(ResourceInformation.VCORES.getName(), vcoreSeconds);
}
@Override
public synchronized long getPreemptedVcoreSeconds() {
ApplicationResourceUsageReportProtoOrBuilder p =
viaProto ? proto : builder;
return (p.getPreemptedVcoreSeconds());
Map<String, Long> tmp = getPreemptedResourceSecondsMap();
if (tmp.containsKey(ResourceInformation.VCORES.getName())) {
return tmp.get(ResourceInformation.VCORES.getName());
}
return 0;
}
private ResourcePBImpl convertFromProtoFormat(ResourceProto p) {
@ -277,4 +313,81 @@ extends ApplicationResourceUsageReport {
maybeInitBuilder();
builder.setClusterUsagePercentage((clusterUsagePerc));
}
@Override
public synchronized void setResourceSecondsMap(
Map<String, Long> resourceSecondsMap) {
this.resourceSecondsMap = resourceSecondsMap;
if (resourceSecondsMap == null) {
return;
}
if (!resourceSecondsMap
.containsKey(ResourceInformation.MEMORY_MB.getName())) {
this.setMemorySeconds(0L);
}
if (!resourceSecondsMap.containsKey(ResourceInformation.VCORES.getName())) {
this.setVcoreSeconds(0L);
}
}
@Override
public synchronized Map<String, Long> getResourceSecondsMap() {
if (this.resourceSecondsMap != null) {
return this.resourceSecondsMap;
}
ApplicationResourceUsageReportProtoOrBuilder p = viaProto ? proto : builder;
this.resourceSecondsMap = ProtoUtils
.convertStringLongMapProtoListToMap(
p.getApplicationResourceUsageMapList());
if (!this.resourceSecondsMap
.containsKey(ResourceInformation.MEMORY_MB.getName())) {
this.setMemorySeconds(p.getMemorySeconds());
}
if (!this.resourceSecondsMap
.containsKey(ResourceInformation.VCORES.getName())) {
this.setVcoreSeconds(p.getVcoreSeconds());
}
this.setMemorySeconds(p.getMemorySeconds());
this.setVcoreSeconds(p.getVcoreSeconds());
return this.resourceSecondsMap;
}
@Override
public synchronized void setPreemptedResourceSecondsMap(
Map<String, Long> preemptedResourceSecondsMap) {
this.preemptedResourceSecondsMap = preemptedResourceSecondsMap;
if (preemptedResourceSecondsMap == null) {
return;
}
if (!preemptedResourceSecondsMap
.containsKey(ResourceInformation.MEMORY_MB.getName())) {
this.setPreemptedMemorySeconds(0L);
}
if (!preemptedResourceSecondsMap
.containsKey(ResourceInformation.VCORES.getName())) {
this.setPreemptedVcoreSeconds(0L);
}
}
@Override
public synchronized Map<String, Long> getPreemptedResourceSecondsMap() {
if (this.preemptedResourceSecondsMap != null) {
return this.preemptedResourceSecondsMap;
}
ApplicationResourceUsageReportProtoOrBuilder p = viaProto ? proto : builder;
this.preemptedResourceSecondsMap = ProtoUtils
.convertStringLongMapProtoListToMap(
p.getApplicationPreemptedResourceUsageMapList());
if (!this.preemptedResourceSecondsMap
.containsKey(ResourceInformation.MEMORY_MB.getName())) {
this.setPreemptedMemorySeconds(p.getPreemptedMemorySeconds());
}
if (!this.preemptedResourceSecondsMap
.containsKey(ResourceInformation.VCORES.getName())) {
this.setPreemptedVcoreSeconds(p.getPreemptedVcoreSeconds());
}
this.setPreemptedMemorySeconds(p.getPreemptedMemorySeconds());
this.setPreemptedVcoreSeconds(p.getPreemptedVcoreSeconds());
return this.preemptedResourceSecondsMap;
}
}

34
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ProtoUtils.java
View File

@ -19,6 +19,10 @@
package org.apache.hadoop.yarn.api.records.impl.pb;
import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.classification.InterfaceStability.Unstable;
@ -45,6 +49,7 @@ import org.apache.hadoop.yarn.api.records.QueueACL;
import org.apache.hadoop.yarn.api.records.QueueState;
import org.apache.hadoop.yarn.api.records.ReservationRequestInterpreter;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.ResourceInformation;
import org.apache.hadoop.yarn.api.records.UpdateContainerError;
import org.apache.hadoop.yarn.api.records.UpdateContainerRequest;
import org.apache.hadoop.yarn.api.records.YarnApplicationAttemptState;
@ -446,6 +451,35 @@ public class ProtoUtils {
public static ResourceTypes convertFromProtoFormat(ResourceTypesProto e) {
return ResourceTypes.valueOf(e.name());
}
public static Map<String, Long> convertStringLongMapProtoListToMap(
List<YarnProtos.StringLongMapProto> pList) {
Resource tmp = Resource.newInstance(0, 0);
Map<String, Long> ret = new HashMap<>();
for (Map.Entry<String, ResourceInformation> entry : tmp.getResources()
.entrySet()) {
ret.put(entry.getKey(), 0L);
}
if (pList != null) {
for (YarnProtos.StringLongMapProto p : pList) {
ret.put(p.getKey(), p.getValue());
}
}
return ret;
}
public static List<YarnProtos.StringLongMapProto> convertMapToStringLongMapProtoList(
Map<String, Long> map) {
List<YarnProtos.StringLongMapProto> ret = new ArrayList<>();
for (Map.Entry<String, Long> entry : map.entrySet()) {
YarnProtos.StringLongMapProto.Builder tmp =
YarnProtos.StringLongMapProto.newBuilder();
tmp.setKey(entry.getKey());
tmp.setValue(entry.getValue());
ret.add(tmp.build());
}
return ret;
}
}

36
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/StringHelper.java
View File

@ -20,9 +20,15 @@ package org.apache.hadoop.yarn.util;
import com.google.common.base.Joiner;
import com.google.common.base.Splitter;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.regex.Pattern;
import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.yarn.api.records.ResourceInformation;
import org.apache.hadoop.yarn.util.resource.ResourceUtils;
/**
* Common string manipulation helpers
@ -174,4 +180,34 @@ public final class StringHelper {
}
sb.append(part);
}
public static String getResourceSecondsString(Map<String, Long> targetMap) {
List<String> strings = new ArrayList<>(targetMap.size());
//completed app report in the timeline server doesn't have usage report
Long memorySeconds = 0L;
Long vcoreSeconds = 0L;
if (targetMap.containsKey(ResourceInformation.MEMORY_MB.getName())) {
memorySeconds = targetMap.get(ResourceInformation.MEMORY_MB.getName());
}
if (targetMap.containsKey(ResourceInformation.VCORES.getName())) {
vcoreSeconds = targetMap.get(ResourceInformation.VCORES.getName());
}
strings.add(memorySeconds + " MB-seconds");
strings.add(vcoreSeconds + " vcore-seconds");
Map<String, ResourceInformation> tmp = ResourceUtils.getResourceTypes();
if (targetMap.size() > 2) {
for (Map.Entry<String, Long> entry : targetMap.entrySet()) {
if (!entry.getKey().equals(ResourceInformation.MEMORY_MB.getName())
&& !entry.getKey().equals(ResourceInformation.VCORES.getName())) {
String units = "";
if (tmp.containsKey(entry.getKey())) {
units = tmp.get(entry.getKey()).getUnits();
}
strings.add(entry.getValue() + " " + entry.getKey() + "-" + units
+ "seconds");
}
}
}
return String.join(", ", strings);
}
}

12
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/BasePBImplRecordsTest.java
View File

@ -37,6 +37,9 @@ public class BasePBImplRecordsTest {
@SuppressWarnings("checkstyle:visibilitymodifier")
protected static HashMap<Type, Object> typeValueCache =
new HashMap<Type, Object>();
@SuppressWarnings("checkstyle:visibilitymodifier")
protected static HashMap<Type, List<String>> excludedPropertiesMap =
new HashMap<>();
private static Random rand = new Random();
private static byte [] bytes = new byte[] {'1', '2', '3', '4'};
@ -167,6 +170,10 @@ public class BasePBImplRecordsTest {
private <R> Map<String, GetSetPair> getGetSetPairs(Class<R> recordClass)
throws Exception {
Map<String, GetSetPair> ret = new HashMap<String, GetSetPair>();
List<String> excluded = null;
if (excludedPropertiesMap.containsKey(recordClass.getClass())) {
excluded = excludedPropertiesMap.get(recordClass.getClass());
}
Method [] methods = recordClass.getDeclaredMethods();
// get all get methods
for (int i = 0; i < methods.length; i++) {
@ -224,6 +231,11 @@ public class BasePBImplRecordsTest {
(gsp.setMethod == null)) {
LOG.info(String.format("Exclude potential property: %s\n", gsp.propertyName));
itr.remove();
} else if ((excluded != null && excluded.contains(gsp.propertyName))) {
LOG.info(String.format(
"Excluding potential property(present in exclusion list): %s\n",
gsp.propertyName));
itr.remove();
} else {
LOG.info(String.format("New property: %s type: %s", gsp.toString(), gsp.type));
gsp.testValue = genTypeValue(gsp.type);

4
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestPBImplRecords.java
View File

@ -343,6 +343,8 @@ import org.junit.Ignore;
import org.junit.Test;
import com.google.common.collect.ImmutableSet;
import java.io.IOException;
import java.util.Arrays;
/**
* Test class for YARN API protocol records.
@ -745,6 +747,8 @@ public class TestPBImplRecords extends BasePBImplRecordsTest {
@Test
public void testApplicationResourceUsageReportPBImpl() throws Exception {
excludedPropertiesMap.put(ApplicationResourceUsageReportPBImpl.class.getClass(),
Arrays.asList("PreemptedResourceSecondsMap", "ResourceSecondsMap"));
validatePBImplRecord(ApplicationResourceUsageReportPBImpl.class,
ApplicationResourceUsageReportProto.class);
}

18
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java
View File

@ -46,6 +46,7 @@ import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.ResourceInformation;
import org.apache.hadoop.yarn.api.records.YarnApplicationAttemptState;
import org.apache.hadoop.yarn.api.records.YarnApplicationState;
import org.apache.hadoop.yarn.api.records.timeline.TimelineEntities;
@ -338,9 +339,20 @@ public class ApplicationHistoryManagerOnTimelineStore extends AbstractService
ApplicationMetricsConstants.APP_MEM_PREEMPT_METRICS);
long preemptedVcoreSeconds = parseLong(entityInfo,
ApplicationMetricsConstants.APP_CPU_PREEMPT_METRICS);
appResources = ApplicationResourceUsageReport.newInstance(0, 0, null,
null, null, memorySeconds, vcoreSeconds, 0, 0,
preemptedMemorySeconds, preemptedVcoreSeconds);
Map<String, Long> resourceSecondsMap = new HashMap<>();
Map<String, Long> preemptedResoureSecondsMap = new HashMap<>();
resourceSecondsMap
.put(ResourceInformation.MEMORY_MB.getName(), memorySeconds);
resourceSecondsMap
.put(ResourceInformation.VCORES.getName(), vcoreSeconds);
preemptedResoureSecondsMap.put(ResourceInformation.MEMORY_MB.getName(),
preemptedMemorySeconds);
preemptedResoureSecondsMap
.put(ResourceInformation.VCORES.getName(), preemptedVcoreSeconds);
appResources = ApplicationResourceUsageReport
.newInstance(0, 0, null, null, null, resourceSecondsMap, 0, 0,
preemptedResoureSecondsMap);
}
if (entityInfo.containsKey(ApplicationMetricsConstants.APP_TAGS_INFO)) {

14
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java
View File

@ -65,8 +65,6 @@ import org.apache.hadoop.yarn.api.records.ResourceUtilization;
import org.apache.hadoop.yarn.api.records.Token;
import org.apache.hadoop.yarn.api.records.URL;
import org.apache.hadoop.yarn.api.records.YarnApplicationState;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.factories.RecordFactory;
import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
import org.apache.hadoop.yarn.nodelabels.CommonNodeLabelsManager;
@ -450,9 +448,9 @@ public class BuilderUtils {
public static ApplicationResourceUsageReport newApplicationResourceUsageReport(
int numUsedContainers, int numReservedContainers, Resource usedResources,
Resource reservedResources, Resource neededResources, long memorySeconds,
long vcoreSeconds, long preemptedMemorySeconds,
long preemptedVcoreSeconds) {
Resource reservedResources, Resource neededResources,
Map<String, Long> resourceSecondsMap,
Map<String, Long> preemptedResourceSecondsMap) {
ApplicationResourceUsageReport report =
recordFactory.newRecordInstance(ApplicationResourceUsageReport.class);
report.setNumUsedContainers(numUsedContainers);
@ -460,10 +458,8 @@ public class BuilderUtils {
report.setUsedResources(usedResources);
report.setReservedResources(reservedResources);
report.setNeededResources(neededResources);
report.setMemorySeconds(memorySeconds);
report.setVcoreSeconds(vcoreSeconds);
report.setPreemptedMemorySeconds(preemptedMemorySeconds);
report.setPreemptedVcoreSeconds(preemptedVcoreSeconds);
report.setResourceSecondsMap(resourceSecondsMap);
report.setPreemptedResourceSecondsMap(preemptedResourceSecondsMap);
return report;
}

8
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
View File

@ -70,6 +70,7 @@ import org.apache.hadoop.yarn.util.Times;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.util.concurrent.SettableFuture;
import org.apache.hadoop.yarn.util.StringHelper;
/**
* This class manages the list of applications for the resource manager.
@ -189,7 +190,12 @@ public class RMAppManager implements EventHandler<RMAppManagerEvent>,
.add("preemptedAMContainers", metrics.getNumAMContainersPreempted())
.add("preemptedNonAMContainers", metrics.getNumNonAMContainersPreempted())
.add("preemptedResources", metrics.getResourcePreempted())
.add("applicationType", app.getApplicationType());
.add("applicationType", app.getApplicationType())
.add("resourceSeconds", StringHelper
.getResourceSecondsString(metrics.getResourceSecondsMap()))
.add("preemptedResourceSeconds", StringHelper
.getResourceSecondsString(
metrics.getPreemptedResourceSecondsMap()));
return summary;
}

10
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java
View File

@ -483,7 +483,7 @@ public class RMServerUtils {
DUMMY_APPLICATION_RESOURCE_USAGE_REPORT =
BuilderUtils.newApplicationResourceUsageReport(-1, -1,
Resources.createResource(-1, -1), Resources.createResource(-1, -1),
Resources.createResource(-1, -1), 0, 0, 0, 0);
Resources.createResource(-1, -1), new HashMap<>(), new HashMap<>());
/**
@ -672,4 +672,12 @@ public class RMServerUtils {
.debug("Converted profile to resource capability for ask " + ask);
}
}
public static Long getOrDefault(Map<String, Long> map, String key,
Long defaultValue) {
if (map.containsKey(key)) {
return map.get(key);
}
return defaultValue;
}
}

7
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java
View File

@ -851,11 +851,8 @@ public abstract class RMStateStore extends AbstractService {
appAttempt.getAppAttemptId(),
appAttempt.getMasterContainer(),
credentials, appAttempt.getStartTime(),
resUsage.getMemorySeconds(),
resUsage.getVcoreSeconds(),
attempMetrics.getPreemptedMemory(),
attempMetrics.getPreemptedVcore()
);
resUsage.getResourceUsageSecondsMap(),
attempMetrics.getPreemptedResourceSecondsMap());
getRMStateStoreEventHandler().handle(
new RMStateStoreAppAttemptEvent(attemptState));

89
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/records/ApplicationAttemptStateData.java
View File

@ -25,23 +25,28 @@ import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.ContainerExitStatus;
import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
import org.apache.hadoop.yarn.api.records.ResourceInformation;
import org.apache.hadoop.yarn.proto.YarnServerResourceManagerRecoveryProtos.ApplicationAttemptStateDataProto;
import org.apache.hadoop.yarn.server.resourcemanager.RMServerUtils;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptState;
import org.apache.hadoop.yarn.util.Records;
import java.util.Map;
/*
* Contains the state data that needs to be persisted for an ApplicationAttempt
*/
@Public
@Unstable
public abstract class ApplicationAttemptStateData {
public static ApplicationAttemptStateData newInstance(
ApplicationAttemptId attemptId, Container container,
Credentials attemptTokens, long startTime, RMAppAttemptState finalState,
String finalTrackingUrl, String diagnostics,
FinalApplicationStatus amUnregisteredFinalStatus, int exitStatus,
long finishTime, long memorySeconds, long vcoreSeconds,
long preemptedMemorySeconds, long preemptedVcoreSeconds) {
long finishTime, Map<String, Long> resourceSecondsMap,
Map<String, Long> preemptedResourceSecondsMap) {
ApplicationAttemptStateData attemptStateData =
Records.newRecord(ApplicationAttemptStateData.class);
attemptStateData.setAttemptId(attemptId);
@ -54,23 +59,33 @@ public abstract class ApplicationAttemptStateData {
attemptStateData.setFinalApplicationStatus(amUnregisteredFinalStatus);
attemptStateData.setAMContainerExitStatus(exitStatus);
attemptStateData.setFinishTime(finishTime);
attemptStateData.setMemorySeconds(memorySeconds);
attemptStateData.setVcoreSeconds(vcoreSeconds);
attemptStateData.setPreemptedMemorySeconds(preemptedMemorySeconds);
attemptStateData.setPreemptedVcoreSeconds(preemptedVcoreSeconds);
attemptStateData.setMemorySeconds(RMServerUtils
.getOrDefault(resourceSecondsMap,
ResourceInformation.MEMORY_MB.getName(), 0L));
attemptStateData.setVcoreSeconds(RMServerUtils
.getOrDefault(resourceSecondsMap, ResourceInformation.VCORES.getName(),
0L));
attemptStateData.setPreemptedMemorySeconds(RMServerUtils
.getOrDefault(preemptedResourceSecondsMap,
ResourceInformation.MEMORY_MB.getName(), 0L));
attemptStateData.setPreemptedVcoreSeconds(RMServerUtils
.getOrDefault(preemptedResourceSecondsMap,
ResourceInformation.VCORES.getName(), 0L));
attemptStateData.setResourceSecondsMap(resourceSecondsMap);
attemptStateData
.setPreemptedResourceSecondsMap(preemptedResourceSecondsMap);
return attemptStateData;
}
public static ApplicationAttemptStateData newInstance(
ApplicationAttemptId attemptId, Container masterContainer,
Credentials attemptTokens, long startTime, long memorySeconds,
long vcoreSeconds, long preemptedMemorySeconds,
long preemptedVcoreSeconds) {
return newInstance(attemptId, masterContainer, attemptTokens,
startTime, null, "N/A", "", null, ContainerExitStatus.INVALID, 0,
memorySeconds, vcoreSeconds,
preemptedMemorySeconds, preemptedVcoreSeconds);
}
Credentials attemptTokens, long startTime,
Map<String, Long> resourceSeondsMap,
Map<String, Long> preemptedResourceSecondsMap) {
return newInstance(attemptId, masterContainer, attemptTokens, startTime,
null, "N/A", "", null, ContainerExitStatus.INVALID, 0,
resourceSeondsMap, preemptedResourceSecondsMap);
}
public abstract ApplicationAttemptStateDataProto getProto();
@ -215,4 +230,50 @@ public abstract class ApplicationAttemptStateData {
@Public
@Unstable
public abstract void setPreemptedVcoreSeconds(long vcoreSeconds);
/**
* Get the aggregated number of resources preempted that the application has
* allocated times the number of seconds the application has been running.
*
* @return map containing the resource name and aggregated preempted
* resource-seconds
*/
@Public
@Unstable
public abstract Map<String, Long> getResourceSecondsMap();
/**
* Set the aggregated number of resources that the application has
* allocated times the number of seconds the application has been running.
*
* @param resourceSecondsMap map containing the resource name and aggregated
* resource-seconds
*/
@Public
@Unstable
public abstract void setResourceSecondsMap(
Map<String, Long> resourceSecondsMap);
/**
* Get the aggregated number of resources preempted that the application has
* allocated times the number of seconds the application has been running.
*
* @return map containing the resource name and aggregated preempted
* resource-seconds
*/
@Public
@Unstable
public abstract Map<String, Long> getPreemptedResourceSecondsMap();
/**
* Set the aggregated number of resources preempted that the application has
* allocated times the number of seconds the application has been running.
*
* @param preemptedResourceSecondsMap map containing the resource name and
* aggregated preempted resource-seconds
*/
@Public
@Unstable
public abstract void setPreemptedResourceSecondsMap(
Map<String, Long> preemptedResourceSecondsMap);
}

50
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/records/impl/pb/ApplicationAttemptStateDataPBImpl.java
View File

@ -20,6 +20,7 @@ package org.apache.hadoop.yarn.server.resourcemanager.recovery.records.impl.pb;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.Map;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@ -55,6 +56,9 @@ public class ApplicationAttemptStateDataPBImpl extends
private Container masterContainer = null;
private ByteBuffer appAttemptTokens = null;
private Map<String, Long> resourceSecondsMap;
private Map<String, Long> preemptedResourceSecondsMap;
public ApplicationAttemptStateDataPBImpl() {
builder = ApplicationAttemptStateDataProto.newBuilder();
}
@ -404,4 +408,50 @@ public class ApplicationAttemptStateDataPBImpl extends
IOUtils.closeStream(dibb);
}
}
@Override
public Map<String, Long> getResourceSecondsMap() {
if (this.resourceSecondsMap != null) {
return this.resourceSecondsMap;
}
ApplicationAttemptStateDataProtoOrBuilder p = viaProto ? proto : builder;
this.resourceSecondsMap = ProtoUtils.convertStringLongMapProtoListToMap(
p.getApplicationResourceUsageMapList());
return this.resourceSecondsMap;
}
@Override
public void setResourceSecondsMap(Map<String, Long> resourceSecondsMap) {
maybeInitBuilder();
builder.clearApplicationResourceUsageMap();
this.resourceSecondsMap = resourceSecondsMap;
if (resourceSecondsMap != null) {
builder.addAllApplicationResourceUsageMap(
ProtoUtils.convertMapToStringLongMapProtoList(resourceSecondsMap));
}
}
@Override
public Map<String, Long> getPreemptedResourceSecondsMap() {
if (this.preemptedResourceSecondsMap != null) {
return this.preemptedResourceSecondsMap;
}
ApplicationAttemptStateDataProtoOrBuilder p = viaProto ? proto : builder;
this.preemptedResourceSecondsMap = ProtoUtils
.convertStringLongMapProtoListToMap(
p.getApplicationResourceUsageMapList());
return this.preemptedResourceSecondsMap;
}
@Override
public void setPreemptedResourceSecondsMap(
Map<String, Long> preemptedResourceSecondsMap) {
maybeInitBuilder();
builder.clearPreemptedResourceUsageMap();
this.preemptedResourceSecondsMap = preemptedResourceSecondsMap;
if (preemptedResourceSecondsMap != null) {
builder.addAllPreemptedResourceUsageMap(ProtoUtils
.convertMapToStringLongMapProtoList(preemptedResourceSecondsMap));
}
}
}

43
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
View File

@ -748,14 +748,10 @@ public class RMAppImpl implements RMApp, Recoverable {
}
RMAppMetrics rmAppMetrics = getRMAppMetrics();
appUsageReport.setMemorySeconds(rmAppMetrics.getMemorySeconds());
appUsageReport.setVcoreSeconds(rmAppMetrics.getVcoreSeconds());
appUsageReport.
setPreemptedMemorySeconds(rmAppMetrics.
getPreemptedMemorySeconds());
appUsageReport.
setPreemptedVcoreSeconds(rmAppMetrics.
getPreemptedVcoreSeconds());
appUsageReport
.setResourceSecondsMap(rmAppMetrics.getResourceSecondsMap());
appUsageReport.setPreemptedResourceSecondsMap(
rmAppMetrics.getPreemptedResourceSecondsMap());
}
if (currentApplicationAttemptId == null) {
@ -1630,10 +1626,9 @@ public class RMAppImpl implements RMApp, Recoverable {
Resource resourcePreempted = Resource.newInstance(0, 0);
int numAMContainerPreempted = 0;
int numNonAMContainerPreempted = 0;
long memorySeconds = 0;
long vcoreSeconds = 0;
long preemptedMemorySeconds = 0;
long preemptedVcoreSeconds = 0;
Map<String, Long> resourceSecondsMap = new HashMap<>();
Map<String, Long> preemptedSecondsMap = new HashMap<>();
for (RMAppAttempt attempt : attempts.values()) {
if (null != attempt) {
RMAppAttemptMetrics attemptMetrics =
@ -1647,17 +1642,25 @@ public class RMAppImpl implements RMApp, Recoverable {
// for both running and finished containers.
AggregateAppResourceUsage resUsage =
attempt.getRMAppAttemptMetrics().getAggregateAppResourceUsage();
memorySeconds += resUsage.getMemorySeconds();
vcoreSeconds += resUsage.getVcoreSeconds();
preemptedMemorySeconds += attemptMetrics.getPreemptedMemory();
preemptedVcoreSeconds += attemptMetrics.getPreemptedVcore();
for (Map.Entry<String, Long> entry : resUsage
.getResourceUsageSecondsMap().entrySet()) {
long value = RMServerUtils
.getOrDefault(resourceSecondsMap, entry.getKey(), 0L);
value += entry.getValue();
resourceSecondsMap.put(entry.getKey(), value);
}
for (Map.Entry<String, Long> entry : attemptMetrics
.getPreemptedResourceSecondsMap().entrySet()) {
long value = RMServerUtils
.getOrDefault(preemptedSecondsMap, entry.getKey(), 0L);
value += entry.getValue();
preemptedSecondsMap.put(entry.getKey(), value);
}
}
}
return new RMAppMetrics(resourcePreempted,
numNonAMContainerPreempted, numAMContainerPreempted,
memorySeconds, vcoreSeconds,
preemptedMemorySeconds, preemptedVcoreSeconds);
return new RMAppMetrics(resourcePreempted, numNonAMContainerPreempted,
numAMContainerPreempted, resourceSecondsMap, preemptedSecondsMap);
}
@Private

41
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppMetrics.java
View File

@ -19,27 +19,27 @@
package org.apache.hadoop.yarn.server.resourcemanager.rmapp;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.ResourceInformation;
import org.apache.hadoop.yarn.server.resourcemanager.RMServerUtils;
import java.util.Map;
public class RMAppMetrics {
final Resource resourcePreempted;
final int numNonAMContainersPreempted;
final int numAMContainersPreempted;
final long memorySeconds;
final long vcoreSeconds;
private final long preemptedMemorySeconds;
private final long preemptedVcoreSeconds;
private final Map<String, Long> resourceSecondsMap;
private final Map<String, Long> preemptedResourceSecondsMap;
public RMAppMetrics(Resource resourcePreempted,
int numNonAMContainersPreempted, int numAMContainersPreempted,
long memorySeconds, long vcoreSeconds, long preemptedMemorySeconds,
long preemptedVcoreSeconds) {
Map<String, Long> resourceSecondsMap,
Map<String, Long> preemptedResourceSecondsMap) {
this.resourcePreempted = resourcePreempted;
this.numNonAMContainersPreempted = numNonAMContainersPreempted;
this.numAMContainersPreempted = numAMContainersPreempted;
this.memorySeconds = memorySeconds;
this.vcoreSeconds = vcoreSeconds;
this.preemptedMemorySeconds = preemptedMemorySeconds;
this.preemptedVcoreSeconds = preemptedVcoreSeconds;
this.resourceSecondsMap = resourceSecondsMap;
this.preemptedResourceSecondsMap = preemptedResourceSecondsMap;
}
public Resource getResourcePreempted() {
@ -55,19 +55,32 @@ public class RMAppMetrics {
}
public long getMemorySeconds() {
return memorySeconds;
return RMServerUtils.getOrDefault(resourceSecondsMap,
ResourceInformation.MEMORY_MB.getName(), 0L);
}
public long getVcoreSeconds() {
return vcoreSeconds;
return RMServerUtils
.getOrDefault(resourceSecondsMap, ResourceInformation.VCORES.getName(),
0L);
}
public long getPreemptedMemorySeconds() {
return preemptedMemorySeconds;
return RMServerUtils.getOrDefault(preemptedResourceSecondsMap,
ResourceInformation.MEMORY_MB.getName(), 0L);
}
public long getPreemptedVcoreSeconds() {
return preemptedVcoreSeconds;
return RMServerUtils.getOrDefault(preemptedResourceSecondsMap,
ResourceInformation.VCORES.getName(), 0L);
}
public Map<String, Long> getResourceSecondsMap() {
return resourceSecondsMap;
}
public Map<String, Long> getPreemptedResourceSecondsMap() {
return preemptedResourceSecondsMap;
}
}

34
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/AggregateAppResourceUsage.java
View File

@ -19,42 +19,38 @@
package org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt;
import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.yarn.api.records.ResourceInformation;
import org.apache.hadoop.yarn.server.resourcemanager.RMServerUtils;
import java.util.HashMap;
import java.util.Map;
@Private
public class AggregateAppResourceUsage {
long memorySeconds;
long vcoreSeconds;
private Map<String, Long> resourceSecondsMap = new HashMap<>();
public AggregateAppResourceUsage(long memorySeconds, long vcoreSeconds) {
this.memorySeconds = memorySeconds;
this.vcoreSeconds = vcoreSeconds;
public AggregateAppResourceUsage(Map<String, Long> resourceSecondsMap) {
this.resourceSecondsMap.putAll(resourceSecondsMap);
}
/**
* @return the memorySeconds
*/
public long getMemorySeconds() {
return memorySeconds;
}
/**
* @param memorySeconds the memorySeconds to set
*/
public void setMemorySeconds(long memorySeconds) {
this.memorySeconds = memorySeconds;
return RMServerUtils.getOrDefault(resourceSecondsMap,
ResourceInformation.MEMORY_MB.getName(), 0L);
}
/**
* @return the vcoreSeconds
*/
public long getVcoreSeconds() {
return vcoreSeconds;
return RMServerUtils
.getOrDefault(resourceSecondsMap, ResourceInformation.VCORES.getName(),
0L);
}
/**
* @param vcoreSeconds the vcoreSeconds to set
*/
public void setVcoreSeconds(long vcoreSeconds) {
this.vcoreSeconds = vcoreSeconds;
public Map<String, Long> getResourceUsageSecondsMap() {
return resourceSecondsMap;
}
}

32
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java
View File

@ -953,12 +953,9 @@ public class RMAppAttemptImpl implements RMAppAttempt, Recoverable {
}
AggregateAppResourceUsage resUsage =
this.attemptMetrics.getAggregateAppResourceUsage();
report.setMemorySeconds(resUsage.getMemorySeconds());
report.setVcoreSeconds(resUsage.getVcoreSeconds());
report.setPreemptedMemorySeconds(
this.attemptMetrics.getPreemptedMemory());
report.setPreemptedVcoreSeconds(
this.attemptMetrics.getPreemptedVcore());
report.setResourceSecondsMap(resUsage.getResourceUsageSecondsMap());
report.setPreemptedResourceSecondsMap(
this.attemptMetrics.getPreemptedResourceSecondsMap());
return report;
} finally {
this.readLock.unlock();
@ -995,11 +992,10 @@ public class RMAppAttemptImpl implements RMAppAttempt, Recoverable {
this.finalStatus = attemptState.getFinalApplicationStatus();
this.startTime = attemptState.getStartTime();
this.finishTime = attemptState.getFinishTime();
this.attemptMetrics.updateAggregateAppResourceUsage(
attemptState.getMemorySeconds(), attemptState.getVcoreSeconds());
this.attemptMetrics
.updateAggregateAppResourceUsage(attemptState.getResourceSecondsMap());
this.attemptMetrics.updateAggregatePreemptedAppResourceUsage(
attemptState.getPreemptedMemorySeconds(),
attemptState.getPreemptedVcoreSeconds());
attemptState.getPreemptedResourceSecondsMap());
}
public void transferStateFromAttempt(RMAppAttempt attempt) {
@ -1375,16 +1371,12 @@ public class RMAppAttemptImpl implements RMAppAttempt, Recoverable {
RMStateStore rmStore = rmContext.getStateStore();
setFinishTime(System.currentTimeMillis());
ApplicationAttemptStateData attemptState =
ApplicationAttemptStateData.newInstance(
applicationAttemptId, getMasterContainer(),
rmStore.getCredentialsFromAppAttempt(this),
startTime, stateToBeStored, finalTrackingUrl, diags.toString(),
finalStatus, exitStatus,
getFinishTime(), resUsage.getMemorySeconds(),
resUsage.getVcoreSeconds(),
this.attemptMetrics.getPreemptedMemory(),
this.attemptMetrics.getPreemptedVcore());
ApplicationAttemptStateData attemptState = ApplicationAttemptStateData
.newInstance(applicationAttemptId, getMasterContainer(),
rmStore.getCredentialsFromAppAttempt(this), startTime,
stateToBeStored, finalTrackingUrl, diags.toString(), finalStatus, exitStatus,
getFinishTime(), resUsage.getResourceUsageSecondsMap(),
this.attemptMetrics.getPreemptedResourceSecondsMap());
LOG.info("Updating application attempt " + applicationAttemptId
+ " with final state: " + targetedFinalState + ", and exit status: "
+ exitStatus);

106
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptMetrics.java
View File

@ -18,6 +18,8 @@
package org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt;
import java.util.HashMap;
import java.util.Map;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicLong;
@ -25,11 +27,13 @@ import java.util.concurrent.locks.ReentrantReadWriteLock;
import java.util.concurrent.locks.ReentrantReadWriteLock.ReadLock;
import java.util.concurrent.locks.ReentrantReadWriteLock.WriteLock;
import org.apache.commons.lang.time.DateUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationResourceUsageReport;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.ResourceInformation;
import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer;
@ -49,10 +53,8 @@ public class RMAppAttemptMetrics {
private ReadLock readLock;
private WriteLock writeLock;
private AtomicLong finishedMemorySeconds = new AtomicLong(0);
private AtomicLong finishedVcoreSeconds = new AtomicLong(0);
private AtomicLong preemptedMemorySeconds = new AtomicLong(0);
private AtomicLong preemptedVcoreSeconds = new AtomicLong(0);
private Map<String, AtomicLong> resourceUsageMap = new HashMap<>();
private Map<String, AtomicLong> preemptedResourceMap = new HashMap<>();
private RMContext rmContext;
private int[][] localityStatistics =
@ -102,11 +104,16 @@ public class RMAppAttemptMetrics {
}
public long getPreemptedMemory() {
return preemptedMemorySeconds.get();
return preemptedResourceMap.get(ResourceInformation.MEMORY_MB.getName())
.get();
}
public long getPreemptedVcore() {
return preemptedVcoreSeconds.get();
return preemptedResourceMap.get(ResourceInformation.VCORES.getName()).get();
}
public Map<String, Long> getPreemptedResourceSecondsMap() {
return convertAtomicLongMaptoLongMap(preemptedResourceMap);
}
public int getNumNonAMContainersPreempted() {
@ -122,35 +129,90 @@ public class RMAppAttemptMetrics {
}
public AggregateAppResourceUsage getAggregateAppResourceUsage() {
long memorySeconds = finishedMemorySeconds.get();
long vcoreSeconds = finishedVcoreSeconds.get();
Map<String, Long> resourcesUsed =
convertAtomicLongMaptoLongMap(resourceUsageMap);
// Only add in the running containers if this is the active attempt.
RMApp rmApp = rmContext.getRMApps().get(attemptId.getApplicationId());
if (null != rmApp) {
RMAppAttempt currentAttempt = rmApp.getCurrentAppAttempt();
if (rmApp != null) {
RMAppAttempt currentAttempt = rmContext.getRMApps().get(attemptId.getApplicationId()).getCurrentAppAttempt();
if (currentAttempt.getAppAttemptId().equals(attemptId)) {
ApplicationResourceUsageReport appResUsageReport = rmContext
.getScheduler().getAppResourceUsageReport(attemptId);
ApplicationResourceUsageReport appResUsageReport =
rmContext.getScheduler().getAppResourceUsageReport(attemptId);
if (appResUsageReport != null) {
memorySeconds += appResUsageReport.getMemorySeconds();
vcoreSeconds += appResUsageReport.getVcoreSeconds();
Map<String, Long> tmp = appResUsageReport.getResourceSecondsMap();
for (Map.Entry<String, Long> entry : tmp.entrySet()) {
if (resourcesUsed.containsKey(entry.getKey())) {
Long value = resourcesUsed.get(entry.getKey());
value += entry.getValue();
resourcesUsed.put(entry.getKey(), value);
} else{
resourcesUsed.put(entry.getKey(), entry.getValue());
}
}
}
}
}
return new AggregateAppResourceUsage(memorySeconds, vcoreSeconds);
return new AggregateAppResourceUsage(resourcesUsed);
}
public void updateAggregateAppResourceUsage(long finishedMemorySeconds,
long finishedVcoreSeconds) {
this.finishedMemorySeconds.addAndGet(finishedMemorySeconds);
this.finishedVcoreSeconds.addAndGet(finishedVcoreSeconds);
public void updateAggregateAppResourceUsage(Resource allocated,
long deltaUsedMillis) {
updateUsageMap(allocated, deltaUsedMillis, resourceUsageMap);
}
public void updateAggregatePreemptedAppResourceUsage(Resource allocated,
long deltaUsedMillis) {
updateUsageMap(allocated, deltaUsedMillis, preemptedResourceMap);
}
public void updateAggregateAppResourceUsage(
Map<String, Long> resourceSecondsMap) {
updateUsageMap(resourceSecondsMap, resourceUsageMap);
}
public void updateAggregatePreemptedAppResourceUsage(
long preemptedMemorySeconds, long preemptedVcoreSeconds) {
this.preemptedMemorySeconds.addAndGet(preemptedMemorySeconds);
this.preemptedVcoreSeconds.addAndGet(preemptedVcoreSeconds);
Map<String, Long> preemptedResourceSecondsMap) {
updateUsageMap(preemptedResourceSecondsMap, preemptedResourceMap);
}
private void updateUsageMap(Resource allocated, long deltaUsedMillis,
Map<String, AtomicLong> targetMap) {
for (Map.Entry<String, ResourceInformation> entry : allocated.getResources()
.entrySet()) {
AtomicLong resourceUsed;
if (!targetMap.containsKey(entry.getKey())) {
resourceUsed = new AtomicLong(0);
targetMap.put(entry.getKey(), resourceUsed);
}
resourceUsed = targetMap.get(entry.getKey());
resourceUsed.addAndGet((entry.getValue().getValue() * deltaUsedMillis)
/ DateUtils.MILLIS_PER_SECOND);
}
}
private void updateUsageMap(Map<String, Long> sourceMap,
Map<String, AtomicLong> targetMap) {
for (Map.Entry<String, Long> entry : sourceMap.entrySet()) {
AtomicLong resourceUsed;
if (!targetMap.containsKey(entry.getKey())) {
resourceUsed = new AtomicLong(0);
targetMap.put(entry.getKey(), resourceUsed);
}
resourceUsed = targetMap.get(entry.getKey());
resourceUsed.set(entry.getValue());
}
}
private Map<String, Long> convertAtomicLongMaptoLongMap(
Map<String, AtomicLong> source) {
Map<String, Long> ret = new HashMap<>();
for (Map.Entry<String, AtomicLong> entry : source.entrySet()) {
ret.put(entry.getKey(), entry.getValue().get());
}
return ret;
}
public void incNumAllocatedContainers(NodeType containerType,

16
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java
View File

@ -25,7 +25,6 @@ import java.util.concurrent.locks.ReentrantReadWriteLock;
import java.util.concurrent.locks.ReentrantReadWriteLock.ReadLock;
import java.util.concurrent.locks.ReentrantReadWriteLock.WriteLock;
import org.apache.commons.lang.time.DateUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
@ -715,20 +714,15 @@ public class RMContainerImpl implements RMContainer {
if (rmAttempt != null) {
long usedMillis = container.finishTime - container.creationTime;
long memorySeconds = resource.getMemorySize()
* usedMillis / DateUtils.MILLIS_PER_SECOND;
long vcoreSeconds = resource.getVirtualCores()
* usedMillis / DateUtils.MILLIS_PER_SECOND;
rmAttempt.getRMAppAttemptMetrics()
.updateAggregateAppResourceUsage(memorySeconds,vcoreSeconds);
.updateAggregateAppResourceUsage(resource, usedMillis);
// If this is a preempted container, update preemption metrics
if (ContainerExitStatus.PREEMPTED == container.finishedStatus
.getExitStatus()) {
rmAttempt.getRMAppAttemptMetrics().updatePreemptionInfo(resource,
container);
.getExitStatus()) {
rmAttempt.getRMAppAttemptMetrics()
.updateAggregatePreemptedAppResourceUsage(memorySeconds,
vcoreSeconds);
.updatePreemptionInfo(resource, container);
rmAttempt.getRMAppAttemptMetrics()
.updateAggregatePreemptedAppResourceUsage(resource, usedMillis);
}
}
}

47
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java
View File

@ -55,11 +55,13 @@ import org.apache.hadoop.yarn.api.records.NMToken;
import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.ResourceInformation;
import org.apache.hadoop.yarn.api.records.ResourceRequest;
import org.apache.hadoop.yarn.api.records.UpdateContainerError;
import org.apache.hadoop.yarn.nodelabels.CommonNodeLabelsManager;
import org.apache.hadoop.yarn.server.api.ContainerType;
import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
import org.apache.hadoop.yarn.server.resourcemanager.RMServerUtils;
import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.AggregateAppResourceUsage;
@ -107,9 +109,7 @@ public class SchedulerApplicationAttempt implements SchedulableEntity {
private static final long MEM_AGGREGATE_ALLOCATION_CACHE_MSECS = 3000;
protected long lastMemoryAggregateAllocationUpdateTime = 0;
private long lastMemorySeconds = 0;
private long lastVcoreSeconds = 0;
private Map<String, Long> lastResourceSecondsMap = new HashMap<>();
protected final AppSchedulingInfo appSchedulingInfo;
protected ApplicationAttemptId attemptId;
protected Map<ContainerId, RMContainer> liveContainers =
@ -1002,22 +1002,24 @@ public class SchedulerApplicationAttempt implements SchedulableEntity {
// recently.
if ((currentTimeMillis - lastMemoryAggregateAllocationUpdateTime)
> MEM_AGGREGATE_ALLOCATION_CACHE_MSECS) {
long memorySeconds = 0;
long vcoreSeconds = 0;
Map<String, Long> resourceSecondsMap = new HashMap<>();
for (RMContainer rmContainer : this.liveContainers.values()) {
long usedMillis = currentTimeMillis - rmContainer.getCreationTime();
Resource resource = rmContainer.getContainer().getResource();
memorySeconds += resource.getMemorySize() * usedMillis /
DateUtils.MILLIS_PER_SECOND;
vcoreSeconds += resource.getVirtualCores() * usedMillis
/ DateUtils.MILLIS_PER_SECOND;
for (Map.Entry<String, ResourceInformation> entry : resource
.getResources().entrySet()) {
long value = RMServerUtils
.getOrDefault(resourceSecondsMap, entry.getKey(), 0L);
value += entry.getValue().getValue() * usedMillis
/ DateUtils.MILLIS_PER_SECOND;
resourceSecondsMap.put(entry.getKey(), value);
}
}
lastMemoryAggregateAllocationUpdateTime = currentTimeMillis;
lastMemorySeconds = memorySeconds;
lastVcoreSeconds = vcoreSeconds;
lastResourceSecondsMap = resourceSecondsMap;
}
return new AggregateAppResourceUsage(lastMemorySeconds, lastVcoreSeconds);
return new AggregateAppResourceUsage(lastResourceSecondsMap);
}
public ApplicationResourceUsageReport getResourceUsageReport() {
@ -1032,6 +1034,11 @@ public class SchedulerApplicationAttempt implements SchedulableEntity {
Resource cluster = rmContext.getScheduler().getClusterResource();
ResourceCalculator calc =
rmContext.getScheduler().getResourceCalculator();
Map<String, Long> preemptedResourceSecondsMaps = new HashMap<>();
preemptedResourceSecondsMaps
.put(ResourceInformation.MEMORY_MB.getName(), 0L);
preemptedResourceSecondsMaps
.put(ResourceInformation.VCORES.getName(), 0L);
float queueUsagePerc = 0.0f;
float clusterUsagePerc = 0.0f;
if (!calc.isInvalidDivisor(cluster)) {
@ -1041,15 +1048,15 @@ public class SchedulerApplicationAttempt implements SchedulableEntity {
queueUsagePerc = calc.divide(cluster, usedResourceClone,
Resources.multiply(cluster, queueCapacityPerc)) * 100;
}
clusterUsagePerc = calc.divide(cluster, usedResourceClone, cluster)
* 100;
clusterUsagePerc =
calc.divide(cluster, usedResourceClone, cluster) * 100;
}
return ApplicationResourceUsageReport.newInstance(liveContainers.size(),
reservedContainers.size(), usedResourceClone, reservedResourceClone,
Resources.add(usedResourceClone, reservedResourceClone),
runningResourceUsage.getMemorySeconds(),
runningResourceUsage.getVcoreSeconds(), queueUsagePerc,
clusterUsagePerc, 0, 0);
return ApplicationResourceUsageReport
.newInstance(liveContainers.size(), reservedContainers.size(),
usedResourceClone, reservedResourceClone,
Resources.add(usedResourceClone, reservedResourceClone),
runningResourceUsage.getResourceUsageSecondsMap(), queueUsagePerc,
clusterUsagePerc, preemptedResourceSecondsMaps);
} finally {
writeLock.unlock();
}

14
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppBlock.java
View File

@ -44,6 +44,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptMetrics;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppAttemptInfo;
import org.apache.hadoop.yarn.server.webapp.AppBlock;
import org.apache.hadoop.yarn.util.StringHelper;
import org.apache.hadoop.yarn.util.resource.Resources;
import org.apache.hadoop.yarn.webapp.hamlet2.Hamlet;
import org.apache.hadoop.yarn.webapp.hamlet2.Hamlet.DIV;
@ -106,15 +107,12 @@ public class RMAppBlock extends AppBlock{
attemptResourcePreempted)
.__("Number of Non-AM Containers Preempted from Current Attempt:",
attemptNumNonAMContainerPreempted)
.__("Aggregate Resource Allocation:",
String.format("%d MB-seconds, %d vcore-seconds",
appMetrics == null ? "N/A" : appMetrics.getMemorySeconds(),
appMetrics == null ? "N/A" : appMetrics.getVcoreSeconds()))
.__("Aggregate Resource Allocation:", appMetrics == null ? "N/A" :
StringHelper
.getResourceSecondsString(appMetrics.getResourceSecondsMap()))
.__("Aggregate Preempted Resource Allocation:",
String.format("%d MB-seconds, %d vcore-seconds",
appMetrics == null ? "N/A" : appMetrics.getPreemptedMemorySeconds(),
appMetrics == null ? "N/A" :
appMetrics.getPreemptedVcoreSeconds()));
appMetrics == null ? "N/A" : StringHelper.getResourceSecondsString(
appMetrics.getPreemptedResourceSecondsMap()));
pdiv.__();
}

28
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java
View File

@ -101,6 +101,7 @@ public class AppInfo {
private long vcoreSeconds;
protected float queueUsagePercentage;
protected float clusterUsagePercentage;
protected Map<String, Long> resourceSecondsMap;
// preemption info fields
private long preemptedResourceMB;
@ -109,6 +110,7 @@ public class AppInfo {
private int numAMContainerPreempted;
private long preemptedMemorySeconds;
private long preemptedVcoreSeconds;
protected Map<String, Long> preemptedResourceSecondsMap;
// list of resource requests
@XmlElement(name = "resourceRequests")
@ -236,8 +238,10 @@ public class AppInfo {
appMetrics.getResourcePreempted().getVirtualCores();
memorySeconds = appMetrics.getMemorySeconds();
vcoreSeconds = appMetrics.getVcoreSeconds();
resourceSecondsMap = appMetrics.getResourceSecondsMap();
preemptedMemorySeconds = appMetrics.getPreemptedMemorySeconds();
preemptedVcoreSeconds = appMetrics.getPreemptedVcoreSeconds();
preemptedResourceSecondsMap = appMetrics.getPreemptedResourceSecondsMap();
ApplicationSubmissionContext appSubmissionContext =
app.getApplicationSubmissionContext();
unmanagedApplication = appSubmissionContext.getUnmanagedAM();
@ -415,6 +419,22 @@ public class AppInfo {
return this.reservedVCores;
}
public long getPreemptedMB() {
return preemptedResourceMB;
}
public long getPreemptedVCores() {
return preemptedResourceVCores;
}
public int getNumNonAMContainersPreempted() {
return numNonAMContainerPreempted;
}
public int getNumAMContainersPreempted() {
return numAMContainerPreempted;
}
public long getMemorySeconds() {
return memorySeconds;
}
@ -423,6 +443,10 @@ public class AppInfo {
return vcoreSeconds;
}
public Map<String, Long> getResourceSecondsMap() {
return resourceSecondsMap;
}
public long getPreemptedMemorySeconds() {
return preemptedMemorySeconds;
}
@ -431,6 +455,10 @@ public class AppInfo {
return preemptedVcoreSeconds;
}
public Map<String, Long> getPreemptedResourceSecondsMap() {
return preemptedResourceSecondsMap;
}
public List<ResourceRequestInfo> getResourceRequests() {
return this.resourceRequests;
}

32
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ResourceInfo.java
View File

@ -20,46 +20,68 @@ package org.apache.hadoop.yarn.server.resourcemanager.webapp.dao;
import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType;
import javax.xml.bind.annotation.XmlElement;
import javax.xml.bind.annotation.XmlRootElement;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.util.resource.Resources;
@XmlRootElement
@XmlAccessorType(XmlAccessType.FIELD)
@XmlAccessorType(XmlAccessType.NONE)
public class ResourceInfo {
@XmlElement
long memory;
@XmlElement
int vCores;
private Resource resources;
public ResourceInfo() {
}
public ResourceInfo(Resource res) {
memory = res.getMemorySize();
vCores = res.getVirtualCores();
resources = Resources.clone(res);
}
public long getMemorySize() {
return memory;
if (resources == null) {
resources = Resource.newInstance(memory, vCores);
}
return resources.getMemorySize();
}
public int getvCores() {
return vCores;
if (resources == null) {
resources = Resource.newInstance(memory, vCores);
}
return resources.getVirtualCores();
}
@Override
public String toString() {
return "<memory:" + memory + ", vCores:" + vCores + ">";
return resources.toString();
}
public void setMemory(int memory) {
if (resources == null) {
resources = Resource.newInstance(memory, vCores);
}
this.memory = memory;
resources.setMemorySize(memory);
}
public void setvCores(int vCores) {
if (resources == null) {
resources = Resource.newInstance(memory, vCores);
}
this.vCores = vCores;
resources.setVirtualCores(vCores);
}
public Resource getResource() {
return Resource.newInstance(memory, vCores);
return Resource.newInstance(resources);
}
}

2
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/SchedulerInfo.java
View File

@ -73,7 +73,7 @@ public class SchedulerInfo {
}
public String getSchedulerResourceTypes() {
return this.schedulingResourceTypes.toString();
return minAllocResource.getResource().getResources().keySet().toString();
}
public int getMaxClusterLevelAppPriority() {

2
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/proto/yarn_server_resourcemanager_recovery.proto
View File

@ -87,6 +87,8 @@ message ApplicationAttemptStateDataProto {
optional int64 finish_time = 12;
optional int64 preempted_memory_seconds = 13;
optional int64 preempted_vcore_seconds = 14;
repeated StringLongMapProto application_resource_usage_map = 15;
repeated StringLongMapProto preempted_resource_usage_map = 16;
}
message EpochProto {

7
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java
View File

@ -35,6 +35,7 @@ import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ConcurrentMap;
import org.apache.commons.logging.Log;
@ -56,6 +57,7 @@ import org.apache.hadoop.yarn.api.records.ExecutionType;
import org.apache.hadoop.yarn.api.records.ExecutionTypeRequest;
import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.ResourceInformation;
import org.apache.hadoop.yarn.api.records.ResourceRequest;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.event.AsyncDispatcher;
@ -828,9 +830,12 @@ public class TestAppManager{
when(app.getState()).thenReturn(RMAppState.RUNNING);
when(app.getApplicationType()).thenReturn("MAPREDUCE");
when(app.getSubmitTime()).thenReturn(1000L);
Map<String, Long> resourceSecondsMap = new HashMap<>();
resourceSecondsMap.put(ResourceInformation.MEMORY_MB.getName(), 16384L);
resourceSecondsMap.put(ResourceInformation.VCORES.getName(), 64L);
RMAppMetrics metrics =
new RMAppMetrics(Resource.newInstance(1234, 56),
10, 1, 16384, 64, 0, 0);
10, 1, resourceSecondsMap, new HashMap<>());
when(app.getRMAppMetrics()).thenReturn(metrics);
RMAppManager.ApplicationSummary.SummaryBuilder summary =

7
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestContainerResourceUsage.java
View File

@ -23,6 +23,7 @@ import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import org.apache.commons.lang.time.DateUtils;
import org.apache.hadoop.security.UserGroupInformation;
@ -32,6 +33,7 @@ import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerState;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.ResourceInformation;
import org.apache.hadoop.yarn.api.records.ResourceRequest;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.server.resourcemanager.recovery.MemoryRMStateStore;
@ -422,6 +424,9 @@ public class TestContainerResourceUsage {
* usedMillis / DateUtils.MILLIS_PER_SECOND;
long vcoreSeconds = resource.getVirtualCores()
* usedMillis / DateUtils.MILLIS_PER_SECOND;
return new AggregateAppResourceUsage(memorySeconds, vcoreSeconds);
Map<String, Long> map = new HashMap<>();
map.put(ResourceInformation.MEMORY_MB.getName(), memorySeconds);
map.put(ResourceInformation.VCORES.getName(), vcoreSeconds);
return new AggregateAppResourceUsage(map);
}
}

9
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java
View File

@ -18,6 +18,7 @@
package org.apache.hadoop.yarn.server.resourcemanager.applicationsmanager;
import java.util.Collection;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
@ -189,7 +190,8 @@ public abstract class MockAsm extends MockApps {
@Override
public RMAppMetrics getRMAppMetrics() {
return new RMAppMetrics(Resource.newInstance(0, 0), 0, 0, 0, 0, 0, 0);
return new RMAppMetrics(Resource.newInstance(0, 0), 0, 0, new HashMap<>(),
new HashMap<>());
}
@Override
@ -337,8 +339,9 @@ public abstract class MockAsm extends MockApps {
public ApplicationReport createAndGetApplicationReport(
String clientUserName, boolean allowAccess) {
ApplicationResourceUsageReport usageReport =
ApplicationResourceUsageReport.newInstance(0, 0, null, null, null,
0, 0, 0, 0, 0, 0);
ApplicationResourceUsageReport
.newInstance(0, 0, null, null, null, new HashMap<>(), 0, 0,
new HashMap<>());
ApplicationReport report = ApplicationReport.newInstance(
getApplicationId(), appAttemptId, getUser(), getQueue(),
getName(), null, 0, null, null, getDiagnostics().toString(),

15
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisher.java
View File

@ -24,6 +24,7 @@ import static org.mockito.Mockito.when;
import java.util.Collection;
import java.util.Collections;
import java.util.EnumSet;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Map;
import java.util.Set;
@ -40,6 +41,7 @@ import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.ResourceInformation;
import org.apache.hadoop.yarn.api.records.ResourceRequest;
import org.apache.hadoop.yarn.api.records.YarnApplicationAttemptState;
import org.apache.hadoop.yarn.api.records.YarnApplicationState;
@ -506,9 +508,16 @@ public class TestSystemMetricsPublisher {
when(app.getCurrentAppAttempt()).thenReturn(appAttempt);
when(app.getFinalApplicationStatus()).thenReturn(
FinalApplicationStatus.UNDEFINED);
when(app.getRMAppMetrics()).thenReturn(
new RMAppMetrics(null, 0, 0, Integer.MAX_VALUE, Long.MAX_VALUE,
Integer.MAX_VALUE, Long.MAX_VALUE));
Map<String, Long> resourceMap = new HashMap<>();
resourceMap
.put(ResourceInformation.MEMORY_MB.getName(), (long) Integer.MAX_VALUE);
resourceMap.put(ResourceInformation.VCORES.getName(), Long.MAX_VALUE);
Map<String, Long> preemptedMap = new HashMap<>();
preemptedMap
.put(ResourceInformation.MEMORY_MB.getName(), (long) Integer.MAX_VALUE);
preemptedMap.put(ResourceInformation.VCORES.getName(), Long.MAX_VALUE);
when(app.getRMAppMetrics())
.thenReturn(new RMAppMetrics(null, 0, 0, resourceMap, preemptedMap));
Set<String> appTags = new HashSet<String>();
appTags.add("test");
appTags.add("tags");

22
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisherForV2.java
View File

@ -29,6 +29,8 @@ import java.io.File;
import java.io.FileReader;
import java.io.IOException;
import java.util.Collections;
import java.util.HashMap;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
@ -46,6 +48,7 @@ import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.ResourceInformation;
import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity;
import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntityType;
import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEvent;
@ -357,15 +360,20 @@ public class TestSystemMetricsPublisherForV2 {
when(app.getDiagnostics()).thenReturn(
new StringBuilder("test diagnostics info"));
RMAppAttempt appAttempt = mock(RMAppAttempt.class);
when(appAttempt.getAppAttemptId()).thenReturn(
ApplicationAttemptId.newInstance(appId, 1));
when(appAttempt.getAppAttemptId())
.thenReturn(ApplicationAttemptId.newInstance(appId, 1));
when(app.getCurrentAppAttempt()).thenReturn(appAttempt);
when(app.getFinalApplicationStatus()).thenReturn(
FinalApplicationStatus.UNDEFINED);
when(app.getFinalApplicationStatus())
.thenReturn(FinalApplicationStatus.UNDEFINED);
Map<String, Long> resourceSecondsMap = new HashMap<>();
resourceSecondsMap
.put(ResourceInformation.MEMORY_MB.getName(), (long) Integer.MAX_VALUE);
resourceSecondsMap
.put(ResourceInformation.VCORES.getName(), Long.MAX_VALUE);
when(app.getRMAppMetrics()).thenReturn(
new RMAppMetrics(Resource.newInstance(0, 0), 0, 0, Integer.MAX_VALUE,
Long.MAX_VALUE, 0, 0));
when(app.getApplicationTags()).thenReturn(Collections.<String> emptySet());
new RMAppMetrics(Resource.newInstance(0, 0), 0, 0, resourceSecondsMap,
new HashMap<>()));
when(app.getApplicationTags()).thenReturn(Collections.<String>emptySet());
ApplicationSubmissionContext appSubmissionContext =
mock(ApplicationSubmissionContext.class);
when(appSubmissionContext.getPriority())

8
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStoreTestBase.java
View File

@ -194,7 +194,7 @@ public class RMStateStoreTestBase {
when(mockAttempt.getRMAppAttemptMetrics())
.thenReturn(mockRmAppAttemptMetrics);
when(mockRmAppAttemptMetrics.getAggregateAppResourceUsage())
.thenReturn(new AggregateAppResourceUsage(0, 0));
.thenReturn(new AggregateAppResourceUsage(new HashMap<>()));
dispatcher.attemptId = attemptId;
store.storeNewApplicationAttempt(mockAttempt);
waitNotify(dispatcher);
@ -292,7 +292,7 @@ public class RMStateStoreTestBase {
when(mockRemovedAttempt.getRMAppAttemptMetrics())
.thenReturn(mockRmAppAttemptMetrics);
when(mockRmAppAttemptMetrics.getAggregateAppResourceUsage())
.thenReturn(new AggregateAppResourceUsage(0,0));
.thenReturn(new AggregateAppResourceUsage(new HashMap<>()));
attempts.put(attemptIdRemoved, mockRemovedAttempt);
store.removeApplication(mockRemovedApp);
@ -369,7 +369,7 @@ public class RMStateStoreTestBase {
oldAttemptState.getStartTime(), RMAppAttemptState.FINISHED,
"myTrackingUrl", "attemptDiagnostics",
FinalApplicationStatus.SUCCEEDED, 100,
oldAttemptState.getFinishTime(), 0, 0, 0, 0);
oldAttemptState.getFinishTime(), new HashMap<>(), new HashMap<>());
store.updateApplicationAttemptState(newAttemptState);
// test updating the state of an app/attempt whose initial state was not
@ -393,7 +393,7 @@ public class RMStateStoreTestBase {
oldAttemptState.getStartTime(), RMAppAttemptState.FINISHED,
"myTrackingUrl", "attemptDiagnostics",
FinalApplicationStatus.SUCCEEDED, 111,
oldAttemptState.getFinishTime(), 0, 0, 0, 0);
oldAttemptState.getFinishTime(), new HashMap<>(), new HashMap<>());
store.updateApplicationAttemptState(dummyAttempt);
// let things settle down

23
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestZKRMStateStore.java
View File

@ -33,12 +33,7 @@ import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.security.token.delegation.DelegationKey;
import org.apache.hadoop.service.Service;
import org.apache.hadoop.test.GenericTestUtils;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
import org.apache.hadoop.yarn.api.records.*;
import org.apache.hadoop.yarn.api.records.impl.pb.ApplicationSubmissionContextPBImpl;
import org.apache.hadoop.yarn.api.records.impl.pb.ContainerPBImpl;
import org.apache.hadoop.yarn.conf.HAUtil;
@ -511,7 +506,7 @@ public class TestZKRMStateStore extends RMStateStoreTestBase {
when(mockAttempt.getRMAppAttemptMetrics())
.thenReturn(mockRmAppAttemptMetrics);
when(mockRmAppAttemptMetrics.getAggregateAppResourceUsage())
.thenReturn(new AggregateAppResourceUsage(0,0));
.thenReturn(new AggregateAppResourceUsage(new HashMap<>()));
store.storeNewApplicationAttempt(mockAttempt);
assertEquals("RMStateStore should have been in fenced state",
true, store.isFencedState());
@ -523,7 +518,7 @@ public class TestZKRMStateStore extends RMStateStoreTestBase {
store.getCredentialsFromAppAttempt(mockAttempt),
startTime, RMAppAttemptState.FINISHED, "testUrl",
"test", FinalApplicationStatus.SUCCEEDED, 100,
finishTime, 0, 0, 0, 0);
finishTime, new HashMap<>(), new HashMap<>());
store.updateApplicationAttemptState(newAttemptState);
assertEquals("RMStateStore should have been in fenced state",
true, store.isFencedState());
@ -751,10 +746,20 @@ public class TestZKRMStateStore extends RMStateStoreTestBase {
private static ApplicationAttemptStateData createFinishedAttempt(
ApplicationAttemptId attemptId, Container container, long startTime,
int amExitStatus) {
Map<String, Long> resourceSecondsMap = new HashMap<>();
Map<String, Long> preemptedResoureSecondsMap = new HashMap<>();
resourceSecondsMap
.put(ResourceInformation.MEMORY_MB.getName(), 0L);
resourceSecondsMap
.put(ResourceInformation.VCORES.getName(), 0L);
preemptedResoureSecondsMap.put(ResourceInformation.MEMORY_MB.getName(),
0L);
preemptedResoureSecondsMap
.put(ResourceInformation.VCORES.getName(), 0L);
return ApplicationAttemptStateData.newInstance(attemptId,
container, null, startTime, RMAppAttemptState.FINISHED,
"myTrackingUrl", "attemptDiagnostics", FinalApplicationStatus.SUCCEEDED,
amExitStatus, 0, 0, 0, 0, 0);
amExitStatus, 0, resourceSecondsMap, preemptedResoureSecondsMap);
}
private ApplicationAttemptId storeAttempt(RMStateStore store,

6
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestAppPage.java
View File

@ -22,6 +22,7 @@ import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
import java.io.IOException;
import java.util.HashMap;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
@ -63,8 +64,9 @@ public class TestAppPage {
when(app.getFinishTime()).thenReturn(0L);
when(app.createApplicationState()).thenReturn(YarnApplicationState.FAILED);
RMAppMetrics appMetrics = new RMAppMetrics(
Resource.newInstance(0, 0), 0, 0, 0, 0, 0, 0);
RMAppMetrics appMetrics =
new RMAppMetrics(Resource.newInstance(0, 0), 0, 0, new HashMap<>(),
new HashMap<>());
when(app.getRMAppMetrics()).thenReturn(appMetrics);
// initialize RM Context, and create RMApp, without creating RMAppAttempt

5
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebAppFairScheduler.java
View File

@ -51,6 +51,7 @@ import org.junit.Test;
import java.io.IOException;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.concurrent.ConcurrentMap;
@ -136,8 +137,8 @@ public class TestRMWebAppFairScheduler {
MockRMApp app = new MockRMApp(i, i, state) {
@Override
public RMAppMetrics getRMAppMetrics() {
return new RMAppMetrics(Resource.newInstance(0, 0),
0, 0, 0, 0, 0, 0);
return new RMAppMetrics(Resource.newInstance(0, 0), 0, 0,
new HashMap<>(), new HashMap<>());
}
@Override
public YarnApplicationState createApplicationState() {

16
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/clientrm/DefaultClientRequestInterceptor.java
View File

@ -27,6 +27,8 @@ import org.apache.hadoop.yarn.api.protocolrecords.CancelDelegationTokenRequest;
import org.apache.hadoop.yarn.api.protocolrecords.CancelDelegationTokenResponse;
import org.apache.hadoop.yarn.api.protocolrecords.FailApplicationAttemptRequest;
import org.apache.hadoop.yarn.api.protocolrecords.FailApplicationAttemptResponse;
import org.apache.hadoop.yarn.api.protocolrecords.GetAllResourceProfilesRequest;
import org.apache.hadoop.yarn.api.protocolrecords.GetAllResourceProfilesResponse;
import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationAttemptReportRequest;
import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationAttemptReportResponse;
import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationAttemptsRequest;
@ -59,6 +61,8 @@ import org.apache.hadoop.yarn.api.protocolrecords.GetQueueInfoRequest;
import org.apache.hadoop.yarn.api.protocolrecords.GetQueueInfoResponse;
import org.apache.hadoop.yarn.api.protocolrecords.GetQueueUserAclsInfoRequest;
import org.apache.hadoop.yarn.api.protocolrecords.GetQueueUserAclsInfoResponse;
import org.apache.hadoop.yarn.api.protocolrecords.GetResourceProfileRequest;
import org.apache.hadoop.yarn.api.protocolrecords.GetResourceProfileResponse;
import org.apache.hadoop.yarn.api.protocolrecords.KillApplicationRequest;
import org.apache.hadoop.yarn.api.protocolrecords.KillApplicationResponse;
import org.apache.hadoop.yarn.api.protocolrecords.MoveApplicationAcrossQueuesRequest;
@ -303,6 +307,18 @@ public class DefaultClientRequestInterceptor
return clientRMProxy.updateApplicationTimeouts(request);
}
@Override
public GetAllResourceProfilesResponse getResourceProfiles(
GetAllResourceProfilesRequest request) throws YarnException, IOException {
return clientRMProxy.getResourceProfiles(request);
}
@Override
public GetResourceProfileResponse getResourceProfile(
GetResourceProfileRequest request) throws YarnException, IOException {
return clientRMProxy.getResourceProfile(request);
}
@VisibleForTesting
public void setRMClient(ApplicationClientProtocol clientRM) {
this.clientRMProxy = clientRM;

15
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/clientrm/FederationClientInterceptor.java
View File

@ -32,6 +32,8 @@ import org.apache.hadoop.yarn.api.protocolrecords.CancelDelegationTokenRequest;
import org.apache.hadoop.yarn.api.protocolrecords.CancelDelegationTokenResponse;
import org.apache.hadoop.yarn.api.protocolrecords.FailApplicationAttemptRequest;
import org.apache.hadoop.yarn.api.protocolrecords.FailApplicationAttemptResponse;
import org.apache.hadoop.yarn.api.protocolrecords.GetAllResourceProfilesRequest;
import org.apache.hadoop.yarn.api.protocolrecords.GetAllResourceProfilesResponse;
import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationAttemptReportRequest;
import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationAttemptReportResponse;
import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationAttemptsRequest;
@ -64,6 +66,8 @@ import org.apache.hadoop.yarn.api.protocolrecords.GetQueueInfoRequest;
import org.apache.hadoop.yarn.api.protocolrecords.GetQueueInfoResponse;
import org.apache.hadoop.yarn.api.protocolrecords.GetQueueUserAclsInfoRequest;
import org.apache.hadoop.yarn.api.protocolrecords.GetQueueUserAclsInfoResponse;
import org.apache.hadoop.yarn.api.protocolrecords.GetResourceProfileRequest;
import org.apache.hadoop.yarn.api.protocolrecords.GetResourceProfileResponse;
import org.apache.hadoop.yarn.api.protocolrecords.KillApplicationRequest;
import org.apache.hadoop.yarn.api.protocolrecords.KillApplicationResponse;
import org.apache.hadoop.yarn.api.protocolrecords.MoveApplicationAcrossQueuesRequest;
@ -709,4 +713,15 @@ public class FederationClientInterceptor
throw new NotImplementedException();
}
@Override
public GetAllResourceProfilesResponse getResourceProfiles(
GetAllResourceProfilesRequest request) throws YarnException, IOException {
throw new NotImplementedException();
}
@Override
public GetResourceProfileResponse getResourceProfile(
GetResourceProfileRequest request) throws YarnException, IOException {
throw new NotImplementedException();
}
}

18
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/clientrm/RouterClientRMService.java
View File

@ -38,6 +38,8 @@ import org.apache.hadoop.yarn.api.protocolrecords.CancelDelegationTokenRequest;
import org.apache.hadoop.yarn.api.protocolrecords.CancelDelegationTokenResponse;
import org.apache.hadoop.yarn.api.protocolrecords.FailApplicationAttemptRequest;
import org.apache.hadoop.yarn.api.protocolrecords.FailApplicationAttemptResponse;
import org.apache.hadoop.yarn.api.protocolrecords.GetAllResourceProfilesRequest;
import org.apache.hadoop.yarn.api.protocolrecords.GetAllResourceProfilesResponse;
import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationAttemptReportRequest;
import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationAttemptReportResponse;
import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationAttemptsRequest;
@ -70,6 +72,8 @@ import org.apache.hadoop.yarn.api.protocolrecords.GetQueueInfoRequest;
import org.apache.hadoop.yarn.api.protocolrecords.GetQueueInfoResponse;
import org.apache.hadoop.yarn.api.protocolrecords.GetQueueUserAclsInfoRequest;
import org.apache.hadoop.yarn.api.protocolrecords.GetQueueUserAclsInfoResponse;
import org.apache.hadoop.yarn.api.protocolrecords.GetResourceProfileRequest;
import org.apache.hadoop.yarn.api.protocolrecords.GetResourceProfileResponse;
import org.apache.hadoop.yarn.api.protocolrecords.KillApplicationRequest;
import org.apache.hadoop.yarn.api.protocolrecords.KillApplicationResponse;
import org.apache.hadoop.yarn.api.protocolrecords.MoveApplicationAcrossQueuesRequest;
@ -403,6 +407,20 @@ public class RouterClientRMService extends AbstractService
return pipeline.getRootInterceptor().updateApplicationTimeouts(request);
}
@Override
public GetAllResourceProfilesResponse getResourceProfiles(
GetAllResourceProfilesRequest request) throws YarnException, IOException {
RequestInterceptorChainWrapper pipeline = getInterceptorChain();
return pipeline.getRootInterceptor().getResourceProfiles(request);
}
@Override
public GetResourceProfileResponse getResourceProfile(
GetResourceProfileRequest request) throws YarnException, IOException {
RequestInterceptorChainWrapper pipeline = getInterceptorChain();
return pipeline.getRootInterceptor().getResourceProfile(request);
}
private RequestInterceptorChainWrapper getInterceptorChain()
throws IOException {
String user = UserGroupInformation.getCurrentUser().getUserName();

16
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/clientrm/PassThroughClientRequestInterceptor.java
View File

@ -24,6 +24,8 @@ import org.apache.hadoop.yarn.api.protocolrecords.CancelDelegationTokenRequest;
import org.apache.hadoop.yarn.api.protocolrecords.CancelDelegationTokenResponse;
import org.apache.hadoop.yarn.api.protocolrecords.FailApplicationAttemptRequest;
import org.apache.hadoop.yarn.api.protocolrecords.FailApplicationAttemptResponse;
import org.apache.hadoop.yarn.api.protocolrecords.GetAllResourceProfilesRequest;
import org.apache.hadoop.yarn.api.protocolrecords.GetAllResourceProfilesResponse;
import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationAttemptReportRequest;
import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationAttemptReportResponse;
import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationAttemptsRequest;
@ -56,6 +58,8 @@ import org.apache.hadoop.yarn.api.protocolrecords.GetQueueInfoRequest;
import org.apache.hadoop.yarn.api.protocolrecords.GetQueueInfoResponse;
import org.apache.hadoop.yarn.api.protocolrecords.GetQueueUserAclsInfoRequest;
import org.apache.hadoop.yarn.api.protocolrecords.GetQueueUserAclsInfoResponse;
import org.apache.hadoop.yarn.api.protocolrecords.GetResourceProfileRequest;
import org.apache.hadoop.yarn.api.protocolrecords.GetResourceProfileResponse;
import org.apache.hadoop.yarn.api.protocolrecords.KillApplicationRequest;
import org.apache.hadoop.yarn.api.protocolrecords.KillApplicationResponse;
import org.apache.hadoop.yarn.api.protocolrecords.MoveApplicationAcrossQueuesRequest;
@ -264,4 +268,16 @@ public class PassThroughClientRequestInterceptor
throws YarnException, IOException {
return getNextInterceptor().updateApplicationTimeouts(request);
}
@Override
public GetAllResourceProfilesResponse getResourceProfiles(
GetAllResourceProfilesRequest request) throws YarnException, IOException {
return getNextInterceptor().getResourceProfiles(request);
}
@Override
public GetResourceProfileResponse getResourceProfile(
GetResourceProfileRequest request) throws YarnException, IOException {
return getNextInterceptor().getResourceProfile(request);
}
}