Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

YARN-7307. Allow client/AM update supported resource types via YARN APIs. (Sunil G via wangda)

This commit is contained in:
Wangda Tan 2017-11-06 10:01:10 -08:00
parent e49509f024
commit 170b6a48c4
9 changed files with 173 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/RegisterApplicationMasterResponse.java
View File

@ -32,6 +32,7 @@ import org.apache.hadoop.yarn.api.records.ApplicationAccessType;
import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.NMToken;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.ResourceTypeInfo;
import org.apache.hadoop.yarn.proto.YarnServiceProtos.SchedulerResourceTypes;
import org.apache.hadoop.yarn.util.Records;
@ -204,4 +205,23 @@ public abstract class RegisterApplicationMasterResponse {
@Unstable
public abstract void setSchedulerResourceTypes(
EnumSet<SchedulerResourceTypes> types);
/**
* Get available resource types supported by RM.
*
* @return a Map of RM settings
*/
@Public
@Unstable
public abstract List<ResourceTypeInfo> getResourceTypes();
/**
* Set the resource types used by RM.
*
* @param types
* a set of the resource types supported by RM.
*/
@Private
@Unstable
public abstract void setResourceTypes(List<ResourceTypeInfo> types);
}

6
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceInformation.java
View File

@ -204,6 +204,12 @@ public class ResourceInformation implements Comparable<ResourceInformation> {
Long.MAX_VALUE);
}
public static ResourceInformation newInstance(String name, String units,
ResourceTypes resourceType) {
return ResourceInformation.newInstance(name, units, 0L, resourceType, 0L,
Long.MAX_VALUE);
}
public static ResourceInformation newInstance(String name, long value) {
return ResourceInformation
.newInstance(name, "", value, ResourceTypes.COUNTABLE, 0L,

11
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
View File

@ -952,6 +952,17 @@ public class YarnConfiguration extends Configuration {
*/
public static final String RM_PROXY_USER_PREFIX = RM_PREFIX + "proxyuser.";
/**
* Enable/disable loading resource-types.xml at client side.
*/
@Public
@Unstable
public static final String YARN_CLIENT_LOAD_RESOURCETYPES_FROM_SERVER = YARN_PREFIX
+ "client.load.resource-types.from-server";
@Public
@Unstable
public static final boolean DEFAULT_YARN_CLIENT_LOAD_RESOURCETYPES_FROM_SERVER = false;
/**
* Timeout in seconds for YARN node graceful decommission.
* This is the maximal time to wait for running containers and applications

23
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java
View File

@ -22,8 +22,6 @@ import com.google.common.annotations.VisibleForTesting;
import org.apache.commons.lang.StringUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.yarn.api.protocolrecords.ResourceTypes;
import org.apache.hadoop.yarn.api.records.Resource;
@ -617,4 +615,25 @@ public class ResourceUtils {
return result;
}
/**
* Reinitialize all resource types from external source (in case of client,
* server will send the updated list and local resourceutils cache will be
* updated as per server's list of resources)
*
* @param resourceTypeInfo
* List of resource types
*/
public static void reinitializeResources(
List<ResourceTypeInfo> resourceTypeInfo) {
Map<String, ResourceInformation> resourceInformationMap = new HashMap<>();
for (ResourceTypeInfo resourceType : resourceTypeInfo) {
resourceInformationMap.put(resourceType.getName(),
ResourceInformation.newInstance(resourceType.getName(),
resourceType.getDefaultUnit(), resourceType.getResourceType()));
}
ResourceUtils
.initializeResourcesFromResourceInformationMap(resourceInformationMap);
}
}

1
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto
View File

@ -48,6 +48,7 @@ message RegisterApplicationMasterResponseProto {
optional string queue = 5;
repeated NMTokenProto nm_tokens_from_previous_attempts = 6;
repeated SchedulerResourceTypes scheduler_resource_types = 7;
repeated ResourceTypeInfoProto resource_types = 9;
}
message FinishApplicationMasterRequestProto {

16
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java
View File

@ -22,6 +22,7 @@ import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.EnumSet;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
@ -101,6 +102,7 @@ import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.api.records.QueueInfo;
import org.apache.hadoop.yarn.api.records.QueueUserACLInfo;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.ResourceInformation;
import org.apache.hadoop.yarn.api.records.ResourceTypeInfo;
import org.apache.hadoop.yarn.api.records.SignalContainerCommand;
import org.apache.hadoop.yarn.api.records.Token;
@ -121,6 +123,7 @@ import org.apache.hadoop.yarn.security.AMRMTokenIdentifier;
import org.apache.hadoop.yarn.security.client.TimelineDelegationTokenIdentifier;
import org.apache.hadoop.yarn.util.ConverterUtils;
import org.apache.hadoop.yarn.util.Records;
import org.apache.hadoop.yarn.util.resource.ResourceUtils;
import org.apache.hadoop.yarn.util.timeline.TimelineUtils;
import com.google.common.annotations.VisibleForTesting;
@ -147,6 +150,7 @@ public class YarnClientImpl extends YarnClient {
String timelineDTRenewer;
private boolean timelineV1ServiceEnabled;
protected boolean timelineServiceBestEffort;
private boolean loadResourceTypesFromServer;
private static final String ROOT = "root";
@ -198,6 +202,11 @@ public class YarnClientImpl extends YarnClient {
timelineServiceBestEffort = conf.getBoolean(
YarnConfiguration.TIMELINE_SERVICE_CLIENT_BEST_EFFORT,
YarnConfiguration.DEFAULT_TIMELINE_SERVICE_CLIENT_BEST_EFFORT);
loadResourceTypesFromServer = conf.getBoolean(
YarnConfiguration.YARN_CLIENT_LOAD_RESOURCETYPES_FROM_SERVER,
YarnConfiguration.DEFAULT_YARN_CLIENT_LOAD_RESOURCETYPES_FROM_SERVER);
super.serviceInit(conf);
}
@ -216,6 +225,13 @@ public class YarnClientImpl extends YarnClient {
} catch (IOException e) {
throw new YarnRuntimeException(e);
}
// Reinitialize local resource types cache from list of resources pulled from
// RM.
if (loadResourceTypesFromServer) {
ResourceUtils.reinitializeResources(getResourceTypeInfo());
}
super.serviceStart();
}

95
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/RegisterApplicationMasterResponsePBImpl.java
View File

@ -18,10 +18,8 @@
package org.apache.hadoop.yarn.api.protocolrecords.impl.pb;
import java.nio.ByteBuffer;
import java.util.*;
import com.google.protobuf.ByteString;
import com.google.protobuf.TextFormat;
import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterResponse;
@ -29,21 +27,28 @@ import org.apache.hadoop.yarn.api.records.ApplicationAccessType;
import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.NMToken;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.ResourceTypeInfo;
import org.apache.hadoop.yarn.api.records.impl.pb.ContainerPBImpl;
import org.apache.hadoop.yarn.api.records.impl.pb.NMTokenPBImpl;
import org.apache.hadoop.yarn.api.records.impl.pb.ProtoUtils;
import org.apache.hadoop.yarn.api.records.impl.pb.ResourcePBImpl;
import org.apache.hadoop.yarn.api.records.impl.pb.ResourceTypeInfoPBImpl;
import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationACLMapProto;
import org.apache.hadoop.yarn.proto.YarnProtos.ContainerProto;
import org.apache.hadoop.yarn.proto.YarnProtos.ResourceProto;
import org.apache.hadoop.yarn.proto.YarnProtos.ResourceTypeInfoProto;
import org.apache.hadoop.yarn.proto.YarnServiceProtos.NMTokenProto;
import org.apache.hadoop.yarn.proto.YarnServiceProtos.RegisterApplicationMasterResponseProto;
import org.apache.hadoop.yarn.proto.YarnServiceProtos.RegisterApplicationMasterResponseProtoOrBuilder;
import org.apache.hadoop.yarn.proto.YarnServiceProtos.SchedulerResourceTypes;
import com.google.protobuf.ByteString;
import com.google.protobuf.TextFormat;
import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.EnumSet;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
@Private
@Unstable
@ -59,6 +64,7 @@ public class RegisterApplicationMasterResponsePBImpl extends
private List<Container> containersFromPreviousAttempts = null;
private List<NMToken> nmTokens = null;
private EnumSet<SchedulerResourceTypes> schedulerResourceTypes = null;
private List<ResourceTypeInfo> resourceTypeInfo = null;
public RegisterApplicationMasterResponsePBImpl() {
builder = RegisterApplicationMasterResponseProto.newBuilder();
@ -123,9 +129,11 @@ public class RegisterApplicationMasterResponsePBImpl extends
if(schedulerResourceTypes != null) {
addSchedulerResourceTypes();
}
if (resourceTypeInfo != null) {
addResourceTypeInfosToProto();
}
}
private void maybeInitBuilder() {
if (viaProto || builder == null) {
builder = RegisterApplicationMasterResponseProto.newBuilder(proto);
@ -456,4 +464,75 @@ public class RegisterApplicationMasterResponsePBImpl extends
private NMToken convertFromProtoFormat(NMTokenProto proto) {
return new NMTokenPBImpl(proto);
}
private ResourceTypeInfoPBImpl convertFromProtoFormat(
ResourceTypeInfoProto p) {
return new ResourceTypeInfoPBImpl(p);
}
private ResourceTypeInfoProto convertToProtoFormat(ResourceTypeInfo t) {
return ((ResourceTypeInfoPBImpl) t).getProto();
}
@Override
public List<ResourceTypeInfo> getResourceTypes() {
initResourceTypeInfosList();
return this.resourceTypeInfo;
}
@Override
public void setResourceTypes(List<ResourceTypeInfo> types) {
if (resourceTypeInfo == null) {
builder.clearResourceTypes();
}
this.resourceTypeInfo = types;
}
private void addResourceTypeInfosToProto() {
maybeInitBuilder();
builder.clearResourceTypes();
if (resourceTypeInfo == null) {
return;
}
Iterable<ResourceTypeInfoProto> iterable = new Iterable<ResourceTypeInfoProto>() {
@Override
public Iterator<ResourceTypeInfoProto> iterator() {
return new Iterator<ResourceTypeInfoProto>() {
Iterator<ResourceTypeInfo> iter = resourceTypeInfo.iterator();
@Override
public boolean hasNext() {
return iter.hasNext();
}
@Override
public ResourceTypeInfoProto next() {
return convertToProtoFormat(iter.next());
}
@Override
public void remove() {
throw new UnsupportedOperationException();
}
};
}
};
builder.addAllResourceTypes(iterable);
}
private void initResourceTypeInfosList() {
if (this.resourceTypeInfo != null) {
return;
}
RegisterApplicationMasterResponseProtoOrBuilder p = viaProto ? proto : builder;
List<ResourceTypeInfoProto> list = p.getResourceTypesList();
resourceTypeInfo = new ArrayList<ResourceTypeInfo>();
for (ResourceTypeInfoProto a : list) {
resourceTypeInfo.add(convertFromProtoFormat(a));
}
}
}

9
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
View File

@ -3433,4 +3433,13 @@
to specify details about the individual resource types.
</description>
</property>
<property>
<description>
Provides an option for client to load supported resource types from RM
instead of depending on local resource-types.xml file.
</description>
<name>yarn.client.load.resource-types.from-server</name>
<value>false</value>
</property>
</configuration>

2
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/DefaultAMSProcessor.java
View File

@ -73,6 +73,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler
.SchedulerNodeReport;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.YarnScheduler;
import org.apache.hadoop.yarn.server.utils.BuilderUtils;
import org.apache.hadoop.yarn.util.resource.ResourceUtils;
import org.apache.hadoop.yarn.util.resource.Resources;
import java.io.IOException;
@ -177,6 +178,7 @@ final class DefaultAMSProcessor implements ApplicationMasterServiceProcessor {
response.setSchedulerResourceTypes(getScheduler()
.getSchedulingResourceTypes());
response.setResourceTypes(ResourceUtils.getResourcesTypeInfo());
}
@Override