From 8ffabfdf4fec104cdaa68133385cfe9cf88b95a8 Mon Sep 17 00:00:00 2001 From: Arun Suresh Date: Thu, 7 Jan 2016 15:18:59 -0800 Subject: [PATCH] YARN-4335. Allow ResourceRequests to specify ExecutionType of a request ask (kkaranasos via asuresh) --- hadoop-yarn-project/CHANGES-yarn-2877.txt | 15 +++++++ .../yarn/api/records/ResourceRequest.java | 41 +++++++++++++++++- .../src/main/proto/yarn_protos.proto | 1 + .../hadoop/yarn/client/api/AMRMClient.java | 43 +++++++++++++++++-- .../impl/pb/ResourceRequestPBImpl.java | 21 +++++++++ 5 files changed, 117 insertions(+), 4 deletions(-) create mode 100644 hadoop-yarn-project/CHANGES-yarn-2877.txt diff --git a/hadoop-yarn-project/CHANGES-yarn-2877.txt b/hadoop-yarn-project/CHANGES-yarn-2877.txt new file mode 100644 index 00000000000..d29ff0fc7a8 --- /dev/null +++ b/hadoop-yarn-project/CHANGES-yarn-2877.txt @@ -0,0 +1,15 @@ +Hadoop YARN Change Log + +Hadoop YARN Change Log for YARN-2877 + +yarn-2877 distributed scheduling (Unreleased) + + INCOMPATIBLE CHANGES + + NEW FEATURES + + IMPROVEMENTS + + YARN-4335. Allow ResourceRequests to specify ExecutionType of a request + ask (kkaranasos via asuresh) + diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceRequest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceRequest.java index 790120a0eb6..8c1fd8d00db 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceRequest.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceRequest.java @@ -79,6 +79,15 @@ public abstract class ResourceRequest implements Comparable { public static ResourceRequest newInstance(Priority priority, String hostName, Resource capability, int numContainers, boolean relaxLocality, String labelExpression) { + return newInstance(priority, hostName, capability, numContainers, + relaxLocality, labelExpression, ExecutionType.GUARANTEED); + } + + @Public + @Stable + public static ResourceRequest newInstance(Priority priority, String hostName, + Resource capability, int numContainers, boolean relaxLocality, String + labelExpression, ExecutionType execType) { ResourceRequest request = Records.newRecord(ResourceRequest.class); request.setPriority(priority); request.setResourceName(hostName); @@ -86,6 +95,7 @@ public abstract class ResourceRequest implements Comparable { request.setNumContainers(numContainers); request.setRelaxLocality(relaxLocality); request.setNodeLabelExpression(labelExpression); + request.setExecutionType(execType); return request; } @@ -221,7 +231,28 @@ public abstract class ResourceRequest implements Comparable { @Public @Stable public abstract boolean getRelaxLocality(); - + + /** + * Set the ExecutionType of the requested container. + * + * @param execType + * ExecutionType of the requested container + */ + @Public + @Stable + public abstract void setExecutionType(ExecutionType execType); + + /** + * Get whether locality relaxation is enabled with this + * ResourceRequest. Defaults to true. + * + * @return whether locality relaxation is enabled with this + * ResourceRequest. + */ + @Public + @Stable + public abstract ExecutionType getExecutionType(); + /** *

For a request at a network hierarchy level, set whether locality can be relaxed * to that level and beyond.

@@ -322,6 +353,14 @@ public abstract class ResourceRequest implements Comparable { return false; } else if (!priority.equals(other.getPriority())) return false; + ExecutionType executionType = getExecutionType(); + if (executionType == null) { + if (other.getExecutionType() != null) { + return false; + } + } else if (executionType != other.getExecutionType()) { + return false; + } if (getNodeLabelExpression() == null) { if (other.getNodeLabelExpression() != null) { return false; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto index 2fe4edae5ef..635f2f061ba 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto @@ -304,6 +304,7 @@ message ResourceRequestProto { optional int32 num_containers = 4; optional bool relax_locality = 5 [default = true]; optional string node_label_expression = 6; + optional ExecutionTypeProto executionType = 7 [default = GUARANTEED]; } enum AMCommandProto { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/AMRMClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/AMRMClient.java index e0bf2d312e0..73c2725aa3c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/AMRMClient.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/AMRMClient.java @@ -34,6 +34,7 @@ import org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterRespo import org.apache.hadoop.yarn.api.records.Container; import org.apache.hadoop.yarn.api.records.ContainerId; +import org.apache.hadoop.yarn.api.records.ExecutionType; import org.apache.hadoop.yarn.api.records.FinalApplicationStatus; import org.apache.hadoop.yarn.api.records.Priority; import org.apache.hadoop.yarn.api.records.Resource; @@ -108,6 +109,7 @@ public abstract class AMRMClient extends final Priority priority; final boolean relaxLocality; final String nodeLabelsExpression; + final ExecutionType executionType; /** * Instantiates a {@link ContainerRequest} with the given constraints and @@ -152,6 +154,33 @@ public abstract class AMRMClient extends String[] racks, Priority priority, boolean relaxLocality) { this(capability, nodes, racks, priority, relaxLocality, null); } + + /** + * Instantiates a {@link ContainerRequest} with the given constraints. + * + * @param capability + * The {@link Resource} to be requested for each container. + * @param nodes + * Any hosts to request that the containers are placed on. + * @param racks + * Any racks to request that the containers are placed on. The + * racks corresponding to any hosts requested will be automatically + * added to this list. + * @param priority + * The priority at which to request the containers. Higher + * priorities have lower numerical values. + * @param relaxLocality + * If true, containers for this request may be assigned on hosts + * and racks other than the ones explicitly requested. + * @param nodeLabelsExpression + * Set node labels to allocate resource, now we only support + * asking for only a single node label + */ + public ContainerRequest(Resource capability, String[] nodes, String[] racks, + Priority priority, boolean relaxLocality, String nodeLabelsExpression) { + this(capability, nodes, racks, priority, relaxLocality, null, + ExecutionType.GUARANTEED); + } /** * Instantiates a {@link ContainerRequest} with the given constraints. @@ -173,10 +202,12 @@ public abstract class AMRMClient extends * @param nodeLabelsExpression * Set node labels to allocate resource, now we only support * asking for only a single node label + * @param executionType + * Set the execution type of the container request. */ - public ContainerRequest(Resource capability, String[] nodes, - String[] racks, Priority priority, boolean relaxLocality, - String nodeLabelsExpression) { + public ContainerRequest(Resource capability, String[] nodes, String[] racks, + Priority priority, boolean relaxLocality, String nodeLabelsExpression, + ExecutionType executionType) { // Validate request Preconditions.checkArgument(capability != null, "The Resource to be requested for each container " + @@ -194,6 +225,7 @@ public abstract class AMRMClient extends this.priority = priority; this.relaxLocality = relaxLocality; this.nodeLabelsExpression = nodeLabelsExpression; + this.executionType = executionType; } public Resource getCapability() { @@ -220,10 +252,15 @@ public abstract class AMRMClient extends return nodeLabelsExpression; } + public ExecutionType getExecutionType() { + return executionType; + } + public String toString() { StringBuilder sb = new StringBuilder(); sb.append("Capability[").append(capability).append("]"); sb.append("Priority[").append(priority).append("]"); + sb.append("ExecutionType[").append(executionType).append("]"); return sb.toString(); } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourceRequestPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourceRequestPBImpl.java index fc09cef3b78..53ae2cde78f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourceRequestPBImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourceRequestPBImpl.java @@ -21,6 +21,7 @@ package org.apache.hadoop.yarn.api.records.impl.pb; import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.classification.InterfaceStability.Unstable; +import org.apache.hadoop.yarn.api.records.ExecutionType; import org.apache.hadoop.yarn.api.records.Priority; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.api.records.ResourceRequest; @@ -206,4 +207,24 @@ public class ResourceRequestPBImpl extends ResourceRequest { } builder.setNodeLabelExpression(nodeLabelExpression); } + + @Override + public ExecutionType getExecutionType() { + ResourceRequestProtoOrBuilder p = viaProto ? proto : builder; + if (!p.hasExecutionType()) { + return null; + } + return ProtoUtils.convertFromProtoFormat(p.getExecutionType()); + } + + @Override + public void setExecutionType(ExecutionType execType) { + maybeInitBuilder(); + if (execType == null) { + builder.clearExecutionType(); + return; + } + builder.setExecutionType(ProtoUtils.convertToProtoFormat(execType)); + } + } \ No newline at end of file