From cd3bdbc70a84c8d6f2b100263fbf78c25a858230 Mon Sep 17 00:00:00 2001 From: Wangda Tan Date: Mon, 13 Jun 2016 21:57:33 -0700 Subject: [PATCH] YARN-4887. Add allocation request ID to AM-RM protocol for identifying resource-requests explicitly. (Subru Krishnan via wangda) (cherry picked from commit eec835ec17f77bf713d90dbc2f77dea45f3ee660) --- .../hadoop/yarn/api/records/Container.java | 45 ++++++++++++++++ .../yarn/api/records/ResourceRequest.java | 52 +++++++++++++++++++ .../src/main/proto/yarn_protos.proto | 2 + .../api/records/impl/pb/ContainerPBImpl.java | 14 +++++ .../impl/pb/ResourceRequestPBImpl.java | 16 +++++- 5 files changed, 128 insertions(+), 1 deletion(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Container.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Container.java index 9a62935344d..707a71d5d44 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Container.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Container.java @@ -20,6 +20,7 @@ package org.apache.hadoop.yarn.api.records; import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.classification.InterfaceAudience.Public; +import org.apache.hadoop.classification.InterfaceStability.Evolving; import org.apache.hadoop.classification.InterfaceStability.Stable; import org.apache.hadoop.classification.InterfaceStability.Unstable; import org.apache.hadoop.yarn.api.ApplicationMasterProtocol; @@ -189,4 +190,48 @@ public abstract class Container implements Comparable { @Private @Unstable public abstract void setExecutionType(ExecutionType executionType); + + /** + * Get the optional ID corresponding to the original {@code + * ResourceRequest{@link #getAllocationRequestId()}}s which is satisfied by + * this allocated {@code Container}. + *

+ * The scheduler may return multiple {@code AllocateResponse}s corresponding + * to the same ID as and when scheduler allocates {@code Container}s. + * Applications can continue to completely ignore the returned ID in + * the response and use the allocation for any of their outstanding requests. + *

+ * + * @return the ID corresponding to the original allocation request + * which is satisfied by this allocation. + */ + @Public + @Evolving + public long getAllocationRequestId() { + throw new UnsupportedOperationException(); + } + + /** + * Set the optional ID corresponding to the original {@code + * ResourceRequest{@link #setAllocationRequestId(long)} + * etAllocationRequestId()}}s which is satisfied by this allocated {@code + * Container}. + *

+ * The scheduler may return multiple {@code AllocateResponse}s corresponding + * to the same ID as and when scheduler allocates {@code Container}s. + * Applications can continue to completely ignore the returned ID in + * the response and use the allocation for any of their outstanding requests. + * If the ID is not set, scheduler will continue to work as previously and all + * allocated {@code Container}(s) will have the default ID, -1. + *

+ * + * @param allocationRequestID the ID corresponding to the original + * allocation request which is satisfied by this + * allocation. + */ + @Private + @Evolving + public void setAllocationRequestId(long allocationRequestID) { + throw new UnsupportedOperationException(); + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceRequest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceRequest.java index fbe7e580dfa..50a76198e91 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceRequest.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceRequest.java @@ -312,6 +312,58 @@ public abstract class ResourceRequest implements Comparable { @Public @Evolving public abstract void setNodeLabelExpression(String nodelabelExpression); + + /** + * Get the optional ID corresponding to this allocation request. This + * ID is an identifier for different {@code ResourceRequest}s from the same + * application. The allocated {@code Container}(s) received as part of the + * {@code AllocateResponse} response will have the ID corresponding to the + * original {@code ResourceRequest} for which the RM made the allocation. + *

+ * The scheduler may return multiple {@code AllocateResponse}s corresponding + * to the same ID as and when scheduler allocates {@code Container}(s). + * Applications can continue to completely ignore the returned ID in + * the response and use the allocation for any of their outstanding requests. + *

+ * If one wishes to replace an entire {@code ResourceRequest} corresponding to + * a specific ID, they can simply cancel the corresponding {@code + * ResourceRequest} and submit a new one afresh. + * + * @return the ID corresponding to this allocation request. + */ + @Public + @Evolving + public long getAllocationRequestId() { + throw new UnsupportedOperationException(); + } + + /** + * Set the optional ID corresponding to this allocation request. This + * ID is an identifier for different {@code ResourceRequest}s from the same + * application. The allocated {@code Container}(s) received as part of the + * {@code AllocateResponse} response will have the ID corresponding to the + * original {@code ResourceRequest} for which the RM made the allocation. + *

+ * The scheduler may return multiple {@code AllocateResponse}s corresponding + * to the same ID as and when scheduler allocates {@code Container}(s). + * Applications can continue to completely ignore the returned ID in + * the response and use the allocation for any of their outstanding requests. + *

+ * If one wishes to replace an entire {@code ResourceRequest} corresponding to + * a specific ID, they can simply cancel the corresponding {@code + * ResourceRequest} and submit a new one afresh. + *

+ * If the ID is not set, scheduler will continue to work as previously and all + * allocated {@code Container}(s) will have the default ID, -1. + * + * @param allocationRequestID the ID corresponding to this allocation + * request. + */ + @Public + @Evolving + public void setAllocationRequestId(long allocationRequestID) { + throw new UnsupportedOperationException(); + } @Override public int hashCode() { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto index 5fb18fb3065..172cea3d9a9 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto @@ -93,6 +93,7 @@ message ContainerProto { optional PriorityProto priority = 5; optional hadoop.common.TokenProto container_token = 6; optional ExecutionTypeProto execution_type = 7 [default = GUARANTEED]; + optional int64 allocation_request_id = 8 [default = -1]; } message ContainerReportProto { @@ -307,6 +308,7 @@ message ResourceRequestProto { optional bool relax_locality = 5 [default = true]; optional string node_label_expression = 6; optional ExecutionTypeRequestProto execution_type_request = 7; + optional int64 allocation_request_id = 8 [default = -1]; } message ExecutionTypeRequestProto { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerPBImpl.java index bd2d93794bb..91b3e5f3f95 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerPBImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerPBImpl.java @@ -262,6 +262,18 @@ public class ContainerPBImpl extends Container { builder.setExecutionType(convertToProtoFormat(executionType)); } + @Override + public long getAllocationRequestId() { + ContainerProtoOrBuilder p = viaProto ? proto : builder; + return (p.getAllocationRequestId()); + } + + @Override + public void setAllocationRequestId(long allocationRequestID) { + maybeInitBuilder(); + builder.setAllocationRequestId(allocationRequestID); + } + private ContainerIdPBImpl convertFromProtoFormat(ContainerIdProto p) { return new ContainerIdPBImpl(p); } @@ -315,6 +327,8 @@ public class ContainerPBImpl extends Container { StringBuilder sb = new StringBuilder(); sb.append("Container: ["); sb.append("ContainerId: ").append(getId()).append(", "); + sb.append("AllocationRequestId: ").append(getAllocationRequestId()) + .append(", "); sb.append("NodeId: ").append(getNodeId()).append(", "); sb.append("NodeHttpAddress: ").append(getNodeHttpAddress()).append(", "); sb.append("Resource: ").append(getResource()).append(", "); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourceRequestPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourceRequestPBImpl.java index b0c4b977577..9890296acdd 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourceRequestPBImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourceRequestPBImpl.java @@ -192,6 +192,18 @@ public class ResourceRequestPBImpl extends ResourceRequest { builder.setRelaxLocality(relaxLocality); } + @Override + public long getAllocationRequestId() { + ResourceRequestProtoOrBuilder p = viaProto ? proto : builder; + return (p.getAllocationRequestId()); + } + + @Override + public void setAllocationRequestId(long allocationRequestID) { + maybeInitBuilder(); + builder.setAllocationRequestId(allocationRequestID); + } + private PriorityPBImpl convertFromProtoFormat(PriorityProto p) { return new PriorityPBImpl(p); } @@ -210,7 +222,9 @@ public class ResourceRequestPBImpl extends ResourceRequest { @Override public String toString() { - return "{Priority: " + getPriority() + ", Capability: " + getCapability() + return "{AllocationRequestId: " + getAllocationRequestId() + + ", Priority: " + getPriority() + + ", Capability: " + getCapability() + ", # Containers: " + getNumContainers() + ", Location: " + getResourceName() + ", Relax Locality: " + getRelaxLocality()