diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/local/LocalContainerAllocator.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/local/LocalContainerAllocator.java index c94f91bbd7a..426dc212f52 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/local/LocalContainerAllocator.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/local/LocalContainerAllocator.java @@ -93,9 +93,9 @@ public class LocalContainerAllocator extends RMCommunicator @SuppressWarnings("unchecked") @Override protected synchronized void heartbeat() throws Exception { - AllocateRequest allocateRequest = AllocateRequest.newInstance( - this.applicationAttemptId, this.lastResponseID, super - .getApplicationProgress(), new ArrayList(), + AllocateRequest allocateRequest = + AllocateRequest.newInstance(this.lastResponseID, + super.getApplicationProgress(), new ArrayList(), new ArrayList(), null); AllocateResponse allocateResponse; try { @@ -143,7 +143,7 @@ public class LocalContainerAllocator extends RMCommunicator LOG.info("Processing the event " + event.toString()); // Assign the same container ID as the AM ContainerId cID = - ContainerId.newInstance(applicationAttemptId, + ContainerId.newInstance(getContext().getApplicationAttemptId(), this.containerId.getId()); Container container = recordFactory.newRecordInstance(Container.class); container.setId(cID); diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMCommunicator.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMCommunicator.java index ca64cc83125..187a6e06205 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMCommunicator.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMCommunicator.java @@ -45,7 +45,6 @@ import org.apache.hadoop.yarn.api.protocolrecords.FinishApplicationMasterRequest import org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterRequest; import org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterResponse; import org.apache.hadoop.yarn.api.records.ApplicationAccessType; -import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.FinalApplicationStatus; import org.apache.hadoop.yarn.api.records.Resource; @@ -63,7 +62,6 @@ public abstract class RMCommunicator extends AbstractService private static final Log LOG = LogFactory.getLog(RMContainerAllocator.class); private int rmPollInterval;//millis protected ApplicationId applicationId; - protected ApplicationAttemptId applicationAttemptId; private final AtomicBoolean stopped; protected Thread allocatorThread; @SuppressWarnings("rawtypes") @@ -91,7 +89,6 @@ public abstract class RMCommunicator extends AbstractService this.context = context; this.eventHandler = context.getEventHandler(); this.applicationId = context.getApplicationID(); - this.applicationAttemptId = context.getApplicationAttemptId(); this.stopped = new AtomicBoolean(false); this.heartbeatCallbacks = new ConcurrentLinkedQueue(); } @@ -142,7 +139,6 @@ public abstract class RMCommunicator extends AbstractService try { RegisterApplicationMasterRequest request = recordFactory.newRecordInstance(RegisterApplicationMasterRequest.class); - request.setApplicationAttemptId(applicationAttemptId); if (serviceAddr != null) { request.setHost(serviceAddr.getHostName()); request.setRpcPort(serviceAddr.getPort()); @@ -193,11 +189,8 @@ public abstract class RMCommunicator extends AbstractService LOG.info("History url is " + historyUrl); FinishApplicationMasterRequest request = - recordFactory.newRecordInstance(FinishApplicationMasterRequest.class); - request.setAppAttemptId(this.applicationAttemptId); - request.setFinalApplicationStatus(finishState); - request.setDiagnostics(sb.toString()); - request.setTrackingUrl(historyUrl); + FinishApplicationMasterRequest.newInstance(finishState, + sb.toString(), historyUrl); scheduler.finishApplicationMaster(request); } catch(Exception are) { LOG.error("Exception while unregistering ", are); diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java index dc134ebbe2e..d6e45931632 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java @@ -1160,14 +1160,6 @@ public class RMContainerAllocator extends RMContainerRequestor TaskAttemptId get(ContainerId cId) { return containerToAttemptMap.get(cId); } - - NodeId getNodeId(TaskAttemptId tId) { - if (tId.getTaskId().getTaskType().equals(TaskType.MAP)) { - return maps.get(tId).getNodeId(); - } else { - return reduces.get(tId).getNodeId(); - } - } ContainerId get(TaskAttemptId tId) { Container taskContainer; diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerRequestor.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerRequestor.java index 16271d492b2..67dd30e1641 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerRequestor.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerRequestor.java @@ -145,10 +145,10 @@ public abstract class RMContainerRequestor extends RMCommunicator { } protected AllocateResponse makeRemoteRequest() throws IOException { - AllocateRequest allocateRequest = AllocateRequest.newInstance( - applicationAttemptId, lastResponseID, super.getApplicationProgress(), - new ArrayList(ask), new ArrayList( - release), null); + AllocateRequest allocateRequest = + AllocateRequest.newInstance(lastResponseID, + super.getApplicationProgress(), new ArrayList(ask), + new ArrayList(release), null); AllocateResponse allocateResponse; try { allocateResponse = scheduler.allocate(allocateRequest); diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MRAppBenchmark.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MRAppBenchmark.java index 3b9292a33d5..a6496d4e96e 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MRAppBenchmark.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MRAppBenchmark.java @@ -234,7 +234,7 @@ public class MRAppBenchmark { for (int i = 0; i < numContainers; i++) { ContainerId containerId = ContainerId.newInstance( - request.getApplicationAttemptId(), + getContext().getApplicationAttemptId(), request.getResponseId() + i); containers.add(Container.newInstance(containerId, NodeId.newInstance("host" + containerId.getId(), 2345), diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 2920618aefb..f7bfbf28207 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -256,6 +256,9 @@ Release 2.1.0-beta - 2013-07-02 YARN-701. Use application tokens irrespective of secure or non-secure mode. (vinodkv via acmurthy) + YARN-918. Remove ApplicationAttemptId from + RegisterApplicationMasterRequestProto. (vinodkv via acmurthy) + NEW FEATURES YARN-482. FS: Extend SchedulingMode to intermediate queues. diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/AllocateRequest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/AllocateRequest.java index 964d3683bd3..9ae4a12f965 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/AllocateRequest.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/AllocateRequest.java @@ -23,10 +23,9 @@ import java.util.List; import org.apache.hadoop.classification.InterfaceAudience.Public; import org.apache.hadoop.classification.InterfaceStability.Stable; import org.apache.hadoop.yarn.api.ApplicationMasterProtocol; -import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; -import org.apache.hadoop.yarn.api.records.ResourceBlacklistRequest; import org.apache.hadoop.yarn.api.records.Container; import org.apache.hadoop.yarn.api.records.ContainerId; +import org.apache.hadoop.yarn.api.records.ResourceBlacklistRequest; import org.apache.hadoop.yarn.api.records.ResourceRequest; import org.apache.hadoop.yarn.util.Records; @@ -36,10 +35,6 @@ import org.apache.hadoop.yarn.util.Records; * *

The request includes: *

    - *
  • - * {@link ApplicationAttemptId} being managed by the - * ApplicationMaster - *
    • *
  • A response id to track duplicate responses.
    • *
  • Progress information.
    • *
  • @@ -61,13 +56,11 @@ public abstract class AllocateRequest { @Public @Stable - public static AllocateRequest newInstance( - ApplicationAttemptId applicationAttemptId, int responseID, - float appProgress, List resourceAsk, - List containersToBeReleased, + public static AllocateRequest newInstance(int responseID, float appProgress, + List resourceAsk, + List containersToBeReleased, ResourceBlacklistRequest resourceBlacklistRequest) { AllocateRequest allocateRequest = Records.newRecord(AllocateRequest.class); - allocateRequest.setApplicationAttemptId(applicationAttemptId); allocateRequest.setResponseId(responseID); allocateRequest.setProgress(appProgress); allocateRequest.setAskList(resourceAsk); @@ -75,27 +68,7 @@ public abstract class AllocateRequest { allocateRequest.setResourceBlacklistRequest(resourceBlacklistRequest); return allocateRequest; } - - /** - * Get the ApplicationAttemptId being managed by the - * ApplicationMaster. - * @return ApplicationAttemptId being managed by the - * ApplicationMaster - */ - @Public - @Stable - public abstract ApplicationAttemptId getApplicationAttemptId(); - /** - * Set the ApplicationAttemptId being managed by the - * ApplicationMaster. - * @param applicationAttemptId ApplicationAttemptId being managed - * by the ApplicationMaster - */ - @Public - @Stable - public abstract void setApplicationAttemptId(ApplicationAttemptId applicationAttemptId); - /** * Get the response id used to track duplicate responses. * @return response id diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/FinishApplicationMasterRequest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/FinishApplicationMasterRequest.java index 882c1227e30..4f20f71208f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/FinishApplicationMasterRequest.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/FinishApplicationMasterRequest.java @@ -21,7 +21,6 @@ package org.apache.hadoop.yarn.api.protocolrecords; import org.apache.hadoop.classification.InterfaceAudience.Public; import org.apache.hadoop.classification.InterfaceStability.Stable; import org.apache.hadoop.yarn.api.ApplicationMasterProtocol; -import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.FinalApplicationStatus; import org.apache.hadoop.yarn.util.Records; @@ -31,10 +30,6 @@ import org.apache.hadoop.yarn.util.Records; * *

    The final request includes details such: *

      - *
    • - * {@link ApplicationAttemptId} being managed by the - * ApplicationMaster - *
      • *
    • Final state of the ApplicationMaster
      • *
    • * Diagnostic information in case of failure of the @@ -53,37 +48,15 @@ public abstract class FinishApplicationMasterRequest { @Public @Stable public static FinishApplicationMasterRequest newInstance( - ApplicationAttemptId appAttemptId, FinalApplicationStatus finalAppStatus, - String diagnostics, String url) { + FinalApplicationStatus finalAppStatus, String diagnostics, String url) { FinishApplicationMasterRequest request = Records.newRecord(FinishApplicationMasterRequest.class); - request.setAppAttemptId(appAttemptId); request.setFinalApplicationStatus(finalAppStatus); request.setDiagnostics(diagnostics); request.setTrackingUrl(url); return request; } - /** - * Get the ApplicationAttemptId being managed by the - * ApplicationMaster. - * @return ApplicationAttemptId being managed by the - * ApplicationMaster - */ - @Public - @Stable - public abstract ApplicationAttemptId getApplicationAttemptId(); - - /** - * Set the ApplicationAttemptId being managed by the - * ApplicationMaster. - * @param applicationAttemptId ApplicationAttemptId being managed - * by the ApplicationMaster - */ - @Public - @Stable - public abstract void setAppAttemptId(ApplicationAttemptId applicationAttemptId); - /** * Get final state of the ApplicationMaster. * @return final state of the ApplicationMaster diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/RegisterApplicationMasterRequest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/RegisterApplicationMasterRequest.java index fde83b2fc02..730eaf95129 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/RegisterApplicationMasterRequest.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/RegisterApplicationMasterRequest.java @@ -21,7 +21,6 @@ package org.apache.hadoop.yarn.api.protocolrecords; import org.apache.hadoop.classification.InterfaceAudience.Public; import org.apache.hadoop.classification.InterfaceStability.Stable; import org.apache.hadoop.yarn.api.ApplicationMasterProtocol; -import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.util.Records; /** @@ -30,10 +29,6 @@ import org.apache.hadoop.yarn.util.Records; * *

      The registration includes details such as: *

        - *
      • - * {@link ApplicationAttemptId} being managed by the - * ApplicationMaster - *
        • *
      • Hostname on which the AM is running.
        • *
      • RPC Port
        • *
      • Tracking URL
        • @@ -57,38 +52,16 @@ public abstract class RegisterApplicationMasterRequest { */ @Public @Stable - public static RegisterApplicationMasterRequest newInstance( - ApplicationAttemptId applicationAttemptId, String host, int port, - String trackingUrl) { + public static RegisterApplicationMasterRequest newInstance(String host, + int port, String trackingUrl) { RegisterApplicationMasterRequest request = Records.newRecord(RegisterApplicationMasterRequest.class); - request.setApplicationAttemptId(applicationAttemptId); request.setHost(host); request.setRpcPort(port); request.setTrackingUrl(trackingUrl); return request; } - /** - * Get the ApplicationAttemptId being managed by the - * ApplicationMaster. - * @return ApplicationAttemptId being managed by the - * ApplicationMaster - */ - @Public - @Stable - public abstract ApplicationAttemptId getApplicationAttemptId(); - - /** - * Set the ApplicationAttemptId being managed by the - * ApplicationMaster. - * @param applicationAttemptId ApplicationAttemptId being managed - * by the ApplicationMaster - */ - @Public - @Stable - public abstract void setApplicationAttemptId(ApplicationAttemptId applicationAttemptId); - /** * Get the host on which the ApplicationMaster is * running. diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto index dd3b3214296..4a6db02ff8c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto @@ -34,10 +34,9 @@ import "yarn_protos.proto"; /////// AM_RM_Protocol /////////////////////////////// ////////////////////////////////////////////////////// message RegisterApplicationMasterRequestProto { - optional ApplicationAttemptIdProto application_attempt_id = 1; - optional string host = 2; - optional int32 rpc_port = 3; - optional string tracking_url = 4; + optional string host = 1; + optional int32 rpc_port = 2; + optional string tracking_url = 3; } message RegisterApplicationMasterResponseProto { @@ -47,22 +46,20 @@ message RegisterApplicationMasterResponseProto { } message FinishApplicationMasterRequestProto { - optional ApplicationAttemptIdProto application_attempt_id = 1; - optional string diagnostics = 2; - optional string tracking_url = 3; - optional FinalApplicationStatusProto final_application_status = 4; + optional string diagnostics = 1; + optional string tracking_url = 2; + optional FinalApplicationStatusProto final_application_status = 3; } message FinishApplicationMasterResponseProto { } message AllocateRequestProto { - optional ApplicationAttemptIdProto application_attempt_id = 1; - repeated ResourceRequestProto ask = 2; - repeated ContainerIdProto release = 3; - optional ResourceBlacklistRequestProto blacklist_request = 4; - optional int32 response_id = 5; - optional float progress = 6; + repeated ResourceRequestProto ask = 1; + repeated ContainerIdProto release = 2; + optional ResourceBlacklistRequestProto blacklist_request = 3; + optional int32 response_id = 4; + optional float progress = 5; } message NMTokenProto { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java index 3cf57010153..df08b0da720 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java @@ -442,8 +442,8 @@ public class ApplicationMaster { LOG.info("Starting ApplicationMaster"); AMRMClientAsync.CallbackHandler allocListener = new RMCallbackHandler(); - resourceManager = - AMRMClientAsync.createAMRMClientAsync(appAttemptID, 1000, allocListener); + resourceManager = + AMRMClientAsync.createAMRMClientAsync(1000, allocListener); resourceManager.init(conf); resourceManager.start(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/AMRMClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/AMRMClient.java index 6d38a344c77..9a0caad4c2b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/AMRMClient.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/AMRMClient.java @@ -29,15 +29,15 @@ import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.service.AbstractService; import org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse; import org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterResponse; -import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.FinalApplicationStatus; import org.apache.hadoop.yarn.api.records.Priority; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.client.api.impl.AMRMClientImpl; import org.apache.hadoop.yarn.exceptions.YarnException; -import com.google.common.collect.ImmutableList; + import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; @InterfaceAudience.Public @InterfaceStability.Stable @@ -49,17 +49,13 @@ public abstract class AMRMClient extends * For usage: * 
            * {@code
-   * AMRMClient.createAMRMClientContainerRequest(appAttemptId)
+   * AMRMClient.createAMRMClientContainerRequest()
    * }
        - * @param appAttemptId the appAttemptId associated with the AMRMClient * @return the newly create AMRMClient instance. */ @Public - public static AMRMClient createAMRMClient( - ApplicationAttemptId appAttemptId) { - Preconditions.checkArgument(appAttemptId != null, - "ApplicationAttempId should not be null"); - AMRMClient client = new AMRMClientImpl(appAttemptId); + public static AMRMClient createAMRMClient() { + AMRMClient client = new AMRMClientImpl(); return client; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/async/AMRMClientAsync.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/async/AMRMClientAsync.java index ae781b6003a..4771f0d0bd6 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/async/AMRMClientAsync.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/async/AMRMClientAsync.java @@ -28,7 +28,6 @@ import org.apache.hadoop.classification.InterfaceAudience.Public; import org.apache.hadoop.classification.InterfaceStability.Stable; import org.apache.hadoop.service.AbstractService; import org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterResponse; -import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.Container; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.ContainerStatus; @@ -96,25 +95,19 @@ extends AbstractService { protected final CallbackHandler handler; protected final AtomicInteger heartbeatIntervalMs = new AtomicInteger(); - public static AMRMClientAsync - createAMRMClientAsync( - ApplicationAttemptId id, - int intervalMs, - CallbackHandler callbackHandler) { - return new AMRMClientAsyncImpl(id, intervalMs, callbackHandler); + public static AMRMClientAsync + createAMRMClientAsync(int intervalMs, CallbackHandler callbackHandler) { + return new AMRMClientAsyncImpl(intervalMs, callbackHandler); } - public static AMRMClientAsync - createAMRMClientAsync( - AMRMClient client, - int intervalMs, - CallbackHandler callbackHandler) { + public static AMRMClientAsync + createAMRMClientAsync(AMRMClient client, int intervalMs, + CallbackHandler callbackHandler) { return new AMRMClientAsyncImpl(client, intervalMs, callbackHandler); } - protected AMRMClientAsync(ApplicationAttemptId id, int intervalMs, - CallbackHandler callbackHandler) { - this(new AMRMClientImpl(id), intervalMs, callbackHandler); + protected AMRMClientAsync(int intervalMs, CallbackHandler callbackHandler) { + this(new AMRMClientImpl(), intervalMs, callbackHandler); } @Private diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/async/impl/AMRMClientAsyncImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/async/impl/AMRMClientAsyncImpl.java index f7002868bc1..eb8b72f1255 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/async/impl/AMRMClientAsyncImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/async/impl/AMRMClientAsyncImpl.java @@ -32,7 +32,6 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse; import org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterResponse; import org.apache.hadoop.yarn.api.records.AMCommand; -import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.Container; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.ContainerStatus; @@ -68,9 +67,8 @@ extends AMRMClientAsync { private volatile Exception savedException; - public AMRMClientAsyncImpl(ApplicationAttemptId id, int intervalMs, - CallbackHandler callbackHandler) { - this(new AMRMClientImpl(id), intervalMs, callbackHandler); + public AMRMClientAsyncImpl(int intervalMs, CallbackHandler callbackHandler) { + this(new AMRMClientImpl(), intervalMs, callbackHandler); } @Private diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/AMRMClientImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/AMRMClientImpl.java index f55da08da1c..a7035b9056c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/AMRMClientImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/AMRMClientImpl.java @@ -46,7 +46,6 @@ import org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse; import org.apache.hadoop.yarn.api.protocolrecords.FinishApplicationMasterRequest; import org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterRequest; import org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterResponse; -import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.FinalApplicationStatus; import org.apache.hadoop.yarn.api.records.NMToken; @@ -61,8 +60,6 @@ import org.apache.hadoop.yarn.client.api.NMTokenCache; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; -import org.apache.hadoop.yarn.factories.RecordFactory; -import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider; import org.apache.hadoop.yarn.util.RackResolver; import com.google.common.annotations.VisibleForTesting; @@ -77,13 +74,9 @@ public class AMRMClientImpl extends AMRMClient { private static final List ANY_LIST = Collections.singletonList(ResourceRequest.ANY); - private final RecordFactory recordFactory = - RecordFactoryProvider.getRecordFactory(null); - private int lastResponseId = 0; protected ApplicationMasterProtocol rmClient; - protected final ApplicationAttemptId appAttemptId; protected Resource clusterAvailableResources; protected int clusterNodeCount; @@ -154,9 +147,8 @@ public class AMRMClientImpl extends AMRMClient { new org.apache.hadoop.yarn.api.records.ResourceRequest.ResourceRequestComparator()); protected final Set release = new TreeSet(); - public AMRMClientImpl(ApplicationAttemptId appAttemptId) { + public AMRMClientImpl() { super(AMRMClientImpl.class.getName()); - this.appAttemptId = appAttemptId; } @Override @@ -193,18 +185,11 @@ public class AMRMClientImpl extends AMRMClient { Preconditions.checkArgument(appHostPort >= 0, "Port number of the host should not be negative"); // do this only once ??? - RegisterApplicationMasterRequest request = recordFactory - .newRecordInstance(RegisterApplicationMasterRequest.class); - synchronized (this) { - request.setApplicationAttemptId(appAttemptId); - } - request.setHost(appHostName); - request.setRpcPort(appHostPort); - if(appTrackingUrl != null) { - request.setTrackingUrl(appTrackingUrl); - } - RegisterApplicationMasterResponse response = rmClient - .registerApplicationMaster(request); + RegisterApplicationMasterRequest request = + RegisterApplicationMasterRequest.newInstance(appHostName, appHostPort, + appTrackingUrl); + RegisterApplicationMasterResponse response = + rmClient.registerApplicationMaster(request); return response; } @@ -233,8 +218,8 @@ public class AMRMClientImpl extends AMRMClient { ask.clear(); release.clear(); allocateRequest = - AllocateRequest.newInstance(appAttemptId, lastResponseId, - progressIndicator, askList, releaseList, null); + AllocateRequest.newInstance(lastResponseId, progressIndicator, + askList, releaseList, null); } allocateResponse = rmClient.allocate(allocateRequest); @@ -292,18 +277,9 @@ public class AMRMClientImpl extends AMRMClient { public void unregisterApplicationMaster(FinalApplicationStatus appStatus, String appMessage, String appTrackingUrl) throws YarnException, IOException { - Preconditions.checkArgument(appStatus != null, - "AppStatus should not be null."); - FinishApplicationMasterRequest request = recordFactory - .newRecordInstance(FinishApplicationMasterRequest.class); - request.setAppAttemptId(appAttemptId); - request.setFinalApplicationStatus(appStatus); - if(appMessage != null) { - request.setDiagnostics(appMessage); - } - if(appTrackingUrl != null) { - request.setTrackingUrl(appTrackingUrl); - } + FinishApplicationMasterRequest request = + FinishApplicationMasterRequest.newInstance(appStatus, appMessage, + appTrackingUrl); rmClient.finishApplicationMaster(request); } @@ -553,7 +529,7 @@ public class AMRMClientImpl extends AMRMClient { if (LOG.isDebugEnabled()) { LOG.debug("addResourceRequest:" + " applicationId=" - + appAttemptId + " priority=" + priority.getPriority() + + " priority=" + priority.getPriority() + " resourceName=" + resourceName + " numContainers=" + resourceRequestInfo.remoteRequest.getNumContainers() + " #asks=" + ask.size()); @@ -587,7 +563,7 @@ public class AMRMClientImpl extends AMRMClient { if (LOG.isDebugEnabled()) { LOG.debug("BEFORE decResourceRequest:" + " applicationId=" - + appAttemptId + " priority=" + priority.getPriority() + + " priority=" + priority.getPriority() + " resourceName=" + resourceName + " numContainers=" + resourceRequestInfo.remoteRequest.getNumContainers() + " #asks=" + ask.size()); @@ -620,7 +596,7 @@ public class AMRMClientImpl extends AMRMClient { if (LOG.isDebugEnabled()) { LOG.info("AFTER decResourceRequest:" + " applicationId=" - + appAttemptId + " priority=" + priority.getPriority() + + " priority=" + priority.getPriority() + " resourceName=" + resourceName + " numContainers=" + resourceRequestInfo.remoteRequest.getNumContainers() + " #asks=" + ask.size()); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMClient.java index e0b4052b359..f4a6dd3f3c8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMClient.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMClient.java @@ -200,7 +200,7 @@ public class TestAMRMClient { AMRMClient amClient = null; try { // start am rm client - amClient = AMRMClient.createAMRMClient(attemptId); + amClient = AMRMClient.createAMRMClient(); amClient.init(conf); amClient.start(); amClient.registerApplicationMaster("Host", 10000, ""); @@ -314,7 +314,7 @@ public class TestAMRMClient { AMRMClientImpl amClient = null; try { // start am rm client - amClient = new AMRMClientImpl(attemptId); + amClient = new AMRMClientImpl(); amClient.init(conf); amClient.start(); amClient.registerApplicationMaster("Host", 10000, ""); @@ -361,7 +361,7 @@ public class TestAMRMClient { // start am rm client amClient = (AMRMClientImpl) AMRMClient - . createAMRMClient(attemptId); + . createAMRMClient(); amClient.init(conf); amClient.start(); amClient.registerApplicationMaster("Host", 10000, ""); @@ -482,7 +482,7 @@ public class TestAMRMClient { AMRMClient amClient = null; try { // start am rm client - amClient = AMRMClient.createAMRMClient(attemptId); + amClient = AMRMClient.createAMRMClient(); amClient.init(conf); amClient.start(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMClientContainerRequest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMClientContainerRequest.java index b87ebb95d0b..71ee9f9f5a7 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMClientContainerRequest.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMClientContainerRequest.java @@ -18,30 +18,27 @@ package org.apache.hadoop.yarn.client.api.impl; +import static org.junit.Assert.assertEquals; + import java.util.Arrays; import java.util.List; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.CommonConfigurationKeysPublic; import org.apache.hadoop.net.DNSToSwitchMapping; -import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; -import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.Priority; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.api.records.ResourceRequest; import org.apache.hadoop.yarn.client.api.AMRMClient; import org.apache.hadoop.yarn.client.api.AMRMClient.ContainerRequest; import org.apache.hadoop.yarn.client.api.InvalidContainerRequestException; -import org.apache.hadoop.yarn.client.api.impl.AMRMClientImpl; import org.junit.Test; -import static org.junit.Assert.assertEquals; - public class TestAMRMClientContainerRequest { @Test public void testFillInRacks() { - AMRMClientImpl client = new AMRMClientImpl( - ApplicationAttemptId.newInstance(ApplicationId.newInstance(0l, 0), 0)); + AMRMClientImpl client = + new AMRMClientImpl(); Configuration conf = new Configuration(); conf.setClass( @@ -63,8 +60,8 @@ public class TestAMRMClientContainerRequest { @Test public void testDisableLocalityRelaxation() { - AMRMClientImpl client = new AMRMClientImpl( - ApplicationAttemptId.newInstance(ApplicationId.newInstance(0l, 0), 0)); + AMRMClientImpl client = + new AMRMClientImpl(); Configuration conf = new Configuration(); conf.setClass( CommonConfigurationKeysPublic.NET_TOPOLOGY_NODE_SWITCH_MAPPING_IMPL_KEY, @@ -130,8 +127,8 @@ public class TestAMRMClientContainerRequest { @Test (expected = InvalidContainerRequestException.class) public void testDifferentLocalityRelaxationSamePriority() { - AMRMClientImpl client = new AMRMClientImpl( - ApplicationAttemptId.newInstance(ApplicationId.newInstance(0l, 0), 0)); + AMRMClientImpl client = + new AMRMClientImpl(); Configuration conf = new Configuration(); conf.setClass( CommonConfigurationKeysPublic.NET_TOPOLOGY_NODE_SWITCH_MAPPING_IMPL_KEY, @@ -151,8 +148,8 @@ public class TestAMRMClientContainerRequest { @Test public void testInvalidValidWhenOldRemoved() { - AMRMClientImpl client = new AMRMClientImpl( - ApplicationAttemptId.newInstance(ApplicationId.newInstance(0l, 0), 0)); + AMRMClientImpl client = + new AMRMClientImpl(); Configuration conf = new Configuration(); conf.setClass( CommonConfigurationKeysPublic.NET_TOPOLOGY_NODE_SWITCH_MAPPING_IMPL_KEY, @@ -190,8 +187,8 @@ public class TestAMRMClientContainerRequest { @Test (expected = InvalidContainerRequestException.class) public void testLocalityRelaxationDifferentLevels() { - AMRMClientImpl client = new AMRMClientImpl( - ApplicationAttemptId.newInstance(ApplicationId.newInstance(0l, 0), 0)); + AMRMClientImpl client = + new AMRMClientImpl(); Configuration conf = new Configuration(); conf.setClass( CommonConfigurationKeysPublic.NET_TOPOLOGY_NODE_SWITCH_MAPPING_IMPL_KEY, diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestNMClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestNMClient.java index 03959e1dc25..b5f82817815 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestNMClient.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestNMClient.java @@ -157,7 +157,7 @@ public class TestNMClient { // start am rm client rmClient = (AMRMClientImpl) AMRMClient - . createAMRMClient(attemptId); + . createAMRMClient(); rmClient.init(conf); rmClient.start(); assertNotNull(rmClient); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/AllocateRequestPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/AllocateRequestPBImpl.java index 4584dc9f202..723ab5cf6cd 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/AllocateRequestPBImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/AllocateRequestPBImpl.java @@ -26,15 +26,12 @@ import java.util.List; import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.classification.InterfaceStability.Unstable; import org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest; -import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.ResourceBlacklistRequest; import org.apache.hadoop.yarn.api.records.ResourceRequest; -import org.apache.hadoop.yarn.api.records.impl.pb.ApplicationAttemptIdPBImpl; import org.apache.hadoop.yarn.api.records.impl.pb.ContainerIdPBImpl; import org.apache.hadoop.yarn.api.records.impl.pb.ResourceBlacklistRequestPBImpl; import org.apache.hadoop.yarn.api.records.impl.pb.ResourceRequestPBImpl; -import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationAttemptIdProto; import org.apache.hadoop.yarn.proto.YarnProtos.ContainerIdProto; import org.apache.hadoop.yarn.proto.YarnProtos.ResourceBlacklistRequestProto; import org.apache.hadoop.yarn.proto.YarnProtos.ResourceRequestProto; @@ -48,7 +45,6 @@ public class AllocateRequestPBImpl extends AllocateRequest { AllocateRequestProto.Builder builder = null; boolean viaProto = false; - private ApplicationAttemptId applicationAttemptID = null; private List ask = null; private List release = null; private ResourceBlacklistRequest blacklistRequest = null; @@ -91,9 +87,6 @@ public class AllocateRequestPBImpl extends AllocateRequest { } private void mergeLocalToBuilder() { - if (this.applicationAttemptID != null) { - builder.setApplicationAttemptId(convertToProtoFormat(this.applicationAttemptID)); - } if (this.ask != null) { addAsksToProto(); } @@ -120,27 +113,6 @@ public class AllocateRequestPBImpl extends AllocateRequest { viaProto = false; } - @Override - public ApplicationAttemptId getApplicationAttemptId() { - AllocateRequestProtoOrBuilder p = viaProto ? proto : builder; - if (this.applicationAttemptID != null) { - return this.applicationAttemptID; - } - if (!p.hasApplicationAttemptId()) { - return null; - } - this.applicationAttemptID = convertFromProtoFormat(p.getApplicationAttemptId()); - return this.applicationAttemptID; - } - - @Override - public void setApplicationAttemptId(ApplicationAttemptId appAttemptId) { - maybeInitBuilder(); - if (appAttemptId == null) - builder.clearApplicationAttemptId(); - this.applicationAttemptID = appAttemptId; - } - @Override public int getResponseId() { AllocateRequestProtoOrBuilder p = viaProto ? proto : builder; @@ -311,14 +283,6 @@ public class AllocateRequestPBImpl extends AllocateRequest { builder.addAllRelease(iterable); } - private ApplicationAttemptIdPBImpl convertFromProtoFormat(ApplicationAttemptIdProto p) { - return new ApplicationAttemptIdPBImpl(p); - } - - private ApplicationAttemptIdProto convertToProtoFormat(ApplicationAttemptId t) { - return ((ApplicationAttemptIdPBImpl)t).getProto(); - } - private ResourceRequestPBImpl convertFromProtoFormat(ResourceRequestProto p) { return new ResourceRequestPBImpl(p); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/FinishApplicationMasterRequestPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/FinishApplicationMasterRequestPBImpl.java index cf52896acc1..233c3114834 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/FinishApplicationMasterRequestPBImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/FinishApplicationMasterRequestPBImpl.java @@ -22,11 +22,8 @@ package org.apache.hadoop.yarn.api.protocolrecords.impl.pb; import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.classification.InterfaceStability.Unstable; import org.apache.hadoop.yarn.api.protocolrecords.FinishApplicationMasterRequest; -import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.FinalApplicationStatus; -import org.apache.hadoop.yarn.api.records.impl.pb.ApplicationAttemptIdPBImpl; import org.apache.hadoop.yarn.api.records.impl.pb.ProtoUtils; -import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationAttemptIdProto; import org.apache.hadoop.yarn.proto.YarnProtos.FinalApplicationStatusProto; import org.apache.hadoop.yarn.proto.YarnServiceProtos.FinishApplicationMasterRequestProto; import org.apache.hadoop.yarn.proto.YarnServiceProtos.FinishApplicationMasterRequestProtoOrBuilder; @@ -38,9 +35,6 @@ public class FinishApplicationMasterRequestPBImpl extends FinishApplicationMaste FinishApplicationMasterRequestProto.Builder builder = null; boolean viaProto = false; - private ApplicationAttemptId appAttemptId = null; - - public FinishApplicationMasterRequestPBImpl() { builder = FinishApplicationMasterRequestProto.newBuilder(); } @@ -78,9 +72,6 @@ public class FinishApplicationMasterRequestPBImpl extends FinishApplicationMaste } private void mergeLocalToBuilder() { - if (this.appAttemptId != null) { - builder.setApplicationAttemptId(convertToProtoFormat(this.appAttemptId)); - } } private void mergeLocalToProto() { @@ -98,27 +89,6 @@ public class FinishApplicationMasterRequestPBImpl extends FinishApplicationMaste viaProto = false; } - @Override - public ApplicationAttemptId getApplicationAttemptId() { - FinishApplicationMasterRequestProtoOrBuilder p = viaProto ? proto : builder; - if (this.appAttemptId != null) { - return this.appAttemptId; - } - if (!p.hasApplicationAttemptId()) { - return null; - } - this.appAttemptId = convertFromProtoFormat(p.getApplicationAttemptId()); - return this.appAttemptId; - } - - @Override - public void setAppAttemptId(ApplicationAttemptId applicationAttemptId) { - maybeInitBuilder(); - if (applicationAttemptId == null) - builder.clearApplicationAttemptId(); - this.appAttemptId = applicationAttemptId; - } - @Override public String getDiagnostics() { FinishApplicationMasterRequestProtoOrBuilder p = viaProto ? proto : builder; @@ -128,6 +98,10 @@ public class FinishApplicationMasterRequestPBImpl extends FinishApplicationMaste @Override public void setDiagnostics(String diagnostics) { maybeInitBuilder(); + if (diagnostics == null) { + builder.clearDiagnostics(); + return; + } builder.setDiagnostics(diagnostics); } @@ -140,6 +114,10 @@ public class FinishApplicationMasterRequestPBImpl extends FinishApplicationMaste @Override public void setTrackingUrl(String url) { maybeInitBuilder(); + if (url == null) { + builder.clearTrackingUrl(); + return; + } builder.setTrackingUrl(url); } @@ -162,14 +140,6 @@ public class FinishApplicationMasterRequestPBImpl extends FinishApplicationMaste builder.setFinalApplicationStatus(convertToProtoFormat(finalState)); } - private ApplicationAttemptIdPBImpl convertFromProtoFormat(ApplicationAttemptIdProto p) { - return new ApplicationAttemptIdPBImpl(p); - } - - private ApplicationAttemptIdProto convertToProtoFormat(ApplicationAttemptId t) { - return ((ApplicationAttemptIdPBImpl)t).getProto(); - } - private FinalApplicationStatus convertFromProtoFormat(FinalApplicationStatusProto s) { return ProtoUtils.convertFromProtoFormat(s); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/RegisterApplicationMasterRequestPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/RegisterApplicationMasterRequestPBImpl.java index 72f452019ad..a2d2024d381 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/RegisterApplicationMasterRequestPBImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/RegisterApplicationMasterRequestPBImpl.java @@ -22,9 +22,6 @@ package org.apache.hadoop.yarn.api.protocolrecords.impl.pb; import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.classification.InterfaceStability.Unstable; import org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterRequest; -import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; -import org.apache.hadoop.yarn.api.records.impl.pb.ApplicationAttemptIdPBImpl; -import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationAttemptIdProto; import org.apache.hadoop.yarn.proto.YarnServiceProtos.RegisterApplicationMasterRequestProto; import org.apache.hadoop.yarn.proto.YarnServiceProtos.RegisterApplicationMasterRequestProtoOrBuilder; @@ -35,9 +32,6 @@ public class RegisterApplicationMasterRequestPBImpl extends RegisterApplicationM RegisterApplicationMasterRequestProto.Builder builder = null; boolean viaProto = false; - private ApplicationAttemptId applicationAttemptId = null; - - public RegisterApplicationMasterRequestPBImpl() { builder = RegisterApplicationMasterRequestProto.newBuilder(); } @@ -75,9 +69,6 @@ public class RegisterApplicationMasterRequestPBImpl extends RegisterApplicationM } private void mergeLocalToBuilder() { - if (this.applicationAttemptId != null && !((ApplicationAttemptIdPBImpl)this.applicationAttemptId).getProto().equals(builder.getApplicationAttemptId())) { - builder.setApplicationAttemptId(convertToProtoFormat(this.applicationAttemptId)); - } } private void mergeLocalToProto() { @@ -94,28 +85,6 @@ public class RegisterApplicationMasterRequestPBImpl extends RegisterApplicationM } viaProto = false; } - - - @Override - public ApplicationAttemptId getApplicationAttemptId() { - RegisterApplicationMasterRequestProtoOrBuilder p = viaProto ? proto : builder; - if (this.applicationAttemptId != null) { - return this.applicationAttemptId; - } - if (!p.hasApplicationAttemptId()) { - return null; - } - this.applicationAttemptId = convertFromProtoFormat(p.getApplicationAttemptId()); - return this.applicationAttemptId; - } - - @Override - public void setApplicationAttemptId(ApplicationAttemptId applicationMaster) { - maybeInitBuilder(); - if (applicationMaster == null) - builder.clearApplicationAttemptId(); - this.applicationAttemptId = applicationMaster; - } @Override public String getHost() { @@ -126,6 +95,10 @@ public class RegisterApplicationMasterRequestPBImpl extends RegisterApplicationM @Override public void setHost(String host) { maybeInitBuilder(); + if (host == null) { + builder.clearHost(); + return; + } builder.setHost(host); } @@ -150,15 +123,10 @@ public class RegisterApplicationMasterRequestPBImpl extends RegisterApplicationM @Override public void setTrackingUrl(String url) { maybeInitBuilder(); + if (url == null) { + builder.clearTrackingUrl(); + return; + } builder.setTrackingUrl(url); } - - private ApplicationAttemptIdPBImpl convertFromProtoFormat(ApplicationAttemptIdProto p) { - return new ApplicationAttemptIdPBImpl(p); - } - - private ApplicationAttemptIdProto convertToProtoFormat(ApplicationAttemptId t) { - return ((ApplicationAttemptIdPBImpl)t).getProto(); - } - } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java index b6c59b61d9f..86033f13778 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java @@ -31,7 +31,6 @@ import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.io.Text; import org.apache.hadoop.net.NetUtils; import org.apache.hadoop.security.SecurityUtil; -import org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest; import org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse; import org.apache.hadoop.yarn.api.records.AMCommand; import org.apache.hadoop.yarn.api.records.ApplicationAccessType; @@ -390,20 +389,6 @@ public class BuilderUtils { url.setFile(file); return url; } - - public static AllocateRequest newAllocateRequest( - ApplicationAttemptId applicationAttemptId, int responseID, - float appProgress, List resourceAsk, - List containersToBeReleased) { - AllocateRequest allocateRequest = recordFactory - .newRecordInstance(AllocateRequest.class); - allocateRequest.setApplicationAttemptId(applicationAttemptId); - allocateRequest.setResponseId(responseID); - allocateRequest.setProgress(appProgress); - allocateRequest.setAskList(resourceAsk); - allocateRequest.setReleaseList(containersToBeReleased); - return allocateRequest; - } public static AllocateResponse newAllocateResponse(int responseId, List completedContainers, diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java index b724b038062..9dfdea8100e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java @@ -167,18 +167,16 @@ public class ApplicationMasterService extends AbstractService implements return result; } - private void authorizeRequest(ApplicationAttemptId appAttemptID) + private ApplicationAttemptId authorizeRequest() throws YarnException { - String appAttemptIDStr = appAttemptID.toString(); - UserGroupInformation remoteUgi; try { remoteUgi = UserGroupInformation.getCurrentUser(); } catch (IOException e) { - String msg = "Cannot obtain the user-name for ApplicationAttemptID: " - + appAttemptIDStr + ". Got exception: " - + StringUtils.stringifyException(e); + String msg = + "Cannot obtain the user-name for authorizing ApplicationMaster. " + + "Got exception: " + StringUtils.stringifyException(e); LOG.warn(msg); throw RPCUtil.getRemoteException(msg); } @@ -190,14 +188,15 @@ public class ApplicationMasterService extends AbstractService implements appTokenIdentifier = selectAMRMTokenIdentifier(remoteUgi); if (appTokenIdentifier == null) { tokenFound = false; - message = "No AMRMToken found for " + appAttemptIDStr; + message = "No AMRMToken found for user " + remoteUgi.getUserName(); } else { tokenFound = true; } } catch (IOException e) { tokenFound = false; message = - "Got exception while looking for AMRMToken for " + appAttemptIDStr; + "Got exception while looking for AMRMToken for user " + + remoteUgi.getUserName(); } if (!tokenFound) { @@ -205,15 +204,7 @@ public class ApplicationMasterService extends AbstractService implements throw RPCUtil.getRemoteException(message); } - ApplicationAttemptId remoteApplicationAttemptId = - appTokenIdentifier.getApplicationAttemptId(); - if (!remoteApplicationAttemptId.equals(appAttemptID)) { - String msg = "Unauthorized request from ApplicationMaster. " - + "Expected ApplicationAttemptID: " + remoteApplicationAttemptId - + " Found: " + appAttemptIDStr; - LOG.warn(msg); - throw RPCUtil.getRemoteException(msg); - } + return appTokenIdentifier.getApplicationAttemptId(); } @Override @@ -221,9 +212,7 @@ public class ApplicationMasterService extends AbstractService implements RegisterApplicationMasterRequest request) throws YarnException, IOException { - ApplicationAttemptId applicationAttemptId = request - .getApplicationAttemptId(); - authorizeRequest(applicationAttemptId); + ApplicationAttemptId applicationAttemptId = authorizeRequest(); ApplicationId appID = applicationAttemptId.getApplicationId(); AllocateResponse lastResponse = responseMap.get(applicationAttemptId); @@ -293,9 +282,7 @@ public class ApplicationMasterService extends AbstractService implements FinishApplicationMasterRequest request) throws YarnException, IOException { - ApplicationAttemptId applicationAttemptId = request - .getApplicationAttemptId(); - authorizeRequest(applicationAttemptId); + ApplicationAttemptId applicationAttemptId = authorizeRequest(); AllocateResponse lastResponse = responseMap.get(applicationAttemptId); if (lastResponse == null) { @@ -343,8 +330,7 @@ public class ApplicationMasterService extends AbstractService implements public AllocateResponse allocate(AllocateRequest request) throws YarnException, IOException { - ApplicationAttemptId appAttemptId = request.getApplicationAttemptId(); - authorizeRequest(appAttemptId); + ApplicationAttemptId appAttemptId = authorizeRequest(); this.amLivelinessMonitor.receivedPing(appAttemptId); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockAM.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockAM.java index 91caaa48a1a..88e094d2af9 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockAM.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockAM.java @@ -96,7 +96,6 @@ public class MockAM { responseId = 0; final RegisterApplicationMasterRequest req = Records.newRecord(RegisterApplicationMasterRequest.class); - req.setApplicationAttemptId(attemptId); req.setHost(""); req.setRpcPort(1); req.setTrackingUrl(""); @@ -174,8 +173,9 @@ public class MockAM { public AllocateResponse allocate( List resourceRequest, List releases) throws Exception { - final AllocateRequest req = AllocateRequest.newInstance(attemptId, - ++responseId, 0F, resourceRequest, releases, null); + final AllocateRequest req = + AllocateRequest.newInstance(++responseId, 0F, resourceRequest, + releases, null); UserGroupInformation ugi = UserGroupInformation.createRemoteUser(attemptId.toString()); Token token = @@ -197,11 +197,8 @@ public class MockAM { public void unregisterAppAttempt() throws Exception { waitForState(RMAppAttemptState.RUNNING); final FinishApplicationMasterRequest req = - Records.newRecord(FinishApplicationMasterRequest.class); - req.setAppAttemptId(attemptId); - req.setDiagnostics(""); - req.setFinalApplicationStatus(FinalApplicationStatus.SUCCEEDED); - req.setTrackingUrl(""); + FinishApplicationMasterRequest.newInstance( + FinalApplicationStatus.SUCCEEDED, "", ""); UserGroupInformation ugi = UserGroupInformation.createRemoteUser(attemptId.toString()); Token token = diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAMAuthorization.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAMAuthorization.java index 642b99596a2..7bae0c173fd 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAMAuthorization.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAMAuthorization.java @@ -52,8 +52,8 @@ import org.apache.hadoop.yarn.ipc.YarnRPC; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptState; -import org.apache.hadoop.yarn.server.utils.BuilderUtils; import org.apache.hadoop.yarn.util.Records; +import org.junit.After; import org.junit.Assert; import org.junit.Test; import org.junit.runner.RunWith; @@ -66,6 +66,7 @@ public class TestAMAuthorization { private static final Log LOG = LogFactory.getLog(TestAMAuthorization.class); private final Configuration conf; + private MockRM rm; @Parameters public static Collection configs() { @@ -82,6 +83,13 @@ public class TestAMAuthorization { UserGroupInformation.setConfiguration(conf); } + @After + public void tearDown() { + if (rm != null) { + rm.stop(); + } + } + public static final class MyContainerManager implements ContainerManagementProtocol { public ByteBuffer containerTokens; @@ -139,7 +147,7 @@ public class TestAMAuthorization { @Test public void testAuthorizedAccess() throws Exception { MyContainerManager containerManager = new MyContainerManager(); - final MockRM rm = + rm = new MockRMWithAMS(conf, containerManager); rm.start(); @@ -183,7 +191,6 @@ public class TestAMAuthorization { RegisterApplicationMasterRequest request = Records .newRecord(RegisterApplicationMasterRequest.class); - request.setApplicationAttemptId(applicationAttemptId); RegisterApplicationMasterResponse response = client.registerApplicationMaster(request); Assert.assertNotNull(response.getClientToAMTokenMasterKey()); @@ -193,14 +200,12 @@ public class TestAMAuthorization { } Assert.assertEquals("Register response has bad ACLs", "*", response.getApplicationACLs().get(ApplicationAccessType.VIEW_APP)); - - rm.stop(); } @Test public void testUnauthorizedAccess() throws Exception { MyContainerManager containerManager = new MyContainerManager(); - MockRM rm = new MockRMWithAMS(conf, containerManager); + rm = new MockRMWithAMS(conf, containerManager); rm.start(); MockNM nm1 = rm.registerNode("localhost:1234", 5120); @@ -242,7 +247,6 @@ public class TestAMAuthorization { RegisterApplicationMasterRequest request = Records .newRecord(RegisterApplicationMasterRequest.class); - request.setApplicationAttemptId(applicationAttemptId); try { client.registerApplicationMaster(request); Assert.fail("Should fail with authorization error"); @@ -260,37 +264,8 @@ public class TestAMAuthorization { + "Available:" + availableAuthMethods)); } - // Now try to validate invalid authorization. - Credentials credentials = containerManager.getContainerCredentials(); - currentUser.addCredentials(credentials); - - // Create a client to the RM. - client = currentUser - .doAs(new PrivilegedAction() { - @Override - public ApplicationMasterProtocol run() { - return (ApplicationMasterProtocol) rpc.getProxy(ApplicationMasterProtocol.class, - serviceAddr, conf); - } - }); - - request = - Records.newRecord(RegisterApplicationMasterRequest.class); - ApplicationAttemptId otherAppAttemptId = BuilderUtils - .newApplicationAttemptId(applicationAttemptId.getApplicationId(), 42); - request.setApplicationAttemptId(otherAppAttemptId); - try { - client.registerApplicationMaster(request); - Assert.fail("Should fail with authorization error"); - } catch (YarnException e) { - Assert.assertTrue(e.getMessage().contains( - "Unauthorized request from ApplicationMaster. " - + "Expected ApplicationAttemptID: " - + applicationAttemptId.toString() + " Found: " - + otherAppAttemptId.toString())); - } finally { - rm.stop(); - } + // TODO: Add validation of invalid authorization when there's more data in + // the AMRMToken } private void waitForLaunchedState(RMAppAttempt attempt) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMRMRPCNodeUpdates.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMRMRPCNodeUpdates.java index f4d3901901a..1b2a14b52ff 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMRMRPCNodeUpdates.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMRMRPCNodeUpdates.java @@ -92,8 +92,8 @@ public class TestAMRMRPCNodeUpdates { dispatcher.await(); } - private AllocateResponse allocate(final AllocateRequest req) throws Exception { - ApplicationAttemptId attemptId = req.getApplicationAttemptId(); + private AllocateResponse allocate(final ApplicationAttemptId attemptId, + final AllocateRequest req) throws Exception { UserGroupInformation ugi = UserGroupInformation.createRemoteUser(attemptId.toString()); Token token = @@ -128,18 +128,20 @@ public class TestAMRMRPCNodeUpdates { am1.registerAppAttempt(); // allocate request returns no updated node - AllocateRequest allocateRequest1 = AllocateRequest.newInstance(attempt1 - .getAppAttemptId(), 0, 0F, null, null, null); - AllocateResponse response1 = allocate(allocateRequest1); + AllocateRequest allocateRequest1 = + AllocateRequest.newInstance(0, 0F, null, null, null); + AllocateResponse response1 = + allocate(attempt1.getAppAttemptId(), allocateRequest1); List updatedNodes = response1.getUpdatedNodes(); Assert.assertEquals(0, updatedNodes.size()); syncNodeHeartbeat(nm4, false); // allocate request returns updated node - allocateRequest1 = AllocateRequest.newInstance(attempt1 - .getAppAttemptId(), response1.getResponseId(), 0F, null, null, null); - response1 = allocate(allocateRequest1); + allocateRequest1 = + AllocateRequest.newInstance(response1.getResponseId(), 0F, null, null, + null); + response1 = allocate(attempt1.getAppAttemptId(), allocateRequest1); updatedNodes = response1.getUpdatedNodes(); Assert.assertEquals(1, updatedNodes.size()); NodeReport nr = updatedNodes.iterator().next(); @@ -147,7 +149,7 @@ public class TestAMRMRPCNodeUpdates { Assert.assertEquals(NodeState.UNHEALTHY, nr.getNodeState()); // resending the allocate request returns the same result - response1 = allocate(allocateRequest1); + response1 = allocate(attempt1.getAppAttemptId(), allocateRequest1); updatedNodes = response1.getUpdatedNodes(); Assert.assertEquals(1, updatedNodes.size()); nr = updatedNodes.iterator().next(); @@ -157,9 +159,10 @@ public class TestAMRMRPCNodeUpdates { syncNodeLost(nm3); // subsequent allocate request returns delta - allocateRequest1 = AllocateRequest.newInstance(attempt1 - .getAppAttemptId(), response1.getResponseId(), 0F, null, null, null); - response1 = allocate(allocateRequest1); + allocateRequest1 = + AllocateRequest.newInstance(response1.getResponseId(), 0F, null, null, + null); + response1 = allocate(attempt1.getAppAttemptId(), allocateRequest1); updatedNodes = response1.getUpdatedNodes(); Assert.assertEquals(1, updatedNodes.size()); nr = updatedNodes.iterator().next(); @@ -177,27 +180,30 @@ public class TestAMRMRPCNodeUpdates { am2.registerAppAttempt(); // allocate request returns no updated node - AllocateRequest allocateRequest2 = AllocateRequest.newInstance(attempt2 - .getAppAttemptId(), 0, 0F, null, null, null); - AllocateResponse response2 = allocate(allocateRequest2); + AllocateRequest allocateRequest2 = + AllocateRequest.newInstance(0, 0F, null, null, null); + AllocateResponse response2 = + allocate(attempt2.getAppAttemptId(), allocateRequest2); updatedNodes = response2.getUpdatedNodes(); Assert.assertEquals(0, updatedNodes.size()); syncNodeHeartbeat(nm4, true); // both AM's should get delta updated nodes - allocateRequest1 = AllocateRequest.newInstance(attempt1 - .getAppAttemptId(), response1.getResponseId(), 0F, null, null, null); - response1 = allocate(allocateRequest1); + allocateRequest1 = + AllocateRequest.newInstance(response1.getResponseId(), 0F, null, null, + null); + response1 = allocate(attempt1.getAppAttemptId(), allocateRequest1); updatedNodes = response1.getUpdatedNodes(); Assert.assertEquals(1, updatedNodes.size()); nr = updatedNodes.iterator().next(); Assert.assertEquals(nm4.getNodeId(), nr.getNodeId()); Assert.assertEquals(NodeState.RUNNING, nr.getNodeState()); - allocateRequest2 = AllocateRequest.newInstance(attempt2 - .getAppAttemptId(), response2.getResponseId(), 0F, null, null, null); - response2 = allocate(allocateRequest2); + allocateRequest2 = + AllocateRequest.newInstance(response2.getResponseId(), 0F, null, null, + null); + response2 = allocate(attempt2.getAppAttemptId(), allocateRequest2); updatedNodes = response2.getUpdatedNodes(); Assert.assertEquals(1, updatedNodes.size()); nr = updatedNodes.iterator().next(); @@ -205,9 +211,10 @@ public class TestAMRMRPCNodeUpdates { Assert.assertEquals(NodeState.RUNNING, nr.getNodeState()); // subsequent allocate calls should return no updated nodes - allocateRequest2 = AllocateRequest.newInstance(attempt2 - .getAppAttemptId(), response2.getResponseId(), 0F, null, null, null); - response2 = allocate(allocateRequest2); + allocateRequest2 = + AllocateRequest.newInstance(response2.getResponseId(), 0F, null, null, + null); + response2 = allocate(attempt2.getAppAttemptId(), allocateRequest2); updatedNodes = response2.getUpdatedNodes(); Assert.assertEquals(0, updatedNodes.size()); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMRMRPCResponseId.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMRMRPCResponseId.java index 860881a9911..dc34461a6d2 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMRMRPCResponseId.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMRMRPCResponseId.java @@ -57,8 +57,8 @@ public class TestAMRMRPCResponseId { } } - private AllocateResponse allocate(final AllocateRequest req) throws Exception { - ApplicationAttemptId attemptId = req.getApplicationAttemptId(); + private AllocateResponse allocate(ApplicationAttemptId attemptId, + final AllocateRequest req) throws Exception { UserGroupInformation ugi = UserGroupInformation.createRemoteUser(attemptId.toString()); org.apache.hadoop.security.token.Token token = @@ -88,25 +88,26 @@ public class TestAMRMRPCResponseId { am.registerAppAttempt(); - AllocateRequest allocateRequest = AllocateRequest.newInstance(attempt - .getAppAttemptId(), 0, 0F, null, null, null); + AllocateRequest allocateRequest = + AllocateRequest.newInstance(0, 0F, null, null, null); - AllocateResponse response = allocate(allocateRequest); + AllocateResponse response = + allocate(attempt.getAppAttemptId(), allocateRequest); Assert.assertEquals(1, response.getResponseId()); Assert.assertTrue(response.getAMCommand() == null); - allocateRequest = AllocateRequest.newInstance(attempt - .getAppAttemptId(), response.getResponseId(), 0F, null, null, null); + allocateRequest = + AllocateRequest.newInstance(response.getResponseId(), 0F, null, null, + null); - response = allocate(allocateRequest); + response = allocate(attempt.getAppAttemptId(), allocateRequest); Assert.assertEquals(2, response.getResponseId()); /* try resending */ - response = allocate(allocateRequest); + response = allocate(attempt.getAppAttemptId(), allocateRequest); Assert.assertEquals(2, response.getResponseId()); /** try sending old request again **/ - allocateRequest = AllocateRequest.newInstance(attempt - .getAppAttemptId(), 0, 0F, null, null, null); - response = allocate(allocateRequest); + allocateRequest = AllocateRequest.newInstance(0, 0F, null, null, null); + response = allocate(attempt.getAppAttemptId(), allocateRequest); Assert.assertTrue(response.getAMCommand() == AMCommand.AM_RESYNC); } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulerUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulerUtils.java index e8a65f2bbbb..6cfb387f0eb 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulerUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulerUtils.java @@ -310,7 +310,6 @@ public class TestSchedulerUtils { RegisterApplicationMasterRequest request = Records .newRecord(RegisterApplicationMasterRequest.class); - request.setApplicationAttemptId(applicationAttemptId); client.registerApplicationMaster(request); ResourceBlacklistRequest blacklistRequest = @@ -318,8 +317,7 @@ public class TestSchedulerUtils { Collections.singletonList(ResourceRequest.ANY), null); AllocateRequest allocateRequest = - AllocateRequest.newInstance(applicationAttemptId, 0, 0.0f, null, null, - blacklistRequest); + AllocateRequest.newInstance(0, 0.0f, null, null, blacklistRequest); boolean error = false; try { client.allocate(allocateRequest); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestAMRMTokens.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestAMRMTokens.java index 3829d66b077..15c2d743796 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestAMRMTokens.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestAMRMTokens.java @@ -43,7 +43,6 @@ import org.apache.hadoop.yarn.server.resourcemanager.TestAMAuthorization.MockRMW import org.apache.hadoop.yarn.server.resourcemanager.TestAMAuthorization.MyContainerManager; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt; -import org.apache.hadoop.yarn.server.utils.BuilderUtils; import org.apache.hadoop.yarn.util.Records; import org.junit.Assert; import org.junit.Test; @@ -118,12 +117,10 @@ public class TestAMRMTokens { RegisterApplicationMasterRequest request = Records.newRecord(RegisterApplicationMasterRequest.class); - request.setApplicationAttemptId(applicationAttemptId); rmClient.registerApplicationMaster(request); FinishApplicationMasterRequest finishAMRequest = Records.newRecord(FinishApplicationMasterRequest.class); - finishAMRequest.setAppAttemptId(applicationAttemptId); finishAMRequest .setFinalApplicationStatus(FinalApplicationStatus.SUCCEEDED); finishAMRequest.setDiagnostics("diagnostics"); @@ -134,11 +131,8 @@ public class TestAMRMTokens { // exception. rpc.stopProxy(rmClient, conf); // To avoid using cached client rmClient = createRMClient(rm, conf, rpc, currentUser); - request.setApplicationAttemptId(BuilderUtils.newApplicationAttemptId( - BuilderUtils.newApplicationId(12345, 78), 987)); AllocateRequest allocateRequest = Records.newRecord(AllocateRequest.class); - allocateRequest.setApplicationAttemptId(applicationAttemptId); try { rmClient.allocate(allocateRequest); Assert.fail("You got to be kidding me! " @@ -206,13 +200,11 @@ public class TestAMRMTokens { RegisterApplicationMasterRequest request = Records.newRecord(RegisterApplicationMasterRequest.class); - request.setApplicationAttemptId(applicationAttemptId); rmClient.registerApplicationMaster(request); // One allocate call. AllocateRequest allocateRequest = Records.newRecord(AllocateRequest.class); - allocateRequest.setApplicationAttemptId(applicationAttemptId); Assert.assertTrue( rmClient.allocate(allocateRequest).getAMCommand() == null); @@ -229,7 +221,6 @@ public class TestAMRMTokens { rpc.stopProxy(rmClient, conf); // To avoid using cached client rmClient = createRMClient(rm, conf, rpc, currentUser); allocateRequest = Records.newRecord(AllocateRequest.class); - allocateRequest.setApplicationAttemptId(applicationAttemptId); Assert.assertTrue( rmClient.allocate(allocateRequest).getAMCommand() == null); } finally {