YARN-441. Removed unused utility methods for collections from two API records. Contributed by Xuan Gong.

MAPREDUCE-5163. Update MR App to not use API utility methods for collections after YARN-441. Contributed by Xuan Gong.


git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1469657 13f79535-47bb-0310-9956-ffa450edef68

hadoop-mapreduce-project/CHANGES.txt

@@ -320,6 +320,9 @@ Release 2.0.5-beta - UNRELEASED
     MAPREDUCE-4932. mapreduce.job#getTaskCompletionEvents incompatible with 
     Hadoop 1. (rkanter via tucu)
 
+    MAPREDUCE-5163. Update MR App to not use API utility methods for collections
+    after YARN-441. (Xuan Gong via vinodkv)
+
 Release 2.0.4-alpha - UNRELEASED
 
   INCOMPATIBLE CHANGES

hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/launcher/ContainerLauncherImpl.java

@@ -157,8 +157,9 @@ public class ContainerLauncherImpl extends AbstractService implements
         startRequest.setContainer(event.getAllocatedContainer());
         StartContainerResponse response = proxy.startContainer(startRequest);
 
-        ByteBuffer portInfo = response
+        ByteBuffer portInfo =
-          .getServiceResponse(ShuffleHandler.MAPREDUCE_SHUFFLE_SERVICEID);
+            response.getAllServiceResponse().get(
+                ShuffleHandler.MAPREDUCE_SHUFFLE_SERVICEID);
         int port = -1;
         if(portInfo != null) {
           port = ShuffleHandler.deserializeMetaData(portInfo);

hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/launcher/TestContainerLauncherImpl.java

@@ -26,7 +26,11 @@ import static org.mockito.Mockito.when;
 import static org.mockito.Mockito.atLeast;
 import org.mockito.ArgumentCaptor;
 
+import java.io.IOException;
 import java.net.InetSocketAddress;
+import java.nio.ByteBuffer;
+import java.util.HashMap;
+import java.util.Map;
 import java.util.concurrent.BrokenBarrierException;
 import java.util.concurrent.CyclicBarrier;
 
@@ -58,6 +62,7 @@ import org.apache.hadoop.yarn.factories.RecordFactory;
 import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
 import org.apache.hadoop.yarn.ipc.YarnRPC;
 import org.apache.hadoop.yarn.util.BuilderUtils;
+import org.junit.Before;
 import org.junit.Test;
 
 public class TestContainerLauncherImpl {
@@ -65,6 +70,15 @@ public class TestContainerLauncherImpl {
   private static final RecordFactory recordFactory =
     RecordFactoryProvider.getRecordFactory(null);
 
+  private Map<String, ByteBuffer> serviceResponse =
+      new HashMap<String, ByteBuffer>();
+
+  @Before
+  public void setup() throws IOException {
+    serviceResponse.clear();
+    serviceResponse.put(ShuffleHandler.MAPREDUCE_SHUFFLE_SERVICEID,
+        ShuffleHandler.serializeMetaData(80));
+  }
   
   private static class ContainerLauncherImplUnderTest extends 
     ContainerLauncherImpl {
@@ -145,8 +159,7 @@ public class TestContainerLauncherImpl {
       String cmAddress = "127.0.0.1:8000";
       StartContainerResponse startResp = 
         recordFactory.newRecordInstance(StartContainerResponse.class);
-      startResp.setServiceResponse(ShuffleHandler.MAPREDUCE_SHUFFLE_SERVICEID, 
+      startResp.setAllServiceResponse(serviceResponse);
-          ShuffleHandler.serializeMetaData(80));
       
 
       LOG.info("inserting launch event");
@@ -210,8 +223,7 @@ public class TestContainerLauncherImpl {
       String cmAddress = "127.0.0.1:8000";
       StartContainerResponse startResp = 
         recordFactory.newRecordInstance(StartContainerResponse.class);
-      startResp.setServiceResponse(ShuffleHandler.MAPREDUCE_SHUFFLE_SERVICEID, 
+      startResp.setAllServiceResponse(serviceResponse);
-          ShuffleHandler.serializeMetaData(80));
 
       LOG.info("inserting cleanup event");
       ContainerLauncherEvent mockCleanupEvent = 
@@ -275,8 +287,7 @@ public class TestContainerLauncherImpl {
       String cmAddress = "127.0.0.1:8000";
       StartContainerResponse startResp =
         recordFactory.newRecordInstance(StartContainerResponse.class);
-      startResp.setServiceResponse(ShuffleHandler.MAPREDUCE_SHUFFLE_SERVICEID,
+      startResp.setAllServiceResponse(serviceResponse);
-          ShuffleHandler.serializeMetaData(80));
 
       LOG.info("inserting launch event");
       ContainerRemoteLaunchEvent mockLaunchEvent =
@@ -333,8 +344,7 @@ public class TestContainerLauncherImpl {
       String cmAddress = "127.0.0.1:8000";
       StartContainerResponse startResp = 
         recordFactory.newRecordInstance(StartContainerResponse.class);
-      startResp.setServiceResponse(ShuffleHandler.MAPREDUCE_SHUFFLE_SERVICEID, 
+      startResp.setAllServiceResponse(serviceResponse);
-          ShuffleHandler.serializeMetaData(80));
       
      
       LOG.info("inserting launch event");

hadoop-yarn-project/CHANGES.txt

@@ -85,6 +85,9 @@ Release 2.0.5-beta - UNRELEASED
     YARN-444. Moved special container exit codes from YarnConfiguration to API
     where they belong. (Sandy Ryza via vinodkv)
 
+    YARN-441. Removed unused utility methods for collections from two API
+    records. (Xuan Gong via vinodkv)
+
   NEW FEATURES
 
     YARN-482. FS: Extend SchedulingMode to intermediate queues. 

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/AllocateRequest.java

@@ -20,10 +20,8 @@ package org.apache.hadoop.yarn.api.protocolrecords;
 
 import java.util.List;
 
-import org.apache.hadoop.classification.InterfaceAudience.Private;
 import org.apache.hadoop.classification.InterfaceAudience.Public;
 import org.apache.hadoop.classification.InterfaceStability.Stable;
-import org.apache.hadoop.classification.InterfaceStability.Unstable;
 import org.apache.hadoop.yarn.api.AMRMProtocol;
 import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
 import org.apache.hadoop.yarn.api.records.Container;
@@ -120,36 +118,16 @@ public interface AllocateRequest {
   @Stable
   List<ResourceRequest> getAskList();
   
-  @Private
-  @Unstable
-  ResourceRequest getAsk(int index);
-  
-  @Private
-  @Unstable
-  int getAskCount();
-  
   /**
-   * Add list of <code>ResourceRequest</code> to update the 
+   * Set list of <code>ResourceRequest</code> to update the
    * <code>ResourceManager</code> about the application's resource requirements.
-   * @param resourceRequest list of <code>ResourceRequest</code> to update the 
+   * @param resourceRequests list of <code>ResourceRequest</code> to update the 
    *                        <code>ResourceManager</code> about the application's 
    *                        resource requirements
    */
   @Public
   @Stable
-  void addAllAsks(List<ResourceRequest> resourceRequest);
+  void setAskList(List<ResourceRequest> resourceRequests);
-
-  @Private
-  @Unstable
-  void addAsk(ResourceRequest request);
-
-  @Private
-  @Unstable
-  void removeAsk(int index);
-
-  @Private
-  @Unstable
-  void clearAsks();
 
   /**
    * Get the list of <code>ContainerId</code> of containers being 
@@ -161,16 +139,8 @@ public interface AllocateRequest {
   @Stable
   List<ContainerId> getReleaseList();
 
-  @Private
-  @Unstable
-  ContainerId getRelease(int index);
-  
-  @Private
-  @Unstable
-  int getReleaseCount();
-
   /**
-   * Add the list of <code>ContainerId</code> of containers being 
+   * Set the list of <code>ContainerId</code> of containers being
    * released by the <code>ApplicationMaster</code>
    * @param releaseContainers list of <code>ContainerId</code> of 
    *                          containers being released by the <
@@ -178,17 +148,5 @@ public interface AllocateRequest {
    */
   @Public
   @Stable
-  void addAllReleases(List<ContainerId> releaseContainers);
+  void setReleaseList(List<ContainerId> releaseContainers);
-  
-  @Private
-  @Unstable
-  void addRelease(ContainerId container);
-  
-  @Private
-  @Unstable
-  void removeRelease(int index);
-  
-  @Private
-  @Unstable
-  void clearReleases();
 }

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/StartContainerResponse.java

@@ -45,43 +45,11 @@ public interface StartContainerResponse {
   Map<String, ByteBuffer> getAllServiceResponse();
 
   /**
-   * Get the response from a single auxiliary service running on the
+   * Set to the list of auxiliary services which have been started on the
-   * <code>NodeManager</code>
-   * 
-   * @param key The auxiliary service name whose response is desired.
-   * @return The opaque blob <code>ByteBuffer</code> returned by the auxiliary
-   * service.
-   */
-  ByteBuffer getServiceResponse(String key);
-
-  /**
-   * Add to the list of auxiliary services which have been started on the
    * <code>NodeManager</code>. This is done only once when the
    * <code>NodeManager</code> starts up
-   * @param serviceResponse A map from auxiliary service names to the opaque
+   * @param serviceResponses A map from auxiliary service names to the opaque
    * blob <code>ByteBuffer</code>s for that auxiliary service
    */
-  void addAllServiceResponse(Map<String, ByteBuffer> serviceResponse);
+  void setAllServiceResponse(Map<String, ByteBuffer> serviceResponses);
-
-  /**
-   * Add to the list of auxiliary services which have been started on the
-   * <code>NodeManager</code>. This is done only once when the
-   * <code>NodeManager</code> starts up
-   * 
-   * @param key The auxiliary service name
-   * @param value The opaque blob <code>ByteBuffer</code> managed by the
-   * auxiliary service
-   */
-  void setServiceResponse(String key, ByteBuffer value);
-
-  /**
-   * Remove a single auxiliary service from the StartContainerResponse object
-   * @param key The auxiliary service to remove
-   */
-  void removeServiceResponse(String key);
-  
-  /**
-   * Remove all the auxiliary services from the StartContainerResponse object
-   */
-  void clearServiceResponse();
 }

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/AllocateRequestPBImpl.java

@@ -25,7 +25,6 @@ import java.util.List;
 
 import org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest;
 import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
-import org.apache.hadoop.yarn.api.records.Container;
 import org.apache.hadoop.yarn.api.records.ContainerId;
 import org.apache.hadoop.yarn.api.records.ProtoBase;
 import org.apache.hadoop.yarn.api.records.ResourceRequest;
@@ -144,14 +143,13 @@ public class AllocateRequestPBImpl extends ProtoBase<AllocateRequestProto> imple
     return this.ask;
   }
   @Override
-  public ResourceRequest getAsk(int index) {
+  public void setAskList(final List<ResourceRequest> resourceRequests) {
+    if(resourceRequests == null) {
+      return;
+    }
     initAsks();
-    return this.ask.get(index);
+    this.ask.clear();
-  }
+    this.ask.addAll(resourceRequests);
-  @Override
-  public int getAskCount() {
-    initAsks();
-    return this.ask.size();
   }
   
   private void initAsks() {
@@ -167,14 +165,6 @@ public class AllocateRequestPBImpl extends ProtoBase<AllocateRequestProto> imple
     }
   }
   
-  @Override
-  public void addAllAsks(final List<ResourceRequest> ask) {
-    if (ask == null)
-      return;
-    initAsks();
-    this.ask.addAll(ask);
-  }
-  
   private void addAsksToProto() {
     maybeInitBuilder();
     builder.clearAsk();
@@ -209,34 +199,18 @@ public class AllocateRequestPBImpl extends ProtoBase<AllocateRequestProto> imple
     builder.addAllAsk(iterable);
   }
   @Override
-  public void addAsk(ResourceRequest ask) {
-    initAsks();
-    this.ask.add(ask);
-  }
-  @Override
-  public void removeAsk(int index) {
-    initAsks();
-    this.ask.remove(index);
-  }
-  @Override
-  public void clearAsks() {
-    initAsks();
-    this.ask.clear();
-  }
-  @Override
   public List<ContainerId> getReleaseList() {
     initReleases();
     return this.release;
   }
   @Override
-  public ContainerId getRelease(int index) {
+  public void setReleaseList(List<ContainerId> releaseContainers) {
+    if(releaseContainers == null) {
+      return;
+    }
     initReleases();
-    return this.release.get(index);
+    this.release.clear();
-  }
+    this.release.addAll(releaseContainers);
-  @Override
-  public int getReleaseCount() {
-    initReleases();
-    return this.release.size();
   }
   
   private void initReleases() {
@@ -252,14 +226,6 @@ public class AllocateRequestPBImpl extends ProtoBase<AllocateRequestProto> imple
     }
   }
   
-  @Override
-  public void addAllReleases(final List<ContainerId> release) {
-    if (release == null)
-      return;
-    initReleases();
-    this.release.addAll(release);
-  }
-  
   private void addReleasesToProto() {
     maybeInitBuilder();
     builder.clearRelease();
@@ -293,21 +259,6 @@ public class AllocateRequestPBImpl extends ProtoBase<AllocateRequestProto> imple
     };
     builder.addAllRelease(iterable);
   }
-  @Override
-  public void addRelease(ContainerId release) {
-    initReleases();
-    this.release.add(release);
-  }
-  @Override
-  public void removeRelease(int index) {
-    initReleases();
-    this.release.remove(index);
-  }
-  @Override
-  public void clearReleases() {
-    initReleases();
-    this.release.clear();
-  }
 
   private ApplicationAttemptIdPBImpl convertFromProtoFormat(ApplicationAttemptIdProto p) {
     return new ApplicationAttemptIdPBImpl(p);

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/StartContainerResponsePBImpl.java

@@ -84,9 +84,14 @@ public class StartContainerResponsePBImpl extends ProtoBase<StartContainerRespon
     return this.serviceResponse;
   }
   @Override
-  public synchronized ByteBuffer getServiceResponse(String key) {
+  public synchronized void setAllServiceResponse(
+      Map<String, ByteBuffer> serviceResponses) {
+    if(serviceResponses == null) {
+      return;
+    }
     initServiceResponse();
-    return this.serviceResponse.get(key);
+    this.serviceResponse.clear();
+    this.serviceResponse.putAll(serviceResponses);
   }
   
   private synchronized void initServiceResponse() {
@@ -102,14 +107,6 @@ public class StartContainerResponsePBImpl extends ProtoBase<StartContainerRespon
     }
   }
   
-  @Override
-  public synchronized void addAllServiceResponse(final Map<String, ByteBuffer> serviceResponse) {
-    if (serviceResponse == null)
-      return;
-    initServiceResponse();
-    this.serviceResponse.putAll(serviceResponse);
-  }
-  
   private synchronized void addServiceResponseToProto() {
     maybeInitBuilder();
     builder.clearServiceResponse();
@@ -143,19 +140,4 @@ public class StartContainerResponsePBImpl extends ProtoBase<StartContainerRespon
     };
     builder.addAllServiceResponse(iterable);
   }
-  @Override
-  public synchronized void setServiceResponse(String key, ByteBuffer val) {
-    initServiceResponse();
-    this.serviceResponse.put(key, val);
-  }
-  @Override
-  public synchronized void removeServiceResponse(String key) {
-    initServiceResponse();
-    this.serviceResponse.remove(key);
-  }
-  @Override
-  public synchronized void clearServiceResponse() {
-    initServiceResponse();
-    this.serviceResponse.clear();
-  }
 }  

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/BuilderUtils.java

@@ -393,8 +393,8 @@ public class BuilderUtils {
     allocateRequest.setApplicationAttemptId(applicationAttemptId);
     allocateRequest.setResponseId(responseID);
     allocateRequest.setProgress(appProgress);
-    allocateRequest.addAllAsks(resourceAsk);
+    allocateRequest.setAskList(resourceAsk);
-    allocateRequest.addAllReleases(containersToBeReleased);
+    allocateRequest.setReleaseList(containersToBeReleased);
     return allocateRequest;
   }
   

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java

@@ -468,7 +468,7 @@ public class ContainerManagerImpl extends CompositeService implements
 
     StartContainerResponse response =
         recordFactory.newRecordInstance(StartContainerResponse.class);
-    response.addAllServiceResponse(auxiliaryServices.getMeta());
+    response.setAllServiceResponse(auxiliaryServices.getMeta());
     // TODO launchedContainer misplaced -> doesn't necessarily mean a container
     // launch. A finished Application will not launch containers.
     metrics.launchedContainer();

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestContainerManagerSecurity.java

@@ -497,7 +497,7 @@ public class TestContainerManagerSecurity {
         .getAllocatedContainers();
 
     // Modify ask to request no more.
-    allocateRequest.clearAsks();
+    allocateRequest.setAskList(new ArrayList<ResourceRequest>());
 
     int waitCounter = 0;
     while ((allocatedContainers == null || allocatedContainers.size() == 0)