diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index 999db136899..b715e97ac2d 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -450,6 +450,9 @@ Release 2.1.0-beta - 2013-07-02 MAPREDUCE-5325. MR changes related to YARN-727. ClientRMProtocol.getAllApplications should accept ApplicationType as a parameter. (Xuan Gong via hitesh) + MAPREDUCE-5412. Update MR app to use multiple containers API of + ContainerManager after YARN-926. (Jian He via vinodkv) + BREAKDOWN OF HADOOP-8562 SUBTASKS MAPREDUCE-4739. Some MapReduce tests fail to find winutils. diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/launcher/ContainerLauncherImpl.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/launcher/ContainerLauncherImpl.java index 28508a92ed2..666f757b540 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/launcher/ContainerLauncherImpl.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/launcher/ContainerLauncherImpl.java @@ -20,7 +20,9 @@ package org.apache.hadoop.mapreduce.v2.app.launcher; import java.io.IOException; import java.nio.ByteBuffer; +import java.util.ArrayList; import java.util.HashSet; +import java.util.List; import java.util.Set; import java.util.concurrent.BlockingQueue; import java.util.concurrent.ConcurrentHashMap; @@ -44,14 +46,15 @@ import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptEventType; import org.apache.hadoop.service.AbstractService; import org.apache.hadoop.util.StringUtils; import org.apache.hadoop.yarn.api.protocolrecords.StartContainerRequest; -import org.apache.hadoop.yarn.api.protocolrecords.StartContainerResponse; -import org.apache.hadoop.yarn.api.protocolrecords.StopContainerRequest; +import org.apache.hadoop.yarn.api.protocolrecords.StartContainersRequest; +import org.apache.hadoop.yarn.api.protocolrecords.StartContainersResponse; +import org.apache.hadoop.yarn.api.protocolrecords.StopContainersRequest; +import org.apache.hadoop.yarn.api.protocolrecords.StopContainersResponse; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.ContainerLaunchContext; import org.apache.hadoop.yarn.client.api.impl.ContainerManagementProtocolProxy; import org.apache.hadoop.yarn.client.api.impl.ContainerManagementProtocolProxy.ContainerManagementProtocolProxyData; import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; -import org.apache.hadoop.yarn.util.Records; import com.google.common.util.concurrent.ThreadFactoryBuilder; @@ -139,13 +142,18 @@ public class ContainerLauncherImpl extends AbstractService implements event.getContainerLaunchContext(); // Now launch the actual container - StartContainerRequest startRequest = Records - .newRecord(StartContainerRequest.class); - startRequest.setContainerLaunchContext(containerLaunchContext); - startRequest.setContainerToken(event.getContainerToken()); - StartContainerResponse response = - proxy.getContainerManagementProtocol().startContainer(startRequest); - + StartContainerRequest startRequest = + StartContainerRequest.newInstance(containerLaunchContext, + event.getContainerToken()); + List list = new ArrayList(); + list.add(startRequest); + StartContainersRequest requestList = StartContainersRequest.newInstance(list); + StartContainersResponse response = + proxy.getContainerManagementProtocol().startContainers(requestList); + if (response.getFailedRequests() != null + && response.getFailedRequests().containsKey(containerID)) { + throw response.getFailedRequests().get(containerID).deSerialize(); + } ByteBuffer portInfo = response.getAllServicesMetaData().get( ShuffleHandler.MAPREDUCE_SHUFFLE_SERVICEID); @@ -192,13 +200,17 @@ public class ContainerLauncherImpl extends AbstractService implements proxy = getCMProxy(this.containerMgrAddress, this.containerID); // kill the remote container if already launched - StopContainerRequest stopRequest = Records - .newRecord(StopContainerRequest.class); - stopRequest.setContainerId(this.containerID); - proxy.getContainerManagementProtocol().stopContainer(stopRequest); - + List ids = new ArrayList(); + ids.add(this.containerID); + StopContainersRequest request = StopContainersRequest.newInstance(ids); + StopContainersResponse response = + proxy.getContainerManagementProtocol().stopContainers(request); + if (response.getFailedRequests() != null + && response.getFailedRequests().containsKey(this.containerID)) { + throw response.getFailedRequests().get(this.containerID) + .deSerialize(); + } } catch (Throwable t) { - // ignore the cleanup failure String message = "cleanup failed for container " + this.containerID + " : " diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/launcher/TestContainerLauncher.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/launcher/TestContainerLauncher.java index 563c31b36e3..dd1060c915b 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/launcher/TestContainerLauncher.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/launcher/TestContainerLauncher.java @@ -24,6 +24,8 @@ import java.io.IOException; import java.lang.reflect.UndeclaredThrowableException; import java.net.InetSocketAddress; import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.List; import java.util.Map; import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.atomic.AtomicInteger; @@ -52,12 +54,13 @@ import org.apache.hadoop.mapreduce.v2.app.job.impl.TaskAttemptImpl; import org.apache.hadoop.mapreduce.v2.util.MRBuilderUtils; import org.apache.hadoop.net.NetUtils; import org.apache.hadoop.yarn.api.ContainerManagementProtocol; -import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusRequest; -import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusResponse; +import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusesRequest; +import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusesResponse; import org.apache.hadoop.yarn.api.protocolrecords.StartContainerRequest; -import org.apache.hadoop.yarn.api.protocolrecords.StartContainerResponse; -import org.apache.hadoop.yarn.api.protocolrecords.StopContainerRequest; -import org.apache.hadoop.yarn.api.protocolrecords.StopContainerResponse; +import org.apache.hadoop.yarn.api.protocolrecords.StartContainersRequest; +import org.apache.hadoop.yarn.api.protocolrecords.StartContainersResponse; +import org.apache.hadoop.yarn.api.protocolrecords.StopContainersRequest; +import org.apache.hadoop.yarn.api.protocolrecords.StopContainersResponse; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ContainerId; @@ -393,18 +396,18 @@ public class TestContainerLauncher { private ContainerStatus status = null; @Override - public GetContainerStatusResponse getContainerStatus( - GetContainerStatusRequest request) throws IOException { - GetContainerStatusResponse response = recordFactory - .newRecordInstance(GetContainerStatusResponse.class); - response.setStatus(status); - return response; + public GetContainerStatusesResponse getContainerStatuses( + GetContainerStatusesRequest request) throws IOException { + List statuses = new ArrayList(); + statuses.add(status); + return GetContainerStatusesResponse.newInstance(statuses, null); } @Override - public StartContainerResponse startContainer(StartContainerRequest request) + public StartContainersResponse startContainers(StartContainersRequest requests) throws IOException { + StartContainerRequest request = requests.getStartContainerRequests().get(0); ContainerTokenIdentifier containerTokenIdentifier = MRApp.newContainerTokenIdentifier(request.getContainerToken()); @@ -412,8 +415,8 @@ public class TestContainerLauncher { Assert.assertEquals(MRApp.NM_HOST + ":" + MRApp.NM_PORT, containerTokenIdentifier.getNmHostAddress()); - StartContainerResponse response = recordFactory - .newRecordInstance(StartContainerResponse.class); + StartContainersResponse response = recordFactory + .newRecordInstance(StartContainersResponse.class); status = recordFactory.newRecordInstance(ContainerStatus.class); try { // make the thread sleep to look like its not going to respond @@ -429,7 +432,7 @@ public class TestContainerLauncher { } @Override - public StopContainerResponse stopContainer(StopContainerRequest request) + public StopContainersResponse stopContainers(StopContainersRequest request) throws IOException { Exception e = new Exception("Dummy function", new Exception( "Dummy function cause")); diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/launcher/TestContainerLauncherImpl.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/launcher/TestContainerLauncherImpl.java index dc6ca6a7949..6f21c87707a 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/launcher/TestContainerLauncherImpl.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/launcher/TestContainerLauncherImpl.java @@ -45,12 +45,12 @@ import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptEventType; import org.apache.hadoop.mapreduce.v2.app.launcher.ContainerLauncher.EventType; import org.apache.hadoop.mapreduce.v2.util.MRBuilderUtils; import org.apache.hadoop.yarn.api.ContainerManagementProtocol; -import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusRequest; -import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusResponse; -import org.apache.hadoop.yarn.api.protocolrecords.StartContainerRequest; -import org.apache.hadoop.yarn.api.protocolrecords.StartContainerResponse; -import org.apache.hadoop.yarn.api.protocolrecords.StopContainerRequest; -import org.apache.hadoop.yarn.api.protocolrecords.StopContainerResponse; +import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusesRequest; +import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusesResponse; +import org.apache.hadoop.yarn.api.protocolrecords.StartContainersRequest; +import org.apache.hadoop.yarn.api.protocolrecords.StartContainersResponse; +import org.apache.hadoop.yarn.api.protocolrecords.StopContainersRequest; +import org.apache.hadoop.yarn.api.protocolrecords.StopContainersResponse; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ContainerId; @@ -162,8 +162,8 @@ public class TestContainerLauncherImpl { try { ContainerId contId = makeContainerId(0l, 0, 0, 1); TaskAttemptId taskAttemptId = makeTaskAttemptId(0l, 0, 0, TaskType.MAP, 0); - StartContainerResponse startResp = - recordFactory.newRecordInstance(StartContainerResponse.class); + StartContainersResponse startResp = + recordFactory.newRecordInstance(StartContainersResponse.class); startResp.setAllServicesMetaData(serviceResponse); @@ -176,14 +176,14 @@ public class TestContainerLauncherImpl { .thenReturn(contId); when(mockLaunchEvent.getTaskAttemptID()).thenReturn(taskAttemptId); when(mockLaunchEvent.getContainerMgrAddress()).thenReturn(cmAddress); - when(mockCM.startContainer(any(StartContainerRequest.class))).thenReturn(startResp); + when(mockCM.startContainers(any(StartContainersRequest.class))).thenReturn(startResp); when(mockLaunchEvent.getContainerToken()).thenReturn( createNewContainerToken(contId, cmAddress)); ut.handle(mockLaunchEvent); ut.waitForPoolToIdle(); - verify(mockCM).startContainer(any(StartContainerRequest.class)); + verify(mockCM).startContainers(any(StartContainersRequest.class)); LOG.info("inserting cleanup event"); ContainerLauncherEvent mockCleanupEvent = @@ -198,7 +198,7 @@ public class TestContainerLauncherImpl { ut.waitForPoolToIdle(); - verify(mockCM).stopContainer(any(StopContainerRequest.class)); + verify(mockCM).stopContainers(any(StopContainersRequest.class)); } finally { ut.stop(); } @@ -224,8 +224,8 @@ public class TestContainerLauncherImpl { ContainerId contId = makeContainerId(0l, 0, 0, 1); TaskAttemptId taskAttemptId = makeTaskAttemptId(0l, 0, 0, TaskType.MAP, 0); String cmAddress = "127.0.0.1:8000"; - StartContainerResponse startResp = - recordFactory.newRecordInstance(StartContainerResponse.class); + StartContainersResponse startResp = + recordFactory.newRecordInstance(StartContainersResponse.class); startResp.setAllServicesMetaData(serviceResponse); LOG.info("inserting cleanup event"); @@ -241,7 +241,7 @@ public class TestContainerLauncherImpl { ut.waitForPoolToIdle(); - verify(mockCM, never()).stopContainer(any(StopContainerRequest.class)); + verify(mockCM, never()).stopContainers(any(StopContainersRequest.class)); LOG.info("inserting launch event"); ContainerRemoteLaunchEvent mockLaunchEvent = @@ -252,14 +252,14 @@ public class TestContainerLauncherImpl { .thenReturn(contId); when(mockLaunchEvent.getTaskAttemptID()).thenReturn(taskAttemptId); when(mockLaunchEvent.getContainerMgrAddress()).thenReturn(cmAddress); - when(mockCM.startContainer(any(StartContainerRequest.class))).thenReturn(startResp); + when(mockCM.startContainers(any(StartContainersRequest.class))).thenReturn(startResp); when(mockLaunchEvent.getContainerToken()).thenReturn( createNewContainerToken(contId, cmAddress)); ut.handle(mockLaunchEvent); ut.waitForPoolToIdle(); - verify(mockCM, never()).startContainer(any(StartContainerRequest.class)); + verify(mockCM, never()).startContainers(any(StartContainersRequest.class)); } finally { ut.stop(); } @@ -286,8 +286,8 @@ public class TestContainerLauncherImpl { ContainerId contId = makeContainerId(0l, 0, 0, 1); TaskAttemptId taskAttemptId = makeTaskAttemptId(0l, 0, 0, TaskType.MAP, 0); String cmAddress = "127.0.0.1:8000"; - StartContainerResponse startResp = - recordFactory.newRecordInstance(StartContainerResponse.class); + StartContainersResponse startResp = + recordFactory.newRecordInstance(StartContainersResponse.class); startResp.setAllServicesMetaData(serviceResponse); LOG.info("inserting launch event"); @@ -299,20 +299,20 @@ public class TestContainerLauncherImpl { .thenReturn(contId); when(mockLaunchEvent.getTaskAttemptID()).thenReturn(taskAttemptId); when(mockLaunchEvent.getContainerMgrAddress()).thenReturn(cmAddress); - when(mockCM.startContainer(any(StartContainerRequest.class))).thenReturn(startResp); + when(mockCM.startContainers(any(StartContainersRequest.class))).thenReturn(startResp); when(mockLaunchEvent.getContainerToken()).thenReturn( createNewContainerToken(contId, cmAddress)); ut.handle(mockLaunchEvent); ut.waitForPoolToIdle(); - verify(mockCM).startContainer(any(StartContainerRequest.class)); + verify(mockCM).startContainers(any(StartContainersRequest.class)); // skip cleanup and make sure stop kills the container } finally { ut.stop(); - verify(mockCM).stopContainer(any(StopContainerRequest.class)); + verify(mockCM).stopContainers(any(StopContainersRequest.class)); } } @@ -341,8 +341,8 @@ public class TestContainerLauncherImpl { ContainerId contId = makeContainerId(0l, 0, 0, 1); TaskAttemptId taskAttemptId = makeTaskAttemptId(0l, 0, 0, TaskType.MAP, 0); String cmAddress = "127.0.0.1:8000"; - StartContainerResponse startResp = - recordFactory.newRecordInstance(StartContainerResponse.class); + StartContainersResponse startResp = + recordFactory.newRecordInstance(StartContainersResponse.class); startResp.setAllServicesMetaData(serviceResponse); @@ -415,7 +415,7 @@ public class TestContainerLauncherImpl { this.completeLaunchBarrier = completeLaunchBarrier; } @Override - public StartContainerResponse startContainer(StartContainerRequest request) + public StartContainersResponse startContainers(StartContainersRequest request) throws IOException { try { startLaunchBarrier.await(); @@ -433,16 +433,14 @@ public class TestContainerLauncherImpl { } @Override - public StopContainerResponse stopContainer(StopContainerRequest request) + public StopContainersResponse stopContainers(StopContainersRequest request) throws IOException { - return null; } @Override - public GetContainerStatusResponse getContainerStatus( - GetContainerStatusRequest request) throws IOException { - + public GetContainerStatusesResponse getContainerStatuses( + GetContainerStatusesRequest request) throws IOException { return null; } } diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 7a42e0c0b2c..330a19916fa 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -206,6 +206,9 @@ Release 2.1.0-beta - 2013-07-02 YARN-918. Remove ApplicationAttemptId from RegisterApplicationMasterRequestProto. (vinodkv via acmurthy) + YARN-926. Modified ContainerManagerProtcol APIs to take in requests for + multiple containers. (Jian He via vinodkv) + NEW FEATURES YARN-482. FS: Extend SchedulingMode to intermediate queues. diff --git a/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml b/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml index 056a2f7e0cb..104ef4a5398 100644 --- a/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml +++ b/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml @@ -131,6 +131,11 @@ + + + + + diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ContainerManagementProtocol.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ContainerManagementProtocol.java index de330c57b5e..7aa43dfb83f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ContainerManagementProtocol.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ContainerManagementProtocol.java @@ -22,17 +22,17 @@ import java.io.IOException; import org.apache.hadoop.classification.InterfaceAudience.Public; import org.apache.hadoop.classification.InterfaceStability.Stable; -import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusRequest; -import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusResponse; +import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusesRequest; +import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusesResponse; import org.apache.hadoop.yarn.api.protocolrecords.StartContainerRequest; -import org.apache.hadoop.yarn.api.protocolrecords.StartContainerResponse; -import org.apache.hadoop.yarn.api.protocolrecords.StopContainerRequest; -import org.apache.hadoop.yarn.api.protocolrecords.StopContainerResponse; +import org.apache.hadoop.yarn.api.protocolrecords.StartContainersRequest; +import org.apache.hadoop.yarn.api.protocolrecords.StartContainersResponse; +import org.apache.hadoop.yarn.api.protocolrecords.StopContainersRequest; +import org.apache.hadoop.yarn.api.protocolrecords.StopContainersResponse; import org.apache.hadoop.yarn.api.records.Container; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.ContainerLaunchContext; import org.apache.hadoop.yarn.api.records.ContainerStatus; -import org.apache.hadoop.yarn.exceptions.InvalidContainerException; import org.apache.hadoop.yarn.exceptions.NMNotYetReadyException; import org.apache.hadoop.yarn.exceptions.YarnException; @@ -50,10 +50,12 @@ import org.apache.hadoop.yarn.exceptions.YarnException; @Public @Stable public interface ContainerManagementProtocol { + /** *

- * The ApplicationMaster requests a NodeManager to - * start a {@link Container} allocated to it using this interface. + * The ApplicationMaster provides a list of + * {@link StartContainerRequest}s to a NodeManager to + * start {@link Container}s allocated to it using this interface. *

* *

@@ -65,82 +67,107 @@ public interface ContainerManagementProtocol { *

* *

- * Currently the NodeManager sends an immediate, empty response - * via {@link StartContainerResponse} to signify acceptance of the request and - * throws an exception in case of errors. The ApplicationMaster - * can use {@link #getContainerStatus(GetContainerStatusRequest)} to get - * updated status of the to-be-launched or launched container. + * The NodeManager sends a response via + * {@link StartContainersResponse} which includes a list of + * {@link Container}s of successfully launched {@link Container}s, a + * containerId-to-exception map for each failed {@link StartContainerRequest} in + * which the exception indicates errors from per container and a + * allServicesMetaData map between the names of auxiliary services and their + * corresponding meta-data. Note: None-container-specific exceptions will + * still be thrown by the API method itself. + *

+ *

+ * The ApplicationMaster can use + * {@link #getContainerStatuses(GetContainerStatusesRequest)} to get updated + * statuses of the to-be-launched or launched containers. *

* * @param request - * request to start a container - * @return empty response to indicate acceptance of the request or an - * exception + * request to start a list of containers + * @return response including conatinerIds of all successfully launched + * containers, a containerId-to-exception map for failed requests and + * a allServicesMetaData map. * @throws YarnException * @throws IOException * @throws NMNotYetReadyException * This exception is thrown when NM starts from scratch but has not * yet connected with RM. - * @throws InvalidContainerException - * This exception is thrown when NM is rejecting start-container - * requests for containers allocated by a previous instance of the - * RM */ @Public @Stable - StartContainerResponse startContainer(StartContainerRequest request) + StartContainersResponse startContainers(StartContainersRequest request) throws YarnException, IOException; /** - *

The ApplicationMaster requests a NodeManager - * to stop a {@link Container} allocated to it using this interface. + *

+ * The ApplicationMaster requests a NodeManager to + * stop a list of {@link Container}s allocated to it using this + * interface. *

* - *

The ApplicationMaster sends a - * {@link StopContainerRequest} which includes the {@link ContainerId} of the - * container to be stopped.

+ *

+ * The ApplicationMaster sends a {@link StopContainersRequest} + * which includes the {@link ContainerId}s of the containers to be stopped. + *

* - *

Currently the NodeManager sends an immediate, empty - * response via {@link StopContainerResponse} to signify acceptance of the - * request and throws an exception in case of errors. The - * ApplicationMaster can use - * {@link #getContainerStatus(GetContainerStatusRequest)} to get updated - * status of the container.

+ *

+ * The NodeManager sends a response via + * {@link StopContainersResponse} which includes a list of {@link ContainerId} + * s of successfully stopped containers, a containerId-to-exception map for + * each failed request in which the exception indicates errors from per + * container. Note: None-container-specific exceptions will still be thrown by + * the API method itself. ApplicationMaster can use + * {@link #getContainerStatuses(GetContainerStatusesRequest)} to get updated + * statuses of the containers. + *

* - * @param request request to stop a container - * @return empty response to indicate acceptance of the request - * or an exception + * @param request + * request to stop a list of containers + * @return response which includes a list of containerIds of successfully + * stopped containers, a containerId-to-exception map for failed + * requests. * @throws YarnException * @throws IOException */ @Public @Stable - StopContainerResponse stopContainer(StopContainerRequest request) + StopContainersResponse stopContainers(StopContainersRequest request) throws YarnException, IOException; /** - *

The api used by the ApplicationMaster to request for - * current status of a Container from the - * NodeManager.

+ *

+ * The API used by the ApplicationMaster to request for current + * statuses of Containers from the NodeManager. + *

+ * + *

+ * The ApplicationMaster sends a + * {@link GetContainerStatusesRequest} which includes the {@link ContainerId}s + * of all containers whose statuses are needed. + *

+ * + *

+ * The NodeManager responds with + * {@link GetContainerStatusesResponse} which includes a list of + * {@link ContainerStatus} of the successfully queried containers and a + * containerId-to-exception map for each failed request in which the exception + * indicates errors from per container. Note: None-container-specific + * exceptions will still be thrown by the API method itself. + *

+ * + * @param request + * request to get ContainerStatuses of containers with + * the specified ContainerIds + * @return response containing the list of ContainerStatus of the + * successfully queried containers and a containerId-to-exception map + * for failed requests. * - *

The ApplicationMaster sends a - * {@link GetContainerStatusRequest} which includes the {@link ContainerId} of - * the container whose status is needed.

- * - *

The NodeManager responds with - *{@link GetContainerStatusResponse} which includes the - *{@link ContainerStatus} of the container.

- * - * @param request request to get ContainerStatus of a container - * with the specified ContainerId - * @return response containing the ContainerStatus of the - * container * @throws YarnException * @throws IOException */ @Public @Stable - GetContainerStatusResponse getContainerStatus( - GetContainerStatusRequest request) throws YarnException, + GetContainerStatusesResponse getContainerStatuses( + GetContainerStatusesRequest request) throws YarnException, IOException; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetContainerStatusResponse.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetContainerStatusResponse.java deleted file mode 100644 index acf70b362e4..00000000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetContainerStatusResponse.java +++ /dev/null @@ -1,61 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.yarn.api.protocolrecords; - -import org.apache.hadoop.classification.InterfaceAudience.Private; -import org.apache.hadoop.classification.InterfaceAudience.Public; -import org.apache.hadoop.classification.InterfaceStability.Stable; -import org.apache.hadoop.classification.InterfaceStability.Unstable; -import org.apache.hadoop.yarn.api.ContainerManagementProtocol; -import org.apache.hadoop.yarn.api.records.ContainerStatus; -import org.apache.hadoop.yarn.util.Records; - -/** - *

The response sent by the NodeManager to the - * ApplicationMaster when asked to obtain the status - * of a container.

- * - * @see ContainerManagementProtocol#getContainerStatus(GetContainerStatusRequest) - */ -@Public -@Stable -public abstract class GetContainerStatusResponse { - - @Private - @Unstable - public static GetContainerStatusResponse newInstance( - ContainerStatus containerStatus) { - GetContainerStatusResponse response = - Records.newRecord(GetContainerStatusResponse.class); - response.setStatus(containerStatus); - return response; - } - - /** - * Get the ContainerStatus of the container. - * @return ContainerStatus of the container - */ - @Public - @Stable - public abstract ContainerStatus getStatus(); - - @Private - @Unstable - public abstract void setStatus(ContainerStatus containerStatus); -} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetContainerStatusRequest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetContainerStatusesRequest.java similarity index 52% rename from hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetContainerStatusRequest.java rename to hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetContainerStatusesRequest.java index 6d0f81e7139..f9f77a3af2f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetContainerStatusRequest.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetContainerStatusesRequest.java @@ -18,6 +18,8 @@ package org.apache.hadoop.yarn.api.protocolrecords; +import java.util.List; + import org.apache.hadoop.classification.InterfaceAudience.Public; import org.apache.hadoop.classification.InterfaceStability.Stable; import org.apache.hadoop.yarn.api.ContainerManagementProtocol; @@ -26,41 +28,48 @@ import org.apache.hadoop.yarn.api.records.ContainerStatus; import org.apache.hadoop.yarn.util.Records; /** - *

The request sent by the ApplicationMaster to the - * NodeManager to get {@link ContainerStatus} of a container.

+ *

+ * The request sent by the ApplicationMaster to the + * NodeManager to get {@link ContainerStatus} of requested + * containers. + *

* - * @see ContainerManagementProtocol#getContainerStatus(GetContainerStatusRequest) + * @see ContainerManagementProtocol#getContainerStatuses(GetContainerStatusesRequest) */ @Public @Stable -public abstract class GetContainerStatusRequest { +public abstract class GetContainerStatusesRequest { @Public @Stable - public static GetContainerStatusRequest newInstance(ContainerId containerId) { - GetContainerStatusRequest request = - Records.newRecord(GetContainerStatusRequest.class); - request.setContainerId(containerId); + public static GetContainerStatusesRequest newInstance( + List containerIds) { + GetContainerStatusesRequest request = + Records.newRecord(GetContainerStatusesRequest.class); + request.setContainerIds(containerIds); return request; } /** - * Get the ContainerId of container for which to obtain the - * ContainerStatus. - * @return ContainerId of container for which to obtain the - * ContainerStatus + * Get the list of ContainerIds of containers for which to obtain + * the ContainerStatus. + * + * @return the list of ContainerIds of containers for which to + * obtain the ContainerStatus. */ @Public @Stable - public abstract ContainerId getContainerId(); - + public abstract List getContainerIds(); + /** - * Set the ContainerId of container for which to obtain the - * ContainerStatus - * @param containerId ContainerId of container for which to - * obtain the ContainerStatus + * Set a list of ContainerIds of containers for which to obtain + * the ContainerStatus + * + * @param containerIds + * a list of ContainerIds of containers for which to + * obtain the ContainerStatus */ @Public @Stable - public abstract void setContainerId(ContainerId containerId); + public abstract void setContainerIds(List containerIds); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetContainerStatusesResponse.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetContainerStatusesResponse.java new file mode 100644 index 00000000000..b0a0f0e4882 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetContainerStatusesResponse.java @@ -0,0 +1,91 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.api.protocolrecords; + +import java.util.List; +import java.util.Map; + +import org.apache.hadoop.classification.InterfaceAudience.Private; +import org.apache.hadoop.classification.InterfaceAudience.Public; +import org.apache.hadoop.classification.InterfaceStability.Stable; +import org.apache.hadoop.classification.InterfaceStability.Unstable; +import org.apache.hadoop.yarn.api.ContainerManagementProtocol; +import org.apache.hadoop.yarn.api.records.ContainerId; +import org.apache.hadoop.yarn.api.records.ContainerStatus; +import org.apache.hadoop.yarn.api.records.SerializedException; +import org.apache.hadoop.yarn.util.Records; + +/** + *

+ * The response sent by the NodeManager to the + * ApplicationMaster when asked to obtain the + * ContainerStatus of requested containers. + *

+ * + * @see ContainerManagementProtocol#getContainerStatuses(GetContainerStatusesRequest) + */ +@Public +@Stable +public abstract class GetContainerStatusesResponse { + + @Private + @Unstable + public static GetContainerStatusesResponse newInstance( + List statuses, + Map failedRequests) { + GetContainerStatusesResponse response = + Records.newRecord(GetContainerStatusesResponse.class); + response.setContainerStatuses(statuses); + response.setFailedRequests(failedRequests); + return response; + } + + /** + * Get the ContainerStatuses of the requested containers. + * + * @return ContainerStatuses of the requested containers. + */ + @Public + @Stable + public abstract List getContainerStatuses(); + + /** + * Set the ContainerStatuses of the requested containers. + */ + @Private + @Unstable + public abstract void setContainerStatuses(List statuses); + + /** + * Get the containerId-to-exception map in which the exception indicates error + * from per container for failed requests + */ + @Public + @Stable + public abstract Map getFailedRequests(); + + /** + * Set the containerId-to-exception map in which the exception indicates error + * from per container for failed requests + */ + @Private + @Unstable + public abstract void setFailedRequests( + Map failedContainers); +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/StartContainerRequest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/StartContainerRequest.java index 6e4f44f09f7..1dcefb2e3ea 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/StartContainerRequest.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/StartContainerRequest.java @@ -36,7 +36,7 @@ import org.apache.hadoop.yarn.util.Records; * necessary binaries/jar/shared-objects etc. via the * {@link ContainerLaunchContext}.

* - * @see ContainerManagementProtocol#startContainer(StartContainerRequest) + * @see ContainerManagementProtocol#startContainers(StartContainersRequest) */ @Public @Stable @@ -81,7 +81,7 @@ public abstract class StartContainerRequest { * @return the container token to be used for authorization during starting * container. * @see NMToken - * @see ContainerManagementProtocol#startContainer(StartContainerRequest) + * @see ContainerManagementProtocol#startContainers(StartContainersRequest) */ @Public @Stable diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/StartContainersRequest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/StartContainersRequest.java new file mode 100644 index 00000000000..1f5b51c43f4 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/StartContainersRequest.java @@ -0,0 +1,76 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.api.protocolrecords; + +import java.util.List; + +import org.apache.hadoop.classification.InterfaceAudience.Public; +import org.apache.hadoop.classification.InterfaceStability.Stable; +import org.apache.hadoop.yarn.api.ContainerManagementProtocol; +import org.apache.hadoop.yarn.api.records.ContainerLaunchContext; +import org.apache.hadoop.yarn.util.Records; + +/** + *

+ * The request which contains a list of {@link StartContainerRequest} sent by + * the ApplicationMaster to the NodeManager to + * start containers. + *

+ * + *

+ * In each {@link StartContainerRequest}, the ApplicationMaster has + * to provide details such as allocated resource capability, security tokens (if + * enabled), command to be executed to start the container, environment for the + * process, necessary binaries/jar/shared-objects etc. via the + * {@link ContainerLaunchContext}. + *

+ * + * @see ContainerManagementProtocol#startContainers(StartContainersRequest) + */ +@Public +@Stable +public abstract class StartContainersRequest { + + @Public + @Stable + public static StartContainersRequest newInstance( + List requests) { + StartContainersRequest request = + Records.newRecord(StartContainersRequest.class); + request.setStartContainerRequests(requests); + return request; + } + + /** + * Get a list of {@link StartContainerRequest} to start containers. + * @return a list of {@link StartContainerRequest} to start containers. + */ + @Public + @Stable + public abstract List getStartContainerRequests(); + + /** + * Set a list of {@link StartContainerRequest} to start containers. + * @param request a list of {@link StartContainerRequest} to start containers + */ + @Public + @Stable + public abstract void setStartContainerRequests( + List request); +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/StartContainerResponse.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/StartContainersResponse.java similarity index 55% rename from hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/StartContainerResponse.java rename to hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/StartContainersResponse.java index 8df11db9b79..8dfda4594f5 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/StartContainerResponse.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/StartContainersResponse.java @@ -19,6 +19,7 @@ package org.apache.hadoop.yarn.api.protocolrecords; import java.nio.ByteBuffer; +import java.util.List; import java.util.Map; import org.apache.hadoop.classification.InterfaceAudience.Private; @@ -26,29 +27,71 @@ import org.apache.hadoop.classification.InterfaceAudience.Public; import org.apache.hadoop.classification.InterfaceStability.Stable; import org.apache.hadoop.classification.InterfaceStability.Unstable; import org.apache.hadoop.yarn.api.ContainerManagementProtocol; +import org.apache.hadoop.yarn.api.records.ContainerId; +import org.apache.hadoop.yarn.api.records.SerializedException; import org.apache.hadoop.yarn.util.Records; /** - *

The response sent by the NodeManager to the - * ApplicationMaster when asked to start an - * allocated container.

+ *

+ * The response sent by the NodeManager to the + * ApplicationMaster when asked to start an allocated + * container. + *

* - * @see ContainerManagementProtocol#startContainer(StartContainerRequest) + * @see ContainerManagementProtocol#startContainers(StartContainersRequest) */ @Public @Stable -public abstract class StartContainerResponse { +public abstract class StartContainersResponse { @Private @Unstable - public static StartContainerResponse newInstance( - Map servicesMetaData) { - StartContainerResponse response = - Records.newRecord(StartContainerResponse.class); + public static StartContainersResponse newInstance( + Map servicesMetaData, + List succeededContainers, + Map failedContainers) { + StartContainersResponse response = + Records.newRecord(StartContainersResponse.class); response.setAllServicesMetaData(servicesMetaData); + response.setSuccessfullyStartedContainers(succeededContainers); + response.setFailedRequests(failedContainers); return response; } + /** + * Get the list of ContainerId s of the containers that are + * started successfully. + * + * @return the list of ContainerId s of the containers that are + * started successfully. + * @see ContainerManagementProtocol#startContainers(StartContainersRequest) + */ + @Public + @Stable + public abstract List getSuccessfullyStartedContainers(); + + @Private + @Unstable + public abstract void setSuccessfullyStartedContainers( + List succeededContainers); + + /** + * Get the containerId-to-exception map in which the exception indicates error + * from per container for failed requests + */ + @Public + @Stable + public abstract Map getFailedRequests(); + + /** + * Set the containerId-to-exception map in which the exception indicates error + * from per container for failed requests + */ + @Private + @Unstable + public abstract void setFailedRequests( + Map failedContainers); + /** *

* Get the meta-data from all auxiliary services running on the @@ -76,8 +119,10 @@ public abstract class StartContainerResponse { * Set to the list of auxiliary services which have been started on the * NodeManager. This is done only once when the * NodeManager starts up - * @param allServicesMetaData A map from auxiliary service names to the opaque - * blob ByteBuffer for that auxiliary service + * + * @param allServicesMetaData + * A map from auxiliary service names to the opaque blob + * ByteBuffer for that auxiliary service */ @Private @Unstable diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/StopContainerResponse.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/StopContainerResponse.java deleted file mode 100644 index c8633f94f73..00000000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/StopContainerResponse.java +++ /dev/null @@ -1,47 +0,0 @@ -/** -* Licensed to the Apache Software Foundation (ASF) under one -* or more contributor license agreements. See the NOTICE file -* distributed with this work for additional information -* regarding copyright ownership. The ASF licenses this file -* to you under the Apache License, Version 2.0 (the -* "License"); you may not use this file except in compliance -* with the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -*/ - -package org.apache.hadoop.yarn.api.protocolrecords; - -import org.apache.hadoop.classification.InterfaceAudience.Private; -import org.apache.hadoop.classification.InterfaceAudience.Public; -import org.apache.hadoop.classification.InterfaceStability.Stable; -import org.apache.hadoop.classification.InterfaceStability.Unstable; -import org.apache.hadoop.yarn.api.ContainerManagementProtocol; -import org.apache.hadoop.yarn.util.Records; - -/** - *

The response sent by the NodeManager to the - * ApplicationMaster when asked to stop an - * allocated container.

- * - *

Currently, this is empty.

- * - * @see ContainerManagementProtocol#stopContainer(StopContainerRequest) - */ -@Public -@Stable -public abstract class StopContainerResponse { - @Private - @Unstable - public static StopContainerResponse newInstance() { - StopContainerResponse response = - Records.newRecord(StopContainerResponse.class); - return response; - } -} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/StopContainerRequest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/StopContainersRequest.java similarity index 61% rename from hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/StopContainerRequest.java rename to hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/StopContainersRequest.java index 1d4a2110ce7..8ea186ca018 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/StopContainerRequest.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/StopContainersRequest.java @@ -18,6 +18,8 @@ package org.apache.hadoop.yarn.api.protocolrecords; +import java.util.List; + import org.apache.hadoop.classification.InterfaceAudience.Public; import org.apache.hadoop.classification.InterfaceStability.Stable; import org.apache.hadoop.yarn.api.ContainerManagementProtocol; @@ -26,36 +28,36 @@ import org.apache.hadoop.yarn.util.Records; /** *

The request sent by the ApplicationMaster to the - * NodeManager to stop a container.

+ * NodeManager to stop containers.

* - * @see ContainerManagementProtocol#stopContainer(StopContainerRequest) + * @see ContainerManagementProtocol#stopContainers(StopContainersRequest) */ @Public @Stable -public abstract class StopContainerRequest { +public abstract class StopContainersRequest { @Public @Stable - public static StopContainerRequest newInstance(ContainerId containerId) { - StopContainerRequest request = - Records.newRecord(StopContainerRequest.class); - request.setContainerId(containerId); + public static StopContainersRequest newInstance(List containerIds) { + StopContainersRequest request = + Records.newRecord(StopContainersRequest.class); + request.setContainerIds(containerIds); return request; } /** - * Get the ContainerId of the container to be stopped. - * @return ContainerId of container to be stopped + * Get the ContainerIds of the containers to be stopped. + * @return ContainerIds of containers to be stopped */ @Public @Stable - public abstract ContainerId getContainerId(); + public abstract List getContainerIds(); /** - * Set the ContainerId of the container to be stopped. - * @param containerId ContainerId of the container to be stopped + * Set the ContainerIds of the containers to be stopped. + * @param containerIds ContainerIds of the containers to be stopped */ @Public @Stable - public abstract void setContainerId(ContainerId containerId); + public abstract void setContainerIds(List containerIds); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/StopContainersResponse.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/StopContainersResponse.java new file mode 100644 index 00000000000..a0f54d1b7df --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/StopContainersResponse.java @@ -0,0 +1,90 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.api.protocolrecords; + +import java.util.List; +import java.util.Map; + +import org.apache.hadoop.classification.InterfaceAudience.Private; +import org.apache.hadoop.classification.InterfaceAudience.Public; +import org.apache.hadoop.classification.InterfaceStability.Stable; +import org.apache.hadoop.classification.InterfaceStability.Unstable; +import org.apache.hadoop.yarn.api.ContainerManagementProtocol; +import org.apache.hadoop.yarn.api.records.ContainerId; +import org.apache.hadoop.yarn.api.records.SerializedException; +import org.apache.hadoop.yarn.util.Records; + +/** + *

+ * The response sent by the NodeManager to the + * ApplicationMaster when asked to stop allocated + * containers. + *

+ * + * @see ContainerManagementProtocol#stopContainers(StopContainersRequest) + */ +@Public +@Stable +public abstract class StopContainersResponse { + @Private + @Unstable + public static StopContainersResponse newInstance( + List succeededRequests, + Map failedRequests) { + StopContainersResponse response = + Records.newRecord(StopContainersResponse.class); + response.setFailedRequests(failedRequests); + response.setSuccessfullyStoppedContainers(succeededRequests); + return response; + } + + /** + * Get the list of containerIds of successfully stopped containers. + * + * @return the list of containerIds of successfully stopped containers. + */ + @Public + @Stable + public abstract List getSuccessfullyStoppedContainers(); + + /** + * Set the list of containerIds of successfully stopped containers. + */ + @Private + @Unstable + public abstract void setSuccessfullyStoppedContainers( + List succeededRequests); + + /** + * Get the containerId-to-exception map in which the exception indicates error + * from per container for failed requests + */ + @Public + @Stable + public abstract Map getFailedRequests(); + + /** + * Set the containerId-to-exception map in which the exception indicates error + * from per container for failed requests + */ + @Private + @Unstable + public abstract void setFailedRequests( + Map failedRequests); +} \ No newline at end of file diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Container.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Container.java index 5cff2ecbed0..279f12759a1 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Container.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Container.java @@ -57,8 +57,8 @@ import org.apache.hadoop.yarn.util.Records; * start/stop containers.

* * @see ApplicationMasterProtocol#allocate(org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest) - * @see ContainerManagementProtocol#startContainer(org.apache.hadoop.yarn.api.protocolrecords.StartContainerRequest) - * @see ContainerManagementProtocol#stopContainer(org.apache.hadoop.yarn.api.protocolrecords.StopContainerRequest) + * @see ContainerManagementProtocol#startContainers(org.apache.hadoop.yarn.api.protocolrecords.StartContainersRequest) + * @see ContainerManagementProtocol#stopContainers(org.apache.hadoop.yarn.api.protocolrecords.StopContainersRequest) */ @Public @Stable @@ -155,7 +155,7 @@ public abstract class Container implements Comparable { * Container includes the ContainerToken.

* * @see ApplicationMasterProtocol#allocate(org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest) - * @see ContainerManagementProtocol#startContainer(org.apache.hadoop.yarn.api.protocolrecords.StartContainerRequest) + * @see ContainerManagementProtocol#startContainers(org.apache.hadoop.yarn.api.protocolrecords.StartContainersRequest) * * @return ContainerToken for the container */ diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerLaunchContext.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerLaunchContext.java index 54c1d07bac0..a648fef0fc5 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerLaunchContext.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerLaunchContext.java @@ -49,7 +49,7 @@ import org.apache.hadoop.yarn.util.Records; * *

* - * @see ContainerManagementProtocol#startContainer(org.apache.hadoop.yarn.api.protocolrecords.StartContainerRequest) + * @see ContainerManagementProtocol#startContainers(org.apache.hadoop.yarn.api.protocolrecords.StartContainersRequest) */ @Public @Stable @@ -132,7 +132,7 @@ public abstract class ContainerLaunchContext { /** *

- * Get application-specific binary service data. This is a map keyed + * Set application-specific binary service data. This is a map keyed * by the name of each {@link AuxiliaryService} that is configured on a * NodeManager and value correspond to the application specific data targeted * for the keyed {@link AuxiliaryService}. All pre-existing Map entries are diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/LocalResource.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/LocalResource.java index a220e6e38e8..f14a136d30d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/LocalResource.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/LocalResource.java @@ -37,7 +37,7 @@ import org.apache.hadoop.yarn.util.Records; * @see LocalResourceVisibility * @see ContainerLaunchContext * @see ApplicationSubmissionContext - * @see ContainerManagementProtocol#startContainer(org.apache.hadoop.yarn.api.protocolrecords.StartContainerRequest) + * @see ContainerManagementProtocol#startContainers(org.apache.hadoop.yarn.api.protocolrecords.StartContainersRequest) */ @Public @Stable diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/LocalResourceType.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/LocalResourceType.java index c1c5eb41942..d1aa45b23a3 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/LocalResourceType.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/LocalResourceType.java @@ -43,7 +43,7 @@ import org.apache.hadoop.yarn.api.ContainerManagementProtocol; * @see LocalResource * @see ContainerLaunchContext * @see ApplicationSubmissionContext - * @see ContainerManagementProtocol#startContainer(org.apache.hadoop.yarn.api.protocolrecords.StartContainerRequest) + * @see ContainerManagementProtocol#startContainers(org.apache.hadoop.yarn.api.protocolrecords.StartContainersRequest) */ @Public @Stable diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/LocalResourceVisibility.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/LocalResourceVisibility.java index 32087eeeb49..d368bfb8139 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/LocalResourceVisibility.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/LocalResourceVisibility.java @@ -43,7 +43,7 @@ import org.apache.hadoop.yarn.api.ContainerManagementProtocol; * @see LocalResource * @see ContainerLaunchContext * @see ApplicationSubmissionContext - * @see ContainerManagementProtocol#startContainer(org.apache.hadoop.yarn.api.protocolrecords.StartContainerRequest) + * @see ContainerManagementProtocol#startContainers(org.apache.hadoop.yarn.api.protocolrecords.StartContainersRequest) */ @Public @Stable diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/SerializedException.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/SerializedException.java new file mode 100644 index 00000000000..aba54d4114f --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/SerializedException.java @@ -0,0 +1,93 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.api.records; + +import org.apache.hadoop.classification.InterfaceAudience.Private; +import org.apache.hadoop.classification.InterfaceStability.Unstable; +import org.apache.hadoop.yarn.util.Records; + +@Private +@Unstable +public abstract class SerializedException { + + @Private + @Unstable + public static SerializedException newInstance(Throwable e) { + SerializedException exception = + Records.newRecord(SerializedException.class); + exception.init(e); + return exception; + } + + /** + * Constructs a new SerializedException with the specified detail + * message and cause. + */ + @Private + @Unstable + public abstract void init(String message, Throwable cause); + + /** + * Constructs a new SerializedException with the specified detail + * message. + */ + @Private + @Unstable + public abstract void init(String message); + + /** + * Constructs a new SerializedException with the specified cause. + */ + @Private + @Unstable + public abstract void init(Throwable cause); + + /** + * Get the detail message string of this exception. + * @return the detail message string of this exception. + */ + @Private + @Unstable + public abstract String getMessage(); + + /** + * Get the backtrace of this exception. + * @return the backtrace of this exception. + */ + @Private + @Unstable + public abstract String getRemoteTrace(); + + /** + * Get the cause of this exception or null if the cause is nonexistent or + * unknown. + * @return the cause of this exception. + */ + @Private + @Unstable + public abstract SerializedException getCause(); + + /** + * Deserialize the exception to a new Throwable. + * @return the Throwable form of this serialized exception. + */ + @Private + @Unstable + public abstract Throwable deSerialize(); +} \ No newline at end of file diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/exceptions/InvalidContainerException.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/exceptions/InvalidContainerException.java index 5f874a815d4..9c9849e5836 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/exceptions/InvalidContainerException.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/exceptions/InvalidContainerException.java @@ -19,12 +19,12 @@ package org.apache.hadoop.yarn.exceptions; import org.apache.hadoop.yarn.api.ContainerManagementProtocol; -import org.apache.hadoop.yarn.api.protocolrecords.StartContainerRequest; +import org.apache.hadoop.yarn.api.protocolrecords.StartContainersRequest; /** * This exception is thrown by a NodeManager that is rejecting start-container * requests via - * {@link ContainerManagementProtocol#startContainer(StartContainerRequest)} + * {@link ContainerManagementProtocol#startContainers(StartContainersRequest)} * for containers allocated by a previous instance of the RM. */ public class InvalidContainerException extends YarnException { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/exceptions/NMNotYetReadyException.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/exceptions/NMNotYetReadyException.java index 01f7df1caa2..5c6781f338d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/exceptions/NMNotYetReadyException.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/exceptions/NMNotYetReadyException.java @@ -19,11 +19,11 @@ package org.apache.hadoop.yarn.exceptions; import org.apache.hadoop.yarn.api.ContainerManagementProtocol; -import org.apache.hadoop.yarn.api.protocolrecords.StartContainerRequest; +import org.apache.hadoop.yarn.api.protocolrecords.StartContainersRequest; /** * This exception is thrown on - * {@link ContainerManagementProtocol#startContainer(StartContainerRequest)} API + * {@link ContainerManagementProtocol#startContainers(StartContainersRequest)} API * when an NM starts from scratch but has not yet connected with RM. */ public class NMNotYetReadyException extends YarnException { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/ApplicationInitializationContext.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/ApplicationInitializationContext.java index b6ec2e1ab2b..da5d45c7354 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/ApplicationInitializationContext.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/ApplicationInitializationContext.java @@ -25,7 +25,7 @@ import org.apache.hadoop.classification.InterfaceAudience.Public; import org.apache.hadoop.classification.InterfaceStability.Evolving; import org.apache.hadoop.classification.InterfaceStability.Unstable; import org.apache.hadoop.yarn.api.ContainerManagementProtocol; -import org.apache.hadoop.yarn.api.protocolrecords.StartContainerRequest; +import org.apache.hadoop.yarn.api.protocolrecords.StartContainersRequest; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ContainerLaunchContext; @@ -70,7 +70,7 @@ public class ApplicationInitializationContext { /** * Get the data sent to the NodeManager via - * {@link ContainerManagementProtocol#startContainer(StartContainerRequest)} + * {@link ContainerManagementProtocol#startContainers(StartContainersRequest)} * as part of {@link ContainerLaunchContext#getServiceData()} * * @return the servicesData for this application. diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/AuxiliaryService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/AuxiliaryService.java index e50098fc445..275f2a91038 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/AuxiliaryService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/AuxiliaryService.java @@ -24,8 +24,8 @@ import org.apache.hadoop.classification.InterfaceAudience.Public; import org.apache.hadoop.classification.InterfaceStability.Evolving; import org.apache.hadoop.service.AbstractService; import org.apache.hadoop.yarn.api.ContainerManagementProtocol; -import org.apache.hadoop.yarn.api.protocolrecords.StartContainerRequest; -import org.apache.hadoop.yarn.api.protocolrecords.StartContainerResponse; +import org.apache.hadoop.yarn.api.protocolrecords.StartContainersRequest; +import org.apache.hadoop.yarn.api.protocolrecords.StartContainersResponse; import org.apache.hadoop.yarn.conf.YarnConfiguration; /** @@ -71,8 +71,8 @@ public abstract class AuxiliaryService extends AbstractService { * *

* The information is passed along to applications via - * {@link StartContainerResponse#getAllServicesMetaData()} that is returned by - * {@link ContainerManagementProtocol#startContainer(StartContainerRequest)} + * {@link StartContainersResponse#getAllServicesMetaData()} that is returned by + * {@link ContainerManagementProtocol#startContainers(StartContainersRequest)} *

* * @return meta-data for this service that should be made available to diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/containermanagement_protocol.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/containermanagement_protocol.proto index 5f02a28a8fe..98f438aa993 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/containermanagement_protocol.proto +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/containermanagement_protocol.proto @@ -30,7 +30,7 @@ option java_generate_equals_and_hash = true; import "yarn_service_protos.proto"; service ContainerManagementProtocolService { - rpc startContainer(StartContainerRequestProto) returns (StartContainerResponseProto); - rpc stopContainer(StopContainerRequestProto) returns (StopContainerResponseProto); - rpc getContainerStatus(GetContainerStatusRequestProto) returns (GetContainerStatusResponseProto); + rpc startContainers(StartContainersRequestProto) returns (StartContainersResponseProto); + rpc stopContainers(StopContainersRequestProto) returns (StopContainersResponseProto); + rpc getContainerStatuses(GetContainerStatusesRequestProto) returns (GetContainerStatusesResponseProto); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto index 4a6db02ff8c..bd009e0d4a6 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto @@ -182,3 +182,37 @@ message GetContainerStatusRequestProto { message GetContainerStatusResponseProto { optional ContainerStatusProto status = 1; } + +//// bulk API records +message StartContainersRequestProto { + repeated StartContainerRequestProto start_container_request = 1; +} + +message ContainerExceptionMapProto { + optional ContainerIdProto container_id = 1; + optional SerializedExceptionProto exception = 2; +} + +message StartContainersResponseProto { + repeated StringBytesMapProto services_meta_data = 1; + repeated ContainerIdProto succeeded_requests = 2; + repeated ContainerExceptionMapProto failed_requests = 3; +} + +message StopContainersRequestProto { + repeated ContainerIdProto container_id = 1; +} + +message StopContainersResponseProto { + repeated ContainerIdProto succeeded_requests = 1; + repeated ContainerExceptionMapProto failed_requests = 2; +} + +message GetContainerStatusesRequestProto { + repeated ContainerIdProto container_id = 1; +} + +message GetContainerStatusesResponseProto { + repeated ContainerStatusProto status = 1; + repeated ContainerExceptionMapProto failed_requests = 2; +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/NMClientImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/NMClientImpl.java index 54a73faf74f..b5f0be11cc8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/NMClientImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/NMClientImpl.java @@ -20,6 +20,8 @@ package org.apache.hadoop.yarn.client.api.impl; import java.io.IOException; import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.List; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; @@ -30,9 +32,14 @@ import org.apache.commons.logging.LogFactory; import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.classification.InterfaceStability.Unstable; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusRequest; +import org.apache.hadoop.security.token.SecretManager.InvalidToken; +import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusesRequest; +import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusesResponse; import org.apache.hadoop.yarn.api.protocolrecords.StartContainerRequest; -import org.apache.hadoop.yarn.api.protocolrecords.StopContainerRequest; +import org.apache.hadoop.yarn.api.protocolrecords.StartContainersRequest; +import org.apache.hadoop.yarn.api.protocolrecords.StartContainersResponse; +import org.apache.hadoop.yarn.api.protocolrecords.StopContainersRequest; +import org.apache.hadoop.yarn.api.protocolrecords.StopContainersResponse; import org.apache.hadoop.yarn.api.records.Container; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.ContainerLaunchContext; @@ -181,11 +188,23 @@ public class NMClientImpl extends NMClient { proxy = cmProxy.getProxy(container.getNodeId().toString(), container.getId()); - allServiceResponse = + StartContainerRequest scRequest = + StartContainerRequest.newInstance(containerLaunchContext, + container.getContainerToken()); + List list = new ArrayList(); + list.add(scRequest); + StartContainersRequest allRequests = + StartContainersRequest.newInstance(list); + StartContainersResponse response = proxy - .getContainerManagementProtocol().startContainer( - StartContainerRequest.newInstance(containerLaunchContext, - container.getContainerToken())).getAllServicesMetaData(); + .getContainerManagementProtocol().startContainers(allRequests); + if (response.getFailedRequests() != null + && response.getFailedRequests().containsKey(container.getId())) { + Throwable t = + response.getFailedRequests().get(container.getId()).deSerialize(); + parseAndThrowException(t); + } + allServiceResponse = response.getAllServicesMetaData(); startingContainer.state = ContainerState.RUNNING; } catch (YarnException e) { startingContainer.state = ContainerState.COMPLETE; @@ -238,11 +257,20 @@ public class NMClientImpl extends NMClient { NodeId nodeId) throws YarnException, IOException { ContainerManagementProtocolProxyData proxy = null; + List containerIds = new ArrayList(); + containerIds.add(containerId); try { proxy = cmProxy.getProxy(nodeId.toString(), containerId); - ContainerStatus containerStatus = - proxy.getContainerManagementProtocol().getContainerStatus( - GetContainerStatusRequest.newInstance(containerId)).getStatus(); + GetContainerStatusesResponse response = + proxy.getContainerManagementProtocol().getContainerStatuses( + GetContainerStatusesRequest.newInstance(containerIds)); + if (response.getFailedRequests() != null + && response.getFailedRequests().containsKey(containerId)) { + Throwable t = + response.getFailedRequests().get(containerId).deSerialize(); + parseAndThrowException(t); + } + ContainerStatus containerStatus = response.getContainerStatuses().get(0); return containerStatus; } finally { if (proxy != null) { @@ -254,10 +282,19 @@ public class NMClientImpl extends NMClient { private void stopContainerInternal(ContainerId containerId, NodeId nodeId) throws IOException, YarnException { ContainerManagementProtocolProxyData proxy = null; + List containerIds = new ArrayList(); + containerIds.add(containerId); try { proxy = cmProxy.getProxy(nodeId.toString(), containerId); - proxy.getContainerManagementProtocol().stopContainer( - StopContainerRequest.newInstance(containerId)); + StopContainersResponse response = + proxy.getContainerManagementProtocol().stopContainers( + StopContainersRequest.newInstance(containerIds)); + if (response.getFailedRequests() != null + && response.getFailedRequests().containsKey(containerId)) { + Throwable t = response.getFailedRequests().get(containerId) + .deSerialize(); + parseAndThrowException(t); + } } finally { if (proxy != null) { cmProxy.mayBeCloseProxy(proxy); @@ -285,4 +322,15 @@ public class NMClientImpl extends NMClient { public AtomicBoolean getCleanupRunningContainers() { return cleanupRunningContainers; } + + private void parseAndThrowException(Throwable t) throws YarnException, + IOException { + if (t instanceof YarnException) { + throw (YarnException) t; + } else if (t instanceof InvalidToken) { + throw (InvalidToken) t; + } else { + throw (IOException) t; + } + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/impl/pb/client/ContainerManagementProtocolPBClientImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/impl/pb/client/ContainerManagementProtocolPBClientImpl.java index f6c0fa6eb10..15397e3518e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/impl/pb/client/ContainerManagementProtocolPBClientImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/impl/pb/client/ContainerManagementProtocolPBClientImpl.java @@ -30,24 +30,24 @@ import org.apache.hadoop.net.NetUtils; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.yarn.api.ContainerManagementProtocol; import org.apache.hadoop.yarn.api.ContainerManagementProtocolPB; -import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusRequest; -import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusResponse; -import org.apache.hadoop.yarn.api.protocolrecords.StartContainerRequest; -import org.apache.hadoop.yarn.api.protocolrecords.StartContainerResponse; -import org.apache.hadoop.yarn.api.protocolrecords.StopContainerRequest; -import org.apache.hadoop.yarn.api.protocolrecords.StopContainerResponse; -import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetContainerStatusRequestPBImpl; -import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetContainerStatusResponsePBImpl; -import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.StartContainerRequestPBImpl; -import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.StartContainerResponsePBImpl; -import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.StopContainerRequestPBImpl; -import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.StopContainerResponsePBImpl; +import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusesRequest; +import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusesResponse; +import org.apache.hadoop.yarn.api.protocolrecords.StartContainersRequest; +import org.apache.hadoop.yarn.api.protocolrecords.StartContainersResponse; +import org.apache.hadoop.yarn.api.protocolrecords.StopContainersRequest; +import org.apache.hadoop.yarn.api.protocolrecords.StopContainersResponse; +import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetContainerStatusesRequestPBImpl; +import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetContainerStatusesResponsePBImpl; +import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.StartContainersRequestPBImpl; +import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.StartContainersResponsePBImpl; +import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.StopContainersRequestPBImpl; +import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.StopContainersResponsePBImpl; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.ipc.RPCUtil; -import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetContainerStatusRequestProto; -import org.apache.hadoop.yarn.proto.YarnServiceProtos.StartContainerRequestProto; -import org.apache.hadoop.yarn.proto.YarnServiceProtos.StopContainerRequestProto; +import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetContainerStatusesRequestProto; +import org.apache.hadoop.yarn.proto.YarnServiceProtos.StartContainersRequestProto; +import org.apache.hadoop.yarn.proto.YarnServiceProtos.StopContainersRequestProto; import com.google.protobuf.ServiceException; @@ -87,45 +87,45 @@ public class ContainerManagementProtocolPBClientImpl implements ContainerManagem } @Override - public GetContainerStatusResponse getContainerStatus( - GetContainerStatusRequest request) throws YarnException, - IOException { - GetContainerStatusRequestProto requestProto = - ((GetContainerStatusRequestPBImpl) request).getProto(); + public StartContainersResponse + startContainers(StartContainersRequest requests) throws YarnException, + IOException { + StartContainersRequestProto requestProto = + ((StartContainersRequestPBImpl) requests).getProto(); try { - return new GetContainerStatusResponsePBImpl(proxy.getContainerStatus( + return new StartContainersResponsePBImpl(proxy.startContainers(null, + requestProto)); + } catch (ServiceException e) { + RPCUtil.unwrapAndThrowException(e); + return null; + } + } + + @Override + public StopContainersResponse stopContainers(StopContainersRequest requests) + throws YarnException, IOException { + StopContainersRequestProto requestProto = + ((StopContainersRequestPBImpl) requests).getProto(); + try { + return new StopContainersResponsePBImpl(proxy.stopContainers(null, + requestProto)); + } catch (ServiceException e) { + RPCUtil.unwrapAndThrowException(e); + return null; + } + } + + @Override + public GetContainerStatusesResponse getContainerStatuses( + GetContainerStatusesRequest request) throws YarnException, IOException { + GetContainerStatusesRequestProto requestProto = + ((GetContainerStatusesRequestPBImpl) request).getProto(); + try { + return new GetContainerStatusesResponsePBImpl(proxy.getContainerStatuses( null, requestProto)); } catch (ServiceException e) { RPCUtil.unwrapAndThrowException(e); return null; } } - - @Override - public StartContainerResponse startContainer(StartContainerRequest request) - throws YarnException, IOException { - StartContainerRequestProto requestProto = - ((StartContainerRequestPBImpl) request).getProto(); - try { - return new StartContainerResponsePBImpl(proxy.startContainer(null, - requestProto)); - } catch (ServiceException e) { - RPCUtil.unwrapAndThrowException(e); - return null; - } - } - - @Override - public StopContainerResponse stopContainer(StopContainerRequest request) - throws YarnException, IOException { - StopContainerRequestProto requestProto = - ((StopContainerRequestPBImpl) request).getProto(); - try { - return new StopContainerResponsePBImpl(proxy.stopContainer(null, - requestProto)); - } catch (ServiceException e) { - RPCUtil.unwrapAndThrowException(e); - return null; - } - } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/impl/pb/service/ContainerManagementProtocolPBServiceImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/impl/pb/service/ContainerManagementProtocolPBServiceImpl.java index 5c2a96a3e6b..2d33e6980f1 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/impl/pb/service/ContainerManagementProtocolPBServiceImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/impl/pb/service/ContainerManagementProtocolPBServiceImpl.java @@ -23,22 +23,22 @@ import java.io.IOException; import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.yarn.api.ContainerManagementProtocol; import org.apache.hadoop.yarn.api.ContainerManagementProtocolPB; -import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusResponse; -import org.apache.hadoop.yarn.api.protocolrecords.StartContainerResponse; -import org.apache.hadoop.yarn.api.protocolrecords.StopContainerResponse; -import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetContainerStatusRequestPBImpl; -import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetContainerStatusResponsePBImpl; -import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.StartContainerRequestPBImpl; -import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.StartContainerResponsePBImpl; -import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.StopContainerRequestPBImpl; -import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.StopContainerResponsePBImpl; +import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusesResponse; +import org.apache.hadoop.yarn.api.protocolrecords.StartContainersResponse; +import org.apache.hadoop.yarn.api.protocolrecords.StopContainersResponse; +import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetContainerStatusesRequestPBImpl; +import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetContainerStatusesResponsePBImpl; +import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.StartContainersRequestPBImpl; +import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.StartContainersResponsePBImpl; +import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.StopContainersRequestPBImpl; +import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.StopContainersResponsePBImpl; import org.apache.hadoop.yarn.exceptions.YarnException; -import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetContainerStatusRequestProto; -import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetContainerStatusResponseProto; -import org.apache.hadoop.yarn.proto.YarnServiceProtos.StartContainerRequestProto; -import org.apache.hadoop.yarn.proto.YarnServiceProtos.StartContainerResponseProto; -import org.apache.hadoop.yarn.proto.YarnServiceProtos.StopContainerRequestProto; -import org.apache.hadoop.yarn.proto.YarnServiceProtos.StopContainerResponseProto; +import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetContainerStatusesRequestProto; +import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetContainerStatusesResponseProto; +import org.apache.hadoop.yarn.proto.YarnServiceProtos.StartContainersRequestProto; +import org.apache.hadoop.yarn.proto.YarnServiceProtos.StartContainersResponseProto; +import org.apache.hadoop.yarn.proto.YarnServiceProtos.StopContainersRequestProto; +import org.apache.hadoop.yarn.proto.YarnServiceProtos.StopContainersResponseProto; import com.google.protobuf.RpcController; import com.google.protobuf.ServiceException; @@ -53,12 +53,12 @@ public class ContainerManagementProtocolPBServiceImpl implements ContainerManage } @Override - public GetContainerStatusResponseProto getContainerStatus(RpcController arg0, - GetContainerStatusRequestProto proto) throws ServiceException { - GetContainerStatusRequestPBImpl request = new GetContainerStatusRequestPBImpl(proto); + public StartContainersResponseProto startContainers(RpcController arg0, + StartContainersRequestProto proto) throws ServiceException { + StartContainersRequestPBImpl request = new StartContainersRequestPBImpl(proto); try { - GetContainerStatusResponse response = real.getContainerStatus(request); - return ((GetContainerStatusResponsePBImpl)response).getProto(); + StartContainersResponse response = real.startContainers(request); + return ((StartContainersResponsePBImpl)response).getProto(); } catch (YarnException e) { throw new ServiceException(e); } catch (IOException e) { @@ -67,12 +67,12 @@ public class ContainerManagementProtocolPBServiceImpl implements ContainerManage } @Override - public StartContainerResponseProto startContainer(RpcController arg0, - StartContainerRequestProto proto) throws ServiceException { - StartContainerRequestPBImpl request = new StartContainerRequestPBImpl(proto); + public StopContainersResponseProto stopContainers(RpcController arg0, + StopContainersRequestProto proto) throws ServiceException { + StopContainersRequestPBImpl request = new StopContainersRequestPBImpl(proto); try { - StartContainerResponse response = real.startContainer(request); - return ((StartContainerResponsePBImpl)response).getProto(); + StopContainersResponse response = real.stopContainers(request); + return ((StopContainersResponsePBImpl)response).getProto(); } catch (YarnException e) { throw new ServiceException(e); } catch (IOException e) { @@ -81,17 +81,17 @@ public class ContainerManagementProtocolPBServiceImpl implements ContainerManage } @Override - public StopContainerResponseProto stopContainer(RpcController arg0, - StopContainerRequestProto proto) throws ServiceException { - StopContainerRequestPBImpl request = new StopContainerRequestPBImpl(proto); + public GetContainerStatusesResponseProto getContainerStatuses( + RpcController arg0, GetContainerStatusesRequestProto proto) + throws ServiceException { + GetContainerStatusesRequestPBImpl request = new GetContainerStatusesRequestPBImpl(proto); try { - StopContainerResponse response = real.stopContainer(request); - return ((StopContainerResponsePBImpl)response).getProto(); + GetContainerStatusesResponse response = real.getContainerStatuses(request); + return ((GetContainerStatusesResponsePBImpl)response).getProto(); } catch (YarnException e) { throw new ServiceException(e); } catch (IOException e) { throw new ServiceException(e); } } - } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetContainerStatusResponsePBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetContainerStatusResponsePBImpl.java deleted file mode 100644 index 2b635f98879..00000000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetContainerStatusResponsePBImpl.java +++ /dev/null @@ -1,130 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.yarn.api.protocolrecords.impl.pb; - - -import org.apache.hadoop.classification.InterfaceAudience.Private; -import org.apache.hadoop.classification.InterfaceStability.Unstable; -import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusResponse; -import org.apache.hadoop.yarn.api.records.ContainerStatus; -import org.apache.hadoop.yarn.api.records.impl.pb.ContainerStatusPBImpl; -import org.apache.hadoop.yarn.proto.YarnProtos.ContainerStatusProto; -import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetContainerStatusResponseProto; -import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetContainerStatusResponseProtoOrBuilder; - -@Private -@Unstable -public class GetContainerStatusResponsePBImpl extends GetContainerStatusResponse { - GetContainerStatusResponseProto proto = GetContainerStatusResponseProto.getDefaultInstance(); - GetContainerStatusResponseProto.Builder builder = null; - boolean viaProto = false; - - private ContainerStatus containerStatus = null; - - - public GetContainerStatusResponsePBImpl() { - builder = GetContainerStatusResponseProto.newBuilder(); - } - - public GetContainerStatusResponsePBImpl(GetContainerStatusResponseProto proto) { - this.proto = proto; - viaProto = true; - } - - public GetContainerStatusResponseProto getProto() { - mergeLocalToProto(); - proto = viaProto ? proto : builder.build(); - viaProto = true; - return proto; - } - - @Override - public int hashCode() { - return getProto().hashCode(); - } - - @Override - public boolean equals(Object other) { - if (other == null) - return false; - if (other.getClass().isAssignableFrom(this.getClass())) { - return this.getProto().equals(this.getClass().cast(other).getProto()); - } - return false; - } - - @Override - public String toString() { - return getProto().toString().replaceAll("\\n", ", ").replaceAll("\\s+", " "); - } - - private void mergeLocalToBuilder() { - if (this.containerStatus != null) { - builder.setStatus(convertToProtoFormat(this.containerStatus)); - } - } - - private void mergeLocalToProto() { - if (viaProto) - maybeInitBuilder(); - mergeLocalToBuilder(); - proto = builder.build(); - viaProto = true; - } - - private void maybeInitBuilder() { - if (viaProto || builder == null) { - builder = GetContainerStatusResponseProto.newBuilder(proto); - } - viaProto = false; - } - - - @Override - public ContainerStatus getStatus() { - GetContainerStatusResponseProtoOrBuilder p = viaProto ? proto : builder; - if (this.containerStatus != null) { - return this.containerStatus; - } - if (!p.hasStatus()) { - return null; - } - this.containerStatus = convertFromProtoFormat(p.getStatus()); - return this.containerStatus; - } - - @Override - public void setStatus(ContainerStatus status) { - maybeInitBuilder(); - if (status == null) - builder.clearStatus(); - this.containerStatus = status; - } - - private ContainerStatusPBImpl convertFromProtoFormat(ContainerStatusProto p) { - return new ContainerStatusPBImpl(p); - } - - private ContainerStatusProto convertToProtoFormat(ContainerStatus t) { - return ((ContainerStatusPBImpl)t).getProto(); - } - - - -} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetContainerStatusRequestPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetContainerStatusesRequestPBImpl.java similarity index 56% rename from hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetContainerStatusRequestPBImpl.java rename to hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetContainerStatusesRequestPBImpl.java index fa88d1f63e4..0c305ca8862 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetContainerStatusRequestPBImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetContainerStatusesRequestPBImpl.java @@ -18,37 +18,41 @@ package org.apache.hadoop.yarn.api.protocolrecords.impl.pb; +import java.util.ArrayList; +import java.util.List; import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.classification.InterfaceStability.Unstable; -import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusRequest; +import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusesRequest; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.impl.pb.ContainerIdPBImpl; import org.apache.hadoop.yarn.proto.YarnProtos.ContainerIdProto; -import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetContainerStatusRequestProto; -import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetContainerStatusRequestProtoOrBuilder; +import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetContainerStatusesRequestProto; +import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetContainerStatusesRequestProtoOrBuilder; @Private @Unstable -public class GetContainerStatusRequestPBImpl extends GetContainerStatusRequest { - GetContainerStatusRequestProto proto = GetContainerStatusRequestProto.getDefaultInstance(); - GetContainerStatusRequestProto.Builder builder = null; +public class GetContainerStatusesRequestPBImpl extends + GetContainerStatusesRequest { + GetContainerStatusesRequestProto proto = GetContainerStatusesRequestProto + .getDefaultInstance(); + GetContainerStatusesRequestProto.Builder builder = null; boolean viaProto = false; - - private ContainerId containerId = null; - - - public GetContainerStatusRequestPBImpl() { - builder = GetContainerStatusRequestProto.newBuilder(); + + private List containerIds = null; + + public GetContainerStatusesRequestPBImpl() { + builder = GetContainerStatusesRequestProto.newBuilder(); } - public GetContainerStatusRequestPBImpl(GetContainerStatusRequestProto proto) { + public GetContainerStatusesRequestPBImpl( + GetContainerStatusesRequestProto proto) { this.proto = proto; viaProto = true; } - - public GetContainerStatusRequestProto getProto() { - mergeLocalToProto(); + + public GetContainerStatusesRequestProto getProto() { + mergeLocalToProto(); proto = viaProto ? proto : builder.build(); viaProto = true; return proto; @@ -71,17 +75,18 @@ public class GetContainerStatusRequestPBImpl extends GetContainerStatusRequest { @Override public String toString() { - return getProto().toString().replaceAll("\\n", ", ").replaceAll("\\s+", " "); + return getProto().toString().replaceAll("\\n", ", ") + .replaceAll("\\s+", " "); } private void mergeLocalToBuilder() { - if (this.containerId != null) { - builder.setContainerId(convertToProtoFormat(this.containerId)); + if (this.containerIds != null) { + addLocalContainerIdsToProto(); } } private void mergeLocalToProto() { - if (viaProto) + if (viaProto) maybeInitBuilder(); mergeLocalToBuilder(); proto = builder.build(); @@ -90,31 +95,47 @@ public class GetContainerStatusRequestPBImpl extends GetContainerStatusRequest { private void maybeInitBuilder() { if (viaProto || builder == null) { - builder = GetContainerStatusRequestProto.newBuilder(proto); + builder = GetContainerStatusesRequestProto.newBuilder(proto); } viaProto = false; } - - - @Override - public ContainerId getContainerId() { - GetContainerStatusRequestProtoOrBuilder p = viaProto ? proto : builder; - if (this.containerId != null) { - return this.containerId; + + private void addLocalContainerIdsToProto() { + maybeInitBuilder(); + builder.clearContainerId(); + if (this.containerIds == null) + return; + List protoList = new ArrayList(); + for (ContainerId id : containerIds) { + protoList.add(convertToProtoFormat(id)); } - if (!p.hasContainerId()) { - return null; + builder.addAllContainerId(protoList); + } + + private void initLocalContainerIds() { + if (this.containerIds != null) { + return; + } + GetContainerStatusesRequestProtoOrBuilder p = viaProto ? proto : builder; + List containerIds = p.getContainerIdList(); + this.containerIds = new ArrayList(); + for (ContainerIdProto id : containerIds) { + this.containerIds.add(convertFromProtoFormat(id)); } - this.containerId = convertFromProtoFormat(p.getContainerId()); - return this.containerId; } @Override - public void setContainerId(ContainerId containerId) { + public List getContainerIds() { + initLocalContainerIds(); + return this.containerIds; + } + + @Override + public void setContainerIds(List containerIds) { maybeInitBuilder(); - if (containerId == null) + if (containerIds == null) builder.clearContainerId(); - this.containerId = containerId; + this.containerIds = containerIds; } private ContainerIdPBImpl convertFromProtoFormat(ContainerIdProto p) { @@ -122,9 +143,7 @@ public class GetContainerStatusRequestPBImpl extends GetContainerStatusRequest { } private ContainerIdProto convertToProtoFormat(ContainerId t) { - return ((ContainerIdPBImpl)t).getProto(); + return ((ContainerIdPBImpl) t).getProto(); } - - -} +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetContainerStatusesResponsePBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetContainerStatusesResponsePBImpl.java new file mode 100644 index 00000000000..18df2146612 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetContainerStatusesResponsePBImpl.java @@ -0,0 +1,223 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.api.protocolrecords.impl.pb; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import org.apache.hadoop.classification.InterfaceAudience.Private; +import org.apache.hadoop.classification.InterfaceStability.Unstable; +import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusesResponse; +import org.apache.hadoop.yarn.api.records.ContainerId; +import org.apache.hadoop.yarn.api.records.ContainerStatus; +import org.apache.hadoop.yarn.api.records.SerializedException; +import org.apache.hadoop.yarn.api.records.impl.pb.ContainerIdPBImpl; +import org.apache.hadoop.yarn.api.records.impl.pb.ContainerStatusPBImpl; +import org.apache.hadoop.yarn.api.records.impl.pb.SerializedExceptionPBImpl; +import org.apache.hadoop.yarn.proto.YarnProtos.ContainerIdProto; +import org.apache.hadoop.yarn.proto.YarnProtos.ContainerStatusProto; +import org.apache.hadoop.yarn.proto.YarnProtos.SerializedExceptionProto; +import org.apache.hadoop.yarn.proto.YarnServiceProtos.ContainerExceptionMapProto; +import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetContainerStatusesResponseProto; +import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetContainerStatusesResponseProtoOrBuilder; + +@Private +@Unstable +public class GetContainerStatusesResponsePBImpl extends + GetContainerStatusesResponse { + GetContainerStatusesResponseProto proto = GetContainerStatusesResponseProto + .getDefaultInstance(); + GetContainerStatusesResponseProto.Builder builder = null; + boolean viaProto = false; + + private List containerStatuses = null; + private Map failedRequests = null; + + public GetContainerStatusesResponsePBImpl() { + builder = GetContainerStatusesResponseProto.newBuilder(); + } + + public GetContainerStatusesResponsePBImpl( + GetContainerStatusesResponseProto proto) { + this.proto = proto; + viaProto = true; + } + + public GetContainerStatusesResponseProto getProto() { + mergeLocalToProto(); + proto = viaProto ? proto : builder.build(); + viaProto = true; + return proto; + } + + @Override + public int hashCode() { + return getProto().hashCode(); + } + + @Override + public boolean equals(Object other) { + if (other == null) + return false; + if (other.getClass().isAssignableFrom(this.getClass())) { + return this.getProto().equals(this.getClass().cast(other).getProto()); + } + return false; + } + + @Override + public String toString() { + return getProto().toString().replaceAll("\\n", ", ") + .replaceAll("\\s+", " "); + } + + private void mergeLocalToBuilder() { + if (this.containerStatuses != null) { + addLocalContainerStatusesToProto(); + } + if (this.failedRequests != null) { + addFailedRequestsToProto(); + } + } + + private void mergeLocalToProto() { + if (viaProto) + maybeInitBuilder(); + mergeLocalToBuilder(); + proto = builder.build(); + viaProto = true; + } + + private void maybeInitBuilder() { + if (viaProto || builder == null) { + builder = GetContainerStatusesResponseProto.newBuilder(proto); + } + viaProto = false; + } + + private void addLocalContainerStatusesToProto() { + maybeInitBuilder(); + builder.clearStatus(); + if (this.containerStatuses == null) + return; + List protoList = + new ArrayList(); + for (ContainerStatus status : containerStatuses) { + protoList.add(convertToProtoFormat(status)); + } + builder.addAllStatus(protoList); + } + + private void addFailedRequestsToProto() { + maybeInitBuilder(); + builder.clearFailedRequests(); + if (this.failedRequests == null) + return; + List protoList = + new ArrayList(); + for (Map.Entry entry : this.failedRequests + .entrySet()) { + protoList.add(ContainerExceptionMapProto.newBuilder() + .setContainerId(convertToProtoFormat(entry.getKey())) + .setException(convertToProtoFormat(entry.getValue())).build()); + } + builder.addAllFailedRequests(protoList); + } + + private void initLocalContainerStatuses() { + if (this.containerStatuses != null) { + return; + } + GetContainerStatusesResponseProtoOrBuilder p = viaProto ? proto : builder; + List statuses = p.getStatusList(); + this.containerStatuses = new ArrayList(); + for (ContainerStatusProto status : statuses) { + this.containerStatuses.add(convertFromProtoFormat(status)); + } + } + + private void initFailedRequests() { + if (this.failedRequests != null) { + return; + } + GetContainerStatusesResponseProtoOrBuilder p = viaProto ? proto : builder; + List protoList = p.getFailedRequestsList(); + this.failedRequests = new HashMap(); + for (ContainerExceptionMapProto ce : protoList) { + this.failedRequests.put(convertFromProtoFormat(ce.getContainerId()), + convertFromProtoFormat(ce.getException())); + } + } + + @Override + public List getContainerStatuses() { + initLocalContainerStatuses(); + return this.containerStatuses; + } + + @Override + public void setContainerStatuses(List statuses) { + maybeInitBuilder(); + if (statuses == null) + builder.clearStatus(); + this.containerStatuses = statuses; + } + + @Override + public Map getFailedRequests() { + initFailedRequests(); + return this.failedRequests; + } + + @Override + public void setFailedRequests( + Map failedRequests) { + maybeInitBuilder(); + if (failedRequests == null) + builder.clearFailedRequests(); + this.failedRequests = failedRequests; + } + + private ContainerStatusPBImpl convertFromProtoFormat(ContainerStatusProto p) { + return new ContainerStatusPBImpl(p); + } + + private ContainerStatusProto convertToProtoFormat(ContainerStatus t) { + return ((ContainerStatusPBImpl) t).getProto(); + } + + private ContainerIdPBImpl convertFromProtoFormat(ContainerIdProto p) { + return new ContainerIdPBImpl(p); + } + + private ContainerIdProto convertToProtoFormat(ContainerId t) { + return ((ContainerIdPBImpl) t).getProto(); + } + + private SerializedExceptionPBImpl convertFromProtoFormat( + SerializedExceptionProto p) { + return new SerializedExceptionPBImpl(p); + } + + private SerializedExceptionProto convertToProtoFormat(SerializedException t) { + return ((SerializedExceptionPBImpl) t).getProto(); + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/StartContainerResponsePBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/StartContainerResponsePBImpl.java deleted file mode 100644 index 8edbfa78690..00000000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/StartContainerResponsePBImpl.java +++ /dev/null @@ -1,176 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.yarn.api.protocolrecords.impl.pb; - - -import java.nio.ByteBuffer; -import java.util.HashMap; -import java.util.Iterator; -import java.util.List; -import java.util.Map; - -import org.apache.hadoop.classification.InterfaceAudience.Private; -import org.apache.hadoop.classification.InterfaceStability.Unstable; -import org.apache.hadoop.yarn.api.protocolrecords.StartContainerResponse; -import org.apache.hadoop.yarn.api.records.impl.pb.ProtoUtils; -import org.apache.hadoop.yarn.proto.YarnProtos.StringBytesMapProto; -import org.apache.hadoop.yarn.proto.YarnServiceProtos.StartContainerResponseProto; -import org.apache.hadoop.yarn.proto.YarnServiceProtos.StartContainerResponseProtoOrBuilder; - -import com.google.protobuf.ByteString; - -@Private -@Unstable -public class StartContainerResponsePBImpl extends StartContainerResponse { - StartContainerResponseProto proto = StartContainerResponseProto.getDefaultInstance(); - StartContainerResponseProto.Builder builder = null; - boolean viaProto = false; - - private Map servicesMetaData = null; - - public StartContainerResponsePBImpl() { - builder = StartContainerResponseProto.newBuilder(); - } - - public StartContainerResponsePBImpl(StartContainerResponseProto proto) { - this.proto = proto; - viaProto = true; - } - - public synchronized StartContainerResponseProto getProto() { - mergeLocalToProto(); - proto = viaProto ? proto : builder.build(); - viaProto = true; - return proto; - } - - @Override - public int hashCode() { - return getProto().hashCode(); - } - - @Override - public boolean equals(Object other) { - if (other == null) - return false; - if (other.getClass().isAssignableFrom(this.getClass())) { - return this.getProto().equals(this.getClass().cast(other).getProto()); - } - return false; - } - - @Override - public String toString() { - return getProto().toString().replaceAll("\\n", ", ").replaceAll("\\s+", " "); - } - - private synchronized void mergeLocalToBuilder() { - if (this.servicesMetaData != null) { - addServicesMetaDataToProto(); - } - } - - protected final ByteBuffer convertFromProtoFormat(ByteString byteString) { - return ProtoUtils.convertFromProtoFormat(byteString); - } - - protected final ByteString convertToProtoFormat(ByteBuffer byteBuffer) { - return ProtoUtils.convertToProtoFormat(byteBuffer); - } - - private synchronized void mergeLocalToProto() { - if (viaProto) { - maybeInitBuilder(); - } - mergeLocalToBuilder(); - proto = builder.build(); - viaProto = true; - } - - private synchronized void maybeInitBuilder() { - if (viaProto || builder == null) { - builder = StartContainerResponseProto.newBuilder(proto); - } - viaProto = false; - } - - - @Override - public synchronized Map getAllServicesMetaData() { - initServicesMetaData(); - return this.servicesMetaData; - } - @Override - public synchronized void setAllServicesMetaData( - Map servicesMetaData) { - if(servicesMetaData == null) { - return; - } - initServicesMetaData(); - this.servicesMetaData.clear(); - this.servicesMetaData.putAll(servicesMetaData); - } - - private synchronized void initServicesMetaData() { - if (this.servicesMetaData != null) { - return; - } - StartContainerResponseProtoOrBuilder p = viaProto ? proto : builder; - List list = p.getServicesMetaDataList(); - this.servicesMetaData = new HashMap(); - - for (StringBytesMapProto c : list) { - this.servicesMetaData.put(c.getKey(), convertFromProtoFormat(c.getValue())); - } - } - - private synchronized void addServicesMetaDataToProto() { - maybeInitBuilder(); - builder.clearServicesMetaData(); - if (servicesMetaData == null) - return; - Iterable iterable = new Iterable() { - - @Override - public synchronized Iterator iterator() { - return new Iterator() { - - Iterator keyIter = servicesMetaData.keySet().iterator(); - - @Override - public synchronized void remove() { - throw new UnsupportedOperationException(); - } - - @Override - public synchronized StringBytesMapProto next() { - String key = keyIter.next(); - return StringBytesMapProto.newBuilder().setKey(key).setValue(convertToProtoFormat(servicesMetaData.get(key))).build(); - } - - @Override - public synchronized boolean hasNext() { - return keyIter.hasNext(); - } - }; - } - }; - builder.addAllServicesMetaData(iterable); - } -} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/StartContainersRequestPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/StartContainersRequestPBImpl.java new file mode 100644 index 00000000000..2233705bb96 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/StartContainersRequestPBImpl.java @@ -0,0 +1,139 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.api.protocolrecords.impl.pb; + +import java.util.ArrayList; +import java.util.List; + +import org.apache.hadoop.yarn.api.protocolrecords.StartContainerRequest; +import org.apache.hadoop.yarn.api.protocolrecords.StartContainersRequest; +import org.apache.hadoop.yarn.proto.YarnServiceProtos.StartContainerRequestProto; +import org.apache.hadoop.yarn.proto.YarnServiceProtos.StartContainersRequestProto; +import org.apache.hadoop.yarn.proto.YarnServiceProtos.StartContainersRequestProtoOrBuilder; + +public class StartContainersRequestPBImpl extends StartContainersRequest { + StartContainersRequestProto proto = StartContainersRequestProto + .getDefaultInstance(); + StartContainersRequestProto.Builder builder = null; + boolean viaProto = false; + + private List requests = null; + + public StartContainersRequestPBImpl() { + builder = StartContainersRequestProto.newBuilder(); + } + + public StartContainersRequestPBImpl(StartContainersRequestProto proto) { + this.proto = proto; + viaProto = true; + } + + public StartContainersRequestProto getProto() { + mergeLocalToProto(); + proto = viaProto ? proto : builder.build(); + viaProto = true; + return proto; + } + + @Override + public int hashCode() { + return getProto().hashCode(); + } + + @Override + public boolean equals(Object other) { + if (other == null) + return false; + if (other.getClass().isAssignableFrom(this.getClass())) { + return this.getProto().equals(this.getClass().cast(other).getProto()); + } + return false; + } + + private void mergeLocalToProto() { + if (viaProto) + maybeInitBuilder(); + mergeLocalToBuilder(); + proto = builder.build(); + viaProto = true; + } + + private void mergeLocalToBuilder() { + if (requests != null) { + addLocalRequestsToProto(); + } + } + + + private void maybeInitBuilder() { + if (viaProto || builder == null) { + builder = StartContainersRequestProto.newBuilder(proto); + } + viaProto = false; + } + + private void addLocalRequestsToProto() { + maybeInitBuilder(); + builder.clearStartContainerRequest(); + List protoList = + new ArrayList(); + for (StartContainerRequest r : this.requests) { + protoList.add(convertToProtoFormat(r)); + } + builder.addAllStartContainerRequest(protoList); + } + + private void initLocalRequests() { + StartContainersRequestProtoOrBuilder p = viaProto ? proto : builder; + List requestList = + p.getStartContainerRequestList(); + this.requests = new ArrayList(); + for (StartContainerRequestProto r : requestList) { + this.requests.add(convertFromProtoFormat(r)); + } + } + + @Override + public void setStartContainerRequests(List requests) { + maybeInitBuilder(); + if (requests == null) { + builder.clearStartContainerRequest(); + } + this.requests = requests; + } + + @Override + public List getStartContainerRequests() { + if (this.requests != null) { + return this.requests; + } + initLocalRequests(); + return this.requests; + } + + private StartContainerRequestPBImpl convertFromProtoFormat( + StartContainerRequestProto p) { + return new StartContainerRequestPBImpl(p); + } + + private StartContainerRequestProto convertToProtoFormat( + StartContainerRequest t) { + return ((StartContainerRequestPBImpl) t).getProto(); + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/StartContainersResponsePBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/StartContainersResponsePBImpl.java new file mode 100644 index 00000000000..1482cd779b0 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/StartContainersResponsePBImpl.java @@ -0,0 +1,319 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.api.protocolrecords.impl.pb; + +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; + +import org.apache.hadoop.classification.InterfaceAudience.Private; +import org.apache.hadoop.classification.InterfaceStability.Unstable; +import org.apache.hadoop.yarn.api.protocolrecords.StartContainersResponse; +import org.apache.hadoop.yarn.api.records.ContainerId; +import org.apache.hadoop.yarn.api.records.SerializedException; +import org.apache.hadoop.yarn.api.records.impl.pb.ContainerIdPBImpl; +import org.apache.hadoop.yarn.api.records.impl.pb.ProtoUtils; +import org.apache.hadoop.yarn.api.records.impl.pb.SerializedExceptionPBImpl; +import org.apache.hadoop.yarn.proto.YarnProtos.ContainerIdProto; +import org.apache.hadoop.yarn.proto.YarnProtos.SerializedExceptionProto; +import org.apache.hadoop.yarn.proto.YarnProtos.StringBytesMapProto; +import org.apache.hadoop.yarn.proto.YarnServiceProtos.ContainerExceptionMapProto; +import org.apache.hadoop.yarn.proto.YarnServiceProtos.StartContainersResponseProto; +import org.apache.hadoop.yarn.proto.YarnServiceProtos.StartContainersResponseProtoOrBuilder; + +import com.google.protobuf.ByteString; + +@Private +@Unstable +public class StartContainersResponsePBImpl extends StartContainersResponse { + StartContainersResponseProto proto = StartContainersResponseProto + .getDefaultInstance(); + StartContainersResponseProto.Builder builder = null; + boolean viaProto = false; + + private Map servicesMetaData = null; + private List succeededContainers = null; + private Map failedContainers = null; + + public StartContainersResponsePBImpl() { + builder = StartContainersResponseProto.newBuilder(); + } + + public StartContainersResponsePBImpl(StartContainersResponseProto proto) { + this.proto = proto; + viaProto = true; + } + + public StartContainersResponseProto getProto() { + mergeLocalToProto(); + proto = viaProto ? proto : builder.build(); + viaProto = true; + return proto; + } + + @Override + public int hashCode() { + return getProto().hashCode(); + } + + @Override + public boolean equals(Object other) { + if (other == null) + return false; + if (other.getClass().isAssignableFrom(this.getClass())) { + return this.getProto().equals(this.getClass().cast(other).getProto()); + } + return false; + } + + @Override + public String toString() { + return getProto().toString().replaceAll("\\n", ", ") + .replaceAll("\\s+", " "); + } + + private void mergeLocalToBuilder() { + if (this.servicesMetaData != null) { + addServicesMetaDataToProto(); + } + if (this.succeededContainers != null) { + addSucceededContainersToProto(); + } + if (this.failedContainers != null) { + addFailedContainersToProto(); + } + } + + protected final ByteBuffer convertFromProtoFormat(ByteString byteString) { + return ProtoUtils.convertFromProtoFormat(byteString); + } + + protected final ByteString convertToProtoFormat(ByteBuffer byteBuffer) { + return ProtoUtils.convertToProtoFormat(byteBuffer); + } + + private ContainerIdPBImpl convertFromProtoFormat(ContainerIdProto p) { + return new ContainerIdPBImpl(p); + } + + private ContainerIdProto convertToProtoFormat(ContainerId t) { + return ((ContainerIdPBImpl) t).getProto(); + } + + private SerializedExceptionPBImpl convertFromProtoFormat( + SerializedExceptionProto p) { + return new SerializedExceptionPBImpl(p); + } + + private SerializedExceptionProto convertToProtoFormat(SerializedException t) { + return ((SerializedExceptionPBImpl) t).getProto(); + } + + private void mergeLocalToProto() { + if (viaProto) { + maybeInitBuilder(); + } + mergeLocalToBuilder(); + proto = builder.build(); + viaProto = true; + } + + private void maybeInitBuilder() { + if (viaProto || builder == null) { + builder = StartContainersResponseProto.newBuilder(proto); + } + viaProto = false; + } + + @Override + public Map getAllServicesMetaData() { + initServicesMetaData(); + return this.servicesMetaData; + } + + @Override + public void setAllServicesMetaData(Map servicesMetaData) { + if (servicesMetaData == null) { + return; + } + initServicesMetaData(); + this.servicesMetaData.clear(); + this.servicesMetaData.putAll(servicesMetaData); + } + + private void initServicesMetaData() { + if (this.servicesMetaData != null) { + return; + } + StartContainersResponseProtoOrBuilder p = viaProto ? proto : builder; + List list = p.getServicesMetaDataList(); + this.servicesMetaData = new HashMap(); + + for (StringBytesMapProto c : list) { + this.servicesMetaData.put(c.getKey(), + convertFromProtoFormat(c.getValue())); + } + } + + private void addServicesMetaDataToProto() { + maybeInitBuilder(); + builder.clearServicesMetaData(); + if (servicesMetaData == null) + return; + Iterable iterable = + new Iterable() { + + @Override + public Iterator iterator() { + return new Iterator() { + + Iterator keyIter = servicesMetaData.keySet().iterator(); + + @Override + public void remove() { + throw new UnsupportedOperationException(); + } + + @Override + public StringBytesMapProto next() { + String key = keyIter.next(); + return StringBytesMapProto.newBuilder().setKey(key) + .setValue(convertToProtoFormat(servicesMetaData.get(key))) + .build(); + } + + @Override + public boolean hasNext() { + return keyIter.hasNext(); + } + }; + } + }; + builder.addAllServicesMetaData(iterable); + } + + private void addFailedContainersToProto() { + maybeInitBuilder(); + builder.clearFailedRequests(); + if (this.failedContainers == null) + return; + List protoList = + new ArrayList(); + + for (Map.Entry entry : this.failedContainers + .entrySet()) { + protoList.add(ContainerExceptionMapProto.newBuilder() + .setContainerId(convertToProtoFormat(entry.getKey())) + .setException(convertToProtoFormat(entry.getValue())).build()); + } + builder.addAllFailedRequests(protoList); + } + + private void addSucceededContainersToProto() { + maybeInitBuilder(); + builder.clearSucceededRequests(); + if (this.succeededContainers == null) { + return; + } + Iterable iterable = new Iterable() { + @Override + public Iterator iterator() { + return new Iterator() { + + Iterator iter = succeededContainers.iterator(); + + @Override + public boolean hasNext() { + return iter.hasNext(); + } + + @Override + public ContainerIdProto next() { + return convertToProtoFormat(iter.next()); + } + + @Override + public void remove() { + throw new UnsupportedOperationException(); + + } + }; + } + }; + builder.addAllSucceededRequests(iterable); + } + + private void initSucceededContainers() { + if (this.succeededContainers != null) + return; + StartContainersResponseProtoOrBuilder p = viaProto ? proto : builder; + List list = p.getSucceededRequestsList(); + this.succeededContainers = new ArrayList(); + for (ContainerIdProto c : list) { + this.succeededContainers.add(convertFromProtoFormat(c)); + } + } + + @Override + public List getSuccessfullyStartedContainers() { + initSucceededContainers(); + return this.succeededContainers; + } + + @Override + public void setSuccessfullyStartedContainers( + List succeededContainers) { + maybeInitBuilder(); + if (succeededContainers == null) { + builder.clearSucceededRequests(); + } + this.succeededContainers = succeededContainers; + } + + private void initFailedContainers() { + if (this.failedContainers != null) { + return; + } + StartContainersResponseProtoOrBuilder p = viaProto ? proto : builder; + List protoList = p.getFailedRequestsList(); + this.failedContainers = new HashMap(); + for (ContainerExceptionMapProto ce : protoList) { + this.failedContainers.put(convertFromProtoFormat(ce.getContainerId()), + convertFromProtoFormat(ce.getException())); + } + } + + @Override + public Map getFailedRequests() { + initFailedContainers(); + return this.failedContainers; + } + + @Override + public void setFailedRequests( + Map failedContainers) { + maybeInitBuilder(); + if (failedContainers == null) + builder.clearFailedRequests(); + this.failedContainers = failedContainers; + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/StopContainerRequestPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/StopContainerRequestPBImpl.java deleted file mode 100644 index dfba527e7cd..00000000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/StopContainerRequestPBImpl.java +++ /dev/null @@ -1,130 +0,0 @@ -/** -* Licensed to the Apache Software Foundation (ASF) under one -* or more contributor license agreements. See the NOTICE file -* distributed with this work for additional information -* regarding copyright ownership. The ASF licenses this file -* to you under the Apache License, Version 2.0 (the -* "License"); you may not use this file except in compliance -* with the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -*/ - -package org.apache.hadoop.yarn.api.protocolrecords.impl.pb; - - -import org.apache.hadoop.classification.InterfaceAudience.Private; -import org.apache.hadoop.classification.InterfaceStability.Unstable; -import org.apache.hadoop.yarn.api.protocolrecords.StopContainerRequest; -import org.apache.hadoop.yarn.api.records.ContainerId; -import org.apache.hadoop.yarn.api.records.impl.pb.ContainerIdPBImpl; -import org.apache.hadoop.yarn.proto.YarnProtos.ContainerIdProto; -import org.apache.hadoop.yarn.proto.YarnServiceProtos.StopContainerRequestProto; -import org.apache.hadoop.yarn.proto.YarnServiceProtos.StopContainerRequestProtoOrBuilder; - -@Private -@Unstable -public class StopContainerRequestPBImpl extends StopContainerRequest { - StopContainerRequestProto proto = StopContainerRequestProto.getDefaultInstance(); - StopContainerRequestProto.Builder builder = null; - boolean viaProto = false; - - private ContainerId containerId = null; - - - public StopContainerRequestPBImpl() { - builder = StopContainerRequestProto.newBuilder(); - } - - public StopContainerRequestPBImpl(StopContainerRequestProto proto) { - this.proto = proto; - viaProto = true; - } - - public StopContainerRequestProto getProto() { - mergeLocalToProto(); - proto = viaProto ? proto : builder.build(); - viaProto = true; - return proto; - } - - @Override - public int hashCode() { - return getProto().hashCode(); - } - - @Override - public boolean equals(Object other) { - if (other == null) - return false; - if (other.getClass().isAssignableFrom(this.getClass())) { - return this.getProto().equals(this.getClass().cast(other).getProto()); - } - return false; - } - - @Override - public String toString() { - return getProto().toString().replaceAll("\\n", ", ").replaceAll("\\s+", " "); - } - - private void mergeLocalToBuilder() { - if (this.containerId != null) { - builder.setContainerId(convertToProtoFormat(this.containerId)); - } - } - - private void mergeLocalToProto() { - if (viaProto) - maybeInitBuilder(); - mergeLocalToBuilder(); - proto = builder.build(); - viaProto = true; - } - - private void maybeInitBuilder() { - if (viaProto || builder == null) { - builder = StopContainerRequestProto.newBuilder(proto); - } - viaProto = false; - } - - - @Override - public ContainerId getContainerId() { - StopContainerRequestProtoOrBuilder p = viaProto ? proto : builder; - if (this.containerId != null) { - return this.containerId; - } - if (!p.hasContainerId()) { - return null; - } - this.containerId = convertFromProtoFormat(p.getContainerId()); - return this.containerId; - } - - @Override - public void setContainerId(ContainerId containerId) { - maybeInitBuilder(); - if (containerId == null) - builder.clearContainerId(); - this.containerId = containerId; - } - - private ContainerIdPBImpl convertFromProtoFormat(ContainerIdProto p) { - return new ContainerIdPBImpl(p); - } - - private ContainerIdProto convertToProtoFormat(ContainerId t) { - return ((ContainerIdPBImpl)t).getProto(); - } - - - -} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/StopContainerResponsePBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/StopContainerResponsePBImpl.java deleted file mode 100644 index 6e3fe216e39..00000000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/StopContainerResponsePBImpl.java +++ /dev/null @@ -1,68 +0,0 @@ -/** -* Licensed to the Apache Software Foundation (ASF) under one -* or more contributor license agreements. See the NOTICE file -* distributed with this work for additional information -* regarding copyright ownership. The ASF licenses this file -* to you under the Apache License, Version 2.0 (the -* "License"); you may not use this file except in compliance -* with the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -*/ - -package org.apache.hadoop.yarn.api.protocolrecords.impl.pb; - - -import org.apache.hadoop.classification.InterfaceAudience.Private; -import org.apache.hadoop.classification.InterfaceStability.Unstable; -import org.apache.hadoop.yarn.api.protocolrecords.StopContainerResponse; -import org.apache.hadoop.yarn.proto.YarnServiceProtos.StopContainerResponseProto; - -@Private -@Unstable -public class StopContainerResponsePBImpl extends StopContainerResponse { - StopContainerResponseProto proto = StopContainerResponseProto.getDefaultInstance(); - StopContainerResponseProto.Builder builder = null; - boolean viaProto = false; - - public StopContainerResponsePBImpl() { - builder = StopContainerResponseProto.newBuilder(); - } - - public StopContainerResponsePBImpl(StopContainerResponseProto proto) { - this.proto = proto; - viaProto = true; - } - - public StopContainerResponseProto getProto() { - proto = viaProto ? proto : builder.build(); - viaProto = true; - return proto; - } - - @Override - public int hashCode() { - return getProto().hashCode(); - } - - @Override - public boolean equals(Object other) { - if (other == null) - return false; - if (other.getClass().isAssignableFrom(this.getClass())) { - return this.getProto().equals(this.getClass().cast(other).getProto()); - } - return false; - } - - @Override - public String toString() { - return getProto().toString().replaceAll("\\n", ", ").replaceAll("\\s+", " "); - } -} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/StopContainersRequestPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/StopContainersRequestPBImpl.java new file mode 100644 index 00000000000..5c758e80629 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/StopContainersRequestPBImpl.java @@ -0,0 +1,146 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.api.protocolrecords.impl.pb; + +import java.util.ArrayList; +import java.util.List; + +import org.apache.hadoop.classification.InterfaceAudience.Private; +import org.apache.hadoop.classification.InterfaceStability.Unstable; +import org.apache.hadoop.yarn.api.protocolrecords.StopContainersRequest; +import org.apache.hadoop.yarn.api.records.ContainerId; +import org.apache.hadoop.yarn.api.records.impl.pb.ContainerIdPBImpl; +import org.apache.hadoop.yarn.proto.YarnProtos.ContainerIdProto; +import org.apache.hadoop.yarn.proto.YarnServiceProtos.StopContainersRequestProto; +import org.apache.hadoop.yarn.proto.YarnServiceProtos.StopContainersRequestProtoOrBuilder; + +@Private +@Unstable +public class StopContainersRequestPBImpl extends StopContainersRequest { + StopContainersRequestProto proto = StopContainersRequestProto + .getDefaultInstance(); + StopContainersRequestProto.Builder builder = null; + boolean viaProto = false; + + private List containerIds = null; + + public StopContainersRequestPBImpl() { + builder = StopContainersRequestProto.newBuilder(); + } + + public StopContainersRequestPBImpl(StopContainersRequestProto proto) { + this.proto = proto; + viaProto = true; + } + + public StopContainersRequestProto getProto() { + mergeLocalToProto(); + proto = viaProto ? proto : builder.build(); + viaProto = true; + return proto; + } + + @Override + public int hashCode() { + return getProto().hashCode(); + } + + @Override + public boolean equals(Object other) { + if (other == null) + return false; + if (other.getClass().isAssignableFrom(this.getClass())) { + return this.getProto().equals(this.getClass().cast(other).getProto()); + } + return false; + } + + @Override + public String toString() { + return getProto().toString().replaceAll("\\n", ", ") + .replaceAll("\\s+", " "); + } + + private void mergeLocalToBuilder() { + if (this.containerIds != null) { + addLocalContainerIdsToProto(); + } + } + + private void mergeLocalToProto() { + if (viaProto) + maybeInitBuilder(); + mergeLocalToBuilder(); + proto = builder.build(); + viaProto = true; + } + + private void maybeInitBuilder() { + if (viaProto || builder == null) { + builder = StopContainersRequestProto.newBuilder(proto); + } + viaProto = false; + } + + private void addLocalContainerIdsToProto() { + maybeInitBuilder(); + builder.clearContainerId(); + if (this.containerIds == null) + return; + List protoList = new ArrayList(); + for (ContainerId id : containerIds) { + protoList.add(convertToProtoFormat(id)); + } + builder.addAllContainerId(protoList); + } + + private void initLocalContainerIds() { + if (this.containerIds != null) { + return; + } + StopContainersRequestProtoOrBuilder p = viaProto ? proto : builder; + List containerIds = p.getContainerIdList(); + this.containerIds = new ArrayList(); + for (ContainerIdProto id : containerIds) { + this.containerIds.add(convertFromProtoFormat(id)); + } + } + + @Override + public List getContainerIds() { + initLocalContainerIds(); + return this.containerIds; + } + + @Override + public void setContainerIds(List containerIds) { + maybeInitBuilder(); + if (containerIds == null) + builder.clearContainerId(); + this.containerIds = containerIds; + } + + private ContainerIdPBImpl convertFromProtoFormat(ContainerIdProto p) { + return new ContainerIdPBImpl(p); + } + + private ContainerIdProto convertToProtoFormat(ContainerId t) { + return ((ContainerIdPBImpl) t).getProto(); + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/StopContainersResponsePBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/StopContainersResponsePBImpl.java new file mode 100644 index 00000000000..5385d0a0ab2 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/StopContainersResponsePBImpl.java @@ -0,0 +1,234 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.api.protocolrecords.impl.pb; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; + +import org.apache.hadoop.classification.InterfaceAudience.Private; +import org.apache.hadoop.classification.InterfaceStability.Unstable; +import org.apache.hadoop.yarn.api.protocolrecords.StopContainersResponse; +import org.apache.hadoop.yarn.api.records.ContainerId; +import org.apache.hadoop.yarn.api.records.SerializedException; +import org.apache.hadoop.yarn.api.records.impl.pb.ContainerIdPBImpl; +import org.apache.hadoop.yarn.api.records.impl.pb.SerializedExceptionPBImpl; +import org.apache.hadoop.yarn.proto.YarnProtos.ContainerIdProto; +import org.apache.hadoop.yarn.proto.YarnProtos.SerializedExceptionProto; +import org.apache.hadoop.yarn.proto.YarnServiceProtos.ContainerExceptionMapProto; +import org.apache.hadoop.yarn.proto.YarnServiceProtos.StopContainersResponseProto; +import org.apache.hadoop.yarn.proto.YarnServiceProtos.StopContainersResponseProtoOrBuilder; + +@Private +@Unstable +public class StopContainersResponsePBImpl extends StopContainersResponse { + StopContainersResponseProto proto = StopContainersResponseProto + .getDefaultInstance(); + StopContainersResponseProto.Builder builder = null; + boolean viaProto = false; + private List succeededRequests = null; + private Map failedRequests = null; + + public StopContainersResponsePBImpl() { + builder = StopContainersResponseProto.newBuilder(); + } + + public StopContainersResponsePBImpl(StopContainersResponseProto proto) { + this.proto = proto; + viaProto = true; + } + + public StopContainersResponseProto getProto() { + mergeLocalToProto(); + proto = viaProto ? proto : builder.build(); + viaProto = true; + return proto; + } + + @Override + public int hashCode() { + return getProto().hashCode(); + } + + @Override + public boolean equals(Object other) { + if (other == null) + return false; + if (other.getClass().isAssignableFrom(this.getClass())) { + return this.getProto().equals(this.getClass().cast(other).getProto()); + } + return false; + } + + @Override + public String toString() { + return getProto().toString().replaceAll("\\n", ", ") + .replaceAll("\\s+", " "); + } + + private void mergeLocalToProto() { + if (viaProto) { + maybeInitBuilder(); + } + mergeLocalToBuilder(); + proto = builder.build(); + viaProto = true; + } + + private void maybeInitBuilder() { + if (viaProto || builder == null) { + builder = StopContainersResponseProto.newBuilder(proto); + } + viaProto = false; + } + + private void mergeLocalToBuilder() { + + if (this.succeededRequests != null) { + addSucceededRequestsToProto(); + } + if (this.failedRequests != null) { + addFailedRequestsToProto(); + } + } + + private void addSucceededRequestsToProto() { + maybeInitBuilder(); + builder.clearSucceededRequests(); + if (this.succeededRequests == null) { + return; + } + Iterable iterable = new Iterable() { + @Override + public Iterator iterator() { + return new Iterator() { + + Iterator iter = succeededRequests.iterator(); + + @Override + public boolean hasNext() { + return iter.hasNext(); + } + + @Override + public ContainerIdProto next() { + return convertToProtoFormat(iter.next()); + } + + @Override + public void remove() { + throw new UnsupportedOperationException(); + + } + }; + } + }; + builder.addAllSucceededRequests(iterable); + } + + private void addFailedRequestsToProto() { + maybeInitBuilder(); + builder.clearFailedRequests(); + if (this.failedRequests == null) + return; + List protoList = + new ArrayList(); + + for (Map.Entry entry : this.failedRequests + .entrySet()) { + protoList.add(ContainerExceptionMapProto.newBuilder() + .setContainerId(convertToProtoFormat(entry.getKey())) + .setException(convertToProtoFormat(entry.getValue())).build()); + } + builder.addAllFailedRequests(protoList); + } + + private void initSucceededRequests() { + if (this.succeededRequests != null) + return; + StopContainersResponseProtoOrBuilder p = viaProto ? proto : builder; + List list = p.getSucceededRequestsList(); + this.succeededRequests = new ArrayList(); + for (ContainerIdProto c : list) { + this.succeededRequests.add(convertFromProtoFormat(c)); + } + } + + private void initFailedRequests() { + if (this.failedRequests != null) { + return; + } + StopContainersResponseProtoOrBuilder p = viaProto ? proto : builder; + List protoList = p.getFailedRequestsList(); + this.failedRequests = new HashMap(); + for (ContainerExceptionMapProto ce : protoList) { + this.failedRequests.put(convertFromProtoFormat(ce.getContainerId()), + convertFromProtoFormat(ce.getException())); + } + } + + @Override + public List getSuccessfullyStoppedContainers() { + initSucceededRequests(); + return this.succeededRequests; + } + + @Override + public void setSuccessfullyStoppedContainers(List succeededRequests) { + maybeInitBuilder(); + if (succeededRequests == null) { + builder.clearSucceededRequests(); + } + this.succeededRequests = succeededRequests; + } + + @Override + public Map getFailedRequests() { + initFailedRequests(); + return this.failedRequests; + } + + @Override + public void setFailedRequests( + Map failedRequests) { + maybeInitBuilder(); + if (failedRequests == null) + builder.clearFailedRequests(); + this.failedRequests = failedRequests; + } + + private ContainerIdPBImpl convertFromProtoFormat(ContainerIdProto p) { + return new ContainerIdPBImpl(p); + } + + private ContainerIdProto convertToProtoFormat(ContainerId t) { + return ((ContainerIdPBImpl) t).getProto(); + } + + private SerializedExceptionPBImpl convertFromProtoFormat( + SerializedExceptionProto p) { + return new SerializedExceptionPBImpl(p); + } + + private SerializedExceptionProto convertToProtoFormat(SerializedException t) { + return ((SerializedExceptionPBImpl) t).getProto(); + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/impl/pb/SerializedExceptionPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/SerializedExceptionPBImpl.java similarity index 58% rename from hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/impl/pb/SerializedExceptionPBImpl.java rename to hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/SerializedExceptionPBImpl.java index 428cfe41ebd..5290013a088 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/impl/pb/SerializedExceptionPBImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/SerializedExceptionPBImpl.java @@ -16,14 +16,19 @@ * limitations under the License. */ -package org.apache.hadoop.yarn.server.api.records.impl.pb; +package org.apache.hadoop.yarn.api.records.impl.pb; +import java.io.IOException; import java.io.PrintWriter; import java.io.StringWriter; +import java.lang.reflect.Constructor; +import java.lang.reflect.InvocationTargetException; +import org.apache.hadoop.yarn.api.records.SerializedException; +import org.apache.hadoop.yarn.exceptions.YarnException; +import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; import org.apache.hadoop.yarn.proto.YarnProtos.SerializedExceptionProto; import org.apache.hadoop.yarn.proto.YarnProtos.SerializedExceptionProtoOrBuilder; -import org.apache.hadoop.yarn.server.api.records.SerializedException; public class SerializedExceptionPBImpl extends SerializedException { @@ -58,7 +63,6 @@ public class SerializedExceptionPBImpl extends SerializedException { if (t.getCause() == null) { } else { builder.setCause(new SerializedExceptionPBImpl(t.getCause()).getProto()); - builder.setClassName(t.getClass().getCanonicalName()); } StringWriter sw = new StringWriter(); PrintWriter pw = new PrintWriter(sw); @@ -68,6 +72,7 @@ public class SerializedExceptionPBImpl extends SerializedException { builder.setTrace(sw.toString()); if (t.getMessage() != null) builder.setMessage(t.getMessage()); + builder.setClassName(t.getClass().getCanonicalName()); } public void init(String message, Throwable t) { @@ -76,6 +81,32 @@ public class SerializedExceptionPBImpl extends SerializedException { builder.setMessage(message); } + @SuppressWarnings("unchecked") + @Override + public Throwable deSerialize() { + + SerializedException cause = getCause(); + SerializedExceptionProtoOrBuilder p = viaProto ? proto : builder; + Class realClass = null; + try { + realClass = Class.forName(p.getClassName()); + } catch (ClassNotFoundException e) { + throw new YarnRuntimeException(e); + } + Class classType = null; + if (YarnException.class.isAssignableFrom(realClass)) { + classType = YarnException.class; + } else if (IOException.class.isAssignableFrom(realClass)) { + classType = IOException.class; + } else if (RuntimeException.class.isAssignableFrom(realClass)) { + classType = RuntimeException.class; + } else { + classType = Exception.class; + } + return instantiateException(realClass.asSubclass(classType), getMessage(), + cause == null ? null : cause.deSerialize()); + } + @Override public String getMessage() { SerializedExceptionProtoOrBuilder p = viaProto ? proto : builder; @@ -110,4 +141,29 @@ public class SerializedExceptionPBImpl extends SerializedException { } viaProto = false; } + + private static T instantiateException( + Class cls, String message, Throwable cause) { + Constructor cn; + T ex = null; + try { + cn = cls.getConstructor(String.class); + cn.setAccessible(true); + ex = cn.newInstance(message); + ex.initCause(cause); + } catch (SecurityException e) { + throw new YarnRuntimeException(e); + } catch (NoSuchMethodException e) { + throw new YarnRuntimeException(e); + } catch (IllegalArgumentException e) { + throw new YarnRuntimeException(e); + } catch (InstantiationException e) { + throw new YarnRuntimeException(e); + } catch (IllegalAccessException e) { + throw new YarnRuntimeException(e); + } catch (InvocationTargetException e) { + throw new YarnRuntimeException(e); + } + return ex; + } } \ No newline at end of file diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/TestContainerLaunchRPC.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/TestContainerLaunchRPC.java index 0c633f169b7..8fe5c3c1684 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/TestContainerLaunchRPC.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/TestContainerLaunchRPC.java @@ -21,6 +21,8 @@ package org.apache.hadoop.yarn; import java.io.IOException; import java.net.InetSocketAddress; import java.net.SocketTimeoutException; +import java.util.ArrayList; +import java.util.List; import junit.framework.Assert; @@ -31,12 +33,13 @@ import org.apache.hadoop.ipc.Server; import org.apache.hadoop.net.NetUtils; import org.apache.hadoop.util.StringUtils; import org.apache.hadoop.yarn.api.ContainerManagementProtocol; -import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusRequest; -import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusResponse; +import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusesRequest; +import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusesResponse; import org.apache.hadoop.yarn.api.protocolrecords.StartContainerRequest; -import org.apache.hadoop.yarn.api.protocolrecords.StartContainerResponse; -import org.apache.hadoop.yarn.api.protocolrecords.StopContainerRequest; -import org.apache.hadoop.yarn.api.protocolrecords.StopContainerResponse; +import org.apache.hadoop.yarn.api.protocolrecords.StartContainersRequest; +import org.apache.hadoop.yarn.api.protocolrecords.StartContainersResponse; +import org.apache.hadoop.yarn.api.protocolrecords.StopContainersRequest; +import org.apache.hadoop.yarn.api.protocolrecords.StopContainersResponse; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ContainerId; @@ -104,12 +107,15 @@ public class TestContainerLaunchRPC { TestRPC.newContainerToken(nodeId, "password".getBytes(), containerTokenIdentifier); - StartContainerRequest scRequest = recordFactory - .newRecordInstance(StartContainerRequest.class); - scRequest.setContainerLaunchContext(containerLaunchContext); - scRequest.setContainerToken(containerToken); + StartContainerRequest scRequest = + StartContainerRequest.newInstance(containerLaunchContext, + containerToken); + List list = new ArrayList(); + list.add(scRequest); + StartContainersRequest allRequests = + StartContainersRequest.newInstance(list); try { - proxy.startContainer(scRequest); + proxy.startContainers(allRequests); } catch (Exception e) { LOG.info(StringUtils.stringifyException(e)); Assert.assertEquals("Error, exception is not: " @@ -129,17 +135,8 @@ public class TestContainerLaunchRPC { private ContainerStatus status = null; @Override - public GetContainerStatusResponse getContainerStatus( - GetContainerStatusRequest request) throws YarnException { - GetContainerStatusResponse response = recordFactory - .newRecordInstance(GetContainerStatusResponse.class); - response.setStatus(status); - return response; - } - - @Override - public StartContainerResponse startContainer(StartContainerRequest request) - throws YarnException, IOException { + public StartContainersResponse startContainers( + StartContainersRequest requests) throws YarnException, IOException { try { // make the thread sleep to look like its not going to respond Thread.sleep(10000); @@ -151,11 +148,22 @@ public class TestContainerLaunchRPC { } @Override - public StopContainerResponse stopContainer(StopContainerRequest request) - throws YarnException { + public StopContainersResponse + stopContainers(StopContainersRequest requests) throws YarnException, + IOException { Exception e = new Exception("Dummy function", new Exception( "Dummy function cause")); throw new YarnException(e); } + + @Override + public GetContainerStatusesResponse getContainerStatuses( + GetContainerStatusesRequest request) throws YarnException, IOException { + List list = new ArrayList(); + list.add(status); + GetContainerStatusesResponse response = + GetContainerStatusesResponse.newInstance(list, null); + return null; + } } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/TestRPC.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/TestRPC.java index 68aeb563fcc..76384d33f07 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/TestRPC.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/TestRPC.java @@ -20,6 +20,8 @@ package org.apache.hadoop.yarn; import java.io.IOException; import java.net.InetSocketAddress; +import java.util.ArrayList; +import java.util.List; import junit.framework.Assert; @@ -33,13 +35,14 @@ import org.apache.hadoop.security.SecurityUtil; import org.apache.hadoop.yarn.api.ApplicationClientProtocol; import org.apache.hadoop.yarn.api.ContainerManagementProtocol; import org.apache.hadoop.yarn.api.ContainerManagementProtocolPB; -import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusRequest; -import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusResponse; +import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusesRequest; +import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusesResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetNewApplicationRequest; import org.apache.hadoop.yarn.api.protocolrecords.StartContainerRequest; -import org.apache.hadoop.yarn.api.protocolrecords.StartContainerResponse; -import org.apache.hadoop.yarn.api.protocolrecords.StopContainerRequest; -import org.apache.hadoop.yarn.api.protocolrecords.StopContainerResponse; +import org.apache.hadoop.yarn.api.protocolrecords.StartContainersRequest; +import org.apache.hadoop.yarn.api.protocolrecords.StartContainersResponse; +import org.apache.hadoop.yarn.api.protocolrecords.StopContainersRequest; +import org.apache.hadoop.yarn.api.protocolrecords.StopContainersResponse; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ContainerId; @@ -122,9 +125,6 @@ public class TestRPC { ApplicationAttemptId.newInstance(applicationId, 0); ContainerId containerId = ContainerId.newInstance(applicationAttemptId, 100); - StartContainerRequest scRequest = - recordFactory.newRecordInstance(StartContainerRequest.class); - scRequest.setContainerLaunchContext(containerLaunchContext); NodeId nodeId = NodeId.newInstance("localhost", 1234); Resource resource = Resource.newInstance(1234, 2); ContainerTokenIdentifier containerTokenIdentifier = @@ -132,22 +132,32 @@ public class TestRPC { resource, System.currentTimeMillis() + 10000, 42, 42); Token containerToken = newContainerToken(nodeId, "password".getBytes(), containerTokenIdentifier); - scRequest.setContainerToken(containerToken); - proxy.startContainer(scRequest); - - GetContainerStatusRequest gcsRequest = - recordFactory.newRecordInstance(GetContainerStatusRequest.class); - gcsRequest.setContainerId(containerId); - GetContainerStatusResponse response = proxy.getContainerStatus(gcsRequest); - ContainerStatus status = response.getStatus(); - + + StartContainerRequest scRequest = + StartContainerRequest.newInstance(containerLaunchContext, + containerToken); + List list = new ArrayList(); + list.add(scRequest); + StartContainersRequest allRequests = + StartContainersRequest.newInstance(list); + proxy.startContainers(allRequests); + + List containerIds = new ArrayList(); + containerIds.add(containerId); + GetContainerStatusesRequest gcsRequest = + GetContainerStatusesRequest.newInstance(containerIds); + GetContainerStatusesResponse response = + proxy.getContainerStatuses(gcsRequest); + List statuses = response.getContainerStatuses(); + //test remote exception boolean exception = false; try { - StopContainerRequest stopRequest = recordFactory.newRecordInstance(StopContainerRequest.class); - stopRequest.setContainerId(containerId); - proxy.stopContainer(stopRequest); - } catch (YarnException e) { + StopContainersRequest stopRequest = + recordFactory.newRecordInstance(StopContainersRequest.class); + stopRequest.setContainerIds(containerIds); + proxy.stopContainers(stopRequest); + } catch (YarnException e) { exception = true; Assert.assertTrue(e.getMessage().contains(EXCEPTION_MSG)); Assert.assertTrue(e.getMessage().contains(EXCEPTION_CAUSE)); @@ -158,46 +168,51 @@ public class TestRPC { Assert.assertTrue(exception); server.stop(); - Assert.assertNotNull(status); - Assert.assertEquals(ContainerState.RUNNING, status.getState()); + Assert.assertNotNull(statuses.get(0)); + Assert.assertEquals(ContainerState.RUNNING, statuses.get(0).getState()); } public class DummyContainerManager implements ContainerManagementProtocol { - private ContainerStatus status = null; - + private List statuses = new ArrayList(); + @Override - public GetContainerStatusResponse getContainerStatus( - GetContainerStatusRequest request) + public GetContainerStatusesResponse getContainerStatuses( + GetContainerStatusesRequest request) throws YarnException { - GetContainerStatusResponse response = - recordFactory.newRecordInstance(GetContainerStatusResponse.class); - response.setStatus(status); + GetContainerStatusesResponse response = + recordFactory.newRecordInstance(GetContainerStatusesResponse.class); + response.setContainerStatuses(statuses); return response; } @Override - public StartContainerResponse startContainer(StartContainerRequest request) - throws YarnException { - Token containerToken = request.getContainerToken(); - ContainerTokenIdentifier tokenId = null; + public StartContainersResponse startContainers( + StartContainersRequest requests) throws YarnException { + StartContainersResponse response = + recordFactory.newRecordInstance(StartContainersResponse.class); + for (StartContainerRequest request : requests.getStartContainerRequests()) { + Token containerToken = request.getContainerToken(); + ContainerTokenIdentifier tokenId = null; + + try { + tokenId = newContainerTokenIdentifier(containerToken); + } catch (IOException e) { + throw RPCUtil.getRemoteException(e); + } + ContainerStatus status = + recordFactory.newRecordInstance(ContainerStatus.class); + status.setState(ContainerState.RUNNING); + status.setContainerId(tokenId.getContainerID()); + status.setExitStatus(0); + statuses.add(status); - try { - tokenId = newContainerTokenIdentifier(containerToken); - } catch (IOException e) { - throw RPCUtil.getRemoteException(e); } - StartContainerResponse response = - recordFactory.newRecordInstance(StartContainerResponse.class); - status = recordFactory.newRecordInstance(ContainerStatus.class); - status.setState(ContainerState.RUNNING); - status.setContainerId(tokenId.getContainerID()); - status.setExitStatus(0); return response; } @Override - public StopContainerResponse stopContainer(StopContainerRequest request) + public StopContainersResponse stopContainers(StopContainersRequest request) throws YarnException { Exception e = new Exception(EXCEPTION_MSG, new Exception(EXCEPTION_CAUSE)); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/SerializedException.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/SerializedException.java deleted file mode 100644 index 9750607d212..00000000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/SerializedException.java +++ /dev/null @@ -1,39 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.yarn.server.api.records; - -import org.apache.hadoop.classification.InterfaceAudience.Private; -import org.apache.hadoop.classification.InterfaceStability.Unstable; - -@Private -@Unstable -public abstract class SerializedException { - - public abstract void init(String message, Throwable cause); - - public abstract void init(String message); - - public abstract void init(Throwable cause); - - public abstract String getMessage(); - - public abstract String getRemoteTrace(); - - public abstract SerializedException getCause(); -} \ No newline at end of file diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/YarnServerBuilderUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/YarnServerBuilderUtils.java index e472746e055..8bdff6291d6 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/YarnServerBuilderUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/YarnServerBuilderUtils.java @@ -22,12 +22,12 @@ import java.util.List; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ContainerId; +import org.apache.hadoop.yarn.api.records.SerializedException; import org.apache.hadoop.yarn.factories.RecordFactory; import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider; import org.apache.hadoop.yarn.server.api.protocolrecords.NodeHeartbeatResponse; import org.apache.hadoop.yarn.server.api.records.MasterKey; import org.apache.hadoop.yarn.server.api.records.NodeAction; -import org.apache.hadoop.yarn.server.api.records.SerializedException; import org.apache.hadoop.yarn.util.Records; /** @@ -59,10 +59,4 @@ public class YarnServerBuilderUtils { } return response; } - - public static SerializedException newSerializedException(Throwable e) { - SerializedException se = Records.newRecord(SerializedException.class); - se.init(e); - return se; - } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/TestYarnServerApiClasses.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/TestYarnServerApiClasses.java index 41fc68e0c8a..f728eb339bf 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/TestYarnServerApiClasses.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/TestYarnServerApiClasses.java @@ -34,6 +34,7 @@ import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.api.records.impl.pb.ApplicationAttemptIdPBImpl; import org.apache.hadoop.yarn.api.records.impl.pb.ApplicationIdPBImpl; import org.apache.hadoop.yarn.api.records.impl.pb.ContainerIdPBImpl; +import org.apache.hadoop.yarn.api.records.impl.pb.SerializedExceptionPBImpl; import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider; import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.NodeHeartbeatRequestPBImpl; import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.NodeHeartbeatResponsePBImpl; @@ -45,7 +46,6 @@ import org.apache.hadoop.yarn.server.api.records.NodeHealthStatus; import org.apache.hadoop.yarn.server.api.records.NodeStatus; import org.apache.hadoop.yarn.server.api.records.impl.pb.MasterKeyPBImpl; import org.apache.hadoop.yarn.server.api.records.impl.pb.NodeStatusPBImpl; -import org.apache.hadoop.yarn.server.api.records.impl.pb.SerializedExceptionPBImpl; import org.junit.Test; /** diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/api/protocolrecords/LocalResourceStatus.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/api/protocolrecords/LocalResourceStatus.java index 70a96cd60f8..d3c5a8f2bcc 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/api/protocolrecords/LocalResourceStatus.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/api/protocolrecords/LocalResourceStatus.java @@ -18,8 +18,8 @@ package org.apache.hadoop.yarn.server.nodemanager.api.protocolrecords; import org.apache.hadoop.yarn.api.records.LocalResource; +import org.apache.hadoop.yarn.api.records.SerializedException; import org.apache.hadoop.yarn.api.records.URL; -import org.apache.hadoop.yarn.server.api.records.SerializedException; public interface LocalResourceStatus { public LocalResource getResource(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/api/protocolrecords/impl/pb/LocalResourceStatusPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/api/protocolrecords/impl/pb/LocalResourceStatusPBImpl.java index 6ecb9348342..bc5fcd4f689 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/api/protocolrecords/impl/pb/LocalResourceStatusPBImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/api/protocolrecords/impl/pb/LocalResourceStatusPBImpl.java @@ -18,9 +18,11 @@ package org.apache.hadoop.yarn.server.nodemanager.api.protocolrecords.impl.pb; import org.apache.hadoop.yarn.api.records.LocalResource; +import org.apache.hadoop.yarn.api.records.SerializedException; import org.apache.hadoop.yarn.api.records.URL; import org.apache.hadoop.yarn.api.records.impl.pb.LocalResourcePBImpl; import org.apache.hadoop.yarn.api.records.impl.pb.ProtoBase; +import org.apache.hadoop.yarn.api.records.impl.pb.SerializedExceptionPBImpl; import org.apache.hadoop.yarn.api.records.impl.pb.URLPBImpl; import org.apache.hadoop.yarn.proto.YarnProtos.LocalResourceProto; import org.apache.hadoop.yarn.proto.YarnProtos.SerializedExceptionProto; @@ -28,8 +30,6 @@ import org.apache.hadoop.yarn.proto.YarnProtos.URLProto; import org.apache.hadoop.yarn.proto.YarnServerNodemanagerServiceProtos.LocalResourceStatusProto; import org.apache.hadoop.yarn.proto.YarnServerNodemanagerServiceProtos.LocalResourceStatusProtoOrBuilder; import org.apache.hadoop.yarn.proto.YarnServerNodemanagerServiceProtos.ResourceStatusTypeProto; -import org.apache.hadoop.yarn.server.api.records.SerializedException; -import org.apache.hadoop.yarn.server.api.records.impl.pb.SerializedExceptionPBImpl; import org.apache.hadoop.yarn.server.nodemanager.api.protocolrecords.LocalResourceStatus; import org.apache.hadoop.yarn.server.nodemanager.api.protocolrecords.ResourceStatusType; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java index 712bc435bb7..d2e75103d32 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java @@ -23,12 +23,12 @@ import static org.apache.hadoop.service.Service.STATE.STARTED; import java.io.IOException; import java.net.InetSocketAddress; import java.nio.ByteBuffer; +import java.util.ArrayList; import java.util.Arrays; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Set; -import java.util.SortedSet; -import java.util.TreeMap; import java.util.concurrent.atomic.AtomicBoolean; import org.apache.commons.logging.Log; @@ -50,25 +50,25 @@ import org.apache.hadoop.service.Service; import org.apache.hadoop.service.ServiceStateChangeListener; import org.apache.hadoop.util.StringUtils; import org.apache.hadoop.yarn.api.ContainerManagementProtocol; -import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusRequest; -import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusResponse; +import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusesRequest; +import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusesResponse; import org.apache.hadoop.yarn.api.protocolrecords.StartContainerRequest; -import org.apache.hadoop.yarn.api.protocolrecords.StartContainerResponse; -import org.apache.hadoop.yarn.api.protocolrecords.StopContainerRequest; -import org.apache.hadoop.yarn.api.protocolrecords.StopContainerResponse; +import org.apache.hadoop.yarn.api.protocolrecords.StartContainersRequest; +import org.apache.hadoop.yarn.api.protocolrecords.StartContainersResponse; +import org.apache.hadoop.yarn.api.protocolrecords.StopContainersRequest; +import org.apache.hadoop.yarn.api.protocolrecords.StopContainersResponse; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.ContainerLaunchContext; import org.apache.hadoop.yarn.api.records.ContainerStatus; import org.apache.hadoop.yarn.api.records.NodeId; +import org.apache.hadoop.yarn.api.records.SerializedException; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.event.AsyncDispatcher; import org.apache.hadoop.yarn.event.EventHandler; import org.apache.hadoop.yarn.exceptions.InvalidContainerException; import org.apache.hadoop.yarn.exceptions.NMNotYetReadyException; import org.apache.hadoop.yarn.exceptions.YarnException; -import org.apache.hadoop.yarn.factories.RecordFactory; -import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider; import org.apache.hadoop.yarn.ipc.RPCUtil; import org.apache.hadoop.yarn.ipc.YarnRPC; import org.apache.hadoop.yarn.security.ContainerTokenIdentifier; @@ -130,8 +130,6 @@ public class ContainerManagerImpl extends CompositeService implements private final NodeStatusUpdater nodeStatusUpdater; - private final RecordFactory recordFactory = RecordFactoryProvider.getRecordFactory(null); - protected LocalDirsHandlerService dirsHandler; protected final AsyncDispatcher dispatcher; private final ApplicationACLsManager aclsManager; @@ -316,18 +314,25 @@ public class ContainerManagerImpl extends CompositeService implements return resultId; } + protected void authorizeUser(UserGroupInformation remoteUgi, + NMTokenIdentifier nmTokenIdentifier) throws YarnException { + if (!remoteUgi.getUserName().equals( + nmTokenIdentifier.getApplicationAttemptId().toString())) { + throw RPCUtil.getRemoteException("Expected applicationAttemptId: " + + remoteUgi.getUserName() + "Found: " + + nmTokenIdentifier.getApplicationAttemptId()); + } + } + /** * @param containerTokenIdentifier * of the container to be started - * @param ugi - * ugi corresponding to the remote end making the api-call * @throws YarnException */ @Private @VisibleForTesting protected void authorizeStartRequest(NMTokenIdentifier nmTokenIdentifier, - ContainerTokenIdentifier containerTokenIdentifier, - UserGroupInformation ugi) throws YarnException { + ContainerTokenIdentifier containerTokenIdentifier) throws YarnException { ContainerId containerId = containerTokenIdentifier.getContainerID(); String containerIDStr = containerId.toString(); @@ -342,12 +347,6 @@ public class ContainerManagerImpl extends CompositeService implements .append(" was used for starting container with container token") .append(" issued for application attempt : ") .append(containerId.getApplicationAttemptId()); - } else if (!ugi.getUserName().equals( - nmTokenIdentifier.getApplicationAttemptId().toString())) { - unauthorized = true; - messageBuilder.append("\nExpected applicationAttemptId: ") - .append(ugi.getUserName()).append(" Found: ") - .append(nmTokenIdentifier.getApplicationAttemptId().toString()); } else if (!this.context.getContainerTokenSecretManager() .isValidStartContainerRequest(containerTokenIdentifier)) { // Is the container being relaunched? Or RPC layer let startCall with @@ -363,7 +362,6 @@ public class ContainerManagerImpl extends CompositeService implements .append(System.currentTimeMillis()).append(" found ") .append(containerTokenIdentifier.getExpiryTimeStamp()); } - if (unauthorized) { String msg = messageBuilder.toString(); LOG.error(msg); @@ -372,18 +370,53 @@ public class ContainerManagerImpl extends CompositeService implements } /** - * Start a container on this NodeManager. + * Start a list of containers on this NodeManager. */ - @SuppressWarnings("unchecked") @Override - public StartContainerResponse startContainer(StartContainerRequest request) - throws YarnException, IOException { - + public StartContainersResponse + startContainers(StartContainersRequest requests) throws YarnException, + IOException { if (blockNewContainerRequests.get()) { throw new NMNotYetReadyException( "Rejecting new containers as NodeManager has not" + " yet connected with ResourceManager"); } + UserGroupInformation remoteUgi = getRemoteUgi(); + NMTokenIdentifier nmTokenIdentifier = selectNMTokenIdentifier(remoteUgi); + authorizeUser(remoteUgi,nmTokenIdentifier); + List succeededContainers = new ArrayList(); + Map failedContainers = + new HashMap(); + for (StartContainerRequest request : requests.getStartContainerRequests()) { + ContainerId containerId = null; + try { + ContainerTokenIdentifier containerTokenIdentifier = + BuilderUtils.newContainerTokenIdentifier(request.getContainerToken()); + verifyAndGetContainerTokenIdentifier(request.getContainerToken(), + containerTokenIdentifier); + containerId = containerTokenIdentifier.getContainerID(); + startContainerInternal(nmTokenIdentifier, containerTokenIdentifier, + request); + succeededContainers.add(containerId); + } catch (YarnException e) { + failedContainers.put(containerId, SerializedException.newInstance(e)); + } catch (InvalidToken ie) { + failedContainers.put(containerId, SerializedException.newInstance(ie)); + throw ie; + } catch (IOException e) { + throw RPCUtil.getRemoteException(e); + } + } + + return StartContainersResponse.newInstance(auxiliaryServices.getMetaData(), + succeededContainers, failedContainers); + } + + @SuppressWarnings("unchecked") + private void startContainerInternal(NMTokenIdentifier nmTokenIdentifier, + ContainerTokenIdentifier containerTokenIdentifier, + StartContainerRequest request) throws YarnException, IOException { + /* * 1) It should save the NMToken into NMTokenSecretManager. This is done * here instead of RPC layer because at the time of opening/authenticating @@ -395,18 +428,8 @@ public class ContainerManagerImpl extends CompositeService implements * belongs to correct Node Manager (part of retrieve password). c) It has * correct RMIdentifier. d) It is not expired. */ - // update NMToken - - UserGroupInformation remoteUgi = getRemoteUgi(); - NMTokenIdentifier nmTokenIdentifier = selectNMTokenIdentifier(remoteUgi); - - // Validate containerToken - ContainerTokenIdentifier containerTokenIdentifier = - verifyAndGetContainerTokenIdentifier(request.getContainerToken()); - - authorizeStartRequest(nmTokenIdentifier, containerTokenIdentifier, - remoteUgi); - + authorizeStartRequest(nmTokenIdentifier, containerTokenIdentifier); + if (containerTokenIdentifier.getRMIdentifer() != nodeStatusUpdater .getRMIdentifier()) { // Is the container coming from unknown RM @@ -415,9 +438,9 @@ public class ContainerManagerImpl extends CompositeService implements .append(" rejected as it is allocated by a previous RM"); throw new InvalidContainerException(sb.toString()); } - + // update NMToken updateNMTokenIdentifier(nmTokenIdentifier); - + ContainerId containerId = containerTokenIdentifier.getContainerID(); String containerIdStr = containerId.toString(); String user = containerTokenIdentifier.getApplicationSubmitter(); @@ -461,26 +484,16 @@ public class ContainerManagerImpl extends CompositeService implements containerTokenIdentifier); NMAuditLogger.logSuccess(user, AuditConstants.START_CONTAINER, "ContainerManageImpl", applicationID, containerId); - StartContainerResponse response = - recordFactory.newRecordInstance(StartContainerResponse.class); - response.setAllServicesMetaData(auxiliaryServices.getMetaData()); // TODO launchedContainer misplaced -> doesn't necessarily mean a container // launch. A finished Application will not launch containers. metrics.launchedContainer(); - metrics.allocateContainer(containerTokenIdentifier.getResource()); - return response; + metrics.allocateContainer(containerTokenIdentifier.getResource()); } protected ContainerTokenIdentifier verifyAndGetContainerTokenIdentifier( - org.apache.hadoop.yarn.api.records.Token token) throws YarnException, + org.apache.hadoop.yarn.api.records.Token token, + ContainerTokenIdentifier containerTokenIdentifier) throws YarnException, InvalidToken { - ContainerTokenIdentifier containerTokenIdentifier = null; - try { - containerTokenIdentifier = - BuilderUtils.newContainerTokenIdentifier(token); - } catch (IOException e) { - throw RPCUtil.getRemoteException(e); - } byte[] password = context.getContainerTokenSecretManager().retrievePassword( containerTokenIdentifier); @@ -528,21 +541,37 @@ public class ContainerManagerImpl extends CompositeService implements } /** - * Stop the container running on this NodeManager. + * Stop a list of containers running on this NodeManager. */ @Override - @SuppressWarnings("unchecked") - public StopContainerResponse stopContainer(StopContainerRequest request) + public StopContainersResponse stopContainers(StopContainersRequest requests) throws YarnException, IOException { - ContainerId containerID = request.getContainerId(); + List succeededRequests = new ArrayList(); + Map failedRequests = + new HashMap(); + UserGroupInformation remoteUgi = getRemoteUgi(); + NMTokenIdentifier identifier = selectNMTokenIdentifier(remoteUgi); + for (ContainerId id : requests.getContainerIds()) { + try { + stopContainerInternal(identifier, id); + succeededRequests.add(id); + } catch (YarnException e) { + failedRequests.put(id, SerializedException.newInstance(e)); + } + } + return StopContainersResponse + .newInstance(succeededRequests, failedRequests); + } + + @SuppressWarnings("unchecked") + private void stopContainerInternal(NMTokenIdentifier nmTokenIdentifier, + ContainerId containerID) throws YarnException { String containerIDStr = containerID.toString(); Container container = this.context.getContainers().get(containerID); - LOG.info("Getting container-status for " + containerIDStr); - authorizeGetAndStopContainerRequest(containerID, container, true); - - StopContainerResponse response = - recordFactory.newRecordInstance(StopContainerResponse.class); + LOG.info("Stopping container with container Id: " + containerIDStr); + authorizeGetAndStopContainerRequest(containerID, container, true, + nmTokenIdentifier); dispatcher.getEventHandler().handle( new ContainerKillEvent(containerID, @@ -555,37 +584,51 @@ public class ContainerManagerImpl extends CompositeService implements // TODO: Move this code to appropriate place once kill_container is // implemented. nodeStatusUpdater.sendOutofBandHeartBeat(); - - return response; } + /** + * Get a list of container statuses running on this NodeManager + */ @Override - public GetContainerStatusResponse getContainerStatus( - GetContainerStatusRequest request) throws YarnException, IOException { + public GetContainerStatusesResponse getContainerStatuses( + GetContainerStatusesRequest request) throws YarnException, IOException { - ContainerId containerID = request.getContainerId(); + List succeededRequests = new ArrayList(); + Map failedRequests = + new HashMap(); + UserGroupInformation remoteUgi = getRemoteUgi(); + NMTokenIdentifier identifier = selectNMTokenIdentifier(remoteUgi); + for (ContainerId id : request.getContainerIds()) { + try { + ContainerStatus status = getContainerStatusInternal(id, identifier); + succeededRequests.add(status); + } catch (YarnException e) { + failedRequests.put(id, SerializedException.newInstance(e)); + } + } + return GetContainerStatusesResponse.newInstance(succeededRequests, + failedRequests); + } + + private ContainerStatus getContainerStatusInternal(ContainerId containerID, + NMTokenIdentifier nmTokenIdentifier) throws YarnException { String containerIDStr = containerID.toString(); Container container = this.context.getContainers().get(containerID); LOG.info("Getting container-status for " + containerIDStr); - authorizeGetAndStopContainerRequest(containerID, container, false); + authorizeGetAndStopContainerRequest(containerID, container, false, + nmTokenIdentifier); ContainerStatus containerStatus = container.cloneAndGetContainerStatus(); LOG.info("Returning " + containerStatus); - GetContainerStatusResponse response = - recordFactory.newRecordInstance(GetContainerStatusResponse.class); - response.setStatus(containerStatus); - return response; + return containerStatus; } @Private @VisibleForTesting protected void authorizeGetAndStopContainerRequest(ContainerId containerId, - Container container, boolean stopRequest) throws YarnException { - - UserGroupInformation remoteUgi = getRemoteUgi(); - NMTokenIdentifier identifier = selectNMTokenIdentifier(remoteUgi); - + Container container, boolean stopRequest, NMTokenIdentifier identifier) + throws YarnException { /* * For get/stop container status; we need to verify that 1) User (NMToken) * application attempt only has started container. 2) Requested containerId diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java index c1f3eb95f46..f0cf9db75a9 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java @@ -103,7 +103,7 @@ public class ContainerImpl implements Container { public ContainerImpl(Configuration conf, Dispatcher dispatcher, ContainerLaunchContext launchContext, Credentials creds, NodeManagerMetrics metrics, - ContainerTokenIdentifier containerTokenIdentifier) throws IOException { + ContainerTokenIdentifier containerTokenIdentifier) { this.daemonConf = conf; this.dispatcher = dispatcher; this.launchContext = launchContext; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ContainerLocalizer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ContainerLocalizer.java index a55d6f0d38d..66f21f6a571 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ContainerLocalizer.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ContainerLocalizer.java @@ -53,6 +53,7 @@ import org.apache.hadoop.security.token.TokenIdentifier; import org.apache.hadoop.util.DiskChecker; import org.apache.hadoop.yarn.YarnUncaughtExceptionHandler; import org.apache.hadoop.yarn.api.records.LocalResource; +import org.apache.hadoop.yarn.api.records.SerializedException; import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.factories.RecordFactory; import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider; @@ -288,11 +289,10 @@ public class ContainerLocalizer { stat.setStatus(ResourceStatusType.FETCH_SUCCESS); } catch (ExecutionException e) { stat.setStatus(ResourceStatusType.FETCH_FAILURE); - stat.setException( - YarnServerBuilderUtils.newSerializedException(e.getCause())); + stat.setException(SerializedException.newInstance(e.getCause())); } catch (CancellationException e) { stat.setStatus(ResourceStatusType.FETCH_FAILURE); - stat.setException(YarnServerBuilderUtils.newSerializedException(e)); + stat.setException(SerializedException.newInstance(e)); } // TODO shouldn't remove until ACK i.remove(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/DummyContainerManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/DummyContainerManager.java index 76ef074778f..a2fd96c041e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/DummyContainerManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/DummyContainerManager.java @@ -195,14 +195,13 @@ public class DummyContainerManager extends ContainerManagerImpl { @Override protected void authorizeStartRequest(NMTokenIdentifier nmTokenIdentifier, - ContainerTokenIdentifier containerTokenIdentifier, - UserGroupInformation ugi) throws YarnException { + ContainerTokenIdentifier containerTokenIdentifier) throws YarnException { // do nothing } @Override protected void authorizeGetAndStopContainerRequest(ContainerId containerId, - Container container, boolean stopRequest) throws YarnException { + Container container, boolean stopRequest, NMTokenIdentifier identifier) throws YarnException { // do nothing } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestContainerManagerWithLCE.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestContainerManagerWithLCE.java index 13dc0991322..cc9b7d9ce0c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestContainerManagerWithLCE.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestContainerManagerWithLCE.java @@ -146,7 +146,7 @@ public class TestContainerManagerWithLCE extends TestContainerManager { @Override public void testContainerLaunchFromPreviousRM() throws InterruptedException, - IOException { + IOException, YarnException { // Don't run the test if the binary is not available. if (!shouldRunTest()) { LOG.info("LCE binary path is not passed. Not running the test"); @@ -155,6 +155,29 @@ public class TestContainerManagerWithLCE extends TestContainerManager { LOG.info("Running testContainerLaunchFromPreviousRM"); super.testContainerLaunchFromPreviousRM(); } + + @Override + public void testMultipleContainersLaunch() throws Exception { + // Don't run the test if the binary is not available. + if (!shouldRunTest()) { + LOG.info("LCE binary path is not passed. Not running the test"); + return; + } + LOG.info("Running testContainerLaunchFromPreviousRM"); + super.testMultipleContainersLaunch(); + } + + @Override + public void testMultipleContainersStopAndGetStatus() throws Exception { + // Don't run the test if the binary is not available. + if (!shouldRunTest()) { + LOG.info("LCE binary path is not passed. Not running the test"); + return; + } + LOG.info("Running testContainerLaunchFromPreviousRM"); + super.testMultipleContainersStopAndGetStatus(); + } + private boolean shouldRunTest() { return System .getProperty(YarnConfiguration.NM_LINUX_CONTAINER_EXECUTOR_PATH) != null; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestEventFlow.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestEventFlow.java index 294c93ed3b8..ba644abf9fe 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestEventFlow.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestEventFlow.java @@ -20,18 +20,19 @@ package org.apache.hadoop.yarn.server.nodemanager; import java.io.File; import java.io.IOException; +import java.util.ArrayList; +import java.util.List; import org.apache.hadoop.fs.FileContext; import org.apache.hadoop.fs.Path; import org.apache.hadoop.yarn.api.protocolrecords.StartContainerRequest; -import org.apache.hadoop.yarn.api.protocolrecords.StopContainerRequest; +import org.apache.hadoop.yarn.api.protocolrecords.StartContainersRequest; +import org.apache.hadoop.yarn.api.protocolrecords.StopContainersRequest; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.ContainerLaunchContext; import org.apache.hadoop.yarn.api.records.ContainerState; -import org.apache.hadoop.yarn.api.records.Token; -import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.event.AsyncDispatcher; import org.apache.hadoop.yarn.event.Dispatcher; @@ -46,7 +47,6 @@ import org.apache.hadoop.yarn.server.nodemanager.metrics.NodeManagerMetrics; import org.apache.hadoop.yarn.server.nodemanager.security.NMContainerTokenSecretManager; import org.apache.hadoop.yarn.server.nodemanager.security.NMTokenSecretManagerInNM; import org.apache.hadoop.yarn.server.security.ApplicationACLsManager; -import org.apache.hadoop.yarn.server.utils.BuilderUtils; import org.junit.Test; @@ -62,7 +62,7 @@ public class TestEventFlow { private static File remoteLogDir = new File("target", TestEventFlow.class.getName() + "-remoteLogDir").getAbsoluteFile(); private static final long SIMULATED_RM_IDENTIFIER = 1234; - + @Test public void testSuccessfulContainerLaunch() throws InterruptedException, IOException, YarnException { @@ -140,21 +140,25 @@ public class TestEventFlow { ContainerId cID = ContainerId.newInstance(applicationAttemptId, 0); String user = "testing"; - StartContainerRequest request = - recordFactory.newRecordInstance(StartContainerRequest.class); - request.setContainerLaunchContext(launchContext); - request.setContainerToken(TestContainerManager.createContainerToken(cID, - SIMULATED_RM_IDENTIFIER, context.getNodeId(), user, - context.getContainerTokenSecretManager())); - containerManager.startContainer(request); + StartContainerRequest scRequest = + StartContainerRequest.newInstance(launchContext, + TestContainerManager.createContainerToken(cID, + SIMULATED_RM_IDENTIFIER, context.getNodeId(), user, + context.getContainerTokenSecretManager())); + List list = new ArrayList(); + list.add(scRequest); + StartContainersRequest allRequests = + StartContainersRequest.newInstance(list); + containerManager.startContainers(allRequests); BaseContainerManagerTest.waitForContainerState(containerManager, cID, ContainerState.RUNNING); - StopContainerRequest stopRequest = - recordFactory.newRecordInstance(StopContainerRequest.class); - stopRequest.setContainerId(cID); - containerManager.stopContainer(stopRequest); + List containerIds = new ArrayList(); + containerIds.add(cID); + StopContainersRequest stopRequest = + StopContainersRequest.newInstance(containerIds); + containerManager.stopContainers(stopRequest); BaseContainerManagerTest.waitForContainerState(containerManager, cID, ContainerState.COMPLETE); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerReboot.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerReboot.java index e07a28d0a9a..fab9e019a47 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerReboot.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerReboot.java @@ -39,8 +39,9 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.UnsupportedFileSystemException; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.yarn.api.ContainerManagementProtocol; -import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusRequest; +import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusesRequest; import org.apache.hadoop.yarn.api.protocolrecords.StartContainerRequest; +import org.apache.hadoop.yarn.api.protocolrecords.StartContainersRequest; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ContainerId; @@ -129,13 +130,17 @@ public class TestNodeManagerReboot { List commands = new ArrayList(); containerLaunchContext.setCommands(commands); - final StartContainerRequest startRequest = - Records.newRecord(StartContainerRequest.class); - startRequest.setContainerLaunchContext(containerLaunchContext); NodeId nodeId = nm.getNMContext().getNodeId(); - startRequest.setContainerToken(TestContainerManager.createContainerToken( - cId, 0, nodeId, destinationFile, nm.getNMContext() - .getContainerTokenSecretManager())); + StartContainerRequest scRequest = + StartContainerRequest.newInstance(containerLaunchContext, + TestContainerManager.createContainerToken( + cId, 0, nodeId, destinationFile, nm.getNMContext() + .getContainerTokenSecretManager())); + List list = new ArrayList(); + list.add(scRequest); + final StartContainersRequest allRequests = + StartContainersRequest.newInstance(list); + final UserGroupInformation currentUser = UserGroupInformation.createRemoteUser(cId.getApplicationAttemptId() .toString()); @@ -145,16 +150,17 @@ public class TestNodeManagerReboot { currentUser.doAs(new PrivilegedExceptionAction() { @Override public Void run() throws YarnException, IOException { - nm.getContainerManager().startContainer(startRequest); + nm.getContainerManager().startContainers(allRequests); return null; } }); - GetContainerStatusRequest request = - Records.newRecord(GetContainerStatusRequest.class); - request.setContainerId(cId); + List containerIds = new ArrayList(); + containerIds.add(cId); + GetContainerStatusesRequest request = + GetContainerStatusesRequest.newInstance(containerIds); Container container = - nm.getNMContext().getContainers().get(request.getContainerId()); + nm.getNMContext().getContainers().get(request.getContainerIds().get(0)); final int MAX_TRIES = 20; int numTries = 0; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerResync.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerResync.java index d01eb6633f0..a05e34143b1 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerResync.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerResync.java @@ -20,6 +20,8 @@ package org.apache.hadoop.yarn.server.nodemanager; import java.io.File; import java.io.IOException; +import java.util.ArrayList; +import java.util.List; import java.util.concurrent.BrokenBarrierException; import java.util.concurrent.ConcurrentMap; import java.util.concurrent.CyclicBarrier; @@ -31,6 +33,7 @@ import org.apache.hadoop.fs.FileContext; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.UnsupportedFileSystemException; import org.apache.hadoop.yarn.api.protocolrecords.StartContainerRequest; +import org.apache.hadoop.yarn.api.protocolrecords.StartContainersRequest; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.ContainerLaunchContext; import org.apache.hadoop.yarn.conf.YarnConfiguration; @@ -282,15 +285,18 @@ public class TestNodeManagerResync { try { while (!isStopped && numContainers < 10) { ContainerId cId = TestNodeManagerShutdown.createContainerId(); - StartContainerRequest startRequest = - recordFactory.newRecordInstance(StartContainerRequest.class); - startRequest.setContainerLaunchContext(containerLaunchContext); - startRequest.setContainerToken(null); + StartContainerRequest scRequest = + StartContainerRequest.newInstance(containerLaunchContext, + null); + List list = new ArrayList(); + list.add(scRequest); + StartContainersRequest allRequests = + StartContainersRequest.newInstance(list); System.out.println("no. of containers to be launched: " + numContainers); numContainers++; try { - getContainerManager().startContainer(startRequest); + getContainerManager().startContainers(allRequests); } catch (YarnException e) { numContainersRejected++; Assert.assertTrue(e.getMessage().contains( diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerShutdown.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerShutdown.java index 20f7f77fffe..66f2e8c4bfa 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerShutdown.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerShutdown.java @@ -25,6 +25,7 @@ import java.io.IOException; import java.io.PrintWriter; import java.net.InetSocketAddress; import java.security.PrivilegedAction; +import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; import java.util.List; @@ -40,8 +41,9 @@ import org.apache.hadoop.net.NetUtils; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.util.Shell; import org.apache.hadoop.yarn.api.ContainerManagementProtocol; -import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusRequest; +import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusesRequest; import org.apache.hadoop.yarn.api.protocolrecords.StartContainerRequest; +import org.apache.hadoop.yarn.api.protocolrecords.StartContainersRequest; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ContainerId; @@ -53,7 +55,6 @@ import org.apache.hadoop.yarn.api.records.LocalResourceType; import org.apache.hadoop.yarn.api.records.LocalResourceVisibility; import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.hadoop.yarn.api.records.URL; -import org.apache.hadoop.yarn.api.records.impl.pb.ProtoUtils; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.event.Dispatcher; import org.apache.hadoop.yarn.exceptions.YarnException; @@ -181,12 +182,6 @@ public class TestNodeManagerShutdown { containerLaunchContext.setLocalResources(localResources); List commands = Arrays.asList(Shell.getRunScriptCommand(scriptFile)); containerLaunchContext.setCommands(commands); - StartContainerRequest startRequest = - recordFactory.newRecordInstance(StartContainerRequest.class); - startRequest.setContainerLaunchContext(containerLaunchContext); - startRequest - .setContainerToken(TestContainerManager.createContainerToken(cId, 0, - nodeId, user, nm.getNMContext().getContainerTokenSecretManager())); final InetSocketAddress containerManagerBindAddress = NetUtils.createSocketAddrForHost("127.0.0.1", 12345); UserGroupInformation currentUser = UserGroupInformation @@ -210,13 +205,22 @@ public class TestNodeManagerShutdown { containerManagerBindAddress, conf); } }); - containerManager.startContainer(startRequest); + StartContainerRequest scRequest = + StartContainerRequest.newInstance(containerLaunchContext, + TestContainerManager.createContainerToken(cId, 0, + nodeId, user, nm.getNMContext().getContainerTokenSecretManager())); + List list = new ArrayList(); + list.add(scRequest); + StartContainersRequest allRequests = + StartContainersRequest.newInstance(list); + containerManager.startContainers(allRequests); - GetContainerStatusRequest request = - recordFactory.newRecordInstance(GetContainerStatusRequest.class); - request.setContainerId(cId); + List containerIds = new ArrayList(); + containerIds.add(cId); + GetContainerStatusesRequest request = + GetContainerStatusesRequest.newInstance(containerIds); ContainerStatus containerStatus = - containerManager.getContainerStatus(request).getStatus(); + containerManager.getContainerStatuses(request).getContainerStatuses().get(0); Assert.assertEquals(ContainerState.RUNNING, containerStatus.getState()); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdater.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdater.java index 2a3e3d579ca..73bea039a2f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdater.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdater.java @@ -45,8 +45,8 @@ import org.apache.hadoop.io.retry.RetryPolicy; import org.apache.hadoop.io.retry.RetryProxy; import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem; import org.apache.hadoop.net.NetUtils; -import org.apache.hadoop.service.ServiceOperations; import org.apache.hadoop.service.Service.STATE; +import org.apache.hadoop.service.ServiceOperations; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ContainerId; @@ -65,7 +65,6 @@ import org.apache.hadoop.yarn.factories.RecordFactory; import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider; import org.apache.hadoop.yarn.security.ContainerTokenIdentifier; import org.apache.hadoop.yarn.server.api.ResourceTracker; -import org.apache.hadoop.yarn.server.api.ServerRMProxy; import org.apache.hadoop.yarn.server.api.protocolrecords.NodeHeartbeatRequest; import org.apache.hadoop.yarn.server.api.protocolrecords.NodeHeartbeatResponse; import org.apache.hadoop.yarn.server.api.protocolrecords.RegisterNodeManagerRequest; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/api/protocolrecords/impl/pb/TestPBRecordImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/api/protocolrecords/impl/pb/TestPBRecordImpl.java index ff9078c909b..d1e6f8d62bd 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/api/protocolrecords/impl/pb/TestPBRecordImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/api/protocolrecords/impl/pb/TestPBRecordImpl.java @@ -21,16 +21,21 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; +import java.io.IOException; import java.net.URISyntaxException; import java.util.ArrayList; +import junit.framework.Assert; + import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.DataInputBuffer; import org.apache.hadoop.io.DataOutputBuffer; import org.apache.hadoop.yarn.api.records.LocalResource; import org.apache.hadoop.yarn.api.records.LocalResourceVisibility; +import org.apache.hadoop.yarn.api.records.SerializedException; import org.apache.hadoop.yarn.api.records.impl.pb.LocalResourcePBImpl; +import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.factories.RecordFactory; import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider; import org.apache.hadoop.yarn.proto.YarnServerNodemanagerServiceProtos.LocalResourceStatusProto; @@ -42,7 +47,6 @@ import org.apache.hadoop.yarn.server.nodemanager.api.protocolrecords.LocalizerAc import org.apache.hadoop.yarn.server.nodemanager.api.protocolrecords.LocalizerHeartbeatResponse; import org.apache.hadoop.yarn.server.nodemanager.api.protocolrecords.LocalizerStatus; import org.apache.hadoop.yarn.server.nodemanager.api.protocolrecords.ResourceStatusType; -import org.apache.hadoop.yarn.server.utils.YarnServerBuilderUtils; import org.apache.hadoop.yarn.util.ConverterUtils; import org.junit.Test; @@ -80,7 +84,7 @@ public class TestPBRecordImpl { e.setStackTrace(new StackTraceElement[] { new StackTraceElement("foo", "bar", "baz", 10), new StackTraceElement("sbb", "one", "onm", 10) }); - ret.setException(YarnServerBuilderUtils.newSerializedException(e)); + ret.setException(SerializedException.newInstance(e)); return ret; } @@ -176,4 +180,33 @@ public class TestPBRecordImpl { assertEquals(createResource(), rsrcD.getResourceSpecs().get(0).getResource()); } + + @Test(timeout=10000) + public void testSerializedExceptionDeSer() throws Exception{ + // without cause + YarnException yarnEx = new YarnException("Yarn_Exception"); + SerializedException serEx = SerializedException.newInstance(yarnEx); + Throwable throwable = serEx.deSerialize(); + Assert.assertEquals(yarnEx.getClass(), throwable.getClass()); + Assert.assertEquals(yarnEx.getMessage(), throwable.getMessage()); + + // with cause + IOException ioe = new IOException("Test_IOException"); + RuntimeException runtimeException = + new RuntimeException("Test_RuntimeException", ioe); + YarnException yarnEx2 = + new YarnException("Test_YarnException", runtimeException); + + SerializedException serEx2 = SerializedException.newInstance(yarnEx2); + Throwable throwable2 = serEx2.deSerialize(); + throwable2.printStackTrace(); + Assert.assertEquals(yarnEx2.getClass(), throwable2.getClass()); + Assert.assertEquals(yarnEx2.getMessage(), throwable2.getMessage()); + + Assert.assertEquals(runtimeException.getClass(), throwable2.getCause().getClass()); + Assert.assertEquals(runtimeException.getMessage(), throwable2.getCause().getMessage()); + + Assert.assertEquals(ioe.getClass(), throwable2.getCause().getCause().getClass()); + Assert.assertEquals(ioe.getMessage(), throwable2.getCause().getCause().getMessage()); + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/BaseContainerManagerTest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/BaseContainerManagerTest.java index cfcf7f6445e..f49f1189736 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/BaseContainerManagerTest.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/BaseContainerManagerTest.java @@ -20,6 +20,8 @@ package org.apache.hadoop.yarn.server.nodemanager.containermanager; import java.io.File; import java.io.IOException; +import java.util.ArrayList; +import java.util.List; import junit.framework.Assert; @@ -32,7 +34,7 @@ import org.apache.hadoop.fs.UnsupportedFileSystemException; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.token.SecretManager.InvalidToken; import org.apache.hadoop.yarn.api.ContainerManagementProtocol; -import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusRequest; +import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusesRequest; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.ContainerState; @@ -189,15 +191,18 @@ public abstract class BaseContainerManagerTest { @Override protected void authorizeGetAndStopContainerRequest(ContainerId containerId, - Container container, boolean stopRequest) throws YarnException { + Container container, boolean stopRequest, NMTokenIdentifier identifier) throws YarnException { // do nothing } - + @Override + protected void authorizeUser(UserGroupInformation remoteUgi, + NMTokenIdentifier nmTokenIdentifier) { + // do nothing + } @Override protected void authorizeStartRequest( NMTokenIdentifier nmTokenIdentifier, - ContainerTokenIdentifier containerTokenIdentifier, - UserGroupInformation ugi) throws YarnException { + ContainerTokenIdentifier containerTokenIdentifier) throws YarnException { // do nothing } @@ -238,18 +243,20 @@ public abstract class BaseContainerManagerTest { public static void waitForContainerState(ContainerManagementProtocol containerManager, ContainerId containerID, ContainerState finalState, int timeOutMax) throws InterruptedException, YarnException, IOException { - GetContainerStatusRequest request = - recordFactory.newRecordInstance(GetContainerStatusRequest.class); - request.setContainerId(containerID); - ContainerStatus containerStatus = - containerManager.getContainerStatus(request).getStatus(); - int timeoutSecs = 0; + List list = new ArrayList(); + list.add(containerID); + GetContainerStatusesRequest request = + GetContainerStatusesRequest.newInstance(list); + ContainerStatus containerStatus = + containerManager.getContainerStatuses(request).getContainerStatuses() + .get(0); + int timeoutSecs = 0; while (!containerStatus.getState().equals(finalState) && timeoutSecs++ < timeOutMax) { Thread.sleep(1000); LOG.info("Waiting for container to get into state " + finalState + ". Current state is " + containerStatus.getState()); - containerStatus = containerManager.getContainerStatus(request).getStatus(); + containerStatus = containerManager.getContainerStatuses(request).getContainerStatuses().get(0); } LOG.info("Container state is " + containerStatus.getState()); Assert.assertEquals("ContainerState is not correct (timedout)", diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManager.java index fdd0ed47447..e5b318ee4cd 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManager.java @@ -24,6 +24,7 @@ import java.io.FileReader; import java.io.IOException; import java.io.PrintWriter; import java.net.InetAddress; +import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; import java.util.List; @@ -37,9 +38,13 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.UnsupportedFileSystemException; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.util.Shell; -import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusRequest; +import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusesRequest; +import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusesResponse; import org.apache.hadoop.yarn.api.protocolrecords.StartContainerRequest; -import org.apache.hadoop.yarn.api.protocolrecords.StopContainerRequest; +import org.apache.hadoop.yarn.api.protocolrecords.StartContainersRequest; +import org.apache.hadoop.yarn.api.protocolrecords.StartContainersResponse; +import org.apache.hadoop.yarn.api.protocolrecords.StopContainersRequest; +import org.apache.hadoop.yarn.api.protocolrecords.StopContainersResponse; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ContainerId; @@ -51,6 +56,7 @@ import org.apache.hadoop.yarn.api.records.LocalResourceType; import org.apache.hadoop.yarn.api.records.LocalResourceVisibility; import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.api.records.SerializedException; import org.apache.hadoop.yarn.api.records.Token; import org.apache.hadoop.yarn.api.records.URL; import org.apache.hadoop.yarn.exceptions.InvalidContainerException; @@ -63,6 +69,7 @@ import org.apache.hadoop.yarn.server.nodemanager.ContainerExecutor.ExitCode; import org.apache.hadoop.yarn.server.nodemanager.DefaultContainerExecutor; import org.apache.hadoop.yarn.server.nodemanager.DeletionService; import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.ApplicationState; +import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container; import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.ContainerLocalizer; import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.ResourceLocalizationService; import org.apache.hadoop.yarn.server.nodemanager.security.NMContainerTokenSecretManager; @@ -88,11 +95,11 @@ public class TestContainerManager extends BaseContainerManagerTest { super.setup(); } - private ContainerId createContainerId() { + private ContainerId createContainerId(int id) { ApplicationId appId = ApplicationId.newInstance(0, 0); ApplicationAttemptId appAttemptId = ApplicationAttemptId.newInstance(appId, 1); - ContainerId containerId = ContainerId.newInstance(appAttemptId, 0); + ContainerId containerId = ContainerId.newInstance(appAttemptId, id); return containerId; } @@ -119,6 +126,14 @@ public class TestContainerManager extends BaseContainerManagerTest { .getKeyId())); return ugi; } + + @Override + protected void authorizeGetAndStopContainerRequest(ContainerId containerId, + Container container, boolean stopRequest, NMTokenIdentifier identifier) throws YarnException { + if(container == null || container.getUser().equals("Fail")){ + throw new YarnException("Reject this container"); + } + } }; } @@ -138,12 +153,17 @@ public class TestContainerManager extends BaseContainerManagerTest { // Just do a query for a non-existing container. boolean throwsException = false; try { - GetContainerStatusRequest request = - recordFactory.newRecordInstance(GetContainerStatusRequest.class); - ContainerId cId = createContainerId(); - request.setContainerId(cId); - containerManager.getContainerStatus(request); - } catch (YarnException e) { + List containerIds = new ArrayList(); + ContainerId id =createContainerId(0); + containerIds.add(id); + GetContainerStatusesRequest request = + GetContainerStatusesRequest.newInstance(containerIds); + GetContainerStatusesResponse response = + containerManager.getContainerStatuses(request); + if(response.getFailedRequests().containsKey(id)){ + throw response.getFailedRequests().get(id).deSerialize(); + } + } catch (Throwable e) { throwsException = true; } Assert.assertTrue(throwsException); @@ -163,7 +183,7 @@ public class TestContainerManager extends BaseContainerManagerTest { fileWriter.close(); // ////// Construct the Container-id - ContainerId cId = createContainerId(); + ContainerId cId = createContainerId(0); // ////// Construct the container-spec. ContainerLaunchContext containerLaunchContext = @@ -182,14 +202,17 @@ public class TestContainerManager extends BaseContainerManagerTest { new HashMap(); localResources.put(destinationFile, rsrc_alpha); containerLaunchContext.setLocalResources(localResources); - StartContainerRequest startRequest = - recordFactory.newRecordInstance(StartContainerRequest.class); - startRequest.setContainerLaunchContext(containerLaunchContext); - startRequest.setContainerToken(createContainerToken(cId, - DUMMY_RM_IDENTIFIER, context.getNodeId(), user, - context.getContainerTokenSecretManager())); - containerManager.startContainer(startRequest); + StartContainerRequest scRequest = + StartContainerRequest.newInstance( + containerLaunchContext, + createContainerToken(cId, DUMMY_RM_IDENTIFIER, context.getNodeId(), + user, context.getContainerTokenSecretManager())); + List list = new ArrayList(); + list.add(scRequest); + StartContainersRequest allRequests = + StartContainersRequest.newInstance(list); + containerManager.startContainers(allRequests); BaseContainerManagerTest.waitForContainerState(containerManager, cId, ContainerState.COMPLETE); @@ -237,7 +260,7 @@ public class TestContainerManager extends BaseContainerManagerTest { new File(tmpDir, "start_file.txt").getAbsoluteFile(); // ////// Construct the Container-id - ContainerId cId = createContainerId(); + ContainerId cId = createContainerId(0); if (Shell.WINDOWS) { fileWriter.println("@echo Hello World!> " + processStartFile); @@ -272,13 +295,17 @@ public class TestContainerManager extends BaseContainerManagerTest { List commands = Arrays.asList(Shell.getRunScriptCommand(scriptFile)); containerLaunchContext.setCommands(commands); - StartContainerRequest startRequest = recordFactory.newRecordInstance(StartContainerRequest.class); - startRequest.setContainerLaunchContext(containerLaunchContext); - startRequest.setContainerToken(createContainerToken(cId, - DUMMY_RM_IDENTIFIER, context.getNodeId(), user, - context.getContainerTokenSecretManager())); - containerManager.startContainer(startRequest); - + StartContainerRequest scRequest = + StartContainerRequest.newInstance(containerLaunchContext, + createContainerToken(cId, + DUMMY_RM_IDENTIFIER, context.getNodeId(), user, + context.getContainerTokenSecretManager())); + List list = new ArrayList(); + list.add(scRequest); + StartContainersRequest allRequests = + StartContainersRequest.newInstance(list); + containerManager.startContainers(allRequests); + int timeoutSecs = 0; while (!processStartFile.exists() && timeoutSecs++ < 20) { Thread.sleep(1000); @@ -305,18 +332,18 @@ public class TestContainerManager extends BaseContainerManagerTest { Assert.assertTrue("Process is not alive!", DefaultContainerExecutor.containerIsAlive(pid)); - StopContainerRequest stopRequest = recordFactory.newRecordInstance(StopContainerRequest.class); - stopRequest.setContainerId(cId); - containerManager.stopContainer(stopRequest); - + List containerIds = new ArrayList(); + containerIds.add(cId); + StopContainersRequest stopRequest = + StopContainersRequest.newInstance(containerIds); + containerManager.stopContainers(stopRequest); BaseContainerManagerTest.waitForContainerState(containerManager, cId, ContainerState.COMPLETE); - GetContainerStatusRequest gcsRequest = - recordFactory.newRecordInstance(GetContainerStatusRequest.class); - gcsRequest.setContainerId(cId); + GetContainerStatusesRequest gcsRequest = + GetContainerStatusesRequest.newInstance(containerIds); ContainerStatus containerStatus = - containerManager.getContainerStatus(gcsRequest).getStatus(); + containerManager.getContainerStatuses(gcsRequest).getContainerStatuses().get(0); int expectedExitCode = Shell.WINDOWS ? ExitCode.FORCE_KILLED.getExitCode() : ExitCode.TERMINATED.getExitCode(); Assert.assertEquals(expectedExitCode, containerStatus.getExitStatus()); @@ -325,7 +352,7 @@ public class TestContainerManager extends BaseContainerManagerTest { Assert.assertFalse("Process is still alive!", DefaultContainerExecutor.containerIsAlive(pid)); } - + private void testContainerLaunchAndExit(int exitCode) throws IOException, InterruptedException, YarnException { @@ -335,7 +362,7 @@ public class TestContainerManager extends BaseContainerManagerTest { new File(tmpDir, "start_file.txt").getAbsoluteFile(); // ////// Construct the Container-id - ContainerId cId = createContainerId(); + ContainerId cId = createContainerId(0); if (Shell.WINDOWS) { fileWriter.println("@echo Hello World!> " + processStartFile); @@ -376,21 +403,26 @@ public class TestContainerManager extends BaseContainerManagerTest { List commands = Arrays.asList(Shell.getRunScriptCommand(scriptFile)); containerLaunchContext.setCommands(commands); - StartContainerRequest startRequest = recordFactory.newRecordInstance(StartContainerRequest.class); - startRequest.setContainerLaunchContext(containerLaunchContext); - startRequest.setContainerToken(createContainerToken(cId, - DUMMY_RM_IDENTIFIER, context.getNodeId(), user, - context.getContainerTokenSecretManager())); - containerManager.startContainer(startRequest); + StartContainerRequest scRequest = + StartContainerRequest.newInstance( + containerLaunchContext, + createContainerToken(cId, DUMMY_RM_IDENTIFIER, context.getNodeId(), + user, context.getContainerTokenSecretManager())); + List list = new ArrayList(); + list.add(scRequest); + StartContainersRequest allRequests = + StartContainersRequest.newInstance(list); + containerManager.startContainers(allRequests); BaseContainerManagerTest.waitForContainerState(containerManager, cId, ContainerState.COMPLETE); - GetContainerStatusRequest gcsRequest = - recordFactory.newRecordInstance(GetContainerStatusRequest.class); - gcsRequest.setContainerId(cId); + List containerIds = new ArrayList(); + containerIds.add(cId); + GetContainerStatusesRequest gcsRequest = + GetContainerStatusesRequest.newInstance(containerIds); ContainerStatus containerStatus = - containerManager.getContainerStatus(gcsRequest).getStatus(); + containerManager.getContainerStatuses(gcsRequest).getContainerStatuses().get(0); // Verify exit status matches exit state of script Assert.assertEquals(exitCode, @@ -439,7 +471,7 @@ public class TestContainerManager extends BaseContainerManagerTest { fileWriter.close(); // ////// Construct the Container-id - ContainerId cId = createContainerId(); + ContainerId cId = createContainerId(0); ApplicationId appId = cId.getApplicationAttemptId().getApplicationId(); // ////// Construct the container-spec. @@ -460,11 +492,17 @@ public class TestContainerManager extends BaseContainerManagerTest { new HashMap(); localResources.put(destinationFile, rsrc_alpha); containerLaunchContext.setLocalResources(localResources); - StartContainerRequest request = recordFactory.newRecordInstance(StartContainerRequest.class); - request.setContainerLaunchContext(containerLaunchContext); - request.setContainerToken(createContainerToken(cId, DUMMY_RM_IDENTIFIER, - context.getNodeId(), user, context.getContainerTokenSecretManager())); - containerManager.startContainer(request); + + StartContainerRequest scRequest = + StartContainerRequest.newInstance( + containerLaunchContext, + createContainerToken(cId, DUMMY_RM_IDENTIFIER, context.getNodeId(), + user, context.getContainerTokenSecretManager())); + List list = new ArrayList(); + list.add(scRequest); + StartContainersRequest allRequests = + StartContainersRequest.newInstance(list); + containerManager.startContainers(allRequests); BaseContainerManagerTest.waitForContainerState(containerManager, cId, ContainerState.COMPLETE); @@ -528,29 +566,37 @@ public class TestContainerManager extends BaseContainerManagerTest { @Test public void testContainerLaunchFromPreviousRM() throws IOException, - InterruptedException { + InterruptedException, YarnException { containerManager.start(); ContainerLaunchContext containerLaunchContext = recordFactory.newRecordInstance(ContainerLaunchContext.class); - ContainerId cId1 = createContainerId(); - ContainerId cId2 = createContainerId(); + ContainerId cId1 = createContainerId(0); + ContainerId cId2 = createContainerId(0); containerLaunchContext .setLocalResources(new HashMap()); // Construct the Container with Invalid RMIdentifier StartContainerRequest startRequest1 = - recordFactory.newRecordInstance(StartContainerRequest.class); - startRequest1.setContainerLaunchContext(containerLaunchContext); + StartContainerRequest.newInstance(containerLaunchContext, + createContainerToken(cId1, + ResourceManagerConstants.RM_INVALID_IDENTIFIER, context.getNodeId(), + user, context.getContainerTokenSecretManager())); + List list = new ArrayList(); + list.add(startRequest1); + StartContainersRequest allRequests = + StartContainersRequest.newInstance(list); + containerManager.startContainers(allRequests); - startRequest1.setContainerToken(createContainerToken(cId1, - ResourceManagerConstants.RM_INVALID_IDENTIFIER, context.getNodeId(), - user, context.getContainerTokenSecretManager())); boolean catchException = false; try { - containerManager.startContainer(startRequest1); - } catch (YarnException e) { + StartContainersResponse response = containerManager.startContainers(allRequests); + if(response.getFailedRequests().containsKey(cId1)) { + throw response.getFailedRequests().get(cId1).deSerialize(); + } + } catch (Throwable e) { + e.printStackTrace(); catchException = true; Assert.assertTrue(e.getMessage().contains( "Container " + cId1 + " rejected as it is allocated by a previous RM")); @@ -563,21 +609,143 @@ public class TestContainerManager extends BaseContainerManagerTest { // Construct the Container with a RMIdentifier within current RM StartContainerRequest startRequest2 = - recordFactory.newRecordInstance(StartContainerRequest.class); - startRequest2.setContainerLaunchContext(containerLaunchContext); - startRequest2.setContainerToken(createContainerToken(cId2, - DUMMY_RM_IDENTIFIER, context.getNodeId(), user, - context.getContainerTokenSecretManager())); + StartContainerRequest.newInstance(containerLaunchContext, + createContainerToken(cId2, + DUMMY_RM_IDENTIFIER, context.getNodeId(), user, + context.getContainerTokenSecretManager())); + List list2 = new ArrayList(); + list.add(startRequest2); + StartContainersRequest allRequests2 = + StartContainersRequest.newInstance(list2); + containerManager.startContainers(allRequests2); + boolean noException = true; try { - containerManager.startContainer(startRequest2); + containerManager.startContainers(allRequests2); } catch (YarnException e) { noException = false; } // Verify that startContainer get no YarnException Assert.assertTrue(noException); } - + + @Test + public void testMultipleContainersLaunch() throws Exception { + containerManager.start(); + + List list = new ArrayList(); + ContainerLaunchContext containerLaunchContext = + recordFactory.newRecordInstance(ContainerLaunchContext.class); + for (int i = 0; i < 10; i++) { + ContainerId cId = createContainerId(i); + long identifier = 0; + if ((i & 1) == 0) + // container with even id fail + identifier = ResourceManagerConstants.RM_INVALID_IDENTIFIER; + else + identifier = DUMMY_RM_IDENTIFIER; + Token containerToken = + createContainerToken(cId, identifier, context.getNodeId(), user, + context.getContainerTokenSecretManager()); + StartContainerRequest request = + StartContainerRequest.newInstance(containerLaunchContext, + containerToken); + list.add(request); + } + StartContainersRequest requestList = + StartContainersRequest.newInstance(list); + + StartContainersResponse response = + containerManager.startContainers(requestList); + + Assert.assertEquals(5, response.getSuccessfullyStartedContainers().size()); + for (ContainerId id : response.getSuccessfullyStartedContainers()) { + // Containers with odd id should succeed. + Assert.assertEquals(1, id.getId() & 1); + } + Assert.assertEquals(5, response.getFailedRequests().size()); + for (Map.Entry entry : response + .getFailedRequests().entrySet()) { + // Containers with even id should fail. + Assert.assertEquals(0, entry.getKey().getId() & 1); + Assert.assertTrue(entry.getValue().getMessage() + .contains( + "Container " + entry.getKey() + " rejected as it is allocated by a previous RM")); + } + } + + @Test + public void testMultipleContainersStopAndGetStatus() throws Exception { + containerManager.start(); + List startRequest = + new ArrayList(); + ContainerLaunchContext containerLaunchContext = + recordFactory.newRecordInstance(ContainerLaunchContext.class); + + List containerIds = new ArrayList(); + for (int i = 0; i < 10; i++) { + ContainerId cId = createContainerId(i); + String user = null; + if ((i & 1) == 0) { + // container with even id fail + user = "Fail"; + } else { + user = "Pass"; + } + Token containerToken = + createContainerToken(cId, DUMMY_RM_IDENTIFIER, context.getNodeId(), + user, context.getContainerTokenSecretManager()); + StartContainerRequest request = + StartContainerRequest.newInstance(containerLaunchContext, + containerToken); + startRequest.add(request); + containerIds.add(cId); + } + // start containers + StartContainersRequest requestList = + StartContainersRequest.newInstance(startRequest); + containerManager.startContainers(requestList); + + // Get container statuses + GetContainerStatusesRequest statusRequest = + GetContainerStatusesRequest.newInstance(containerIds); + GetContainerStatusesResponse statusResponse = + containerManager.getContainerStatuses(statusRequest); + Assert.assertEquals(5, statusResponse.getContainerStatuses().size()); + for (ContainerStatus status : statusResponse.getContainerStatuses()) { + // Containers with odd id should succeed + Assert.assertEquals(1, status.getContainerId().getId() & 1); + } + Assert.assertEquals(5, statusResponse.getFailedRequests().size()); + for (Map.Entry entry : statusResponse + .getFailedRequests().entrySet()) { + // Containers with even id should fail. + Assert.assertEquals(0, entry.getKey().getId() & 1); + Assert.assertTrue(entry.getValue().getMessage() + .contains("Reject this container")); + } + + // stop containers + StopContainersRequest stopRequest = + StopContainersRequest.newInstance(containerIds); + StopContainersResponse stopResponse = + containerManager.stopContainers(stopRequest); + Assert.assertEquals(5, stopResponse.getSuccessfullyStoppedContainers() + .size()); + for (ContainerId id : stopResponse.getSuccessfullyStoppedContainers()) { + // Containers with odd id should succeed. + Assert.assertEquals(1, id.getId() & 1); + } + Assert.assertEquals(5, stopResponse.getFailedRequests().size()); + for (Map.Entry entry : stopResponse + .getFailedRequests().entrySet()) { + // Containers with even id should fail. + Assert.assertEquals(0, entry.getKey().getId() & 1); + Assert.assertTrue(entry.getValue().getMessage() + .contains("Reject this container")); + } + } + public static Token createContainerToken(ContainerId cId, long rmIdentifier, NodeId nodeId, String user, NMContainerTokenSecretManager containerTokenSecretManager) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java index 9e17c47b777..ee6217cfb6f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java @@ -42,9 +42,10 @@ import org.apache.hadoop.util.Shell; import org.apache.hadoop.util.Shell.ExitCodeException; import org.apache.hadoop.util.StringUtils; import org.apache.hadoop.yarn.api.ApplicationConstants.Environment; -import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusRequest; +import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusesRequest; import org.apache.hadoop.yarn.api.protocolrecords.StartContainerRequest; -import org.apache.hadoop.yarn.api.protocolrecords.StopContainerRequest; +import org.apache.hadoop.yarn.api.protocolrecords.StartContainersRequest; +import org.apache.hadoop.yarn.api.protocolrecords.StopContainersRequest; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ContainerId; @@ -399,10 +400,14 @@ public class TestContainerLaunch extends BaseContainerManagerTest { // set up the rest of the container List commands = Arrays.asList(Shell.getRunScriptCommand(scriptFile)); containerLaunchContext.setCommands(commands); - StartContainerRequest startRequest = recordFactory.newRecordInstance(StartContainerRequest.class); - startRequest.setContainerLaunchContext(containerLaunchContext); - startRequest.setContainerToken(createContainerToken(cId)); - containerManager.startContainer(startRequest); + StartContainerRequest scRequest = + StartContainerRequest.newInstance(containerLaunchContext, + createContainerToken(cId)); + List list = new ArrayList(); + list.add(scRequest); + StartContainersRequest allRequests = + StartContainersRequest.newInstance(list); + containerManager.startContainers(allRequests); int timeoutSecs = 0; while (!processStartFile.exists() && timeoutSecs++ < 20) { @@ -465,18 +470,20 @@ public class TestContainerLaunch extends BaseContainerManagerTest { Assert.assertTrue("Process is not alive!", DefaultContainerExecutor.containerIsAlive(pid)); - StopContainerRequest stopRequest = recordFactory.newRecordInstance(StopContainerRequest.class); - stopRequest.setContainerId(cId); - containerManager.stopContainer(stopRequest); + // Now test the stop functionality. + List containerIds = new ArrayList(); + containerIds.add(cId); + StopContainersRequest stopRequest = + StopContainersRequest.newInstance(containerIds); + containerManager.stopContainers(stopRequest); BaseContainerManagerTest.waitForContainerState(containerManager, cId, ContainerState.COMPLETE); - GetContainerStatusRequest gcsRequest = - recordFactory.newRecordInstance(GetContainerStatusRequest.class); - gcsRequest.setContainerId(cId); + GetContainerStatusesRequest gcsRequest = + GetContainerStatusesRequest.newInstance(containerIds); ContainerStatus containerStatus = - containerManager.getContainerStatus(gcsRequest).getStatus(); + containerManager.getContainerStatuses(gcsRequest).getContainerStatuses().get(0); int expectedExitCode = Shell.WINDOWS ? ExitCode.FORCE_KILLED.getExitCode() : ExitCode.TERMINATED.getExitCode(); Assert.assertEquals(expectedExitCode, containerStatus.getExitStatus()); @@ -544,11 +551,15 @@ public class TestContainerLaunch extends BaseContainerManagerTest { List commands = Arrays.asList(Shell.getRunScriptCommand(scriptFile)); containerLaunchContext.setCommands(commands); Token containerToken = createContainerToken(cId); - StartContainerRequest startRequest = - recordFactory.newRecordInstance(StartContainerRequest.class); - startRequest.setContainerLaunchContext(containerLaunchContext); - startRequest.setContainerToken(containerToken); - containerManager.startContainer(startRequest); + + StartContainerRequest scRequest = + StartContainerRequest.newInstance(containerLaunchContext, + containerToken); + List list = new ArrayList(); + list.add(scRequest); + StartContainersRequest allRequests = + StartContainersRequest.newInstance(list); + containerManager.startContainers(allRequests); int timeoutSecs = 0; while (!processStartFile.exists() && timeoutSecs++ < 20) { @@ -559,19 +570,22 @@ public class TestContainerLaunch extends BaseContainerManagerTest { processStartFile.exists()); // Now test the stop functionality. - StopContainerRequest stopRequest = recordFactory.newRecordInstance(StopContainerRequest.class); - stopRequest.setContainerId(cId); - containerManager.stopContainer(stopRequest); + List containerIds = new ArrayList(); + containerIds.add(cId); + StopContainersRequest stopRequest = + StopContainersRequest.newInstance(containerIds); + containerManager.stopContainers(stopRequest); BaseContainerManagerTest.waitForContainerState(containerManager, cId, ContainerState.COMPLETE); // container stop sends a sigterm followed by a sigkill - GetContainerStatusRequest gcsRequest = - recordFactory.newRecordInstance(GetContainerStatusRequest.class); - gcsRequest.setContainerId(cId); + GetContainerStatusesRequest gcsRequest = + GetContainerStatusesRequest.newInstance(containerIds); + ContainerStatus containerStatus = - containerManager.getContainerStatus(gcsRequest).getStatus(); + containerManager.getContainerStatuses(gcsRequest) + .getContainerStatuses().get(0); Assert.assertEquals(ExitCode.FORCE_KILLED.getExitCode(), containerStatus.getExitStatus()); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/MockLocalResourceStatus.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/MockLocalResourceStatus.java index 882068a7339..1eeab7c81ae 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/MockLocalResourceStatus.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/MockLocalResourceStatus.java @@ -18,8 +18,8 @@ package org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer; import org.apache.hadoop.yarn.api.records.LocalResource; +import org.apache.hadoop.yarn.api.records.SerializedException; import org.apache.hadoop.yarn.api.records.URL; -import org.apache.hadoop.yarn.server.api.records.SerializedException; import org.apache.hadoop.yarn.server.nodemanager.api.protocolrecords.LocalResourceStatus; import org.apache.hadoop.yarn.server.nodemanager.api.protocolrecords.ResourceStatusType; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestResourceLocalizationService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestResourceLocalizationService.java index d4832e0c98f..26578bf5376 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestResourceLocalizationService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestResourceLocalizationService.java @@ -80,6 +80,7 @@ import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.LocalResource; import org.apache.hadoop.yarn.api.records.LocalResourceType; import org.apache.hadoop.yarn.api.records.LocalResourceVisibility; +import org.apache.hadoop.yarn.api.records.SerializedException; import org.apache.hadoop.yarn.api.records.URL; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.event.AsyncDispatcher; @@ -1013,8 +1014,8 @@ public class TestResourceLocalizationService { String localizerId, LocalResourceRequest req) { LocalizerStatus status = createLocalizerStatus(localizerId); LocalResourceStatus resourceStatus = new LocalResourceStatusPBImpl(); - resourceStatus.setException(YarnServerBuilderUtils - .newSerializedException(new YarnException("test"))); + resourceStatus.setException(SerializedException + .newInstance(new YarnException("test"))); resourceStatus.setStatus(ResourceStatusType.FETCH_FAILURE); resourceStatus.setResource(req); status.addResourceStatus(resourceStatus); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java index 152b988cb50..5179f3f965f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java @@ -63,6 +63,7 @@ import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.security.Credentials; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.yarn.api.protocolrecords.StartContainerRequest; +import org.apache.hadoop.yarn.api.protocolrecords.StartContainersRequest; import org.apache.hadoop.yarn.api.records.ApplicationAccessType; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationId; @@ -809,14 +810,18 @@ public class TestLogAggregationService extends BaseContainerManagerTest { commands.add("/bin/bash"); commands.add(scriptFile.getAbsolutePath()); containerLaunchContext.setCommands(commands); - StartContainerRequest startRequest = - recordFactory.newRecordInstance(StartContainerRequest.class); - startRequest.setContainerLaunchContext(containerLaunchContext); - startRequest.setContainerToken(TestContainerManager.createContainerToken( - cId, DUMMY_RM_IDENTIFIER, context.getNodeId(), user, - context.getContainerTokenSecretManager())); - this.containerManager.startContainer(startRequest); + StartContainerRequest scRequest = + StartContainerRequest.newInstance(containerLaunchContext, + TestContainerManager.createContainerToken( + cId, DUMMY_RM_IDENTIFIER, context.getNodeId(), user, + context.getContainerTokenSecretManager())); + List list = new ArrayList(); + list.add(scRequest); + StartContainersRequest allRequests = + StartContainersRequest.newInstance(list); + this.containerManager.startContainers(allRequests); + BaseContainerManagerTest.waitForContainerState(this.containerManager, cId, ContainerState.COMPLETE); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/TestContainersMonitor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/TestContainersMonitor.java index cd73fab4068..e0a4bfe19ba 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/TestContainersMonitor.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/TestContainersMonitor.java @@ -40,8 +40,9 @@ import org.apache.commons.logging.LogFactory; import org.apache.hadoop.fs.FileUtil; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.UnsupportedFileSystemException; -import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusRequest; +import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusesRequest; import org.apache.hadoop.yarn.api.protocolrecords.StartContainerRequest; +import org.apache.hadoop.yarn.api.protocolrecords.StartContainersRequest; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ContainerId; @@ -227,9 +228,6 @@ public class TestContainersMonitor extends BaseContainerManagerTest { commands.add(scriptFile.getAbsolutePath()); containerLaunchContext.setCommands(commands); Resource r = BuilderUtils.newResource(8 * 1024 * 1024, 1); - StartContainerRequest startRequest = - recordFactory.newRecordInstance(StartContainerRequest.class); - startRequest.setContainerLaunchContext(containerLaunchContext); ContainerTokenIdentifier containerIdentifier = new ContainerTokenIdentifier(cId, context.getNodeId().toString(), user, r, System.currentTimeMillis() + 120000, 123, DUMMY_RM_IDENTIFIER); @@ -237,8 +235,14 @@ public class TestContainersMonitor extends BaseContainerManagerTest { BuilderUtils.newContainerToken(context.getNodeId(), containerManager.getContext().getContainerTokenSecretManager() .createPassword(containerIdentifier), containerIdentifier); - startRequest.setContainerToken(containerToken); - containerManager.startContainer(startRequest); + StartContainerRequest scRequest = + StartContainerRequest.newInstance(containerLaunchContext, + containerToken); + List list = new ArrayList(); + list.add(scRequest); + StartContainersRequest allRequests = + StartContainersRequest.newInstance(list); + containerManager.startContainers(allRequests); int timeoutSecs = 0; while (!processStartFile.exists() && timeoutSecs++ < 20) { @@ -260,11 +264,12 @@ public class TestContainersMonitor extends BaseContainerManagerTest { BaseContainerManagerTest.waitForContainerState(containerManager, cId, ContainerState.COMPLETE, 60); - GetContainerStatusRequest gcsRequest = - recordFactory.newRecordInstance(GetContainerStatusRequest.class); - gcsRequest.setContainerId(cId); + List containerIds = new ArrayList(); + containerIds.add(cId); + GetContainerStatusesRequest gcsRequest = + GetContainerStatusesRequest.newInstance(containerIds); ContainerStatus containerStatus = - containerManager.getContainerStatus(gcsRequest).getStatus(); + containerManager.getContainerStatuses(gcsRequest).getContainerStatuses().get(0); Assert.assertEquals(ExitCode.TERMINATED.getExitCode(), containerStatus.getExitStatus()); String expectedMsgPattern = diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/AMLauncher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/AMLauncher.java index c1f02da939e..7dc19b5829d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/AMLauncher.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/AMLauncher.java @@ -22,6 +22,8 @@ import java.io.IOException; import java.net.InetSocketAddress; import java.nio.ByteBuffer; import java.security.PrivilegedAction; +import java.util.ArrayList; +import java.util.List; import java.util.Map; import org.apache.commons.logging.Log; @@ -33,11 +35,15 @@ import org.apache.hadoop.net.NetUtils; import org.apache.hadoop.security.Credentials; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.token.Token; +import org.apache.hadoop.security.token.SecretManager.InvalidToken; import org.apache.hadoop.util.StringUtils; import org.apache.hadoop.yarn.api.ApplicationConstants; import org.apache.hadoop.yarn.api.ContainerManagementProtocol; import org.apache.hadoop.yarn.api.protocolrecords.StartContainerRequest; -import org.apache.hadoop.yarn.api.protocolrecords.StopContainerRequest; +import org.apache.hadoop.yarn.api.protocolrecords.StartContainersRequest; +import org.apache.hadoop.yarn.api.protocolrecords.StartContainersResponse; +import org.apache.hadoop.yarn.api.protocolrecords.StopContainersRequest; +import org.apache.hadoop.yarn.api.protocolrecords.StopContainersResponse; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext; import org.apache.hadoop.yarn.api.records.Container; @@ -46,8 +52,6 @@ import org.apache.hadoop.yarn.api.records.ContainerLaunchContext; import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.hadoop.yarn.event.EventHandler; import org.apache.hadoop.yarn.exceptions.YarnException; -import org.apache.hadoop.yarn.factories.RecordFactory; -import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider; import org.apache.hadoop.yarn.ipc.YarnRPC; import org.apache.hadoop.yarn.security.AMRMTokenIdentifier; import org.apache.hadoop.yarn.server.resourcemanager.RMContext; @@ -68,8 +72,6 @@ public class AMLauncher implements Runnable { private final RMAppAttempt application; private final Configuration conf; - private final RecordFactory recordFactory = - RecordFactoryProvider.getRecordFactory(null); private final AMLauncherEventType eventType; private final RMContext rmContext; private final Container masterContainer; @@ -102,22 +104,42 @@ public class AMLauncher implements Runnable { + " for AM " + application.getAppAttemptId()); ContainerLaunchContext launchContext = createAMContainerLaunchContext(applicationContext, masterContainerID); - StartContainerRequest request = - recordFactory.newRecordInstance(StartContainerRequest.class); - request.setContainerLaunchContext(launchContext); - request.setContainerToken(masterContainer.getContainerToken()); - containerMgrProxy.startContainer(request); - LOG.info("Done launching container " + masterContainer - + " for AM " + application.getAppAttemptId()); + + StartContainerRequest scRequest = + StartContainerRequest.newInstance(launchContext, + masterContainer.getContainerToken()); + List list = new ArrayList(); + list.add(scRequest); + StartContainersRequest allRequests = + StartContainersRequest.newInstance(list); + + StartContainersResponse response = + containerMgrProxy.startContainers(allRequests); + if (response.getFailedRequests() != null + && response.getFailedRequests().containsKey(masterContainerID)) { + Throwable t = + response.getFailedRequests().get(masterContainerID).deSerialize(); + parseAndThrowException(t); + } else { + LOG.info("Done launching container " + masterContainer + " for AM " + + application.getAppAttemptId()); + } } private void cleanup() throws IOException, YarnException { connect(); ContainerId containerId = masterContainer.getId(); - StopContainerRequest stopRequest = - recordFactory.newRecordInstance(StopContainerRequest.class); - stopRequest.setContainerId(containerId); - containerMgrProxy.stopContainer(stopRequest); + List containerIds = new ArrayList(); + containerIds.add(containerId); + StopContainersRequest stopRequest = + StopContainersRequest.newInstance(containerIds); + StopContainersResponse response = + containerMgrProxy.stopContainers(stopRequest); + if (response.getFailedRequests() != null + && response.getFailedRequests().containsKey(containerId)) { + Throwable t = response.getFailedRequests().get(containerId).deSerialize(); + parseAndThrowException(t); + } } // Protected. For tests. @@ -254,4 +276,15 @@ public class AMLauncher implements Runnable { break; } } + + private void parseAndThrowException(Throwable t) throws YarnException, + IOException { + if (t instanceof YarnException) { + throw (YarnException) t; + } else if (t instanceof InvalidToken) { + throw (InvalidToken) t; + } else { + throw (IOException) t; + } + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/Application.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/Application.java index 4d6cbf2f8b5..3c76f0bb144 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/Application.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/Application.java @@ -35,7 +35,8 @@ import org.apache.commons.logging.LogFactory; import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.net.NetworkTopology; import org.apache.hadoop.yarn.api.protocolrecords.StartContainerRequest; -import org.apache.hadoop.yarn.api.protocolrecords.StopContainerRequest; +import org.apache.hadoop.yarn.api.protocolrecords.StartContainersRequest; +import org.apache.hadoop.yarn.api.protocolrecords.StopContainersRequest; import org.apache.hadoop.yarn.api.protocolrecords.SubmitApplicationRequest; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationId; @@ -212,9 +213,11 @@ public class Application { NodeManager nodeManager = task.getNodeManager(); ContainerId containerId = task.getContainerId(); task.stop(); - StopContainerRequest stopRequest = recordFactory.newRecordInstance(StopContainerRequest.class); - stopRequest.setContainerId(containerId); - nodeManager.stopContainer(stopRequest); + List containerIds = new ArrayList(); + containerIds.add(containerId); + StopContainersRequest stopRequest = + StopContainersRequest.newInstance(containerIds); + nodeManager.stopContainers(stopRequest); Resources.subtractFrom(used, requestSpec.get(task.getPriority())); @@ -339,10 +342,15 @@ public class Application { updateResourceRequests(requests.get(priority), type, task); // Launch the container - StartContainerRequest startRequest = recordFactory.newRecordInstance(StartContainerRequest.class); - startRequest.setContainerLaunchContext(createCLC()); - startRequest.setContainerToken(container.getContainerToken()); - nodeManager.startContainer(startRequest); + StartContainerRequest scRequest = + StartContainerRequest.newInstance(createCLC(), + container.getContainerToken()); + List list = + new ArrayList(); + list.add(scRequest); + StartContainersRequest allRequests = + StartContainersRequest.newInstance(list); + nodeManager.startContainers(allRequests); break; } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/NodeManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/NodeManager.java index 7772b04e6a9..2c9d67845a4 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/NodeManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/NodeManager.java @@ -31,12 +31,13 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.yarn.api.ContainerManagementProtocol; -import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusRequest; -import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusResponse; +import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusesRequest; +import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusesResponse; import org.apache.hadoop.yarn.api.protocolrecords.StartContainerRequest; -import org.apache.hadoop.yarn.api.protocolrecords.StartContainerResponse; -import org.apache.hadoop.yarn.api.protocolrecords.StopContainerRequest; -import org.apache.hadoop.yarn.api.protocolrecords.StopContainerResponse; +import org.apache.hadoop.yarn.api.protocolrecords.StartContainersRequest; +import org.apache.hadoop.yarn.api.protocolrecords.StartContainersResponse; +import org.apache.hadoop.yarn.api.protocolrecords.StopContainersRequest; +import org.apache.hadoop.yarn.api.protocolrecords.StopContainersResponse; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.Container; import org.apache.hadoop.yarn.api.records.ContainerId; @@ -158,61 +159,60 @@ public class NodeManager implements ContainerManagementProtocol { } @Override - synchronized public StartContainerResponse startContainer( - StartContainerRequest request) + synchronized public StartContainersResponse startContainers( + StartContainersRequest requests) throws YarnException { - Token containerToken = request.getContainerToken(); - ContainerTokenIdentifier tokenId = null; + for (StartContainerRequest request : requests.getStartContainerRequests()) { + Token containerToken = request.getContainerToken(); + ContainerTokenIdentifier tokenId = null; - try { - tokenId = BuilderUtils.newContainerTokenIdentifier(containerToken); - } catch (IOException e) { - throw RPCUtil.getRemoteException(e); - } - - ContainerId containerID = tokenId.getContainerID(); - ApplicationId applicationId = - containerID.getApplicationAttemptId().getApplicationId(); - - List applicationContainers = containers.get(applicationId); - if (applicationContainers == null) { - applicationContainers = new ArrayList(); - containers.put(applicationId, applicationContainers); - } - - // Sanity check - for (Container container : applicationContainers) { - if (container.getId().compareTo(containerID) - == 0) { - throw new IllegalStateException( - "Container " + containerID + - " already setup on node " + containerManagerAddress); + try { + tokenId = BuilderUtils.newContainerTokenIdentifier(containerToken); + } catch (IOException e) { + throw RPCUtil.getRemoteException(e); } - } - Container container = - BuilderUtils.newContainer(containerID, - this.nodeId, nodeHttpAddress, - tokenId.getResource(), - null, null // DKDC - Doesn't matter + ContainerId containerID = tokenId.getContainerID(); + ApplicationId applicationId = + containerID.getApplicationAttemptId().getApplicationId(); + + List applicationContainers = containers.get(applicationId); + if (applicationContainers == null) { + applicationContainers = new ArrayList(); + containers.put(applicationId, applicationContainers); + } + + // Sanity check + for (Container container : applicationContainers) { + if (container.getId().compareTo(containerID) == 0) { + throw new IllegalStateException("Container " + containerID + + " already setup on node " + containerManagerAddress); + } + } + + Container container = + BuilderUtils.newContainer(containerID, this.nodeId, nodeHttpAddress, + tokenId.getResource(), null, null // DKDC - Doesn't matter ); - ContainerStatus containerStatus = - BuilderUtils.newContainerStatus(container.getId(), ContainerState.NEW, - "", -1000); - applicationContainers.add(container); - containerStatusMap.put(container, containerStatus); - Resources.subtractFrom(available, tokenId.getResource()); - Resources.addTo(used, tokenId.getResource()); - - if(LOG.isDebugEnabled()) { - LOG.debug("startContainer:" + " node=" + containerManagerAddress - + " application=" + applicationId + " container=" + container - + " available=" + available + " used=" + used); - } + ContainerStatus containerStatus = + BuilderUtils.newContainerStatus(container.getId(), + ContainerState.NEW, "", -1000); + applicationContainers.add(container); + containerStatusMap.put(container, containerStatus); + Resources.subtractFrom(available, tokenId.getResource()); + Resources.addTo(used, tokenId.getResource()); - StartContainerResponse response = recordFactory.newRecordInstance(StartContainerResponse.class); + if (LOG.isDebugEnabled()) { + LOG.debug("startContainer:" + " node=" + containerManagerAddress + + " application=" + applicationId + " container=" + container + + " available=" + available + " used=" + used); + } + + } + StartContainersResponse response = + StartContainersResponse.newInstance(null, null, null); return response; } @@ -225,76 +225,79 @@ public class NodeManager implements ContainerManagementProtocol { } @Override - synchronized public StopContainerResponse stopContainer(StopContainerRequest request) + synchronized public StopContainersResponse stopContainers(StopContainersRequest request) throws YarnException { - ContainerId containerID = request.getContainerId(); - String applicationId = String.valueOf( - containerID.getApplicationAttemptId().getApplicationId().getId()); - - // Mark the container as COMPLETE - List applicationContainers = containers.get(applicationId); - for (Container c : applicationContainers) { - if (c.getId().compareTo(containerID) == 0) { - ContainerStatus containerStatus = containerStatusMap.get(c); - containerStatus.setState(ContainerState.COMPLETE); - containerStatusMap.put(c, containerStatus); + for (ContainerId containerID : request.getContainerIds()) { + String applicationId = + String.valueOf(containerID.getApplicationAttemptId() + .getApplicationId().getId()); + + // Mark the container as COMPLETE + List applicationContainers = containers.get(applicationId); + for (Container c : applicationContainers) { + if (c.getId().compareTo(containerID) == 0) { + ContainerStatus containerStatus = containerStatusMap.get(c); + containerStatus.setState(ContainerState.COMPLETE); + containerStatusMap.put(c, containerStatus); + } + } + + // Send a heartbeat + try { + heartbeat(); + } catch (IOException ioe) { + throw RPCUtil.getRemoteException(ioe); + } + + // Remove container and update status + int ctr = 0; + Container container = null; + for (Iterator i = applicationContainers.iterator(); i + .hasNext();) { + container = i.next(); + if (container.getId().compareTo(containerID) == 0) { + i.remove(); + ++ctr; + } + } + + if (ctr != 1) { + throw new IllegalStateException("Container " + containerID + + " stopped " + ctr + " times!"); + } + + Resources.addTo(available, container.getResource()); + Resources.subtractFrom(used, container.getResource()); + + if (LOG.isDebugEnabled()) { + LOG.debug("stopContainer:" + " node=" + containerManagerAddress + + " application=" + applicationId + " container=" + containerID + + " available=" + available + " used=" + used); } } - - // Send a heartbeat - try { - heartbeat(); - } catch (IOException ioe) { - throw RPCUtil.getRemoteException(ioe); - } - - // Remove container and update status - int ctr = 0; - Container container = null; - for (Iterator i=applicationContainers.iterator(); i.hasNext();) { - container = i.next(); - if (container.getId().compareTo(containerID) == 0) { - i.remove(); - ++ctr; - } - } - - if (ctr != 1) { - throw new IllegalStateException("Container " + containerID + - " stopped " + ctr + " times!"); - } - - Resources.addTo(available, container.getResource()); - Resources.subtractFrom(used, container.getResource()); - - if(LOG.isDebugEnabled()) { - LOG.debug("stopContainer:" + " node=" + containerManagerAddress - + " application=" + applicationId + " container=" + containerID - + " available=" + available + " used=" + used); - } - - StopContainerResponse response = recordFactory.newRecordInstance(StopContainerResponse.class); - return response; + return StopContainersResponse.newInstance(null,null); } @Override - synchronized public GetContainerStatusResponse getContainerStatus(GetContainerStatusRequest request) throws YarnException { - ContainerId containerId = request.getContainerId(); - List appContainers = - containers.get( - containerId.getApplicationAttemptId().getApplicationId()); - Container container = null; - for (Container c : appContainers) { - if (c.getId().equals(containerId)) { - container = c; + synchronized public GetContainerStatusesResponse getContainerStatuses( + GetContainerStatusesRequest request) throws YarnException { + List statuses = new ArrayList(); + for (ContainerId containerId : request.getContainerIds()) { + List appContainers = + containers.get(containerId.getApplicationAttemptId() + .getApplicationId()); + Container container = null; + for (Container c : appContainers) { + if (c.getId().equals(containerId)) { + container = c; + } + } + if (container != null + && containerStatusMap.get(container).getState() != null) { + statuses.add(containerStatusMap.get(container)); } } - GetContainerStatusResponse response = - recordFactory.newRecordInstance(GetContainerStatusResponse.class); - if (container != null && containerStatusMap.get(container).getState() != null) { - response.setStatus(containerStatusMap.get(container)); - } - return response; + return GetContainerStatusesResponse.newInstance(statuses, null); } public static org.apache.hadoop.yarn.server.api.records.NodeStatus diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAMAuthorization.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAMAuthorization.java index 7bae0c173fd..af07fb87bb8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAMAuthorization.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAMAuthorization.java @@ -36,14 +36,14 @@ import org.apache.hadoop.security.Credentials; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.yarn.api.ApplicationMasterProtocol; import org.apache.hadoop.yarn.api.ContainerManagementProtocol; -import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusRequest; -import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusResponse; +import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusesRequest; +import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusesResponse; import org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterRequest; import org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterResponse; -import org.apache.hadoop.yarn.api.protocolrecords.StartContainerRequest; -import org.apache.hadoop.yarn.api.protocolrecords.StartContainerResponse; -import org.apache.hadoop.yarn.api.protocolrecords.StopContainerRequest; -import org.apache.hadoop.yarn.api.protocolrecords.StopContainerResponse; +import org.apache.hadoop.yarn.api.protocolrecords.StartContainersRequest; +import org.apache.hadoop.yarn.api.protocolrecords.StartContainersResponse; +import org.apache.hadoop.yarn.api.protocolrecords.StopContainersRequest; +import org.apache.hadoop.yarn.api.protocolrecords.StopContainersResponse; import org.apache.hadoop.yarn.api.records.ApplicationAccessType; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.conf.YarnConfiguration; @@ -98,23 +98,23 @@ public class TestAMAuthorization { } @Override - public StartContainerResponse - startContainer(StartContainerRequest request) + public StartContainersResponse + startContainers(StartContainersRequest request) throws YarnException { - containerTokens = request.getContainerLaunchContext().getTokens(); - return null; + containerTokens = request.getStartContainerRequests().get(0).getContainerLaunchContext().getTokens(); + return StartContainersResponse.newInstance(null, null, null); } @Override - public StopContainerResponse stopContainer(StopContainerRequest request) + public StopContainersResponse stopContainers(StopContainersRequest request) throws YarnException { - return null; + return StopContainersResponse.newInstance(null, null); } @Override - public GetContainerStatusResponse getContainerStatus( - GetContainerStatusRequest request) throws YarnException { - return null; + public GetContainerStatusesResponse getContainerStatuses( + GetContainerStatusesRequest request) throws YarnException { + return GetContainerStatusesResponse.newInstance(null, null); } public Credentials getContainerCredentials() throws IOException { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterLauncher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterLauncher.java index ad55b0c22eb..27dbe869b88 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterLauncher.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterLauncher.java @@ -19,7 +19,10 @@ package org.apache.hadoop.yarn.server.resourcemanager; import java.io.IOException; +import java.nio.ByteBuffer; import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; import java.util.Map; import org.apache.commons.logging.Log; @@ -27,16 +30,18 @@ import org.apache.commons.logging.LogFactory; import org.apache.hadoop.yarn.api.ApplicationConstants; import org.apache.hadoop.yarn.api.ContainerManagementProtocol; import org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse; -import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusRequest; -import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusResponse; +import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusesRequest; +import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusesResponse; import org.apache.hadoop.yarn.api.protocolrecords.StartContainerRequest; -import org.apache.hadoop.yarn.api.protocolrecords.StartContainerResponse; -import org.apache.hadoop.yarn.api.protocolrecords.StopContainerRequest; -import org.apache.hadoop.yarn.api.protocolrecords.StopContainerResponse; +import org.apache.hadoop.yarn.api.protocolrecords.StartContainersRequest; +import org.apache.hadoop.yarn.api.protocolrecords.StartContainersResponse; +import org.apache.hadoop.yarn.api.protocolrecords.StopContainersRequest; +import org.apache.hadoop.yarn.api.protocolrecords.StopContainersResponse; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.ContainerState; import org.apache.hadoop.yarn.api.records.ResourceRequest; +import org.apache.hadoop.yarn.api.records.SerializedException; import org.apache.hadoop.yarn.api.records.Token; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.YarnException; @@ -69,9 +74,10 @@ public class TestApplicationMasterLauncher { int maxAppAttempts; @Override - public StartContainerResponse - startContainer(StartContainerRequest request) + public StartContainersResponse + startContainers(StartContainersRequest requests) throws YarnException { + StartContainerRequest request = requests.getStartContainerRequests().get(0); LOG.info("Container started by MyContainerManager: " + request); launched = true; Map env = @@ -95,11 +101,13 @@ public class TestApplicationMasterLauncher { Long.parseLong(env.get(ApplicationConstants.APP_SUBMIT_TIME_ENV)); maxAppAttempts = Integer.parseInt(env.get(ApplicationConstants.MAX_APP_ATTEMPTS_ENV)); - return null; + return StartContainersResponse.newInstance( + new HashMap(), new ArrayList(), + new HashMap()); } @Override - public StopContainerResponse stopContainer(StopContainerRequest request) + public StopContainersResponse stopContainers(StopContainersRequest request) throws YarnException { LOG.info("Container cleaned up by MyContainerManager"); cleanedup = true; @@ -107,11 +115,10 @@ public class TestApplicationMasterLauncher { } @Override - public GetContainerStatusResponse getContainerStatus( - GetContainerStatusRequest request) throws YarnException { + public GetContainerStatusesResponse getContainerStatuses( + GetContainerStatusesRequest request) throws YarnException { return null; } - } @Test diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestClientToAMTokens.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestClientToAMTokens.java index 0492471ef39..fc2fda85202 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestClientToAMTokens.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestClientToAMTokens.java @@ -19,7 +19,9 @@ package org.apache.hadoop.yarn.server.resourcemanager.security; import static org.junit.Assert.fail; +import static org.mockito.Matchers.any; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; import java.io.IOException; import java.lang.annotation.Annotation; @@ -50,6 +52,8 @@ import org.apache.hadoop.yarn.api.ContainerManagementProtocol; import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportRequest; import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportResponse; import org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterResponse; +import org.apache.hadoop.yarn.api.protocolrecords.StartContainersRequest; +import org.apache.hadoop.yarn.api.protocolrecords.StartContainersResponse; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationReport; import org.apache.hadoop.yarn.event.Dispatcher; @@ -158,6 +162,9 @@ public class TestClientToAMTokens { ContainerManagementProtocol containerManager = mock(ContainerManagementProtocol.class); + StartContainersResponse mockResponse = mock(StartContainersResponse.class); + when(containerManager.startContainers((StartContainersRequest) any())) + .thenReturn(mockResponse); final DrainDispatcher dispatcher = new DrainDispatcher(); MockRM rm = new MockRMWithCustomAMLauncher(conf, containerManager) { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestContainerManagerSecurity.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestContainerManagerSecurity.java index 7781d50c8ee..743bf8afeab 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestContainerManagerSecurity.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestContainerManagerSecurity.java @@ -23,6 +23,8 @@ import static org.junit.Assert.fail; import java.io.IOException; import java.net.InetSocketAddress; import java.security.PrivilegedAction; +import java.util.ArrayList; +import java.util.List; import junit.framework.Assert; @@ -32,15 +34,20 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.CommonConfigurationKeysPublic; import org.apache.hadoop.net.NetUtils; import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.security.token.SecretManager.InvalidToken; import org.apache.hadoop.yarn.api.ContainerManagementProtocol; -import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusRequest; +import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusesRequest; +import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusesResponse; import org.apache.hadoop.yarn.api.protocolrecords.StartContainerRequest; +import org.apache.hadoop.yarn.api.protocolrecords.StartContainersRequest; +import org.apache.hadoop.yarn.api.protocolrecords.StartContainersResponse; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.ContainerLaunchContext; import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.api.records.SerializedException; import org.apache.hadoop.yarn.api.records.Token; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.YarnException; @@ -328,18 +335,21 @@ public class TestContainerManagerSecurity { ContainerId containerId, ApplicationAttemptId appAttemptId, NodeId nodeId, boolean isExceptionExpected) throws Exception { - GetContainerStatusRequest request = - Records.newRecord(GetContainerStatusRequest.class); - request.setContainerId(containerId); - + List containerIds = new ArrayList(); + containerIds.add(containerId); + GetContainerStatusesRequest request = + GetContainerStatusesRequest.newInstance(containerIds); ContainerManagementProtocol proxy = null; - try { proxy = getContainerManagementProtocolProxy(rpc, nmToken, nodeId, appAttemptId.toString()); - proxy.getContainerStatus(request); - + GetContainerStatusesResponse statuses = proxy.getContainerStatuses(request); + if (statuses.getFailedRequests() != null + && statuses.getFailedRequests().containsKey(containerId)) { + parseAndThrowException(statuses.getFailedRequests().get(containerId) + .deSerialize()); + } } finally { if (proxy != null) { rpc.stopProxy(proxy, conf); @@ -352,17 +362,21 @@ public class TestContainerManagerSecurity { org.apache.hadoop.yarn.api.records.Token containerToken, NodeId nodeId, String user) throws Exception { - StartContainerRequest request = - Records.newRecord(StartContainerRequest.class); - request.setContainerToken(containerToken); ContainerLaunchContext context = Records.newRecord(ContainerLaunchContext.class); - request.setContainerLaunchContext(context); - + StartContainerRequest scRequest = + StartContainerRequest.newInstance(context,containerToken); + List list = new ArrayList(); + list.add(scRequest); + StartContainersRequest allRequests = + StartContainersRequest.newInstance(list); ContainerManagementProtocol proxy = null; try { proxy = getContainerManagementProtocolProxy(rpc, nmToken, nodeId, user); - proxy.startContainer(request); + StartContainersResponse response = proxy.startContainers(allRequests); + for(SerializedException ex : response.getFailedRequests().values()){ + parseAndThrowException(ex.deSerialize()); + } } finally { if (proxy != null) { rpc.stopProxy(proxy, conf); @@ -370,6 +384,17 @@ public class TestContainerManagerSecurity { } } + private void parseAndThrowException(Throwable t) throws YarnException, + IOException { + if (t instanceof YarnException) { + throw (YarnException) t; + } else if (t instanceof InvalidToken) { + throw (InvalidToken) t; + } else { + throw (IOException) t; + } + } + protected ContainerManagementProtocol getContainerManagementProtocolProxy( final YarnRPC rpc, org.apache.hadoop.yarn.api.records.Token nmToken, NodeId nodeId, String user) {