From 46b2898a1ab12397992b96f15ccc2102febdfe91 Mon Sep 17 00:00:00 2001 From: Vinod Kumar Vavilapalli Date: Wed, 1 May 2013 23:23:20 +0000 Subject: [PATCH] YARN-618. Modified RM_INVALID_IDENTIFIER to be -1 instead of zero. Contributed by Jian He. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1478230 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-yarn-project/CHANGES.txt | 3 +++ .../server/api/ResourceManagerConstants.java | 7 ++++- .../server/nodemanager/TestEventFlow.java | 8 ++++++ .../BaseContainerManagerTest.java | 7 +++++ .../TestContainerManager.java | 26 +++++-------------- .../launcher/TestContainerLaunch.java | 2 ++ .../TestLogAggregationService.java | 1 + .../monitor/TestContainersMonitor.java | 2 ++ 8 files changed, 36 insertions(+), 20 deletions(-) diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 0fac5c700e2..22524f982bd 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -201,6 +201,9 @@ Release 2.0.5-beta - UNRELEASED to separate out various validation checks depending on whether they rely on RM configuration or not. (Zhijie Shen via vinodkv) + YARN-618. Modified RM_INVALID_IDENTIFIER to be -1 instead of zero. (Jian He + via vinodkv) + OPTIMIZATIONS BUG FIXES diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/ResourceManagerConstants.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/ResourceManagerConstants.java index 3842574a2c2..2daf374ee6e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/ResourceManagerConstants.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/ResourceManagerConstants.java @@ -20,5 +20,10 @@ package org.apache.hadoop.yarn.server.api; public interface ResourceManagerConstants { - public static final long RM_INVALID_IDENTIFIER = 0; + /** + * This states the invalid identifier of Resource Manager. This is used as a + * default value for initializing RM identifier. Currently, RM is using time + * stamp as RM identifier. + */ + public static final long RM_INVALID_IDENTIFIER = -1; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestEventFlow.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestEventFlow.java index 292d00fadf1..48c74f38cae 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestEventFlow.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestEventFlow.java @@ -58,6 +58,7 @@ public class TestEventFlow { TestEventFlow.class.getName() + "-localLogDir").getAbsoluteFile(); private static File remoteLogDir = new File("target", TestEventFlow.class.getName() + "-remoteLogDir").getAbsoluteFile(); + private static final long SIMULATED_RM_IDENTIFIER = 1234; @Test public void testSuccessfulContainerLaunch() throws InterruptedException, @@ -100,6 +101,11 @@ public class TestEventFlow { protected void startStatusUpdater() { return; // Don't start any updating thread. } + + @Override + public long getRMIdentifier() { + return SIMULATED_RM_IDENTIFIER; + } }; DummyContainerManager containerManager = @@ -124,6 +130,8 @@ public class TestEventFlow { when(mockContainer.getId()).thenReturn(cID); when(mockContainer.getResource()).thenReturn(recordFactory .newRecordInstance(Resource.class)); + when(mockContainer.getRMIdentifer()).thenReturn(SIMULATED_RM_IDENTIFIER); + launchContext.setUser("testing"); StartContainerRequest request = recordFactory.newRecordInstance(StartContainerRequest.class); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/BaseContainerManagerTest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/BaseContainerManagerTest.java index 5fd11d58e75..bb17a213ed4 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/BaseContainerManagerTest.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/BaseContainerManagerTest.java @@ -98,6 +98,7 @@ public abstract class BaseContainerManagerTest { protected String user = "nobody"; protected NodeHealthCheckerService nodeHealthChecker; protected LocalDirsHandlerService dirsHandler; + protected final long DUMMY_RM_IDENTIFIER = 1234; protected NodeStatusUpdater nodeStatusUpdater = new NodeStatusUpdaterImpl( context, new AsyncDispatcher(), null, metrics) { @@ -110,6 +111,12 @@ public abstract class BaseContainerManagerTest { protected void startStatusUpdater() { return; // Don't start any updating thread. } + + @Override + public long getRMIdentifier() { + // There is no real RM registration, simulate and set RMIdentifier + return DUMMY_RM_IDENTIFIER; + } }; protected ContainerManagerImpl containerManager = null; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManager.java index df3d9173c93..cce121493af 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManager.java @@ -26,7 +26,6 @@ import java.io.File; import java.io.FileReader; import java.io.IOException; import java.io.PrintWriter; -import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; import java.util.List; @@ -58,12 +57,8 @@ import org.apache.hadoop.yarn.exceptions.YarnRemoteException; import org.apache.hadoop.yarn.server.api.ResourceManagerConstants; import org.apache.hadoop.yarn.server.nodemanager.CMgrCompletedAppsEvent; import org.apache.hadoop.yarn.server.nodemanager.ContainerExecutor.ExitCode; -import org.apache.hadoop.yarn.server.nodemanager.ContainerExecutor.Signal; import org.apache.hadoop.yarn.server.nodemanager.DefaultContainerExecutor; import org.apache.hadoop.yarn.server.nodemanager.DeletionService; -import org.apache.hadoop.yarn.server.nodemanager.LocalRMInterface; -import org.apache.hadoop.yarn.server.nodemanager.NodeStatusUpdater; -import org.apache.hadoop.yarn.server.nodemanager.NodeStatusUpdaterImpl; import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.ApplicationState; import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.ContainerLocalizer; import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.ResourceLocalizationService; @@ -161,6 +156,7 @@ public class TestContainerManager extends BaseContainerManagerTest { when(mockContainer.getNodeId()).thenReturn(context.getNodeId()); when(mockContainer.getNodeHttpAddress()).thenReturn( context.getNodeId().getHost() + ":12345"); + when(mockContainer.getRMIdentifer()).thenReturn(super.DUMMY_RM_IDENTIFIER); StartContainerRequest startRequest = recordFactory.newRecordInstance(StartContainerRequest.class); @@ -259,6 +255,7 @@ public class TestContainerManager extends BaseContainerManagerTest { when(mockContainer.getNodeId()).thenReturn(context.getNodeId()); when(mockContainer.getNodeHttpAddress()).thenReturn( context.getNodeId().getHost() + ":12345"); + when(mockContainer.getRMIdentifer()).thenReturn(super.DUMMY_RM_IDENTIFIER); StartContainerRequest startRequest = recordFactory.newRecordInstance(StartContainerRequest.class); startRequest.setContainerLaunchContext(containerLaunchContext); @@ -370,6 +367,8 @@ public class TestContainerManager extends BaseContainerManagerTest { when(mockContainer.getNodeId()).thenReturn(context.getNodeId()); when(mockContainer.getNodeHttpAddress()).thenReturn( context.getNodeId().getHost() + ":12345"); + when(mockContainer.getRMIdentifer()).thenReturn(super.DUMMY_RM_IDENTIFIER); + StartContainerRequest startRequest = recordFactory.newRecordInstance(StartContainerRequest.class); startRequest.setContainerLaunchContext(containerLaunchContext); startRequest.setContainer(mockContainer); @@ -470,6 +469,7 @@ public class TestContainerManager extends BaseContainerManagerTest { when(mockContainer.getNodeId()).thenReturn(context.getNodeId()); when(mockContainer.getNodeHttpAddress()).thenReturn( context.getNodeId().getHost() + ":12345"); + when(mockContainer.getRMIdentifer()).thenReturn(super.DUMMY_RM_IDENTIFIER); // containerLaunchContext.command = new ArrayList(); @@ -541,19 +541,6 @@ public class TestContainerManager extends BaseContainerManagerTest { @Test public void testContainerLaunchFromPreviousRM() throws IOException, InterruptedException { - // There is no real RM registration, simulate and set RMIdentifier - NodeStatusUpdater nodeStatusUpdater = mock(NodeStatusUpdater.class); - when(nodeStatusUpdater.getRMIdentifier()).thenReturn((long) 1234); - containerManager = - new ContainerManagerImpl(context, exec, delSrvc, nodeStatusUpdater, - metrics, new ApplicationACLsManager(conf), dirsHandler) { - @Override - public void setBlockNewContainerRequests( - boolean blockNewContainerRequests) { - // do nothing - } - }; - containerManager.init(conf); containerManager.start(); ContainerLaunchContext containerLaunchContext = @@ -595,7 +582,8 @@ public class TestContainerManager extends BaseContainerManagerTest { // Construct the Container with a RMIdentifier within current RM Container mockContainer2 = mock(Container.class); when(mockContainer2.getId()).thenReturn(cId2); - when(mockContainer2.getRMIdentifer()).thenReturn((long) 1234); + when(mockContainer2.getRMIdentifer()).thenReturn(super.DUMMY_RM_IDENTIFIER); + when(mockContainer2.getResource()).thenReturn(mockResource); StartContainerRequest startRequest2 = recordFactory.newRecordInstance(StartContainerRequest.class); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java index 05293c55746..9e9d6569d66 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java @@ -180,6 +180,7 @@ public class TestContainerLaunch extends BaseContainerManagerTest { when(mockContainer.getNodeId()).thenReturn(context.getNodeId()); when(mockContainer.getNodeHttpAddress()).thenReturn( context.getNodeId().getHost() + ":12345"); + when(mockContainer.getRMIdentifer()).thenReturn(super.DUMMY_RM_IDENTIFIER); Map userSetEnv = new HashMap(); userSetEnv.put(Environment.CONTAINER_ID.name(), "user_set_container_id"); @@ -376,6 +377,7 @@ public class TestContainerLaunch extends BaseContainerManagerTest { when(mockContainer.getNodeId()).thenReturn(context.getNodeId()); when(mockContainer.getNodeHttpAddress()).thenReturn( context.getNodeId().getHost() + ":12345"); + when(mockContainer.getRMIdentifer()).thenReturn(super.DUMMY_RM_IDENTIFIER); containerLaunchContext.setUser(user); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java index ccbf9f76bf1..4ae97a397bb 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java @@ -691,6 +691,7 @@ public class TestLogAggregationService extends BaseContainerManagerTest { ContainerId cId = BuilderUtils.newContainerId(appAttemptId, 0); when(mockContainer.getId()).thenReturn(cId); + when(mockContainer.getRMIdentifer()).thenReturn(super.DUMMY_RM_IDENTIFIER); containerLaunchContext.setUser(this.user); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/TestContainersMonitor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/TestContainersMonitor.java index ad4a8185737..c56af0e9163 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/TestContainersMonitor.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/TestContainersMonitor.java @@ -216,6 +216,8 @@ public class TestContainersMonitor extends BaseContainerManagerTest { when(mockContainer.getNodeId()).thenReturn(context.getNodeId()); when(mockContainer.getNodeHttpAddress()).thenReturn( context.getNodeId().getHost() + ":12345"); + when(mockContainer.getRMIdentifer()).thenReturn( + super.DUMMY_RM_IDENTIFIER); containerLaunchContext.setUser(user); URL resource_alpha =