From fbdc7ed67dbd8499aacdd8f1af9696b58e784e4d Mon Sep 17 00:00:00 2001 From: Vinod Kumar Vavilapalli Date: Wed, 1 May 2013 23:48:32 +0000 Subject: [PATCH] YARN-618. Modified RM_INVALID_IDENTIFIER to be -1 instead of zero. Contributed by Jian He. svn merge --ignore-ancestry -c 1478230 ../../trunk/ git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/branch-2@1478235 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-yarn-project/CHANGES.txt | 3 +++ .../server/api/ResourceManagerConstants.java | 7 +++++- .../server/nodemanager/TestEventFlow.java | 8 +++++++ .../BaseContainerManagerTest.java | 7 ++++++ .../TestContainerManager.java | 23 +++++++------------ .../launcher/TestContainerLaunch.java | 2 ++ .../TestLogAggregationService.java | 1 + .../monitor/TestContainersMonitor.java | 2 ++ 8 files changed, 37 insertions(+), 16 deletions(-) diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index fce7fbff0c8..ae06849fc49 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -126,6 +126,9 @@ Release 2.0.5-beta - UNRELEASED to separate out various validation checks depending on whether they rely on RM configuration or not. (Zhijie Shen via vinodkv) + YARN-618. Modified RM_INVALID_IDENTIFIER to be -1 instead of zero. (Jian He + via vinodkv) + OPTIMIZATIONS BUG FIXES diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/ResourceManagerConstants.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/ResourceManagerConstants.java index 3842574a2c2..2daf374ee6e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/ResourceManagerConstants.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/ResourceManagerConstants.java @@ -20,5 +20,10 @@ package org.apache.hadoop.yarn.server.api; public interface ResourceManagerConstants { - public static final long RM_INVALID_IDENTIFIER = 0; + /** + * This states the invalid identifier of Resource Manager. This is used as a + * default value for initializing RM identifier. Currently, RM is using time + * stamp as RM identifier. + */ + public static final long RM_INVALID_IDENTIFIER = -1; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestEventFlow.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestEventFlow.java index 292d00fadf1..48c74f38cae 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestEventFlow.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestEventFlow.java @@ -58,6 +58,7 @@ public class TestEventFlow { TestEventFlow.class.getName() + "-localLogDir").getAbsoluteFile(); private static File remoteLogDir = new File("target", TestEventFlow.class.getName() + "-remoteLogDir").getAbsoluteFile(); + private static final long SIMULATED_RM_IDENTIFIER = 1234; @Test public void testSuccessfulContainerLaunch() throws InterruptedException, @@ -100,6 +101,11 @@ public class TestEventFlow { protected void startStatusUpdater() { return; // Don't start any updating thread. } + + @Override + public long getRMIdentifier() { + return SIMULATED_RM_IDENTIFIER; + } }; DummyContainerManager containerManager = @@ -124,6 +130,8 @@ public class TestEventFlow { when(mockContainer.getId()).thenReturn(cID); when(mockContainer.getResource()).thenReturn(recordFactory .newRecordInstance(Resource.class)); + when(mockContainer.getRMIdentifer()).thenReturn(SIMULATED_RM_IDENTIFIER); + launchContext.setUser("testing"); StartContainerRequest request = recordFactory.newRecordInstance(StartContainerRequest.class); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/BaseContainerManagerTest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/BaseContainerManagerTest.java index 36e433be342..7da531ece56 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/BaseContainerManagerTest.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/BaseContainerManagerTest.java @@ -98,6 +98,7 @@ public abstract class BaseContainerManagerTest { protected String user = "nobody"; protected NodeHealthCheckerService nodeHealthChecker; protected LocalDirsHandlerService dirsHandler; + protected final long DUMMY_RM_IDENTIFIER = 1234; protected NodeStatusUpdater nodeStatusUpdater = new NodeStatusUpdaterImpl( context, new AsyncDispatcher(), null, metrics) { @@ -110,6 +111,12 @@ public abstract class BaseContainerManagerTest { protected void startStatusUpdater() { return; // Don't start any updating thread. } + + @Override + public long getRMIdentifier() { + // There is no real RM registration, simulate and set RMIdentifier + return DUMMY_RM_IDENTIFIER; + } }; protected ContainerManagerImpl containerManager = null; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManager.java index 4177491951c..c86fbc7e928 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManager.java @@ -59,7 +59,6 @@ import org.apache.hadoop.yarn.server.nodemanager.CMgrCompletedAppsEvent; import org.apache.hadoop.yarn.server.nodemanager.ContainerExecutor.ExitCode; import org.apache.hadoop.yarn.server.nodemanager.ContainerExecutor.Signal; import org.apache.hadoop.yarn.server.nodemanager.DeletionService; -import org.apache.hadoop.yarn.server.nodemanager.NodeStatusUpdater; import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.ApplicationState; import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.ContainerLocalizer; import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.ResourceLocalizationService; @@ -157,6 +156,7 @@ public class TestContainerManager extends BaseContainerManagerTest { when(mockContainer.getNodeId()).thenReturn(context.getNodeId()); when(mockContainer.getNodeHttpAddress()).thenReturn( context.getNodeId().getHost() + ":12345"); + when(mockContainer.getRMIdentifer()).thenReturn(super.DUMMY_RM_IDENTIFIER); StartContainerRequest startRequest = recordFactory.newRecordInstance(StartContainerRequest.class); @@ -250,6 +250,8 @@ public class TestContainerManager extends BaseContainerManagerTest { when(mockContainer.getNodeId()).thenReturn(context.getNodeId()); when(mockContainer.getNodeHttpAddress()).thenReturn( context.getNodeId().getHost() + ":12345"); + when(mockContainer.getRMIdentifer()).thenReturn(super.DUMMY_RM_IDENTIFIER); + StartContainerRequest startRequest = recordFactory.newRecordInstance(StartContainerRequest.class); startRequest.setContainerLaunchContext(containerLaunchContext); startRequest.setContainer(mockContainer); @@ -357,6 +359,8 @@ public class TestContainerManager extends BaseContainerManagerTest { when(mockContainer.getNodeId()).thenReturn(context.getNodeId()); when(mockContainer.getNodeHttpAddress()).thenReturn( context.getNodeId().getHost() + ":12345"); + when(mockContainer.getRMIdentifer()).thenReturn(super.DUMMY_RM_IDENTIFIER); + StartContainerRequest startRequest = recordFactory.newRecordInstance(StartContainerRequest.class); startRequest.setContainerLaunchContext(containerLaunchContext); startRequest.setContainer(mockContainer); @@ -457,6 +461,7 @@ public class TestContainerManager extends BaseContainerManagerTest { when(mockContainer.getNodeId()).thenReturn(context.getNodeId()); when(mockContainer.getNodeHttpAddress()).thenReturn( context.getNodeId().getHost() + ":12345"); + when(mockContainer.getRMIdentifer()).thenReturn(super.DUMMY_RM_IDENTIFIER); // containerLaunchContext.command = new ArrayList(); @@ -528,19 +533,6 @@ public class TestContainerManager extends BaseContainerManagerTest { @Test public void testContainerLaunchFromPreviousRM() throws IOException, InterruptedException { - // There is no real RM registration, simulate and set RMIdentifier - NodeStatusUpdater nodeStatusUpdater = mock(NodeStatusUpdater.class); - when(nodeStatusUpdater.getRMIdentifier()).thenReturn((long) 1234); - containerManager = - new ContainerManagerImpl(context, exec, delSrvc, nodeStatusUpdater, - metrics, new ApplicationACLsManager(conf), dirsHandler) { - @Override - public void setBlockNewContainerRequests( - boolean blockNewContainerRequests) { - // do nothing - } - }; - containerManager.init(conf); containerManager.start(); ContainerLaunchContext containerLaunchContext = @@ -582,7 +574,8 @@ public class TestContainerManager extends BaseContainerManagerTest { // Construct the Container with a RMIdentifier within current RM Container mockContainer2 = mock(Container.class); when(mockContainer2.getId()).thenReturn(cId2); - when(mockContainer2.getRMIdentifer()).thenReturn((long) 1234); + when(mockContainer2.getRMIdentifer()).thenReturn(super.DUMMY_RM_IDENTIFIER); + when(mockContainer2.getResource()).thenReturn(mockResource); StartContainerRequest startRequest2 = recordFactory.newRecordInstance(StartContainerRequest.class); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java index 0cc6a687c07..f70112336bc 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java @@ -170,6 +170,7 @@ public class TestContainerLaunch extends BaseContainerManagerTest { when(mockContainer.getNodeId()).thenReturn(context.getNodeId()); when(mockContainer.getNodeHttpAddress()).thenReturn( context.getNodeId().getHost() + ":12345"); + when(mockContainer.getRMIdentifer()).thenReturn(super.DUMMY_RM_IDENTIFIER); Map userSetEnv = new HashMap(); userSetEnv.put(Environment.CONTAINER_ID.name(), "user_set_container_id"); @@ -348,6 +349,7 @@ public class TestContainerLaunch extends BaseContainerManagerTest { when(mockContainer.getNodeId()).thenReturn(context.getNodeId()); when(mockContainer.getNodeHttpAddress()).thenReturn( context.getNodeId().getHost() + ":12345"); + when(mockContainer.getRMIdentifer()).thenReturn(super.DUMMY_RM_IDENTIFIER); containerLaunchContext.setUser(user); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java index ccbf9f76bf1..4ae97a397bb 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java @@ -691,6 +691,7 @@ public class TestLogAggregationService extends BaseContainerManagerTest { ContainerId cId = BuilderUtils.newContainerId(appAttemptId, 0); when(mockContainer.getId()).thenReturn(cId); + when(mockContainer.getRMIdentifer()).thenReturn(super.DUMMY_RM_IDENTIFIER); containerLaunchContext.setUser(this.user); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/TestContainersMonitor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/TestContainersMonitor.java index ad4a8185737..c56af0e9163 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/TestContainersMonitor.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/TestContainersMonitor.java @@ -216,6 +216,8 @@ public class TestContainersMonitor extends BaseContainerManagerTest { when(mockContainer.getNodeId()).thenReturn(context.getNodeId()); when(mockContainer.getNodeHttpAddress()).thenReturn( context.getNodeId().getHost() + ":12345"); + when(mockContainer.getRMIdentifer()).thenReturn( + super.DUMMY_RM_IDENTIFIER); containerLaunchContext.setUser(user); URL resource_alpha =