From a70140e1bae8fc06f05771306f7117449953b786 Mon Sep 17 00:00:00 2001 From: Vinod Kumar Vavilapalli Date: Tue, 19 Nov 2013 00:52:04 +0000 Subject: [PATCH] YARN-709. Added tests to verify validity of delegation tokens and logging of appsummary after RM restart. Contributed by Jian He. svn merge --ignore-ancestry -c 1543269 ../../trunk/ git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/branch-2@1543271 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-yarn-project/CHANGES.txt | 3 + .../server/resourcemanager/RMAppManager.java | 10 +++- .../yarn/server/resourcemanager/MockRM.java | 4 ++ .../server/resourcemanager/TestRMRestart.java | 56 ++++++++++++++++++- 4 files changed, 69 insertions(+), 4 deletions(-) diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index cd1a8427f1f..91eb325a0bf 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -82,6 +82,9 @@ Release 2.3.0 - UNRELEASED YARN-1222. Make improvements in ZKRMStateStore for fencing (Karthik Kambatla via bikas) + YARN-709. Added tests to verify validity of delegation tokens and logging of + appsummary after RM restart. (Jian He via vinodkv) + OPTIMIZATIONS BUG FIXES diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java index 55b748d8952..9df9fa29603 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java @@ -54,6 +54,8 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.YarnScheduler; import org.apache.hadoop.yarn.server.security.ApplicationACLsManager; import org.apache.hadoop.yarn.server.utils.BuilderUtils; +import com.google.common.annotations.VisibleForTesting; + /** * This class manages the list of applications for the resource manager. */ @@ -165,6 +167,11 @@ public class RMAppManager implements EventHandler, } } + @VisibleForTesting + public void logApplicationSummary(ApplicationId appId) { + ApplicationSummary.logAppSummary(rmContext.getRMApps().get(appId)); + } + protected synchronized void setCompletedAppsMax(int max) { this.completedAppsMax = max; } @@ -351,8 +358,7 @@ public class RMAppManager implements EventHandler, case APP_COMPLETED: { finishApplication(applicationId); - ApplicationSummary.logAppSummary( - rmContext.getRMApps().get(applicationId)); + logApplicationSummary(applicationId); checkAppNumCompletedLimit(); } break; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java index cb41ca4e45e..c9a8e6a9ecb 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java @@ -421,6 +421,10 @@ public class MockRM extends ResourceManager { return this.clientToAMSecretManager; } + public RMAppManager getRMAppManager() { + return this.rmAppManager; + } + @Override protected void startWepApp() { // override to disable webapp diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java index f87f6894bbd..b1d1e19a898 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java @@ -18,6 +18,11 @@ package org.apache.hadoop.yarn.server.resourcemanager; +import static org.mockito.Matchers.isA; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; + import java.io.IOException; import java.net.InetSocketAddress; import java.net.UnknownHostException; @@ -577,7 +582,14 @@ public class TestRMRestart { rm1.waitForState(am2.getApplicationAttemptId(), RMAppAttemptState.KILLED); // restart rm - MockRM rm2 = new MockRM(conf, memStore); + + MockRM rm2 = new MockRM(conf, memStore) { + @Override + protected RMAppManager createRMAppManager() { + return spy(super.createRMAppManager()); + } + }; + rm2.start(); GetApplicationsRequest request1 = @@ -620,6 +632,10 @@ public class TestRMRestart { rm2.getClientRMService().getApplications(request2); List appList2 = response2.getApplicationList(); Assert.assertTrue(3 == appList2.size()); + + // check application summary is logged for the completed apps after RM restart. + verify(rm2.getRMAppManager(), times(3)).logApplicationSummary( + isA(ApplicationId.class)); } private MockAM launchAM(RMApp app, MockRM rm, MockNM nm) @@ -920,7 +936,6 @@ public class TestRMRestart { @Test public void testRMDelegationTokenRestoredOnRMRestart() throws Exception { conf.setInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, 2); - conf.set( CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION, "kerberos"); @@ -1063,6 +1078,43 @@ public class TestRMRestart { rm2.stop(); } + // This is to test submit an application to the new RM with the old delegation + // token got from previous RM. + @Test + public void testAppSubmissionWithOldDelegationTokenAfterRMRestart() + throws Exception { + conf.setInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, 2); + conf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION, + "kerberos"); + conf.set(YarnConfiguration.RM_ADDRESS, "localhost:8032"); + UserGroupInformation.setConfiguration(conf); + MemoryRMStateStore memStore = new MemoryRMStateStore(); + memStore.init(conf); + + MockRM rm1 = new TestSecurityMockRM(conf, memStore); + rm1.start(); + + GetDelegationTokenRequest request1 = + GetDelegationTokenRequest.newInstance("renewer1"); + UserGroupInformation.getCurrentUser().setAuthenticationMethod( + AuthMethod.KERBEROS); + GetDelegationTokenResponse response1 = + rm1.getClientRMService().getDelegationToken(request1); + Token token1 = + ConverterUtils.convertFromYarn(response1.getRMDelegationToken(), rmAddr); + + // start new RM + MockRM rm2 = new TestSecurityMockRM(conf, memStore); + rm2.start(); + + // submit an app with the old delegation token got from previous RM. + Credentials ts = new Credentials(); + ts.addToken(token1.getService(), token1); + RMApp app = rm2.submitApp(200, "name", "user", + new HashMap(), false, "default", 1, ts); + rm2.waitForState(app.getApplicationId(), RMAppState.ACCEPTED); + } + @Test public void testRMStateStoreDispatcherDrainedOnRMStop() throws Exception { MemoryRMStateStore memStore = new MemoryRMStateStore() {