From 6706e58f618cf8995634e7ab8bc62d3a6c6a5821 Mon Sep 17 00:00:00 2001 From: Vinod Kumar Vavilapalli Date: Tue, 14 May 2013 02:50:27 +0000 Subject: [PATCH] MAPREDUCE-5240. Fix a bug in MRAppMaster because of which OutputCommitter could not access credentials set by the user. Contributed by Vinod Kumar Vavilapalli. svn merge --ignore-ancestry -c 1482175 ../../trunk/ git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/branch-2@1482176 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-mapreduce-project/CHANGES.txt | 3 ++ .../hadoop/mapreduce/v2/app/MRAppMaster.java | 5 ++- .../mapreduce/v2/app/TestMRAppMaster.java | 42 +++++++++++++------ 3 files changed, 35 insertions(+), 15 deletions(-) diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index b61f81aae23..77a27cb0268 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -248,6 +248,9 @@ Release 2.0.5-beta - UNRELEASED MAPREDUCE-5220. Setter methods in TaskCompletionEvent are public in MR1 and protected in MR2. (sandyr via tucu) + MAPREDUCE-5240. Fix a bug in MRAppMaster because of which OutputCommitter + could not access credentials set by the user. (vinodkv) + Release 2.0.4-alpha - 2013-04-25 INCOMPATIBLE CHANGES diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/MRAppMaster.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/MRAppMaster.java index 21524fc62c4..4d07698b95d 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/MRAppMaster.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/MRAppMaster.java @@ -1304,7 +1304,7 @@ public class MRAppMaster extends CompositeService { Integer.parseInt(maxAppAttempts)); ShutdownHookManager.get().addShutdownHook( new MRAppMasterShutdownHook(appMaster), SHUTDOWN_HOOK_PRIORITY); - YarnConfiguration conf = new YarnConfiguration(new JobConf()); + JobConf conf = new JobConf(new YarnConfiguration()); conf.addResource(new Path(MRJobConfig.JOB_CONF_FILE)); String jobUserName = System .getenv(ApplicationConstants.Environment.USER.name()); @@ -1357,7 +1357,7 @@ public class MRAppMaster extends CompositeService { } protected static void initAndStartAppMaster(final MRAppMaster appMaster, - final YarnConfiguration conf, String jobUserName) throws IOException, + final JobConf conf, String jobUserName) throws IOException, InterruptedException { UserGroupInformation.setConfiguration(conf); // Security framework already loaded the tokens into current UGI, just use @@ -1367,6 +1367,7 @@ public class MRAppMaster extends CompositeService { UserGroupInformation appMasterUgi = UserGroupInformation .createRemoteUser(jobUserName); appMasterUgi.addCredentials(credentials); + conf.getCredentials().addAll(credentials); appMasterUgi.doAs(new PrivilegedExceptionAction() { @Override public Object run() throws Exception { diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestMRAppMaster.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestMRAppMaster.java index 0479175db4e..e29c5ac9c87 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestMRAppMaster.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestMRAppMaster.java @@ -40,6 +40,7 @@ import org.apache.hadoop.fs.FileContext; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.Text; +import org.apache.hadoop.mapred.JobConf; import org.apache.hadoop.mapreduce.MRJobConfig; import org.apache.hadoop.mapreduce.OutputCommitter; import org.apache.hadoop.mapreduce.TypeConverter; @@ -61,7 +62,6 @@ import org.apache.hadoop.yarn.YarnException; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ContainerId; -import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.event.EventHandler; import org.apache.hadoop.yarn.util.BuilderUtils; import org.apache.hadoop.yarn.util.ConverterUtils; @@ -113,7 +113,7 @@ public class TestMRAppMaster { MRAppMasterTest appMaster = new MRAppMasterTest(applicationAttemptId, containerId, "host", -1, -1, System.currentTimeMillis(), MRJobConfig.DEFAULT_MR_AM_MAX_ATTEMPTS); - YarnConfiguration conf = new YarnConfiguration(); + JobConf conf = new JobConf(); conf.set(MRJobConfig.MR_AM_STAGING_DIR, stagingDir); MRAppMaster.initAndStartAppMaster(appMaster, conf, userName); assertEquals(stagingDir + Path.SEPARATOR + userName + Path.SEPARATOR @@ -126,7 +126,7 @@ public class TestMRAppMaster { String applicationAttemptIdStr = "appattempt_1317529182569_0004_000002"; String containerIdStr = "container_1317529182569_0004_000002_1"; String userName = "TestAppMasterUser"; - YarnConfiguration conf = new YarnConfiguration(); + JobConf conf = new JobConf(); conf.set(MRJobConfig.MR_AM_STAGING_DIR, stagingDir); ApplicationAttemptId applicationAttemptId = ConverterUtils .toApplicationAttemptId(applicationAttemptIdStr); @@ -161,7 +161,7 @@ public class TestMRAppMaster { String applicationAttemptIdStr = "appattempt_1317529182569_0004_000002"; String containerIdStr = "container_1317529182569_0004_000002_1"; String userName = "TestAppMasterUser"; - YarnConfiguration conf = new YarnConfiguration(); + JobConf conf = new JobConf(); conf.set(MRJobConfig.MR_AM_STAGING_DIR, stagingDir); ApplicationAttemptId applicationAttemptId = ConverterUtils .toApplicationAttemptId(applicationAttemptIdStr); @@ -197,7 +197,7 @@ public class TestMRAppMaster { String applicationAttemptIdStr = "appattempt_1317529182569_0004_000002"; String containerIdStr = "container_1317529182569_0004_000002_1"; String userName = "TestAppMasterUser"; - YarnConfiguration conf = new YarnConfiguration(); + JobConf conf = new JobConf(); conf.set(MRJobConfig.MR_AM_STAGING_DIR, stagingDir); ApplicationAttemptId applicationAttemptId = ConverterUtils .toApplicationAttemptId(applicationAttemptIdStr); @@ -233,7 +233,7 @@ public class TestMRAppMaster { String applicationAttemptIdStr = "appattempt_1317529182569_0004_000002"; String containerIdStr = "container_1317529182569_0004_000002_1"; String userName = "TestAppMasterUser"; - YarnConfiguration conf = new YarnConfiguration(); + JobConf conf = new JobConf(); conf.set(MRJobConfig.MR_AM_STAGING_DIR, stagingDir); ApplicationAttemptId applicationAttemptId = ConverterUtils .toApplicationAttemptId(applicationAttemptIdStr); @@ -278,7 +278,7 @@ public class TestMRAppMaster { ApplicationAttemptId applicationAttemptId = ConverterUtils .toApplicationAttemptId(applicationAttemptIdStr); ContainerId containerId = ConverterUtils.toContainerId(containerIdStr); - YarnConfiguration conf = new YarnConfiguration(); + JobConf conf = new JobConf(); conf.set(MRJobConfig.MR_AM_STAGING_DIR, stagingDir); File stagingDir = @@ -343,10 +343,12 @@ public class TestMRAppMaster { new Token(identifier, password, kind, service); Text tokenAlias = new Text("myToken"); credentials.addToken(tokenAlias, myToken); + Text keyAlias = new Text("mySecretKeyAlias"); + credentials.addSecretKey(keyAlias, "mySecretKey".getBytes()); Token storedToken = credentials.getToken(tokenAlias); - YarnConfiguration conf = new YarnConfiguration(); + JobConf conf = new JobConf(); Path tokenFilePath = new Path(testDir.getAbsolutePath(), "tokens-file"); Map newEnv = new HashMap(); @@ -379,14 +381,28 @@ public class TestMRAppMaster { // Now validate the credentials Credentials appMasterCreds = appMaster.credentials; Assert.assertNotNull(appMasterCreds); + Assert.assertEquals(1, appMasterCreds.numberOfSecretKeys()); + Assert.assertEquals(1, appMasterCreds.numberOfTokens()); + + // Validate the tokens Token usedToken = appMasterCreds.getToken(tokenAlias); Assert.assertNotNull(usedToken); - Assert - .assertEquals("MyIdentifier", new String(storedToken.getIdentifier())); - Assert.assertEquals("MyPassword", new String(storedToken.getPassword())); - Assert.assertEquals("MyTokenKind", storedToken.getKind().toString()); - Assert.assertEquals("host:port", storedToken.getService().toString()); + Assert.assertEquals(storedToken, usedToken); + + // Validate the keys + byte[] usedKey = appMasterCreds.getSecretKey(keyAlias); + Assert.assertNotNull(usedKey); + Assert.assertEquals("mySecretKey", new String(usedKey)); + + // The credentials should also be added to conf so that OuputCommitter can + // access it + Credentials confCredentials = conf.getCredentials(); + Assert.assertEquals(1, confCredentials.numberOfSecretKeys()); + Assert.assertEquals(1, confCredentials.numberOfTokens()); + Assert.assertEquals(storedToken, confCredentials.getToken(tokenAlias)); + Assert.assertEquals("mySecretKey", + new String(confCredentials.getSecretKey(keyAlias))); } }