diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index e1742a94d61..26adf9eccc5 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -282,6 +282,9 @@ Release 2.1.0-beta - UNRELEASED MAPREDUCE-5283. Over 10 different tests have near identical implementations of AppContext (Sandy Ryza via jlowe) + MAPREDUCE-5199. Removing ApplicationTokens file as it is no longer needed. + (Daryn Sharp via vinodkv) + OPTIMIZATIONS MAPREDUCE-4974. Optimising the LineRecordReader initialize() method diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/MRAppMaster.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/MRAppMaster.java index 43e7e19a232..edfd5addcd4 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/MRAppMaster.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/MRAppMaster.java @@ -18,7 +18,6 @@ package org.apache.hadoop.mapreduce.v2.app; -import java.io.File; import java.io.IOException; import java.lang.reflect.Constructor; import java.lang.reflect.InvocationTargetException; @@ -37,7 +36,6 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FSDataInputStream; -import org.apache.hadoop.fs.FileContext; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.mapred.FileOutputCommitter; @@ -107,6 +105,7 @@ import org.apache.hadoop.mapreduce.v2.util.MRBuilderUtils; import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem; import org.apache.hadoop.security.Credentials; import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.security.token.Token; import org.apache.hadoop.util.ReflectionUtils; import org.apache.hadoop.util.ShutdownHookManager; import org.apache.hadoop.util.StringInterner; @@ -125,6 +124,7 @@ import org.apache.hadoop.yarn.event.AsyncDispatcher; import org.apache.hadoop.yarn.event.Dispatcher; import org.apache.hadoop.yarn.event.Event; import org.apache.hadoop.yarn.event.EventHandler; +import org.apache.hadoop.yarn.security.ApplicationTokenIdentifier; import org.apache.hadoop.yarn.service.AbstractService; import org.apache.hadoop.yarn.service.CompositeService; import org.apache.hadoop.yarn.service.Service; @@ -192,7 +192,7 @@ public class MRAppMaster extends CompositeService { private SpeculatorEventDispatcher speculatorEventDispatcher; private Job job; - private Credentials fsTokens = new Credentials(); // Filled during init + private Credentials jobCredentials = new Credentials(); // Filled during init protected UserGroupInformation currentUser; // Will be setup during init private volatile boolean isLastAMRetry = false; @@ -231,7 +231,7 @@ public class MRAppMaster extends CompositeService { protected void serviceInit(final Configuration conf) throws Exception { conf.setBoolean(Dispatcher.DISPATCHER_EXIT_ON_ERROR_KEY, true); - downloadTokensAndSetupUGI(conf); + initJobCredentialsAndUGI(conf); isLastAMRetry = appAttemptID.getAttemptId() >= maxAppAttempts; LOG.info("The specific max attempts: " + maxAppAttempts + @@ -470,7 +470,7 @@ public class MRAppMaster extends CompositeService { } protected Credentials getCredentials() { - return fsTokens; + return jobCredentials; } /** @@ -590,7 +590,7 @@ public class MRAppMaster extends CompositeService { // create single job Job newJob = new JobImpl(jobId, appAttemptID, conf, dispatcher.getEventHandler(), - taskAttemptListener, jobTokenSecretManager, fsTokens, clock, + taskAttemptListener, jobTokenSecretManager, jobCredentials, clock, completedTasksFromPreviousRun, metrics, committer, newApiCommitter, currentUser.getUserName(), appSubmitTime, amInfos, context, @@ -607,22 +607,11 @@ public class MRAppMaster extends CompositeService { * Obtain the tokens needed by the job and put them in the UGI * @param conf */ - protected void downloadTokensAndSetupUGI(Configuration conf) { + protected void initJobCredentialsAndUGI(Configuration conf) { try { this.currentUser = UserGroupInformation.getCurrentUser(); - - // Read the file-system tokens from the localized tokens-file. - Path jobSubmitDir = - FileContext.getLocalFSFileContext().makeQualified( - new Path(new File(MRJobConfig.JOB_SUBMIT_DIR) - .getAbsolutePath())); - Path jobTokenFile = - new Path(jobSubmitDir, MRJobConfig.APPLICATION_TOKENS_FILE); - fsTokens.addAll(Credentials.readTokenStorageFile(jobTokenFile, conf)); - LOG.info("jobSubmitDir=" + jobSubmitDir + " jobTokenFile=" - + jobTokenFile); - currentUser.addCredentials(fsTokens); // For use by AppMaster itself. + this.jobCredentials = ((JobConf)conf).getCredentials(); } catch (IOException e) { throw new YarnRuntimeException(e); } @@ -1034,7 +1023,7 @@ public class MRAppMaster extends CompositeService { // are reducers as the shuffle secret would be app attempt specific. int numReduceTasks = getConfig().getInt(MRJobConfig.NUM_REDUCES, 0); boolean shuffleKeyValidForRecovery = (numReduceTasks > 0 && - TokenCache.getShuffleSecretKey(fsTokens) != null); + TokenCache.getShuffleSecretKey(jobCredentials) != null); if (recoveryEnabled && recoverySupportedByCommitter && shuffleKeyValidForRecovery) { @@ -1372,9 +1361,23 @@ public class MRAppMaster extends CompositeService { // them Credentials credentials = UserGroupInformation.getCurrentUser().getCredentials(); + LOG.info("Executing with tokens:"); + for (Token token : credentials.getAllTokens()) { + LOG.info(token); + } + UserGroupInformation appMasterUgi = UserGroupInformation .createRemoteUser(jobUserName); appMasterUgi.addCredentials(credentials); + + // Now remove the AM->RM token so tasks don't have it + Iterator> iter = credentials.getAllTokens().iterator(); + while (iter.hasNext()) { + Token token = iter.next(); + if (token.getKind().equals(ApplicationTokenIdentifier.KIND_NAME)) { + iter.remove(); + } + } conf.getCredentials().addAll(credentials); appMasterUgi.doAs(new PrivilegedExceptionAction() { @Override diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java index 4645e861f12..505fbcf720e 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java @@ -594,7 +594,7 @@ public class JobImpl implements org.apache.hadoop.mapreduce.v2.app.job.Job, private float cleanupProgress; private boolean isUber = false; - private Credentials fsTokens; + private Credentials jobCredentials; private Token jobToken; private JobTokenSecretManager jobTokenSecretManager; @@ -604,7 +604,7 @@ public class JobImpl implements org.apache.hadoop.mapreduce.v2.app.job.Job, Configuration conf, EventHandler eventHandler, TaskAttemptListener taskAttemptListener, JobTokenSecretManager jobTokenSecretManager, - Credentials fsTokenCredentials, Clock clock, + Credentials jobCredentials, Clock clock, Map completedTasksFromPreviousRun, MRAppMetrics metrics, OutputCommitter committer, boolean newApiCommitter, String userName, long appSubmitTime, List amInfos, AppContext appContext, @@ -631,7 +631,7 @@ public class JobImpl implements org.apache.hadoop.mapreduce.v2.app.job.Job, this.readLock = readWriteLock.readLock(); this.writeLock = readWriteLock.writeLock(); - this.fsTokens = fsTokenCredentials; + this.jobCredentials = jobCredentials; this.jobTokenSecretManager = jobTokenSecretManager; this.aclsManager = new JobACLsManager(conf); @@ -1414,11 +1414,11 @@ public class JobImpl implements org.apache.hadoop.mapreduce.v2.app.job.Job, // If the job client did not setup the shuffle secret then reuse // the job token secret for the shuffle. - if (TokenCache.getShuffleSecretKey(job.fsTokens) == null) { + if (TokenCache.getShuffleSecretKey(job.jobCredentials) == null) { LOG.warn("Shuffle secret key missing from job credentials." + " Using job token secret as shuffle secret."); TokenCache.setShuffleSecretKey(job.jobToken.getPassword(), - job.fsTokens); + job.jobCredentials); } } @@ -1431,7 +1431,7 @@ public class JobImpl implements org.apache.hadoop.mapreduce.v2.app.job.Job, job.remoteJobConfFile, job.conf, splits[i], job.taskAttemptListener, - job.jobToken, job.fsTokens, + job.jobToken, job.jobCredentials, job.clock, job.applicationAttemptId.getAttemptId(), job.metrics, job.appContext); @@ -1449,7 +1449,7 @@ public class JobImpl implements org.apache.hadoop.mapreduce.v2.app.job.Job, job.remoteJobConfFile, job.conf, job.numMapTasks, job.taskAttemptListener, job.jobToken, - job.fsTokens, job.clock, + job.jobCredentials, job.clock, job.applicationAttemptId.getAttemptId(), job.metrics, job.appContext); job.addTask(task); diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MRApp.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MRApp.java index 6e241377958..07867bec598 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MRApp.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MRApp.java @@ -141,7 +141,7 @@ public class MRApp extends MRAppMaster { } @Override - protected void downloadTokensAndSetupUGI(Configuration conf) { + protected void initJobCredentialsAndUGI(Configuration conf) { // Fake a shuffle secret that normally is provided by the job client. String shuffleSecret = "fake-shuffle-secret"; TokenCache.setShuffleSecretKey(shuffleSecret.getBytes(), getCredentials()); diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestMRAppMaster.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestMRAppMaster.java index 8b4448ab47e..889481fbb91 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestMRAppMaster.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestMRAppMaster.java @@ -58,11 +58,11 @@ import org.apache.hadoop.security.Credentials; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.token.Token; import org.apache.hadoop.security.token.TokenIdentifier; -import org.apache.hadoop.yarn.YarnRuntimeException; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.event.EventHandler; +import org.apache.hadoop.yarn.security.ApplicationTokenIdentifier; import org.apache.hadoop.yarn.util.ConverterUtils; import org.apache.log4j.Level; import org.apache.log4j.LogManager; @@ -344,6 +344,13 @@ public class TestMRAppMaster { new Token(identifier, password, kind, service); Text tokenAlias = new Text("myToken"); credentials.addToken(tokenAlias, myToken); + + Text appTokenService = new Text("localhost:0"); + Token appToken = + new Token(identifier, password, + ApplicationTokenIdentifier.KIND_NAME, appTokenService); + credentials.addToken(appTokenService, appToken); + Text keyAlias = new Text("mySecretKeyAlias"); credentials.addSecretKey(keyAlias, "mySecretKey".getBytes()); Token storedToken = @@ -379,13 +386,13 @@ public class TestMRAppMaster { System.currentTimeMillis(), 1, false, true); MRAppMaster.initAndStartAppMaster(appMaster, conf, userName); - // Now validate the credentials - Credentials appMasterCreds = appMaster.credentials; + // Now validate the task credentials + Credentials appMasterCreds = appMaster.getCredentials(); Assert.assertNotNull(appMasterCreds); Assert.assertEquals(1, appMasterCreds.numberOfSecretKeys()); Assert.assertEquals(1, appMasterCreds.numberOfTokens()); - // Validate the tokens + // Validate the tokens - app token should not be present Token usedToken = appMasterCreds.getToken(tokenAlias); Assert.assertNotNull(usedToken); @@ -397,13 +404,24 @@ public class TestMRAppMaster { Assert.assertEquals("mySecretKey", new String(usedKey)); // The credentials should also be added to conf so that OuputCommitter can - // access it + // access it - app token should not be present Credentials confCredentials = conf.getCredentials(); Assert.assertEquals(1, confCredentials.numberOfSecretKeys()); Assert.assertEquals(1, confCredentials.numberOfTokens()); Assert.assertEquals(storedToken, confCredentials.getToken(tokenAlias)); Assert.assertEquals("mySecretKey", new String(confCredentials.getSecretKey(keyAlias))); + + // Verify the AM's ugi - app token should be present + Credentials ugiCredentials = appMaster.getUgi().getCredentials(); + Assert.assertEquals(1, ugiCredentials.numberOfSecretKeys()); + Assert.assertEquals(2, ugiCredentials.numberOfTokens()); + Assert.assertEquals(storedToken, ugiCredentials.getToken(tokenAlias)); + Assert.assertEquals(appToken, ugiCredentials.getToken(appTokenService)); + Assert.assertEquals("mySecretKey", + new String(ugiCredentials.getSecretKey(keyAlias))); + + } } @@ -416,7 +434,6 @@ class MRAppMasterTest extends MRAppMaster { ContainerAllocator mockContainerAllocator; CommitterEventHandler mockCommitterEventHandler; RMHeartbeatHandler mockRMHeartbeatHandler; - Credentials credentials; public MRAppMasterTest(ApplicationAttemptId applicationAttemptId, ContainerId containerId, String host, int port, int httpPort, @@ -445,15 +462,6 @@ class MRAppMasterTest extends MRAppMaster { this.conf = conf; } - @Override - protected void downloadTokensAndSetupUGI(Configuration conf) { - try { - this.currentUser = UserGroupInformation.getCurrentUser(); - } catch (IOException e) { - throw new YarnRuntimeException(e); - } - } - @Override protected ContainerAllocator createContainerAllocator( final ClientService clientService, final AppContext context) { @@ -477,7 +485,6 @@ class MRAppMasterTest extends MRAppMaster { try { UserGroupInformation ugi = UserGroupInformation.getCurrentUser(); String user = ugi.getShortUserName(); - this.credentials = ugi.getCredentials(); stagingDirPath = MRApps.getStagingAreaDir(conf, user); } catch (Exception e) { fail(e.getMessage()); @@ -487,4 +494,12 @@ class MRAppMasterTest extends MRAppMaster { } } + @Override + public Credentials getCredentials() { + return super.getCredentials(); + } + + public UserGroupInformation getUgi() { + return currentUser; + } } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRecovery.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRecovery.java index df9454f4146..110cd86a33d 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRecovery.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRecovery.java @@ -1595,7 +1595,7 @@ public class TestRecovery { } @Override - protected void downloadTokensAndSetupUGI(Configuration conf) { + protected void initJobCredentialsAndUGI(Configuration conf) { // do NOT put a shuffle secret in the job credentials } } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestStagingCleanup.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestStagingCleanup.java index 5648bd8f5f0..7c6525ffc89 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestStagingCleanup.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestStagingCleanup.java @@ -270,7 +270,7 @@ import org.junit.Test; } @Override - protected void downloadTokensAndSetupUGI(Configuration conf) { + protected void initJobCredentialsAndUGI(Configuration conf) { } public boolean getTestIsLastAMRetry(){ diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java index 6b90796d0cc..0e8586000c9 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java @@ -593,12 +593,6 @@ public interface MRJobConfig { public static final String APPLICATION_MASTER_CLASS = "org.apache.hadoop.mapreduce.v2.app.MRAppMaster"; - // The token file for the application. Should contain tokens for access to - // remote file system and may optionally contain application specific tokens. - // For now, generated by the AppManagers and used by NodeManagers and the - // Containers. - public static final String APPLICATION_TOKENS_FILE = "appTokens"; - public static final String MAPREDUCE_V2_CHILD_CLASS = "org.apache.hadoop.mapred.YarnChild"; diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java index 4686d9f2136..7feca251c43 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java @@ -64,7 +64,6 @@ import org.apache.hadoop.security.SecurityUtil; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.authorize.AccessControlList; import org.apache.hadoop.security.token.Token; -import org.apache.hadoop.yarn.YarnRuntimeException; import org.apache.hadoop.yarn.api.ApplicationConstants; import org.apache.hadoop.yarn.api.ApplicationConstants.Environment; import org.apache.hadoop.yarn.api.records.ApplicationAccessType; @@ -179,7 +178,7 @@ public class YARNRunner implements ClientProtocol { } @VisibleForTesting - void addHistoyToken(Credentials ts) throws IOException, InterruptedException { + void addHistoryToken(Credentials ts) throws IOException, InterruptedException { /* check if we have a hsproxy, if not, no need */ MRClientProtocol hsProxy = clientCache.getInitializedHSProxy(); if (UserGroupInformation.isSecurityEnabled() && (hsProxy != null)) { @@ -279,17 +278,8 @@ public class YARNRunner implements ClientProtocol { public JobStatus submitJob(JobID jobId, String jobSubmitDir, Credentials ts) throws IOException, InterruptedException { - addHistoyToken(ts); + addHistoryToken(ts); - // Upload only in security mode: TODO - Path applicationTokensFile = - new Path(jobSubmitDir, MRJobConfig.APPLICATION_TOKENS_FILE); - try { - ts.writeTokenStorageFile(applicationTokensFile, conf); - } catch (IOException e) { - throw new YarnRuntimeException(e); - } - // Construct necessary information to start the MR AM ApplicationSubmissionContext appContext = createApplicationSubmissionContext(conf, jobSubmitDir, ts); @@ -383,8 +373,7 @@ public class YARNRunner implements ClientProtocol { // TODO gross hack for (String s : new String[] { MRJobConfig.JOB_SPLIT, - MRJobConfig.JOB_SPLIT_METAINFO, - MRJobConfig.APPLICATION_TOKENS_FILE }) { + MRJobConfig.JOB_SPLIT_METAINFO }) { localResources.put( MRJobConfig.JOB_SUBMIT_DIR + "/" + s, createApplicationResource(defaultFileContext, diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestYARNRunner.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestYARNRunner.java index 71e3289af61..e25f3cd24a5 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestYARNRunner.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestYARNRunner.java @@ -306,13 +306,13 @@ public class TestYARNRunner extends TestCase { YARNRunner yarnRunner = new YARNRunner(conf, rmDelegate, clientCache); // No HS token if no RM token - yarnRunner.addHistoyToken(creds); + yarnRunner.addHistoryToken(creds); verify(mockHsProxy, times(0)).getDelegationToken( any(GetDelegationTokenRequest.class)); // No HS token if RM token, but secirity disabled. creds.addToken(new Text("rmdt"), token); - yarnRunner.addHistoyToken(creds); + yarnRunner.addHistoryToken(creds); verify(mockHsProxy, times(0)).getDelegationToken( any(GetDelegationTokenRequest.class)); @@ -322,18 +322,18 @@ public class TestYARNRunner extends TestCase { creds = new Credentials(); // No HS token if no RM token, security enabled - yarnRunner.addHistoyToken(creds); + yarnRunner.addHistoryToken(creds); verify(mockHsProxy, times(0)).getDelegationToken( any(GetDelegationTokenRequest.class)); // HS token if RM token present, security enabled creds.addToken(new Text("rmdt"), token); - yarnRunner.addHistoyToken(creds); + yarnRunner.addHistoryToken(creds); verify(mockHsProxy, times(1)).getDelegationToken( any(GetDelegationTokenRequest.class)); // No additional call to get HS token if RM and HS token present - yarnRunner.addHistoyToken(creds); + yarnRunner.addHistoryToken(creds); verify(mockHsProxy, times(1)).getDelegationToken( any(GetDelegationTokenRequest.class)); } finally { @@ -407,10 +407,6 @@ public class TestYARNRunner extends TestCase { out = new FileOutputStream(jobsplitmetainfo); out.close(); - File appTokens = new File(testWorkDir, MRJobConfig.APPLICATION_TOKENS_FILE); - out = new FileOutputStream(appTokens); - out.close(); - ApplicationSubmissionContext submissionContext = yarnRunner.createApplicationSubmissionContext(jobConf, testWorkDir.toString(), new Credentials()); @@ -477,10 +473,6 @@ public class TestYARNRunner extends TestCase { out = new FileOutputStream(jobsplitmetainfo); out.close(); - File appTokens = new File(testWorkDir, MRJobConfig.APPLICATION_TOKENS_FILE); - out = new FileOutputStream(appTokens); - out.close(); - @SuppressWarnings("unused") ApplicationSubmissionContext submissionContext = yarnRunner.createApplicationSubmissionContext(jobConf, testWorkDir.toString(), new Credentials());