Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

MAPREDUCE-3850. Avoid redundant calls for tokens in TokenCache (Daryn Sharp via bobby) 

git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1335585 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Robert Joseph Evans 2012-05-08 15:25:08 +00:00
parent aa60da6c2e
commit 67a514b647
3 changed files with 29 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
hadoop-mapreduce-project/CHANGES.txt
View File

@ -325,6 +325,9 @@ Release 0.23.3 - UNRELEASED
OPTIMIZATIONS OPTIMIZATIONS
MAPREDUCE-3850. Avoid redundant calls for tokens in TokenCache (Daryn
Sharp via bobby)
BUG FIXES BUG FIXES
MAPREDUCE-4092. commitJob Exception does not fail job (Jon Eagles via MAPREDUCE-4092. commitJob Exception does not fail job (Jon Eagles via

7
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/security/TokenCache.java
View File

@ -19,7 +19,9 @@
package org.apache.hadoop.mapreduce.security; package org.apache.hadoop.mapreduce.security;
import java.io.IOException; import java.io.IOException;
import java.util.HashSet;
import java.util.List; import java.util.List;
import java.util.Set;
import org.apache.commons.logging.Log; import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory; import org.apache.commons.logging.LogFactory;
@ -92,8 +94,11 @@ public class TokenCache {
static void obtainTokensForNamenodesInternal(Credentials credentials, static void obtainTokensForNamenodesInternal(Credentials credentials,
Path[] ps, Configuration conf) throws IOException { Path[] ps, Configuration conf) throws IOException {
Set<FileSystem> fsSet = new HashSet<FileSystem>();
for(Path p: ps) { for(Path p: ps) {
FileSystem fs = FileSystem.get(p.toUri(), conf); fsSet.add(p.getFileSystem(conf));
}
for (FileSystem fs : fsSet) {
obtainTokensForNamenodesInternal(fs, credentials, conf); obtainTokensForNamenodesInternal(fs, credentials, conf);
} }
} }

20
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/security/TestTokenCache.java
View File

@ -251,6 +251,26 @@ public class TestTokenCache {
return mockFs; return mockFs;
} }
@Test
public void testSingleTokenFetch() throws Exception {
Configuration conf = new Configuration();
conf.set(YarnConfiguration.RM_PRINCIPAL, "mapred/host@REALM");
String renewer = Master.getMasterPrincipal(conf);
Credentials credentials = new Credentials();
FileSystem mockFs = mock(FileSystem.class);
when(mockFs.getCanonicalServiceName()).thenReturn("host:0");
when(mockFs.getUri()).thenReturn(new URI("mockfs://host:0"));
Path mockPath = mock(Path.class);
when(mockPath.getFileSystem(conf)).thenReturn(mockFs);
Path[] paths = new Path[]{ mockPath, mockPath };
when(mockFs.getDelegationTokens("me", credentials)).thenReturn(null);
TokenCache.obtainTokensForNamenodesInternal(credentials, paths, conf);
verify(mockFs, times(1)).getDelegationTokens(renewer, credentials);
}
@Test @Test
public void testCleanUpTokenReferral() throws Exception { public void testCleanUpTokenReferral() throws Exception {
Configuration conf = new Configuration(); Configuration conf = new Configuration();