diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index 74cbb3b800f..868eca2cc36 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -325,6 +325,9 @@ Release 0.23.3 - UNRELEASED OPTIMIZATIONS + MAPREDUCE-3850. Avoid redundant calls for tokens in TokenCache (Daryn + Sharp via bobby) + BUG FIXES MAPREDUCE-4092. commitJob Exception does not fail job (Jon Eagles via diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/security/TokenCache.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/security/TokenCache.java index ef25939ebdb..9e8c1909a13 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/security/TokenCache.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/security/TokenCache.java @@ -19,7 +19,9 @@ package org.apache.hadoop.mapreduce.security; import java.io.IOException; +import java.util.HashSet; import java.util.List; +import java.util.Set; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -92,8 +94,11 @@ public class TokenCache { static void obtainTokensForNamenodesInternal(Credentials credentials, Path[] ps, Configuration conf) throws IOException { + Set fsSet = new HashSet(); for(Path p: ps) { - FileSystem fs = FileSystem.get(p.toUri(), conf); + fsSet.add(p.getFileSystem(conf)); + } + for (FileSystem fs : fsSet) { obtainTokensForNamenodesInternal(fs, credentials, conf); } } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/security/TestTokenCache.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/security/TestTokenCache.java index b0e9350dbed..1ae2ecde1e2 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/security/TestTokenCache.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/security/TestTokenCache.java @@ -251,6 +251,26 @@ public class TestTokenCache { return mockFs; } + @Test + public void testSingleTokenFetch() throws Exception { + Configuration conf = new Configuration(); + conf.set(YarnConfiguration.RM_PRINCIPAL, "mapred/host@REALM"); + String renewer = Master.getMasterPrincipal(conf); + Credentials credentials = new Credentials(); + + FileSystem mockFs = mock(FileSystem.class); + when(mockFs.getCanonicalServiceName()).thenReturn("host:0"); + when(mockFs.getUri()).thenReturn(new URI("mockfs://host:0")); + + Path mockPath = mock(Path.class); + when(mockPath.getFileSystem(conf)).thenReturn(mockFs); + + Path[] paths = new Path[]{ mockPath, mockPath }; + when(mockFs.getDelegationTokens("me", credentials)).thenReturn(null); + TokenCache.obtainTokensForNamenodesInternal(credentials, paths, conf); + verify(mockFs, times(1)).getDelegationTokens(renewer, credentials); + } + @Test public void testCleanUpTokenReferral() throws Exception { Configuration conf = new Configuration();