diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 07949054db3..1f6f5da93fd 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -264,6 +264,9 @@ Release 2.8.0 - UNRELEASED YARN-3148. Allow CORS related headers to passthrough in WebAppProxyServlet. (Varun Saxena via devaraj) + YARN-3834. Scrub debug logging of tokens during resource localization. + (Chris Nauroth via xgong) + OPTIMIZATIONS YARN-3339. TestDockerContainerExecutor should pull a single image and not diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java index 54c31c2a88d..d6e0903a248 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java @@ -51,6 +51,7 @@ import java.util.concurrent.ScheduledThreadPoolExecutor; import java.util.concurrent.ThreadFactory; import java.util.concurrent.TimeUnit; +import org.apache.commons.codec.digest.DigestUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.classification.InterfaceAudience.Private; @@ -1208,7 +1209,7 @@ public class ResourceLocalizationService extends CompositeService if (LOG.isDebugEnabled()) { for (Token tk : credentials .getAllTokens()) { - LOG.debug(tk.getService() + " : " + tk.encodeToUrlString()); + LOG.debug(tk + " : " + buildTokenFingerprint(tk)); } } if (UserGroupInformation.isSecurityEnabled()) { @@ -1228,6 +1229,32 @@ public class ResourceLocalizationService extends CompositeService } + /** + * Returns a fingerprint of a token. The fingerprint is suitable for use in + * logging, because it cannot be used to determine the secret. The + * fingerprint is built using the first 10 bytes of a SHA-256 hash of the + * string encoding of the token. The returned string contains the hex + * representation of each byte, delimited by a space. + * + * @param tk token + * @return token fingerprint + * @throws IOException if there is an I/O error + */ + @VisibleForTesting + static String buildTokenFingerprint(Token tk) + throws IOException { + char[] digest = DigestUtils.sha256Hex(tk.encodeToUrlString()).toCharArray(); + StringBuilder fingerprint = new StringBuilder(); + for (int i = 0; i < 10; ++i) { + if (i > 0) { + fingerprint.append(' '); + } + fingerprint.append(digest[2 * i]); + fingerprint.append(digest[2 * i + 1]); + } + return fingerprint.toString(); + } + static class CacheCleanup extends Thread { private final Dispatcher dispatcher; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestResourceLocalizationService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestResourceLocalizationService.java index a02b2b09bc5..c515506ed95 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestResourceLocalizationService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestResourceLocalizationService.java @@ -2035,7 +2035,7 @@ public class TestResourceLocalizationService { } private static Container getMockContainer(ApplicationId appId, int id, - String user) { + String user) throws IOException { Container c = mock(Container.class); ApplicationAttemptId appAttemptId = BuilderUtils.newApplicationAttemptId(appId, 1); @@ -2043,7 +2043,13 @@ public class TestResourceLocalizationService { when(c.getUser()).thenReturn(user); when(c.getContainerId()).thenReturn(cId); Credentials creds = new Credentials(); - creds.addToken(new Text("tok" + id), getToken(id)); + Token tk = getToken(id); + String fingerprint = ResourceLocalizationService.buildTokenFingerprint(tk); + assertNotNull(fingerprint); + assertTrue( + "Expected token fingerprint of 10 hex bytes delimited by space.", + fingerprint.matches("^(([0-9a-f]){2} ){9}([0-9a-f]){2}$")); + creds.addToken(new Text("tok" + id), tk); when(c.getCredentials()).thenReturn(creds); when(c.toString()).thenReturn(cId.toString()); return c;