Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

YARN-3834. Scrub debug logging of tokens during resource localization. Contributed by Chris Nauroth 

(cherry picked from commit 6c7a9d502a)
This commit is contained in:
Xuan 2015-06-21 17:13:44 -07:00
parent 175c5829f0
commit c3254a63a9
3 changed files with 39 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
hadoop-yarn-project/CHANGES.txt
View File

@ -264,6 +264,9 @@ Release 2.8.0 - UNRELEASED
YARN-3148. Allow CORS related headers to passthrough in WebAppProxyServlet.
(Varun Saxena via devaraj)
YARN-3834. Scrub debug logging of tokens during resource localization.
(Chris Nauroth via xgong)
OPTIMIZATIONS
YARN-3339. TestDockerContainerExecutor should pull a single image and not

29
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java
View File

@ -51,6 +51,7 @@ import java.util.concurrent.ScheduledThreadPoolExecutor;
import java.util.concurrent.ThreadFactory;
import java.util.concurrent.TimeUnit;
import org.apache.commons.codec.digest.DigestUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience.Private;
@ -1208,7 +1209,7 @@ public class ResourceLocalizationService extends CompositeService
if (LOG.isDebugEnabled()) {
for (Token<? extends TokenIdentifier> tk : credentials
.getAllTokens()) {
LOG.debug(tk.getService() + " : " + tk.encodeToUrlString());
LOG.debug(tk + " : " + buildTokenFingerprint(tk));
}
}
if (UserGroupInformation.isSecurityEnabled()) {
@ -1228,6 +1229,32 @@ public class ResourceLocalizationService extends CompositeService
}
/**
* Returns a fingerprint of a token. The fingerprint is suitable for use in
* logging, because it cannot be used to determine the secret. The
* fingerprint is built using the first 10 bytes of a SHA-256 hash of the
* string encoding of the token. The returned string contains the hex
* representation of each byte, delimited by a space.
*
* @param tk token
* @return token fingerprint
* @throws IOException if there is an I/O error
*/
@VisibleForTesting
static String buildTokenFingerprint(Token<? extends TokenIdentifier> tk)
throws IOException {
char[] digest = DigestUtils.sha256Hex(tk.encodeToUrlString()).toCharArray();
StringBuilder fingerprint = new StringBuilder();
for (int i = 0; i < 10; ++i) {
if (i > 0) {
fingerprint.append(' ');
}
fingerprint.append(digest[2 * i]);
fingerprint.append(digest[2 * i + 1]);
}
return fingerprint.toString();
}
static class CacheCleanup extends Thread {
private final Dispatcher dispatcher;

10
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestResourceLocalizationService.java
View File

@ -2035,7 +2035,7 @@ public class TestResourceLocalizationService {
}
private static Container getMockContainer(ApplicationId appId, int id,
String user) {
String user) throws IOException {
Container c = mock(Container.class);
ApplicationAttemptId appAttemptId =
BuilderUtils.newApplicationAttemptId(appId, 1);
@ -2043,7 +2043,13 @@ public class TestResourceLocalizationService {
when(c.getUser()).thenReturn(user);
when(c.getContainerId()).thenReturn(cId);
Credentials creds = new Credentials();
creds.addToken(new Text("tok" + id), getToken(id));
Token<? extends TokenIdentifier> tk = getToken(id);
String fingerprint = ResourceLocalizationService.buildTokenFingerprint(tk);
assertNotNull(fingerprint);
assertTrue(
"Expected token fingerprint of 10 hex bytes delimited by space.",
fingerprint.matches("^(([0-9a-f]){2} ){9}([0-9a-f]){2}$"));
creds.addToken(new Text("tok" + id), tk);
when(c.getCredentials()).thenReturn(creds);
when(c.toString()).thenReturn(cId.toString());
return c;