diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java index 0baca07a24e..ca17f8d64c7 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java @@ -316,6 +316,9 @@ public class CommonConfigurationKeysPublic { /** See core-default.xml */ public static final String HADOOP_SECURITY_DNS_NAMESERVER_KEY = "hadoop.security.dns.nameserver"; + /** See core-default.xml */ + public static final String HADOOP_TOKEN_FILES = + "hadoop.token.files"; @Deprecated /** Only used by HttpServer. */ diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java index 90d396f3f7c..2ea80dd8902 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java @@ -18,6 +18,7 @@ package org.apache.hadoop.security; import static org.apache.hadoop.fs.CommonConfigurationKeys.HADOOP_USER_GROUP_METRICS_PERCENTILES_INTERVALS; +import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_TOKEN_FILES; import static org.apache.hadoop.util.PlatformName.IBM_JAVA; import java.io.File; @@ -66,6 +67,7 @@ import org.apache.hadoop.security.authentication.util.KerberosUtil; import org.apache.hadoop.security.token.Token; import org.apache.hadoop.security.token.TokenIdentifier; import org.apache.hadoop.util.Shell; +import org.apache.hadoop.util.StringUtils; import org.apache.hadoop.util.Time; import com.google.common.annotations.VisibleForTesting; @@ -811,6 +813,26 @@ public class UserGroupInformation { } loginUser = proxyUser == null ? realUser : createProxyUser(proxyUser, realUser); + String tokenFileLocation = System.getProperty(HADOOP_TOKEN_FILES); + if (tokenFileLocation == null) { + tokenFileLocation = conf.get(HADOOP_TOKEN_FILES); + } + if (tokenFileLocation != null) { + for (String tokenFileName: + StringUtils.getTrimmedStrings(tokenFileLocation)) { + if (tokenFileName.length() > 0) { + File tokenFile = new File(tokenFileName); + if (tokenFile.exists() && tokenFile.isFile()) { + Credentials cred = Credentials.readTokenStorageFile( + tokenFile, conf); + loginUser.addCredentials(cred); + } else { + LOG.info("tokenFile("+tokenFileName+") does not exist"); + } + } + } + } + String fileLocation = System.getenv(HADOOP_TOKEN_FILE_LOCATION); if (fileLocation != null) { // Load the token storage file and put all of the tokens into the diff --git a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml index abdb0cf9a77..a3be860a727 100644 --- a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml +++ b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml @@ -415,6 +415,12 @@ Maps kerberos principals to local user names + + hadoop.token.files + + List of token cache files that have delegation tokens for hadoop service + + io.file.buffer.size diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestUserGroupInformation.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestUserGroupInformation.java index 5cfa29a34ee..29cb6df86b3 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestUserGroupInformation.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestUserGroupInformation.java @@ -18,6 +18,7 @@ package org.apache.hadoop.security; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.CommonConfigurationKeysPublic; +import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.Text; import org.apache.hadoop.metrics2.MetricsRecordBuilder; import org.apache.hadoop.security.SaslRpcServer.AuthMethod; @@ -35,6 +36,7 @@ import javax.security.auth.login.AppConfigurationEntry; import javax.security.auth.login.LoginContext; import java.io.BufferedReader; +import java.io.File; import java.io.IOException; import java.io.InputStreamReader; import java.security.PrivilegedExceptionAction; @@ -812,7 +814,9 @@ public class TestUserGroupInformation { */ @Test public void testPrivateTokenExclusion() throws Exception { - UserGroupInformation ugi = UserGroupInformation.getCurrentUser(); + UserGroupInformation ugi = + UserGroupInformation.createUserForTesting( + "privateUser", new String[] { "PRIVATEUSERS" }); TestTokenIdentifier tokenId = new TestTokenIdentifier(); Token token = new Token( tokenId.getBytes(), "password".getBytes(), @@ -892,4 +896,46 @@ public class TestUserGroupInformation { } } } + + @Test + public void testExternalTokenFiles() throws Exception { + StringBuilder tokenFullPathnames = new StringBuilder(); + String tokenFilenames = "token1,token2"; + String tokenFiles[] = StringUtils.getTrimmedStrings(tokenFilenames); + final File testDir = new File("target", + TestUserGroupInformation.class.getName() + "-tmpDir").getAbsoluteFile(); + String testDirPath = testDir.getAbsolutePath(); + + // create path for token files + for (String tokenFile: tokenFiles) { + if (tokenFullPathnames.length() > 0) { + tokenFullPathnames.append(","); + } + tokenFullPathnames.append(testDirPath).append("/").append(tokenFile); + } + + // create new token and store it + TestTokenIdentifier tokenId = new TestTokenIdentifier(); + Credentials cred1 = new Credentials(); + Token token1 = new Token( + tokenId.getBytes(), "password".getBytes(), + tokenId.getKind(), new Text("token-service1")); + cred1.addToken(token1.getService(), token1); + cred1.writeTokenStorageFile(new Path(testDirPath, tokenFiles[0]), conf); + + Credentials cred2 = new Credentials(); + Token token2 = new Token( + tokenId.getBytes(), "password".getBytes(), + tokenId.getKind(), new Text("token-service2")); + cred2.addToken(token2.getService(), token2); + cred2.writeTokenStorageFile(new Path(testDirPath, tokenFiles[1]), conf); + + // set property for token external token files + System.setProperty("hadoop.token.files", tokenFullPathnames.toString()); + UserGroupInformation.setLoginUser(null); + UserGroupInformation tokenUgi = UserGroupInformation.getLoginUser(); + Collection> credsugiTokens = tokenUgi.getTokens(); + assertTrue(credsugiTokens.contains(token1)); + assertTrue(credsugiTokens.contains(token2)); + } } diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java index 03b372e4a18..bf1602b05e3 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java @@ -302,7 +302,7 @@ public class WebHdfsFileSystem extends FileSystem // the first getAuthParams() for a non-token op will either get the // internal token from the ugi or lazy fetch one protected synchronized Token getDelegationToken() throws IOException { - if (canRefreshDelegationToken && delegationToken == null) { + if (delegationToken == null) { Token token = tokenSelector.selectToken( new Text(getCanonicalServiceName()), ugi.getTokens()); // ugi tokens are usually indicative of a task which can't @@ -312,11 +312,13 @@ public class WebHdfsFileSystem extends FileSystem LOG.debug("Using UGI token: {}", token); canRefreshDelegationToken = false; } else { - token = getDelegationToken(null); - if (token != null) { - LOG.debug("Fetched new token: {}", token); - } else { // security is disabled - canRefreshDelegationToken = false; + if (canRefreshDelegationToken) { + token = getDelegationToken(null); + if (token != null) { + LOG.debug("Fetched new token: " + token); + } else { // security is disabled + canRefreshDelegationToken = false; + } } } setDelegationToken(token); diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/resources/DelegationParam.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/resources/DelegationParam.java index 5329580289e..fda14438045 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/resources/DelegationParam.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/resources/DelegationParam.java @@ -17,8 +17,6 @@ */ package org.apache.hadoop.hdfs.web.resources; -import org.apache.hadoop.security.UserGroupInformation; - /** Represents delegation token used for authentication. */ public class DelegationParam extends StringParam { /** Parameter name. */ @@ -33,8 +31,7 @@ public class DelegationParam extends StringParam { * @param str a string representation of the parameter value. */ public DelegationParam(final String str) { - super(DOMAIN, UserGroupInformation.isSecurityEnabled() - && str != null && !str.equals(DEFAULT)? str: null); + super(DOMAIN, str != null && !str.equals(DEFAULT)? str: null); } @Override diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHdfsUrl.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHdfsUrl.java index 2913a97d309..24c13af61ea 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHdfsUrl.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHdfsUrl.java @@ -195,7 +195,7 @@ public class TestWebHdfsUrl { checkQueryParams( new String[]{ GetOpParam.Op.GETFILESTATUS.toQueryString(), - new UserParam(ugi.getShortUserName()).toString() + new DelegationParam(tokenString).toString() }, fileStatusUrl); } @@ -280,8 +280,7 @@ public class TestWebHdfsUrl { checkQueryParams( new String[]{ GetOpParam.Op.GETFILESTATUS.toQueryString(), - new UserParam(ugi.getRealUser().getShortUserName()).toString(), - new DoAsParam(ugi.getShortUserName()).toString() + new DelegationParam(tokenString).toString() }, fileStatusUrl); }