diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index f0e4bdd0ede..8c29b120d7b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -885,6 +885,9 @@ Release 0.23.3 - UNRELEASED HDFS-2652. Add support for host-based delegation tokens. (Daryn Sharp via szetszwo) + HDFS-3308. Uses canonical URI to select delegation tokens in HftpFileSystem + and WebHdfsFileSystem. (Daryn Sharp via szetszwo) + Release 0.23.2 - UNRELEASED INCOMPATIBLE CHANGES diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/HftpFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/HftpFileSystem.java index 7151e9f9472..cadde41514c 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/HftpFileSystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/HftpFileSystem.java @@ -168,7 +168,7 @@ public void initialize(final URI name, final Configuration conf) protected void initDelegationToken() throws IOException { // look for hftp token, then try hdfs - Token token = selectDelegationToken(); + Token token = selectDelegationToken(ugi); // if we don't already have a token, go get one over https boolean createdToken = false; @@ -189,8 +189,9 @@ protected void initDelegationToken() throws IOException { } } - protected Token selectDelegationToken() { - return hftpTokenSelector.selectToken(getUri(), ugi.getTokens(), getConf()); + protected Token selectDelegationToken( + UserGroupInformation ugi) { + return hftpTokenSelector.selectToken(getCanonicalUri(), ugi.getTokens(), getConf()); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java index 415bf6c12dd..4e44e03e050 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java @@ -165,7 +165,7 @@ public synchronized void initialize(URI uri, Configuration conf } catch (URISyntaxException e) { throw new IllegalArgumentException(e); } - this.nnAddr = NetUtils.createSocketAddrForHost(uri.getHost(), uri.getPort()); + this.nnAddr = NetUtils.createSocketAddr(uri.getAuthority(), getDefaultPort()); this.workingDir = getHomeDirectory(); if (UserGroupInformation.isSecurityEnabled()) { @@ -175,7 +175,7 @@ public synchronized void initialize(URI uri, Configuration conf protected void initDelegationToken() throws IOException { // look for webhdfs token, then try hdfs - Token token = selectDelegationToken(); + Token token = selectDelegationToken(ugi); //since we don't already have a token, go get one boolean createdToken = false; @@ -196,8 +196,9 @@ protected void initDelegationToken() throws IOException { } } - protected Token selectDelegationToken() { - return DT_SELECTOR.selectToken(getUri(), ugi.getTokens(), getConf()); + protected Token selectDelegationToken( + UserGroupInformation ugi) { + return DT_SELECTOR.selectToken(getCanonicalUri(), ugi.getTokens(), getConf()); } @Override diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestHftpDelegationToken.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestHftpDelegationToken.java index e4071222410..d292aede5cd 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestHftpDelegationToken.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestHftpDelegationToken.java @@ -21,6 +21,7 @@ import static org.apache.hadoop.fs.CommonConfigurationKeys.HADOOP_SECURITY_AUTHENTICATION; +import java.io.IOException; import java.lang.reflect.Field; import java.net.URI; import java.security.PrivilegedExceptionAction; @@ -73,34 +74,58 @@ public void testSelectHdfsDelegationToken() throws Exception { SecurityUtilTestHelper.setTokenServiceUseIp(true); Configuration conf = new Configuration(); - URI hftpUri = URI.create("hftp://localhost:0"); - UserGroupInformation ugi = UserGroupInformation.getCurrentUser(); - Token token = null; + conf.setClass("fs.hftp.impl", MyHftpFileSystem.class, FileSystem.class); + // test with implicit default port + URI fsUri = URI.create("hftp://localhost"); + MyHftpFileSystem fs = (MyHftpFileSystem) FileSystem.get(fsUri, conf); + checkTokenSelection(fs, conf); + + // test with explicit default port + fsUri = URI.create("hftp://localhost:"+fs.getDefaultPort()); + fs = (MyHftpFileSystem) FileSystem.get(fsUri, conf); + checkTokenSelection(fs, conf); + + // test with non-default port + fsUri = URI.create("hftp://localhost:"+(fs.getDefaultPort()-1)); + fs = (MyHftpFileSystem) FileSystem.get(fsUri, conf); + checkTokenSelection(fs, conf); + + } + + private void checkTokenSelection(MyHftpFileSystem fs, + Configuration conf) throws IOException { + int port = fs.getCanonicalUri().getPort(); + UserGroupInformation ugi = + UserGroupInformation.createUserForTesting(fs.getUri().getAuthority(), new String[]{}); + + // use ip-based tokens + SecurityUtilTestHelper.setTokenServiceUseIp(true); + // test fallback to hdfs token Token hdfsToken = new Token( new byte[0], new byte[0], DelegationTokenIdentifier.HDFS_DELEGATION_KIND, new Text("127.0.0.1:8020")); ugi.addToken(hdfsToken); - - HftpFileSystem fs = (HftpFileSystem) FileSystem.get(hftpUri, conf); - token = fs.selectDelegationToken(); + + // test fallback to hdfs token + Token token = fs.selectDelegationToken(ugi); assertNotNull(token); assertEquals(hdfsToken, token); - + // test hftp is favored over hdfs Token hftpToken = new Token( new byte[0], new byte[0], - HftpFileSystem.TOKEN_KIND, new Text("127.0.0.1:0")); + HftpFileSystem.TOKEN_KIND, new Text("127.0.0.1:"+port)); ugi.addToken(hftpToken); - token = fs.selectDelegationToken(); + token = fs.selectDelegationToken(ugi); assertNotNull(token); assertEquals(hftpToken, token); // switch to using host-based tokens, no token should match SecurityUtilTestHelper.setTokenServiceUseIp(false); - token = fs.selectDelegationToken(); + token = fs.selectDelegationToken(ugi); assertNull(token); // test fallback to hdfs token @@ -109,17 +134,31 @@ public void testSelectHdfsDelegationToken() throws Exception { DelegationTokenIdentifier.HDFS_DELEGATION_KIND, new Text("localhost:8020")); ugi.addToken(hdfsToken); - token = fs.selectDelegationToken(); + token = fs.selectDelegationToken(ugi); assertNotNull(token); assertEquals(hdfsToken, token); // test hftp is favored over hdfs hftpToken = new Token( new byte[0], new byte[0], - HftpFileSystem.TOKEN_KIND, new Text("localhost:0")); + HftpFileSystem.TOKEN_KIND, new Text("localhost:"+port)); ugi.addToken(hftpToken); - token = fs.selectDelegationToken(); + token = fs.selectDelegationToken(ugi); assertNotNull(token); assertEquals(hftpToken, token); } + + static class MyHftpFileSystem extends HftpFileSystem { + @Override + public URI getCanonicalUri() { + return super.getCanonicalUri(); + } + @Override + public int getDefaultPort() { + return super.getDefaultPort(); + } + // don't automatically get a token + @Override + protected void initDelegationToken() throws IOException {} + } } \ No newline at end of file diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHdfsUrl.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHdfsUrl.java index 1dde0997dd1..015ce4dde2b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHdfsUrl.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHdfsUrl.java @@ -98,38 +98,63 @@ private String generateUrlQueryPrefix(HttpOpParam.Op op, String username) { } @Test - public void testSelectDelegationToken() throws Exception { + public void testSelectHdfsDelegationToken() throws Exception { SecurityUtilTestHelper.setTokenServiceUseIp(true); Configuration conf = new Configuration(); - URI webHdfsUri = URI.create("webhdfs://localhost:0"); - UserGroupInformation ugi = UserGroupInformation.getCurrentUser(); - Token token = null; + conf.setClass("fs.webhdfs.impl", MyWebHdfsFileSystem.class, FileSystem.class); + // test with implicit default port + URI fsUri = URI.create("webhdfs://localhost"); + MyWebHdfsFileSystem fs = (MyWebHdfsFileSystem) FileSystem.get(fsUri, conf); + checkTokenSelection(fs, conf); + + // test with explicit default port + fsUri = URI.create("webhdfs://localhost:"+fs.getDefaultPort()); + fs = (MyWebHdfsFileSystem) FileSystem.get(fsUri, conf); + checkTokenSelection(fs, conf); + + // test with non-default port + fsUri = URI.create("webhdfs://localhost:"+(fs.getDefaultPort()-1)); + fs = (MyWebHdfsFileSystem) FileSystem.get(fsUri, conf); + checkTokenSelection(fs, conf); + + } + + private void checkTokenSelection(MyWebHdfsFileSystem fs, + Configuration conf) throws IOException { + int port = fs.getCanonicalUri().getPort(); + // can't clear tokens from ugi, so create a new user everytime + UserGroupInformation ugi = + UserGroupInformation.createUserForTesting(fs.getUri().getAuthority(), new String[]{}); + + // use ip-based tokens + SecurityUtilTestHelper.setTokenServiceUseIp(true); + // test fallback to hdfs token Token hdfsToken = new Token( new byte[0], new byte[0], DelegationTokenIdentifier.HDFS_DELEGATION_KIND, new Text("127.0.0.1:8020")); ugi.addToken(hdfsToken); - - WebHdfsFileSystem fs = (WebHdfsFileSystem) FileSystem.get(webHdfsUri, conf); - token = fs.selectDelegationToken(); + + // test fallback to hdfs token + Token token = fs.selectDelegationToken(ugi); assertNotNull(token); assertEquals(hdfsToken, token); - + // test webhdfs is favored over hdfs Token webHdfsToken = new Token( new byte[0], new byte[0], - WebHdfsFileSystem.TOKEN_KIND, new Text("127.0.0.1:0")); + WebHdfsFileSystem.TOKEN_KIND, new Text("127.0.0.1:"+port)); ugi.addToken(webHdfsToken); - token = fs.selectDelegationToken(); + token = fs.selectDelegationToken(ugi); assertNotNull(token); assertEquals(webHdfsToken, token); // switch to using host-based tokens, no token should match SecurityUtilTestHelper.setTokenServiceUseIp(false); - token = fs.selectDelegationToken(); + token = fs.selectDelegationToken(ugi); assertNull(token); // test fallback to hdfs token @@ -138,18 +163,31 @@ public void testSelectDelegationToken() throws Exception { DelegationTokenIdentifier.HDFS_DELEGATION_KIND, new Text("localhost:8020")); ugi.addToken(hdfsToken); - token = fs.selectDelegationToken(); + token = fs.selectDelegationToken(ugi); assertNotNull(token); assertEquals(hdfsToken, token); // test webhdfs is favored over hdfs webHdfsToken = new Token( new byte[0], new byte[0], - WebHdfsFileSystem.TOKEN_KIND, new Text("localhost:0")); + WebHdfsFileSystem.TOKEN_KIND, new Text("localhost:"+port)); ugi.addToken(webHdfsToken); - token = fs.selectDelegationToken(); + token = fs.selectDelegationToken(ugi); assertNotNull(token); assertEquals(webHdfsToken, token); } - -} + + static class MyWebHdfsFileSystem extends WebHdfsFileSystem { + @Override + public URI getCanonicalUri() { + return super.getCanonicalUri(); + } + @Override + public int getDefaultPort() { + return super.getDefaultPort(); + } + // don't automatically get a token + @Override + protected void initDelegationToken() throws IOException {} + } +} \ No newline at end of file