svn merge -c 1328541 from trunk for HDFS-3308. Uses canonical URI to select delegation tokens in HftpFileSystem and WebHdfsFileSystem.

git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/branch-2@1328542 13f79535-47bb-0310-9956-ffa450edef68

hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt

@@ -764,6 +764,9 @@ Release 0.23.3 - UNRELEASED
     HDFS-2652. Add support for host-based delegation tokens.  (Daryn Sharp via
     szetszwo)
 
+    HDFS-3308. Uses canonical URI to select delegation tokens in HftpFileSystem
+    and WebHdfsFileSystem.  (Daryn Sharp via szetszwo)
+
 Release 0.23.2 - UNRELEASED
 
   INCOMPATIBLE CHANGES

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/HftpFileSystem.java

@@ -168,7 +168,7 @@ public class HftpFileSystem extends FileSystem
 
   protected void initDelegationToken() throws IOException {
     // look for hftp token, then try hdfs
-    Token<?> token = selectDelegationToken();
+    Token<?> token = selectDelegationToken(ugi);
 
     // if we don't already have a token, go get one over https
     boolean createdToken = false;
@@ -189,8 +189,9 @@ public class HftpFileSystem extends FileSystem
     }
   }
 
-  protected Token<DelegationTokenIdentifier> selectDelegationToken() {
+  protected Token<DelegationTokenIdentifier> selectDelegationToken(
-  	return hftpTokenSelector.selectToken(getUri(), ugi.getTokens(), getConf());
+      UserGroupInformation ugi) {
+  	return hftpTokenSelector.selectToken(getCanonicalUri(), ugi.getTokens(), getConf());
   }
   
 

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java

@@ -165,7 +165,7 @@ public class WebHdfsFileSystem extends FileSystem
     } catch (URISyntaxException e) {
       throw new IllegalArgumentException(e);
     }
-    this.nnAddr = NetUtils.createSocketAddrForHost(uri.getHost(), uri.getPort());
+    this.nnAddr = NetUtils.createSocketAddr(uri.getAuthority(), getDefaultPort());
     this.workingDir = getHomeDirectory();
 
     if (UserGroupInformation.isSecurityEnabled()) {
@@ -175,7 +175,7 @@ public class WebHdfsFileSystem extends FileSystem
 
   protected void initDelegationToken() throws IOException {
     // look for webhdfs token, then try hdfs
-    Token<?> token = selectDelegationToken();
+    Token<?> token = selectDelegationToken(ugi);
 
     //since we don't already have a token, go get one
     boolean createdToken = false;
@@ -196,8 +196,9 @@ public class WebHdfsFileSystem extends FileSystem
     }
   }
 
-  protected Token<DelegationTokenIdentifier> selectDelegationToken() {
+  protected Token<DelegationTokenIdentifier> selectDelegationToken(
-    return DT_SELECTOR.selectToken(getUri(), ugi.getTokens(), getConf());
+      UserGroupInformation ugi) {
+    return DT_SELECTOR.selectToken(getCanonicalUri(), ugi.getTokens(), getConf());
   }
 
   @Override

hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestHftpDelegationToken.java

@@ -21,6 +21,7 @@ package org.apache.hadoop.hdfs;
 import static 
   org.apache.hadoop.fs.CommonConfigurationKeys.HADOOP_SECURITY_AUTHENTICATION;
 
+import java.io.IOException;
 import java.lang.reflect.Field;
 import java.net.URI;
 import java.security.PrivilegedExceptionAction;
@@ -73,34 +74,58 @@ public class TestHftpDelegationToken {
     SecurityUtilTestHelper.setTokenServiceUseIp(true);
 
     Configuration conf = new Configuration();
-    URI hftpUri = URI.create("hftp://localhost:0");
+    conf.setClass("fs.hftp.impl", MyHftpFileSystem.class, FileSystem.class);
-    UserGroupInformation ugi = UserGroupInformation.getCurrentUser();
-    Token<?> token = null;
     
+    // test with implicit default port 
+    URI fsUri = URI.create("hftp://localhost");
+    MyHftpFileSystem fs = (MyHftpFileSystem) FileSystem.get(fsUri, conf);
+    checkTokenSelection(fs, conf);
+
+    // test with explicit default port
+    fsUri = URI.create("hftp://localhost:"+fs.getDefaultPort());
+    fs = (MyHftpFileSystem) FileSystem.get(fsUri, conf);
+    checkTokenSelection(fs, conf);
+    
+    // test with non-default port
+    fsUri = URI.create("hftp://localhost:"+(fs.getDefaultPort()-1));
+    fs = (MyHftpFileSystem) FileSystem.get(fsUri, conf);
+    checkTokenSelection(fs, conf);
+
+  }
+  
+  private void checkTokenSelection(MyHftpFileSystem fs,
+                                   Configuration conf) throws IOException {
+    int port = fs.getCanonicalUri().getPort();
+    UserGroupInformation ugi =
+        UserGroupInformation.createUserForTesting(fs.getUri().getAuthority(), new String[]{});
+
+    // use ip-based tokens
+    SecurityUtilTestHelper.setTokenServiceUseIp(true);
+
     // test fallback to hdfs token
     Token<?> hdfsToken = new Token<TokenIdentifier>(
         new byte[0], new byte[0],
         DelegationTokenIdentifier.HDFS_DELEGATION_KIND,
         new Text("127.0.0.1:8020"));
     ugi.addToken(hdfsToken);
-    
+
-    HftpFileSystem fs = (HftpFileSystem) FileSystem.get(hftpUri, conf);
+    // test fallback to hdfs token
-    token = fs.selectDelegationToken();
+    Token<?> token = fs.selectDelegationToken(ugi);
     assertNotNull(token);
     assertEquals(hdfsToken, token);
-    
+
     // test hftp is favored over hdfs
     Token<?> hftpToken = new Token<TokenIdentifier>(
         new byte[0], new byte[0],
-        HftpFileSystem.TOKEN_KIND, new Text("127.0.0.1:0"));
+        HftpFileSystem.TOKEN_KIND, new Text("127.0.0.1:"+port));
     ugi.addToken(hftpToken);
-    token = fs.selectDelegationToken();
+    token = fs.selectDelegationToken(ugi);
     assertNotNull(token);
     assertEquals(hftpToken, token);
     
     // switch to using host-based tokens, no token should match
     SecurityUtilTestHelper.setTokenServiceUseIp(false);
-    token = fs.selectDelegationToken();
+    token = fs.selectDelegationToken(ugi);
     assertNull(token);
     
     // test fallback to hdfs token
@@ -109,17 +134,31 @@ public class TestHftpDelegationToken {
         DelegationTokenIdentifier.HDFS_DELEGATION_KIND,
         new Text("localhost:8020"));
     ugi.addToken(hdfsToken);
-    token = fs.selectDelegationToken();
+    token = fs.selectDelegationToken(ugi);
     assertNotNull(token);
     assertEquals(hdfsToken, token);
 
     // test hftp is favored over hdfs
     hftpToken = new Token<TokenIdentifier>(
         new byte[0], new byte[0],
-        HftpFileSystem.TOKEN_KIND, new Text("localhost:0"));
+        HftpFileSystem.TOKEN_KIND, new Text("localhost:"+port));
     ugi.addToken(hftpToken);
-    token = fs.selectDelegationToken();
+    token = fs.selectDelegationToken(ugi);
     assertNotNull(token);
     assertEquals(hftpToken, token);
   }
+  
+  static class MyHftpFileSystem extends HftpFileSystem {
+    @Override
+    public URI getCanonicalUri() {
+      return super.getCanonicalUri();
+    }
+    @Override
+    public int getDefaultPort() {
+      return super.getDefaultPort();
+    }
+    // don't automatically get a token
+    @Override
+    protected void initDelegationToken() throws IOException {}
+  }
 }

hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHdfsUrl.java

@@ -98,38 +98,63 @@ public class TestWebHdfsUrl {
   }
   
   @Test
-  public void testSelectDelegationToken() throws Exception {
+  public void testSelectHdfsDelegationToken() throws Exception {
     SecurityUtilTestHelper.setTokenServiceUseIp(true);
 
     Configuration conf = new Configuration();
-    URI webHdfsUri = URI.create("webhdfs://localhost:0");
+    conf.setClass("fs.webhdfs.impl", MyWebHdfsFileSystem.class, FileSystem.class);
-    UserGroupInformation ugi = UserGroupInformation.getCurrentUser();
-    Token<?> token = null;
     
+    // test with implicit default port 
+    URI fsUri = URI.create("webhdfs://localhost");
+    MyWebHdfsFileSystem fs = (MyWebHdfsFileSystem) FileSystem.get(fsUri, conf);
+    checkTokenSelection(fs, conf);
+
+    // test with explicit default port
+    fsUri = URI.create("webhdfs://localhost:"+fs.getDefaultPort());
+    fs = (MyWebHdfsFileSystem) FileSystem.get(fsUri, conf);
+    checkTokenSelection(fs, conf);
+    
+    // test with non-default port
+    fsUri = URI.create("webhdfs://localhost:"+(fs.getDefaultPort()-1));
+    fs = (MyWebHdfsFileSystem) FileSystem.get(fsUri, conf);
+    checkTokenSelection(fs, conf);
+
+  }
+  
+  private void checkTokenSelection(MyWebHdfsFileSystem fs,
+                                   Configuration conf) throws IOException {
+    int port = fs.getCanonicalUri().getPort();
+    // can't clear tokens from ugi, so create a new user everytime
+    UserGroupInformation ugi =
+        UserGroupInformation.createUserForTesting(fs.getUri().getAuthority(), new String[]{});
+
+    // use ip-based tokens
+    SecurityUtilTestHelper.setTokenServiceUseIp(true);
+
     // test fallback to hdfs token
     Token<?> hdfsToken = new Token<TokenIdentifier>(
         new byte[0], new byte[0],
         DelegationTokenIdentifier.HDFS_DELEGATION_KIND,
         new Text("127.0.0.1:8020"));
     ugi.addToken(hdfsToken);
-    
+
-    WebHdfsFileSystem fs = (WebHdfsFileSystem) FileSystem.get(webHdfsUri, conf);
+    // test fallback to hdfs token
-    token = fs.selectDelegationToken();
+    Token<?> token = fs.selectDelegationToken(ugi);
     assertNotNull(token);
     assertEquals(hdfsToken, token);
-    
+
     // test webhdfs is favored over hdfs
     Token<?> webHdfsToken = new Token<TokenIdentifier>(
         new byte[0], new byte[0],
-        WebHdfsFileSystem.TOKEN_KIND, new Text("127.0.0.1:0"));
+        WebHdfsFileSystem.TOKEN_KIND, new Text("127.0.0.1:"+port));
     ugi.addToken(webHdfsToken);
-    token = fs.selectDelegationToken();
+    token = fs.selectDelegationToken(ugi);
     assertNotNull(token);
     assertEquals(webHdfsToken, token);
     
     // switch to using host-based tokens, no token should match
     SecurityUtilTestHelper.setTokenServiceUseIp(false);
-    token = fs.selectDelegationToken();
+    token = fs.selectDelegationToken(ugi);
     assertNull(token);
     
     // test fallback to hdfs token
@@ -138,18 +163,31 @@ public class TestWebHdfsUrl {
         DelegationTokenIdentifier.HDFS_DELEGATION_KIND,
         new Text("localhost:8020"));
     ugi.addToken(hdfsToken);
-    token = fs.selectDelegationToken();
+    token = fs.selectDelegationToken(ugi);
     assertNotNull(token);
     assertEquals(hdfsToken, token);
 
     // test webhdfs is favored over hdfs
     webHdfsToken = new Token<TokenIdentifier>(
         new byte[0], new byte[0],
-        WebHdfsFileSystem.TOKEN_KIND, new Text("localhost:0"));
+        WebHdfsFileSystem.TOKEN_KIND, new Text("localhost:"+port));
     ugi.addToken(webHdfsToken);
-    token = fs.selectDelegationToken();
+    token = fs.selectDelegationToken(ugi);
     assertNotNull(token);
     assertEquals(webHdfsToken, token);
   }
-
+  
-}
+  static class MyWebHdfsFileSystem extends WebHdfsFileSystem {
+    @Override
+    public URI getCanonicalUri() {
+      return super.getCanonicalUri();
+    }
+    @Override
+    public int getDefaultPort() {
+      return super.getDefaultPort();
+    }
+    // don't automatically get a token
+    @Override
+    protected void initDelegationToken() throws IOException {}
+  }
+}