From e360e796eff6e1faa5781c13e2ba70b751cda168 Mon Sep 17 00:00:00 2001 From: Tsz-wo Sze Date: Mon, 23 Apr 2012 21:29:18 +0000 Subject: [PATCH] svn merge -c 1329462 from trunk for HDFS-3312. In HftpFileSystem, the namenode URI is non-secure but the delegation tokens have to use secure URI. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/branch-2@1329464 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 + .../apache/hadoop/hdfs/HftpFileSystem.java | 26 ++++--- .../apache/hadoop/hdfs/HsftpFileSystem.java | 9 ++- .../hadoop/hdfs/TestHftpDelegationToken.java | 78 ++++++++++++++++--- 4 files changed, 94 insertions(+), 22 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 7a081ea83ed..b781253a029 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -767,6 +767,9 @@ Release 0.23.3 - UNRELEASED HDFS-3308. Uses canonical URI to select delegation tokens in HftpFileSystem and WebHdfsFileSystem. (Daryn Sharp via szetszwo) + HDFS-3312. In HftpFileSystem, the namenode URI is non-secure but the + delegation tokens have to use secure URI. (Daryn Sharp via szetszwo) + Release 0.23.2 - UNRELEASED INCOMPATIBLE CHANGES diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/HftpFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/HftpFileSystem.java index cadde41514c..82b321d2dd4 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/HftpFileSystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/HftpFileSystem.java @@ -94,8 +94,8 @@ public class HftpFileSystem extends FileSystem protected UserGroupInformation ugi; private URI hftpURI; - protected InetSocketAddress nnAddr; - protected InetSocketAddress nnSecureAddr; + protected URI nnUri; + protected URI nnSecureUri; public static final String HFTP_TIMEZONE = "UTC"; public static final String HFTP_DATE_FORMAT = "yyyy-MM-dd'T'HH:mm:ssZ"; @@ -139,11 +139,19 @@ public class HftpFileSystem extends FileSystem return NetUtils.createSocketAddrForHost(uri.getHost(), getDefaultSecurePort()); } + protected URI getNamenodeUri(URI uri) { + return DFSUtil.createUri("http", getNamenodeAddr(uri)); + } + + protected URI getNamenodeSecureUri(URI uri) { + return DFSUtil.createUri("https", getNamenodeSecureAddr(uri)); + } + @Override public String getCanonicalServiceName() { // unlike other filesystems, hftp's service is the secure port, not the // actual port in the uri - return SecurityUtil.buildTokenService(nnSecureAddr).toString(); + return SecurityUtil.buildTokenService(nnSecureUri).toString(); } @Override @@ -152,8 +160,8 @@ public class HftpFileSystem extends FileSystem super.initialize(name, conf); setConf(conf); this.ugi = UserGroupInformation.getCurrentUser(); - this.nnAddr = getNamenodeAddr(name); - this.nnSecureAddr = getNamenodeSecureAddr(name); + this.nnUri = getNamenodeUri(name); + this.nnSecureUri = getNamenodeSecureUri(name); try { this.hftpURI = new URI(name.getScheme(), name.getAuthority(), null, null, null); @@ -191,7 +199,7 @@ public class HftpFileSystem extends FileSystem protected Token selectDelegationToken( UserGroupInformation ugi) { - return hftpTokenSelector.selectToken(getCanonicalUri(), ugi.getTokens(), getConf()); + return hftpTokenSelector.selectToken(nnSecureUri, ugi.getTokens(), getConf()); } @@ -222,7 +230,7 @@ public class HftpFileSystem extends FileSystem ugi.reloginFromKeytab(); return ugi.doAs(new PrivilegedExceptionAction>() { public Token run() throws IOException { - final String nnHttpUrl = DFSUtil.createUri("https", nnSecureAddr).toString(); + final String nnHttpUrl = nnSecureUri.toString(); Credentials c; try { c = DelegationTokenFetcher.getDTfromRemote(nnHttpUrl, renewer); @@ -264,8 +272,8 @@ public class HftpFileSystem extends FileSystem * @throws IOException on error constructing the URL */ protected URL getNamenodeURL(String path, String query) throws IOException { - final URL url = new URL("http", nnAddr.getHostName(), - nnAddr.getPort(), path + '?' + query); + final URL url = new URL("http", nnUri.getHost(), + nnUri.getPort(), path + '?' + query); if (LOG.isTraceEnabled()) { LOG.trace("url=" + url); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/HsftpFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/HsftpFileSystem.java index b6fe12d66b8..141ebb2bbd4 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/HsftpFileSystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/HsftpFileSystem.java @@ -132,12 +132,17 @@ public class HsftpFileSystem extends HftpFileSystem { return getNamenodeAddr(uri); } + @Override + protected URI getNamenodeUri(URI uri) { + return getNamenodeSecureUri(uri); + } + @Override protected HttpURLConnection openConnection(String path, String query) throws IOException { query = addDelegationTokenParam(query); - final URL url = new URL("https", nnAddr.getHostName(), - nnAddr.getPort(), path + '?' + query); + final URL url = new URL("https", nnUri.getHost(), + nnUri.getPort(), path + '?' + query); HttpsURLConnection conn = (HttpsURLConnection)URLUtils.openConnection(url); // bypass hostname verification conn.setHostnameVerifier(new DummyHostnameVerifier()); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestHftpDelegationToken.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestHftpDelegationToken.java index d292aede5cd..809073d0cec 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestHftpDelegationToken.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestHftpDelegationToken.java @@ -70,32 +70,74 @@ public class TestHftpDelegationToken { } @Test - public void testSelectHdfsDelegationToken() throws Exception { + public void testSelectHftpDelegationToken() throws Exception { SecurityUtilTestHelper.setTokenServiceUseIp(true); Configuration conf = new Configuration(); conf.setClass("fs.hftp.impl", MyHftpFileSystem.class, FileSystem.class); + int httpPort = 80; + int httpsPort = 443; + conf.setInt(DFSConfigKeys.DFS_NAMENODE_HTTP_PORT_KEY, httpPort); + conf.setInt(DFSConfigKeys.DFS_NAMENODE_HTTPS_PORT_KEY, httpsPort); + // test with implicit default port URI fsUri = URI.create("hftp://localhost"); - MyHftpFileSystem fs = (MyHftpFileSystem) FileSystem.get(fsUri, conf); - checkTokenSelection(fs, conf); + MyHftpFileSystem fs = (MyHftpFileSystem) FileSystem.newInstance(fsUri, conf); + assertEquals(httpPort, fs.getCanonicalUri().getPort()); + checkTokenSelection(fs, httpsPort, conf); // should still use secure port // test with explicit default port - fsUri = URI.create("hftp://localhost:"+fs.getDefaultPort()); - fs = (MyHftpFileSystem) FileSystem.get(fsUri, conf); - checkTokenSelection(fs, conf); + fsUri = URI.create("hftp://localhost:"+httpPort); + fs = (MyHftpFileSystem) FileSystem.newInstance(fsUri, conf); + assertEquals(httpPort, fs.getCanonicalUri().getPort()); + checkTokenSelection(fs, httpsPort, conf); // should still use secure port // test with non-default port - fsUri = URI.create("hftp://localhost:"+(fs.getDefaultPort()-1)); - fs = (MyHftpFileSystem) FileSystem.get(fsUri, conf); - checkTokenSelection(fs, conf); + fsUri = URI.create("hftp://localhost:"+(httpPort+1)); + fs = (MyHftpFileSystem) FileSystem.newInstance(fsUri, conf); + assertEquals(httpPort+1, fs.getCanonicalUri().getPort()); + checkTokenSelection(fs, httpsPort, conf); // should still use secure port + + conf.setInt(DFSConfigKeys.DFS_NAMENODE_HTTPS_PORT_KEY, 5); + } + @Test + public void testSelectHsftpDelegationToken() throws Exception { + SecurityUtilTestHelper.setTokenServiceUseIp(true); + + Configuration conf = new Configuration(); + conf.setClass("fs.hsftp.impl", MyHsftpFileSystem.class, FileSystem.class); + + int httpPort = 80; + int httpsPort = 443; + conf.setInt(DFSConfigKeys.DFS_NAMENODE_HTTP_PORT_KEY, httpPort); + conf.setInt(DFSConfigKeys.DFS_NAMENODE_HTTPS_PORT_KEY, httpsPort); + + // test with implicit default port + URI fsUri = URI.create("hsftp://localhost"); + MyHsftpFileSystem fs = (MyHsftpFileSystem) FileSystem.newInstance(fsUri, conf); + assertEquals(httpsPort, fs.getCanonicalUri().getPort()); + checkTokenSelection(fs, httpsPort, conf); + + // test with explicit default port + fsUri = URI.create("hsftp://localhost:"+httpsPort); + fs = (MyHsftpFileSystem) FileSystem.newInstance(fsUri, conf); + assertEquals(httpsPort, fs.getCanonicalUri().getPort()); + checkTokenSelection(fs, httpsPort, conf); + + // test with non-default port + fsUri = URI.create("hsftp://localhost:"+(httpsPort+1)); + fs = (MyHsftpFileSystem) FileSystem.newInstance(fsUri, conf); + assertEquals(httpsPort+1, fs.getCanonicalUri().getPort()); + checkTokenSelection(fs, httpsPort+1, conf); + + conf.setInt(DFSConfigKeys.DFS_NAMENODE_HTTPS_PORT_KEY, 5); } - private void checkTokenSelection(MyHftpFileSystem fs, + private void checkTokenSelection(HftpFileSystem fs, + int port, Configuration conf) throws IOException { - int port = fs.getCanonicalUri().getPort(); UserGroupInformation ugi = UserGroupInformation.createUserForTesting(fs.getUri().getAuthority(), new String[]{}); @@ -161,4 +203,18 @@ public class TestHftpDelegationToken { @Override protected void initDelegationToken() throws IOException {} } + + static class MyHsftpFileSystem extends HsftpFileSystem { + @Override + public URI getCanonicalUri() { + return super.getCanonicalUri(); + } + @Override + public int getDefaultPort() { + return super.getDefaultPort(); + } + // don't automatically get a token + @Override + protected void initDelegationToken() throws IOException {} + } } \ No newline at end of file