From 8c505adddbaec558c25f3dd26f92eb6107150281 Mon Sep 17 00:00:00 2001 From: Jing Zhao Date: Wed, 29 Jan 2014 21:53:32 +0000 Subject: [PATCH] HDFS-5842. Cannot create hftp filesystem when using a proxy user ugi and a doAs on a secure cluster. Contributed by Jing Zhao. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1562603 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 + .../hdfs/tools/DelegationTokenFetcher.java | 11 ++- .../hadoop/hdfs/web/HftpFileSystem.java | 75 ++++++++++++------- 3 files changed, 60 insertions(+), 29 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index d2188dd8abb..01af9fb4fd0 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -602,6 +602,9 @@ Release 2.4.0 - UNRELEASED HDFS-5728. Block recovery will fail if the metafile does not have crc for all chunks of the block (Vinay via kihwal) + HDFS-5842. Cannot create hftp filesystem when using a proxy user ugi and a doAs + on a secure cluster. (jing9) + BREAKDOWN OF HDFS-2832 SUBTASKS AND RELATED JIRAS HDFS-4985. Add storage type to the protocol and expose it in block report diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DelegationTokenFetcher.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DelegationTokenFetcher.java index bc76cc07879..ecd87b91d39 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DelegationTokenFetcher.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DelegationTokenFetcher.java @@ -185,8 +185,8 @@ public Object run() throws Exception { } else { // otherwise we are fetching if (webUrl != null) { - Credentials creds = getDTfromRemote(connectionFactory, new URI(webUrl), - renewer); + Credentials creds = getDTfromRemote(connectionFactory, new URI( + webUrl), renewer, null); creds.writeTokenStorageFile(tokenFile, conf); for (Token token : creds.getAllTokens()) { if(LOG.isDebugEnabled()) { @@ -213,12 +213,17 @@ public Object run() throws Exception { } static public Credentials getDTfromRemote(URLConnectionFactory factory, - URI nnUri, String renewer) throws IOException { + URI nnUri, String renewer, String proxyUser) throws IOException { StringBuilder buf = new StringBuilder(nnUri.toString()) .append(GetDelegationTokenServlet.PATH_SPEC); + String separator = "?"; if (renewer != null) { buf.append("?").append(GetDelegationTokenServlet.RENEWER).append("=") .append(renewer); + separator = "&"; + } + if (proxyUser != null) { + buf.append(separator).append("doas=").append(proxyUser); } boolean isHttps = nnUri.getScheme().equals("https"); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/HftpFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/HftpFileSystem.java index 076b4b51d86..83adc226dce 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/HftpFileSystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/HftpFileSystem.java @@ -57,7 +57,6 @@ import org.apache.hadoop.security.Credentials; import org.apache.hadoop.security.SecurityUtil; import org.apache.hadoop.security.UserGroupInformation; -import org.apache.hadoop.security.authentication.client.AuthenticationException; import org.apache.hadoop.security.token.Token; import org.apache.hadoop.security.token.TokenIdentifier; import org.apache.hadoop.util.Progressable; @@ -234,17 +233,23 @@ public synchronized void setDelegationToken(Token } @Override - public synchronized Token getDelegationToken(final String renewer - ) throws IOException { + public synchronized Token getDelegationToken(final String renewer) + throws IOException { try { - //Renew TGT if needed - ugi.checkTGTAndReloginFromKeytab(); - return ugi.doAs(new PrivilegedExceptionAction>() { + // Renew TGT if needed + UserGroupInformation connectUgi = ugi.getRealUser(); + final String proxyUser = connectUgi == null ? null : ugi + .getShortUserName(); + if (connectUgi == null) { + connectUgi = ugi; + } + return connectUgi.doAs(new PrivilegedExceptionAction>() { @Override public Token run() throws IOException { Credentials c; try { - c = DelegationTokenFetcher.getDTfromRemote(connectionFactory, nnUri, renewer); + c = DelegationTokenFetcher.getDTfromRemote(connectionFactory, + nnUri, renewer, proxyUser); } catch (IOException e) { if (e.getCause() instanceof ConnectException) { LOG.warn("Couldn't connect to " + nnUri + @@ -299,13 +304,13 @@ protected URL getNamenodeURL(String path, String query) throws IOException { * @return user_shortname,group1,group2... */ private String getEncodedUgiParameter() { - StringBuilder ugiParamenter = new StringBuilder( + StringBuilder ugiParameter = new StringBuilder( ServletUtil.encodeQueryValue(ugi.getShortUserName())); for(String g: ugi.getGroupNames()) { - ugiParamenter.append(","); - ugiParamenter.append(ServletUtil.encodeQueryValue(g)); + ugiParameter.append(","); + ugiParameter.append(ServletUtil.encodeQueryValue(g)); } - return ugiParamenter.toString(); + return ugiParameter.toString(); } /** @@ -675,30 +680,48 @@ public ContentSummary getContentSummary(Path f) throws IOException { @SuppressWarnings("unchecked") @Override - public long renewDelegationToken(Token token) throws IOException { + public long renewDelegationToken(final Token token) throws IOException { // update the kerberos credentials, if they are coming from a keytab - UserGroupInformation.getLoginUser().checkTGTAndReloginFromKeytab(); - InetSocketAddress serviceAddr = SecurityUtil.getTokenServiceAddr(token); + UserGroupInformation connectUgi = ugi.getRealUser(); + if (connectUgi == null) { + connectUgi = ugi; + } try { - return DelegationTokenFetcher.renewDelegationToken(connectionFactory, - DFSUtil.createUri(getUnderlyingProtocol(), serviceAddr), - (Token) token); - } catch (AuthenticationException e) { + return connectUgi.doAs(new PrivilegedExceptionAction() { + @Override + public Long run() throws Exception { + InetSocketAddress serviceAddr = SecurityUtil + .getTokenServiceAddr(token); + return DelegationTokenFetcher.renewDelegationToken(connectionFactory, + DFSUtil.createUri(getUnderlyingProtocol(), serviceAddr), + (Token) token); + } + }); + } catch (InterruptedException e) { throw new IOException(e); } } @SuppressWarnings("unchecked") @Override - public void cancelDelegationToken(Token token) throws IOException { - // update the kerberos credentials, if they are coming from a keytab - UserGroupInformation.getLoginUser().checkTGTAndReloginFromKeytab(); - InetSocketAddress serviceAddr = SecurityUtil.getTokenServiceAddr(token); + public void cancelDelegationToken(final Token token) throws IOException { + UserGroupInformation connectUgi = ugi.getRealUser(); + if (connectUgi == null) { + connectUgi = ugi; + } try { - DelegationTokenFetcher.cancelDelegationToken(connectionFactory, DFSUtil - .createUri(getUnderlyingProtocol(), serviceAddr), - (Token) token); - } catch (AuthenticationException e) { + connectUgi.doAs(new PrivilegedExceptionAction() { + @Override + public Void run() throws Exception { + InetSocketAddress serviceAddr = SecurityUtil + .getTokenServiceAddr(token); + DelegationTokenFetcher.cancelDelegationToken(connectionFactory, + DFSUtil.createUri(getUnderlyingProtocol(), serviceAddr), + (Token) token); + return null; + } + }); + } catch (InterruptedException e) { throw new IOException(e); } }