HDFS-5842. Merge change r1562603 from trunk.

git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/branch-2@1562608 13f79535-47bb-0310-9956-ffa450edef68

hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt

@@ -316,6 +316,9 @@ Release 2.4.0 - UNRELEASED
     HDFS-5721. sharedEditsImage in Namenode#initializeSharedEdits() should be
     closed before method returns (Ted Yu via todd)
 
+    HDFS-5842. Cannot create hftp filesystem when using a proxy user ugi and a doAs 
+    on a secure cluster. (jing9)
+
   BREAKDOWN OF HDFS-2832 SUBTASKS AND RELATED JIRAS
 
     HDFS-4985. Add storage type to the protocol and expose it in block report

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DelegationTokenFetcher.java

@@ -186,8 +186,8 @@ public class DelegationTokenFetcher {
             } else {
               // otherwise we are fetching
               if (webUrl != null) {
-                Credentials creds = getDTfromRemote(connectionFactory, new URI(webUrl),
+                Credentials creds = getDTfromRemote(connectionFactory, new URI(
-                    renewer);
+                    webUrl), renewer, null);
                 creds.writeTokenStorageFile(tokenFile, conf);
                 for (Token<?> token : creds.getAllTokens()) {
                   System.out.println("Fetched token via " + webUrl + " for "
@@ -210,12 +210,17 @@ public class DelegationTokenFetcher {
   }
   
   static public Credentials getDTfromRemote(URLConnectionFactory factory,
-      URI nnUri, String renewer) throws IOException {
+      URI nnUri, String renewer, String proxyUser) throws IOException {
     StringBuilder buf = new StringBuilder(nnUri.toString())
         .append(GetDelegationTokenServlet.PATH_SPEC);
+    String separator = "?";
     if (renewer != null) {
       buf.append("?").append(GetDelegationTokenServlet.RENEWER).append("=")
           .append(renewer);
+      separator = "&";
+    }
+    if (proxyUser != null) {
+      buf.append(separator).append("doas=").append(proxyUser);
     }
 
     boolean isHttps = nnUri.getScheme().equals("https");

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/HftpFileSystem.java

@@ -57,7 +57,6 @@ import org.apache.hadoop.net.NetUtils;
 import org.apache.hadoop.security.Credentials;
 import org.apache.hadoop.security.SecurityUtil;
 import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hadoop.security.authentication.client.AuthenticationException;
 import org.apache.hadoop.security.token.Token;
 import org.apache.hadoop.security.token.TokenIdentifier;
 import org.apache.hadoop.util.Progressable;
@@ -234,17 +233,23 @@ public class HftpFileSystem extends FileSystem
   }
 
   @Override
-  public synchronized Token<?> getDelegationToken(final String renewer
+  public synchronized Token<?> getDelegationToken(final String renewer)
-                                                  ) throws IOException {
+      throws IOException {
     try {
-      //Renew TGT if needed
+      // Renew TGT if needed
-      ugi.checkTGTAndReloginFromKeytab();
+      UserGroupInformation connectUgi = ugi.getRealUser();
-      return ugi.doAs(new PrivilegedExceptionAction<Token<?>>() {
+      final String proxyUser = connectUgi == null ? null : ugi
+          .getShortUserName();
+      if (connectUgi == null) {
+        connectUgi = ugi;
+      }
+      return connectUgi.doAs(new PrivilegedExceptionAction<Token<?>>() {
         @Override
         public Token<?> run() throws IOException {
           Credentials c;
           try {
-            c = DelegationTokenFetcher.getDTfromRemote(connectionFactory, nnUri, renewer);
+            c = DelegationTokenFetcher.getDTfromRemote(connectionFactory,
+                nnUri, renewer, proxyUser);
           } catch (IOException e) {
             if (e.getCause() instanceof ConnectException) {
               LOG.warn("Couldn't connect to " + nnUri +
@@ -299,13 +304,13 @@ public class HftpFileSystem extends FileSystem
    * @return user_shortname,group1,group2...
    */
   private String getEncodedUgiParameter() {
-    StringBuilder ugiParamenter = new StringBuilder(
+    StringBuilder ugiParameter = new StringBuilder(
         ServletUtil.encodeQueryValue(ugi.getShortUserName()));
     for(String g: ugi.getGroupNames()) {
-      ugiParamenter.append(",");
+      ugiParameter.append(",");
-      ugiParamenter.append(ServletUtil.encodeQueryValue(g));
+      ugiParameter.append(ServletUtil.encodeQueryValue(g));
     }
-    return ugiParamenter.toString();
+    return ugiParameter.toString();
   }
 
   /**
@@ -675,30 +680,48 @@ public class HftpFileSystem extends FileSystem
 
   @SuppressWarnings("unchecked")
   @Override
-  public long renewDelegationToken(Token<?> token) throws IOException {
+  public long renewDelegationToken(final Token<?> token) throws IOException {
     // update the kerberos credentials, if they are coming from a keytab
-    UserGroupInformation.getLoginUser().checkTGTAndReloginFromKeytab();
+    UserGroupInformation connectUgi = ugi.getRealUser();
-    InetSocketAddress serviceAddr = SecurityUtil.getTokenServiceAddr(token);
+    if (connectUgi == null) {
+      connectUgi = ugi;
+    }
     try {
-      return DelegationTokenFetcher.renewDelegationToken(connectionFactory,
+      return connectUgi.doAs(new PrivilegedExceptionAction<Long>() {
-          DFSUtil.createUri(getUnderlyingProtocol(), serviceAddr),
+        @Override
-          (Token<DelegationTokenIdentifier>) token);
+        public Long run() throws Exception {
-    } catch (AuthenticationException e) {
+          InetSocketAddress serviceAddr = SecurityUtil
+              .getTokenServiceAddr(token);
+          return DelegationTokenFetcher.renewDelegationToken(connectionFactory,
+              DFSUtil.createUri(getUnderlyingProtocol(), serviceAddr),
+              (Token<DelegationTokenIdentifier>) token);
+        }
+      });
+    } catch (InterruptedException e) {
       throw new IOException(e);
     }
   }
 
   @SuppressWarnings("unchecked")
   @Override
-  public void cancelDelegationToken(Token<?> token) throws IOException {
+  public void cancelDelegationToken(final Token<?> token) throws IOException {
-    // update the kerberos credentials, if they are coming from a keytab
+    UserGroupInformation connectUgi = ugi.getRealUser();
-    UserGroupInformation.getLoginUser().checkTGTAndReloginFromKeytab();
+    if (connectUgi == null) {
-    InetSocketAddress serviceAddr = SecurityUtil.getTokenServiceAddr(token);
+      connectUgi = ugi;
+    }
     try {
-      DelegationTokenFetcher.cancelDelegationToken(connectionFactory, DFSUtil
+      connectUgi.doAs(new PrivilegedExceptionAction<Void>() {
-          .createUri(getUnderlyingProtocol(), serviceAddr),
+        @Override
-          (Token<DelegationTokenIdentifier>) token);
+        public Void run() throws Exception {
-    } catch (AuthenticationException e) {
+          InetSocketAddress serviceAddr = SecurityUtil
+              .getTokenServiceAddr(token);
+          DelegationTokenFetcher.cancelDelegationToken(connectionFactory,
+              DFSUtil.createUri(getUnderlyingProtocol(), serviceAddr),
+              (Token<DelegationTokenIdentifier>) token);
+          return null;
+        }
+      });
+    } catch (InterruptedException e) {
       throw new IOException(e);
     }
   }