HADOOP-11181. Generalized o.a.h.s.t.d.DelegationTokenManager to handle all sub-classes of AbstractDelegationTokenIdentifier. Contributed by Zhijie Shen.

(cherry picked from commit cdce88376a)

hadoop-common-project/hadoop-common/CHANGES.txt

@@ -229,6 +229,10 @@ Release 2.6.0 - UNRELEASED
 
     HADOOP-11184. Update Hadoop's lz4 to version r123. (cmccabe)
 
+    HADOOP-11181. Generalized o.a.h.s.t.d.DelegationTokenManager to handle all
+    sub-classes of AbstractDelegationTokenIdentifier. (zjshen)
+
+
   OPTIMIZATIONS
 
     HADOOP-10838. Byte array native checksumming. (James Thomas via todd)

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/AbstractDelegationTokenIdentifier.java

@@ -53,26 +53,9 @@ extends TokenIdentifier {
   }
   
   public AbstractDelegationTokenIdentifier(Text owner, Text renewer, Text realUser) {
-    if (owner == null) {
+    setOwner(owner);
-      this.owner = new Text();
+    setRenewer(renewer);
-    } else {
+    setRealUser(realUser);
-      this.owner = owner;
-    }
-    if (renewer == null) {
-      this.renewer = new Text();
-    } else {
-      HadoopKerberosName renewerKrbName = new HadoopKerberosName(renewer.toString());
-      try {
-        this.renewer = new Text(renewerKrbName.getShortName());
-      } catch (IOException e) {
-        throw new RuntimeException(e);
-      }
-    }
-    if (realUser == null) {
-      this.realUser = new Text();
-    } else {
-      this.realUser = realUser;
-    }
     issueDate = 0;
     maxDate = 0;
   }
@@ -107,14 +90,43 @@ extends TokenIdentifier {
     return owner;
   }
 
+  public void setOwner(Text owner) {
+    if (owner == null) {
+      this.owner = new Text();
+    } else {
+      this.owner = owner;
+    }
+  }
+
   public Text getRenewer() {
     return renewer;
   }
-  
+
+  public void setRenewer(Text renewer) {
+    if (renewer == null) {
+      this.renewer = new Text();
+    } else {
+      HadoopKerberosName renewerKrbName = new HadoopKerberosName(renewer.toString());
+      try {
+        this.renewer = new Text(renewerKrbName.getShortName());
+      } catch (IOException e) {
+        throw new RuntimeException(e);
+      }
+    }
+  }
+
   public Text getRealUser() {
     return realUser;
   }
-  
+
+  public void setRealUser(Text realUser) {
+    if (realUser == null) {
+      this.realUser = new Text();
+    } else {
+      this.realUser = realUser;
+    }
+  }
+
   public void setIssueDate(long issueDate) {
     this.issueDate = issueDate;
   }

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/AbstractDelegationTokenSecretManager.java

@@ -648,4 +648,17 @@ extends AbstractDelegationTokenIdentifier>
       }
     }
   }
+
+  /**
+   * Decode the token identifier. The subclass can customize the way to decode
+   * the token identifier.
+   * 
+   * @param token the token where to extract the identifier
+   * @return the delegation token identifier
+   * @throws IOException
+   */
+  public TokenIdent decodeTokenIdentifier(Token<TokenIdent> token) throws IOException {
+    return token.decodeIdentifier();
+  }
+
 }

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/DelegationTokenAuthenticationHandler.java

@@ -28,6 +28,7 @@ import org.apache.hadoop.security.authentication.server.AuthenticationHandler;
 import org.apache.hadoop.security.authentication.server.AuthenticationToken;
 import org.apache.hadoop.security.authentication.server.KerberosAuthenticationHandler;
 import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.security.token.delegation.AbstractDelegationTokenIdentifier;
 import org.apache.hadoop.security.token.delegation.AbstractDelegationTokenSecretManager;
 import org.apache.hadoop.util.HttpExceptionUtils;
 import org.codehaus.jackson.map.ObjectMapper;
@@ -216,8 +217,7 @@ public abstract class DelegationTokenAuthenticationHandler
                 );
                 requestContinues = false;
               } else {
-                Token<DelegationTokenIdentifier> dt =
+                Token<AbstractDelegationTokenIdentifier> dt = new Token();
-                    new Token<DelegationTokenIdentifier>();
                 try {
                   dt.decodeFromUrlString(tokenToRenew);
                   long expirationTime = tokenManager.renewToken(dt,
@@ -240,8 +240,7 @@ public abstract class DelegationTokenAuthenticationHandler
                 );
                 requestContinues = false;
               } else {
-                Token<DelegationTokenIdentifier> dt =
+                Token<AbstractDelegationTokenIdentifier> dt = new Token();
-                    new Token<DelegationTokenIdentifier>();
                 try {
                   dt.decodeFromUrlString(tokenToCancel);
                   tokenManager.cancelToken(dt, (requestUgi != null)
@@ -303,6 +302,7 @@ public abstract class DelegationTokenAuthenticationHandler
    * @throws IOException thrown if an IO error occurred.
    * @throws AuthenticationException thrown if the authentication failed.
    */
+  @SuppressWarnings("unchecked")
   @Override
   public AuthenticationToken authenticate(HttpServletRequest request,
       HttpServletResponse response)
@@ -311,8 +311,7 @@ public abstract class DelegationTokenAuthenticationHandler
     String delegationParam = getDelegationToken(request);
     if (delegationParam != null) {
       try {
-        Token<DelegationTokenIdentifier> dt =
+        Token<AbstractDelegationTokenIdentifier> dt = new Token();
-            new Token<DelegationTokenIdentifier>();
         dt.decodeFromUrlString(delegationParam);
         UserGroupInformation ugi = tokenManager.verifyToken(dt);
         final String shortName = ugi.getShortUserName();

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/DelegationTokenManager.java

@@ -27,6 +27,7 @@ import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.io.Text;
 import org.apache.hadoop.security.UserGroupInformation;
 import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.security.token.delegation.AbstractDelegationTokenIdentifier;
 import org.apache.hadoop.security.token.delegation.AbstractDelegationTokenSecretManager;
 import org.apache.hadoop.security.token.delegation.ZKDelegationTokenSecretManager;
 
@@ -76,6 +77,13 @@ public class DelegationTokenManager {
     public DelegationTokenIdentifier createIdentifier() {
       return new DelegationTokenIdentifier(tokenKind);
     }
+
+    @Override
+    public DelegationTokenIdentifier decodeTokenIdentifier(
+        Token<DelegationTokenIdentifier> token) throws IOException {
+      return DelegationTokenManager.decodeToken(token, tokenKind);
+    }
+
   }
 
   private static class ZKSecretManager
@@ -92,11 +100,16 @@ public class DelegationTokenManager {
     public DelegationTokenIdentifier createIdentifier() {
       return new DelegationTokenIdentifier(tokenKind);
     }
+
+    @Override
+    public DelegationTokenIdentifier decodeTokenIdentifier(
+        Token<DelegationTokenIdentifier> token) throws IOException {
+      return DelegationTokenManager.decodeToken(token, tokenKind);
+    }
   }
 
   private AbstractDelegationTokenSecretManager secretManager = null;
   private boolean managedSecretManager;
-  private Text tokenKind;
 
   public DelegationTokenManager(Configuration conf, Text tokenKind) {
     if (conf.getBoolean(ENABLE_ZK_KEY, false)) {
@@ -104,7 +117,6 @@ public class DelegationTokenManager {
     } else {
       this.secretManager = new DelegationTokenSecretManager(conf, tokenKind);
     }
-    this.tokenKind = tokenKind;
     managedSecretManager = true;
   }
 
@@ -121,7 +133,6 @@ public class DelegationTokenManager {
       AbstractDelegationTokenSecretManager secretManager) {
     this.secretManager.stopThreads();
     this.secretManager = secretManager;
-    this.tokenKind = secretManager.createIdentifier().getKind();
     managedSecretManager = false;
   }
 
@@ -143,8 +154,8 @@ public class DelegationTokenManager {
   }
 
   @SuppressWarnings("unchecked")
-  public Token<DelegationTokenIdentifier> createToken(UserGroupInformation ugi,
+  public Token<? extends AbstractDelegationTokenIdentifier> createToken(
-      String renewer) {
+      UserGroupInformation ugi, String renewer) {
     renewer = (renewer == null) ? ugi.getShortUserName() : renewer;
     String user = ugi.getUserName();
     Text owner = new Text(user);
@@ -152,19 +163,24 @@ public class DelegationTokenManager {
     if (ugi.getRealUser() != null) {
       realUser = new Text(ugi.getRealUser().getUserName());
     }
-    DelegationTokenIdentifier tokenIdentifier = new DelegationTokenIdentifier(
+    AbstractDelegationTokenIdentifier tokenIdentifier =
-        tokenKind, owner, new Text(renewer), realUser);
+        (AbstractDelegationTokenIdentifier) secretManager.createIdentifier();
-    return new Token<DelegationTokenIdentifier>(tokenIdentifier, secretManager);
+    tokenIdentifier.setOwner(owner);
+    tokenIdentifier.setRenewer(new Text(renewer));
+    tokenIdentifier.setRealUser(realUser);
+    return new Token(tokenIdentifier, secretManager);
   }
 
   @SuppressWarnings("unchecked")
-  public long renewToken(Token<DelegationTokenIdentifier> token, String renewer)
+  public long renewToken(
-      throws IOException {
+      Token<? extends AbstractDelegationTokenIdentifier> token, String renewer)
+          throws IOException {
     return secretManager.renewToken(token, renewer);
   }
 
   @SuppressWarnings("unchecked")
-  public void cancelToken(Token<DelegationTokenIdentifier> token,
+  public void cancelToken(
+      Token<? extends AbstractDelegationTokenIdentifier> token,
       String canceler) throws IOException {
     canceler = (canceler != null) ? canceler :
                verifyToken(token).getShortUserName();
@@ -172,13 +188,10 @@ public class DelegationTokenManager {
   }
 
   @SuppressWarnings("unchecked")
-  public UserGroupInformation verifyToken(Token<DelegationTokenIdentifier>
+  public UserGroupInformation verifyToken(
-      token) throws IOException {
+      Token<? extends AbstractDelegationTokenIdentifier> token)
-    ByteArrayInputStream buf = new ByteArrayInputStream(token.getIdentifier());
+          throws IOException {
-    DataInputStream dis = new DataInputStream(buf);
+    AbstractDelegationTokenIdentifier id = secretManager.decodeTokenIdentifier(token);
-    DelegationTokenIdentifier id = new DelegationTokenIdentifier(tokenKind);
-    id.readFields(dis);
-    dis.close();
     secretManager.verifyToken(id, token.getPassword());
     return id.getUser();
   }
@@ -188,4 +201,15 @@ public class DelegationTokenManager {
   public AbstractDelegationTokenSecretManager getDelegationTokenSecretManager() {
     return secretManager;
   }
+
+  private static DelegationTokenIdentifier decodeToken(
+      Token<DelegationTokenIdentifier> token, Text tokenKind)
+          throws IOException {
+    ByteArrayInputStream buf = new ByteArrayInputStream(token.getIdentifier());
+    DataInputStream dis = new DataInputStream(buf);
+    DelegationTokenIdentifier id = new DelegationTokenIdentifier(tokenKind);
+    id.readFields(dis);
+    dis.close();
+    return id;
+  }
 }

hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/token/delegation/TestZKDelegationTokenSecretManager.java

@@ -32,6 +32,7 @@ public class TestZKDelegationTokenSecretManager {
 
   private static final long DAY_IN_SECS = 86400;
 
+  @SuppressWarnings("unchecked")
   @Test
   public void testZKDelTokSecretManager() throws Exception {
     TestingServer zkServer = new TestingServer();
@@ -54,11 +55,13 @@ public class TestZKDelegationTokenSecretManager {
       tm2.init();
 
       Token<DelegationTokenIdentifier> token =
-          tm1.createToken(UserGroupInformation.getCurrentUser(), "foo");
+          (Token<DelegationTokenIdentifier>) tm1.createToken(
+              UserGroupInformation.getCurrentUser(), "foo");
       Assert.assertNotNull(token);
       tm2.verifyToken(token);
 
-      token = tm2.createToken(UserGroupInformation.getCurrentUser(), "bar");
+      token = (Token<DelegationTokenIdentifier>) tm2.createToken(
+          UserGroupInformation.getCurrentUser(), "bar");
       Assert.assertNotNull(token);
       tm1.verifyToken(token);
     } finally {

hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/token/delegation/web/TestDelegationTokenAuthenticationHandlerWithMocks.java

@@ -202,6 +202,7 @@ public class TestDelegationTokenAuthenticationHandlerWithMocks {
     Assert.assertEquals(expectedTokenKind, dt.getKind());
   }
 
+  @SuppressWarnings("unchecked")
   private void testCancelToken() throws Exception {
     DelegationTokenAuthenticator.DelegationTokenOperation op =
         DelegationTokenAuthenticator.DelegationTokenOperation.
@@ -220,7 +221,7 @@ public class TestDelegationTokenAuthenticationHandlerWithMocks {
 
     Mockito.reset(response);
     Token<DelegationTokenIdentifier> token =
-        handler.getTokenManager().createToken(
+        (Token<DelegationTokenIdentifier>) handler.getTokenManager().createToken(
             UserGroupInformation.getCurrentUser(), "foo");
     Mockito.when(request.getQueryString()).thenReturn(
         DelegationTokenAuthenticator.OP_PARAM + "=" + op.toString() + "&" +
@@ -239,6 +240,7 @@ public class TestDelegationTokenAuthenticationHandlerWithMocks {
     }
   }
 
+  @SuppressWarnings("unchecked")
   private void testRenewToken() throws Exception {
     DelegationTokenAuthenticator.DelegationTokenOperation op =
         DelegationTokenAuthenticator.DelegationTokenOperation.
@@ -271,7 +273,7 @@ public class TestDelegationTokenAuthenticationHandlerWithMocks {
     PrintWriter pwriter = new PrintWriter(writer);
     Mockito.when(response.getWriter()).thenReturn(pwriter);
     Token<DelegationTokenIdentifier> dToken =
-        handler.getTokenManager().createToken(
+        (Token<DelegationTokenIdentifier>) handler.getTokenManager().createToken(
             UserGroupInformation.getCurrentUser(), "user");
     Mockito.when(request.getQueryString()).
         thenReturn(DelegationTokenAuthenticator.OP_PARAM + "=" + op.toString() +
@@ -292,11 +294,12 @@ public class TestDelegationTokenAuthenticationHandlerWithMocks {
     testInvalidDelegationTokenHeader();
   }
 
+  @SuppressWarnings("unchecked")
   private void testValidDelegationTokenQueryString() throws Exception {
     HttpServletRequest request = Mockito.mock(HttpServletRequest.class);
     HttpServletResponse response = Mockito.mock(HttpServletResponse.class);
     Token<DelegationTokenIdentifier> dToken =
-        handler.getTokenManager().createToken(
+        (Token<DelegationTokenIdentifier>) handler.getTokenManager().createToken(
             UserGroupInformation.getCurrentUser(), "user");
     Mockito.when(request.getQueryString()).thenReturn(
         DelegationTokenAuthenticator.DELEGATION_PARAM + "=" +
@@ -311,11 +314,12 @@ public class TestDelegationTokenAuthenticationHandlerWithMocks {
     Assert.assertTrue(token.isExpired());
   }
 
+  @SuppressWarnings("unchecked")
   private void testValidDelegationTokenHeader() throws Exception {
     HttpServletRequest request = Mockito.mock(HttpServletRequest.class);
     HttpServletResponse response = Mockito.mock(HttpServletResponse.class);
     Token<DelegationTokenIdentifier> dToken =
-        handler.getTokenManager().createToken(
+        (Token<DelegationTokenIdentifier>) handler.getTokenManager().createToken(
             UserGroupInformation.getCurrentUser(), "user");
     Mockito.when(request.getHeader(Mockito.eq(
         DelegationTokenAuthenticator.DELEGATION_TOKEN_HEADER))).thenReturn(

hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/token/delegation/web/TestDelegationTokenManager.java

@@ -18,6 +18,8 @@
 package org.apache.hadoop.security.token.delegation.web;
 
 import java.io.IOException;
+import java.util.Arrays;
+import java.util.Collection;
 
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.io.Text;
@@ -25,11 +27,26 @@ import org.apache.hadoop.security.UserGroupInformation;
 import org.apache.hadoop.security.token.Token;
 import org.junit.Assert;
 import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
 
+@RunWith(Parameterized.class)
 public class TestDelegationTokenManager {
 
   private static final long DAY_IN_SECS = 86400;
 
+  @Parameterized.Parameters
+  public static Collection<Object[]> headers() {
+    return Arrays.asList(new Object[][] { { false }, { true } });
+  }
+
+  private boolean enableZKKey;
+
+  public TestDelegationTokenManager(boolean enableZKKey) {
+    this.enableZKKey = enableZKKey;
+  }
+
+  @SuppressWarnings("unchecked")
   @Test
   public void testDTManager() throws Exception {
     Configuration conf = new Configuration(false);
@@ -37,11 +54,13 @@ public class TestDelegationTokenManager {
     conf.setLong(DelegationTokenManager.MAX_LIFETIME, DAY_IN_SECS);
     conf.setLong(DelegationTokenManager.RENEW_INTERVAL, DAY_IN_SECS);
     conf.setLong(DelegationTokenManager.REMOVAL_SCAN_INTERVAL, DAY_IN_SECS);
+    conf.getBoolean(DelegationTokenManager.ENABLE_ZK_KEY, enableZKKey);
     DelegationTokenManager tm =
         new DelegationTokenManager(conf, new Text("foo"));
     tm.init();
     Token<DelegationTokenIdentifier> token =
-        tm.createToken(UserGroupInformation.getCurrentUser(), "foo");
+        (Token<DelegationTokenIdentifier>) tm.createToken(
+            UserGroupInformation.getCurrentUser(), "foo");
     Assert.assertNotNull(token);
     tm.verifyToken(token);
     Assert.assertTrue(tm.renewToken(token, "foo") > System.currentTimeMillis());

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesDelegationTokens.java

@@ -738,7 +738,8 @@ public class TestRMWebServicesDelegationTokens extends JerseyTest {
     Token<RMDelegationTokenIdentifier> realToken =
         new Token<RMDelegationTokenIdentifier>();
     realToken.decodeFromUrlString(encodedToken);
-    RMDelegationTokenIdentifier ident = realToken.decodeIdentifier();
+    RMDelegationTokenIdentifier ident = rm.getRMContext()
+      .getRMDelegationTokenSecretManager().decodeTokenIdentifier(realToken);
     rm.getRMContext().getRMDelegationTokenSecretManager()
       .verifyToken(ident, realToken.getPassword());
     assertTrue(rm.getRMContext().getRMDelegationTokenSecretManager()
@@ -749,7 +750,8 @@ public class TestRMWebServicesDelegationTokens extends JerseyTest {
     Token<RMDelegationTokenIdentifier> realToken =
         new Token<RMDelegationTokenIdentifier>();
     realToken.decodeFromUrlString(encodedToken);
-    RMDelegationTokenIdentifier ident = realToken.decodeIdentifier();
+    RMDelegationTokenIdentifier ident = rm.getRMContext()
+      .getRMDelegationTokenSecretManager().decodeTokenIdentifier(realToken);
     boolean exceptionCaught = false;
     try {
       rm.getRMContext().getRMDelegationTokenSecretManager()