diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index bfaa9b3cfb4..77a2677ded8 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -229,6 +229,10 @@ Release 2.6.0 - UNRELEASED HADOOP-11184. Update Hadoop's lz4 to version r123. (cmccabe) + HADOOP-11181. Generalized o.a.h.s.t.d.DelegationTokenManager to handle all + sub-classes of AbstractDelegationTokenIdentifier. (zjshen) + + OPTIMIZATIONS HADOOP-10838. Byte array native checksumming. (James Thomas via todd) diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/AbstractDelegationTokenIdentifier.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/AbstractDelegationTokenIdentifier.java index 089cd99fd5c..168a77b736f 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/AbstractDelegationTokenIdentifier.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/AbstractDelegationTokenIdentifier.java @@ -53,26 +53,9 @@ extends TokenIdentifier { } public AbstractDelegationTokenIdentifier(Text owner, Text renewer, Text realUser) { - if (owner == null) { - this.owner = new Text(); - } else { - this.owner = owner; - } - if (renewer == null) { - this.renewer = new Text(); - } else { - HadoopKerberosName renewerKrbName = new HadoopKerberosName(renewer.toString()); - try { - this.renewer = new Text(renewerKrbName.getShortName()); - } catch (IOException e) { - throw new RuntimeException(e); - } - } - if (realUser == null) { - this.realUser = new Text(); - } else { - this.realUser = realUser; - } + setOwner(owner); + setRenewer(renewer); + setRealUser(realUser); issueDate = 0; maxDate = 0; } @@ -107,14 +90,43 @@ extends TokenIdentifier { return owner; } + public void setOwner(Text owner) { + if (owner == null) { + this.owner = new Text(); + } else { + this.owner = owner; + } + } + public Text getRenewer() { return renewer; } - + + public void setRenewer(Text renewer) { + if (renewer == null) { + this.renewer = new Text(); + } else { + HadoopKerberosName renewerKrbName = new HadoopKerberosName(renewer.toString()); + try { + this.renewer = new Text(renewerKrbName.getShortName()); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + } + public Text getRealUser() { return realUser; } - + + public void setRealUser(Text realUser) { + if (realUser == null) { + this.realUser = new Text(); + } else { + this.realUser = realUser; + } + } + public void setIssueDate(long issueDate) { this.issueDate = issueDate; } diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/AbstractDelegationTokenSecretManager.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/AbstractDelegationTokenSecretManager.java index ac1dd2d7c79..7b6bf351a93 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/AbstractDelegationTokenSecretManager.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/AbstractDelegationTokenSecretManager.java @@ -648,4 +648,17 @@ extends AbstractDelegationTokenIdentifier> } } } + + /** + * Decode the token identifier. The subclass can customize the way to decode + * the token identifier. + * + * @param token the token where to extract the identifier + * @return the delegation token identifier + * @throws IOException + */ + public TokenIdent decodeTokenIdentifier(Token token) throws IOException { + return token.decodeIdentifier(); + } + } diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/DelegationTokenAuthenticationHandler.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/DelegationTokenAuthenticationHandler.java index 5a31d6dc294..b73b0629ed6 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/DelegationTokenAuthenticationHandler.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/DelegationTokenAuthenticationHandler.java @@ -28,6 +28,7 @@ import org.apache.hadoop.security.authentication.server.AuthenticationHandler; import org.apache.hadoop.security.authentication.server.AuthenticationToken; import org.apache.hadoop.security.authentication.server.KerberosAuthenticationHandler; import org.apache.hadoop.security.token.Token; +import org.apache.hadoop.security.token.delegation.AbstractDelegationTokenIdentifier; import org.apache.hadoop.security.token.delegation.AbstractDelegationTokenSecretManager; import org.apache.hadoop.util.HttpExceptionUtils; import org.codehaus.jackson.map.ObjectMapper; @@ -216,8 +217,7 @@ public abstract class DelegationTokenAuthenticationHandler ); requestContinues = false; } else { - Token dt = - new Token(); + Token dt = new Token(); try { dt.decodeFromUrlString(tokenToRenew); long expirationTime = tokenManager.renewToken(dt, @@ -240,8 +240,7 @@ public abstract class DelegationTokenAuthenticationHandler ); requestContinues = false; } else { - Token dt = - new Token(); + Token dt = new Token(); try { dt.decodeFromUrlString(tokenToCancel); tokenManager.cancelToken(dt, (requestUgi != null) @@ -303,6 +302,7 @@ public abstract class DelegationTokenAuthenticationHandler * @throws IOException thrown if an IO error occurred. * @throws AuthenticationException thrown if the authentication failed. */ + @SuppressWarnings("unchecked") @Override public AuthenticationToken authenticate(HttpServletRequest request, HttpServletResponse response) @@ -311,8 +311,7 @@ public abstract class DelegationTokenAuthenticationHandler String delegationParam = getDelegationToken(request); if (delegationParam != null) { try { - Token dt = - new Token(); + Token dt = new Token(); dt.decodeFromUrlString(delegationParam); UserGroupInformation ugi = tokenManager.verifyToken(dt); final String shortName = ugi.getShortUserName(); diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/DelegationTokenManager.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/DelegationTokenManager.java index dbde0a29f08..5d862494843 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/DelegationTokenManager.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/DelegationTokenManager.java @@ -27,6 +27,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.io.Text; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.token.Token; +import org.apache.hadoop.security.token.delegation.AbstractDelegationTokenIdentifier; import org.apache.hadoop.security.token.delegation.AbstractDelegationTokenSecretManager; import org.apache.hadoop.security.token.delegation.ZKDelegationTokenSecretManager; @@ -76,6 +77,13 @@ public class DelegationTokenManager { public DelegationTokenIdentifier createIdentifier() { return new DelegationTokenIdentifier(tokenKind); } + + @Override + public DelegationTokenIdentifier decodeTokenIdentifier( + Token token) throws IOException { + return DelegationTokenManager.decodeToken(token, tokenKind); + } + } private static class ZKSecretManager @@ -92,11 +100,16 @@ public class DelegationTokenManager { public DelegationTokenIdentifier createIdentifier() { return new DelegationTokenIdentifier(tokenKind); } + + @Override + public DelegationTokenIdentifier decodeTokenIdentifier( + Token token) throws IOException { + return DelegationTokenManager.decodeToken(token, tokenKind); + } } private AbstractDelegationTokenSecretManager secretManager = null; private boolean managedSecretManager; - private Text tokenKind; public DelegationTokenManager(Configuration conf, Text tokenKind) { if (conf.getBoolean(ENABLE_ZK_KEY, false)) { @@ -104,7 +117,6 @@ public class DelegationTokenManager { } else { this.secretManager = new DelegationTokenSecretManager(conf, tokenKind); } - this.tokenKind = tokenKind; managedSecretManager = true; } @@ -121,7 +133,6 @@ public class DelegationTokenManager { AbstractDelegationTokenSecretManager secretManager) { this.secretManager.stopThreads(); this.secretManager = secretManager; - this.tokenKind = secretManager.createIdentifier().getKind(); managedSecretManager = false; } @@ -143,8 +154,8 @@ public class DelegationTokenManager { } @SuppressWarnings("unchecked") - public Token createToken(UserGroupInformation ugi, - String renewer) { + public Token createToken( + UserGroupInformation ugi, String renewer) { renewer = (renewer == null) ? ugi.getShortUserName() : renewer; String user = ugi.getUserName(); Text owner = new Text(user); @@ -152,19 +163,24 @@ public class DelegationTokenManager { if (ugi.getRealUser() != null) { realUser = new Text(ugi.getRealUser().getUserName()); } - DelegationTokenIdentifier tokenIdentifier = new DelegationTokenIdentifier( - tokenKind, owner, new Text(renewer), realUser); - return new Token(tokenIdentifier, secretManager); + AbstractDelegationTokenIdentifier tokenIdentifier = + (AbstractDelegationTokenIdentifier) secretManager.createIdentifier(); + tokenIdentifier.setOwner(owner); + tokenIdentifier.setRenewer(new Text(renewer)); + tokenIdentifier.setRealUser(realUser); + return new Token(tokenIdentifier, secretManager); } @SuppressWarnings("unchecked") - public long renewToken(Token token, String renewer) - throws IOException { + public long renewToken( + Token token, String renewer) + throws IOException { return secretManager.renewToken(token, renewer); } @SuppressWarnings("unchecked") - public void cancelToken(Token token, + public void cancelToken( + Token token, String canceler) throws IOException { canceler = (canceler != null) ? canceler : verifyToken(token).getShortUserName(); @@ -172,13 +188,10 @@ public class DelegationTokenManager { } @SuppressWarnings("unchecked") - public UserGroupInformation verifyToken(Token - token) throws IOException { - ByteArrayInputStream buf = new ByteArrayInputStream(token.getIdentifier()); - DataInputStream dis = new DataInputStream(buf); - DelegationTokenIdentifier id = new DelegationTokenIdentifier(tokenKind); - id.readFields(dis); - dis.close(); + public UserGroupInformation verifyToken( + Token token) + throws IOException { + AbstractDelegationTokenIdentifier id = secretManager.decodeTokenIdentifier(token); secretManager.verifyToken(id, token.getPassword()); return id.getUser(); } @@ -188,4 +201,15 @@ public class DelegationTokenManager { public AbstractDelegationTokenSecretManager getDelegationTokenSecretManager() { return secretManager; } + + private static DelegationTokenIdentifier decodeToken( + Token token, Text tokenKind) + throws IOException { + ByteArrayInputStream buf = new ByteArrayInputStream(token.getIdentifier()); + DataInputStream dis = new DataInputStream(buf); + DelegationTokenIdentifier id = new DelegationTokenIdentifier(tokenKind); + id.readFields(dis); + dis.close(); + return id; + } } diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/token/delegation/TestZKDelegationTokenSecretManager.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/token/delegation/TestZKDelegationTokenSecretManager.java index 076c87ae689..eece2de06b0 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/token/delegation/TestZKDelegationTokenSecretManager.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/token/delegation/TestZKDelegationTokenSecretManager.java @@ -32,6 +32,7 @@ public class TestZKDelegationTokenSecretManager { private static final long DAY_IN_SECS = 86400; + @SuppressWarnings("unchecked") @Test public void testZKDelTokSecretManager() throws Exception { TestingServer zkServer = new TestingServer(); @@ -54,11 +55,13 @@ public class TestZKDelegationTokenSecretManager { tm2.init(); Token token = - tm1.createToken(UserGroupInformation.getCurrentUser(), "foo"); + (Token) tm1.createToken( + UserGroupInformation.getCurrentUser(), "foo"); Assert.assertNotNull(token); tm2.verifyToken(token); - token = tm2.createToken(UserGroupInformation.getCurrentUser(), "bar"); + token = (Token) tm2.createToken( + UserGroupInformation.getCurrentUser(), "bar"); Assert.assertNotNull(token); tm1.verifyToken(token); } finally { diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/token/delegation/web/TestDelegationTokenAuthenticationHandlerWithMocks.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/token/delegation/web/TestDelegationTokenAuthenticationHandlerWithMocks.java index 889b0545001..600078964e2 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/token/delegation/web/TestDelegationTokenAuthenticationHandlerWithMocks.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/token/delegation/web/TestDelegationTokenAuthenticationHandlerWithMocks.java @@ -202,6 +202,7 @@ public class TestDelegationTokenAuthenticationHandlerWithMocks { Assert.assertEquals(expectedTokenKind, dt.getKind()); } + @SuppressWarnings("unchecked") private void testCancelToken() throws Exception { DelegationTokenAuthenticator.DelegationTokenOperation op = DelegationTokenAuthenticator.DelegationTokenOperation. @@ -220,7 +221,7 @@ public class TestDelegationTokenAuthenticationHandlerWithMocks { Mockito.reset(response); Token token = - handler.getTokenManager().createToken( + (Token) handler.getTokenManager().createToken( UserGroupInformation.getCurrentUser(), "foo"); Mockito.when(request.getQueryString()).thenReturn( DelegationTokenAuthenticator.OP_PARAM + "=" + op.toString() + "&" + @@ -239,6 +240,7 @@ public class TestDelegationTokenAuthenticationHandlerWithMocks { } } + @SuppressWarnings("unchecked") private void testRenewToken() throws Exception { DelegationTokenAuthenticator.DelegationTokenOperation op = DelegationTokenAuthenticator.DelegationTokenOperation. @@ -271,7 +273,7 @@ public class TestDelegationTokenAuthenticationHandlerWithMocks { PrintWriter pwriter = new PrintWriter(writer); Mockito.when(response.getWriter()).thenReturn(pwriter); Token dToken = - handler.getTokenManager().createToken( + (Token) handler.getTokenManager().createToken( UserGroupInformation.getCurrentUser(), "user"); Mockito.when(request.getQueryString()). thenReturn(DelegationTokenAuthenticator.OP_PARAM + "=" + op.toString() + @@ -292,11 +294,12 @@ public class TestDelegationTokenAuthenticationHandlerWithMocks { testInvalidDelegationTokenHeader(); } + @SuppressWarnings("unchecked") private void testValidDelegationTokenQueryString() throws Exception { HttpServletRequest request = Mockito.mock(HttpServletRequest.class); HttpServletResponse response = Mockito.mock(HttpServletResponse.class); Token dToken = - handler.getTokenManager().createToken( + (Token) handler.getTokenManager().createToken( UserGroupInformation.getCurrentUser(), "user"); Mockito.when(request.getQueryString()).thenReturn( DelegationTokenAuthenticator.DELEGATION_PARAM + "=" + @@ -311,11 +314,12 @@ public class TestDelegationTokenAuthenticationHandlerWithMocks { Assert.assertTrue(token.isExpired()); } + @SuppressWarnings("unchecked") private void testValidDelegationTokenHeader() throws Exception { HttpServletRequest request = Mockito.mock(HttpServletRequest.class); HttpServletResponse response = Mockito.mock(HttpServletResponse.class); Token dToken = - handler.getTokenManager().createToken( + (Token) handler.getTokenManager().createToken( UserGroupInformation.getCurrentUser(), "user"); Mockito.when(request.getHeader(Mockito.eq( DelegationTokenAuthenticator.DELEGATION_TOKEN_HEADER))).thenReturn( diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/token/delegation/web/TestDelegationTokenManager.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/token/delegation/web/TestDelegationTokenManager.java index 496b762bc09..38ca11860e4 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/token/delegation/web/TestDelegationTokenManager.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/token/delegation/web/TestDelegationTokenManager.java @@ -18,6 +18,8 @@ package org.apache.hadoop.security.token.delegation.web; import java.io.IOException; +import java.util.Arrays; +import java.util.Collection; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.io.Text; @@ -25,11 +27,26 @@ import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.token.Token; import org.junit.Assert; import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; +@RunWith(Parameterized.class) public class TestDelegationTokenManager { private static final long DAY_IN_SECS = 86400; + @Parameterized.Parameters + public static Collection headers() { + return Arrays.asList(new Object[][] { { false }, { true } }); + } + + private boolean enableZKKey; + + public TestDelegationTokenManager(boolean enableZKKey) { + this.enableZKKey = enableZKKey; + } + + @SuppressWarnings("unchecked") @Test public void testDTManager() throws Exception { Configuration conf = new Configuration(false); @@ -37,11 +54,13 @@ public class TestDelegationTokenManager { conf.setLong(DelegationTokenManager.MAX_LIFETIME, DAY_IN_SECS); conf.setLong(DelegationTokenManager.RENEW_INTERVAL, DAY_IN_SECS); conf.setLong(DelegationTokenManager.REMOVAL_SCAN_INTERVAL, DAY_IN_SECS); + conf.getBoolean(DelegationTokenManager.ENABLE_ZK_KEY, enableZKKey); DelegationTokenManager tm = new DelegationTokenManager(conf, new Text("foo")); tm.init(); Token token = - tm.createToken(UserGroupInformation.getCurrentUser(), "foo"); + (Token) tm.createToken( + UserGroupInformation.getCurrentUser(), "foo"); Assert.assertNotNull(token); tm.verifyToken(token); Assert.assertTrue(tm.renewToken(token, "foo") > System.currentTimeMillis()); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesDelegationTokens.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesDelegationTokens.java index 62612affeb4..56779bc678c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesDelegationTokens.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesDelegationTokens.java @@ -738,7 +738,8 @@ public class TestRMWebServicesDelegationTokens extends JerseyTest { Token realToken = new Token(); realToken.decodeFromUrlString(encodedToken); - RMDelegationTokenIdentifier ident = realToken.decodeIdentifier(); + RMDelegationTokenIdentifier ident = rm.getRMContext() + .getRMDelegationTokenSecretManager().decodeTokenIdentifier(realToken); rm.getRMContext().getRMDelegationTokenSecretManager() .verifyToken(ident, realToken.getPassword()); assertTrue(rm.getRMContext().getRMDelegationTokenSecretManager() @@ -749,7 +750,8 @@ public class TestRMWebServicesDelegationTokens extends JerseyTest { Token realToken = new Token(); realToken.decodeFromUrlString(encodedToken); - RMDelegationTokenIdentifier ident = realToken.decodeIdentifier(); + RMDelegationTokenIdentifier ident = rm.getRMContext() + .getRMDelegationTokenSecretManager().decodeTokenIdentifier(realToken); boolean exceptionCaught = false; try { rm.getRMContext().getRMDelegationTokenSecretManager()