diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index b0fc7c12bd7..92515c6c370 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -765,6 +765,12 @@ Release 2.1.0-beta - 2013-07-02 YARN-245. Fixed NodeManager to handle duplicate responses from ResourceManager. (Mayank Bansal via vinodkv) + YARN-932. TestResourceLocalizationService.testLocalizationInit can fail on + JDK7. (Karthik Kambatla via Sandy Ryza) + + YARN-961. Changed ContainerManager to enforce Token auth irrespective of + security. (Omkar Vinit Joshi via vinodkv) + BREAKDOWN OF HADOOP-8562/YARN-191 SUBTASKS AND RELATED JIRAS YARN-158. Yarn creating package-info.java must not depend on sh. @@ -830,9 +836,6 @@ Release 2.1.0-beta - 2013-07-02 YARN-909. Disable TestLinuxContainerExecutorWithMocks on Windows. (Chuan Liu via cnauroth) - YARN-932. TestResourceLocalizationService.testLocalizationInit can fail on - JDK7. (Karthik Kambatla via Sandy Ryza) - Release 2.0.5-alpha - 06/06/2013 INCOMPATIBLE CHANGES diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java index d2e75103d32..26998c938ca 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java @@ -40,6 +40,7 @@ import org.apache.hadoop.io.DataInputByteBuffer; import org.apache.hadoop.ipc.Server; import org.apache.hadoop.net.NetUtils; import org.apache.hadoop.security.Credentials; +import org.apache.hadoop.security.SaslRpcServer; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.authorize.PolicyProvider; import org.apache.hadoop.security.token.SecretManager.InvalidToken; @@ -230,6 +231,13 @@ public class ContainerManagerImpl extends CompositeService implements // Enqueue user dirs in deletion context Configuration conf = getConfig(); + Configuration serverConf = new Configuration(conf); + + // always enforce it to be token-based. + serverConf.set( + CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION, + SaslRpcServer.AuthMethod.TOKEN.toString()); + YarnRPC rpc = YarnRPC.create(conf); InetSocketAddress initialAddress = conf.getSocketAddr( @@ -238,8 +246,8 @@ public class ContainerManagerImpl extends CompositeService implements YarnConfiguration.DEFAULT_NM_PORT); server = - rpc.getServer(ContainerManagementProtocol.class, this, initialAddress, conf, - this.context.getNMTokenSecretManager(), + rpc.getServer(ContainerManagementProtocol.class, this, initialAddress, + serverConf, this.context.getNMTokenSecretManager(), conf.getInt(YarnConfiguration.NM_CONTAINER_MGR_THREAD_COUNT, YarnConfiguration.DEFAULT_NM_CONTAINER_MGR_THREAD_COUNT)); @@ -249,7 +257,7 @@ public class ContainerManagerImpl extends CompositeService implements false)) { refreshServiceAcls(conf, new NMPolicyProvider()); } - + LOG.info("Blocking new container-requests as container manager rpc" + " server is still starting."); this.setBlockNewContainerRequests(true); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java index 9dfdea8100e..14456f535a9 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java @@ -33,6 +33,7 @@ import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.CommonConfigurationKeysPublic; import org.apache.hadoop.ipc.Server; +import org.apache.hadoop.security.SaslRpcServer; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.authorize.PolicyProvider; import org.apache.hadoop.security.token.TokenIdentifier; @@ -119,12 +120,11 @@ public class ApplicationMasterService extends AbstractService implements YarnConfiguration.DEFAULT_RM_SCHEDULER_PORT); Configuration serverConf = conf; - if (!UserGroupInformation.isSecurityEnabled()) { - // If the auth is not-simple, enforce it to be token-based. - serverConf = new Configuration(conf); - serverConf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION, - UserGroupInformation.AuthenticationMethod.TOKEN.toString()); - } + // If the auth is not-simple, enforce it to be token-based. + serverConf = new Configuration(conf); + serverConf.set( + CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION, + SaslRpcServer.AuthMethod.TOKEN.toString()); this.server = rpc.getServer(ApplicationMasterProtocol.class, this, masterServiceAddress, serverConf, this.rmContext.getAMRMTokenSecretManager(), diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAMAuthorization.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAMAuthorization.java index af07fb87bb8..14acd01d187 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAMAuthorization.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAMAuthorization.java @@ -253,15 +253,14 @@ public class TestAMAuthorization { } catch (Exception e) { // Because there are no tokens, the request should be rejected as the // server side will assume we are trying simple auth. - String availableAuthMethods; + String expectedMessage = ""; if (UserGroupInformation.isSecurityEnabled()) { - availableAuthMethods = "[TOKEN, KERBEROS]"; + expectedMessage = "Client cannot authenticate via:[TOKEN]"; } else { - availableAuthMethods = "[TOKEN]"; + expectedMessage = + "SIMPLE authentication is not enabled. Available:[TOKEN]"; } - Assert.assertTrue(e.getCause().getMessage().contains( - "SIMPLE authentication is not enabled. " - + "Available:" + availableAuthMethods)); + Assert.assertTrue(e.getCause().getMessage().contains(expectedMessage)); } // TODO: Add validation of invalid authorization when there's more data in diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestContainerManagerSecurity.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestContainerManagerSecurity.java index 743bf8afeab..1657f1655d5 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestContainerManagerSecurity.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestContainerManagerSecurity.java @@ -24,6 +24,8 @@ import java.io.IOException; import java.net.InetSocketAddress; import java.security.PrivilegedAction; import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; import java.util.List; import junit.framework.Assert; @@ -63,7 +65,11 @@ import org.apache.hadoop.yarn.server.resourcemanager.security.RMContainerTokenSe import org.apache.hadoop.yarn.util.ConverterUtils; import org.apache.hadoop.yarn.util.Records; import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; +import org.junit.runners.Parameterized.Parameters; +@RunWith(Parameterized.class) public class TestContainerManagerSecurity { static Log LOG = LogFactory.getLog(TestContainerManagerSecurity.class); @@ -71,28 +77,33 @@ public class TestContainerManagerSecurity { .getRecordFactory(null); private static MiniYARNCluster yarnCluster; - static final Configuration conf = new Configuration(); + private Configuration conf; - @Test (timeout = 1000000) - public void testContainerManagerWithSecurityEnabled() throws Exception { - conf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION, + @Parameters + public static Collection configs() { + Configuration configurationWithoutSecurity = new Configuration(); + configurationWithoutSecurity.set( + CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION, "simple"); + + Configuration configurationWithSecurity = new Configuration(); + configurationWithSecurity.set( + CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION, "kerberos"); - testContainerManager(); + return Arrays.asList(new Object[][] { { configurationWithoutSecurity }, + { configurationWithSecurity } }); } - @Test (timeout=1000000) - public void testContainerManagerWithSecurityDisabled() throws Exception { - conf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION, - "simple"); - testContainerManager(); + public TestContainerManagerSecurity(Configuration conf) { + conf.setLong(YarnConfiguration.RM_AM_EXPIRY_INTERVAL_MS, 100000L); + UserGroupInformation.setConfiguration(conf); + this.conf = conf; } - private void testContainerManager() throws Exception { + @Test (timeout = 1000000) + public void testContainerManager() throws Exception { try { yarnCluster = new MiniYARNCluster(TestContainerManagerSecurity.class .getName(), 1, 1, 1); - conf.setLong(YarnConfiguration.RM_AM_EXPIRY_INTERVAL_MS, 100000L); - UserGroupInformation.setConfiguration(conf); yarnCluster.init(conf); yarnCluster.start(); @@ -184,6 +195,18 @@ public class TestContainerManagerSecurity { } while (tempManager.getCurrentKey().getKeyId() == nmTokenSecretManagerRM .getCurrentKey().getKeyId()); + // Testing that NM rejects the requests when we don't send any token. + if (UserGroupInformation.isSecurityEnabled()) { + sb = new StringBuilder("Client cannot authenticate via:[TOKEN]"); + } else { + sb = + new StringBuilder( + "SIMPLE authentication is not enabled. Available:[TOKEN]"); + } + String errorMsg = testStartContainer(rpc, validAppAttemptId, validNode, + validContainerToken, null, true); + Assert.assertTrue(errorMsg.contains(sb.toString())); + org.apache.hadoop.yarn.api.records.Token invalidNMToken = tempManager.createNMToken(validAppAttemptId, validNode, user); sb = new StringBuilder("Given NMToken for application : "); @@ -402,7 +425,9 @@ public class TestContainerManagerSecurity { UserGroupInformation ugi = UserGroupInformation.createRemoteUser(user); final InetSocketAddress addr = NetUtils.createSocketAddr(nodeId.getHost(), nodeId.getPort()); - ugi.addToken(ConverterUtils.convertFromYarn(nmToken, addr)); + if (nmToken != null) { + ugi.addToken(ConverterUtils.convertFromYarn(nmToken, addr)); + } proxy = ugi .doAs(new PrivilegedAction() {