From 3ff2148482794f41a2d4d09f92bf4135a8c4cb7e Mon Sep 17 00:00:00 2001 From: Jonathan Hung Date: Mon, 22 Jul 2019 10:46:45 -0700 Subject: [PATCH] YARN-9668. UGI conf doesn't read user overridden configurations on RM and NM startup. (Contributed by Jonanthan Hung) --- .../yarn/server/nodemanager/NodeManager.java | 1 + .../server/nodemanager/TestNodeManager.java | 28 +++++++++++++++++ .../resourcemanager/ResourceManager.java | 1 + .../server/resourcemanager/TestRMRestart.java | 5 ++-- .../resourcemanager/TestResourceManager.java | 30 ++++++++++++++++++- 5 files changed, 62 insertions(+), 3 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java index cd4171adbb0..da4fda21f10 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java @@ -350,6 +350,7 @@ public class NodeManager extends CompositeService @Override protected void serviceInit(Configuration conf) throws Exception { + UserGroupInformation.setConfiguration(conf); rmWorkPreservingRestartEnabled = conf.getBoolean(YarnConfiguration .RM_WORK_PRESERVING_RECOVERY_ENABLED, YarnConfiguration.DEFAULT_RM_WORK_PRESERVING_RECOVERY_ENABLED); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManager.java index b31215b0f3d..ece00868fd9 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManager.java @@ -23,6 +23,7 @@ import static org.junit.Assert.fail; import java.io.IOException; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.CommonConfigurationKeysPublic; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerEvent; @@ -30,7 +31,9 @@ import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Cont import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerState; import org.apache.hadoop.yarn.server.nodemanager.nodelabels.NodeLabelsProvider; import org.junit.Assert; +import org.junit.Rule; import org.junit.Test; +import org.junit.rules.ExpectedException; public class TestNodeManager { @@ -42,6 +45,9 @@ public class TestNodeManager { } } + @Rule + public ExpectedException thrown = ExpectedException.none(); + @Test public void testContainerExecutorInitCall() { NodeManager nm = new NodeManager(); @@ -170,4 +176,26 @@ public class TestNodeManager { e.printStackTrace(); } } + + /** + * Test whether NodeManager passes user-provided conf to + * UserGroupInformation class. If it reads this (incorrect) + * AuthenticationMethod enum an exception is thrown. + */ + @Test + public void testUserProvidedUGIConf() throws Exception { + thrown.expect(IllegalArgumentException.class); + thrown.expectMessage("Invalid attribute value for " + + CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION + + " of DUMMYAUTH"); + Configuration dummyConf = new YarnConfiguration(); + dummyConf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION, + "DUMMYAUTH"); + NodeManager dummyNodeManager = new NodeManager(); + try { + dummyNodeManager.init(dummyConf); + } finally { + dummyNodeManager.stop(); + } + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java index d1a4e34fb71..f54bd451c3c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java @@ -248,6 +248,7 @@ public class ResourceManager extends CompositeService @Override protected void serviceInit(Configuration conf) throws Exception { this.conf = conf; + UserGroupInformation.setConfiguration(conf); this.rmContext = new RMContextImpl(); rmContext.setResourceManager(this); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java index d5880ca9e4e..03dd04aa6c0 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java @@ -2098,9 +2098,10 @@ public class TestRMRestart extends ParameterizedSchedulerTestBase { conf.setInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, 2); conf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION, "kerberos"); + UserGroupInformation.setConfiguration(conf); // start RM - MockRM rm1 = createMockRM(conf); + MockRM rm1 = new TestSecurityMockRM(conf); rm1.start(); final MockNM nm1 = new MockNM("127.0.0.1:1234", 15120, rm1.getResourceTrackerService()); @@ -2108,7 +2109,7 @@ public class TestRMRestart extends ParameterizedSchedulerTestBase { RMApp app0 = rm1.submitApp(200); final MockAM am0 = MockRM.launchAndRegisterAM(app0, rm1, nm1); - MockRM rm2 = new MockRM(conf, rm1.getRMStateStore()) { + MockRM rm2 = new TestSecurityMockRM(conf, rm1.getRMStateStore()) { @Override protected ResourceTrackerService createResourceTrackerService() { return new ResourceTrackerService(this.rmContext, diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceManager.java index a66c583ccf7..254de9a6111 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceManager.java @@ -27,6 +27,7 @@ import java.util.concurrent.TimeoutException; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.fs.CommonConfigurationKeysPublic; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.http.lib.StaticUserWebFilter; import org.apache.hadoop.net.NetworkTopology; @@ -51,13 +52,18 @@ import org.apache.hadoop.yarn.util.resource.Resources; import org.junit.After; import org.junit.Assert; import org.junit.Before; +import org.junit.Rule; import org.junit.Test; +import org.junit.rules.ExpectedException; public class TestResourceManager { private static final Log LOG = LogFactory.getLog(TestResourceManager.class); private ResourceManager resourceManager = null; - + + @Rule + public ExpectedException thrown = ExpectedException.none(); + @Before public void setUp() throws Exception { Configuration conf = new YarnConfiguration(); @@ -328,4 +334,26 @@ public class TestResourceManager { } } + /** + * Test whether ResourceManager passes user-provided conf to + * UserGroupInformation class. If it reads this (incorrect) + * AuthenticationMethod enum an exception is thrown. + */ + @Test + public void testUserProvidedUGIConf() throws Exception { + thrown.expect(IllegalArgumentException.class); + thrown.expectMessage("Invalid attribute value for " + + CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION + + " of DUMMYAUTH"); + Configuration dummyConf = new YarnConfiguration(); + dummyConf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION, + "DUMMYAUTH"); + ResourceManager dummyResourceManager = new ResourceManager(); + try { + dummyResourceManager.init(dummyConf); + } finally { + dummyResourceManager.stop(); + } + } + }