diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 6a0d38ff6a3..b638ab058a4 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -622,6 +622,9 @@ Release 2.6.0 - UNRELEASED test-failures: added a dummy in-memory labels-manager. (Wangda Tan via vinodkv) + YARN-2715. Fixed ResourceManager to respect common configurations for proxy + users/groups beyond just the YARN level config. (Zhijie Shen via vinodkv) + BREAKDOWN OF YARN-1051 SUBTASKS AND RELATED JIRAS YARN-1707. Introduce APIs to add/remove/resize queues in the diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index 992d3ea7fc5..59e303ffa7a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -31,6 +31,7 @@ import org.apache.hadoop.classification.InterfaceStability.Unstable; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.http.HttpConfig; import org.apache.hadoop.net.NetUtils; +import org.apache.hadoop.security.authorize.ProxyUsers; import org.apache.hadoop.util.StringUtils; import org.apache.hadoop.yarn.api.ApplicationConstants; @@ -560,6 +561,11 @@ public class YarnConfiguration extends Configuration { public static final String DEFAULT_RM_NODEMANAGER_MINIMUM_VERSION = "NONE"; + /** + * RM proxy users' prefix + */ + public static final String RM_PROXY_USER_PREFIX = RM_PREFIX + "proxyuser."; + //////////////////////////////// // Node Manager Configs //////////////////////////////// diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/security/http/RMAuthenticationFilterInitializer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/security/http/RMAuthenticationFilterInitializer.java index f0baf2be34e..a62cda39a30 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/security/http/RMAuthenticationFilterInitializer.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/security/http/RMAuthenticationFilterInitializer.java @@ -35,6 +35,7 @@ import org.apache.hadoop.security.SecurityUtil; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.authentication.server.AuthenticationFilter; import org.apache.hadoop.security.authentication.server.KerberosAuthenticationHandler; +import org.apache.hadoop.security.authorize.ProxyUsers; import org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticationHandler; import org.apache.hadoop.yarn.security.client.RMDelegationTokenIdentifier; @@ -42,14 +43,12 @@ import org.apache.hadoop.yarn.security.client.RMDelegationTokenIdentifier; public class RMAuthenticationFilterInitializer extends FilterInitializer { String configPrefix; - String proxyPrefix; String signatureSecretFileProperty; String kerberosPrincipalProperty; String cookiePath; public RMAuthenticationFilterInitializer() { this.configPrefix = "hadoop.http.authentication."; - this.proxyPrefix = "yarn.resourcemanager.webapp.proxyuser."; this.signatureSecretFileProperty = AuthenticationFilter.SIGNATURE_SECRET + ".file"; this.kerberosPrincipalProperty = KerberosAuthenticationHandler.PRINCIPAL; @@ -62,15 +61,18 @@ public class RMAuthenticationFilterInitializer extends FilterInitializer { // setting the cookie path to root '/' so it is used for all resources. filterConfig.put(AuthenticationFilter.COOKIE_PATH, cookiePath); + // Before conf object is passed in, RM has already processed it and used RM + // specific configs to overwrite hadoop common ones. Hence we just need to + // source hadoop.proxyuser configs here. for (Map.Entry entry : conf) { String propName = entry.getKey(); if (propName.startsWith(configPrefix)) { String value = conf.get(propName); String name = propName.substring(configPrefix.length()); filterConfig.put(name, value); - } else if (propName.startsWith(proxyPrefix)) { + } else if (propName.startsWith(ProxyUsers.CONF_HADOOP_PROXYUSER)) { String value = conf.get(propName); - String name = propName.substring("yarn.resourcemanager.webapp.".length()); + String name = propName.substring("hadoop.".length()); filterConfig.put(name, value); } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java index ecdcb209254..7ec3e796772 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java @@ -421,9 +421,13 @@ public class AdminService extends CompositeService implements throwStandbyException(); } + // Accept hadoop common configs in core-site.xml as well as RM specific + // configurations in yarn-site.xml Configuration conf = getConfiguration(new Configuration(false), - YarnConfiguration.CORE_SITE_CONFIGURATION_FILE); + YarnConfiguration.CORE_SITE_CONFIGURATION_FILE, + YarnConfiguration.YARN_SITE_CONFIGURATION_FILE); + RMServerUtils.processRMProxyUsersConf(conf); ProxyUsers.refreshSuperUserGroupsConfiguration(conf); RMAuditLogger.logSuccess(user.getShortUserName(), argName, "AdminService"); @@ -594,11 +598,13 @@ public class AdminService extends CompositeService implements } private synchronized Configuration getConfiguration(Configuration conf, - String confFileName) throws YarnException, IOException { - InputStream confFileInputStream = this.rmContext.getConfigurationProvider() - .getConfigurationInputStream(conf, confFileName); - if (confFileInputStream != null) { - conf.addResource(confFileInputStream); + String... confFileNames) throws YarnException, IOException { + for (String confFileName : confFileNames) { + InputStream confFileInputStream = this.rmContext.getConfigurationProvider() + .getConfigurationInputStream(conf, confFileName); + if (confFileInputStream != null) { + conf.addResource(confFileInputStream); + } } return conf; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java index 40d86e51bf0..c80778cf0a2 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java @@ -21,12 +21,16 @@ package org.apache.hadoop.yarn.server.resourcemanager; import java.io.IOException; import java.util.ArrayList; import java.util.EnumSet; +import java.util.HashMap; import java.util.List; +import java.util.Map; import org.apache.commons.logging.Log; +import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.security.AccessControlException; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.authorize.AccessControlList; +import org.apache.hadoop.security.authorize.ProxyUsers; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationResourceUsageReport; import org.apache.hadoop.yarn.api.records.ContainerId; @@ -36,6 +40,7 @@ import org.apache.hadoop.yarn.api.records.ResourceBlacklistRequest; import org.apache.hadoop.yarn.api.records.ResourceRequest; import org.apache.hadoop.yarn.api.records.YarnApplicationAttemptState; import org.apache.hadoop.yarn.api.records.YarnApplicationState; +import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.InvalidContainerReleaseException; import org.apache.hadoop.yarn.exceptions.InvalidResourceBlacklistRequestException; import org.apache.hadoop.yarn.exceptions.InvalidResourceRequestException; @@ -248,4 +253,26 @@ public class RMServerUtils { BuilderUtils.newApplicationResourceUsageReport(-1, -1, Resources.createResource(-1, -1), Resources.createResource(-1, -1), Resources.createResource(-1, -1), 0, 0); + + + + /** + * Find all configs whose name starts with + * YarnConfiguration.RM_PROXY_USER_PREFIX, and add a record for each one by + * replacing the prefix with ProxyUsers.CONF_HADOOP_PROXYUSER + */ + public static void processRMProxyUsersConf(Configuration conf) { + Map rmProxyUsers = new HashMap(); + for (Map.Entry entry : conf) { + String propName = entry.getKey(); + if (propName.startsWith(YarnConfiguration.RM_PROXY_USER_PREFIX)) { + rmProxyUsers.put(ProxyUsers.CONF_HADOOP_PROXYUSER + "." + + propName.substring(YarnConfiguration.RM_PROXY_USER_PREFIX.length()), + entry.getValue()); + } + } + for (Map.Entry entry : rmProxyUsers.entrySet()) { + conf.set(entry.getKey(), entry.getValue()); + } + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java index 51ed2b1a9b9..57c8fce1dbc 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java @@ -23,7 +23,9 @@ import java.io.InputStream; import java.net.InetSocketAddress; import java.security.PrivilegedExceptionAction; import java.util.ArrayList; +import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.concurrent.BlockingQueue; import java.util.concurrent.LinkedBlockingQueue; @@ -212,6 +214,9 @@ public class ResourceManager extends CompositeService implements Recoverable { .refresh(); // Do refreshSuperUserGroupsConfiguration with loaded core-site.xml + // Or use RM specific configurations to overwrite the common ones first + // if they exist + RMServerUtils.processRMProxyUsersConf(conf); ProxyUsers.refreshSuperUserGroupsConfiguration(this.conf); // load yarn-site.xml diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAdminService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAdminService.java index 7161d0c4dbe..1bd3dbfd447 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAdminService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAdminService.java @@ -357,6 +357,24 @@ public class TestRMAdminService { .get("hadoop.proxyuser.test.hosts").size() == 1); Assert.assertTrue(ProxyUsers.getDefaultImpersonationProvider().getProxyHosts() .get("hadoop.proxyuser.test.hosts").contains("test_hosts")); + + Configuration yarnConf = new Configuration(false); + yarnConf.set("yarn.resourcemanager.proxyuser.test.groups", "test_groups_1"); + yarnConf.set("yarn.resourcemanager.proxyuser.test.hosts", "test_hosts_1"); + uploadConfiguration(yarnConf, "yarn-site.xml"); + + // RM specific configs will overwrite the common ones + rm.adminService.refreshSuperUserGroupsConfiguration( + RefreshSuperUserGroupsConfigurationRequest.newInstance()); + Assert.assertTrue(ProxyUsers.getDefaultImpersonationProvider().getProxyGroups() + .get("hadoop.proxyuser.test.groups").size() == 1); + Assert.assertTrue(ProxyUsers.getDefaultImpersonationProvider().getProxyGroups() + .get("hadoop.proxyuser.test.groups").contains("test_groups_1")); + + Assert.assertTrue(ProxyUsers.getDefaultImpersonationProvider().getProxyHosts() + .get("hadoop.proxyuser.test.hosts").size() == 1); + Assert.assertTrue(ProxyUsers.getDefaultImpersonationProvider().getProxyHosts() + .get("hadoop.proxyuser.test.hosts").contains("test_hosts_1")); } @Test diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMProxyUsersConf.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMProxyUsersConf.java new file mode 100644 index 00000000000..86a9fe48f8e --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMProxyUsersConf.java @@ -0,0 +1,105 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager; + +import java.util.Arrays; +import java.util.Collection; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.security.authorize.AuthorizationException; +import org.apache.hadoop.security.authorize.ProxyUsers; +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.junit.Assert; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; + +@RunWith(Parameterized.class) +public class TestRMProxyUsersConf { + + private static final UserGroupInformation FOO_USER = + UserGroupInformation.createUserForTesting("foo", new String[] { "foo_group" }); + private static final UserGroupInformation BAR_USER = + UserGroupInformation.createUserForTesting("bar", new String[] { "bar_group" }); + + @Parameterized.Parameters + public static Collection headers() { + return Arrays.asList(new Object[][] { { 0 }, { 1 }, { 2 } }); + } + + private Configuration conf; + + public TestRMProxyUsersConf(int round) { + conf = new YarnConfiguration(); + switch (round) { + case 0: + // hadoop.proxyuser prefix + conf.set("hadoop.proxyuser.foo.hosts", "localhost"); + conf.set("hadoop.proxyuser.foo.users", "bar"); + conf.set("hadoop.proxyuser.foo.groups", "bar_group"); + break; + case 1: + // yarn.resourcemanager.proxyuser prefix + conf.set("yarn.resourcemanager.proxyuser.foo.hosts", "localhost"); + conf.set("yarn.resourcemanager.proxyuser.foo.users", "bar"); + conf.set("yarn.resourcemanager.proxyuser.foo.groups", "bar_group"); + break; + case 2: + // hadoop.proxyuser prefix has been overwritten by + // yarn.resourcemanager.proxyuser prefix + conf.set("hadoop.proxyuser.foo.hosts", "xyz"); + conf.set("hadoop.proxyuser.foo.users", "xyz"); + conf.set("hadoop.proxyuser.foo.groups", "xyz"); + conf.set("yarn.resourcemanager.proxyuser.foo.hosts", "localhost"); + conf.set("yarn.resourcemanager.proxyuser.foo.users", "bar"); + conf.set("yarn.resourcemanager.proxyuser.foo.groups", "bar_group"); + break; + default: + break; + } + } + + @Test + public void testProxyUserConfiguration() throws Exception { + MockRM rm = null; + try { + rm = new MockRM(conf); + rm.start(); + // wait for web server starting + Thread.sleep(10000); + UserGroupInformation proxyUser = + UserGroupInformation.createProxyUser( + BAR_USER.getShortUserName(), FOO_USER); + try { + ProxyUsers.getDefaultImpersonationProvider().authorize(proxyUser, + "localhost"); + } catch (AuthorizationException e) { + // Exception is not expected + Assert.fail(); + } + } finally { + if (rm != null) { + rm.stop(); + rm.close(); + } + } + } + +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesDelegationTokenAuthentication.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesDelegationTokenAuthentication.java index a9ae0b933a2..36e24ecf189 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesDelegationTokenAuthentication.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesDelegationTokenAuthentication.java @@ -158,8 +158,8 @@ public class TestRMWebServicesDelegationTokenAuthentication { rmconf.set(YarnConfiguration.NM_WEBAPP_SPNEGO_KEYTAB_FILE_KEY, httpSpnegoKeytabFile.getAbsolutePath()); rmconf.setBoolean("mockrm.webapp.enabled", true); - rmconf.set("yarn.resourcemanager.webapp.proxyuser.client.hosts", "*"); - rmconf.set("yarn.resourcemanager.webapp.proxyuser.client.groups", "*"); + rmconf.set("yarn.resourcemanager.proxyuser.client.hosts", "*"); + rmconf.set("yarn.resourcemanager.proxyuser.client.groups", "*"); UserGroupInformation.setConfiguration(rmconf); rm = new MockRM(rmconf); rm.start();