diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index b07ecd77144..9a947f713cd 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -110,6 +110,10 @@ Release 2.4.0 - UNRELEASED YARN-1639. Modified RM HA configuration handling to have a way of not requiring separate configuration files for each RM. (Xuan Gong via vinodkv) + YARN-1668. Modified RM HA handling of admin-acls to be available across RM + failover by making using of a remote configuration-provider. (Xuan Gong via + vinodkv) + OPTIMIZATIONS BUG FIXES diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index a324e97b297..af385f81db2 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -40,8 +40,10 @@ public class YarnConfiguration extends Configuration { @Private public static final String CS_CONFIGURATION_FILE= "capacity-scheduler.xml"; + @Private + public static final String YARN_SITE_XML_FILE = "yarn-site.xml"; + private static final String YARN_DEFAULT_XML_FILE = "yarn-default.xml"; - private static final String YARN_SITE_XML_FILE = "yarn-site.xml"; static { Configuration.addDefaultResource(YARN_DEFAULT_XML_FILE); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java index c7fe0e2e6e9..9a33b706f90 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java @@ -74,6 +74,7 @@ import org.apache.hadoop.yarn.server.api.protocolrecords.UpdateNodeResourceRespo import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode; import org.apache.hadoop.yarn.server.resourcemanager.security.authorize.RMPolicyProvider; +import com.google.common.annotations.VisibleForTesting; import com.google.protobuf.BlockingService; public class AdminService extends CompositeService implements @@ -407,14 +408,22 @@ public class AdminService extends CompositeService implements @Override public RefreshAdminAclsResponse refreshAdminAcls( - RefreshAdminAclsRequest request) throws YarnException { - UserGroupInformation user = checkAcls("refreshAdminAcls"); + RefreshAdminAclsRequest request) throws YarnException, IOException { + String argName = "refreshAdminAcls"; + UserGroupInformation user = checkAcls(argName); - Configuration conf = new Configuration(); + if (!isRMActive()) { + RMAuditLogger.logFailure(user.getShortUserName(), argName, + adminAcl.toString(), "AdminService", + "ResourceManager is not active. Can not refresh user-groups."); + throwStandbyException(); + } + Configuration conf = + getConfiguration(YarnConfiguration.YARN_SITE_XML_FILE); adminAcl = new AccessControlList(conf.get( YarnConfiguration.YARN_ADMIN_ACL, YarnConfiguration.DEFAULT_YARN_ADMIN_ACL)); - RMAuditLogger.logSuccess(user.getShortUserName(), "refreshAdminAcls", + RMAuditLogger.logSuccess(user.getShortUserName(), argName, "AdminService"); return recordFactory.newRecordInstance(RefreshAdminAclsResponse.class); @@ -504,4 +513,9 @@ public class AdminService extends CompositeService implements throws YarnException, IOException { return this.configurationProvider.getConfiguration(confFileName); } + + @VisibleForTesting + public AccessControlList getAccessControlList() { + return this.adminAcl; + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAdminService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAdminService.java index d800f5067d9..4b7018528fe 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAdminService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAdminService.java @@ -31,6 +31,7 @@ import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.YarnException; +import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshAdminAclsRequest; import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshQueuesRequest; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration; @@ -50,6 +51,7 @@ public class TestRMAdminService { @Before public void setup() throws IOException { + Configuration.addDefaultResource(YarnConfiguration.CS_CONFIGURATION_FILE); fs = FileSystem.get(configuration); workingPath = new Path(new File("target", this.getClass().getSimpleName() @@ -72,6 +74,7 @@ public class TestRMAdminService { fs.delete(workingPath, true); fs.delete(tmpDir, true); } + @Test public void testAdminRefreshQueuesWithLocalConfigurationProvider() throws IOException, YarnException { @@ -95,7 +98,6 @@ public class TestRMAdminService { @Test public void testAdminRefreshQueuesWithFileSystemBasedConfigurationProvider() throws IOException, YarnException { - Configuration.addDefaultResource(YarnConfiguration.CS_CONFIGURATION_FILE); configuration.set(YarnConfiguration.RM_CONFIGURATION_PROVIDER_CLASS, "org.apache.hadoop.yarn.FileSystemBasedConfigurationProvider"); rm = new MockRM(configuration); @@ -134,6 +136,58 @@ public class TestRMAdminService { Assert.assertTrue(maxAppsAfter != maxAppsBefore); } + @Test + public void testAdminAclsWithLocalConfigurationProvider() { + rm = new MockRM(configuration); + rm.init(configuration); + rm.start(); + + try { + rm.adminService.refreshAdminAcls(RefreshAdminAclsRequest.newInstance()); + } catch (Exception ex) { + fail("Using localConfigurationProvider. Should not get any exception."); + } + } + + @Test + public void testAdminAclsWithFileSystemBasedConfigurationProvider() + throws IOException, YarnException { + configuration.set(YarnConfiguration.RM_CONFIGURATION_PROVIDER_CLASS, + "org.apache.hadoop.yarn.FileSystemBasedConfigurationProvider"); + rm = new MockRM(configuration); + rm.init(configuration); + rm.start(); + + // clean the remoteDirectory + cleanRemoteDirectory(); + + try { + rm.adminService.refreshAdminAcls(RefreshAdminAclsRequest.newInstance()); + fail("FileSystemBasedConfigurationProvider is used." + + " Should get an exception here"); + } catch (Exception ex) { + Assert.assertTrue(ex.getMessage().contains( + "Can not find Configuration: yarn-site.xml")); + } + + String aclStringBefore = + rm.adminService.getAccessControlList().getAclString().trim(); + + YarnConfiguration yarnConf = new YarnConfiguration(); + yarnConf.set(YarnConfiguration.YARN_ADMIN_ACL, "world:anyone:rwcda"); + String yarnConfFile = writeConfigurationXML(yarnConf, "yarn-site.xml"); + + // upload the file into Remote File System + uploadToRemoteFileSystem(new Path(yarnConfFile)); + rm.adminService.refreshAdminAcls(RefreshAdminAclsRequest.newInstance()); + + String aclStringAfter = + rm.adminService.getAccessControlList().getAclString().trim(); + + Assert.assertTrue(!aclStringAfter.equals(aclStringBefore)); + Assert.assertEquals(aclStringAfter, "world:anyone:rwcda"); + } + private String writeConfigurationXML(Configuration conf, String confXMLName) throws IOException { DataOutputStream output = null;