From 0b546420da3c7ae708d361ddf52a3cf8d24494b9 Mon Sep 17 00:00:00 2001 From: Xuan Date: Tue, 1 Aug 2017 08:48:04 -0700 Subject: [PATCH] YARN-6322: Disable queue refresh when configuration mutation is enabled. Contributed by Jonathan Hung --- .../server/resourcemanager/AdminService.java | 12 +++++++++- .../scheduler/MutableConfScheduler.java | 12 ++++++++-- .../MutableConfigurationProvider.java | 4 +++- .../scheduler/capacity/CapacityScheduler.java | 9 ++++++-- .../conf/MutableCSConfigurationProvider.java | 11 ++++----- .../resourcemanager/TestRMAdminService.java | 23 +++++++++++++++++++ .../TestMutableCSConfigurationProvider.java | 14 ++++++----- 7 files changed, 67 insertions(+), 18 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java index 3457ae329bf..fd9e849f3c4 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java @@ -29,6 +29,7 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.CommonConfigurationKeysPublic; import org.apache.hadoop.ha.HAServiceProtocol; @@ -92,6 +93,8 @@ import org.apache.hadoop.yarn.server.resourcemanager.resource.DynamicResourceConfiguration; import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode; import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeResourceUpdateEvent; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.MutableConfScheduler; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler; import org.apache.hadoop.yarn.server.resourcemanager.security.authorize.RMPolicyProvider; import com.google.common.annotations.VisibleForTesting; @@ -384,6 +387,12 @@ public RefreshQueuesResponse refreshQueues(RefreshQueuesRequest request) RefreshQueuesResponse response = recordFactory.newRecordInstance(RefreshQueuesResponse.class); try { + ResourceScheduler scheduler = rm.getRMContext().getScheduler(); + if (scheduler instanceof MutableConfScheduler + && ((MutableConfScheduler) scheduler).isConfigurationMutable()) { + throw new IOException("Scheduler configuration is mutable. " + + operation + " is not allowed in this scenario."); + } refreshQueues(); RMAuditLogger.logSuccess(user.getShortUserName(), operation, "AdminService"); @@ -393,7 +402,8 @@ public RefreshQueuesResponse refreshQueues(RefreshQueuesRequest request) } } - private void refreshQueues() throws IOException, YarnException { + @Private + public void refreshQueues() throws IOException, YarnException { rm.getRMContext().getScheduler().reinitialize(getConfig(), this.rm.getRMContext()); // refresh the reservation system diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfScheduler.java index 007dc29af7d..313bf6aee1c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfScheduler.java @@ -19,6 +19,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.webapp.dao.SchedConfUpdateInfo; import java.io.IOException; @@ -33,10 +34,11 @@ public interface MutableConfScheduler extends ResourceScheduler { * Update the scheduler's configuration. * @param user Caller of this update * @param confUpdate configuration update - * @throws IOException if update is invalid + * @throws IOException if scheduler could not be reinitialized + * @throws YarnException if reservation system could not be reinitialized */ void updateConfiguration(UserGroupInformation user, - SchedConfUpdateInfo confUpdate) throws IOException; + SchedConfUpdateInfo confUpdate) throws IOException, YarnException; /** * Get the scheduler configuration. @@ -50,4 +52,10 @@ void updateConfiguration(UserGroupInformation user, * @return the queue object */ Queue getQueue(String queueName); + + /** + * Return whether the scheduler configuration is mutable. + * @return whether scheduler configuration is mutable or not. + */ + boolean isConfigurationMutable(); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfigurationProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfigurationProvider.java index 1f134677da0..9baf1ad1ee2 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfigurationProvider.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfigurationProvider.java @@ -19,6 +19,7 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler; import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.webapp.dao.SchedConfUpdateInfo; import java.io.IOException; @@ -39,8 +40,9 @@ public interface MutableConfigurationProvider { * @param user User issuing the request * @param confUpdate Key-value pairs for configurations to be updated. * @throws IOException if scheduler could not be reinitialized + * @throws YarnException if reservation system could not be reinitialized */ void mutateConfiguration(UserGroupInformation user, SchedConfUpdateInfo - confUpdate) throws IOException; + confUpdate) throws IOException, YarnException; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java index d38f4b4ef2d..f46c41f5d93 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java @@ -2620,8 +2620,8 @@ public long getMaximumApplicationLifetime(String queueName) { @Override public void updateConfiguration(UserGroupInformation user, - SchedConfUpdateInfo confUpdate) throws IOException { - if (csConfProvider instanceof MutableConfigurationProvider) { + SchedConfUpdateInfo confUpdate) throws IOException, YarnException { + if (isConfigurationMutable()) { ((MutableConfigurationProvider) csConfProvider).mutateConfiguration( user, confUpdate); } else { @@ -2629,4 +2629,9 @@ public void updateConfiguration(UserGroupInformation user, "provider does not support updating configuration."); } } + + @Override + public boolean isConfigurationMutable() { + return csConfProvider instanceof MutableConfigurationProvider; + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/MutableCSConfigurationProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/MutableCSConfigurationProvider.java index 9ccc146f19a..d03b2e23d27 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/MutableCSConfigurationProvider.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/MutableCSConfigurationProvider.java @@ -25,6 +25,7 @@ import org.apache.hadoop.security.AccessControlException; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.server.resourcemanager.RMContext; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ConfigurationMutationACLPolicy; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ConfigurationMutationACLPolicyFactory; @@ -58,7 +59,6 @@ public class MutableCSConfigurationProvider implements CSConfigurationProvider, private YarnConfigurationStore confStore; private ConfigurationMutationACLPolicy aclMutationPolicy; private RMContext rmContext; - private Configuration conf; public MutableCSConfigurationProvider(RMContext rmContext) { this.rmContext = rmContext; @@ -96,7 +96,6 @@ public void init(Configuration config) throws IOException { this.aclMutationPolicy = ConfigurationMutationACLPolicyFactory .getPolicy(config); aclMutationPolicy.init(config, rmContext); - this.conf = config; } @Override @@ -109,7 +108,7 @@ public CapacitySchedulerConfiguration loadConfiguration(Configuration @Override public synchronized void mutateConfiguration(UserGroupInformation user, - SchedConfUpdateInfo confUpdate) throws IOException { + SchedConfUpdateInfo confUpdate) throws IOException, YarnException { if (!aclMutationPolicy.isMutationAllowed(user, confUpdate)) { throw new AccessControlException("User is not admin of all modified" + " queues."); @@ -126,8 +125,8 @@ public synchronized void mutateConfiguration(UserGroupInformation user, } } try { - rmContext.getScheduler().reinitialize(conf, rmContext); - } catch (IOException e) { + rmContext.getRMAdminService().refreshQueues(); + } catch (IOException | YarnException e) { schedConf = oldConf; confStore.confirmMutation(id, false); throw e; @@ -148,7 +147,7 @@ public void recoverConf() throws IOException { } } try { - rmContext.getScheduler().reinitialize(conf, rmContext); + rmContext.getScheduler().reinitialize(schedConf, rmContext); } catch (IOException e) { schedConf = oldConf; confStore.confirmMutation(mutation.getId(), false); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAdminService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAdminService.java index 588f16deefc..620ca58a77c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAdminService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAdminService.java @@ -197,6 +197,29 @@ public void testAdminRefreshQueuesWithFileSystemBasedConfigurationProvider() Assert.assertTrue(maxAppsAfter != maxAppsBefore); } + @Test + public void testAdminRefreshQueuesWithMutableSchedulerConfiguration() { + configuration.set(CapacitySchedulerConfiguration.CS_CONF_PROVIDER, + CapacitySchedulerConfiguration.STORE_CS_CONF_PROVIDER); + + try { + rm = new MockRM(configuration); + rm.init(configuration); + rm.start(); + } catch (Exception ex) { + fail("Should not get any exceptions"); + } + + try { + rm.adminService.refreshQueues(RefreshQueuesRequest.newInstance()); + fail("Expected exception while calling refreshQueues when scheduler" + + " configuration is mutable."); + } catch (Exception ex) { + assertTrue(ex.getMessage().endsWith("Scheduler configuration is " + + "mutable. refreshQueues is not allowed in this scenario.")); + } + } + @Test public void testAdminRefreshNodesWithoutConfiguration() throws IOException, YarnException { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestMutableCSConfigurationProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestMutableCSConfigurationProvider.java index 9104f1635e0..635a184e8ff 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestMutableCSConfigurationProvider.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestMutableCSConfigurationProvider.java @@ -20,6 +20,8 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.yarn.exceptions.YarnException; +import org.apache.hadoop.yarn.server.resourcemanager.AdminService; import org.apache.hadoop.yarn.server.resourcemanager.RMContext; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration; @@ -34,7 +36,6 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNull; -import static org.mockito.Matchers.any; import static org.mockito.Mockito.doNothing; import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.mock; @@ -50,6 +51,7 @@ public class TestMutableCSConfigurationProvider { private SchedConfUpdateInfo goodUpdate; private SchedConfUpdateInfo badUpdate; private CapacityScheduler cs; + private AdminService adminService; private static final UserGroupInformation TEST_USER = UserGroupInformation .createUserForTesting("testUser", new String[] {}); @@ -61,6 +63,8 @@ public void setUp() { when(rmContext.getScheduler()).thenReturn(cs); when(cs.getConfiguration()).thenReturn( new CapacitySchedulerConfiguration()); + adminService = mock(AdminService.class); + when(rmContext.getRMAdminService()).thenReturn(adminService); confProvider = new MutableCSConfigurationProvider(rmContext); goodUpdate = new SchedConfUpdateInfo(); Map goodUpdateMap = new HashMap<>(); @@ -78,22 +82,20 @@ public void setUp() { } @Test - public void testInMemoryBackedProvider() throws IOException { + public void testInMemoryBackedProvider() throws IOException, YarnException { Configuration conf = new Configuration(); confProvider.init(conf); assertNull(confProvider.loadConfiguration(conf) .get("yarn.scheduler.capacity.root.a.goodKey")); - doNothing().when(cs).reinitialize(any(Configuration.class), - any(RMContext.class)); + doNothing().when(adminService).refreshQueues(); confProvider.mutateConfiguration(TEST_USER, goodUpdate); assertEquals("goodVal", confProvider.loadConfiguration(conf) .get("yarn.scheduler.capacity.root.a.goodKey")); assertNull(confProvider.loadConfiguration(conf).get( "yarn.scheduler.capacity.root.a.badKey")); - doThrow(new IOException()).when(cs).reinitialize(any(Configuration.class), - any(RMContext.class)); + doThrow(new IOException()).when(adminService).refreshQueues(); try { confProvider.mutateConfiguration(TEST_USER, badUpdate); } catch (IOException e) {