diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java index ef709fce00e..0ef856de229 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java @@ -101,12 +101,15 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeEventType; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetrics; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerNode; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.conf.YarnConfigurationStore; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.conf.YarnConfigurationStoreFactory; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.constraint.AllocationTagsManager; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.constraint.MemoryPlacementConstraintManager; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.constraint.PlacementConstraintManagerService; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.SchedulerEvent; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.SchedulerEventType; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.placement.MultiNodeSortingManager; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.MutableConfScheduler; import org.apache.hadoop.yarn.server.resourcemanager.security.DelegationTokenRenewer; import org.apache.hadoop.yarn.server.resourcemanager.security.QueueACLsManager; import org.apache.hadoop.yarn.server.resourcemanager.timelineservice.RMTimelineCollectorManager; @@ -1525,6 +1528,8 @@ public class ResourceManager extends CompositeService if (argv.length >= 1) { if (argv[0].equals("-format-state-store")) { deleteRMStateStore(conf); + } else if (argv[0].equals("-format-conf-store")) { + deleteRMConfStore(conf); } else if (argv[0].equals("-remove-application-from-state-store") && argv.length == 2) { removeApplication(conf, argv[1]); @@ -1617,6 +1622,45 @@ public class ResourceManager extends CompositeService } } + /** + * Deletes the YarnConfigurationStore. + * + * @param conf + * @throws Exception + */ + @VisibleForTesting + static void deleteRMConfStore(Configuration conf) throws Exception { + ResourceManager rm = new ResourceManager(); + rm.conf = conf; + ResourceScheduler scheduler = rm.createScheduler(); + RMContextImpl rmContext = new RMContextImpl(); + rmContext.setResourceManager(rm); + + boolean isConfigurationMutable = false; + String confProviderStr = conf.get( + YarnConfiguration.SCHEDULER_CONFIGURATION_STORE_CLASS, + YarnConfiguration.DEFAULT_CONFIGURATION_STORE); + switch (confProviderStr) { + case YarnConfiguration.MEMORY_CONFIGURATION_STORE: + case YarnConfiguration.LEVELDB_CONFIGURATION_STORE: + case YarnConfiguration.ZK_CONFIGURATION_STORE: + case YarnConfiguration.FS_CONFIGURATION_STORE: + isConfigurationMutable = true; + break; + default: + } + + if (scheduler instanceof MutableConfScheduler && isConfigurationMutable) { + YarnConfigurationStore confStore = YarnConfigurationStoreFactory + .getStore(conf); + confStore.initialize(conf, conf, rmContext); + confStore.format(); + } else { + System.out.println("Scheduler Configuration format only " + + "supported by MutableConfScheduler."); + } + } + @VisibleForTesting static void removeApplication(Configuration conf, String applicationId) throws Exception { @@ -1637,7 +1681,9 @@ public class ResourceManager extends CompositeService private static void printUsage(PrintStream out) { out.println("Usage: yarn resourcemanager [-format-state-store]"); out.println(" " - + "[-remove-application-from-state-store ]" + "\n"); + + "[-remove-application-from-state-store ]"); + out.println(" " + + "[-format-conf-store]" + "\n"); } protected RMAppLifetimeMonitor createRMAppLifetimeMonitor() { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/MutableCSConfigurationProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/MutableCSConfigurationProvider.java index 7eab65f8693..3e66439a1c3 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/MutableCSConfigurationProvider.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/MutableCSConfigurationProvider.java @@ -70,26 +70,7 @@ public class MutableCSConfigurationProvider implements CSConfigurationProvider, @Override public void init(Configuration config) throws IOException { - String store = config.get( - YarnConfiguration.SCHEDULER_CONFIGURATION_STORE_CLASS, - YarnConfiguration.MEMORY_CONFIGURATION_STORE); - switch (store) { - case YarnConfiguration.MEMORY_CONFIGURATION_STORE: - this.confStore = new InMemoryConfigurationStore(); - break; - case YarnConfiguration.LEVELDB_CONFIGURATION_STORE: - this.confStore = new LeveldbConfigurationStore(); - break; - case YarnConfiguration.ZK_CONFIGURATION_STORE: - this.confStore = new ZKConfigurationStore(); - break; - case YarnConfiguration.FS_CONFIGURATION_STORE: - this.confStore = new FSSchedulerConfigurationStore(); - break; - default: - this.confStore = YarnConfigurationStoreFactory.getStore(config); - break; - } + this.confStore = YarnConfigurationStoreFactory.getStore(config); Configuration initialSchedConf = new Configuration(false); initialSchedConf.addResource(YarnConfiguration.CS_CONFIGURATION_FILE); this.schedConf = new Configuration(false); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/YarnConfigurationStoreFactory.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/YarnConfigurationStoreFactory.java index af637d41f8e..52753a7bc6d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/YarnConfigurationStoreFactory.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/YarnConfigurationStoreFactory.java @@ -37,10 +37,24 @@ public final class YarnConfigurationStoreFactory { } public static YarnConfigurationStore getStore(Configuration conf) { - Class storeClass = - conf.getClass(YarnConfiguration.SCHEDULER_CONFIGURATION_STORE_CLASS, - InMemoryConfigurationStore.class, YarnConfigurationStore.class); - LOG.info("Using YarnConfigurationStore implementation - " + storeClass); - return ReflectionUtils.newInstance(storeClass, conf); + String store = conf.get( + YarnConfiguration.SCHEDULER_CONFIGURATION_STORE_CLASS, + YarnConfiguration.MEMORY_CONFIGURATION_STORE); + switch (store) { + case YarnConfiguration.MEMORY_CONFIGURATION_STORE: + return new InMemoryConfigurationStore(); + case YarnConfiguration.LEVELDB_CONFIGURATION_STORE: + return new LeveldbConfigurationStore(); + case YarnConfiguration.ZK_CONFIGURATION_STORE: + return new ZKConfigurationStore(); + case YarnConfiguration.FS_CONFIGURATION_STORE: + return new FSSchedulerConfigurationStore(); + default: + Class storeClass = + conf.getClass(YarnConfiguration.SCHEDULER_CONFIGURATION_STORE_CLASS, + InMemoryConfigurationStore.class, YarnConfigurationStore.class); + LOG.info("Using YarnConfigurationStore implementation - " + storeClass); + return ReflectionUtils.newInstance(storeClass, conf); + } } } \ No newline at end of file diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMStoreCommands.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMStoreCommands.java index 02033518082..e2c78345d53 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMStoreCommands.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMStoreCommands.java @@ -19,6 +19,8 @@ package org.apache.hadoop.yarn.server.resourcemanager; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; @@ -66,6 +68,39 @@ public class TestRMStoreCommands { } } + @Test + public void testFormatConfStoreCmdForZK() throws Exception { + try (TestingServer curatorTestingServer = + TestZKRMStateStore.setupCuratorServer(); + CuratorFramework curatorFramework = TestZKRMStateStore. + setupCuratorFramework(curatorTestingServer)) { + Configuration conf = TestZKRMStateStore.createHARMConf("rm1,rm2", "rm1", + 1234, false, curatorTestingServer); + conf.set(YarnConfiguration.SCHEDULER_CONFIGURATION_STORE_CLASS, + YarnConfiguration.ZK_CONFIGURATION_STORE); + + ResourceManager rm = new MockRM(conf); + rm.start(); + + String confStorePath = conf.get( + YarnConfiguration.RM_SCHEDCONF_STORE_ZK_PARENT_PATH, + YarnConfiguration.DEFAULT_RM_SCHEDCONF_STORE_ZK_PARENT_PATH) + + "/CONF_STORE"; + assertNotNull("Failed to initialize ZKConfigurationStore", + curatorFramework.checkExists().forPath(confStorePath)); + + rm.close(); + try { + ResourceManager.deleteRMConfStore(conf); + } catch (Exception e) { + fail("Exception should not be thrown during format rm conf store" + + " operation."); + } + assertNull("Failed to format ZKConfigurationStore", + curatorFramework.checkExists().forPath(confStorePath)); + } + } + @Test public void testRemoveApplicationFromStateStoreCmdForZK() throws Exception { StateChangeRequestInfo req = new StateChangeRequestInfo( diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/YarnCommands.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/YarnCommands.md index e17538ccdfc..cfa1ce665cb 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/YarnCommands.md +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/YarnCommands.md @@ -193,7 +193,7 @@ Usage: `yarn resourcemanager [-format-state-store]` |:---- |:---- | | -format-state-store | Formats the RMStateStore. This will clear the RMStateStore and is useful if past applications are no longer needed. This should be run only when the ResourceManager is not running. | | -remove-application-from-state-store \ | Remove the application from RMStateStore. This should be run only when the ResourceManager is not running. | - +| -format-conf-store | Formats the YarnConfigurationStore. This will clear the persisted scheduler configuration under YarnConfigurationStore. This should be run only when the ResourceManager is not running. | Start the ResourceManager ### `rmadmin`