diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/conf/capacity-scheduler.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/conf/capacity-scheduler.xml index 62654cacc48..38526d18dbf 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/conf/capacity-scheduler.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/conf/capacity-scheduler.xml @@ -207,4 +207,14 @@ + + + yarn.scheduler.capacity.application.fail-fast + false + + Whether RM should fail during recovery if previous applications' + queue is no longer valid. + + + diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java index 4b68678362b..3566b4a1ed9 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java @@ -749,7 +749,7 @@ public class CapacityScheduler extends if (queue == null) { //During a restart, this indicates a queue was removed, which is //not presently supported - if (!YarnConfiguration.shouldRMFailFast(getConfig())) { + if (!getConfiguration().shouldAppFailFast(getConfig())) { this.rmContext.getDispatcher().getEventHandler().handle( new RMAppEvent(applicationId, RMAppEventType.KILL, "Application killed on recovery as it" @@ -770,7 +770,7 @@ public class CapacityScheduler extends if (!(queue instanceof LeafQueue)) { // During RM restart, this means leaf queue was converted to a parent // queue, which is not supported for running apps. - if (!YarnConfiguration.shouldRMFailFast(getConfig())) { + if (!getConfiguration().shouldAppFailFast(getConfig())) { this.rmContext.getDispatcher().getEventHandler().handle( new RMAppEvent(applicationId, RMAppEventType.KILL, "Application killed on recovery as it was " @@ -829,7 +829,7 @@ public class CapacityScheduler extends return autoCreateLeafQueue(placementContext); } catch (YarnException | IOException e) { if (isRecovery) { - if (!YarnConfiguration.shouldRMFailFast(getConfig())) { + if (!getConfiguration().shouldAppFailFast(getConfig())) { LOG.error("Could not auto-create leaf queue " + queueName + " due to : ", e); this.rmContext.getDispatcher().getEventHandler().handle( diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java index c807d7befa3..5c9904b23ca 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java @@ -248,6 +248,12 @@ public class CapacitySchedulerConfiguration extends ReservationSchedulerConfigur public static final String SCHEDULE_ASYNCHRONOUSLY_MAXIMUM_PENDING_BACKLOGS = SCHEDULE_ASYNCHRONOUSLY_PREFIX + ".maximum-pending-backlogs"; + @Private + public static final String APP_FAIL_FAST = PREFIX + "application.fail-fast"; + + @Private + public static final boolean DEFAULT_APP_FAIL_FAST = false; + @Private public static final Integer DEFAULT_SCHEDULE_ASYNCHRONOUSLY_MAXIMUM_PENDING_BACKLOGS = 100; @@ -1287,6 +1293,10 @@ public class CapacitySchedulerConfiguration extends ReservationSchedulerConfigur return getBoolean(LAZY_PREEMPTION_ENABLED, DEFAULT_LAZY_PREEMPTION_ENABLED); } + public boolean shouldAppFailFast(Configuration conf) { + return conf.getBoolean(APP_FAIL_FAST, DEFAULT_APP_FAIL_FAST); + } + private static final String PREEMPTION_CONFIG_PREFIX = "yarn.resourcemanager.monitor.capacity.preemption."; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestWorkPreservingRMRestart.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestWorkPreservingRMRestart.java index e4c83e32dc7..88c19a1d075 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestWorkPreservingRMRestart.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestWorkPreservingRMRestart.java @@ -760,6 +760,7 @@ public class TestWorkPreservingRMRestart extends ParameterizedSchedulerTestBase MockMemoryRMStateStore memStore, RMState state) throws Exception { // Restart RM with fail-fast as false. App should be killed. csConf.setBoolean(YarnConfiguration.RM_FAIL_FAST, false); + csConf.setBoolean(CapacitySchedulerConfiguration.APP_FAIL_FAST, false); rm2 = new MockRM(csConf, memStore); rm2.start(); @@ -794,6 +795,7 @@ public class TestWorkPreservingRMRestart extends ParameterizedSchedulerTestBase // Now restart RM with fail-fast as true. QueueException should be thrown. csConf.setBoolean(YarnConfiguration.RM_FAIL_FAST, true); + csConf.setBoolean(CapacitySchedulerConfiguration.APP_FAIL_FAST, true); MockRM rm = new MockRM(csConf, memStore2); try { rm.start();