Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

YARN-10625. FairScheduler: add global flag to disable AM-preemption. Contributed by Peter Bacsko

This commit is contained in:
Szilard Nemeth 2021-02-16 18:06:57 +01:00
parent 1bb4101b59
commit 9882a22365
3 changed files with 43 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
View File

@ -1442,6 +1442,13 @@ public class FairScheduler extends
+ " ms instead");
}
boolean globalAmPreemption = conf.getBoolean(
FairSchedulerConfiguration.AM_PREEMPTION,
FairSchedulerConfiguration.DEFAULT_AM_PREEMPTION);
if (!globalAmPreemption) {
LOG.info("AM preemption is DISABLED globally");
}
rootMetrics = FSQueueMetrics.forQueue("root", null, true, conf);
fsOpDurations = FSOpDurations.getInstance(true);

14
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerConfiguration.java
View File

@ -176,6 +176,8 @@ public class FairSchedulerConfiguration extends Configuration {
public static final String PREEMPTION = CONF_PREFIX + "preemption";
public static final boolean DEFAULT_PREEMPTION = false;
protected static final String AM_PREEMPTION =
CONF_PREFIX + "am.preemption";
protected static final String AM_PREEMPTION_PREFIX =
CONF_PREFIX + "am.preemption.";
protected static final boolean DEFAULT_AM_PREEMPTION = true;
@ -407,7 +409,17 @@ public class FairSchedulerConfiguration extends Configuration {
}
public boolean getAMPreemptionEnabled(String queueName) {
return getBoolean(AM_PREEMPTION_PREFIX + queueName, DEFAULT_AM_PREEMPTION);
String propertyName = AM_PREEMPTION_PREFIX + queueName;
if (get(propertyName) != null) {
boolean amPreemptionEnabled =
getBoolean(propertyName, DEFAULT_AM_PREEMPTION);
LOG.debug("AM preemption enabled for queue {}: {}",
queueName, amPreemptionEnabled);
return amPreemptionEnabled;
}
return getBoolean(AM_PREEMPTION, DEFAULT_AM_PREEMPTION);
}
public float getPreemptionUtilizationThreshold() {

24
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerPreemption.java
View File

@ -35,7 +35,9 @@ import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.TestName;
import org.junit.runner.RunWith;
import org.junit.runners.Parameterized;
@ -53,6 +55,8 @@ import java.util.List;
public class TestFairSchedulerPreemption extends FairSchedulerTestBase {
private static final File ALLOC_FILE = new File(TEST_DIR, "test-queues");
private static final int GB = 1024;
private static final String TC_DISABLE_AM_PREEMPTION_GLOBALLY =
"testDisableAMPreemptionGlobally";
// Scheduler clock
private final ControlledClock clock = new ControlledClock();
@ -69,6 +73,9 @@ public class TestFairSchedulerPreemption extends FairSchedulerTestBase {
// Starving app that is expected to instigate preemption
private FSAppAttempt starvingApp;
@Rule
public TestName testName = new TestName();
@Parameterized.Parameters(name = "{0}")
public static Collection<Object[]> getParameters() {
return Arrays.asList(new Object[][] {
@ -95,6 +102,10 @@ public class TestFairSchedulerPreemption extends FairSchedulerTestBase {
conf.setFloat(FairSchedulerConfiguration.PREEMPTION_THRESHOLD, 0f);
conf.setInt(FairSchedulerConfiguration.WAIT_TIME_BEFORE_KILL, 0);
conf.setLong(FairSchedulerConfiguration.UPDATE_INTERVAL_MS, 60_000L);
String testMethod = testName.getMethodName();
if (testMethod.startsWith(TC_DISABLE_AM_PREEMPTION_GLOBALLY)) {
conf.setBoolean(FairSchedulerConfiguration.AM_PREEMPTION, false);
}
setupCluster();
}
@ -417,13 +428,24 @@ public class TestFairSchedulerPreemption extends FairSchedulerTestBase {
@Test
public void testDisableAMPreemption() {
testDisableAMPreemption(false);
}
@Test
public void testDisableAMPreemptionGlobally() {
testDisableAMPreemption(true);
}
private void testDisableAMPreemption(boolean global) {
takeAllResources("root.preemptable.child-1");
setNumAMContainersPerNode(2);
RMContainer container = greedyApp.getLiveContainers().stream()
.filter(rmContainer -> rmContainer.isAMContainer())
.findFirst()
.get();
greedyApp.setEnableAMPreemption(false);
if (!global) {
greedyApp.setEnableAMPreemption(false);
}
assertFalse(greedyApp.canContainerBePreempted(container, null));
}