From 907919d28c1b7e4496d189b46ecbb86a10d41339 Mon Sep 17 00:00:00 2001 From: Jason Lowe Date: Mon, 9 Apr 2018 17:19:21 -0500 Subject: [PATCH] YARN-7667. Docker Stop grace period should be configurable. Contributed by Eric Badger --- .../apache/hadoop/yarn/conf/YarnConfiguration.java | 14 ++++++++++++++ .../src/main/resources/yarn-default.xml | 8 ++++++++ .../linux/runtime/DockerLinuxContainerRuntime.java | 8 +++++++- .../linux/runtime/TestDockerContainerRuntime.java | 14 +++++++++++--- 4 files changed, 40 insertions(+), 4 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index 2590b6f84cf..d2a71bcd1d0 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -1951,6 +1951,20 @@ public static boolean isAclEnabled(Configuration conf) { */ public static final boolean DEFAULT_NM_DOCKER_ALLOW_DELAYED_REMOVAL = false; + /** + * A configurable value to pass to the Docker Stop command. This value + * defines the number of seconds between the docker stop command sending + * a SIGTERM and a SIGKILL. + */ + public static final String NM_DOCKER_STOP_GRACE_PERIOD = + DOCKER_CONTAINER_RUNTIME_PREFIX + "stop.grace-period"; + + /** + * The default value for the grace period between the SIGTERM and the + * SIGKILL in the Docker Stop command. + */ + public static final int DEFAULT_NM_DOCKER_STOP_GRACE_PERIOD = 10; + /** The mode in which the Java Container Sandbox should run detailed by * the JavaSandboxLinuxContainerRuntime. */ public static final String YARN_CONTAINER_SANDBOX = diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml index 81b6658d61e..def0816f614 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml @@ -1786,6 +1786,14 @@ false + + A configurable value to pass to the Docker Stop command. This value + defines the number of seconds between the docker stop command sending + a SIGTERM and a SIGKILL. + yarn.nodemanager.runtime.linux.docker.stop.grace-period + 10 + + The mode in which the Java Container Sandbox should run detailed by the JavaSandboxLinuxContainerRuntime. diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java index 02904930a97..132ae38be27 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java @@ -245,6 +245,7 @@ public class DockerLinuxContainerRuntime implements LinuxContainerRuntime { private int userRemappingGidThreshold; private Set capabilities; private boolean delayedRemovalAllowed; + private int dockerStopGracePeriod; /** * Return whether the given environment variables indicate that the operation @@ -348,6 +349,10 @@ public void initialize(Configuration conf, Context nmContext) delayedRemovalAllowed = conf.getBoolean( YarnConfiguration.NM_DOCKER_ALLOW_DELAYED_REMOVAL, YarnConfiguration.DEFAULT_NM_DOCKER_ALLOW_DELAYED_REMOVAL); + + dockerStopGracePeriod = conf.getInt( + YarnConfiguration.NM_DOCKER_STOP_GRACE_PERIOD, + YarnConfiguration.DEFAULT_NM_DOCKER_STOP_GRACE_PERIOD); } private Set getDockerCapabilitiesFromConf() throws @@ -1138,7 +1143,8 @@ private void handleContainerStop(String containerId, Map env) DockerCommandExecutor.getContainerStatus(containerId, conf, privilegedOperationExecutor); if (DockerCommandExecutor.isStoppable(containerStatus)) { - DockerStopCommand dockerStopCommand = new DockerStopCommand(containerId); + DockerStopCommand dockerStopCommand = new DockerStopCommand( + containerId).setGracePeriod(dockerStopGracePeriod); DockerCommandExecutor.executeDockerCommand(dockerStopCommand, containerId, env, conf, privilegedOperationExecutor, false); } else { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java index 132a2025bf7..2d80592f02e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java @@ -146,6 +146,7 @@ public class TestDockerContainerRuntime { private final String whitelistedUser = "yoda"; private String[] testCapabilities; private final String signalPid = "1234"; + private int dockerStopGracePeriod; @Before public void setup() { @@ -166,6 +167,10 @@ public void setup() { env.put("FROM_CLIENT", "1"); image = "busybox:latest"; + dockerStopGracePeriod = conf.getInt( + YarnConfiguration.NM_DOCKER_STOP_GRACE_PERIOD, + YarnConfiguration.DEFAULT_NM_DOCKER_STOP_GRACE_PERIOD); + env.put(DockerLinuxContainerRuntime.ENV_DOCKER_CONTAINER_IMAGE, image); when(container.getContainerId()).thenReturn(cId); when(cId.toString()).thenReturn(containerId); @@ -1308,10 +1313,11 @@ public void testDockerStopOnTermSignalWhenRunning() List dockerCommands = getDockerCommandsForSignal( ContainerExecutor.Signal.TERM, DockerCommandExecutor.DockerContainerStatus.RUNNING); - Assert.assertEquals(3, dockerCommands.size()); + Assert.assertEquals(4, dockerCommands.size()); Assert.assertEquals("[docker-command-execution]", dockerCommands.get(0)); Assert.assertEquals(" docker-command=stop", dockerCommands.get(1)); Assert.assertEquals(" name=container_id", dockerCommands.get(2)); + Assert.assertEquals(" time=10", dockerCommands.get(3)); } @Test @@ -1321,10 +1327,11 @@ public void testDockerStopOnKillSignalWhenRunning() List dockerCommands = getDockerCommandsForSignal( ContainerExecutor.Signal.KILL, DockerCommandExecutor.DockerContainerStatus.RUNNING); - Assert.assertEquals(3, dockerCommands.size()); + Assert.assertEquals(4, dockerCommands.size()); Assert.assertEquals("[docker-command-execution]", dockerCommands.get(0)); Assert.assertEquals(" docker-command=stop", dockerCommands.get(1)); Assert.assertEquals(" name=container_id", dockerCommands.get(2)); + Assert.assertEquals(" time=10", dockerCommands.get(3)); } @Test @@ -1884,7 +1891,8 @@ public void signalContainer(ContainerRuntimeContext ctx) || ContainerExecutor.Signal.TERM.equals(signal)) { if (DockerCommandExecutor.isStoppable(containerStatus)) { DockerStopCommand dockerStopCommand = - new DockerStopCommand(containerName); + new DockerStopCommand(containerName) + .setGracePeriod(dockerStopGracePeriod); DockerCommandExecutor.executeDockerCommand(dockerStopCommand, containerName, environment, conf, mockExecutor, false); }