From fa7a0b269a8613627819c41ab0e9c02a55e278b3 Mon Sep 17 00:00:00 2001 From: Eric Badger Date: Mon, 11 Mar 2019 12:43:14 -0500 Subject: [PATCH] YARN-8805. Automatically convert the launch command to the exec form when using entrypoint support --- .../yarn/service/provider/ProviderUtils.java | 17 ++++++++++++ .../docker/DockerProviderService.java | 6 +++++ .../provider/TestAbstractProviderService.java | 27 ++++++++++++++++--- .../service/provider/TestProviderUtils.java | 9 +++++++ 4 files changed, 55 insertions(+), 4 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/ProviderUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/ProviderUtils.java index 88883f71f0f..ea1fb0c51da 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/ProviderUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/ProviderUtils.java @@ -47,9 +47,12 @@ import java.io.File; import java.io.FileNotFoundException; import java.io.IOException; import java.io.OutputStream; +import java.util.ArrayList; import java.util.HashMap; +import java.util.List; import java.util.Map; import java.util.concurrent.ExecutionException; +import java.util.regex.Matcher; import java.util.regex.Pattern; import static org.apache.hadoop.yarn.service.api.ServiceApiConstants.COMPONENT_ID; @@ -149,6 +152,20 @@ public class ProviderUtils implements YarnServiceConstants { return content; } + public static String replaceSpacesWithDelimiter(String content, + String delimiter) { + List parts = new ArrayList(); + Matcher m = Pattern.compile("([^\"]\\S*|\".+?\")\\s*").matcher(content); + while (m.find()) { + String part = m.group(1); + if(part.startsWith("\"") && part.endsWith("\"")) { + part = part.replaceAll("^\"|\"$", ""); + } + parts.add(part); + } + return String.join(delimiter, parts); + } + // configs will be substituted by corresponding env in tokenMap public static void substituteMapWithTokens(Map configs, Map tokenMap) { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/docker/DockerProviderService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/docker/DockerProviderService.java index 6027a66abca..9b4138e57d6 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/docker/DockerProviderService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/docker/DockerProviderService.java @@ -25,6 +25,7 @@ import org.apache.hadoop.yarn.service.provider.ProviderUtils; import org.apache.hadoop.yarn.api.records.Container; import org.apache.hadoop.yarn.service.api.records.Service; import org.apache.hadoop.yarn.service.utils.SliderFileSystem; + import org.apache.hadoop.yarn.service.containerlaunch.AbstractLauncher; import org.apache.hadoop.yarn.service.containerlaunch.CommandLineBuilder; import org.apache.hadoop.yarn.service.containerlaunch.ContainerLaunchService; @@ -84,6 +85,11 @@ public class DockerProviderService extends AbstractProviderService if (useEntryPoint) { String launchCommand = compLaunchContext.getLaunchCommand(); if (!StringUtils.isEmpty(launchCommand)) { + if(launchCommand.contains(" ")) { + // convert space delimiter command to exec format + launchCommand = ProviderUtils + .replaceSpacesWithDelimiter(launchCommand, ","); + } launcher.addCommand(launchCommand); } } else { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/provider/TestAbstractProviderService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/provider/TestAbstractProviderService.java index 81ccc7fed47..f0525aae232 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/provider/TestAbstractProviderService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/provider/TestAbstractProviderService.java @@ -82,7 +82,7 @@ public class TestAbstractProviderService { Component component = serviceContext.scheduler.getAllComponents().entrySet() .iterator().next().getValue(); ContainerLaunchService.ComponentLaunchContext clc = - createEntryPointCLCFor(testService, component); + createEntryPointCLCFor(testService, component, "sleep,9000"); ComponentInstance instance = component.getAllComponentInstances().iterator() .next(); @@ -95,13 +95,32 @@ public class TestAbstractProviderService { launcher.getCommands()); } + @Test + public void testBuildContainerLaunchCommandWithSpace() throws Exception { + AbstractProviderService providerService = new DockerProviderService(); + Component component = serviceContext.scheduler.getAllComponents().entrySet() + .iterator().next().getValue(); + ContainerLaunchService.ComponentLaunchContext clc = + createEntryPointCLCFor(testService, component, "ls -l \" space\""); + + ComponentInstance instance = component.getAllComponentInstances().iterator() + .next(); + Container container = mock(Container.class); + providerService.buildContainerLaunchCommand(launcher, testService, instance, + rule.getFs(), serviceContext.scheduler.getConfig(), container, clc, + null); + + Assert.assertEquals("commands don't match.", + Lists.newArrayList("ls,-l, space"), launcher.getCommands()); + } + @Test public void testBuildContainerLaunchContext() throws Exception { AbstractProviderService providerService = new DockerProviderService(); Component component = serviceContext.scheduler.getAllComponents().entrySet() .iterator().next().getValue(); ContainerLaunchService.ComponentLaunchContext clc = - createEntryPointCLCFor(testService, component); + createEntryPointCLCFor(testService, component, "sleep,9000"); ComponentInstance instance = component.getAllComponentInstances().iterator() .next(); @@ -118,8 +137,8 @@ public class TestAbstractProviderService { } private static ContainerLaunchService.ComponentLaunchContext - createEntryPointCLCFor(Service service, Component component) { - String launchCmd = "sleep,9000"; + createEntryPointCLCFor(Service service, Component component, + String launchCmd) { Artifact artifact = new Artifact(); artifact.setType(Artifact.TypeEnum.DOCKER); artifact.setId("example"); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/provider/TestProviderUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/provider/TestProviderUtils.java index 0f7f37594cc..84c3b6e020d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/provider/TestProviderUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/provider/TestProviderUtils.java @@ -168,4 +168,13 @@ public class TestProviderUtils { Assert.assertEquals(resolved.getResolvedRsrcPaths().get("destFile1"), "destFile1"); } + + @Test + public void testReplaceSpacesWithDelimiter() { + String command = "ls -l \" space\""; + String expected = "ls,-l, space"; + String actual = ProviderUtils.replaceSpacesWithDelimiter(command, ","); + Assert.assertEquals("replaceSpaceWithDelimiter produces unexpected result.", + expected, actual); + } }