diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index 744643960a9..e9ff71eafb6 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -1369,6 +1369,30 @@ public class YarnConfiguration extends Configuration { /** Default list for users allowed to run privileged containers is empty. */ public static final String DEFAULT_NM_DOCKER_PRIVILEGED_CONTAINERS_ACL = ""; + /** The set of networks allowed when launching containers using the + * DockerContainerRuntime. */ + public static final String NM_DOCKER_ALLOWED_CONTAINER_NETWORKS = + DOCKER_CONTAINER_RUNTIME_PREFIX + "allowed-container-networks"; + + /** The set of networks allowed when launching containers using the + * DockerContainerRuntime. */ + public static final String[] DEFAULT_NM_DOCKER_ALLOWED_CONTAINER_NETWORKS = + {"host", "none", "bridge"}; + + /** The network used when launching containers using the + * DockerContainerRuntime when no network is specified in the request. This + * network must be one of the (configurable) set of allowed container + * networks. */ + public static final String NM_DOCKER_DEFAULT_CONTAINER_NETWORK = + DOCKER_CONTAINER_RUNTIME_PREFIX + "default-container-network"; + + /** The network used when launching containers using the + * DockerContainerRuntime when no network is specified in the request and + * no default network is configured. + * . */ + public static final String DEFAULT_NM_DOCKER_DEFAULT_CONTAINER_NETWORK = + "host"; + /** The path to the Linux container executor.*/ public static final String NM_LINUX_CONTAINER_EXECUTOR_PATH = NM_PREFIX + "linux-container-executor.path"; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml index 703c03df0fb..b3b2e2d7071 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml @@ -1510,6 +1510,22 @@ + + The set of networks allowed when launching containers using the + DockerContainerRuntime. + yarn.nodemanager.runtime.linux.docker.allowed-container-networks + host,none,bridge + + + + The network used when launching containers using the + DockerContainerRuntime when no network is specified in the request + . This network must be one of the (configurable) set of allowed container + networks. + yarn.nodemanager.runtime.linux.docker.default-container-network + host + + This flag determines whether memory limit will be set for the Windows Job Object of the containers launched by the default container executor. diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java index 43d8b4e4ac3..4398e54a78c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java @@ -44,7 +44,6 @@ import org.apache.hadoop.yarn.server.nodemanager.containermanager.runtime.Contai import org.apache.hadoop.yarn.server.nodemanager.containermanager.runtime.ContainerRuntimeConstants; import org.apache.hadoop.yarn.server.nodemanager.containermanager.runtime.ContainerRuntimeContext; - import java.nio.file.Files; import java.nio.file.Paths; import java.util.ArrayList; @@ -73,6 +72,8 @@ public class DockerLinuxContainerRuntime implements LinuxContainerRuntime { public static final String ENV_DOCKER_CONTAINER_RUN_OVERRIDE_DISABLE = "YARN_CONTAINER_RUNTIME_DOCKER_RUN_OVERRIDE_DISABLE"; @InterfaceAudience.Private + public static final String ENV_DOCKER_CONTAINER_NETWORK = + "YARN_CONTAINER_RUNTIME_DOCKER_CONTAINER_NETWORK"; public static final String ENV_DOCKER_CONTAINER_RUN_PRIVILEGED_CONTAINER = "YARN_CONTAINER_RUNTIME_DOCKER_RUN_PRIVILEGED_CONTAINER"; @InterfaceAudience.Private @@ -82,6 +83,8 @@ public class DockerLinuxContainerRuntime implements LinuxContainerRuntime { private Configuration conf; private DockerClient dockerClient; private PrivilegedOperationExecutor privilegedOperationExecutor; + private Set allowedNetworks = new HashSet<>(); + private String defaultNetwork; private CGroupsHandler cGroupsHandler; private AccessControlList privilegedContainersAcl; @@ -122,6 +125,26 @@ public class DockerLinuxContainerRuntime implements LinuxContainerRuntime { throws ContainerExecutionException { this.conf = conf; dockerClient = new DockerClient(conf); + allowedNetworks.clear(); + allowedNetworks.addAll(Arrays.asList( + conf.getStrings(YarnConfiguration.NM_DOCKER_ALLOWED_CONTAINER_NETWORKS, + YarnConfiguration.DEFAULT_NM_DOCKER_ALLOWED_CONTAINER_NETWORKS))); + defaultNetwork = conf.get( + YarnConfiguration.NM_DOCKER_DEFAULT_CONTAINER_NETWORK, + YarnConfiguration.DEFAULT_NM_DOCKER_DEFAULT_CONTAINER_NETWORK); + + if(!allowedNetworks.contains(defaultNetwork)) { + String message = "Default network: " + defaultNetwork + + " is not in the set of allowed networks: " + allowedNetworks; + + if (LOG.isWarnEnabled()) { + LOG.warn(message + ". Please check " + + "configuration"); + } + + throw new ContainerExecutionException(message); + } + privilegedContainersAcl = new AccessControlList(conf.get( YarnConfiguration.NM_DOCKER_PRIVILEGED_CONTAINERS_ACL, YarnConfiguration.DEFAULT_NM_DOCKER_PRIVILEGED_CONTAINERS_ACL)); @@ -133,6 +156,18 @@ public class DockerLinuxContainerRuntime implements LinuxContainerRuntime { } + private void validateContainerNetworkType(String network) + throws ContainerExecutionException { + if (allowedNetworks.contains(network)) { + return; + } + + String msg = "Disallowed network: '" + network + + "' specified. Allowed networks: are " + allowedNetworks + .toString(); + throw new ContainerExecutionException(msg); + } + public void addCGroupParentIfRequired(String resourcesOptions, String containerIdStr, DockerRunCommand runCommand) throws ContainerExecutionException { @@ -260,6 +295,13 @@ public class DockerLinuxContainerRuntime implements LinuxContainerRuntime { Map environment = container.getLaunchContext() .getEnvironment(); String imageName = environment.get(ENV_DOCKER_CONTAINER_IMAGE); + String network = environment.get(ENV_DOCKER_CONTAINER_NETWORK); + + if(network == null || network.isEmpty()) { + network = defaultNetwork; + } + + validateContainerNetworkType(network); if (imageName == null) { throw new ContainerExecutionException(ENV_DOCKER_CONTAINER_IMAGE @@ -293,7 +335,7 @@ public class DockerLinuxContainerRuntime implements LinuxContainerRuntime { runAsUser, imageName) .detachOnRun() .setContainerWorkDir(containerWorkDir.toString()) - .setNetworkType("host") + .setNetworkType(network) .setCapabilities(capabilities) .addMountLocation("/etc/passwd", "/etc/password:ro"); List allDirs = new ArrayList<>(containerLocalDirs); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java index 538b03fcbb4..3c0b67d658a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java @@ -47,7 +47,7 @@ import java.io.IOException; import java.nio.charset.Charset; import java.nio.file.Files; import java.nio.file.Paths; -import java.util.ArrayList; +import java.util.*; import java.util.Arrays; import java.util.Collections; import java.util.HashMap; @@ -64,37 +64,37 @@ public class TestDockerContainerRuntime { private static final Log LOG = LogFactory .getLog(TestDockerContainerRuntime.class); private Configuration conf; - PrivilegedOperationExecutor mockExecutor; - CGroupsHandler mockCGroupsHandler; - String containerId; - Container container; - ContainerId cId; - ContainerLaunchContext context; - HashMap env; - String image; - String runAsUser; - String user; - String appId; - String containerIdStr = containerId; - Path containerWorkDir; - Path nmPrivateContainerScriptPath; - Path nmPrivateTokensPath; - Path pidFilePath; - List localDirs; - List logDirs; - List containerLocalDirs; - List containerLogDirs; - Map> localizedResources; - String resourcesOptions; - ContainerRuntimeContext.Builder builder; - String submittingUser = "anakin"; - String whitelistedUser = "yoda"; + private PrivilegedOperationExecutor mockExecutor; + private CGroupsHandler mockCGroupsHandler; + private String containerId; + private Container container; + private ContainerId cId; + private ContainerLaunchContext context; + private HashMap env; + private String image; + private String runAsUser; + private String user; + private String appId; + private String containerIdStr = containerId; + private Path containerWorkDir; + private Path nmPrivateContainerScriptPath; + private Path nmPrivateTokensPath; + private Path pidFilePath; + private List localDirs; + private List logDirs; + private List containerLocalDirs; + private List containerLogDirs; + private Map> localizedResources; + private String resourcesOptions; + private ContainerRuntimeContext.Builder builder; + private final String submittingUser = "anakin"; + private final String whitelistedUser = "yoda"; + private String[] testCapabilities; @Before public void setup() { String tmpPath = new StringBuffer(System.getProperty("test.build.data")) - .append - ('/').append("hadoop.tmp.dir").toString(); + .append('/').append("hadoop.tmp.dir").toString(); conf = new Configuration(); conf.set("hadoop.tmp.dir", tmpPath); @@ -138,6 +138,10 @@ public class TestDockerContainerRuntime { localizedResources.put(new Path("/test_local_dir/test_resource_file"), Collections.singletonList("test_dir/test_resource_file")); + testCapabilities = new String[] {"NET_BIND_SERVICE", "SYS_CHROOT"}; + conf.setStrings(YarnConfiguration.NM_DOCKER_CONTAINER_CAPABILITIES, + testCapabilities); + builder = new ContainerRuntimeContext .Builder(container); @@ -187,6 +191,10 @@ public class TestDockerContainerRuntime { .executePrivilegedOperation(anyList(), opCaptor.capture(), any( File.class), any(Map.class), eq(false), eq(false)); + //verification completed. we need to isolate specific invications. + // hence, reset mock here + Mockito.reset(mockExecutor); + PrivilegedOperation op = opCaptor.getValue(); Assert.assertEquals(PrivilegedOperation.OperationType @@ -217,24 +225,7 @@ public class TestDockerContainerRuntime { return op; } - @Test - public void testDockerContainerLaunch() - throws ContainerExecutionException, PrivilegedOperationException, - IOException { - DockerLinuxContainerRuntime runtime = new DockerLinuxContainerRuntime( - mockExecutor, mockCGroupsHandler); - runtime.initialize(conf); - - String[] testCapabilities = {"NET_BIND_SERVICE", "SYS_CHROOT"}; - - conf.setStrings(YarnConfiguration.NM_DOCKER_CONTAINER_CAPABILITIES, - testCapabilities); - runtime.launchContainer(builder.build()); - - PrivilegedOperation op = capturePrivilegedOperationAndVerifyArgs(); - List args = op.getArguments(); - String dockerCommandFile = args.get(11); - + private String getExpectedTestCapabilitiesArgumentString() { /* Ordering of capabilities depends on HashSet ordering. */ Set capabilitySet = new HashSet<>(Arrays.asList(testCapabilities)); StringBuilder expectedCapabilitiesString = new StringBuilder( @@ -245,12 +236,28 @@ public class TestDockerContainerRuntime { .append(" "); } + return expectedCapabilitiesString.toString(); + } + + @Test + public void testDockerContainerLaunch() + throws ContainerExecutionException, PrivilegedOperationException, + IOException { + DockerLinuxContainerRuntime runtime = new DockerLinuxContainerRuntime( + mockExecutor, mockCGroupsHandler); + runtime.initialize(conf); + runtime.launchContainer(builder.build()); + + PrivilegedOperation op = capturePrivilegedOperationAndVerifyArgs(); + List args = op.getArguments(); + String dockerCommandFile = args.get(11); + //This is the expected docker invocation for this case StringBuffer expectedCommandTemplate = new StringBuffer("run --name=%1$s ") .append("--user=%2$s -d ") .append("--workdir=%3$s ") .append("--net=host ") - .append(expectedCapabilitiesString) + .append(getExpectedTestCapabilitiesArgumentString()) .append("-v /etc/passwd:/etc/password:ro ") .append("-v %4$s:%4$s ") .append("-v %5$s:%5$s ") @@ -269,6 +276,208 @@ public class TestDockerContainerRuntime { Assert.assertEquals(expectedCommand, dockerCommands.get(0)); } + @Test + public void testAllowedNetworksConfiguration() throws + ContainerExecutionException { + //the default network configuration should cause + // no exception should be thrown. + + DockerLinuxContainerRuntime runtime = + new DockerLinuxContainerRuntime(mockExecutor, mockCGroupsHandler); + runtime.initialize(conf); + + //invalid default network configuration - sdn2 is included in allowed + // networks + + String[] networks = {"host", "none", "bridge", "sdn1"}; + String invalidDefaultNetwork = "sdn2"; + + conf.setStrings(YarnConfiguration.NM_DOCKER_ALLOWED_CONTAINER_NETWORKS, + networks); + conf.set(YarnConfiguration.NM_DOCKER_DEFAULT_CONTAINER_NETWORK, + invalidDefaultNetwork); + + try { + runtime = + new DockerLinuxContainerRuntime(mockExecutor, mockCGroupsHandler); + runtime.initialize(conf); + Assert.fail("Invalid default network configuration should did not " + + "trigger initialization failure."); + } catch (ContainerExecutionException e) { + LOG.info("Caught expected exception : " + e); + } + + //valid default network configuration - sdn1 is included in allowed + // networks - no exception should be thrown. + + String validDefaultNetwork = "sdn1"; + + conf.set(YarnConfiguration.NM_DOCKER_DEFAULT_CONTAINER_NETWORK, + validDefaultNetwork); + runtime = + new DockerLinuxContainerRuntime(mockExecutor, mockCGroupsHandler); + runtime.initialize(conf); + } + + @Test + @SuppressWarnings("unchecked") + public void testContainerLaunchWithNetworkingDefaults() + throws ContainerExecutionException, IOException, + PrivilegedOperationException { + DockerLinuxContainerRuntime runtime = + new DockerLinuxContainerRuntime(mockExecutor, mockCGroupsHandler); + runtime.initialize(conf); + + Random randEngine = new Random(); + String disallowedNetwork = "sdn" + Integer.toString(randEngine.nextInt()); + + try { + env.put("YARN_CONTAINER_RUNTIME_DOCKER_CONTAINER_NETWORK", + disallowedNetwork); + runtime.launchContainer(builder.build()); + Assert.fail("Network was expected to be disallowed: " + + disallowedNetwork); + } catch (ContainerExecutionException e) { + LOG.info("Caught expected exception: " + e); + } + + int size = YarnConfiguration + .DEFAULT_NM_DOCKER_ALLOWED_CONTAINER_NETWORKS.length; + String allowedNetwork = YarnConfiguration + .DEFAULT_NM_DOCKER_ALLOWED_CONTAINER_NETWORKS[randEngine.nextInt(size)]; + env.put("YARN_CONTAINER_RUNTIME_DOCKER_CONTAINER_NETWORK", + allowedNetwork); + + //this should cause no failures. + + runtime.launchContainer(builder.build()); + PrivilegedOperation op = capturePrivilegedOperationAndVerifyArgs(); + List args = op.getArguments(); + String dockerCommandFile = args.get(11); + + //This is the expected docker invocation for this case + StringBuffer expectedCommandTemplate = + new StringBuffer("run --name=%1$s ").append("--user=%2$s -d ") + .append("--workdir=%3$s ") + .append("--net=" + allowedNetwork + " ") + .append(getExpectedTestCapabilitiesArgumentString()) + .append("-v /etc/passwd:/etc/password:ro ") + .append("-v %4$s:%4$s ").append("-v %5$s:%5$s ") + .append("-v %6$s:%6$s ").append("%7$s ") + .append("bash %8$s/launch_container.sh"); + + String expectedCommand = String + .format(expectedCommandTemplate.toString(), containerId, runAsUser, + containerWorkDir, containerLocalDirs.get(0), containerWorkDir, + containerLogDirs.get(0), image, containerWorkDir); + + List dockerCommands = Files + .readAllLines(Paths.get(dockerCommandFile), Charset.forName("UTF-8")); + + Assert.assertEquals(1, dockerCommands.size()); + Assert.assertEquals(expectedCommand, dockerCommands.get(0)); + } + + @Test + @SuppressWarnings("unchecked") + public void testContainerLaunchWithCustomNetworks() + throws ContainerExecutionException, IOException, + PrivilegedOperationException { + DockerLinuxContainerRuntime runtime = + new DockerLinuxContainerRuntime(mockExecutor, mockCGroupsHandler); + + String customNetwork1 = "sdn1"; + String customNetwork2 = "sdn2"; + String customNetwork3 = "sdn3"; + + String[] networks = {"host", "none", "bridge", customNetwork1, + customNetwork2}; + + //customized set of allowed networks + conf.setStrings(YarnConfiguration.NM_DOCKER_ALLOWED_CONTAINER_NETWORKS, + networks); + //default network is "sdn1" + conf.set(YarnConfiguration.NM_DOCKER_DEFAULT_CONTAINER_NETWORK, + customNetwork1); + + //this should cause no failures. + runtime.initialize(conf); + runtime.launchContainer(builder.build()); + PrivilegedOperation op = capturePrivilegedOperationAndVerifyArgs(); + List args = op.getArguments(); + String dockerCommandFile = args.get(11); + + //This is the expected docker invocation for this case. customNetwork1 + // ("sdn1") is the expected network to be used in this case + StringBuffer expectedCommandTemplate = + new StringBuffer("run --name=%1$s ").append("--user=%2$s -d ") + .append("--workdir=%3$s ") + .append("--net=" + customNetwork1 + " ") + .append(getExpectedTestCapabilitiesArgumentString()) + .append("-v /etc/passwd:/etc/password:ro ") + .append("-v %4$s:%4$s ").append("-v %5$s:%5$s ") + .append("-v %6$s:%6$s ").append("%7$s ") + .append("bash %8$s/launch_container.sh"); + + String expectedCommand = String + .format(expectedCommandTemplate.toString(), containerId, runAsUser, + containerWorkDir, containerLocalDirs.get(0), containerWorkDir, + containerLogDirs.get(0), image, containerWorkDir); + + List dockerCommands = Files + .readAllLines(Paths.get(dockerCommandFile), Charset.forName("UTF-8")); + + Assert.assertEquals(1, dockerCommands.size()); + Assert.assertEquals(expectedCommand, dockerCommands.get(0)); + + + //now set an explicit (non-default) allowedNetwork and ensure that it is + // used. + + env.put("YARN_CONTAINER_RUNTIME_DOCKER_CONTAINER_NETWORK", + customNetwork2); + runtime.launchContainer(builder.build()); + + op = capturePrivilegedOperationAndVerifyArgs(); + args = op.getArguments(); + dockerCommandFile = args.get(11); + + //This is the expected docker invocation for this case. customNetwork2 + // ("sdn2") is the expected network to be used in this case + expectedCommandTemplate = + new StringBuffer("run --name=%1$s ").append("--user=%2$s -d ") + .append("--workdir=%3$s ") + .append("--net=" + customNetwork2 + " ") + .append(getExpectedTestCapabilitiesArgumentString()) + .append("-v /etc/passwd:/etc/password:ro ") + .append("-v %4$s:%4$s ").append("-v %5$s:%5$s ") + .append("-v %6$s:%6$s ").append("%7$s ") + .append("bash %8$s/launch_container.sh"); + + expectedCommand = String + .format(expectedCommandTemplate.toString(), containerId, runAsUser, + containerWorkDir, containerLocalDirs.get(0), containerWorkDir, + containerLogDirs.get(0), image, containerWorkDir); + + dockerCommands = Files + .readAllLines(Paths.get(dockerCommandFile), Charset.forName("UTF-8")); + + Assert.assertEquals(1, dockerCommands.size()); + Assert.assertEquals(expectedCommand, dockerCommands.get(0)); + + //disallowed network should trigger a launch failure + + env.put("YARN_CONTAINER_RUNTIME_DOCKER_CONTAINER_NETWORK", + customNetwork3); + try { + runtime.launchContainer(builder.build()); + Assert.fail("Disallowed network : " + customNetwork3 + + "did not trigger launch failure."); + } catch (ContainerExecutionException e) { + LOG.info("Caught expected exception : " + e); + } + } + @Test public void testLaunchPrivilegedContainersInvalidEnvVar() throws ContainerExecutionException, PrivilegedOperationException, @@ -342,7 +551,6 @@ public class TestDockerContainerRuntime { } } - @Test public void testLaunchPrivilegedContainersEnabledAndUserNotInWhitelist()