YARN-6804. Allow custom hostname for docker containers in native services. Contributed by Billie Rinaldi
This commit is contained in:
Jian He
parent
a68b5b31cf
commit
ac9489f7fc
|
|
@ -393,6 +393,10 @@
|
|||
<groupId>org.apache.hadoop</groupId>
|
||||
<artifactId>hadoop-yarn-common</artifactId>
|
||||
</exclusion>
|
||||
<exclusion>
|
||||
<groupId>org.apache.hadoop</groupId>
|
||||
<artifactId>hadoop-yarn-registry</artifactId>
|
||||
</exclusion>
|
||||
<exclusion>
|
||||
<groupId>org.apache.hadoop</groupId>
|
||||
<artifactId>hadoop-yarn-server-common</artifactId>
|
||||
|
|
|
|||
|
|
@ -213,6 +213,6 @@ public class RegistryPathUtils {
|
|||
* @return a string suitable for use in registry paths.
|
||||
*/
|
||||
public static String encodeYarnID(String yarnId) {
|
||||
return yarnId.replace("_", "-");
|
||||
return yarnId.replace("container", "ctr").replace("_", "-");
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -19,7 +19,7 @@
|
|||
package org.apache.hadoop.registry.client.types;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
|
||||
import com.fasterxml.jackson.databind.annotation.JsonSerialize;
|
||||
import com.fasterxml.jackson.annotation.JsonInclude;
|
||||
import com.google.common.base.Preconditions;
|
||||
import org.apache.hadoop.classification.InterfaceAudience;
|
||||
import org.apache.hadoop.classification.InterfaceStability;
|
||||
|
|
@ -46,7 +46,7 @@ import java.util.Map;
|
|||
@InterfaceAudience.Public
|
||||
@InterfaceStability.Evolving
|
||||
@JsonIgnoreProperties(ignoreUnknown = true)
|
||||
@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL)
|
||||
@JsonInclude(JsonInclude.Include.NON_NULL)
|
||||
public final class Endpoint implements Cloneable {
|
||||
|
||||
/**
|
||||
|
|
|
|||
|
|
@ -20,7 +20,7 @@ package org.apache.hadoop.registry.client.types;
|
|||
|
||||
import com.fasterxml.jackson.annotation.JsonAnyGetter;
|
||||
import com.fasterxml.jackson.annotation.JsonAnySetter;
|
||||
import com.fasterxml.jackson.databind.annotation.JsonSerialize;
|
||||
import com.fasterxml.jackson.annotation.JsonInclude;
|
||||
import com.google.common.base.Preconditions;
|
||||
import org.apache.hadoop.classification.InterfaceAudience;
|
||||
import org.apache.hadoop.classification.InterfaceStability;
|
||||
|
|
@ -37,7 +37,7 @@ import java.util.Map;
|
|||
*/
|
||||
@InterfaceAudience.Public
|
||||
@InterfaceStability.Evolving
|
||||
@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL)
|
||||
@JsonInclude(JsonInclude.Include.NON_NULL)
|
||||
public class ServiceRecord implements Cloneable {
|
||||
|
||||
/**
|
||||
|
|
|
|||
|
|
@ -51,6 +51,10 @@
|
|||
<groupId>org.apache.hadoop</groupId>
|
||||
<artifactId>hadoop-yarn-api</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.apache.hadoop</groupId>
|
||||
<artifactId>hadoop-yarn-registry</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>javax.xml.bind</groupId>
|
||||
<artifactId>jaxb-api</artifactId>
|
||||
|
|
|
|||
|
|
@ -27,6 +27,7 @@ import org.apache.hadoop.classification.InterfaceAudience;
|
|||
import org.apache.hadoop.classification.InterfaceStability;
|
||||
import org.apache.hadoop.conf.Configuration;
|
||||
import org.apache.hadoop.fs.Path;
|
||||
import org.apache.hadoop.registry.client.binding.RegistryPathUtils;
|
||||
import org.apache.hadoop.security.UserGroupInformation;
|
||||
import org.apache.hadoop.security.authorize.AccessControlList;
|
||||
import org.apache.hadoop.util.StringUtils;
|
||||
|
|
@ -101,6 +102,11 @@ import static org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.r
|
|||
* property.
|
||||
* </li>
|
||||
* <li>
|
||||
* {@code YARN_CONTAINER_RUNTIME_DOCKER_CONTAINER_HOSTNAME} sets the
|
||||
* hostname to be used by the Docker container. If not specified, a
|
||||
* hostname will be derived from the container ID.
|
||||
* </li>
|
||||
* <li>
|
||||
* {@code YARN_CONTAINER_RUNTIME_DOCKER_RUN_PRIVILEGED_CONTAINER}
|
||||
* controls whether the Docker container is a privileged container. In order
|
||||
* to use privileged containers, the
|
||||
|
|
@ -134,6 +140,10 @@ public class DockerLinuxContainerRuntime implements LinuxContainerRuntime {
|
|||
"^(([a-zA-Z0-9.-]+)(:\\d+)?/)?([a-z0-9_./-]+)(:[\\w.-]+)?$";
|
||||
private static final Pattern dockerImagePattern =
|
||||
Pattern.compile(DOCKER_IMAGE_PATTERN);
|
||||
public static final String HOSTNAME_PATTERN =
|
||||
"^[a-zA-Z0-9][a-zA-Z0-9_.-]+$";
|
||||
private static final Pattern hostnamePattern = Pattern.compile(
|
||||
HOSTNAME_PATTERN);
|
||||
|
||||
@InterfaceAudience.Private
|
||||
public static final String ENV_DOCKER_CONTAINER_IMAGE =
|
||||
|
|
@ -147,6 +157,10 @@ public class DockerLinuxContainerRuntime implements LinuxContainerRuntime {
|
|||
@InterfaceAudience.Private
|
||||
public static final String ENV_DOCKER_CONTAINER_NETWORK =
|
||||
"YARN_CONTAINER_RUNTIME_DOCKER_CONTAINER_NETWORK";
|
||||
@InterfaceAudience.Private
|
||||
public static final String ENV_DOCKER_CONTAINER_HOSTNAME =
|
||||
"YARN_CONTAINER_RUNTIME_DOCKER_CONTAINER_HOSTNAME";
|
||||
@InterfaceAudience.Private
|
||||
public static final String ENV_DOCKER_CONTAINER_RUN_PRIVILEGED_CONTAINER =
|
||||
"YARN_CONTAINER_RUNTIME_DOCKER_RUN_PRIVILEGED_CONTAINER";
|
||||
@InterfaceAudience.Private
|
||||
|
|
@ -211,9 +225,7 @@ public class DockerLinuxContainerRuntime implements LinuxContainerRuntime {
|
|||
this.privilegedOperationExecutor = privilegedOperationExecutor;
|
||||
|
||||
if (cGroupsHandler == null) {
|
||||
if (LOG.isInfoEnabled()) {
|
||||
LOG.info("cGroupsHandler is null - cgroups not in use.");
|
||||
}
|
||||
LOG.info("cGroupsHandler is null - cgroups not in use.");
|
||||
} else {
|
||||
this.cGroupsHandler = cGroupsHandler;
|
||||
}
|
||||
|
|
@ -267,6 +279,29 @@ public class DockerLinuxContainerRuntime implements LinuxContainerRuntime {
|
|||
throw new ContainerExecutionException(msg);
|
||||
}
|
||||
|
||||
public static void validateHostname(String hostname) throws
|
||||
ContainerExecutionException {
|
||||
if (hostname != null && !hostname.isEmpty()) {
|
||||
if (!hostnamePattern.matcher(hostname).matches()) {
|
||||
throw new ContainerExecutionException("Hostname '" + hostname
|
||||
+ "' doesn't match docker hostname pattern");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/** Set a DNS friendly hostname. */
|
||||
private void setHostname(DockerRunCommand runCommand, String
|
||||
containerIdStr, String name)
|
||||
throws ContainerExecutionException {
|
||||
if (name == null || name.isEmpty()) {
|
||||
name = RegistryPathUtils.encodeYarnID(containerIdStr);
|
||||
validateHostname(name);
|
||||
}
|
||||
|
||||
LOG.info("setting hostname in container to: " + name);
|
||||
runCommand.setHostname(name);
|
||||
}
|
||||
|
||||
/**
|
||||
* If CGROUPS in enabled and not set to none, then set the CGROUP parent for
|
||||
* the command instance.
|
||||
|
|
@ -343,10 +378,8 @@ public class DockerLinuxContainerRuntime implements LinuxContainerRuntime {
|
|||
return false;
|
||||
}
|
||||
|
||||
if (LOG.isInfoEnabled()) {
|
||||
LOG.info("Privileged container requested for : " + container
|
||||
.getContainerId().toString());
|
||||
}
|
||||
LOG.info("Privileged container requested for : " + container
|
||||
.getContainerId().toString());
|
||||
|
||||
//Ok, so we have been asked to run a privileged container. Security
|
||||
// checks need to be run. Each violation is an error.
|
||||
|
|
@ -375,10 +408,8 @@ public class DockerLinuxContainerRuntime implements LinuxContainerRuntime {
|
|||
throw new ContainerExecutionException(message);
|
||||
}
|
||||
|
||||
if (LOG.isInfoEnabled()) {
|
||||
LOG.info("All checks pass. Launching privileged container for : "
|
||||
+ container.getContainerId().toString());
|
||||
}
|
||||
LOG.info("All checks pass. Launching privileged container for : "
|
||||
+ container.getContainerId().toString());
|
||||
|
||||
return true;
|
||||
}
|
||||
|
|
@ -413,6 +444,7 @@ public class DockerLinuxContainerRuntime implements LinuxContainerRuntime {
|
|||
.getEnvironment();
|
||||
String imageName = environment.get(ENV_DOCKER_CONTAINER_IMAGE);
|
||||
String network = environment.get(ENV_DOCKER_CONTAINER_NETWORK);
|
||||
String hostname = environment.get(ENV_DOCKER_CONTAINER_HOSTNAME);
|
||||
|
||||
if(network == null || network.isEmpty()) {
|
||||
network = defaultNetwork;
|
||||
|
|
@ -420,6 +452,8 @@ public class DockerLinuxContainerRuntime implements LinuxContainerRuntime {
|
|||
|
||||
validateContainerNetworkType(network);
|
||||
|
||||
validateHostname(hostname);
|
||||
|
||||
validateImageName(imageName);
|
||||
|
||||
String containerIdStr = container.getContainerId().toString();
|
||||
|
|
@ -450,12 +484,13 @@ public class DockerLinuxContainerRuntime implements LinuxContainerRuntime {
|
|||
runAsUser, imageName)
|
||||
.detachOnRun()
|
||||
.setContainerWorkDir(containerWorkDir.toString())
|
||||
.setNetworkType(network)
|
||||
.setCapabilities(capabilities)
|
||||
.setNetworkType(network);
|
||||
setHostname(runCommand, containerIdStr, hostname);
|
||||
runCommand.setCapabilities(capabilities)
|
||||
.addMountLocation(CGROUPS_ROOT_DIRECTORY,
|
||||
CGROUPS_ROOT_DIRECTORY + ":ro", false);
|
||||
List<String> allDirs = new ArrayList<>(containerLocalDirs);
|
||||
|
||||
List<String> allDirs = new ArrayList<>(containerLocalDirs);
|
||||
allDirs.addAll(filecacheDirs);
|
||||
allDirs.add(containerWorkDir.toString());
|
||||
allDirs.addAll(containerLogDirs);
|
||||
|
|
@ -493,9 +528,7 @@ public class DockerLinuxContainerRuntime implements LinuxContainerRuntime {
|
|||
ENV_DOCKER_CONTAINER_RUN_OVERRIDE_DISABLE);
|
||||
|
||||
if (disableOverride != null && disableOverride.equals("true")) {
|
||||
if (LOG.isInfoEnabled()) {
|
||||
LOG.info("command override disabled");
|
||||
}
|
||||
LOG.info("command override disabled");
|
||||
} else {
|
||||
List<String> overrideCommands = new ArrayList<>();
|
||||
Path launchDst =
|
||||
|
|
|
|||
|
|
@ -91,6 +91,12 @@ public class DockerRunCommand extends DockerCommand {
|
|||
|
||||
return this;
|
||||
}
|
||||
|
||||
public DockerRunCommand setHostname(String hostname) {
|
||||
super.addCommandArguments("--hostname=" + hostname);
|
||||
return this;
|
||||
}
|
||||
|
||||
public DockerRunCommand addDevice(String sourceDevice, String
|
||||
destinationDevice) {
|
||||
super.addCommandArguments("--device=" + sourceDevice + ":" +
|
||||
|
|
|
|||
|
|
@ -1215,6 +1215,7 @@ char* sanitize_docker_command(const char *line) {
|
|||
{"rm", no_argument, 0, 'r' },
|
||||
{"workdir", required_argument, 0, 'w' },
|
||||
{"net", required_argument, 0, 'e' },
|
||||
{"hostname", required_argument, 0, 'h' },
|
||||
{"cgroup-parent", required_argument, 0, 'g' },
|
||||
{"privileged", no_argument, 0, 'p' },
|
||||
{"cap-add", required_argument, 0, 'a' },
|
||||
|
|
@ -1256,6 +1257,9 @@ char* sanitize_docker_command(const char *line) {
|
|||
case 'e':
|
||||
quote_and_append_arg(&output, &output_size, "--net=", optarg);
|
||||
break;
|
||||
case 'h':
|
||||
quote_and_append_arg(&output, &output_size, "--hostname=", optarg);
|
||||
break;
|
||||
case 'v':
|
||||
quote_and_append_arg(&output, &output_size, "-v ", optarg);
|
||||
break;
|
||||
|
|
|
|||
|
|
@ -1088,17 +1088,17 @@ void test_trim_function() {
|
|||
void test_sanitize_docker_command() {
|
||||
|
||||
char *input[] = {
|
||||
"run --name=cname --user=nobody -d --workdir=/yarn/local/cdir --privileged --rm --device=/sys/fs/cgroup/device:/sys/fs/cgroup/device --detach=true --cgroup-parent=/sys/fs/cgroup/cpu/yarn/cid --net=host --cap-drop=ALL --cap-add=SYS_CHROOT --cap-add=MKNOD --cap-add=SETFCAP --cap-add=SETPCAP --cap-add=FSETID --cap-add=CHOWN --cap-add=AUDIT_WRITE --cap-add=SETGID --cap-add=NET_RAW --cap-add=FOWNER --cap-add=SETUID --cap-add=DAC_OVERRIDE --cap-add=KILL --cap-add=NET_BIND_SERVICE -v /sys/fs/cgroup:/sys/fs/cgroup:ro -v /yarn/local/cdir:/yarn/local/cdir -v /yarn/local/usercache/test/:/yarn/local/usercache/test/ ubuntu bash /yarn/local/usercache/test/appcache/aid/cid/launch_container.sh",
|
||||
"run --name=$CID --user=nobody -d --workdir=/yarn/local/cdir --privileged --rm --device=/sys/fs/cgroup/device:/sys/fs/cgroup/device --detach=true --cgroup-parent=/sys/fs/cgroup/cpu/yarn/cid --net=host --cap-drop=ALL --cap-add=SYS_CHROOT --cap-add=MKNOD --cap-add=SETFCAP --cap-add=SETPCAP --cap-add=FSETID --cap-add=CHOWN --cap-add=AUDIT_WRITE --cap-add=SETGID --cap-add=NET_RAW --cap-add=FOWNER --cap-add=SETUID --cap-add=DAC_OVERRIDE --cap-add=KILL --cap-add=NET_BIND_SERVICE -v /sys/fs/cgroup:/sys/fs/cgroup:ro -v /yarn/local/cdir:/yarn/local/cdir -v /yarn/local/usercache/test/:/yarn/local/usercache/test/ ubuntu bash /yarn/local/usercache/test/appcache/aid/cid/launch_container.sh",
|
||||
"run --name=cname --user=nobody -d --workdir=/yarn/local/cdir --privileged --rm --device=/sys/fs/cgroup/device:/sys/fs/cgroup/device --detach=true --cgroup-parent=/sys/fs/cgroup/cpu/yarn/cid --net=host --cap-drop=ALL --cap-add=SYS_CHROOT --cap-add=MKNOD --cap-add=SETFCAP --cap-add=SETPCAP --cap-add=FSETID --cap-add=CHOWN --cap-add=AUDIT_WRITE --cap-add=SETGID --cap-add=NET_RAW --cap-add=FOWNER --cap-add=SETUID --cap-add=DAC_OVERRIDE --cap-add=KILL --cap-add=NET_BIND_SERVICE -v /sys/fs/cgroup:/sys/fs/cgroup:ro -v /yarn/local/cdir:/yarn/local/cdir -v /yarn/local/usercache/test/:/yarn/local/usercache/test/ ubuntu || touch /tmp/file # bash /yarn/local/usercache/test/appcache/aid/cid/launch_container.sh",
|
||||
"run --name=cname --user=nobody -d --workdir=/yarn/local/cdir --privileged --rm --device=/sys/fs/cgroup/device:/sys/fs/cgroup/device --detach=true --cgroup-parent=/sys/fs/cgroup/cpu/yarn/cid --net=host --cap-drop=ALL --cap-add=SYS_CHROOT --cap-add=MKNOD --cap-add=SETFCAP --cap-add=SETPCAP --cap-add=FSETID --cap-add=CHOWN --cap-add=AUDIT_WRITE --cap-add=SETGID --cap-add=NET_RAW --cap-add=FOWNER --cap-add=SETUID --cap-add=DAC_OVERRIDE --cap-add=KILL --cap-add=NET_BIND_SERVICE -v /sys/fs/cgroup:/sys/fs/cgroup:ro -v /yarn/local/cdir:/yarn/local/cdir -v /yarn/local/usercache/test/:/yarn/local/usercache/test/ ubuntu' || touch /tmp/file # bash /yarn/local/usercache/test/appcache/aid/cid/launch_container.sh",
|
||||
"run --name=cname --user=nobody -d --workdir=/yarn/local/cdir --privileged --rm --device=/sys/fs/cgroup/device:/sys/fs/cgroup/device --detach=true --cgroup-parent=/sys/fs/cgroup/cpu/yarn/cid --net=host --hostname=test.host.name --cap-drop=ALL --cap-add=SYS_CHROOT --cap-add=MKNOD --cap-add=SETFCAP --cap-add=SETPCAP --cap-add=FSETID --cap-add=CHOWN --cap-add=AUDIT_WRITE --cap-add=SETGID --cap-add=NET_RAW --cap-add=FOWNER --cap-add=SETUID --cap-add=DAC_OVERRIDE --cap-add=KILL --cap-add=NET_BIND_SERVICE -v /sys/fs/cgroup:/sys/fs/cgroup:ro -v /yarn/local/cdir:/yarn/local/cdir -v /yarn/local/usercache/test/:/yarn/local/usercache/test/ ubuntu bash /yarn/local/usercache/test/appcache/aid/cid/launch_container.sh",
|
||||
"run --name=$CID --user=nobody -d --workdir=/yarn/local/cdir --privileged --rm --device=/sys/fs/cgroup/device:/sys/fs/cgroup/device --detach=true --cgroup-parent=/sys/fs/cgroup/cpu/yarn/cid --net=host --hostname=test.host.name --cap-drop=ALL --cap-add=SYS_CHROOT --cap-add=MKNOD --cap-add=SETFCAP --cap-add=SETPCAP --cap-add=FSETID --cap-add=CHOWN --cap-add=AUDIT_WRITE --cap-add=SETGID --cap-add=NET_RAW --cap-add=FOWNER --cap-add=SETUID --cap-add=DAC_OVERRIDE --cap-add=KILL --cap-add=NET_BIND_SERVICE -v /sys/fs/cgroup:/sys/fs/cgroup:ro -v /yarn/local/cdir:/yarn/local/cdir -v /yarn/local/usercache/test/:/yarn/local/usercache/test/ ubuntu bash /yarn/local/usercache/test/appcache/aid/cid/launch_container.sh",
|
||||
"run --name=cname --user=nobody -d --workdir=/yarn/local/cdir --privileged --rm --device=/sys/fs/cgroup/device:/sys/fs/cgroup/device --detach=true --cgroup-parent=/sys/fs/cgroup/cpu/yarn/cid --net=host --hostname=test.host.name --cap-drop=ALL --cap-add=SYS_CHROOT --cap-add=MKNOD --cap-add=SETFCAP --cap-add=SETPCAP --cap-add=FSETID --cap-add=CHOWN --cap-add=AUDIT_WRITE --cap-add=SETGID --cap-add=NET_RAW --cap-add=FOWNER --cap-add=SETUID --cap-add=DAC_OVERRIDE --cap-add=KILL --cap-add=NET_BIND_SERVICE -v /sys/fs/cgroup:/sys/fs/cgroup:ro -v /yarn/local/cdir:/yarn/local/cdir -v /yarn/local/usercache/test/:/yarn/local/usercache/test/ ubuntu || touch /tmp/file # bash /yarn/local/usercache/test/appcache/aid/cid/launch_container.sh",
|
||||
"run --name=cname --user=nobody -d --workdir=/yarn/local/cdir --privileged --rm --device=/sys/fs/cgroup/device:/sys/fs/cgroup/device --detach=true --cgroup-parent=/sys/fs/cgroup/cpu/yarn/cid --net=host --hostname=test.host.name --cap-drop=ALL --cap-add=SYS_CHROOT --cap-add=MKNOD --cap-add=SETFCAP --cap-add=SETPCAP --cap-add=FSETID --cap-add=CHOWN --cap-add=AUDIT_WRITE --cap-add=SETGID --cap-add=NET_RAW --cap-add=FOWNER --cap-add=SETUID --cap-add=DAC_OVERRIDE --cap-add=KILL --cap-add=NET_BIND_SERVICE -v /sys/fs/cgroup:/sys/fs/cgroup:ro -v /yarn/local/cdir:/yarn/local/cdir -v /yarn/local/usercache/test/:/yarn/local/usercache/test/ ubuntu' || touch /tmp/file # bash /yarn/local/usercache/test/appcache/aid/cid/launch_container.sh",
|
||||
"run ''''''''"
|
||||
};
|
||||
char *expected_output[] = {
|
||||
"run --name='cname' --user='nobody' -d --workdir='/yarn/local/cdir' --privileged --rm --device='/sys/fs/cgroup/device:/sys/fs/cgroup/device' --detach='true' --cgroup-parent='/sys/fs/cgroup/cpu/yarn/cid' --net='host' --cap-drop='ALL' --cap-add='SYS_CHROOT' --cap-add='MKNOD' --cap-add='SETFCAP' --cap-add='SETPCAP' --cap-add='FSETID' --cap-add='CHOWN' --cap-add='AUDIT_WRITE' --cap-add='SETGID' --cap-add='NET_RAW' --cap-add='FOWNER' --cap-add='SETUID' --cap-add='DAC_OVERRIDE' --cap-add='KILL' --cap-add='NET_BIND_SERVICE' -v '/sys/fs/cgroup:/sys/fs/cgroup:ro' -v '/yarn/local/cdir:/yarn/local/cdir' -v '/yarn/local/usercache/test/:/yarn/local/usercache/test/' 'ubuntu' 'bash' '/yarn/local/usercache/test/appcache/aid/cid/launch_container.sh' ",
|
||||
"run --name='$CID' --user='nobody' -d --workdir='/yarn/local/cdir' --privileged --rm --device='/sys/fs/cgroup/device:/sys/fs/cgroup/device' --detach='true' --cgroup-parent='/sys/fs/cgroup/cpu/yarn/cid' --net='host' --cap-drop='ALL' --cap-add='SYS_CHROOT' --cap-add='MKNOD' --cap-add='SETFCAP' --cap-add='SETPCAP' --cap-add='FSETID' --cap-add='CHOWN' --cap-add='AUDIT_WRITE' --cap-add='SETGID' --cap-add='NET_RAW' --cap-add='FOWNER' --cap-add='SETUID' --cap-add='DAC_OVERRIDE' --cap-add='KILL' --cap-add='NET_BIND_SERVICE' -v '/sys/fs/cgroup:/sys/fs/cgroup:ro' -v '/yarn/local/cdir:/yarn/local/cdir' -v '/yarn/local/usercache/test/:/yarn/local/usercache/test/' 'ubuntu' 'bash' '/yarn/local/usercache/test/appcache/aid/cid/launch_container.sh' ",
|
||||
"run --name='cname' --user='nobody' -d --workdir='/yarn/local/cdir' --privileged --rm --device='/sys/fs/cgroup/device:/sys/fs/cgroup/device' --detach='true' --cgroup-parent='/sys/fs/cgroup/cpu/yarn/cid' --net='host' --cap-drop='ALL' --cap-add='SYS_CHROOT' --cap-add='MKNOD' --cap-add='SETFCAP' --cap-add='SETPCAP' --cap-add='FSETID' --cap-add='CHOWN' --cap-add='AUDIT_WRITE' --cap-add='SETGID' --cap-add='NET_RAW' --cap-add='FOWNER' --cap-add='SETUID' --cap-add='DAC_OVERRIDE' --cap-add='KILL' --cap-add='NET_BIND_SERVICE' -v '/sys/fs/cgroup:/sys/fs/cgroup:ro' -v '/yarn/local/cdir:/yarn/local/cdir' -v '/yarn/local/usercache/test/:/yarn/local/usercache/test/' 'ubuntu' '||' 'touch' '/tmp/file' '#' 'bash' '/yarn/local/usercache/test/appcache/aid/cid/launch_container.sh' ",
|
||||
"run --name='cname' --user='nobody' -d --workdir='/yarn/local/cdir' --privileged --rm --device='/sys/fs/cgroup/device:/sys/fs/cgroup/device' --detach='true' --cgroup-parent='/sys/fs/cgroup/cpu/yarn/cid' --net='host' --cap-drop='ALL' --cap-add='SYS_CHROOT' --cap-add='MKNOD' --cap-add='SETFCAP' --cap-add='SETPCAP' --cap-add='FSETID' --cap-add='CHOWN' --cap-add='AUDIT_WRITE' --cap-add='SETGID' --cap-add='NET_RAW' --cap-add='FOWNER' --cap-add='SETUID' --cap-add='DAC_OVERRIDE' --cap-add='KILL' --cap-add='NET_BIND_SERVICE' -v '/sys/fs/cgroup:/sys/fs/cgroup:ro' -v '/yarn/local/cdir:/yarn/local/cdir' -v '/yarn/local/usercache/test/:/yarn/local/usercache/test/' 'ubuntu'\"'\"'' '||' 'touch' '/tmp/file' '#' 'bash' '/yarn/local/usercache/test/appcache/aid/cid/launch_container.sh' ",
|
||||
"run --name='cname' --user='nobody' -d --workdir='/yarn/local/cdir' --privileged --rm --device='/sys/fs/cgroup/device:/sys/fs/cgroup/device' --detach='true' --cgroup-parent='/sys/fs/cgroup/cpu/yarn/cid' --net='host' --hostname='test.host.name' --cap-drop='ALL' --cap-add='SYS_CHROOT' --cap-add='MKNOD' --cap-add='SETFCAP' --cap-add='SETPCAP' --cap-add='FSETID' --cap-add='CHOWN' --cap-add='AUDIT_WRITE' --cap-add='SETGID' --cap-add='NET_RAW' --cap-add='FOWNER' --cap-add='SETUID' --cap-add='DAC_OVERRIDE' --cap-add='KILL' --cap-add='NET_BIND_SERVICE' -v '/sys/fs/cgroup:/sys/fs/cgroup:ro' -v '/yarn/local/cdir:/yarn/local/cdir' -v '/yarn/local/usercache/test/:/yarn/local/usercache/test/' 'ubuntu' 'bash' '/yarn/local/usercache/test/appcache/aid/cid/launch_container.sh' ",
|
||||
"run --name='$CID' --user='nobody' -d --workdir='/yarn/local/cdir' --privileged --rm --device='/sys/fs/cgroup/device:/sys/fs/cgroup/device' --detach='true' --cgroup-parent='/sys/fs/cgroup/cpu/yarn/cid' --net='host' --hostname='test.host.name' --cap-drop='ALL' --cap-add='SYS_CHROOT' --cap-add='MKNOD' --cap-add='SETFCAP' --cap-add='SETPCAP' --cap-add='FSETID' --cap-add='CHOWN' --cap-add='AUDIT_WRITE' --cap-add='SETGID' --cap-add='NET_RAW' --cap-add='FOWNER' --cap-add='SETUID' --cap-add='DAC_OVERRIDE' --cap-add='KILL' --cap-add='NET_BIND_SERVICE' -v '/sys/fs/cgroup:/sys/fs/cgroup:ro' -v '/yarn/local/cdir:/yarn/local/cdir' -v '/yarn/local/usercache/test/:/yarn/local/usercache/test/' 'ubuntu' 'bash' '/yarn/local/usercache/test/appcache/aid/cid/launch_container.sh' ",
|
||||
"run --name='cname' --user='nobody' -d --workdir='/yarn/local/cdir' --privileged --rm --device='/sys/fs/cgroup/device:/sys/fs/cgroup/device' --detach='true' --cgroup-parent='/sys/fs/cgroup/cpu/yarn/cid' --net='host' --hostname='test.host.name' --cap-drop='ALL' --cap-add='SYS_CHROOT' --cap-add='MKNOD' --cap-add='SETFCAP' --cap-add='SETPCAP' --cap-add='FSETID' --cap-add='CHOWN' --cap-add='AUDIT_WRITE' --cap-add='SETGID' --cap-add='NET_RAW' --cap-add='FOWNER' --cap-add='SETUID' --cap-add='DAC_OVERRIDE' --cap-add='KILL' --cap-add='NET_BIND_SERVICE' -v '/sys/fs/cgroup:/sys/fs/cgroup:ro' -v '/yarn/local/cdir:/yarn/local/cdir' -v '/yarn/local/usercache/test/:/yarn/local/usercache/test/' 'ubuntu' '||' 'touch' '/tmp/file' '#' 'bash' '/yarn/local/usercache/test/appcache/aid/cid/launch_container.sh' ",
|
||||
"run --name='cname' --user='nobody' -d --workdir='/yarn/local/cdir' --privileged --rm --device='/sys/fs/cgroup/device:/sys/fs/cgroup/device' --detach='true' --cgroup-parent='/sys/fs/cgroup/cpu/yarn/cid' --net='host' --hostname='test.host.name' --cap-drop='ALL' --cap-add='SYS_CHROOT' --cap-add='MKNOD' --cap-add='SETFCAP' --cap-add='SETPCAP' --cap-add='FSETID' --cap-add='CHOWN' --cap-add='AUDIT_WRITE' --cap-add='SETGID' --cap-add='NET_RAW' --cap-add='FOWNER' --cap-add='SETUID' --cap-add='DAC_OVERRIDE' --cap-add='KILL' --cap-add='NET_BIND_SERVICE' -v '/sys/fs/cgroup:/sys/fs/cgroup:ro' -v '/yarn/local/cdir:/yarn/local/cdir' -v '/yarn/local/usercache/test/:/yarn/local/usercache/test/' 'ubuntu'\"'\"'' '||' 'touch' '/tmp/file' '#' 'bash' '/yarn/local/usercache/test/appcache/aid/cid/launch_container.sh' ",
|
||||
"run ''\"'\"''\"'\"''\"'\"''\"'\"''\"'\"''\"'\"''\"'\"''\"'\"'' ",
|
||||
};
|
||||
|
||||
|
|
|
|||
|
|
@ -25,6 +25,7 @@ import org.apache.commons.logging.LogFactory;
|
|||
import org.apache.hadoop.conf.Configuration;
|
||||
import org.apache.hadoop.fs.FileUtil;
|
||||
import org.apache.hadoop.fs.Path;
|
||||
import org.apache.hadoop.registry.client.binding.RegistryPathUtils;
|
||||
import org.apache.hadoop.yarn.api.records.ContainerId;
|
||||
import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
|
||||
import org.apache.hadoop.yarn.conf.YarnConfiguration;
|
||||
|
|
@ -69,6 +70,7 @@ public class TestDockerContainerRuntime {
|
|||
private PrivilegedOperationExecutor mockExecutor;
|
||||
private CGroupsHandler mockCGroupsHandler;
|
||||
private String containerId;
|
||||
private String defaultHostname;
|
||||
private Container container;
|
||||
private ContainerId cId;
|
||||
private ContainerLaunchContext context;
|
||||
|
|
@ -108,6 +110,7 @@ public class TestDockerContainerRuntime {
|
|||
.mock(PrivilegedOperationExecutor.class);
|
||||
mockCGroupsHandler = Mockito.mock(CGroupsHandler.class);
|
||||
containerId = "container_id";
|
||||
defaultHostname = RegistryPathUtils.encodeYarnID(containerId);
|
||||
container = mock(Container.class);
|
||||
cId = mock(ContainerId.class);
|
||||
context = mock(ContainerLaunchContext.class);
|
||||
|
|
@ -287,6 +290,7 @@ public class TestDockerContainerRuntime {
|
|||
.append("--user=%2$s -d ")
|
||||
.append("--workdir=%3$s ")
|
||||
.append("--net=host ")
|
||||
.append("--hostname=" + defaultHostname + " ")
|
||||
.append(getExpectedTestCapabilitiesArgumentString())
|
||||
.append(getExpectedCGroupsMountString())
|
||||
.append("-v %4$s:%4$s ")
|
||||
|
|
@ -365,7 +369,7 @@ public class TestDockerContainerRuntime {
|
|||
String disallowedNetwork = "sdn" + Integer.toString(randEngine.nextInt());
|
||||
|
||||
try {
|
||||
env.put("YARN_CONTAINER_RUNTIME_DOCKER_CONTAINER_NETWORK",
|
||||
env.put(DockerLinuxContainerRuntime.ENV_DOCKER_CONTAINER_NETWORK,
|
||||
disallowedNetwork);
|
||||
runtime.launchContainer(builder.build());
|
||||
Assert.fail("Network was expected to be disallowed: " +
|
||||
|
|
@ -378,8 +382,11 @@ public class TestDockerContainerRuntime {
|
|||
.DEFAULT_NM_DOCKER_ALLOWED_CONTAINER_NETWORKS.length;
|
||||
String allowedNetwork = YarnConfiguration
|
||||
.DEFAULT_NM_DOCKER_ALLOWED_CONTAINER_NETWORKS[randEngine.nextInt(size)];
|
||||
env.put("YARN_CONTAINER_RUNTIME_DOCKER_CONTAINER_NETWORK",
|
||||
env.put(DockerLinuxContainerRuntime.ENV_DOCKER_CONTAINER_NETWORK,
|
||||
allowedNetwork);
|
||||
String expectedHostname = "test.hostname";
|
||||
env.put(DockerLinuxContainerRuntime.ENV_DOCKER_CONTAINER_HOSTNAME,
|
||||
expectedHostname);
|
||||
|
||||
//this should cause no failures.
|
||||
|
||||
|
|
@ -393,6 +400,7 @@ public class TestDockerContainerRuntime {
|
|||
new StringBuffer("run --name=%1$s ").append("--user=%2$s -d ")
|
||||
.append("--workdir=%3$s ")
|
||||
.append("--net=" + allowedNetwork + " ")
|
||||
.append("--hostname=" + expectedHostname + " ")
|
||||
.append(getExpectedTestCapabilitiesArgumentString())
|
||||
.append(getExpectedCGroupsMountString())
|
||||
.append("-v %4$s:%4$s ").append("-v %5$s:%5$s ")
|
||||
|
|
@ -448,6 +456,7 @@ public class TestDockerContainerRuntime {
|
|||
new StringBuffer("run --name=%1$s ").append("--user=%2$s -d ")
|
||||
.append("--workdir=%3$s ")
|
||||
.append("--net=" + customNetwork1 + " ")
|
||||
.append("--hostname=" + defaultHostname + " ")
|
||||
.append(getExpectedTestCapabilitiesArgumentString())
|
||||
.append(getExpectedCGroupsMountString())
|
||||
.append("-v %4$s:%4$s ").append("-v %5$s:%5$s ")
|
||||
|
|
@ -471,7 +480,7 @@ public class TestDockerContainerRuntime {
|
|||
//now set an explicit (non-default) allowedNetwork and ensure that it is
|
||||
// used.
|
||||
|
||||
env.put("YARN_CONTAINER_RUNTIME_DOCKER_CONTAINER_NETWORK",
|
||||
env.put(DockerLinuxContainerRuntime.ENV_DOCKER_CONTAINER_NETWORK,
|
||||
customNetwork2);
|
||||
runtime.launchContainer(builder.build());
|
||||
|
||||
|
|
@ -485,6 +494,7 @@ public class TestDockerContainerRuntime {
|
|||
new StringBuffer("run --name=%1$s ").append("--user=%2$s -d ")
|
||||
.append("--workdir=%3$s ")
|
||||
.append("--net=" + customNetwork2 + " ")
|
||||
.append("--hostname=" + defaultHostname + " ")
|
||||
.append(getExpectedTestCapabilitiesArgumentString())
|
||||
.append(getExpectedCGroupsMountString())
|
||||
.append("-v %4$s:%4$s ").append("-v %5$s:%5$s ")
|
||||
|
|
@ -505,7 +515,7 @@ public class TestDockerContainerRuntime {
|
|||
|
||||
//disallowed network should trigger a launch failure
|
||||
|
||||
env.put("YARN_CONTAINER_RUNTIME_DOCKER_CONTAINER_NETWORK",
|
||||
env.put(DockerLinuxContainerRuntime.ENV_DOCKER_CONTAINER_NETWORK,
|
||||
customNetwork3);
|
||||
try {
|
||||
runtime.launchContainer(builder.build());
|
||||
|
|
@ -524,8 +534,8 @@ public class TestDockerContainerRuntime {
|
|||
mockExecutor, mockCGroupsHandler);
|
||||
runtime.initialize(conf);
|
||||
|
||||
env.put("YARN_CONTAINER_RUNTIME_DOCKER_RUN_PRIVILEGED_CONTAINER",
|
||||
"invalid-value");
|
||||
env.put(DockerLinuxContainerRuntime
|
||||
.ENV_DOCKER_CONTAINER_RUN_PRIVILEGED_CONTAINER, "invalid-value");
|
||||
runtime.launchContainer(builder.build());
|
||||
|
||||
PrivilegedOperation op = capturePrivilegedOperationAndVerifyArgs();
|
||||
|
|
@ -552,8 +562,8 @@ public class TestDockerContainerRuntime {
|
|||
mockExecutor, mockCGroupsHandler);
|
||||
runtime.initialize(conf);
|
||||
|
||||
env.put("YARN_CONTAINER_RUNTIME_DOCKER_RUN_PRIVILEGED_CONTAINER",
|
||||
"true");
|
||||
env.put(DockerLinuxContainerRuntime
|
||||
.ENV_DOCKER_CONTAINER_RUN_PRIVILEGED_CONTAINER, "true");
|
||||
|
||||
try {
|
||||
runtime.launchContainer(builder.build());
|
||||
|
|
@ -575,8 +585,8 @@ public class TestDockerContainerRuntime {
|
|||
mockExecutor, mockCGroupsHandler);
|
||||
runtime.initialize(conf);
|
||||
|
||||
env.put("YARN_CONTAINER_RUNTIME_DOCKER_RUN_PRIVILEGED_CONTAINER",
|
||||
"true");
|
||||
env.put(DockerLinuxContainerRuntime
|
||||
.ENV_DOCKER_CONTAINER_RUN_PRIVILEGED_CONTAINER, "true");
|
||||
//By default
|
||||
// yarn.nodemanager.runtime.linux.docker.privileged-containers.acl
|
||||
// is empty. So we expect this launch to fail.
|
||||
|
|
@ -605,8 +615,8 @@ public class TestDockerContainerRuntime {
|
|||
mockExecutor, mockCGroupsHandler);
|
||||
runtime.initialize(conf);
|
||||
|
||||
env.put("YARN_CONTAINER_RUNTIME_DOCKER_RUN_PRIVILEGED_CONTAINER",
|
||||
"true");
|
||||
env.put(DockerLinuxContainerRuntime
|
||||
.ENV_DOCKER_CONTAINER_RUN_PRIVILEGED_CONTAINER, "true");
|
||||
|
||||
try {
|
||||
runtime.launchContainer(builder.build());
|
||||
|
|
@ -632,8 +642,8 @@ public class TestDockerContainerRuntime {
|
|||
mockExecutor, mockCGroupsHandler);
|
||||
runtime.initialize(conf);
|
||||
|
||||
env.put("YARN_CONTAINER_RUNTIME_DOCKER_RUN_PRIVILEGED_CONTAINER",
|
||||
"true");
|
||||
env.put(DockerLinuxContainerRuntime
|
||||
.ENV_DOCKER_CONTAINER_RUN_PRIVILEGED_CONTAINER, "true");
|
||||
|
||||
runtime.launchContainer(builder.build());
|
||||
PrivilegedOperation op = capturePrivilegedOperationAndVerifyArgs();
|
||||
|
|
@ -927,4 +937,24 @@ public class TestDockerContainerRuntime {
|
|||
}
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testDockerHostnamePattern() throws Exception {
|
||||
String[] validNames = {"ab", "a.b.c.d", "a1-b.cd.ef", "0AB.", "C_D-"};
|
||||
|
||||
String[] invalidNames = {"a", "a#.b.c", "-a.b.c", "a@b.c", "a/b/c"};
|
||||
|
||||
for (String name : validNames) {
|
||||
DockerLinuxContainerRuntime.validateHostname(name);
|
||||
}
|
||||
|
||||
for (String name : invalidNames) {
|
||||
try {
|
||||
DockerLinuxContainerRuntime.validateHostname(name);
|
||||
Assert.fail(name + " is an invalid hostname and should fail the regex");
|
||||
} catch (ContainerExecutionException ce) {
|
||||
continue;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in New Issue