YARN-8027. Setting hostname of docker container breaks for --net=host in docker 1.13. Contributed by Jim Brennan
This commit is contained in:
parent
a08921ca6c
commit
f480367af6
|
@ -133,7 +133,8 @@ import static org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.r
|
|||
* <li>
|
||||
* {@code YARN_CONTAINER_RUNTIME_DOCKER_CONTAINER_HOSTNAME} sets the
|
||||
* hostname to be used by the Docker container. If not specified, a
|
||||
* hostname will be derived from the container ID.
|
||||
* hostname will be derived from the container ID. This variable is
|
||||
* ignored if the network is 'host' and Registry DNS is not enabled.
|
||||
* </li>
|
||||
* <li>
|
||||
* {@code YARN_CONTAINER_RUNTIME_DOCKER_RUN_PRIVILEGED_CONTAINER}
|
||||
|
@ -785,7 +786,12 @@ public class DockerLinuxContainerRuntime implements LinuxContainerRuntime {
|
|||
.detachOnRun()
|
||||
.setContainerWorkDir(containerWorkDir.toString())
|
||||
.setNetworkType(network);
|
||||
setHostname(runCommand, containerIdStr, hostname);
|
||||
// Only add hostname if network is not host or if Registry DNS is enabled.
|
||||
if (!network.equalsIgnoreCase("host") ||
|
||||
conf.getBoolean(RegistryConstants.KEY_DNS_ENABLED,
|
||||
RegistryConstants.DEFAULT_DNS_ENABLED)) {
|
||||
setHostname(runCommand, containerIdStr, hostname);
|
||||
}
|
||||
runCommand.setCapabilities(capabilities);
|
||||
|
||||
runCommand.addAllReadWriteMountLocations(containerLogDirs);
|
||||
|
|
|
@ -25,6 +25,7 @@ import org.apache.hadoop.conf.Configuration;
|
|||
import org.apache.hadoop.fs.FileUtil;
|
||||
import org.apache.hadoop.fs.Path;
|
||||
import org.apache.hadoop.io.DataOutputBuffer;
|
||||
import org.apache.hadoop.registry.client.api.RegistryConstants;
|
||||
import org.apache.hadoop.registry.client.binding.RegistryPathUtils;
|
||||
import org.apache.hadoop.security.Credentials;
|
||||
import org.apache.hadoop.util.Shell;
|
||||
|
@ -202,6 +203,8 @@ public class TestDockerContainerRuntime {
|
|||
LOG.info("Could not run id -G command: " + e);
|
||||
}
|
||||
uidGidPair = uid + ":" + gid;
|
||||
// Prevent gid threshold failures for these tests
|
||||
conf.setInt(YarnConfiguration.NM_DOCKER_USER_REMAPPING_GID_THRESHOLD, 0);
|
||||
|
||||
user = "user";
|
||||
appId = "app_id";
|
||||
|
@ -340,7 +343,7 @@ public class TestDockerContainerRuntime {
|
|||
List<String> dockerCommands = Files.readAllLines(Paths.get
|
||||
(dockerCommandFile), Charset.forName("UTF-8"));
|
||||
|
||||
int expected = 15;
|
||||
int expected = 14;
|
||||
int counter = 0;
|
||||
Assert.assertEquals(expected, dockerCommands.size());
|
||||
Assert.assertEquals("[docker-command-execution]",
|
||||
|
@ -352,7 +355,6 @@ public class TestDockerContainerRuntime {
|
|||
Assert.assertEquals(" docker-command=run", dockerCommands.get(counter++));
|
||||
Assert.assertEquals(" group-add=" + String.join(",", groups),
|
||||
dockerCommands.get(counter++));
|
||||
Assert.assertEquals(" hostname=ctr-id", dockerCommands.get(counter++));
|
||||
Assert
|
||||
.assertEquals(" image=busybox:latest", dockerCommands.get(counter++));
|
||||
Assert.assertEquals(
|
||||
|
@ -390,7 +392,7 @@ public class TestDockerContainerRuntime {
|
|||
List<String> dockerCommands = Files.readAllLines(
|
||||
Paths.get(dockerCommandFile), Charset.forName("UTF-8"));
|
||||
|
||||
Assert.assertEquals(15, dockerCommands.size());
|
||||
Assert.assertEquals(14, dockerCommands.size());
|
||||
int counter = 0;
|
||||
Assert.assertEquals("[docker-command-execution]",
|
||||
dockerCommands.get(counter++));
|
||||
|
@ -401,8 +403,6 @@ public class TestDockerContainerRuntime {
|
|||
Assert.assertEquals(" docker-command=run", dockerCommands.get(counter++));
|
||||
Assert.assertEquals(" group-add=" + String.join(",", groups),
|
||||
dockerCommands.get(counter++));
|
||||
Assert.assertEquals(" hostname=ctr-id",
|
||||
dockerCommands.get(counter++));
|
||||
Assert
|
||||
.assertEquals(" image=busybox:latest", dockerCommands.get(counter++));
|
||||
Assert.assertEquals(
|
||||
|
@ -488,10 +488,7 @@ public class TestDockerContainerRuntime {
|
|||
LOG.info("Caught expected exception: " + e);
|
||||
}
|
||||
|
||||
int size = YarnConfiguration
|
||||
.DEFAULT_NM_DOCKER_ALLOWED_CONTAINER_NETWORKS.length;
|
||||
String allowedNetwork = YarnConfiguration
|
||||
.DEFAULT_NM_DOCKER_ALLOWED_CONTAINER_NETWORKS[randEngine.nextInt(size)];
|
||||
String allowedNetwork = "bridge";
|
||||
env.put(DockerLinuxContainerRuntime.ENV_DOCKER_CONTAINER_NETWORK,
|
||||
allowedNetwork);
|
||||
String expectedHostname = "test.hostname";
|
||||
|
@ -542,6 +539,64 @@ public class TestDockerContainerRuntime {
|
|||
dockerCommands.get(counter));
|
||||
}
|
||||
|
||||
@Test
|
||||
@SuppressWarnings("unchecked")
|
||||
public void testContainerLaunchWithHostDnsNetwork()
|
||||
throws ContainerExecutionException, IOException,
|
||||
PrivilegedOperationException {
|
||||
// Make it look like Registry DNS is enabled so we can test whether
|
||||
// hostname goes through
|
||||
conf.setBoolean(RegistryConstants.KEY_DNS_ENABLED, true);
|
||||
DockerLinuxContainerRuntime runtime =
|
||||
new DockerLinuxContainerRuntime(mockExecutor, mockCGroupsHandler);
|
||||
runtime.initialize(conf, null);
|
||||
|
||||
String expectedHostname = "test.hostname";
|
||||
env.put(DockerLinuxContainerRuntime.ENV_DOCKER_CONTAINER_HOSTNAME,
|
||||
expectedHostname);
|
||||
|
||||
runtime.launchContainer(builder.build());
|
||||
PrivilegedOperation op = capturePrivilegedOperationAndVerifyArgs();
|
||||
List<String> args = op.getArguments();
|
||||
String dockerCommandFile = args.get(11);
|
||||
|
||||
//This is the expected docker invocation for this case
|
||||
List<String> dockerCommands = Files
|
||||
.readAllLines(Paths.get(dockerCommandFile), Charset.forName("UTF-8"));
|
||||
int expected = 15;
|
||||
int counter = 0;
|
||||
Assert.assertEquals(expected, dockerCommands.size());
|
||||
Assert.assertEquals("[docker-command-execution]",
|
||||
dockerCommands.get(counter++));
|
||||
Assert.assertEquals(" cap-add=SYS_CHROOT,NET_BIND_SERVICE",
|
||||
dockerCommands.get(counter++));
|
||||
Assert.assertEquals(" cap-drop=ALL", dockerCommands.get(counter++));
|
||||
Assert.assertEquals(" detach=true", dockerCommands.get(counter++));
|
||||
Assert.assertEquals(" docker-command=run", dockerCommands.get(counter++));
|
||||
Assert.assertEquals(" group-add=" + String.join(",", groups),
|
||||
dockerCommands.get(counter++));
|
||||
Assert.assertEquals(" hostname=test.hostname",
|
||||
dockerCommands.get(counter++));
|
||||
Assert
|
||||
.assertEquals(" image=busybox:latest", dockerCommands.get(counter++));
|
||||
Assert.assertEquals(
|
||||
" launch-command=bash,/test_container_work_dir/launch_container.sh",
|
||||
dockerCommands.get(counter++));
|
||||
Assert.assertEquals(" name=container_id", dockerCommands.get(counter++));
|
||||
Assert
|
||||
.assertEquals(" net=host", dockerCommands.get(counter++));
|
||||
Assert.assertEquals(" ro-mounts=/test_filecache_dir:/test_filecache_dir,"
|
||||
+ "/test_user_filecache_dir:/test_user_filecache_dir",
|
||||
dockerCommands.get(counter++));
|
||||
Assert.assertEquals(
|
||||
" rw-mounts=/test_container_log_dir:/test_container_log_dir,"
|
||||
+ "/test_application_local_dir:/test_application_local_dir",
|
||||
dockerCommands.get(counter++));
|
||||
Assert.assertEquals(" user=" + uidGidPair, dockerCommands.get(counter++));
|
||||
Assert.assertEquals(" workdir=/test_container_work_dir",
|
||||
dockerCommands.get(counter));
|
||||
}
|
||||
|
||||
@Test
|
||||
@SuppressWarnings("unchecked")
|
||||
public void testContainerLaunchWithCustomNetworks()
|
||||
|
@ -686,7 +741,7 @@ public class TestDockerContainerRuntime {
|
|||
List<String> dockerCommands = Files.readAllLines(Paths.get
|
||||
(dockerCommandFile), Charset.forName("UTF-8"));
|
||||
|
||||
int expected = 15;
|
||||
int expected = 14;
|
||||
Assert.assertEquals(expected, dockerCommands.size());
|
||||
|
||||
String command = dockerCommands.get(0);
|
||||
|
@ -737,7 +792,7 @@ public class TestDockerContainerRuntime {
|
|||
List<String> dockerCommands = Files.readAllLines(
|
||||
Paths.get(dockerCommandFile), Charset.forName("UTF-8"));
|
||||
|
||||
int expected = 16;
|
||||
int expected = 15;
|
||||
int counter = 0;
|
||||
Assert.assertEquals(expected, dockerCommands.size());
|
||||
Assert.assertEquals("[docker-command-execution]",
|
||||
|
@ -749,7 +804,6 @@ public class TestDockerContainerRuntime {
|
|||
Assert.assertEquals(" docker-command=run", dockerCommands.get(counter++));
|
||||
Assert.assertEquals(" group-add=" + String.join(",", groups),
|
||||
dockerCommands.get(counter++));
|
||||
Assert.assertEquals(" hostname=ctr-id", dockerCommands.get(counter++));
|
||||
Assert
|
||||
.assertEquals(" image=busybox:latest", dockerCommands.get(counter++));
|
||||
Assert.assertEquals(
|
||||
|
@ -789,7 +843,7 @@ public class TestDockerContainerRuntime {
|
|||
List<String> dockerCommands = Files.readAllLines(
|
||||
Paths.get(dockerCommandFile), Charset.forName("UTF-8"));
|
||||
|
||||
int expected = 15;
|
||||
int expected = 14;
|
||||
Assert.assertEquals(expected, dockerCommands.size());
|
||||
|
||||
String command = dockerCommands.get(0);
|
||||
|
@ -895,7 +949,7 @@ public class TestDockerContainerRuntime {
|
|||
List<String> dockerCommands = Files.readAllLines(Paths.get
|
||||
(dockerCommandFile), Charset.forName("UTF-8"));
|
||||
|
||||
int expected = 16;
|
||||
int expected = 15;
|
||||
int counter = 0;
|
||||
Assert.assertEquals(expected, dockerCommands.size());
|
||||
Assert.assertEquals("[docker-command-execution]",
|
||||
|
@ -907,7 +961,6 @@ public class TestDockerContainerRuntime {
|
|||
Assert.assertEquals(" docker-command=run", dockerCommands.get(counter++));
|
||||
Assert.assertEquals(" group-add=" + String.join(",", groups),
|
||||
dockerCommands.get(counter++));
|
||||
Assert.assertEquals(" hostname=ctr-id", dockerCommands.get(counter++));
|
||||
Assert
|
||||
.assertEquals(" image=busybox:latest", dockerCommands.get(counter++));
|
||||
Assert.assertEquals(
|
||||
|
@ -1012,7 +1065,7 @@ public class TestDockerContainerRuntime {
|
|||
List<String> dockerCommands = Files.readAllLines(Paths.get
|
||||
(dockerCommandFile), Charset.forName("UTF-8"));
|
||||
|
||||
int expected = 15;
|
||||
int expected = 14;
|
||||
int counter = 0;
|
||||
Assert.assertEquals(expected, dockerCommands.size());
|
||||
Assert.assertEquals("[docker-command-execution]",
|
||||
|
@ -1024,7 +1077,6 @@ public class TestDockerContainerRuntime {
|
|||
Assert.assertEquals(" docker-command=run", dockerCommands.get(counter++));
|
||||
Assert.assertEquals(" group-add=" + String.join(",", groups),
|
||||
dockerCommands.get(counter++));
|
||||
Assert.assertEquals(" hostname=ctr-id", dockerCommands.get(counter++));
|
||||
Assert.assertEquals(" image=busybox:latest",
|
||||
dockerCommands.get(counter++));
|
||||
Assert.assertEquals(
|
||||
|
@ -1085,7 +1137,7 @@ public class TestDockerContainerRuntime {
|
|||
List<String> dockerCommands = Files.readAllLines(Paths.get
|
||||
(dockerCommandFile), Charset.forName("UTF-8"));
|
||||
|
||||
int expected = 15;
|
||||
int expected = 14;
|
||||
int counter = 0;
|
||||
Assert.assertEquals(expected, dockerCommands.size());
|
||||
Assert.assertEquals("[docker-command-execution]",
|
||||
|
@ -1097,7 +1149,6 @@ public class TestDockerContainerRuntime {
|
|||
Assert.assertEquals(" docker-command=run", dockerCommands.get(counter++));
|
||||
Assert.assertEquals(" group-add=" + String.join(",", groups),
|
||||
dockerCommands.get(counter++));
|
||||
Assert.assertEquals(" hostname=ctr-id", dockerCommands.get(counter++));
|
||||
Assert.assertEquals(" image=busybox:latest",
|
||||
dockerCommands.get(counter++));
|
||||
Assert.assertEquals(
|
||||
|
@ -1140,7 +1191,7 @@ public class TestDockerContainerRuntime {
|
|||
List<String> dockerCommands = Files.readAllLines(
|
||||
Paths.get(dockerCommandFile), Charset.forName("UTF-8"));
|
||||
|
||||
int expected = 15;
|
||||
int expected = 14;
|
||||
int counter = 0;
|
||||
Assert.assertEquals(expected, dockerCommands.size());
|
||||
Assert.assertEquals("[docker-command-execution]",
|
||||
|
@ -1152,7 +1203,6 @@ public class TestDockerContainerRuntime {
|
|||
Assert.assertEquals(" docker-command=run", dockerCommands.get(counter++));
|
||||
Assert.assertEquals(" group-add=" + String.join(",", groups),
|
||||
dockerCommands.get(counter++));
|
||||
Assert.assertEquals(" hostname=ctr-id", dockerCommands.get(counter++));
|
||||
Assert.assertEquals(" image=busybox:latest",
|
||||
dockerCommands.get(counter++));
|
||||
Assert.assertEquals(
|
||||
|
@ -1635,7 +1685,7 @@ public class TestDockerContainerRuntime {
|
|||
List<String> dockerCommands = Files.readAllLines(Paths.get
|
||||
(dockerCommandFile), Charset.forName("UTF-8"));
|
||||
|
||||
int expected = 16;
|
||||
int expected = 15;
|
||||
int counter = 0;
|
||||
Assert.assertEquals(expected, dockerCommands.size());
|
||||
Assert.assertEquals("[docker-command-execution]",
|
||||
|
@ -1647,7 +1697,6 @@ public class TestDockerContainerRuntime {
|
|||
Assert.assertEquals(" docker-command=run", dockerCommands.get(counter++));
|
||||
Assert.assertEquals(" group-add=" + String.join(",", groups),
|
||||
dockerCommands.get(counter++));
|
||||
Assert.assertEquals(" hostname=ctr-id", dockerCommands.get(counter++));
|
||||
Assert
|
||||
.assertEquals(" image=busybox:latest", dockerCommands.get(counter++));
|
||||
Assert.assertEquals(
|
||||
|
@ -1775,7 +1824,7 @@ public class TestDockerContainerRuntime {
|
|||
List<String> dockerCommands = Files
|
||||
.readAllLines(Paths.get(dockerCommandFile), Charset.forName("UTF-8"));
|
||||
|
||||
int expected = 16;
|
||||
int expected = 15;
|
||||
int counter = 0;
|
||||
Assert.assertEquals(expected, dockerCommands.size());
|
||||
Assert.assertEquals("[docker-command-execution]",
|
||||
|
@ -1789,7 +1838,6 @@ public class TestDockerContainerRuntime {
|
|||
dockerCommands.get(counter++));
|
||||
Assert.assertEquals(" group-add=" + String.join(",", groups),
|
||||
dockerCommands.get(counter++));
|
||||
Assert.assertEquals(" hostname=ctr-id", dockerCommands.get(counter++));
|
||||
Assert.assertEquals(" image=busybox:latest",
|
||||
dockerCommands.get(counter++));
|
||||
Assert.assertEquals(
|
||||
|
|
Loading…
Reference in New Issue