From cc2a2a8e063ecdb23216830be8418736af79fe7b Mon Sep 17 00:00:00 2001 From: Wei-Chiu Chuang Date: Fri, 13 Apr 2018 10:13:59 -0700 Subject: [PATCH] Revert "YARN-7810. Update TestDockerContainerRuntime to test with current user credential." This reverts commit 724bffdb89cd62cc90a1f49c7c5e40998dc1cc0f. --- .../runtime/TestDockerContainerRuntime.java | 178 ++++++++---------- 1 file changed, 82 insertions(+), 96 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java index ab38ea2f9b3..aef94a72980 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java @@ -81,8 +81,7 @@ public class TestDockerContainerRuntime { private HashMap env; private String image; private String uidGidPair; - private String runAsUser = System.getProperty("user.name"); - private String[] groups = {}; + private String runAsUser; private String user; private String appId; private String containerIdStr = containerId; @@ -131,37 +130,8 @@ public void setup() { when(context.getEnvironment()).thenReturn(env); when(container.getUser()).thenReturn(submittingUser); - // Get the running user's uid and gid for remap - String uid = ""; - String gid = ""; - Shell.ShellCommandExecutor shexec1 = new Shell.ShellCommandExecutor( - new String[]{"id", "-u", runAsUser}); - Shell.ShellCommandExecutor shexec2 = new Shell.ShellCommandExecutor( - new String[]{"id", "-g", runAsUser}); - Shell.ShellCommandExecutor shexec3 = new Shell.ShellCommandExecutor( - new String[]{"id", "-G", runAsUser}); - try { - shexec1.execute(); - // get rid of newline at the end - uid = shexec1.getOutput().replaceAll("\n$", ""); - } catch (Exception e) { - LOG.info("Could not run id -u command: " + e); - } - try { - shexec2.execute(); - // get rid of newline at the end - gid = shexec2.getOutput().replaceAll("\n$", ""); - } catch (Exception e) { - LOG.info("Could not run id -g command: " + e); - } - try { - shexec3.execute(); - groups = shexec3.getOutput().replace("\n", " ").split(" "); - } catch (Exception e) { - LOG.info("Could not run id -G command: " + e); - } - uidGidPair = uid + ":" + gid; - + uidGidPair = ""; + runAsUser = "run_as_user"; user = "user"; appId = "app_id"; containerIdStr = containerId; @@ -331,7 +301,7 @@ public void testDockerContainerLaunch() List dockerCommands = Files.readAllLines(Paths.get (dockerCommandFile), Charset.forName("UTF-8")); - int expected = 14; + int expected = 13; int counter = 0; Assert.assertEquals(expected, dockerCommands.size()); Assert.assertEquals("[docker-command-execution]", @@ -341,8 +311,6 @@ public void testDockerContainerLaunch() Assert.assertEquals(" cap-drop=ALL", dockerCommands.get(counter++)); Assert.assertEquals(" detach=true", dockerCommands.get(counter++)); Assert.assertEquals(" docker-command=run", dockerCommands.get(counter++)); - Assert.assertEquals(" group-add=" + String.join(",", groups), - dockerCommands.get(counter++)); Assert.assertEquals(" hostname=ctr-id", dockerCommands.get(counter++)); Assert .assertEquals(" image=busybox:latest", dockerCommands.get(counter++)); @@ -358,7 +326,7 @@ public void testDockerContainerLaunch() + "/test_container_log_dir:/test_container_log_dir," + "/test_user_local_dir:/test_user_local_dir", dockerCommands.get(counter++)); - Assert.assertEquals(" user=" + uidGidPair, dockerCommands.get(counter++)); + Assert.assertEquals(" user=run_as_user", dockerCommands.get(counter++)); Assert.assertEquals(" workdir=/test_container_work_dir", dockerCommands.get(counter++)); } @@ -369,6 +337,13 @@ public void testContainerLaunchWithUserRemapping() IOException { conf.setBoolean(YarnConfiguration.NM_DOCKER_ENABLE_USER_REMAPPING, true); + Shell.ShellCommandExecutor shexec = new Shell.ShellCommandExecutor( + new String[]{"whoami"}); + shexec.execute(); + // get rid of newline at the end + runAsUser = shexec.getOutput().replaceAll("\n$", ""); + builder.setExecutionAttribute(RUN_AS_USER, runAsUser); + DockerLinuxContainerRuntime runtime = new DockerLinuxContainerRuntime( mockExecutor, mockCGroupsHandler); runtime.initialize(conf); @@ -378,6 +353,37 @@ public void testContainerLaunchWithUserRemapping() List args = op.getArguments(); String dockerCommandFile = args.get(11); + String uid = ""; + String gid = ""; + String[] groups = {}; + Shell.ShellCommandExecutor shexec1 = new Shell.ShellCommandExecutor( + new String[]{"id", "-u", runAsUser}); + Shell.ShellCommandExecutor shexec2 = new Shell.ShellCommandExecutor( + new String[]{"id", "-g", runAsUser}); + Shell.ShellCommandExecutor shexec3 = new Shell.ShellCommandExecutor( + new String[]{"id", "-G", runAsUser}); + try { + shexec1.execute(); + // get rid of newline at the end + uid = shexec1.getOutput().replaceAll("\n$", ""); + } catch (Exception e) { + LOG.info("Could not run id -u command: " + e); + } + try { + shexec2.execute(); + // get rid of newline at the end + gid = shexec2.getOutput().replaceAll("\n$", ""); + } catch (Exception e) { + LOG.info("Could not run id -g command: " + e); + } + try { + shexec3.execute(); + groups = shexec3.getOutput().replace("\n", " ").split(" "); + } catch (Exception e) { + LOG.info("Could not run id -G command: " + e); + } + uidGidPair = uid + ":" + gid; + List dockerCommands = Files.readAllLines( Paths.get(dockerCommandFile), Charset.forName("UTF-8")); @@ -499,7 +505,7 @@ public void testContainerLaunchWithNetworkingDefaults() //This is the expected docker invocation for this case List dockerCommands = Files .readAllLines(Paths.get(dockerCommandFile), Charset.forName("UTF-8")); - int expected = 14; + int expected = 13; int counter = 0; Assert.assertEquals(expected, dockerCommands.size()); Assert.assertEquals("[docker-command-execution]", @@ -509,8 +515,6 @@ public void testContainerLaunchWithNetworkingDefaults() Assert.assertEquals(" cap-drop=ALL", dockerCommands.get(counter++)); Assert.assertEquals(" detach=true", dockerCommands.get(counter++)); Assert.assertEquals(" docker-command=run", dockerCommands.get(counter++)); - Assert.assertEquals(" group-add=" + String.join(",", groups), - dockerCommands.get(counter++)); Assert.assertEquals(" hostname=test.hostname", dockerCommands.get(counter++)); Assert @@ -528,7 +532,7 @@ public void testContainerLaunchWithNetworkingDefaults() + "/test_container_log_dir:/test_container_log_dir," + "/test_user_local_dir:/test_user_local_dir", dockerCommands.get(counter++)); - Assert.assertEquals(" user=" + uidGidPair, dockerCommands.get(counter++)); + Assert.assertEquals(" user=run_as_user", dockerCommands.get(counter++)); Assert.assertEquals(" workdir=/test_container_work_dir", dockerCommands.get(counter++)); } @@ -567,7 +571,7 @@ public void testContainerLaunchWithCustomNetworks() List dockerCommands = Files .readAllLines(Paths.get(dockerCommandFile), Charset.forName("UTF-8")); - int expected = 14; + int expected = 13; int counter = 0; Assert.assertEquals(expected, dockerCommands.size()); Assert.assertEquals("[docker-command-execution]", @@ -577,8 +581,6 @@ public void testContainerLaunchWithCustomNetworks() Assert.assertEquals(" cap-drop=ALL", dockerCommands.get(counter++)); Assert.assertEquals(" detach=true", dockerCommands.get(counter++)); Assert.assertEquals(" docker-command=run", dockerCommands.get(counter++)); - Assert.assertEquals(" group-add=" + String.join(",", groups), - dockerCommands.get(counter++)); Assert.assertEquals(" hostname=ctr-id", dockerCommands.get(counter++)); Assert .assertEquals(" image=busybox:latest", dockerCommands.get(counter++)); @@ -594,7 +596,7 @@ public void testContainerLaunchWithCustomNetworks() + "/test_container_log_dir:/test_container_log_dir," + "/test_user_local_dir:/test_user_local_dir", dockerCommands.get(counter++)); - Assert.assertEquals(" user=" + uidGidPair, dockerCommands.get(counter++)); + Assert.assertEquals(" user=run_as_user", dockerCommands.get(counter++)); Assert.assertEquals(" workdir=/test_container_work_dir", dockerCommands.get(counter++)); @@ -622,8 +624,6 @@ public void testContainerLaunchWithCustomNetworks() Assert.assertEquals(" cap-drop=ALL", dockerCommands.get(counter++)); Assert.assertEquals(" detach=true", dockerCommands.get(counter++)); Assert.assertEquals(" docker-command=run", dockerCommands.get(counter++)); - Assert.assertEquals(" group-add=" + String.join(",", groups), - dockerCommands.get(counter++)); Assert.assertEquals(" hostname=ctr-id", dockerCommands.get(counter++)); Assert .assertEquals(" image=busybox:latest", dockerCommands.get(counter++)); @@ -640,7 +640,7 @@ public void testContainerLaunchWithCustomNetworks() + "/test_container_log_dir:/test_container_log_dir," + "/test_user_local_dir:/test_user_local_dir", dockerCommands.get(counter++)); - Assert.assertEquals(" user=" + uidGidPair, dockerCommands.get(counter++)); + Assert.assertEquals(" user=run_as_user", dockerCommands.get(counter++)); Assert.assertEquals(" workdir=/test_container_work_dir", dockerCommands.get(counter++)); @@ -677,7 +677,7 @@ public void testLaunchPrivilegedContainersInvalidEnvVar() List dockerCommands = Files.readAllLines(Paths.get (dockerCommandFile), Charset.forName("UTF-8")); - int expected = 14; + int expected = 13; Assert.assertEquals(expected, dockerCommands.size()); String command = dockerCommands.get(0); @@ -786,7 +786,7 @@ public void testLaunchPrivilegedContainersWithEnabledSettingAndDefaultACL() List dockerCommands = Files.readAllLines(Paths.get (dockerCommandFile), Charset.forName("UTF-8")); - int expected = 15; + int expected = 14; int counter = 0; Assert.assertEquals(expected, dockerCommands.size()); Assert.assertEquals("[docker-command-execution]", @@ -796,8 +796,6 @@ public void testLaunchPrivilegedContainersWithEnabledSettingAndDefaultACL() Assert.assertEquals(" cap-drop=ALL", dockerCommands.get(counter++)); Assert.assertEquals(" detach=true", dockerCommands.get(counter++)); Assert.assertEquals(" docker-command=run", dockerCommands.get(counter++)); - Assert.assertEquals(" group-add=" + String.join(",", groups), - dockerCommands.get(counter++)); Assert.assertEquals(" hostname=ctr-id", dockerCommands.get(counter++)); Assert .assertEquals(" image=busybox:latest", dockerCommands.get(counter++)); @@ -814,7 +812,7 @@ public void testLaunchPrivilegedContainersWithEnabledSettingAndDefaultACL() + "/test_container_log_dir:/test_container_log_dir," + "/test_user_local_dir:/test_user_local_dir", dockerCommands.get(counter++)); - Assert.assertEquals(" user=" + uidGidPair, dockerCommands.get(counter++)); + Assert.assertEquals(" user=run_as_user", dockerCommands.get(counter++)); Assert.assertEquals(" workdir=/test_container_work_dir", dockerCommands.get(counter++)); } @@ -905,39 +903,33 @@ public void testMountSourceTarget() List dockerCommands = Files.readAllLines(Paths.get (dockerCommandFile), Charset.forName("UTF-8")); - int expected = 15; - int counter = 0; - Assert.assertEquals(expected, dockerCommands.size()); - Assert.assertEquals("[docker-command-execution]", - dockerCommands.get(counter++)); + Assert.assertEquals(14, dockerCommands.size()); + Assert.assertEquals("[docker-command-execution]", dockerCommands.get(0)); Assert.assertEquals(" cap-add=SYS_CHROOT,NET_BIND_SERVICE", - dockerCommands.get(counter++)); - Assert.assertEquals(" cap-drop=ALL", dockerCommands.get(counter++)); - Assert.assertEquals(" detach=true", dockerCommands.get(counter++)); - Assert.assertEquals(" docker-command=run", dockerCommands.get(counter++)); - Assert.assertEquals(" group-add=" + String.join(",", groups), - dockerCommands.get(counter++)); - Assert.assertEquals(" hostname=ctr-id", dockerCommands.get(counter++)); - Assert.assertEquals(" image=busybox:latest", - dockerCommands.get(counter++)); + dockerCommands.get(1)); + Assert.assertEquals(" cap-drop=ALL", dockerCommands.get(2)); + Assert.assertEquals(" detach=true", dockerCommands.get(3)); + Assert.assertEquals(" docker-command=run", dockerCommands.get(4)); + Assert.assertEquals(" hostname=ctr-id", dockerCommands.get(5)); + Assert.assertEquals(" image=busybox:latest", dockerCommands.get(6)); Assert.assertEquals( " launch-command=bash,/test_container_work_dir/launch_container.sh", - dockerCommands.get(counter++)); - Assert.assertEquals(" name=container_id", dockerCommands.get(counter++)); - Assert.assertEquals(" net=host", dockerCommands.get(counter++)); + dockerCommands.get(7)); + Assert.assertEquals(" name=container_id", dockerCommands.get(8)); + Assert.assertEquals(" net=host", dockerCommands.get(9)); Assert.assertEquals( " ro-mounts=/test_local_dir/test_resource_file:test_mount", - dockerCommands.get(counter++)); + dockerCommands.get(10)); Assert.assertEquals( " rw-mounts=/test_container_local_dir:/test_container_local_dir," + "/test_filecache_dir:/test_filecache_dir," + "/test_container_work_dir:/test_container_work_dir," + "/test_container_log_dir:/test_container_log_dir," + "/test_user_local_dir:/test_user_local_dir", - dockerCommands.get(counter++)); - Assert.assertEquals(" user=" + uidGidPair, dockerCommands.get(counter++)); + dockerCommands.get(11)); + Assert.assertEquals(" user=run_as_user", dockerCommands.get(12)); Assert.assertEquals(" workdir=/test_container_work_dir", - dockerCommands.get(counter++)); + dockerCommands.get(13)); } @Test @@ -981,40 +973,34 @@ public void testMountMultiple() List dockerCommands = Files.readAllLines(Paths.get (dockerCommandFile), Charset.forName("UTF-8")); - int expected = 15; - int counter = 0; - Assert.assertEquals(expected, dockerCommands.size()); - Assert.assertEquals("[docker-command-execution]", - dockerCommands.get(counter++)); + Assert.assertEquals(14, dockerCommands.size()); + Assert.assertEquals("[docker-command-execution]", dockerCommands.get(0)); Assert.assertEquals(" cap-add=SYS_CHROOT,NET_BIND_SERVICE", - dockerCommands.get(counter++)); - Assert.assertEquals(" cap-drop=ALL", dockerCommands.get(counter++)); - Assert.assertEquals(" detach=true", dockerCommands.get(counter++)); - Assert.assertEquals(" docker-command=run", dockerCommands.get(counter++)); - Assert.assertEquals(" group-add=" + String.join(",", groups), - dockerCommands.get(counter++)); - Assert.assertEquals(" hostname=ctr-id", dockerCommands.get(counter++)); - Assert.assertEquals(" image=busybox:latest", - dockerCommands.get(counter++)); + dockerCommands.get(1)); + Assert.assertEquals(" cap-drop=ALL", dockerCommands.get(2)); + Assert.assertEquals(" detach=true", dockerCommands.get(3)); + Assert.assertEquals(" docker-command=run", dockerCommands.get(4)); + Assert.assertEquals(" hostname=ctr-id", dockerCommands.get(5)); + Assert.assertEquals(" image=busybox:latest", dockerCommands.get(6)); Assert.assertEquals( " launch-command=bash,/test_container_work_dir/launch_container.sh", - dockerCommands.get(counter++)); - Assert.assertEquals(" name=container_id", dockerCommands.get(counter++)); - Assert.assertEquals(" net=host", dockerCommands.get(counter++)); + dockerCommands.get(7)); + Assert.assertEquals(" name=container_id", dockerCommands.get(8)); + Assert.assertEquals(" net=host", dockerCommands.get(9)); Assert.assertEquals( " ro-mounts=/test_local_dir/test_resource_file:test_mount1," + "/test_local_dir/test_resource_file:test_mount2", - dockerCommands.get(counter++)); + dockerCommands.get(10)); Assert.assertEquals( " rw-mounts=/test_container_local_dir:/test_container_local_dir," + "/test_filecache_dir:/test_filecache_dir," + "/test_container_work_dir:/test_container_work_dir," + "/test_container_log_dir:/test_container_log_dir," + "/test_user_local_dir:/test_user_local_dir", - dockerCommands.get(counter++)); - Assert.assertEquals(" user=" + uidGidPair, dockerCommands.get(counter++)); + dockerCommands.get(11)); + Assert.assertEquals(" user=run_as_user", dockerCommands.get(12)); Assert.assertEquals(" workdir=/test_container_work_dir", - dockerCommands.get(counter++)); + dockerCommands.get(13)); } @@ -1034,7 +1020,7 @@ public void testContainerLivelinessCheck() PrivilegedOperation op = capturePrivilegedOperation(); Assert.assertEquals(op.getOperationType(), PrivilegedOperation.OperationType.SIGNAL_CONTAINER); - Assert.assertEquals(runAsUser, op.getArguments().get(0)); + Assert.assertEquals("run_as_user", op.getArguments().get(0)); Assert.assertEquals("user", op.getArguments().get(1)); Assert.assertEquals("2", op.getArguments().get(2)); Assert.assertEquals("1234", op.getArguments().get(3));