From 80135297462b6b1961940cb396daa983c690c6db Mon Sep 17 00:00:00 2001 From: Jason Lowe Date: Tue, 3 May 2016 17:27:28 +0000 Subject: [PATCH] YARN-4834. ProcfsBasedProcessTree doesn't track daemonized processes. Contributed by Nathan Roberts (cherry picked from commit c6b48391680c1b81a86aabc3ad4c725bfade6d2e) --- .../yarn/util/ProcfsBasedProcessTree.java | 19 ++++++++- .../yarn/util/TestProcfsBasedProcessTree.java | 39 +++++++++++-------- 2 files changed, 41 insertions(+), 17 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ProcfsBasedProcessTree.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ProcfsBasedProcessTree.java index 2345c6273ad..7510836a226 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ProcfsBasedProcessTree.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ProcfsBasedProcessTree.java @@ -216,7 +216,16 @@ public class ProcfsBasedProcessTree extends ResourceCalculatorProcessTree { String pID = entry.getKey(); if (!pID.equals("1")) { ProcessInfo pInfo = entry.getValue(); - ProcessInfo parentPInfo = allProcessInfo.get(pInfo.getPpid()); + String ppid = pInfo.getPpid(); + // If parent is init and process is not session leader, + // attach to sessionID + if (ppid.equals("1")) { + String sid = pInfo.getSessionId().toString(); + if (!pID.equals(sid)) { + ppid = sid; + } + } + ProcessInfo parentPInfo = allProcessInfo.get(ppid); if (parentPInfo != null) { parentPInfo.addChild(pInfo); } @@ -573,6 +582,14 @@ public class ProcfsBasedProcessTree extends ResourceCalculatorProcessTree { return pTree.substring(0, pTree.length()) + "]"; } +/** + * Returns boolean indicating whether pid + * is in process tree. + */ + public boolean contains(String pid) { + return processTree.containsKey(pid); + } + /** * * Class containing information of a process. diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestProcfsBasedProcessTree.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestProcfsBasedProcessTree.java index 59cef07d28c..c65a7874402 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestProcfsBasedProcessTree.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestProcfsBasedProcessTree.java @@ -36,6 +36,7 @@ import java.util.Vector; import java.util.regex.Matcher; import java.util.regex.Pattern; +import org.apache.commons.io.FileUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; @@ -65,7 +66,7 @@ public class TestProcfsBasedProcessTree { TestProcfsBasedProcessTree.class.getName() + "-localDir"); private ShellCommandExecutor shexec = null; - private String pidFile, lowestDescendant; + private String pidFile, lowestDescendant, lostDescendant; private String shellScript; private static final int N = 6; // Controls the RogueTask @@ -144,19 +145,17 @@ public class TestProcfsBasedProcessTree { lowestDescendant = TEST_ROOT_DIR + File.separator + "lowestDescendantPidFile"; + lostDescendant = + TEST_ROOT_DIR + File.separator + "lostDescendantPidFile"; // write to shell-script - try { - FileWriter fWriter = new FileWriter(shellScript); - fWriter.write("# rogue task\n" + "sleep 1\n" + "echo hello\n" - + "if [ $1 -ne 0 ]\n" + "then\n" + " sh " + shellScript - + " $(($1-1))\n" + "else\n" + " echo $$ > " + lowestDescendant + "\n" - + " while true\n do\n" + " sleep 5\n" + " done\n" + "fi"); - fWriter.close(); - } catch (IOException ioe) { - LOG.info("Error: " + ioe); - return; - } + File file = new File(shellScript); + FileUtils.writeStringToFile(file, "# rogue task\n" + "sleep 1\n" + "echo hello\n" + + "if [ $1 -ne 0 ]\n" + "then\n" + " sh " + shellScript + + " $(($1-1))\n" + "else\n" + " echo $$ > " + lowestDescendant + "\n" + + "(sleep 300&\n" + + "echo $! > " + lostDescendant + ")\n" + + " while true\n do\n" + " sleep 5\n" + " done\n" + "fi"); Thread t = new RogueTaskThread(); t.start(); @@ -179,6 +178,12 @@ public class TestProcfsBasedProcessTree { p.updateProcessTree(); // reconstruct LOG.info("ProcessTree: " + p.toString()); + // Verify the orphaned pid is In process tree + String lostpid = getPidFromPidFile(lostDescendant); + LOG.info("Orphaned pid: " + lostpid); + Assert.assertTrue("Child process owned by init escaped process tree.", + p.contains(lostpid)); + // Get the process-tree dump String processTreeDump = p.getProcessTreeDump(); @@ -230,10 +235,12 @@ public class TestProcfsBasedProcessTree { Assert.assertTrue( "vmem for the gone-process is " + p.getVirtualMemorySize() - + " . It should be zero.", p.getVirtualMemorySize() == 0); + + " . It should be UNAVAILABLE(-1).", + p.getVirtualMemorySize() == UNAVAILABLE); Assert.assertTrue( "vmem (old API) for the gone-process is " + p.getCumulativeVmem() - + " . It should be zero.", p.getCumulativeVmem() == 0); + + " . It should be UNAVAILABLE(-1).", + p.getCumulativeVmem() == UNAVAILABLE); Assert.assertTrue(p.toString().equals("[ ]")); } @@ -247,7 +254,7 @@ public class TestProcfsBasedProcessTree { } protected void destroyProcessTree(String pid) throws IOException { - sendSignal(pid, 9); + sendSignal("-"+pid, 9); } /** @@ -917,7 +924,7 @@ public class TestProcfsBasedProcessTree { private static void sendSignal(String pid, int signal) throws IOException { ShellCommandExecutor shexec = null; - String[] arg = { "kill", "-" + signal, pid }; + String[] arg = { "kill", "-" + signal, "--", pid }; shexec = new ShellCommandExecutor(arg); shexec.execute(); }