Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

YARN-3344. Fix warning - procfs stat file is not in the expected format. (Ravindra Kumar Naik and Akira Ajisaka via kasha) 

(cherry picked from commit 77d5ce9069)
(cherry picked from commit 0ac8c098e8)
This commit is contained in:
Karthik Kambatla 2016-05-25 10:50:45 -07:00
parent 35456bb7c9
commit b491d24bb6
2 changed files with 39 additions and 38 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ProcfsBasedProcessTree.java
View File

@ -57,10 +57,10 @@ public class ProcfsBasedProcessTree extends ResourceCalculatorProcessTree {
private static final String PROCFS = "/proc/"; private static final String PROCFS = "/proc/";
private static final Pattern PROCFS_STAT_FILE_FORMAT = Pattern .compile( private static final Pattern PROCFS_STAT_FILE_FORMAT = Pattern.compile(
"^([0-9-]+)\\s([^\\s]+)\\s[^\\s]\\s([0-9-]+)\\s([0-9-]+)\\s([0-9-]+)\\s" + "^([\\d-]+)\\s\\(([^)]+)\\)\\s[^\\s]\\s([\\d-]+)\\s([\\d-]+)\\s" +
"([0-9-]+\\s){7}([0-9]+)\\s([0-9]+)\\s([0-9-]+\\s){7}([0-9]+)\\s([0-9]+)" + "([\\d-]+)\\s([\\d-]+\\s){7}(\\d+)\\s(\\d+)\\s([\\d-]+\\s){7}(\\d+)\\s" +
"(\\s[0-9-]+){15}"); "(\\d+)(\\s[\\d-]+){15}");
public static final String PROCFS_STAT_FILE = "stat"; public static final String PROCFS_STAT_FILE = "stat";
public static final String PROCFS_CMDLINE_FILE = "cmdline"; public static final String PROCFS_CMDLINE_FILE = "cmdline";
@ -539,8 +539,9 @@ public class ProcfsBasedProcessTree extends ResourceCalculatorProcessTree {
Matcher m = PROCFS_STAT_FILE_FORMAT.matcher(str); Matcher m = PROCFS_STAT_FILE_FORMAT.matcher(str);
boolean mat = m.find(); boolean mat = m.find();
if (mat) { if (mat) {
String processName = "(" + m.group(2) + ")";
// Set (name) (ppid) (pgrpId) (session) (utime) (stime) (vsize) (rss) // Set (name) (ppid) (pgrpId) (session) (utime) (stime) (vsize) (rss)
pinfo.updateProcessInfo(m.group(2), m.group(3), pinfo.updateProcessInfo(processName, m.group(3),
Integer.parseInt(m.group(4)), Integer.parseInt(m.group(5)), Integer.parseInt(m.group(4)), Integer.parseInt(m.group(5)),
Long.parseLong(m.group(7)), new BigInteger(m.group(8)), Long.parseLong(m.group(7)), new BigInteger(m.group(8)),
Long.parseLong(m.group(10)), Long.parseLong(m.group(11))); Long.parseLong(m.group(10)), Long.parseLong(m.group(11)));

66
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestProcfsBasedProcessTree.java
View File

@ -415,17 +415,17 @@ public class TestProcfsBasedProcessTree {
// assuming processes 100, 200, 300 are in tree and 400 is not. // assuming processes 100, 200, 300 are in tree and 400 is not.
ProcessStatInfo[] procInfos = new ProcessStatInfo[4]; ProcessStatInfo[] procInfos = new ProcessStatInfo[4];
procInfos[0] = procInfos[0] =
new ProcessStatInfo(new String[] { "100", "proc1", "1", "100", "100", new ProcessStatInfo(new String[]{"100", "proc1", "1", "100", "100",
"100000", "100", "1000", "200" }); "100000", "100", "1000", "200"});
procInfos[1] = procInfos[1] =
new ProcessStatInfo(new String[] { "200", "proc2", "100", "100", new ProcessStatInfo(new String[]{"200", "process two", "100", "100",
"100", "200000", "200", "2000", "400" }); "100", "200000", "200", "2000", "400"});
procInfos[2] = procInfos[2] =
new ProcessStatInfo(new String[] { "300", "proc3", "200", "100", new ProcessStatInfo(new String[]{"300", "proc3", "200", "100",
"100", "300000", "300", "3000", "600" }); "100", "300000", "300", "3000", "600"});
procInfos[3] = procInfos[3] =
new ProcessStatInfo(new String[] { "400", "proc4", "1", "400", "400", new ProcessStatInfo(new String[]{"400", "proc4", "1", "400", "400",
"400000", "400", "4000", "800" }); "400000", "400", "4000", "800"});
ProcessTreeSmapMemInfo[] memInfo = new ProcessTreeSmapMemInfo[4]; ProcessTreeSmapMemInfo[] memInfo = new ProcessTreeSmapMemInfo[4];
memInfo[0] = new ProcessTreeSmapMemInfo("100"); memInfo[0] = new ProcessTreeSmapMemInfo("100");
@ -482,11 +482,11 @@ public class TestProcfsBasedProcessTree {
// test the cpu time again to see if it cumulates // test the cpu time again to see if it cumulates
procInfos[0] = procInfos[0] =
new ProcessStatInfo(new String[] { "100", "proc1", "1", "100", "100", new ProcessStatInfo(new String[]{"100", "proc1", "1", "100", "100",
"100000", "100", "2000", "300" }); "100000", "100", "2000", "300"});
procInfos[1] = procInfos[1] =
new ProcessStatInfo(new String[] { "200", "proc2", "100", "100", new ProcessStatInfo(new String[]{"200", "process two", "100", "100",
"100", "200000", "200", "3000", "500" }); "100", "200000", "200", "3000", "500"});
writeStatFiles(procfsRootDir, pids, procInfos, memInfo); writeStatFiles(procfsRootDir, pids, procInfos, memInfo);
long elapsedTimeBetweenUpdatesMsec = 200000; long elapsedTimeBetweenUpdatesMsec = 200000;
@ -559,17 +559,17 @@ public class TestProcfsBasedProcessTree {
// assuming 100, 200 and 400 are in tree, 300 is not. // assuming 100, 200 and 400 are in tree, 300 is not.
ProcessStatInfo[] procInfos = new ProcessStatInfo[4]; ProcessStatInfo[] procInfos = new ProcessStatInfo[4];
procInfos[0] = procInfos[0] =
new ProcessStatInfo(new String[] { "100", "proc1", "1", "100", "100", new ProcessStatInfo(new String[]{"100", "proc1", "1", "100", "100",
"100000", "100" }); "100000", "100"});
procInfos[1] = procInfos[1] =
new ProcessStatInfo(new String[] { "200", "proc2", "100", "100", new ProcessStatInfo(new String[]{"200", "process two", "100", "100",
"100", "200000", "200" }); "100", "200000", "200"});
procInfos[2] = procInfos[2] =
new ProcessStatInfo(new String[] { "300", "proc3", "1", "300", "300", new ProcessStatInfo(new String[]{"300", "proc3", "1", "300", "300",
"300000", "300" }); "300000", "300"});
procInfos[3] = procInfos[3] =
new ProcessStatInfo(new String[] { "400", "proc4", "100", "100", new ProcessStatInfo(new String[]{"400", "proc4", "100", "100",
"100", "400000", "400" }); "100", "400000", "400"});
// write smap information invariably for testing // write smap information invariably for testing
ProcessTreeSmapMemInfo[] memInfo = new ProcessTreeSmapMemInfo[4]; ProcessTreeSmapMemInfo[] memInfo = new ProcessTreeSmapMemInfo[4];
memInfo[0] = new ProcessTreeSmapMemInfo("100"); memInfo[0] = new ProcessTreeSmapMemInfo("100");
@ -811,23 +811,23 @@ public class TestProcfsBasedProcessTree {
// Processes 200, 300, 400 and 500 are descendants of 100. 600 is not. // Processes 200, 300, 400 and 500 are descendants of 100. 600 is not.
ProcessStatInfo[] procInfos = new ProcessStatInfo[numProcesses]; ProcessStatInfo[] procInfos = new ProcessStatInfo[numProcesses];
procInfos[0] = procInfos[0] =
new ProcessStatInfo(new String[] { "100", "proc1", "1", "100", "100", new ProcessStatInfo(new String[]{"100", "proc1", "1", "100", "100",
"100000", "100", "1000", "200" }); "100000", "100", "1000", "200"});
procInfos[1] = procInfos[1] =
new ProcessStatInfo(new String[] { "200", "proc2", "100", "100", new ProcessStatInfo(new String[]{"200", "process two", "100", "100",
"100", "200000", "200", "2000", "400" }); "100", "200000", "200", "2000", "400"});
procInfos[2] = procInfos[2] =
new ProcessStatInfo(new String[] { "300", "proc3", "200", "100", new ProcessStatInfo(new String[]{"300", "proc3", "200", "100",
"100", "300000", "300", "3000", "600" }); "100", "300000", "300", "3000", "600"});
procInfos[3] = procInfos[3] =
new ProcessStatInfo(new String[] { "400", "proc4", "200", "100", new ProcessStatInfo(new String[]{"400", "proc4", "200", "100",
"100", "400000", "400", "4000", "800" }); "100", "400000", "400", "4000", "800"});
procInfos[4] = procInfos[4] =
new ProcessStatInfo(new String[] { "500", "proc5", "400", "100", new ProcessStatInfo(new String[]{"500", "proc5", "400", "100",
"100", "400000", "400", "4000", "800" }); "100", "400000", "400", "4000", "800"});
procInfos[5] = procInfos[5] =
new ProcessStatInfo(new String[] { "600", "proc6", "1", "1", "1", new ProcessStatInfo(new String[]{"600", "proc6", "1", "1", "1",
"400000", "400", "4000", "800" }); "400000", "400", "4000", "800"});
ProcessTreeSmapMemInfo[] memInfos = new ProcessTreeSmapMemInfo[6]; ProcessTreeSmapMemInfo[] memInfos = new ProcessTreeSmapMemInfo[6];
memInfos[0] = new ProcessTreeSmapMemInfo("100"); memInfos[0] = new ProcessTreeSmapMemInfo("100");
@ -839,7 +839,7 @@ public class TestProcfsBasedProcessTree {
String[] cmdLines = new String[numProcesses]; String[] cmdLines = new String[numProcesses];
cmdLines[0] = "proc1 arg1 arg2"; cmdLines[0] = "proc1 arg1 arg2";
cmdLines[1] = "proc2 arg3 arg4"; cmdLines[1] = "process two arg3 arg4";
cmdLines[2] = "proc3 arg5 arg6"; cmdLines[2] = "proc3 arg5 arg6";
cmdLines[3] = "proc4 arg7 arg8"; cmdLines[3] = "proc4 arg7 arg8";
cmdLines[4] = "proc5 arg9 arg10"; cmdLines[4] = "proc5 arg9 arg10";