Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

YARN-3344. Fix warning - procfs stat file is not in the expected format. (Ravindra Kumar Naik and Akira Ajisaka via kasha) 

(cherry picked from commit 77d5ce9069)
This commit is contained in:
Karthik Kambatla 2016-05-25 10:50:45 -07:00
parent 71e5b36763
commit 0ac8c098e8
2 changed files with 39 additions and 38 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ProcfsBasedProcessTree.java
View File

@ -57,10 +57,10 @@ public class ProcfsBasedProcessTree extends ResourceCalculatorProcessTree {
private static final String PROCFS = "/proc/"; private static final String PROCFS = "/proc/";
private static final Pattern PROCFS_STAT_FILE_FORMAT = Pattern .compile( private static final Pattern PROCFS_STAT_FILE_FORMAT = Pattern.compile(
"^([0-9-]+)\\s([^\\s]+)\\s[^\\s]\\s([0-9-]+)\\s([0-9-]+)\\s([0-9-]+)\\s" + "^([\\d-]+)\\s\\(([^)]+)\\)\\s[^\\s]\\s([\\d-]+)\\s([\\d-]+)\\s" +
"([0-9-]+\\s){7}([0-9]+)\\s([0-9]+)\\s([0-9-]+\\s){7}([0-9]+)\\s([0-9]+)" + "([\\d-]+)\\s([\\d-]+\\s){7}(\\d+)\\s(\\d+)\\s([\\d-]+\\s){7}(\\d+)\\s" +
"(\\s[0-9-]+){15}"); "(\\d+)(\\s[\\d-]+){15}");
public static final String PROCFS_STAT_FILE = "stat"; public static final String PROCFS_STAT_FILE = "stat";
public static final String PROCFS_CMDLINE_FILE = "cmdline"; public static final String PROCFS_CMDLINE_FILE = "cmdline";
@ -537,8 +537,9 @@ private static ProcessInfo constructProcessInfo(ProcessInfo pinfo,
Matcher m = PROCFS_STAT_FILE_FORMAT.matcher(str); Matcher m = PROCFS_STAT_FILE_FORMAT.matcher(str);
boolean mat = m.find(); boolean mat = m.find();
if (mat) { if (mat) {
String processName = "(" + m.group(2) + ")";
// Set (name) (ppid) (pgrpId) (session) (utime) (stime) (vsize) (rss) // Set (name) (ppid) (pgrpId) (session) (utime) (stime) (vsize) (rss)
pinfo.updateProcessInfo(m.group(2), m.group(3), pinfo.updateProcessInfo(processName, m.group(3),
Integer.parseInt(m.group(4)), Integer.parseInt(m.group(5)), Integer.parseInt(m.group(4)), Integer.parseInt(m.group(5)),
Long.parseLong(m.group(7)), new BigInteger(m.group(8)), Long.parseLong(m.group(7)), new BigInteger(m.group(8)),
Long.parseLong(m.group(10)), Long.parseLong(m.group(11))); Long.parseLong(m.group(10)), Long.parseLong(m.group(11)));

66
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestProcfsBasedProcessTree.java
View File

@ -414,17 +414,17 @@ public void testCpuAndMemoryForProcessTree() throws IOException {
// assuming processes 100, 200, 300 are in tree and 400 is not. // assuming processes 100, 200, 300 are in tree and 400 is not.
ProcessStatInfo[] procInfos = new ProcessStatInfo[4]; ProcessStatInfo[] procInfos = new ProcessStatInfo[4];
procInfos[0] = procInfos[0] =
new ProcessStatInfo(new String[] { "100", "proc1", "1", "100", "100", new ProcessStatInfo(new String[]{"100", "proc1", "1", "100", "100",
"100000", "100", "1000", "200" }); "100000", "100", "1000", "200"});
procInfos[1] = procInfos[1] =
new ProcessStatInfo(new String[] { "200", "proc2", "100", "100", new ProcessStatInfo(new String[]{"200", "process two", "100", "100",
"100", "200000", "200", "2000", "400" }); "100", "200000", "200", "2000", "400"});
procInfos[2] = procInfos[2] =
new ProcessStatInfo(new String[] { "300", "proc3", "200", "100", new ProcessStatInfo(new String[]{"300", "proc3", "200", "100",
"100", "300000", "300", "3000", "600" }); "100", "300000", "300", "3000", "600"});
procInfos[3] = procInfos[3] =
new ProcessStatInfo(new String[] { "400", "proc4", "1", "400", "400", new ProcessStatInfo(new String[]{"400", "proc4", "1", "400", "400",
"400000", "400", "4000", "800" }); "400000", "400", "4000", "800"});
ProcessTreeSmapMemInfo[] memInfo = new ProcessTreeSmapMemInfo[4]; ProcessTreeSmapMemInfo[] memInfo = new ProcessTreeSmapMemInfo[4];
memInfo[0] = new ProcessTreeSmapMemInfo("100"); memInfo[0] = new ProcessTreeSmapMemInfo("100");
@ -481,11 +481,11 @@ public void testCpuAndMemoryForProcessTree() throws IOException {
// test the cpu time again to see if it cumulates // test the cpu time again to see if it cumulates
procInfos[0] = procInfos[0] =
new ProcessStatInfo(new String[] { "100", "proc1", "1", "100", "100", new ProcessStatInfo(new String[]{"100", "proc1", "1", "100", "100",
"100000", "100", "2000", "300" }); "100000", "100", "2000", "300"});
procInfos[1] = procInfos[1] =
new ProcessStatInfo(new String[] { "200", "proc2", "100", "100", new ProcessStatInfo(new String[]{"200", "process two", "100", "100",
"100", "200000", "200", "3000", "500" }); "100", "200000", "200", "3000", "500"});
writeStatFiles(procfsRootDir, pids, procInfos, memInfo); writeStatFiles(procfsRootDir, pids, procInfos, memInfo);
long elapsedTimeBetweenUpdatesMsec = 200000; long elapsedTimeBetweenUpdatesMsec = 200000;
@ -558,17 +558,17 @@ private void testMemForOlderProcesses(boolean smapEnabled) throws IOException {
// assuming 100, 200 and 400 are in tree, 300 is not. // assuming 100, 200 and 400 are in tree, 300 is not.
ProcessStatInfo[] procInfos = new ProcessStatInfo[4]; ProcessStatInfo[] procInfos = new ProcessStatInfo[4];
procInfos[0] = procInfos[0] =
new ProcessStatInfo(new String[] { "100", "proc1", "1", "100", "100", new ProcessStatInfo(new String[]{"100", "proc1", "1", "100", "100",
"100000", "100" }); "100000", "100"});
procInfos[1] = procInfos[1] =
new ProcessStatInfo(new String[] { "200", "proc2", "100", "100", new ProcessStatInfo(new String[]{"200", "process two", "100", "100",
"100", "200000", "200" }); "100", "200000", "200"});
procInfos[2] = procInfos[2] =
new ProcessStatInfo(new String[] { "300", "proc3", "1", "300", "300", new ProcessStatInfo(new String[]{"300", "proc3", "1", "300", "300",
"300000", "300" }); "300000", "300"});
procInfos[3] = procInfos[3] =
new ProcessStatInfo(new String[] { "400", "proc4", "100", "100", new ProcessStatInfo(new String[]{"400", "proc4", "100", "100",
"100", "400000", "400" }); "100", "400000", "400"});
// write smap information invariably for testing // write smap information invariably for testing
ProcessTreeSmapMemInfo[] memInfo = new ProcessTreeSmapMemInfo[4]; ProcessTreeSmapMemInfo[] memInfo = new ProcessTreeSmapMemInfo[4];
memInfo[0] = new ProcessTreeSmapMemInfo("100"); memInfo[0] = new ProcessTreeSmapMemInfo("100");
@ -804,23 +804,23 @@ public void testProcessTreeDump() throws IOException {
// Processes 200, 300, 400 and 500 are descendants of 100. 600 is not. // Processes 200, 300, 400 and 500 are descendants of 100. 600 is not.
ProcessStatInfo[] procInfos = new ProcessStatInfo[numProcesses]; ProcessStatInfo[] procInfos = new ProcessStatInfo[numProcesses];
procInfos[0] = procInfos[0] =
new ProcessStatInfo(new String[] { "100", "proc1", "1", "100", "100", new ProcessStatInfo(new String[]{"100", "proc1", "1", "100", "100",
"100000", "100", "1000", "200" }); "100000", "100", "1000", "200"});
procInfos[1] = procInfos[1] =
new ProcessStatInfo(new String[] { "200", "proc2", "100", "100", new ProcessStatInfo(new String[]{"200", "process two", "100", "100",
"100", "200000", "200", "2000", "400" }); "100", "200000", "200", "2000", "400"});
procInfos[2] = procInfos[2] =
new ProcessStatInfo(new String[] { "300", "proc3", "200", "100", new ProcessStatInfo(new String[]{"300", "proc3", "200", "100",
"100", "300000", "300", "3000", "600" }); "100", "300000", "300", "3000", "600"});
procInfos[3] = procInfos[3] =
new ProcessStatInfo(new String[] { "400", "proc4", "200", "100", new ProcessStatInfo(new String[]{"400", "proc4", "200", "100",
"100", "400000", "400", "4000", "800" }); "100", "400000", "400", "4000", "800"});
procInfos[4] = procInfos[4] =
new ProcessStatInfo(new String[] { "500", "proc5", "400", "100", new ProcessStatInfo(new String[]{"500", "proc5", "400", "100",
"100", "400000", "400", "4000", "800" }); "100", "400000", "400", "4000", "800"});
procInfos[5] = procInfos[5] =
new ProcessStatInfo(new String[] { "600", "proc6", "1", "1", "1", new ProcessStatInfo(new String[]{"600", "proc6", "1", "1", "1",
"400000", "400", "4000", "800" }); "400000", "400", "4000", "800"});
ProcessTreeSmapMemInfo[] memInfos = new ProcessTreeSmapMemInfo[6]; ProcessTreeSmapMemInfo[] memInfos = new ProcessTreeSmapMemInfo[6];
memInfos[0] = new ProcessTreeSmapMemInfo("100"); memInfos[0] = new ProcessTreeSmapMemInfo("100");
@ -832,7 +832,7 @@ public void testProcessTreeDump() throws IOException {
String[] cmdLines = new String[numProcesses]; String[] cmdLines = new String[numProcesses];
cmdLines[0] = "proc1 arg1 arg2"; cmdLines[0] = "proc1 arg1 arg2";
cmdLines[1] = "proc2 arg3 arg4"; cmdLines[1] = "process two arg3 arg4";
cmdLines[2] = "proc3 arg5 arg6"; cmdLines[2] = "proc3 arg5 arg6";
cmdLines[3] = "proc4 arg7 arg8"; cmdLines[3] = "proc4 arg7 arg8";
cmdLines[4] = "proc5 arg9 arg10"; cmdLines[4] = "proc5 arg9 arg10";