Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

YARN-3344. Fix warning - procfs stat file is not in the expected format. (Ravindra Kumar Naik and Akira Ajisaka via kasha) 

(cherry picked from commit 77d5ce9069)
(cherry picked from commit 0ac8c098e8)
This commit is contained in:
Karthik Kambatla 2016-05-25 10:50:45 -07:00
parent 35456bb7c9
commit b491d24bb6
2 changed files with 39 additions and 38 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ProcfsBasedProcessTree.java
View File

@ -57,10 +57,10 @@ public class ProcfsBasedProcessTree extends ResourceCalculatorProcessTree {
private static final String PROCFS = "/proc/";
private static final Pattern PROCFS_STAT_FILE_FORMAT = Pattern .compile(
"^([0-9-]+)\\s([^\\s]+)\\s[^\\s]\\s([0-9-]+)\\s([0-9-]+)\\s([0-9-]+)\\s" +
"([0-9-]+\\s){7}([0-9]+)\\s([0-9]+)\\s([0-9-]+\\s){7}([0-9]+)\\s([0-9]+)" +
"(\\s[0-9-]+){15}");
private static final Pattern PROCFS_STAT_FILE_FORMAT = Pattern.compile(
"^([\\d-]+)\\s\\(([^)]+)\\)\\s[^\\s]\\s([\\d-]+)\\s([\\d-]+)\\s" +
"([\\d-]+)\\s([\\d-]+\\s){7}(\\d+)\\s(\\d+)\\s([\\d-]+\\s){7}(\\d+)\\s" +
"(\\d+)(\\s[\\d-]+){15}");
public static final String PROCFS_STAT_FILE = "stat";
public static final String PROCFS_CMDLINE_FILE = "cmdline";
@ -539,8 +539,9 @@ private static ProcessInfo constructProcessInfo(ProcessInfo pinfo,
Matcher m = PROCFS_STAT_FILE_FORMAT.matcher(str);
boolean mat = m.find();
if (mat) {
String processName = "(" + m.group(2) + ")";
// Set (name) (ppid) (pgrpId) (session) (utime) (stime) (vsize) (rss)
pinfo.updateProcessInfo(m.group(2), m.group(3),
pinfo.updateProcessInfo(processName, m.group(3),
Integer.parseInt(m.group(4)), Integer.parseInt(m.group(5)),
Long.parseLong(m.group(7)), new BigInteger(m.group(8)),
Long.parseLong(m.group(10)), Long.parseLong(m.group(11)));

66
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestProcfsBasedProcessTree.java
View File

@ -415,17 +415,17 @@ public void testCpuAndMemoryForProcessTree() throws IOException {
// assuming processes 100, 200, 300 are in tree and 400 is not.
ProcessStatInfo[] procInfos = new ProcessStatInfo[4];
procInfos[0] =
new ProcessStatInfo(new String[] { "100", "proc1", "1", "100", "100",
"100000", "100", "1000", "200" });
new ProcessStatInfo(new String[]{"100", "proc1", "1", "100", "100",
"100000", "100", "1000", "200"});
procInfos[1] =
new ProcessStatInfo(new String[] { "200", "proc2", "100", "100",
"100", "200000", "200", "2000", "400" });
new ProcessStatInfo(new String[]{"200", "process two", "100", "100",
"100", "200000", "200", "2000", "400"});
procInfos[2] =
new ProcessStatInfo(new String[] { "300", "proc3", "200", "100",
"100", "300000", "300", "3000", "600" });
new ProcessStatInfo(new String[]{"300", "proc3", "200", "100",
"100", "300000", "300", "3000", "600"});
procInfos[3] =
new ProcessStatInfo(new String[] { "400", "proc4", "1", "400", "400",
"400000", "400", "4000", "800" });
new ProcessStatInfo(new String[]{"400", "proc4", "1", "400", "400",
"400000", "400", "4000", "800"});
ProcessTreeSmapMemInfo[] memInfo = new ProcessTreeSmapMemInfo[4];
memInfo[0] = new ProcessTreeSmapMemInfo("100");
@ -482,11 +482,11 @@ public void testCpuAndMemoryForProcessTree() throws IOException {
// test the cpu time again to see if it cumulates
procInfos[0] =
new ProcessStatInfo(new String[] { "100", "proc1", "1", "100", "100",
"100000", "100", "2000", "300" });
new ProcessStatInfo(new String[]{"100", "proc1", "1", "100", "100",
"100000", "100", "2000", "300"});
procInfos[1] =
new ProcessStatInfo(new String[] { "200", "proc2", "100", "100",
"100", "200000", "200", "3000", "500" });
new ProcessStatInfo(new String[]{"200", "process two", "100", "100",
"100", "200000", "200", "3000", "500"});
writeStatFiles(procfsRootDir, pids, procInfos, memInfo);
long elapsedTimeBetweenUpdatesMsec = 200000;
@ -559,17 +559,17 @@ private void testMemForOlderProcesses(boolean smapEnabled) throws IOException {
// assuming 100, 200 and 400 are in tree, 300 is not.
ProcessStatInfo[] procInfos = new ProcessStatInfo[4];
procInfos[0] =
new ProcessStatInfo(new String[] { "100", "proc1", "1", "100", "100",
"100000", "100" });
new ProcessStatInfo(new String[]{"100", "proc1", "1", "100", "100",
"100000", "100"});
procInfos[1] =
new ProcessStatInfo(new String[] { "200", "proc2", "100", "100",
"100", "200000", "200" });
new ProcessStatInfo(new String[]{"200", "process two", "100", "100",
"100", "200000", "200"});
procInfos[2] =
new ProcessStatInfo(new String[] { "300", "proc3", "1", "300", "300",
"300000", "300" });
new ProcessStatInfo(new String[]{"300", "proc3", "1", "300", "300",
"300000", "300"});
procInfos[3] =
new ProcessStatInfo(new String[] { "400", "proc4", "100", "100",
"100", "400000", "400" });
new ProcessStatInfo(new String[]{"400", "proc4", "100", "100",
"100", "400000", "400"});
// write smap information invariably for testing
ProcessTreeSmapMemInfo[] memInfo = new ProcessTreeSmapMemInfo[4];
memInfo[0] = new ProcessTreeSmapMemInfo("100");
@ -811,23 +811,23 @@ public void testProcessTreeDump() throws IOException {
// Processes 200, 300, 400 and 500 are descendants of 100. 600 is not.
ProcessStatInfo[] procInfos = new ProcessStatInfo[numProcesses];
procInfos[0] =
new ProcessStatInfo(new String[] { "100", "proc1", "1", "100", "100",
"100000", "100", "1000", "200" });
new ProcessStatInfo(new String[]{"100", "proc1", "1", "100", "100",
"100000", "100", "1000", "200"});
procInfos[1] =
new ProcessStatInfo(new String[] { "200", "proc2", "100", "100",
"100", "200000", "200", "2000", "400" });
new ProcessStatInfo(new String[]{"200", "process two", "100", "100",
"100", "200000", "200", "2000", "400"});
procInfos[2] =
new ProcessStatInfo(new String[] { "300", "proc3", "200", "100",
"100", "300000", "300", "3000", "600" });
new ProcessStatInfo(new String[]{"300", "proc3", "200", "100",
"100", "300000", "300", "3000", "600"});
procInfos[3] =
new ProcessStatInfo(new String[] { "400", "proc4", "200", "100",
"100", "400000", "400", "4000", "800" });
new ProcessStatInfo(new String[]{"400", "proc4", "200", "100",
"100", "400000", "400", "4000", "800"});
procInfos[4] =
new ProcessStatInfo(new String[] { "500", "proc5", "400", "100",
"100", "400000", "400", "4000", "800" });
new ProcessStatInfo(new String[]{"500", "proc5", "400", "100",
"100", "400000", "400", "4000", "800"});
procInfos[5] =
new ProcessStatInfo(new String[] { "600", "proc6", "1", "1", "1",
"400000", "400", "4000", "800" });
new ProcessStatInfo(new String[]{"600", "proc6", "1", "1", "1",
"400000", "400", "4000", "800"});
ProcessTreeSmapMemInfo[] memInfos = new ProcessTreeSmapMemInfo[6];
memInfos[0] = new ProcessTreeSmapMemInfo("100");
@ -839,7 +839,7 @@ public void testProcessTreeDump() throws IOException {
String[] cmdLines = new String[numProcesses];
cmdLines[0] = "proc1 arg1 arg2";
cmdLines[1] = "proc2 arg3 arg4";
cmdLines[1] = "process two arg3 arg4";
cmdLines[2] = "proc3 arg5 arg6";
cmdLines[3] = "proc4 arg7 arg8";
cmdLines[4] = "proc5 arg9 arg10";