Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

YARN-8444: NodeResourceMonitor crashes on bad swapFree value. Contributed by Jim Brennan. 

(cherry picked from commit 6432128622)
This commit is contained in:
Eric E Payne 2018-06-22 17:15:29 +00:00
parent 6b5b58a495
commit 1974dfa44d
2 changed files with 77 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/SysInfoLinux.java
View File

@ -216,6 +216,21 @@ public class SysInfoLinux extends SysInfo {
readProcMemInfoFile(false);
}
/**
*
* Wrapper for Long.parseLong() that returns zero if the value is
* invalid. Under some circumstances, swapFree in /proc/meminfo can
* go negative, reported as a very large decimal value.
*/
private long safeParseLong(String strVal) {
long parsedVal;
try {
parsedVal = Long.parseLong(strVal);
} catch (NumberFormatException nfe) {
parsedVal = 0;
}
return parsedVal;
}
/**
* Read /proc/meminfo, parse and compute memory information.
* @param readAgain if false, read only on the first time
@ -252,9 +267,9 @@ public class SysInfoLinux extends SysInfo {
} else if (mat.group(1).equals(SWAPTOTAL_STRING)) {
swapSize = Long.parseLong(mat.group(2));
} else if (mat.group(1).equals(MEMFREE_STRING)) {
ramSizeFree = Long.parseLong(mat.group(2));
ramSizeFree = safeParseLong(mat.group(2));
} else if (mat.group(1).equals(SWAPFREE_STRING)) {
swapSizeFree = Long.parseLong(mat.group(2));
swapSizeFree = safeParseLong(mat.group(2));
} else if (mat.group(1).equals(INACTIVE_STRING)) {
inactiveSize = Long.parseLong(mat.group(2));
} else if (mat.group(1).equals(INACTIVEFILE_STRING)) {

60
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestSysInfoLinux.java
View File

@ -161,6 +161,36 @@ public class TestSysInfoLinux {
"DirectMap2M: 2027520 kB\n" +
"DirectMap1G: 132120576 kB\n";
static final String MEMINFO_FORMAT3 =
"MemTotal: %d kB\n" +
"MemFree: %s kB\n" +
"Buffers: 138244 kB\n" +
"Cached: 947780 kB\n" +
"SwapCached: 142880 kB\n" +
"Active: 3229888 kB\n" +
"Inactive: %d kB\n" +
"SwapTotal: %d kB\n" +
"SwapFree: %s kB\n" +
"Dirty: 122012 kB\n" +
"Writeback: 0 kB\n" +
"AnonPages: 2710792 kB\n" +
"Mapped: 24740 kB\n" +
"Slab: 132528 kB\n" +
"SReclaimable: 105096 kB\n" +
"SUnreclaim: 27432 kB\n" +
"PageTables: 11448 kB\n" +
"NFS_Unstable: 0 kB\n" +
"Bounce: 0 kB\n" +
"CommitLimit: 4125904 kB\n" +
"Committed_AS: 4143556 kB\n" +
"VmallocTotal: 34359738367 kB\n" +
"VmallocUsed: 1632 kB\n" +
"VmallocChunk: 34359736375 kB\n" +
"HugePages_Total: %d\n" +
"HugePages_Free: 0\n" +
"HugePages_Rsvd: 0\n" +
"Hugepagesize: 2048 kB";
static final String CPUINFO_FORMAT =
"processor : %s\n" +
"vendor_id : AuthenticAMD\n" +
@ -384,6 +414,36 @@ public class TestSysInfoLinux {
(nrHugePages * 2048) + swapTotal));
}
/**
* Test parsing /proc/meminfo
* @throws IOException
*/
@Test
public void parsingProcMemFileWithBadValues() throws IOException {
long memTotal = 4058864L;
long memFree = 0L; // bad value should return 0
long inactive = 567732L;
long swapTotal = 2096472L;
long swapFree = 0L; // bad value should return 0
int nrHugePages = 10;
String badFreeValue = "18446744073709551596";
File tempFile = new File(FAKE_MEMFILE);
tempFile.deleteOnExit();
FileWriter fWriter = new FileWriter(FAKE_MEMFILE);
fWriter.write(String.format(MEMINFO_FORMAT3,
memTotal, badFreeValue, inactive, swapTotal, badFreeValue, nrHugePages));
fWriter.close();
assertEquals(plugin.getAvailablePhysicalMemorySize(),
1024L * (memFree + inactive));
assertEquals(plugin.getAvailableVirtualMemorySize(),
1024L * (memFree + inactive + swapFree));
assertEquals(plugin.getPhysicalMemorySize(),
1024L * (memTotal - (nrHugePages * 2048)));
assertEquals(plugin.getVirtualMemorySize(),
1024L * (memTotal - (nrHugePages * 2048) + swapTotal));
}
@Test
public void testCoreCounts() throws IOException {