Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

YARN-4768. getAvailablePhysicalMemorySize can be inaccurate on linux. Contributed by Nathan Roberts. 

(cherry picked from commit 6b1c1cb01c)
This commit is contained in:
Eric Payne 2016-05-10 21:28:02 +00:00
parent f47ea0cc3f
commit de4daf9d22
2 changed files with 119 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

39
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/SysInfoLinux.java
View File

@ -53,7 +53,7 @@ public class SysInfoLinux extends SysInfo {
*/
private static final String PROCFS_MEMFILE = "/proc/meminfo";
private static final Pattern PROCFS_MEMFILE_FORMAT =
Pattern.compile("^([a-zA-Z]*):[ \t]*([0-9]*)[ \t]kB");
Pattern.compile("^([a-zA-Z_()]*):[ \t]*([0-9]*)[ \t]*(kB)?");
// We need the values for the following keys in meminfo
private static final String MEMTOTAL_STRING = "MemTotal";
@ -61,6 +61,12 @@ public class SysInfoLinux extends SysInfo {
private static final String MEMFREE_STRING = "MemFree";
private static final String SWAPFREE_STRING = "SwapFree";
private static final String INACTIVE_STRING = "Inactive";
private static final String INACTIVEFILE_STRING = "Inactive(file)";
private static final String HARDWARECORRUPTED_STRING = "HardwareCorrupted";
private static final String HUGEPAGESTOTAL_STRING = "HugePages_Total";
private static final String HUGEPAGESIZE_STRING = "Hugepagesize";
/**
* Patterns for parsing /proc/cpuinfo.
@ -122,7 +128,13 @@ public class SysInfoLinux extends SysInfo {
private long swapSize = 0;
private long ramSizeFree = 0; // free ram space on the machine (kB)
private long swapSizeFree = 0; // free swap space on the machine (kB)
private long inactiveSize = 0; // inactive cache memory (kB)
private long inactiveSize = 0; // inactive memory (kB)
private long inactiveFileSize = -1; // inactive cache memory, -1 if not there
private long hardwareCorruptSize = 0; // RAM corrupt and not available
private long hugePagesTotal = 0; // # of hugepages reserved
private long hugePageSize = 0; // # size of each hugepage
/* number of logical processors on the system. */
private int numProcessors = 0;
/* number of physical cores on the system. */
@ -245,6 +257,14 @@ public class SysInfoLinux extends SysInfo {
swapSizeFree = Long.parseLong(mat.group(2));
} else if (mat.group(1).equals(INACTIVE_STRING)) {
inactiveSize = Long.parseLong(mat.group(2));
} else if (mat.group(1).equals(INACTIVEFILE_STRING)) {
inactiveFileSize = Long.parseLong(mat.group(2));
} else if (mat.group(1).equals(HARDWARECORRUPTED_STRING)) {
hardwareCorruptSize = Long.parseLong(mat.group(2));
} else if (mat.group(1).equals(HUGEPAGESTOTAL_STRING)) {
hugePagesTotal = Long.parseLong(mat.group(2));
} else if (mat.group(1).equals(HUGEPAGESIZE_STRING)) {
hugePageSize = Long.parseLong(mat.group(2));
}
}
str = in.readLine();
@ -554,28 +574,31 @@ public class SysInfoLinux extends SysInfo {
@Override
public long getPhysicalMemorySize() {
readProcMemInfoFile();
return ramSize * 1024;
return (ramSize
- hardwareCorruptSize
- (hugePagesTotal * hugePageSize)) * 1024;
}
/** {@inheritDoc} */
@Override
public long getVirtualMemorySize() {
readProcMemInfoFile();
return (ramSize + swapSize) * 1024;
return getPhysicalMemorySize() + (swapSize * 1024);
}
/** {@inheritDoc} */
@Override
public long getAvailablePhysicalMemorySize() {
readProcMemInfoFile(true);
return (ramSizeFree + inactiveSize) * 1024;
long inactive = inactiveFileSize != -1
? inactiveFileSize
: inactiveSize;
return (ramSizeFree + inactive) * 1024;
}
/** {@inheritDoc} */
@Override
public long getAvailableVirtualMemorySize() {
readProcMemInfoFile(true);
return (ramSizeFree + swapSizeFree + inactiveSize) * 1024;
return getAvailablePhysicalMemorySize() + (swapSizeFree * 1024);
}
/** {@inheritDoc} */

92
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestSysInfoLinux.java
View File

@ -29,6 +29,7 @@ import org.apache.hadoop.test.GenericTestUtils;
import org.junit.Test;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
/**
* A JUnit test to test {@link SysInfoLinux}
@ -110,11 +111,56 @@ public class TestSysInfoLinux {
"VmallocTotal: 34359738367 kB\n" +
"VmallocUsed: 1632 kB\n" +
"VmallocChunk: 34359736375 kB\n" +
"HugePages_Total: 0\n" +
"HugePages_Total: %d\n" +
"HugePages_Free: 0\n" +
"HugePages_Rsvd: 0\n" +
"Hugepagesize: 2048 kB";
static final String MEMINFO_FORMAT_2 =
"MemTotal: %d kB\n" +
"MemFree: %d kB\n" +
"Buffers: 129976 kB\n" +
"Cached: 32317676 kB\n" +
"SwapCached: 0 kB\n" +
"Active: 88938588 kB\n" +
"Inactive: %d kB\n" +
"Active(anon): 77502200 kB\n" +
"Inactive(anon): 6385336 kB\n" +
"Active(file): 11436388 kB\n" +
"Inactive(file): %d kB\n" +
"Unevictable: 0 kB\n" +
"Mlocked: 0 kB\n" +
"SwapTotal: %d kB\n" +
"SwapFree: %d kB\n" +
"Dirty: 575864 kB\n" +
"Writeback: 16 kB\n" +
"AnonPages: 83886180 kB\n" +
"Mapped: 108640 kB\n" +
"Shmem: 1880 kB\n" +
"Slab: 2413448 kB\n" +
"SReclaimable: 2194488 kB\n" +
"SUnreclaim: 218960 kB\n" +
"KernelStack: 31496 kB\n" +
"PageTables: 195176 kB\n" +
"NFS_Unstable: 0 kB\n" +
"Bounce: 0 kB\n" +
"WritebackTmp: 0 kB\n" +
"CommitLimit: 97683468 kB\n" +
"Committed_AS: 94553560 kB\n" +
"VmallocTotal: 34359738367 kB\n" +
"VmallocUsed: 498580 kB\n" +
"VmallocChunk: 34256922296 kB\n" +
"HardwareCorrupted: %d kB\n" +
"AnonHugePages: 0 kB\n" +
"HugePages_Total: %d\n" +
"HugePages_Free: 0\n" +
"HugePages_Rsvd: 0\n" +
"HugePages_Surp: 0\n" +
"Hugepagesize: 2048 kB\n" +
"DirectMap4k: 4096 kB\n" +
"DirectMap2M: 2027520 kB\n" +
"DirectMap1G: 132120576 kB\n";
static final String CPUINFO_FORMAT =
"processor : %s\n" +
"vendor_id : AuthenticAMD\n" +
@ -285,19 +331,57 @@ public class TestSysInfoLinux {
long inactive = 567732L;
long swapTotal = 2096472L;
long swapFree = 1818480L;
int nrHugePages = 10;
File tempFile = new File(FAKE_MEMFILE);
tempFile.deleteOnExit();
FileWriter fWriter = new FileWriter(FAKE_MEMFILE);
fWriter.write(String.format(MEMINFO_FORMAT,
memTotal, memFree, inactive, swapTotal, swapFree));
memTotal, memFree, inactive, swapTotal, swapFree, nrHugePages));
fWriter.close();
assertEquals(plugin.getAvailablePhysicalMemorySize(),
1024L * (memFree + inactive));
assertEquals(plugin.getAvailableVirtualMemorySize(),
1024L * (memFree + inactive + swapFree));
assertEquals(plugin.getPhysicalMemorySize(), 1024L * memTotal);
assertEquals(plugin.getVirtualMemorySize(), 1024L * (memTotal + swapTotal));
assertEquals(plugin.getPhysicalMemorySize(),
1024L * (memTotal - (nrHugePages * 2048)));
assertEquals(plugin.getVirtualMemorySize(),
1024L * (memTotal - (nrHugePages * 2048) + swapTotal));
}
/**
* Test parsing /proc/meminfo with Inactive(file) present
* @throws IOException
*/
@Test
public void parsingProcMemFile2() throws IOException {
long memTotal = 131403836L;
long memFree = 11257036L;
long inactive = 27396032L;
long inactiveFile = 21010696L;
long swapTotal = 31981552L;
long swapFree = 1818480L;
long hardwareCorrupt = 31960904L;
int nrHugePages = 10;
File tempFile = new File(FAKE_MEMFILE);
tempFile.deleteOnExit();
FileWriter fWriter = new FileWriter(FAKE_MEMFILE);
fWriter.write(String.format(MEMINFO_FORMAT_2,
memTotal, memFree, inactive, inactiveFile, swapTotal, swapFree,
hardwareCorrupt, nrHugePages));
fWriter.close();
assertEquals(plugin.getAvailablePhysicalMemorySize(),
1024L * (memFree + inactiveFile));
assertFalse(plugin.getAvailablePhysicalMemorySize() ==
1024L * (memFree + inactive));
assertEquals(plugin.getAvailableVirtualMemorySize(),
1024L * (memFree + inactiveFile + swapFree));
assertEquals(plugin.getPhysicalMemorySize(),
1024L * (memTotal - hardwareCorrupt - (nrHugePages * 2048)));
assertEquals(plugin.getVirtualMemorySize(),
1024L * (memTotal - hardwareCorrupt -
(nrHugePages * 2048) + swapTotal));
}
@Test