Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

YARN-4768. getAvailablePhysicalMemorySize can be inaccurate on linux. Contributed by Nathan Roberts.

This commit is contained in:
Eric Payne 2016-05-10 21:28:02 +00:00
parent 025219b12f
commit 6b1c1cb01c
2 changed files with 119 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

39
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/SysInfoLinux.java
View File

@ -53,7 +53,7 @@ public class SysInfoLinux extends SysInfo {
*/ */
private static final String PROCFS_MEMFILE = "/proc/meminfo"; private static final String PROCFS_MEMFILE = "/proc/meminfo";
private static final Pattern PROCFS_MEMFILE_FORMAT = private static final Pattern PROCFS_MEMFILE_FORMAT =
Pattern.compile("^([a-zA-Z]*):[ \t]*([0-9]*)[ \t]kB"); Pattern.compile("^([a-zA-Z_()]*):[ \t]*([0-9]*)[ \t]*(kB)?");
// We need the values for the following keys in meminfo // We need the values for the following keys in meminfo
private static final String MEMTOTAL_STRING = "MemTotal"; private static final String MEMTOTAL_STRING = "MemTotal";
@ -61,6 +61,12 @@ public class SysInfoLinux extends SysInfo {
private static final String MEMFREE_STRING = "MemFree"; private static final String MEMFREE_STRING = "MemFree";
private static final String SWAPFREE_STRING = "SwapFree"; private static final String SWAPFREE_STRING = "SwapFree";
private static final String INACTIVE_STRING = "Inactive"; private static final String INACTIVE_STRING = "Inactive";
private static final String INACTIVEFILE_STRING = "Inactive(file)";
private static final String HARDWARECORRUPTED_STRING = "HardwareCorrupted";
private static final String HUGEPAGESTOTAL_STRING = "HugePages_Total";
private static final String HUGEPAGESIZE_STRING = "Hugepagesize";
/** /**
* Patterns for parsing /proc/cpuinfo. * Patterns for parsing /proc/cpuinfo.
@ -122,7 +128,13 @@ public class SysInfoLinux extends SysInfo {
private long swapSize = 0; private long swapSize = 0;
private long ramSizeFree = 0; // free ram space on the machine (kB) private long ramSizeFree = 0; // free ram space on the machine (kB)
private long swapSizeFree = 0; // free swap space on the machine (kB) private long swapSizeFree = 0; // free swap space on the machine (kB)
private long inactiveSize = 0; // inactive cache memory (kB) private long inactiveSize = 0; // inactive memory (kB)
private long inactiveFileSize = -1; // inactive cache memory, -1 if not there
private long hardwareCorruptSize = 0; // RAM corrupt and not available
private long hugePagesTotal = 0; // # of hugepages reserved
private long hugePageSize = 0; // # size of each hugepage
/* number of logical processors on the system. */ /* number of logical processors on the system. */
private int numProcessors = 0; private int numProcessors = 0;
/* number of physical cores on the system. */ /* number of physical cores on the system. */
@ -245,6 +257,14 @@ private void readProcMemInfoFile(boolean readAgain) {
swapSizeFree = Long.parseLong(mat.group(2)); swapSizeFree = Long.parseLong(mat.group(2));
} else if (mat.group(1).equals(INACTIVE_STRING)) { } else if (mat.group(1).equals(INACTIVE_STRING)) {
inactiveSize = Long.parseLong(mat.group(2)); inactiveSize = Long.parseLong(mat.group(2));
} else if (mat.group(1).equals(INACTIVEFILE_STRING)) {
inactiveFileSize = Long.parseLong(mat.group(2));
} else if (mat.group(1).equals(HARDWARECORRUPTED_STRING)) {
hardwareCorruptSize = Long.parseLong(mat.group(2));
} else if (mat.group(1).equals(HUGEPAGESTOTAL_STRING)) {
hugePagesTotal = Long.parseLong(mat.group(2));
} else if (mat.group(1).equals(HUGEPAGESIZE_STRING)) {
hugePageSize = Long.parseLong(mat.group(2));
} }
} }
str = in.readLine(); str = in.readLine();
@ -554,28 +574,31 @@ int readDiskBlockInformation(String diskName, int defSector) {
@Override @Override
public long getPhysicalMemorySize() { public long getPhysicalMemorySize() {
readProcMemInfoFile(); readProcMemInfoFile();
return ramSize * 1024; return (ramSize
- hardwareCorruptSize
- (hugePagesTotal * hugePageSize)) * 1024;
} }
/** {@inheritDoc} */ /** {@inheritDoc} */
@Override @Override
public long getVirtualMemorySize() { public long getVirtualMemorySize() {
readProcMemInfoFile(); return getPhysicalMemorySize() + (swapSize * 1024);
return (ramSize + swapSize) * 1024;
} }
/** {@inheritDoc} */ /** {@inheritDoc} */
@Override @Override
public long getAvailablePhysicalMemorySize() { public long getAvailablePhysicalMemorySize() {
readProcMemInfoFile(true); readProcMemInfoFile(true);
return (ramSizeFree + inactiveSize) * 1024; long inactive = inactiveFileSize != -1
? inactiveFileSize
: inactiveSize;
return (ramSizeFree + inactive) * 1024;
} }
/** {@inheritDoc} */ /** {@inheritDoc} */
@Override @Override
public long getAvailableVirtualMemorySize() { public long getAvailableVirtualMemorySize() {
readProcMemInfoFile(true); return getAvailablePhysicalMemorySize() + (swapSizeFree * 1024);
return (ramSizeFree + swapSizeFree + inactiveSize) * 1024;
} }
/** {@inheritDoc} */ /** {@inheritDoc} */

92
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestSysInfoLinux.java
View File

@ -29,6 +29,7 @@
import org.junit.Test; import org.junit.Test;
import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
/** /**
* A JUnit test to test {@link SysInfoLinux} * A JUnit test to test {@link SysInfoLinux}
@ -110,11 +111,56 @@ int readDiskBlockInformation(String diskName, int defSector) {
"VmallocTotal: 34359738367 kB\n" + "VmallocTotal: 34359738367 kB\n" +
"VmallocUsed: 1632 kB\n" + "VmallocUsed: 1632 kB\n" +
"VmallocChunk: 34359736375 kB\n" + "VmallocChunk: 34359736375 kB\n" +
"HugePages_Total: 0\n" + "HugePages_Total: %d\n" +
"HugePages_Free: 0\n" + "HugePages_Free: 0\n" +
"HugePages_Rsvd: 0\n" + "HugePages_Rsvd: 0\n" +
"Hugepagesize: 2048 kB"; "Hugepagesize: 2048 kB";
static final String MEMINFO_FORMAT_2 =
"MemTotal: %d kB\n" +
"MemFree: %d kB\n" +
"Buffers: 129976 kB\n" +
"Cached: 32317676 kB\n" +
"SwapCached: 0 kB\n" +
"Active: 88938588 kB\n" +
"Inactive: %d kB\n" +
"Active(anon): 77502200 kB\n" +
"Inactive(anon): 6385336 kB\n" +
"Active(file): 11436388 kB\n" +
"Inactive(file): %d kB\n" +
"Unevictable: 0 kB\n" +
"Mlocked: 0 kB\n" +
"SwapTotal: %d kB\n" +
"SwapFree: %d kB\n" +
"Dirty: 575864 kB\n" +
"Writeback: 16 kB\n" +
"AnonPages: 83886180 kB\n" +
"Mapped: 108640 kB\n" +
"Shmem: 1880 kB\n" +
"Slab: 2413448 kB\n" +
"SReclaimable: 2194488 kB\n" +
"SUnreclaim: 218960 kB\n" +
"KernelStack: 31496 kB\n" +
"PageTables: 195176 kB\n" +
"NFS_Unstable: 0 kB\n" +
"Bounce: 0 kB\n" +
"WritebackTmp: 0 kB\n" +
"CommitLimit: 97683468 kB\n" +
"Committed_AS: 94553560 kB\n" +
"VmallocTotal: 34359738367 kB\n" +
"VmallocUsed: 498580 kB\n" +
"VmallocChunk: 34256922296 kB\n" +
"HardwareCorrupted: %d kB\n" +
"AnonHugePages: 0 kB\n" +
"HugePages_Total: %d\n" +
"HugePages_Free: 0\n" +
"HugePages_Rsvd: 0\n" +
"HugePages_Surp: 0\n" +
"Hugepagesize: 2048 kB\n" +
"DirectMap4k: 4096 kB\n" +
"DirectMap2M: 2027520 kB\n" +
"DirectMap1G: 132120576 kB\n";
static final String CPUINFO_FORMAT = static final String CPUINFO_FORMAT =
"processor : %s\n" + "processor : %s\n" +
"vendor_id : AuthenticAMD\n" + "vendor_id : AuthenticAMD\n" +
@ -285,19 +331,57 @@ public void parsingProcMemFile() throws IOException {
long inactive = 567732L; long inactive = 567732L;
long swapTotal = 2096472L; long swapTotal = 2096472L;
long swapFree = 1818480L; long swapFree = 1818480L;
int nrHugePages = 10;
File tempFile = new File(FAKE_MEMFILE); File tempFile = new File(FAKE_MEMFILE);
tempFile.deleteOnExit(); tempFile.deleteOnExit();
FileWriter fWriter = new FileWriter(FAKE_MEMFILE); FileWriter fWriter = new FileWriter(FAKE_MEMFILE);
fWriter.write(String.format(MEMINFO_FORMAT, fWriter.write(String.format(MEMINFO_FORMAT,
memTotal, memFree, inactive, swapTotal, swapFree)); memTotal, memFree, inactive, swapTotal, swapFree, nrHugePages));
fWriter.close(); fWriter.close();
assertEquals(plugin.getAvailablePhysicalMemorySize(), assertEquals(plugin.getAvailablePhysicalMemorySize(),
1024L * (memFree + inactive)); 1024L * (memFree + inactive));
assertEquals(plugin.getAvailableVirtualMemorySize(), assertEquals(plugin.getAvailableVirtualMemorySize(),
1024L * (memFree + inactive + swapFree)); 1024L * (memFree + inactive + swapFree));
assertEquals(plugin.getPhysicalMemorySize(), 1024L * memTotal); assertEquals(plugin.getPhysicalMemorySize(),
assertEquals(plugin.getVirtualMemorySize(), 1024L * (memTotal + swapTotal)); 1024L * (memTotal - (nrHugePages * 2048)));
assertEquals(plugin.getVirtualMemorySize(),
1024L * (memTotal - (nrHugePages * 2048) + swapTotal));
}
/**
* Test parsing /proc/meminfo with Inactive(file) present
* @throws IOException
*/
@Test
public void parsingProcMemFile2() throws IOException {
long memTotal = 131403836L;
long memFree = 11257036L;
long inactive = 27396032L;
long inactiveFile = 21010696L;
long swapTotal = 31981552L;
long swapFree = 1818480L;
long hardwareCorrupt = 31960904L;
int nrHugePages = 10;
File tempFile = new File(FAKE_MEMFILE);
tempFile.deleteOnExit();
FileWriter fWriter = new FileWriter(FAKE_MEMFILE);
fWriter.write(String.format(MEMINFO_FORMAT_2,
memTotal, memFree, inactive, inactiveFile, swapTotal, swapFree,
hardwareCorrupt, nrHugePages));
fWriter.close();
assertEquals(plugin.getAvailablePhysicalMemorySize(),
1024L * (memFree + inactiveFile));
assertFalse(plugin.getAvailablePhysicalMemorySize() ==
1024L * (memFree + inactive));
assertEquals(plugin.getAvailableVirtualMemorySize(),
1024L * (memFree + inactiveFile + swapFree));
assertEquals(plugin.getPhysicalMemorySize(),
1024L * (memTotal - hardwareCorrupt - (nrHugePages * 2048)));
assertEquals(plugin.getVirtualMemorySize(),
1024L * (memTotal - hardwareCorrupt -
(nrHugePages * 2048) + swapTotal));
} }
@Test @Test