diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/SysInfoLinux.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/SysInfoLinux.java index 2852bea0423..bba16316d62 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/SysInfoLinux.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/SysInfoLinux.java @@ -53,7 +53,7 @@ public class SysInfoLinux extends SysInfo { */ private static final String PROCFS_MEMFILE = "/proc/meminfo"; private static final Pattern PROCFS_MEMFILE_FORMAT = - Pattern.compile("^([a-zA-Z]*):[ \t]*([0-9]*)[ \t]kB"); + Pattern.compile("^([a-zA-Z_()]*):[ \t]*([0-9]*)[ \t]*(kB)?"); // We need the values for the following keys in meminfo private static final String MEMTOTAL_STRING = "MemTotal"; @@ -61,6 +61,12 @@ public class SysInfoLinux extends SysInfo { private static final String MEMFREE_STRING = "MemFree"; private static final String SWAPFREE_STRING = "SwapFree"; private static final String INACTIVE_STRING = "Inactive"; + private static final String INACTIVEFILE_STRING = "Inactive(file)"; + private static final String HARDWARECORRUPTED_STRING = "HardwareCorrupted"; + private static final String HUGEPAGESTOTAL_STRING = "HugePages_Total"; + private static final String HUGEPAGESIZE_STRING = "Hugepagesize"; + + /** * Patterns for parsing /proc/cpuinfo. @@ -122,7 +128,13 @@ public class SysInfoLinux extends SysInfo { private long swapSize = 0; private long ramSizeFree = 0; // free ram space on the machine (kB) private long swapSizeFree = 0; // free swap space on the machine (kB) - private long inactiveSize = 0; // inactive cache memory (kB) + private long inactiveSize = 0; // inactive memory (kB) + private long inactiveFileSize = -1; // inactive cache memory, -1 if not there + private long hardwareCorruptSize = 0; // RAM corrupt and not available + private long hugePagesTotal = 0; // # of hugepages reserved + private long hugePageSize = 0; // # size of each hugepage + + /* number of logical processors on the system. */ private int numProcessors = 0; /* number of physical cores on the system. */ @@ -245,6 +257,14 @@ public class SysInfoLinux extends SysInfo { swapSizeFree = Long.parseLong(mat.group(2)); } else if (mat.group(1).equals(INACTIVE_STRING)) { inactiveSize = Long.parseLong(mat.group(2)); + } else if (mat.group(1).equals(INACTIVEFILE_STRING)) { + inactiveFileSize = Long.parseLong(mat.group(2)); + } else if (mat.group(1).equals(HARDWARECORRUPTED_STRING)) { + hardwareCorruptSize = Long.parseLong(mat.group(2)); + } else if (mat.group(1).equals(HUGEPAGESTOTAL_STRING)) { + hugePagesTotal = Long.parseLong(mat.group(2)); + } else if (mat.group(1).equals(HUGEPAGESIZE_STRING)) { + hugePageSize = Long.parseLong(mat.group(2)); } } str = in.readLine(); @@ -554,28 +574,31 @@ public class SysInfoLinux extends SysInfo { @Override public long getPhysicalMemorySize() { readProcMemInfoFile(); - return ramSize * 1024; + return (ramSize + - hardwareCorruptSize + - (hugePagesTotal * hugePageSize)) * 1024; } /** {@inheritDoc} */ @Override public long getVirtualMemorySize() { - readProcMemInfoFile(); - return (ramSize + swapSize) * 1024; + return getPhysicalMemorySize() + (swapSize * 1024); } /** {@inheritDoc} */ @Override public long getAvailablePhysicalMemorySize() { readProcMemInfoFile(true); - return (ramSizeFree + inactiveSize) * 1024; + long inactive = inactiveFileSize != -1 + ? inactiveFileSize + : inactiveSize; + return (ramSizeFree + inactive) * 1024; } /** {@inheritDoc} */ @Override public long getAvailableVirtualMemorySize() { - readProcMemInfoFile(true); - return (ramSizeFree + swapSizeFree + inactiveSize) * 1024; + return getAvailablePhysicalMemorySize() + (swapSizeFree * 1024); } /** {@inheritDoc} */ diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestSysInfoLinux.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestSysInfoLinux.java index 8a6df0c0511..a646a41271c 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestSysInfoLinux.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestSysInfoLinux.java @@ -29,6 +29,7 @@ import org.apache.hadoop.test.GenericTestUtils; import org.junit.Test; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; /** * A JUnit test to test {@link SysInfoLinux} @@ -110,11 +111,56 @@ public class TestSysInfoLinux { "VmallocTotal: 34359738367 kB\n" + "VmallocUsed: 1632 kB\n" + "VmallocChunk: 34359736375 kB\n" + - "HugePages_Total: 0\n" + + "HugePages_Total: %d\n" + "HugePages_Free: 0\n" + "HugePages_Rsvd: 0\n" + "Hugepagesize: 2048 kB"; + static final String MEMINFO_FORMAT_2 = + "MemTotal: %d kB\n" + + "MemFree: %d kB\n" + + "Buffers: 129976 kB\n" + + "Cached: 32317676 kB\n" + + "SwapCached: 0 kB\n" + + "Active: 88938588 kB\n" + + "Inactive: %d kB\n" + + "Active(anon): 77502200 kB\n" + + "Inactive(anon): 6385336 kB\n" + + "Active(file): 11436388 kB\n" + + "Inactive(file): %d kB\n" + + "Unevictable: 0 kB\n" + + "Mlocked: 0 kB\n" + + "SwapTotal: %d kB\n" + + "SwapFree: %d kB\n" + + "Dirty: 575864 kB\n" + + "Writeback: 16 kB\n" + + "AnonPages: 83886180 kB\n" + + "Mapped: 108640 kB\n" + + "Shmem: 1880 kB\n" + + "Slab: 2413448 kB\n" + + "SReclaimable: 2194488 kB\n" + + "SUnreclaim: 218960 kB\n" + + "KernelStack: 31496 kB\n" + + "PageTables: 195176 kB\n" + + "NFS_Unstable: 0 kB\n" + + "Bounce: 0 kB\n" + + "WritebackTmp: 0 kB\n" + + "CommitLimit: 97683468 kB\n" + + "Committed_AS: 94553560 kB\n" + + "VmallocTotal: 34359738367 kB\n" + + "VmallocUsed: 498580 kB\n" + + "VmallocChunk: 34256922296 kB\n" + + "HardwareCorrupted: %d kB\n" + + "AnonHugePages: 0 kB\n" + + "HugePages_Total: %d\n" + + "HugePages_Free: 0\n" + + "HugePages_Rsvd: 0\n" + + "HugePages_Surp: 0\n" + + "Hugepagesize: 2048 kB\n" + + "DirectMap4k: 4096 kB\n" + + "DirectMap2M: 2027520 kB\n" + + "DirectMap1G: 132120576 kB\n"; + static final String CPUINFO_FORMAT = "processor : %s\n" + "vendor_id : AuthenticAMD\n" + @@ -285,19 +331,57 @@ public class TestSysInfoLinux { long inactive = 567732L; long swapTotal = 2096472L; long swapFree = 1818480L; + int nrHugePages = 10; File tempFile = new File(FAKE_MEMFILE); tempFile.deleteOnExit(); FileWriter fWriter = new FileWriter(FAKE_MEMFILE); fWriter.write(String.format(MEMINFO_FORMAT, - memTotal, memFree, inactive, swapTotal, swapFree)); + memTotal, memFree, inactive, swapTotal, swapFree, nrHugePages)); fWriter.close(); assertEquals(plugin.getAvailablePhysicalMemorySize(), 1024L * (memFree + inactive)); assertEquals(plugin.getAvailableVirtualMemorySize(), 1024L * (memFree + inactive + swapFree)); - assertEquals(plugin.getPhysicalMemorySize(), 1024L * memTotal); - assertEquals(plugin.getVirtualMemorySize(), 1024L * (memTotal + swapTotal)); + assertEquals(plugin.getPhysicalMemorySize(), + 1024L * (memTotal - (nrHugePages * 2048))); + assertEquals(plugin.getVirtualMemorySize(), + 1024L * (memTotal - (nrHugePages * 2048) + swapTotal)); + } + + /** + * Test parsing /proc/meminfo with Inactive(file) present + * @throws IOException + */ + @Test + public void parsingProcMemFile2() throws IOException { + long memTotal = 131403836L; + long memFree = 11257036L; + long inactive = 27396032L; + long inactiveFile = 21010696L; + long swapTotal = 31981552L; + long swapFree = 1818480L; + long hardwareCorrupt = 31960904L; + int nrHugePages = 10; + File tempFile = new File(FAKE_MEMFILE); + tempFile.deleteOnExit(); + FileWriter fWriter = new FileWriter(FAKE_MEMFILE); + fWriter.write(String.format(MEMINFO_FORMAT_2, + memTotal, memFree, inactive, inactiveFile, swapTotal, swapFree, + hardwareCorrupt, nrHugePages)); + + fWriter.close(); + assertEquals(plugin.getAvailablePhysicalMemorySize(), + 1024L * (memFree + inactiveFile)); + assertFalse(plugin.getAvailablePhysicalMemorySize() == + 1024L * (memFree + inactive)); + assertEquals(plugin.getAvailableVirtualMemorySize(), + 1024L * (memFree + inactiveFile + swapFree)); + assertEquals(plugin.getPhysicalMemorySize(), + 1024L * (memTotal - hardwareCorrupt - (nrHugePages * 2048))); + assertEquals(plugin.getVirtualMemorySize(), + 1024L * (memTotal - hardwareCorrupt - + (nrHugePages * 2048) + swapTotal)); } @Test