Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

YARN-3304. Cleaning up ResourceCalculatorProcessTree APIs for public use and removing inconsistencies in the default values. Contributed by Junping Du and Karthik Kambatla. 

(cherry picked from commit c358368f51)
This commit is contained in:
Vinod Kumar Vavilapalli 2015-03-30 10:09:40 -07:00
parent a0ed29a058
commit 35af6f1802
11 changed files with 187 additions and 154 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Task.java
View File

@ -170,7 +170,7 @@ abstract public class Task implements Writable, Configurable {
skipRanges.skipRangeIterator();
private ResourceCalculatorProcessTree pTree;
private long initCpuCumulativeTime = 0;
private long initCpuCumulativeTime = ResourceCalculatorProcessTree.UNAVAILABLE;
protected JobConf conf;
protected MapOutputFile mapOutputFile;
@ -844,14 +844,26 @@ abstract public class Task implements Writable, Configurable {
}
pTree.updateProcessTree();
long cpuTime = pTree.getCumulativeCpuTime();
long pMem = pTree.getCumulativeRssmem();
long vMem = pTree.getCumulativeVmem();
long pMem = pTree.getRssMemorySize();
long vMem = pTree.getVirtualMemorySize();
// Remove the CPU time consumed previously by JVM reuse
if (cpuTime != ResourceCalculatorProcessTree.UNAVAILABLE &&
initCpuCumulativeTime != ResourceCalculatorProcessTree.UNAVAILABLE) {
cpuTime -= initCpuCumulativeTime;
}
if (cpuTime != ResourceCalculatorProcessTree.UNAVAILABLE) {
counters.findCounter(TaskCounter.CPU_MILLISECONDS).setValue(cpuTime);
}
if (pMem != ResourceCalculatorProcessTree.UNAVAILABLE) {
counters.findCounter(TaskCounter.PHYSICAL_MEMORY_BYTES).setValue(pMem);
}
if (vMem != ResourceCalculatorProcessTree.UNAVAILABLE) {
counters.findCounter(TaskCounter.VIRTUAL_MEMORY_BYTES).setValue(vMem);
}
}
/**
* An updater that tracks the amount of time this task has spent in GC.

4
hadoop-yarn-project/CHANGES.txt
View File

@ -741,6 +741,10 @@ Release 2.7.0 - UNRELEASED
YARN-2213. Change proxy-user cookie log in AmIpFilter to DEBUG.
(Varun Saxena via xgong)
YARN-3304. Cleaning up ResourceCalculatorProcessTree APIs for public use and
removing inconsistencies in the default values. (Junping Du and Karthik
Kambatla via vinodkv)
Release 2.6.0 - 2014-11-18
INCOMPATIBLE CHANGES

3
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/CpuTimeTracker.java
View File

@ -26,7 +26,8 @@ import java.math.BigInteger;
@InterfaceAudience.Private
@InterfaceStability.Unstable
public class CpuTimeTracker {
public static final int UNAVAILABLE = -1;
public static final int UNAVAILABLE =
ResourceCalculatorProcessTree.UNAVAILABLE;
final long MINIMUM_UPDATE_INTERVAL;
// CPU used time since system is on (ms)

78
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ProcfsBasedProcessTree.java
View File

@ -140,7 +140,7 @@ public class ProcfsBasedProcessTree extends ResourceCalculatorProcessTree {
static private String deadPid = "-1";
private String pid = deadPid;
static private Pattern numberPattern = Pattern.compile("[1-9][0-9]*");
private Long cpuTime = 0L;
private long cpuTime = UNAVAILABLE;
protected Map<String, ProcessInfo> processTree =
new HashMap<String, ProcessInfo>();
@ -340,66 +340,53 @@ public class ProcfsBasedProcessTree extends ResourceCalculatorProcessTree {
return ret.toString();
}
/**
* Get the cumulative virtual memory used by all the processes in the
* process-tree that are older than the passed in age.
*
* @param olderThanAge processes above this age are included in the
* memory addition
* @return cumulative virtual memory used by the process-tree in bytes,
* for processes older than this age.
*/
@Override
public long getCumulativeVmem(int olderThanAge) {
long total = 0;
public long getVirtualMemorySize(int olderThanAge) {
long total = UNAVAILABLE;
for (ProcessInfo p : processTree.values()) {
if ((p != null) && (p.getAge() > olderThanAge)) {
if (total == UNAVAILABLE ) {
total = 0;
}
total += p.getVmem();
}
}
return total;
}
/**
* Get the cumulative resident set size (rss) memory used by all the processes
* in the process-tree that are older than the passed in age.
*
* @param olderThanAge processes above this age are included in the
* memory addition
* @return cumulative rss memory used by the process-tree in bytes,
* for processes older than this age. return 0 if it cannot be
* calculated
*/
@Override
public long getCumulativeRssmem(int olderThanAge) {
public long getRssMemorySize(int olderThanAge) {
if (PAGE_SIZE < 0) {
return 0;
return UNAVAILABLE;
}
if (smapsEnabled) {
return getSmapBasedCumulativeRssmem(olderThanAge);
return getSmapBasedRssMemorySize(olderThanAge);
}
boolean isAvailable = false;
long totalPages = 0;
for (ProcessInfo p : processTree.values()) {
if ((p != null) && (p.getAge() > olderThanAge)) {
totalPages += p.getRssmemPage();
isAvailable = true;
}
}
return totalPages * PAGE_SIZE; // convert # pages to byte
return isAvailable ? totalPages * PAGE_SIZE : UNAVAILABLE; // convert # pages to byte
}
/**
* Get the cumulative resident set size (RSS) memory used by all the processes
* Get the resident set size (RSS) memory used by all the processes
* in the process-tree that are older than the passed in age. RSS is
* calculated based on SMAP information. Skip mappings with "r--s", "r-xs"
* permissions to get real RSS usage of the process.
*
* @param olderThanAge
* processes above this age are included in the memory addition
* @return cumulative rss memory used by the process-tree in bytes, for
* processes older than this age. return 0 if it cannot be calculated
* @return rss memory used by the process-tree in bytes, for
* processes older than this age. return {@link #UNAVAILABLE} if it cannot
* be calculated.
*/
private long getSmapBasedCumulativeRssmem(int olderThanAge) {
long total = 0;
private long getSmapBasedRssMemorySize(int olderThanAge) {
long total = UNAVAILABLE;
for (ProcessInfo p : processTree.values()) {
if ((p != null) && (p.getAge() > olderThanAge)) {
ProcessTreeSmapMemInfo procMemInfo = processSMAPTree.get(p.getPid());
@ -412,6 +399,9 @@ public class ProcfsBasedProcessTree extends ResourceCalculatorProcessTree {
.equalsIgnoreCase(READ_EXECUTE_WITH_SHARED_PERMISSION)) {
continue;
}
if (total == UNAVAILABLE){
total = 0;
}
total +=
Math.min(info.sharedDirty, info.pss) + info.privateDirty
+ info.privateClean;
@ -429,30 +419,34 @@ public class ProcfsBasedProcessTree extends ResourceCalculatorProcessTree {
}
}
}
total = (total * KB_TO_BYTES); // convert to bytes
if (total > 0) {
total *= KB_TO_BYTES; // convert to bytes
}
LOG.info("SmapBasedCumulativeRssmem (bytes) : " + total);
return total; // size
}
/**
* Get the CPU time in millisecond used by all the processes in the
* process-tree since the process-tree created
*
* @return cumulative CPU time in millisecond since the process-tree created
* return 0 if it cannot be calculated
*/
@Override
public long getCumulativeCpuTime() {
if (JIFFY_LENGTH_IN_MILLIS < 0) {
return 0;
return UNAVAILABLE;
}
long incJiffies = 0;
boolean isAvailable = false;
for (ProcessInfo p : processTree.values()) {
if (p != null) {
incJiffies += p.getDtime();
// data is available
isAvailable = true;
}
}
if (isAvailable) {
// reset cpuTime to 0 instead of UNAVAILABLE
if (cpuTime == UNAVAILABLE) {
cpuTime = 0L;
}
cpuTime += incJiffies * JIFFY_LENGTH_IN_MILLIS;
}
return cpuTime;
}
@ -1031,8 +1025,8 @@ public class ProcfsBasedProcessTree extends ResourceCalculatorProcessTree {
System.out.println("Cpu usage " + procfsBasedProcessTree
.getCpuUsagePercent());
System.out.println("Vmem usage in bytes " + procfsBasedProcessTree
.getCumulativeVmem());
.getVirtualMemorySize());
System.out.println("Rss mem usage in bytes " + procfsBasedProcessTree
.getCumulativeRssmem());
.getRssMemorySize());
}
}

62
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ResourceCalculatorProcessTree.java
View File

@ -23,19 +23,23 @@ import java.lang.reflect.Constructor;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience.Public;
import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.classification.InterfaceStability.Evolving;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.Configured;
/**
* Interface class to obtain process resource usage
*
* NOTE: This class should not be used by external users, but only by external
* developers to extend and include their own process-tree implementation,
* especially for platforms other than Linux and Windows.
*/
@Public
@Evolving
public abstract class ResourceCalculatorProcessTree extends Configured {
static final Log LOG = LogFactory
.getLog(ResourceCalculatorProcessTree.class);
public static final int UNAVAILABLE = -1;
/**
* Create process-tree instance with specified root process.
@ -65,63 +69,64 @@ public abstract class ResourceCalculatorProcessTree extends Configured {
public abstract String getProcessTreeDump();
/**
* Get the cumulative virtual memory used by all the processes in the
* Get the virtual memory used by all the processes in the
* process-tree.
*
* @return cumulative virtual memory used by the process-tree in bytes.
* @return virtual memory used by the process-tree in bytes,
* {@link #UNAVAILABLE} if it cannot be calculated.
*/
public long getCumulativeVmem() {
return getCumulativeVmem(0);
public long getVirtualMemorySize() {
return getVirtualMemorySize(0);
}
/**
* Get the cumulative resident set size (rss) memory used by all the processes
* Get the resident set size (rss) memory used by all the processes
* in the process-tree.
*
* @return cumulative rss memory used by the process-tree in bytes. return 0
* if it cannot be calculated
* @return rss memory used by the process-tree in bytes,
* {@link #UNAVAILABLE} if it cannot be calculated.
*/
public long getCumulativeRssmem() {
return getCumulativeRssmem(0);
public long getRssMemorySize() {
return getRssMemorySize(0);
}
/**
* Get the cumulative virtual memory used by all the processes in the
* Get the virtual memory used by all the processes in the
* process-tree that are older than the passed in age.
*
* @param olderThanAge processes above this age are included in the
* memory addition
* @return cumulative virtual memory used by the process-tree in bytes,
* for processes older than this age. return 0 if it cannot be
* calculated
* @return virtual memory used by the process-tree in bytes for
* processes older than the specified age, {@link #UNAVAILABLE} if it
* cannot be calculated.
*/
public long getCumulativeVmem(int olderThanAge) {
return 0;
public long getVirtualMemorySize(int olderThanAge) {
return UNAVAILABLE;
}
/**
* Get the cumulative resident set size (rss) memory used by all the processes
* Get the resident set size (rss) memory used by all the processes
* in the process-tree that are older than the passed in age.
*
* @param olderThanAge processes above this age are included in the
* memory addition
* @return cumulative rss memory used by the process-tree in bytes,
* for processes older than this age. return 0 if it cannot be
* calculated
* @return rss memory used by the process-tree in bytes for
* processes older than specified age, {@link #UNAVAILABLE} if it cannot be
* calculated.
*/
public long getCumulativeRssmem(int olderThanAge) {
return 0;
public long getRssMemorySize(int olderThanAge) {
return UNAVAILABLE;
}
/**
* Get the CPU time in millisecond used by all the processes in the
* process-tree since the process-tree was created
*
* @return cumulative CPU time in millisecond since the process-tree created
* return 0 if it cannot be calculated
* @return cumulative CPU time in millisecond since the process-tree
* created, {@link #UNAVAILABLE} if it cannot be calculated.
*/
public long getCumulativeCpuTime() {
return 0;
return UNAVAILABLE;
}
/**
@ -129,11 +134,11 @@ public abstract class ResourceCalculatorProcessTree extends Configured {
* average between samples as a ratio of overall CPU cycles similar to top.
* Thus, if 2 out of 4 cores are used this should return 200.0.
*
* @return percentage CPU usage since the process-tree was created
* return {@link CpuTimeTracker#UNAVAILABLE} if it cannot be calculated
* @return percentage CPU usage since the process-tree was created,
* {@link #UNAVAILABLE} if it cannot be calculated.
*/
public float getCpuUsagePercent() {
return -1;
return UNAVAILABLE;
}
/** Verify that the tree process id is same as its process group id.
@ -153,6 +158,7 @@ public abstract class ResourceCalculatorProcessTree extends Configured {
* @return ResourceCalculatorProcessTree or null if ResourceCalculatorPluginTree
* is not available for this system.
*/
@Private
public static ResourceCalculatorProcessTree getResourceCalculatorProcessTree(
String pid, Class<? extends ResourceCalculatorProcessTree> clazz, Configuration conf) {

19
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/WindowsBasedProcessTree.java
View File

@ -45,7 +45,7 @@ public class WindowsBasedProcessTree extends ResourceCalculatorProcessTree {
}
private String taskProcessId = null;
private long cpuTimeMs = 0;
private long cpuTimeMs = UNAVAILABLE;
private Map<String, ProcessInfo> processTree =
new HashMap<String, ProcessInfo>();
@ -173,10 +173,13 @@ public class WindowsBasedProcessTree extends ResourceCalculatorProcessTree {
}
@Override
public long getCumulativeVmem(int olderThanAge) {
long total = 0;
public long getVirtualMemorySize(int olderThanAge) {
long total = UNAVAILABLE;
for (ProcessInfo p : processTree.values()) {
if ((p != null) && (p.age > olderThanAge)) {
if (total == UNAVAILABLE) {
total = 0;
}
total += p.vmem;
}
}
@ -184,10 +187,13 @@ public class WindowsBasedProcessTree extends ResourceCalculatorProcessTree {
}
@Override
public long getCumulativeRssmem(int olderThanAge) {
long total = 0;
public long getRssMemorySize(int olderThanAge) {
long total = UNAVAILABLE;
for (ProcessInfo p : processTree.values()) {
if ((p != null) && (p.age > olderThanAge)) {
if (total == UNAVAILABLE) {
total = 0;
}
total += p.workingSet;
}
}
@ -197,6 +203,9 @@ public class WindowsBasedProcessTree extends ResourceCalculatorProcessTree {
@Override
public long getCumulativeCpuTime() {
for (ProcessInfo p : processTree.values()) {
if (cpuTimeMs == UNAVAILABLE) {
cpuTimeMs = 0;
}
cpuTimeMs += p.cpuTimeMsDelta;
}
return cpuTimeMs;

85
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestProcfsBasedProcessTree.java
View File

@ -19,6 +19,7 @@
package org.apache.hadoop.yarn.util;
import static org.apache.hadoop.yarn.util.ProcfsBasedProcessTree.KB_TO_BYTES;
import static org.apache.hadoop.yarn.util.ResourceCalculatorProcessTree.UNAVAILABLE;
import static org.junit.Assert.fail;
import static org.junit.Assume.assumeTrue;
@ -226,8 +227,8 @@ public class TestProcfsBasedProcessTree {
p.updateProcessTree();
Assert.assertFalse("ProcessTree must have been gone", isAlive(pid));
Assert.assertTrue(
"Cumulative vmem for the gone-process is " + p.getCumulativeVmem()
+ " . It should be zero.", p.getCumulativeVmem() == 0);
"vmem for the gone-process is " + p.getVirtualMemorySize()
+ " . It should be zero.", p.getVirtualMemorySize() == 0);
Assert.assertTrue(p.toString().equals("[ ]"));
}
@ -429,16 +430,16 @@ public class TestProcfsBasedProcessTree {
// build the process tree.
processTree.updateProcessTree();
// verify cumulative memory
Assert.assertEquals("Cumulative virtual memory does not match", 600000L,
processTree.getCumulativeVmem());
// verify virtual memory
Assert.assertEquals("Virtual memory does not match", 600000L,
processTree.getVirtualMemorySize());
// verify rss memory
long cumuRssMem =
ProcfsBasedProcessTree.PAGE_SIZE > 0
? 600L * ProcfsBasedProcessTree.PAGE_SIZE : 0L;
Assert.assertEquals("Cumulative rss memory does not match", cumuRssMem,
processTree.getCumulativeRssmem());
Assert.assertEquals("rss memory does not match", cumuRssMem,
processTree.getRssMemorySize());
// verify cumulative cpu time
long cumuCpuTime =
@ -456,8 +457,8 @@ public class TestProcfsBasedProcessTree {
setSmapsInProceTree(processTree, true);
// RSS=Min(shared_dirty,PSS)+PrivateClean+PrivateDirty (exclude r-xs,
// r--s)
Assert.assertEquals("Cumulative rss memory does not match",
(100 * KB_TO_BYTES * 3), processTree.getCumulativeRssmem());
Assert.assertEquals("rss memory does not match",
(100 * KB_TO_BYTES * 3), processTree.getRssMemorySize());
// test the cpu time again to see if it cumulates
procInfos[0] =
@ -563,9 +564,9 @@ public class TestProcfsBasedProcessTree {
new SystemClock());
setSmapsInProceTree(processTree, smapEnabled);
// verify cumulative memory
// verify virtual memory
Assert.assertEquals("Cumulative memory does not match", 700000L,
processTree.getCumulativeVmem());
processTree.getVirtualMemorySize());
// write one more process as child of 100.
String[] newPids = { "500" };
setupPidDirs(procfsRootDir, newPids);
@ -581,34 +582,34 @@ public class TestProcfsBasedProcessTree {
// check memory includes the new process.
processTree.updateProcessTree();
Assert.assertEquals("Cumulative vmem does not include new process",
1200000L, processTree.getCumulativeVmem());
Assert.assertEquals("vmem does not include new process",
1200000L, processTree.getVirtualMemorySize());
if (!smapEnabled) {
long cumuRssMem =
ProcfsBasedProcessTree.PAGE_SIZE > 0
? 1200L * ProcfsBasedProcessTree.PAGE_SIZE : 0L;
Assert.assertEquals("Cumulative rssmem does not include new process",
cumuRssMem, processTree.getCumulativeRssmem());
Assert.assertEquals("rssmem does not include new process",
cumuRssMem, processTree.getRssMemorySize());
} else {
Assert.assertEquals("Cumulative rssmem does not include new process",
100 * KB_TO_BYTES * 4, processTree.getCumulativeRssmem());
Assert.assertEquals("rssmem does not include new process",
100 * KB_TO_BYTES * 4, processTree.getRssMemorySize());
}
// however processes older than 1 iteration will retain the older value
Assert.assertEquals(
"Cumulative vmem shouldn't have included new process", 700000L,
processTree.getCumulativeVmem(1));
"vmem shouldn't have included new process", 700000L,
processTree.getVirtualMemorySize(1));
if (!smapEnabled) {
long cumuRssMem =
ProcfsBasedProcessTree.PAGE_SIZE > 0
? 700L * ProcfsBasedProcessTree.PAGE_SIZE : 0L;
Assert.assertEquals(
"Cumulative rssmem shouldn't have included new process", cumuRssMem,
processTree.getCumulativeRssmem(1));
"rssmem shouldn't have included new process", cumuRssMem,
processTree.getRssMemorySize(1));
} else {
Assert.assertEquals(
"Cumulative rssmem shouldn't have included new process",
100 * KB_TO_BYTES * 3, processTree.getCumulativeRssmem(1));
"rssmem shouldn't have included new process",
100 * KB_TO_BYTES * 3, processTree.getRssMemorySize(1));
}
// one more process
@ -629,49 +630,49 @@ public class TestProcfsBasedProcessTree {
// processes older than 2 iterations should be same as before.
Assert.assertEquals(
"Cumulative vmem shouldn't have included new processes", 700000L,
processTree.getCumulativeVmem(2));
"vmem shouldn't have included new processes", 700000L,
processTree.getVirtualMemorySize(2));
if (!smapEnabled) {
long cumuRssMem =
ProcfsBasedProcessTree.PAGE_SIZE > 0
? 700L * ProcfsBasedProcessTree.PAGE_SIZE : 0L;
Assert.assertEquals(
"Cumulative rssmem shouldn't have included new processes",
cumuRssMem, processTree.getCumulativeRssmem(2));
"rssmem shouldn't have included new processes",
cumuRssMem, processTree.getRssMemorySize(2));
} else {
Assert.assertEquals(
"Cumulative rssmem shouldn't have included new processes",
100 * KB_TO_BYTES * 3, processTree.getCumulativeRssmem(2));
"rssmem shouldn't have included new processes",
100 * KB_TO_BYTES * 3, processTree.getRssMemorySize(2));
}
// processes older than 1 iteration should not include new process,
// but include process 500
Assert.assertEquals(
"Cumulative vmem shouldn't have included new processes", 1200000L,
processTree.getCumulativeVmem(1));
"vmem shouldn't have included new processes", 1200000L,
processTree.getVirtualMemorySize(1));
if (!smapEnabled) {
long cumuRssMem =
ProcfsBasedProcessTree.PAGE_SIZE > 0
? 1200L * ProcfsBasedProcessTree.PAGE_SIZE : 0L;
Assert.assertEquals(
"Cumulative rssmem shouldn't have included new processes",
cumuRssMem, processTree.getCumulativeRssmem(1));
"rssmem shouldn't have included new processes",
cumuRssMem, processTree.getRssMemorySize(1));
} else {
Assert.assertEquals(
"Cumulative rssmem shouldn't have included new processes",
100 * KB_TO_BYTES * 4, processTree.getCumulativeRssmem(1));
"rssmem shouldn't have included new processes",
100 * KB_TO_BYTES * 4, processTree.getRssMemorySize(1));
}
// no processes older than 3 iterations, this should be 0
// no processes older than 3 iterations
Assert.assertEquals(
"Getting non-zero vmem for processes older than 3 iterations", 0L,
processTree.getCumulativeVmem(3));
"Getting non-zero vmem for processes older than 3 iterations",
UNAVAILABLE, processTree.getVirtualMemorySize(3));
Assert.assertEquals(
"Getting non-zero rssmem for processes older than 3 iterations", 0L,
processTree.getCumulativeRssmem(3));
"Getting non-zero rssmem for processes older than 3 iterations",
UNAVAILABLE, processTree.getRssMemorySize(3));
Assert.assertEquals(
"Getting non-zero rssmem for processes older than 3 iterations", 0L,
processTree.getCumulativeRssmem(3));
"Getting non-zero rssmem for processes older than 3 iterations",
UNAVAILABLE, processTree.getRssMemorySize(3));
} finally {
FileUtil.fullyDelete(procfsRootDir);
}

4
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestResourceCalculatorProcessTree.java
View File

@ -41,11 +41,11 @@ public class TestResourceCalculatorProcessTree {
return "Empty tree for testing";
}
public long getCumulativeRssmem(int age) {
public long getRssMemorySize(int age) {
return 0;
}
public long getCumulativeVmem(int age) {
public long getVirtualMemorySize(int age) {
return 0;
}

24
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestWindowsBasedProcessTree.java
View File

@ -53,26 +53,26 @@ public class TestWindowsBasedProcessTree {
WindowsBasedProcessTreeTester pTree = new WindowsBasedProcessTreeTester("-1");
pTree.infoStr = "3524,1024,1024,500\r\n2844,1024,1024,500\r\n";
pTree.updateProcessTree();
assertTrue(pTree.getCumulativeVmem() == 2048);
assertTrue(pTree.getCumulativeVmem(0) == 2048);
assertTrue(pTree.getCumulativeRssmem() == 2048);
assertTrue(pTree.getCumulativeRssmem(0) == 2048);
assertTrue(pTree.getVirtualMemorySize() == 2048);
assertTrue(pTree.getVirtualMemorySize(0) == 2048);
assertTrue(pTree.getRssMemorySize() == 2048);
assertTrue(pTree.getRssMemorySize(0) == 2048);
assertTrue(pTree.getCumulativeCpuTime() == 1000);
pTree.infoStr = "3524,1024,1024,1000\r\n2844,1024,1024,1000\r\n1234,1024,1024,1000\r\n";
pTree.updateProcessTree();
assertTrue(pTree.getCumulativeVmem() == 3072);
assertTrue(pTree.getCumulativeVmem(1) == 2048);
assertTrue(pTree.getCumulativeRssmem() == 3072);
assertTrue(pTree.getCumulativeRssmem(1) == 2048);
assertTrue(pTree.getVirtualMemorySize() == 3072);
assertTrue(pTree.getVirtualMemorySize(1) == 2048);
assertTrue(pTree.getRssMemorySize() == 3072);
assertTrue(pTree.getRssMemorySize(1) == 2048);
assertTrue(pTree.getCumulativeCpuTime() == 3000);
pTree.infoStr = "3524,1024,1024,1500\r\n2844,1024,1024,1500\r\n";
pTree.updateProcessTree();
assertTrue(pTree.getCumulativeVmem() == 2048);
assertTrue(pTree.getCumulativeVmem(2) == 2048);
assertTrue(pTree.getCumulativeRssmem() == 2048);
assertTrue(pTree.getCumulativeRssmem(2) == 2048);
assertTrue(pTree.getVirtualMemorySize() == 2048);
assertTrue(pTree.getVirtualMemorySize(2) == 2048);
assertTrue(pTree.getRssMemorySize() == 2048);
assertTrue(pTree.getRssMemorySize(2) == 2048);
assertTrue(pTree.getCumulativeCpuTime() == 4000);
}
}

6
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainerMetrics.java
View File

@ -188,14 +188,20 @@ public class ContainerMetrics implements MetricsSource {
}
public void recordMemoryUsage(int memoryMBs) {
if (memoryMBs >= 0) {
this.pMemMBsStat.add(memoryMBs);
}
}
public void recordCpuUsage(
int totalPhysicalCpuPercent, int milliVcoresUsed) {
if (totalPhysicalCpuPercent >=0) {
this.cpuCoreUsagePercent.add(totalPhysicalCpuPercent);
}
if (milliVcoresUsed >= 0) {
this.milliVcoresUsed.add(milliVcoresUsed);
}
}
public void recordProcessId(String processId) {
registry.tag(PROCESSID_INFO, processId);

12
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
View File

@ -333,10 +333,10 @@ public class ContainersMonitorImpl extends AbstractService implements
// method provided just for easy testing purposes
boolean isProcessTreeOverLimit(ResourceCalculatorProcessTree pTree,
String containerId, long limit) {
long currentMemUsage = pTree.getCumulativeVmem();
long currentMemUsage = pTree.getVirtualMemorySize();
// as processes begin with an age 1, we want to see if there are processes
// more than 1 iteration old.
long curMemUsageOfAgedProcesses = pTree.getCumulativeVmem(1);
long curMemUsageOfAgedProcesses = pTree.getVirtualMemorySize(1);
return isProcessTreeOverLimit(containerId, currentMemUsage,
curMemUsageOfAgedProcesses, limit);
}
@ -437,8 +437,8 @@ public class ContainersMonitorImpl extends AbstractService implements
+ " ContainerId = " + containerId);
ResourceCalculatorProcessTree pTree = ptInfo.getProcessTree();
pTree.updateProcessTree(); // update process-tree
long currentVmemUsage = pTree.getCumulativeVmem();
long currentPmemUsage = pTree.getCumulativeRssmem();
long currentVmemUsage = pTree.getVirtualMemorySize();
long currentPmemUsage = pTree.getRssMemorySize();
// if machine has 6 cores and 3 are used,
// cpuUsagePercentPerCore should be 300% and
// cpuUsageTotalCoresPercentage should be 50%
@ -451,8 +451,8 @@ public class ContainersMonitorImpl extends AbstractService implements
* maxVCoresAllottedForContainers /nodeCpuPercentageForYARN);
// as processes begin with an age 1, we want to see if there
// are processes more than 1 iteration old.
long curMemUsageOfAgedProcesses = pTree.getCumulativeVmem(1);
long curRssMemUsageOfAgedProcesses = pTree.getCumulativeRssmem(1);
long curMemUsageOfAgedProcesses = pTree.getVirtualMemorySize(1);
long curRssMemUsageOfAgedProcesses = pTree.getRssMemorySize(1);
long vmemLimit = ptInfo.getVmemLimit();
long pmemLimit = ptInfo.getPmemLimit();
LOG.info(String.format(