Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

YARN-7580. ContainersMonitorImpl logged message lacks detail when exceeding memory limits. Contributed by Wilfred Spiegelenburg.

This commit is contained in:
Miklos Szegedi 2017-12-29 12:35:49 -08:00
parent 6e3e1b8cde
commit b82049b4f0
2 changed files with 15 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
View File

@ -650,26 +650,34 @@ public class ContainersMonitorImpl extends AbstractService implements
if (isVmemCheckEnabled()
&& isProcessTreeOverLimit(containerId.toString(),
currentVmemUsage, curMemUsageOfAgedProcesses, vmemLimit)) {
// The current usage (age=0) is always higher than the aged usage. We
// do not show the aged size in the message, base the delta on the
// current usage
long delta = currentVmemUsage - vmemLimit;
// Container (the root process) is still alive and overflowing
// memory.
// Dump the process-tree and then clean it up.
msg = formatErrorMessage("virtual",
formatUsageString(currentVmemUsage, vmemLimit,
currentPmemUsage, pmemLimit),
pId, containerId, pTree);
pId, containerId, pTree, delta);
isMemoryOverLimit = true;
containerExitStatus = ContainerExitStatus.KILLED_EXCEEDED_VMEM;
} else if (isPmemCheckEnabled()
&& isProcessTreeOverLimit(containerId.toString(),
currentPmemUsage, curRssMemUsageOfAgedProcesses,
pmemLimit)) {
// The current usage (age=0) is always higher than the aged usage. We
// do not show the aged size in the message, base the delta on the
// current usage
long delta = currentPmemUsage - pmemLimit;
// Container (the root process) is still alive and overflowing
// memory.
// Dump the process-tree and then clean it up.
msg = formatErrorMessage("physical",
formatUsageString(currentVmemUsage, vmemLimit,
currentPmemUsage, pmemLimit),
pId, containerId, pTree);
pId, containerId, pTree, delta);
isMemoryOverLimit = true;
containerExitStatus = ContainerExitStatus.KILLED_EXCEEDED_PMEM;
}
@ -726,11 +734,11 @@ public class ContainersMonitorImpl extends AbstractService implements
*/
private String formatErrorMessage(String memTypeExceeded,
String usageString, String pId, ContainerId containerId,
ResourceCalculatorProcessTree pTree) {
ResourceCalculatorProcessTree pTree, long delta) {
return
String.format("Container [pid=%s,containerID=%s] is " +
"running beyond %s memory limits. ",
pId, containerId, memTypeExceeded) +
"running %dB beyond the '%S' memory limit. ",
pId, containerId, delta, memTypeExceeded) +
"Current usage: " + usageString +
". Killing container.\n" +
"Dump of the process-tree for " + containerId + " :\n" +

4
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/TestContainersMonitor.java
View File

@ -332,8 +332,8 @@ public class TestContainersMonitor extends BaseContainerManagerTest {
Assert.assertEquals(ContainerExitStatus.KILLED_EXCEEDED_VMEM,
containerStatus.getExitStatus());
String expectedMsgPattern =
"Container \\[pid=" + pid + ",containerID=" + cId
+ "\\] is running beyond virtual memory limits. Current usage: "
"Container \\[pid=" + pid + ",containerID=" + cId + "\\] is running "
+ "[0-9]+B beyond the 'VIRTUAL' memory limit. Current usage: "
+ "[0-9.]+ ?[KMGTPE]?B of [0-9.]+ ?[KMGTPE]?B physical memory used; "
+ "[0-9.]+ ?[KMGTPE]?B of [0-9.]+ ?[KMGTPE]?B virtual memory used. "
+ "Killing container.\nDump of the process-tree for "