From a1e2e74dcbf0809d11aabd67311c45277926af4c Mon Sep 17 00:00:00 2001 From: Eli Collins Date: Thu, 19 Jan 2012 22:49:40 +0000 Subject: [PATCH] MAPREDUCE-3692. svn merge -c 1233605 from trunk git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/branch-0.23@1233622 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-mapreduce-project/CHANGES.txt | 2 ++ .../org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java | 2 +- .../apache/hadoop/mapreduce/v2/app/job/impl/TaskImpl.java | 2 +- .../org/apache/hadoop/yarn/ipc/ProtoOverHadoopRpcEngine.java | 5 ++--- .../containermanager/application/ApplicationImpl.java | 2 +- .../containermanager/container/ContainerImpl.java | 2 +- .../containermanager/localizer/LocalizedResource.java | 2 +- .../hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java | 2 +- .../resourcemanager/rmapp/attempt/RMAppAttemptImpl.java | 2 +- .../server/resourcemanager/rmcontainer/RMContainerImpl.java | 2 +- .../yarn/server/resourcemanager/rmnode/RMNodeImpl.java | 2 +- .../server/resourcemanager/scheduler/fifo/FifoScheduler.java | 4 ++-- 12 files changed, 15 insertions(+), 14 deletions(-) diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index f84d90b0ba4..764975f0554 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -119,6 +119,8 @@ Release 0.23.1 - Unreleased assign only one off-switch container in a single scheduling iteration. (Arun C Murthy via vinodkv) + MAPREDUCE-3692. yarn-resourcemanager out and log files can get big. (eli) + OPTIMIZATIONS MAPREDUCE-3567. Extraneous JobConf objects in AM heap. (Vinod Kumar diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java index 808e92d7624..a0e8613e206 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java @@ -632,7 +632,7 @@ public class JobImpl implements org.apache.hadoop.mapreduce.v2.app.job.Job, * The only entry point to change the Job. */ public void handle(JobEvent event) { - LOG.info("Processing " + event.getJobId() + " of type " + event.getType()); + LOG.debug("Processing " + event.getJobId() + " of type " + event.getType()); try { writeLock.lock(); JobState oldState = getState(); diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskImpl.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskImpl.java index e376c9e887d..fb2d32f4ab7 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskImpl.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskImpl.java @@ -537,7 +537,7 @@ public abstract class TaskImpl implements Task, EventHandler { @Override public void handle(TaskEvent event) { - LOG.info("Processing " + event.getTaskID() + " of type " + event.getType()); + LOG.debug("Processing " + event.getTaskID() + " of type " + event.getType()); try { writeLock.lock(); TaskState oldState = getState(); diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/ipc/ProtoOverHadoopRpcEngine.java b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/ipc/ProtoOverHadoopRpcEngine.java index 936899fa567..e68b420f2e8 100644 --- a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/ipc/ProtoOverHadoopRpcEngine.java +++ b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/ipc/ProtoOverHadoopRpcEngine.java @@ -323,11 +323,10 @@ public class ProtoOverHadoopRpcEngine implements RpcEngine { ProtoSpecificRequestWritable request = (ProtoSpecificRequestWritable) writableRequest; ProtoSpecificRpcRequest rpcRequest = request.message; String methodName = rpcRequest.getMethodName(); - System.out.println("Call: protocol=" + protocol.getCanonicalName() + ", method=" - + methodName); - if (verbose) + if (verbose) { log("Call: protocol=" + protocol.getCanonicalName() + ", method=" + methodName); + } MethodDescriptor methodDescriptor = service.getDescriptorForType() .findMethodByName(methodName); if (methodDescriptor == null) { diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationImpl.java b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationImpl.java index afdbe5b0823..d1bcaf2ef9d 100644 --- a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationImpl.java +++ b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationImpl.java @@ -373,7 +373,7 @@ public class ApplicationImpl implements Application { try { ApplicationId applicationID = event.getApplicationID(); - LOG.info("Processing " + applicationID + " of type " + event.getType()); + LOG.debug("Processing " + applicationID + " of type " + event.getType()); ApplicationState oldState = stateMachine.getCurrentState(); ApplicationState newState = null; diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java index 2c2d2baaa43..1cbdbaa8146 100644 --- a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java +++ b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java @@ -811,7 +811,7 @@ public class ContainerImpl implements Container { this.writeLock.lock(); ContainerId containerID = event.getContainerID(); - LOG.info("Processing " + containerID + " of type " + event.getType()); + LOG.debug("Processing " + containerID + " of type " + event.getType()); ContainerState oldState = stateMachine.getCurrentState(); ContainerState newState = null; diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/LocalizedResource.java b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/LocalizedResource.java index 883b4bcf559..bee9c2d3daa 100644 --- a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/LocalizedResource.java +++ b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/LocalizedResource.java @@ -181,7 +181,7 @@ public class LocalizedResource implements EventHandler { this.writeLock.lock(); Path resourcePath = event.getLocalResourceRequest().getPath(); - LOG.info("Processing " + resourcePath + " of type " + event.getType()); + LOG.debug("Processing " + resourcePath + " of type " + event.getType()); ResourceState oldState = this.stateMachine.getCurrentState(); ResourceState newState = null; diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java index f6cf29ac65a..c8cdf463d5b 100644 --- a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java +++ b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java @@ -413,7 +413,7 @@ public class RMAppImpl implements RMApp { try { ApplicationId appID = event.getApplicationId(); - LOG.info("Processing event for " + appID + " of type " + LOG.debug("Processing event for " + appID + " of type " + event.getType()); final RMAppState oldState = getState(); try { diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java index 0f695fda9f1..9882c615878 100644 --- a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java +++ b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java @@ -468,7 +468,7 @@ public class RMAppAttemptImpl implements RMAppAttempt { try { ApplicationAttemptId appAttemptID = event.getApplicationAttemptId(); - LOG.info("Processing event for " + appAttemptID + " of type " + LOG.debug("Processing event for " + appAttemptID + " of type " + event.getType()); final RMAppAttemptState oldState = getAppAttemptState(); try { diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java index ffdd23c4386..d845edeb614 100644 --- a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java +++ b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java @@ -192,7 +192,7 @@ public class RMContainerImpl implements RMContainer { @Override public void handle(RMContainerEvent event) { - LOG.info("Processing " + event.getContainerId() + " of type " + event.getType()); + LOG.debug("Processing " + event.getContainerId() + " of type " + event.getType()); try { writeLock.lock(); RMContainerState oldState = getState(); diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java index dd3e25fe167..7f2b48f85be 100644 --- a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java +++ b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java @@ -283,7 +283,7 @@ public class RMNodeImpl implements RMNode, EventHandler { } public void handle(RMNodeEvent event) { - LOG.info("Processing " + event.getNodeId() + " of type " + event.getType()); + LOG.debug("Processing " + event.getNodeId() + " of type " + event.getType()); try { writeLock.lock(); RMNodeState oldState = getState(); diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java index 0f6a8a84c8e..48e01a72fd0 100644 --- a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java +++ b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java @@ -575,12 +575,12 @@ public class FifoScheduler implements ResourceScheduler { if (Resources.greaterThanOrEqual(node.getAvailableResource(), minimumAllocation)) { - LOG.info("Node heartbeat " + rmNode.getNodeID() + + LOG.debug("Node heartbeat " + rmNode.getNodeID() + " available resource = " + node.getAvailableResource()); assignContainers(node); - LOG.info("Node after allocation " + rmNode.getNodeID() + " resource = " + LOG.debug("Node after allocation " + rmNode.getNodeID() + " resource = " + node.getAvailableResource()); }