MAPREDUCE-2907. svn merge -c r1179178 --ignore-ancestry ../../trunk/

git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/branch-0.23@1179181 13f79535-47bb-0310-9956-ffa450edef68

hadoop-mapreduce-project/CHANGES.txt

@@ -312,6 +312,9 @@ Release 0.23.0 - Unreleased
     MAPREDUCE-3013. Removed YarnConfiguration.YARN_SECURITY_INFO and its usage
     as it doesn't affect security any more. (vinodkv)
 
+    MAPREDUCE-2907. Changed log level for various messages in ResourceManager
+    from INFO to DEBUG. (Ravi Prakash via vinodkv)
+
   OPTIMIZATIONS
 
     MAPREDUCE-2026. Make JobTracker.getJobCounters() and

hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java

@@ -991,10 +991,9 @@ public class JobImpl implements org.apache.hadoop.mapreduce.v2.app.job.Job,
       String user = 
         UserGroupInformation.getCurrentUser().getShortUserName();
       Path path = MRApps.getStagingAreaDir(job.conf, user);
-      LOG.info("DEBUG --- startJobs:"
+      if(LOG.isDebugEnabled()) {
-          + " parent="
+        LOG.debug("startJobs: parent=" + path + " child=" + oldJobIDString);
-          + path + " child="
+      }
-          + oldJobIDString);
 
       job.remoteJobSubmitDir =
           FileSystem.get(job.conf).makeQualified(

hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ResourceMgrDelegate.java

@@ -258,7 +258,7 @@ public class ResourceMgrDelegate {
     String user = 
       UserGroupInformation.getCurrentUser().getShortUserName();
     Path path = MRApps.getStagingAreaDir(conf, user);
-    LOG.info("DEBUG --- getStagingAreaDir: dir=" + path);
+    LOG.debug("getStagingAreaDir: dir=" + path);
     return path.toString();
   }
 

hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java

@@ -464,21 +464,25 @@ implements ResourceScheduler, CapacitySchedulerContext {
 
       if (!ask.isEmpty()) {
 
-        LOG.info("DEBUG --- allocate: pre-update" +
+        if(LOG.isDebugEnabled()) {
+          LOG.debug("allocate: pre-update" +
             " applicationAttemptId=" + applicationAttemptId + 
             " application=" + application);
+        }
         application.showRequests();
   
         // Update application requests
         application.updateResourceRequests(ask);
   
-        LOG.info("DEBUG --- allocate: post-update");
+        LOG.debug("allocate: post-update");
         application.showRequests();
       }
 
-      LOG.info("DEBUG --- allocate:" +
+      if(LOG.isDebugEnabled()) {
+        LOG.debug("allocate:" +
           " applicationAttemptId=" + applicationAttemptId + 
           " #ask=" + ask.size());
+      }
 
       return new Allocation(
           application.pullNewlyAllocatedContainers(), 
@@ -547,14 +551,16 @@ implements ResourceScheduler, CapacitySchedulerContext {
     // Process completed containers
     for (ContainerStatus completedContainer : completedContainers) {
       ContainerId containerId = completedContainer.getContainerId();
-      LOG.info("DEBUG --- Container FINISHED: " + containerId);
+      LOG.debug("Container FINISHED: " + containerId);
       completedContainer(getRMContainer(containerId), 
           completedContainer, RMContainerEventType.FINISHED);
     }
 
     // Now node data structures are upto date and ready for scheduling.
-    LOG.info("DEBUG -- Node being looked for scheduling " + nm
+    if(LOG.isDebugEnabled()) {
+      LOG.debug("Node being looked for scheduling " + nm
         + " availableResource: " + node.getAvailableResource());
+    }
 
     // Assign new containers...
     // 1. Check for reserved applications

hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java

@@ -184,9 +184,10 @@ public class LeafQueue implements CSQueue {
         maxActiveApplications, maxActiveApplicationsPerUser,
         state, acls);
 
-    LOG.info("DEBUG --- LeafQueue:" +
+    if(LOG.isDebugEnabled()) {
-        " name=" + queueName + 
+      LOG.debug("LeafQueue:" + " name=" + queueName
-        ", fullname=" + getQueuePath());
+        + ", fullname=" + getQueuePath());
+    }
 
     this.pendingApplications = 
         new TreeSet<SchedulerApp>(applicationComparator);
@@ -673,9 +674,10 @@ public class LeafQueue implements CSQueue {
   public synchronized Resource 
   assignContainers(Resource clusterResource, SchedulerNode node) {
 
-    LOG.info("DEBUG --- assignContainers:" +
+    if(LOG.isDebugEnabled()) {
-        " node=" + node.getHostName() + 
+      LOG.debug("assignContainers: node=" + node.getHostName()
-        " #applications=" + activeApplications.size());
+        + " #applications=" + activeApplications.size());
+    }
     
     // Check for reserved resources
     RMContainer reservedContainer = node.getReservedContainer();
@@ -689,8 +691,10 @@ public class LeafQueue implements CSQueue {
     // Try to assign containers to applications in order
     for (SchedulerApp application : activeApplications) {
       
-      LOG.info("DEBUG --- pre-assignContainers for application "
+      if(LOG.isDebugEnabled()) {
+        LOG.debug("pre-assignContainers for application "
         + application.getApplicationId());
+      }
       application.showRequests();
 
       synchronized (application) {
@@ -750,8 +754,10 @@ public class LeafQueue implements CSQueue {
         }
       }
 
-      LOG.info("DEBUG --- post-assignContainers for application "
+      if(LOG.isDebugEnabled()) {
+        LOG.debug("post-assignContainers for application "
           + application.getApplicationId());
+      }
       application.showRequests();
     }
   
@@ -1094,11 +1100,10 @@ public class LeafQueue implements CSQueue {
       SchedulerApp application, Priority priority, 
       ResourceRequest request, NodeType type, RMContainer rmContainer) {
     if (LOG.isDebugEnabled()) {
-      LOG.info("DEBUG --- assignContainers:" +
+      LOG.debug("assignContainers: node=" + node.getHostName()
-          " node=" + node.getHostName() + 
+        + " application=" + application.getApplicationId().getId()
-          " application=" + application.getApplicationId().getId() + 
+        + " priority=" + priority.getPriority()
-          " priority=" + priority.getPriority() + 
+        + " request=" + request + " type=" + type);
-          " request=" + request + " type=" + type);
     }
     Resource capability = request.getCapability();
 

hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java

@@ -197,7 +197,9 @@ public class ParentQueue implements CSQueue {
     
     this.childQueues.clear();
     this.childQueues.addAll(childQueues);
-    LOG.info("DEBUG --- setChildQueues: " + getChildQueuesToPrint());
+    if (LOG.isDebugEnabled()) {
+      LOG.debug("setChildQueues: " + getChildQueuesToPrint());
+    }
   }
   
   @Override
@@ -497,8 +499,10 @@ public class ParentQueue implements CSQueue {
     Resource assigned = Resources.createResource(0);
 
     while (canAssign(node)) {
-      LOG.info("DEBUG --- Trying to assign containers to child-queue of " + 
+      if (LOG.isDebugEnabled()) {
-          getQueueName());
+        LOG.debug("Trying to assign containers to child-queue of "
+          + getQueueName());
+      }
       
       // Are we over maximum-capacity for this queue?
       if (!assignToQueue(clusterResource)) {
@@ -527,10 +531,11 @@ public class ParentQueue implements CSQueue {
         break;
       }
 
-      LOG.info("DEBUG ---" +
+      if (LOG.isDebugEnabled()) {
-      		" parentQ=" + getQueueName() + 
+        LOG.debug("ParentQ=" + getQueueName()
-      		" assignedSoFarInThisIteration=" + assigned + 
+          + " assignedSoFarInThisIteration=" + assigned
-      		" utilization=" + getUtilization());
+          + " utilization=" + getUtilization());
+      }
 
       // Do not assign more than one container if this isn't the root queue
       if (!rootQueue) {
@@ -571,13 +576,15 @@ public class ParentQueue implements CSQueue {
     // Try to assign to most 'under-served' sub-queue
     for (Iterator<CSQueue> iter=childQueues.iterator(); iter.hasNext();) {
       CSQueue childQueue = iter.next();
-      LOG.info("DEBUG --- Trying to assign to" +
+      if(LOG.isDebugEnabled()) {
-      		" queue: " + childQueue.getQueuePath() + 
+        LOG.debug("Trying to assign to queue: " + childQueue.getQueuePath()
-      		" stats: " + childQueue);
+          + " stats: " + childQueue);
+      }
       assigned = childQueue.assignContainers(cluster, node);
-      LOG.info("DEBUG --- Assignedto" +
+      if(LOG.isDebugEnabled()) {
-          " queue: " + childQueue.getQueuePath() + 
+        LOG.debug("Assignedto queue: " + childQueue.getQueuePath()
-          " stats: " + childQueue + " --> " + assigned.getMemory());
+          + " stats: " + childQueue + " --> " + assigned.getMemory());
+      }
 
       // If we do assign, remove the queue and re-insert in-order to re-sort
       if (Resources.greaterThan(assigned, Resources.none())) {
@@ -602,8 +609,10 @@ public class ParentQueue implements CSQueue {
     return sb.toString();
   }
   void printChildQueues() {
-    LOG.info("DEBUG --- printChildQueues - queue: " + getQueuePath() + 
+    if (LOG.isDebugEnabled()) {
-        " child-queues: " + getChildQueuesToPrint());
+      LOG.debug("printChildQueues - queue: " + getQueuePath()
+        + " child-queues: " + getChildQueuesToPrint());
+    }
   }
   
   @Override

hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java

@@ -595,7 +595,7 @@ public class FifoScheduler implements ResourceScheduler {
     // Process completed containers
     for (ContainerStatus completedContainer : completedContainers) {
       ContainerId containerId = completedContainer.getContainerId();
-      LOG.info("DEBUG --- Container FINISHED: " + containerId);
+      LOG.debug("Container FINISHED: " + containerId);
       containerCompleted(getRMContainer(containerId), 
           completedContainer, RMContainerEventType.FINISHED);
     }

hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/Application.java

@@ -164,9 +164,10 @@ public class Application {
     if (requests == null) {
       requests = new HashMap<String, ResourceRequest>();
       this.requests.put(priority, requests);
-      LOG.info("DEBUG --- Added" +
+      if(LOG.isDebugEnabled()) {
-      		" priority=" + priority + 
+        LOG.debug("Added priority=" + priority + " application="
-      		" application=" + applicationId);
+          + applicationId);
+      }
     }
     
     final Resource capability = requestSpec.get(priority);
@@ -182,9 +183,10 @@ public class Application {
     LOG.info("Added task " + task.getTaskId() + " to application " + 
         applicationId + " at priority " + priority);
     
-    LOG.info("DEBUG --- addTask:" +
+    if(LOG.isDebugEnabled()) {
-    		" application=" + applicationId + 
+      LOG.debug("addTask: application=" + applicationId
-    		" #asks=" + ask.size());
+        + " #asks=" + ask.size());
+    }
     
     // Create resource requests
     for (String host : task.getHosts()) {
@@ -245,23 +247,24 @@ public class Application {
         org.apache.hadoop.yarn.util.BuilderUtils.newResourceRequest(
             request)); // clone to ensure the RM doesn't manipulate the same obj
     
-    LOG.info("DEBUG --- addResourceRequest:" +
+    if(LOG.isDebugEnabled()) {
-    		" applicationId=" + applicationId.getId() +
+      LOG.debug("addResourceRequest: applicationId=" + applicationId.getId()
-    		" priority=" + priority.getPriority() + 
+        + " priority=" + priority.getPriority()
-        " resourceName=" + resourceName + 
+        + " resourceName=" + resourceName + " capability=" + capability
-        " capability=" + capability +
+        + " numContainers=" + request.getNumContainers()
-        " numContainers=" + request.getNumContainers() + 
+        + " #asks=" + ask.size());
-        " #asks=" + ask.size());
+    }
   }
   
   public synchronized List<Container> getResources() throws IOException {
-    LOG.info("DEBUG --- getResources begin:" +
+    if(LOG.isDebugEnabled()) {
-        " application=" + applicationId + 
+      LOG.debug("getResources begin:" + " application=" + applicationId
-        " #ask=" + ask.size());
+        + " #ask=" + ask.size());
+
       for (ResourceRequest request : ask) {
-      LOG.info("DEBUG --- getResources:" +
+        LOG.debug("getResources:" + " application=" + applicationId
-          " application=" + applicationId + 
+          + " ask-request=" + request);
-          " ask-request=" + request);
+      }
     }
     
     // Get resources from the ResourceManager
@@ -280,9 +283,10 @@ public class Application {
     // Clear state for next interaction with ResourceManager
     ask.clear();
     
-    LOG.info("DEBUG --- getResources() for " + applicationId + ":" +
+    if(LOG.isDebugEnabled()) {
-    		" ask=" + ask.size() + 
+      LOG.debug("getResources() for " + applicationId + ":"
-    		" recieved=" + containers.size());
+        + " ask=" + ask.size() + " recieved=" + containers.size());
+    }
     
     return containers;
   }
@@ -353,22 +357,22 @@ public class Application {
       NodeType type, Task task) {
     if (type == NodeType.NODE_LOCAL) {
       for (String host : task.getHosts()) {
-        LOG.info("DEBUG --- updateResourceRequests:" +
+        if(LOG.isDebugEnabled()) {
-            " application=" + applicationId +
+          LOG.debug("updateResourceRequests:" + " application=" + applicationId
-        		" type=" + type + 
+            + " type=" + type + " host=" + host
-        		" host=" + host + 
+            + " request=" + ((requests == null) ? "null" : requests.get(host)));
-        		" request=" + ((requests == null) ? "null" : requests.get(host)));
+        }
         updateResourceRequest(requests.get(host));
       }
     }
     
     if (type == NodeType.NODE_LOCAL || type == NodeType.RACK_LOCAL) {
       for (String rack : task.getRacks()) {
-        LOG.info("DEBUG --- updateResourceRequests:" +
+        if(LOG.isDebugEnabled()) {
-            " application=" + applicationId +
+          LOG.debug("updateResourceRequests:" + " application=" + applicationId
-            " type=" + type + 
+            + " type=" + type + " rack=" + rack
-            " rack=" + rack + 
+            + " request=" + ((requests == null) ? "null" : requests.get(rack)));
-            " request=" + ((requests == null) ? "null" : requests.get(rack)));
+        }
         updateResourceRequest(requests.get(rack));
       }
     }
@@ -378,9 +382,10 @@ public class Application {
             org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode.ANY)
             );
     
-    LOG.info("DEBUG --- updateResourceRequests:" +
+    if(LOG.isDebugEnabled()) {
-        " application=" + applicationId +
+      LOG.debug("updateResourceRequests:" + " application=" + applicationId
-    		" #asks=" + ask.size());
+        + " #asks=" + ask.size());
+    }
   }
   
   private void updateResourceRequest(ResourceRequest request) {
@@ -392,9 +397,10 @@ public class Application {
         org.apache.hadoop.yarn.util.BuilderUtils.newResourceRequest(
         request)); // clone to ensure the RM doesn't manipulate the same obj
 
-    LOG.info("DEBUG --- updateResourceRequest:" +
+    if(LOG.isDebugEnabled()) {
-        " application=" + applicationId +
+      LOG.debug("updateResourceRequest:" + " application=" + applicationId
-    		" request=" + request);
+        + " request=" + request);
+    }
   }
 
   private ContainerLaunchContext createCLC(Container container) {

hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/NodeManager.java

@@ -194,12 +194,11 @@ public class NodeManager implements ContainerManager {
     Resources.subtractFrom(available, containerLaunchContext.getResource());
     Resources.addTo(used, containerLaunchContext.getResource());
     
-    LOG.info("DEBUG --- startContainer:" +
+    if(LOG.isDebugEnabled()) {
-        " node=" + containerManagerAddress +
+      LOG.debug("startContainer:" + " node=" + containerManagerAddress
-        " application=" + applicationId + 
+        + " application=" + applicationId + " container=" + container
-        " container=" + container +
+        + " available=" + available + " used=" + used);
-        " available=" + available +
+    }
-        " used=" + used);
 
     StartContainerResponse response = recordFactory.newRecordInstance(StartContainerResponse.class);
     return response;
@@ -254,12 +253,11 @@ public class NodeManager implements ContainerManager {
     Resources.addTo(available, container.getResource());
     Resources.subtractFrom(used, container.getResource());
 
-    LOG.info("DEBUG --- stopContainer:" +
+    if(LOG.isDebugEnabled()) {
-        " node=" + containerManagerAddress +
+      LOG.debug("stopContainer:" + " node=" + containerManagerAddress
-        " application=" + applicationId + 
+        + " application=" + applicationId + " container=" + containerID
-        " container=" + containerID +
+        + " available=" + available + " used=" + used);
-        " available=" + available +
+    }
-        " used=" + used);
 
     StopContainerResponse response = recordFactory.newRecordInstance(StopContainerResponse.class);
     return response;