YARN-2679. Add metric for container launch duration. (Zhihai Xu via kasha)

hadoop-yarn-project/CHANGES.txt

@@ -87,6 +87,9 @@ Release 2.7.0 - UNRELEASED
     YARN-2604. Scheduler should consider max-allocation-* in conjunction
     with the largest node. (Robert Kanter via kasha)
 
+    YARN-2679. Add metric for container launch duration. (Zhihai Xu via kasha)
+
+
   OPTIMIZATIONS
 
   BUG FIXES

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java

@@ -74,7 +74,9 @@ import org.apache.hadoop.yarn.state.MultipleArcTransition;
 import org.apache.hadoop.yarn.state.SingleArcTransition;
 import org.apache.hadoop.yarn.state.StateMachine;
 import org.apache.hadoop.yarn.state.StateMachineFactory;
+import org.apache.hadoop.yarn.util.Clock;
 import org.apache.hadoop.yarn.util.ConverterUtils;
+import org.apache.hadoop.yarn.util.SystemClock;
 
 public class ContainerImpl implements Container {
 
@@ -92,6 +94,8 @@ public class ContainerImpl implements Container {
   private int exitCode = ContainerExitStatus.INVALID;
   private final StringBuilder diagnostics;
   private boolean wasLaunched;
+  private long containerLaunchStartTime;
+  private static Clock clock = new SystemClock();
 
   /** The NM-wide configuration - not specific to this container */
   private final Configuration daemonConf;
@@ -521,6 +525,7 @@ public class ContainerImpl implements Container {
       // try to recover a container that was previously launched
       launcherEvent = ContainersLauncherEventType.RECOVER_CONTAINER;
     }
+    containerLaunchStartTime = clock.getTime();
     dispatcher.getEventHandler().handle(
         new ContainersLauncherEvent(this, launcherEvent));
   }
@@ -781,6 +786,8 @@ public class ContainerImpl implements Container {
       container.sendContainerMonitorStartEvent();
       container.metrics.runningContainer();
       container.wasLaunched  = true;
+      long duration = clock.getTime() - container.containerLaunchStartTime;
+      container.metrics.addContainerLaunchDuration(duration);
 
       if (container.recoveredAsKilled) {
         LOG.info("Killing " + container.containerId

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/metrics/NodeManagerMetrics.java

@@ -23,6 +23,7 @@ import org.apache.hadoop.metrics2.annotation.Metrics;
 import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
 import org.apache.hadoop.metrics2.lib.MutableCounterInt;
 import org.apache.hadoop.metrics2.lib.MutableGaugeInt;
+import org.apache.hadoop.metrics2.lib.MutableRate;
 import org.apache.hadoop.metrics2.source.JvmMetrics;
 import org.apache.hadoop.yarn.api.records.Resource;
 
@@ -43,6 +44,8 @@ public class NodeManagerMetrics {
   @Metric("Current allocated Virtual Cores")
       MutableGaugeInt allocatedVCores;
   @Metric MutableGaugeInt availableVCores;
+  @Metric("Container launch duration")
+      MutableRate containerLaunchDuration;
 
   public static NodeManagerMetrics create() {
     return create(DefaultMetricsSystem.instance());
@@ -107,7 +110,11 @@ public class NodeManagerMetrics {
     availableGB.incr(res.getMemory() / 1024);
     availableVCores.incr(res.getVirtualCores());
   }
-  
+
+  public void addContainerLaunchDuration(long value) {
+    containerLaunchDuration.add(value);
+  }
+
   public int getRunningContainers() {
     return containersRunning.value();
   }

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/metrics/TestNodeManagerMetrics.java

@@ -59,6 +59,7 @@ public class TestNodeManagerMetrics {
 
     metrics.initingContainer();
     metrics.runningContainer();
+    metrics.addContainerLaunchDuration(1);
 
     checkMetrics(5, 1, 1, 1, 1, 1, 2, 2, 6, 4, 12);
   }