YARN-3379. Fixed missing data in localityTable and ResourceRequests table in RM WebUI. Contributed by Xuan Gong

(cherry picked from commit 4e886eb9cb)

(cherry picked from commit 3f0c9e5fe3)

hadoop-yarn-project/CHANGES.txt

@@ -214,6 +214,9 @@ Release 2.6.1 - 2015-09-09
     YARN-3740. Fixed the typo in the configuration name:
     APPLICATION_HISTORY_PREFIX_MAX_APPS. (Xuan Gong via zjshen)
 
+    YARN-3379. Fixed missing data in localityTable and ResourceRequests table
+    in RM WebUI. (Xuan Gong via jianhe)
+
 Release 2.6.0 - 2014-11-18
 
   INCOMPATIBLE CHANGES

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppAttemptBlock.java

@@ -43,13 +43,13 @@ import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.TABLE;
 import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.TBODY;
 import org.apache.hadoop.yarn.webapp.view.HtmlBlock;
 import org.apache.hadoop.yarn.webapp.view.InfoBlock;
-
 import com.google.inject.Inject;
 
 public class AppAttemptBlock extends HtmlBlock {
 
   private static final Log LOG = LogFactory.getLog(AppAttemptBlock.class);
   protected ApplicationBaseProtocol appBaseProt;
+  protected ApplicationAttemptId appAttemptId = null;
 
   @Inject
   public AppAttemptBlock(ApplicationBaseProtocol appBaseProt, ViewContext ctx) {
@@ -65,7 +65,6 @@ public class AppAttemptBlock extends HtmlBlock {
       return;
     }
 
-    ApplicationAttemptId appAttemptId = null;
     try {
       appAttemptId = ConverterUtils.toApplicationAttemptId(attemptid);
     } catch (IllegalArgumentException e) {

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppBlock.java

@@ -21,15 +21,9 @@ package org.apache.hadoop.yarn.server.webapp;
 import static org.apache.hadoop.yarn.util.StringHelper.join;
 import static org.apache.hadoop.yarn.webapp.YarnWebParams.APPLICATION_ID;
 import static org.apache.hadoop.yarn.webapp.YarnWebParams.WEB_UI_TYPE;
-import static org.apache.hadoop.yarn.webapp.view.JQueryUI._EVEN;
-import static org.apache.hadoop.yarn.webapp.view.JQueryUI._INFO_WRAP;
-import static org.apache.hadoop.yarn.webapp.view.JQueryUI._ODD;
-import static org.apache.hadoop.yarn.webapp.view.JQueryUI._TH;
 
 import java.security.PrivilegedExceptionAction;
 import java.util.Collection;
-import java.util.List;
-
 import org.apache.commons.lang.StringEscapeUtils;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
@@ -43,12 +37,9 @@ import org.apache.hadoop.yarn.api.protocolrecords.GetContainerReportRequest;
 import org.apache.hadoop.yarn.api.records.ApplicationAttemptReport;
 import org.apache.hadoop.yarn.api.records.ApplicationId;
 import org.apache.hadoop.yarn.api.records.ApplicationReport;
-import org.apache.hadoop.yarn.api.records.ApplicationResourceUsageReport;
 import org.apache.hadoop.yarn.api.records.ContainerId;
 import org.apache.hadoop.yarn.api.records.ContainerReport;
 import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
-import org.apache.hadoop.yarn.api.records.Resource;
-import org.apache.hadoop.yarn.api.records.ResourceRequest;
 import org.apache.hadoop.yarn.api.records.YarnApplicationState;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.exceptions.ContainerNotFoundException;
@@ -57,10 +48,8 @@ import org.apache.hadoop.yarn.server.webapp.dao.AppInfo;
 import org.apache.hadoop.yarn.server.webapp.dao.ContainerInfo;
 import org.apache.hadoop.yarn.util.Apps;
 import org.apache.hadoop.yarn.util.Times;
-import org.apache.hadoop.yarn.util.resource.Resources;
 import org.apache.hadoop.yarn.webapp.YarnWebParams;
 import org.apache.hadoop.yarn.webapp.hamlet.Hamlet;
-import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.DIV;
 import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.TABLE;
 import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.TBODY;
 import org.apache.hadoop.yarn.webapp.view.HtmlBlock;
@@ -73,9 +62,11 @@ public class AppBlock extends HtmlBlock {
   private static final Log LOG = LogFactory.getLog(AppBlock.class);
   protected ApplicationBaseProtocol appBaseProt;
   protected Configuration conf;
+  protected ApplicationId appID = null;
 
   @Inject
-  AppBlock(ApplicationBaseProtocol appBaseProt, ViewContext ctx, Configuration conf) {
+  protected AppBlock(ApplicationBaseProtocol appBaseProt, ViewContext ctx,
+      Configuration conf) {
     super(ctx);
     this.appBaseProt = appBaseProt;
     this.conf = conf;
@@ -90,7 +81,6 @@ public class AppBlock extends HtmlBlock {
       return;
     }
 
-    ApplicationId appID = null;
     try {
       appID = Apps.toAppID(aid);
     } catch (Exception e) {
@@ -217,31 +207,7 @@ public class AppBlock extends HtmlBlock {
       return;
     }
 
-    //TODO:YARN-3284
-    //The preemption metrics will be exposed from ApplicationReport
-    // and ApplicationAttemptReport
-    ApplicationResourceUsageReport usageReport =
-        appReport.getApplicationResourceUsageReport();
-    DIV<Hamlet> pdiv = html.
-        _(InfoBlock.class).
-        div(_INFO_WRAP);
-    info("Application Overview").clear();
-    info("Application Metrics")
-        ._("Total Resource Preempted:",
-          Resources.none()) // TODO: YARN-3284
-        ._("Total Number of Non-AM Containers Preempted:",
-          String.valueOf(0)) // TODO: YARN-3284
-        ._("Total Number of AM Containers Preempted:",
-          String.valueOf(0)) // TODO: YARN-3284
-        ._("Resource Preempted from Current Attempt:",
-          Resources.none()) // TODO: YARN-3284
-        ._("Number of Non-AM Containers Preempted from Current Attempt:",
-          0) // TODO: YARN-3284
-        ._("Aggregate Resource Allocation:",
-          String.format("%d MB-seconds, %d vcore-seconds", usageReport == null
-            ? 0 : usageReport.getMemorySeconds(), usageReport == null ? 0
-            : usageReport.getVcoreSeconds()));
-    pdiv._();
+    createApplicationMetricsTable(html);
 
     html._(InfoBlock.class);
 
@@ -352,4 +318,9 @@ public class AppBlock extends HtmlBlock {
     }
     return status.toString();
   }
+
+  // The preemption metrics only need to be shown in RM WebUI
+  protected void createApplicationMetricsTable(Block html) {
+
+  }
 }

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/AppAttemptPage.java

@@ -23,7 +23,6 @@ import static org.apache.hadoop.yarn.webapp.view.JQueryUI.DATATABLES;
 import static org.apache.hadoop.yarn.webapp.view.JQueryUI.DATATABLES_ID;
 import static org.apache.hadoop.yarn.webapp.view.JQueryUI.initID;
 
-import org.apache.hadoop.yarn.server.webapp.AppAttemptBlock;
 import org.apache.hadoop.yarn.server.webapp.WebPageUtils;
 import org.apache.hadoop.yarn.webapp.SubView;
 import org.apache.hadoop.yarn.webapp.YarnWebParams;
@@ -49,7 +48,7 @@ public class AppAttemptPage extends RmView {
 
   @Override
   protected Class<? extends SubView> content() {
-    return AppAttemptBlock.class;
+    return RMAppAttemptBlock.class;
   }
 
 }

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/AppPage.java

@@ -23,7 +23,6 @@ import static org.apache.hadoop.yarn.webapp.view.JQueryUI.DATATABLES;
 import static org.apache.hadoop.yarn.webapp.view.JQueryUI.DATATABLES_ID;
 import static org.apache.hadoop.yarn.webapp.view.JQueryUI.initID;
 
-import org.apache.hadoop.yarn.server.webapp.AppBlock;
 import org.apache.hadoop.yarn.server.webapp.WebPageUtils;
 import org.apache.hadoop.yarn.webapp.SubView;
 import org.apache.hadoop.yarn.webapp.YarnWebParams;
@@ -50,6 +49,6 @@ public class AppPage extends RmView {
 
   @Override 
   protected Class<? extends SubView> content() {
-    return AppBlock.class;
+    return RMAppBlock.class;
   }
 }

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppAttemptBlock.java

@@ -0,0 +1,56 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.webapp;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt;
+import org.apache.hadoop.yarn.server.webapp.AppAttemptBlock;
+
+import com.google.inject.Inject;
+
+public class RMAppAttemptBlock extends AppAttemptBlock{
+
+  private final ResourceManager rm;
+  protected Configuration conf;
+
+  @Inject
+  RMAppAttemptBlock(ViewContext ctx, ResourceManager rm, Configuration conf) {
+    super(rm.getClientRMService(), ctx);
+    this.rm = rm;
+    this.conf = conf;
+  }
+
+  @Override
+  protected void render(Block html) {
+    super.render(html);
+  }
+
+  private RMAppAttempt getRMAppAttempt() {
+    ApplicationId appId = this.appAttemptId.getApplicationId();
+    RMAppAttempt attempt = null;
+    RMApp rmApp = rm.getRMContext().getRMApps().get(appId);
+    if (rmApp != null) { 
+      attempt = rmApp.getAppAttempts().get(appAttemptId);
+    }
+    return attempt;
+  }
+}

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppBlock.java

@@ -0,0 +1,94 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.webapp;
+
+import static org.apache.hadoop.yarn.webapp.view.JQueryUI._INFO_WRAP;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppMetrics;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptMetrics;
+import org.apache.hadoop.yarn.server.webapp.AppBlock;
+import org.apache.hadoop.yarn.util.resource.Resources;
+import org.apache.hadoop.yarn.webapp.hamlet.Hamlet;
+import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.DIV;
+import org.apache.hadoop.yarn.webapp.view.InfoBlock;
+
+import com.google.inject.Inject;
+
+public class RMAppBlock extends AppBlock{
+
+  private final ResourceManager rm;
+
+  @Inject
+  RMAppBlock(ViewContext ctx, Configuration conf, ResourceManager rm) {
+    super(rm.getClientRMService(), ctx, conf);
+    this.rm = rm;
+  }
+
+  @Override
+  protected void render(Block html) {
+    super.render(html);
+  }
+
+  @Override
+  protected void createApplicationMetricsTable(Block html){
+    RMApp rmApp = this.rm.getRMContext().getRMApps().get(appID);
+    RMAppMetrics appMetrics = rmApp == null ? null : rmApp.getRMAppMetrics();
+    // Get attempt metrics and fields, it is possible currentAttempt of RMApp is
+    // null. In that case, we will assume resource preempted and number of Non
+    // AM container preempted on that attempt is 0
+    RMAppAttemptMetrics attemptMetrics;
+    if (rmApp == null || null == rmApp.getCurrentAppAttempt()) {
+      attemptMetrics = null;
+    } else {
+      attemptMetrics = rmApp.getCurrentAppAttempt().getRMAppAttemptMetrics();
+    }
+    Resource attemptResourcePreempted =
+        attemptMetrics == null ? Resources.none() : attemptMetrics
+          .getResourcePreempted();
+    int attemptNumNonAMContainerPreempted =
+        attemptMetrics == null ? 0 : attemptMetrics
+          .getNumNonAMContainersPreempted();
+    DIV<Hamlet> pdiv = html.
+        _(InfoBlock.class).
+        div(_INFO_WRAP);
+    info("Application Overview").clear();
+    info("Application Metrics")
+        ._("Total Resource Preempted:",
+          appMetrics == null ? "N/A" : appMetrics.getResourcePreempted())
+        ._("Total Number of Non-AM Containers Preempted:",
+          appMetrics == null ? "N/A"
+              : appMetrics.getNumNonAMContainersPreempted())
+        ._("Total Number of AM Containers Preempted:",
+          appMetrics == null ? "N/A"
+              : appMetrics.getNumAMContainersPreempted())
+        ._("Resource Preempted from Current Attempt:",
+          attemptResourcePreempted)
+        ._("Number of Non-AM Containers Preempted from Current Attempt:",
+          attemptNumNonAMContainerPreempted)
+        ._("Aggregate Resource Allocation:",
+          String.format("%d MB-seconds, %d vcore-seconds",
+              appMetrics == null ? "N/A" : appMetrics.getMemorySeconds(),
+              appMetrics == null ? "N/A" : appMetrics.getVcoreSeconds()));
+    pdiv._();
+  }
+}