diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 6641bb24b55..e7d847f635c 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -214,6 +214,9 @@ Release 2.6.1 - 2015-09-09 YARN-3740. Fixed the typo in the configuration name: APPLICATION_HISTORY_PREFIX_MAX_APPS. (Xuan Gong via zjshen) + YARN-3379. Fixed missing data in localityTable and ResourceRequests table + in RM WebUI. (Xuan Gong via jianhe) + Release 2.6.0 - 2014-11-18 INCOMPATIBLE CHANGES diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppAttemptBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppAttemptBlock.java index ae5f092cbc8..abf7c866f3d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppAttemptBlock.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppAttemptBlock.java @@ -43,13 +43,13 @@ import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.TABLE; import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.TBODY; import org.apache.hadoop.yarn.webapp.view.HtmlBlock; import org.apache.hadoop.yarn.webapp.view.InfoBlock; - import com.google.inject.Inject; public class AppAttemptBlock extends HtmlBlock { private static final Log LOG = LogFactory.getLog(AppAttemptBlock.class); protected ApplicationBaseProtocol appBaseProt; + protected ApplicationAttemptId appAttemptId = null; @Inject public AppAttemptBlock(ApplicationBaseProtocol appBaseProt, ViewContext ctx) { @@ -65,7 +65,6 @@ public class AppAttemptBlock extends HtmlBlock { return; } - ApplicationAttemptId appAttemptId = null; try { appAttemptId = ConverterUtils.toApplicationAttemptId(attemptid); } catch (IllegalArgumentException e) { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppBlock.java index e62c783a4aa..93b24a30b0d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppBlock.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppBlock.java @@ -21,15 +21,9 @@ package org.apache.hadoop.yarn.server.webapp; import static org.apache.hadoop.yarn.util.StringHelper.join; import static org.apache.hadoop.yarn.webapp.YarnWebParams.APPLICATION_ID; import static org.apache.hadoop.yarn.webapp.YarnWebParams.WEB_UI_TYPE; -import static org.apache.hadoop.yarn.webapp.view.JQueryUI._EVEN; -import static org.apache.hadoop.yarn.webapp.view.JQueryUI._INFO_WRAP; -import static org.apache.hadoop.yarn.webapp.view.JQueryUI._ODD; -import static org.apache.hadoop.yarn.webapp.view.JQueryUI._TH; import java.security.PrivilegedExceptionAction; import java.util.Collection; -import java.util.List; - import org.apache.commons.lang.StringEscapeUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -43,12 +37,9 @@ import org.apache.hadoop.yarn.api.protocolrecords.GetContainerReportRequest; import org.apache.hadoop.yarn.api.records.ApplicationAttemptReport; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ApplicationReport; -import org.apache.hadoop.yarn.api.records.ApplicationResourceUsageReport; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.ContainerReport; import org.apache.hadoop.yarn.api.records.FinalApplicationStatus; -import org.apache.hadoop.yarn.api.records.Resource; -import org.apache.hadoop.yarn.api.records.ResourceRequest; import org.apache.hadoop.yarn.api.records.YarnApplicationState; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.ContainerNotFoundException; @@ -57,10 +48,8 @@ import org.apache.hadoop.yarn.server.webapp.dao.AppInfo; import org.apache.hadoop.yarn.server.webapp.dao.ContainerInfo; import org.apache.hadoop.yarn.util.Apps; import org.apache.hadoop.yarn.util.Times; -import org.apache.hadoop.yarn.util.resource.Resources; import org.apache.hadoop.yarn.webapp.YarnWebParams; import org.apache.hadoop.yarn.webapp.hamlet.Hamlet; -import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.DIV; import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.TABLE; import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.TBODY; import org.apache.hadoop.yarn.webapp.view.HtmlBlock; @@ -73,9 +62,11 @@ public class AppBlock extends HtmlBlock { private static final Log LOG = LogFactory.getLog(AppBlock.class); protected ApplicationBaseProtocol appBaseProt; protected Configuration conf; + protected ApplicationId appID = null; @Inject - AppBlock(ApplicationBaseProtocol appBaseProt, ViewContext ctx, Configuration conf) { + protected AppBlock(ApplicationBaseProtocol appBaseProt, ViewContext ctx, + Configuration conf) { super(ctx); this.appBaseProt = appBaseProt; this.conf = conf; @@ -90,7 +81,6 @@ public class AppBlock extends HtmlBlock { return; } - ApplicationId appID = null; try { appID = Apps.toAppID(aid); } catch (Exception e) { @@ -217,31 +207,7 @@ public class AppBlock extends HtmlBlock { return; } - //TODO:YARN-3284 - //The preemption metrics will be exposed from ApplicationReport - // and ApplicationAttemptReport - ApplicationResourceUsageReport usageReport = - appReport.getApplicationResourceUsageReport(); - DIV pdiv = html. - _(InfoBlock.class). - div(_INFO_WRAP); - info("Application Overview").clear(); - info("Application Metrics") - ._("Total Resource Preempted:", - Resources.none()) // TODO: YARN-3284 - ._("Total Number of Non-AM Containers Preempted:", - String.valueOf(0)) // TODO: YARN-3284 - ._("Total Number of AM Containers Preempted:", - String.valueOf(0)) // TODO: YARN-3284 - ._("Resource Preempted from Current Attempt:", - Resources.none()) // TODO: YARN-3284 - ._("Number of Non-AM Containers Preempted from Current Attempt:", - 0) // TODO: YARN-3284 - ._("Aggregate Resource Allocation:", - String.format("%d MB-seconds, %d vcore-seconds", usageReport == null - ? 0 : usageReport.getMemorySeconds(), usageReport == null ? 0 - : usageReport.getVcoreSeconds())); - pdiv._(); + createApplicationMetricsTable(html); html._(InfoBlock.class); @@ -352,4 +318,9 @@ public class AppBlock extends HtmlBlock { } return status.toString(); } + + // The preemption metrics only need to be shown in RM WebUI + protected void createApplicationMetricsTable(Block html) { + + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/AppAttemptPage.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/AppAttemptPage.java index 92eae48485d..2bbe4ff57be 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/AppAttemptPage.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/AppAttemptPage.java @@ -23,7 +23,6 @@ import static org.apache.hadoop.yarn.webapp.view.JQueryUI.DATATABLES; import static org.apache.hadoop.yarn.webapp.view.JQueryUI.DATATABLES_ID; import static org.apache.hadoop.yarn.webapp.view.JQueryUI.initID; -import org.apache.hadoop.yarn.server.webapp.AppAttemptBlock; import org.apache.hadoop.yarn.server.webapp.WebPageUtils; import org.apache.hadoop.yarn.webapp.SubView; import org.apache.hadoop.yarn.webapp.YarnWebParams; @@ -49,7 +48,7 @@ public class AppAttemptPage extends RmView { @Override protected Class content() { - return AppAttemptBlock.class; + return RMAppAttemptBlock.class; } } \ No newline at end of file diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/AppPage.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/AppPage.java index 77ae44c8d38..b0698afd33d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/AppPage.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/AppPage.java @@ -23,7 +23,6 @@ import static org.apache.hadoop.yarn.webapp.view.JQueryUI.DATATABLES; import static org.apache.hadoop.yarn.webapp.view.JQueryUI.DATATABLES_ID; import static org.apache.hadoop.yarn.webapp.view.JQueryUI.initID; -import org.apache.hadoop.yarn.server.webapp.AppBlock; import org.apache.hadoop.yarn.server.webapp.WebPageUtils; import org.apache.hadoop.yarn.webapp.SubView; import org.apache.hadoop.yarn.webapp.YarnWebParams; @@ -50,6 +49,6 @@ public class AppPage extends RmView { @Override protected Class content() { - return AppBlock.class; + return RMAppBlock.class; } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppAttemptBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppAttemptBlock.java new file mode 100644 index 00000000000..7dc70301c56 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppAttemptBlock.java @@ -0,0 +1,56 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.webapp; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager; +import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; +import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt; +import org.apache.hadoop.yarn.server.webapp.AppAttemptBlock; + +import com.google.inject.Inject; + +public class RMAppAttemptBlock extends AppAttemptBlock{ + + private final ResourceManager rm; + protected Configuration conf; + + @Inject + RMAppAttemptBlock(ViewContext ctx, ResourceManager rm, Configuration conf) { + super(rm.getClientRMService(), ctx); + this.rm = rm; + this.conf = conf; + } + + @Override + protected void render(Block html) { + super.render(html); + } + + private RMAppAttempt getRMAppAttempt() { + ApplicationId appId = this.appAttemptId.getApplicationId(); + RMAppAttempt attempt = null; + RMApp rmApp = rm.getRMContext().getRMApps().get(appId); + if (rmApp != null) { + attempt = rmApp.getAppAttempts().get(appAttemptId); + } + return attempt; + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppBlock.java new file mode 100644 index 00000000000..64c57476f79 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppBlock.java @@ -0,0 +1,94 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.webapp; + +import static org.apache.hadoop.yarn.webapp.view.JQueryUI._INFO_WRAP; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager; +import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; +import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppMetrics; +import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptMetrics; +import org.apache.hadoop.yarn.server.webapp.AppBlock; +import org.apache.hadoop.yarn.util.resource.Resources; +import org.apache.hadoop.yarn.webapp.hamlet.Hamlet; +import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.DIV; +import org.apache.hadoop.yarn.webapp.view.InfoBlock; + +import com.google.inject.Inject; + +public class RMAppBlock extends AppBlock{ + + private final ResourceManager rm; + + @Inject + RMAppBlock(ViewContext ctx, Configuration conf, ResourceManager rm) { + super(rm.getClientRMService(), ctx, conf); + this.rm = rm; + } + + @Override + protected void render(Block html) { + super.render(html); + } + + @Override + protected void createApplicationMetricsTable(Block html){ + RMApp rmApp = this.rm.getRMContext().getRMApps().get(appID); + RMAppMetrics appMetrics = rmApp == null ? null : rmApp.getRMAppMetrics(); + // Get attempt metrics and fields, it is possible currentAttempt of RMApp is + // null. In that case, we will assume resource preempted and number of Non + // AM container preempted on that attempt is 0 + RMAppAttemptMetrics attemptMetrics; + if (rmApp == null || null == rmApp.getCurrentAppAttempt()) { + attemptMetrics = null; + } else { + attemptMetrics = rmApp.getCurrentAppAttempt().getRMAppAttemptMetrics(); + } + Resource attemptResourcePreempted = + attemptMetrics == null ? Resources.none() : attemptMetrics + .getResourcePreempted(); + int attemptNumNonAMContainerPreempted = + attemptMetrics == null ? 0 : attemptMetrics + .getNumNonAMContainersPreempted(); + DIV pdiv = html. + _(InfoBlock.class). + div(_INFO_WRAP); + info("Application Overview").clear(); + info("Application Metrics") + ._("Total Resource Preempted:", + appMetrics == null ? "N/A" : appMetrics.getResourcePreempted()) + ._("Total Number of Non-AM Containers Preempted:", + appMetrics == null ? "N/A" + : appMetrics.getNumNonAMContainersPreempted()) + ._("Total Number of AM Containers Preempted:", + appMetrics == null ? "N/A" + : appMetrics.getNumAMContainersPreempted()) + ._("Resource Preempted from Current Attempt:", + attemptResourcePreempted) + ._("Number of Non-AM Containers Preempted from Current Attempt:", + attemptNumNonAMContainerPreempted) + ._("Aggregate Resource Allocation:", + String.format("%d MB-seconds, %d vcore-seconds", + appMetrics == null ? "N/A" : appMetrics.getMemorySeconds(), + appMetrics == null ? "N/A" : appMetrics.getVcoreSeconds())); + pdiv._(); + } +}