From c184466939444fb584af410a32aceeda3d47ece6 Mon Sep 17 00:00:00 2001 From: Xuan Date: Fri, 24 Apr 2015 09:41:59 -0700 Subject: [PATCH] YARN-3511. Add errors and warnings page to ATS. Contributed by Varun Vasudev (cherry picked from commit eee9facbbae52cb62dfca01b8bbe676b8e289863) --- hadoop-yarn-project/CHANGES.txt | 2 + .../webapp/AHSController.java | 4 ++ .../webapp/AHSErrorsAndWarningsPage.java | 57 +++++++++++++++++++ .../webapp/AHSWebApp.java | 1 + .../webapp/NavBlock.java | 30 +++++++++- .../server/webapp/ErrorsAndWarningsBlock.java | 23 +++++++- .../resourcemanager/webapp/NavBlock.java | 2 +- 7 files changed, 114 insertions(+), 5 deletions(-) create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/AHSErrorsAndWarningsPage.java diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 1de4e2df390..aca570ec88a 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -118,6 +118,8 @@ Release 2.8.0 - UNRELEASED YARN-3503. Expose disk utilization percentage and bad local and log dir counts in NM metrics. (Varun Vasudev via jianhe) + YARN-3511. Add errors and warnings page to ATS. (Varun Vasudev via xgong) + OPTIMIZATIONS YARN-3339. TestDockerContainerExecutor should pull a single image and not diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/AHSController.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/AHSController.java index 4e00bc8fe0b..4037f51756a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/AHSController.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/AHSController.java @@ -52,4 +52,8 @@ public class AHSController extends Controller { public void logs() { render(AHSLogsPage.class); } + + public void errorsAndWarnings() { + render(AHSErrorsAndWarningsPage.class); + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/AHSErrorsAndWarningsPage.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/AHSErrorsAndWarningsPage.java new file mode 100644 index 00000000000..3798ee5b3b2 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/AHSErrorsAndWarningsPage.java @@ -0,0 +1,57 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.applicationhistoryservice.webapp; + +import org.apache.hadoop.yarn.server.webapp.ErrorsAndWarningsBlock; +import org.apache.hadoop.yarn.webapp.SubView; + +import static org.apache.hadoop.yarn.webapp.view.JQueryUI.*; + +/** + * Class to display the Errors and Warnings page for the AHS. + */ +public class AHSErrorsAndWarningsPage extends AHSView { + + @Override + protected Class content() { + return ErrorsAndWarningsBlock.class; + } + + @Override + protected void preHead(Page.HTML<_> html) { + commonPreHead(html); + String title = "Errors and Warnings in the Application History Server"; + setTitle(title); + String tableId = "messages"; + set(DATATABLES_ID, tableId); + set(initID(DATATABLES, tableId), tablesInit()); + setTableStyles(html, tableId, ".message {width:50em}", + ".count {width:8em}", ".lasttime {width:16em}"); + } + + private String tablesInit() { + StringBuilder b = tableInit().append(", aoColumnDefs: ["); + b.append("{'sType': 'string', 'aTargets': [ 0 ]}"); + b.append(", {'sType': 'string', 'bSearchable': true, 'aTargets': [ 1 ]}"); + b.append(", {'sType': 'numeric', 'bSearchable': false, 'aTargets': [ 2 ]}"); + b.append(", {'sType': 'date', 'aTargets': [ 3 ] }]"); + b.append(", aaSorting: [[3, 'desc']]}"); + return b.toString(); + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/AHSWebApp.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/AHSWebApp.java index 4b579c60e56..80c45502cbd 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/AHSWebApp.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/AHSWebApp.java @@ -64,5 +64,6 @@ public class AHSWebApp extends WebApp implements YarnWebParams { route( pajoin("/logs", NM_NODENAME, CONTAINER_ID, ENTITY_STRING, APP_OWNER, CONTAINER_LOG_TYPE), AHSController.class, "logs"); + route("/errors-and-warnings", AHSController.class, "errorsAndWarnings"); } } \ No newline at end of file diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/NavBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/NavBlock.java index e84ddec2207..498503f1aac 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/NavBlock.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/NavBlock.java @@ -18,14 +18,28 @@ package org.apache.hadoop.yarn.server.applicationhistoryservice.webapp; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.commons.logging.impl.Log4JLogger; import org.apache.hadoop.yarn.api.records.YarnApplicationState; +import org.apache.hadoop.yarn.util.Log4jWarningErrorMetricsAppender; +import org.apache.hadoop.yarn.webapp.hamlet.Hamlet; import org.apache.hadoop.yarn.webapp.view.HtmlBlock; public class NavBlock extends HtmlBlock { @Override public void render(Block html) { - html. + boolean addErrorsAndWarningsLink = false; + Log log = LogFactory.getLog(NavBlock.class); + if (log instanceof Log4JLogger) { + Log4jWarningErrorMetricsAppender appender = + Log4jWarningErrorMetricsAppender.findAppender(); + if (appender != null) { + addErrorsAndWarningsLink = true; + } + } + Hamlet.DIV nav = html. div("#nav"). h3("Application History"). ul(). @@ -45,7 +59,17 @@ public class NavBlock extends HtmlBlock { _(). _(). _(). - _(). - _(); + _(); + + Hamlet.UL> tools = nav.h3("Tools").ul(); + tools.li().a("/conf", "Configuration")._() + .li().a("/logs", "Local logs")._() + .li().a("/stacks", "Server stacks")._() + .li().a("/jmx?qry=Hadoop:*", "Server metrics")._(); + + if (addErrorsAndWarningsLink) { + tools.li().a(url("errors-and-warnings"), "Errors/Warnings")._(); + } + tools._()._(); } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/ErrorsAndWarningsBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/ErrorsAndWarningsBlock.java index 773a9e7f4a8..56db66c3605 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/ErrorsAndWarningsBlock.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/ErrorsAndWarningsBlock.java @@ -22,7 +22,10 @@ import com.google.inject.Inject; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.commons.logging.impl.Log4JLogger; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.util.Time; +import org.apache.hadoop.yarn.security.AdminACLsManager; import org.apache.hadoop.yarn.util.Log4jWarningErrorMetricsAppender; import org.apache.hadoop.yarn.util.Times; import org.apache.hadoop.yarn.webapp.hamlet.Hamlet; @@ -36,9 +39,10 @@ import java.util.Map; public class ErrorsAndWarningsBlock extends HtmlBlock { long cutoffPeriodSeconds; + final private AdminACLsManager adminAclsManager; @Inject - ErrorsAndWarningsBlock(ViewContext ctx) { + ErrorsAndWarningsBlock(ViewContext ctx, Configuration conf) { super(ctx); // default is to show all errors and warnings cutoffPeriodSeconds = Time.now() / 1000; @@ -51,12 +55,29 @@ public class ErrorsAndWarningsBlock extends HtmlBlock { } catch (NumberFormatException ne) { cutoffPeriodSeconds = Time.now() / 1000; } + adminAclsManager = new AdminACLsManager(conf); } @Override protected void render(Block html) { Log log = LogFactory.getLog(ErrorsAndWarningsBlock.class); + boolean isAdmin = false; + UserGroupInformation callerUGI = this.getCallerUGI(); + + if (adminAclsManager.areACLsEnabled()) { + if (callerUGI != null && adminAclsManager.isAdmin(callerUGI)) { + isAdmin = true; + } + } else { + isAdmin = true; + } + + if (!isAdmin) { + html.div().p()._("This page is for admins only.")._()._(); + return; + } + if (log instanceof Log4JLogger) { html._(ErrorMetrics.class); html._(WarningMetrics.class); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/NavBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/NavBlock.java index 7a8ab8d7991..ca55175c244 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/NavBlock.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/NavBlock.java @@ -33,7 +33,7 @@ public class NavBlock extends HtmlBlock { @Override public void render(Block html) { boolean addErrorsAndWarningsLink = false; - Log log = LogFactory.getLog(RMErrorsAndWarningsPage.class); + Log log = LogFactory.getLog(NavBlock.class); if (log instanceof Log4JLogger) { Log4jWarningErrorMetricsAppender appender = Log4jWarningErrorMetricsAppender.findAppender();