From a6dfb7b0453268d332e45f33ffb26ab7a507dee9 Mon Sep 17 00:00:00 2001 From: Vinod Kumar Vavilapalli Date: Tue, 28 Jan 2014 20:03:04 +0000 Subject: [PATCH] YARN-1413. Implemented serving of aggregated-logs in the ApplicationHistory server. Contributed by Mayank Bansal. svn merge --ignore-ancestry -c 1556752 ../YARN-321 git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/branch-2@1562206 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-yarn-project/CHANGES.txt | 3 + .../apache/hadoop/yarn/util/StringHelper.java | 4 ++ .../hadoop/yarn/webapp/util/WebAppUtils.java | 11 ++++ .../webapp/AHSController.java | 6 ++ .../webapp/AHSLogsPage.java | 55 +++++++++++++++++++ .../webapp/AHSWebApp.java | 2 + .../yarn/server/webapp/AppAttemptBlock.java | 7 ++- .../hadoop/yarn/server/webapp/AppBlock.java | 4 +- .../yarn/server/webapp/ContainerBlock.java | 8 ++- .../rmcontainer/RMContainerImpl.java | 12 ++-- 10 files changed, 98 insertions(+), 14 deletions(-) create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/AHSLogsPage.java diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index b70ec3d8ac2..ac11f9c9b73 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -516,6 +516,9 @@ Branch YARN-321: Generic ApplicationHistoryService YARN-1534. Fixed failure of test TestAHSWebApp. (Shinichi Yamashita via vinodkv) + YARN-1413. Implemented serving of aggregated-logs in the ApplicationHistory + server. (Mayank Bansal via vinodkv) + Release 2.2.0 - 2013-10-13 INCOMPATIBLE CHANGES diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/StringHelper.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/StringHelper.java index fd285b4141d..a48b3c0965b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/StringHelper.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/StringHelper.java @@ -178,4 +178,8 @@ public final class StringHelper { public static String percent(double value) { return String.format("%.2f", value * 100); } + + public static String getPartUrl(String url, String part) { + return url.substring(url.indexOf(part)); + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/util/WebAppUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/util/WebAppUtils.java index 5d17659a739..4a288c44af9 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/util/WebAppUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/util/WebAppUtils.java @@ -17,6 +17,8 @@ */ package org.apache.hadoop.yarn.webapp.util; +import static org.apache.hadoop.yarn.util.StringHelper.join; + import java.net.InetAddress; import java.net.InetSocketAddress; import java.net.UnknownHostException; @@ -27,7 +29,9 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.http.HttpConfig; import org.apache.hadoop.http.HttpConfig.Policy; import org.apache.hadoop.net.NetUtils; +import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.util.ConverterUtils; import com.google.common.base.Joiner; @@ -170,4 +174,11 @@ public class WebAppUtils { return schemePrefix + url; } } + + public static String getLogUrl(String nodeHttpAddress, String allocatedNode, + ContainerId containerId, String user) { + return join(HttpConfig.getSchemePrefix(), nodeHttpAddress, "/logs", "/", + allocatedNode, "/", ConverterUtils.toString(containerId), "/", + ConverterUtils.toString(containerId), "/", user); + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/AHSController.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/AHSController.java index b7badfe24db..7bbb39176c5 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/AHSController.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/AHSController.java @@ -47,4 +47,10 @@ public class AHSController extends Controller { render(ContainerPage.class); } + /** + * Render the logs page. + */ + public void logs() { + render(AHSLogsPage.class); + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/AHSLogsPage.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/AHSLogsPage.java new file mode 100644 index 00000000000..8821bc02dcb --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/AHSLogsPage.java @@ -0,0 +1,55 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.server.applicationhistoryservice.webapp; + +import static org.apache.hadoop.yarn.webapp.YarnWebParams.CONTAINER_ID; +import static org.apache.hadoop.yarn.webapp.YarnWebParams.ENTITY_STRING; + +import org.apache.hadoop.yarn.webapp.SubView; +import org.apache.hadoop.yarn.webapp.log.AggregatedLogsBlock; + +public class AHSLogsPage extends AHSView { + /* + * (non-Javadoc) + * + * @see + * org.apache.hadoop.yarn.server.applicationhistoryservice.webapp.AHSView# + * preHead(org.apache.hadoop .yarn.webapp.hamlet.Hamlet.HTML) + */ + @Override + protected void preHead(Page.HTML<_> html) { + String logEntity = $(ENTITY_STRING); + if (logEntity == null || logEntity.isEmpty()) { + logEntity = $(CONTAINER_ID); + } + if (logEntity == null || logEntity.isEmpty()) { + logEntity = "UNKNOWN"; + } + commonPreHead(html); + } + + /** + * The content of this page is the AggregatedLogsBlock + * + * @return AggregatedLogsBlock.class + */ + @Override + protected Class content() { + return AggregatedLogsBlock.class; + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/AHSWebApp.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/AHSWebApp.java index 61b7287bfb6..1ebbc5e95c3 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/AHSWebApp.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/AHSWebApp.java @@ -45,5 +45,7 @@ public class AHSWebApp extends WebApp implements YarnWebParams { route(pajoin("/appattempt", APPLICATION_ATTEMPT_ID), AHSController.class, "appattempt"); route(pajoin("/container", CONTAINER_ID), AHSController.class, "container"); + route(pajoin("/logs", NM_NODENAME, CONTAINER_ID, ENTITY_STRING, APP_OWNER, + CONTAINER_LOG_TYPE), AHSController.class, "logs"); } } \ No newline at end of file diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppAttemptBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppAttemptBlock.java index 1eac7b90a61..bdef62c09b6 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppAttemptBlock.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppAttemptBlock.java @@ -18,6 +18,7 @@ package org.apache.hadoop.yarn.server.webapp; import static org.apache.hadoop.yarn.util.StringHelper.join; +import static org.apache.hadoop.yarn.util.StringHelper.getPartUrl; import static org.apache.hadoop.yarn.webapp.YarnWebParams.APPLICATION_ATTEMPT_ID; import java.io.IOException; @@ -126,6 +127,8 @@ public class AppAttemptBlock extends HtmlBlock { StringBuilder containersTableData = new StringBuilder("[\n"); for (ContainerReport containerReport : containers) { + String logURL = containerReport.getLogUrl(); + logURL = getPartUrl(logURL, "log"); ContainerInfo container = new ContainerInfo(containerReport); // ConatinerID numerical value parsed by parseHadoopID in yarn.dt.plugins.js containersTableData @@ -141,9 +144,9 @@ public class AppAttemptBlock extends HtmlBlock { .append("\",\"") .append(container.getContainerExitStatus()) .append("\",\"") - .append(container.getLogUrl() == null ? "N/A" : "Logs") + .append(logURL == null ? "N/A" : "Logs") .append("\"],\n"); } if (containersTableData.charAt(containersTableData.length() - 2) == ',') { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppBlock.java index d2509d6832b..3d7877df22a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppBlock.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppBlock.java @@ -19,6 +19,7 @@ package org.apache.hadoop.yarn.server.webapp; import static org.apache.hadoop.yarn.util.StringHelper.join; +import static org.apache.hadoop.yarn.util.StringHelper.getPartUrl; import static org.apache.hadoop.yarn.webapp.YarnWebParams.APPLICATION_ID; import java.io.IOException; @@ -147,7 +148,8 @@ public class AppBlock extends HtmlBlock { if (containerReport != null) { ContainerInfo container = new ContainerInfo(containerReport); startTime = container.getStartedTime(); - logsLink = container.getLogUrl(); + logsLink = containerReport.getLogUrl(); + logsLink = getPartUrl(logsLink,"log"); } String nodeLink = null; if (appAttempt.getHost() != null && appAttempt.getRpcPort() >= 0 diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/ContainerBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/ContainerBlock.java index eeef4187187..c7abe1cc421 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/ContainerBlock.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/ContainerBlock.java @@ -18,6 +18,7 @@ package org.apache.hadoop.yarn.server.webapp; import static org.apache.hadoop.yarn.util.StringHelper.join; +import static org.apache.hadoop.yarn.util.StringHelper.getPartUrl; import static org.apache.hadoop.yarn.webapp.YarnWebParams.CONTAINER_ID; import java.io.IOException; @@ -76,8 +77,10 @@ public class ContainerBlock extends HtmlBlock { puts("Container not found: " + containerid); return; } + ContainerInfo container = new ContainerInfo(containerReport); - + String logURL = containerReport.getLogUrl(); + logURL = getPartUrl(logURL,"log"); setTitle(join("Container ", containerid)); info("Container Overview"). @@ -91,8 +94,7 @@ public class ContainerBlock extends HtmlBlock { container.getFinishedTime()))). _("Resource:", container.getAllocatedMB() + " Memory, " + container.getAllocatedVCores() + " VCores"). - _("Logs:", container.getLogUrl() == null ? "#" : root_url(container.getLogUrl()), - container.getLogUrl() == null ? "N/A" : container.getLogUrl()). + _("Logs:", logURL == null ? "#" : url(logURL), "Logs"). _("Diagnostics:", container.getDiagnosticsInfo()); html._(InfoBlock.class); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java index adffb997e03..057c9ace7e3 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java @@ -18,8 +18,6 @@ package org.apache.hadoop.yarn.server.resourcemanager.rmcontainer; -import static org.apache.hadoop.yarn.util.StringHelper.join; - import java.util.EnumSet; import java.util.concurrent.locks.ReentrantReadWriteLock; import java.util.concurrent.locks.ReentrantReadWriteLock.ReadLock; @@ -27,7 +25,6 @@ import java.util.concurrent.locks.ReentrantReadWriteLock.WriteLock; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.http.HttpConfig; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.Container; import org.apache.hadoop.yarn.api.records.ContainerId; @@ -46,7 +43,7 @@ import org.apache.hadoop.yarn.state.InvalidStateTransitonException; import org.apache.hadoop.yarn.state.SingleArcTransition; import org.apache.hadoop.yarn.state.StateMachine; import org.apache.hadoop.yarn.state.StateMachineFactory; -import org.apache.hadoop.yarn.util.ConverterUtils; +import org.apache.hadoop.yarn.webapp.util.WebAppUtils; @SuppressWarnings({"unchecked", "rawtypes"}) public class RMContainerImpl implements RMContainer { @@ -366,10 +363,9 @@ public class RMContainerImpl implements RMContainer { public void transition(RMContainerImpl container, RMContainerEvent event) { // The logs of running containers should be found on NM webUI // The logs should be accessible after the container is launched - container.logURL = join(HttpConfig.getSchemePrefix(), - container.container.getNodeHttpAddress(), "/node", "/containerlogs/", - ConverterUtils.toString(container.containerId), "/", - container.user); + container.logURL = WebAppUtils.getLogUrl(container.container + .getNodeHttpAddress(), container.getAllocatedNode().toString(), + container.containerId, container.user); // Unregister from containerAllocationExpirer. container.containerAllocationExpirer.unregister(container .getContainerId());