From c27601fefebd0af887a12d684bfc6f90d9fc0321 Mon Sep 17 00:00:00 2001 From: Vinod Kumar Vavilapalli Date: Thu, 3 Nov 2011 08:02:19 +0000 Subject: [PATCH] MAPREDUCE-3297. Moved log related components into yarn-common so that HistoryServer and clients can use them without depending on the yarn-server-nodemanager module. Contributed by Siddharth Seth. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1196986 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-mapreduce-project/CHANGES.txt | 16 ++ .../hadoop-mapreduce-client-core/pom.xml | 4 - .../apache/hadoop/mapreduce/tools/CLI.java | 2 +- .../mapreduce/v2/hs/webapp/HsController.java | 2 +- .../mapreduce/v2/hs/webapp/HsLogsPage.java | 6 +- .../mapreduce/v2/hs/webapp/HsWebApp.java | 8 +- .../mapreduce/v2/hs/webapp/TestHSWebApp.java | 10 +- .../dev-support/findbugs-exclude.xml | 2 +- .../logaggregation/AggregatedLogFormat.java | 2 +- .../ContainerLogsRetentionPolicy.java | 2 +- .../logaggregation/LogAggregationUtils.java | 107 ++++++++++ .../yarn}/logaggregation/LogDumper.java | 14 +- .../security/ApplicationACLsManager.java | 0 .../hadoop/yarn/webapp/YarnWebParams.java} | 14 +- .../yarn/webapp/log/AggregatedLogsBlock.java | 183 ++++++++++++++++++ .../webapp/log}/AggregatedLogsNavBlock.java | 4 +- .../yarn/webapp/log/AggregatedLogsPage.java | 45 +++++ .../application/ApplicationImpl.java | 2 +- .../logaggregation/AppLogAggregatorImpl.java | 7 +- .../logaggregation/LogAggregationService.java | 88 +-------- .../event/LogHandlerAppStartedEvent.java | 2 +- .../webapp/AggregatedLogsBlock.java | 183 ------------------ .../webapp/AggregatedLogsPage.java | 44 ----- .../webapp/AllApplicationsPage.java | 3 +- .../nodemanager/webapp/AllContainersPage.java | 3 +- .../nodemanager/webapp/ApplicationPage.java | 5 +- .../nodemanager/webapp/ContainerLogsPage.java | 5 +- .../nodemanager/webapp/ContainerPage.java | 5 +- .../nodemanager/webapp/NMController.java | 3 +- .../server/nodemanager/webapp/NavBlock.java | 3 +- .../server/nodemanager/webapp/WebServer.java | 3 +- .../TestLogAggregationService.java | 6 +- .../TestNonAggregatingLogHandler.java | 2 +- 33 files changed, 421 insertions(+), 364 deletions(-) rename hadoop-mapreduce-project/hadoop-yarn/{hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager => hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn}/logaggregation/AggregatedLogFormat.java (99%) rename hadoop-mapreduce-project/hadoop-yarn/{hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager => hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn}/logaggregation/ContainerLogsRetentionPolicy.java (91%) create mode 100644 hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/LogAggregationUtils.java rename hadoop-mapreduce-project/hadoop-yarn/{hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager => hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn}/logaggregation/LogDumper.java (93%) rename hadoop-mapreduce-project/hadoop-yarn/{hadoop-yarn-server/hadoop-yarn-server-common => hadoop-yarn-common}/src/main/java/org/apache/hadoop/yarn/server/security/ApplicationACLsManager.java (100%) rename hadoop-mapreduce-project/hadoop-yarn/{hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMWebParams.java => hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/YarnWebParams.java} (73%) create mode 100644 hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/log/AggregatedLogsBlock.java rename hadoop-mapreduce-project/hadoop-yarn/{hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp => hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/log}/AggregatedLogsNavBlock.java (88%) create mode 100644 hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/log/AggregatedLogsPage.java diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index 0a80cc36ab6..5cb5fec264f 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -50,6 +50,22 @@ Trunk (unreleased changes) MAPREDUCE-3014. Rename and invert logic of '-cbuild' profile to 'native' and off by default. (tucu) +Release 0.23.1 - Unreleased + + INCOMPATIBLE CHANGES + + NEW FEATURES + + IMPROVEMENTS + + MAPREDUCE-3297. Moved log related components into yarn-common so that + HistoryServer and clients can use them without depending on the + yarn-server-nodemanager module. (Siddharth Seth via vinodkv) + + OPTIMIZATIONS + + BUG FIXES + Release 0.23.0 - 2011-11-01 INCOMPATIBLE CHANGES diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/pom.xml b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/pom.xml index 138e4332aae..2f083f5275a 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/pom.xml +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/pom.xml @@ -34,10 +34,6 @@ org.apache.hadoop hadoop-yarn-common - - org.apache.hadoop - hadoop-yarn-server-nodemanager - org.apache.hadoop hadoop-hdfs diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/tools/CLI.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/tools/CLI.java index 7b255289b02..27ccb8a27e8 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/tools/CLI.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/tools/CLI.java @@ -46,7 +46,7 @@ import org.apache.hadoop.mapreduce.v2.LogParams; import org.apache.hadoop.security.AccessControlException; import org.apache.hadoop.util.Tool; import org.apache.hadoop.util.ToolRunner; -import org.apache.hadoop.yarn.server.nodemanager.containermanager.logaggregation.LogDumper; +import org.apache.hadoop.yarn.logaggregation.LogDumper; /** * Interprets the map reduce cli options diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/webapp/HsController.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/webapp/HsController.java index d78f026d2f6..b7bdfa4e8f4 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/webapp/HsController.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/webapp/HsController.java @@ -23,8 +23,8 @@ import java.io.IOException; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.mapreduce.v2.app.webapp.App; import org.apache.hadoop.mapreduce.v2.app.webapp.AppController; -import org.apache.hadoop.yarn.server.nodemanager.webapp.AggregatedLogsPage; import org.apache.hadoop.yarn.webapp.View; +import org.apache.hadoop.yarn.webapp.log.AggregatedLogsPage; import com.google.inject.Inject; diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/webapp/HsLogsPage.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/webapp/HsLogsPage.java index 3d6caa90e2b..909a316dbfe 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/webapp/HsLogsPage.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/webapp/HsLogsPage.java @@ -17,11 +17,11 @@ */ package org.apache.hadoop.mapreduce.v2.hs.webapp; -import static org.apache.hadoop.yarn.server.nodemanager.webapp.NMWebParams.CONTAINER_ID; -import static org.apache.hadoop.yarn.server.nodemanager.webapp.NMWebParams.ENTITY_STRING; +import static org.apache.hadoop.yarn.webapp.YarnWebParams.CONTAINER_ID; +import static org.apache.hadoop.yarn.webapp.YarnWebParams.ENTITY_STRING; -import org.apache.hadoop.yarn.server.nodemanager.webapp.AggregatedLogsBlock; import org.apache.hadoop.yarn.webapp.SubView; +import org.apache.hadoop.yarn.webapp.log.AggregatedLogsBlock; public class HsLogsPage extends HsView { diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/webapp/HsWebApp.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/webapp/HsWebApp.java index 01eb3f06ae0..a9f08c5dc0b 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/webapp/HsWebApp.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/webapp/HsWebApp.java @@ -18,11 +18,11 @@ package org.apache.hadoop.mapreduce.v2.hs.webapp; -import static org.apache.hadoop.yarn.server.nodemanager.webapp.NMWebParams.CONTAINER_ID; -import static org.apache.hadoop.yarn.server.nodemanager.webapp.NMWebParams.NM_NODENAME; -import static org.apache.hadoop.yarn.server.nodemanager.webapp.NMWebParams.ENTITY_STRING; -import static org.apache.hadoop.yarn.server.nodemanager.webapp.NMWebParams.APP_OWNER; import static org.apache.hadoop.yarn.util.StringHelper.pajoin; +import static org.apache.hadoop.yarn.webapp.YarnWebParams.APP_OWNER; +import static org.apache.hadoop.yarn.webapp.YarnWebParams.CONTAINER_ID; +import static org.apache.hadoop.yarn.webapp.YarnWebParams.ENTITY_STRING; +import static org.apache.hadoop.yarn.webapp.YarnWebParams.NM_NODENAME; import org.apache.hadoop.mapreduce.v2.app.AppContext; import org.apache.hadoop.mapreduce.v2.app.webapp.AMParams; diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestHSWebApp.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestHSWebApp.java index 063cb6bdda9..2e6bb9cf36e 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestHSWebApp.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestHSWebApp.java @@ -22,10 +22,10 @@ import static org.apache.hadoop.mapreduce.v2.app.webapp.AMParams.APP_ID; import static org.apache.hadoop.mapreduce.v2.app.webapp.AMParams.ATTEMPT_STATE; import static org.apache.hadoop.mapreduce.v2.app.webapp.AMParams.JOB_ID; import static org.apache.hadoop.mapreduce.v2.app.webapp.AMParams.TASK_TYPE; -import static org.apache.hadoop.yarn.server.nodemanager.webapp.NMWebParams.CONTAINER_ID; -import static org.apache.hadoop.yarn.server.nodemanager.webapp.NMWebParams.NM_NODENAME; -import static org.apache.hadoop.yarn.server.nodemanager.webapp.NMWebParams.ENTITY_STRING; -import static org.apache.hadoop.yarn.server.nodemanager.webapp.NMWebParams.APP_OWNER; +import static org.apache.hadoop.yarn.webapp.YarnWebParams.APP_OWNER; +import static org.apache.hadoop.yarn.webapp.YarnWebParams.CONTAINER_ID; +import static org.apache.hadoop.yarn.webapp.YarnWebParams.ENTITY_STRING; +import static org.apache.hadoop.yarn.webapp.YarnWebParams.NM_NODENAME; import static org.junit.Assert.assertEquals; import java.io.IOException; @@ -44,8 +44,8 @@ import org.apache.hadoop.yarn.Clock; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.event.EventHandler; -import org.apache.hadoop.yarn.server.nodemanager.webapp.AggregatedLogsPage; import org.apache.hadoop.yarn.util.BuilderUtils; +import org.apache.hadoop.yarn.webapp.log.AggregatedLogsPage; import org.apache.hadoop.yarn.webapp.test.WebAppTests; import org.junit.Test; diff --git a/hadoop-mapreduce-project/hadoop-yarn/dev-support/findbugs-exclude.xml b/hadoop-mapreduce-project/hadoop-yarn/dev-support/findbugs-exclude.xml index b545bf72d08..4b7d45f66a4 100644 --- a/hadoop-mapreduce-project/hadoop-yarn/dev-support/findbugs-exclude.xml +++ b/hadoop-mapreduce-project/hadoop-yarn/dev-support/findbugs-exclude.xml @@ -208,7 +208,7 @@ - + diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/AggregatedLogFormat.java b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/AggregatedLogFormat.java similarity index 99% rename from hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/AggregatedLogFormat.java rename to hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/AggregatedLogFormat.java index 1693b8337ee..b49ecc784cc 100644 --- a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/AggregatedLogFormat.java +++ b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/AggregatedLogFormat.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package org.apache.hadoop.yarn.server.nodemanager.containermanager.logaggregation; +package org.apache.hadoop.yarn.logaggregation; import java.io.DataInput; import java.io.DataInputStream; diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/ContainerLogsRetentionPolicy.java b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/ContainerLogsRetentionPolicy.java similarity index 91% rename from hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/ContainerLogsRetentionPolicy.java rename to hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/ContainerLogsRetentionPolicy.java index 82701c3e4cd..b2ccd2c0730 100644 --- a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/ContainerLogsRetentionPolicy.java +++ b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/ContainerLogsRetentionPolicy.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package org.apache.hadoop.yarn.server.nodemanager.containermanager.logaggregation; +package org.apache.hadoop.yarn.logaggregation; public enum ContainerLogsRetentionPolicy { APPLICATION_MASTER_ONLY, AM_AND_FAILED_CONTAINERS_ONLY, ALL_CONTAINERS diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/LogAggregationUtils.java b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/LogAggregationUtils.java new file mode 100644 index 00000000000..e8a2bd1a3f8 --- /dev/null +++ b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/LogAggregationUtils.java @@ -0,0 +1,107 @@ +/** +* Licensed to the Apache Software Foundation (ASF) under one +* or more contributor license agreements. See the NOTICE file +* distributed with this work for additional information +* regarding copyright ownership. The ASF licenses this file +* to you under the Apache License, Version 2.0 (the +* "License"); you may not use this file except in compliance +* with the License. You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ + +package org.apache.hadoop.yarn.logaggregation; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.hadoop.yarn.api.records.NodeId; +import org.apache.hadoop.yarn.conf.YarnConfiguration; + +public class LogAggregationUtils { + + /** + * Constructs the full filename for an application's log file per node. + * @param remoteRootLogDir + * @param appId + * @param user + * @param nodeId + * @param suffix + * @return the remote log file. + */ + public static Path getRemoteNodeLogFileForApp(Path remoteRootLogDir, + ApplicationId appId, String user, NodeId nodeId, String suffix) { + return new Path(getRemoteAppLogDir(remoteRootLogDir, appId, user, suffix), + getNodeString(nodeId)); + } + + /** + * Gets the remote app log dir. + * @param remoteRootLogDir + * @param appId + * @param user + * @param suffix + * @return the remote application specific log dir. + */ + public static Path getRemoteAppLogDir(Path remoteRootLogDir, + ApplicationId appId, String user, String suffix) { + return new Path(getRemoteLogSuffixedDir(remoteRootLogDir, user, suffix), + appId.toString()); + } + + /** + * Gets the remote suffixed log dir for the user. + * @param remoteRootLogDir + * @param user + * @param suffix + * @return the remote suffixed log dir. + */ + public static Path getRemoteLogSuffixedDir(Path remoteRootLogDir, + String user, String suffix) { + if (suffix == null || suffix.isEmpty()) { + return getRemoteLogUserDir(remoteRootLogDir, user); + } + // TODO Maybe support suffix to be more than a single file. + return new Path(getRemoteLogUserDir(remoteRootLogDir, user), suffix); + } + + // TODO Add a utility method to list available log files. Ignore the + // temporary ones. + + /** + * Gets the remote log user dir. + * @param remoteRootLogDir + * @param user + * @return the remote per user log dir. + */ + public static Path getRemoteLogUserDir(Path remoteRootLogDir, String user) { + return new Path(remoteRootLogDir, user); + } + + /** + * Returns the suffix component of the log dir. + * @param conf + * @return the suffix which will be appended to the user log dir. + */ + public static String getRemoteNodeLogDirSuffix(Configuration conf) { + return conf.get(YarnConfiguration.NM_REMOTE_APP_LOG_DIR_SUFFIX, + YarnConfiguration.DEFAULT_NM_REMOTE_APP_LOG_DIR_SUFFIX); + } + + + /** + * Converts a nodeId to a form used in the app log file name. + * @param nodeId + * @return the node string to be used to construct the file name. + */ + private static String getNodeString(NodeId nodeId) { + return nodeId.toString().replace(":", "_"); + } + +} diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/LogDumper.java b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/LogDumper.java similarity index 93% rename from hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/LogDumper.java rename to hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/LogDumper.java index f5e5640c52d..20206d6f941 100644 --- a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/LogDumper.java +++ b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/LogDumper.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package org.apache.hadoop.yarn.server.nodemanager.containermanager.logaggregation; +package org.apache.hadoop.yarn.logaggregation; import java.io.DataInputStream; import java.io.DataOutputStream; @@ -41,8 +41,8 @@ import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.factories.RecordFactory; import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider; -import org.apache.hadoop.yarn.server.nodemanager.containermanager.logaggregation.AggregatedLogFormat.LogKey; -import org.apache.hadoop.yarn.server.nodemanager.containermanager.logaggregation.AggregatedLogFormat.LogReader; +import org.apache.hadoop.yarn.logaggregation.AggregatedLogFormat.LogKey; +import org.apache.hadoop.yarn.logaggregation.AggregatedLogFormat.LogReader; import org.apache.hadoop.yarn.util.ConverterUtils; public class LogDumper extends Configured implements Tool { @@ -117,7 +117,7 @@ public class LogDumper extends Configured implements Tool { YarnConfiguration.DEFAULT_NM_REMOTE_APP_LOG_DIR)); AggregatedLogFormat.LogReader reader = new AggregatedLogFormat.LogReader(getConf(), - LogAggregationService.getRemoteNodeLogFileForApp( + LogAggregationUtils.getRemoteNodeLogFileForApp( remoteRootLogDir, appId, appOwner, @@ -135,10 +135,10 @@ public class LogDumper extends Configured implements Tool { Path remoteRootLogDir = new Path(getConf().get(YarnConfiguration.NM_REMOTE_APP_LOG_DIR, YarnConfiguration.DEFAULT_NM_REMOTE_APP_LOG_DIR)); - String suffix = LogAggregationService.getRemoteNodeLogDirSuffix(getConf()); + String suffix = LogAggregationUtils.getRemoteNodeLogDirSuffix(getConf()); AggregatedLogFormat.LogReader reader = new AggregatedLogFormat.LogReader(getConf(), - LogAggregationService.getRemoteNodeLogFileForApp(remoteRootLogDir, + LogAggregationUtils.getRemoteNodeLogFileForApp(remoteRootLogDir, ConverterUtils.toApplicationId(appId), jobOwner, ConverterUtils.toNodeId(nodeId), suffix)); DataOutputStream out = new DataOutputStream(System.out); @@ -185,7 +185,7 @@ public class LogDumper extends Configured implements Tool { YarnConfiguration.DEFAULT_NM_REMOTE_APP_LOG_DIR_SUFFIX); //TODO Change this to get a list of files from the LAS. Path remoteAppLogDir = - LogAggregationService.getRemoteAppLogDir(remoteRootLogDir, appId, user, + LogAggregationUtils.getRemoteAppLogDir(remoteRootLogDir, appId, user, logDirSuffix); RemoteIterator nodeFiles = FileContext.getFileContext().listStatus(remoteAppLogDir); diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/security/ApplicationACLsManager.java b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/security/ApplicationACLsManager.java similarity index 100% rename from hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/security/ApplicationACLsManager.java rename to hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/security/ApplicationACLsManager.java diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMWebParams.java b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/YarnWebParams.java similarity index 73% rename from hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMWebParams.java rename to hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/YarnWebParams.java index eac16f4ccfa..fbc0887e129 100644 --- a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMWebParams.java +++ b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/YarnWebParams.java @@ -16,13 +16,13 @@ * limitations under the License. */ -package org.apache.hadoop.yarn.server.nodemanager.webapp; +package org.apache.hadoop.yarn.webapp; -public interface NMWebParams { +public interface YarnWebParams { String NM_NODENAME = "nm.id"; - String APPLICATION_ID = "nm.appId"; - String CONTAINER_ID = "nm.containerId"; - String CONTAINER_LOG_TYPE= "nm.containerLogType"; - String ENTITY_STRING = "nm.entityString"; - String APP_OWNER = "nm.appOwner"; + String APPLICATION_ID = "app.id"; + String CONTAINER_ID = "container.id"; + String CONTAINER_LOG_TYPE= "log.type"; + String ENTITY_STRING = "entity.string"; + String APP_OWNER = "app.owner"; } diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/log/AggregatedLogsBlock.java b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/log/AggregatedLogsBlock.java new file mode 100644 index 00000000000..5698e04dc85 --- /dev/null +++ b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/log/AggregatedLogsBlock.java @@ -0,0 +1,183 @@ +package org.apache.hadoop.yarn.webapp.log; + +import static org.apache.hadoop.yarn.webapp.YarnWebParams.APP_OWNER; +import static org.apache.hadoop.yarn.webapp.YarnWebParams.CONTAINER_ID; +import static org.apache.hadoop.yarn.webapp.YarnWebParams.ENTITY_STRING; +import static org.apache.hadoop.yarn.webapp.YarnWebParams.NM_NODENAME; + +import java.io.DataInputStream; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.util.Map; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.yarn.api.records.ApplicationAccessType; +import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.hadoop.yarn.api.records.ContainerId; +import org.apache.hadoop.yarn.api.records.NodeId; +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.logaggregation.AggregatedLogFormat; +import org.apache.hadoop.yarn.logaggregation.AggregatedLogFormat.LogKey; +import org.apache.hadoop.yarn.logaggregation.LogAggregationUtils; +import org.apache.hadoop.yarn.server.security.ApplicationACLsManager; +import org.apache.hadoop.yarn.util.ConverterUtils; +import org.apache.hadoop.yarn.webapp.view.HtmlBlock; + +import com.google.inject.Inject; + +public class AggregatedLogsBlock extends HtmlBlock { + + private final Configuration conf; + + @Inject + AggregatedLogsBlock(Configuration conf) { + this.conf = conf; + } + + @Override + protected void render(Block html) { + ContainerId containerId = verifyAndGetContainerId(html); + NodeId nodeId = verifyAndGetNodeId(html); + String appOwner = verifyAndGetAppOwner(html); + if (containerId == null || nodeId == null || appOwner == null + || appOwner.isEmpty()) { + return; + } + + ApplicationId applicationId = + containerId.getApplicationAttemptId().getApplicationId(); + String logEntity = $(ENTITY_STRING); + if (logEntity == null || logEntity.isEmpty()) { + logEntity = containerId.toString(); + } + + if (!conf.getBoolean(YarnConfiguration.NM_LOG_AGGREGATION_ENABLED, + YarnConfiguration.DEFAULT_NM_LOG_AGGREGATION_ENABLED)) { + html.h1() + ._("Aggregation is not enabled. Try the nodemanager at " + nodeId) + ._(); + return; + } + + Path remoteRootLogDir = + new Path(conf.get(YarnConfiguration.NM_REMOTE_APP_LOG_DIR, + YarnConfiguration.DEFAULT_NM_REMOTE_APP_LOG_DIR)); + AggregatedLogFormat.LogReader reader = null; + try { + reader = + new AggregatedLogFormat.LogReader(conf, + LogAggregationUtils.getRemoteNodeLogFileForApp( + remoteRootLogDir, applicationId, appOwner, nodeId, + LogAggregationUtils.getRemoteNodeLogDirSuffix(conf))); + } catch (FileNotFoundException e) { + // ACLs not available till the log file is opened. + html.h1() + ._("Logs not available for " + + logEntity + + ". Aggregation may not be complete, " + + "Check back later or try the nodemanager at " + + nodeId)._(); + return; + } catch (IOException e) { + html.h1()._("Error getting logs for " + logEntity)._(); + LOG.error("Error getting logs for " + logEntity, e); + return; + } + + String owner = null; + Map appAcls = null; + try { + owner = reader.getApplicationOwner(); + appAcls = reader.getApplicationAcls(); + } catch (IOException e) { + html.h1()._("Error getting logs for " + logEntity)._(); + LOG.error("Error getting logs for " + logEntity, e); + return; + } + ApplicationACLsManager aclsManager = new ApplicationACLsManager(conf); + aclsManager.addApplication(applicationId, appAcls); + + String remoteUser = request().getRemoteUser(); + UserGroupInformation callerUGI = null; + if (remoteUser != null) { + callerUGI = UserGroupInformation.createRemoteUser(remoteUser); + } + if (callerUGI != null + && !aclsManager.checkAccess(callerUGI, ApplicationAccessType.VIEW_APP, + owner, applicationId)) { + html.h1() + ._("User [" + remoteUser + + "] is not authorized to view the logs for " + logEntity)._(); + return; + } + + DataInputStream valueStream; + LogKey key = new LogKey(); + try { + valueStream = reader.next(key); + while (valueStream != null + && !key.toString().equals(containerId.toString())) { + valueStream = reader.next(key); + } + if (valueStream == null) { + html.h1()._( + "Logs not available for " + logEntity + + ". Could be caused by the rentention policy")._(); + return; + } + writer().write("
");
+      AggregatedLogFormat.LogReader.readAcontainerLogs(valueStream, writer());
+      writer().write("
"); + return; + } catch (IOException e) { + html.h1()._("Error getting logs for " + logEntity)._(); + LOG.error("Error getting logs for " + logEntity, e); + return; + } + } + + private ContainerId verifyAndGetContainerId(Block html) { + String containerIdStr = $(CONTAINER_ID); + if (containerIdStr == null || containerIdStr.isEmpty()) { + html.h1()._("Cannot get container logs without a ContainerId")._(); + return null; + } + ContainerId containerId = null; + try { + containerId = ConverterUtils.toContainerId(containerIdStr); + } catch (IllegalArgumentException e) { + html.h1() + ._("Cannot get container logs for invalid containerId: " + + containerIdStr)._(); + return null; + } + return containerId; + } + + private NodeId verifyAndGetNodeId(Block html) { + String nodeIdStr = $(NM_NODENAME); + if (nodeIdStr == null || nodeIdStr.isEmpty()) { + html.h1()._("Cannot get container logs without a NodeId")._(); + return null; + } + NodeId nodeId = null; + try { + nodeId = ConverterUtils.toNodeId(nodeIdStr); + } catch (IllegalArgumentException e) { + html.h1()._("Cannot get container logs. Invalid nodeId: " + nodeIdStr) + ._(); + return null; + } + return nodeId; + } + + private String verifyAndGetAppOwner(Block html) { + String appOwner = $(APP_OWNER); + if (appOwner == null || appOwner.isEmpty()) { + html.h1()._("Cannot get container logs without an app owner")._(); + } + return appOwner; + } +} \ No newline at end of file diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/AggregatedLogsNavBlock.java b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/log/AggregatedLogsNavBlock.java similarity index 88% rename from hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/AggregatedLogsNavBlock.java rename to hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/log/AggregatedLogsNavBlock.java index bde6db93c82..2b785525adb 100644 --- a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/AggregatedLogsNavBlock.java +++ b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/log/AggregatedLogsNavBlock.java @@ -16,11 +16,11 @@ * limitations under the License. */ -package org.apache.hadoop.yarn.server.nodemanager.webapp; +package org.apache.hadoop.yarn.webapp.log; import org.apache.hadoop.yarn.webapp.view.HtmlBlock; -public class AggregatedLogsNavBlock extends HtmlBlock implements NMWebParams { +public class AggregatedLogsNavBlock extends HtmlBlock { @Override protected void render(Block html) { diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/log/AggregatedLogsPage.java b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/log/AggregatedLogsPage.java new file mode 100644 index 00000000000..add4fc68538 --- /dev/null +++ b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/log/AggregatedLogsPage.java @@ -0,0 +1,45 @@ +package org.apache.hadoop.yarn.webapp.log; + +import static org.apache.hadoop.yarn.webapp.YarnWebParams.CONTAINER_ID; +import static org.apache.hadoop.yarn.webapp.YarnWebParams.ENTITY_STRING; +import static org.apache.hadoop.yarn.util.StringHelper.join; +import static org.apache.hadoop.yarn.webapp.view.JQueryUI.ACCORDION; +import static org.apache.hadoop.yarn.webapp.view.JQueryUI.ACCORDION_ID; +import static org.apache.hadoop.yarn.webapp.view.JQueryUI.THEMESWITCHER_ID; +import static org.apache.hadoop.yarn.webapp.view.JQueryUI.initID; + + +import org.apache.hadoop.yarn.webapp.SubView; +import org.apache.hadoop.yarn.webapp.view.TwoColumnLayout; + + +public class AggregatedLogsPage extends TwoColumnLayout { + + /* (non-Javadoc) + * @see org.apache.hadoop.yarn.server.nodemanager.webapp.NMView#preHead(org.apache.hadoop.yarn.webapp.hamlet.Hamlet.HTML) + */ + @Override + protected void preHead(Page.HTML<_> html) { + String logEntity = $(ENTITY_STRING); + if (logEntity == null || logEntity.isEmpty()) { + logEntity = $(CONTAINER_ID); + } + if (logEntity == null || logEntity.isEmpty()) { + logEntity = "UNKNOWN"; + } + set(TITLE, join("Logs for ", logEntity)); + set(ACCORDION_ID, "nav"); + set(initID(ACCORDION, "nav"), "{autoHeight:false, active:0}"); + set(THEMESWITCHER_ID, "themeswitcher"); + } + + @Override + protected Class content() { + return AggregatedLogsBlock.class; + } + + @Override + protected Class nav() { + return AggregatedLogsNavBlock.class; + } +} diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationImpl.java b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationImpl.java index 9cc5d6f786e..afdbe5b0823 100644 --- a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationImpl.java +++ b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationImpl.java @@ -32,6 +32,7 @@ import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ApplicationAccessType; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.event.Dispatcher; +import org.apache.hadoop.yarn.logaggregation.ContainerLogsRetentionPolicy; import org.apache.hadoop.yarn.server.nodemanager.Context; import org.apache.hadoop.yarn.server.nodemanager.containermanager.AuxServicesEvent; import org.apache.hadoop.yarn.server.nodemanager.containermanager.AuxServicesEventType; @@ -41,7 +42,6 @@ import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Cont import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.ResourceLocalizationService; import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.event.ApplicationLocalizationEvent; import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.event.LocalizationEventType; -import org.apache.hadoop.yarn.server.nodemanager.containermanager.logaggregation.ContainerLogsRetentionPolicy; import org.apache.hadoop.yarn.server.nodemanager.containermanager.loghandler.event.LogHandlerAppFinishedEvent; import org.apache.hadoop.yarn.server.nodemanager.containermanager.loghandler.event.LogHandlerAppStartedEvent; import org.apache.hadoop.yarn.server.security.ApplicationACLsManager; diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/AppLogAggregatorImpl.java b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/AppLogAggregatorImpl.java index 5db1b5de50e..c41162bbec0 100644 --- a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/AppLogAggregatorImpl.java +++ b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/AppLogAggregatorImpl.java @@ -35,12 +35,13 @@ import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ApplicationAccessType; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.event.Dispatcher; +import org.apache.hadoop.yarn.logaggregation.ContainerLogsRetentionPolicy; +import org.apache.hadoop.yarn.logaggregation.AggregatedLogFormat.LogKey; +import org.apache.hadoop.yarn.logaggregation.AggregatedLogFormat.LogValue; +import org.apache.hadoop.yarn.logaggregation.AggregatedLogFormat.LogWriter; import org.apache.hadoop.yarn.server.nodemanager.DeletionService; import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.ApplicationEvent; import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.ApplicationEventType; -import org.apache.hadoop.yarn.server.nodemanager.containermanager.logaggregation.AggregatedLogFormat.LogKey; -import org.apache.hadoop.yarn.server.nodemanager.containermanager.logaggregation.AggregatedLogFormat.LogValue; -import org.apache.hadoop.yarn.server.nodemanager.containermanager.logaggregation.AggregatedLogFormat.LogWriter; import org.apache.hadoop.yarn.util.ConverterUtils; public class AppLogAggregatorImpl implements AppLogAggregator { diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/LogAggregationService.java b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/LogAggregationService.java index d651cb98531..95885d4e07e 100644 --- a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/LogAggregationService.java +++ b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/LogAggregationService.java @@ -43,6 +43,8 @@ import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.event.Dispatcher; +import org.apache.hadoop.yarn.logaggregation.ContainerLogsRetentionPolicy; +import org.apache.hadoop.yarn.logaggregation.LogAggregationUtils; import org.apache.hadoop.yarn.server.nodemanager.Context; import org.apache.hadoop.yarn.server.nodemanager.DeletionService; import org.apache.hadoop.yarn.server.nodemanager.containermanager.loghandler.LogHandler; @@ -138,83 +140,7 @@ public class LogAggregationService extends AbstractService implements super.stop(); } - /** - * Constructs the full filename for an application's log file per node. - * @param remoteRootLogDir - * @param appId - * @param user - * @param nodeId - * @param suffix - * @return the remote log file. - */ - public static Path getRemoteNodeLogFileForApp(Path remoteRootLogDir, - ApplicationId appId, String user, NodeId nodeId, String suffix) { - return new Path(getRemoteAppLogDir(remoteRootLogDir, appId, user, suffix), - getNodeString(nodeId)); - } - - /** - * Gets the remote app log dir. - * @param remoteRootLogDir - * @param appId - * @param user - * @param suffix - * @return the remote application specific log dir. - */ - public static Path getRemoteAppLogDir(Path remoteRootLogDir, - ApplicationId appId, String user, String suffix) { - return new Path(getRemoteLogSuffixedDir(remoteRootLogDir, user, suffix), - appId.toString()); - } - - /** - * Gets the remote suffixed log dir for the user. - * @param remoteRootLogDir - * @param user - * @param suffix - * @return the remote suffixed log dir. - */ - private static Path getRemoteLogSuffixedDir(Path remoteRootLogDir, - String user, String suffix) { - if (suffix == null || suffix.isEmpty()) { - return getRemoteLogUserDir(remoteRootLogDir, user); - } - // TODO Maybe support suffix to be more than a single file. - return new Path(getRemoteLogUserDir(remoteRootLogDir, user), suffix); - } - - // TODO Add a utility method to list available log files. Ignore the - // temporary ones. - /** - * Gets the remote log user dir. - * @param remoteRootLogDir - * @param user - * @return the remote per user log dir. - */ - private static Path getRemoteLogUserDir(Path remoteRootLogDir, String user) { - return new Path(remoteRootLogDir, user); - } - - /** - * Returns the suffix component of the log dir. - * @param conf - * @return the suffix which will be appended to the user log dir. - */ - public static String getRemoteNodeLogDirSuffix(Configuration conf) { - return conf.get(YarnConfiguration.NM_REMOTE_APP_LOG_DIR_SUFFIX, - YarnConfiguration.DEFAULT_NM_REMOTE_APP_LOG_DIR_SUFFIX); - } - - - /** - * Converts a nodeId to a form used in the app log file name. - * @param nodeId - * @return the node string to be used to construct the file name. - */ - private static String getNodeString(NodeId nodeId) { - return nodeId.toString().replace(":", "_"); - } @@ -268,7 +194,7 @@ public class LogAggregationService extends AbstractService implements } Path getRemoteNodeLogFileForApp(ApplicationId appId, String user) { - return LogAggregationService.getRemoteNodeLogFileForApp( + return LogAggregationUtils.getRemoteNodeLogFileForApp( this.remoteRootLogDir, appId, user, this.nodeId, this.remoteRootLogDirSuffix); } @@ -299,7 +225,7 @@ public class LogAggregationService extends AbstractService implements } try { userDir = - getRemoteLogUserDir( + LogAggregationUtils.getRemoteLogUserDir( LogAggregationService.this.remoteRootLogDir, user); userDir = userDir.makeQualified(remoteFS.getUri(), @@ -312,7 +238,7 @@ public class LogAggregationService extends AbstractService implements } try { suffixDir = - getRemoteLogSuffixedDir( + LogAggregationUtils.getRemoteLogSuffixedDir( LogAggregationService.this.remoteRootLogDir, user, LogAggregationService.this.remoteRootLogDirSuffix); suffixDir = @@ -326,8 +252,8 @@ public class LogAggregationService extends AbstractService implements } try { appDir = - getRemoteAppLogDir(LogAggregationService.this.remoteRootLogDir, - appId, user, + LogAggregationUtils.getRemoteAppLogDir( + LogAggregationService.this.remoteRootLogDir, appId, user, LogAggregationService.this.remoteRootLogDirSuffix); appDir = appDir.makeQualified(remoteFS.getUri(), diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/loghandler/event/LogHandlerAppStartedEvent.java b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/loghandler/event/LogHandlerAppStartedEvent.java index a598bed264f..6c076748a40 100644 --- a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/loghandler/event/LogHandlerAppStartedEvent.java +++ b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/loghandler/event/LogHandlerAppStartedEvent.java @@ -23,7 +23,7 @@ import java.util.Map; import org.apache.hadoop.security.Credentials; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ApplicationAccessType; -import org.apache.hadoop.yarn.server.nodemanager.containermanager.logaggregation.ContainerLogsRetentionPolicy; +import org.apache.hadoop.yarn.logaggregation.ContainerLogsRetentionPolicy; public class LogHandlerAppStartedEvent extends LogHandlerEvent { diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/AggregatedLogsBlock.java b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/AggregatedLogsBlock.java index 09c983bd52e..5d5f1e35f1a 100644 --- a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/AggregatedLogsBlock.java +++ b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/AggregatedLogsBlock.java @@ -15,186 +15,3 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.hadoop.yarn.server.nodemanager.webapp; - -import static org.apache.hadoop.yarn.server.nodemanager.webapp.NMWebParams.CONTAINER_ID; -import static org.apache.hadoop.yarn.server.nodemanager.webapp.NMWebParams.NM_NODENAME; -import static org.apache.hadoop.yarn.server.nodemanager.webapp.NMWebParams.ENTITY_STRING; -import static org.apache.hadoop.yarn.server.nodemanager.webapp.NMWebParams.APP_OWNER; - -import java.io.DataInputStream; -import java.io.FileNotFoundException; -import java.io.IOException; -import java.util.Map; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.security.UserGroupInformation; -import org.apache.hadoop.yarn.api.records.ApplicationAccessType; -import org.apache.hadoop.yarn.api.records.ApplicationId; -import org.apache.hadoop.yarn.api.records.ContainerId; -import org.apache.hadoop.yarn.api.records.NodeId; -import org.apache.hadoop.yarn.conf.YarnConfiguration; -import org.apache.hadoop.yarn.server.nodemanager.containermanager.logaggregation.AggregatedLogFormat; -import org.apache.hadoop.yarn.server.nodemanager.containermanager.logaggregation.LogAggregationService; -import org.apache.hadoop.yarn.server.nodemanager.containermanager.logaggregation.AggregatedLogFormat.LogKey; -import org.apache.hadoop.yarn.server.security.ApplicationACLsManager; -import org.apache.hadoop.yarn.util.ConverterUtils; -import org.apache.hadoop.yarn.webapp.view.HtmlBlock; - -import com.google.inject.Inject; - -public class AggregatedLogsBlock extends HtmlBlock { - - private final Configuration conf; - - @Inject - AggregatedLogsBlock(Configuration conf) { - this.conf = conf; - } - - @Override - protected void render(Block html) { - ContainerId containerId = verifyAndGetContainerId(html); - NodeId nodeId = verifyAndGetNodeId(html); - String appOwner = verifyAndGetAppOwner(html); - if (containerId == null || nodeId == null || appOwner == null - || appOwner.isEmpty()) { - return; - } - - ApplicationId applicationId = - containerId.getApplicationAttemptId().getApplicationId(); - String logEntity = $(ENTITY_STRING); - if (logEntity == null || logEntity.isEmpty()) { - logEntity = containerId.toString(); - } - - if (!conf.getBoolean(YarnConfiguration.NM_LOG_AGGREGATION_ENABLED, - YarnConfiguration.DEFAULT_NM_LOG_AGGREGATION_ENABLED)) { - html.h1() - ._("Aggregation is not enabled. Try the nodemanager at " + nodeId) - ._(); - return; - } - - Path remoteRootLogDir = - new Path(conf.get(YarnConfiguration.NM_REMOTE_APP_LOG_DIR, - YarnConfiguration.DEFAULT_NM_REMOTE_APP_LOG_DIR)); - AggregatedLogFormat.LogReader reader = null; - try { - reader = - new AggregatedLogFormat.LogReader(conf, - LogAggregationService.getRemoteNodeLogFileForApp( - remoteRootLogDir, applicationId, appOwner, nodeId, - LogAggregationService.getRemoteNodeLogDirSuffix(conf))); - } catch (FileNotFoundException e) { - // ACLs not available till the log file is opened. - html.h1() - ._("Logs not available for " - + logEntity - + ". Aggregation may not be complete, " - + "Check back later or try the nodemanager at " - + nodeId)._(); - return; - } catch (IOException e) { - html.h1()._("Error getting logs for " + logEntity)._(); - LOG.error("Error getting logs for " + logEntity, e); - return; - } - - String owner = null; - Map appAcls = null; - try { - owner = reader.getApplicationOwner(); - appAcls = reader.getApplicationAcls(); - } catch (IOException e) { - html.h1()._("Error getting logs for " + logEntity)._(); - LOG.error("Error getting logs for " + logEntity, e); - return; - } - ApplicationACLsManager aclsManager = new ApplicationACLsManager(conf); - aclsManager.addApplication(applicationId, appAcls); - - String remoteUser = request().getRemoteUser(); - UserGroupInformation callerUGI = null; - if (remoteUser != null) { - callerUGI = UserGroupInformation.createRemoteUser(remoteUser); - } - if (callerUGI != null - && !aclsManager.checkAccess(callerUGI, ApplicationAccessType.VIEW_APP, - owner, applicationId)) { - html.h1() - ._("User [" + remoteUser - + "] is not authorized to view the logs for " + logEntity)._(); - return; - } - - DataInputStream valueStream; - LogKey key = new LogKey(); - try { - valueStream = reader.next(key); - while (valueStream != null - && !key.toString().equals(containerId.toString())) { - valueStream = reader.next(key); - } - if (valueStream == null) { - html.h1()._( - "Logs not available for " + logEntity - + ". Could be caused by the rentention policy")._(); - return; - } - writer().write("
");
-      AggregatedLogFormat.LogReader.readAcontainerLogs(valueStream, writer());
-      writer().write("
"); - return; - } catch (IOException e) { - html.h1()._("Error getting logs for " + logEntity)._(); - LOG.error("Error getting logs for " + logEntity, e); - return; - } - } - - private ContainerId verifyAndGetContainerId(Block html) { - String containerIdStr = $(CONTAINER_ID); - if (containerIdStr == null || containerIdStr.isEmpty()) { - html.h1()._("Cannot get container logs without a ContainerId")._(); - return null; - } - ContainerId containerId = null; - try { - containerId = ConverterUtils.toContainerId(containerIdStr); - } catch (IllegalArgumentException e) { - html.h1() - ._("Cannot get container logs for invalid containerId: " - + containerIdStr)._(); - return null; - } - return containerId; - } - - private NodeId verifyAndGetNodeId(Block html) { - String nodeIdStr = $(NM_NODENAME); - if (nodeIdStr == null || nodeIdStr.isEmpty()) { - html.h1()._("Cannot get container logs without a NodeId")._(); - return null; - } - NodeId nodeId = null; - try { - nodeId = ConverterUtils.toNodeId(nodeIdStr); - } catch (IllegalArgumentException e) { - html.h1()._("Cannot get container logs. Invalid nodeId: " + nodeIdStr) - ._(); - return null; - } - return nodeId; - } - - private String verifyAndGetAppOwner(Block html) { - String appOwner = $(APP_OWNER); - if (appOwner == null || appOwner.isEmpty()) { - html.h1()._("Cannot get container logs without an app owner")._(); - } - return appOwner; - } -} \ No newline at end of file diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/AggregatedLogsPage.java b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/AggregatedLogsPage.java index ff3a82275a6..5d5f1e35f1a 100644 --- a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/AggregatedLogsPage.java +++ b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/AggregatedLogsPage.java @@ -15,47 +15,3 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.hadoop.yarn.server.nodemanager.webapp; - -import static org.apache.hadoop.yarn.server.nodemanager.webapp.NMWebParams.CONTAINER_ID; -import static org.apache.hadoop.yarn.server.nodemanager.webapp.NMWebParams.ENTITY_STRING; -import static org.apache.hadoop.yarn.util.StringHelper.join; -import static org.apache.hadoop.yarn.webapp.view.JQueryUI.ACCORDION; -import static org.apache.hadoop.yarn.webapp.view.JQueryUI.ACCORDION_ID; -import static org.apache.hadoop.yarn.webapp.view.JQueryUI.THEMESWITCHER_ID; -import static org.apache.hadoop.yarn.webapp.view.JQueryUI.initID; - - -import org.apache.hadoop.yarn.webapp.SubView; - - -public class AggregatedLogsPage extends NMView { - - /* (non-Javadoc) - * @see org.apache.hadoop.yarn.server.nodemanager.webapp.NMView#preHead(org.apache.hadoop.yarn.webapp.hamlet.Hamlet.HTML) - */ - @Override - protected void preHead(Page.HTML<_> html) { - String logEntity = $(ENTITY_STRING); - if (logEntity == null || logEntity.isEmpty()) { - logEntity = $(CONTAINER_ID); - } - if (logEntity == null || logEntity.isEmpty()) { - logEntity = "UNKNOWN"; - } - set(TITLE, join("Logs for ", logEntity)); - set(ACCORDION_ID, "nav"); - set(initID(ACCORDION, "nav"), "{autoHeight:false, active:0}"); - set(THEMESWITCHER_ID, "themeswitcher"); - } - - @Override - protected Class content() { - return AggregatedLogsBlock.class; - } - - @Override - protected Class nav() { - return AggregatedLogsNavBlock.class; - } -} diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/AllApplicationsPage.java b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/AllApplicationsPage.java index a35f81ffb20..49b75373d78 100644 --- a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/AllApplicationsPage.java +++ b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/AllApplicationsPage.java @@ -29,6 +29,7 @@ import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.server.nodemanager.Context; import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.Application; import org.apache.hadoop.yarn.util.ConverterUtils; +import org.apache.hadoop.yarn.webapp.YarnWebParams; import org.apache.hadoop.yarn.webapp.SubView; import org.apache.hadoop.yarn.webapp.hamlet.Hamlet; import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.BODY; @@ -60,7 +61,7 @@ public class AllApplicationsPage extends NMView { } public static class AllApplicationsBlock extends HtmlBlock implements - NMWebParams { + YarnWebParams { private final Context nmContext; diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/AllContainersPage.java b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/AllContainersPage.java index 9c2dc47ba28..13c8951aa6b 100644 --- a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/AllContainersPage.java +++ b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/AllContainersPage.java @@ -29,6 +29,7 @@ import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.server.nodemanager.Context; import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container; import org.apache.hadoop.yarn.util.ConverterUtils; +import org.apache.hadoop.yarn.webapp.YarnWebParams; import org.apache.hadoop.yarn.webapp.SubView; import org.apache.hadoop.yarn.webapp.hamlet.Hamlet; import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.BODY; @@ -60,7 +61,7 @@ public class AllContainersPage extends NMView { } public static class AllContainersBlock extends HtmlBlock implements - NMWebParams { + YarnWebParams { private final Context nmContext; diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/ApplicationPage.java b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/ApplicationPage.java index 4fd26be49b8..fc02120c6e9 100644 --- a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/ApplicationPage.java +++ b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/ApplicationPage.java @@ -34,6 +34,7 @@ import org.apache.hadoop.yarn.server.nodemanager.Context; import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.Application; import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container; import org.apache.hadoop.yarn.util.ConverterUtils; +import org.apache.hadoop.yarn.webapp.YarnWebParams; import org.apache.hadoop.yarn.webapp.SubView; import org.apache.hadoop.yarn.webapp.hamlet.Hamlet; import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.TABLE; @@ -42,7 +43,7 @@ import org.apache.hadoop.yarn.webapp.view.InfoBlock; import com.google.inject.Inject; -public class ApplicationPage extends NMView implements NMWebParams { +public class ApplicationPage extends NMView implements YarnWebParams { @Override protected void preHead(Page.HTML<_> html) { commonPreHead(html); @@ -61,7 +62,7 @@ public class ApplicationPage extends NMView implements NMWebParams { } public static class ApplicationBlock extends HtmlBlock implements - NMWebParams { + YarnWebParams { private final Context nmContext; private final Configuration conf; diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/ContainerLogsPage.java b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/ContainerLogsPage.java index 84d2a4a2718..faf0cbc47fe 100644 --- a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/ContainerLogsPage.java +++ b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/ContainerLogsPage.java @@ -18,8 +18,8 @@ package org.apache.hadoop.yarn.server.nodemanager.webapp; -import static org.apache.hadoop.yarn.server.nodemanager.webapp.NMWebParams.CONTAINER_ID; import static org.apache.hadoop.yarn.util.StringHelper.join; +import static org.apache.hadoop.yarn.webapp.YarnWebParams.CONTAINER_ID; import static org.apache.hadoop.yarn.webapp.view.JQueryUI.ACCORDION; import static org.apache.hadoop.yarn.webapp.view.JQueryUI.ACCORDION_ID; import static org.apache.hadoop.yarn.webapp.view.JQueryUI.THEMESWITCHER_ID; @@ -49,6 +49,7 @@ import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Cont import org.apache.hadoop.yarn.server.nodemanager.containermanager.launcher.ContainerLaunch; import org.apache.hadoop.yarn.server.security.ApplicationACLsManager; import org.apache.hadoop.yarn.util.ConverterUtils; +import org.apache.hadoop.yarn.webapp.YarnWebParams; import org.apache.hadoop.yarn.webapp.SubView; import org.apache.hadoop.yarn.webapp.view.HtmlBlock; @@ -84,7 +85,7 @@ public class ContainerLogsPage extends NMView { } public static class ContainersLogsBlock extends HtmlBlock implements - NMWebParams { + YarnWebParams { private final Configuration conf; private final LocalDirAllocator logsSelector; private final Context nmContext; diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/ContainerPage.java b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/ContainerPage.java index aec44295a76..6633aa03495 100644 --- a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/ContainerPage.java +++ b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/ContainerPage.java @@ -28,6 +28,7 @@ import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.server.nodemanager.Context; import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container; import org.apache.hadoop.yarn.util.ConverterUtils; +import org.apache.hadoop.yarn.webapp.YarnWebParams; import org.apache.hadoop.yarn.webapp.SubView; import org.apache.hadoop.yarn.webapp.hamlet.Hamlet; import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.DIV; @@ -36,7 +37,7 @@ import org.apache.hadoop.yarn.webapp.view.InfoBlock; import com.google.inject.Inject; -public class ContainerPage extends NMView implements NMWebParams { +public class ContainerPage extends NMView implements YarnWebParams { @Override protected void preHead(Page.HTML<_> html) { @@ -50,7 +51,7 @@ public class ContainerPage extends NMView implements NMWebParams { return ContainerBlock.class; } - public static class ContainerBlock extends HtmlBlock implements NMWebParams { + public static class ContainerBlock extends HtmlBlock implements YarnWebParams { private final Context nmContext; diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMController.java b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMController.java index cf69b0f4ee2..033271afde9 100644 --- a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMController.java +++ b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMController.java @@ -28,10 +28,11 @@ import org.apache.hadoop.yarn.server.nodemanager.Context; import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.Application; import org.apache.hadoop.yarn.util.ConverterUtils; import org.apache.hadoop.yarn.webapp.Controller; +import org.apache.hadoop.yarn.webapp.YarnWebParams; import com.google.inject.Inject; -public class NMController extends Controller implements NMWebParams { +public class NMController extends Controller implements YarnWebParams { private Context nmContext; private Configuration nmConf; diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NavBlock.java b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NavBlock.java index 01ea4aa9ee4..d6710e95d18 100644 --- a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NavBlock.java +++ b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NavBlock.java @@ -18,9 +18,10 @@ package org.apache.hadoop.yarn.server.nodemanager.webapp; +import org.apache.hadoop.yarn.webapp.YarnWebParams; import org.apache.hadoop.yarn.webapp.view.HtmlBlock; -public class NavBlock extends HtmlBlock implements NMWebParams { +public class NavBlock extends HtmlBlock implements YarnWebParams { @Override protected void render(Block html) { diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/WebServer.java b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/WebServer.java index 5df3b9edbd9..25730158777 100644 --- a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/WebServer.java +++ b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/WebServer.java @@ -29,6 +29,7 @@ import org.apache.hadoop.yarn.server.nodemanager.Context; import org.apache.hadoop.yarn.server.nodemanager.ResourceView; import org.apache.hadoop.yarn.server.security.ApplicationACLsManager; import org.apache.hadoop.yarn.service.AbstractService; +import org.apache.hadoop.yarn.webapp.YarnWebParams; import org.apache.hadoop.yarn.webapp.WebApp; import org.apache.hadoop.yarn.webapp.WebApps; @@ -76,7 +77,7 @@ public class WebServer extends AbstractService { super.stop(); } - public static class NMWebApp extends WebApp implements NMWebParams { + public static class NMWebApp extends WebApp implements YarnWebParams { private final ResourceView resourceView; private final ApplicationACLsManager aclsManager; diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java index bc420e4f921..a4202a9462d 100644 --- a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java +++ b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java @@ -63,13 +63,15 @@ import org.apache.hadoop.yarn.event.DrainDispatcher; import org.apache.hadoop.yarn.event.EventHandler; import org.apache.hadoop.yarn.factories.RecordFactory; import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider; +import org.apache.hadoop.yarn.logaggregation.AggregatedLogFormat; +import org.apache.hadoop.yarn.logaggregation.ContainerLogsRetentionPolicy; +import org.apache.hadoop.yarn.logaggregation.AggregatedLogFormat.LogKey; +import org.apache.hadoop.yarn.logaggregation.AggregatedLogFormat.LogReader; import org.apache.hadoop.yarn.server.nodemanager.CMgrCompletedAppsEvent; import org.apache.hadoop.yarn.server.nodemanager.DeletionService; import org.apache.hadoop.yarn.server.nodemanager.containermanager.BaseContainerManagerTest; import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.ApplicationEvent; import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.ApplicationEventType; -import org.apache.hadoop.yarn.server.nodemanager.containermanager.logaggregation.AggregatedLogFormat.LogKey; -import org.apache.hadoop.yarn.server.nodemanager.containermanager.logaggregation.AggregatedLogFormat.LogReader; import org.apache.hadoop.yarn.server.nodemanager.containermanager.loghandler.event.LogHandlerAppFinishedEvent; import org.apache.hadoop.yarn.server.nodemanager.containermanager.loghandler.event.LogHandlerAppStartedEvent; import org.apache.hadoop.yarn.server.nodemanager.containermanager.loghandler.event.LogHandlerContainerFinishedEvent; diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/loghandler/TestNonAggregatingLogHandler.java b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/loghandler/TestNonAggregatingLogHandler.java index ba7e4b93f6f..5fa7bcb3b1c 100644 --- a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/loghandler/TestNonAggregatingLogHandler.java +++ b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/loghandler/TestNonAggregatingLogHandler.java @@ -35,10 +35,10 @@ import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.event.Dispatcher; import org.apache.hadoop.yarn.event.DrainDispatcher; import org.apache.hadoop.yarn.event.EventHandler; +import org.apache.hadoop.yarn.logaggregation.ContainerLogsRetentionPolicy; import org.apache.hadoop.yarn.server.nodemanager.DeletionService; import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.ApplicationEvent; import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.ApplicationEventType; -import org.apache.hadoop.yarn.server.nodemanager.containermanager.logaggregation.ContainerLogsRetentionPolicy; import org.apache.hadoop.yarn.server.nodemanager.containermanager.loghandler.event.LogHandlerAppFinishedEvent; import org.apache.hadoop.yarn.server.nodemanager.containermanager.loghandler.event.LogHandlerAppStartedEvent; import org.apache.hadoop.yarn.server.nodemanager.containermanager.loghandler.event.LogHandlerContainerFinishedEvent;