From 6ba0375b21c4ce07d2b6b592c4963f705c35222b Mon Sep 17 00:00:00 2001 From: Siddharth Seth Date: Thu, 2 Feb 2012 01:43:49 +0000 Subject: [PATCH] MAPREDUCE-3744. Fix the yarn logs command line. Improve error messages for mapred job -logs. (Contributed by Jason Lowe) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1239433 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-mapreduce-project/CHANGES.txt | 3 + .../apache/hadoop/mapreduce/tools/CLI.java | 2 +- .../hadoop/yarn/logaggregation/LogDumper.java | 55 ++++++++++++------- .../yarn/logaggregation/TestLogDumper.java | 47 ++++++++++++++++ 4 files changed, 87 insertions(+), 20 deletions(-) create mode 100644 hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/TestLogDumper.java diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index 99b4d92fde3..1ed09ada0e0 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -654,6 +654,9 @@ Release 0.23.1 - Unreleased MAPREDUCE-3752. Modified application limits to include queue max-capacities besides the usual user limits. (Arun C Murthy via vinodkv) + MAPREDUCE-3744. Fix the yarn logs command line. Improve error messages for + mapred job -logs. (Jason Lowe via sseth) + Release 0.23.0 - 2011-11-01 INCOMPATIBLE CHANGES diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/tools/CLI.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/tools/CLI.java index b73a9ea2bfd..c219247823e 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/tools/CLI.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/tools/CLI.java @@ -345,7 +345,7 @@ public class CLI extends Configured implements Tool { LogParams logParams = cluster.getLogParams(jobID, taskAttemptID); LogDumper logDumper = new LogDumper(); logDumper.setConf(getConf()); - logDumper.dumpAContainersLogs(logParams.getApplicationId(), + exitCode = logDumper.dumpAContainersLogs(logParams.getApplicationId(), logParams.getContainerId(), logParams.getNodeId(), logParams.getOwner()); } catch (IOException e) { diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/LogDumper.java b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/LogDumper.java index 20206d6f941..c81b772e785 100644 --- a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/LogDumper.java +++ b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/LogDumper.java @@ -21,6 +21,7 @@ package org.apache.hadoop.yarn.logaggregation; import java.io.DataInputStream; import java.io.DataOutputStream; import java.io.EOFException; +import java.io.FileNotFoundException; import java.io.IOException; import org.apache.commons.cli.CommandLine; @@ -103,14 +104,15 @@ public class LogDumper extends Configured implements Tool { if (appOwner == null || appOwner.isEmpty()) { appOwner = UserGroupInformation.getCurrentUser().getShortUserName(); } + int resultCode = 0; if (containerIdStr == null && nodeAddress == null) { - dumpAllContainersLogs(appId, appOwner, out); + resultCode = dumpAllContainersLogs(appId, appOwner, out); } else if ((containerIdStr == null && nodeAddress != null) || (containerIdStr != null && nodeAddress == null)) { System.out.println("ContainerId or NodeAddress cannot be null!"); HelpFormatter formatter = new HelpFormatter(); formatter.printHelp("general options are: ", opts); - return -1; + resultCode = -1; } else { Path remoteRootLogDir = new Path(getConf().get(YarnConfiguration.NM_REMOTE_APP_LOG_DIR, @@ -122,27 +124,33 @@ public class LogDumper extends Configured implements Tool { appId, appOwner, ConverterUtils.toNodeId(nodeAddress), - getConf().get(YarnConfiguration.NM_REMOTE_APP_LOG_DIR_SUFFIX, - YarnConfiguration.DEFAULT_NM_REMOTE_APP_LOG_DIR_SUFFIX))); - return dumpAContainerLogs(containerIdStr, reader, out); + LogAggregationUtils.getRemoteNodeLogDirSuffix(getConf()))); + resultCode = dumpAContainerLogs(containerIdStr, reader, out); } - return 0; + return resultCode; } - public void dumpAContainersLogs(String appId, String containerId, + public int dumpAContainersLogs(String appId, String containerId, String nodeId, String jobOwner) throws IOException { Path remoteRootLogDir = new Path(getConf().get(YarnConfiguration.NM_REMOTE_APP_LOG_DIR, YarnConfiguration.DEFAULT_NM_REMOTE_APP_LOG_DIR)); String suffix = LogAggregationUtils.getRemoteNodeLogDirSuffix(getConf()); - AggregatedLogFormat.LogReader reader = - new AggregatedLogFormat.LogReader(getConf(), - LogAggregationUtils.getRemoteNodeLogFileForApp(remoteRootLogDir, - ConverterUtils.toApplicationId(appId), jobOwner, - ConverterUtils.toNodeId(nodeId), suffix)); + Path logPath = LogAggregationUtils.getRemoteNodeLogFileForApp( + remoteRootLogDir, ConverterUtils.toApplicationId(appId), jobOwner, + ConverterUtils.toNodeId(nodeId), suffix); + AggregatedLogFormat.LogReader reader; + try { + reader = new AggregatedLogFormat.LogReader(getConf(), logPath); + } catch (FileNotFoundException fnfe) { + System.out.println("Logs not available at " + logPath.toString()); + System.out.println( + "Log aggregation has not completed or is not enabled."); + return -1; + } DataOutputStream out = new DataOutputStream(System.out); - dumpAContainerLogs(containerId, reader, out); + return dumpAContainerLogs(containerId, reader, out); } private int dumpAContainerLogs(String containerIdStr, @@ -174,21 +182,28 @@ public class LogDumper extends Configured implements Tool { return 0; } - private void dumpAllContainersLogs(ApplicationId appId, String appOwner, + private int dumpAllContainersLogs(ApplicationId appId, String appOwner, DataOutputStream out) throws IOException { Path remoteRootLogDir = new Path(getConf().get(YarnConfiguration.NM_REMOTE_APP_LOG_DIR, YarnConfiguration.DEFAULT_NM_REMOTE_APP_LOG_DIR)); String user = appOwner; String logDirSuffix = - getConf().get(YarnConfiguration.NM_REMOTE_APP_LOG_DIR, - YarnConfiguration.DEFAULT_NM_REMOTE_APP_LOG_DIR_SUFFIX); + LogAggregationUtils.getRemoteNodeLogDirSuffix(getConf()); //TODO Change this to get a list of files from the LAS. Path remoteAppLogDir = LogAggregationUtils.getRemoteAppLogDir(remoteRootLogDir, appId, user, logDirSuffix); - RemoteIterator nodeFiles = - FileContext.getFileContext().listStatus(remoteAppLogDir); + RemoteIterator nodeFiles; + try { + nodeFiles = FileContext.getFileContext().listStatus(remoteAppLogDir); + } catch (FileNotFoundException fnf) { + System.out.println("Logs not available at " + + remoteAppLogDir.toString()); + System.out.println( + "Log aggregation has not completed or is not enabled."); + return -1; + } while (nodeFiles.hasNext()) { FileStatus thisNodeFile = nodeFiles.next(); AggregatedLogFormat.LogReader reader = @@ -217,12 +232,14 @@ public class LogDumper extends Configured implements Tool { reader.close(); } } + return 0; } public static void main(String[] args) throws Exception { Configuration conf = new YarnConfiguration(); LogDumper logDumper = new LogDumper(); logDumper.setConf(conf); - logDumper.run(args); + int exitCode = logDumper.run(args); + System.exit(exitCode); } } diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/TestLogDumper.java b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/TestLogDumper.java new file mode 100644 index 00000000000..527a551f2d6 --- /dev/null +++ b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/TestLogDumper.java @@ -0,0 +1,47 @@ +/** +* Licensed to the Apache Software Foundation (ASF) under one +* or more contributor license agreements. See the NOTICE file +* distributed with this work for additional information +* regarding copyright ownership. The ASF licenses this file +* to you under the Apache License, Version 2.0 (the +* "License"); you may not use this file except in compliance +* with the License. You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ + +package org.apache.hadoop.yarn.logaggregation; + +import static org.junit.Assert.assertTrue; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.LocalFileSystem; +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.junit.Test; + +public class TestLogDumper { + @Test + public void testFailResultCodes() throws Exception { + Configuration conf = new YarnConfiguration(); + conf.setClass("fs.file.impl", LocalFileSystem.class, FileSystem.class); + LogDumper dumper = new LogDumper(); + dumper.setConf(conf); + + // verify dumping a non-existent application's logs returns a failure code + int exitCode = dumper.run( new String[] { + "-applicationId", "application_0_0" } ); + assertTrue("Should return an error code", exitCode != 0); + + // verify dumping a non-existent container log is a failure code + exitCode = dumper.dumpAContainersLogs("application_0_0", "container_0_0", + "nonexistentnode:1234", "nobody"); + assertTrue("Should return an error code", exitCode != 0); + } +}