From 63231a2a3008794c5ebcbc9d3855638fc5f28216 Mon Sep 17 00:00:00 2001 From: Junping Du Date: Wed, 6 Sep 2017 13:19:34 -0700 Subject: [PATCH] YARN-7148. TestLogsCLI fails in trunk and branch-2 and javadoc error. Contributed by Xuan Gong. --- .../hadoop/yarn/client/cli/TestLogsCLI.java | 16 ++-- .../logaggregation/AggregatedLogFormat.java | 78 ++++++++++--------- .../logaggregation/LogAggregationUtils.java | 32 ++++---- .../yarn/logaggregation/LogCLIHelpers.java | 41 ++++++++-- .../LogAggregationFileController.java | 11 +-- .../tfile/LogAggregationTFileController.java | 1 - 6 files changed, 103 insertions(+), 76 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java index 509a790b2a1..d58732e006a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java @@ -285,7 +285,7 @@ public void testHelpMessage() throws Exception { @Test (timeout = 15000) public void testFetchFinishedApplictionLogs() throws Exception { String remoteLogRootDir = "target/logs/"; - Configuration configuration = new Configuration(); + Configuration configuration = new YarnConfiguration(); configuration.setBoolean(YarnConfiguration.LOG_AGGREGATION_ENABLED, true); configuration .set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR, remoteLogRootDir); @@ -646,7 +646,7 @@ public void testGetRunningContainerLogs() throws Exception { any(ContainerId.class)); // create local logs - Configuration configuration = new Configuration(); + Configuration configuration = new YarnConfiguration(); FileSystem fs = FileSystem.get(configuration); String rootLogDir = "target/LocalLogs"; Path rootLogDirPath = new Path(rootLogDir); @@ -790,7 +790,7 @@ public void testFetchApplictionLogsAsAnotherUser() throws Exception { UserGroupInformation testUgi = UserGroupInformation .createRemoteUser(testUser); - Configuration configuration = new Configuration(); + Configuration configuration = new YarnConfiguration(); configuration.setBoolean(YarnConfiguration.LOG_AGGREGATION_ENABLED, true); configuration .set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR, remoteLogRootDir); @@ -921,7 +921,7 @@ public void testFetchApplictionLogsAsAnotherUser() throws Exception { public void testLogsCLIWithInvalidArgs() throws Exception { String localDir = "target/SaveLogs"; Path localPath = new Path(localDir); - Configuration configuration = new Configuration(); + Configuration configuration = new YarnConfiguration(); FileSystem fs = FileSystem.get(configuration); ApplicationId appId = ApplicationId.newInstance(0, 1); YarnClient mockYarnClient = @@ -992,7 +992,7 @@ public void testSaveContainerLogsLocally() throws Exception { String localDir = "target/SaveLogs"; Path localPath = new Path(localDir); - Configuration configuration = new Configuration(); + Configuration configuration = new YarnConfiguration(); configuration.setBoolean(YarnConfiguration.LOG_AGGREGATION_ENABLED, true); configuration .set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR, remoteLogRootDir); @@ -1083,7 +1083,7 @@ private String readContainerContent(Path containerPath, @Test (timeout = 15000) public void testPrintContainerLogMetadata() throws Exception { String remoteLogRootDir = "target/logs/"; - Configuration configuration = new Configuration(); + Configuration configuration = new YarnConfiguration(); configuration.setBoolean(YarnConfiguration.LOG_AGGREGATION_ENABLED, true); configuration .set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR, remoteLogRootDir); @@ -1188,7 +1188,7 @@ public void testPrintContainerLogMetadata() throws Exception { @Test (timeout = 15000) public void testListNodeInfo() throws Exception { String remoteLogRootDir = "target/logs/"; - Configuration configuration = new Configuration(); + Configuration configuration = new YarnConfiguration(); configuration.setBoolean(YarnConfiguration.LOG_AGGREGATION_ENABLED, true); configuration .set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR, remoteLogRootDir); @@ -1239,7 +1239,7 @@ public void testListNodeInfo() throws Exception { @Test (timeout = 15000) public void testFetchApplictionLogsHar() throws Exception { String remoteLogRootDir = "target/logs/"; - Configuration configuration = new Configuration(); + Configuration configuration = new YarnConfiguration(); configuration.setBoolean(YarnConfiguration.LOG_AGGREGATION_ENABLED, true); configuration .set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR, remoteLogRootDir); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/AggregatedLogFormat.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/AggregatedLogFormat.java index 3c1dcdc1234..9bec147228a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/AggregatedLogFormat.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/AggregatedLogFormat.java @@ -583,9 +583,9 @@ public LogReader(Configuration conf, Path remoteAppLogFile) /** * Returns the owner of the application. - * + * * @return the application owner. - * @throws IOException + * @throws IOException if we can not get the application owner. */ public String getApplicationOwner() throws IOException { TFile.Reader.Scanner ownerScanner = null; @@ -610,9 +610,9 @@ public String getApplicationOwner() throws IOException { /** * Returns ACLs for the application. An empty map is returned if no ACLs are * found. - * + * * @return a map of the Application ACLs. - * @throws IOException + * @throws IOException if we can not get the application acls. */ public Map getApplicationAcls() throws IOException { @@ -656,9 +656,10 @@ public Map getApplicationAcls() /** * Read the next key and return the value-stream. * - * @param key - * @return the valueStream if there are more keys or null otherwise. - * @throws IOException + * @param key the log key + * @return the valueStream if there are more keys or null otherwise + * @throws IOException if we can not get the dataInputStream + * for the next key */ public DataInputStream next(LogKey key) throws IOException { if (!this.atBeginning) { @@ -683,10 +684,10 @@ public DataInputStream next(LogKey key) throws IOException { * Get a ContainerLogsReader to read the logs for * the specified container. * - * @param containerId + * @param containerId the containerId * @return object to read the container's logs or null if the * logs could not be found - * @throws IOException + * @throws IOException if we can not get the container log reader. */ @Private public ContainerLogsReader getContainerLogsReader( @@ -752,10 +753,10 @@ public ContainerLogsReader getContainerLogsReader( /** * Writes all logs for a single container to the provided writer. - * @param valueStream - * @param writer - * @param logUploadedTime - * @throws IOException + * @param valueStream the valueStream + * @param writer the log writer + * @param logUploadedTime the time stamp + * @throws IOException if we can not read the container logs. */ public static void readAcontainerLogs(DataInputStream valueStream, Writer writer, long logUploadedTime) throws IOException { @@ -780,9 +781,9 @@ public static void readAcontainerLogs(DataInputStream valueStream, /** * Writes all logs for a single container to the provided writer. - * @param valueStream - * @param writer - * @throws IOException + * @param valueStream the value stream + * @param writer the log writer + * @throws IOException if we can not read the container logs. */ public static void readAcontainerLogs(DataInputStream valueStream, Writer writer) throws IOException { @@ -849,10 +850,11 @@ private static void readContainerLogs(DataInputStream valueStream, * Keep calling this till you get a {@link EOFException} for getting logs of * all types for a single container. * - * @param valueStream - * @param out - * @param logUploadedTime - * @throws IOException + * @param valueStream the value stream + * @param out the print stream + * @param logUploadedTime the time stamp + * @throws IOException if we can not read the container log by specifying + * the container log type. */ public static void readAContainerLogsForALogType( DataInputStream valueStream, PrintStream out, long logUploadedTime) @@ -864,11 +866,11 @@ public static void readAContainerLogsForALogType( * Keep calling this till you get a {@link EOFException} for getting logs of * all types for a single container for the specific bytes. * - * @param valueStream - * @param out - * @param logUploadedTime - * @param bytes - * @throws IOException + * @param valueStream the value stream + * @param out the output print stream + * @param logUploadedTime the log upload time stamp + * @param bytes the output size of the log + * @throws IOException if we can not read the container log */ public static void readAContainerLogsForALogType( DataInputStream valueStream, PrintStream out, long logUploadedTime, @@ -880,9 +882,9 @@ public static void readAContainerLogsForALogType( * Keep calling this till you get a {@link EOFException} for getting logs of * all types for a single container. * - * @param valueStream - * @param out - * @throws IOException + * @param valueStream the value stream + * @param out the output print stream + * @throws IOException if we can not read the container log */ public static void readAContainerLogsForALogType( DataInputStream valueStream, PrintStream out) @@ -893,11 +895,11 @@ public static void readAContainerLogsForALogType( /** * Keep calling this till you get a {@link EOFException} for getting logs of * the specific types for a single container. - * @param valueStream - * @param out - * @param logUploadedTime - * @param logType - * @throws IOException + * @param valueStream the value stream + * @param out the output print stream + * @param logUploadedTime the log uploaded time stamp + * @param logType the given log type + * @throws IOException if we can not read the container logs */ public static int readContainerLogsForALogType( DataInputStream valueStream, PrintStream out, long logUploadedTime, @@ -909,11 +911,11 @@ public static int readContainerLogsForALogType( /** * Keep calling this till you get a {@link EOFException} for getting logs of * the specific types for a single container. - * @param valueStream - * @param out - * @param logUploadedTime - * @param logType - * @throws IOException + * @param valueStream the value stream + * @param out the output print stream + * @param logUploadedTime the log uploaded time stamp + * @param logType the given log type + * @throws IOException if we can not read the container logs */ public static int readContainerLogsForALogType( DataInputStream valueStream, PrintStream out, long logUploadedTime, diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/LogAggregationUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/LogAggregationUtils.java index e8a28dec223..6d04c29406a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/LogAggregationUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/LogAggregationUtils.java @@ -38,11 +38,11 @@ public class LogAggregationUtils { /** * Constructs the full filename for an application's log file per node. - * @param remoteRootLogDir - * @param appId - * @param user - * @param nodeId - * @param suffix + * @param remoteRootLogDir the aggregated remote root log dir + * @param appId the application Id + * @param user the application owner + * @param nodeId the node id + * @param suffix the log dir suffix * @return the remote log file. */ public static Path getRemoteNodeLogFileForApp(Path remoteRootLogDir, @@ -53,10 +53,10 @@ public static Path getRemoteNodeLogFileForApp(Path remoteRootLogDir, /** * Gets the remote app log dir. - * @param remoteRootLogDir - * @param appId - * @param user - * @param suffix + * @param remoteRootLogDir the aggregated log remote root log dir + * @param appId the application id + * @param user the application owner + * @param suffix the log directory suffix * @return the remote application specific log dir. */ public static Path getRemoteAppLogDir(Path remoteRootLogDir, @@ -67,9 +67,9 @@ public static Path getRemoteAppLogDir(Path remoteRootLogDir, /** * Gets the remote suffixed log dir for the user. - * @param remoteRootLogDir - * @param user - * @param suffix + * @param remoteRootLogDir the aggregated log remote root log dir + * @param user the application owner + * @param suffix the log dir suffix * @return the remote suffixed log dir. */ public static Path getRemoteLogSuffixedDir(Path remoteRootLogDir, @@ -83,8 +83,8 @@ public static Path getRemoteLogSuffixedDir(Path remoteRootLogDir, /** * Gets the remote log user dir. - * @param remoteRootLogDir - * @param user + * @param remoteRootLogDir the aggregated log remote root log dir + * @param user the application owner * @return the remote per user log dir. */ public static Path getRemoteLogUserDir(Path remoteRootLogDir, String user) { @@ -93,7 +93,7 @@ public static Path getRemoteLogUserDir(Path remoteRootLogDir, String user) { /** * Returns the suffix component of the log dir. - * @param conf + * @param conf the configuration * @return the suffix which will be appended to the user log dir. */ public static String getRemoteNodeLogDirSuffix(Configuration conf) { @@ -104,7 +104,7 @@ public static String getRemoteNodeLogDirSuffix(Configuration conf) { /** * Converts a nodeId to a form used in the app log file name. - * @param nodeId + * @param nodeId the nodeId * @return the node string to be used to construct the file name. */ @VisibleForTesting diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/LogCLIHelpers.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/LogCLIHelpers.java index 03acb336521..0068eaeeb9a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/LogCLIHelpers.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/LogCLIHelpers.java @@ -128,8 +128,17 @@ public int dumpAContainerLogsForLogType(ContainerLogsRequest options) @VisibleForTesting public int dumpAContainerLogsForLogType(ContainerLogsRequest options, boolean outputFailure) throws IOException { - boolean foundAnyLogs = this.getFileController(options.getAppId(), - options.getAppOwner()).readAggregatedLogs(options, null); + LogAggregationFileController fc = null; + try { + fc = this.getFileController( + options.getAppId(), options.getAppOwner()); + } catch (IOException ex) { + System.err.println(ex); + } + boolean foundAnyLogs = false; + if (fc != null) { + foundAnyLogs = fc.readAggregatedLogs(options, null); + } if (!foundAnyLogs) { if (outputFailure) { containerLogNotFound(options.getContainerId()); @@ -142,9 +151,17 @@ public int dumpAContainerLogsForLogType(ContainerLogsRequest options, @Private public int dumpAContainerLogsForLogTypeWithoutNodeId( ContainerLogsRequest options) throws IOException { - boolean foundAnyLogs = getFileController(options.getAppId(), - options.getAppOwner()).readAggregatedLogs( - options, null); + LogAggregationFileController fc = null; + try { + fc = this.getFileController( + options.getAppId(), options.getAppOwner()); + } catch (IOException ex) { + System.err.println(ex); + } + boolean foundAnyLogs = false; + if (fc != null) { + foundAnyLogs = fc.readAggregatedLogs(options, null); + } if (!foundAnyLogs) { containerLogNotFound(options.getContainerId()); return -1; @@ -155,9 +172,17 @@ public int dumpAContainerLogsForLogTypeWithoutNodeId( @Private public int dumpAllContainersLogs(ContainerLogsRequest options) throws IOException { - boolean foundAnyLogs = getFileController(options.getAppId(), - options.getAppOwner()).readAggregatedLogs( - options, null); + LogAggregationFileController fc = null; + try { + fc = this.getFileController( + options.getAppId(), options.getAppOwner()); + } catch (IOException ex) { + System.err.println(ex); + } + boolean foundAnyLogs = false; + if (fc != null) { + foundAnyLogs = fc.readAggregatedLogs(options, null); + } if (!foundAnyLogs) { emptyLogDir(LogAggregationUtils.getRemoteAppLogDir( conf, options.getAppId(), options.getAppOwner()) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/LogAggregationFileController.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/LogAggregationFileController.java index 87344a7f68a..aafdb664b39 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/LogAggregationFileController.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/LogAggregationFileController.java @@ -201,6 +201,7 @@ protected void closePrintStream(OutputStream out) { * Output container log. * @param logRequest {@link ContainerLogsRequest} * @param os the output stream + * @return true if we can read the aggregated logs successfully * @throws IOException if we can not access the log file. */ public abstract boolean readAggregatedLogs(ContainerLogsRequest logRequest, @@ -228,9 +229,9 @@ public abstract void renderAggregatedLogsBlock(Block html, /** * Returns the owner of the application. * - * @param the aggregatedLog path. - * @return the application owner. - * @throws IOException + * @param aggregatedLogPath the aggregatedLog path + * @return the application owner + * @throws IOException if we can not get the application owner */ public abstract String getApplicationOwner(Path aggregatedLogPath) throws IOException; @@ -239,9 +240,9 @@ public abstract String getApplicationOwner(Path aggregatedLogPath) * Returns ACLs for the application. An empty map is returned if no ACLs are * found. * - * @param the aggregatedLog path. + * @param aggregatedLogPath the aggregatedLog path. * @return a map of the Application ACLs. - * @throws IOException + * @throws IOException if we can not get the application acls */ public abstract Map getApplicationAcls( Path aggregatedLogPath) throws IOException; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/tfile/LogAggregationTFileController.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/tfile/LogAggregationTFileController.java index d2038e2d2e5..aaed538fc01 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/tfile/LogAggregationTFileController.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/tfile/LogAggregationTFileController.java @@ -172,7 +172,6 @@ public boolean readAggregatedLogs(ContainerLogsRequest logRequest, byte[] buf = new byte[65535]; while (nodeFiles != null && nodeFiles.hasNext()) { final FileStatus thisNodeFile = nodeFiles.next(); - LOG.error(thisNodeFile.getPath().toString()); String nodeName = thisNodeFile.getPath().getName(); if (nodeName.equals(appId + ".har")) { Path p = new Path("har:///"