From 6afa044dda511f603d7646e7e1252b3a04391292 Mon Sep 17 00:00:00 2001 From: Junping Du Date: Fri, 17 Jun 2016 08:24:24 -0700 Subject: [PATCH] YARN-5251. Yarn CLI to obtain App logs for last 'n' bytes fails. Contributed by Xuan Gong. (cherry picked from commit c35fa4a0e57b99415e284c7c2f3b1a0bebc610c2) --- .../hadoop/yarn/client/cli/TestLogsCLI.java | 2 +- .../logaggregation/AggregatedLogFormat.java | 26 ++++++++++--------- .../TestAggregatedLogFormat.java | 2 +- .../webapp/AHSWebServices.java | 11 ++++---- .../nodemanager/webapp/NMWebServices.java | 9 +++---- .../TestLogAggregationService.java | 2 +- 6 files changed, 27 insertions(+), 25 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java index df1c6b7dd07..6683fc75c1a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java @@ -400,7 +400,7 @@ public ContainerReport getContainerReport(String containerIdStr) String logMessage = "Hello container_0_0001_01_000003 in stdout!"; int fileContentSize = logMessage.getBytes().length; - int tailContentSize = "End of LogType:syslog\n\n".getBytes().length; + int tailContentSize = "\nEnd of LogType:syslog\n\n".getBytes().length; // specify how many bytes we should get from logs // specify a position number, it would get the first n bytes from diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/AggregatedLogFormat.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/AggregatedLogFormat.java index 2fb18ed03dc..7508dd5bb09 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/AggregatedLogFormat.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/AggregatedLogFormat.java @@ -787,20 +787,19 @@ private static void readContainerLogs(DataInputStream valueStream, long toSkip = 0; long totalBytesToRead = fileLength; + long skipAfterRead = 0; if (bytes < 0) { long absBytes = Math.abs(bytes); if (absBytes < fileLength) { toSkip = fileLength - absBytes; totalBytesToRead = absBytes; } - long skippedBytes = valueStream.skip(toSkip); - if (skippedBytes != toSkip) { - throw new IOException("The bytes were skipped are " - + "different from the caller requested"); - } + org.apache.hadoop.io.IOUtils.skipFully( + valueStream, toSkip); } else { if (bytes < fileLength) { totalBytesToRead = bytes; + skipAfterRead = fileLength - bytes; } } @@ -818,7 +817,9 @@ private static void readContainerLogs(DataInputStream valueStream, pendingRead > buf.length ? buf.length : (int) pendingRead; len = valueStream.read(buf, 0, toRead); } - out.println("End of LogType:" + fileType); + org.apache.hadoop.io.IOUtils.skipFully( + valueStream, skipAfterRead); + out.println("\nEnd of LogType:" + fileType); out.println(""); } @@ -913,20 +914,19 @@ public static int readContainerLogsForALogType( long toSkip = 0; long totalBytesToRead = fileLength; + long skipAfterRead = 0; if (bytes < 0) { long absBytes = Math.abs(bytes); if (absBytes < fileLength) { toSkip = fileLength - absBytes; totalBytesToRead = absBytes; } - long skippedBytes = valueStream.skip(toSkip); - if (skippedBytes != toSkip) { - throw new IOException("The bytes were skipped are " - + "different from the caller requested"); - } + org.apache.hadoop.io.IOUtils.skipFully( + valueStream, toSkip); } else { if (bytes < fileLength) { totalBytesToRead = bytes; + skipAfterRead = fileLength - bytes; } } @@ -942,7 +942,9 @@ public static int readContainerLogsForALogType( toRead = pendingRead > buf.length ? buf.length : (int) pendingRead; len = valueStream.read(buf, 0, toRead); } - out.println("End of LogType:" + fileType); + org.apache.hadoop.io.IOUtils.skipFully( + valueStream, skipAfterRead); + out.println("\nEnd of LogType:" + fileType); out.println(""); return 0; } else { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/TestAggregatedLogFormat.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/TestAggregatedLogFormat.java index 21e1655ccea..45dd8abca1f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/TestAggregatedLogFormat.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/TestAggregatedLogFormat.java @@ -258,7 +258,7 @@ private void testReadAcontainerLog(boolean logUploadedTime) throws Exception { .currentTimeMillis())).length() : 0) + ("\nLogLength:" + numChars).length() + "\nLog Contents:\n".length() + numChars + "\n".length() - + "End of LogType:stdout\n".length(); + + "\nEnd of LogType:stdout\n".length(); Assert.assertTrue("LogType not matched", s.contains("LogType:stdout")); Assert.assertTrue("log file:stderr should not be aggregated.", !s.contains("LogType:stderr")); Assert.assertTrue("log file:logs should not be aggregated.", !s.contains("LogType:logs")); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/AHSWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/AHSWebServices.java index 692b1720c89..b56fae971e0 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/AHSWebServices.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/AHSWebServices.java @@ -404,20 +404,19 @@ public void write(OutputStream os) throws IOException, long toSkip = 0; long totalBytesToRead = fileLength; + long skipAfterRead = 0; if (bytes < 0) { long absBytes = Math.abs(bytes); if (absBytes < fileLength) { toSkip = fileLength - absBytes; totalBytesToRead = absBytes; } - long skippedBytes = valueStream.skip(toSkip); - if (skippedBytes != toSkip) { - throw new IOException("The bytes were skipped are " - + "different from the caller requested"); - } + org.apache.hadoop.io.IOUtils.skipFully( + valueStream, toSkip); } else { if (bytes < fileLength) { totalBytesToRead = bytes; + skipAfterRead = fileLength - bytes; } } @@ -435,6 +434,8 @@ public void write(OutputStream os) throws IOException, : (int) pendingRead; len = valueStream.read(buf, 0, toRead); } + org.apache.hadoop.io.IOUtils.skipFully( + valueStream, skipAfterRead); sb = new StringBuilder(); sb.append("\nEnd of LogType:" + fileType + "\n"); b = sb.toString().getBytes(Charset.forName("UTF-8")); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMWebServices.java index 3a30392ee5e..4e38e906b51 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMWebServices.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMWebServices.java @@ -264,20 +264,18 @@ public void write(OutputStream os) throws IOException, byte[] buf = new byte[bufferSize]; long toSkip = 0; long totalBytesToRead = fileLength; + long skipAfterRead = 0; if (bytes < 0) { long absBytes = Math.abs(bytes); if (absBytes < fileLength) { toSkip = fileLength - absBytes; totalBytesToRead = absBytes; } - long skippedBytes = fis.skip(toSkip); - if (skippedBytes != toSkip) { - throw new IOException("The bytes were skipped are different " - + "from the caller requested"); - } + org.apache.hadoop.io.IOUtils.skipFully(fis, toSkip); } else { if (bytes < fileLength) { totalBytesToRead = bytes; + skipAfterRead = fileLength - bytes; } } @@ -295,6 +293,7 @@ public void write(OutputStream os) throws IOException, : (int) pendingRead; len = fis.read(buf, 0, toRead); } + org.apache.hadoop.io.IOUtils.skipFully(fis, skipAfterRead); os.flush(); } finally { IOUtils.closeQuietly(fis); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java index c98e3664bda..1edb841f116 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java @@ -948,7 +948,7 @@ private LogFileStatusInLastCycle verifyContainerLogs( Assert.assertEquals(numOfLogsPerContainer, thisContainerMap.size()); for (String fileType : logFiles) { String expectedValue = - containerStr + " Hello " + fileType + "!End of LogType:" + containerStr + " Hello " + fileType + "!\nEnd of LogType:" + fileType; LOG.info("Expected log-content : " + new String(expectedValue)); String foundValue = thisContainerMap.remove(fileType);