Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

YARN-5213. Fix a bug in LogCLIHelpers which cause TestLogsCLI#testFetchApplictionLogs fails intermittently. Contributed by Xuan Gong. 

(cherry picked from commit dc2f4b6ac8)
This commit is contained in:
Junping Du 2016-07-19 07:17:58 -07:00
parent 2153c9f04c
commit ad8180fc6c
2 changed files with 46 additions and 31 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

66
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java
View File

@ -329,26 +329,26 @@ public class TestLogsCLI {
int exitCode = cli.run(new String[] { "-applicationId", appId.toString() }); int exitCode = cli.run(new String[] { "-applicationId", appId.toString() });
assertTrue(exitCode == 0); assertTrue(exitCode == 0);
assertTrue(sysOutStream.toString().contains( assertTrue(sysOutStream.toString().contains(
"Hello container_0_0001_01_000001 in syslog!")); logMessage(containerId1, "syslog")));
assertTrue(sysOutStream.toString().contains( assertTrue(sysOutStream.toString().contains(
"Hello container_0_0001_01_000002 in syslog!")); logMessage(containerId2, "syslog")));
assertTrue(sysOutStream.toString().contains( assertTrue(sysOutStream.toString().contains(
"Hello container_0_0001_01_000003 in syslog!")); logMessage(containerId3, "syslog")));
assertTrue(sysOutStream.toString().contains( assertTrue(sysOutStream.toString().contains(
"Hello container_0_0001_01_000003 in stdout!")); logMessage(containerId3, "stdout")));
sysOutStream.reset(); sysOutStream.reset();
exitCode = cli.run(new String[] {"-applicationId", appId.toString(), exitCode = cli.run(new String[] {"-applicationId", appId.toString(),
"-log_files", ".*"}); "-log_files", ".*"});
assertTrue(exitCode == 0); assertTrue(exitCode == 0);
assertTrue(sysOutStream.toString().contains( assertTrue(sysOutStream.toString().contains(
"Hello container_0_0001_01_000001 in syslog!")); logMessage(containerId1, "syslog")));
assertTrue(sysOutStream.toString().contains( assertTrue(sysOutStream.toString().contains(
"Hello container_0_0001_01_000002 in syslog!")); logMessage(containerId2, "syslog")));
assertTrue(sysOutStream.toString().contains( assertTrue(sysOutStream.toString().contains(
"Hello container_0_0001_01_000003 in syslog!")); logMessage(containerId3, "syslog")));
assertTrue(sysOutStream.toString().contains( assertTrue(sysOutStream.toString().contains(
"Hello container_0_0001_01_000003 in stdout!")); logMessage(containerId3, "stdout")));
int fullSize = sysOutStream.toByteArray().length; int fullSize = sysOutStream.toByteArray().length;
sysOutStream.reset(); sysOutStream.reset();
@ -356,13 +356,13 @@ public class TestLogsCLI {
"-log_files", "std*"}); "-log_files", "std*"});
assertTrue(exitCode == 0); assertTrue(exitCode == 0);
assertFalse(sysOutStream.toString().contains( assertFalse(sysOutStream.toString().contains(
"Hello container_0_0001_01_000001 in syslog!")); logMessage(containerId1, "syslog")));
assertFalse(sysOutStream.toString().contains( assertFalse(sysOutStream.toString().contains(
"Hello container_0_0001_01_000002 in syslog!")); logMessage(containerId2, "syslog")));
assertFalse(sysOutStream.toString().contains( assertFalse(sysOutStream.toString().contains(
"Hello container_0_0001_01_000003 in syslog!")); logMessage(containerId3, "syslog")));
assertTrue(sysOutStream.toString().contains( assertTrue(sysOutStream.toString().contains(
"Hello container_0_0001_01_000003 in stdout!")); logMessage(containerId3, "stdout")));
sysOutStream.reset(); sysOutStream.reset();
exitCode = cli.run(new String[] {"-applicationId", appId.toString(), exitCode = cli.run(new String[] {"-applicationId", appId.toString(),
@ -390,7 +390,7 @@ public class TestLogsCLI {
containerId1.toString() }); containerId1.toString() });
assertTrue(exitCode == 0); assertTrue(exitCode == 0);
assertTrue(sysOutStream.toString().contains( assertTrue(sysOutStream.toString().contains(
"Hello container_0_0001_01_000001 in syslog!")); logMessage(containerId1, "syslog")));
assertTrue(sysOutStream.toString().contains("Log Upload Time")); assertTrue(sysOutStream.toString().contains("Log Upload Time"));
assertTrue(!sysOutStream.toString().contains( assertTrue(!sysOutStream.toString().contains(
"Logs for container " + containerId1.toString() "Logs for container " + containerId1.toString()
@ -413,9 +413,9 @@ public class TestLogsCLI {
String fullContext = sysOutStream.toString(); String fullContext = sysOutStream.toString();
sysOutStream.reset(); sysOutStream.reset();
String logMessage = "Hello container_0_0001_01_000003 in stdout!"; String logMessage = logMessage(containerId3, "stdout");
int fileContentSize = logMessage.getBytes().length; int fileContentSize = logMessage.getBytes().length;
int tailContentSize = "\nEnd of LogType:syslog\n\n".getBytes().length; int tailContentSize = "\nEnd of LogType:stdout\n\n".getBytes().length;
// specify how many bytes we should get from logs // specify how many bytes we should get from logs
// specify a position number, it would get the first n bytes from // specify a position number, it would get the first n bytes from
@ -470,9 +470,9 @@ public class TestLogsCLI {
containerId3.toString() }); containerId3.toString() });
assertTrue(exitCode == 0); assertTrue(exitCode == 0);
assertTrue(sysOutStream.toString().contains( assertTrue(sysOutStream.toString().contains(
"Hello container_0_0001_01_000003 in syslog!")); logMessage(containerId3, "syslog")));
assertTrue(sysOutStream.toString().contains( assertTrue(sysOutStream.toString().contains(
"Hello container_0_0001_01_000003 in stdout!")); logMessage(containerId3, "stdout")));
sysOutStream.reset(); sysOutStream.reset();
// set -log_files option as stdout // set -log_files option as stdout
@ -483,9 +483,9 @@ public class TestLogsCLI {
containerId3.toString() , "-log_files", "stdout"}); containerId3.toString() , "-log_files", "stdout"});
assertTrue(exitCode == 0); assertTrue(exitCode == 0);
assertTrue(sysOutStream.toString().contains( assertTrue(sysOutStream.toString().contains(
"Hello container_0_0001_01_000003 in stdout!")); logMessage(containerId3, "stdout")));
assertTrue(!sysOutStream.toString().contains( assertTrue(!sysOutStream.toString().contains(
"Hello container_0_0001_01_000003 in syslog!")); logMessage(containerId3, "syslog")));
sysOutStream.reset(); sysOutStream.reset();
YarnClient mockYarnClientWithException = YarnClient mockYarnClientWithException =
@ -498,9 +498,9 @@ public class TestLogsCLI {
"-containerId", containerId3.toString() }); "-containerId", containerId3.toString() });
assertTrue(exitCode == 0); assertTrue(exitCode == 0);
assertTrue(sysOutStream.toString().contains( assertTrue(sysOutStream.toString().contains(
"Hello container_0_0001_01_000003 in syslog!")); logMessage(containerId3, "syslog")));
assertTrue(sysOutStream.toString().contains( assertTrue(sysOutStream.toString().contains(
"Hello container_0_0001_01_000003 in stdout!")); logMessage(containerId3, "stdout")));
assertTrue(sysOutStream.toString().contains( assertTrue(sysOutStream.toString().contains(
containerId3 + " on " + LogAggregationUtils.getNodeString(nodeId))); containerId3 + " on " + LogAggregationUtils.getNodeString(nodeId)));
sysOutStream.reset(); sysOutStream.reset();
@ -510,9 +510,9 @@ public class TestLogsCLI {
cli.run(new String[] { "-containerId", containerId3.toString() }); cli.run(new String[] { "-containerId", containerId3.toString() });
assertTrue(exitCode == 0); assertTrue(exitCode == 0);
assertTrue(sysOutStream.toString().contains( assertTrue(sysOutStream.toString().contains(
"Hello container_0_0001_01_000003 in syslog!")); logMessage(containerId3, "syslog")));
assertTrue(sysOutStream.toString().contains( assertTrue(sysOutStream.toString().contains(
"Hello container_0_0001_01_000003 in stdout!")); logMessage(containerId3, "stdout")));
assertTrue(sysOutStream.toString().contains( assertTrue(sysOutStream.toString().contains(
containerId3 + " on " + LogAggregationUtils.getNodeString(nodeId))); containerId3 + " on " + LogAggregationUtils.getNodeString(nodeId)));
sysOutStream.reset(); sysOutStream.reset();
@ -662,7 +662,7 @@ public class TestLogsCLI {
"-appOwner", testUser}); "-appOwner", testUser});
assertTrue(exitCode == 0); assertTrue(exitCode == 0);
assertTrue(sysOutStream.toString().contains( assertTrue(sysOutStream.toString().contains(
"Hello " + containerId + " in syslog!")); logMessage(containerId, "syslog")));
sysOutStream.reset(); sysOutStream.reset();
// Verify that we can not get the application logs // Verify that we can not get the application logs
@ -684,8 +684,8 @@ public class TestLogsCLI {
exitCode = cli.run(new String[] { exitCode = cli.run(new String[] {
"-applicationId", appId.toString()}); "-applicationId", appId.toString()});
assertTrue(exitCode == 0); assertTrue(exitCode == 0);
assertTrue(sysOutStream.toString().contains("Hello " assertTrue(sysOutStream.toString().contains(
+ containerId + " in syslog!")); logMessage(containerId, "syslog")));
sysOutStream.reset(); sysOutStream.reset();
// Verify that we could get the err message "Can not find the appOwner" // Verify that we could get the err message "Can not find the appOwner"
@ -847,16 +847,14 @@ public class TestLogsCLI {
assertTrue(container1Dir[0].getPath().getName().equals( assertTrue(container1Dir[0].getPath().getName().equals(
containerId1.toString())); containerId1.toString()));
String container1= readContainerContent(container1Dir[0].getPath(), fs); String container1= readContainerContent(container1Dir[0].getPath(), fs);
assertTrue(container1.contains("Hello " + containerId1 assertTrue(container1.contains(logMessage(containerId1, "syslog")));
+ " in syslog!"));
FileStatus[] container2Dir = fs.listStatus(nodeDir[1].getPath()); FileStatus[] container2Dir = fs.listStatus(nodeDir[1].getPath());
assertTrue(container2Dir.length == 1); assertTrue(container2Dir.length == 1);
assertTrue(container2Dir[0].getPath().getName().equals( assertTrue(container2Dir[0].getPath().getName().equals(
containerId2.toString())); containerId2.toString()));
String container2= readContainerContent(container2Dir[0].getPath(), fs); String container2= readContainerContent(container2Dir[0].getPath(), fs);
assertTrue(container2.contains("Hello " + containerId2 assertTrue(container2.contains(logMessage(containerId2, "syslog")));
+ " in syslog!"));
} finally { } finally {
fs.delete(new Path(remoteLogRootDir), true); fs.delete(new Path(remoteLogRootDir), true);
fs.delete(new Path(rootLogDir), true); fs.delete(new Path(rootLogDir), true);
@ -1139,11 +1137,17 @@ public class TestLogsCLI {
for (String logType : logTypes) { for (String logType : logTypes) {
Writer writer = Writer writer =
new FileWriter(new File(containerLogsDir.toString(), logType)); new FileWriter(new File(containerLogsDir.toString(), logType));
writer.write("Hello " + containerId + " in " + logType + "!"); writer.write(logMessage(containerId, logType));
writer.close(); writer.close();
} }
} }
private static String logMessage(ContainerId containerId, String logType) {
StringBuilder sb = new StringBuilder();
sb.append("Hello " + containerId + " in " + logType + "!");
return sb.toString();
}
private static void uploadContainerLogIntoRemoteDir(UserGroupInformation ugi, private static void uploadContainerLogIntoRemoteDir(UserGroupInformation ugi,
Configuration configuration, List<String> rootLogDirs, NodeId nodeId, Configuration configuration, List<String> rootLogDirs, NodeId nodeId,
ContainerId containerId, Path appDir, FileSystem fs) throws Exception { ContainerId containerId, Path appDir, FileSystem fs) throws Exception {

11
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/LogCLIHelpers.java
View File

@ -157,10 +157,18 @@ public class LogCLIHelpers implements Configurable {
AggregatedLogFormat.LogReader reader = null; AggregatedLogFormat.LogReader reader = null;
PrintStream out = createPrintStream(localDir, fileName, containerId); PrintStream out = createPrintStream(localDir, fileName, containerId);
try { try {
reader = new AggregatedLogFormat.LogReader(getConf(),
thisNodeFile.getPath());
if (getContainerLogsStream(containerId, reader) == null) {
continue;
}
String containerString = String.format(CONTAINER_ON_NODE_PATTERN, String containerString = String.format(CONTAINER_ON_NODE_PATTERN,
containerId, thisNodeFile.getPath().getName()); containerId, thisNodeFile.getPath().getName());
out.println(containerString); out.println(containerString);
out.println(StringUtils.repeat("=", containerString.length())); out.println(StringUtils.repeat("=", containerString.length()));
// We have to re-create reader object to reset the stream index
// after calling getContainerLogsStream which would move the stream
// index to the end of the log file.
reader = reader =
new AggregatedLogFormat.LogReader(getConf(), new AggregatedLogFormat.LogReader(getConf(),
thisNodeFile.getPath()); thisNodeFile.getPath());
@ -220,6 +228,9 @@ public class LogCLIHelpers implements Configurable {
if (getContainerLogsStream(containerId, reader) == null) { if (getContainerLogsStream(containerId, reader) == null) {
continue; continue;
} }
// We have to re-create reader object to reset the stream index
// after calling getContainerLogsStream which would move the stream
// index to the end of the log file.
reader = reader =
new AggregatedLogFormat.LogReader(getConf(), new AggregatedLogFormat.LogReader(getConf(),
thisNodeFile.getPath()); thisNodeFile.getPath());