YARN-2734. Skipped sub-folders in the local log dir when aggregating logs. Contributed by Xuan Gong.

(cherry picked from commit caecd9fffe)
This commit is contained in:
Zhijie Shen 2014-10-26 12:57:31 -07:00
parent 259db7a726
commit ab7a21569c
3 changed files with 18 additions and 0 deletions

View File

@ -694,6 +694,9 @@ Release 2.6.0 - UNRELEASED
YARN-2723. Fix rmadmin -replaceLabelsOnNode does not correctly parse port. YARN-2723. Fix rmadmin -replaceLabelsOnNode does not correctly parse port.
(Naganarasimha G R via xgong) (Naganarasimha G R via xgong)
YARN-2734. Skipped sub-folders in the local log dir when aggregating logs.
(Xuan Gong via zjshen)
Release 2.5.1 - 2014-09-05 Release 2.5.1 - 2014-09-05
INCOMPATIBLE CHANGES INCOMPATIBLE CHANGES

View File

@ -213,6 +213,12 @@ public class AggregatedLogFormat {
Collections.sort(fileList); Collections.sort(fileList);
for (File logFile : fileList) { for (File logFile : fileList) {
// We only aggregate top level files.
// Ignore anything inside sub-folders.
if (logFile.isDirectory()) {
LOG.warn(logFile.getAbsolutePath() + " is a directory. Ignore it.");
continue;
}
FileInputStream in = null; FileInputStream in = null;
try { try {

View File

@ -194,6 +194,14 @@ public class TestAggregatedLogFormat {
int numChars = 80000; int numChars = 80000;
// create a sub-folder under srcFilePath
// and create file logs in this sub-folder.
// We only aggregate top level files.
// So, this log file should be ignored.
Path subDir = new Path(srcFilePath, "subDir");
fs.mkdirs(subDir);
writeSrcFile(subDir, "logs", numChars);
// create file stderr and stdout in containerLogDir // create file stderr and stdout in containerLogDir
writeSrcFile(srcFilePath, "stderr", numChars); writeSrcFile(srcFilePath, "stderr", numChars);
writeSrcFile(srcFilePath, "stdout", numChars); writeSrcFile(srcFilePath, "stdout", numChars);
@ -238,6 +246,7 @@ public class TestAggregatedLogFormat {
+ "\nLog Contents:\n".length() + numChars; + "\nLog Contents:\n".length() + numChars;
Assert.assertTrue("LogType not matched", s.contains("LogType:stdout")); Assert.assertTrue("LogType not matched", s.contains("LogType:stdout"));
Assert.assertTrue("log file:stderr should not be aggregated.", !s.contains("LogType:stderr")); Assert.assertTrue("log file:stderr should not be aggregated.", !s.contains("LogType:stderr"));
Assert.assertTrue("log file:logs should not be aggregated.", !s.contains("LogType:logs"));
Assert.assertTrue("LogLength not matched", s.contains("LogLength:" + numChars)); Assert.assertTrue("LogLength not matched", s.contains("LogLength:" + numChars));
Assert.assertTrue("Log Contents not matched", s.contains("Log Contents")); Assert.assertTrue("Log Contents not matched", s.contains("Log Contents"));