From cb3974524c5b265e70d94d0d40b24e2e9224f050 Mon Sep 17 00:00:00 2001 From: Zhijie Shen Date: Sun, 26 Oct 2014 12:57:31 -0700 Subject: [PATCH] YARN-2734. Skipped sub-folders in the local log dir when aggregating logs. Contributed by Xuan Gong. (cherry picked from commit caecd9fffe7c6216be31f3ab65349182045451fa) --- hadoop-yarn-project/CHANGES.txt | 3 +++ .../hadoop/yarn/logaggregation/AggregatedLogFormat.java | 6 ++++++ .../yarn/logaggregation/TestAggregatedLogFormat.java | 9 +++++++++ 3 files changed, 18 insertions(+) diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 0c9e525586d..74d1dffcbf1 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -730,6 +730,9 @@ Release 2.6.0 - UNRELEASED YARN-2723. Fix rmadmin -replaceLabelsOnNode does not correctly parse port. (Naganarasimha G R via xgong) + YARN-2734. Skipped sub-folders in the local log dir when aggregating logs. + (Xuan Gong via zjshen) + Release 2.5.1 - 2014-09-05 INCOMPATIBLE CHANGES diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/AggregatedLogFormat.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/AggregatedLogFormat.java index 035d61de119..22219be1e08 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/AggregatedLogFormat.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/AggregatedLogFormat.java @@ -213,6 +213,12 @@ public void write(DataOutputStream out, Set pendingUploadFiles) Collections.sort(fileList); for (File logFile : fileList) { + // We only aggregate top level files. + // Ignore anything inside sub-folders. + if (logFile.isDirectory()) { + LOG.warn(logFile.getAbsolutePath() + " is a directory. Ignore it."); + continue; + } FileInputStream in = null; try { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/TestAggregatedLogFormat.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/TestAggregatedLogFormat.java index fa9de74a63f..1d607b1572e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/TestAggregatedLogFormat.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/TestAggregatedLogFormat.java @@ -194,6 +194,14 @@ public void testReadAcontainerLogs1() throws Exception { int numChars = 80000; + // create a sub-folder under srcFilePath + // and create file logs in this sub-folder. + // We only aggregate top level files. + // So, this log file should be ignored. + Path subDir = new Path(srcFilePath, "subDir"); + fs.mkdirs(subDir); + writeSrcFile(subDir, "logs", numChars); + // create file stderr and stdout in containerLogDir writeSrcFile(srcFilePath, "stderr", numChars); writeSrcFile(srcFilePath, "stdout", numChars); @@ -238,6 +246,7 @@ public void testReadAcontainerLogs1() throws Exception { + "\nLog Contents:\n".length() + numChars; Assert.assertTrue("LogType not matched", s.contains("LogType:stdout")); Assert.assertTrue("log file:stderr should not be aggregated.", !s.contains("LogType:stderr")); + Assert.assertTrue("log file:logs should not be aggregated.", !s.contains("LogType:logs")); Assert.assertTrue("LogLength not matched", s.contains("LogLength:" + numChars)); Assert.assertTrue("Log Contents not matched", s.contains("Log Contents"));