YARN-2734. Skipped sub-folders in the local log dir when aggregating logs. Contributed by Xuan Gong.

(cherry picked from commit caecd9fffe)

hadoop-yarn-project/CHANGES.txt

@@ -730,6 +730,9 @@ Release 2.6.0 - UNRELEASED
     YARN-2723. Fix rmadmin -replaceLabelsOnNode does not correctly parse port.
     (Naganarasimha G R via xgong)
 
+    YARN-2734. Skipped sub-folders in the local log dir when aggregating logs.
+    (Xuan Gong via zjshen)
+
 Release 2.5.1 - 2014-09-05
 
   INCOMPATIBLE CHANGES

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/AggregatedLogFormat.java

@@ -213,6 +213,12 @@ public void write(DataOutputStream out, Set<File> pendingUploadFiles)
       Collections.sort(fileList);
 
       for (File logFile : fileList) {
+        // We only aggregate top level files.
+        // Ignore anything inside sub-folders.
+        if (logFile.isDirectory()) {
+          LOG.warn(logFile.getAbsolutePath() + " is a directory. Ignore it.");
+          continue;
+        }
 
         FileInputStream in = null;
         try {

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/TestAggregatedLogFormat.java

@@ -194,6 +194,14 @@ public void testReadAcontainerLogs1() throws Exception {
 
     int numChars = 80000;
 
+    // create a sub-folder under srcFilePath
+    // and create file logs in this sub-folder.
+    // We only aggregate top level files.
+    // So, this log file should be ignored.
+    Path subDir = new Path(srcFilePath, "subDir");
+    fs.mkdirs(subDir);
+    writeSrcFile(subDir, "logs", numChars);
+
     // create file stderr and stdout in containerLogDir
     writeSrcFile(srcFilePath, "stderr", numChars);
     writeSrcFile(srcFilePath, "stdout", numChars);
@@ -238,6 +246,7 @@ public void testReadAcontainerLogs1() throws Exception {
             + "\nLog Contents:\n".length() + numChars;
     Assert.assertTrue("LogType not matched", s.contains("LogType:stdout"));
     Assert.assertTrue("log file:stderr should not be aggregated.", !s.contains("LogType:stderr"));
+    Assert.assertTrue("log file:logs should not be aggregated.", !s.contains("LogType:logs"));
     Assert.assertTrue("LogLength not matched", s.contains("LogLength:" + numChars));
     Assert.assertTrue("Log Contents not matched", s.contains("Log Contents"));