YARN-5200. Enhanced "yarn logs" to be able to get a list of containers whose logs are aggregated via a "show_container_log_info" option. Contributed by Xuan Gong.

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/LogsCLI.java

@@ -78,14 +78,30 @@ public class LogsCLI extends Configured implements Tool {
   private static final String APP_OWNER_OPTION = "appOwner";
   private static final String AM_CONTAINER_OPTION = "am";
   private static final String CONTAINER_LOG_FILES = "logFiles";
-  private static final String SHOW_META_INFO = "show_meta_info";
   private static final String LIST_NODES_OPTION = "list_nodes";
+  private static final String SHOW_APPLICATION_LOG_INFO
+      = "show_application_log_info";
+  private static final String SHOW_CONTAINER_LOG_INFO
+      = "show_container_log_info";
   private static final String OUT_OPTION = "out";
   private static final String SIZE_OPTION = "size";
   public static final String HELP_CMD = "help";
+  private PrintStream outStream = System.out;
+  private YarnClient yarnClient = null;
 
   @Override
   public int run(String[] args) throws Exception {
+    try {
+      yarnClient = createYarnClient();
+      return runCommand(args);
+    } finally {
+      if (yarnClient != null) {
+        yarnClient.close();
+      }
+    }
+  }
+
+  private int runCommand(String[] args) throws Exception {
     Options opts = createCommandOpts();
     Options printOpts = createPrintOpts(opts);
     if (args.length < 1) {
@@ -102,8 +118,9 @@ public class LogsCLI extends Configured implements Tool {
     String nodeAddress = null;
     String appOwner = null;
     boolean getAMContainerLogs = false;
-    boolean showMetaInfo = false;
     boolean nodesList = false;
+    boolean showApplicationLogInfo = false;
+    boolean showContainerLogInfo = false;
     String[] logFiles = null;
     List<String> amContainersList = new ArrayList<String>();
     String localDir = null;
@@ -115,9 +132,11 @@ public class LogsCLI extends Configured implements Tool {
       nodeAddress = commandLine.getOptionValue(NODE_ADDRESS_OPTION);
       appOwner = commandLine.getOptionValue(APP_OWNER_OPTION);
       getAMContainerLogs = commandLine.hasOption(AM_CONTAINER_OPTION);
-      showMetaInfo = commandLine.hasOption(SHOW_META_INFO);
       nodesList = commandLine.hasOption(LIST_NODES_OPTION);
       localDir = commandLine.getOptionValue(OUT_OPTION);
+      showApplicationLogInfo = commandLine.hasOption(
+          SHOW_APPLICATION_LOG_INFO);
+      showContainerLogInfo = commandLine.hasOption(SHOW_CONTAINER_LOG_INFO);
       if (getAMContainerLogs) {
         try {
           amContainersList = parseAMContainer(commandLine, printOpts);
@@ -172,6 +191,12 @@ public class LogsCLI extends Configured implements Tool {
       }
     }
 
+    if (showApplicationLogInfo && showContainerLogInfo) {
+      System.err.println("Invalid options. Can only accept one of "
+          + "show_application_log_info/show_container_log_info.");
+      return -1;
+    }
+
     LogCLIHelpers logCliHelper = new LogCLIHelpers();
     logCliHelper.setConf(getConf());
 
@@ -215,14 +240,17 @@ public class LogsCLI extends Configured implements Tool {
         isApplicationFinished(appState), appOwner, nodeAddress, null,
         containerIdStr, localDir, logs, bytes);
 
-    if (showMetaInfo) {
+    if (showContainerLogInfo) {
-      return showMetaInfo(request, logCliHelper);
+      return showContainerLogInfo(request, logCliHelper);
     }
 
     if (nodesList) {
       return showNodeLists(request, logCliHelper);
     }
 
+    if (showApplicationLogInfo) {
+      return showApplicationLogInfo(request, logCliHelper);
+    }
     // To get am logs
     if (getAMContainerLogs) {
       return fetchAMContainerLogs(request, amContainersList,
@@ -246,21 +274,15 @@ public class LogsCLI extends Configured implements Tool {
 
   private ApplicationReport getApplicationReport(ApplicationId appId)
       throws IOException, YarnException {
-    YarnClient yarnClient = createYarnClient();
+    return yarnClient.getApplicationReport(appId);
-
-    try {
-      return yarnClient.getApplicationReport(appId);
-    } finally {
-      yarnClient.close();
-    }
   }
   
   @VisibleForTesting
   protected YarnClient createYarnClient() {
-    YarnClient yarnClient = YarnClient.createYarnClient();
+    YarnClient client = YarnClient.createYarnClient();
-    yarnClient.init(getConf());
+    client.init(getConf());
-    yarnClient.start();
+    client.start();
-    return yarnClient;
+    return client;
   }
 
   public static void main(String[] args) throws Exception {
@@ -272,7 +294,7 @@ public class LogsCLI extends Configured implements Tool {
   }
 
   private void printHelpMessage(Options options) {
-    System.out.println("Retrieve logs for completed YARN applications.");
+    outStream.println("Retrieve logs for YARN applications.");
     HelpFormatter formatter = new HelpFormatter();
     formatter.printHelp("yarn logs -applicationId <application ID> [OPTIONS]",
         new Options());
@@ -336,9 +358,9 @@ public class LogsCLI extends Configured implements Tool {
     return false;
   }
 
-  private List<String> getContainerLogFiles(Configuration conf,
+  private List<PerLogFileInfo> getContainerLogFiles(Configuration conf,
       String containerIdStr, String nodeHttpAddress) throws IOException {
-    List<String> logFiles = new ArrayList<>();
+    List<PerLogFileInfo> logFileInfos = new ArrayList<>();
     Client webServiceClient = Client.create();
     try {
       WebResource webResource = webServiceClient
@@ -354,7 +376,9 @@ public class LogsCLI extends Configured implements Tool {
           JSONObject json = response.getEntity(JSONObject.class);
           JSONArray array = json.getJSONArray("containerLogInfo");
           for (int i = 0; i < array.length(); i++) {
-            logFiles.add(array.getJSONObject(i).getString("fileName"));
+            String fileName = array.getJSONObject(i).getString("fileName");
+            String fileSize = array.getJSONObject(i).getString("fileSize");
+            logFileInfos.add(new PerLogFileInfo(fileName, fileSize));
           }
         } catch (Exception e) {
           System.err.println("Unable to parse json from webservice. Error:");
@@ -367,7 +391,7 @@ public class LogsCLI extends Configured implements Tool {
       System.err.println("Unable to fetch log files list");
       throw new IOException(ex);
     }
-    return logFiles;
+    return logFileInfos;
   }
 
   @Private
@@ -378,15 +402,26 @@ public class LogsCLI extends Configured implements Tool {
     String containerIdStr = request.getContainerId().toString();
     String localDir = request.getOutputLocalDir();
     String nodeHttpAddress = request.getNodeHttpAddress();
+    if (nodeHttpAddress == null || nodeHttpAddress.isEmpty()) {
+      System.err.println("Can not get the logs for the container: "
+          + containerIdStr);
+      System.err.println("The node http address is required to get container "
+          + "logs for the Running application.");
+      return -1;
+    }
     String nodeId = request.getNodeId();
     PrintStream out = logCliHelper.createPrintStream(localDir, nodeId,
         containerIdStr);
     try {
       // fetch all the log files for the container
       // filter the log files based on the given --logFiles pattern
-      List<String> allLogs=
+      List<PerLogFileInfo> allLogFileInfos=
           getContainerLogFiles(getConf(), containerIdStr, nodeHttpAddress);
-      List<String> matchedFiles = getMatchedLogFiles(request, allLogs);
+      List<String> fileNames = new ArrayList<String>();
+      for (PerLogFileInfo fileInfo : allLogFileInfos) {
+        fileNames.add(fileInfo.getFileName());
+      }
+      List<String> matchedFiles = getMatchedLogFiles(request, fileNames);
       if (matchedFiles.isEmpty()) {
         System.err.println("Can not find any log file matching the pattern: "
             + request.getLogTypes() + " for the container: " + containerIdStr
@@ -397,8 +432,8 @@ public class LogsCLI extends Configured implements Tool {
       newOptions.setLogTypes(matchedFiles);
 
       Client webServiceClient = Client.create();
-      String containerString = "\n\nContainer: " + containerIdStr + " on "
+      String containerString = String.format(
-          + nodeId;
+          LogCLIHelpers.CONTAINER_ON_NODE_PATTERN, containerIdStr, nodeId);
       out.println(containerString);
       out.println(StringUtils.repeat("=", containerString.length()));
       boolean foundAnyLogs = false;
@@ -477,13 +512,8 @@ public class LogsCLI extends Configured implements Tool {
   @VisibleForTesting
   public ContainerReport getContainerReport(String containerIdStr)
       throws YarnException, IOException {
-    YarnClient yarnClient = createYarnClient();
+    return yarnClient.getContainerReport(
-    try {
+        ContainerId.fromString(containerIdStr));
-      return yarnClient.getContainerReport(
-          ContainerId.fromString(containerIdStr));
-    } finally {
-      yarnClient.close();
-    }
   }
 
   private boolean isApplicationFinished(YarnApplicationState appState) {
@@ -508,8 +538,10 @@ public class LogsCLI extends Configured implements Tool {
         for (JSONObject amContainer : amContainersList) {
           ContainerLogsRequest amRequest = new ContainerLogsRequest(request);
           amRequest.setContainerId(amContainer.getString("containerId"));
-          amRequest.setNodeHttpAddress(
+          String httpAddress = amContainer.getString("nodeHttpAddress");
-              amContainer.getString("nodeHttpAddress"));
+          if (httpAddress != null && !httpAddress.isEmpty()) {
+            amRequest.setNodeHttpAddress(httpAddress);
+          }
           amRequest.setNodeId(amContainer.getString("nodeId"));
           requests.add(amRequest);
         }
@@ -545,8 +577,8 @@ public class LogsCLI extends Configured implements Tool {
       for (ContainerLogsRequest amRequest : requests) {
         outputAMContainerLogs(amRequest, conf, logCliHelper);
       }
-      System.out.println();      
+      outStream.println();
-      System.out.println("Specified ALL for -am option. "
+      outStream.println("Specified ALL for -am option. "
           + "Printed logs for all am containers.");
     } else {
       for (String amContainer : amContainers) {
@@ -602,15 +634,13 @@ public class LogsCLI extends Configured implements Tool {
     }
   }
 
-  private int showMetaInfo(ContainerLogsRequest request,
+  private int showContainerLogInfo(ContainerLogsRequest request,
-      LogCLIHelpers logCliHelper) throws IOException {
+      LogCLIHelpers logCliHelper) throws IOException, YarnException {
     if (!request.isAppFinished()) {
-      System.err.println("The -show_meta_info command can be only used "
+      return printContainerInfoFromRunningApplication(request);
-          + "with finished applications");
-      return -1;
     } else {
-      logCliHelper.printLogMetadata(request, System.out, System.err);
+      return logCliHelper.printAContainerLogMetadata(
-      return 0;
+          request, System.out, System.err);
     }
   }
 
@@ -626,6 +656,33 @@ public class LogsCLI extends Configured implements Tool {
     }
   }
 
+  private int showApplicationLogInfo(ContainerLogsRequest request,
+      LogCLIHelpers logCliHelper) throws IOException, YarnException {
+    String appState = "Application State: "
+        + (request.isAppFinished() ? "Completed." : "Running.");
+    if (!request.isAppFinished()) {
+      List<ContainerReport> reports =
+          getContainerReportsFromRunningApplication(request);
+      List<ContainerReport> filterReports = filterContainersInfo(
+          request, reports);
+      if (filterReports.isEmpty()) {
+        System.err.println("Can not find any containers for the application:"
+            + request.getAppId() + ".");
+        return -1;
+      }
+      outStream.println(appState);
+      for (ContainerReport report : filterReports) {
+        outStream.println(String.format(LogCLIHelpers.CONTAINER_ON_NODE_PATTERN,
+            report.getContainerId(), report.getAssignedNode()));
+      }
+      return 0;
+    } else {
+      outStream.println(appState);
+      logCliHelper.printContainersList(request, System.out, System.err);
+      return 0;
+    }
+  }
+
   private Options createCommandOpts() {
     Options opts = new Options();
     opts.addOption(HELP_CMD, false, "Displays help for all commands.");
@@ -661,13 +718,16 @@ public class LogsCLI extends Configured implements Tool {
     logFileOpt.setArgs(Option.UNLIMITED_VALUES);
     logFileOpt.setArgName("Log File Name");
     opts.addOption(logFileOpt);
-    opts.addOption(SHOW_META_INFO, false, "Show the log metadata, "
+    opts.addOption(SHOW_CONTAINER_LOG_INFO, false,
+        "Show the container log metadata, "
         + "including log-file names, the size of the log files. "
         + "You can combine this with --containerId to get log metadata for "
         + "the specific container, or with --nodeAddress to get log metadata "
-        + "for all the containers on the specific NodeManager. "
+        + "for all the containers on the specific NodeManager.");
-        + "Currently, this option can only be used for finished "
+    opts.addOption(SHOW_APPLICATION_LOG_INFO, false, "Show the "
-        + "applications.");
+        + "containerIds which belong to the specific Application. "
+        + "You can combine this with --nodeAddress to get containerIds "
+        + "for all the containers on the specific NodeManager.");
     opts.addOption(LIST_NODES_OPTION, false,
         "Show the list of nodes that successfully aggregated logs. "
         + "This option can only be used with finished applications.");
@@ -695,8 +755,9 @@ public class LogsCLI extends Configured implements Tool {
     printOpts.addOption(commandOpts.getOption(APP_OWNER_OPTION));
     printOpts.addOption(commandOpts.getOption(AM_CONTAINER_OPTION));
     printOpts.addOption(commandOpts.getOption(CONTAINER_LOG_FILES));
-    printOpts.addOption(commandOpts.getOption(SHOW_META_INFO));
     printOpts.addOption(commandOpts.getOption(LIST_NODES_OPTION));
+    printOpts.addOption(commandOpts.getOption(SHOW_APPLICATION_LOG_INFO));
+    printOpts.addOption(commandOpts.getOption(SHOW_CONTAINER_LOG_INFO));
     printOpts.addOption(commandOpts.getOption(OUT_OPTION));
     printOpts.addOption(commandOpts.getOption(SIZE_OPTION));
     return printOpts;
@@ -801,12 +862,14 @@ public class LogsCLI extends Configured implements Tool {
       // the ContainerReport. In the containerReport, we could get
       // nodeAddress and nodeHttpAddress
       ContainerReport report = getContainerReport(containerIdStr);
-      nodeHttpAddress =
+      nodeHttpAddress = report.getNodeHttpAddress();
-          report.getNodeHttpAddress().replaceFirst(
+      if (nodeHttpAddress != null && !nodeHttpAddress.isEmpty()) {
-            WebAppUtils.getHttpSchemePrefix(getConf()), "");
+        nodeHttpAddress = nodeHttpAddress.replaceFirst(
+                WebAppUtils.getHttpSchemePrefix(getConf()), "");
+        request.setNodeHttpAddress(nodeHttpAddress);
+      }
       nodeId = report.getAssignedNode().toString();
       request.setNodeId(nodeId);
-      request.setNodeHttpAddress(nodeHttpAddress);
     } catch (IOException | YarnException ex) {
       if (isAppFinished) {
         return printContainerLogsForFinishedApplicationWithoutNodeId(
@@ -942,32 +1005,140 @@ public class LogsCLI extends Configured implements Tool {
           ContainerLogsRequest options) throws YarnException, IOException {
     List<ContainerLogsRequest> newOptionsList =
         new ArrayList<ContainerLogsRequest>();
-    YarnClient yarnClient = createYarnClient();
+    List<ContainerReport> reports =
-    try {
+        getContainerReportsFromRunningApplication(options);
-      List<ApplicationAttemptReport> attempts =
+    for (ContainerReport container : reports) {
-          yarnClient.getApplicationAttempts(options.getAppId());
+      ContainerLogsRequest newOptions = new ContainerLogsRequest(options);
-      for (ApplicationAttemptReport attempt : attempts) {
+      newOptions.setContainerId(container.getContainerId().toString());
-        List<ContainerReport> containers = yarnClient.getContainers(
+      newOptions.setNodeId(container.getAssignedNode().toString());
-            attempt.getApplicationAttemptId());
+      String httpAddress = container.getNodeHttpAddress();
-        for (ContainerReport container : containers) {
+      if (httpAddress != null && !httpAddress.isEmpty()) {
-          ContainerLogsRequest newOptions = new ContainerLogsRequest(options);
+        newOptions.setNodeHttpAddress(httpAddress
-          newOptions.setContainerId(container.getContainerId().toString());
+            .replaceFirst(WebAppUtils.getHttpSchemePrefix(getConf()), ""));
-          newOptions.setNodeId(container.getAssignedNode().toString());
+      }
-          newOptions.setNodeHttpAddress(container.getNodeHttpAddress()
+      // if we do not specify the value for CONTAINER_LOG_FILES option,
-              .replaceFirst(WebAppUtils.getHttpSchemePrefix(getConf()), ""));
+      // we will only output syslog
-          // if we do not specify the value for CONTAINER_LOG_FILES option,
+      List<String> logFiles = newOptions.getLogTypes();
-          // we will only output syslog
+      if (logFiles == null || logFiles.isEmpty()) {
-          List<String> logFiles = newOptions.getLogTypes();
+        logFiles = Arrays.asList("syslog");
-          if (logFiles == null || logFiles.isEmpty()) {
+        newOptions.setLogTypes(logFiles);
-            logFiles = Arrays.asList("syslog");
+      }
-            newOptions.setLogTypes(logFiles);
+      newOptionsList.add(newOptions);
+    }
+    return newOptionsList;
+  }
+
+  private List<ContainerReport> getContainerReportsFromRunningApplication(
+      ContainerLogsRequest options) throws YarnException, IOException {
+    List<ContainerReport> reports = new ArrayList<ContainerReport>();
+    List<ApplicationAttemptReport> attempts =
+        yarnClient.getApplicationAttempts(options.getAppId());
+    for (ApplicationAttemptReport attempt : attempts) {
+      List<ContainerReport> containers = yarnClient.getContainers(
+          attempt.getApplicationAttemptId());
+      reports.addAll(containers);
+    }
+    return reports;
+  }
+
+  // filter the containerReports based on the nodeId and ContainerId
+  private List<ContainerReport> filterContainersInfo(
+      ContainerLogsRequest options, List<ContainerReport> containers) {
+    List<ContainerReport> filterReports = new ArrayList<ContainerReport>(
+        containers);
+    String nodeId = options.getNodeId();
+    boolean filterBasedOnNodeId = (nodeId != null && !nodeId.isEmpty());
+    String containerId = options.getContainerId();
+    boolean filterBasedOnContainerId = (containerId != null
+        && !containerId.isEmpty());
+
+    if (filterBasedOnNodeId || filterBasedOnContainerId) {
+    // filter the reports based on the containerId and.or nodeId
+      for(ContainerReport report : containers) {
+        if (filterBasedOnContainerId) {
+          if (!report.getContainerId().toString()
+              .equalsIgnoreCase(containerId)) {
+            filterReports.remove(report);
+          }
+        }
+
+        if (filterBasedOnNodeId) {
+          if (!report.getAssignedNode().toString().equalsIgnoreCase(nodeId)) {
+            filterReports.remove(report);
           }
-          newOptionsList.add(newOptions);
         }
       }
-      return newOptionsList;
+    }
-    } finally {
+    return filterReports;
-      yarnClient.close();
+  }
+
+  private int printContainerInfoFromRunningApplication(
+      ContainerLogsRequest options) throws YarnException, IOException {
+    String containerIdStr = options.getContainerId();
+    String nodeIdStr = options.getNodeId();
+    List<ContainerReport> reports =
+        getContainerReportsFromRunningApplication(options);
+    List<ContainerReport> filteredReports = filterContainersInfo(
+        options, reports);
+    if (filteredReports.isEmpty()) {
+      StringBuilder sb = new StringBuilder();
+      if (containerIdStr != null && !containerIdStr.isEmpty()) {
+        sb.append("Trying to get container with ContainerId: "
+            + containerIdStr + "\n");
+      }
+      if (nodeIdStr != null && !nodeIdStr.isEmpty()) {
+        sb.append("Trying to get container from NodeManager: "
+            + nodeIdStr + "\n");
+      }
+      sb.append("Can not find any matched containers for the application: "
+          + options.getAppId());
+      System.err.println(sb.toString());
+      return -1;
+    }
+    for (ContainerReport report : filteredReports) {
+      String nodeId = report.getAssignedNode().toString();
+      String nodeHttpAddress = report.getNodeHttpAddress().replaceFirst(
+          WebAppUtils.getHttpSchemePrefix(getConf()), "");
+      String containerId = report.getContainerId().toString();
+      String containerString = String.format(
+          LogCLIHelpers.CONTAINER_ON_NODE_PATTERN, containerId, nodeId);
+      outStream.println(containerString);
+      outStream.println(StringUtils.repeat("=", containerString.length()));
+      outStream.printf(LogCLIHelpers.PER_LOG_FILE_INFO_PATTERN,
+          "LogType", "LogLength");
+      outStream.println(StringUtils.repeat("=", containerString.length()));
+      List<PerLogFileInfo> infos = getContainerLogFiles(
+          getConf(), containerId, nodeHttpAddress);
+      for (PerLogFileInfo info : infos) {
+        outStream.printf(LogCLIHelpers.PER_LOG_FILE_INFO_PATTERN,
+            info.getFileName(), info.getFileLength());
+      }
+    }
+    return 0;
+  }
+
+  private static class PerLogFileInfo {
+    private String fileName;
+    private String fileLength;
+    public PerLogFileInfo(String fileName, String fileLength) {
+      setFileName(fileName);
+      setFileLength(fileLength);
+    }
+
+    public String getFileName() {
+      return fileName;
+    }
+
+    public void setFileName(String fileName) {
+      this.fileName = fileName;
+    }
+
+    public String getFileLength() {
+      return fileLength;
+    }
+
+    public void setFileLength(String fileLength) {
+      this.fileLength = fileLength;
     }
   }
 }

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java

@@ -182,7 +182,7 @@ public class TestLogsCLI {
     assertTrue(exitCode == -1);
     ByteArrayOutputStream baos = new ByteArrayOutputStream();
     PrintWriter pw = new PrintWriter(baos);
-    pw.println("Retrieve logs for completed YARN applications.");
+    pw.println("Retrieve logs for YARN applications.");
     pw.println("usage: yarn logs -applicationId <application ID> [OPTIONS]");
     pw.println();
     pw.println("general options are:");
@@ -217,14 +217,18 @@ public class TestLogsCLI {
     pw.println("                                 container logs. The container logs will");
     pw.println("                                 be stored based on the node the container");
     pw.println("                                 ran on.");
-    pw.println(" -show_meta_info                 Show the log metadata, including log-file");
+    pw.println(" -show_application_log_info      Show the containerIds which belong to the");
-    pw.println("                                 names, the size of the log files. You can");
+    pw.println("                                 specific Application. You can combine");
-    pw.println("                                 combine this with --containerId to get");
+    pw.println("                                 this with --nodeAddress to get");
-    pw.println("                                 log metadata for the specific container,");
+    pw.println("                                 containerIds for all the containers on");
-    pw.println("                                 or with --nodeAddress to get log metadata");
+    pw.println("                                 the specific NodeManager.");
-    pw.println("                                 for all the containers on the specific");
+    pw.println(" -show_container_log_info        Show the container log metadata,");
-    pw.println("                                 NodeManager. Currently, this option can");
+    pw.println("                                 including log-file names, the size of the");
-    pw.println("                                 only be used for finished applications.");
+    pw.println("                                 log files. You can combine this with");
+    pw.println("                                 --containerId to get log metadata for the");
+    pw.println("                                 specific container, or with --nodeAddress");
+    pw.println("                                 to get log metadata for all the");
+    pw.println("                                 containers on the specific NodeManager.");
     pw.println(" -size <size>                    Prints the log file's first 'n' bytes or");
     pw.println("                                 the last 'n' bytes. Use negative values");
     pw.println("                                 as bytes to read from the end and");
@@ -698,9 +702,8 @@ public class TestLogsCLI {
           "-applicationId", appTest.toString()});
       assertTrue(exitCode == -1);
       assertTrue(sysErrStream.toString().contains(
-        "Guessed logs' owner is " + priorityUser + " and current user "
+          "Can not find the logs for the application: "
-            + UserGroupInformation.getCurrentUser().getUserName()
+          + appTest.toString()));
-            + " does not have permission to access"));
       sysErrStream.reset();
     } finally {
       fs.delete(new Path(remoteLogRootDir), true);
@@ -842,49 +845,77 @@ public class TestLogsCLI {
     LogsCLI cli = new LogsCLIForTest(mockYarnClient);
     cli.setConf(configuration);
 
-    cli.run(new String[] { "-applicationId", appId.toString(),
+    int result = cli.run(new String[] {"-applicationId", appId.toString(),
-        "-show_meta_info" });
+        "-show_container_log_info", "-show_application_log_info"});
+    assertTrue(result == -1);
+    assertTrue(sysErrStream.toString().contains("Invalid options. "
+        + "Can only accept one of show_application_log_info/"
+        + "show_container_log_info."));
+
+    cli.run(new String[] {"-applicationId", appId.toString(),
+        "-show_container_log_info"});
     assertTrue(sysOutStream.toString().contains(
         "Container: container_0_0001_01_000001 on localhost_"));
     assertTrue(sysOutStream.toString().contains(
         "Container: container_0_0001_01_000002 on localhost_"));
     assertTrue(sysOutStream.toString().contains(
-        "LogType:syslog"));
+        "syslog"));
     assertTrue(sysOutStream.toString().contains(
-        "LogLength:43"));
+        "43"));
     sysOutStream.reset();
 
-    cli.run(new String[] { "-applicationId", appId.toString(),
+    cli.run(new String[] {"-applicationId", appId.toString(),
-        "-show_meta_info", "-containerId", "container_0_0001_01_000001" });
+        "-show_container_log_info", "-containerId",
+        "container_0_0001_01_000001"});
     assertTrue(sysOutStream.toString().contains(
         "Container: container_0_0001_01_000001 on localhost_"));
     assertFalse(sysOutStream.toString().contains(
         "Container: container_0_0001_01_000002 on localhost_"));
     assertTrue(sysOutStream.toString().contains(
-        "LogType:syslog"));
+        "syslog"));
     assertTrue(sysOutStream.toString().contains(
-        "LogLength:43"));
+        "43"));
     sysOutStream.reset();
 
-    cli.run(new String[] { "-applicationId", appId.toString(),
+    cli.run(new String[] {"-applicationId", appId.toString(),
-        "-show_meta_info", "-nodeAddress", "localhost" });
+        "-show_container_log_info", "-nodeAddress", "localhost"});
     assertTrue(sysOutStream.toString().contains(
         "Container: container_0_0001_01_000001 on localhost_"));
     assertTrue(sysOutStream.toString().contains(
         "Container: container_0_0001_01_000002 on localhost_"));
     assertTrue(sysOutStream.toString().contains(
-        "LogType:syslog"));
+        "syslog"));
     assertTrue(sysOutStream.toString().contains(
-        "LogLength:43"));
+        "43"));
     sysOutStream.reset();
 
-    cli.run(new String[] { "-applicationId", appId.toString(),
+    cli.run(new String[] {"-applicationId", appId.toString(),
-        "-show_meta_info", "-nodeAddress", "localhost", "-containerId",
+        "-show_container_log_info", "-nodeAddress", "localhost",
-        "container_1234" });
+        "-containerId", "container_1234"});
     assertTrue(sysErrStream.toString().contains(
         "Invalid ContainerId specified"));
     sysErrStream.reset();
 
+    cli.run(new String[] {"-applicationId", appId.toString(),
+        "-show_application_log_info"});
+    assertTrue(sysOutStream.toString().contains(
+        "Application State: Completed."));
+    assertTrue(sysOutStream.toString().contains(
+        "container_0_0001_01_000001 on localhost"));
+    assertTrue(sysOutStream.toString().contains(
+        "container_0_0001_01_000002 on localhost"));
+    sysOutStream.reset();
+
+    cli.run(new String[] {"-applicationId", appId.toString(),
+        "-show_application_log_info", "-nodeAddress", "localhost"});
+    assertTrue(sysOutStream.toString().contains(
+        "Application State: Completed."));
+    assertTrue(sysOutStream.toString().contains(
+        "container_0_0001_01_000001 on localhost"));
+    assertTrue(sysOutStream.toString().contains(
+        "container_0_0001_01_000002 on localhost"));
+    sysOutStream.reset();
+
     fs.delete(new Path(remoteLogRootDir), true);
     fs.delete(new Path(rootLogDir), true);
   }

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/AggregatedLogFormat.java

@@ -49,6 +49,7 @@ import org.apache.commons.io.input.BoundedInputStream;
 import org.apache.commons.io.output.WriterOutputStream;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
+import org.apache.commons.math3.util.Pair;
 import org.apache.hadoop.classification.InterfaceAudience.Private;
 import org.apache.hadoop.classification.InterfaceAudience.Public;
 import org.apache.hadoop.classification.InterfaceStability.Evolving;
@@ -959,25 +960,21 @@ public class AggregatedLogFormat {
     }
 
     @Private
-    public static String readContainerMetaDataAndSkipData(
+    public static Pair<String, String> readContainerMetaDataAndSkipData(
         DataInputStream valueStream, PrintStream out) throws IOException {
 
       String fileType = valueStream.readUTF();
       String fileLengthStr = valueStream.readUTF();
       long fileLength = Long.parseLong(fileLengthStr);
-      if (out != null) {
+      Pair<String, String> logMeta = new Pair<String, String>(
-        out.print("LogType:");
+          fileType, fileLengthStr);
-        out.println(fileType);
-        out.print("LogLength:");
-        out.println(fileLengthStr);
-      }
       long totalSkipped = 0;
       long currSkipped = 0;
       while (currSkipped != -1 && totalSkipped < fileLength) {
         currSkipped = valueStream.skip(fileLength - totalSkipped);
         totalSkipped += currSkipped;
       }
-      return fileType;
+      return logMeta;
     }
 
     public void close() {

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/LogCLIHelpers.java

@@ -32,6 +32,7 @@ import java.util.List;
 import java.util.Set;
 import org.apache.commons.io.IOUtils;
 import org.apache.commons.lang.StringUtils;
+import org.apache.commons.math3.util.Pair;
 import org.apache.hadoop.classification.InterfaceAudience.Private;
 import org.apache.hadoop.conf.Configurable;
 import org.apache.hadoop.conf.Configuration;
@@ -50,6 +51,11 @@ import com.google.common.annotations.VisibleForTesting;
 
 public class LogCLIHelpers implements Configurable {
 
+  public static final String PER_LOG_FILE_INFO_PATTERN =
+      "%20s\t%20s" + System.getProperty("line.separator");
+  public static final String CONTAINER_ON_NODE_PATTERN =
+      "Container: %s on %s";
+
   private Configuration conf;
 
   @Private
@@ -151,8 +157,8 @@ public class LogCLIHelpers implements Configurable {
         AggregatedLogFormat.LogReader reader = null;
         PrintStream out = createPrintStream(localDir, fileName, containerId);
         try {
-          String containerString = "\n\nContainer: " + containerId + " on "
+          String containerString = String.format(CONTAINER_ON_NODE_PATTERN,
-              + thisNodeFile.getPath().getName();
+              containerId, thisNodeFile.getPath().getName());
           out.println(containerString);
           out.println(StringUtils.repeat("=", containerString.length()));
           reader =
@@ -219,8 +225,10 @@ public class LogCLIHelpers implements Configurable {
               thisNodeFile.getPath());
           out = createPrintStream(localDir, thisNodeFile.getPath().getName(),
               containerId);
-          out.println(containerId + " on " + thisNodeFile.getPath().getName());
+          String containerString = String.format(CONTAINER_ON_NODE_PATTERN,
-          out.println(StringUtils.repeat("=", containerId.length()));
+              containerId, thisNodeFile.getPath().getName());
+          out.println(containerString);
+          out.println(StringUtils.repeat("=", containerString.length()));
           if (logType == null || logType.isEmpty()) {
             if (dumpAContainerLogs(containerId, reader, out,
                 thisNodeFile.getModificationTime(), options.getBytes()) > -1) {
@@ -355,9 +363,9 @@ public class LogCLIHelpers implements Configurable {
             PrintStream out = createPrintStream(localDir,
                 thisNodeFile.getPath().getName(), key.toString());
             try {
-              String containerString =
+              String containerString = String.format(
-                  "\n\nContainer: " + key + " on "
+                  CONTAINER_ON_NODE_PATTERN, key,
-                  + thisNodeFile.getPath().getName();
+                  thisNodeFile.getPath().getName());
               out.println(containerString);
               out.println(StringUtils.repeat("=", containerString.length()));
               while (true) {
@@ -400,7 +408,7 @@ public class LogCLIHelpers implements Configurable {
   }
 
   @Private
-  public void printLogMetadata(ContainerLogsRequest options,
+  public int printAContainerLogMetadata(ContainerLogsRequest options,
       PrintStream out, PrintStream err)
       throws IOException {
     ApplicationId appId = options.getAppId();
@@ -413,7 +421,7 @@ public class LogCLIHelpers implements Configurable {
     RemoteIterator<FileStatus> nodeFiles = getRemoteNodeFileDir(
         appId, appOwner);
     if (nodeFiles == null) {
-      return;
+      return -1;
     }
     boolean foundAnyLogs = false;
     while (nodeFiles.hasNext()) {
@@ -434,16 +442,21 @@ public class LogCLIHelpers implements Configurable {
           valueStream = reader.next(key);
           while (valueStream != null) {
             if (getAllContainers || (key.toString().equals(containerIdStr))) {
-              String containerString =
+              String containerString = String.format(CONTAINER_ON_NODE_PATTERN,
-                  "\n\nContainer: " + key + " on "
+                  key, thisNodeFile.getPath().getName());
-                  + thisNodeFile.getPath().getName();
               out.println(containerString);
               out.println("Log Upload Time:"
                   + thisNodeFile.getModificationTime());
               out.println(StringUtils.repeat("=", containerString.length()));
+              out.printf(PER_LOG_FILE_INFO_PATTERN, "LogType", "LogLength");
+              out.println(StringUtils.repeat("=", containerString.length()));
               while (true) {
                 try {
-                  LogReader.readContainerMetaDataAndSkipData(valueStream, out);
+                  Pair<String, String> logMeta =
+                      LogReader.readContainerMetaDataAndSkipData(
+                          valueStream, out);
+                  out.printf(PER_LOG_FILE_INFO_PATTERN,
+                      logMeta.getFirst(), logMeta.getSecond());
                 } catch (EOFException eof) {
                   break;
                 }
@@ -473,7 +486,9 @@ public class LogCLIHelpers implements Configurable {
         err.println("Can not find log metadata for container: "
             + containerIdStr);
       }
+      return -1;
     }
+    return 0;
   }
 
   @Private
@@ -501,6 +516,60 @@ public class LogCLIHelpers implements Configurable {
     }
   }
 
+  @Private
+  public void printContainersList(ContainerLogsRequest options,
+      PrintStream out, PrintStream err) throws IOException {
+    ApplicationId appId = options.getAppId();
+    String appOwner = options.getAppOwner();
+    String nodeId = options.getNodeId();
+    String nodeIdStr = (nodeId == null) ? null
+        : LogAggregationUtils.getNodeString(nodeId);
+    RemoteIterator<FileStatus> nodeFiles = getRemoteNodeFileDir(
+        appId, appOwner);
+    if (nodeFiles == null) {
+      return;
+    }
+    boolean foundAnyLogs = false;
+    while (nodeFiles.hasNext()) {
+      FileStatus thisNodeFile = nodeFiles.next();
+      if (nodeIdStr != null) {
+        if (!thisNodeFile.getPath().getName().contains(nodeIdStr)) {
+          continue;
+        }
+      }
+      if (!thisNodeFile.getPath().getName()
+          .endsWith(LogAggregationUtils.TMP_FILE_SUFFIX)) {
+        AggregatedLogFormat.LogReader reader =
+            new AggregatedLogFormat.LogReader(getConf(),
+            thisNodeFile.getPath());
+        try {
+          DataInputStream valueStream;
+          LogKey key = new LogKey();
+          valueStream = reader.next(key);
+          while (valueStream != null) {
+            out.println(String.format(CONTAINER_ON_NODE_PATTERN, key,
+                thisNodeFile.getPath().getName()));
+            foundAnyLogs = true;
+            // Next container
+            key = new LogKey();
+            valueStream = reader.next(key);
+          }
+        } finally {
+          reader.close();
+        }
+      }
+    }
+    if (!foundAnyLogs) {
+      if (nodeId != null) {
+        err.println("Can not find information for any containers on "
+            + nodeId);
+      } else {
+        err.println("Can not find any container information for "
+            + "the application: " + appId);
+      }
+    }
+  }
+
   private RemoteIterator<FileStatus> getRemoteNodeFileDir(ApplicationId appId,
       String appOwner) throws IOException {
     Path remoteAppLogDir = getRemoteAppLogDir(appId, appOwner);
@@ -621,7 +690,7 @@ public class LogCLIHelpers implements Configurable {
               while (true) {
                 try {
                   String logFile = LogReader.readContainerMetaDataAndSkipData(
-                      valueStream, null);
+                      valueStream, null).getFirst();
                   logTypes.add(logFile);
                 } catch (EOFException eof) {
                   break;