YARN-5224. Added new web-services /containers/{containerid}/logs & /containers/{containerid}/logs/{filename} and using them in "yarn logs" CLI to get logs of finished containers of a running application. Contributed by Xuan Gong.

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/LogsCLI.java

@@ -20,7 +20,6 @@ package org.apache.hadoop.yarn.client.cli;
 
 import java.io.IOException;
 import java.io.PrintStream;
-import java.io.StringReader;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collection;
@@ -29,9 +28,6 @@ import java.util.List;
 import java.util.Set;
 import java.util.regex.Pattern;
 import javax.ws.rs.core.MediaType;
-import javax.xml.parsers.DocumentBuilder;
-import javax.xml.parsers.DocumentBuilderFactory;
-
 import org.apache.commons.cli.CommandLine;
 import org.apache.commons.cli.CommandLineParser;
 import org.apache.commons.cli.GnuParser;
@@ -71,9 +67,6 @@ import com.sun.jersey.api.client.ClientHandlerException;
 import com.sun.jersey.api.client.ClientResponse;
 import com.sun.jersey.api.client.UniformInterfaceException;
 import com.sun.jersey.api.client.WebResource;
-import org.w3c.dom.Document;
-import org.w3c.dom.NodeList;
-import org.xml.sax.InputSource;
 
 @Public
 @Evolving
@@ -353,23 +346,19 @@ public class LogsCLI extends Configured implements Tool {
           .resource(WebAppUtils.getHttpSchemePrefix(conf) + nodeHttpAddress);
       ClientResponse response =
           webResource.path("ws").path("v1").path("node").path("containers")
-              .path(containerIdStr).accept(MediaType.APPLICATION_XML)
+              .path(containerIdStr).path("logs")
+              .accept(MediaType.APPLICATION_JSON)
               .get(ClientResponse.class);
       if (response.getStatusInfo().getStatusCode() ==
           ClientResponse.Status.OK.getStatusCode()) {
         try {
-          String xml = response.getEntity(String.class);
+          JSONObject json = response.getEntity(JSONObject.class);
-          DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance();
+          JSONArray array = json.getJSONArray("containerLogInfo");
-          DocumentBuilder db = dbf.newDocumentBuilder();
+          for (int i = 0; i < array.length(); i++) {
-          InputSource is = new InputSource();
+            logFiles.add(array.getJSONObject(i).getString("fileName"));
-          is.setCharacterStream(new StringReader(xml));
-          Document dom = db.parse(is);
-          NodeList elements = dom.getElementsByTagName("containerLogFiles");
-          for (int i = 0; i < elements.getLength(); i++) {
-            logFiles.add(elements.item(i).getTextContent());
           }
         } catch (Exception e) {
-          System.err.println("Unable to parse xml from webservice. Error:");
+          System.err.println("Unable to parse json from webservice. Error:");
           System.err.println(e.getMessage());
           throw new IOException(e);
         }
@@ -425,7 +414,8 @@ public class LogsCLI extends Configured implements Tool {
                   + nodeHttpAddress);
           ClientResponse response =
               webResource.path("ws").path("v1").path("node")
-                .path("containerlogs").path(containerIdStr).path(logFile)
+                .path("containers").path(containerIdStr).path("logs")
+                .path(logFile)
                 .queryParam("size", Long.toString(request.getBytes()))
                 .accept(MediaType.TEXT_PLAIN).get(ClientResponse.class);
           out.println(response.getEntity(String.class));

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/AHSWebServices.java

@@ -76,7 +76,7 @@ import com.google.inject.Singleton;
 public class AHSWebServices extends WebServices {
 
   private static final String NM_DOWNLOAD_URI_STR =
-      "/ws/v1/node/containerlogs";
+      "/ws/v1/node/containers";
   private static final Joiner JOINER = Joiner.on("");
   private static final Joiner DOT_JOINER = Joiner.on(". ");
   private final Configuration conf;
@@ -256,7 +256,7 @@ public class AHSWebServices extends WebServices {
     String nodeId = containerInfo.getNodeId();
     if (isRunningState(appInfo.getAppState())) {
       String nodeHttpAddress = containerInfo.getNodeHttpAddress();
-      String uri = "/" + containerId.toString() + "/" + filename;
+      String uri = "/" + containerId.toString() + "/logs/" + filename;
       String resURI = JOINER.join(nodeHttpAddress, NM_DOWNLOAD_URI_STR, uri);
       String query = req.getQueryString();
       if (query != null && !query.isEmpty()) {

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/TestAHSWebServices.java

@@ -711,8 +711,9 @@ public class TestAHSWebServices extends JerseyTestBase {
     String redirectURL = getRedirectURL(requestURI.toString());
     assertTrue(redirectURL != null);
     assertTrue(redirectURL.contains("test:1234"));
-    assertTrue(redirectURL.contains("ws/v1/node/containerlogs"));
+    assertTrue(redirectURL.contains("ws/v1/node/containers"));
     assertTrue(redirectURL.contains(containerId1.toString()));
+    assertTrue(redirectURL.contains("/logs/" + fileName));
     assertTrue(redirectURL.contains("user.name=" + user));
   }
 

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMWebServices.java

@@ -53,9 +53,9 @@ import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Cont
 import org.apache.hadoop.yarn.server.nodemanager.webapp.dao.AppInfo;
 import org.apache.hadoop.yarn.server.nodemanager.webapp.dao.AppsInfo;
 import org.apache.hadoop.yarn.server.nodemanager.webapp.dao.ContainerInfo;
+import org.apache.hadoop.yarn.server.nodemanager.webapp.dao.ContainerLogsInfo;
 import org.apache.hadoop.yarn.server.nodemanager.webapp.dao.ContainersInfo;
 import org.apache.hadoop.yarn.server.nodemanager.webapp.dao.NodeInfo;
-import org.apache.hadoop.yarn.util.ConverterUtils;
 import org.apache.hadoop.yarn.webapp.BadRequestException;
 import org.apache.hadoop.yarn.webapp.NotFoundException;
 import org.apache.hadoop.yarn.webapp.WebApp;
@@ -195,6 +195,68 @@ public class NMWebServices {
 
   }
 
+  /**
+   * Returns log file's name as well as current file size for a container.
+   *
+   * @param hsr
+   *    HttpServletRequest
+   * @param containerIdStr
+   *    The container ID
+   * @return
+   *    The log file's name and current file size
+   */
+  @GET
+  @Path("/containers/{containerid}/logs")
+  @Produces({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML })
+  public ContainerLogsInfo getContainerLogsInfo(@javax.ws.rs.core.Context
+      HttpServletRequest hsr,
+      @PathParam("containerid") String containerIdStr) {
+    ContainerId containerId = null;
+    init();
+    try {
+      containerId = ContainerId.fromString(containerIdStr);
+    } catch (Exception e) {
+      throw new BadRequestException("invalid container id, " + containerIdStr);
+    }
+    try {
+      return new ContainerLogsInfo(this.nmContext, containerId,
+          hsr.getRemoteUser());
+    } catch (YarnException ex) {
+      throw new WebApplicationException(ex);
+    }
+  }
+
+  /**
+   * Returns the contents of a container's log file in plain text.
+   *
+   * Only works for containers that are still in the NodeManager's memory, so
+   * logs are no longer available after the corresponding application is no
+   * longer running.
+   *
+   * @param containerIdStr
+   *    The container ID
+   * @param filename
+   *    The name of the log file
+   * @param format
+   *    The content type
+   * @param size
+   *    the size of the log file
+   * @return
+   *    The contents of the container's log file
+   */
+  @GET
+  @Path("/containers/{containerid}/logs/{filename}")
+  @Produces({ MediaType.TEXT_PLAIN })
+  @Public
+  @Unstable
+  public Response getContainerLogFile(
+      @PathParam("containerid") String containerIdStr,
+      @PathParam("filename") String filename,
+      @QueryParam("format") String format,
+      @QueryParam("size") String size) {
+    return getLogs(containerIdStr, filename, format, size);
+  }
+
   /**
    * Returns the contents of a container's log file in plain text. 
    *

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/dao/ContainerLogsInfo.java

@@ -0,0 +1,112 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.nodemanager.webapp.dao;
+
+import java.io.File;
+import java.util.ArrayList;
+import java.util.List;
+import javax.xml.bind.annotation.XmlAccessType;
+import javax.xml.bind.annotation.XmlAccessorType;
+import javax.xml.bind.annotation.XmlElement;
+import javax.xml.bind.annotation.XmlRootElement;
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.server.nodemanager.Context;
+import org.apache.hadoop.yarn.server.nodemanager.webapp.ContainerLogsUtils;
+
+/**
+ * {@code ContainerLogsInfo} includes the log meta-data of containers.
+ * <p>
+ * The container log meta-data includes details such as:
+ * <ul>
+ *   <li>The filename of the container log.</li>
+ *   <li>The size of the container log.</li>
+ * </ul>
+ */
+
+@XmlRootElement(name = "containerLogsInfo")
+@XmlAccessorType(XmlAccessType.FIELD)
+public class ContainerLogsInfo {
+
+  @XmlElement(name = "containerLogInfo")
+  protected List<ContainerLogInfo> containerLogsInfo;
+
+  //JAXB needs this
+  public ContainerLogsInfo() {}
+
+  public ContainerLogsInfo(final Context nmContext,
+      final ContainerId containerId, String remoteUser)
+      throws YarnException {
+    this.containerLogsInfo = getContainerLogsInfo(
+        containerId, remoteUser, nmContext);
+  }
+
+  public List<ContainerLogInfo> getContainerLogsInfo() {
+    return this.containerLogsInfo;
+  }
+
+  private static List<ContainerLogInfo> getContainerLogsInfo(ContainerId id,
+      String remoteUser, Context nmContext) throws YarnException {
+    List<ContainerLogInfo> logFiles = new ArrayList<ContainerLogInfo>();
+    List<File> logDirs = ContainerLogsUtils.getContainerLogDirs(
+        id, remoteUser, nmContext);
+    for (File containerLogsDir : logDirs) {
+      File[] logs = containerLogsDir.listFiles();
+      if (logs != null) {
+        for (File log : logs) {
+          if (log.isFile()) {
+            ContainerLogInfo logMeta = new ContainerLogInfo(
+                log.getName(), log.length());
+            logFiles.add(logMeta);
+          }
+        }
+      }
+    }
+    return logFiles;
+  }
+
+  private static class ContainerLogInfo {
+    private String fileName;
+    private long fileSize;
+
+    //JAXB needs this
+    public ContainerLogInfo() {}
+
+    public ContainerLogInfo(String fileName, long fileSize) {
+      this.setFileName(fileName);
+      this.setFileSize(fileSize);
+    }
+
+    public String getFileName() {
+      return fileName;
+    }
+
+    public void setFileName(String fileName) {
+      this.fileName = fileName;
+    }
+
+    public long getFileSize() {
+      return fileSize;
+    }
+
+    public void setFileSize(long fileSize) {
+      this.fileSize = fileSize;
+    }
+  }
+}

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServices.java

@@ -311,13 +311,29 @@ public class TestNMWebServices extends JerseyTestBase {
     verifyNodesXML(nodes);
   }
 
-  @Test
+  @Test (timeout = 5000)
-  public void testContainerLogs() throws IOException {
+  public void testContainerLogsWithNewAPI() throws IOException, JSONException{
-    WebResource r = resource();
     final ContainerId containerId = BuilderUtils.newContainerId(0, 0, 0, 0);
-    final String containerIdStr = BuilderUtils.newContainerId(0, 0, 0, 0)
+    WebResource r = resource();
-        .toString();
+    r = r.path("ws").path("v1").path("node").path("containers")
-    final ApplicationAttemptId appAttemptId = containerId.getApplicationAttemptId();
+        .path(containerId.toString()).path("logs");
+    testContainerLogs(r, containerId);
+  }
+
+  @Test (timeout = 5000)
+  public void testContainerLogsWithOldAPI() throws IOException, JSONException{
+    final ContainerId containerId = BuilderUtils.newContainerId(1, 1, 0, 1);
+    WebResource r = resource();
+    r = r.path("ws").path("v1").path("node").path("containerlogs")
+        .path(containerId.toString());
+    testContainerLogs(r, containerId);
+  }
+
+  private void testContainerLogs(WebResource r, ContainerId containerId)
+      throws IOException, JSONException {
+    final String containerIdStr = containerId.toString();
+    final ApplicationAttemptId appAttemptId = containerId
+        .getApplicationAttemptId();
     final ApplicationId appId = appAttemptId.getApplicationId();
     final String appIdStr = appId.toString();
     final String filename = "logfile1";
@@ -343,8 +359,7 @@ public class TestNMWebServices extends JerseyTestBase {
     pw.close();
 
     // ask for it
-    ClientResponse response = r.path("ws").path("v1").path("node")
+    ClientResponse response = r.path(filename)
-        .path("containerlogs").path(containerIdStr).path(filename)
         .accept(MediaType.TEXT_PLAIN).get(ClientResponse.class);
     String responseText = response.getEntity(String.class);
     assertEquals(logMessage, responseText);
@@ -353,8 +368,7 @@ public class TestNMWebServices extends JerseyTestBase {
     // specify how many bytes we should get from logs
     // specify a position number, it would get the first n bytes from
     // container log
-    response = r.path("ws").path("v1").path("node")
+    response = r.path(filename)
-        .path("containerlogs").path(containerIdStr).path(filename)
         .queryParam("size", "5")
         .accept(MediaType.TEXT_PLAIN).get(ClientResponse.class);
     responseText = response.getEntity(String.class);
@@ -364,8 +378,7 @@ public class TestNMWebServices extends JerseyTestBase {
 
     // specify the bytes which is larger than the actual file size,
     // we would get the full logs
-    response = r.path("ws").path("v1").path("node")
+    response = r.path(filename)
-        .path("containerlogs").path(containerIdStr).path(filename)
         .queryParam("size", "10000")
         .accept(MediaType.TEXT_PLAIN).get(ClientResponse.class);
     responseText = response.getEntity(String.class);
@@ -374,8 +387,7 @@ public class TestNMWebServices extends JerseyTestBase {
 
     // specify a negative number, it would get the last n bytes from
     // container log
-    response = r.path("ws").path("v1").path("node")
+    response = r.path(filename)
-        .path("containerlogs").path(containerIdStr).path(filename)
         .queryParam("size", "-5")
         .accept(MediaType.TEXT_PLAIN).get(ClientResponse.class);
     responseText = response.getEntity(String.class);
@@ -384,8 +396,7 @@ public class TestNMWebServices extends JerseyTestBase {
         logMessage.getBytes().length - 5, 5), responseText);
     assertTrue(fullTextSize >= responseText.getBytes().length);
 
-    response = r.path("ws").path("v1").path("node")
+    response = r.path(filename)
-        .path("containerlogs").path(containerIdStr).path(filename)
         .queryParam("size", "-10000")
         .accept(MediaType.TEXT_PLAIN).get(ClientResponse.class);
     responseText = response.getEntity(String.class);
@@ -394,8 +405,7 @@ public class TestNMWebServices extends JerseyTestBase {
     assertEquals(logMessage, responseText);
 
     // ask and download it
-    response = r.path("ws").path("v1").path("node").path("containerlogs")
+    response = r.path(filename)
-        .path(containerIdStr).path(filename)
         .queryParam("format", "octet-stream")
         .accept(MediaType.TEXT_PLAIN).get(ClientResponse.class);
     responseText = response.getEntity(String.class);
@@ -404,8 +414,7 @@ public class TestNMWebServices extends JerseyTestBase {
     assertEquals("application/octet-stream", response.getType().toString());
 
     // specify a invalid format value
-    response = r.path("ws").path("v1").path("node").path("containerlogs")
+    response = r.path(filename)
-        .path(containerIdStr).path(filename)
         .queryParam("format", "123")
         .accept(MediaType.TEXT_PLAIN).get(ClientResponse.class);
     responseText = response.getEntity(String.class);
@@ -414,19 +423,29 @@ public class TestNMWebServices extends JerseyTestBase {
     assertEquals(400, response.getStatus());
 
     // ask for file that doesn't exist
-    response = r.path("ws").path("v1").path("node")
+    response = r.path("uhhh")
-        .path("containerlogs").path(containerIdStr).path("uhhh")
         .accept(MediaType.TEXT_PLAIN).get(ClientResponse.class);
-    assertResponseStatusCode(Status.NOT_FOUND, response.getStatusInfo());
+    assertEquals(Status.NOT_FOUND.getStatusCode(),
+        response.getStatus());
     responseText = response.getEntity(String.class);
     assertTrue(responseText.contains("Cannot find this log on the local disk."));
 
+    // Get container log files' name
+    WebResource r1 = resource();
+    response = r1.path("ws").path("v1").path("node")
+        .path("containers").path(containerIdStr)
+        .path("logs").accept(MediaType.APPLICATION_JSON)
+        .get(ClientResponse.class);
+    assertEquals(200, response.getStatus());
+    JSONObject json = response.getEntity(JSONObject.class);
+    assertEquals(json.getJSONObject("containerLogInfo")
+        .getString("fileName"), filename);
+
     // After container is completed, it is removed from nmContext
     nmContext.getContainers().remove(containerId);
     Assert.assertNull(nmContext.getContainers().get(containerId));
     response =
-        r.path("ws").path("v1").path("node").path("containerlogs")
+        r.path(filename).accept(MediaType.TEXT_PLAIN)
-            .path(containerIdStr).path(filename).accept(MediaType.TEXT_PLAIN)
             .get(ClientResponse.class);
     responseText = response.getEntity(String.class);
     assertEquals(logMessage, responseText);