diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/LogsCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/LogsCLI.java index ad4964af6d3..4a2dac0f3fb 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/LogsCLI.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/LogsCLI.java @@ -20,7 +20,6 @@ package org.apache.hadoop.yarn.client.cli; import java.io.IOException; import java.io.PrintStream; -import java.io.StringReader; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; @@ -29,9 +28,6 @@ import java.util.List; import java.util.Set; import java.util.regex.Pattern; import javax.ws.rs.core.MediaType; -import javax.xml.parsers.DocumentBuilder; -import javax.xml.parsers.DocumentBuilderFactory; - import org.apache.commons.cli.CommandLine; import org.apache.commons.cli.CommandLineParser; import org.apache.commons.cli.GnuParser; @@ -71,9 +67,6 @@ import com.sun.jersey.api.client.ClientHandlerException; import com.sun.jersey.api.client.ClientResponse; import com.sun.jersey.api.client.UniformInterfaceException; import com.sun.jersey.api.client.WebResource; -import org.w3c.dom.Document; -import org.w3c.dom.NodeList; -import org.xml.sax.InputSource; @Public @Evolving @@ -353,22 +346,20 @@ public class LogsCLI extends Configured implements Tool { .resource(WebAppUtils.getHttpSchemePrefix(conf) + nodeHttpAddress); ClientResponse response = webResource.path("ws").path("v1").path("node").path("containers") - .path(containerIdStr).accept(MediaType.APPLICATION_XML) + .path(containerIdStr).path("logs") + .accept(MediaType.APPLICATION_JSON) .get(ClientResponse.class); - if (response.getClientResponseStatus().equals(ClientResponse.Status.OK)) { + if (response.getClientResponseStatus().equals( + ClientResponse.Status.OK)) { try { - String xml = response.getEntity(String.class); - DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance(); - DocumentBuilder db = dbf.newDocumentBuilder(); - InputSource is = new InputSource(); - is.setCharacterStream(new StringReader(xml)); - Document dom = db.parse(is); - NodeList elements = dom.getElementsByTagName("containerLogFiles"); - for (int i = 0; i < elements.getLength(); i++) { - logFiles.add(elements.item(i).getTextContent()); + JSONObject json = + response.getEntity(JSONObject.class); + JSONArray array = json.getJSONArray("containerLogInfo"); + for (int i = 0; i < array.length(); i++) { + logFiles.add(array.getJSONObject(i).getString("fileName")); } } catch (Exception e) { - System.err.println("Unable to parse xml from webservice. Error:"); + System.err.println("Unable to parse json from webservice. Error:"); System.err.println(e.getMessage()); throw new IOException(e); } @@ -424,7 +415,8 @@ public class LogsCLI extends Configured implements Tool { + nodeHttpAddress); ClientResponse response = webResource.path("ws").path("v1").path("node") - .path("containerlogs").path(containerIdStr).path(logFile) + .path("containers").path(containerIdStr).path("logs") + .path(logFile) .queryParam("size", Long.toString(request.getBytes())) .accept(MediaType.TEXT_PLAIN).get(ClientResponse.class); out.println(response.getEntity(String.class)); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/AHSWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/AHSWebServices.java index b56fae971e0..04fe7bc006f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/AHSWebServices.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/AHSWebServices.java @@ -76,7 +76,7 @@ import com.google.inject.Singleton; public class AHSWebServices extends WebServices { private static final String NM_DOWNLOAD_URI_STR = - "/ws/v1/node/containerlogs"; + "/ws/v1/node/containers"; private static final Joiner JOINER = Joiner.on(""); private static final Joiner DOT_JOINER = Joiner.on(". "); private final Configuration conf; @@ -256,7 +256,7 @@ public class AHSWebServices extends WebServices { String nodeId = containerInfo.getNodeId(); if (isRunningState(appInfo.getAppState())) { String nodeHttpAddress = containerInfo.getNodeHttpAddress(); - String uri = "/" + containerId.toString() + "/" + filename; + String uri = "/" + containerId.toString() + "/logs/" + filename; String resURI = JOINER.join(nodeHttpAddress, NM_DOWNLOAD_URI_STR, uri); String query = req.getQueryString(); if (query != null && !query.isEmpty()) { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/TestAHSWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/TestAHSWebServices.java index 71b02757c8a..8721a1db6f5 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/TestAHSWebServices.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/TestAHSWebServices.java @@ -715,8 +715,9 @@ public class TestAHSWebServices extends JerseyTestBase { String redirectURL = getRedirectURL(requestURI.toString()); assertTrue(redirectURL != null); assertTrue(redirectURL.contains("test:1234")); - assertTrue(redirectURL.contains("ws/v1/node/containerlogs")); + assertTrue(redirectURL.contains("ws/v1/node/containers")); assertTrue(redirectURL.contains(containerId1.toString())); + assertTrue(redirectURL.contains("/logs/" + fileName)); assertTrue(redirectURL.contains("user.name=" + user)); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMWebServices.java index 4e38e906b51..afbf0d4cfed 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMWebServices.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMWebServices.java @@ -53,9 +53,9 @@ import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Cont import org.apache.hadoop.yarn.server.nodemanager.webapp.dao.AppInfo; import org.apache.hadoop.yarn.server.nodemanager.webapp.dao.AppsInfo; import org.apache.hadoop.yarn.server.nodemanager.webapp.dao.ContainerInfo; +import org.apache.hadoop.yarn.server.nodemanager.webapp.dao.ContainerLogsInfo; import org.apache.hadoop.yarn.server.nodemanager.webapp.dao.ContainersInfo; import org.apache.hadoop.yarn.server.nodemanager.webapp.dao.NodeInfo; -import org.apache.hadoop.yarn.util.ConverterUtils; import org.apache.hadoop.yarn.webapp.BadRequestException; import org.apache.hadoop.yarn.webapp.NotFoundException; import org.apache.hadoop.yarn.webapp.WebApp; @@ -194,7 +194,69 @@ public class NMWebServices { .toString(), webapp.name(), hsr.getRemoteUser()); } - + + /** + * Returns log file's name as well as current file size for a container. + * + * @param hsr + * HttpServletRequest + * @param containerIdStr + * The container ID + * @return + * The log file's name and current file size + */ + @GET + @Path("/containers/{containerid}/logs") + @Produces({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML }) + public ContainerLogsInfo getContainerLogsInfo(@javax.ws.rs.core.Context + HttpServletRequest hsr, + @PathParam("containerid") String containerIdStr) { + ContainerId containerId = null; + init(); + try { + containerId = ContainerId.fromString(containerIdStr); + } catch (Exception e) { + throw new BadRequestException("invalid container id, " + containerIdStr); + } + try { + return new ContainerLogsInfo(this.nmContext, containerId, + hsr.getRemoteUser()); + } catch (YarnException ex) { + throw new WebApplicationException(ex); + } + } + + /** + * Returns the contents of a container's log file in plain text. + * + * Only works for containers that are still in the NodeManager's memory, so + * logs are no longer available after the corresponding application is no + * longer running. + * + * @param containerIdStr + * The container ID + * @param filename + * The name of the log file + * @param format + * The content type + * @param size + * the size of the log file + * @return + * The contents of the container's log file + */ + @GET + @Path("/containers/{containerid}/logs/{filename}") + @Produces({ MediaType.TEXT_PLAIN }) + @Public + @Unstable + public Response getContainerLogFile( + @PathParam("containerid") String containerIdStr, + @PathParam("filename") String filename, + @QueryParam("format") String format, + @QueryParam("size") String size) { + return getLogs(containerIdStr, filename, format, size); + } + /** * Returns the contents of a container's log file in plain text. * diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/dao/ContainerLogsInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/dao/ContainerLogsInfo.java new file mode 100644 index 00000000000..a986e28e5f0 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/dao/ContainerLogsInfo.java @@ -0,0 +1,112 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.nodemanager.webapp.dao; + +import java.io.File; +import java.util.ArrayList; +import java.util.List; +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlElement; +import javax.xml.bind.annotation.XmlRootElement; +import org.apache.hadoop.yarn.api.records.ContainerId; +import org.apache.hadoop.yarn.exceptions.YarnException; +import org.apache.hadoop.yarn.server.nodemanager.Context; +import org.apache.hadoop.yarn.server.nodemanager.webapp.ContainerLogsUtils; + +/** + * {@code ContainerLogsInfo} includes the log meta-data of containers. + *

+ * The container log meta-data includes details such as: + *

+ */ + +@XmlRootElement(name = "containerLogsInfo") +@XmlAccessorType(XmlAccessType.FIELD) +public class ContainerLogsInfo { + + @XmlElement(name = "containerLogInfo") + protected List containerLogsInfo; + + //JAXB needs this + public ContainerLogsInfo() {} + + public ContainerLogsInfo(final Context nmContext, + final ContainerId containerId, String remoteUser) + throws YarnException { + this.containerLogsInfo = getContainerLogsInfo( + containerId, remoteUser, nmContext); + } + + public List getContainerLogsInfo() { + return this.containerLogsInfo; + } + + private static List getContainerLogsInfo(ContainerId id, + String remoteUser, Context nmContext) throws YarnException { + List logFiles = new ArrayList(); + List logDirs = ContainerLogsUtils.getContainerLogDirs( + id, remoteUser, nmContext); + for (File containerLogsDir : logDirs) { + File[] logs = containerLogsDir.listFiles(); + if (logs != null) { + for (File log : logs) { + if (log.isFile()) { + ContainerLogInfo logMeta = new ContainerLogInfo( + log.getName(), log.length()); + logFiles.add(logMeta); + } + } + } + } + return logFiles; + } + + private static class ContainerLogInfo { + private String fileName; + private long fileSize; + + //JAXB needs this + public ContainerLogInfo() {} + + public ContainerLogInfo(String fileName, long fileSize) { + this.setFileName(fileName); + this.setFileSize(fileSize); + } + + public String getFileName() { + return fileName; + } + + public void setFileName(String fileName) { + this.fileName = fileName; + } + + public long getFileSize() { + return fileSize; + } + + public void setFileSize(long fileSize) { + this.fileSize = fileSize; + } + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServices.java index a4305da4563..d9782994ebc 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServices.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServices.java @@ -312,13 +312,29 @@ public class TestNMWebServices extends JerseyTestBase { verifyNodesXML(nodes); } - @Test - public void testContainerLogs() throws IOException { - WebResource r = resource(); + @Test (timeout = 5000) + public void testContainerLogsWithNewAPI() throws IOException, JSONException{ final ContainerId containerId = BuilderUtils.newContainerId(0, 0, 0, 0); - final String containerIdStr = BuilderUtils.newContainerId(0, 0, 0, 0) - .toString(); - final ApplicationAttemptId appAttemptId = containerId.getApplicationAttemptId(); + WebResource r = resource(); + r = r.path("ws").path("v1").path("node").path("containers") + .path(containerId.toString()).path("logs"); + testContainerLogs(r, containerId); + } + + @Test (timeout = 5000) + public void testContainerLogsWithOldAPI() throws IOException, JSONException{ + final ContainerId containerId = BuilderUtils.newContainerId(1, 1, 0, 1); + WebResource r = resource(); + r = r.path("ws").path("v1").path("node").path("containerlogs") + .path(containerId.toString()); + testContainerLogs(r, containerId); + } + + private void testContainerLogs(WebResource r, ContainerId containerId) + throws IOException, JSONException { + final String containerIdStr = containerId.toString(); + final ApplicationAttemptId appAttemptId = containerId + .getApplicationAttemptId(); final ApplicationId appId = appAttemptId.getApplicationId(); final String appIdStr = appId.toString(); final String filename = "logfile1"; @@ -344,8 +360,7 @@ public class TestNMWebServices extends JerseyTestBase { pw.close(); // ask for it - ClientResponse response = r.path("ws").path("v1").path("node") - .path("containerlogs").path(containerIdStr).path(filename) + ClientResponse response = r.path(filename) .accept(MediaType.TEXT_PLAIN).get(ClientResponse.class); String responseText = response.getEntity(String.class); assertEquals(logMessage, responseText); @@ -354,8 +369,7 @@ public class TestNMWebServices extends JerseyTestBase { // specify how many bytes we should get from logs // specify a position number, it would get the first n bytes from // container log - response = r.path("ws").path("v1").path("node") - .path("containerlogs").path(containerIdStr).path(filename) + response = r.path(filename) .queryParam("size", "5") .accept(MediaType.TEXT_PLAIN).get(ClientResponse.class); responseText = response.getEntity(String.class); @@ -365,8 +379,7 @@ public class TestNMWebServices extends JerseyTestBase { // specify the bytes which is larger than the actual file size, // we would get the full logs - response = r.path("ws").path("v1").path("node") - .path("containerlogs").path(containerIdStr).path(filename) + response = r.path(filename) .queryParam("size", "10000") .accept(MediaType.TEXT_PLAIN).get(ClientResponse.class); responseText = response.getEntity(String.class); @@ -375,8 +388,7 @@ public class TestNMWebServices extends JerseyTestBase { // specify a negative number, it would get the last n bytes from // container log - response = r.path("ws").path("v1").path("node") - .path("containerlogs").path(containerIdStr).path(filename) + response = r.path(filename) .queryParam("size", "-5") .accept(MediaType.TEXT_PLAIN).get(ClientResponse.class); responseText = response.getEntity(String.class); @@ -385,8 +397,7 @@ public class TestNMWebServices extends JerseyTestBase { logMessage.getBytes().length - 5, 5), responseText); assertTrue(fullTextSize >= responseText.getBytes().length); - response = r.path("ws").path("v1").path("node") - .path("containerlogs").path(containerIdStr).path(filename) + response = r.path(filename) .queryParam("size", "-10000") .accept(MediaType.TEXT_PLAIN).get(ClientResponse.class); responseText = response.getEntity(String.class); @@ -395,8 +406,7 @@ public class TestNMWebServices extends JerseyTestBase { assertEquals(logMessage, responseText); // ask and download it - response = r.path("ws").path("v1").path("node").path("containerlogs") - .path(containerIdStr).path(filename) + response = r.path(filename) .queryParam("format", "octet-stream") .accept(MediaType.TEXT_PLAIN).get(ClientResponse.class); responseText = response.getEntity(String.class); @@ -405,8 +415,7 @@ public class TestNMWebServices extends JerseyTestBase { assertEquals("application/octet-stream", response.getType().toString()); // specify a invalid format value - response = r.path("ws").path("v1").path("node").path("containerlogs") - .path(containerIdStr).path(filename) + response = r.path(filename) .queryParam("format", "123") .accept(MediaType.TEXT_PLAIN).get(ClientResponse.class); responseText = response.getEntity(String.class); @@ -415,20 +424,28 @@ public class TestNMWebServices extends JerseyTestBase { assertEquals(400, response.getStatus()); // ask for file that doesn't exist - response = r.path("ws").path("v1").path("node") - .path("containerlogs").path(containerIdStr).path("uhhh") + response = r.path("uhhh") .accept(MediaType.TEXT_PLAIN).get(ClientResponse.class); Assert.assertEquals(Status.NOT_FOUND.getStatusCode(), response.getStatus()); responseText = response.getEntity(String.class); assertTrue(responseText.contains("Cannot find this log on the local disk.")); - + + // Get container log files' name + WebResource r1 = resource(); + response = r1.path("ws").path("v1").path("node") + .path("containers").path(containerIdStr) + .path("logs").accept(MediaType.APPLICATION_JSON) + .get(ClientResponse.class); + assertEquals(200, response.getStatus()); + JSONObject json = response.getEntity(JSONObject.class); + assertEquals(json.getJSONObject("containerLogInfo") + .getString("fileName"), filename); + // After container is completed, it is removed from nmContext nmContext.getContainers().remove(containerId); Assert.assertNull(nmContext.getContainers().get(containerId)); - response = - r.path("ws").path("v1").path("node").path("containerlogs") - .path(containerIdStr).path(filename).accept(MediaType.TEXT_PLAIN) - .get(ClientResponse.class); + response = r.path(filename).accept(MediaType.TEXT_PLAIN) + .get(ClientResponse.class); responseText = response.getEntity(String.class); assertEquals(logMessage, responseText); }