From f3c0a210b6628480429d2024d988c5242d54e7ba Mon Sep 17 00:00:00 2001 From: Xuan Date: Tue, 15 Sep 2015 14:36:30 -0700 Subject: [PATCH] YARN-4149. yarn logs -am should provide an option to fetch all the log files. Contributed by Varun Vasudev (cherry picked from commit 34ef1a092bcab369bb845481efffb8c47adef53a) --- hadoop-yarn-project/CHANGES.txt | 3 + .../hadoop/yarn/client/cli/LogsCLI.java | 104 ++++++++++++++++-- .../hadoop/yarn/client/cli/TestLogsCLI.java | 3 +- .../nodemanager/webapp/NMWebServices.java | 12 +- .../nodemanager/webapp/dao/ContainerInfo.java | 46 +++++++- .../webapp/TestNMWebServicesContainers.java | 13 ++- 6 files changed, 157 insertions(+), 24 deletions(-) diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 4909b274d97..f85e5607033 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -387,6 +387,9 @@ Release 2.8.0 - UNRELEASED YARN-3717. Expose app/am/queue's node-label-expression to RM web UI / CLI / REST-API. (Naganarasimha G R via wangda) + YARN-4149. yarn logs -am should provide an option to fetch all the log files + (Varun Vasudev via xgong) + OPTIMIZATIONS YARN-3339. TestDockerContainerExecutor should pull a single image and not diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/LogsCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/LogsCLI.java index 25481f87606..edee8eef745 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/LogsCLI.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/LogsCLI.java @@ -19,12 +19,15 @@ package org.apache.hadoop.yarn.client.cli; import java.io.IOException; +import java.io.StringReader; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.List; import javax.ws.rs.core.MediaType; +import javax.xml.parsers.DocumentBuilder; +import javax.xml.parsers.DocumentBuilderFactory; import org.apache.commons.cli.CommandLine; import org.apache.commons.cli.CommandLineParser; @@ -61,6 +64,9 @@ import com.sun.jersey.api.client.ClientHandlerException; import com.sun.jersey.api.client.ClientResponse; import com.sun.jersey.api.client.UniformInterfaceException; import com.sun.jersey.api.client.WebResource; +import org.w3c.dom.Document; +import org.w3c.dom.NodeList; +import org.xml.sax.InputSource; @Public @Evolving @@ -105,7 +111,8 @@ public class LogsCLI extends Configured implements Tool { opts.addOption(amOption); Option logFileOpt = new Option(CONTAINER_LOG_FILES, true, "Work with -am/-containerId and specify comma-separated value " - + "to get specified Container log files"); + + "to get specified container log files. Use \"ALL\" to fetch all the " + + "log files for the container."); logFileOpt.setValueSeparator(','); logFileOpt.setArgs(Option.UNLIMITED_VALUES); logFileOpt.setArgName("Log File Name"); @@ -248,8 +255,8 @@ public class LogsCLI extends Configured implements Tool { logFiles, logCliHelper, appOwner, true); } else { System.out - .println("Can not get AMContainers logs for the application:" - + appId); + .println( + "Can not get AMContainers logs for the application:" + appId); System.out.println("This application:" + appId + " is finished." + " Please enable the application history service. Or Using " + "yarn logs -applicationId -containerId " @@ -264,9 +271,18 @@ public class LogsCLI extends Configured implements Tool { // if we provide the node address and the application is in the final // state, we could directly get logs from HDFS. if (nodeAddress != null && isApplicationFinished(appState)) { + // if user specified "ALL" as the logFiles param, pass null + // to logCliHelper so that it fetches all the logs + List logs; + if (logFiles == null) { + logs = null; + } else if (fetchAllLogFiles(logFiles)) { + logs = null; + } else { + logs = Arrays.asList(logFiles); + } return logCliHelper.dumpAContainersLogsForALogType(appIdStr, - containerIdStr, nodeAddress, appOwner, logFiles == null ? null - : Arrays.asList(logFiles)); + containerIdStr, nodeAddress, appOwner, logs); } try { // If the nodeAddress is not provided, we will try to get @@ -288,10 +304,14 @@ public class LogsCLI extends Configured implements Tool { containerIdStr, nodeHttpAddress, nodeId, logFiles, logCliHelper, appOwner); } else { + String [] requestedLogFiles = logFiles; + if(fetchAllLogFiles(logFiles)) { + requestedLogFiles = null; + } // If the application is in the final state, we will directly // get the container logs from HDFS. printContainerLogsForFinishedApplication(appIdStr, containerIdStr, - nodeId, logFiles, logCliHelper, appOwner); + nodeId, requestedLogFiles, logCliHelper, appOwner); } return resultCode; } catch (IOException | YarnException ex) { @@ -401,15 +421,69 @@ public class LogsCLI extends Configured implements Tool { return amContainersList; } + private boolean fetchAllLogFiles(String[] logFiles) { + if(logFiles != null) { + List logs = Arrays.asList(logFiles); + if(logs.contains("ALL")) { + return true; + } + } + return false; + } + + private String[] getContainerLogFiles(Configuration conf, + String containerIdStr, String nodeHttpAddress) throws IOException { + List logFiles = new ArrayList<>(); + Client webServiceClient = Client.create(); + try { + WebResource webResource = webServiceClient + .resource(WebAppUtils.getHttpSchemePrefix(conf) + nodeHttpAddress); + ClientResponse response = + webResource.path("ws").path("v1").path("node").path("containers") + .path(containerIdStr).accept(MediaType.APPLICATION_XML) + .get(ClientResponse.class); + if (response.getClientResponseStatus().equals(ClientResponse.Status.OK)) { + try { + String xml = response.getEntity(String.class); + DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance(); + DocumentBuilder db = dbf.newDocumentBuilder(); + InputSource is = new InputSource(); + is.setCharacterStream(new StringReader(xml)); + Document dom = db.parse(is); + NodeList elements = dom.getElementsByTagName("containerLogFiles"); + for (int i = 0; i < elements.getLength(); i++) { + logFiles.add(elements.item(i).getTextContent()); + } + } catch (Exception e) { + System.out.println("Unable to parse xml from webservice. Error:"); + System.out.println(e.getMessage()); + throw new IOException(e); + } + } + + } catch (ClientHandlerException | UniformInterfaceException ex) { + System.out.println("Unable to fetch log files list"); + throw new IOException(ex); + } + return logFiles.toArray(new String[0]); + } + private void printContainerLogsFromRunningApplication(Configuration conf, String appId, String containerIdStr, String nodeHttpAddress, String nodeId, String[] logFiles, LogCLIHelpers logCliHelper, String appOwner) throws IOException { + String [] requestedLogFiles = logFiles; + // fetch all the log files for the container + if (fetchAllLogFiles(logFiles)) { + requestedLogFiles = + getContainerLogFiles(getConf(), containerIdStr, nodeHttpAddress); + } Client webServiceClient = Client.create(); String containerString = "\n\nContainer: " + containerIdStr; System.out.println(containerString); System.out.println(StringUtils.repeat("=", containerString.length())); - for (String logFile : logFiles) { + + for (String logFile : requestedLogFiles) { System.out.println("LogType:" + logFile); System.out.println("Log Upload Time:" + Times.format(System.currentTimeMillis())); @@ -432,7 +506,7 @@ public class LogsCLI extends Configured implements Tool { } // for the case, we have already uploaded partial logs in HDFS logCliHelper.dumpAContainersLogsForALogType(appId, containerIdStr, nodeId, - appOwner, Arrays.asList(logFiles)); + appOwner, Arrays.asList(requestedLogFiles)); } private void printContainerLogsForFinishedApplication(String appId, @@ -551,15 +625,25 @@ public class LogsCLI extends Configured implements Tool { } } if (nodeId != null && !nodeId.isEmpty()) { + String [] requestedLogFilesList = null; + if(!fetchAllLogFiles(logFiles)) { + requestedLogFilesList = logFiles; + } printContainerLogsForFinishedApplication(appId, containerId, nodeId, - logFiles, logCliHelper, appOwner); + requestedLogFilesList, logCliHelper, appOwner); } } } else { if (nodeHttpAddress != null && containerId != null && !nodeHttpAddress.isEmpty() && !containerId.isEmpty()) { + String [] requestedLogFiles = logFiles; + // fetch all the log files for the AM + if (fetchAllLogFiles(logFiles)) { + requestedLogFiles = + getContainerLogFiles(getConf(), containerId, nodeHttpAddress); + } printContainerLogsFromRunningApplication(conf, appId, containerId, - nodeHttpAddress, nodeId, logFiles, logCliHelper, appOwner); + nodeHttpAddress, nodeId, requestedLogFiles, logCliHelper, appOwner); } } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java index a3538116ab9..aec7caec9dd 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java @@ -168,7 +168,8 @@ public class TestLogsCLI { pw.println(" -help Displays help for all commands."); pw.println(" -logFiles Work with -am/-containerId and specify"); pw.println(" comma-separated value to get specified"); - pw.println(" Container log files"); + pw.println(" container log files. Use \"ALL\" to fetch"); + pw.println(" all the log files for the container."); pw.println(" -nodeAddress NodeAddress in the format nodename:port"); pw.close(); String appReportStr = baos.toString("UTF-8"); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMWebServices.java index 16f2c685c1e..de6d219deaa 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMWebServices.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMWebServices.java @@ -129,7 +129,7 @@ public class NMWebServices { String msg = "Error: You must specify a non-empty string for the user"; throw new BadRequestException(msg); } - if (!appInfo.getUser().toString().equals(userQuery)) { + if (!appInfo.getUser().equals(userQuery)) { continue; } } @@ -158,7 +158,8 @@ public class NMWebServices { @GET @Path("/containers") @Produces({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML }) - public ContainersInfo getNodeContainers() { + public ContainersInfo getNodeContainers(@javax.ws.rs.core.Context + HttpServletRequest hsr) { init(); ContainersInfo allContainers = new ContainersInfo(); for (Entry entry : this.nmContext.getContainers() @@ -168,7 +169,7 @@ public class NMWebServices { continue; } ContainerInfo info = new ContainerInfo(this.nmContext, entry.getValue(), - uriInfo.getBaseUri().toString(), webapp.name()); + uriInfo.getBaseUri().toString(), webapp.name(), hsr.getRemoteUser()); allContainers.add(info); } return allContainers; @@ -177,7 +178,8 @@ public class NMWebServices { @GET @Path("/containers/{containerid}") @Produces({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML }) - public ContainerInfo getNodeContainer(@PathParam("containerid") String id) { + public ContainerInfo getNodeContainer(@javax.ws.rs.core.Context + HttpServletRequest hsr, @PathParam("containerid") String id) { ContainerId containerId = null; init(); try { @@ -191,7 +193,7 @@ public class NMWebServices { throw new NotFoundException("container with id, " + id + ", not found"); } return new ContainerInfo(this.nmContext, container, uriInfo.getBaseUri() - .toString(), webapp.name()); + .toString(), webapp.name(), hsr.getRemoteUser()); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/dao/ContainerInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/dao/ContainerInfo.java index 78982deb19f..e462eba4b80 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/dao/ContainerInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/dao/ContainerInfo.java @@ -21,16 +21,19 @@ package org.apache.hadoop.yarn.server.nodemanager.webapp.dao; import static org.apache.hadoop.yarn.util.StringHelper.join; import static org.apache.hadoop.yarn.util.StringHelper.ujoin; -import javax.xml.bind.annotation.XmlAccessType; -import javax.xml.bind.annotation.XmlAccessorType; -import javax.xml.bind.annotation.XmlRootElement; -import javax.xml.bind.annotation.XmlTransient; +import javax.xml.bind.annotation.*; import org.apache.hadoop.yarn.api.records.ContainerExitStatus; +import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.ContainerStatus; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.server.nodemanager.Context; import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container; +import org.apache.hadoop.yarn.server.nodemanager.webapp.ContainerLogsUtils; + +import java.io.File; +import java.util.ArrayList; +import java.util.List; @XmlRootElement(name = "container") @XmlAccessorType(XmlAccessType.FIELD) @@ -50,15 +53,18 @@ public class ContainerInfo { @XmlTransient protected String exitStatus; + @XmlElementWrapper + protected List containerLogFiles; + public ContainerInfo() { } // JAXB needs this public ContainerInfo(final Context nmContext, final Container container) { - this(nmContext, container, "", ""); + this(nmContext, container, "", "", ""); } public ContainerInfo(final Context nmContext, final Container container, - String requestUri, String pathPrefix) { + String requestUri, String pathPrefix, String remoteUser) { this.id = container.getContainerId().toString(); this.nodeId = nmContext.getNodeId().toString(); @@ -90,6 +96,8 @@ public class ContainerInfo { } this.containerLogsLink = join(requestUri, pathPrefix, this.containerLogsShortLink); + this.containerLogFiles = + getContainerLogFiles(container.getContainerId(), remoteUser, nmContext); } public String getId() { @@ -136,4 +144,30 @@ public class ContainerInfo { return this.totalVCoresNeeded; } + public List getContainerLogFiles() { + return this.containerLogFiles; + } + + private List getContainerLogFiles(ContainerId id, String remoteUser, + Context nmContext) { + List logFiles = new ArrayList<>(); + try { + List logDirs = + ContainerLogsUtils.getContainerLogDirs(id, remoteUser, nmContext); + for (File containerLogsDir : logDirs) { + File[] logs = containerLogsDir.listFiles(); + if (logs != null) { + for (File log : logs) { + if (log.isFile()) { + logFiles.add(log.getName()); + } + } + } + } + } catch (Exception ye) { + return logFiles; + } + return logFiles; + } + } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServicesContainers.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServicesContainers.java index 3bfd440a094..0ed56d3f0b1 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServicesContainers.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServicesContainers.java @@ -32,6 +32,7 @@ import javax.ws.rs.core.MediaType; import javax.xml.parsers.DocumentBuilder; import javax.xml.parsers.DocumentBuilderFactory; +import com.sun.jersey.api.client.filter.LoggingFilter; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileUtil; import org.apache.hadoop.util.NodeHealthScriptRunner; @@ -245,6 +246,7 @@ public class TestNMWebServicesContainers extends JerseyTestBase { Application app2 = new MockApp(2); nmContext.getApplications().put(app2.getAppId(), app2); addAppContainers(app2); + client().addFilter(new LoggingFilter()); ClientResponse response = r.path("ws").path("v1").path("node").path(path) .accept(media).get(ClientResponse.class); @@ -412,6 +414,7 @@ public class TestNMWebServicesContainers extends JerseyTestBase { Application app2 = new MockApp(2); nmContext.getApplications().put(app2.getAppId(), app2); addAppContainers(app2); + client().addFilter(new LoggingFilter(System.out)); for (String id : hash.keySet()) { ClientResponse response = r.path("ws").path("v1").path("node") @@ -471,12 +474,16 @@ public class TestNMWebServicesContainers extends JerseyTestBase { WebServicesTestUtils.getXmlInt(element, "totalMemoryNeededMB"), WebServicesTestUtils.getXmlInt(element, "totalVCoresNeeded"), WebServicesTestUtils.getXmlString(element, "containerLogsLink")); + // verify that the container log files element exists + assertTrue("containerLogFiles missing", + WebServicesTestUtils.getXmlString(element, "containerLogFiles") + != null); } } public void verifyNodeContainerInfo(JSONObject info, Container cont) throws JSONException, Exception { - assertEquals("incorrect number of elements", 9, info.length()); + assertEquals("incorrect number of elements", 10, info.length()); verifyNodeContainerInfoGeneric(cont, info.getString("id"), info.getString("state"), info.getString("user"), @@ -484,6 +491,9 @@ public class TestNMWebServicesContainers extends JerseyTestBase { info.getString("nodeId"), info.getInt("totalMemoryNeededMB"), info.getInt("totalVCoresNeeded"), info.getString("containerLogsLink")); + // verify that the container log files element exists + assertTrue("containerLogFiles missing", + info.getJSONArray("containerLogFiles") != null); } public void verifyNodeContainerInfoGeneric(Container cont, String id, @@ -514,5 +524,4 @@ public class TestNMWebServicesContainers extends JerseyTestBase { cont.getUser()); assertTrue("containerLogsLink wrong", logsLink.contains(shortLink)); } - }