diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/LogsCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/LogsCLI.java index 039ba3c6800..b0e5f221bd8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/LogsCLI.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/LogsCLI.java @@ -23,9 +23,11 @@ import java.io.PrintStream; import java.io.StringReader; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collection; import java.util.Collections; import java.util.List; - +import java.util.Set; +import java.util.regex.Pattern; import javax.ws.rs.core.MediaType; import javax.xml.parsers.DocumentBuilder; import javax.xml.parsers.DocumentBuilderFactory; @@ -53,6 +55,7 @@ import org.apache.hadoop.yarn.client.api.YarnClient; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.logaggregation.LogCLIHelpers; +import org.apache.hadoop.yarn.logaggregation.ContainerLogsRequest; import org.apache.hadoop.yarn.util.ConverterUtils; import org.apache.hadoop.yarn.util.Times; import org.apache.hadoop.yarn.webapp.util.WebAppUtils; @@ -181,9 +184,16 @@ public class LogsCLI extends Configured implements Tool { } } + List logs = new ArrayList(); + if (fetchAllLogFiles(logFiles)) { + logs.add(".*"); + } else if (logFiles != null && logFiles.length > 0) { + logs = Arrays.asList(logFiles); + } + ContainerLogsRequest request = new ContainerLogsRequest(appId, - isApplicationFinished(appState), appOwner, - nodeAddress, null, containerIdStr); + isApplicationFinished(appState), appOwner, nodeAddress, null, + containerIdStr, localDir, logs); if (showMetaInfo) { return showMetaInfo(request, logCliHelper); @@ -196,7 +206,7 @@ public class LogsCLI extends Configured implements Tool { // To get am logs if (getAMContainerLogs) { return fetchAMContainerLogs(request, amContainersList, - logFiles, logCliHelper, localDir); + logCliHelper); } int resultCode = 0; @@ -208,12 +218,10 @@ public class LogsCLI extends Configured implements Tool { + " does not have the container:" + containerId); return -1; } - return fetchContainerLogs(request, logFiles, - logCliHelper, localDir); + return fetchContainerLogs(request, logCliHelper); } else { if (nodeAddress == null) { - resultCode = fetchApplicationLogs(appId, appOwner, - logCliHelper, localDir); + resultCode = fetchApplicationLogs(request, logCliHelper); } else { System.err.println("Should at least provide ContainerId!"); printHelpMessage(printOpts); @@ -307,14 +315,14 @@ public class LogsCLI extends Configured implements Tool { private boolean fetchAllLogFiles(String[] logFiles) { if(logFiles != null) { List logs = Arrays.asList(logFiles); - if(logs.contains("ALL")) { + if(logs.contains("ALL") || logs.contains(".*")) { return true; } } return false; } - private String[] getContainerLogFiles(Configuration conf, + private List getContainerLogFiles(Configuration conf, String containerIdStr, String nodeHttpAddress) throws IOException { List logFiles = new ArrayList<>(); Client webServiceClient = Client.create(); @@ -348,32 +356,37 @@ public class LogsCLI extends Configured implements Tool { System.err.println("Unable to fetch log files list"); throw new IOException(ex); } - return logFiles.toArray(new String[0]); + return logFiles; } private void printContainerLogsFromRunningApplication(Configuration conf, - ContainerLogsRequest request, String[] logFiles, - LogCLIHelpers logCliHelper, String localDir) throws IOException { - String appId = request.getAppId().toString(); + ContainerLogsRequest request, LogCLIHelpers logCliHelper) + throws IOException { String containerIdStr = request.getContainerId().toString(); - String[] requestedLogFiles = logFiles; + String localDir = request.getOutputLocalDir(); String nodeHttpAddress = request.getNodeHttpAddress(); String nodeId = request.getNodeId(); - String appOwner = request.getAppOwner(); PrintStream out = logCliHelper.createPrintStream(localDir, nodeId, containerIdStr); try { // fetch all the log files for the container - if (fetchAllLogFiles(logFiles)) { - requestedLogFiles = - getContainerLogFiles(getConf(), containerIdStr, nodeHttpAddress); + // filter the log files based on the given --logFiles pattern + List allLogs= + getContainerLogFiles(getConf(), containerIdStr, nodeHttpAddress); + List matchedFiles = getMatchedLogFiles( + request, allLogs, true); + if (matchedFiles.isEmpty()) { + return; } + ContainerLogsRequest newOptions = new ContainerLogsRequest(request); + newOptions.setLogTypes(matchedFiles); + Client webServiceClient = Client.create(); String containerString = "\n\nContainer: " + containerIdStr; out.println(containerString); out.println(StringUtils.repeat("=", containerString.length())); - for (String logFile : requestedLogFiles) { + for (String logFile : newOptions.getLogTypes()) { out.println("LogType:" + logFile); out.println("Log Upload Time:" + Times.format(System.currentTimeMillis())); @@ -395,31 +408,34 @@ public class LogsCLI extends Configured implements Tool { + nodeId); } } + // for the case, we have already uploaded partial logs in HDFS + logCliHelper.dumpAContainersLogsForALogType(newOptions, false); } finally { logCliHelper.closePrintStream(out); } - // for the case, we have already uploaded partial logs in HDFS - logCliHelper.dumpAContainersLogsForALogType(appId, containerIdStr, nodeId, - appOwner, Arrays.asList(requestedLogFiles), false, localDir); } - private void printContainerLogsForFinishedApplication( - ContainerLogsRequest request, String[] logFiles, - LogCLIHelpers logCliHelper, String localDir) + private int printContainerLogsForFinishedApplication( + ContainerLogsRequest request, LogCLIHelpers logCliHelper) throws IOException { - logCliHelper.dumpAContainersLogsForALogType(request.getAppId().toString(), - request.getContainerId().toString(), request.getNodeId(), - request.getAppOwner(), logFiles != null ? Arrays.asList(logFiles) - : null, localDir); + ContainerLogsRequest newOptions = getMatchedLogOptions( + request, logCliHelper); + if (newOptions == null) { + return -1; + } + return logCliHelper.dumpAContainersLogsForALogType(newOptions); } private int printContainerLogsForFinishedApplicationWithoutNodeId( - String appId, String containerId, String[] logFiles, - LogCLIHelpers logCliHelper, String appOwner, String localDir) + ContainerLogsRequest request, LogCLIHelpers logCliHelper) throws IOException { - return logCliHelper.dumpAContainersLogsForALogTypeWithoutNodeId(appId, - containerId, appOwner, logFiles != null ? - Arrays.asList(logFiles) : null, localDir); + ContainerLogsRequest newOptions = getMatchedLogOptions( + request, logCliHelper); + if (newOptions == null) { + return -1; + } + return logCliHelper.dumpAContainersLogsForALogTypeWithoutNodeId( + newOptions); } private ContainerReport getContainerReport(String containerIdStr) @@ -441,8 +457,7 @@ public class LogsCLI extends Configured implements Tool { private int printAMContainerLogs(Configuration conf, ContainerLogsRequest request, List amContainers, - String[] logFiles, LogCLIHelpers logCliHelper, String localDir) - throws Exception { + LogCLIHelpers logCliHelper) throws Exception { List amContainersList = null; List requests = new ArrayList(); @@ -491,8 +506,7 @@ public class LogsCLI extends Configured implements Tool { if (amContainers.contains("ALL")) { for (ContainerLogsRequest amRequest : requests) { - outputAMContainerLogs(amRequest, conf, logFiles, - logCliHelper, localDir); + outputAMContainerLogs(amRequest, conf, logCliHelper); } System.out.println(); System.out.println("Specified ALL for -am option. " @@ -502,11 +516,11 @@ public class LogsCLI extends Configured implements Tool { int amContainerId = Integer.parseInt(amContainer.trim()); if (amContainerId == -1) { outputAMContainerLogs(requests.get(requests.size() - 1), conf, - logFiles, logCliHelper, localDir); + logCliHelper); } else { if (amContainerId <= requests.size()) { outputAMContainerLogs(requests.get(amContainerId - 1), conf, - logFiles, logCliHelper, localDir); + logCliHelper); } } } @@ -515,8 +529,7 @@ public class LogsCLI extends Configured implements Tool { } private void outputAMContainerLogs(ContainerLogsRequest request, - Configuration conf, String[] logFiles, - LogCLIHelpers logCliHelper, String localDir) throws Exception { + Configuration conf, LogCLIHelpers logCliHelper) throws Exception { String nodeHttpAddress = request.getNodeHttpAddress(); String containerId = request.getContainerId(); String nodeId = request.getNodeId(); @@ -534,25 +547,15 @@ public class LogsCLI extends Configured implements Tool { } } if (nodeId != null && !nodeId.isEmpty()) { - String[] requestedLogFilesList = null; - if(!fetchAllLogFiles(logFiles)) { - requestedLogFilesList = logFiles; - } printContainerLogsForFinishedApplication(request, - requestedLogFilesList, logCliHelper, localDir); + logCliHelper); } } } else { if (nodeHttpAddress != null && containerId != null && !nodeHttpAddress.isEmpty() && !containerId.isEmpty()) { - String[] requestedLogFiles = logFiles; - // fetch all the log files for the AM - if (fetchAllLogFiles(logFiles)) { - requestedLogFiles = - getContainerLogFiles(getConf(), containerId, nodeHttpAddress); - } printContainerLogsFromRunningApplication(conf, - request, requestedLogFiles, logCliHelper, localDir); + request, logCliHelper); } } } @@ -564,9 +567,7 @@ public class LogsCLI extends Configured implements Tool { + "with finished applications"); return -1; } else { - logCliHelper.printLogMetadata(request.getAppId(), - request.getContainerId(), request.getNodeId(), - request.getAppOwner(), System.out, System.err); + logCliHelper.printLogMetadata(request, System.out, System.err); return 0; } } @@ -578,8 +579,7 @@ public class LogsCLI extends Configured implements Tool { + "finished applications"); return -1; } else { - logCliHelper.printNodesList(request.getAppId(), request.getAppOwner(), - System.out, System.err); + logCliHelper.printNodesList(request, System.out, System.err); return 0; } } @@ -614,7 +614,7 @@ public class LogsCLI extends Configured implements Tool { Option logFileOpt = new Option(CONTAINER_LOG_FILES, true, "Work with -am/-containerId and specify comma-separated value " + "to get specified container log files. Use \"ALL\" to fetch all the " - + "log files for the container."); + + "log files for the container. It also supports Java Regex."); logFileOpt.setValueSeparator(','); logFileOpt.setArgs(Option.UNLIMITED_VALUES); logFileOpt.setArgName("Log File Name"); @@ -687,13 +687,15 @@ public class LogsCLI extends Configured implements Tool { } private int fetchAMContainerLogs(ContainerLogsRequest request, - List amContainersList, String[] logFiles, - LogCLIHelpers logCliHelper, String localDir) throws Exception { + List amContainersList, LogCLIHelpers logCliHelper) + throws Exception { + List logFiles = request.getLogTypes(); // if we do not specify the value for CONTAINER_LOG_FILES option, // we will only output syslog - if (logFiles == null || logFiles.length == 0) { - logFiles = new String[] {"syslog"}; + if (logFiles == null || logFiles.isEmpty()) { + logFiles = Arrays.asList("syslog"); } + request.setLogTypes(logFiles); // If the application is running, we will call the RM WebService // to get the AppAttempts which includes the nodeHttpAddress // and containerId for all the AM Containers. @@ -701,7 +703,7 @@ public class LogsCLI extends Configured implements Tool { // related logs if (!request.isAppFinished()) { return printAMContainerLogs(getConf(), request, amContainersList, - logFiles, logCliHelper, localDir); + logCliHelper); } else { // If the application is in the final state, we will call RM webservice // to get all AppAttempts information first. If we get nothing, @@ -712,7 +714,7 @@ public class LogsCLI extends Configured implements Tool { if (getConf().getBoolean(YarnConfiguration.APPLICATION_HISTORY_ENABLED, YarnConfiguration.DEFAULT_APPLICATION_HISTORY_ENABLED)) { return printAMContainerLogs(getConf(), request, amContainersList, - logFiles, logCliHelper, localDir); + logCliHelper); } else { ApplicationId appId = request.getAppId(); String appOwner = request.getAppOwner(); @@ -729,29 +731,21 @@ public class LogsCLI extends Configured implements Tool { } private int fetchContainerLogs(ContainerLogsRequest request, - String[] logFiles, LogCLIHelpers logCliHelper, String localDir) - throws IOException { + LogCLIHelpers logCliHelper) throws IOException { int resultCode = 0; String appIdStr = request.getAppId().toString(); String containerIdStr = request.getContainerId(); String nodeAddress = request.getNodeId(); String appOwner = request.getAppOwner(); boolean isAppFinished = request.isAppFinished(); + List logFiles = request.getLogTypes(); // if we provide the node address and the application is in the final // state, we could directly get logs from HDFS. if (nodeAddress != null && isAppFinished) { - // if user specified "ALL" as the logFiles param, pass null + // if user specified "ALL" as the logFiles param, pass empty list // to logCliHelper so that it fetches all the logs - List logs; - if (logFiles == null) { - logs = null; - } else if (fetchAllLogFiles(logFiles)) { - logs = null; - } else { - logs = Arrays.asList(logFiles); - } - return logCliHelper.dumpAContainersLogsForALogType(appIdStr, - containerIdStr, nodeAddress, appOwner, logs, localDir); + return printContainerLogsForFinishedApplication( + request, logCliHelper); } String nodeHttpAddress = null; String nodeId = null; @@ -768,13 +762,8 @@ public class LogsCLI extends Configured implements Tool { request.setNodeHttpAddress(nodeHttpAddress); } catch (IOException | YarnException ex) { if (isAppFinished) { - String[] requestedLogFiles = logFiles; - if(fetchAllLogFiles(logFiles)) { - requestedLogFiles = null; - } return printContainerLogsForFinishedApplicationWithoutNodeId( - appIdStr, containerIdStr, requestedLogFiles, logCliHelper, - appOwner, localDir); + request, logCliHelper); } else { System.err.println("Unable to get logs for this container:" + containerIdStr + "for the application:" + appIdStr @@ -790,31 +779,39 @@ public class LogsCLI extends Configured implements Tool { // we will provide the NodeHttpAddress and get the container logs // by calling NodeManager webservice. if (!isAppFinished) { - if (logFiles == null || logFiles.length == 0) { - logFiles = new String[] {"syslog"}; + // if we do not specify the value for CONTAINER_LOG_FILES option, + // we will only output syslog + if (logFiles == null || logFiles.isEmpty()) { + logFiles = Arrays.asList("syslog"); } + request.setLogTypes(logFiles); printContainerLogsFromRunningApplication(getConf(), request, - logFiles, logCliHelper, localDir); + logCliHelper); } else { - String[] requestedLogFiles = logFiles; - if(fetchAllLogFiles(logFiles)) { - requestedLogFiles = null; - } // If the application is in the final state, we will directly // get the container logs from HDFS. - printContainerLogsForFinishedApplication(request, - requestedLogFiles, logCliHelper, localDir); + resultCode = printContainerLogsForFinishedApplication( + request, logCliHelper); } return resultCode; } - private int fetchApplicationLogs(ApplicationId appId, String appOwner, - LogCLIHelpers logCliHelper, String localDir) throws IOException { - int resultCode = - logCliHelper.dumpAllContainersLogs(appId, appOwner, localDir); + private int fetchApplicationLogs(ContainerLogsRequest options, + LogCLIHelpers logCliHelper) throws IOException { + // TODO: YARN-5141. To get container logs for the Running applications. + int resultCode = 0; + ContainerLogsRequest newOptions = getMatchedLogOptions( + options, logCliHelper); + if (newOptions == null) { + resultCode = -1; + } else { + resultCode = + logCliHelper.dumpAllContainersLogs(newOptions); + } if (resultCode == -1) { System.err.println("Can not find the logs for the application: " - + appId + " with the appOwner: " + appOwner); + + options.getAppId() + " with the appOwner: " + + options.getAppOwner()); } return resultCode; } @@ -833,80 +830,53 @@ public class LogsCLI extends Configured implements Tool { return appOwner; } - private static class ContainerLogsRequest { - private ApplicationId appId; - private String containerId; - private String nodeId; - private String nodeHttpAddress; - private String appOwner; - private boolean appFinished; - - public ContainerLogsRequest(ContainerLogsRequest request) { - this.setAppId(request.getAppId()); - this.setAppFinished(request.isAppFinished()); - this.setAppOwner(request.getAppOwner()); - this.setNodeId(request.getNodeId()); - this.setNodeHttpAddress(request.getNodeHttpAddress()); - this.setContainerId(request.getContainerId()); + private ContainerLogsRequest getMatchedLogOptions( + ContainerLogsRequest request, LogCLIHelpers logCliHelper) + throws IOException { + ContainerLogsRequest newOptions = new ContainerLogsRequest(request); + if (request.getLogTypes() != null && !request.getLogTypes().isEmpty()) { + List matchedFiles = new ArrayList(); + if (!request.getLogTypes().contains(".*")) { + Set files = logCliHelper.listContainerLogs(request); + matchedFiles = getMatchedLogFiles( + request, files, true); + if (matchedFiles.isEmpty()) { + return null; + } + } + newOptions.setLogTypes(matchedFiles); } + return newOptions; + } - public ContainerLogsRequest(ApplicationId applicationId, - boolean isAppFinished, String owner, - String address, String httpAddress, String container) { - this.setAppId(applicationId); - this.setAppFinished(isAppFinished); - this.setAppOwner(owner); - this.setNodeId(address); - this.setNodeHttpAddress(httpAddress); - this.setContainerId(container); + private List getMatchedLogFiles(ContainerLogsRequest options, + Collection candidate, boolean printError) throws IOException { + List matchedFiles = new ArrayList(); + List filePattern = options.getLogTypes(); + for (String file : candidate) { + if (isFileMatching(file, filePattern)) { + matchedFiles.add(file); + } } + if (matchedFiles.isEmpty()) { + if (printError) { + System.err.println("Can not find any log file matching the pattern: " + + options.getLogTypes() + " for the application: " + + options.getAppId()); + } + } + return matchedFiles; + } - public ApplicationId getAppId() { - return appId; - } - - public void setAppId(ApplicationId appId) { - this.appId = appId; - } - - public String getContainerId() { - return containerId; - } - - public void setContainerId(String containerId) { - this.containerId = containerId; - } - - public String getNodeId() { - return nodeId; - } - - public void setNodeId(String nodeAddress) { - this.nodeId = nodeAddress; - } - - public String getAppOwner() { - return appOwner; - } - - public void setAppOwner(String appOwner) { - this.appOwner = appOwner; - } - - public String getNodeHttpAddress() { - return nodeHttpAddress; - } - - public void setNodeHttpAddress(String nodeHttpAddress) { - this.nodeHttpAddress = nodeHttpAddress; - } - - public boolean isAppFinished() { - return appFinished; - } - - public void setAppFinished(boolean appFinished) { - this.appFinished = appFinished; + private boolean isFileMatching(String fileType, + List logTypes) { + for (String logType : logTypes) { + Pattern filterPattern = Pattern.compile(logType); + boolean match = filterPattern.matcher(fileType).find(); + if (match) { + return true; + } } + return false; } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java index d649ce78ec2..c1af3249756 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java @@ -181,7 +181,8 @@ public class TestLogsCLI { pw.println(" -logFiles Work with -am/-containerId and specify"); pw.println(" comma-separated value to get specified"); pw.println(" container log files. Use \"ALL\" to fetch"); - pw.println(" all the log files for the container."); + pw.println(" all the log files for the container. It"); + pw.println(" also supports Java Regex."); pw.println(" -nodeAddress NodeAddress in the format nodename:port"); pw.println(" -out Local directory for storing individual"); pw.println(" container logs. The container logs will"); @@ -288,6 +289,39 @@ public class TestLogsCLI { "Hello container_0_0001_01_000003 in stdout!")); sysOutStream.reset(); + exitCode = cli.run(new String[] {"-applicationId", appId.toString(), + "-logFiles", ".*"}); + assertTrue(exitCode == 0); + assertTrue(sysOutStream.toString().contains( + "Hello container_0_0001_01_000001 in syslog!")); + assertTrue(sysOutStream.toString().contains( + "Hello container_0_0001_01_000002 in syslog!")); + assertTrue(sysOutStream.toString().contains( + "Hello container_0_0001_01_000003 in syslog!")); + assertTrue(sysOutStream.toString().contains( + "Hello container_0_0001_01_000003 in stdout!")); + sysOutStream.reset(); + + exitCode = cli.run(new String[] {"-applicationId", appId.toString(), + "-logFiles", "std*"}); + assertTrue(exitCode == 0); + assertFalse(sysOutStream.toString().contains( + "Hello container_0_0001_01_000001 in syslog!")); + assertFalse(sysOutStream.toString().contains( + "Hello container_0_0001_01_000002 in syslog!")); + assertFalse(sysOutStream.toString().contains( + "Hello container_0_0001_01_000003 in syslog!")); + assertTrue(sysOutStream.toString().contains( + "Hello container_0_0001_01_000003 in stdout!")); + sysOutStream.reset(); + + exitCode = cli.run(new String[] {"-applicationId", appId.toString(), + "-logFiles", "123"}); + assertTrue(exitCode == -1); + assertTrue(sysErrStream.toString().contains( + "Can not find any log file matching the pattern: [123]")); + sysErrStream.reset(); + // uploaded two logs for container1. The first log is empty. // The second one is not empty. // We can still successfully read logs for container1. diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/AggregatedLogFormat.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/AggregatedLogFormat.java index 646aa6dfdb6..d636200975b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/AggregatedLogFormat.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/AggregatedLogFormat.java @@ -875,23 +875,25 @@ public class AggregatedLogFormat { } @Private - public static void readContainerMetaDataAndSkipData( + public static String readContainerMetaDataAndSkipData( DataInputStream valueStream, PrintStream out) throws IOException { String fileType = valueStream.readUTF(); String fileLengthStr = valueStream.readUTF(); long fileLength = Long.parseLong(fileLengthStr); - out.print("LogType:"); - out.println(fileType); - out.print("LogLength:"); - out.println(fileLengthStr); - + if (out != null) { + out.print("LogType:"); + out.println(fileType); + out.print("LogLength:"); + out.println(fileLengthStr); + } long totalSkipped = 0; long currSkipped = 0; while (currSkipped != -1 && totalSkipped < fileLength) { currSkipped = valueStream.skip(fileLength - totalSkipped); totalSkipped += currSkipped; } + return fileType; } public void close() { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/ContainerLogsRequest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/ContainerLogsRequest.java new file mode 100644 index 00000000000..b0a7fdc3ecb --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/ContainerLogsRequest.java @@ -0,0 +1,124 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.logaggregation; + +import java.util.List; +import org.apache.hadoop.yarn.api.records.ApplicationId; + +public class ContainerLogsRequest { + private ApplicationId appId; + private String containerId; + private String nodeId; + private String nodeHttpAddress; + private String appOwner; + private boolean appFinished; + private String outputLocalDir; + private List logTypes; + + public ContainerLogsRequest() {} + + public ContainerLogsRequest(ContainerLogsRequest request) { + this.setAppId(request.getAppId()); + this.setAppFinished(request.isAppFinished()); + this.setAppOwner(request.getAppOwner()); + this.setNodeId(request.getNodeId()); + this.setNodeHttpAddress(request.getNodeHttpAddress()); + this.setContainerId(request.getContainerId()); + this.setOutputLocalDir(request.getOutputLocalDir()); + this.setLogTypes(request.getLogTypes()); + } + + public ContainerLogsRequest(ApplicationId applicationId, + boolean isAppFinished, String owner, + String address, String httpAddress, String container, String localDir, + List logs) { + this.setAppId(applicationId); + this.setAppFinished(isAppFinished); + this.setAppOwner(owner); + this.setNodeId(address); + this.setNodeHttpAddress(httpAddress); + this.setContainerId(container); + this.setOutputLocalDir(localDir); + this.setLogTypes(logs); + } + + public ApplicationId getAppId() { + return appId; + } + + public void setAppId(ApplicationId appId) { + this.appId = appId; + } + + public String getContainerId() { + return containerId; + } + + public void setContainerId(String containerId) { + this.containerId = containerId; + } + + public String getNodeId() { + return nodeId; + } + + public void setNodeId(String nodeAddress) { + this.nodeId = nodeAddress; + } + + public String getAppOwner() { + return appOwner; + } + + public void setAppOwner(String appOwner) { + this.appOwner = appOwner; + } + + public String getNodeHttpAddress() { + return nodeHttpAddress; + } + + public void setNodeHttpAddress(String nodeHttpAddress) { + this.nodeHttpAddress = nodeHttpAddress; + } + + public boolean isAppFinished() { + return appFinished; + } + + public void setAppFinished(boolean appFinished) { + this.appFinished = appFinished; + } + + public String getOutputLocalDir() { + return outputLocalDir; + } + + public void setOutputLocalDir(String outputLocalDir) { + this.outputLocalDir = outputLocalDir; + } + + public List getLogTypes() { + return logTypes; + } + + public void setLogTypes(List logTypes) { + this.logTypes = logTypes; + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/LogCLIHelpers.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/LogCLIHelpers.java index 339df9d18bc..22147aed555 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/LogCLIHelpers.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/LogCLIHelpers.java @@ -26,7 +26,10 @@ import java.io.PrintStream; import java.nio.file.AccessDeniedException; import java.nio.file.Files; import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.HashSet; import java.util.List; +import java.util.Set; import org.apache.commons.io.IOUtils; import org.apache.commons.lang.StringUtils; import org.apache.hadoop.classification.InterfaceAudience.Private; @@ -55,8 +58,14 @@ public class LogCLIHelpers implements Configurable { @VisibleForTesting public int dumpAContainersLogs(String appId, String containerId, String nodeId, String jobOwner) throws IOException { - return dumpAContainersLogsForALogType(appId, containerId, nodeId, jobOwner, - null, null); + ContainerLogsRequest options = new ContainerLogsRequest(); + options.setAppId(ConverterUtils.toApplicationId(appId)); + options.setContainerId(containerId); + options.setNodeId(nodeId); + options.setAppOwner(jobOwner); + List logs = new ArrayList(); + options.setLogTypes(logs); + return dumpAContainersLogsForALogType(options, false); } @Private @@ -108,19 +117,21 @@ public class LogCLIHelpers implements Configurable { @Private @VisibleForTesting - public int dumpAContainersLogsForALogType(String appId, String containerId, - String nodeId, String jobOwner, List logType, String localDir) + public int dumpAContainersLogsForALogType(ContainerLogsRequest options) throws IOException { - return dumpAContainersLogsForALogType(appId, containerId, nodeId, - jobOwner, logType, true, localDir); + return dumpAContainersLogsForALogType(options, true); } @Private @VisibleForTesting - public int dumpAContainersLogsForALogType(String appId, String containerId, - String nodeId, String jobOwner, List logType, - boolean outputFailure, String localDir) throws IOException { - ApplicationId applicationId = ConverterUtils.toApplicationId(appId); + public int dumpAContainersLogsForALogType(ContainerLogsRequest options, + boolean outputFailure) throws IOException { + ApplicationId applicationId = options.getAppId(); + String jobOwner = options.getAppOwner(); + String nodeId = options.getNodeId(); + String containerId = options.getContainerId(); + String localDir = options.getOutputLocalDir(); + List logType = options.getLogTypes(); RemoteIterator nodeFiles = getRemoteNodeFileDir( applicationId, jobOwner); if (nodeFiles == null) { @@ -147,7 +158,7 @@ public class LogCLIHelpers implements Configurable { reader = new AggregatedLogFormat.LogReader(getConf(), thisNodeFile.getPath()); - if (logType == null) { + if (logType == null || logType.isEmpty()) { if (dumpAContainerLogs(containerId, reader, out, thisNodeFile.getModificationTime()) > -1) { foundContainerLogs = true; @@ -176,10 +187,13 @@ public class LogCLIHelpers implements Configurable { } @Private - public int dumpAContainersLogsForALogTypeWithoutNodeId(String appId, - String containerId, String jobOwner, List logType, - String localDir) throws IOException { - ApplicationId applicationId = ConverterUtils.toApplicationId(appId); + public int dumpAContainersLogsForALogTypeWithoutNodeId( + ContainerLogsRequest options) throws IOException { + ApplicationId applicationId = options.getAppId(); + String jobOwner = options.getAppOwner(); + String containerId = options.getContainerId(); + String localDir = options.getOutputLocalDir(); + List logType = options.getLogTypes(); RemoteIterator nodeFiles = getRemoteNodeFileDir( applicationId, jobOwner); if (nodeFiles == null) { @@ -206,7 +220,7 @@ public class LogCLIHelpers implements Configurable { containerId); out.println(containerId); out.println(StringUtils.repeat("=", containerId.length())); - if (logType == null) { + if (logType == null || logType.isEmpty()) { if (dumpAContainerLogs(containerId, reader, out, thisNodeFile.getModificationTime()) > -1) { foundContainerLogs = true; @@ -303,8 +317,12 @@ public class LogCLIHelpers implements Configurable { } @Private - public int dumpAllContainersLogs(ApplicationId appId, String appOwner, - String localDir) throws IOException { + public int dumpAllContainersLogs(ContainerLogsRequest options) + throws IOException { + ApplicationId appId = options.getAppId(); + String appOwner = options.getAppOwner(); + String localDir = options.getOutputLocalDir(); + List logTypes = options.getLogTypes(); RemoteIterator nodeFiles = getRemoteNodeFileDir( appId, appOwner); if (nodeFiles == null) { @@ -341,9 +359,18 @@ public class LogCLIHelpers implements Configurable { out.println(StringUtils.repeat("=", containerString.length())); while (true) { try { - LogReader.readAContainerLogsForALogType(valueStream, out, - thisNodeFile.getModificationTime()); - foundAnyLogs = true; + if (logTypes == null || logTypes.isEmpty()) { + LogReader.readAContainerLogsForALogType(valueStream, out, + thisNodeFile.getModificationTime()); + foundAnyLogs = true; + } else { + int result = LogReader.readContainerLogsForALogType( + valueStream, out, thisNodeFile.getModificationTime(), + logTypes); + if (result == 0) { + foundAnyLogs = true; + } + } } catch (EOFException eof) { break; } @@ -369,10 +396,13 @@ public class LogCLIHelpers implements Configurable { } @Private - public void printLogMetadata(ApplicationId appId, - String containerIdStr, String nodeId, String appOwner, + public void printLogMetadata(ContainerLogsRequest options, PrintStream out, PrintStream err) throws IOException { + ApplicationId appId = options.getAppId(); + String appOwner = options.getAppOwner(); + String nodeId = options.getNodeId(); + String containerIdStr = options.getContainerId(); boolean getAllContainers = (containerIdStr == null); String nodeIdStr = (nodeId == null) ? null : LogAggregationUtils.getNodeString(nodeId); @@ -443,8 +473,10 @@ public class LogCLIHelpers implements Configurable { } @Private - public void printNodesList(ApplicationId appId, String appOwner, + public void printNodesList(ContainerLogsRequest options, PrintStream out, PrintStream err) throws IOException { + ApplicationId appId = options.getAppId(); + String appOwner = options.getAppOwner(); RemoteIterator nodeFiles = getRemoteNodeFileDir( appId, appOwner); if (nodeFiles == null) { @@ -547,4 +579,63 @@ public class LogCLIHelpers implements Configurable { IOUtils.closeQuietly(out); } } + + @Private + public Set listContainerLogs(ContainerLogsRequest options) + throws IOException { + Set logTypes = new HashSet(); + ApplicationId appId = options.getAppId(); + String appOwner = options.getAppOwner(); + String nodeId = options.getNodeId(); + String containerIdStr = options.getContainerId(); + boolean getAllContainers = (containerIdStr == null); + String nodeIdStr = (nodeId == null) ? null + : LogAggregationUtils.getNodeString(nodeId); + RemoteIterator nodeFiles = getRemoteNodeFileDir( + appId, appOwner); + if (nodeFiles == null) { + return logTypes; + } + while (nodeFiles.hasNext()) { + FileStatus thisNodeFile = nodeFiles.next(); + if (nodeIdStr != null) { + if (!thisNodeFile.getPath().getName().contains(nodeIdStr)) { + continue; + } + } + if (!thisNodeFile.getPath().getName() + .endsWith(LogAggregationUtils.TMP_FILE_SUFFIX)) { + AggregatedLogFormat.LogReader reader = + new AggregatedLogFormat.LogReader(getConf(), + thisNodeFile.getPath()); + try { + DataInputStream valueStream; + LogKey key = new LogKey(); + valueStream = reader.next(key); + while (valueStream != null) { + if (getAllContainers || (key.toString().equals(containerIdStr))) { + while (true) { + try { + String logFile = LogReader.readContainerMetaDataAndSkipData( + valueStream, null); + logTypes.add(logFile); + } catch (EOFException eof) { + break; + } + } + if (!getAllContainers) { + break; + } + } + // Next container + key = new LogKey(); + valueStream = reader.next(key); + } + } finally { + reader.close(); + } + } + } + return logTypes; + } }