From adcecd417b55ba38c49c7e8626e0a60511ea802e Mon Sep 17 00:00:00 2001 From: Junping Du Date: Fri, 5 Aug 2016 10:54:36 -0700 Subject: [PATCH] YARN-5470. Differentiate exactly match with regex in yarn log CLI. Contributed by Xuan Gong. (cherry picked from commit e605d47df05619c6b1c18aca59f709899498da75) --- .../hadoop/yarn/client/cli/LogsCLI.java | 125 ++++++++++-------- .../hadoop/yarn/client/cli/TestLogsCLI.java | 69 ++++++++-- .../logaggregation/ContainerLogsRequest.java | 10 +- .../yarn/logaggregation/LogCLIHelpers.java | 8 +- 4 files changed, 137 insertions(+), 75 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/LogsCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/LogsCLI.java index 12338fd1d06..985f878574f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/LogsCLI.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/LogsCLI.java @@ -25,6 +25,7 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; import java.util.Collections; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; @@ -90,6 +91,7 @@ public class LogsCLI extends Configured implements Tool { = "show_container_log_info"; private static final String OUT_OPTION = "out"; private static final String SIZE_OPTION = "size"; + private static final String REGEX_OPTION = "regex"; public static final String HELP_CMD = "help"; private PrintStream outStream = System.out; private YarnClient yarnClient = null; @@ -126,6 +128,7 @@ public class LogsCLI extends Configured implements Tool { boolean nodesList = false; boolean showApplicationLogInfo = false; boolean showContainerLogInfo = false; + boolean useRegex = false; String[] logFiles = null; List amContainersList = new ArrayList(); String localDir = null; @@ -142,6 +145,7 @@ public class LogsCLI extends Configured implements Tool { showApplicationLogInfo = commandLine.hasOption( SHOW_APPLICATION_LOG_INFO); showContainerLogInfo = commandLine.hasOption(SHOW_CONTAINER_LOG_INFO); + useRegex = commandLine.hasOption(REGEX_OPTION); if (getAMContainerLogs) { try { amContainersList = parseAMContainer(commandLine, printOpts); @@ -243,11 +247,11 @@ public class LogsCLI extends Configured implements Tool { } } - List logs = new ArrayList(); - if (fetchAllLogFiles(logFiles)) { - logs.add(".*"); + Set logs = new HashSet(); + if (fetchAllLogFiles(logFiles, useRegex)) { + logs.add("ALL"); } else if (logFiles != null && logFiles.length > 0) { - logs = Arrays.asList(logFiles); + logs.addAll(Arrays.asList(logFiles)); } ContainerLogsRequest request = new ContainerLogsRequest(appId, @@ -268,15 +272,15 @@ public class LogsCLI extends Configured implements Tool { // To get am logs if (getAMContainerLogs) { return fetchAMContainerLogs(request, amContainersList, - logCliHelper); + logCliHelper, useRegex); } int resultCode = 0; if (containerIdStr != null) { - return fetchContainerLogs(request, logCliHelper); + return fetchContainerLogs(request, logCliHelper, useRegex); } else { if (nodeAddress == null) { - resultCode = fetchApplicationLogs(request, logCliHelper); + resultCode = fetchApplicationLogs(request, logCliHelper, useRegex); } else { System.err.println("Should at least provide ContainerId!"); printHelpMessage(printOpts); @@ -362,7 +366,7 @@ public class LogsCLI extends Configured implements Tool { return amContainersList; } - private boolean fetchAllLogFiles(String[] logFiles) { + private boolean fetchAllLogFiles(String[] logFiles, boolean useRegex) { // If no value is specified for the PER_CONTAINER_LOG_FILES_OPTION option, // we will assume all logs. @@ -371,7 +375,8 @@ public class LogsCLI extends Configured implements Tool { } List logs = Arrays.asList(logFiles); - if (logs.contains("ALL") || logs.contains(".*")) { + if (logs.contains("ALL") || logs.contains("*")|| + (logs.contains(".*") && useRegex)) { return true; } @@ -418,8 +423,8 @@ public class LogsCLI extends Configured implements Tool { @Private @VisibleForTesting public int printContainerLogsFromRunningApplication(Configuration conf, - ContainerLogsRequest request, LogCLIHelpers logCliHelper) - throws IOException { + ContainerLogsRequest request, LogCLIHelpers logCliHelper, + boolean useRegex) throws IOException { String containerIdStr = request.getContainerId().toString(); String localDir = request.getOutputLocalDir(); String nodeHttpAddress = request.getNodeHttpAddress(); @@ -442,7 +447,8 @@ public class LogsCLI extends Configured implements Tool { for (PerLogFileInfo fileInfo : allLogFileInfos) { fileNames.add(fileInfo.getFileName()); } - List matchedFiles = getMatchedLogFiles(request, fileNames); + Set matchedFiles = getMatchedLogFiles(request, fileNames, + useRegex); if (matchedFiles.isEmpty()) { System.err.println("Can not find any log file matching the pattern: " + request.getLogTypes() + " for the container: " + containerIdStr @@ -504,10 +510,10 @@ public class LogsCLI extends Configured implements Tool { } private int printContainerLogsForFinishedApplication( - ContainerLogsRequest request, LogCLIHelpers logCliHelper) - throws IOException { + ContainerLogsRequest request, LogCLIHelpers logCliHelper, + boolean useRegex) throws IOException { ContainerLogsRequest newOptions = getMatchedLogOptions( - request, logCliHelper); + request, logCliHelper, useRegex); if (newOptions == null) { System.err.println("Can not find any log file matching the pattern: " + request.getLogTypes() + " for the container: " @@ -519,10 +525,10 @@ public class LogsCLI extends Configured implements Tool { } private int printContainerLogsForFinishedApplicationWithoutNodeId( - ContainerLogsRequest request, LogCLIHelpers logCliHelper) - throws IOException { + ContainerLogsRequest request, LogCLIHelpers logCliHelper, + boolean useRegex) throws IOException { ContainerLogsRequest newOptions = getMatchedLogOptions( - request, logCliHelper); + request, logCliHelper, useRegex); if (newOptions == null) { System.err.println("Can not find any log file matching the pattern: " + request.getLogTypes() + " for the container: " @@ -550,7 +556,7 @@ public class LogsCLI extends Configured implements Tool { private int printAMContainerLogs(Configuration conf, ContainerLogsRequest request, List amContainers, - LogCLIHelpers logCliHelper) throws Exception { + LogCLIHelpers logCliHelper, boolean useRegex) throws Exception { List amContainersList = null; List requests = new ArrayList(); @@ -614,7 +620,7 @@ public class LogsCLI extends Configured implements Tool { if (amContainers.contains("ALL")) { for (ContainerLogsRequest amRequest : requests) { - outputAMContainerLogs(amRequest, conf, logCliHelper); + outputAMContainerLogs(amRequest, conf, logCliHelper, useRegex); } outStream.println(); outStream.println("Specified ALL for -am option. " @@ -624,11 +630,11 @@ public class LogsCLI extends Configured implements Tool { int amContainerId = Integer.parseInt(amContainer.trim()); if (amContainerId == -1) { outputAMContainerLogs(requests.get(requests.size() - 1), conf, - logCliHelper); + logCliHelper, useRegex); } else { if (amContainerId <= requests.size()) { outputAMContainerLogs(requests.get(amContainerId - 1), conf, - logCliHelper); + logCliHelper, useRegex); } else { System.err.println(String.format("ERROR: Specified AM containerId" + " (%s) exceeds the number of AM containers (%s).", @@ -642,7 +648,8 @@ public class LogsCLI extends Configured implements Tool { } private void outputAMContainerLogs(ContainerLogsRequest request, - Configuration conf, LogCLIHelpers logCliHelper) throws Exception { + Configuration conf, LogCLIHelpers logCliHelper, boolean useRegex) + throws Exception { String nodeHttpAddress = request.getNodeHttpAddress(); String containerId = request.getContainerId(); String nodeId = request.getNodeId(); @@ -651,10 +658,10 @@ public class LogsCLI extends Configured implements Tool { if (containerId != null && !containerId.isEmpty()) { if (nodeId != null && !nodeId.isEmpty()) { printContainerLogsForFinishedApplication(request, - logCliHelper); + logCliHelper, useRegex); } else { printContainerLogsForFinishedApplicationWithoutNodeId( - request, logCliHelper); + request, logCliHelper, useRegex); } } } else { @@ -664,7 +671,7 @@ public class LogsCLI extends Configured implements Tool { .getContainerState(); request.setContainerState(containerState); printContainerLogsFromRunningApplication(conf, - request, logCliHelper); + request, logCliHelper, useRegex); } } } @@ -747,12 +754,15 @@ public class LogsCLI extends Configured implements Tool { opts.addOption(amOption); Option logFileOpt = new Option(PER_CONTAINER_LOG_FILES_OPTION, true, "Specify comma-separated value " - + "to get specified container log files. Use \"ALL\" to fetch all the " - + "log files for the container. It also supports Java Regex."); + + "to get exact matched log files. Use \"ALL\" or \"*\"to " + + "fetch all the log files for the container. Specific -regex " + + "for using java regex to find matched log files."); logFileOpt.setValueSeparator(','); logFileOpt.setArgs(Option.UNLIMITED_VALUES); logFileOpt.setArgName("Log File Name"); opts.addOption(logFileOpt); + opts.addOption(REGEX_OPTION, false, "Work with -log_files to find " + + "matched files by using java regex."); opts.addOption(SHOW_CONTAINER_LOG_INFO, false, "Show the container log metadata, " + "including log-file names, the size of the log files. " @@ -795,6 +805,7 @@ public class LogsCLI extends Configured implements Tool { printOpts.addOption(commandOpts.getOption(SHOW_CONTAINER_LOG_INFO)); printOpts.addOption(commandOpts.getOption(OUT_OPTION)); printOpts.addOption(commandOpts.getOption(SIZE_OPTION)); + printOpts.addOption(commandOpts.getOption(REGEX_OPTION)); return printOpts; } @@ -830,14 +841,14 @@ public class LogsCLI extends Configured implements Tool { } private int fetchAMContainerLogs(ContainerLogsRequest request, - List amContainersList, LogCLIHelpers logCliHelper) - throws Exception { + List amContainersList, LogCLIHelpers logCliHelper, + boolean useRegex) throws Exception { return printAMContainerLogs(getConf(), request, amContainersList, - logCliHelper); + logCliHelper, useRegex); } private int fetchContainerLogs(ContainerLogsRequest request, - LogCLIHelpers logCliHelper) throws IOException { + LogCLIHelpers logCliHelper, boolean useRegex) throws IOException { int resultCode = 0; String appIdStr = request.getAppId().toString(); String containerIdStr = request.getContainerId(); @@ -851,10 +862,10 @@ public class LogsCLI extends Configured implements Tool { // to logCliHelper so that it fetches all the logs if (nodeAddress != null && !nodeAddress.isEmpty()) { return printContainerLogsForFinishedApplication( - request, logCliHelper); + request, logCliHelper, useRegex); } else { return printContainerLogsForFinishedApplicationWithoutNodeId( - request, logCliHelper); + request, logCliHelper, useRegex); } } String nodeHttpAddress = null; @@ -876,7 +887,7 @@ public class LogsCLI extends Configured implements Tool { } catch (IOException | YarnException ex) { if (isAppFinished) { return printContainerLogsForFinishedApplicationWithoutNodeId( - request, logCliHelper); + request, logCliHelper, useRegex); } else { System.err.println("Unable to get logs for this container:" + containerIdStr + "for the application:" + appIdStr @@ -893,18 +904,19 @@ public class LogsCLI extends Configured implements Tool { // by calling NodeManager webservice. if (!isAppFinished) { resultCode = printContainerLogsFromRunningApplication(getConf(), request, - logCliHelper); + logCliHelper, useRegex); } else { // If the application is in the final state, we will directly // get the container logs from HDFS. resultCode = printContainerLogsForFinishedApplication( - request, logCliHelper); + request, logCliHelper, useRegex); } return resultCode; } private int fetchApplicationLogs(ContainerLogsRequest options, - LogCLIHelpers logCliHelper) throws IOException, YarnException { + LogCLIHelpers logCliHelper, boolean useRegex) throws IOException, + YarnException { // If the application has finished, we would fetch the logs // from HDFS. // If the application is still running, we would get the full @@ -913,7 +925,7 @@ public class LogsCLI extends Configured implements Tool { int resultCode = -1; if (options.isAppFinished()) { ContainerLogsRequest newOptions = getMatchedLogOptions( - options, logCliHelper); + options, logCliHelper, useRegex); if (newOptions == null) { System.err.println("Can not find any log file matching the pattern: " + options.getLogTypes() + " for the application: " @@ -927,7 +939,7 @@ public class LogsCLI extends Configured implements Tool { getContainersLogRequestForRunningApplication(options); for (ContainerLogsRequest container : containerLogRequests) { int result = printContainerLogsFromRunningApplication(getConf(), - container, logCliHelper); + container, logCliHelper, useRegex); if (result == 0) { resultCode = 0; } @@ -956,14 +968,14 @@ public class LogsCLI extends Configured implements Tool { } private ContainerLogsRequest getMatchedLogOptions( - ContainerLogsRequest request, LogCLIHelpers logCliHelper) - throws IOException { + ContainerLogsRequest request, LogCLIHelpers logCliHelper, + boolean useRegex) throws IOException { ContainerLogsRequest newOptions = new ContainerLogsRequest(request); if (request.getLogTypes() != null && !request.getLogTypes().isEmpty()) { - List matchedFiles = new ArrayList(); - if (!request.getLogTypes().contains(".*")) { + Set matchedFiles = new HashSet(); + if (!request.getLogTypes().contains("ALL")) { Set files = logCliHelper.listContainerLogs(request); - matchedFiles = getMatchedLogFiles(request, files); + matchedFiles = getMatchedLogFiles(request, files, useRegex); if (matchedFiles.isEmpty()) { return null; } @@ -973,20 +985,29 @@ public class LogsCLI extends Configured implements Tool { return newOptions; } - private List getMatchedLogFiles(ContainerLogsRequest options, - Collection candidate) throws IOException { - List matchedFiles = new ArrayList(); - List filePattern = options.getLogTypes(); + private Set getMatchedLogFiles(ContainerLogsRequest options, + Collection candidate, boolean useRegex) throws IOException { + Set matchedFiles = new HashSet(); + Set filePattern = options.getLogTypes(); + if (options.getLogTypes().contains("ALL")) { + return new HashSet(candidate); + } for (String file : candidate) { - if (isFileMatching(file, filePattern)) { - matchedFiles.add(file); + if (useRegex) { + if (isFileMatching(file, filePattern)) { + matchedFiles.add(file); + } + } else { + if (filePattern.contains(file)) { + matchedFiles.add(file); + } } } return matchedFiles; } private boolean isFileMatching(String fileType, - List logTypes) { + Set logTypes) { for (String logType : logTypes) { Pattern filterPattern = Pattern.compile(logType); boolean match = filterPattern.matcher(fileType).find(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java index 329d8569e43..467b2043ab5 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java @@ -22,6 +22,7 @@ import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; import static org.mockito.Matchers.any; +import static org.mockito.Matchers.anyBoolean; import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.mock; @@ -46,7 +47,7 @@ import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; - +import java.util.Set; import org.apache.commons.io.IOUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileStatus; @@ -211,14 +212,17 @@ public class TestLogsCLI { pw.println(" aggregated logs. This option can only be"); pw.println(" used with finished applications."); pw.println(" -log_files Specify comma-separated value to get"); - pw.println(" specified container log files. Use \"ALL\""); - pw.println(" to fetch all the log files for the"); - pw.println(" container. It also supports Java Regex."); + pw.println(" exact matched log files. Use \"ALL\" or"); + pw.println(" \"*\"to fetch all the log files for the"); + pw.println(" container. Specific -regex for using java"); + pw.println(" regex to find matched log files."); pw.println(" -nodeAddress NodeAddress in the format nodename:port"); pw.println(" -out Local directory for storing individual"); pw.println(" container logs. The container logs will"); pw.println(" be stored based on the node the container"); pw.println(" ran on."); + pw.println(" -regex Work with -log_files to find matched"); + pw.println(" files by using java regex."); pw.println(" -show_application_log_info Show the containerIds which belong to the"); pw.println(" specific Application. You can combine"); pw.println(" this with --nodeAddress to get"); @@ -286,6 +290,7 @@ public class TestLogsCLI { // create two logs for container3 in localLogDir logTypes.add("stdout"); + logTypes.add("stdout1234"); createContainerLogInLocalDir(appLogsDir, containerId3, fs, logTypes); Path path = @@ -336,10 +341,12 @@ public class TestLogsCLI { logMessage(containerId3, "syslog"))); assertTrue(sysOutStream.toString().contains( logMessage(containerId3, "stdout"))); + assertTrue(sysOutStream.toString().contains( + logMessage(containerId3, "stdout1234"))); sysOutStream.reset(); exitCode = cli.run(new String[] {"-applicationId", appId.toString(), - "-log_files", ".*"}); + "-log_files", ".*", "-regex"}); assertTrue(exitCode == 0); assertTrue(sysOutStream.toString().contains( logMessage(containerId1, "syslog"))); @@ -349,11 +356,28 @@ public class TestLogsCLI { logMessage(containerId3, "syslog"))); assertTrue(sysOutStream.toString().contains( logMessage(containerId3, "stdout"))); + assertTrue(sysOutStream.toString().contains( + logMessage(containerId3, "stdout1234"))); + sysOutStream.reset(); + + exitCode = cli.run(new String[] {"-applicationId", appId.toString(), + "-log_files", "*"}); + assertTrue(exitCode == 0); + assertTrue(sysOutStream.toString().contains( + logMessage(containerId1, "syslog"))); + assertTrue(sysOutStream.toString().contains( + logMessage(containerId2, "syslog"))); + assertTrue(sysOutStream.toString().contains( + logMessage(containerId3, "syslog"))); + assertTrue(sysOutStream.toString().contains( + logMessage(containerId3, "stdout"))); + assertTrue(sysOutStream.toString().contains( + logMessage(containerId3, "stdout1234"))); int fullSize = sysOutStream.toByteArray().length; sysOutStream.reset(); exitCode = cli.run(new String[] {"-applicationId", appId.toString(), - "-log_files", "std*"}); + "-log_files", "stdout"}); assertTrue(exitCode == 0); assertFalse(sysOutStream.toString().contains( logMessage(containerId1, "syslog"))); @@ -363,6 +387,23 @@ public class TestLogsCLI { logMessage(containerId3, "syslog"))); assertTrue(sysOutStream.toString().contains( logMessage(containerId3, "stdout"))); + assertFalse(sysOutStream.toString().contains( + logMessage(containerId3, "stdout1234"))); + sysOutStream.reset(); + + exitCode = cli.run(new String[] {"-applicationId", appId.toString(), + "-log_files", "std*", "-regex"}); + assertTrue(exitCode == 0); + assertFalse(sysOutStream.toString().contains( + logMessage(containerId1, "syslog"))); + assertFalse(sysOutStream.toString().contains( + logMessage(containerId2, "syslog"))); + assertFalse(sysOutStream.toString().contains( + logMessage(containerId3, "syslog"))); + assertTrue(sysOutStream.toString().contains( + logMessage(containerId3, "stdout"))); + assertTrue(sysOutStream.toString().contains( + logMessage(containerId3, "stdout1234"))); sysOutStream.reset(); exitCode = cli.run(new String[] {"-applicationId", appId.toString(), @@ -376,7 +417,7 @@ public class TestLogsCLI { // specify the bytes which is larger than the actual file size, // we would get the full logs exitCode = cli.run(new String[] {"-applicationId", appId.toString(), - "-log_files", ".*", "-size", "10000" }); + "-log_files", "*", "-size", "10000" }); assertTrue(exitCode == 0); assertTrue(sysOutStream.toByteArray().length == fullSize); sysOutStream.reset(); @@ -568,7 +609,7 @@ public class TestLogsCLI { LogsCLI cli = spy(new LogsCLIForTest(mockYarnClient)); doReturn(0).when(cli).printContainerLogsFromRunningApplication( any(Configuration.class), any(ContainerLogsRequest.class), - any(LogCLIHelpers.class)); + any(LogCLIHelpers.class), anyBoolean()); cli.setConf(new YarnConfiguration()); int exitCode = cli.run(new String[] {"-applicationId", appId.toString()}); @@ -581,16 +622,16 @@ public class TestLogsCLI { // printContainerLogsFromRunningApplication twice verify(cli, times(2)).printContainerLogsFromRunningApplication( any(Configuration.class), logsRequestCaptor.capture(), - any(LogCLIHelpers.class)); + any(LogCLIHelpers.class), anyBoolean()); - // Verify that the log-type is * + // Verify that the log-type is "ALL" List capturedRequests = logsRequestCaptor.getAllValues(); Assert.assertEquals(2, capturedRequests.size()); - List logTypes0 = capturedRequests.get(0).getLogTypes(); - List logTypes1 = capturedRequests.get(1).getLogTypes(); - Assert.assertEquals(".*", logTypes0.get(0)); - Assert.assertEquals(".*", logTypes1.get(0)); + Set logTypes0 = capturedRequests.get(0).getLogTypes(); + Set logTypes1 = capturedRequests.get(1).getLogTypes(); + Assert.assertTrue(logTypes0.contains("ALL") && (logTypes0.size() == 1)); + Assert.assertTrue(logTypes1.contains("ALL") && (logTypes1.size() == 1)); } @Test (timeout = 15000) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/ContainerLogsRequest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/ContainerLogsRequest.java index af65fef444e..30aeb6cc78b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/ContainerLogsRequest.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/ContainerLogsRequest.java @@ -18,7 +18,7 @@ package org.apache.hadoop.yarn.logaggregation; -import java.util.List; +import java.util.Set; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ContainerState; @@ -30,7 +30,7 @@ public class ContainerLogsRequest { private String appOwner; private boolean appFinished; private String outputLocalDir; - private List logTypes; + private Set logTypes; private long bytes; private ContainerState containerState; @@ -52,7 +52,7 @@ public class ContainerLogsRequest { public ContainerLogsRequest(ApplicationId applicationId, boolean isAppFinished, String owner, String address, String httpAddress, String container, String localDir, - List logs, long bytes, ContainerState containerState) { + Set logs, long bytes, ContainerState containerState) { this.setAppId(applicationId); this.setAppFinished(isAppFinished); this.setAppOwner(owner); @@ -121,11 +121,11 @@ public class ContainerLogsRequest { this.outputLocalDir = outputLocalDir; } - public List getLogTypes() { + public Set getLogTypes() { return logTypes; } - public void setLogTypes(List logTypes) { + public void setLogTypes(Set logTypes) { this.logTypes = logTypes; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/LogCLIHelpers.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/LogCLIHelpers.java index 13934d2113d..3ea3fa65c55 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/LogCLIHelpers.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/LogCLIHelpers.java @@ -67,7 +67,7 @@ public class LogCLIHelpers implements Configurable { options.setContainerId(containerId); options.setNodeId(nodeId); options.setAppOwner(jobOwner); - List logs = new ArrayList(); + Set logs = new HashSet(); options.setLogTypes(logs); options.setBytes(Long.MAX_VALUE); return dumpAContainerLogsForLogType(options, false); @@ -136,7 +136,7 @@ public class LogCLIHelpers implements Configurable { String nodeId = options.getNodeId(); String containerId = options.getContainerId(); String localDir = options.getOutputLocalDir(); - List logType = options.getLogTypes(); + List logType = new ArrayList(options.getLogTypes()); RemoteIterator nodeFiles = getRemoteNodeFileDir( applicationId, jobOwner); if (nodeFiles == null) { @@ -208,7 +208,7 @@ public class LogCLIHelpers implements Configurable { String jobOwner = options.getAppOwner(); String containerId = options.getContainerId(); String localDir = options.getOutputLocalDir(); - List logType = options.getLogTypes(); + List logType = new ArrayList(options.getLogTypes()); RemoteIterator nodeFiles = getRemoteNodeFileDir( applicationId, jobOwner); if (nodeFiles == null) { @@ -344,7 +344,7 @@ public class LogCLIHelpers implements Configurable { ApplicationId appId = options.getAppId(); String appOwner = options.getAppOwner(); String localDir = options.getOutputLocalDir(); - List logTypes = options.getLogTypes(); + List logTypes = new ArrayList(options.getLogTypes()); RemoteIterator nodeFiles = getRemoteNodeFileDir( appId, appOwner); if (nodeFiles == null) {