Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

YARN-5200. Enhanced "yarn logs" to be able to get a list of containers whose logs are aggregated via a "show_container_log_info" option. Contributed by Xuan Gong.

This commit is contained in:
Vinod Kumar Vavilapalli 2016-07-13 10:47:43 -07:00
parent 56142171b9
commit eb47163234
4 changed files with 392 additions and 124 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

321
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/LogsCLI.java
View File

@ -78,14 +78,30 @@ public class LogsCLI extends Configured implements Tool {
private static final String APP_OWNER_OPTION = "appOwner";
private static final String AM_CONTAINER_OPTION = "am";
private static final String CONTAINER_LOG_FILES = "logFiles";
private static final String SHOW_META_INFO = "show_meta_info";
private static final String LIST_NODES_OPTION = "list_nodes";
private static final String SHOW_APPLICATION_LOG_INFO
= "show_application_log_info";
private static final String SHOW_CONTAINER_LOG_INFO
= "show_container_log_info";
private static final String OUT_OPTION = "out";
private static final String SIZE_OPTION = "size";
public static final String HELP_CMD = "help";
private PrintStream outStream = System.out;
private YarnClient yarnClient = null;
@Override
public int run(String[] args) throws Exception {
try {
yarnClient = createYarnClient();
return runCommand(args);
} finally {
if (yarnClient != null) {
yarnClient.close();
}
}
}
private int runCommand(String[] args) throws Exception {
Options opts = createCommandOpts();
Options printOpts = createPrintOpts(opts);
if (args.length < 1) {
@ -102,8 +118,9 @@ public class LogsCLI extends Configured implements Tool {
String nodeAddress = null;
String appOwner = null;
boolean getAMContainerLogs = false;
boolean showMetaInfo = false;
boolean nodesList = false;
boolean showApplicationLogInfo = false;
boolean showContainerLogInfo = false;
String[] logFiles = null;
List<String> amContainersList = new ArrayList<String>();
String localDir = null;
@ -115,9 +132,11 @@ public class LogsCLI extends Configured implements Tool {
nodeAddress = commandLine.getOptionValue(NODE_ADDRESS_OPTION);
appOwner = commandLine.getOptionValue(APP_OWNER_OPTION);
getAMContainerLogs = commandLine.hasOption(AM_CONTAINER_OPTION);
showMetaInfo = commandLine.hasOption(SHOW_META_INFO);
nodesList = commandLine.hasOption(LIST_NODES_OPTION);
localDir = commandLine.getOptionValue(OUT_OPTION);
showApplicationLogInfo = commandLine.hasOption(
SHOW_APPLICATION_LOG_INFO);
showContainerLogInfo = commandLine.hasOption(SHOW_CONTAINER_LOG_INFO);
if (getAMContainerLogs) {
try {
amContainersList = parseAMContainer(commandLine, printOpts);
@ -172,6 +191,12 @@ public class LogsCLI extends Configured implements Tool {
}
}
if (showApplicationLogInfo && showContainerLogInfo) {
System.err.println("Invalid options. Can only accept one of "
+ "show_application_log_info/show_container_log_info.");
return -1;
}
LogCLIHelpers logCliHelper = new LogCLIHelpers();
logCliHelper.setConf(getConf());
@ -215,14 +240,17 @@ public class LogsCLI extends Configured implements Tool {
isApplicationFinished(appState), appOwner, nodeAddress, null,
containerIdStr, localDir, logs, bytes);
if (showMetaInfo) {
return showMetaInfo(request, logCliHelper);
if (showContainerLogInfo) {
return showContainerLogInfo(request, logCliHelper);
}
if (nodesList) {
return showNodeLists(request, logCliHelper);
}
if (showApplicationLogInfo) {
return showApplicationLogInfo(request, logCliHelper);
}
// To get am logs
if (getAMContainerLogs) {
return fetchAMContainerLogs(request, amContainersList,
@ -246,21 +274,15 @@ public class LogsCLI extends Configured implements Tool {
private ApplicationReport getApplicationReport(ApplicationId appId)
throws IOException, YarnException {
YarnClient yarnClient = createYarnClient();
try {
return yarnClient.getApplicationReport(appId);
} finally {
yarnClient.close();
}
return yarnClient.getApplicationReport(appId);
}
@VisibleForTesting
protected YarnClient createYarnClient() {
YarnClient yarnClient = YarnClient.createYarnClient();
yarnClient.init(getConf());
yarnClient.start();
return yarnClient;
YarnClient client = YarnClient.createYarnClient();
client.init(getConf());
client.start();
return client;
}
public static void main(String[] args) throws Exception {
@ -272,7 +294,7 @@ public class LogsCLI extends Configured implements Tool {
}
private void printHelpMessage(Options options) {
System.out.println("Retrieve logs for completed YARN applications.");
outStream.println("Retrieve logs for YARN applications.");
HelpFormatter formatter = new HelpFormatter();
formatter.printHelp("yarn logs -applicationId <application ID> [OPTIONS]",
new Options());
@ -336,9 +358,9 @@ public class LogsCLI extends Configured implements Tool {
return false;
}
private List<String> getContainerLogFiles(Configuration conf,
private List<PerLogFileInfo> getContainerLogFiles(Configuration conf,
String containerIdStr, String nodeHttpAddress) throws IOException {
List<String> logFiles = new ArrayList<>();
List<PerLogFileInfo> logFileInfos = new ArrayList<>();
Client webServiceClient = Client.create();
try {
WebResource webResource = webServiceClient
@ -354,7 +376,9 @@ public class LogsCLI extends Configured implements Tool {
JSONObject json = response.getEntity(JSONObject.class);
JSONArray array = json.getJSONArray("containerLogInfo");
for (int i = 0; i < array.length(); i++) {
logFiles.add(array.getJSONObject(i).getString("fileName"));
String fileName = array.getJSONObject(i).getString("fileName");
String fileSize = array.getJSONObject(i).getString("fileSize");
logFileInfos.add(new PerLogFileInfo(fileName, fileSize));
}
} catch (Exception e) {
System.err.println("Unable to parse json from webservice. Error:");
@ -367,7 +391,7 @@ public class LogsCLI extends Configured implements Tool {
System.err.println("Unable to fetch log files list");
throw new IOException(ex);
}
return logFiles;
return logFileInfos;
}
@Private
@ -378,15 +402,26 @@ public class LogsCLI extends Configured implements Tool {
String containerIdStr = request.getContainerId().toString();
String localDir = request.getOutputLocalDir();
String nodeHttpAddress = request.getNodeHttpAddress();
if (nodeHttpAddress == null || nodeHttpAddress.isEmpty()) {
System.err.println("Can not get the logs for the container: "
+ containerIdStr);
System.err.println("The node http address is required to get container "
+ "logs for the Running application.");
return -1;
}
String nodeId = request.getNodeId();
PrintStream out = logCliHelper.createPrintStream(localDir, nodeId,
containerIdStr);
try {
// fetch all the log files for the container
// filter the log files based on the given --logFiles pattern
List<String> allLogs=
List<PerLogFileInfo> allLogFileInfos=
getContainerLogFiles(getConf(), containerIdStr, nodeHttpAddress);
List<String> matchedFiles = getMatchedLogFiles(request, allLogs);
List<String> fileNames = new ArrayList<String>();
for (PerLogFileInfo fileInfo : allLogFileInfos) {
fileNames.add(fileInfo.getFileName());
}
List<String> matchedFiles = getMatchedLogFiles(request, fileNames);
if (matchedFiles.isEmpty()) {
System.err.println("Can not find any log file matching the pattern: "
+ request.getLogTypes() + " for the container: " + containerIdStr
@ -397,8 +432,8 @@ public class LogsCLI extends Configured implements Tool {
newOptions.setLogTypes(matchedFiles);
Client webServiceClient = Client.create();
String containerString = "\n\nContainer: " + containerIdStr + " on "
+ nodeId;
String containerString = String.format(
LogCLIHelpers.CONTAINER_ON_NODE_PATTERN, containerIdStr, nodeId);
out.println(containerString);
out.println(StringUtils.repeat("=", containerString.length()));
boolean foundAnyLogs = false;
@ -477,13 +512,8 @@ public class LogsCLI extends Configured implements Tool {
@VisibleForTesting
public ContainerReport getContainerReport(String containerIdStr)
throws YarnException, IOException {
YarnClient yarnClient = createYarnClient();
try {
return yarnClient.getContainerReport(
ContainerId.fromString(containerIdStr));
} finally {
yarnClient.close();
}
return yarnClient.getContainerReport(
ContainerId.fromString(containerIdStr));
}
private boolean isApplicationFinished(YarnApplicationState appState) {
@ -508,8 +538,10 @@ public class LogsCLI extends Configured implements Tool {
for (JSONObject amContainer : amContainersList) {
ContainerLogsRequest amRequest = new ContainerLogsRequest(request);
amRequest.setContainerId(amContainer.getString("containerId"));
amRequest.setNodeHttpAddress(
amContainer.getString("nodeHttpAddress"));
String httpAddress = amContainer.getString("nodeHttpAddress");
if (httpAddress != null && !httpAddress.isEmpty()) {
amRequest.setNodeHttpAddress(httpAddress);
}
amRequest.setNodeId(amContainer.getString("nodeId"));
requests.add(amRequest);
}
@ -545,8 +577,8 @@ public class LogsCLI extends Configured implements Tool {
for (ContainerLogsRequest amRequest : requests) {
outputAMContainerLogs(amRequest, conf, logCliHelper);
}
System.out.println();
System.out.println("Specified ALL for -am option. "
outStream.println();
outStream.println("Specified ALL for -am option. "
+ "Printed logs for all am containers.");
} else {
for (String amContainer : amContainers) {
@ -602,15 +634,13 @@ public class LogsCLI extends Configured implements Tool {
}
}
private int showMetaInfo(ContainerLogsRequest request,
LogCLIHelpers logCliHelper) throws IOException {
private int showContainerLogInfo(ContainerLogsRequest request,
LogCLIHelpers logCliHelper) throws IOException, YarnException {
if (!request.isAppFinished()) {
System.err.println("The -show_meta_info command can be only used "
+ "with finished applications");
return -1;
return printContainerInfoFromRunningApplication(request);
} else {
logCliHelper.printLogMetadata(request, System.out, System.err);
return 0;
return logCliHelper.printAContainerLogMetadata(
request, System.out, System.err);
}
}
@ -626,6 +656,33 @@ public class LogsCLI extends Configured implements Tool {
}
}
private int showApplicationLogInfo(ContainerLogsRequest request,
LogCLIHelpers logCliHelper) throws IOException, YarnException {
String appState = "Application State: "
+ (request.isAppFinished() ? "Completed." : "Running.");
if (!request.isAppFinished()) {
List<ContainerReport> reports =
getContainerReportsFromRunningApplication(request);
List<ContainerReport> filterReports = filterContainersInfo(
request, reports);
if (filterReports.isEmpty()) {
System.err.println("Can not find any containers for the application:"
+ request.getAppId() + ".");
return -1;
}
outStream.println(appState);
for (ContainerReport report : filterReports) {
outStream.println(String.format(LogCLIHelpers.CONTAINER_ON_NODE_PATTERN,
report.getContainerId(), report.getAssignedNode()));
}
return 0;
} else {
outStream.println(appState);
logCliHelper.printContainersList(request, System.out, System.err);
return 0;
}
}
private Options createCommandOpts() {
Options opts = new Options();
opts.addOption(HELP_CMD, false, "Displays help for all commands.");
@ -661,13 +718,16 @@ public class LogsCLI extends Configured implements Tool {
logFileOpt.setArgs(Option.UNLIMITED_VALUES);
logFileOpt.setArgName("Log File Name");
opts.addOption(logFileOpt);
opts.addOption(SHOW_META_INFO, false, "Show the log metadata, "
opts.addOption(SHOW_CONTAINER_LOG_INFO, false,
"Show the container log metadata, "
+ "including log-file names, the size of the log files. "
+ "You can combine this with --containerId to get log metadata for "
+ "the specific container, or with --nodeAddress to get log metadata "
+ "for all the containers on the specific NodeManager. "
+ "Currently, this option can only be used for finished "
+ "applications.");
+ "for all the containers on the specific NodeManager.");
opts.addOption(SHOW_APPLICATION_LOG_INFO, false, "Show the "
+ "containerIds which belong to the specific Application. "
+ "You can combine this with --nodeAddress to get containerIds "
+ "for all the containers on the specific NodeManager.");
opts.addOption(LIST_NODES_OPTION, false,
"Show the list of nodes that successfully aggregated logs. "
+ "This option can only be used with finished applications.");
@ -695,8 +755,9 @@ public class LogsCLI extends Configured implements Tool {
printOpts.addOption(commandOpts.getOption(APP_OWNER_OPTION));
printOpts.addOption(commandOpts.getOption(AM_CONTAINER_OPTION));
printOpts.addOption(commandOpts.getOption(CONTAINER_LOG_FILES));
printOpts.addOption(commandOpts.getOption(SHOW_META_INFO));
printOpts.addOption(commandOpts.getOption(LIST_NODES_OPTION));
printOpts.addOption(commandOpts.getOption(SHOW_APPLICATION_LOG_INFO));
printOpts.addOption(commandOpts.getOption(SHOW_CONTAINER_LOG_INFO));
printOpts.addOption(commandOpts.getOption(OUT_OPTION));
printOpts.addOption(commandOpts.getOption(SIZE_OPTION));
return printOpts;
@ -801,12 +862,14 @@ public class LogsCLI extends Configured implements Tool {
// the ContainerReport. In the containerReport, we could get
// nodeAddress and nodeHttpAddress
ContainerReport report = getContainerReport(containerIdStr);
nodeHttpAddress =
report.getNodeHttpAddress().replaceFirst(
WebAppUtils.getHttpSchemePrefix(getConf()), "");
nodeHttpAddress = report.getNodeHttpAddress();
if (nodeHttpAddress != null && !nodeHttpAddress.isEmpty()) {
nodeHttpAddress = nodeHttpAddress.replaceFirst(
WebAppUtils.getHttpSchemePrefix(getConf()), "");
request.setNodeHttpAddress(nodeHttpAddress);
}
nodeId = report.getAssignedNode().toString();
request.setNodeId(nodeId);
request.setNodeHttpAddress(nodeHttpAddress);
} catch (IOException | YarnException ex) {
if (isAppFinished) {
return printContainerLogsForFinishedApplicationWithoutNodeId(
@ -942,32 +1005,140 @@ public class LogsCLI extends Configured implements Tool {
ContainerLogsRequest options) throws YarnException, IOException {
List<ContainerLogsRequest> newOptionsList =
new ArrayList<ContainerLogsRequest>();
YarnClient yarnClient = createYarnClient();
try {
List<ApplicationAttemptReport> attempts =
yarnClient.getApplicationAttempts(options.getAppId());
for (ApplicationAttemptReport attempt : attempts) {
List<ContainerReport> containers = yarnClient.getContainers(
attempt.getApplicationAttemptId());
for (ContainerReport container : containers) {
ContainerLogsRequest newOptions = new ContainerLogsRequest(options);
newOptions.setContainerId(container.getContainerId().toString());
newOptions.setNodeId(container.getAssignedNode().toString());
newOptions.setNodeHttpAddress(container.getNodeHttpAddress()
.replaceFirst(WebAppUtils.getHttpSchemePrefix(getConf()), ""));
// if we do not specify the value for CONTAINER_LOG_FILES option,
// we will only output syslog
List<String> logFiles = newOptions.getLogTypes();
if (logFiles == null || logFiles.isEmpty()) {
logFiles = Arrays.asList("syslog");
newOptions.setLogTypes(logFiles);
List<ContainerReport> reports =
getContainerReportsFromRunningApplication(options);
for (ContainerReport container : reports) {
ContainerLogsRequest newOptions = new ContainerLogsRequest(options);
newOptions.setContainerId(container.getContainerId().toString());
newOptions.setNodeId(container.getAssignedNode().toString());
String httpAddress = container.getNodeHttpAddress();
if (httpAddress != null && !httpAddress.isEmpty()) {
newOptions.setNodeHttpAddress(httpAddress
.replaceFirst(WebAppUtils.getHttpSchemePrefix(getConf()), ""));
}
// if we do not specify the value for CONTAINER_LOG_FILES option,
// we will only output syslog
List<String> logFiles = newOptions.getLogTypes();
if (logFiles == null || logFiles.isEmpty()) {
logFiles = Arrays.asList("syslog");
newOptions.setLogTypes(logFiles);
}
newOptionsList.add(newOptions);
}
return newOptionsList;
}
private List<ContainerReport> getContainerReportsFromRunningApplication(
ContainerLogsRequest options) throws YarnException, IOException {
List<ContainerReport> reports = new ArrayList<ContainerReport>();
List<ApplicationAttemptReport> attempts =
yarnClient.getApplicationAttempts(options.getAppId());
for (ApplicationAttemptReport attempt : attempts) {
List<ContainerReport> containers = yarnClient.getContainers(
attempt.getApplicationAttemptId());
reports.addAll(containers);
}
return reports;
}
// filter the containerReports based on the nodeId and ContainerId
private List<ContainerReport> filterContainersInfo(
ContainerLogsRequest options, List<ContainerReport> containers) {
List<ContainerReport> filterReports = new ArrayList<ContainerReport>(
containers);
String nodeId = options.getNodeId();
boolean filterBasedOnNodeId = (nodeId != null && !nodeId.isEmpty());
String containerId = options.getContainerId();
boolean filterBasedOnContainerId = (containerId != null
&& !containerId.isEmpty());
if (filterBasedOnNodeId || filterBasedOnContainerId) {
// filter the reports based on the containerId and.or nodeId
for(ContainerReport report : containers) {
if (filterBasedOnContainerId) {
if (!report.getContainerId().toString()
.equalsIgnoreCase(containerId)) {
filterReports.remove(report);
}
}
if (filterBasedOnNodeId) {
if (!report.getAssignedNode().toString().equalsIgnoreCase(nodeId)) {
filterReports.remove(report);
}
newOptionsList.add(newOptions);
}
}
return newOptionsList;
} finally {
yarnClient.close();
}
return filterReports;
}
private int printContainerInfoFromRunningApplication(
ContainerLogsRequest options) throws YarnException, IOException {
String containerIdStr = options.getContainerId();
String nodeIdStr = options.getNodeId();
List<ContainerReport> reports =
getContainerReportsFromRunningApplication(options);
List<ContainerReport> filteredReports = filterContainersInfo(
options, reports);
if (filteredReports.isEmpty()) {
StringBuilder sb = new StringBuilder();
if (containerIdStr != null && !containerIdStr.isEmpty()) {
sb.append("Trying to get container with ContainerId: "
+ containerIdStr + "\n");
}
if (nodeIdStr != null && !nodeIdStr.isEmpty()) {
sb.append("Trying to get container from NodeManager: "
+ nodeIdStr + "\n");
}
sb.append("Can not find any matched containers for the application: "
+ options.getAppId());
System.err.println(sb.toString());
return -1;
}
for (ContainerReport report : filteredReports) {
String nodeId = report.getAssignedNode().toString();
String nodeHttpAddress = report.getNodeHttpAddress().replaceFirst(
WebAppUtils.getHttpSchemePrefix(getConf()), "");
String containerId = report.getContainerId().toString();
String containerString = String.format(
LogCLIHelpers.CONTAINER_ON_NODE_PATTERN, containerId, nodeId);
outStream.println(containerString);
outStream.println(StringUtils.repeat("=", containerString.length()));
outStream.printf(LogCLIHelpers.PER_LOG_FILE_INFO_PATTERN,
"LogType", "LogLength");
outStream.println(StringUtils.repeat("=", containerString.length()));
List<PerLogFileInfo> infos = getContainerLogFiles(
getConf(), containerId, nodeHttpAddress);
for (PerLogFileInfo info : infos) {
outStream.printf(LogCLIHelpers.PER_LOG_FILE_INFO_PATTERN,
info.getFileName(), info.getFileLength());
}
}
return 0;
}
private static class PerLogFileInfo {
private String fileName;
private String fileLength;
public PerLogFileInfo(String fileName, String fileLength) {
setFileName(fileName);
setFileLength(fileLength);
}
public String getFileName() {
return fileName;
}
public void setFileName(String fileName) {
this.fileName = fileName;
}
public String getFileLength() {
return fileLength;
}
public void setFileLength(String fileLength) {
this.fileLength = fileLength;
}
}
}

85
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java
View File

@ -182,7 +182,7 @@ public class TestLogsCLI {
assertTrue(exitCode == -1);
ByteArrayOutputStream baos = new ByteArrayOutputStream();
PrintWriter pw = new PrintWriter(baos);
pw.println("Retrieve logs for completed YARN applications.");
pw.println("Retrieve logs for YARN applications.");
pw.println("usage: yarn logs -applicationId <application ID> [OPTIONS]");
pw.println();
pw.println("general options are:");
@ -217,14 +217,18 @@ public class TestLogsCLI {
pw.println(" container logs. The container logs will");
pw.println(" be stored based on the node the container");
pw.println(" ran on.");
pw.println(" -show_meta_info Show the log metadata, including log-file");
pw.println(" names, the size of the log files. You can");
pw.println(" combine this with --containerId to get");
pw.println(" log metadata for the specific container,");
pw.println(" or with --nodeAddress to get log metadata");
pw.println(" for all the containers on the specific");
pw.println(" NodeManager. Currently, this option can");
pw.println(" only be used for finished applications.");
pw.println(" -show_application_log_info Show the containerIds which belong to the");
pw.println(" specific Application. You can combine");
pw.println(" this with --nodeAddress to get");
pw.println(" containerIds for all the containers on");
pw.println(" the specific NodeManager.");
pw.println(" -show_container_log_info Show the container log metadata,");
pw.println(" including log-file names, the size of the");
pw.println(" log files. You can combine this with");
pw.println(" --containerId to get log metadata for the");
pw.println(" specific container, or with --nodeAddress");
pw.println(" to get log metadata for all the");
pw.println(" containers on the specific NodeManager.");
pw.println(" -size <size> Prints the log file's first 'n' bytes or");
pw.println(" the last 'n' bytes. Use negative values");
pw.println(" as bytes to read from the end and");
@ -698,9 +702,8 @@ public class TestLogsCLI {
"-applicationId", appTest.toString()});
assertTrue(exitCode == -1);
assertTrue(sysErrStream.toString().contains(
"Guessed logs' owner is " + priorityUser + " and current user "
+ UserGroupInformation.getCurrentUser().getUserName()
+ " does not have permission to access"));
"Can not find the logs for the application: "
+ appTest.toString()));
sysErrStream.reset();
} finally {
fs.delete(new Path(remoteLogRootDir), true);
@ -842,49 +845,77 @@ public class TestLogsCLI {
LogsCLI cli = new LogsCLIForTest(mockYarnClient);
cli.setConf(configuration);
cli.run(new String[] { "-applicationId", appId.toString(),
"-show_meta_info" });
int result = cli.run(new String[] {"-applicationId", appId.toString(),
"-show_container_log_info", "-show_application_log_info"});
assertTrue(result == -1);
assertTrue(sysErrStream.toString().contains("Invalid options. "
+ "Can only accept one of show_application_log_info/"
+ "show_container_log_info."));
cli.run(new String[] {"-applicationId", appId.toString(),
"-show_container_log_info"});
assertTrue(sysOutStream.toString().contains(
"Container: container_0_0001_01_000001 on localhost_"));
assertTrue(sysOutStream.toString().contains(
"Container: container_0_0001_01_000002 on localhost_"));
assertTrue(sysOutStream.toString().contains(
"LogType:syslog"));
"syslog"));
assertTrue(sysOutStream.toString().contains(
"LogLength:43"));
"43"));
sysOutStream.reset();
cli.run(new String[] { "-applicationId", appId.toString(),
"-show_meta_info", "-containerId", "container_0_0001_01_000001" });
cli.run(new String[] {"-applicationId", appId.toString(),
"-show_container_log_info", "-containerId",
"container_0_0001_01_000001"});
assertTrue(sysOutStream.toString().contains(
"Container: container_0_0001_01_000001 on localhost_"));
assertFalse(sysOutStream.toString().contains(
"Container: container_0_0001_01_000002 on localhost_"));
assertTrue(sysOutStream.toString().contains(
"LogType:syslog"));
"syslog"));
assertTrue(sysOutStream.toString().contains(
"LogLength:43"));
"43"));
sysOutStream.reset();
cli.run(new String[] { "-applicationId", appId.toString(),
"-show_meta_info", "-nodeAddress", "localhost" });
cli.run(new String[] {"-applicationId", appId.toString(),
"-show_container_log_info", "-nodeAddress", "localhost"});
assertTrue(sysOutStream.toString().contains(
"Container: container_0_0001_01_000001 on localhost_"));
assertTrue(sysOutStream.toString().contains(
"Container: container_0_0001_01_000002 on localhost_"));
assertTrue(sysOutStream.toString().contains(
"LogType:syslog"));
"syslog"));
assertTrue(sysOutStream.toString().contains(
"LogLength:43"));
"43"));
sysOutStream.reset();
cli.run(new String[] { "-applicationId", appId.toString(),
"-show_meta_info", "-nodeAddress", "localhost", "-containerId",
"container_1234" });
cli.run(new String[] {"-applicationId", appId.toString(),
"-show_container_log_info", "-nodeAddress", "localhost",
"-containerId", "container_1234"});
assertTrue(sysErrStream.toString().contains(
"Invalid ContainerId specified"));
sysErrStream.reset();
cli.run(new String[] {"-applicationId", appId.toString(),
"-show_application_log_info"});
assertTrue(sysOutStream.toString().contains(
"Application State: Completed."));
assertTrue(sysOutStream.toString().contains(
"container_0_0001_01_000001 on localhost"));
assertTrue(sysOutStream.toString().contains(
"container_0_0001_01_000002 on localhost"));
sysOutStream.reset();
cli.run(new String[] {"-applicationId", appId.toString(),
"-show_application_log_info", "-nodeAddress", "localhost"});
assertTrue(sysOutStream.toString().contains(
"Application State: Completed."));
assertTrue(sysOutStream.toString().contains(
"container_0_0001_01_000001 on localhost"));
assertTrue(sysOutStream.toString().contains(
"container_0_0001_01_000002 on localhost"));
sysOutStream.reset();
fs.delete(new Path(remoteLogRootDir), true);
fs.delete(new Path(rootLogDir), true);
}

13
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/AggregatedLogFormat.java
View File

@ -49,6 +49,7 @@ import org.apache.commons.io.input.BoundedInputStream;
import org.apache.commons.io.output.WriterOutputStream;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.commons.math3.util.Pair;
import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.classification.InterfaceAudience.Public;
import org.apache.hadoop.classification.InterfaceStability.Evolving;
@ -959,25 +960,21 @@ public class AggregatedLogFormat {
}
@Private
public static String readContainerMetaDataAndSkipData(
public static Pair<String, String> readContainerMetaDataAndSkipData(
DataInputStream valueStream, PrintStream out) throws IOException {
String fileType = valueStream.readUTF();
String fileLengthStr = valueStream.readUTF();
long fileLength = Long.parseLong(fileLengthStr);
if (out != null) {
out.print("LogType:");
out.println(fileType);
out.print("LogLength:");
out.println(fileLengthStr);
}
Pair<String, String> logMeta = new Pair<String, String>(
fileType, fileLengthStr);
long totalSkipped = 0;
long currSkipped = 0;
while (currSkipped != -1 && totalSkipped < fileLength) {
currSkipped = valueStream.skip(fileLength - totalSkipped);
totalSkipped += currSkipped;
}
return fileType;
return logMeta;
}
public void close() {

97
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/LogCLIHelpers.java
View File

@ -32,6 +32,7 @@ import java.util.List;
import java.util.Set;
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang.StringUtils;
import org.apache.commons.math3.util.Pair;
import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.conf.Configurable;
import org.apache.hadoop.conf.Configuration;
@ -50,6 +51,11 @@ import com.google.common.annotations.VisibleForTesting;
public class LogCLIHelpers implements Configurable {
public static final String PER_LOG_FILE_INFO_PATTERN =
"%20s\t%20s" + System.getProperty("line.separator");
public static final String CONTAINER_ON_NODE_PATTERN =
"Container: %s on %s";
private Configuration conf;
@Private
@ -151,8 +157,8 @@ public class LogCLIHelpers implements Configurable {
AggregatedLogFormat.LogReader reader = null;
PrintStream out = createPrintStream(localDir, fileName, containerId);
try {
String containerString = "\n\nContainer: " + containerId + " on "
+ thisNodeFile.getPath().getName();
String containerString = String.format(CONTAINER_ON_NODE_PATTERN,
containerId, thisNodeFile.getPath().getName());
out.println(containerString);
out.println(StringUtils.repeat("=", containerString.length()));
reader =
@ -219,8 +225,10 @@ public class LogCLIHelpers implements Configurable {
thisNodeFile.getPath());
out = createPrintStream(localDir, thisNodeFile.getPath().getName(),
containerId);
out.println(containerId + " on " + thisNodeFile.getPath().getName());
out.println(StringUtils.repeat("=", containerId.length()));
String containerString = String.format(CONTAINER_ON_NODE_PATTERN,
containerId, thisNodeFile.getPath().getName());
out.println(containerString);
out.println(StringUtils.repeat("=", containerString.length()));
if (logType == null || logType.isEmpty()) {
if (dumpAContainerLogs(containerId, reader, out,
thisNodeFile.getModificationTime(), options.getBytes()) > -1) {
@ -355,9 +363,9 @@ public class LogCLIHelpers implements Configurable {
PrintStream out = createPrintStream(localDir,
thisNodeFile.getPath().getName(), key.toString());
try {
String containerString =
"\n\nContainer: " + key + " on "
+ thisNodeFile.getPath().getName();
String containerString = String.format(
CONTAINER_ON_NODE_PATTERN, key,
thisNodeFile.getPath().getName());
out.println(containerString);
out.println(StringUtils.repeat("=", containerString.length()));
while (true) {
@ -400,7 +408,7 @@ public class LogCLIHelpers implements Configurable {
}
@Private
public void printLogMetadata(ContainerLogsRequest options,
public int printAContainerLogMetadata(ContainerLogsRequest options,
PrintStream out, PrintStream err)
throws IOException {
ApplicationId appId = options.getAppId();
@ -413,7 +421,7 @@ public class LogCLIHelpers implements Configurable {
RemoteIterator<FileStatus> nodeFiles = getRemoteNodeFileDir(
appId, appOwner);
if (nodeFiles == null) {
return;
return -1;
}
boolean foundAnyLogs = false;
while (nodeFiles.hasNext()) {
@ -434,16 +442,21 @@ public class LogCLIHelpers implements Configurable {
valueStream = reader.next(key);
while (valueStream != null) {
if (getAllContainers || (key.toString().equals(containerIdStr))) {
String containerString =
"\n\nContainer: " + key + " on "
+ thisNodeFile.getPath().getName();
String containerString = String.format(CONTAINER_ON_NODE_PATTERN,
key, thisNodeFile.getPath().getName());
out.println(containerString);
out.println("Log Upload Time:"
+ thisNodeFile.getModificationTime());
out.println(StringUtils.repeat("=", containerString.length()));
out.printf(PER_LOG_FILE_INFO_PATTERN, "LogType", "LogLength");
out.println(StringUtils.repeat("=", containerString.length()));
while (true) {
try {
LogReader.readContainerMetaDataAndSkipData(valueStream, out);
Pair<String, String> logMeta =
LogReader.readContainerMetaDataAndSkipData(
valueStream, out);
out.printf(PER_LOG_FILE_INFO_PATTERN,
logMeta.getFirst(), logMeta.getSecond());
} catch (EOFException eof) {
break;
}
@ -473,7 +486,9 @@ public class LogCLIHelpers implements Configurable {
err.println("Can not find log metadata for container: "
+ containerIdStr);
}
return -1;
}
return 0;
}
@Private
@ -501,6 +516,60 @@ public class LogCLIHelpers implements Configurable {
}
}
@Private
public void printContainersList(ContainerLogsRequest options,
PrintStream out, PrintStream err) throws IOException {
ApplicationId appId = options.getAppId();
String appOwner = options.getAppOwner();
String nodeId = options.getNodeId();
String nodeIdStr = (nodeId == null) ? null
: LogAggregationUtils.getNodeString(nodeId);
RemoteIterator<FileStatus> nodeFiles = getRemoteNodeFileDir(
appId, appOwner);
if (nodeFiles == null) {
return;
}
boolean foundAnyLogs = false;
while (nodeFiles.hasNext()) {
FileStatus thisNodeFile = nodeFiles.next();
if (nodeIdStr != null) {
if (!thisNodeFile.getPath().getName().contains(nodeIdStr)) {
continue;
}
}
if (!thisNodeFile.getPath().getName()
.endsWith(LogAggregationUtils.TMP_FILE_SUFFIX)) {
AggregatedLogFormat.LogReader reader =
new AggregatedLogFormat.LogReader(getConf(),
thisNodeFile.getPath());
try {
DataInputStream valueStream;
LogKey key = new LogKey();
valueStream = reader.next(key);
while (valueStream != null) {
out.println(String.format(CONTAINER_ON_NODE_PATTERN, key,
thisNodeFile.getPath().getName()));
foundAnyLogs = true;
// Next container
key = new LogKey();
valueStream = reader.next(key);
}
} finally {
reader.close();
}
}
}
if (!foundAnyLogs) {
if (nodeId != null) {
err.println("Can not find information for any containers on "
+ nodeId);
} else {
err.println("Can not find any container information for "
+ "the application: " + appId);
}
}
}
private RemoteIterator<FileStatus> getRemoteNodeFileDir(ApplicationId appId,
String appOwner) throws IOException {
Path remoteAppLogDir = getRemoteAppLogDir(appId, appOwner);
@ -621,7 +690,7 @@ public class LogCLIHelpers implements Configurable {
while (true) {
try {
String logFile = LogReader.readContainerMetaDataAndSkipData(
valueStream, null);
valueStream, null).getFirst();
logTypes.add(logFile);
} catch (EOFException eof) {
break;