From 2aa6f317f69e2d9a60be8badc87cdec86e7ef770 Mon Sep 17 00:00:00 2001 From: Junping Du Date: Wed, 15 Feb 2017 09:05:14 -0800 Subject: [PATCH] YARN-6174. Log files pattern should be same for both running and finished container. Contributed by Xuan Gong. (cherry picked from commit ce2d5bfa5f84e7e563980796549b56ef1e4bbf1e) --- .../ContainerLogAggregationType.java | 31 +++++++++++++++++++ .../yarn/logaggregation/LogToolUtils.java | 11 ++++--- .../webapp/AHSWebServices.java | 13 ++++---- .../webapp/TestAHSWebServices.java | 26 ++++++++++------ .../server/webapp/dao/ContainerLogsInfo.java | 8 ++--- .../nodemanager/webapp/NMWebServices.java | 11 ++++--- .../webapp/dao/NMContainerLogsInfo.java | 4 +-- .../nodemanager/webapp/TestNMWebServices.java | 19 ++++++------ 8 files changed, 82 insertions(+), 41 deletions(-) create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/ContainerLogAggregationType.java diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/ContainerLogAggregationType.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/ContainerLogAggregationType.java new file mode 100644 index 00000000000..664448bb865 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/ContainerLogAggregationType.java @@ -0,0 +1,31 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.logaggregation; + +/** + * Enumeration of various aggregation type of a container log. + */ +public enum ContainerLogAggregationType { + + /** The log is from NodeManager local log directory. */ + LOCAL, + + /** The log is from Remote FileSystem application log directory. */ + AGGREGATED +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/LogToolUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/LogToolUtils.java index a4def73ad0e..a519bf8d46c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/LogToolUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/LogToolUtils.java @@ -141,7 +141,7 @@ public final class LogToolUtils { public static void outputContainerLog(String containerId, String nodeId, String fileName, long fileLength, long outputSize, String lastModifiedTime, InputStream fis, OutputStream os, - byte[] buf, ContainerLogType logType) throws IOException { + byte[] buf, ContainerLogAggregationType logType) throws IOException { long toSkip = 0; long totalBytesToRead = fileLength; long skipAfterRead = 0; @@ -171,9 +171,9 @@ public final class LogToolUtils { LogToolUtils.CONTAINER_ON_NODE_PATTERN, containerId, nodeId); sb.append(containerStr + "\n"); - sb.append("LogType: " + logType + "\n"); + sb.append("LogAggregationType: " + logType + "\n"); sb.append(StringUtils.repeat("=", containerStr.length()) + "\n"); - sb.append("FileName:" + fileName + "\n"); + sb.append("LogType:" + fileName + "\n"); sb.append("LogLastModifiedTime:" + lastModifiedTime + "\n"); sb.append("LogLength:" + Long.toString(fileLength) + "\n"); sb.append("LogContents:\n"); @@ -240,9 +240,10 @@ public final class LogToolUtils { LogToolUtils.outputContainerLog(containerId, nodeId, fileType, fileLength, outputSize, Times.format(thisNodeFile.getModificationTime()), - valueStream, os, buf, ContainerLogType.AGGREGATED); + valueStream, os, buf, + ContainerLogAggregationType.AGGREGATED); StringBuilder sb = new StringBuilder(); - String endOfFile = "End of LogFile:" + fileType; + String endOfFile = "End of LogType:" + fileType; sb.append("\n" + endOfFile + "\n"); sb.append(StringUtils.repeat("*", endOfFile.length() + 50) + "\n\n"); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/AHSWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/AHSWebServices.java index 54d880de41f..13c1d5e4763 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/AHSWebServices.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/AHSWebServices.java @@ -54,7 +54,7 @@ import org.apache.hadoop.yarn.api.records.YarnApplicationState; import org.apache.hadoop.yarn.api.ApplicationBaseProtocol; import org.apache.hadoop.yarn.api.records.timeline.TimelineAbout; import org.apache.hadoop.yarn.logaggregation.ContainerLogMeta; -import org.apache.hadoop.yarn.logaggregation.ContainerLogType; +import org.apache.hadoop.yarn.logaggregation.ContainerLogAggregationType; import org.apache.hadoop.yarn.logaggregation.LogToolUtils; import org.apache.hadoop.yarn.server.webapp.WebServices; import org.apache.hadoop.yarn.server.webapp.YarnWebServiceParams; @@ -501,13 +501,14 @@ public class AHSWebServices extends WebServices { boolean findLogs = LogToolUtils.outputAggregatedContainerLog(conf, appId, appOwner, containerIdStr, nodeId, logFile, bytes, os, buf); if (!findLogs) { - throw new IOException("Can not find logs for container:" - + containerIdStr); + os.write(("Can not find logs for container:" + + containerIdStr).getBytes(Charset.forName("UTF-8"))); } else { if (printEmptyLocalContainerLog) { StringBuilder sb = new StringBuilder(); sb.append(containerIdStr + "\n"); - sb.append("LogType: " + ContainerLogType.LOCAL + "\n"); + sb.append("LogAggregationType: " + + ContainerLogAggregationType.LOCAL + "\n"); sb.append("LogContents:\n"); sb.append(getNoRedirectWarning() + "\n"); os.write(sb.toString().getBytes(Charset.forName("UTF-8"))); @@ -539,14 +540,14 @@ public class AHSWebServices extends WebServices { List containersLogsInfo = new ArrayList<>(); for (ContainerLogMeta meta : containerLogMeta) { ContainerLogsInfo logInfo = new ContainerLogsInfo(meta, - ContainerLogType.AGGREGATED); + ContainerLogAggregationType.AGGREGATED); containersLogsInfo.add(logInfo); } if (emptyLocalContainerLogMeta) { ContainerLogMeta emptyMeta = new ContainerLogMeta( containerIdStr, "N/A"); ContainerLogsInfo empty = new ContainerLogsInfo(emptyMeta, - ContainerLogType.LOCAL); + ContainerLogAggregationType.LOCAL); containersLogsInfo.add(empty); } GenericEntity> meta = new GenericEntity>(){}); assertTrue(responseText.size() == 2); for (ContainerLogsInfo logInfo : responseText) { - if(logInfo.getLogType().equals(ContainerLogType.AGGREGATED.toString())) { + if(logInfo.getLogType().equals( + ContainerLogAggregationType.AGGREGATED.toString())) { List logMeta = logInfo .getContainerLogsInfo(); assertTrue(logMeta.size() == 1); @@ -834,7 +837,8 @@ public class TestAHSWebServices extends JerseyTestBase { assertEquals(logMeta.get(0).getFileSize(), String.valueOf( content.length())); } else { - assertEquals(logInfo.getLogType(), ContainerLogType.LOCAL.toString()); + assertEquals(logInfo.getLogType(), + ContainerLogAggregationType.LOCAL.toString()); } } @@ -852,7 +856,8 @@ public class TestAHSWebServices extends JerseyTestBase { List>(){}); assertTrue(responseText.size() == 2); for (ContainerLogsInfo logInfo : responseText) { - if(logInfo.getLogType().equals(ContainerLogType.AGGREGATED.toString())) { + if(logInfo.getLogType().equals( + ContainerLogAggregationType.AGGREGATED.toString())) { List logMeta = logInfo .getContainerLogsInfo(); assertTrue(logMeta.size() == 1); @@ -860,7 +865,8 @@ public class TestAHSWebServices extends JerseyTestBase { assertEquals(logMeta.get(0).getFileSize(), String.valueOf( content.length())); } else { - assertEquals(logInfo.getLogType(), ContainerLogType.LOCAL.toString()); + assertEquals(logInfo.getLogType(), + ContainerLogAggregationType.LOCAL.toString()); } } } @@ -889,7 +895,7 @@ public class TestAHSWebServices extends JerseyTestBase { List>(){}); assertTrue(responseText.size() == 1); assertEquals(responseText.get(0).getLogType(), - ContainerLogType.AGGREGATED.toString()); + ContainerLogAggregationType.AGGREGATED.toString()); List logMeta = responseText.get(0) .getContainerLogsInfo(); assertTrue(logMeta.size() == 1); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/dao/ContainerLogsInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/dao/ContainerLogsInfo.java index fd3c43b9b64..bc3ab393bc5 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/dao/ContainerLogsInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/dao/ContainerLogsInfo.java @@ -26,7 +26,7 @@ import javax.xml.bind.annotation.XmlElement; import javax.xml.bind.annotation.XmlRootElement; import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.logaggregation.ContainerLogMeta; -import org.apache.hadoop.yarn.logaggregation.ContainerLogType; +import org.apache.hadoop.yarn.logaggregation.ContainerLogAggregationType; import org.apache.hadoop.yarn.logaggregation.PerContainerLogFileInfo; /** @@ -48,7 +48,7 @@ public class ContainerLogsInfo { @XmlElement(name = "containerLogInfo") protected List containerLogsInfo; - @XmlElement(name = "logType") + @XmlElement(name = "logAggregationType") protected String logType; @XmlElement(name = "containerId") @@ -60,8 +60,8 @@ public class ContainerLogsInfo { //JAXB needs this public ContainerLogsInfo() {} - public ContainerLogsInfo(ContainerLogMeta logMeta, ContainerLogType logType) - throws YarnException { + public ContainerLogsInfo(ContainerLogMeta logMeta, + ContainerLogAggregationType logType) throws YarnException { this.containerLogsInfo = new ArrayList( logMeta.getContainerLogMeta()); this.logType = logType.toString(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMWebServices.java index 080a737a288..655dc6bbc51 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMWebServices.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMWebServices.java @@ -54,7 +54,7 @@ import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.factories.RecordFactory; import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider; import org.apache.hadoop.yarn.logaggregation.ContainerLogMeta; -import org.apache.hadoop.yarn.logaggregation.ContainerLogType; +import org.apache.hadoop.yarn.logaggregation.ContainerLogAggregationType; import org.apache.hadoop.yarn.logaggregation.LogToolUtils; import org.apache.hadoop.yarn.server.nodemanager.Context; import org.apache.hadoop.yarn.server.nodemanager.ResourceView; @@ -244,7 +244,7 @@ public class NMWebServices { List containersLogsInfo = new ArrayList<>(); containersLogsInfo.add(new NMContainerLogsInfo( this.nmContext, containerId, - hsr.getRemoteUser(), ContainerLogType.LOCAL)); + hsr.getRemoteUser(), ContainerLogAggregationType.LOCAL)); // check whether we have aggregated logs in RemoteFS. If exists, show the // the log meta for the aggregated logs as well. ApplicationId appId = containerId.getApplicationAttemptId() @@ -259,7 +259,7 @@ public class NMWebServices { if (!containerLogMeta.isEmpty()) { for (ContainerLogMeta logMeta : containerLogMeta) { containersLogsInfo.add(new ContainerLogsInfo(logMeta, - ContainerLogType.AGGREGATED)); + ContainerLogAggregationType.AGGREGATED)); } } } catch (IOException ex) { @@ -421,9 +421,10 @@ public class NMWebServices { byte[] buf = new byte[bufferSize]; LogToolUtils.outputContainerLog(containerId.toString(), nmContext.getNodeId().toString(), outputFileName, fileLength, - bytes, lastModifiedTime, fis, os, buf, ContainerLogType.LOCAL); + bytes, lastModifiedTime, fis, os, buf, + ContainerLogAggregationType.LOCAL); StringBuilder sb = new StringBuilder(); - String endOfFile = "End of LogFile:" + outputFileName; + String endOfFile = "End of LogType:" + outputFileName; sb.append(endOfFile + "."); if (isRunning) { sb.append("This log file belongs to a running container (" diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/dao/NMContainerLogsInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/dao/NMContainerLogsInfo.java index d2bc6f2f443..5415e04eb47 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/dao/NMContainerLogsInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/dao/NMContainerLogsInfo.java @@ -26,7 +26,7 @@ import javax.xml.bind.annotation.XmlAccessorType; import javax.xml.bind.annotation.XmlRootElement; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.exceptions.YarnException; -import org.apache.hadoop.yarn.logaggregation.ContainerLogType; +import org.apache.hadoop.yarn.logaggregation.ContainerLogAggregationType; import org.apache.hadoop.yarn.logaggregation.PerContainerLogFileInfo; import org.apache.hadoop.yarn.server.nodemanager.Context; import org.apache.hadoop.yarn.server.nodemanager.webapp.ContainerLogsUtils; @@ -47,7 +47,7 @@ public class NMContainerLogsInfo extends ContainerLogsInfo { public NMContainerLogsInfo(final Context nmContext, final ContainerId containerId, String remoteUser, - ContainerLogType logType) throws YarnException { + ContainerLogAggregationType logType) throws YarnException { this.logType = logType.toString(); this.containerId = containerId.toString(); this.nodeId = nmContext.getNodeId().toString(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServices.java index 7bf3824a941..68a68272369 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServices.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServices.java @@ -48,7 +48,7 @@ import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.event.AsyncDispatcher; -import org.apache.hadoop.yarn.logaggregation.ContainerLogType; +import org.apache.hadoop.yarn.logaggregation.ContainerLogAggregationType; import org.apache.hadoop.yarn.logaggregation.PerContainerLogFileInfo; import org.apache.hadoop.yarn.logaggregation.TestContainerLogsUtils; import org.apache.hadoop.yarn.server.nodemanager.Context; @@ -522,7 +522,7 @@ public class TestNMWebServices extends JerseyTestBase { List>(){}); assertTrue(responseList.size() == 1); assertEquals(responseList.get(0).getLogType(), - ContainerLogType.LOCAL.toString()); + ContainerLogAggregationType.LOCAL.toString()); List logMeta = responseList.get(0) .getContainerLogsInfo(); assertTrue(logMeta.size() == 1); @@ -549,12 +549,13 @@ public class TestNMWebServices extends JerseyTestBase { assertEquals(responseList.size(), 2); for (ContainerLogsInfo logInfo : responseList) { if(logInfo.getLogType().equals( - ContainerLogType.AGGREGATED.toString())) { + ContainerLogAggregationType.AGGREGATED.toString())) { List meta = logInfo.getContainerLogsInfo(); assertTrue(meta.size() == 1); assertEquals(meta.get(0).getFileName(), aggregatedLogFile); } else { - assertEquals(logInfo.getLogType(), ContainerLogType.LOCAL.toString()); + assertEquals(logInfo.getLogType(), + ContainerLogAggregationType.LOCAL.toString()); List meta = logInfo.getContainerLogsInfo(); assertTrue(meta.size() == 1); assertEquals(meta.get(0).getFileName(), filename); @@ -569,11 +570,11 @@ public class TestNMWebServices extends JerseyTestBase { response = r.path(filename) .accept(MediaType.TEXT_PLAIN).get(ClientResponse.class); responseText = response.getEntity(String.class); - assertTrue(responseText.contains("LogType: " - + ContainerLogType.AGGREGATED)); + assertTrue(responseText.contains("LogAggregationType: " + + ContainerLogAggregationType.AGGREGATED)); assertTrue(responseText.contains(aggregatedLogMessage)); - assertTrue(responseText.contains("LogType: " - + ContainerLogType.LOCAL)); + assertTrue(responseText.contains("LogAggregationType: " + + ContainerLogAggregationType.LOCAL)); assertTrue(responseText.contains(logMessage)); } finally { FileUtil.fullyDelete(tempLogDir); @@ -676,7 +677,7 @@ public class TestNMWebServices extends JerseyTestBase { private String getLogContext(String fullMessage) { String prefix = "LogContents:\n"; - String postfix = "End of LogFile:"; + String postfix = "End of LogType:"; int prefixIndex = fullMessage.indexOf(prefix) + prefix.length(); int postfixIndex = fullMessage.indexOf(postfix); return fullMessage.substring(prefixIndex, postfixIndex);