diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index d01fb04e7d0..00c0d7e5603 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -498,6 +498,9 @@ Release 2.4.0 - UNRELEASED YARN-1685. Fixed few bugs related to handling of containers' log-URLs on ResourceManager and history-service. (Zhijie Shen via vinodkv) + YARN-1206. Fixed AM container log to show on NM web page after application + finishes if log-aggregation is disabled. (Rohith Sharmaks via jianhe) + Release 2.3.1 - UNRELEASED INCOMPATIBLE CHANGES diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/ContainerLogsUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/ContainerLogsUtils.java index 4754daf0ada..3d0f19d0a3b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/ContainerLogsUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/ContainerLogsUtils.java @@ -56,13 +56,18 @@ public class ContainerLogsUtils { public static List getContainerLogDirs(ContainerId containerId, String remoteUser, Context context) throws YarnException { Container container = context.getContainers().get(containerId); - if (container == null) { - throw new YarnException("Container does not exist."); - } Application application = getApplicationForContainer(containerId, context); checkAccess(remoteUser, application, context); - checkState(container.getContainerState()); + // It is not required to have null check for container ( container == null ) + // and throw back exception.Because when container is completed, NodeManager + // remove container information from its NMContext.Configuring log + // aggregation to false, container log view request is forwarded to NM. NM + // does not have completed container information,but still NM serve request for + // reading container logs. + if (container != null) { + checkState(container.getContainerState()); + } return getContainerLogDirs(containerId, context.getLocalDirsHandler()); } @@ -91,14 +96,12 @@ public class ContainerLogsUtils { public static File getContainerLogFile(ContainerId containerId, String fileName, String remoteUser, Context context) throws YarnException { Container container = context.getContainers().get(containerId); - if (container == null) { - throw new NotFoundException("Container with id " + containerId - + " not found."); - } Application application = getApplicationForContainer(containerId, context); checkAccess(remoteUser, application, context); - checkState(container.getContainerState()); + if (container != null) { + checkState(container.getContainerState()); + } try { LocalDirsHandlerService dirsHandler = context.getLocalDirsHandler(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestContainerLogsPage.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestContainerLogsPage.java index 6c21ff72cb4..447ea8c8bb4 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestContainerLogsPage.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestContainerLogsPage.java @@ -49,10 +49,13 @@ import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider; import org.apache.hadoop.yarn.server.nodemanager.Context; import org.apache.hadoop.yarn.server.nodemanager.LocalDirsHandlerService; import org.apache.hadoop.yarn.server.nodemanager.NodeHealthCheckerService; +import org.apache.hadoop.yarn.server.nodemanager.NodeManager; +import org.apache.hadoop.yarn.server.nodemanager.NodeManager.NMContext; import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.Application; import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container; import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerState; import org.apache.hadoop.yarn.server.nodemanager.webapp.ContainerLogsPage.ContainersLogsBlock; +import org.apache.hadoop.yarn.server.security.ApplicationACLsManager; import org.apache.hadoop.yarn.server.utils.BuilderUtils; import org.apache.hadoop.yarn.webapp.YarnWebParams; import org.apache.hadoop.yarn.webapp.test.WebAppTests; @@ -74,6 +77,7 @@ public class TestContainerLogsPage { NodeHealthCheckerService healthChecker = new NodeHealthCheckerService(); healthChecker.init(conf); LocalDirsHandlerService dirsHandler = healthChecker.getDiskHandler(); + NMContext nmContext = new NodeManager.NMContext(null, null, dirsHandler, new ApplicationACLsManager(conf)); // Add an application and the corresponding containers RecordFactory recordFactory = RecordFactoryProvider.getRecordFactory(conf); String user = "nobody"; @@ -87,9 +91,21 @@ public class TestContainerLogsPage { appId, 1); ContainerId container1 = BuilderUtils.newContainerId(recordFactory, appId, appAttemptId, 0); + nmContext.getApplications().put(appId, app); + + MockContainer container = + new MockContainer(appAttemptId, new AsyncDispatcher(), conf, user, + appId, 1); + container.setState(ContainerState.RUNNING); + nmContext.getContainers().put(container1, container); List files = null; - files = ContainerLogsUtils.getContainerLogDirs( - container1, dirsHandler); + files = ContainerLogsUtils.getContainerLogDirs(container1, user, nmContext); + Assert.assertTrue(!(files.get(0).toString().contains("file:"))); + + // After container is completed, it is removed from nmContext + nmContext.getContainers().remove(container1); + Assert.assertNull(nmContext.getContainers().get(container1)); + files = ContainerLogsUtils.getContainerLogDirs(container1, user, nmContext); Assert.assertTrue(!(files.get(0).toString().contains("file:"))); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServices.java index eab68491d08..2f08bb331a5 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServices.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServices.java @@ -352,6 +352,16 @@ public class TestNMWebServices extends JerseyTest { Assert.assertEquals(Status.NOT_FOUND.getStatusCode(), response.getStatus()); responseText = response.getEntity(String.class); assertTrue(responseText.contains("Cannot find this log on the local disk.")); + + // After container is completed, it is removed from nmContext + nmContext.getContainers().remove(containerId); + Assert.assertNull(nmContext.getContainers().get(containerId)); + response = + r.path("ws").path("v1").path("node").path("containerlogs") + .path(containerIdStr).path(filename).accept(MediaType.TEXT_PLAIN) + .get(ClientResponse.class); + responseText = response.getEntity(String.class); + assertEquals(logMessage, responseText); } public void verifyNodesXML(NodeList nodes) throws JSONException, Exception {