diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 97801d315f2..22c525fe2bf 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -34,6 +34,9 @@ Release 2.1.0-alpha - Unreleased YARN-12. Fix findbugs warnings in FairScheduler. (Junping Du via acmurthy) + YARN-22. Fix ContainerLogs to work if the log-dir is specified as a URI. + (Mayank Bansal via sseth) + Release 0.23.3 - Unreleased INCOMPATIBLE CHANGES diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/ContainerLogsPage.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/ContainerLogsPage.java index ee019255193..94c7f0e91d5 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/ContainerLogsPage.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/ContainerLogsPage.java @@ -29,6 +29,8 @@ import java.io.FileReader; import java.io.IOException; import java.io.InputStreamReader; +import java.net.URI; +import java.net.URISyntaxException; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; @@ -53,6 +55,7 @@ import org.apache.hadoop.yarn.webapp.YarnWebParams; import org.apache.hadoop.yarn.webapp.SubView; import org.apache.hadoop.yarn.webapp.view.HtmlBlock; +import org.mortbay.log.Log; import com.google.inject.Inject; @@ -198,12 +201,14 @@ private void printLogs(Block html, ContainerId containerId, if (!$(CONTAINER_LOG_TYPE).isEmpty()) { File logFile = null; try { - logFile = - new File(this.dirsHandler.getLogPathToRead( - ContainerLaunch.getRelativeContainerLogDir( + URI logPathURI = new URI(this.dirsHandler.getLogPathToRead( + ContainerLaunch.getRelativeContainerLogDir( applicationId.toString(), containerId.toString()) - + Path.SEPARATOR + $(CONTAINER_LOG_TYPE)) - .toUri().getPath()); + + Path.SEPARATOR + $(CONTAINER_LOG_TYPE)).toString()); + logFile = new File(logPathURI.getPath()); + } catch (URISyntaxException e) { + html.h1("Cannot find this log on the local disk."); + return; } catch (Exception e) { html.h1("Cannot find this log on the local disk."); return; @@ -278,14 +283,16 @@ private void printLogs(Block html, ContainerId containerId, boolean foundLogFile = false; for (File containerLogsDir : containerLogsDirs) { File[] logFiles = containerLogsDir.listFiles(); - Arrays.sort(logFiles); - for (File logFile : logFiles) { - foundLogFile = true; - html.p() - .a(url("containerlogs", $(CONTAINER_ID), $(APP_OWNER), - logFile.getName(), "?start=-4096"), - logFile.getName() + " : Total file length is " - + logFile.length() + " bytes.")._(); + if (logFiles != null) { + Arrays.sort(logFiles); + for (File logFile : logFiles) { + foundLogFile = true; + html.p() + .a(url("containerlogs", $(CONTAINER_ID), $(APP_OWNER), + logFile.getName(), "?start=-4096"), + logFile.getName() + " : Total file length is " + + logFile.length() + " bytes.")._(); + } } } if (!foundLogFile) { @@ -297,13 +304,17 @@ private void printLogs(Block html, ContainerId containerId, } static List getContainerLogDirs(ContainerId containerId, - LocalDirsHandlerService dirsHandler) { + LocalDirsHandlerService dirsHandler) { List logDirs = dirsHandler.getLogDirs(); List containerLogDirs = new ArrayList(logDirs.size()); for (String logDir : logDirs) { - String appIdStr = - ConverterUtils.toString( - containerId.getApplicationAttemptId().getApplicationId()); + try { + logDir = new URI(logDir).getPath(); + } catch (URISyntaxException e) { + Log.warn(e.getMessage()); + } + String appIdStr = ConverterUtils.toString(containerId + .getApplicationAttemptId().getApplicationId()); File appLogDir = new File(logDir, appIdStr); String containerIdStr = ConverterUtils.toString(containerId); containerLogDirs.add(new File(appLogDir, containerIdStr)); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestContainerLogsPage.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestContainerLogsPage.java new file mode 100644 index 00000000000..28985f5a5a3 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestContainerLogsPage.java @@ -0,0 +1,71 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.nodemanager.webapp; + +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import java.io.File; +import java.io.IOException; +import java.util.List; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; +import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.hadoop.yarn.api.records.ContainerId; +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.factories.RecordFactory; +import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider; +import org.apache.hadoop.yarn.server.nodemanager.LocalDirsHandlerService; +import org.apache.hadoop.yarn.server.nodemanager.NodeHealthCheckerService; +import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.Application; +import org.apache.hadoop.yarn.util.BuilderUtils; +import org.junit.Assert; +import org.junit.Test; + +public class TestContainerLogsPage { + + @Test + public void testContainerLogDirs() throws IOException { + String logdirwithFile = "file:///target/" + + TestNMWebServer.class.getSimpleName() + "LogDir"; + Configuration conf = new Configuration(); + conf.set(YarnConfiguration.NM_LOG_DIRS, logdirwithFile); + NodeHealthCheckerService healthChecker = new NodeHealthCheckerService(); + healthChecker.init(conf); + LocalDirsHandlerService dirsHandler = healthChecker.getDiskHandler(); + // Add an application and the corresponding containers + RecordFactory recordFactory = RecordFactoryProvider.getRecordFactory(conf); + String user = "nobody"; + long clusterTimeStamp = 1234; + ApplicationId appId = BuilderUtils.newApplicationId(recordFactory, + clusterTimeStamp, 1); + Application app = mock(Application.class); + when(app.getUser()).thenReturn(user); + when(app.getAppId()).thenReturn(appId); + ApplicationAttemptId appAttemptId = BuilderUtils.newApplicationAttemptId( + appId, 1); + ContainerId container1 = BuilderUtils.newContainerId(recordFactory, appId, + appAttemptId, 0); + List files = null; + files = ContainerLogsPage.ContainersLogsBlock.getContainerLogDirs( + container1, dirsHandler); + Assert.assertTrue(!(files.get(0).toString().contains("file:"))); + } +}