From f40389ae08572c60fc36c7a6d71c6b9c47f5b902 Mon Sep 17 00:00:00 2001 From: Zhijie Shen Date: Tue, 28 Oct 2014 14:10:16 -0700 Subject: [PATCH] YARN-2741. Made NM web UI serve logs on the drive other than C: on Windows. Contributed by Craig Welch. (cherry picked from commit 8984e9b1774033e379b57da1bd30a5c81888c7a3) --- hadoop-yarn-project/CHANGES.txt | 3 + .../webapp/ContainerLogsUtils.java | 10 +-- .../webapp/TestContainerLogsPage.java | 82 +++++++++++++++++++ 3 files changed, 87 insertions(+), 8 deletions(-) diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index d58fdb8b703..46c899383cf 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -754,6 +754,9 @@ Release 2.6.0 - UNRELEASED YARN-2758. Update TestApplicationHistoryClientService to use the new generic history store. (Zhijie Shen via xgong) + YARN-2741. Made NM web UI serve logs on the drive other than C: on Windows. (Craig + Welch via zjshen) + Release 2.5.1 - 2014-09-05 INCOMPATIBLE CHANGES diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/ContainerLogsUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/ContainerLogsUtils.java index 3d0f19d0a3b..c588a89ffe8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/ContainerLogsUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/ContainerLogsUtils.java @@ -77,11 +77,7 @@ public class ContainerLogsUtils { List logDirs = dirsHandler.getLogDirs(); List containerLogDirs = new ArrayList(logDirs.size()); for (String logDir : logDirs) { - try { - logDir = new URI(logDir).getPath(); - } catch (URISyntaxException e) { - throw new YarnException("Internal error", e); - } + logDir = new File(logDir).toURI().getPath(); String appIdStr = ConverterUtils.toString(containerId .getApplicationAttemptId().getApplicationId()); File appLogDir = new File(logDir, appIdStr); @@ -109,11 +105,9 @@ public class ContainerLogsUtils { application.getAppId().toString(), containerId.toString()); Path logPath = dirsHandler.getLogPathToRead( relativeContainerLogDir + Path.SEPARATOR + fileName); - URI logPathURI = new URI(logPath.toString()); + URI logPathURI = new File(logPath.toString()).toURI(); File logFile = new File(logPathURI.getPath()); return logFile; - } catch (URISyntaxException e) { - throw new YarnException("Internal error", e); } catch (IOException e) { LOG.warn("Failed to find log file", e); throw new NotFoundException("Cannot find this log on the local disk."); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestContainerLogsPage.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestContainerLogsPage.java index 9305b8459e6..b1d439779ac 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestContainerLogsPage.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestContainerLogsPage.java @@ -28,6 +28,7 @@ import java.io.File; import java.io.FileOutputStream; import java.io.IOException; import java.io.PrintWriter; +import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -36,11 +37,16 @@ import java.util.concurrent.ConcurrentMap; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.CommonConfigurationKeysPublic; +import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.nativeio.NativeIO; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ContainerId; +import org.apache.hadoop.yarn.api.records.impl.pb.ContainerIdPBImpl; +import org.apache.hadoop.yarn.api.records.impl.pb.ApplicationAttemptIdPBImpl; +import org.apache.hadoop.yarn.api.records.impl.pb.ApplicationIdPBImpl; +import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerImpl; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.event.AsyncDispatcher; import org.apache.hadoop.yarn.exceptions.YarnException; @@ -210,4 +216,80 @@ public class TestContainerLogsPage { } } } + + @Test + public void testLogDirWithDriveLetter() throws Exception { + //To verify that logs paths which include drive letters (Windows) + //do not lose their drive letter specification + LocalDirsHandlerService localDirs = mock(LocalDirsHandlerService.class); + List logDirs = new ArrayList(); + logDirs.add("F:/nmlogs"); + when(localDirs.getLogDirs()).thenReturn(logDirs); + + ApplicationIdPBImpl appId = mock(ApplicationIdPBImpl.class); + when(appId.toString()).thenReturn("app_id_1"); + + ApplicationAttemptIdPBImpl appAttemptId = + mock(ApplicationAttemptIdPBImpl.class); + when(appAttemptId.getApplicationId()).thenReturn(appId); + + ContainerId containerId = mock(ContainerIdPBImpl.class); + when(containerId.getApplicationAttemptId()).thenReturn(appAttemptId); + + List logDirFiles = ContainerLogsUtils.getContainerLogDirs( + containerId, localDirs); + + Assert.assertTrue("logDir lost drive letter " + + logDirFiles.get(0), + logDirFiles.get(0).toString().indexOf("F:" + File.separator + + "nmlogs") > -1); + } + + @Test + public void testLogFileWithDriveLetter() throws Exception { + + ContainerImpl container = mock(ContainerImpl.class); + + ApplicationIdPBImpl appId = mock(ApplicationIdPBImpl.class); + when(appId.toString()).thenReturn("appId"); + + Application app = mock(Application.class); + when(app.getAppId()).thenReturn(appId); + + ApplicationAttemptIdPBImpl appAttemptId = + mock(ApplicationAttemptIdPBImpl.class); + when(appAttemptId.getApplicationId()).thenReturn(appId); + + ConcurrentMap applications = + new ConcurrentHashMap(); + applications.put(appId, app); + + ContainerId containerId = mock(ContainerIdPBImpl.class); + when(containerId.toString()).thenReturn("containerId"); + when(containerId.getApplicationAttemptId()).thenReturn(appAttemptId); + + ConcurrentMap containers = + new ConcurrentHashMap(); + + containers.put(containerId, container); + + LocalDirsHandlerService localDirs = mock(LocalDirsHandlerService.class); + when(localDirs.getLogPathToRead("appId" + Path.SEPARATOR + "containerId" + + Path.SEPARATOR + "fileName")) + .thenReturn(new Path("F:/nmlogs/appId/containerId/fileName")); + + NMContext context = mock(NMContext.class); + when(context.getLocalDirsHandler()).thenReturn(localDirs); + when(context.getApplications()).thenReturn(applications); + when(context.getContainers()).thenReturn(containers); + + File logFile = ContainerLogsUtils.getContainerLogFile(containerId, + "fileName", null, context); + + Assert.assertTrue("logFile lost drive letter " + + logFile, + logFile.toString().indexOf("F:" + File.separator + "nmlogs") > -1); + + } + }