From 49e1292ea3e4d00ab0b0191bd8c4ea4d2afed671 Mon Sep 17 00:00:00 2001 From: Eric Yang Date: Mon, 6 May 2019 19:48:45 -0400 Subject: [PATCH] YARN-9524. Fixed TestAHSWebService and TestLogsCLI unit tests. Contributed by Prabhu Joseph --- .../hadoop/yarn/client/cli/TestLogsCLI.java | 27 ++++++++--- .../logaggregation/LogAggregationUtils.java | 48 ++++++++++++++----- .../yarn/logaggregation/LogCLIHelpers.java | 34 ++++++++++++- .../LogAggregationFileController.java | 2 +- .../LogAggregationIndexedFileController.java | 2 +- 5 files changed, 92 insertions(+), 21 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java index f9061eb6e70..801cf40d933 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java @@ -407,7 +407,7 @@ public class TestLogsCLI { Path path = new Path(remoteLogRootDir + ugi.getShortUserName() - + "/logs/application_0_0001"); + + "/bucket_logs/0001/application_0_0001"); if (fs.exists(path)) { fs.delete(path, true); } @@ -925,7 +925,6 @@ public class TestLogsCLI { public void testFetchApplictionLogsAsAnotherUser() throws Exception { String remoteLogRootDir = "target/logs/"; String rootLogDir = "target/LocalLogs"; - String testUser = "test"; UserGroupInformation testUgi = UserGroupInformation .createRemoteUser(testUser); @@ -966,9 +965,9 @@ public class TestLogsCLI { // create container logs in localLogDir for app createContainerLogInLocalDir(appLogsDir, containerId, fs, logTypes); - // create the remote app dir for app - // but for a different user testUser" - Path path = new Path(remoteLogRootDir + testUser + "/logs/" + appId); + // create the remote app dir for app but for a different user testUser + Path path = new Path(remoteLogRootDir + testUser + "/bucket_logs/0001/" + + appId); if (fs.exists(path)) { fs.delete(path, true); } @@ -1016,6 +1015,22 @@ public class TestLogsCLI { logMessage(containerId, "syslog"))); sysOutStream.reset(); + // Verify appOwner guessed correctly with older log dir dtructure + path = new Path(remoteLogRootDir + testUser + "/logs/" + appId); + if (fs.exists(path)) { + fs.delete(path, true); + } + assertTrue(fs.mkdirs(path)); + uploadContainerLogIntoRemoteDir(testUgi, configuration, rootLogDirs, + nodeId, containerId, path, fs); + + exitCode = cli.run(new String[] { + "-applicationId", appId.toString()}); + assertTrue(exitCode == 0); + assertTrue(sysOutStream.toString().contains( + logMessage(containerId, "syslog"))); + sysOutStream.reset(); + // Verify that we could get the err message "Can not find the appOwner" // if we do not specify the appOwner, can not get appReport, and // the app does not exist in remote dir. @@ -1034,7 +1049,7 @@ public class TestLogsCLI { System.currentTimeMillis(), 1000); String priorityUser = "priority"; Path pathWithoutPerm = new Path(remoteLogRootDir + priorityUser - + "/logs/" + appTest); + + "/bucket_logs/1000/" + appTest); if (fs.exists(pathWithoutPerm)) { fs.delete(pathWithoutPerm, true); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/LogAggregationUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/LogAggregationUtils.java index 3f5151b3e1b..deff2aaa75a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/LogAggregationUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/LogAggregationUtils.java @@ -83,6 +83,30 @@ public class LogAggregationUtils { suffix), appId.toString()); } + public static Path getOlderRemoteAppLogDir(Configuration conf, + ApplicationId appId, String user, Path remoteRootLogDir, String suffix) + throws IOException { + org.apache.hadoop.fs.Path remoteAppDir = null; + if (user == null) { + org.apache.hadoop.fs.Path qualifiedRemoteRootLogDir = + FileContext.getFileContext(conf).makeQualified(remoteRootLogDir); + FileContext fc = FileContext.getFileContext( + qualifiedRemoteRootLogDir.toUri(), conf); + org.apache.hadoop.fs.Path toMatch = LogAggregationUtils + .getOlderRemoteAppLogDir(appId, "*", remoteRootLogDir, suffix); + FileStatus[] matching = fc.util().globStatus(toMatch); + if (matching == null || matching.length != 1) { + throw new IOException("Can not find remote application directory for " + + "the application:" + appId); + } + remoteAppDir = matching[0].getPath(); + } else { + remoteAppDir = LogAggregationUtils.getOlderRemoteAppLogDir( + appId, user, remoteRootLogDir, suffix); + } + return remoteAppDir; + } + /** * Gets the remote suffixed log dir for the user. * @param remoteRootLogDir the aggregated log remote root log dir @@ -274,9 +298,9 @@ public class LogAggregationUtils { StringBuilder diagnosticsMsg = new StringBuilder(); // Get Node Files from new app log dir - Path remoteAppLogDir = getRemoteAppLogDir(conf, appId, appOwner, - remoteRootLogDir, suffix); try { + Path remoteAppLogDir = getRemoteAppLogDir(conf, appId, appOwner, + remoteRootLogDir, suffix); nodeFilesCur = getNodeFiles(conf, remoteAppLogDir, appId, appOwner); } catch (IOException ex) { diagnosticsMsg.append(ex.getMessage() + "\n"); @@ -284,9 +308,9 @@ public class LogAggregationUtils { // Get Node Files from old app log dir if (isOlderPathEnabled(conf)) { - remoteAppLogDir = getOlderRemoteAppLogDir(appId, appOwner, - remoteRootLogDir, suffix); try { + Path remoteAppLogDir = getOlderRemoteAppLogDir(conf, appId, appOwner, + remoteRootLogDir, suffix); nodeFilesPrev = getNodeFiles(conf, remoteAppLogDir, appId, appOwner); } catch (IOException ex) { @@ -338,14 +362,14 @@ public class LogAggregationUtils { org.apache.hadoop.fs.Path remoteRootLogDir, String suffix) throws IOException { StringBuilder diagnosticsMsg = new StringBuilder(); - Path remoteAppLogDir = getRemoteAppLogDir(conf, appId, appOwner, - remoteRootLogDir, suffix); List nodeFiles = new ArrayList<>(); - Path qualifiedLogDir = - FileContext.getFileContext(conf).makeQualified(remoteAppLogDir); // Get Node Files from new app log dir try { + Path remoteAppLogDir = getRemoteAppLogDir(conf, appId, appOwner, + remoteRootLogDir, suffix); + Path qualifiedLogDir = + FileContext.getFileContext(conf).makeQualified(remoteAppLogDir); nodeFiles.addAll(Arrays.asList(FileContext.getFileContext( qualifiedLogDir.toUri(), conf).util().listStatus(remoteAppLogDir))); } catch (IOException ex) { @@ -354,11 +378,11 @@ public class LogAggregationUtils { // Get Node Files from old app log dir if (isOlderPathEnabled(conf)) { - remoteAppLogDir = getOlderRemoteAppLogDir(appId, appOwner, - remoteRootLogDir, suffix); - qualifiedLogDir = FileContext.getFileContext(conf). - makeQualified(remoteAppLogDir); try { + Path remoteAppLogDir = getOlderRemoteAppLogDir(conf, appId, appOwner, + remoteRootLogDir, suffix); + Path qualifiedLogDir = FileContext.getFileContext(conf). + makeQualified(remoteAppLogDir); nodeFiles.addAll(Arrays.asList(FileContext.getFileContext( qualifiedLogDir.toUri(), conf).util().listStatus(remoteAppLogDir))); } catch (IOException ex) { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/LogCLIHelpers.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/LogCLIHelpers.java index 9dae7b9ce26..8a72d80722e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/LogCLIHelpers.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/LogCLIHelpers.java @@ -88,6 +88,7 @@ public class LogCLIHelpers implements Configurable { String suffix = LogAggregationUtils.getRemoteNodeLogDirSuffix(conf); Path fullPath = LogAggregationUtils.getRemoteAppLogDir(remoteRootLogDir, appId, bestGuess, suffix); + FileContext fc = FileContext.getFileContext(remoteRootLogDir.toUri(), conf); String pathAccess = fullPath.toString(); @@ -95,19 +96,50 @@ public class LogCLIHelpers implements Configurable { if (fc.util().exists(fullPath)) { return bestGuess; } + + boolean scanOldPath = LogAggregationUtils.isOlderPathEnabled(conf); + if (scanOldPath) { + Path olderAppPath = LogAggregationUtils.getOlderRemoteAppLogDir(appId, + bestGuess, remoteRootLogDir, suffix); + if (fc.util().exists(olderAppPath)) { + return bestGuess; + } + } + Path toMatch = LogAggregationUtils. getRemoteAppLogDir(remoteRootLogDir, appId, "*", suffix); + pathAccess = toMatch.toString(); FileStatus[] matching = fc.util().globStatus(toMatch); if (matching == null || matching.length != 1) { + if (scanOldPath) { + toMatch = LogAggregationUtils.getOlderRemoteAppLogDir(appId, "*", + remoteRootLogDir, suffix); + try { + matching = fc.util().globStatus(toMatch); + if (matching != null && matching.length == 1) { + //fetch the user from the old path /app-logs/user[/suffix]/app_id + Path parent = matching[0].getPath().getParent(); + //skip the suffix too + if (suffix != null && !StringUtils.isEmpty(suffix)) { + parent = parent.getParent(); + } + return parent.getName(); + } + } catch (IOException e) { + // Ignore IOException from accessing older app log dir + } + } return null; } - //fetch the user from the full path /app-logs/user[/suffix]/app_id + //fetch the user from the full path /app-logs/user[/suffix]/bucket/app_id Path parent = matching[0].getPath().getParent(); //skip the suffix too if (suffix != null && !StringUtils.isEmpty(suffix)) { parent = parent.getParent(); } + //skip the bucket + parent = parent.getParent(); return parent.getName(); } catch (AccessControlException | AccessDeniedException ex) { logDirNoAccessPermission(pathAccess, bestGuess, ex.getMessage()); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/LogAggregationFileController.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/LogAggregationFileController.java index 053a5631a63..2547d95bbe9 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/LogAggregationFileController.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/LogAggregationFileController.java @@ -470,7 +470,7 @@ public abstract class LogAggregationFileController { */ public Path getOlderRemoteAppLogDir(ApplicationId appId, String appOwner) throws IOException { - return LogAggregationUtils.getOlderRemoteAppLogDir(appId, appOwner, + return LogAggregationUtils.getOlderRemoteAppLogDir(conf, appId, appOwner, this.remoteRootLogDir, this.remoteRootLogDirSuffix); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/ifile/LogAggregationIndexedFileController.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/ifile/LogAggregationIndexedFileController.java index 9bb4f9d3375..96e06379b83 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/ifile/LogAggregationIndexedFileController.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/ifile/LogAggregationIndexedFileController.java @@ -822,7 +822,7 @@ public class LogAggregationIndexedFileController @Override public Path getOlderRemoteAppLogDir(ApplicationId appId, String user) throws IOException { - return LogAggregationUtils.getOlderRemoteAppLogDir(appId, user, + return LogAggregationUtils.getOlderRemoteAppLogDir(conf, appId, user, this.remoteRootLogDir, this.remoteRootLogDirSuffix); }