Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

YARN-9524. Fixed TestAHSWebService and TestLogsCLI unit tests. 

Contributed by Prabhu Joseph
This commit is contained in:
Eric Yang 2019-05-06 19:48:45 -04:00
parent 93f2283a69
commit 49e1292ea3
5 changed files with 92 additions and 21 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

27
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java
View File

@ -407,7 +407,7 @@ public class TestLogsCLI {
Path path =
new Path(remoteLogRootDir + ugi.getShortUserName()
+ "/logs/application_0_0001");
+ "/bucket_logs/0001/application_0_0001");
if (fs.exists(path)) {
fs.delete(path, true);
}
@ -925,7 +925,6 @@ public class TestLogsCLI {
public void testFetchApplictionLogsAsAnotherUser() throws Exception {
String remoteLogRootDir = "target/logs/";
String rootLogDir = "target/LocalLogs";
String testUser = "test";
UserGroupInformation testUgi = UserGroupInformation
.createRemoteUser(testUser);
@ -966,9 +965,9 @@ public class TestLogsCLI {
// create container logs in localLogDir for app
createContainerLogInLocalDir(appLogsDir, containerId, fs, logTypes);
// create the remote app dir for app
// but for a different user testUser"
Path path = new Path(remoteLogRootDir + testUser + "/logs/" + appId);
// create the remote app dir for app but for a different user testUser
Path path = new Path(remoteLogRootDir + testUser + "/bucket_logs/0001/"
+ appId);
if (fs.exists(path)) {
fs.delete(path, true);
}
@ -1016,6 +1015,22 @@ public class TestLogsCLI {
logMessage(containerId, "syslog")));
sysOutStream.reset();
// Verify appOwner guessed correctly with older log dir dtructure
path = new Path(remoteLogRootDir + testUser + "/logs/" + appId);
if (fs.exists(path)) {
fs.delete(path, true);
}
assertTrue(fs.mkdirs(path));
uploadContainerLogIntoRemoteDir(testUgi, configuration, rootLogDirs,
nodeId, containerId, path, fs);
exitCode = cli.run(new String[] {
"-applicationId", appId.toString()});
assertTrue(exitCode == 0);
assertTrue(sysOutStream.toString().contains(
logMessage(containerId, "syslog")));
sysOutStream.reset();
// Verify that we could get the err message "Can not find the appOwner"
// if we do not specify the appOwner, can not get appReport, and
// the app does not exist in remote dir.
@ -1034,7 +1049,7 @@ public class TestLogsCLI {
System.currentTimeMillis(), 1000);
String priorityUser = "priority";
Path pathWithoutPerm = new Path(remoteLogRootDir + priorityUser
+ "/logs/" + appTest);
+ "/bucket_logs/1000/" + appTest);
if (fs.exists(pathWithoutPerm)) {
fs.delete(pathWithoutPerm, true);
}

48
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/LogAggregationUtils.java
View File

@ -83,6 +83,30 @@ public class LogAggregationUtils {
suffix), appId.toString());
}
public static Path getOlderRemoteAppLogDir(Configuration conf,
ApplicationId appId, String user, Path remoteRootLogDir, String suffix)
throws IOException {
org.apache.hadoop.fs.Path remoteAppDir = null;
if (user == null) {
org.apache.hadoop.fs.Path qualifiedRemoteRootLogDir =
FileContext.getFileContext(conf).makeQualified(remoteRootLogDir);
FileContext fc = FileContext.getFileContext(
qualifiedRemoteRootLogDir.toUri(), conf);
org.apache.hadoop.fs.Path toMatch = LogAggregationUtils
.getOlderRemoteAppLogDir(appId, "*", remoteRootLogDir, suffix);
FileStatus[] matching = fc.util().globStatus(toMatch);
if (matching == null || matching.length != 1) {
throw new IOException("Can not find remote application directory for "
+ "the application:" + appId);
}
remoteAppDir = matching[0].getPath();
} else {
remoteAppDir = LogAggregationUtils.getOlderRemoteAppLogDir(
appId, user, remoteRootLogDir, suffix);
}
return remoteAppDir;
}
/**
* Gets the remote suffixed log dir for the user.
* @param remoteRootLogDir the aggregated log remote root log dir
@ -274,9 +298,9 @@ public class LogAggregationUtils {
StringBuilder diagnosticsMsg = new StringBuilder();
// Get Node Files from new app log dir
Path remoteAppLogDir = getRemoteAppLogDir(conf, appId, appOwner,
remoteRootLogDir, suffix);
try {
Path remoteAppLogDir = getRemoteAppLogDir(conf, appId, appOwner,
remoteRootLogDir, suffix);
nodeFilesCur = getNodeFiles(conf, remoteAppLogDir, appId, appOwner);
} catch (IOException ex) {
diagnosticsMsg.append(ex.getMessage() + "\n");
@ -284,9 +308,9 @@ public class LogAggregationUtils {
// Get Node Files from old app log dir
if (isOlderPathEnabled(conf)) {
remoteAppLogDir = getOlderRemoteAppLogDir(appId, appOwner,
remoteRootLogDir, suffix);
try {
Path remoteAppLogDir = getOlderRemoteAppLogDir(conf, appId, appOwner,
remoteRootLogDir, suffix);
nodeFilesPrev = getNodeFiles(conf,
remoteAppLogDir, appId, appOwner);
} catch (IOException ex) {
@ -338,14 +362,14 @@ public class LogAggregationUtils {
org.apache.hadoop.fs.Path remoteRootLogDir, String suffix)
throws IOException {
StringBuilder diagnosticsMsg = new StringBuilder();
Path remoteAppLogDir = getRemoteAppLogDir(conf, appId, appOwner,
remoteRootLogDir, suffix);
List<FileStatus> nodeFiles = new ArrayList<>();
Path qualifiedLogDir =
FileContext.getFileContext(conf).makeQualified(remoteAppLogDir);
// Get Node Files from new app log dir
try {
Path remoteAppLogDir = getRemoteAppLogDir(conf, appId, appOwner,
remoteRootLogDir, suffix);
Path qualifiedLogDir =
FileContext.getFileContext(conf).makeQualified(remoteAppLogDir);
nodeFiles.addAll(Arrays.asList(FileContext.getFileContext(
qualifiedLogDir.toUri(), conf).util().listStatus(remoteAppLogDir)));
} catch (IOException ex) {
@ -354,11 +378,11 @@ public class LogAggregationUtils {
// Get Node Files from old app log dir
if (isOlderPathEnabled(conf)) {
remoteAppLogDir = getOlderRemoteAppLogDir(appId, appOwner,
remoteRootLogDir, suffix);
qualifiedLogDir = FileContext.getFileContext(conf).
makeQualified(remoteAppLogDir);
try {
Path remoteAppLogDir = getOlderRemoteAppLogDir(conf, appId, appOwner,
remoteRootLogDir, suffix);
Path qualifiedLogDir = FileContext.getFileContext(conf).
makeQualified(remoteAppLogDir);
nodeFiles.addAll(Arrays.asList(FileContext.getFileContext(
qualifiedLogDir.toUri(), conf).util().listStatus(remoteAppLogDir)));
} catch (IOException ex) {

34
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/LogCLIHelpers.java
View File

@ -88,6 +88,7 @@ public class LogCLIHelpers implements Configurable {
String suffix = LogAggregationUtils.getRemoteNodeLogDirSuffix(conf);
Path fullPath = LogAggregationUtils.getRemoteAppLogDir(remoteRootLogDir,
appId, bestGuess, suffix);
FileContext fc =
FileContext.getFileContext(remoteRootLogDir.toUri(), conf);
String pathAccess = fullPath.toString();
@ -95,19 +96,50 @@ public class LogCLIHelpers implements Configurable {
if (fc.util().exists(fullPath)) {
return bestGuess;
}
boolean scanOldPath = LogAggregationUtils.isOlderPathEnabled(conf);
if (scanOldPath) {
Path olderAppPath = LogAggregationUtils.getOlderRemoteAppLogDir(appId,
bestGuess, remoteRootLogDir, suffix);
if (fc.util().exists(olderAppPath)) {
return bestGuess;
}
}
Path toMatch = LogAggregationUtils.
getRemoteAppLogDir(remoteRootLogDir, appId, "*", suffix);
pathAccess = toMatch.toString();
FileStatus[] matching = fc.util().globStatus(toMatch);
if (matching == null || matching.length != 1) {
if (scanOldPath) {
toMatch = LogAggregationUtils.getOlderRemoteAppLogDir(appId, "*",
remoteRootLogDir, suffix);
try {
matching = fc.util().globStatus(toMatch);
if (matching != null && matching.length == 1) {
//fetch the user from the old path /app-logs/user[/suffix]/app_id
Path parent = matching[0].getPath().getParent();
//skip the suffix too
if (suffix != null && !StringUtils.isEmpty(suffix)) {
parent = parent.getParent();
}
return parent.getName();
}
} catch (IOException e) {
// Ignore IOException from accessing older app log dir
}
}
return null;
}
//fetch the user from the full path /app-logs/user[/suffix]/app_id
//fetch the user from the full path /app-logs/user[/suffix]/bucket/app_id
Path parent = matching[0].getPath().getParent();
//skip the suffix too
if (suffix != null && !StringUtils.isEmpty(suffix)) {
parent = parent.getParent();
}
//skip the bucket
parent = parent.getParent();
return parent.getName();
} catch (AccessControlException | AccessDeniedException ex) {
logDirNoAccessPermission(pathAccess, bestGuess, ex.getMessage());

2
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/LogAggregationFileController.java
View File

@ -470,7 +470,7 @@ public abstract class LogAggregationFileController {
*/
public Path getOlderRemoteAppLogDir(ApplicationId appId, String appOwner)
throws IOException {
return LogAggregationUtils.getOlderRemoteAppLogDir(appId, appOwner,
return LogAggregationUtils.getOlderRemoteAppLogDir(conf, appId, appOwner,
this.remoteRootLogDir, this.remoteRootLogDirSuffix);
}

2
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/ifile/LogAggregationIndexedFileController.java
View File

@ -822,7 +822,7 @@ public class LogAggregationIndexedFileController
@Override
public Path getOlderRemoteAppLogDir(ApplicationId appId, String user)
throws IOException {
return LogAggregationUtils.getOlderRemoteAppLogDir(appId, user,
return LogAggregationUtils.getOlderRemoteAppLogDir(conf, appId, user,
this.remoteRootLogDir, this.remoteRootLogDirSuffix);
}