YARN-4842. Fixed "yarn logs" command to guess (and thus not require) the appOwner argument when viewing another user's logs. Contributed by Ram Venkatesh and Xuan Gong.
This commit is contained in:
Vinod Kumar Vavilapalli
parent
996a210ab0
commit
87f5e35133
|
|
@ -151,14 +151,12 @@ public class LogsCLI extends Configured implements Tool {
|
|||
LogCLIHelpers logCliHelper = new LogCLIHelpers();
|
||||
logCliHelper.setConf(getConf());
|
||||
|
||||
if (appOwner == null || appOwner.isEmpty()) {
|
||||
appOwner = UserGroupInformation.getCurrentUser().getShortUserName();
|
||||
}
|
||||
|
||||
boolean appStateObtainedSuccessfully = true;
|
||||
YarnApplicationState appState = YarnApplicationState.NEW;
|
||||
ApplicationReport appReport = null;
|
||||
try {
|
||||
appState = getApplicationState(appId);
|
||||
appReport = getApplicationReport(appId);
|
||||
appState = appReport.getYarnApplicationState();
|
||||
if (appState == YarnApplicationState.NEW
|
||||
|| appState == YarnApplicationState.NEW_SAVING
|
||||
|| appState == YarnApplicationState.SUBMITTED) {
|
||||
|
|
@ -171,6 +169,16 @@ public class LogsCLI extends Configured implements Tool {
|
|||
+ " Attempting to fetch logs directly from the filesystem.");
|
||||
}
|
||||
|
||||
if (appOwner == null || appOwner.isEmpty()) {
|
||||
appOwner = guessAppOwner(appReport, appId);
|
||||
if (appOwner == null) {
|
||||
System.err.println("Can not find the appOwner. "
|
||||
+ "Please specify the correct appOwner");
|
||||
System.err.println("Could not locate application logs for " + appId);
|
||||
return -1;
|
||||
}
|
||||
}
|
||||
|
||||
if (showMetaInfo) {
|
||||
return showMetaInfo(appState, appStateObtainedSuccessfully,
|
||||
logCliHelper, appId, containerIdStr, nodeAddress, appOwner);
|
||||
|
|
@ -201,6 +209,10 @@ public class LogsCLI extends Configured implements Tool {
|
|||
if (nodeAddress == null) {
|
||||
resultCode =
|
||||
logCliHelper.dumpAllContainersLogs(appId, appOwner, System.out);
|
||||
if (resultCode == -1) {
|
||||
System.err.println("Can not find the logs for the application: "
|
||||
+ appId + " with the appOwner: " + appOwner);
|
||||
}
|
||||
} else {
|
||||
System.err.println("Should at least provide ContainerId!");
|
||||
printHelpMessage(printOpts);
|
||||
|
|
@ -210,13 +222,12 @@ public class LogsCLI extends Configured implements Tool {
|
|||
return resultCode;
|
||||
}
|
||||
|
||||
private YarnApplicationState getApplicationState(ApplicationId appId)
|
||||
private ApplicationReport getApplicationReport(ApplicationId appId)
|
||||
throws IOException, YarnException {
|
||||
YarnClient yarnClient = createYarnClient();
|
||||
|
||||
try {
|
||||
ApplicationReport appReport = yarnClient.getApplicationReport(appId);
|
||||
return appReport.getYarnApplicationState();
|
||||
return yarnClient.getApplicationReport(appId);
|
||||
} finally {
|
||||
yarnClient.close();
|
||||
}
|
||||
|
|
@ -693,11 +704,12 @@ public class LogsCLI extends Configured implements Tool {
|
|||
amContainersList, logFiles, logCliHelper, appOwner, true);
|
||||
} else {
|
||||
System.err.println("Can not get AMContainers logs for "
|
||||
+ "the application:" + appId);
|
||||
System.err.println("This application:" + appId + " is finished."
|
||||
+ " Please enable the application history service. Or Using "
|
||||
+ "yarn logs -applicationId <appId> -containerId <containerId> "
|
||||
+ "--nodeAddress <nodeHttpAddress> to get the container logs");
|
||||
+ "the application:" + appId + " with the appOwner:" + appOwner);
|
||||
System.err.println("This application:" + appId + " has finished."
|
||||
+ " Please enable the application-history service or explicitly"
|
||||
+ " use 'yarn logs -applicationId <appId> "
|
||||
+ "-containerId <containerId> --nodeAddress <nodeHttpAddress>' "
|
||||
+ "to get the container logs.");
|
||||
return -1;
|
||||
}
|
||||
}
|
||||
|
|
@ -750,7 +762,8 @@ public class LogsCLI extends Configured implements Tool {
|
|||
appOwner);
|
||||
} else if (!isApplicationFinished(appState)) {
|
||||
System.err.println("Unable to get logs for this container:"
|
||||
+ containerIdStr + "for the application:" + appIdStr);
|
||||
+ containerIdStr + "for the application:" + appIdStr
|
||||
+ " with the appOwner: " + appOwner);
|
||||
System.err.println("The application: " + appIdStr
|
||||
+ " is still running, and we can not get Container report "
|
||||
+ "for the container: " + containerIdStr +". Please try later "
|
||||
|
|
@ -821,4 +834,18 @@ public class LogsCLI extends Configured implements Tool {
|
|||
return isAppFinished;
|
||||
}
|
||||
}
|
||||
|
||||
private String guessAppOwner(ApplicationReport appReport,
|
||||
ApplicationId appId) throws IOException {
|
||||
String appOwner = null;
|
||||
if (appReport != null) {
|
||||
//always use the app owner from the app report if possible
|
||||
appOwner = appReport.getUser();
|
||||
} else {
|
||||
appOwner = UserGroupInformation.getCurrentUser().getShortUserName();
|
||||
appOwner = LogCLIHelpers.getOwnerForAppIdOrNull(
|
||||
appId, appOwner, getConf());
|
||||
}
|
||||
return appOwner;
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -42,11 +42,11 @@ import java.util.HashSet;
|
|||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
import org.junit.Assert;
|
||||
import org.apache.hadoop.conf.Configuration;
|
||||
import org.apache.hadoop.fs.FileSystem;
|
||||
import org.apache.hadoop.fs.LocalFileSystem;
|
||||
import org.apache.hadoop.fs.Path;
|
||||
import org.apache.hadoop.fs.permission.FsPermission;
|
||||
import org.apache.hadoop.security.UserGroupInformation;
|
||||
import org.apache.hadoop.yarn.api.records.ApplicationAccessType;
|
||||
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
|
||||
|
|
@ -55,22 +55,21 @@ import org.apache.hadoop.yarn.api.records.ApplicationReport;
|
|||
import org.apache.hadoop.yarn.api.records.ContainerId;
|
||||
import org.apache.hadoop.yarn.api.records.NodeId;
|
||||
import org.apache.hadoop.yarn.api.records.YarnApplicationState;
|
||||
import org.apache.hadoop.yarn.api.records.impl.pb.ApplicationAttemptIdPBImpl;
|
||||
import org.apache.hadoop.yarn.api.records.impl.pb.ApplicationIdPBImpl;
|
||||
import org.apache.hadoop.yarn.api.records.impl.pb.ContainerIdPBImpl;
|
||||
import org.apache.hadoop.yarn.client.api.YarnClient;
|
||||
import org.apache.hadoop.yarn.conf.YarnConfiguration;
|
||||
import org.apache.hadoop.yarn.exceptions.YarnException;
|
||||
import org.apache.hadoop.yarn.logaggregation.AggregatedLogFormat;
|
||||
import org.apache.hadoop.yarn.logaggregation.LogAggregationUtils;
|
||||
import org.apache.hadoop.yarn.logaggregation.LogCLIHelpers;
|
||||
import org.junit.Assert;
|
||||
import org.junit.Before;
|
||||
import org.junit.Test;
|
||||
|
||||
public class TestLogsCLI {
|
||||
|
||||
ByteArrayOutputStream sysOutStream;
|
||||
private PrintStream sysOut;
|
||||
|
||||
|
||||
ByteArrayOutputStream sysErrStream;
|
||||
private PrintStream sysErr;
|
||||
|
||||
|
|
@ -79,7 +78,7 @@ public class TestLogsCLI {
|
|||
sysOutStream = new ByteArrayOutputStream();
|
||||
sysOut = new PrintStream(sysOutStream);
|
||||
System.setOut(sysOut);
|
||||
|
||||
|
||||
sysErrStream = new ByteArrayOutputStream();
|
||||
sysErr = new PrintStream(sysErrStream);
|
||||
System.setErr(sysErr);
|
||||
|
|
@ -91,16 +90,18 @@ public class TestLogsCLI {
|
|||
conf.setClass("fs.file.impl", LocalFileSystem.class, FileSystem.class);
|
||||
LogCLIHelpers cliHelper = new LogCLIHelpers();
|
||||
cliHelper.setConf(conf);
|
||||
YarnClient mockYarnClient = createMockYarnClient(YarnApplicationState.FINISHED);
|
||||
YarnClient mockYarnClient = createMockYarnClient(
|
||||
YarnApplicationState.FINISHED,
|
||||
UserGroupInformation.getCurrentUser().getShortUserName());
|
||||
LogsCLI dumper = new LogsCLIForTest(mockYarnClient);
|
||||
dumper.setConf(conf);
|
||||
|
||||
|
||||
// verify dumping a non-existent application's logs returns a failure code
|
||||
int exitCode = dumper.run( new String[] {
|
||||
"-applicationId", "application_0_0" } );
|
||||
assertTrue("Should return an error code", exitCode != 0);
|
||||
|
||||
// verify dumping a non-existent container log is a failure code
|
||||
|
||||
// verify dumping a non-existent container log is a failure code
|
||||
exitCode = cliHelper.dumpAContainersLogs("application_0_0", "container_0_0",
|
||||
"nonexistentnode:1234", "nobody");
|
||||
assertTrue("Should return an error code", exitCode != 0);
|
||||
|
|
@ -109,10 +110,12 @@ public class TestLogsCLI {
|
|||
@Test(timeout = 5000l)
|
||||
public void testInvalidApplicationId() throws Exception {
|
||||
Configuration conf = new YarnConfiguration();
|
||||
YarnClient mockYarnClient = createMockYarnClient(YarnApplicationState.FINISHED);
|
||||
YarnClient mockYarnClient = createMockYarnClient(
|
||||
YarnApplicationState.FINISHED,
|
||||
UserGroupInformation.getCurrentUser().getShortUserName());
|
||||
LogsCLI cli = new LogsCLIForTest(mockYarnClient);
|
||||
cli.setConf(conf);
|
||||
|
||||
|
||||
int exitCode = cli.run( new String[] { "-applicationId", "not_an_app_id"});
|
||||
assertTrue(exitCode == -1);
|
||||
assertTrue(sysErrStream.toString().startsWith("Invalid ApplicationId specified"));
|
||||
|
|
@ -137,7 +140,9 @@ public class TestLogsCLI {
|
|||
@Test(timeout = 5000l)
|
||||
public void testHelpMessage() throws Exception {
|
||||
Configuration conf = new YarnConfiguration();
|
||||
YarnClient mockYarnClient = createMockYarnClient(YarnApplicationState.FINISHED);
|
||||
YarnClient mockYarnClient = createMockYarnClient(
|
||||
YarnApplicationState.FINISHED,
|
||||
UserGroupInformation.getCurrentUser().getShortUserName());
|
||||
LogsCLI dumper = new LogsCLIForTest(mockYarnClient);
|
||||
dumper.setConf(conf);
|
||||
|
||||
|
|
@ -187,7 +192,7 @@ public class TestLogsCLI {
|
|||
String appReportStr = baos.toString("UTF-8");
|
||||
Assert.assertEquals(appReportStr, sysOutStream.toString());
|
||||
}
|
||||
|
||||
|
||||
@Test (timeout = 15000)
|
||||
public void testFetchApplictionLogs() throws Exception {
|
||||
String remoteLogRootDir = "target/logs/";
|
||||
|
|
@ -200,13 +205,13 @@ public class TestLogsCLI {
|
|||
FileSystem fs = FileSystem.get(configuration);
|
||||
|
||||
UserGroupInformation ugi = UserGroupInformation.getCurrentUser();
|
||||
ApplicationId appId = ApplicationIdPBImpl.newInstance(0, 1);
|
||||
ApplicationId appId = ApplicationId.newInstance(0, 1);
|
||||
ApplicationAttemptId appAttemptId =
|
||||
ApplicationAttemptIdPBImpl.newInstance(appId, 1);
|
||||
ContainerId containerId0 = ContainerIdPBImpl.newContainerId(appAttemptId, 0);
|
||||
ContainerId containerId1 = ContainerIdPBImpl.newContainerId(appAttemptId, 1);
|
||||
ContainerId containerId2 = ContainerIdPBImpl.newContainerId(appAttemptId, 2);
|
||||
ContainerId containerId3 = ContainerIdPBImpl.newContainerId(appAttemptId, 3);
|
||||
ApplicationAttemptId.newInstance(appId, 1);
|
||||
ContainerId containerId0 = ContainerId.newContainerId(appAttemptId, 0);
|
||||
ContainerId containerId1 = ContainerId.newContainerId(appAttemptId, 1);
|
||||
ContainerId containerId2 = ContainerId.newContainerId(appAttemptId, 2);
|
||||
ContainerId containerId3 = ContainerId.newContainerId(appAttemptId, 3);
|
||||
NodeId nodeId = NodeId.newInstance("localhost", 1234);
|
||||
|
||||
// create local logs
|
||||
|
|
@ -222,6 +227,7 @@ public class TestLogsCLI {
|
|||
fs.delete(appLogsDir, true);
|
||||
}
|
||||
assertTrue(fs.mkdirs(appLogsDir));
|
||||
|
||||
List<String> rootLogDirs = Arrays.asList(rootLogDir);
|
||||
|
||||
List<String> logTypes = new ArrayList<String>();
|
||||
|
|
@ -258,7 +264,8 @@ public class TestLogsCLI {
|
|||
containerId3, path, fs);
|
||||
|
||||
YarnClient mockYarnClient =
|
||||
createMockYarnClient(YarnApplicationState.FINISHED);
|
||||
createMockYarnClient(
|
||||
YarnApplicationState.FINISHED, ugi.getShortUserName());
|
||||
LogsCLI cli = new LogsCLIForTest(mockYarnClient);
|
||||
cli.setConf(configuration);
|
||||
|
||||
|
|
@ -348,6 +355,143 @@ public class TestLogsCLI {
|
|||
fs.delete(new Path(rootLogDir), true);
|
||||
}
|
||||
|
||||
@Test (timeout = 15000)
|
||||
public void testFetchApplictionLogsAsAnotherUser() throws Exception {
|
||||
String remoteLogRootDir = "target/logs/";
|
||||
String rootLogDir = "target/LocalLogs";
|
||||
|
||||
String testUser = "test";
|
||||
UserGroupInformation testUgi = UserGroupInformation
|
||||
.createRemoteUser(testUser);
|
||||
|
||||
Configuration configuration = new Configuration();
|
||||
configuration.setBoolean(YarnConfiguration.LOG_AGGREGATION_ENABLED, true);
|
||||
configuration
|
||||
.set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR, remoteLogRootDir);
|
||||
configuration.setBoolean(YarnConfiguration.YARN_ACL_ENABLE, true);
|
||||
configuration.set(YarnConfiguration.YARN_ADMIN_ACL, "admin");
|
||||
FileSystem fs = FileSystem.get(configuration);
|
||||
|
||||
ApplicationId appId = ApplicationId.newInstance(0, 1);
|
||||
ApplicationAttemptId appAttemptId =
|
||||
ApplicationAttemptId.newInstance(appId, 1);
|
||||
ContainerId containerId = ContainerId
|
||||
.newContainerId(appAttemptId, 1);
|
||||
NodeId nodeId = NodeId.newInstance("localhost", 1234);
|
||||
|
||||
try {
|
||||
Path rootLogDirPath = new Path(rootLogDir);
|
||||
if (fs.exists(rootLogDirPath)) {
|
||||
fs.delete(rootLogDirPath, true);
|
||||
}
|
||||
assertTrue(fs.mkdirs(rootLogDirPath));
|
||||
|
||||
// create local app dir for app
|
||||
final Path appLogsDir = new Path(rootLogDirPath, appId.toString());
|
||||
if (fs.exists(appLogsDir)) {
|
||||
fs.delete(appLogsDir, true);
|
||||
}
|
||||
assertTrue(fs.mkdirs(appLogsDir));
|
||||
|
||||
List<String> rootLogDirs = Arrays.asList(rootLogDir);
|
||||
List<String> logTypes = new ArrayList<String>();
|
||||
logTypes.add("syslog");
|
||||
|
||||
// create container logs in localLogDir for app
|
||||
createContainerLogInLocalDir(appLogsDir, containerId, fs, logTypes);
|
||||
|
||||
// create the remote app dir for app
|
||||
// but for a different user testUser"
|
||||
Path path = new Path(remoteLogRootDir + testUser + "/logs/" + appId);
|
||||
if (fs.exists(path)) {
|
||||
fs.delete(path, true);
|
||||
}
|
||||
assertTrue(fs.mkdirs(path));
|
||||
|
||||
// upload container logs for app into remote dir
|
||||
uploadContainerLogIntoRemoteDir(testUgi, configuration, rootLogDirs,
|
||||
nodeId, containerId, path, fs);
|
||||
|
||||
YarnClient mockYarnClient = createMockYarnClient(
|
||||
YarnApplicationState.FINISHED, testUgi.getShortUserName());
|
||||
LogsCLI cli = new LogsCLIForTest(mockYarnClient);
|
||||
cli.setConf(configuration);
|
||||
|
||||
// Verify that we can get the application logs by specifying
|
||||
// a correct appOwner
|
||||
int exitCode = cli.run(new String[] {
|
||||
"-applicationId", appId.toString(),
|
||||
"-appOwner", testUser});
|
||||
assertTrue(exitCode == 0);
|
||||
assertTrue(sysOutStream.toString().contains(
|
||||
"Hello " + containerId + " in syslog!"));
|
||||
sysOutStream.reset();
|
||||
|
||||
// Verify that we can not get the application logs
|
||||
// if an invalid user is specified
|
||||
exitCode = cli.run(new String[] {
|
||||
"-applicationId", appId.toString(),
|
||||
"-appOwner", "invalid"});
|
||||
assertTrue(exitCode == -1);
|
||||
assertTrue(sysErrStream.toString().contains("Can not find the logs "
|
||||
+ "for the application: " + appId.toString()));
|
||||
sysErrStream.reset();
|
||||
|
||||
// Verify that we do not specify appOwner, and can not
|
||||
// get appReport from RM, we still can figure out the appOwner
|
||||
// and can get app logs successfully.
|
||||
YarnClient mockYarnClient2 = createMockYarnClientUnknownApp();
|
||||
cli = new LogsCLIForTest(mockYarnClient2);
|
||||
cli.setConf(configuration);
|
||||
exitCode = cli.run(new String[] {
|
||||
"-applicationId", appId.toString()});
|
||||
assertTrue(exitCode == 0);
|
||||
assertTrue(sysOutStream.toString().contains("Hello "
|
||||
+ containerId + " in syslog!"));
|
||||
sysOutStream.reset();
|
||||
|
||||
// Verify that we could get the err message "Can not find the appOwner"
|
||||
// if we do not specify the appOwner, can not get appReport, and
|
||||
// the app does not exist in remote dir.
|
||||
ApplicationId appId2 = ApplicationId.newInstance(
|
||||
System.currentTimeMillis(), 2);
|
||||
exitCode = cli.run(new String[] {
|
||||
"-applicationId", appId2.toString()});
|
||||
assertTrue(exitCode == -1);
|
||||
assertTrue(sysErrStream.toString().contains(
|
||||
"Can not find the appOwner"));
|
||||
sysErrStream.reset();
|
||||
|
||||
// Verify that we could not get appOwner
|
||||
// because we don't have file-system permissions
|
||||
ApplicationId appTest = ApplicationId.newInstance(
|
||||
System.currentTimeMillis(), 1000);
|
||||
String priorityUser = "priority";
|
||||
Path pathWithoutPerm = new Path(remoteLogRootDir + priorityUser
|
||||
+ "/logs/" + appTest);
|
||||
if (fs.exists(pathWithoutPerm)) {
|
||||
fs.delete(pathWithoutPerm, true);
|
||||
}
|
||||
// The user will not have read permission for this directory.
|
||||
// To mimic the scenario that the user can not get file status
|
||||
FsPermission permission = FsPermission
|
||||
.createImmutable((short) 01300);
|
||||
assertTrue(fs.mkdirs(pathWithoutPerm, permission));
|
||||
|
||||
exitCode = cli.run(new String[] {
|
||||
"-applicationId", appTest.toString()});
|
||||
assertTrue(exitCode == -1);
|
||||
assertTrue(sysErrStream.toString().contains(
|
||||
"Guessed logs' owner is " + priorityUser + " and current user "
|
||||
+ UserGroupInformation.getCurrentUser().getUserName()
|
||||
+ " does not have permission to access"));
|
||||
sysErrStream.reset();
|
||||
} finally {
|
||||
fs.delete(new Path(remoteLogRootDir), true);
|
||||
fs.delete(new Path(rootLogDir), true);
|
||||
}
|
||||
}
|
||||
|
||||
@Test (timeout = 15000)
|
||||
public void testPrintContainerLogMetadata() throws Exception {
|
||||
String remoteLogRootDir = "target/logs/";
|
||||
|
|
@ -380,7 +524,8 @@ public class TestLogsCLI {
|
|||
appId, containerIds, nodeIds);
|
||||
|
||||
YarnClient mockYarnClient =
|
||||
createMockYarnClient(YarnApplicationState.FINISHED);
|
||||
createMockYarnClient(YarnApplicationState.FINISHED,
|
||||
UserGroupInformation.getCurrentUser().getShortUserName());
|
||||
LogsCLI cli = new LogsCLIForTest(mockYarnClient);
|
||||
cli.setConf(configuration);
|
||||
|
||||
|
|
@ -466,7 +611,8 @@ public class TestLogsCLI {
|
|||
appId, containerIds, nodeIds);
|
||||
|
||||
YarnClient mockYarnClient =
|
||||
createMockYarnClient(YarnApplicationState.FINISHED);
|
||||
createMockYarnClient(YarnApplicationState.FINISHED,
|
||||
UserGroupInformation.getCurrentUser().getShortUserName());
|
||||
LogsCLI cli = new LogsCLIForTest(mockYarnClient);
|
||||
cli.setConf(configuration);
|
||||
|
||||
|
|
@ -508,7 +654,8 @@ public class TestLogsCLI {
|
|||
assertTrue(fs.exists(harPath));
|
||||
|
||||
YarnClient mockYarnClient =
|
||||
createMockYarnClient(YarnApplicationState.FINISHED);
|
||||
createMockYarnClient(YarnApplicationState.FINISHED,
|
||||
ugi.getShortUserName());
|
||||
LogsCLI cli = new LogsCLIForTest(mockYarnClient);
|
||||
cli.setConf(configuration);
|
||||
int exitCode = cli.run(new String[]{"-applicationId",
|
||||
|
|
@ -630,10 +777,12 @@ public class TestLogsCLI {
|
|||
writer.close();
|
||||
}
|
||||
|
||||
private YarnClient createMockYarnClient(YarnApplicationState appState)
|
||||
private YarnClient createMockYarnClient(YarnApplicationState appState,
|
||||
String user)
|
||||
throws YarnException, IOException {
|
||||
YarnClient mockClient = mock(YarnClient.class);
|
||||
ApplicationReport mockAppReport = mock(ApplicationReport.class);
|
||||
doReturn(user).when(mockAppReport).getUser();
|
||||
doReturn(appState).when(mockAppReport).getYarnApplicationState();
|
||||
doReturn(mockAppReport).when(mockClient).getApplicationReport(
|
||||
any(ApplicationId.class));
|
||||
|
|
@ -659,9 +808,9 @@ public class TestLogsCLI {
|
|||
}
|
||||
|
||||
private static class LogsCLIForTest extends LogsCLI {
|
||||
|
||||
|
||||
private YarnClient yarnClient;
|
||||
|
||||
|
||||
public LogsCLIForTest(YarnClient yarnClient) {
|
||||
super();
|
||||
this.yarnClient = yarnClient;
|
||||
|
|
|
|||
|
|
@ -23,6 +23,7 @@ import java.io.EOFException;
|
|||
import java.io.FileNotFoundException;
|
||||
import java.io.IOException;
|
||||
import java.io.PrintStream;
|
||||
import java.nio.file.AccessDeniedException;
|
||||
import java.util.List;
|
||||
|
||||
import org.apache.commons.lang.StringUtils;
|
||||
|
|
@ -34,6 +35,8 @@ import org.apache.hadoop.fs.FileStatus;
|
|||
import org.apache.hadoop.fs.HarFs;
|
||||
import org.apache.hadoop.fs.Path;
|
||||
import org.apache.hadoop.fs.RemoteIterator;
|
||||
import org.apache.hadoop.security.AccessControlException;
|
||||
import org.apache.hadoop.security.UserGroupInformation;
|
||||
import org.apache.hadoop.yarn.api.records.ApplicationId;
|
||||
import org.apache.hadoop.yarn.conf.YarnConfiguration;
|
||||
import org.apache.hadoop.yarn.logaggregation.AggregatedLogFormat.LogKey;
|
||||
|
|
@ -54,6 +57,53 @@ public class LogCLIHelpers implements Configurable {
|
|||
null);
|
||||
}
|
||||
|
||||
@Private
|
||||
@VisibleForTesting
|
||||
/**
|
||||
* Return the owner for a given AppId
|
||||
* @param remoteRootLogDir
|
||||
* @param appId
|
||||
* @param bestGuess
|
||||
* @param conf
|
||||
* @return the owner or null
|
||||
* @throws IOException
|
||||
*/
|
||||
public static String getOwnerForAppIdOrNull(
|
||||
ApplicationId appId, String bestGuess,
|
||||
Configuration conf) throws IOException {
|
||||
Path remoteRootLogDir = new Path(conf.get(
|
||||
YarnConfiguration.NM_REMOTE_APP_LOG_DIR,
|
||||
YarnConfiguration.DEFAULT_NM_REMOTE_APP_LOG_DIR));
|
||||
String suffix = LogAggregationUtils.getRemoteNodeLogDirSuffix(conf);
|
||||
Path fullPath = LogAggregationUtils.getRemoteAppLogDir(remoteRootLogDir,
|
||||
appId, bestGuess, suffix);
|
||||
FileContext fc =
|
||||
FileContext.getFileContext(remoteRootLogDir.toUri(), conf);
|
||||
String pathAccess = fullPath.toString();
|
||||
try {
|
||||
if (fc.util().exists(fullPath)) {
|
||||
return bestGuess;
|
||||
}
|
||||
Path toMatch = LogAggregationUtils.
|
||||
getRemoteAppLogDir(remoteRootLogDir, appId, "*", suffix);
|
||||
pathAccess = toMatch.toString();
|
||||
FileStatus[] matching = fc.util().globStatus(toMatch);
|
||||
if (matching == null || matching.length != 1) {
|
||||
return null;
|
||||
}
|
||||
//fetch the user from the full path /app-logs/user[/suffix]/app_id
|
||||
Path parent = matching[0].getPath().getParent();
|
||||
//skip the suffix too
|
||||
if (suffix != null && !StringUtils.isEmpty(suffix)) {
|
||||
parent = parent.getParent();
|
||||
}
|
||||
return parent.getName();
|
||||
} catch (AccessControlException | AccessDeniedException ex) {
|
||||
logDirNoAccessPermission(pathAccess, bestGuess, ex.getMessage());
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
@Private
|
||||
@VisibleForTesting
|
||||
public int dumpAContainersLogsForALogType(String appId, String containerId,
|
||||
|
|
@ -93,12 +143,12 @@ public class LogCLIHelpers implements Configurable {
|
|||
thisNodeFile.getPath());
|
||||
if (logType == null) {
|
||||
if (dumpAContainerLogs(containerId, reader, System.out,
|
||||
thisNodeFile.getModificationTime()) > -1) {
|
||||
thisNodeFile.getModificationTime()) > -1) {
|
||||
foundContainerLogs = true;
|
||||
}
|
||||
} else {
|
||||
if (dumpAContainerLogsForALogType(containerId, reader, System.out,
|
||||
thisNodeFile.getModificationTime(), logType) > -1) {
|
||||
thisNodeFile.getModificationTime(), logType) > -1) {
|
||||
foundContainerLogs = true;
|
||||
}
|
||||
}
|
||||
|
|
@ -182,7 +232,7 @@ public class LogCLIHelpers implements Configurable {
|
|||
while (true) {
|
||||
try {
|
||||
LogReader.readAContainerLogsForALogType(valueStream, out,
|
||||
logUploadedTime);
|
||||
logUploadedTime);
|
||||
foundContainerLogs = true;
|
||||
} catch (EOFException eof) {
|
||||
break;
|
||||
|
|
@ -249,9 +299,10 @@ public class LogCLIHelpers implements Configurable {
|
|||
continue;
|
||||
}
|
||||
if (!thisNodeFile.getPath().getName()
|
||||
.endsWith(LogAggregationUtils.TMP_FILE_SUFFIX)) {
|
||||
.endsWith(LogAggregationUtils.TMP_FILE_SUFFIX)) {
|
||||
AggregatedLogFormat.LogReader reader =
|
||||
new AggregatedLogFormat.LogReader(getConf(), thisNodeFile.getPath());
|
||||
new AggregatedLogFormat.LogReader(getConf(),
|
||||
thisNodeFile.getPath());
|
||||
try {
|
||||
|
||||
DataInputStream valueStream;
|
||||
|
|
@ -261,13 +312,14 @@ public class LogCLIHelpers implements Configurable {
|
|||
while (valueStream != null) {
|
||||
|
||||
String containerString =
|
||||
"\n\nContainer: " + key + " on " + thisNodeFile.getPath().getName();
|
||||
"\n\nContainer: " + key + " on "
|
||||
+ thisNodeFile.getPath().getName();
|
||||
out.println(containerString);
|
||||
out.println(StringUtils.repeat("=", containerString.length()));
|
||||
while (true) {
|
||||
try {
|
||||
LogReader.readAContainerLogsForALogType(valueStream, out,
|
||||
thisNodeFile.getModificationTime());
|
||||
thisNodeFile.getModificationTime());
|
||||
foundAnyLogs = true;
|
||||
} catch (EOFException eof) {
|
||||
break;
|
||||
|
|
@ -283,7 +335,7 @@ public class LogCLIHelpers implements Configurable {
|
|||
}
|
||||
}
|
||||
}
|
||||
if (! foundAnyLogs) {
|
||||
if (!foundAnyLogs) {
|
||||
emptyLogDir(getRemoteAppLogDir(appId, appOwner).toString());
|
||||
return -1;
|
||||
}
|
||||
|
|
@ -398,6 +450,9 @@ public class LogCLIHelpers implements Configurable {
|
|||
getConf()).listStatus(remoteAppLogDir);
|
||||
} catch (FileNotFoundException fnf) {
|
||||
logDirNotExist(remoteAppLogDir.toString());
|
||||
} catch (AccessControlException | AccessDeniedException ace) {
|
||||
logDirNoAccessPermission(remoteAppLogDir.toString(), appOwner,
|
||||
ace.getMessage());
|
||||
}
|
||||
return nodeFiles;
|
||||
}
|
||||
|
|
@ -426,7 +481,7 @@ public class LogCLIHelpers implements Configurable {
|
|||
|
||||
private static void containerLogNotFound(String containerId) {
|
||||
System.err.println("Logs for container " + containerId
|
||||
+ " are not present in this log-file.");
|
||||
+ " are not present in this log-file.");
|
||||
}
|
||||
|
||||
private static void logDirNotExist(String remoteAppLogDir) {
|
||||
|
|
@ -437,4 +492,13 @@ public class LogCLIHelpers implements Configurable {
|
|||
private static void emptyLogDir(String remoteAppLogDir) {
|
||||
System.err.println(remoteAppLogDir + " does not have any log files.");
|
||||
}
|
||||
|
||||
private static void logDirNoAccessPermission(String remoteAppLogDir,
|
||||
String appOwner, String errorMessage) throws IOException {
|
||||
System.err.println("Guessed logs' owner is " + appOwner
|
||||
+ " and current user "
|
||||
+ UserGroupInformation.getCurrentUser().getUserName() + " does not "
|
||||
+ "have permission to access " + remoteAppLogDir
|
||||
+ ". Error message found: " + errorMessage);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in New Issue