YARN-4842. Fixed "yarn logs" command to guess (and thus not require) the appOwner argument when viewing another user's logs. Contributed by Ram Venkatesh and Xuan Gong.
(cherry picked from commit 87f5e35133)
This commit is contained in:
Vinod Kumar Vavilapalli
parent
007d6d1d57
commit
e0d6a9632d
|
|
@ -151,14 +151,12 @@ public class LogsCLI extends Configured implements Tool {
|
|||
LogCLIHelpers logCliHelper = new LogCLIHelpers();
|
||||
logCliHelper.setConf(getConf());
|
||||
|
||||
if (appOwner == null || appOwner.isEmpty()) {
|
||||
appOwner = UserGroupInformation.getCurrentUser().getShortUserName();
|
||||
}
|
||||
|
||||
boolean appStateObtainedSuccessfully = true;
|
||||
YarnApplicationState appState = YarnApplicationState.NEW;
|
||||
ApplicationReport appReport = null;
|
||||
try {
|
||||
appState = getApplicationState(appId);
|
||||
appReport = getApplicationReport(appId);
|
||||
appState = appReport.getYarnApplicationState();
|
||||
if (appState == YarnApplicationState.NEW
|
||||
|| appState == YarnApplicationState.NEW_SAVING
|
||||
|| appState == YarnApplicationState.SUBMITTED) {
|
||||
|
|
@ -171,6 +169,16 @@ public class LogsCLI extends Configured implements Tool {
|
|||
+ " Attempting to fetch logs directly from the filesystem.");
|
||||
}
|
||||
|
||||
if (appOwner == null || appOwner.isEmpty()) {
|
||||
appOwner = guessAppOwner(appReport, appId);
|
||||
if (appOwner == null) {
|
||||
System.err.println("Can not find the appOwner. "
|
||||
+ "Please specify the correct appOwner");
|
||||
System.err.println("Could not locate application logs for " + appId);
|
||||
return -1;
|
||||
}
|
||||
}
|
||||
|
||||
if (showMetaInfo) {
|
||||
return showMetaInfo(appState, appStateObtainedSuccessfully,
|
||||
logCliHelper, appId, containerIdStr, nodeAddress, appOwner);
|
||||
|
|
@ -201,6 +209,10 @@ public class LogsCLI extends Configured implements Tool {
|
|||
if (nodeAddress == null) {
|
||||
resultCode =
|
||||
logCliHelper.dumpAllContainersLogs(appId, appOwner, System.out);
|
||||
if (resultCode == -1) {
|
||||
System.err.println("Can not find the logs for the application: "
|
||||
+ appId + " with the appOwner: " + appOwner);
|
||||
}
|
||||
} else {
|
||||
System.err.println("Should at least provide ContainerId!");
|
||||
printHelpMessage(printOpts);
|
||||
|
|
@ -210,13 +222,12 @@ public class LogsCLI extends Configured implements Tool {
|
|||
return resultCode;
|
||||
}
|
||||
|
||||
private YarnApplicationState getApplicationState(ApplicationId appId)
|
||||
private ApplicationReport getApplicationReport(ApplicationId appId)
|
||||
throws IOException, YarnException {
|
||||
YarnClient yarnClient = createYarnClient();
|
||||
|
||||
try {
|
||||
ApplicationReport appReport = yarnClient.getApplicationReport(appId);
|
||||
return appReport.getYarnApplicationState();
|
||||
return yarnClient.getApplicationReport(appId);
|
||||
} finally {
|
||||
yarnClient.close();
|
||||
}
|
||||
|
|
@ -693,11 +704,12 @@ public class LogsCLI extends Configured implements Tool {
|
|||
amContainersList, logFiles, logCliHelper, appOwner, true);
|
||||
} else {
|
||||
System.err.println("Can not get AMContainers logs for "
|
||||
+ "the application:" + appId);
|
||||
System.err.println("This application:" + appId + " is finished."
|
||||
+ " Please enable the application history service. Or Using "
|
||||
+ "yarn logs -applicationId <appId> -containerId <containerId> "
|
||||
+ "--nodeAddress <nodeHttpAddress> to get the container logs");
|
||||
+ "the application:" + appId + " with the appOwner:" + appOwner);
|
||||
System.err.println("This application:" + appId + " has finished."
|
||||
+ " Please enable the application-history service or explicitly"
|
||||
+ " use 'yarn logs -applicationId <appId> "
|
||||
+ "-containerId <containerId> --nodeAddress <nodeHttpAddress>' "
|
||||
+ "to get the container logs.");
|
||||
return -1;
|
||||
}
|
||||
}
|
||||
|
|
@ -750,7 +762,8 @@ public class LogsCLI extends Configured implements Tool {
|
|||
appOwner);
|
||||
} else if (!isApplicationFinished(appState)) {
|
||||
System.err.println("Unable to get logs for this container:"
|
||||
+ containerIdStr + "for the application:" + appIdStr);
|
||||
+ containerIdStr + "for the application:" + appIdStr
|
||||
+ " with the appOwner: " + appOwner);
|
||||
System.err.println("The application: " + appIdStr
|
||||
+ " is still running, and we can not get Container report "
|
||||
+ "for the container: " + containerIdStr +". Please try later "
|
||||
|
|
@ -821,4 +834,18 @@ public class LogsCLI extends Configured implements Tool {
|
|||
return isAppFinished;
|
||||
}
|
||||
}
|
||||
|
||||
private String guessAppOwner(ApplicationReport appReport,
|
||||
ApplicationId appId) throws IOException {
|
||||
String appOwner = null;
|
||||
if (appReport != null) {
|
||||
//always use the app owner from the app report if possible
|
||||
appOwner = appReport.getUser();
|
||||
} else {
|
||||
appOwner = UserGroupInformation.getCurrentUser().getShortUserName();
|
||||
appOwner = LogCLIHelpers.getOwnerForAppIdOrNull(
|
||||
appId, appOwner, getConf());
|
||||
}
|
||||
return appOwner;
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -42,11 +42,11 @@ import java.util.HashSet;
|
|||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
import org.junit.Assert;
|
||||
import org.apache.hadoop.conf.Configuration;
|
||||
import org.apache.hadoop.fs.FileSystem;
|
||||
import org.apache.hadoop.fs.LocalFileSystem;
|
||||
import org.apache.hadoop.fs.Path;
|
||||
import org.apache.hadoop.fs.permission.FsPermission;
|
||||
import org.apache.hadoop.security.UserGroupInformation;
|
||||
import org.apache.hadoop.yarn.api.records.ApplicationAccessType;
|
||||
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
|
||||
|
|
@ -55,22 +55,21 @@ import org.apache.hadoop.yarn.api.records.ApplicationReport;
|
|||
import org.apache.hadoop.yarn.api.records.ContainerId;
|
||||
import org.apache.hadoop.yarn.api.records.NodeId;
|
||||
import org.apache.hadoop.yarn.api.records.YarnApplicationState;
|
||||
import org.apache.hadoop.yarn.api.records.impl.pb.ApplicationAttemptIdPBImpl;
|
||||
import org.apache.hadoop.yarn.api.records.impl.pb.ApplicationIdPBImpl;
|
||||
import org.apache.hadoop.yarn.api.records.impl.pb.ContainerIdPBImpl;
|
||||
import org.apache.hadoop.yarn.client.api.YarnClient;
|
||||
import org.apache.hadoop.yarn.conf.YarnConfiguration;
|
||||
import org.apache.hadoop.yarn.exceptions.YarnException;
|
||||
import org.apache.hadoop.yarn.logaggregation.AggregatedLogFormat;
|
||||
import org.apache.hadoop.yarn.logaggregation.LogAggregationUtils;
|
||||
import org.apache.hadoop.yarn.logaggregation.LogCLIHelpers;
|
||||
import org.junit.Assert;
|
||||
import org.junit.Before;
|
||||
import org.junit.Test;
|
||||
|
||||
public class TestLogsCLI {
|
||||
|
||||
ByteArrayOutputStream sysOutStream;
|
||||
private PrintStream sysOut;
|
||||
|
||||
|
||||
ByteArrayOutputStream sysErrStream;
|
||||
private PrintStream sysErr;
|
||||
|
||||
|
|
@ -79,7 +78,7 @@ public class TestLogsCLI {
|
|||
sysOutStream = new ByteArrayOutputStream();
|
||||
sysOut = new PrintStream(sysOutStream);
|
||||
System.setOut(sysOut);
|
||||
|
||||
|
||||
sysErrStream = new ByteArrayOutputStream();
|
||||
sysErr = new PrintStream(sysErrStream);
|
||||
System.setErr(sysErr);
|
||||
|
|
@ -91,16 +90,18 @@ public class TestLogsCLI {
|
|||
conf.setClass("fs.file.impl", LocalFileSystem.class, FileSystem.class);
|
||||
LogCLIHelpers cliHelper = new LogCLIHelpers();
|
||||
cliHelper.setConf(conf);
|
||||
YarnClient mockYarnClient = createMockYarnClient(YarnApplicationState.FINISHED);
|
||||
YarnClient mockYarnClient = createMockYarnClient(
|
||||
YarnApplicationState.FINISHED,
|
||||
UserGroupInformation.getCurrentUser().getShortUserName());
|
||||
LogsCLI dumper = new LogsCLIForTest(mockYarnClient);
|
||||
dumper.setConf(conf);
|
||||
|
||||
|
||||
// verify dumping a non-existent application's logs returns a failure code
|
||||
int exitCode = dumper.run( new String[] {
|
||||
"-applicationId", "application_0_0" } );
|
||||
assertTrue("Should return an error code", exitCode != 0);
|
||||
|
||||
// verify dumping a non-existent container log is a failure code
|
||||
|
||||
// verify dumping a non-existent container log is a failure code
|
||||
exitCode = cliHelper.dumpAContainersLogs("application_0_0", "container_0_0",
|
||||
"nonexistentnode:1234", "nobody");
|
||||
assertTrue("Should return an error code", exitCode != 0);
|
||||
|
|
@ -109,10 +110,12 @@ public class TestLogsCLI {
|
|||
@Test(timeout = 5000l)
|
||||
public void testInvalidApplicationId() throws Exception {
|
||||
Configuration conf = new YarnConfiguration();
|
||||
YarnClient mockYarnClient = createMockYarnClient(YarnApplicationState.FINISHED);
|
||||
YarnClient mockYarnClient = createMockYarnClient(
|
||||
YarnApplicationState.FINISHED,
|
||||
UserGroupInformation.getCurrentUser().getShortUserName());
|
||||
LogsCLI cli = new LogsCLIForTest(mockYarnClient);
|
||||
cli.setConf(conf);
|
||||
|
||||
|
||||
int exitCode = cli.run( new String[] { "-applicationId", "not_an_app_id"});
|
||||
assertTrue(exitCode == -1);
|
||||
assertTrue(sysErrStream.toString().startsWith("Invalid ApplicationId specified"));
|
||||
|
|
@ -137,7 +140,9 @@ public class TestLogsCLI {
|
|||
@Test(timeout = 5000l)
|
||||
public void testHelpMessage() throws Exception {
|
||||
Configuration conf = new YarnConfiguration();
|
||||
YarnClient mockYarnClient = createMockYarnClient(YarnApplicationState.FINISHED);
|
||||
YarnClient mockYarnClient = createMockYarnClient(
|
||||
YarnApplicationState.FINISHED,
|
||||
UserGroupInformation.getCurrentUser().getShortUserName());
|
||||
LogsCLI dumper = new LogsCLIForTest(mockYarnClient);
|
||||
dumper.setConf(conf);
|
||||
|
||||
|
|
@ -187,7 +192,7 @@ public class TestLogsCLI {
|
|||
String appReportStr = baos.toString("UTF-8");
|
||||
Assert.assertEquals(appReportStr, sysOutStream.toString());
|
||||
}
|
||||
|
||||
|
||||
@Test (timeout = 15000)
|
||||
public void testFetchApplictionLogs() throws Exception {
|
||||
String remoteLogRootDir = "target/logs/";
|
||||
|
|
@ -200,13 +205,13 @@ public class TestLogsCLI {
|
|||
FileSystem fs = FileSystem.get(configuration);
|
||||
|
||||
UserGroupInformation ugi = UserGroupInformation.getCurrentUser();
|
||||
ApplicationId appId = ApplicationIdPBImpl.newInstance(0, 1);
|
||||
ApplicationId appId = ApplicationId.newInstance(0, 1);
|
||||
ApplicationAttemptId appAttemptId =
|
||||
ApplicationAttemptIdPBImpl.newInstance(appId, 1);
|
||||
ContainerId containerId0 = ContainerIdPBImpl.newContainerId(appAttemptId, 0);
|
||||
ContainerId containerId1 = ContainerIdPBImpl.newContainerId(appAttemptId, 1);
|
||||
ContainerId containerId2 = ContainerIdPBImpl.newContainerId(appAttemptId, 2);
|
||||
ContainerId containerId3 = ContainerIdPBImpl.newContainerId(appAttemptId, 3);
|
||||
ApplicationAttemptId.newInstance(appId, 1);
|
||||
ContainerId containerId0 = ContainerId.newContainerId(appAttemptId, 0);
|
||||
ContainerId containerId1 = ContainerId.newContainerId(appAttemptId, 1);
|
||||
ContainerId containerId2 = ContainerId.newContainerId(appAttemptId, 2);
|
||||
ContainerId containerId3 = ContainerId.newContainerId(appAttemptId, 3);
|
||||
NodeId nodeId = NodeId.newInstance("localhost", 1234);
|
||||
|
||||
// create local logs
|
||||
|
|
@ -222,6 +227,7 @@ public class TestLogsCLI {
|
|||
fs.delete(appLogsDir, true);
|
||||
}
|
||||
assertTrue(fs.mkdirs(appLogsDir));
|
||||
|
||||
List<String> rootLogDirs = Arrays.asList(rootLogDir);
|
||||
|
||||
List<String> logTypes = new ArrayList<String>();
|
||||
|
|
@ -258,7 +264,8 @@ public class TestLogsCLI {
|
|||
containerId3, path, fs);
|
||||
|
||||
YarnClient mockYarnClient =
|
||||
createMockYarnClient(YarnApplicationState.FINISHED);
|
||||
createMockYarnClient(
|
||||
YarnApplicationState.FINISHED, ugi.getShortUserName());
|
||||
LogsCLI cli = new LogsCLIForTest(mockYarnClient);
|
||||
cli.setConf(configuration);
|
||||
|
||||
|
|
@ -348,6 +355,143 @@ public class TestLogsCLI {
|
|||
fs.delete(new Path(rootLogDir), true);
|
||||
}
|
||||
|
||||
@Test (timeout = 15000)
|
||||
public void testFetchApplictionLogsAsAnotherUser() throws Exception {
|
||||
String remoteLogRootDir = "target/logs/";
|
||||
String rootLogDir = "target/LocalLogs";
|
||||
|
||||
String testUser = "test";
|
||||
UserGroupInformation testUgi = UserGroupInformation
|
||||
.createRemoteUser(testUser);
|
||||
|
||||
Configuration configuration = new Configuration();
|
||||
configuration.setBoolean(YarnConfiguration.LOG_AGGREGATION_ENABLED, true);
|
||||
configuration
|
||||
.set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR, remoteLogRootDir);
|
||||
configuration.setBoolean(YarnConfiguration.YARN_ACL_ENABLE, true);
|
||||
configuration.set(YarnConfiguration.YARN_ADMIN_ACL, "admin");
|
||||
FileSystem fs = FileSystem.get(configuration);
|
||||
|
||||
ApplicationId appId = ApplicationId.newInstance(0, 1);
|
||||
ApplicationAttemptId appAttemptId =
|
||||
ApplicationAttemptId.newInstance(appId, 1);
|
||||
ContainerId containerId = ContainerId
|
||||
.newContainerId(appAttemptId, 1);
|
||||
NodeId nodeId = NodeId.newInstance("localhost", 1234);
|
||||
|
||||
try {
|
||||
Path rootLogDirPath = new Path(rootLogDir);
|
||||
if (fs.exists(rootLogDirPath)) {
|
||||
fs.delete(rootLogDirPath, true);
|
||||
}
|
||||
assertTrue(fs.mkdirs(rootLogDirPath));
|
||||
|
||||
// create local app dir for app
|
||||
final Path appLogsDir = new Path(rootLogDirPath, appId.toString());
|
||||
if (fs.exists(appLogsDir)) {
|
||||
fs.delete(appLogsDir, true);
|
||||
}
|
||||
assertTrue(fs.mkdirs(appLogsDir));
|
||||
|
||||
List<String> rootLogDirs = Arrays.asList(rootLogDir);
|
||||
List<String> logTypes = new ArrayList<String>();
|
||||
logTypes.add("syslog");
|
||||
|
||||
// create container logs in localLogDir for app
|
||||
createContainerLogInLocalDir(appLogsDir, containerId, fs, logTypes);
|
||||
|
||||
// create the remote app dir for app
|
||||
// but for a different user testUser"
|
||||
Path path = new Path(remoteLogRootDir + testUser + "/logs/" + appId);
|
||||
if (fs.exists(path)) {
|
||||
fs.delete(path, true);
|
||||
}
|
||||
assertTrue(fs.mkdirs(path));
|
||||
|
||||
// upload container logs for app into remote dir
|
||||
uploadContainerLogIntoRemoteDir(testUgi, configuration, rootLogDirs,
|
||||
nodeId, containerId, path, fs);
|
||||
|
||||
YarnClient mockYarnClient = createMockYarnClient(
|
||||
YarnApplicationState.FINISHED, testUgi.getShortUserName());
|
||||
LogsCLI cli = new LogsCLIForTest(mockYarnClient);
|
||||
cli.setConf(configuration);
|
||||
|
||||
// Verify that we can get the application logs by specifying
|
||||
// a correct appOwner
|
||||
int exitCode = cli.run(new String[] {
|
||||
"-applicationId", appId.toString(),
|
||||
"-appOwner", testUser});
|
||||
assertTrue(exitCode == 0);
|
||||
assertTrue(sysOutStream.toString().contains(
|
||||
"Hello " + containerId + " in syslog!"));
|
||||
sysOutStream.reset();
|
||||
|
||||
// Verify that we can not get the application logs
|
||||
// if an invalid user is specified
|
||||
exitCode = cli.run(new String[] {
|
||||
"-applicationId", appId.toString(),
|
||||
"-appOwner", "invalid"});
|
||||
assertTrue(exitCode == -1);
|
||||
assertTrue(sysErrStream.toString().contains("Can not find the logs "
|
||||
+ "for the application: " + appId.toString()));
|
||||
sysErrStream.reset();
|
||||
|
||||
// Verify that we do not specify appOwner, and can not
|
||||
// get appReport from RM, we still can figure out the appOwner
|
||||
// and can get app logs successfully.
|
||||
YarnClient mockYarnClient2 = createMockYarnClientUnknownApp();
|
||||
cli = new LogsCLIForTest(mockYarnClient2);
|
||||
cli.setConf(configuration);
|
||||
exitCode = cli.run(new String[] {
|
||||
"-applicationId", appId.toString()});
|
||||
assertTrue(exitCode == 0);
|
||||
assertTrue(sysOutStream.toString().contains("Hello "
|
||||
+ containerId + " in syslog!"));
|
||||
sysOutStream.reset();
|
||||
|
||||
// Verify that we could get the err message "Can not find the appOwner"
|
||||
// if we do not specify the appOwner, can not get appReport, and
|
||||
// the app does not exist in remote dir.
|
||||
ApplicationId appId2 = ApplicationId.newInstance(
|
||||
System.currentTimeMillis(), 2);
|
||||
exitCode = cli.run(new String[] {
|
||||
"-applicationId", appId2.toString()});
|
||||
assertTrue(exitCode == -1);
|
||||
assertTrue(sysErrStream.toString().contains(
|
||||
"Can not find the appOwner"));
|
||||
sysErrStream.reset();
|
||||
|
||||
// Verify that we could not get appOwner
|
||||
// because we don't have file-system permissions
|
||||
ApplicationId appTest = ApplicationId.newInstance(
|
||||
System.currentTimeMillis(), 1000);
|
||||
String priorityUser = "priority";
|
||||
Path pathWithoutPerm = new Path(remoteLogRootDir + priorityUser
|
||||
+ "/logs/" + appTest);
|
||||
if (fs.exists(pathWithoutPerm)) {
|
||||
fs.delete(pathWithoutPerm, true);
|
||||
}
|
||||
// The user will not have read permission for this directory.
|
||||
// To mimic the scenario that the user can not get file status
|
||||
FsPermission permission = FsPermission
|
||||
.createImmutable((short) 01300);
|
||||
assertTrue(fs.mkdirs(pathWithoutPerm, permission));
|
||||
|
||||
exitCode = cli.run(new String[] {
|
||||
"-applicationId", appTest.toString()});
|
||||
assertTrue(exitCode == -1);
|
||||
assertTrue(sysErrStream.toString().contains(
|
||||
"Guessed logs' owner is " + priorityUser + " and current user "
|
||||
+ UserGroupInformation.getCurrentUser().getUserName()
|
||||
+ " does not have permission to access"));
|
||||
sysErrStream.reset();
|
||||
} finally {
|
||||
fs.delete(new Path(remoteLogRootDir), true);
|
||||
fs.delete(new Path(rootLogDir), true);
|
||||
}
|
||||
}
|
||||
|
||||
@Test (timeout = 15000)
|
||||
public void testPrintContainerLogMetadata() throws Exception {
|
||||
String remoteLogRootDir = "target/logs/";
|
||||
|
|
@ -380,7 +524,8 @@ public class TestLogsCLI {
|
|||
appId, containerIds, nodeIds);
|
||||
|
||||
YarnClient mockYarnClient =
|
||||
createMockYarnClient(YarnApplicationState.FINISHED);
|
||||
createMockYarnClient(YarnApplicationState.FINISHED,
|
||||
UserGroupInformation.getCurrentUser().getShortUserName());
|
||||
LogsCLI cli = new LogsCLIForTest(mockYarnClient);
|
||||
cli.setConf(configuration);
|
||||
|
||||
|
|
@ -466,7 +611,8 @@ public class TestLogsCLI {
|
|||
appId, containerIds, nodeIds);
|
||||
|
||||
YarnClient mockYarnClient =
|
||||
createMockYarnClient(YarnApplicationState.FINISHED);
|
||||
createMockYarnClient(YarnApplicationState.FINISHED,
|
||||
UserGroupInformation.getCurrentUser().getShortUserName());
|
||||
LogsCLI cli = new LogsCLIForTest(mockYarnClient);
|
||||
cli.setConf(configuration);
|
||||
|
||||
|
|
@ -508,7 +654,8 @@ public class TestLogsCLI {
|
|||
assertTrue(fs.exists(harPath));
|
||||
|
||||
YarnClient mockYarnClient =
|
||||
createMockYarnClient(YarnApplicationState.FINISHED);
|
||||
createMockYarnClient(YarnApplicationState.FINISHED,
|
||||
ugi.getShortUserName());
|
||||
LogsCLI cli = new LogsCLIForTest(mockYarnClient);
|
||||
cli.setConf(configuration);
|
||||
int exitCode = cli.run(new String[]{"-applicationId",
|
||||
|
|
@ -630,10 +777,12 @@ public class TestLogsCLI {
|
|||
writer.close();
|
||||
}
|
||||
|
||||
private YarnClient createMockYarnClient(YarnApplicationState appState)
|
||||
private YarnClient createMockYarnClient(YarnApplicationState appState,
|
||||
String user)
|
||||
throws YarnException, IOException {
|
||||
YarnClient mockClient = mock(YarnClient.class);
|
||||
ApplicationReport mockAppReport = mock(ApplicationReport.class);
|
||||
doReturn(user).when(mockAppReport).getUser();
|
||||
doReturn(appState).when(mockAppReport).getYarnApplicationState();
|
||||
doReturn(mockAppReport).when(mockClient).getApplicationReport(
|
||||
any(ApplicationId.class));
|
||||
|
|
@ -659,9 +808,9 @@ public class TestLogsCLI {
|
|||
}
|
||||
|
||||
private static class LogsCLIForTest extends LogsCLI {
|
||||
|
||||
|
||||
private YarnClient yarnClient;
|
||||
|
||||
|
||||
public LogsCLIForTest(YarnClient yarnClient) {
|
||||
super();
|
||||
this.yarnClient = yarnClient;
|
||||
|
|
|
|||
|
|
@ -23,6 +23,7 @@ import java.io.EOFException;
|
|||
import java.io.FileNotFoundException;
|
||||
import java.io.IOException;
|
||||
import java.io.PrintStream;
|
||||
import java.nio.file.AccessDeniedException;
|
||||
import java.util.List;
|
||||
|
||||
import org.apache.commons.lang.StringUtils;
|
||||
|
|
@ -34,6 +35,8 @@ import org.apache.hadoop.fs.FileStatus;
|
|||
import org.apache.hadoop.fs.HarFs;
|
||||
import org.apache.hadoop.fs.Path;
|
||||
import org.apache.hadoop.fs.RemoteIterator;
|
||||
import org.apache.hadoop.security.AccessControlException;
|
||||
import org.apache.hadoop.security.UserGroupInformation;
|
||||
import org.apache.hadoop.yarn.api.records.ApplicationId;
|
||||
import org.apache.hadoop.yarn.conf.YarnConfiguration;
|
||||
import org.apache.hadoop.yarn.logaggregation.AggregatedLogFormat.LogKey;
|
||||
|
|
@ -54,6 +57,53 @@ public class LogCLIHelpers implements Configurable {
|
|||
null);
|
||||
}
|
||||
|
||||
@Private
|
||||
@VisibleForTesting
|
||||
/**
|
||||
* Return the owner for a given AppId
|
||||
* @param remoteRootLogDir
|
||||
* @param appId
|
||||
* @param bestGuess
|
||||
* @param conf
|
||||
* @return the owner or null
|
||||
* @throws IOException
|
||||
*/
|
||||
public static String getOwnerForAppIdOrNull(
|
||||
ApplicationId appId, String bestGuess,
|
||||
Configuration conf) throws IOException {
|
||||
Path remoteRootLogDir = new Path(conf.get(
|
||||
YarnConfiguration.NM_REMOTE_APP_LOG_DIR,
|
||||
YarnConfiguration.DEFAULT_NM_REMOTE_APP_LOG_DIR));
|
||||
String suffix = LogAggregationUtils.getRemoteNodeLogDirSuffix(conf);
|
||||
Path fullPath = LogAggregationUtils.getRemoteAppLogDir(remoteRootLogDir,
|
||||
appId, bestGuess, suffix);
|
||||
FileContext fc =
|
||||
FileContext.getFileContext(remoteRootLogDir.toUri(), conf);
|
||||
String pathAccess = fullPath.toString();
|
||||
try {
|
||||
if (fc.util().exists(fullPath)) {
|
||||
return bestGuess;
|
||||
}
|
||||
Path toMatch = LogAggregationUtils.
|
||||
getRemoteAppLogDir(remoteRootLogDir, appId, "*", suffix);
|
||||
pathAccess = toMatch.toString();
|
||||
FileStatus[] matching = fc.util().globStatus(toMatch);
|
||||
if (matching == null || matching.length != 1) {
|
||||
return null;
|
||||
}
|
||||
//fetch the user from the full path /app-logs/user[/suffix]/app_id
|
||||
Path parent = matching[0].getPath().getParent();
|
||||
//skip the suffix too
|
||||
if (suffix != null && !StringUtils.isEmpty(suffix)) {
|
||||
parent = parent.getParent();
|
||||
}
|
||||
return parent.getName();
|
||||
} catch (AccessControlException | AccessDeniedException ex) {
|
||||
logDirNoAccessPermission(pathAccess, bestGuess, ex.getMessage());
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
@Private
|
||||
@VisibleForTesting
|
||||
public int dumpAContainersLogsForALogType(String appId, String containerId,
|
||||
|
|
@ -93,12 +143,12 @@ public class LogCLIHelpers implements Configurable {
|
|||
thisNodeFile.getPath());
|
||||
if (logType == null) {
|
||||
if (dumpAContainerLogs(containerId, reader, System.out,
|
||||
thisNodeFile.getModificationTime()) > -1) {
|
||||
thisNodeFile.getModificationTime()) > -1) {
|
||||
foundContainerLogs = true;
|
||||
}
|
||||
} else {
|
||||
if (dumpAContainerLogsForALogType(containerId, reader, System.out,
|
||||
thisNodeFile.getModificationTime(), logType) > -1) {
|
||||
thisNodeFile.getModificationTime(), logType) > -1) {
|
||||
foundContainerLogs = true;
|
||||
}
|
||||
}
|
||||
|
|
@ -182,7 +232,7 @@ public class LogCLIHelpers implements Configurable {
|
|||
while (true) {
|
||||
try {
|
||||
LogReader.readAContainerLogsForALogType(valueStream, out,
|
||||
logUploadedTime);
|
||||
logUploadedTime);
|
||||
foundContainerLogs = true;
|
||||
} catch (EOFException eof) {
|
||||
break;
|
||||
|
|
@ -249,9 +299,10 @@ public class LogCLIHelpers implements Configurable {
|
|||
continue;
|
||||
}
|
||||
if (!thisNodeFile.getPath().getName()
|
||||
.endsWith(LogAggregationUtils.TMP_FILE_SUFFIX)) {
|
||||
.endsWith(LogAggregationUtils.TMP_FILE_SUFFIX)) {
|
||||
AggregatedLogFormat.LogReader reader =
|
||||
new AggregatedLogFormat.LogReader(getConf(), thisNodeFile.getPath());
|
||||
new AggregatedLogFormat.LogReader(getConf(),
|
||||
thisNodeFile.getPath());
|
||||
try {
|
||||
|
||||
DataInputStream valueStream;
|
||||
|
|
@ -261,13 +312,14 @@ public class LogCLIHelpers implements Configurable {
|
|||
while (valueStream != null) {
|
||||
|
||||
String containerString =
|
||||
"\n\nContainer: " + key + " on " + thisNodeFile.getPath().getName();
|
||||
"\n\nContainer: " + key + " on "
|
||||
+ thisNodeFile.getPath().getName();
|
||||
out.println(containerString);
|
||||
out.println(StringUtils.repeat("=", containerString.length()));
|
||||
while (true) {
|
||||
try {
|
||||
LogReader.readAContainerLogsForALogType(valueStream, out,
|
||||
thisNodeFile.getModificationTime());
|
||||
thisNodeFile.getModificationTime());
|
||||
foundAnyLogs = true;
|
||||
} catch (EOFException eof) {
|
||||
break;
|
||||
|
|
@ -283,7 +335,7 @@ public class LogCLIHelpers implements Configurable {
|
|||
}
|
||||
}
|
||||
}
|
||||
if (! foundAnyLogs) {
|
||||
if (!foundAnyLogs) {
|
||||
emptyLogDir(getRemoteAppLogDir(appId, appOwner).toString());
|
||||
return -1;
|
||||
}
|
||||
|
|
@ -398,6 +450,9 @@ public class LogCLIHelpers implements Configurable {
|
|||
getConf()).listStatus(remoteAppLogDir);
|
||||
} catch (FileNotFoundException fnf) {
|
||||
logDirNotExist(remoteAppLogDir.toString());
|
||||
} catch (AccessControlException | AccessDeniedException ace) {
|
||||
logDirNoAccessPermission(remoteAppLogDir.toString(), appOwner,
|
||||
ace.getMessage());
|
||||
}
|
||||
return nodeFiles;
|
||||
}
|
||||
|
|
@ -426,7 +481,7 @@ public class LogCLIHelpers implements Configurable {
|
|||
|
||||
private static void containerLogNotFound(String containerId) {
|
||||
System.err.println("Logs for container " + containerId
|
||||
+ " are not present in this log-file.");
|
||||
+ " are not present in this log-file.");
|
||||
}
|
||||
|
||||
private static void logDirNotExist(String remoteAppLogDir) {
|
||||
|
|
@ -437,4 +492,13 @@ public class LogCLIHelpers implements Configurable {
|
|||
private static void emptyLogDir(String remoteAppLogDir) {
|
||||
System.err.println(remoteAppLogDir + " does not have any log files.");
|
||||
}
|
||||
|
||||
private static void logDirNoAccessPermission(String remoteAppLogDir,
|
||||
String appOwner, String errorMessage) throws IOException {
|
||||
System.err.println("Guessed logs' owner is " + appOwner
|
||||
+ " and current user "
|
||||
+ UserGroupInformation.getCurrentUser().getUserName() + " does not "
|
||||
+ "have permission to access " + remoteAppLogDir
|
||||
+ ". Error message found: " + errorMessage);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in New Issue