YARN-4842. Fixed "yarn logs" command to guess (and thus not require) the appOwner argument when viewing another user's logs. Contributed by Ram Venkatesh and Xuan Gong.
(cherry picked from commit 87f5e351337a905af5215af76c72b9312616cd4f)
This commit is contained in:
Vinod Kumar Vavilapalli
parent
007d6d1d57
commit
e0d6a9632d
@ -151,14 +151,12 @@ public int run(String[] args) throws Exception {
|
||||
LogCLIHelpers logCliHelper = new LogCLIHelpers();
|
||||
logCliHelper.setConf(getConf());
|
||||
|
||||
if (appOwner == null || appOwner.isEmpty()) {
|
||||
appOwner = UserGroupInformation.getCurrentUser().getShortUserName();
|
||||
}
|
||||
|
||||
boolean appStateObtainedSuccessfully = true;
|
||||
YarnApplicationState appState = YarnApplicationState.NEW;
|
||||
ApplicationReport appReport = null;
|
||||
try {
|
||||
appState = getApplicationState(appId);
|
||||
appReport = getApplicationReport(appId);
|
||||
appState = appReport.getYarnApplicationState();
|
||||
if (appState == YarnApplicationState.NEW
|
||||
|| appState == YarnApplicationState.NEW_SAVING
|
||||
|| appState == YarnApplicationState.SUBMITTED) {
|
||||
@ -171,6 +169,16 @@ public int run(String[] args) throws Exception {
|
||||
+ " Attempting to fetch logs directly from the filesystem.");
|
||||
}
|
||||
|
||||
if (appOwner == null || appOwner.isEmpty()) {
|
||||
appOwner = guessAppOwner(appReport, appId);
|
||||
if (appOwner == null) {
|
||||
System.err.println("Can not find the appOwner. "
|
||||
+ "Please specify the correct appOwner");
|
||||
System.err.println("Could not locate application logs for " + appId);
|
||||
return -1;
|
||||
}
|
||||
}
|
||||
|
||||
if (showMetaInfo) {
|
||||
return showMetaInfo(appState, appStateObtainedSuccessfully,
|
||||
logCliHelper, appId, containerIdStr, nodeAddress, appOwner);
|
||||
@ -201,6 +209,10 @@ public int run(String[] args) throws Exception {
|
||||
if (nodeAddress == null) {
|
||||
resultCode =
|
||||
logCliHelper.dumpAllContainersLogs(appId, appOwner, System.out);
|
||||
if (resultCode == -1) {
|
||||
System.err.println("Can not find the logs for the application: "
|
||||
+ appId + " with the appOwner: " + appOwner);
|
||||
}
|
||||
} else {
|
||||
System.err.println("Should at least provide ContainerId!");
|
||||
printHelpMessage(printOpts);
|
||||
@ -210,13 +222,12 @@ public int run(String[] args) throws Exception {
|
||||
return resultCode;
|
||||
}
|
||||
|
||||
private YarnApplicationState getApplicationState(ApplicationId appId)
|
||||
private ApplicationReport getApplicationReport(ApplicationId appId)
|
||||
throws IOException, YarnException {
|
||||
YarnClient yarnClient = createYarnClient();
|
||||
|
||||
try {
|
||||
ApplicationReport appReport = yarnClient.getApplicationReport(appId);
|
||||
return appReport.getYarnApplicationState();
|
||||
return yarnClient.getApplicationReport(appId);
|
||||
} finally {
|
||||
yarnClient.close();
|
||||
}
|
||||
@ -693,11 +704,12 @@ private int fetchAMContainerLogs(String[] logFiles,
|
||||
amContainersList, logFiles, logCliHelper, appOwner, true);
|
||||
} else {
|
||||
System.err.println("Can not get AMContainers logs for "
|
||||
+ "the application:" + appId);
|
||||
System.err.println("This application:" + appId + " is finished."
|
||||
+ " Please enable the application history service. Or Using "
|
||||
+ "yarn logs -applicationId <appId> -containerId <containerId> "
|
||||
+ "--nodeAddress <nodeHttpAddress> to get the container logs");
|
||||
+ "the application:" + appId + " with the appOwner:" + appOwner);
|
||||
System.err.println("This application:" + appId + " has finished."
|
||||
+ " Please enable the application-history service or explicitly"
|
||||
+ " use 'yarn logs -applicationId <appId> "
|
||||
+ "-containerId <containerId> --nodeAddress <nodeHttpAddress>' "
|
||||
+ "to get the container logs.");
|
||||
return -1;
|
||||
}
|
||||
}
|
||||
@ -750,7 +762,8 @@ private int fetchContainerLogs(YarnApplicationState appState,
|
||||
appOwner);
|
||||
} else if (!isApplicationFinished(appState)) {
|
||||
System.err.println("Unable to get logs for this container:"
|
||||
+ containerIdStr + "for the application:" + appIdStr);
|
||||
+ containerIdStr + "for the application:" + appIdStr
|
||||
+ " with the appOwner: " + appOwner);
|
||||
System.err.println("The application: " + appIdStr
|
||||
+ " is still running, and we can not get Container report "
|
||||
+ "for the container: " + containerIdStr +". Please try later "
|
||||
@ -821,4 +834,18 @@ public boolean isAppFinished() {
|
||||
return isAppFinished;
|
||||
}
|
||||
}
|
||||
|
||||
private String guessAppOwner(ApplicationReport appReport,
|
||||
ApplicationId appId) throws IOException {
|
||||
String appOwner = null;
|
||||
if (appReport != null) {
|
||||
//always use the app owner from the app report if possible
|
||||
appOwner = appReport.getUser();
|
||||
} else {
|
||||
appOwner = UserGroupInformation.getCurrentUser().getShortUserName();
|
||||
appOwner = LogCLIHelpers.getOwnerForAppIdOrNull(
|
||||
appId, appOwner, getConf());
|
||||
}
|
||||
return appOwner;
|
||||
}
|
||||
}
|
||||
|
||||
@ -42,11 +42,11 @@
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
import org.junit.Assert;
|
||||
import org.apache.hadoop.conf.Configuration;
|
||||
import org.apache.hadoop.fs.FileSystem;
|
||||
import org.apache.hadoop.fs.LocalFileSystem;
|
||||
import org.apache.hadoop.fs.Path;
|
||||
import org.apache.hadoop.fs.permission.FsPermission;
|
||||
import org.apache.hadoop.security.UserGroupInformation;
|
||||
import org.apache.hadoop.yarn.api.records.ApplicationAccessType;
|
||||
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
|
||||
@ -55,22 +55,21 @@
|
||||
import org.apache.hadoop.yarn.api.records.ContainerId;
|
||||
import org.apache.hadoop.yarn.api.records.NodeId;
|
||||
import org.apache.hadoop.yarn.api.records.YarnApplicationState;
|
||||
import org.apache.hadoop.yarn.api.records.impl.pb.ApplicationAttemptIdPBImpl;
|
||||
import org.apache.hadoop.yarn.api.records.impl.pb.ApplicationIdPBImpl;
|
||||
import org.apache.hadoop.yarn.api.records.impl.pb.ContainerIdPBImpl;
|
||||
import org.apache.hadoop.yarn.client.api.YarnClient;
|
||||
import org.apache.hadoop.yarn.conf.YarnConfiguration;
|
||||
import org.apache.hadoop.yarn.exceptions.YarnException;
|
||||
import org.apache.hadoop.yarn.logaggregation.AggregatedLogFormat;
|
||||
import org.apache.hadoop.yarn.logaggregation.LogAggregationUtils;
|
||||
import org.apache.hadoop.yarn.logaggregation.LogCLIHelpers;
|
||||
import org.junit.Assert;
|
||||
import org.junit.Before;
|
||||
import org.junit.Test;
|
||||
|
||||
public class TestLogsCLI {
|
||||
|
||||
ByteArrayOutputStream sysOutStream;
|
||||
private PrintStream sysOut;
|
||||
|
||||
|
||||
ByteArrayOutputStream sysErrStream;
|
||||
private PrintStream sysErr;
|
||||
|
||||
@ -79,7 +78,7 @@ public void setUp() {
|
||||
sysOutStream = new ByteArrayOutputStream();
|
||||
sysOut = new PrintStream(sysOutStream);
|
||||
System.setOut(sysOut);
|
||||
|
||||
|
||||
sysErrStream = new ByteArrayOutputStream();
|
||||
sysErr = new PrintStream(sysErrStream);
|
||||
System.setErr(sysErr);
|
||||
@ -91,16 +90,18 @@ public void testFailResultCodes() throws Exception {
|
||||
conf.setClass("fs.file.impl", LocalFileSystem.class, FileSystem.class);
|
||||
LogCLIHelpers cliHelper = new LogCLIHelpers();
|
||||
cliHelper.setConf(conf);
|
||||
YarnClient mockYarnClient = createMockYarnClient(YarnApplicationState.FINISHED);
|
||||
YarnClient mockYarnClient = createMockYarnClient(
|
||||
YarnApplicationState.FINISHED,
|
||||
UserGroupInformation.getCurrentUser().getShortUserName());
|
||||
LogsCLI dumper = new LogsCLIForTest(mockYarnClient);
|
||||
dumper.setConf(conf);
|
||||
|
||||
|
||||
// verify dumping a non-existent application's logs returns a failure code
|
||||
int exitCode = dumper.run( new String[] {
|
||||
"-applicationId", "application_0_0" } );
|
||||
assertTrue("Should return an error code", exitCode != 0);
|
||||
|
||||
// verify dumping a non-existent container log is a failure code
|
||||
|
||||
// verify dumping a non-existent container log is a failure code
|
||||
exitCode = cliHelper.dumpAContainersLogs("application_0_0", "container_0_0",
|
||||
"nonexistentnode:1234", "nobody");
|
||||
assertTrue("Should return an error code", exitCode != 0);
|
||||
@ -109,10 +110,12 @@ public void testFailResultCodes() throws Exception {
|
||||
@Test(timeout = 5000l)
|
||||
public void testInvalidApplicationId() throws Exception {
|
||||
Configuration conf = new YarnConfiguration();
|
||||
YarnClient mockYarnClient = createMockYarnClient(YarnApplicationState.FINISHED);
|
||||
YarnClient mockYarnClient = createMockYarnClient(
|
||||
YarnApplicationState.FINISHED,
|
||||
UserGroupInformation.getCurrentUser().getShortUserName());
|
||||
LogsCLI cli = new LogsCLIForTest(mockYarnClient);
|
||||
cli.setConf(conf);
|
||||
|
||||
|
||||
int exitCode = cli.run( new String[] { "-applicationId", "not_an_app_id"});
|
||||
assertTrue(exitCode == -1);
|
||||
assertTrue(sysErrStream.toString().startsWith("Invalid ApplicationId specified"));
|
||||
@ -137,7 +140,9 @@ public void testUnknownApplicationId() throws Exception {
|
||||
@Test(timeout = 5000l)
|
||||
public void testHelpMessage() throws Exception {
|
||||
Configuration conf = new YarnConfiguration();
|
||||
YarnClient mockYarnClient = createMockYarnClient(YarnApplicationState.FINISHED);
|
||||
YarnClient mockYarnClient = createMockYarnClient(
|
||||
YarnApplicationState.FINISHED,
|
||||
UserGroupInformation.getCurrentUser().getShortUserName());
|
||||
LogsCLI dumper = new LogsCLIForTest(mockYarnClient);
|
||||
dumper.setConf(conf);
|
||||
|
||||
@ -187,7 +192,7 @@ public void testHelpMessage() throws Exception {
|
||||
String appReportStr = baos.toString("UTF-8");
|
||||
Assert.assertEquals(appReportStr, sysOutStream.toString());
|
||||
}
|
||||
|
||||
|
||||
@Test (timeout = 15000)
|
||||
public void testFetchApplictionLogs() throws Exception {
|
||||
String remoteLogRootDir = "target/logs/";
|
||||
@ -200,13 +205,13 @@ public void testFetchApplictionLogs() throws Exception {
|
||||
FileSystem fs = FileSystem.get(configuration);
|
||||
|
||||
UserGroupInformation ugi = UserGroupInformation.getCurrentUser();
|
||||
ApplicationId appId = ApplicationIdPBImpl.newInstance(0, 1);
|
||||
ApplicationId appId = ApplicationId.newInstance(0, 1);
|
||||
ApplicationAttemptId appAttemptId =
|
||||
ApplicationAttemptIdPBImpl.newInstance(appId, 1);
|
||||
ContainerId containerId0 = ContainerIdPBImpl.newContainerId(appAttemptId, 0);
|
||||
ContainerId containerId1 = ContainerIdPBImpl.newContainerId(appAttemptId, 1);
|
||||
ContainerId containerId2 = ContainerIdPBImpl.newContainerId(appAttemptId, 2);
|
||||
ContainerId containerId3 = ContainerIdPBImpl.newContainerId(appAttemptId, 3);
|
||||
ApplicationAttemptId.newInstance(appId, 1);
|
||||
ContainerId containerId0 = ContainerId.newContainerId(appAttemptId, 0);
|
||||
ContainerId containerId1 = ContainerId.newContainerId(appAttemptId, 1);
|
||||
ContainerId containerId2 = ContainerId.newContainerId(appAttemptId, 2);
|
||||
ContainerId containerId3 = ContainerId.newContainerId(appAttemptId, 3);
|
||||
NodeId nodeId = NodeId.newInstance("localhost", 1234);
|
||||
|
||||
// create local logs
|
||||
@ -222,6 +227,7 @@ public void testFetchApplictionLogs() throws Exception {
|
||||
fs.delete(appLogsDir, true);
|
||||
}
|
||||
assertTrue(fs.mkdirs(appLogsDir));
|
||||
|
||||
List<String> rootLogDirs = Arrays.asList(rootLogDir);
|
||||
|
||||
List<String> logTypes = new ArrayList<String>();
|
||||
@ -258,7 +264,8 @@ public void testFetchApplictionLogs() throws Exception {
|
||||
containerId3, path, fs);
|
||||
|
||||
YarnClient mockYarnClient =
|
||||
createMockYarnClient(YarnApplicationState.FINISHED);
|
||||
createMockYarnClient(
|
||||
YarnApplicationState.FINISHED, ugi.getShortUserName());
|
||||
LogsCLI cli = new LogsCLIForTest(mockYarnClient);
|
||||
cli.setConf(configuration);
|
||||
|
||||
@ -348,6 +355,143 @@ public void testFetchApplictionLogs() throws Exception {
|
||||
fs.delete(new Path(rootLogDir), true);
|
||||
}
|
||||
|
||||
@Test (timeout = 15000)
|
||||
public void testFetchApplictionLogsAsAnotherUser() throws Exception {
|
||||
String remoteLogRootDir = "target/logs/";
|
||||
String rootLogDir = "target/LocalLogs";
|
||||
|
||||
String testUser = "test";
|
||||
UserGroupInformation testUgi = UserGroupInformation
|
||||
.createRemoteUser(testUser);
|
||||
|
||||
Configuration configuration = new Configuration();
|
||||
configuration.setBoolean(YarnConfiguration.LOG_AGGREGATION_ENABLED, true);
|
||||
configuration
|
||||
.set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR, remoteLogRootDir);
|
||||
configuration.setBoolean(YarnConfiguration.YARN_ACL_ENABLE, true);
|
||||
configuration.set(YarnConfiguration.YARN_ADMIN_ACL, "admin");
|
||||
FileSystem fs = FileSystem.get(configuration);
|
||||
|
||||
ApplicationId appId = ApplicationId.newInstance(0, 1);
|
||||
ApplicationAttemptId appAttemptId =
|
||||
ApplicationAttemptId.newInstance(appId, 1);
|
||||
ContainerId containerId = ContainerId
|
||||
.newContainerId(appAttemptId, 1);
|
||||
NodeId nodeId = NodeId.newInstance("localhost", 1234);
|
||||
|
||||
try {
|
||||
Path rootLogDirPath = new Path(rootLogDir);
|
||||
if (fs.exists(rootLogDirPath)) {
|
||||
fs.delete(rootLogDirPath, true);
|
||||
}
|
||||
assertTrue(fs.mkdirs(rootLogDirPath));
|
||||
|
||||
// create local app dir for app
|
||||
final Path appLogsDir = new Path(rootLogDirPath, appId.toString());
|
||||
if (fs.exists(appLogsDir)) {
|
||||
fs.delete(appLogsDir, true);
|
||||
}
|
||||
assertTrue(fs.mkdirs(appLogsDir));
|
||||
|
||||
List<String> rootLogDirs = Arrays.asList(rootLogDir);
|
||||
List<String> logTypes = new ArrayList<String>();
|
||||
logTypes.add("syslog");
|
||||
|
||||
// create container logs in localLogDir for app
|
||||
createContainerLogInLocalDir(appLogsDir, containerId, fs, logTypes);
|
||||
|
||||
// create the remote app dir for app
|
||||
// but for a different user testUser"
|
||||
Path path = new Path(remoteLogRootDir + testUser + "/logs/" + appId);
|
||||
if (fs.exists(path)) {
|
||||
fs.delete(path, true);
|
||||
}
|
||||
assertTrue(fs.mkdirs(path));
|
||||
|
||||
// upload container logs for app into remote dir
|
||||
uploadContainerLogIntoRemoteDir(testUgi, configuration, rootLogDirs,
|
||||
nodeId, containerId, path, fs);
|
||||
|
||||
YarnClient mockYarnClient = createMockYarnClient(
|
||||
YarnApplicationState.FINISHED, testUgi.getShortUserName());
|
||||
LogsCLI cli = new LogsCLIForTest(mockYarnClient);
|
||||
cli.setConf(configuration);
|
||||
|
||||
// Verify that we can get the application logs by specifying
|
||||
// a correct appOwner
|
||||
int exitCode = cli.run(new String[] {
|
||||
"-applicationId", appId.toString(),
|
||||
"-appOwner", testUser});
|
||||
assertTrue(exitCode == 0);
|
||||
assertTrue(sysOutStream.toString().contains(
|
||||
"Hello " + containerId + " in syslog!"));
|
||||
sysOutStream.reset();
|
||||
|
||||
// Verify that we can not get the application logs
|
||||
// if an invalid user is specified
|
||||
exitCode = cli.run(new String[] {
|
||||
"-applicationId", appId.toString(),
|
||||
"-appOwner", "invalid"});
|
||||
assertTrue(exitCode == -1);
|
||||
assertTrue(sysErrStream.toString().contains("Can not find the logs "
|
||||
+ "for the application: " + appId.toString()));
|
||||
sysErrStream.reset();
|
||||
|
||||
// Verify that we do not specify appOwner, and can not
|
||||
// get appReport from RM, we still can figure out the appOwner
|
||||
// and can get app logs successfully.
|
||||
YarnClient mockYarnClient2 = createMockYarnClientUnknownApp();
|
||||
cli = new LogsCLIForTest(mockYarnClient2);
|
||||
cli.setConf(configuration);
|
||||
exitCode = cli.run(new String[] {
|
||||
"-applicationId", appId.toString()});
|
||||
assertTrue(exitCode == 0);
|
||||
assertTrue(sysOutStream.toString().contains("Hello "
|
||||
+ containerId + " in syslog!"));
|
||||
sysOutStream.reset();
|
||||
|
||||
// Verify that we could get the err message "Can not find the appOwner"
|
||||
// if we do not specify the appOwner, can not get appReport, and
|
||||
// the app does not exist in remote dir.
|
||||
ApplicationId appId2 = ApplicationId.newInstance(
|
||||
System.currentTimeMillis(), 2);
|
||||
exitCode = cli.run(new String[] {
|
||||
"-applicationId", appId2.toString()});
|
||||
assertTrue(exitCode == -1);
|
||||
assertTrue(sysErrStream.toString().contains(
|
||||
"Can not find the appOwner"));
|
||||
sysErrStream.reset();
|
||||
|
||||
// Verify that we could not get appOwner
|
||||
// because we don't have file-system permissions
|
||||
ApplicationId appTest = ApplicationId.newInstance(
|
||||
System.currentTimeMillis(), 1000);
|
||||
String priorityUser = "priority";
|
||||
Path pathWithoutPerm = new Path(remoteLogRootDir + priorityUser
|
||||
+ "/logs/" + appTest);
|
||||
if (fs.exists(pathWithoutPerm)) {
|
||||
fs.delete(pathWithoutPerm, true);
|
||||
}
|
||||
// The user will not have read permission for this directory.
|
||||
// To mimic the scenario that the user can not get file status
|
||||
FsPermission permission = FsPermission
|
||||
.createImmutable((short) 01300);
|
||||
assertTrue(fs.mkdirs(pathWithoutPerm, permission));
|
||||
|
||||
exitCode = cli.run(new String[] {
|
||||
"-applicationId", appTest.toString()});
|
||||
assertTrue(exitCode == -1);
|
||||
assertTrue(sysErrStream.toString().contains(
|
||||
"Guessed logs' owner is " + priorityUser + " and current user "
|
||||
+ UserGroupInformation.getCurrentUser().getUserName()
|
||||
+ " does not have permission to access"));
|
||||
sysErrStream.reset();
|
||||
} finally {
|
||||
fs.delete(new Path(remoteLogRootDir), true);
|
||||
fs.delete(new Path(rootLogDir), true);
|
||||
}
|
||||
}
|
||||
|
||||
@Test (timeout = 15000)
|
||||
public void testPrintContainerLogMetadata() throws Exception {
|
||||
String remoteLogRootDir = "target/logs/";
|
||||
@ -380,7 +524,8 @@ public void testPrintContainerLogMetadata() throws Exception {
|
||||
appId, containerIds, nodeIds);
|
||||
|
||||
YarnClient mockYarnClient =
|
||||
createMockYarnClient(YarnApplicationState.FINISHED);
|
||||
createMockYarnClient(YarnApplicationState.FINISHED,
|
||||
UserGroupInformation.getCurrentUser().getShortUserName());
|
||||
LogsCLI cli = new LogsCLIForTest(mockYarnClient);
|
||||
cli.setConf(configuration);
|
||||
|
||||
@ -466,7 +611,8 @@ public void testListNodeInfo() throws Exception {
|
||||
appId, containerIds, nodeIds);
|
||||
|
||||
YarnClient mockYarnClient =
|
||||
createMockYarnClient(YarnApplicationState.FINISHED);
|
||||
createMockYarnClient(YarnApplicationState.FINISHED,
|
||||
UserGroupInformation.getCurrentUser().getShortUserName());
|
||||
LogsCLI cli = new LogsCLIForTest(mockYarnClient);
|
||||
cli.setConf(configuration);
|
||||
|
||||
@ -508,7 +654,8 @@ public void testFetchApplictionLogsHar() throws Exception {
|
||||
assertTrue(fs.exists(harPath));
|
||||
|
||||
YarnClient mockYarnClient =
|
||||
createMockYarnClient(YarnApplicationState.FINISHED);
|
||||
createMockYarnClient(YarnApplicationState.FINISHED,
|
||||
ugi.getShortUserName());
|
||||
LogsCLI cli = new LogsCLIForTest(mockYarnClient);
|
||||
cli.setConf(configuration);
|
||||
int exitCode = cli.run(new String[]{"-applicationId",
|
||||
@ -630,10 +777,12 @@ private static void uploadEmptyContainerLogIntoRemoteDir(UserGroupInformation ug
|
||||
writer.close();
|
||||
}
|
||||
|
||||
private YarnClient createMockYarnClient(YarnApplicationState appState)
|
||||
private YarnClient createMockYarnClient(YarnApplicationState appState,
|
||||
String user)
|
||||
throws YarnException, IOException {
|
||||
YarnClient mockClient = mock(YarnClient.class);
|
||||
ApplicationReport mockAppReport = mock(ApplicationReport.class);
|
||||
doReturn(user).when(mockAppReport).getUser();
|
||||
doReturn(appState).when(mockAppReport).getYarnApplicationState();
|
||||
doReturn(mockAppReport).when(mockClient).getApplicationReport(
|
||||
any(ApplicationId.class));
|
||||
@ -659,9 +808,9 @@ private YarnClient createMockYarnClientUnknownApp() throws YarnException,
|
||||
}
|
||||
|
||||
private static class LogsCLIForTest extends LogsCLI {
|
||||
|
||||
|
||||
private YarnClient yarnClient;
|
||||
|
||||
|
||||
public LogsCLIForTest(YarnClient yarnClient) {
|
||||
super();
|
||||
this.yarnClient = yarnClient;
|
||||
|
||||
@ -23,6 +23,7 @@
|
||||
import java.io.FileNotFoundException;
|
||||
import java.io.IOException;
|
||||
import java.io.PrintStream;
|
||||
import java.nio.file.AccessDeniedException;
|
||||
import java.util.List;
|
||||
|
||||
import org.apache.commons.lang.StringUtils;
|
||||
@ -34,6 +35,8 @@
|
||||
import org.apache.hadoop.fs.HarFs;
|
||||
import org.apache.hadoop.fs.Path;
|
||||
import org.apache.hadoop.fs.RemoteIterator;
|
||||
import org.apache.hadoop.security.AccessControlException;
|
||||
import org.apache.hadoop.security.UserGroupInformation;
|
||||
import org.apache.hadoop.yarn.api.records.ApplicationId;
|
||||
import org.apache.hadoop.yarn.conf.YarnConfiguration;
|
||||
import org.apache.hadoop.yarn.logaggregation.AggregatedLogFormat.LogKey;
|
||||
@ -54,6 +57,53 @@ public int dumpAContainersLogs(String appId, String containerId,
|
||||
null);
|
||||
}
|
||||
|
||||
@Private
|
||||
@VisibleForTesting
|
||||
/**
|
||||
* Return the owner for a given AppId
|
||||
* @param remoteRootLogDir
|
||||
* @param appId
|
||||
* @param bestGuess
|
||||
* @param conf
|
||||
* @return the owner or null
|
||||
* @throws IOException
|
||||
*/
|
||||
public static String getOwnerForAppIdOrNull(
|
||||
ApplicationId appId, String bestGuess,
|
||||
Configuration conf) throws IOException {
|
||||
Path remoteRootLogDir = new Path(conf.get(
|
||||
YarnConfiguration.NM_REMOTE_APP_LOG_DIR,
|
||||
YarnConfiguration.DEFAULT_NM_REMOTE_APP_LOG_DIR));
|
||||
String suffix = LogAggregationUtils.getRemoteNodeLogDirSuffix(conf);
|
||||
Path fullPath = LogAggregationUtils.getRemoteAppLogDir(remoteRootLogDir,
|
||||
appId, bestGuess, suffix);
|
||||
FileContext fc =
|
||||
FileContext.getFileContext(remoteRootLogDir.toUri(), conf);
|
||||
String pathAccess = fullPath.toString();
|
||||
try {
|
||||
if (fc.util().exists(fullPath)) {
|
||||
return bestGuess;
|
||||
}
|
||||
Path toMatch = LogAggregationUtils.
|
||||
getRemoteAppLogDir(remoteRootLogDir, appId, "*", suffix);
|
||||
pathAccess = toMatch.toString();
|
||||
FileStatus[] matching = fc.util().globStatus(toMatch);
|
||||
if (matching == null || matching.length != 1) {
|
||||
return null;
|
||||
}
|
||||
//fetch the user from the full path /app-logs/user[/suffix]/app_id
|
||||
Path parent = matching[0].getPath().getParent();
|
||||
//skip the suffix too
|
||||
if (suffix != null && !StringUtils.isEmpty(suffix)) {
|
||||
parent = parent.getParent();
|
||||
}
|
||||
return parent.getName();
|
||||
} catch (AccessControlException | AccessDeniedException ex) {
|
||||
logDirNoAccessPermission(pathAccess, bestGuess, ex.getMessage());
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
@Private
|
||||
@VisibleForTesting
|
||||
public int dumpAContainersLogsForALogType(String appId, String containerId,
|
||||
@ -93,12 +143,12 @@ public int dumpAContainersLogsForALogType(String appId, String containerId,
|
||||
thisNodeFile.getPath());
|
||||
if (logType == null) {
|
||||
if (dumpAContainerLogs(containerId, reader, System.out,
|
||||
thisNodeFile.getModificationTime()) > -1) {
|
||||
thisNodeFile.getModificationTime()) > -1) {
|
||||
foundContainerLogs = true;
|
||||
}
|
||||
} else {
|
||||
if (dumpAContainerLogsForALogType(containerId, reader, System.out,
|
||||
thisNodeFile.getModificationTime(), logType) > -1) {
|
||||
thisNodeFile.getModificationTime(), logType) > -1) {
|
||||
foundContainerLogs = true;
|
||||
}
|
||||
}
|
||||
@ -182,7 +232,7 @@ public int dumpAContainerLogs(String containerIdStr,
|
||||
while (true) {
|
||||
try {
|
||||
LogReader.readAContainerLogsForALogType(valueStream, out,
|
||||
logUploadedTime);
|
||||
logUploadedTime);
|
||||
foundContainerLogs = true;
|
||||
} catch (EOFException eof) {
|
||||
break;
|
||||
@ -249,9 +299,10 @@ public int dumpAllContainersLogs(ApplicationId appId, String appOwner,
|
||||
continue;
|
||||
}
|
||||
if (!thisNodeFile.getPath().getName()
|
||||
.endsWith(LogAggregationUtils.TMP_FILE_SUFFIX)) {
|
||||
.endsWith(LogAggregationUtils.TMP_FILE_SUFFIX)) {
|
||||
AggregatedLogFormat.LogReader reader =
|
||||
new AggregatedLogFormat.LogReader(getConf(), thisNodeFile.getPath());
|
||||
new AggregatedLogFormat.LogReader(getConf(),
|
||||
thisNodeFile.getPath());
|
||||
try {
|
||||
|
||||
DataInputStream valueStream;
|
||||
@ -261,13 +312,14 @@ public int dumpAllContainersLogs(ApplicationId appId, String appOwner,
|
||||
while (valueStream != null) {
|
||||
|
||||
String containerString =
|
||||
"\n\nContainer: " + key + " on " + thisNodeFile.getPath().getName();
|
||||
"\n\nContainer: " + key + " on "
|
||||
+ thisNodeFile.getPath().getName();
|
||||
out.println(containerString);
|
||||
out.println(StringUtils.repeat("=", containerString.length()));
|
||||
while (true) {
|
||||
try {
|
||||
LogReader.readAContainerLogsForALogType(valueStream, out,
|
||||
thisNodeFile.getModificationTime());
|
||||
thisNodeFile.getModificationTime());
|
||||
foundAnyLogs = true;
|
||||
} catch (EOFException eof) {
|
||||
break;
|
||||
@ -283,7 +335,7 @@ public int dumpAllContainersLogs(ApplicationId appId, String appOwner,
|
||||
}
|
||||
}
|
||||
}
|
||||
if (! foundAnyLogs) {
|
||||
if (!foundAnyLogs) {
|
||||
emptyLogDir(getRemoteAppLogDir(appId, appOwner).toString());
|
||||
return -1;
|
||||
}
|
||||
@ -398,6 +450,9 @@ private RemoteIterator<FileStatus> getRemoteNodeFileDir(ApplicationId appId,
|
||||
getConf()).listStatus(remoteAppLogDir);
|
||||
} catch (FileNotFoundException fnf) {
|
||||
logDirNotExist(remoteAppLogDir.toString());
|
||||
} catch (AccessControlException | AccessDeniedException ace) {
|
||||
logDirNoAccessPermission(remoteAppLogDir.toString(), appOwner,
|
||||
ace.getMessage());
|
||||
}
|
||||
return nodeFiles;
|
||||
}
|
||||
@ -426,7 +481,7 @@ public Configuration getConf() {
|
||||
|
||||
private static void containerLogNotFound(String containerId) {
|
||||
System.err.println("Logs for container " + containerId
|
||||
+ " are not present in this log-file.");
|
||||
+ " are not present in this log-file.");
|
||||
}
|
||||
|
||||
private static void logDirNotExist(String remoteAppLogDir) {
|
||||
@ -437,4 +492,13 @@ private static void logDirNotExist(String remoteAppLogDir) {
|
||||
private static void emptyLogDir(String remoteAppLogDir) {
|
||||
System.err.println(remoteAppLogDir + " does not have any log files.");
|
||||
}
|
||||
|
||||
private static void logDirNoAccessPermission(String remoteAppLogDir,
|
||||
String appOwner, String errorMessage) throws IOException {
|
||||
System.err.println("Guessed logs' owner is " + appOwner
|
||||
+ " and current user "
|
||||
+ UserGroupInformation.getCurrentUser().getUserName() + " does not "
|
||||
+ "have permission to access " + remoteAppLogDir
|
||||
+ ". Error message found: " + errorMessage);
|
||||
}
|
||||
}
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user