|
|
@ -89,8 +89,10 @@ import org.apache.hadoop.yarn.logaggregation.filecontroller.LogAggregationFileCo
|
|
|
|
import org.apache.hadoop.yarn.logaggregation.filecontroller.LogAggregationFileControllerContext;
|
|
|
|
import org.apache.hadoop.yarn.logaggregation.filecontroller.LogAggregationFileControllerContext;
|
|
|
|
import org.apache.hadoop.yarn.logaggregation.filecontroller.LogAggregationFileControllerFactory;
|
|
|
|
import org.apache.hadoop.yarn.logaggregation.filecontroller.LogAggregationFileControllerFactory;
|
|
|
|
import org.apache.hadoop.yarn.logaggregation.filecontroller.ifile.LogAggregationIndexedFileController;
|
|
|
|
import org.apache.hadoop.yarn.logaggregation.filecontroller.ifile.LogAggregationIndexedFileController;
|
|
|
|
|
|
|
|
import org.apache.hadoop.yarn.webapp.util.WebServiceClient;
|
|
|
|
import org.codehaus.jettison.json.JSONArray;
|
|
|
|
import org.codehaus.jettison.json.JSONArray;
|
|
|
|
import org.codehaus.jettison.json.JSONObject;
|
|
|
|
import org.codehaus.jettison.json.JSONObject;
|
|
|
|
|
|
|
|
import org.junit.After;
|
|
|
|
import org.junit.Assert;
|
|
|
|
import org.junit.Assert;
|
|
|
|
import org.junit.Before;
|
|
|
|
import org.junit.Before;
|
|
|
|
import org.junit.Test;
|
|
|
|
import org.junit.Test;
|
|
|
@ -109,8 +111,10 @@ public class TestLogsCLI {
|
|
|
|
ByteArrayOutputStream sysErrStream;
|
|
|
|
ByteArrayOutputStream sysErrStream;
|
|
|
|
private PrintStream sysErr;
|
|
|
|
private PrintStream sysErr;
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
private Configuration conf;
|
|
|
|
|
|
|
|
|
|
|
|
@Before
|
|
|
|
@Before
|
|
|
|
public void setUp() {
|
|
|
|
public void setUp() throws Exception {
|
|
|
|
sysOutStream = new ByteArrayOutputStream();
|
|
|
|
sysOutStream = new ByteArrayOutputStream();
|
|
|
|
sysOut = new PrintStream(sysOutStream);
|
|
|
|
sysOut = new PrintStream(sysOutStream);
|
|
|
|
System.setOut(sysOut);
|
|
|
|
System.setOut(sysOut);
|
|
|
@ -118,11 +122,17 @@ public class TestLogsCLI {
|
|
|
|
sysErrStream = new ByteArrayOutputStream();
|
|
|
|
sysErrStream = new ByteArrayOutputStream();
|
|
|
|
sysErr = new PrintStream(sysErrStream);
|
|
|
|
sysErr = new PrintStream(sysErrStream);
|
|
|
|
System.setErr(sysErr);
|
|
|
|
System.setErr(sysErr);
|
|
|
|
|
|
|
|
conf = new YarnConfiguration();
|
|
|
|
|
|
|
|
WebServiceClient.initialize(conf);
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
@After
|
|
|
|
|
|
|
|
public void tearDown() {
|
|
|
|
|
|
|
|
WebServiceClient.destroy();
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
@Test(timeout = 5000l)
|
|
|
|
@Test(timeout = 5000l)
|
|
|
|
public void testFailResultCodes() throws Exception {
|
|
|
|
public void testFailResultCodes() throws Exception {
|
|
|
|
Configuration conf = new YarnConfiguration();
|
|
|
|
|
|
|
|
conf.setClass("fs.file.impl", LocalFileSystem.class, FileSystem.class);
|
|
|
|
conf.setClass("fs.file.impl", LocalFileSystem.class, FileSystem.class);
|
|
|
|
LogCLIHelpers cliHelper = new LogCLIHelpers();
|
|
|
|
LogCLIHelpers cliHelper = new LogCLIHelpers();
|
|
|
|
cliHelper.setConf(conf);
|
|
|
|
cliHelper.setConf(conf);
|
|
|
@ -145,7 +155,6 @@ public class TestLogsCLI {
|
|
|
|
|
|
|
|
|
|
|
|
@Test(timeout = 10000l)
|
|
|
|
@Test(timeout = 10000l)
|
|
|
|
public void testInvalidOpts() throws Exception {
|
|
|
|
public void testInvalidOpts() throws Exception {
|
|
|
|
Configuration conf = new YarnConfiguration();
|
|
|
|
|
|
|
|
YarnClient mockYarnClient = createMockYarnClient(
|
|
|
|
YarnClient mockYarnClient = createMockYarnClient(
|
|
|
|
YarnApplicationState.FINISHED,
|
|
|
|
YarnApplicationState.FINISHED,
|
|
|
|
UserGroupInformation.getCurrentUser().getShortUserName());
|
|
|
|
UserGroupInformation.getCurrentUser().getShortUserName());
|
|
|
@ -160,7 +169,6 @@ public class TestLogsCLI {
|
|
|
|
|
|
|
|
|
|
|
|
@Test(timeout = 5000l)
|
|
|
|
@Test(timeout = 5000l)
|
|
|
|
public void testInvalidApplicationId() throws Exception {
|
|
|
|
public void testInvalidApplicationId() throws Exception {
|
|
|
|
Configuration conf = new YarnConfiguration();
|
|
|
|
|
|
|
|
YarnClient mockYarnClient = createMockYarnClient(
|
|
|
|
YarnClient mockYarnClient = createMockYarnClient(
|
|
|
|
YarnApplicationState.FINISHED,
|
|
|
|
YarnApplicationState.FINISHED,
|
|
|
|
UserGroupInformation.getCurrentUser().getShortUserName());
|
|
|
|
UserGroupInformation.getCurrentUser().getShortUserName());
|
|
|
@ -174,7 +182,6 @@ public class TestLogsCLI {
|
|
|
|
|
|
|
|
|
|
|
|
@Test(timeout = 5000L)
|
|
|
|
@Test(timeout = 5000L)
|
|
|
|
public void testInvalidAMContainerId() throws Exception {
|
|
|
|
public void testInvalidAMContainerId() throws Exception {
|
|
|
|
Configuration conf = new YarnConfiguration();
|
|
|
|
|
|
|
|
conf.setBoolean(YarnConfiguration.APPLICATION_HISTORY_ENABLED, true);
|
|
|
|
conf.setBoolean(YarnConfiguration.APPLICATION_HISTORY_ENABLED, true);
|
|
|
|
YarnClient mockYarnClient =
|
|
|
|
YarnClient mockYarnClient =
|
|
|
|
createMockYarnClient(YarnApplicationState.FINISHED,
|
|
|
|
createMockYarnClient(YarnApplicationState.FINISHED,
|
|
|
@ -196,7 +203,6 @@ public class TestLogsCLI {
|
|
|
|
|
|
|
|
|
|
|
|
@Test
|
|
|
|
@Test
|
|
|
|
public void testAMContainerInfoFetchFromTimelineReader() throws Exception {
|
|
|
|
public void testAMContainerInfoFetchFromTimelineReader() throws Exception {
|
|
|
|
Configuration conf = new YarnConfiguration();
|
|
|
|
|
|
|
|
conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, true);
|
|
|
|
conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, true);
|
|
|
|
conf.set(YarnConfiguration.TIMELINE_SERVICE_VERSIONS, "2.0f");
|
|
|
|
conf.set(YarnConfiguration.TIMELINE_SERVICE_VERSIONS, "2.0f");
|
|
|
|
YarnClient mockYarnClient =
|
|
|
|
YarnClient mockYarnClient =
|
|
|
@ -232,7 +238,6 @@ public class TestLogsCLI {
|
|
|
|
|
|
|
|
|
|
|
|
@Test(timeout = 5000l)
|
|
|
|
@Test(timeout = 5000l)
|
|
|
|
public void testUnknownApplicationId() throws Exception {
|
|
|
|
public void testUnknownApplicationId() throws Exception {
|
|
|
|
Configuration conf = new YarnConfiguration();
|
|
|
|
|
|
|
|
YarnClient mockYarnClient = createMockYarnClientUnknownApp();
|
|
|
|
YarnClient mockYarnClient = createMockYarnClientUnknownApp();
|
|
|
|
LogsCLI cli = new LogsCLIForTest(mockYarnClient);
|
|
|
|
LogsCLI cli = new LogsCLIForTest(mockYarnClient);
|
|
|
|
cli.setConf(conf);
|
|
|
|
cli.setConf(conf);
|
|
|
@ -248,7 +253,6 @@ public class TestLogsCLI {
|
|
|
|
|
|
|
|
|
|
|
|
@Test (timeout = 10000)
|
|
|
|
@Test (timeout = 10000)
|
|
|
|
public void testHelpMessage() throws Exception {
|
|
|
|
public void testHelpMessage() throws Exception {
|
|
|
|
Configuration conf = new YarnConfiguration();
|
|
|
|
|
|
|
|
YarnClient mockYarnClient = createMockYarnClient(
|
|
|
|
YarnClient mockYarnClient = createMockYarnClient(
|
|
|
|
YarnApplicationState.FINISHED,
|
|
|
|
YarnApplicationState.FINISHED,
|
|
|
|
UserGroupInformation.getCurrentUser().getShortUserName());
|
|
|
|
UserGroupInformation.getCurrentUser().getShortUserName());
|
|
|
@ -372,13 +376,12 @@ public class TestLogsCLI {
|
|
|
|
@Test (timeout = 15000)
|
|
|
|
@Test (timeout = 15000)
|
|
|
|
public void testFetchFinishedApplictionLogs() throws Exception {
|
|
|
|
public void testFetchFinishedApplictionLogs() throws Exception {
|
|
|
|
String remoteLogRootDir = "target/logs/";
|
|
|
|
String remoteLogRootDir = "target/logs/";
|
|
|
|
Configuration configuration = new YarnConfiguration();
|
|
|
|
conf.setBoolean(YarnConfiguration.LOG_AGGREGATION_ENABLED, true);
|
|
|
|
configuration.setBoolean(YarnConfiguration.LOG_AGGREGATION_ENABLED, true);
|
|
|
|
conf
|
|
|
|
configuration
|
|
|
|
|
|
|
|
.set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR, remoteLogRootDir);
|
|
|
|
.set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR, remoteLogRootDir);
|
|
|
|
configuration.setBoolean(YarnConfiguration.YARN_ACL_ENABLE, true);
|
|
|
|
conf.setBoolean(YarnConfiguration.YARN_ACL_ENABLE, true);
|
|
|
|
configuration.set(YarnConfiguration.YARN_ADMIN_ACL, "admin");
|
|
|
|
conf.set(YarnConfiguration.YARN_ADMIN_ACL, "admin");
|
|
|
|
FileSystem fs = FileSystem.get(configuration);
|
|
|
|
FileSystem fs = FileSystem.get(conf);
|
|
|
|
|
|
|
|
|
|
|
|
UserGroupInformation ugi = UserGroupInformation.getCurrentUser();
|
|
|
|
UserGroupInformation ugi = UserGroupInformation.getCurrentUser();
|
|
|
|
ApplicationId appId = ApplicationId.newInstance(0, 1);
|
|
|
|
ApplicationId appId = ApplicationId.newInstance(0, 1);
|
|
|
@ -431,15 +434,15 @@ public class TestLogsCLI {
|
|
|
|
// the first two logs is empty. When we try to read first two logs,
|
|
|
|
// the first two logs is empty. When we try to read first two logs,
|
|
|
|
// we will meet EOF exception, but it will not impact other logs.
|
|
|
|
// we will meet EOF exception, but it will not impact other logs.
|
|
|
|
// Other logs should be read successfully.
|
|
|
|
// Other logs should be read successfully.
|
|
|
|
uploadEmptyContainerLogIntoRemoteDir(ugi, configuration, rootLogDirs, nodeId,
|
|
|
|
uploadEmptyContainerLogIntoRemoteDir(ugi, conf, rootLogDirs, nodeId,
|
|
|
|
containerId0, path, fs);
|
|
|
|
containerId0, path, fs);
|
|
|
|
uploadEmptyContainerLogIntoRemoteDir(ugi, configuration, rootLogDirs, nodeId,
|
|
|
|
uploadEmptyContainerLogIntoRemoteDir(ugi, conf, rootLogDirs, nodeId,
|
|
|
|
containerId1, path, fs);
|
|
|
|
containerId1, path, fs);
|
|
|
|
uploadContainerLogIntoRemoteDir(ugi, configuration, rootLogDirs, nodeId,
|
|
|
|
uploadContainerLogIntoRemoteDir(ugi, conf, rootLogDirs, nodeId,
|
|
|
|
containerId1, path, fs);
|
|
|
|
containerId1, path, fs);
|
|
|
|
uploadContainerLogIntoRemoteDir(ugi, configuration, rootLogDirs, nodeId,
|
|
|
|
uploadContainerLogIntoRemoteDir(ugi, conf, rootLogDirs, nodeId,
|
|
|
|
containerId2, path, fs);
|
|
|
|
containerId2, path, fs);
|
|
|
|
uploadContainerLogIntoRemoteDir(ugi, configuration, rootLogDirs, nodeId,
|
|
|
|
uploadContainerLogIntoRemoteDir(ugi, conf, rootLogDirs, nodeId,
|
|
|
|
containerId3, path, fs);
|
|
|
|
containerId3, path, fs);
|
|
|
|
|
|
|
|
|
|
|
|
YarnClient mockYarnClient =
|
|
|
|
YarnClient mockYarnClient =
|
|
|
@ -455,7 +458,7 @@ public class TestLogsCLI {
|
|
|
|
return mockReport;
|
|
|
|
return mockReport;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
};
|
|
|
|
};
|
|
|
|
cli.setConf(configuration);
|
|
|
|
cli.setConf(conf);
|
|
|
|
|
|
|
|
|
|
|
|
int exitCode = cli.run(new String[] { "-applicationId", appId.toString() });
|
|
|
|
int exitCode = cli.run(new String[] { "-applicationId", appId.toString() });
|
|
|
|
LOG.info(sysOutStream.toString());
|
|
|
|
LOG.info(sysOutStream.toString());
|
|
|
@ -723,7 +726,7 @@ public class TestLogsCLI {
|
|
|
|
YarnClient mockYarnClientWithException =
|
|
|
|
YarnClient mockYarnClientWithException =
|
|
|
|
createMockYarnClientWithException();
|
|
|
|
createMockYarnClientWithException();
|
|
|
|
cli = new LogsCLIForTest(mockYarnClientWithException);
|
|
|
|
cli = new LogsCLIForTest(mockYarnClientWithException);
|
|
|
|
cli.setConf(configuration);
|
|
|
|
cli.setConf(conf);
|
|
|
|
|
|
|
|
|
|
|
|
exitCode =
|
|
|
|
exitCode =
|
|
|
|
cli.run(new String[] { "-applicationId", appId.toString(),
|
|
|
|
cli.run(new String[] { "-applicationId", appId.toString(),
|
|
|
@ -846,8 +849,7 @@ public class TestLogsCLI {
|
|
|
|
any(ContainerId.class));
|
|
|
|
any(ContainerId.class));
|
|
|
|
|
|
|
|
|
|
|
|
// create local logs
|
|
|
|
// create local logs
|
|
|
|
Configuration configuration = new YarnConfiguration();
|
|
|
|
FileSystem fs = FileSystem.get(conf);
|
|
|
|
FileSystem fs = FileSystem.get(configuration);
|
|
|
|
|
|
|
|
String rootLogDir = "target/LocalLogs";
|
|
|
|
String rootLogDir = "target/LocalLogs";
|
|
|
|
Path rootLogDirPath = new Path(rootLogDir);
|
|
|
|
Path rootLogDirPath = new Path(rootLogDir);
|
|
|
|
if (fs.exists(rootLogDirPath)) {
|
|
|
|
if (fs.exists(rootLogDirPath)) {
|
|
|
@ -1006,13 +1008,12 @@ public class TestLogsCLI {
|
|
|
|
UserGroupInformation testUgi = UserGroupInformation
|
|
|
|
UserGroupInformation testUgi = UserGroupInformation
|
|
|
|
.createRemoteUser(testUser);
|
|
|
|
.createRemoteUser(testUser);
|
|
|
|
|
|
|
|
|
|
|
|
Configuration configuration = new YarnConfiguration();
|
|
|
|
conf.setBoolean(YarnConfiguration.LOG_AGGREGATION_ENABLED, true);
|
|
|
|
configuration.setBoolean(YarnConfiguration.LOG_AGGREGATION_ENABLED, true);
|
|
|
|
conf
|
|
|
|
configuration
|
|
|
|
|
|
|
|
.set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR, remoteLogRootDir);
|
|
|
|
.set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR, remoteLogRootDir);
|
|
|
|
configuration.setBoolean(YarnConfiguration.YARN_ACL_ENABLE, true);
|
|
|
|
conf.setBoolean(YarnConfiguration.YARN_ACL_ENABLE, true);
|
|
|
|
configuration.set(YarnConfiguration.YARN_ADMIN_ACL, "admin");
|
|
|
|
conf.set(YarnConfiguration.YARN_ADMIN_ACL, "admin");
|
|
|
|
FileSystem fs = FileSystem.get(configuration);
|
|
|
|
FileSystem fs = FileSystem.get(conf);
|
|
|
|
|
|
|
|
|
|
|
|
ApplicationId appId = ApplicationId.newInstance(0, 1);
|
|
|
|
ApplicationId appId = ApplicationId.newInstance(0, 1);
|
|
|
|
ApplicationAttemptId appAttemptId =
|
|
|
|
ApplicationAttemptId appAttemptId =
|
|
|
@ -1052,13 +1053,13 @@ public class TestLogsCLI {
|
|
|
|
assertTrue(fs.mkdirs(path));
|
|
|
|
assertTrue(fs.mkdirs(path));
|
|
|
|
|
|
|
|
|
|
|
|
// upload container logs for app into remote dir
|
|
|
|
// upload container logs for app into remote dir
|
|
|
|
uploadContainerLogIntoRemoteDir(testUgi, configuration, rootLogDirs,
|
|
|
|
uploadContainerLogIntoRemoteDir(testUgi, conf, rootLogDirs,
|
|
|
|
nodeId, containerId, path, fs);
|
|
|
|
nodeId, containerId, path, fs);
|
|
|
|
|
|
|
|
|
|
|
|
YarnClient mockYarnClient = createMockYarnClient(
|
|
|
|
YarnClient mockYarnClient = createMockYarnClient(
|
|
|
|
YarnApplicationState.FINISHED, testUgi.getShortUserName());
|
|
|
|
YarnApplicationState.FINISHED, testUgi.getShortUserName());
|
|
|
|
LogsCLI cli = new LogsCLIForTest(mockYarnClient);
|
|
|
|
LogsCLI cli = new LogsCLIForTest(mockYarnClient);
|
|
|
|
cli.setConf(configuration);
|
|
|
|
cli.setConf(conf);
|
|
|
|
|
|
|
|
|
|
|
|
// Verify that we can get the application logs by specifying
|
|
|
|
// Verify that we can get the application logs by specifying
|
|
|
|
// a correct appOwner
|
|
|
|
// a correct appOwner
|
|
|
@ -1085,7 +1086,7 @@ public class TestLogsCLI {
|
|
|
|
// and can get app logs successfully.
|
|
|
|
// and can get app logs successfully.
|
|
|
|
YarnClient mockYarnClient2 = createMockYarnClientUnknownApp();
|
|
|
|
YarnClient mockYarnClient2 = createMockYarnClientUnknownApp();
|
|
|
|
cli = new LogsCLIForTest(mockYarnClient2);
|
|
|
|
cli = new LogsCLIForTest(mockYarnClient2);
|
|
|
|
cli.setConf(configuration);
|
|
|
|
cli.setConf(conf);
|
|
|
|
exitCode = cli.run(new String[] {
|
|
|
|
exitCode = cli.run(new String[] {
|
|
|
|
"-applicationId", appId.toString()});
|
|
|
|
"-applicationId", appId.toString()});
|
|
|
|
assertTrue(exitCode == 0);
|
|
|
|
assertTrue(exitCode == 0);
|
|
|
@ -1099,7 +1100,7 @@ public class TestLogsCLI {
|
|
|
|
fs.delete(path, true);
|
|
|
|
fs.delete(path, true);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
assertTrue(fs.mkdirs(path));
|
|
|
|
assertTrue(fs.mkdirs(path));
|
|
|
|
uploadContainerLogIntoRemoteDir(testUgi, configuration, rootLogDirs,
|
|
|
|
uploadContainerLogIntoRemoteDir(testUgi, conf, rootLogDirs,
|
|
|
|
nodeId, containerId, path, fs);
|
|
|
|
nodeId, containerId, path, fs);
|
|
|
|
|
|
|
|
|
|
|
|
exitCode = cli.run(new String[] {
|
|
|
|
exitCode = cli.run(new String[] {
|
|
|
@ -1154,14 +1155,13 @@ public class TestLogsCLI {
|
|
|
|
public void testLogsCLIWithInvalidArgs() throws Exception {
|
|
|
|
public void testLogsCLIWithInvalidArgs() throws Exception {
|
|
|
|
String localDir = "target/SaveLogs";
|
|
|
|
String localDir = "target/SaveLogs";
|
|
|
|
Path localPath = new Path(localDir);
|
|
|
|
Path localPath = new Path(localDir);
|
|
|
|
Configuration configuration = new YarnConfiguration();
|
|
|
|
FileSystem fs = FileSystem.get(conf);
|
|
|
|
FileSystem fs = FileSystem.get(configuration);
|
|
|
|
|
|
|
|
ApplicationId appId = ApplicationId.newInstance(0, 1);
|
|
|
|
ApplicationId appId = ApplicationId.newInstance(0, 1);
|
|
|
|
YarnClient mockYarnClient =
|
|
|
|
YarnClient mockYarnClient =
|
|
|
|
createMockYarnClient(YarnApplicationState.FINISHED,
|
|
|
|
createMockYarnClient(YarnApplicationState.FINISHED,
|
|
|
|
UserGroupInformation.getCurrentUser().getShortUserName());
|
|
|
|
UserGroupInformation.getCurrentUser().getShortUserName());
|
|
|
|
LogsCLI cli = new LogsCLIForTest(mockYarnClient);
|
|
|
|
LogsCLI cli = new LogsCLIForTest(mockYarnClient);
|
|
|
|
cli.setConf(configuration);
|
|
|
|
cli.setConf(conf);
|
|
|
|
|
|
|
|
|
|
|
|
// Specify an invalid applicationId
|
|
|
|
// Specify an invalid applicationId
|
|
|
|
int exitCode = cli.run(new String[] {"-applicationId",
|
|
|
|
int exitCode = cli.run(new String[] {"-applicationId",
|
|
|
@ -1222,7 +1222,6 @@ public class TestLogsCLI {
|
|
|
|
String remoteLogRootDir = "target/logs/";
|
|
|
|
String remoteLogRootDir = "target/logs/";
|
|
|
|
String jobUser = "user1";
|
|
|
|
String jobUser = "user1";
|
|
|
|
String loggedUser = "user2";
|
|
|
|
String loggedUser = "user2";
|
|
|
|
Configuration conf = new YarnConfiguration();
|
|
|
|
|
|
|
|
conf.setBoolean(YarnConfiguration.LOG_AGGREGATION_ENABLED, true);
|
|
|
|
conf.setBoolean(YarnConfiguration.LOG_AGGREGATION_ENABLED, true);
|
|
|
|
conf.set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR, remoteLogRootDir);
|
|
|
|
conf.set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR, remoteLogRootDir);
|
|
|
|
conf.setBoolean(YarnConfiguration.YARN_ACL_ENABLE, true);
|
|
|
|
conf.setBoolean(YarnConfiguration.YARN_ACL_ENABLE, true);
|
|
|
@ -1268,7 +1267,6 @@ public class TestLogsCLI {
|
|
|
|
String remoteLogRootDir1 = "target/logs1/";
|
|
|
|
String remoteLogRootDir1 = "target/logs1/";
|
|
|
|
String jobUser = "user1";
|
|
|
|
String jobUser = "user1";
|
|
|
|
String loggedUser = "user2";
|
|
|
|
String loggedUser = "user2";
|
|
|
|
Configuration conf = new YarnConfiguration();
|
|
|
|
|
|
|
|
conf.setBoolean(YarnConfiguration.LOG_AGGREGATION_ENABLED, true);
|
|
|
|
conf.setBoolean(YarnConfiguration.LOG_AGGREGATION_ENABLED, true);
|
|
|
|
conf.set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR, remoteLogRootDir);
|
|
|
|
conf.set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR, remoteLogRootDir);
|
|
|
|
String controllerName = "indexed";
|
|
|
|
String controllerName = "indexed";
|
|
|
@ -1303,14 +1301,13 @@ public class TestLogsCLI {
|
|
|
|
String localDir = "target/SaveLogs";
|
|
|
|
String localDir = "target/SaveLogs";
|
|
|
|
Path localPath = new Path(localDir);
|
|
|
|
Path localPath = new Path(localDir);
|
|
|
|
|
|
|
|
|
|
|
|
Configuration configuration = new YarnConfiguration();
|
|
|
|
conf.setBoolean(YarnConfiguration.LOG_AGGREGATION_ENABLED, true);
|
|
|
|
configuration.setBoolean(YarnConfiguration.LOG_AGGREGATION_ENABLED, true);
|
|
|
|
conf
|
|
|
|
configuration
|
|
|
|
|
|
|
|
.set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR, remoteLogRootDir);
|
|
|
|
.set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR, remoteLogRootDir);
|
|
|
|
configuration.setBoolean(YarnConfiguration.YARN_ACL_ENABLE, true);
|
|
|
|
conf.setBoolean(YarnConfiguration.YARN_ACL_ENABLE, true);
|
|
|
|
configuration.set(YarnConfiguration.YARN_ADMIN_ACL, "admin");
|
|
|
|
conf.set(YarnConfiguration.YARN_ADMIN_ACL, "admin");
|
|
|
|
|
|
|
|
|
|
|
|
FileSystem fs = FileSystem.get(configuration);
|
|
|
|
FileSystem fs = FileSystem.get(conf);
|
|
|
|
ApplicationId appId = ApplicationId.newInstance(0, 1);
|
|
|
|
ApplicationId appId = ApplicationId.newInstance(0, 1);
|
|
|
|
ApplicationAttemptId appAttemptId =
|
|
|
|
ApplicationAttemptId appAttemptId =
|
|
|
|
ApplicationAttemptId.newInstance(appId, 1);
|
|
|
|
ApplicationAttemptId.newInstance(appId, 1);
|
|
|
@ -1329,14 +1326,14 @@ public class TestLogsCLI {
|
|
|
|
nodeIds.add(nodeId2);
|
|
|
|
nodeIds.add(nodeId2);
|
|
|
|
|
|
|
|
|
|
|
|
try {
|
|
|
|
try {
|
|
|
|
createContainerLogs(configuration, remoteLogRootDir, rootLogDir, fs,
|
|
|
|
createContainerLogs(conf, remoteLogRootDir, rootLogDir, fs,
|
|
|
|
appId, containerIds, nodeIds);
|
|
|
|
appId, containerIds, nodeIds);
|
|
|
|
|
|
|
|
|
|
|
|
YarnClient mockYarnClient =
|
|
|
|
YarnClient mockYarnClient =
|
|
|
|
createMockYarnClient(YarnApplicationState.FINISHED,
|
|
|
|
createMockYarnClient(YarnApplicationState.FINISHED,
|
|
|
|
UserGroupInformation.getCurrentUser().getShortUserName());
|
|
|
|
UserGroupInformation.getCurrentUser().getShortUserName());
|
|
|
|
LogsCLI cli = new LogsCLIForTest(mockYarnClient);
|
|
|
|
LogsCLI cli = new LogsCLIForTest(mockYarnClient);
|
|
|
|
cli.setConf(configuration);
|
|
|
|
cli.setConf(conf);
|
|
|
|
int exitCode = cli.run(new String[] {"-applicationId",
|
|
|
|
int exitCode = cli.run(new String[] {"-applicationId",
|
|
|
|
appId.toString(),
|
|
|
|
appId.toString(),
|
|
|
|
"-out" , localPath.toString()});
|
|
|
|
"-out" , localPath.toString()});
|
|
|
@ -1394,13 +1391,12 @@ public class TestLogsCLI {
|
|
|
|
@Test (timeout = 15000)
|
|
|
|
@Test (timeout = 15000)
|
|
|
|
public void testPrintContainerLogMetadata() throws Exception {
|
|
|
|
public void testPrintContainerLogMetadata() throws Exception {
|
|
|
|
String remoteLogRootDir = "target/logs/";
|
|
|
|
String remoteLogRootDir = "target/logs/";
|
|
|
|
Configuration configuration = new YarnConfiguration();
|
|
|
|
conf.setBoolean(YarnConfiguration.LOG_AGGREGATION_ENABLED, true);
|
|
|
|
configuration.setBoolean(YarnConfiguration.LOG_AGGREGATION_ENABLED, true);
|
|
|
|
conf
|
|
|
|
configuration
|
|
|
|
|
|
|
|
.set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR, remoteLogRootDir);
|
|
|
|
.set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR, remoteLogRootDir);
|
|
|
|
configuration.setBoolean(YarnConfiguration.YARN_ACL_ENABLE, true);
|
|
|
|
conf.setBoolean(YarnConfiguration.YARN_ACL_ENABLE, true);
|
|
|
|
configuration.set(YarnConfiguration.YARN_ADMIN_ACL, "admin");
|
|
|
|
conf.set(YarnConfiguration.YARN_ADMIN_ACL, "admin");
|
|
|
|
FileSystem fs = FileSystem.get(configuration);
|
|
|
|
FileSystem fs = FileSystem.get(conf);
|
|
|
|
String rootLogDir = "target/LocalLogs";
|
|
|
|
String rootLogDir = "target/LocalLogs";
|
|
|
|
|
|
|
|
|
|
|
|
ApplicationId appId = ApplicationId.newInstance(0, 1);
|
|
|
|
ApplicationId appId = ApplicationId.newInstance(0, 1);
|
|
|
@ -1419,14 +1415,14 @@ public class TestLogsCLI {
|
|
|
|
nodeIds.add(nodeId);
|
|
|
|
nodeIds.add(nodeId);
|
|
|
|
nodeIds.add(nodeId);
|
|
|
|
nodeIds.add(nodeId);
|
|
|
|
|
|
|
|
|
|
|
|
createContainerLogs(configuration, remoteLogRootDir, rootLogDir, fs,
|
|
|
|
createContainerLogs(conf, remoteLogRootDir, rootLogDir, fs,
|
|
|
|
appId, containerIds, nodeIds);
|
|
|
|
appId, containerIds, nodeIds);
|
|
|
|
|
|
|
|
|
|
|
|
YarnClient mockYarnClient =
|
|
|
|
YarnClient mockYarnClient =
|
|
|
|
createMockYarnClient(YarnApplicationState.FINISHED,
|
|
|
|
createMockYarnClient(YarnApplicationState.FINISHED,
|
|
|
|
UserGroupInformation.getCurrentUser().getShortUserName());
|
|
|
|
UserGroupInformation.getCurrentUser().getShortUserName());
|
|
|
|
LogsCLI cli = new LogsCLIForTest(mockYarnClient);
|
|
|
|
LogsCLI cli = new LogsCLIForTest(mockYarnClient);
|
|
|
|
cli.setConf(configuration);
|
|
|
|
cli.setConf(conf);
|
|
|
|
|
|
|
|
|
|
|
|
cli.run(new String[] {"-applicationId", appId.toString(),
|
|
|
|
cli.run(new String[] {"-applicationId", appId.toString(),
|
|
|
|
"-show_container_log_info"});
|
|
|
|
"-show_container_log_info"});
|
|
|
@ -1499,12 +1495,11 @@ public class TestLogsCLI {
|
|
|
|
@Test (timeout = 15000)
|
|
|
|
@Test (timeout = 15000)
|
|
|
|
public void testListNodeInfo() throws Exception {
|
|
|
|
public void testListNodeInfo() throws Exception {
|
|
|
|
String remoteLogRootDir = "target/logs/";
|
|
|
|
String remoteLogRootDir = "target/logs/";
|
|
|
|
Configuration configuration = new YarnConfiguration();
|
|
|
|
conf.setBoolean(YarnConfiguration.LOG_AGGREGATION_ENABLED, true);
|
|
|
|
configuration.setBoolean(YarnConfiguration.LOG_AGGREGATION_ENABLED, true);
|
|
|
|
conf
|
|
|
|
configuration
|
|
|
|
|
|
|
|
.set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR, remoteLogRootDir);
|
|
|
|
.set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR, remoteLogRootDir);
|
|
|
|
configuration.setBoolean(YarnConfiguration.YARN_ACL_ENABLE, true);
|
|
|
|
conf.setBoolean(YarnConfiguration.YARN_ACL_ENABLE, true);
|
|
|
|
configuration.set(YarnConfiguration.YARN_ADMIN_ACL, "admin");
|
|
|
|
conf.set(YarnConfiguration.YARN_ADMIN_ACL, "admin");
|
|
|
|
|
|
|
|
|
|
|
|
ApplicationId appId = ApplicationId.newInstance(0, 1);
|
|
|
|
ApplicationId appId = ApplicationId.newInstance(0, 1);
|
|
|
|
ApplicationAttemptId appAttemptId =
|
|
|
|
ApplicationAttemptId appAttemptId =
|
|
|
@ -1524,16 +1519,16 @@ public class TestLogsCLI {
|
|
|
|
nodeIds.add(nodeId2);
|
|
|
|
nodeIds.add(nodeId2);
|
|
|
|
|
|
|
|
|
|
|
|
String rootLogDir = "target/LocalLogs";
|
|
|
|
String rootLogDir = "target/LocalLogs";
|
|
|
|
FileSystem fs = FileSystem.get(configuration);
|
|
|
|
FileSystem fs = FileSystem.get(conf);
|
|
|
|
|
|
|
|
|
|
|
|
createContainerLogs(configuration, remoteLogRootDir, rootLogDir, fs,
|
|
|
|
createContainerLogs(conf, remoteLogRootDir, rootLogDir, fs,
|
|
|
|
appId, containerIds, nodeIds);
|
|
|
|
appId, containerIds, nodeIds);
|
|
|
|
|
|
|
|
|
|
|
|
YarnClient mockYarnClient =
|
|
|
|
YarnClient mockYarnClient =
|
|
|
|
createMockYarnClient(YarnApplicationState.FINISHED,
|
|
|
|
createMockYarnClient(YarnApplicationState.FINISHED,
|
|
|
|
UserGroupInformation.getCurrentUser().getShortUserName());
|
|
|
|
UserGroupInformation.getCurrentUser().getShortUserName());
|
|
|
|
LogsCLI cli = new LogsCLIForTest(mockYarnClient);
|
|
|
|
LogsCLI cli = new LogsCLIForTest(mockYarnClient);
|
|
|
|
cli.setConf(configuration);
|
|
|
|
cli.setConf(conf);
|
|
|
|
|
|
|
|
|
|
|
|
cli.run(new String[] { "-applicationId", appId.toString(),
|
|
|
|
cli.run(new String[] { "-applicationId", appId.toString(),
|
|
|
|
"-list_nodes" });
|
|
|
|
"-list_nodes" });
|
|
|
@ -1550,13 +1545,12 @@ public class TestLogsCLI {
|
|
|
|
@Test (timeout = 15000)
|
|
|
|
@Test (timeout = 15000)
|
|
|
|
public void testFetchApplictionLogsHar() throws Exception {
|
|
|
|
public void testFetchApplictionLogsHar() throws Exception {
|
|
|
|
String remoteLogRootDir = "target/logs/";
|
|
|
|
String remoteLogRootDir = "target/logs/";
|
|
|
|
Configuration configuration = new YarnConfiguration();
|
|
|
|
conf.setBoolean(YarnConfiguration.LOG_AGGREGATION_ENABLED, true);
|
|
|
|
configuration.setBoolean(YarnConfiguration.LOG_AGGREGATION_ENABLED, true);
|
|
|
|
conf
|
|
|
|
configuration
|
|
|
|
|
|
|
|
.set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR, remoteLogRootDir);
|
|
|
|
.set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR, remoteLogRootDir);
|
|
|
|
configuration.setBoolean(YarnConfiguration.YARN_ACL_ENABLE, true);
|
|
|
|
conf.setBoolean(YarnConfiguration.YARN_ACL_ENABLE, true);
|
|
|
|
configuration.set(YarnConfiguration.YARN_ADMIN_ACL, "admin");
|
|
|
|
conf.set(YarnConfiguration.YARN_ADMIN_ACL, "admin");
|
|
|
|
FileSystem fs = FileSystem.get(configuration);
|
|
|
|
FileSystem fs = FileSystem.get(conf);
|
|
|
|
UserGroupInformation ugi = UserGroupInformation.getCurrentUser();
|
|
|
|
UserGroupInformation ugi = UserGroupInformation.getCurrentUser();
|
|
|
|
URL harUrl = ClassLoader.getSystemClassLoader()
|
|
|
|
URL harUrl = ClassLoader.getSystemClassLoader()
|
|
|
|
.getResource("application_1440536969523_0001.har");
|
|
|
|
.getResource("application_1440536969523_0001.har");
|
|
|
@ -1576,7 +1570,7 @@ public class TestLogsCLI {
|
|
|
|
createMockYarnClient(YarnApplicationState.FINISHED,
|
|
|
|
createMockYarnClient(YarnApplicationState.FINISHED,
|
|
|
|
ugi.getShortUserName());
|
|
|
|
ugi.getShortUserName());
|
|
|
|
LogsCLI cli = new LogsCLIForTest(mockYarnClient);
|
|
|
|
LogsCLI cli = new LogsCLIForTest(mockYarnClient);
|
|
|
|
cli.setConf(configuration);
|
|
|
|
cli.setConf(conf);
|
|
|
|
int exitCode = cli.run(new String[]{"-applicationId",
|
|
|
|
int exitCode = cli.run(new String[]{"-applicationId",
|
|
|
|
"application_1440536969523_0001"});
|
|
|
|
"application_1440536969523_0001"});
|
|
|
|
assertTrue(exitCode == 0);
|
|
|
|
assertTrue(exitCode == 0);
|
|
|
|