YARN-9606. Set sslfactory for AuthenticatedURL() while creating LogsCLI#webServiceClient.

Contributed by Bilwa S T. Reviewed by Akira Ajisaka.
This commit is contained in:
Prabhu Joseph 2020-05-19 17:53:23 +05:30 committed by Prabhu Joseph
parent 1983eea62d
commit cef0756929
6 changed files with 81 additions and 99 deletions

View File

@ -27,17 +27,13 @@
import com.sun.jersey.api.client.WebResource; import com.sun.jersey.api.client.WebResource;
import com.sun.jersey.api.client.WebResource.Builder; import com.sun.jersey.api.client.WebResource.Builder;
import com.sun.jersey.api.client.filter.ClientFilter; import com.sun.jersey.api.client.filter.ClientFilter;
import com.sun.jersey.client.urlconnection.HttpURLConnectionFactory;
import com.sun.jersey.client.urlconnection.URLConnectionClientHandler;
import java.io.File; import java.io.File;
import java.io.IOException; import java.io.IOException;
import java.io.InputStream; import java.io.InputStream;
import java.io.PrintStream; import java.io.PrintStream;
import java.net.ConnectException; import java.net.ConnectException;
import java.net.HttpURLConnection;
import java.net.SocketException; import java.net.SocketException;
import java.net.SocketTimeoutException; import java.net.SocketTimeoutException;
import java.net.URL;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Arrays; import java.util.Arrays;
import java.util.Collection; import java.util.Collection;
@ -68,8 +64,6 @@
import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.Configured; import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.authentication.client.AuthenticatedURL;
import org.apache.hadoop.security.authentication.client.AuthenticationException;
import org.apache.hadoop.util.Tool; import org.apache.hadoop.util.Tool;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptReport; import org.apache.hadoop.yarn.api.records.ApplicationAttemptReport;
import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ApplicationId;
@ -89,6 +83,7 @@
import org.apache.hadoop.yarn.server.metrics.AppAttemptMetricsConstants; import org.apache.hadoop.yarn.server.metrics.AppAttemptMetricsConstants;
import org.apache.hadoop.yarn.util.Apps; import org.apache.hadoop.yarn.util.Apps;
import org.apache.hadoop.yarn.webapp.util.WebAppUtils; import org.apache.hadoop.yarn.webapp.util.WebAppUtils;
import org.apache.hadoop.yarn.webapp.util.WebServiceClient;
import org.apache.hadoop.yarn.webapp.util.YarnWebServiceUtils; import org.apache.hadoop.yarn.webapp.util.YarnWebServiceUtils;
import org.codehaus.jettison.json.JSONArray; import org.codehaus.jettison.json.JSONArray;
import org.codehaus.jettison.json.JSONException; import org.codehaus.jettison.json.JSONException;
@ -139,21 +134,7 @@ public class LogsCLI extends Configured implements Tool {
@Override @Override
public int run(String[] args) throws Exception { public int run(String[] args) throws Exception {
try { try {
webServiceClient = new Client(new URLConnectionClientHandler( webServiceClient = WebServiceClient.getWebServiceClient().createClient();
new HttpURLConnectionFactory() {
@Override
public HttpURLConnection getHttpURLConnection(URL url)
throws IOException {
AuthenticatedURL.Token token = new AuthenticatedURL.Token();
HttpURLConnection conn = null;
try {
conn = new AuthenticatedURL().openConnection(url, token);
} catch (AuthenticationException e) {
throw new IOException(e);
}
return conn;
}
}));
return runCommand(args); return runCommand(args);
} finally { } finally {
if (yarnClient != null) { if (yarnClient != null) {
@ -411,7 +392,9 @@ public static void main(String[] args) throws Exception {
Configuration conf = new YarnConfiguration(); Configuration conf = new YarnConfiguration();
LogsCLI logDumper = new LogsCLI(); LogsCLI logDumper = new LogsCLI();
logDumper.setConf(conf); logDumper.setConf(conf);
WebServiceClient.initialize(conf);
int exitCode = logDumper.run(args); int exitCode = logDumper.run(args);
WebServiceClient.destroy();
System.exit(exitCode); System.exit(exitCode);
} }

View File

@ -89,8 +89,10 @@
import org.apache.hadoop.yarn.logaggregation.filecontroller.LogAggregationFileControllerContext; import org.apache.hadoop.yarn.logaggregation.filecontroller.LogAggregationFileControllerContext;
import org.apache.hadoop.yarn.logaggregation.filecontroller.LogAggregationFileControllerFactory; import org.apache.hadoop.yarn.logaggregation.filecontroller.LogAggregationFileControllerFactory;
import org.apache.hadoop.yarn.logaggregation.filecontroller.ifile.LogAggregationIndexedFileController; import org.apache.hadoop.yarn.logaggregation.filecontroller.ifile.LogAggregationIndexedFileController;
import org.apache.hadoop.yarn.webapp.util.WebServiceClient;
import org.codehaus.jettison.json.JSONArray; import org.codehaus.jettison.json.JSONArray;
import org.codehaus.jettison.json.JSONObject; import org.codehaus.jettison.json.JSONObject;
import org.junit.After;
import org.junit.Assert; import org.junit.Assert;
import org.junit.Before; import org.junit.Before;
import org.junit.Test; import org.junit.Test;
@ -109,8 +111,10 @@ public class TestLogsCLI {
ByteArrayOutputStream sysErrStream; ByteArrayOutputStream sysErrStream;
private PrintStream sysErr; private PrintStream sysErr;
private Configuration conf;
@Before @Before
public void setUp() { public void setUp() throws Exception {
sysOutStream = new ByteArrayOutputStream(); sysOutStream = new ByteArrayOutputStream();
sysOut = new PrintStream(sysOutStream); sysOut = new PrintStream(sysOutStream);
System.setOut(sysOut); System.setOut(sysOut);
@ -118,11 +122,17 @@ public void setUp() {
sysErrStream = new ByteArrayOutputStream(); sysErrStream = new ByteArrayOutputStream();
sysErr = new PrintStream(sysErrStream); sysErr = new PrintStream(sysErrStream);
System.setErr(sysErr); System.setErr(sysErr);
conf = new YarnConfiguration();
WebServiceClient.initialize(conf);
}
@After
public void tearDown() {
WebServiceClient.destroy();
} }
@Test(timeout = 5000l) @Test(timeout = 5000l)
public void testFailResultCodes() throws Exception { public void testFailResultCodes() throws Exception {
Configuration conf = new YarnConfiguration();
conf.setClass("fs.file.impl", LocalFileSystem.class, FileSystem.class); conf.setClass("fs.file.impl", LocalFileSystem.class, FileSystem.class);
LogCLIHelpers cliHelper = new LogCLIHelpers(); LogCLIHelpers cliHelper = new LogCLIHelpers();
cliHelper.setConf(conf); cliHelper.setConf(conf);
@ -145,7 +155,6 @@ public void testFailResultCodes() throws Exception {
@Test(timeout = 10000l) @Test(timeout = 10000l)
public void testInvalidOpts() throws Exception { public void testInvalidOpts() throws Exception {
Configuration conf = new YarnConfiguration();
YarnClient mockYarnClient = createMockYarnClient( YarnClient mockYarnClient = createMockYarnClient(
YarnApplicationState.FINISHED, YarnApplicationState.FINISHED,
UserGroupInformation.getCurrentUser().getShortUserName()); UserGroupInformation.getCurrentUser().getShortUserName());
@ -160,7 +169,6 @@ public void testInvalidOpts() throws Exception {
@Test(timeout = 5000l) @Test(timeout = 5000l)
public void testInvalidApplicationId() throws Exception { public void testInvalidApplicationId() throws Exception {
Configuration conf = new YarnConfiguration();
YarnClient mockYarnClient = createMockYarnClient( YarnClient mockYarnClient = createMockYarnClient(
YarnApplicationState.FINISHED, YarnApplicationState.FINISHED,
UserGroupInformation.getCurrentUser().getShortUserName()); UserGroupInformation.getCurrentUser().getShortUserName());
@ -174,7 +182,6 @@ public void testInvalidApplicationId() throws Exception {
@Test(timeout = 5000L) @Test(timeout = 5000L)
public void testInvalidAMContainerId() throws Exception { public void testInvalidAMContainerId() throws Exception {
Configuration conf = new YarnConfiguration();
conf.setBoolean(YarnConfiguration.APPLICATION_HISTORY_ENABLED, true); conf.setBoolean(YarnConfiguration.APPLICATION_HISTORY_ENABLED, true);
YarnClient mockYarnClient = YarnClient mockYarnClient =
createMockYarnClient(YarnApplicationState.FINISHED, createMockYarnClient(YarnApplicationState.FINISHED,
@ -196,7 +203,6 @@ public void testInvalidAMContainerId() throws Exception {
@Test @Test
public void testAMContainerInfoFetchFromTimelineReader() throws Exception { public void testAMContainerInfoFetchFromTimelineReader() throws Exception {
Configuration conf = new YarnConfiguration();
conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, true); conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, true);
conf.set(YarnConfiguration.TIMELINE_SERVICE_VERSIONS, "2.0f"); conf.set(YarnConfiguration.TIMELINE_SERVICE_VERSIONS, "2.0f");
YarnClient mockYarnClient = YarnClient mockYarnClient =
@ -232,7 +238,6 @@ public void testAMContainerInfoFetchFromTimelineReader() throws Exception {
@Test(timeout = 5000l) @Test(timeout = 5000l)
public void testUnknownApplicationId() throws Exception { public void testUnknownApplicationId() throws Exception {
Configuration conf = new YarnConfiguration();
YarnClient mockYarnClient = createMockYarnClientUnknownApp(); YarnClient mockYarnClient = createMockYarnClientUnknownApp();
LogsCLI cli = new LogsCLIForTest(mockYarnClient); LogsCLI cli = new LogsCLIForTest(mockYarnClient);
cli.setConf(conf); cli.setConf(conf);
@ -248,7 +253,6 @@ public void testUnknownApplicationId() throws Exception {
@Test (timeout = 10000) @Test (timeout = 10000)
public void testHelpMessage() throws Exception { public void testHelpMessage() throws Exception {
Configuration conf = new YarnConfiguration();
YarnClient mockYarnClient = createMockYarnClient( YarnClient mockYarnClient = createMockYarnClient(
YarnApplicationState.FINISHED, YarnApplicationState.FINISHED,
UserGroupInformation.getCurrentUser().getShortUserName()); UserGroupInformation.getCurrentUser().getShortUserName());
@ -372,13 +376,12 @@ public void testHelpMessage() throws Exception {
@Test (timeout = 15000) @Test (timeout = 15000)
public void testFetchFinishedApplictionLogs() throws Exception { public void testFetchFinishedApplictionLogs() throws Exception {
String remoteLogRootDir = "target/logs/"; String remoteLogRootDir = "target/logs/";
Configuration configuration = new YarnConfiguration(); conf.setBoolean(YarnConfiguration.LOG_AGGREGATION_ENABLED, true);
configuration.setBoolean(YarnConfiguration.LOG_AGGREGATION_ENABLED, true); conf
configuration
.set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR, remoteLogRootDir); .set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR, remoteLogRootDir);
configuration.setBoolean(YarnConfiguration.YARN_ACL_ENABLE, true); conf.setBoolean(YarnConfiguration.YARN_ACL_ENABLE, true);
configuration.set(YarnConfiguration.YARN_ADMIN_ACL, "admin"); conf.set(YarnConfiguration.YARN_ADMIN_ACL, "admin");
FileSystem fs = FileSystem.get(configuration); FileSystem fs = FileSystem.get(conf);
UserGroupInformation ugi = UserGroupInformation.getCurrentUser(); UserGroupInformation ugi = UserGroupInformation.getCurrentUser();
ApplicationId appId = ApplicationId.newInstance(0, 1); ApplicationId appId = ApplicationId.newInstance(0, 1);
@ -431,15 +434,15 @@ public void testFetchFinishedApplictionLogs() throws Exception {
// the first two logs is empty. When we try to read first two logs, // the first two logs is empty. When we try to read first two logs,
// we will meet EOF exception, but it will not impact other logs. // we will meet EOF exception, but it will not impact other logs.
// Other logs should be read successfully. // Other logs should be read successfully.
uploadEmptyContainerLogIntoRemoteDir(ugi, configuration, rootLogDirs, nodeId, uploadEmptyContainerLogIntoRemoteDir(ugi, conf, rootLogDirs, nodeId,
containerId0, path, fs); containerId0, path, fs);
uploadEmptyContainerLogIntoRemoteDir(ugi, configuration, rootLogDirs, nodeId, uploadEmptyContainerLogIntoRemoteDir(ugi, conf, rootLogDirs, nodeId,
containerId1, path, fs); containerId1, path, fs);
uploadContainerLogIntoRemoteDir(ugi, configuration, rootLogDirs, nodeId, uploadContainerLogIntoRemoteDir(ugi, conf, rootLogDirs, nodeId,
containerId1, path, fs); containerId1, path, fs);
uploadContainerLogIntoRemoteDir(ugi, configuration, rootLogDirs, nodeId, uploadContainerLogIntoRemoteDir(ugi, conf, rootLogDirs, nodeId,
containerId2, path, fs); containerId2, path, fs);
uploadContainerLogIntoRemoteDir(ugi, configuration, rootLogDirs, nodeId, uploadContainerLogIntoRemoteDir(ugi, conf, rootLogDirs, nodeId,
containerId3, path, fs); containerId3, path, fs);
YarnClient mockYarnClient = YarnClient mockYarnClient =
@ -455,7 +458,7 @@ public ContainerReport getContainerReport(String containerIdStr)
return mockReport; return mockReport;
} }
}; };
cli.setConf(configuration); cli.setConf(conf);
int exitCode = cli.run(new String[] { "-applicationId", appId.toString() }); int exitCode = cli.run(new String[] { "-applicationId", appId.toString() });
LOG.info(sysOutStream.toString()); LOG.info(sysOutStream.toString());
@ -681,7 +684,7 @@ public ContainerReport getContainerReport(String containerIdStr)
YarnClient mockYarnClientWithException = YarnClient mockYarnClientWithException =
createMockYarnClientWithException(); createMockYarnClientWithException();
cli = new LogsCLIForTest(mockYarnClientWithException); cli = new LogsCLIForTest(mockYarnClientWithException);
cli.setConf(configuration); cli.setConf(conf);
exitCode = exitCode =
cli.run(new String[] { "-applicationId", appId.toString(), cli.run(new String[] { "-applicationId", appId.toString(),
@ -804,8 +807,7 @@ public void testGetRunningContainerLogs() throws Exception {
any(ContainerId.class)); any(ContainerId.class));
// create local logs // create local logs
Configuration configuration = new YarnConfiguration(); FileSystem fs = FileSystem.get(conf);
FileSystem fs = FileSystem.get(configuration);
String rootLogDir = "target/LocalLogs"; String rootLogDir = "target/LocalLogs";
Path rootLogDirPath = new Path(rootLogDir); Path rootLogDirPath = new Path(rootLogDir);
if (fs.exists(rootLogDirPath)) { if (fs.exists(rootLogDirPath)) {
@ -964,13 +966,12 @@ public void testFetchApplictionLogsAsAnotherUser() throws Exception {
UserGroupInformation testUgi = UserGroupInformation UserGroupInformation testUgi = UserGroupInformation
.createRemoteUser(testUser); .createRemoteUser(testUser);
Configuration configuration = new YarnConfiguration(); conf.setBoolean(YarnConfiguration.LOG_AGGREGATION_ENABLED, true);
configuration.setBoolean(YarnConfiguration.LOG_AGGREGATION_ENABLED, true); conf
configuration
.set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR, remoteLogRootDir); .set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR, remoteLogRootDir);
configuration.setBoolean(YarnConfiguration.YARN_ACL_ENABLE, true); conf.setBoolean(YarnConfiguration.YARN_ACL_ENABLE, true);
configuration.set(YarnConfiguration.YARN_ADMIN_ACL, "admin"); conf.set(YarnConfiguration.YARN_ADMIN_ACL, "admin");
FileSystem fs = FileSystem.get(configuration); FileSystem fs = FileSystem.get(conf);
ApplicationId appId = ApplicationId.newInstance(0, 1); ApplicationId appId = ApplicationId.newInstance(0, 1);
ApplicationAttemptId appAttemptId = ApplicationAttemptId appAttemptId =
@ -1010,13 +1011,13 @@ public void testFetchApplictionLogsAsAnotherUser() throws Exception {
assertTrue(fs.mkdirs(path)); assertTrue(fs.mkdirs(path));
// upload container logs for app into remote dir // upload container logs for app into remote dir
uploadContainerLogIntoRemoteDir(testUgi, configuration, rootLogDirs, uploadContainerLogIntoRemoteDir(testUgi, conf, rootLogDirs,
nodeId, containerId, path, fs); nodeId, containerId, path, fs);
YarnClient mockYarnClient = createMockYarnClient( YarnClient mockYarnClient = createMockYarnClient(
YarnApplicationState.FINISHED, testUgi.getShortUserName()); YarnApplicationState.FINISHED, testUgi.getShortUserName());
LogsCLI cli = new LogsCLIForTest(mockYarnClient); LogsCLI cli = new LogsCLIForTest(mockYarnClient);
cli.setConf(configuration); cli.setConf(conf);
// Verify that we can get the application logs by specifying // Verify that we can get the application logs by specifying
// a correct appOwner // a correct appOwner
@ -1043,7 +1044,7 @@ public void testFetchApplictionLogsAsAnotherUser() throws Exception {
// and can get app logs successfully. // and can get app logs successfully.
YarnClient mockYarnClient2 = createMockYarnClientUnknownApp(); YarnClient mockYarnClient2 = createMockYarnClientUnknownApp();
cli = new LogsCLIForTest(mockYarnClient2); cli = new LogsCLIForTest(mockYarnClient2);
cli.setConf(configuration); cli.setConf(conf);
exitCode = cli.run(new String[] { exitCode = cli.run(new String[] {
"-applicationId", appId.toString()}); "-applicationId", appId.toString()});
assertTrue(exitCode == 0); assertTrue(exitCode == 0);
@ -1057,7 +1058,7 @@ public void testFetchApplictionLogsAsAnotherUser() throws Exception {
fs.delete(path, true); fs.delete(path, true);
} }
assertTrue(fs.mkdirs(path)); assertTrue(fs.mkdirs(path));
uploadContainerLogIntoRemoteDir(testUgi, configuration, rootLogDirs, uploadContainerLogIntoRemoteDir(testUgi, conf, rootLogDirs,
nodeId, containerId, path, fs); nodeId, containerId, path, fs);
exitCode = cli.run(new String[] { exitCode = cli.run(new String[] {
@ -1112,14 +1113,13 @@ public void testFetchApplictionLogsAsAnotherUser() throws Exception {
public void testLogsCLIWithInvalidArgs() throws Exception { public void testLogsCLIWithInvalidArgs() throws Exception {
String localDir = "target/SaveLogs"; String localDir = "target/SaveLogs";
Path localPath = new Path(localDir); Path localPath = new Path(localDir);
Configuration configuration = new YarnConfiguration(); FileSystem fs = FileSystem.get(conf);
FileSystem fs = FileSystem.get(configuration);
ApplicationId appId = ApplicationId.newInstance(0, 1); ApplicationId appId = ApplicationId.newInstance(0, 1);
YarnClient mockYarnClient = YarnClient mockYarnClient =
createMockYarnClient(YarnApplicationState.FINISHED, createMockYarnClient(YarnApplicationState.FINISHED,
UserGroupInformation.getCurrentUser().getShortUserName()); UserGroupInformation.getCurrentUser().getShortUserName());
LogsCLI cli = new LogsCLIForTest(mockYarnClient); LogsCLI cli = new LogsCLIForTest(mockYarnClient);
cli.setConf(configuration); cli.setConf(conf);
// Specify an invalid applicationId // Specify an invalid applicationId
int exitCode = cli.run(new String[] {"-applicationId", int exitCode = cli.run(new String[] {"-applicationId",
@ -1180,7 +1180,6 @@ public void testGuessAppOwnerWithCustomSuffix() throws Exception {
String remoteLogRootDir = "target/logs/"; String remoteLogRootDir = "target/logs/";
String jobUser = "user1"; String jobUser = "user1";
String loggedUser = "user2"; String loggedUser = "user2";
Configuration conf = new YarnConfiguration();
conf.setBoolean(YarnConfiguration.LOG_AGGREGATION_ENABLED, true); conf.setBoolean(YarnConfiguration.LOG_AGGREGATION_ENABLED, true);
conf.set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR, remoteLogRootDir); conf.set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR, remoteLogRootDir);
conf.setBoolean(YarnConfiguration.YARN_ACL_ENABLE, true); conf.setBoolean(YarnConfiguration.YARN_ACL_ENABLE, true);
@ -1226,7 +1225,6 @@ public void testGuessAppOwnerWithCustomAppLogDir() throws Exception {
String remoteLogRootDir1 = "target/logs1/"; String remoteLogRootDir1 = "target/logs1/";
String jobUser = "user1"; String jobUser = "user1";
String loggedUser = "user2"; String loggedUser = "user2";
Configuration conf = new YarnConfiguration();
conf.setBoolean(YarnConfiguration.LOG_AGGREGATION_ENABLED, true); conf.setBoolean(YarnConfiguration.LOG_AGGREGATION_ENABLED, true);
conf.set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR, remoteLogRootDir); conf.set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR, remoteLogRootDir);
String controllerName = "indexed"; String controllerName = "indexed";
@ -1261,14 +1259,13 @@ public void testSaveContainerLogsLocally() throws Exception {
String localDir = "target/SaveLogs"; String localDir = "target/SaveLogs";
Path localPath = new Path(localDir); Path localPath = new Path(localDir);
Configuration configuration = new YarnConfiguration(); conf.setBoolean(YarnConfiguration.LOG_AGGREGATION_ENABLED, true);
configuration.setBoolean(YarnConfiguration.LOG_AGGREGATION_ENABLED, true); conf
configuration
.set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR, remoteLogRootDir); .set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR, remoteLogRootDir);
configuration.setBoolean(YarnConfiguration.YARN_ACL_ENABLE, true); conf.setBoolean(YarnConfiguration.YARN_ACL_ENABLE, true);
configuration.set(YarnConfiguration.YARN_ADMIN_ACL, "admin"); conf.set(YarnConfiguration.YARN_ADMIN_ACL, "admin");
FileSystem fs = FileSystem.get(configuration); FileSystem fs = FileSystem.get(conf);
ApplicationId appId = ApplicationId.newInstance(0, 1); ApplicationId appId = ApplicationId.newInstance(0, 1);
ApplicationAttemptId appAttemptId = ApplicationAttemptId appAttemptId =
ApplicationAttemptId.newInstance(appId, 1); ApplicationAttemptId.newInstance(appId, 1);
@ -1287,14 +1284,14 @@ public void testSaveContainerLogsLocally() throws Exception {
nodeIds.add(nodeId2); nodeIds.add(nodeId2);
try { try {
createContainerLogs(configuration, remoteLogRootDir, rootLogDir, fs, createContainerLogs(conf, remoteLogRootDir, rootLogDir, fs,
appId, containerIds, nodeIds); appId, containerIds, nodeIds);
YarnClient mockYarnClient = YarnClient mockYarnClient =
createMockYarnClient(YarnApplicationState.FINISHED, createMockYarnClient(YarnApplicationState.FINISHED,
UserGroupInformation.getCurrentUser().getShortUserName()); UserGroupInformation.getCurrentUser().getShortUserName());
LogsCLI cli = new LogsCLIForTest(mockYarnClient); LogsCLI cli = new LogsCLIForTest(mockYarnClient);
cli.setConf(configuration); cli.setConf(conf);
int exitCode = cli.run(new String[] {"-applicationId", int exitCode = cli.run(new String[] {"-applicationId",
appId.toString(), appId.toString(),
"-out" , localPath.toString()}); "-out" , localPath.toString()});
@ -1352,13 +1349,12 @@ private String readContainerContent(Path containerPath,
@Test (timeout = 15000) @Test (timeout = 15000)
public void testPrintContainerLogMetadata() throws Exception { public void testPrintContainerLogMetadata() throws Exception {
String remoteLogRootDir = "target/logs/"; String remoteLogRootDir = "target/logs/";
Configuration configuration = new YarnConfiguration(); conf.setBoolean(YarnConfiguration.LOG_AGGREGATION_ENABLED, true);
configuration.setBoolean(YarnConfiguration.LOG_AGGREGATION_ENABLED, true); conf
configuration
.set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR, remoteLogRootDir); .set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR, remoteLogRootDir);
configuration.setBoolean(YarnConfiguration.YARN_ACL_ENABLE, true); conf.setBoolean(YarnConfiguration.YARN_ACL_ENABLE, true);
configuration.set(YarnConfiguration.YARN_ADMIN_ACL, "admin"); conf.set(YarnConfiguration.YARN_ADMIN_ACL, "admin");
FileSystem fs = FileSystem.get(configuration); FileSystem fs = FileSystem.get(conf);
String rootLogDir = "target/LocalLogs"; String rootLogDir = "target/LocalLogs";
ApplicationId appId = ApplicationId.newInstance(0, 1); ApplicationId appId = ApplicationId.newInstance(0, 1);
@ -1377,14 +1373,14 @@ public void testPrintContainerLogMetadata() throws Exception {
nodeIds.add(nodeId); nodeIds.add(nodeId);
nodeIds.add(nodeId); nodeIds.add(nodeId);
createContainerLogs(configuration, remoteLogRootDir, rootLogDir, fs, createContainerLogs(conf, remoteLogRootDir, rootLogDir, fs,
appId, containerIds, nodeIds); appId, containerIds, nodeIds);
YarnClient mockYarnClient = YarnClient mockYarnClient =
createMockYarnClient(YarnApplicationState.FINISHED, createMockYarnClient(YarnApplicationState.FINISHED,
UserGroupInformation.getCurrentUser().getShortUserName()); UserGroupInformation.getCurrentUser().getShortUserName());
LogsCLI cli = new LogsCLIForTest(mockYarnClient); LogsCLI cli = new LogsCLIForTest(mockYarnClient);
cli.setConf(configuration); cli.setConf(conf);
cli.run(new String[] {"-applicationId", appId.toString(), cli.run(new String[] {"-applicationId", appId.toString(),
"-show_container_log_info"}); "-show_container_log_info"});
@ -1457,12 +1453,11 @@ public void testPrintContainerLogMetadata() throws Exception {
@Test (timeout = 15000) @Test (timeout = 15000)
public void testListNodeInfo() throws Exception { public void testListNodeInfo() throws Exception {
String remoteLogRootDir = "target/logs/"; String remoteLogRootDir = "target/logs/";
Configuration configuration = new YarnConfiguration(); conf.setBoolean(YarnConfiguration.LOG_AGGREGATION_ENABLED, true);
configuration.setBoolean(YarnConfiguration.LOG_AGGREGATION_ENABLED, true); conf
configuration
.set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR, remoteLogRootDir); .set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR, remoteLogRootDir);
configuration.setBoolean(YarnConfiguration.YARN_ACL_ENABLE, true); conf.setBoolean(YarnConfiguration.YARN_ACL_ENABLE, true);
configuration.set(YarnConfiguration.YARN_ADMIN_ACL, "admin"); conf.set(YarnConfiguration.YARN_ADMIN_ACL, "admin");
ApplicationId appId = ApplicationId.newInstance(0, 1); ApplicationId appId = ApplicationId.newInstance(0, 1);
ApplicationAttemptId appAttemptId = ApplicationAttemptId appAttemptId =
@ -1482,16 +1477,16 @@ public void testListNodeInfo() throws Exception {
nodeIds.add(nodeId2); nodeIds.add(nodeId2);
String rootLogDir = "target/LocalLogs"; String rootLogDir = "target/LocalLogs";
FileSystem fs = FileSystem.get(configuration); FileSystem fs = FileSystem.get(conf);
createContainerLogs(configuration, remoteLogRootDir, rootLogDir, fs, createContainerLogs(conf, remoteLogRootDir, rootLogDir, fs,
appId, containerIds, nodeIds); appId, containerIds, nodeIds);
YarnClient mockYarnClient = YarnClient mockYarnClient =
createMockYarnClient(YarnApplicationState.FINISHED, createMockYarnClient(YarnApplicationState.FINISHED,
UserGroupInformation.getCurrentUser().getShortUserName()); UserGroupInformation.getCurrentUser().getShortUserName());
LogsCLI cli = new LogsCLIForTest(mockYarnClient); LogsCLI cli = new LogsCLIForTest(mockYarnClient);
cli.setConf(configuration); cli.setConf(conf);
cli.run(new String[] { "-applicationId", appId.toString(), cli.run(new String[] { "-applicationId", appId.toString(),
"-list_nodes" }); "-list_nodes" });
@ -1508,13 +1503,12 @@ public void testListNodeInfo() throws Exception {
@Test (timeout = 15000) @Test (timeout = 15000)
public void testFetchApplictionLogsHar() throws Exception { public void testFetchApplictionLogsHar() throws Exception {
String remoteLogRootDir = "target/logs/"; String remoteLogRootDir = "target/logs/";
Configuration configuration = new YarnConfiguration(); conf.setBoolean(YarnConfiguration.LOG_AGGREGATION_ENABLED, true);
configuration.setBoolean(YarnConfiguration.LOG_AGGREGATION_ENABLED, true); conf
configuration
.set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR, remoteLogRootDir); .set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR, remoteLogRootDir);
configuration.setBoolean(YarnConfiguration.YARN_ACL_ENABLE, true); conf.setBoolean(YarnConfiguration.YARN_ACL_ENABLE, true);
configuration.set(YarnConfiguration.YARN_ADMIN_ACL, "admin"); conf.set(YarnConfiguration.YARN_ADMIN_ACL, "admin");
FileSystem fs = FileSystem.get(configuration); FileSystem fs = FileSystem.get(conf);
UserGroupInformation ugi = UserGroupInformation.getCurrentUser(); UserGroupInformation ugi = UserGroupInformation.getCurrentUser();
URL harUrl = ClassLoader.getSystemClassLoader() URL harUrl = ClassLoader.getSystemClassLoader()
.getResource("application_1440536969523_0001.har"); .getResource("application_1440536969523_0001.har");
@ -1534,7 +1528,7 @@ public void testFetchApplictionLogsHar() throws Exception {
createMockYarnClient(YarnApplicationState.FINISHED, createMockYarnClient(YarnApplicationState.FINISHED,
ugi.getShortUserName()); ugi.getShortUserName());
LogsCLI cli = new LogsCLIForTest(mockYarnClient); LogsCLI cli = new LogsCLIForTest(mockYarnClient);
cli.setConf(configuration); cli.setConf(conf);
int exitCode = cli.run(new String[]{"-applicationId", int exitCode = cli.run(new String[]{"-applicationId",
"application_1440536969523_0001"}); "application_1440536969523_0001"});
assertTrue(exitCode == 0); assertTrue(exitCode == 0);

View File

@ -15,7 +15,7 @@
* See the License for the specific language governing permissions and * See the License for the specific language governing permissions and
* limitations under the License. * limitations under the License.
*/ */
package org.apache.hadoop.yarn.server.webapp; package org.apache.hadoop.yarn.webapp.util;
import java.io.IOException; import java.io.IOException;
import java.net.HttpURLConnection; import java.net.HttpURLConnection;
@ -34,12 +34,11 @@
import com.sun.jersey.client.urlconnection.URLConnectionClientHandler; import com.sun.jersey.client.urlconnection.URLConnectionClientHandler;
/** /**
* Utility for handling Web client. * Utility for handling Web client.
* *
*/ */
public class WebServiceClient { public class WebServiceClient {
@VisibleForTesting private static SSLFactory sslFactory = null;
protected static SSLFactory sslFactory = null;
private static volatile WebServiceClient instance = null; private static volatile WebServiceClient instance = null;
private static boolean isHttps = false; private static boolean isHttps = false;
@ -68,6 +67,11 @@ public static WebServiceClient getWebServiceClient() {
return instance; return instance;
} }
@VisibleForTesting
SSLFactory getSSLFactory() {
return sslFactory;
}
/** /**
* Start SSL factory. * Start SSL factory.
* *

View File

@ -15,7 +15,7 @@
* See the License for the specific language governing permissions and * See the License for the specific language governing permissions and
* limitations under the License. * limitations under the License.
*/ */
package org.apache.hadoop.yarn.server.webapp; package org.apache.hadoop.yarn.webapp.util;
import java.io.File; import java.io.File;
import java.net.HttpURLConnection; import java.net.HttpURLConnection;
@ -37,7 +37,7 @@ public class TestWebServiceClient {
private static final String BASEDIR = System.getProperty("test.build.dir", private static final String BASEDIR = System.getProperty("test.build.dir",
"target/test-dir") + "/" + TestWebServiceClient.class.getSimpleName(); "target/test-dir") + "/" + TestWebServiceClient.class.getSimpleName();
static final String SSL_SERVER_KEYSTORE_PROP_PREFIX = "ssl.server.keystore"; static final String SSL_SERVER_KEYSTORE_PROP_PREFIX = "ssl.server.keystore";
static final String SSL_SERVER_TRUSTSTORE_PROP_PREFIX = static final String SSL_SERVER_TRUSTSTORE_PROP_PREFIX =
"ssl.server.truststore"; "ssl.server.truststore";
static final String SERVLET_NAME_ECHO = "echo"; static final String SERVLET_NAME_ECHO = "echo";
static final String SERVLET_PATH_ECHO = "/" + SERVLET_NAME_ECHO; static final String SERVLET_PATH_ECHO = "/" + SERVLET_NAME_ECHO;
@ -47,7 +47,8 @@ public void testGetWebServiceClient() throws Exception {
Configuration conf = new Configuration(); Configuration conf = new Configuration();
conf.set(YarnConfiguration.YARN_HTTP_POLICY_KEY, "HTTPS_ONLY"); conf.set(YarnConfiguration.YARN_HTTP_POLICY_KEY, "HTTPS_ONLY");
WebServiceClient.initialize(conf); WebServiceClient.initialize(conf);
Assert.assertNotNull(WebServiceClient.sslFactory); WebServiceClient client = WebServiceClient.getWebServiceClient();
Assert.assertNotNull(client.getSSLFactory());
WebServiceClient.destroy(); WebServiceClient.destroy();
} }

View File

@ -36,11 +36,11 @@
import org.apache.hadoop.yarn.server.router.clientrm.RouterClientRMService; import org.apache.hadoop.yarn.server.router.clientrm.RouterClientRMService;
import org.apache.hadoop.yarn.server.router.rmadmin.RouterRMAdminService; import org.apache.hadoop.yarn.server.router.rmadmin.RouterRMAdminService;
import org.apache.hadoop.yarn.server.router.webapp.RouterWebApp; import org.apache.hadoop.yarn.server.router.webapp.RouterWebApp;
import org.apache.hadoop.yarn.server.webapp.WebServiceClient;
import org.apache.hadoop.yarn.webapp.WebApp; import org.apache.hadoop.yarn.webapp.WebApp;
import org.apache.hadoop.yarn.webapp.WebApps; import org.apache.hadoop.yarn.webapp.WebApps;
import org.apache.hadoop.yarn.webapp.WebApps.Builder; import org.apache.hadoop.yarn.webapp.WebApps.Builder;
import org.apache.hadoop.yarn.webapp.util.WebAppUtils; import org.apache.hadoop.yarn.webapp.util.WebAppUtils;
import org.apache.hadoop.yarn.webapp.util.WebServiceClient;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;

View File

@ -50,10 +50,10 @@
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.NodeInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.NodeInfo;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.NodesInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.NodesInfo;
import org.apache.hadoop.yarn.server.uam.UnmanagedApplicationManager; import org.apache.hadoop.yarn.server.uam.UnmanagedApplicationManager;
import org.apache.hadoop.yarn.server.webapp.WebServiceClient;
import org.apache.hadoop.yarn.webapp.BadRequestException; import org.apache.hadoop.yarn.webapp.BadRequestException;
import org.apache.hadoop.yarn.webapp.ForbiddenException; import org.apache.hadoop.yarn.webapp.ForbiddenException;
import org.apache.hadoop.yarn.webapp.NotFoundException; import org.apache.hadoop.yarn.webapp.NotFoundException;
import org.apache.hadoop.yarn.webapp.util.WebServiceClient;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;