HADOOP-12847. hadoop daemonlog should support https and SPNEGO for Kerberized cluster. (Wei-Chiu Chuang via Yongjun Zhang)
This commit is contained in:
Yongjun Zhang
parent
ef3e0c6fd6
commit
41c02757a5
|
|
@ -17,20 +17,38 @@
|
|||
*/
|
||||
package org.apache.hadoop.log;
|
||||
|
||||
import java.io.*;
|
||||
import java.net.*;
|
||||
import java.io.BufferedReader;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStreamReader;
|
||||
import java.io.PrintWriter;
|
||||
import java.net.URL;
|
||||
import java.net.URLConnection;
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
import javax.servlet.*;
|
||||
import javax.servlet.http.*;
|
||||
import javax.net.ssl.HttpsURLConnection;
|
||||
import javax.net.ssl.SSLSocketFactory;
|
||||
import javax.servlet.ServletException;
|
||||
import javax.servlet.http.HttpServlet;
|
||||
import javax.servlet.http.HttpServletRequest;
|
||||
import javax.servlet.http.HttpServletResponse;
|
||||
|
||||
import com.google.common.annotations.VisibleForTesting;
|
||||
import com.google.common.base.Charsets;
|
||||
import org.apache.commons.logging.*;
|
||||
import org.apache.commons.logging.impl.*;
|
||||
import org.apache.commons.logging.Log;
|
||||
import org.apache.commons.logging.LogFactory;
|
||||
import org.apache.commons.logging.impl.Jdk14Logger;
|
||||
import org.apache.commons.logging.impl.Log4JLogger;
|
||||
import org.apache.hadoop.HadoopIllegalArgumentException;
|
||||
import org.apache.hadoop.classification.InterfaceAudience;
|
||||
import org.apache.hadoop.classification.InterfaceStability;
|
||||
import org.apache.hadoop.conf.Configuration;
|
||||
import org.apache.hadoop.conf.Configured;
|
||||
import org.apache.hadoop.http.HttpServer2;
|
||||
import org.apache.hadoop.security.authentication.client.AuthenticatedURL;
|
||||
import org.apache.hadoop.security.authentication.client.KerberosAuthenticator;
|
||||
import org.apache.hadoop.security.ssl.SSLFactory;
|
||||
import org.apache.hadoop.util.ServletUtil;
|
||||
import org.apache.hadoop.util.Tool;
|
||||
|
||||
/**
|
||||
* Change log level in runtime.
|
||||
|
|
@ -38,43 +56,252 @@ import org.apache.hadoop.util.ServletUtil;
|
|||
@InterfaceStability.Evolving
|
||||
public class LogLevel {
|
||||
public static final String USAGES = "\nUsage: General options are:\n"
|
||||
+ "\t[-getlevel <host:httpPort> <name>]\n"
|
||||
+ "\t[-setlevel <host:httpPort> <name> <level>]\n";
|
||||
+ "\t[-getlevel <host:port> <classname> [-protocol (http|https)]\n"
|
||||
+ "\t[-setlevel <host:port> <classname> <level> "
|
||||
+ "[-protocol (http|https)]\n";
|
||||
|
||||
public static final String PROTOCOL_HTTP = "http";
|
||||
public static final String PROTOCOL_HTTPS = "https";
|
||||
/**
|
||||
* A command line implementation
|
||||
*/
|
||||
public static void main(String[] args) {
|
||||
if (args.length == 3 && "-getlevel".equals(args[0])) {
|
||||
process("http://" + args[1] + "/logLevel?log=" + args[2]);
|
||||
return;
|
||||
}
|
||||
else if (args.length == 4 && "-setlevel".equals(args[0])) {
|
||||
process("http://" + args[1] + "/logLevel?log=" + args[2]
|
||||
+ "&level=" + args[3]);
|
||||
return;
|
||||
}
|
||||
|
||||
System.err.println(USAGES);
|
||||
System.exit(-1);
|
||||
public static void main(String[] args) throws Exception {
|
||||
CLI cli = new CLI(new Configuration());
|
||||
System.exit(cli.run(args));
|
||||
}
|
||||
|
||||
private static void process(String urlstring) {
|
||||
try {
|
||||
URL url = new URL(urlstring);
|
||||
System.out.println("Connecting to " + url);
|
||||
URLConnection connection = url.openConnection();
|
||||
connection.connect();
|
||||
/**
|
||||
* Valid command line options.
|
||||
*/
|
||||
private enum Operations {
|
||||
GETLEVEL,
|
||||
SETLEVEL,
|
||||
UNKNOWN
|
||||
}
|
||||
|
||||
BufferedReader in = new BufferedReader(new InputStreamReader(
|
||||
connection.getInputStream(), Charsets.UTF_8));
|
||||
for(String line; (line = in.readLine()) != null; )
|
||||
private static void printUsage() {
|
||||
System.err.println(USAGES);
|
||||
}
|
||||
|
||||
public static boolean isValidProtocol(String protocol) {
|
||||
return ((protocol.equals(PROTOCOL_HTTP) ||
|
||||
protocol.equals(PROTOCOL_HTTPS)));
|
||||
}
|
||||
|
||||
@VisibleForTesting
|
||||
static class CLI extends Configured implements Tool {
|
||||
private Operations operation = Operations.UNKNOWN;
|
||||
private String protocol;
|
||||
private String hostName;
|
||||
private String className;
|
||||
private String level;
|
||||
|
||||
CLI(Configuration conf) {
|
||||
setConf(conf);
|
||||
}
|
||||
|
||||
@Override
|
||||
public int run(String[] args) throws Exception {
|
||||
try {
|
||||
parseArguments(args);
|
||||
sendLogLevelRequest();
|
||||
} catch (HadoopIllegalArgumentException e) {
|
||||
printUsage();
|
||||
throw e;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
/**
|
||||
* Send HTTP/HTTPS request to the daemon.
|
||||
* @throws HadoopIllegalArgumentException if arguments are invalid.
|
||||
* @throws Exception if unable to connect
|
||||
*/
|
||||
private void sendLogLevelRequest()
|
||||
throws HadoopIllegalArgumentException, Exception {
|
||||
switch (operation) {
|
||||
case GETLEVEL:
|
||||
doGetLevel();
|
||||
break;
|
||||
case SETLEVEL:
|
||||
doSetLevel();
|
||||
break;
|
||||
default:
|
||||
throw new HadoopIllegalArgumentException(
|
||||
"Expect either -getlevel or -setlevel");
|
||||
}
|
||||
}
|
||||
|
||||
public void parseArguments(String[] args) throws
|
||||
HadoopIllegalArgumentException {
|
||||
if (args.length == 0) {
|
||||
throw new HadoopIllegalArgumentException("No arguments specified");
|
||||
}
|
||||
int nextArgIndex = 0;
|
||||
while (nextArgIndex < args.length) {
|
||||
if (args[nextArgIndex].equals("-getlevel")) {
|
||||
nextArgIndex = parseGetLevelArgs(args, nextArgIndex);
|
||||
} else if (args[nextArgIndex].equals("-setlevel")) {
|
||||
nextArgIndex = parseSetLevelArgs(args, nextArgIndex);
|
||||
} else if (args[nextArgIndex].equals("-protocol")) {
|
||||
nextArgIndex = parseProtocolArgs(args, nextArgIndex);
|
||||
} else {
|
||||
throw new HadoopIllegalArgumentException(
|
||||
"Unexpected argument " + args[nextArgIndex]);
|
||||
}
|
||||
}
|
||||
|
||||
// if operation is never specified in the arguments
|
||||
if (operation == Operations.UNKNOWN) {
|
||||
throw new HadoopIllegalArgumentException(
|
||||
"Must specify either -getlevel or -setlevel");
|
||||
}
|
||||
|
||||
// if protocol is unspecified, set it as http.
|
||||
if (protocol == null) {
|
||||
protocol = PROTOCOL_HTTP;
|
||||
}
|
||||
}
|
||||
|
||||
private int parseGetLevelArgs(String[] args, int index) throws
|
||||
HadoopIllegalArgumentException {
|
||||
// fail if multiple operations are specified in the arguments
|
||||
if (operation != Operations.UNKNOWN) {
|
||||
throw new HadoopIllegalArgumentException(
|
||||
"Redundant -getlevel command");
|
||||
}
|
||||
// check number of arguments is sufficient
|
||||
if (index+2 >= args.length) {
|
||||
throw new HadoopIllegalArgumentException(
|
||||
"-getlevel needs two parameters");
|
||||
}
|
||||
operation = Operations.GETLEVEL;
|
||||
hostName = args[index+1];
|
||||
className = args[index+2];
|
||||
return index+3;
|
||||
}
|
||||
|
||||
private int parseSetLevelArgs(String[] args, int index) throws
|
||||
HadoopIllegalArgumentException {
|
||||
// fail if multiple operations are specified in the arguments
|
||||
if (operation != Operations.UNKNOWN) {
|
||||
throw new HadoopIllegalArgumentException(
|
||||
"Redundant -setlevel command");
|
||||
}
|
||||
// check number of arguments is sufficient
|
||||
if (index+3 >= args.length) {
|
||||
throw new HadoopIllegalArgumentException(
|
||||
"-setlevel needs three parameters");
|
||||
}
|
||||
operation = Operations.SETLEVEL;
|
||||
hostName = args[index+1];
|
||||
className = args[index+2];
|
||||
level = args[index+3];
|
||||
return index+4;
|
||||
}
|
||||
|
||||
private int parseProtocolArgs(String[] args, int index) throws
|
||||
HadoopIllegalArgumentException {
|
||||
// make sure only -protocol is specified
|
||||
if (protocol != null) {
|
||||
throw new HadoopIllegalArgumentException(
|
||||
"Redundant -protocol command");
|
||||
}
|
||||
// check number of arguments is sufficient
|
||||
if (index+1 >= args.length) {
|
||||
throw new HadoopIllegalArgumentException(
|
||||
"-protocol needs one parameter");
|
||||
}
|
||||
// check protocol is valid
|
||||
protocol = args[index+1];
|
||||
if (!isValidProtocol(protocol)) {
|
||||
throw new HadoopIllegalArgumentException(
|
||||
"Invalid protocol: " + protocol);
|
||||
}
|
||||
return index+2;
|
||||
}
|
||||
|
||||
/**
|
||||
* Send HTTP/HTTPS request to get log level.
|
||||
*
|
||||
* @throws HadoopIllegalArgumentException if arguments are invalid.
|
||||
* @throws Exception if unable to connect
|
||||
*/
|
||||
private void doGetLevel() throws Exception {
|
||||
process(protocol + "://" + hostName + "/logLevel?log=" + className);
|
||||
}
|
||||
|
||||
/**
|
||||
* Send HTTP/HTTPS request to set log level.
|
||||
*
|
||||
* @throws HadoopIllegalArgumentException if arguments are invalid.
|
||||
* @throws Exception if unable to connect
|
||||
*/
|
||||
private void doSetLevel() throws Exception {
|
||||
process(protocol + "://" + hostName + "/logLevel?log=" + className
|
||||
+ "&level=" + level);
|
||||
}
|
||||
|
||||
/**
|
||||
* Connect to the URL. Supports HTTP/HTTPS and supports SPNEGO
|
||||
* authentication. It falls back to simple authentication if it fails to
|
||||
* initiate SPNEGO.
|
||||
*
|
||||
* @param url the URL address of the daemon servlet
|
||||
* @return a connected connection
|
||||
* @throws Exception if it can not establish a connection.
|
||||
*/
|
||||
private URLConnection connect(URL url) throws Exception {
|
||||
AuthenticatedURL.Token token = new AuthenticatedURL.Token();
|
||||
AuthenticatedURL aUrl;
|
||||
SSLFactory clientSslFactory;
|
||||
URLConnection connection;
|
||||
// If https is chosen, configures SSL client.
|
||||
if (PROTOCOL_HTTPS.equals(url.getProtocol())) {
|
||||
clientSslFactory = new SSLFactory(
|
||||
SSLFactory.Mode.CLIENT, this.getConf());
|
||||
clientSslFactory.init();
|
||||
SSLSocketFactory sslSocketF = clientSslFactory.createSSLSocketFactory();
|
||||
|
||||
aUrl = new AuthenticatedURL(
|
||||
new KerberosAuthenticator(), clientSslFactory);
|
||||
connection = aUrl.openConnection(url, token);
|
||||
HttpsURLConnection httpsConn = (HttpsURLConnection) connection;
|
||||
httpsConn.setSSLSocketFactory(sslSocketF);
|
||||
} else {
|
||||
aUrl = new AuthenticatedURL(new KerberosAuthenticator());
|
||||
connection = aUrl.openConnection(url, token);
|
||||
}
|
||||
|
||||
connection.connect();
|
||||
return connection;
|
||||
}
|
||||
|
||||
/**
|
||||
* Configures the client to send HTTP/HTTPS request to the URL.
|
||||
* Supports SPENGO for authentication.
|
||||
* @param urlString URL and query string to the daemon's web UI
|
||||
* @throws Exception if unable to connect
|
||||
*/
|
||||
private void process(String urlString) throws Exception {
|
||||
URL url = new URL(urlString);
|
||||
System.out.println("Connecting to " + url);
|
||||
|
||||
URLConnection connection = connect(url);
|
||||
|
||||
// read from the servlet
|
||||
BufferedReader in = new BufferedReader(
|
||||
new InputStreamReader(connection.getInputStream(), Charsets.UTF_8));
|
||||
for (String line;;) {
|
||||
line = in.readLine();
|
||||
if (line == null) {
|
||||
break;
|
||||
}
|
||||
if (line.startsWith(MARKER)) {
|
||||
System.out.println(TAG.matcher(line).replaceAll(""));
|
||||
}
|
||||
}
|
||||
in.close();
|
||||
} catch (IOException ioe) {
|
||||
System.err.println("" + ioe);
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -164,14 +164,34 @@ Commands useful for administrators of a hadoop cluster.
|
|||
|
||||
Usage:
|
||||
|
||||
hadoop daemonlog -getlevel <host:httpport> <classname>
|
||||
hadoop daemonlog -setlevel <host:httpport> <classname> <level>
|
||||
hadoop daemonlog -getlevel <host:port> <classname> [-protocol (http|https)]
|
||||
hdoop daemonlog -setlevel <host:port> <classname> <level> [-protocol (http|https)]
|
||||
|
||||
| COMMAND\_OPTION | Description |
|
||||
|:---- |:---- |
|
||||
| `-getlevel` *host:httpport* *classname* | Prints the log level of the log identified by a qualified *classname*, in the daemon running at *host:httpport*. This command internally connects to `http://<host:httpport>/logLevel?log=<classname>` |
|
||||
| `-setlevel` *host:httpport* *classname* *level* | Sets the log level of the log identified by a qualified *classname*, in the daemon running at *host:httpport*. This command internally connects to `http://<host:httpport>/logLevel?log=<classname>&level=<level>` |
|
||||
| `-getlevel` *host:port* *classname* [-protocol (http|https)] | Prints the log level of the log identified by a qualified *classname*, in the daemon running at *host:port*. The `-protocol` flag specifies the protocol for connection. |
|
||||
| `-setlevel` *host:port* *classname* *level* [-protocol (http|https)] | Sets the log level of the log identified by a qualified *classname*, in the daemon running at *host:port*. The `-protocol` flag specifies the protocol for connection. |
|
||||
|
||||
Get/Set the log level for a Log identified by a qualified class name in the daemon.
|
||||
|
||||
Example: $ bin/hadoop daemonlog -setlevel 127.0.0.1:50070 org.apache.hadoop.hdfs.server.namenode.NameNode DEBUG
|
||||
Get/Set the log level for a Log identified by a qualified class name in the daemon dynamically.
|
||||
By default, the command sends a HTTP request, but this can be overridden by using argument `-protocol https` to send a HTTPS request.
|
||||
|
||||
Example:
|
||||
|
||||
$ bin/hadoop daemonlog -setlevel 127.0.0.1:50070 org.apache.hadoop.hdfs.server.namenode.NameNode DEBUG
|
||||
$ bin/hadoop daemonlog -getlevel 127.0.0.1:50470 org.apache.hadoop.hdfs.server.namenode.NameNode DEBUG -protocol https
|
||||
|
||||
Note that the setting is not permanent and will be reset when the daemon is restarted.
|
||||
This command works by sending a HTTP/HTTPS request to the daemon's internal Jetty servlet, so it supports the following daemons:
|
||||
|
||||
* HDFS
|
||||
* name node
|
||||
* secondary name node
|
||||
* data node
|
||||
* journal node
|
||||
* YARN
|
||||
* resource manager
|
||||
* node manager
|
||||
* Timeline server
|
||||
|
||||
However, the command does not support KMS server, because its web interface is based on Tomcat, which does not support the servlet.
|
||||
|
|
|
|||
|
|
@ -17,99 +17,404 @@
|
|||
*/
|
||||
package org.apache.hadoop.log;
|
||||
|
||||
import java.io.*;
|
||||
import java.net.*;
|
||||
import java.io.File;
|
||||
import java.net.SocketException;
|
||||
import java.net.URI;
|
||||
import java.util.concurrent.Callable;
|
||||
|
||||
import org.apache.commons.logging.Log;
|
||||
import org.apache.commons.logging.LogFactory;
|
||||
import org.apache.commons.logging.impl.Log4JLogger;
|
||||
import org.apache.hadoop.HadoopIllegalArgumentException;
|
||||
import org.apache.hadoop.conf.Configuration;
|
||||
import org.apache.hadoop.fs.CommonConfigurationKeys;
|
||||
import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
|
||||
import org.apache.hadoop.fs.FileUtil;
|
||||
import org.apache.hadoop.http.HttpServer2;
|
||||
import org.apache.hadoop.log.LogLevel.CLI;
|
||||
import org.apache.hadoop.minikdc.KerberosSecurityTestcase;
|
||||
import org.apache.hadoop.net.NetUtils;
|
||||
|
||||
import junit.framework.TestCase;
|
||||
|
||||
import org.apache.commons.logging.*;
|
||||
import org.apache.commons.logging.impl.*;
|
||||
import org.apache.log4j.*;
|
||||
import org.apache.hadoop.security.UserGroupInformation;
|
||||
import org.apache.hadoop.security.authentication.KerberosTestUtils;
|
||||
import org.apache.hadoop.security.authentication.client.KerberosAuthenticator;
|
||||
import org.apache.hadoop.security.authorize.AccessControlList;
|
||||
import org.apache.hadoop.security.ssl.KeyStoreTestUtil;
|
||||
import org.junit.Assert;
|
||||
|
||||
public class TestLogLevel extends TestCase {
|
||||
static final PrintStream out = System.out;
|
||||
import org.apache.hadoop.test.GenericTestUtils;
|
||||
import org.apache.log4j.Level;
|
||||
import org.apache.log4j.Logger;
|
||||
import org.junit.AfterClass;
|
||||
import org.junit.Before;
|
||||
import org.junit.BeforeClass;
|
||||
import org.junit.Test;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
public void testDynamicLogLevel() throws Exception {
|
||||
String logName = TestLogLevel.class.getName();
|
||||
Log testlog = LogFactory.getLog(logName);
|
||||
import javax.net.ssl.SSLException;
|
||||
|
||||
//only test Log4JLogger
|
||||
if (testlog instanceof Log4JLogger) {
|
||||
Logger log = ((Log4JLogger)testlog).getLogger();
|
||||
log.debug("log.debug1");
|
||||
log.info("log.info1");
|
||||
log.error("log.error1");
|
||||
Assert.assertNotEquals("Get default Log Level which shouldn't be ERROR.",
|
||||
Level.ERROR, log.getEffectiveLevel());
|
||||
import static org.junit.Assert.assertEquals;
|
||||
import static org.junit.Assert.assertFalse;
|
||||
import static org.junit.Assert.assertTrue;
|
||||
import static org.junit.Assert.fail;
|
||||
|
||||
HttpServer2 server = new HttpServer2.Builder().setName("..")
|
||||
.addEndpoint(new URI("http://localhost:0")).setFindPort(true)
|
||||
.build();
|
||||
/**
|
||||
* Test LogLevel.
|
||||
*/
|
||||
public class TestLogLevel extends KerberosSecurityTestcase {
|
||||
private static final File BASEDIR = GenericTestUtils.getRandomizedTestDir();
|
||||
private static String keystoresDir;
|
||||
private static String sslConfDir;
|
||||
private static Configuration conf;
|
||||
private static Configuration sslConf;
|
||||
private final String logName = TestLogLevel.class.getName();
|
||||
private String clientPrincipal;
|
||||
private String serverPrincipal;
|
||||
private final Log testlog = LogFactory.getLog(logName);
|
||||
private final Logger log = ((Log4JLogger)testlog).getLogger();
|
||||
private final static String PRINCIPAL = "loglevel.principal";
|
||||
private final static String KEYTAB = "loglevel.keytab";
|
||||
|
||||
server.start();
|
||||
String authority = NetUtils.getHostPortString(server
|
||||
.getConnectorAddress(0));
|
||||
|
||||
//servlet
|
||||
URL url = new URL("http://" + authority + "/logLevel?log=" + logName
|
||||
+ "&level=" + Level.ERROR);
|
||||
out.println("*** Connecting to " + url);
|
||||
URLConnection connection = url.openConnection();
|
||||
connection.connect();
|
||||
|
||||
BufferedReader in = new BufferedReader(new InputStreamReader(
|
||||
connection.getInputStream()));
|
||||
for(String line; (line = in.readLine()) != null; out.println(line));
|
||||
in.close();
|
||||
|
||||
log.debug("log.debug2");
|
||||
log.info("log.info2");
|
||||
log.error("log.error2");
|
||||
assertEquals("Try setting log level: ERROR from servlet.", Level.ERROR,
|
||||
log.getEffectiveLevel());
|
||||
|
||||
//command line
|
||||
String[] args = {"-setlevel", authority, logName, Level.DEBUG.toString()};
|
||||
LogLevel.main(args);
|
||||
log.debug("log.debug3");
|
||||
log.info("log.info3");
|
||||
log.error("log.error3");
|
||||
assertEquals("Try setting log level: DEBUG via command line", Level.DEBUG,
|
||||
log.getEffectiveLevel());
|
||||
|
||||
// Test mixed upper case and lower case in level string.
|
||||
String[] args2 = {"-setlevel", authority, logName, "Info"};
|
||||
LogLevel.main(args2);
|
||||
log.debug("log.debug4");
|
||||
log.info("log.info4");
|
||||
log.error("log.error4");
|
||||
assertEquals("Try setting log level: Info via command line.", Level.INFO,
|
||||
log.getEffectiveLevel());
|
||||
|
||||
// Test "Error" instead of "ERROR" should work for servlet
|
||||
URL newUrl = new URL("http://" + authority + "/logLevel?log=" + logName
|
||||
+ "&level=" + "Error");
|
||||
out.println("*** Connecting to " + newUrl);
|
||||
connection = newUrl.openConnection();
|
||||
connection.connect();
|
||||
|
||||
BufferedReader in2 = new BufferedReader(new InputStreamReader(
|
||||
connection.getInputStream()));
|
||||
for(String line; (line = in2.readLine()) != null; out.println(line));
|
||||
in2.close();
|
||||
|
||||
log.debug("log.debug5");
|
||||
log.info("log.info5");
|
||||
log.error("log.error5");
|
||||
assertEquals("Try setting log level: Error via servlet.", Level.ERROR,
|
||||
log.getEffectiveLevel());
|
||||
@BeforeClass
|
||||
public static void setUp() throws Exception {
|
||||
org.slf4j.Logger logger =
|
||||
LoggerFactory.getLogger(KerberosAuthenticator.class);
|
||||
GenericTestUtils.setLogLevel(logger, Level.DEBUG);
|
||||
FileUtil.fullyDelete(BASEDIR);
|
||||
if (!BASEDIR.mkdirs()) {
|
||||
throw new Exception("unable to create the base directory for testing");
|
||||
}
|
||||
else {
|
||||
out.println(testlog.getClass() + " not tested.");
|
||||
conf = new Configuration();
|
||||
|
||||
setupSSL(BASEDIR);
|
||||
}
|
||||
|
||||
static private void setupSSL(File base) throws Exception {
|
||||
keystoresDir = base.getAbsolutePath();
|
||||
sslConfDir = KeyStoreTestUtil.getClasspathDir(TestLogLevel.class);
|
||||
KeyStoreTestUtil.setupSSLConfig(keystoresDir, sslConfDir, conf, false);
|
||||
|
||||
sslConf = KeyStoreTestUtil.getSslConfig();
|
||||
}
|
||||
|
||||
@Before
|
||||
public void setupKerberos() throws Exception {
|
||||
File keytabFile = new File(KerberosTestUtils.getKeytabFile());
|
||||
clientPrincipal = KerberosTestUtils.getClientPrincipal();
|
||||
serverPrincipal = KerberosTestUtils.getServerPrincipal();
|
||||
clientPrincipal = clientPrincipal.substring(0,
|
||||
clientPrincipal.lastIndexOf("@"));
|
||||
serverPrincipal = serverPrincipal.substring(0,
|
||||
serverPrincipal.lastIndexOf("@"));
|
||||
getKdc().createPrincipal(keytabFile, clientPrincipal, serverPrincipal);
|
||||
}
|
||||
|
||||
@AfterClass
|
||||
public static void tearDown() throws Exception {
|
||||
KeyStoreTestUtil.cleanupSSLConfig(keystoresDir, sslConfDir);
|
||||
FileUtil.fullyDelete(BASEDIR);
|
||||
}
|
||||
|
||||
/**
|
||||
* Test client command line options. Does not validate server behavior.
|
||||
* @throws Exception
|
||||
*/
|
||||
@Test(timeout=120000)
|
||||
public void testCommandOptions() throws Exception {
|
||||
final String className = this.getClass().getName();
|
||||
|
||||
assertFalse(validateCommand(new String[] {"-foo" }));
|
||||
// fail due to insufficient number of arguments
|
||||
assertFalse(validateCommand(new String[] {}));
|
||||
assertFalse(validateCommand(new String[] {"-getlevel" }));
|
||||
assertFalse(validateCommand(new String[] {"-setlevel" }));
|
||||
assertFalse(validateCommand(new String[] {"-getlevel", "foo.bar:8080" }));
|
||||
|
||||
// valid command arguments
|
||||
assertTrue(validateCommand(
|
||||
new String[] {"-getlevel", "foo.bar:8080", className }));
|
||||
assertTrue(validateCommand(
|
||||
new String[] {"-setlevel", "foo.bar:8080", className, "DEBUG" }));
|
||||
assertTrue(validateCommand(
|
||||
new String[] {"-getlevel", "foo.bar:8080", className, "-protocol",
|
||||
"http" }));
|
||||
assertTrue(validateCommand(
|
||||
new String[] {"-getlevel", "foo.bar:8080", className, "-protocol",
|
||||
"https" }));
|
||||
assertTrue(validateCommand(
|
||||
new String[] {"-setlevel", "foo.bar:8080", className, "DEBUG",
|
||||
"-protocol", "http" }));
|
||||
assertTrue(validateCommand(
|
||||
new String[] {"-setlevel", "foo.bar:8080", className, "DEBUG",
|
||||
"-protocol", "https" }));
|
||||
|
||||
// fail due to the extra argument
|
||||
assertFalse(validateCommand(
|
||||
new String[] {"-getlevel", "foo.bar:8080", className, "-protocol",
|
||||
"https", "blah" }));
|
||||
assertFalse(validateCommand(
|
||||
new String[] {"-setlevel", "foo.bar:8080", className, "DEBUG",
|
||||
"-protocol", "https", "blah" }));
|
||||
assertFalse(validateCommand(
|
||||
new String[] {"-getlevel", "foo.bar:8080", className, "-protocol",
|
||||
"https", "-protocol", "https" }));
|
||||
assertFalse(validateCommand(
|
||||
new String[] {"-getlevel", "foo.bar:8080", className,
|
||||
"-setlevel", "foo.bar:8080", className }));
|
||||
}
|
||||
|
||||
/**
|
||||
* Check to see if a command can be accepted.
|
||||
*
|
||||
* @param args a String array of arguments
|
||||
* @return true if the command can be accepted, false if not.
|
||||
*/
|
||||
private boolean validateCommand(String[] args) throws Exception {
|
||||
CLI cli = new CLI(sslConf);
|
||||
try {
|
||||
cli.parseArguments(args);
|
||||
} catch (HadoopIllegalArgumentException e) {
|
||||
return false;
|
||||
} catch (Exception e) {
|
||||
// this is used to verify the command arguments only.
|
||||
// no HadoopIllegalArgumentException = the arguments are good.
|
||||
return true;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
/**
|
||||
* Creates and starts a Jetty server binding at an ephemeral port to run
|
||||
* LogLevel servlet.
|
||||
* @param protocol "http" or "https"
|
||||
* @param isSpnego true if SPNEGO is enabled
|
||||
* @return a created HttpServer2 object
|
||||
* @throws Exception if unable to create or start a Jetty server
|
||||
*/
|
||||
private HttpServer2 createServer(String protocol, boolean isSpnego)
|
||||
throws Exception {
|
||||
HttpServer2.Builder builder = new HttpServer2.Builder()
|
||||
.setName("..")
|
||||
.addEndpoint(new URI(protocol + "://localhost:0"))
|
||||
.setFindPort(true)
|
||||
.setConf(conf);
|
||||
if (isSpnego) {
|
||||
// Set up server Kerberos credentials.
|
||||
// Since the server may fall back to simple authentication,
|
||||
// use ACL to make sure the connection is Kerberos/SPNEGO authenticated.
|
||||
builder.setSecurityEnabled(true)
|
||||
.setUsernameConfKey(PRINCIPAL)
|
||||
.setKeytabConfKey(KEYTAB)
|
||||
.setACL(new AccessControlList(clientPrincipal));
|
||||
}
|
||||
|
||||
// if using HTTPS, configure keystore/truststore properties.
|
||||
if (protocol.equals(LogLevel.PROTOCOL_HTTPS)) {
|
||||
builder = builder.
|
||||
keyPassword(sslConf.get("ssl.server.keystore.keypassword"))
|
||||
.keyStore(sslConf.get("ssl.server.keystore.location"),
|
||||
sslConf.get("ssl.server.keystore.password"),
|
||||
sslConf.get("ssl.server.keystore.type", "jks"))
|
||||
.trustStore(sslConf.get("ssl.server.truststore.location"),
|
||||
sslConf.get("ssl.server.truststore.password"),
|
||||
sslConf.get("ssl.server.truststore.type", "jks"));
|
||||
}
|
||||
HttpServer2 server = builder.build();
|
||||
// Enable SPNEGO for LogLevel servlet
|
||||
if (isSpnego) {
|
||||
server.addInternalServlet("logLevel", "/logLevel", LogLevel.Servlet.class,
|
||||
true);
|
||||
}
|
||||
server.start();
|
||||
return server;
|
||||
}
|
||||
|
||||
private void testDynamicLogLevel(final String bindProtocol,
|
||||
final String connectProtocol, final boolean isSpnego)
|
||||
throws Exception {
|
||||
testDynamicLogLevel(bindProtocol, connectProtocol, isSpnego,
|
||||
Level.DEBUG.toString());
|
||||
}
|
||||
|
||||
/**
|
||||
* Run both client and server using the given protocol.
|
||||
*
|
||||
* @param bindProtocol specify either http or https for server
|
||||
* @param connectProtocol specify either http or https for client
|
||||
* @param isSpnego true if SPNEGO is enabled
|
||||
* @throws Exception
|
||||
*/
|
||||
private void testDynamicLogLevel(final String bindProtocol,
|
||||
final String connectProtocol, final boolean isSpnego,
|
||||
final String newLevel) throws Exception {
|
||||
if (!LogLevel.isValidProtocol(bindProtocol)) {
|
||||
throw new Exception("Invalid server protocol " + bindProtocol);
|
||||
}
|
||||
if (!LogLevel.isValidProtocol(connectProtocol)) {
|
||||
throw new Exception("Invalid client protocol " + connectProtocol);
|
||||
}
|
||||
Level oldLevel = log.getEffectiveLevel();
|
||||
Assert.assertNotEquals("Get default Log Level which shouldn't be ERROR.",
|
||||
Level.ERROR, oldLevel);
|
||||
|
||||
// configs needed for SPNEGO at server side
|
||||
if (isSpnego) {
|
||||
conf.set(PRINCIPAL, KerberosTestUtils.getServerPrincipal());
|
||||
conf.set(KEYTAB, KerberosTestUtils.getKeytabFile());
|
||||
conf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION,
|
||||
"kerberos");
|
||||
conf.setBoolean(CommonConfigurationKeys.HADOOP_SECURITY_AUTHORIZATION,
|
||||
true);
|
||||
UserGroupInformation.setConfiguration(conf);
|
||||
}
|
||||
|
||||
final HttpServer2 server = createServer(bindProtocol, isSpnego);
|
||||
// get server port
|
||||
final String authority = NetUtils.getHostPortString(server
|
||||
.getConnectorAddress(0));
|
||||
|
||||
KerberosTestUtils.doAsClient(new Callable<Void>() {
|
||||
@Override
|
||||
public Void call() throws Exception {
|
||||
// client command line
|
||||
getLevel(connectProtocol, authority);
|
||||
setLevel(connectProtocol, authority, newLevel);
|
||||
return null;
|
||||
}
|
||||
});
|
||||
server.stop();
|
||||
// restore log level
|
||||
GenericTestUtils.setLogLevel(log, oldLevel);
|
||||
}
|
||||
|
||||
/**
|
||||
* Run LogLevel command line to start a client to get log level of this test
|
||||
* class.
|
||||
*
|
||||
* @param protocol specify either http or https
|
||||
* @param authority daemon's web UI address
|
||||
* @throws Exception if unable to connect
|
||||
*/
|
||||
private void getLevel(String protocol, String authority) throws Exception {
|
||||
String[] getLevelArgs = {"-getlevel", authority, logName, "-protocol",
|
||||
protocol};
|
||||
CLI cli = new CLI(sslConf);
|
||||
cli.run(getLevelArgs);
|
||||
}
|
||||
|
||||
/**
|
||||
* Run LogLevel command line to start a client to set log level of this test
|
||||
* class to debug.
|
||||
*
|
||||
* @param protocol specify either http or https
|
||||
* @param authority daemon's web UI address
|
||||
* @throws Exception if unable to run or log level does not change as expected
|
||||
*/
|
||||
private void setLevel(String protocol, String authority, String newLevel)
|
||||
throws Exception {
|
||||
String[] setLevelArgs = {"-setlevel", authority, logName,
|
||||
newLevel, "-protocol", protocol};
|
||||
CLI cli = new CLI(sslConf);
|
||||
cli.run(setLevelArgs);
|
||||
|
||||
assertEquals("new level not equal to expected: ", newLevel.toUpperCase(),
|
||||
log.getEffectiveLevel().toString());
|
||||
}
|
||||
|
||||
/**
|
||||
* Test setting log level to "Info".
|
||||
*
|
||||
* @throws Exception
|
||||
*/
|
||||
@Test(timeout=60000)
|
||||
public void testInfoLogLevel() throws Exception {
|
||||
testDynamicLogLevel(LogLevel.PROTOCOL_HTTP, LogLevel.PROTOCOL_HTTP, false,
|
||||
"Info");
|
||||
}
|
||||
|
||||
/**
|
||||
* Test setting log level to "Error".
|
||||
*
|
||||
* @throws Exception
|
||||
*/
|
||||
@Test(timeout=60000)
|
||||
public void testErrorLogLevel() throws Exception {
|
||||
testDynamicLogLevel(LogLevel.PROTOCOL_HTTP, LogLevel.PROTOCOL_HTTP, false,
|
||||
"Error");
|
||||
}
|
||||
|
||||
/**
|
||||
* Server runs HTTP, no SPNEGO.
|
||||
*
|
||||
* @throws Exception
|
||||
*/
|
||||
@Test(timeout=60000)
|
||||
public void testLogLevelByHttp() throws Exception {
|
||||
testDynamicLogLevel(LogLevel.PROTOCOL_HTTP, LogLevel.PROTOCOL_HTTP, false);
|
||||
try {
|
||||
testDynamicLogLevel(LogLevel.PROTOCOL_HTTP, LogLevel.PROTOCOL_HTTPS,
|
||||
false);
|
||||
fail("A HTTPS Client should not have succeeded in connecting to a " +
|
||||
"HTTP server");
|
||||
} catch (SSLException e) {
|
||||
GenericTestUtils.assertExceptionContains("Unrecognized SSL message", e);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Server runs HTTP + SPNEGO.
|
||||
*
|
||||
* @throws Exception
|
||||
*/
|
||||
@Test(timeout=60000)
|
||||
public void testLogLevelByHttpWithSpnego() throws Exception {
|
||||
testDynamicLogLevel(LogLevel.PROTOCOL_HTTP, LogLevel.PROTOCOL_HTTP, true);
|
||||
try {
|
||||
testDynamicLogLevel(LogLevel.PROTOCOL_HTTP, LogLevel.PROTOCOL_HTTPS,
|
||||
true);
|
||||
fail("A HTTPS Client should not have succeeded in connecting to a " +
|
||||
"HTTP server");
|
||||
} catch (SSLException e) {
|
||||
GenericTestUtils.assertExceptionContains("Unrecognized SSL message", e);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Server runs HTTPS, no SPNEGO.
|
||||
*
|
||||
* @throws Exception
|
||||
*/
|
||||
@Test(timeout=60000)
|
||||
public void testLogLevelByHttps() throws Exception {
|
||||
testDynamicLogLevel(LogLevel.PROTOCOL_HTTPS, LogLevel.PROTOCOL_HTTPS,
|
||||
false);
|
||||
try {
|
||||
testDynamicLogLevel(LogLevel.PROTOCOL_HTTPS, LogLevel.PROTOCOL_HTTP,
|
||||
false);
|
||||
fail("A HTTP Client should not have succeeded in connecting to a " +
|
||||
"HTTPS server");
|
||||
} catch (SocketException e) {
|
||||
GenericTestUtils.assertExceptionContains(
|
||||
"Unexpected end of file from server", e);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Server runs HTTPS + SPNEGO.
|
||||
*
|
||||
* @throws Exception
|
||||
*/
|
||||
@Test(timeout=60000)
|
||||
public void testLogLevelByHttpsWithSpnego() throws Exception {
|
||||
testDynamicLogLevel(LogLevel.PROTOCOL_HTTPS, LogLevel.PROTOCOL_HTTPS,
|
||||
true);
|
||||
try {
|
||||
testDynamicLogLevel(LogLevel.PROTOCOL_HTTPS, LogLevel.PROTOCOL_HTTP,
|
||||
true);
|
||||
fail("A HTTP Client should not have succeeded in connecting to a " +
|
||||
"HTTPS server");
|
||||
} catch (SocketException e) {
|
||||
GenericTestUtils.assertExceptionContains(
|
||||
"Unexpected end of file from server", e);
|
||||
}
|
||||
}
|
||||
}
|
||||
Loading…
Reference in New Issue