HBASE-22184 [security] Support get|set LogLevel in HTTPS mode.
Signed-off-by: Reid Chan <reidchan@apache.org>
This commit is contained in:
Wei-Chiu Chuang
Reid Chan
parent
5537de38b3
commit
a04cb3b2a5
|
|
@ -27,6 +27,8 @@ import java.net.URLConnection;
|
|||
import java.util.Objects;
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
import javax.net.ssl.HttpsURLConnection;
|
||||
import javax.net.ssl.SSLSocketFactory;
|
||||
import javax.servlet.ServletException;
|
||||
import javax.servlet.http.HttpServlet;
|
||||
import javax.servlet.http.HttpServletRequest;
|
||||
|
|
@ -40,6 +42,7 @@ import org.apache.hadoop.conf.Configured;
|
|||
import org.apache.hadoop.hbase.http.HttpServer;
|
||||
import org.apache.hadoop.security.authentication.client.AuthenticatedURL;
|
||||
import org.apache.hadoop.security.authentication.client.KerberosAuthenticator;
|
||||
import org.apache.hadoop.security.ssl.SSLFactory;
|
||||
import org.apache.hadoop.util.ServletUtil;
|
||||
import org.apache.hadoop.util.Tool;
|
||||
import org.apache.log4j.LogManager;
|
||||
|
|
@ -59,10 +62,12 @@ import org.apache.hbase.thirdparty.com.google.common.base.Charsets;
|
|||
@InterfaceAudience.Private
|
||||
public final class LogLevel {
|
||||
private static final String USAGES = "\nUsage: General options are:\n"
|
||||
+ "\t[-getlevel <host:port> <classname>\n"
|
||||
+ "\t[-setlevel <host:port> <classname> <level> ";
|
||||
+ "\t[-getlevel <host:port> <classname> [-protocol (http|https)]\n"
|
||||
+ "\t[-setlevel <host:port> <classname> <level> [-protocol (http|https)]";
|
||||
|
||||
public static final String PROTOCOL_HTTP = "http";
|
||||
public static final String PROTOCOL_HTTPS = "https";
|
||||
|
||||
/**
|
||||
* A command line implementation
|
||||
*/
|
||||
|
|
@ -85,9 +90,15 @@ public final class LogLevel {
|
|||
System.exit(-1);
|
||||
}
|
||||
|
||||
public static boolean isValidProtocol(String protocol) {
|
||||
return ((protocol.equals(PROTOCOL_HTTP) ||
|
||||
protocol.equals(PROTOCOL_HTTPS)));
|
||||
}
|
||||
|
||||
@VisibleForTesting
|
||||
static class CLI extends Configured implements Tool {
|
||||
private Operations operation = Operations.UNKNOWN;
|
||||
private String protocol;
|
||||
private String hostName;
|
||||
private String className;
|
||||
private String level;
|
||||
|
|
@ -141,6 +152,9 @@ public final class LogLevel {
|
|||
case "-setlevel":
|
||||
nextArgIndex = parseSetLevelArgs(args, nextArgIndex);
|
||||
break;
|
||||
case "-protocol":
|
||||
nextArgIndex = parseProtocolArgs(args, nextArgIndex);
|
||||
break;
|
||||
default:
|
||||
throw new HadoopIllegalArgumentException(
|
||||
"Unexpected argument " + args[nextArgIndex]);
|
||||
|
|
@ -152,6 +166,11 @@ public final class LogLevel {
|
|||
throw new HadoopIllegalArgumentException(
|
||||
"Must specify either -getlevel or -setlevel");
|
||||
}
|
||||
|
||||
// if protocol is unspecified, set it as http.
|
||||
if (protocol == null) {
|
||||
protocol = PROTOCOL_HTTP;
|
||||
}
|
||||
}
|
||||
|
||||
private int parseGetLevelArgs(String[] args, int index) throws
|
||||
|
|
@ -187,6 +206,27 @@ public final class LogLevel {
|
|||
return index + 4;
|
||||
}
|
||||
|
||||
private int parseProtocolArgs(String[] args, int index) throws
|
||||
HadoopIllegalArgumentException {
|
||||
// make sure only -protocol is specified
|
||||
if (protocol != null) {
|
||||
throw new HadoopIllegalArgumentException(
|
||||
"Redundant -protocol command");
|
||||
}
|
||||
// check number of arguments is sufficient
|
||||
if (index + 1 >= args.length) {
|
||||
throw new HadoopIllegalArgumentException(
|
||||
"-protocol needs one parameter");
|
||||
}
|
||||
// check protocol is valid
|
||||
protocol = args[index + 1];
|
||||
if (!isValidProtocol(protocol)) {
|
||||
throw new HadoopIllegalArgumentException(
|
||||
"Invalid protocol: " + protocol);
|
||||
}
|
||||
return index + 2;
|
||||
}
|
||||
|
||||
/**
|
||||
* Send HTTP request to get log level.
|
||||
*
|
||||
|
|
@ -194,7 +234,7 @@ public final class LogLevel {
|
|||
* @throws Exception if unable to connect
|
||||
*/
|
||||
private void doGetLevel() throws Exception {
|
||||
process(PROTOCOL_HTTP + "://" + hostName + "/logLevel?log=" + className);
|
||||
process(protocol + "://" + hostName + "/logLevel?log=" + className);
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -204,7 +244,7 @@ public final class LogLevel {
|
|||
* @throws Exception if unable to connect
|
||||
*/
|
||||
private void doSetLevel() throws Exception {
|
||||
process(PROTOCOL_HTTP + "://" + hostName + "/logLevel?log=" + className
|
||||
process(protocol + "://" + hostName + "/logLevel?log=" + className
|
||||
+ "&level=" + level);
|
||||
}
|
||||
|
||||
|
|
@ -220,10 +260,22 @@ public final class LogLevel {
|
|||
private URLConnection connect(URL url) throws Exception {
|
||||
AuthenticatedURL.Token token = new AuthenticatedURL.Token();
|
||||
AuthenticatedURL aUrl;
|
||||
SSLFactory clientSslFactory;
|
||||
URLConnection connection;
|
||||
// If https is chosen, configures SSL client.
|
||||
if (PROTOCOL_HTTPS.equals(url.getProtocol())) {
|
||||
clientSslFactory = new SSLFactory(SSLFactory.Mode.CLIENT, this.getConf());
|
||||
clientSslFactory.init();
|
||||
SSLSocketFactory sslSocketF = clientSslFactory.createSSLSocketFactory();
|
||||
|
||||
aUrl = new AuthenticatedURL(new KerberosAuthenticator());
|
||||
connection = aUrl.openConnection(url, token);
|
||||
aUrl = new AuthenticatedURL(new KerberosAuthenticator(), clientSslFactory);
|
||||
connection = aUrl.openConnection(url, token);
|
||||
HttpsURLConnection httpsConn = (HttpsURLConnection) connection;
|
||||
httpsConn.setSSLSocketFactory(sslSocketF);
|
||||
} else {
|
||||
aUrl = new AuthenticatedURL(new KerberosAuthenticator());
|
||||
connection = aUrl.openConnection(url, token);
|
||||
}
|
||||
connection.connect();
|
||||
return connection;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -17,17 +17,19 @@
|
|||
*/
|
||||
package org.apache.hadoop.hbase.http.log;
|
||||
|
||||
import static org.apache.hadoop.hbase.http.log.LogLevel.PROTOCOL_HTTP;
|
||||
import static org.junit.Assert.assertEquals;
|
||||
import static org.junit.Assert.assertFalse;
|
||||
import static org.junit.Assert.assertNotEquals;
|
||||
import static org.junit.Assert.assertTrue;
|
||||
import static org.junit.Assert.fail;
|
||||
|
||||
import java.io.File;
|
||||
import java.net.BindException;
|
||||
import java.net.SocketException;
|
||||
import java.net.URI;
|
||||
import java.security.PrivilegedExceptionAction;
|
||||
import java.util.Properties;
|
||||
import javax.net.ssl.SSLException;
|
||||
|
||||
import org.apache.commons.io.FileUtils;
|
||||
import org.apache.hadoop.HadoopIllegalArgumentException;
|
||||
|
|
@ -37,14 +39,18 @@ import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
|
|||
import org.apache.hadoop.fs.FileUtil;
|
||||
import org.apache.hadoop.hbase.HBaseClassTestRule;
|
||||
import org.apache.hadoop.hbase.HBaseCommonTestingUtility;
|
||||
import org.apache.hadoop.hbase.http.HttpConfig;
|
||||
import org.apache.hadoop.hbase.http.HttpServer;
|
||||
import org.apache.hadoop.hbase.http.log.LogLevel.CLI;
|
||||
import org.apache.hadoop.hbase.http.ssl.KeyStoreTestUtil;
|
||||
import org.apache.hadoop.hbase.testclassification.MiscTests;
|
||||
import org.apache.hadoop.hbase.testclassification.SmallTests;
|
||||
import org.apache.hadoop.hdfs.DFSConfigKeys;
|
||||
import org.apache.hadoop.minikdc.MiniKdc;
|
||||
import org.apache.hadoop.net.NetUtils;
|
||||
import org.apache.hadoop.security.UserGroupInformation;
|
||||
import org.apache.hadoop.security.authorize.AccessControlList;
|
||||
import org.apache.hadoop.security.ssl.SSLFactory;
|
||||
import org.apache.hadoop.test.GenericTestUtils;
|
||||
import org.apache.log4j.Level;
|
||||
import org.apache.log4j.LogManager;
|
||||
|
|
@ -66,8 +72,11 @@ public class TestLogLevel {
|
|||
HBaseClassTestRule.forClass(TestLogLevel.class);
|
||||
|
||||
private static File BASEDIR;
|
||||
private static String keystoresDir;
|
||||
private static String sslConfDir;
|
||||
private static Configuration serverConf;
|
||||
private static Configuration clientConf;
|
||||
private static Configuration sslConf;
|
||||
private static final String logName = TestLogLevel.class.getName();
|
||||
private static final Logger log = LogManager.getLogger(logName);
|
||||
private final static String PRINCIPAL = "loglevel.principal";
|
||||
|
|
@ -94,6 +103,8 @@ public class TestLogLevel {
|
|||
serverConf = new Configuration();
|
||||
clientConf = new Configuration();
|
||||
|
||||
setupSSL(BASEDIR);
|
||||
|
||||
kdc = setupMiniKdc();
|
||||
// Create two principles: a client and a HTTP principal
|
||||
kdc.createPrincipal(KEYTAB_FILE, clientPrincipal, HTTP_PRINCIPAL);
|
||||
|
|
@ -132,6 +143,35 @@ public class TestLogLevel {
|
|||
return kdc;
|
||||
}
|
||||
|
||||
static private void setupSSL(File base) throws Exception {
|
||||
Configuration conf = new Configuration();
|
||||
conf.set(DFSConfigKeys.DFS_HTTP_POLICY_KEY, HttpConfig.Policy.HTTPS_ONLY.name());
|
||||
conf.set(DFSConfigKeys.DFS_NAMENODE_HTTPS_ADDRESS_KEY, "localhost:0");
|
||||
conf.set(DFSConfigKeys.DFS_DATANODE_HTTPS_ADDRESS_KEY, "localhost:0");
|
||||
|
||||
keystoresDir = base.getAbsolutePath();
|
||||
sslConfDir = KeyStoreTestUtil.getClasspathDir(TestLogLevel.class);
|
||||
KeyStoreTestUtil.setupSSLConfig(keystoresDir, sslConfDir, conf, false);
|
||||
|
||||
sslConf = getSslConfig();
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the SSL configuration.
|
||||
* This method is copied from KeyStoreTestUtil#getSslConfig() in Hadoop.
|
||||
* @return {@link Configuration} instance with ssl configs loaded.
|
||||
*/
|
||||
private static Configuration getSslConfig(){
|
||||
Configuration sslConf = new Configuration(false);
|
||||
String sslServerConfFile = "ssl-server.xml";
|
||||
String sslClientConfFile = "ssl-client.xml";
|
||||
sslConf.addResource(sslServerConfFile);
|
||||
sslConf.addResource(sslClientConfFile);
|
||||
sslConf.set(SSLFactory.SSL_SERVER_CONF_KEY, sslServerConfFile);
|
||||
sslConf.set(SSLFactory.SSL_CLIENT_CONF_KEY, sslClientConfFile);
|
||||
return sslConf;
|
||||
}
|
||||
|
||||
@AfterClass
|
||||
public static void tearDown() {
|
||||
if (kdc != null) {
|
||||
|
|
@ -199,15 +239,16 @@ public class TestLogLevel {
|
|||
/**
|
||||
* Creates and starts a Jetty server binding at an ephemeral port to run
|
||||
* LogLevel servlet.
|
||||
* @param protocol "http" or "https"
|
||||
* @param isSpnego true if SPNEGO is enabled
|
||||
* @return a created HttpServer object
|
||||
* @throws Exception if unable to create or start a Jetty server
|
||||
*/
|
||||
private HttpServer createServer(boolean isSpnego)
|
||||
private HttpServer createServer(String protocol, boolean isSpnego)
|
||||
throws Exception {
|
||||
HttpServer.Builder builder = new HttpServer.Builder()
|
||||
.setName("..")
|
||||
.addEndpoint(new URI(PROTOCOL_HTTP + "://localhost:0"))
|
||||
.addEndpoint(new URI(protocol + "://localhost:0"))
|
||||
.setFindPort(true)
|
||||
.setConf(serverConf);
|
||||
if (isSpnego) {
|
||||
|
|
@ -220,24 +261,46 @@ public class TestLogLevel {
|
|||
.setACL(new AccessControlList("client"));
|
||||
}
|
||||
|
||||
// if using HTTPS, configure keystore/truststore properties.
|
||||
if (protocol.equals(LogLevel.PROTOCOL_HTTPS)) {
|
||||
builder = builder.
|
||||
keyPassword(sslConf.get("ssl.server.keystore.keypassword"))
|
||||
.keyStore(sslConf.get("ssl.server.keystore.location"),
|
||||
sslConf.get("ssl.server.keystore.password"),
|
||||
sslConf.get("ssl.server.keystore.type", "jks"))
|
||||
.trustStore(sslConf.get("ssl.server.truststore.location"),
|
||||
sslConf.get("ssl.server.truststore.password"),
|
||||
sslConf.get("ssl.server.truststore.type", "jks"));
|
||||
}
|
||||
|
||||
HttpServer server = builder.build();
|
||||
server.start();
|
||||
return server;
|
||||
}
|
||||
|
||||
private void testDynamicLogLevel(final boolean isSpnego)
|
||||
private void testDynamicLogLevel(final String bindProtocol, final String connectProtocol,
|
||||
final boolean isSpnego)
|
||||
throws Exception {
|
||||
testDynamicLogLevel(isSpnego, Level.DEBUG.toString());
|
||||
testDynamicLogLevel(bindProtocol, connectProtocol, isSpnego, Level.DEBUG.toString());
|
||||
}
|
||||
|
||||
/**
|
||||
* Run both client and server using the given protocol.
|
||||
*
|
||||
* @param bindProtocol specify either http or https for server
|
||||
* @param connectProtocol specify either http or https for client
|
||||
* @param isSpnego true if SPNEGO is enabled
|
||||
* @throws Exception if client can't accesss server.
|
||||
*/
|
||||
private void testDynamicLogLevel(final boolean isSpnego, final String newLevel)
|
||||
private void testDynamicLogLevel(final String bindProtocol, final String connectProtocol,
|
||||
final boolean isSpnego, final String newLevel)
|
||||
throws Exception {
|
||||
if (!LogLevel.isValidProtocol(bindProtocol)) {
|
||||
throw new Exception("Invalid server protocol " + bindProtocol);
|
||||
}
|
||||
if (!LogLevel.isValidProtocol(connectProtocol)) {
|
||||
throw new Exception("Invalid client protocol " + connectProtocol);
|
||||
}
|
||||
Level oldLevel = log.getEffectiveLevel();
|
||||
assertNotEquals("Get default Log Level which shouldn't be ERROR.",
|
||||
Level.ERROR, oldLevel);
|
||||
|
|
@ -255,7 +318,7 @@ public class TestLogLevel {
|
|||
UserGroupInformation.setConfiguration(serverConf);
|
||||
}
|
||||
|
||||
final HttpServer server = createServer(isSpnego);
|
||||
final HttpServer server = createServer(bindProtocol, isSpnego);
|
||||
// get server port
|
||||
final String authority = NetUtils.getHostPortString(server.getConnectorAddress(0));
|
||||
|
||||
|
|
@ -266,8 +329,8 @@ public class TestLogLevel {
|
|||
try {
|
||||
clientUGI.doAs((PrivilegedExceptionAction<Void>) () -> {
|
||||
// client command line
|
||||
getLevel(authority);
|
||||
setLevel(authority, newLevel);
|
||||
getLevel(connectProtocol, authority);
|
||||
setLevel(connectProtocol, authority, newLevel);
|
||||
return null;
|
||||
});
|
||||
} finally {
|
||||
|
|
@ -283,11 +346,12 @@ public class TestLogLevel {
|
|||
* Run LogLevel command line to start a client to get log level of this test
|
||||
* class.
|
||||
*
|
||||
* @param protocol specify either http or https
|
||||
* @param authority daemon's web UI address
|
||||
* @throws Exception if unable to connect
|
||||
*/
|
||||
private void getLevel(String authority) throws Exception {
|
||||
String[] getLevelArgs = {"-getlevel", authority, logName};
|
||||
private void getLevel(String protocol, String authority) throws Exception {
|
||||
String[] getLevelArgs = {"-getlevel", authority, logName, "-protocol", protocol};
|
||||
CLI cli = new CLI(clientConf);
|
||||
cli.run(getLevelArgs);
|
||||
}
|
||||
|
|
@ -296,12 +360,13 @@ public class TestLogLevel {
|
|||
* Run LogLevel command line to start a client to set log level of this test
|
||||
* class to debug.
|
||||
*
|
||||
* @param protocol specify either http or https
|
||||
* @param authority daemon's web UI address
|
||||
* @throws Exception if unable to run or log level does not change as expected
|
||||
*/
|
||||
private void setLevel(String authority, String newLevel)
|
||||
private void setLevel(String protocol, String authority, String newLevel)
|
||||
throws Exception {
|
||||
String[] setLevelArgs = {"-setlevel", authority, logName, newLevel};
|
||||
String[] setLevelArgs = {"-setlevel", authority, logName, newLevel, "-protocol", protocol};
|
||||
CLI cli = new CLI(clientConf);
|
||||
cli.run(setLevelArgs);
|
||||
|
||||
|
|
@ -316,7 +381,7 @@ public class TestLogLevel {
|
|||
*/
|
||||
@Test(timeout=60000)
|
||||
public void testInfoLogLevel() throws Exception {
|
||||
testDynamicLogLevel(true, "INFO");
|
||||
testDynamicLogLevel(LogLevel.PROTOCOL_HTTP, LogLevel.PROTOCOL_HTTP, true, "INFO");
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -326,26 +391,86 @@ public class TestLogLevel {
|
|||
*/
|
||||
@Test(timeout=60000)
|
||||
public void testErrorLogLevel() throws Exception {
|
||||
testDynamicLogLevel(true, "ERROR");
|
||||
testDynamicLogLevel(LogLevel.PROTOCOL_HTTP, LogLevel.PROTOCOL_HTTP, true, "ERROR");
|
||||
}
|
||||
|
||||
/**
|
||||
* Server runs HTTP, no SPNEGO.
|
||||
*
|
||||
* @throws Exception if http client can't access http server.
|
||||
* @throws Exception if http client can't access http server,
|
||||
* or http client can access https server.
|
||||
*/
|
||||
@Test(timeout=60000)
|
||||
public void testLogLevelByHttp() throws Exception {
|
||||
testDynamicLogLevel(false);
|
||||
testDynamicLogLevel(LogLevel.PROTOCOL_HTTP, LogLevel.PROTOCOL_HTTP, false);
|
||||
try {
|
||||
testDynamicLogLevel(LogLevel.PROTOCOL_HTTP, LogLevel.PROTOCOL_HTTPS,
|
||||
false);
|
||||
fail("A HTTPS Client should not have succeeded in connecting to a " +
|
||||
"HTTP server");
|
||||
} catch (SSLException e) {
|
||||
GenericTestUtils.assertExceptionContains("Unrecognized SSL message", e);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Server runs HTTP + SPNEGO.
|
||||
*
|
||||
* @throws Exception if http client can't access http server.
|
||||
* @throws Exception if http client can't access http server,
|
||||
* or http client can access https server.
|
||||
*/
|
||||
@Test(timeout=60000)
|
||||
public void testLogLevelByHttpWithSpnego() throws Exception {
|
||||
testDynamicLogLevel(true);
|
||||
testDynamicLogLevel(LogLevel.PROTOCOL_HTTP, LogLevel.PROTOCOL_HTTP, true);
|
||||
try {
|
||||
testDynamicLogLevel(LogLevel.PROTOCOL_HTTP, LogLevel.PROTOCOL_HTTPS,
|
||||
true);
|
||||
fail("A HTTPS Client should not have succeeded in connecting to a " +
|
||||
"HTTP server");
|
||||
} catch (SSLException e) {
|
||||
GenericTestUtils.assertExceptionContains("Unrecognized SSL message", e);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Server runs HTTPS, no SPNEGO.
|
||||
*
|
||||
* @throws Exception if https client can't access https server,
|
||||
* or https client can access http server.
|
||||
*/
|
||||
@Test(timeout=60000)
|
||||
public void testLogLevelByHttps() throws Exception {
|
||||
testDynamicLogLevel(LogLevel.PROTOCOL_HTTPS, LogLevel.PROTOCOL_HTTPS,
|
||||
false);
|
||||
try {
|
||||
testDynamicLogLevel(LogLevel.PROTOCOL_HTTPS, LogLevel.PROTOCOL_HTTP,
|
||||
false);
|
||||
fail("A HTTP Client should not have succeeded in connecting to a " +
|
||||
"HTTPS server");
|
||||
} catch (SocketException e) {
|
||||
GenericTestUtils.assertExceptionContains(
|
||||
"Unexpected end of file from server", e);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Server runs HTTPS + SPNEGO.
|
||||
*
|
||||
* @throws Exception if https client can't access https server,
|
||||
* or https client can access http server.
|
||||
*/
|
||||
@Test(timeout=60000)
|
||||
public void testLogLevelByHttpsWithSpnego() throws Exception {
|
||||
testDynamicLogLevel(LogLevel.PROTOCOL_HTTPS, LogLevel.PROTOCOL_HTTPS,
|
||||
true);
|
||||
try {
|
||||
testDynamicLogLevel(LogLevel.PROTOCOL_HTTPS, LogLevel.PROTOCOL_HTTP,
|
||||
true);
|
||||
fail("A HTTP Client should not have succeeded in connecting to a " +
|
||||
"HTTPS server");
|
||||
} catch (SocketException e) {
|
||||
GenericTestUtils.assertExceptionContains(
|
||||
"Unexpected end of file from server", e);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in New Issue