HBASE-22184 [security] Support get|set LogLevel in HTTPS mode.

Signed-off-by: Reid Chan <reidchan@apache.org>

hbase-http/src/main/java/org/apache/hadoop/hbase/http/log/LogLevel.java

@@ -27,6 +27,8 @@ import java.net.URLConnection;
 import java.util.Objects;
 import java.util.regex.Pattern;
 
+import javax.net.ssl.HttpsURLConnection;
+import javax.net.ssl.SSLSocketFactory;
 import javax.servlet.ServletException;
 import javax.servlet.http.HttpServlet;
 import javax.servlet.http.HttpServletRequest;
@@ -40,6 +42,7 @@ import org.apache.hadoop.conf.Configured;
 import org.apache.hadoop.hbase.http.HttpServer;
 import org.apache.hadoop.security.authentication.client.AuthenticatedURL;
 import org.apache.hadoop.security.authentication.client.KerberosAuthenticator;
+import org.apache.hadoop.security.ssl.SSLFactory;
 import org.apache.hadoop.util.ServletUtil;
 import org.apache.hadoop.util.Tool;
 import org.apache.log4j.LogManager;
@@ -59,10 +62,12 @@ import org.apache.hbase.thirdparty.com.google.common.base.Charsets;
 @InterfaceAudience.Private
 public final class LogLevel {
   private static final String USAGES = "\nUsage: General options are:\n"
-      + "\t[-getlevel <host:port> <classname>\n"
+      + "\t[-getlevel <host:port> <classname> [-protocol (http|https)]\n"
-      + "\t[-setlevel <host:port> <classname> <level> ";
+      + "\t[-setlevel <host:port> <classname> <level> [-protocol (http|https)]";
 
   public static final String PROTOCOL_HTTP = "http";
+  public static final String PROTOCOL_HTTPS = "https";
+
   /**
    * A command line implementation
    */
@@ -85,9 +90,15 @@ public final class LogLevel {
     System.exit(-1);
   }
 
+  public static boolean isValidProtocol(String protocol) {
+    return ((protocol.equals(PROTOCOL_HTTP) ||
+        protocol.equals(PROTOCOL_HTTPS)));
+  }
+
   @VisibleForTesting
   static class CLI extends Configured implements Tool {
     private Operations operation = Operations.UNKNOWN;
+    private String protocol;
     private String hostName;
     private String className;
     private String level;
@@ -141,6 +152,9 @@ public final class LogLevel {
           case "-setlevel":
             nextArgIndex = parseSetLevelArgs(args, nextArgIndex);
             break;
+          case "-protocol":
+            nextArgIndex = parseProtocolArgs(args, nextArgIndex);
+            break;
           default:
             throw new HadoopIllegalArgumentException(
                 "Unexpected argument " + args[nextArgIndex]);
@@ -152,6 +166,11 @@ public final class LogLevel {
         throw new HadoopIllegalArgumentException(
             "Must specify either -getlevel or -setlevel");
       }
+
+      // if protocol is unspecified, set it as http.
+      if (protocol == null) {
+        protocol = PROTOCOL_HTTP;
+      }
     }
 
     private int parseGetLevelArgs(String[] args, int index) throws
@@ -187,6 +206,27 @@ public final class LogLevel {
       return index + 4;
     }
 
+    private int parseProtocolArgs(String[] args, int index) throws
+        HadoopIllegalArgumentException {
+      // make sure only -protocol is specified
+      if (protocol != null) {
+        throw new HadoopIllegalArgumentException(
+            "Redundant -protocol command");
+      }
+      // check number of arguments is sufficient
+      if (index + 1 >= args.length) {
+        throw new HadoopIllegalArgumentException(
+            "-protocol needs one parameter");
+      }
+      // check protocol is valid
+      protocol = args[index + 1];
+      if (!isValidProtocol(protocol)) {
+        throw new HadoopIllegalArgumentException(
+            "Invalid protocol: " + protocol);
+      }
+      return index + 2;
+    }
+
     /**
      * Send HTTP request to get log level.
      *
@@ -194,7 +234,7 @@ public final class LogLevel {
      * @throws Exception if unable to connect
      */
     private void doGetLevel() throws Exception {
-      process(PROTOCOL_HTTP + "://" + hostName + "/logLevel?log=" + className);
+      process(protocol + "://" + hostName + "/logLevel?log=" + className);
     }
 
     /**
@@ -204,7 +244,7 @@ public final class LogLevel {
      * @throws Exception if unable to connect
      */
     private void doSetLevel() throws Exception {
-      process(PROTOCOL_HTTP + "://" + hostName + "/logLevel?log=" + className
+      process(protocol + "://" + hostName + "/logLevel?log=" + className
           + "&level=" + level);
     }
 
@@ -220,10 +260,22 @@ public final class LogLevel {
     private URLConnection connect(URL url) throws Exception {
       AuthenticatedURL.Token token = new AuthenticatedURL.Token();
       AuthenticatedURL aUrl;
+      SSLFactory clientSslFactory;
       URLConnection connection;
+      // If https is chosen, configures SSL client.
+      if (PROTOCOL_HTTPS.equals(url.getProtocol())) {
+        clientSslFactory = new SSLFactory(SSLFactory.Mode.CLIENT, this.getConf());
+        clientSslFactory.init();
+        SSLSocketFactory sslSocketF = clientSslFactory.createSSLSocketFactory();
 
-      aUrl = new AuthenticatedURL(new KerberosAuthenticator());
+        aUrl = new AuthenticatedURL(new KerberosAuthenticator(), clientSslFactory);
-      connection = aUrl.openConnection(url, token);
+        connection = aUrl.openConnection(url, token);
+        HttpsURLConnection httpsConn = (HttpsURLConnection) connection;
+        httpsConn.setSSLSocketFactory(sslSocketF);
+      } else {
+        aUrl = new AuthenticatedURL(new KerberosAuthenticator());
+        connection = aUrl.openConnection(url, token);
+      }
       connection.connect();
       return connection;
     }

hbase-http/src/test/java/org/apache/hadoop/hbase/http/log/TestLogLevel.java

@@ -17,17 +17,19 @@
  */
 package org.apache.hadoop.hbase.http.log;
 
-import static org.apache.hadoop.hbase.http.log.LogLevel.PROTOCOL_HTTP;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertNotEquals;
 import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
 
 import java.io.File;
 import java.net.BindException;
+import java.net.SocketException;
 import java.net.URI;
 import java.security.PrivilegedExceptionAction;
 import java.util.Properties;
+import javax.net.ssl.SSLException;
 
 import org.apache.commons.io.FileUtils;
 import org.apache.hadoop.HadoopIllegalArgumentException;
@@ -37,14 +39,18 @@ import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
 import org.apache.hadoop.fs.FileUtil;
 import org.apache.hadoop.hbase.HBaseClassTestRule;
 import org.apache.hadoop.hbase.HBaseCommonTestingUtility;
+import org.apache.hadoop.hbase.http.HttpConfig;
 import org.apache.hadoop.hbase.http.HttpServer;
 import org.apache.hadoop.hbase.http.log.LogLevel.CLI;
+import org.apache.hadoop.hbase.http.ssl.KeyStoreTestUtil;
 import org.apache.hadoop.hbase.testclassification.MiscTests;
 import org.apache.hadoop.hbase.testclassification.SmallTests;
+import org.apache.hadoop.hdfs.DFSConfigKeys;
 import org.apache.hadoop.minikdc.MiniKdc;
 import org.apache.hadoop.net.NetUtils;
 import org.apache.hadoop.security.UserGroupInformation;
 import org.apache.hadoop.security.authorize.AccessControlList;
+import org.apache.hadoop.security.ssl.SSLFactory;
 import org.apache.hadoop.test.GenericTestUtils;
 import org.apache.log4j.Level;
 import org.apache.log4j.LogManager;
@@ -66,8 +72,11 @@ public class TestLogLevel {
       HBaseClassTestRule.forClass(TestLogLevel.class);
 
   private static File BASEDIR;
+  private static String keystoresDir;
+  private static String sslConfDir;
   private static Configuration serverConf;
   private static Configuration clientConf;
+  private static Configuration sslConf;
   private static final String logName = TestLogLevel.class.getName();
   private static final Logger log = LogManager.getLogger(logName);
   private final static String PRINCIPAL = "loglevel.principal";
@@ -94,6 +103,8 @@ public class TestLogLevel {
     serverConf = new Configuration();
     clientConf = new Configuration();
 
+    setupSSL(BASEDIR);
+
     kdc = setupMiniKdc();
     // Create two principles: a client and a HTTP principal
     kdc.createPrincipal(KEYTAB_FILE, clientPrincipal, HTTP_PRINCIPAL);
@@ -132,6 +143,35 @@ public class TestLogLevel {
     return kdc;
   }
 
+  static private void setupSSL(File base) throws Exception {
+    Configuration conf = new Configuration();
+    conf.set(DFSConfigKeys.DFS_HTTP_POLICY_KEY, HttpConfig.Policy.HTTPS_ONLY.name());
+    conf.set(DFSConfigKeys.DFS_NAMENODE_HTTPS_ADDRESS_KEY, "localhost:0");
+    conf.set(DFSConfigKeys.DFS_DATANODE_HTTPS_ADDRESS_KEY, "localhost:0");
+
+    keystoresDir = base.getAbsolutePath();
+    sslConfDir = KeyStoreTestUtil.getClasspathDir(TestLogLevel.class);
+    KeyStoreTestUtil.setupSSLConfig(keystoresDir, sslConfDir, conf, false);
+
+    sslConf = getSslConfig();
+  }
+
+  /**
+   * Get the SSL configuration.
+   * This method is copied from KeyStoreTestUtil#getSslConfig() in Hadoop.
+   * @return {@link Configuration} instance with ssl configs loaded.
+   */
+  private static Configuration getSslConfig(){
+    Configuration sslConf = new Configuration(false);
+    String sslServerConfFile = "ssl-server.xml";
+    String sslClientConfFile = "ssl-client.xml";
+    sslConf.addResource(sslServerConfFile);
+    sslConf.addResource(sslClientConfFile);
+    sslConf.set(SSLFactory.SSL_SERVER_CONF_KEY, sslServerConfFile);
+    sslConf.set(SSLFactory.SSL_CLIENT_CONF_KEY, sslClientConfFile);
+    return sslConf;
+  }
+
   @AfterClass
   public static void tearDown() {
     if (kdc != null) {
@@ -199,15 +239,16 @@ public class TestLogLevel {
   /**
    * Creates and starts a Jetty server binding at an ephemeral port to run
    * LogLevel servlet.
+   * @param protocol "http" or "https"
    * @param isSpnego true if SPNEGO is enabled
    * @return a created HttpServer object
    * @throws Exception if unable to create or start a Jetty server
    */
-  private HttpServer createServer(boolean isSpnego)
+  private HttpServer createServer(String protocol, boolean isSpnego)
       throws Exception {
     HttpServer.Builder builder = new HttpServer.Builder()
         .setName("..")
-        .addEndpoint(new URI(PROTOCOL_HTTP + "://localhost:0"))
+        .addEndpoint(new URI(protocol + "://localhost:0"))
         .setFindPort(true)
         .setConf(serverConf);
     if (isSpnego) {
@@ -220,24 +261,46 @@ public class TestLogLevel {
           .setACL(new AccessControlList("client"));
     }
 
+    // if using HTTPS, configure keystore/truststore properties.
+    if (protocol.equals(LogLevel.PROTOCOL_HTTPS)) {
+      builder = builder.
+          keyPassword(sslConf.get("ssl.server.keystore.keypassword"))
+          .keyStore(sslConf.get("ssl.server.keystore.location"),
+              sslConf.get("ssl.server.keystore.password"),
+              sslConf.get("ssl.server.keystore.type", "jks"))
+          .trustStore(sslConf.get("ssl.server.truststore.location"),
+              sslConf.get("ssl.server.truststore.password"),
+              sslConf.get("ssl.server.truststore.type", "jks"));
+    }
+
     HttpServer server = builder.build();
     server.start();
     return server;
   }
 
-  private void testDynamicLogLevel(final boolean isSpnego)
+  private void testDynamicLogLevel(final String bindProtocol, final String connectProtocol,
+      final boolean isSpnego)
       throws Exception {
-    testDynamicLogLevel(isSpnego, Level.DEBUG.toString());
+    testDynamicLogLevel(bindProtocol, connectProtocol, isSpnego, Level.DEBUG.toString());
   }
 
   /**
    * Run both client and server using the given protocol.
    *
+   * @param bindProtocol specify either http or https for server
+   * @param connectProtocol specify either http or https for client
    * @param isSpnego true if SPNEGO is enabled
    * @throws Exception if client can't accesss server.
    */
-  private void testDynamicLogLevel(final boolean isSpnego, final String newLevel)
+  private void testDynamicLogLevel(final String bindProtocol, final String connectProtocol,
+      final boolean isSpnego, final String newLevel)
       throws Exception {
+    if (!LogLevel.isValidProtocol(bindProtocol)) {
+      throw new Exception("Invalid server protocol " + bindProtocol);
+    }
+    if (!LogLevel.isValidProtocol(connectProtocol)) {
+      throw new Exception("Invalid client protocol " + connectProtocol);
+    }
     Level oldLevel = log.getEffectiveLevel();
     assertNotEquals("Get default Log Level which shouldn't be ERROR.",
         Level.ERROR, oldLevel);
@@ -255,7 +318,7 @@ public class TestLogLevel {
       UserGroupInformation.setConfiguration(serverConf);
     }
 
-    final HttpServer server = createServer(isSpnego);
+    final HttpServer server = createServer(bindProtocol, isSpnego);
     // get server port
     final String authority = NetUtils.getHostPortString(server.getConnectorAddress(0));
 
@@ -266,8 +329,8 @@ public class TestLogLevel {
     try {
       clientUGI.doAs((PrivilegedExceptionAction<Void>) () -> {
         // client command line
-        getLevel(authority);
+        getLevel(connectProtocol, authority);
-        setLevel(authority, newLevel);
+        setLevel(connectProtocol, authority, newLevel);
         return null;
       });
     } finally {
@@ -283,11 +346,12 @@ public class TestLogLevel {
    * Run LogLevel command line to start a client to get log level of this test
    * class.
    *
+   * @param protocol specify either http or https
    * @param authority daemon's web UI address
    * @throws Exception if unable to connect
    */
-  private void getLevel(String authority) throws Exception {
+  private void getLevel(String protocol, String authority) throws Exception {
-    String[] getLevelArgs = {"-getlevel", authority, logName};
+    String[] getLevelArgs = {"-getlevel", authority, logName, "-protocol", protocol};
     CLI cli = new CLI(clientConf);
     cli.run(getLevelArgs);
   }
@@ -296,12 +360,13 @@ public class TestLogLevel {
    * Run LogLevel command line to start a client to set log level of this test
    * class to debug.
    *
+   * @param protocol specify either http or https
    * @param authority daemon's web UI address
    * @throws Exception if unable to run or log level does not change as expected
    */
-  private void setLevel(String authority, String newLevel)
+  private void setLevel(String protocol, String authority, String newLevel)
       throws Exception {
-    String[] setLevelArgs = {"-setlevel", authority, logName, newLevel};
+    String[] setLevelArgs = {"-setlevel", authority, logName, newLevel, "-protocol", protocol};
     CLI cli = new CLI(clientConf);
     cli.run(setLevelArgs);
 
@@ -316,7 +381,7 @@ public class TestLogLevel {
    */
   @Test(timeout=60000)
   public void testInfoLogLevel() throws Exception {
-    testDynamicLogLevel(true, "INFO");
+    testDynamicLogLevel(LogLevel.PROTOCOL_HTTP, LogLevel.PROTOCOL_HTTP, true, "INFO");
   }
 
   /**
@@ -326,26 +391,86 @@ public class TestLogLevel {
    */
   @Test(timeout=60000)
   public void testErrorLogLevel() throws Exception {
-    testDynamicLogLevel(true, "ERROR");
+    testDynamicLogLevel(LogLevel.PROTOCOL_HTTP, LogLevel.PROTOCOL_HTTP, true, "ERROR");
   }
 
   /**
    * Server runs HTTP, no SPNEGO.
    *
-   * @throws Exception if http client can't access http server.
+   * @throws Exception if http client can't access http server,
+   *   or http client can access https server.
    */
   @Test(timeout=60000)
   public void testLogLevelByHttp() throws Exception {
-    testDynamicLogLevel(false);
+    testDynamicLogLevel(LogLevel.PROTOCOL_HTTP, LogLevel.PROTOCOL_HTTP, false);
+    try {
+      testDynamicLogLevel(LogLevel.PROTOCOL_HTTP, LogLevel.PROTOCOL_HTTPS,
+          false);
+      fail("A HTTPS Client should not have succeeded in connecting to a " +
+          "HTTP server");
+    } catch (SSLException e) {
+      GenericTestUtils.assertExceptionContains("Unrecognized SSL message", e);
+    }
   }
 
   /**
    * Server runs HTTP + SPNEGO.
    *
-   * @throws Exception if http client can't access http server.
+   * @throws Exception if http client can't access http server,
+   *   or http client can access https server.
    */
   @Test(timeout=60000)
   public void testLogLevelByHttpWithSpnego() throws Exception {
-    testDynamicLogLevel(true);
+    testDynamicLogLevel(LogLevel.PROTOCOL_HTTP, LogLevel.PROTOCOL_HTTP, true);
+    try {
+      testDynamicLogLevel(LogLevel.PROTOCOL_HTTP, LogLevel.PROTOCOL_HTTPS,
+          true);
+      fail("A HTTPS Client should not have succeeded in connecting to a " +
+          "HTTP server");
+    } catch (SSLException e) {
+      GenericTestUtils.assertExceptionContains("Unrecognized SSL message", e);
+    }
+  }
+
+  /**
+   * Server runs HTTPS, no SPNEGO.
+   *
+   * @throws Exception if https client can't access https server,
+   *   or https client can access http server.
+   */
+  @Test(timeout=60000)
+  public void testLogLevelByHttps() throws Exception {
+    testDynamicLogLevel(LogLevel.PROTOCOL_HTTPS, LogLevel.PROTOCOL_HTTPS,
+        false);
+    try {
+      testDynamicLogLevel(LogLevel.PROTOCOL_HTTPS, LogLevel.PROTOCOL_HTTP,
+          false);
+      fail("A HTTP Client should not have succeeded in connecting to a " +
+          "HTTPS server");
+    } catch (SocketException e) {
+      GenericTestUtils.assertExceptionContains(
+          "Unexpected end of file from server", e);
+    }
+  }
+
+  /**
+   * Server runs HTTPS + SPNEGO.
+   *
+   * @throws Exception if https client can't access https server,
+   *   or https client can access http server.
+   */
+  @Test(timeout=60000)
+  public void testLogLevelByHttpsWithSpnego() throws Exception {
+    testDynamicLogLevel(LogLevel.PROTOCOL_HTTPS, LogLevel.PROTOCOL_HTTPS,
+        true);
+    try {
+      testDynamicLogLevel(LogLevel.PROTOCOL_HTTPS, LogLevel.PROTOCOL_HTTP,
+          true);
+      fail("A HTTP Client should not have succeeded in connecting to a " +
+          "HTTPS server");
+    }  catch (SocketException e) {
+      GenericTestUtils.assertExceptionContains(
+          "Unexpected end of file from server", e);
+    }
   }
 }