HADOOP-12765. HttpServer2 should switch to using the non-blocking SslSelectChannelConnector to prevent performance degradation when handling SSL connections. Contributed by Min Shen. Branch-2 patch contributed by Wei-Chiu Chuang.

hadoop-common-project/hadoop-common/pom.xml

@@ -105,6 +105,11 @@
       <scope>compile</scope>
     </dependency>
 
+    <dependency>
+      <groupId>org.mortbay.jetty</groupId>
+      <artifactId>jetty-sslengine</artifactId>
+      <scope>compile</scope>
+    </dependency>
     <dependency>
       <groupId>javax.servlet.jsp</groupId>
       <artifactId>jsp-api</artifactId>

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpServer2.java

@@ -56,7 +56,7 @@ import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.CommonConfigurationKeys;
 import org.apache.hadoop.security.AuthenticationFilterInitializer;
 import org.apache.hadoop.security.authentication.util.SignerSecretProvider;
-import org.apache.hadoop.security.ssl.SslSocketConnectorSecure;
+import org.apache.hadoop.security.ssl.SslSelectChannelConnectorSecure;
 import org.apache.hadoop.jmx.JMXJsonServlet;
 import org.apache.hadoop.log.LogLevel;
 import org.apache.hadoop.metrics.MetricsServlet;
@@ -78,7 +78,7 @@ import org.mortbay.jetty.handler.ContextHandlerCollection;
 import org.mortbay.jetty.handler.HandlerCollection;
 import org.mortbay.jetty.handler.RequestLogHandler;
 import org.mortbay.jetty.nio.SelectChannelConnector;
-import org.mortbay.jetty.security.SslSocketConnector;
+import org.mortbay.jetty.security.SslSelectChannelConnector;
 import org.mortbay.jetty.servlet.AbstractSessionManager;
 import org.mortbay.jetty.servlet.Context;
 import org.mortbay.jetty.servlet.DefaultServlet;
@@ -333,8 +333,24 @@ public final class HttpServer2 implements FilterContainer {
         if ("http".equals(scheme)) {
           listener = HttpServer2.createDefaultChannelConnector();
         } else if ("https".equals(scheme)) {
-          SslSocketConnector c = new SslSocketConnectorSecure();
+          listener = createHttpsChannelConnector();
-          c.setHeaderBufferSize(1024*64);
+
+        } else {
+          throw new HadoopIllegalArgumentException(
+              "unknown scheme for endpoint:" + ep);
+        }
+        listener.setHost(ep.getHost());
+        listener.setPort(ep.getPort() == -1 ? 0 : ep.getPort());
+        server.addListener(listener);
+      }
+      server.loadListeners();
+      return server;
+    }
+
+    private Connector createHttpsChannelConnector() {
+      SslSelectChannelConnector c = new SslSelectChannelConnectorSecure();
+      configureChannelConnector(c);
+
       c.setNeedClientAuth(needsClientAuth);
       c.setKeyPassword(keyPassword);
 
@@ -354,19 +370,7 @@ public final class HttpServer2 implements FilterContainer {
         c.setExcludeCipherSuites(excludeCiphers.split(","));
         LOG.info("Excluded Cipher List:" + excludeCiphers);
       }
-
+      return c;
-          listener = c;
-
-        } else {
-          throw new HadoopIllegalArgumentException(
-              "unknown scheme for endpoint:" + ep);
-        }
-        listener.setHost(ep.getHost());
-        listener.setPort(ep.getPort() == -1 ? 0 : ep.getPort());
-        server.addListener(listener);
-      }
-      server.loadListeners();
-      return server;
     }
   }
 
@@ -541,21 +545,25 @@ public final class HttpServer2 implements FilterContainer {
     }
   }
 
-  @InterfaceAudience.Private
+  private static void configureChannelConnector(SelectChannelConnector c) {
-  public static Connector createDefaultChannelConnector() {
+    c.setLowResourceMaxIdleTime(10000);
-    SelectChannelConnector ret = new SelectChannelConnectorWithSafeStartup();
+    c.setAcceptQueueSize(128);
-    ret.setLowResourceMaxIdleTime(10000);
+    c.setResolveNames(false);
-    ret.setAcceptQueueSize(128);
+    c.setUseDirectBuffers(false);
-    ret.setResolveNames(false);
-    ret.setUseDirectBuffers(false);
     if(Shell.WINDOWS) {
       // result of setting the SO_REUSEADDR flag is different on Windows
       // http://msdn.microsoft.com/en-us/library/ms740621(v=vs.85).aspx
       // without this 2 NN's can start on the same machine and listen on
       // the same port with indeterminate routing of incoming requests to them
-      ret.setReuseAddress(false);
+      c.setReuseAddress(false);
     }
-    ret.setHeaderBufferSize(1024*64);
+    c.setHeaderBufferSize(1024*64);
+  }
+
+  @InterfaceAudience.Private
+  public static Connector createDefaultChannelConnector() {
+    SelectChannelConnector ret = new SelectChannelConnectorWithSafeStartup();
+    configureChannelConnector(ret);
     return ret;
   }
 

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ssl/SslSelectChannelConnectorSecure.java

@@ -18,41 +18,41 @@
 
 package org.apache.hadoop.security.ssl;
 
-import org.mortbay.jetty.security.SslSocketConnector;
-
-import javax.net.ssl.SSLServerSocket;
 import java.io.IOException;
-import java.net.ServerSocket;
 import java.util.ArrayList;
 
+import javax.net.ssl.SSLEngine;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.mortbay.jetty.security.SslSelectChannelConnector;
+
 /**
- * This subclass of the Jetty SslSocketConnector exists solely to control
+ * This subclass of the Jetty SslSelectChannelConnector exists solely to
- * the TLS protocol versions allowed.  This is fallout from the POODLE
+ * control the TLS protocol versions allowed.  This is fallout from the
- * vulnerability (CVE-2014-3566), which requires that SSLv3 be disabled.
+ * POODLE vulnerability (CVE-2014-3566), which requires that SSLv3 be disabled.
  * Only TLS 1.0 and later protocols are allowed.
  */
-public class SslSocketConnectorSecure extends SslSocketConnector {
+@InterfaceAudience.Private
+public class SslSelectChannelConnectorSecure extends SslSelectChannelConnector {
 
-  public SslSocketConnectorSecure() {
+  public SslSelectChannelConnectorSecure() {
     super();
   }
 
   /**
-   * Create a new ServerSocket that will not accept SSLv3 connections,
+   * Disable SSLv3 protocol.
-   * but will accept TLSv1.x connections.
    */
-  protected ServerSocket newServerSocket(String host, int port,int backlog)
+  @Override
-          throws IOException {
+  protected SSLEngine createSSLEngine() throws IOException {
-    SSLServerSocket socket = (SSLServerSocket)
+    SSLEngine engine = super.createSSLEngine();
-            super.newServerSocket(host, port, backlog);
     ArrayList<String> nonSSLProtocols = new ArrayList<String>();
-    for (String p : socket.getEnabledProtocols()) {
+    for (String p : engine.getEnabledProtocols()) {
       if (!p.contains("SSLv3")) {
         nonSSLProtocols.add(p);
       }
     }
-    socket.setEnabledProtocols(nonSSLProtocols.toArray(
+    engine.setEnabledProtocols(nonSSLProtocols.toArray(
         new String[nonSSLProtocols.size()]));
-    return socket;
+    return engine;
   }
 }

hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/MiniKMS.java

@@ -18,14 +18,15 @@
 package org.apache.hadoop.crypto.key.kms.server;
 
 import com.google.common.base.Preconditions;
+
 import org.apache.commons.io.IOUtils;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.crypto.key.kms.KMSRESTConstants;
 import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.security.ssl.SslSocketConnectorSecure;
+import org.apache.hadoop.security.ssl.SslSelectChannelConnectorSecure;
 import org.mortbay.jetty.Connector;
 import org.mortbay.jetty.Server;
-import org.mortbay.jetty.security.SslSocketConnector;
+import org.mortbay.jetty.security.SslSelectChannelConnector;
 import org.mortbay.jetty.webapp.WebAppContext;
 
 import java.io.File;
@@ -52,7 +53,7 @@ public class MiniKMS {
       if (!ssl) {
         server.getConnectors()[0].setHost(host);
       } else {
-        SslSocketConnector c = new SslSocketConnectorSecure();
+        SslSelectChannelConnector c = new SslSelectChannelConnectorSecure();
         c.setHost(host);
         c.setNeedClientAuth(false);
         c.setKeystore(keyStore);
@@ -69,7 +70,7 @@ public class MiniKMS {
 
   private static URL getJettyURL(Server server) {
     boolean ssl = server.getConnectors()[0].getClass()
-        == SslSocketConnectorSecure.class;
+        == SslSelectChannelConnectorSecure.class;
     try {
       String scheme = (ssl) ? "https" : "http";
       return new URL(scheme + "://" +

hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/test/TestJettyHelper.java

@@ -24,14 +24,14 @@ import java.net.ServerSocket;
 import java.net.URL;
 import java.net.UnknownHostException;
 
-import org.apache.hadoop.security.ssl.SslSocketConnectorSecure;
+import org.apache.hadoop.security.ssl.SslSelectChannelConnectorSecure;
 import org.junit.Test;
 import org.junit.rules.MethodRule;
 import org.junit.runners.model.FrameworkMethod;
 import org.junit.runners.model.Statement;
 import org.mortbay.jetty.Connector;
 import org.mortbay.jetty.Server;
-import org.mortbay.jetty.security.SslSocketConnector;
+import org.mortbay.jetty.security.SslSelectChannelConnector;
 
 public class TestJettyHelper implements MethodRule {
   private boolean ssl;
@@ -93,7 +93,7 @@ public class TestJettyHelper implements MethodRule {
         server.getConnectors()[0].setHost(host);
         server.getConnectors()[0].setPort(port);
       } else {
-        SslSocketConnector c = new SslSocketConnectorSecure();
+        SslSelectChannelConnector c = new SslSelectChannelConnectorSecure();
         c.setHost(host);
         c.setPort(port);
         c.setNeedClientAuth(false);

hadoop-project/pom.xml

@@ -498,6 +498,11 @@
         <artifactId>jetty-util</artifactId>
         <version>${jetty.version}</version>
       </dependency>
+      <dependency>
+        <groupId>org.mortbay.jetty</groupId>
+        <artifactId>jetty-sslengine</artifactId>
+        <version>${jetty.version}</version>
+      </dependency>
       <dependency>
         <groupId>org.apache.tomcat.embed</groupId>
         <artifactId>tomcat-embed-core</artifactId>