HADOOP-12668. Support excluding weak Ciphers in HttpServer2 through ssl-server.conf. Contributed by Vijay Singh.

Change-Id: Ie46a5427d29188935427f67e55203c19fcd83335
(cherry picked from commit 6d4a4e785b)

hadoop-common-project/hadoop-common/src/main/conf/ssl-server.xml.example

@@ -75,4 +75,14 @@
   </description>
 </property>
 
+<property>
+  <name>ssl.server.exclude.cipher.list</name>
+  <value>TLS_ECDHE_RSA_WITH_RC4_128_SHA,SSL_DHE_RSA_EXPORT_WITH_DES40_CBC_SHA,
+  SSL_RSA_WITH_DES_CBC_SHA,SSL_DHE_RSA_WITH_DES_CBC_SHA,
+  SSL_RSA_EXPORT_WITH_RC4_40_MD5,SSL_RSA_EXPORT_WITH_DES40_CBC_SHA,
+  SSL_RSA_WITH_RC4_128_MD5</value>
+  <description>Optional. The weak security cipher suites that you want excluded
+  from SSL communication.</description>
+</property>
+
 </configuration>

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpServer2.java

@@ -171,6 +171,7 @@ public final class HttpServer2 implements FilterContainer {
     private String hostName;
     private boolean disallowFallbackToRandomSignerSecretProvider;
     private String authFilterConfigurationPrefix = "hadoop.http.authentication.";
+    private String excludeCiphers;
 
     public Builder setName(String name){
       this.name = name;
@@ -275,6 +276,11 @@ public final class HttpServer2 implements FilterContainer {
       return this;
     }
 
+    public Builder excludeCiphers(String pExcludeCiphers) {
+      this.excludeCiphers = pExcludeCiphers;
+      return this;
+    }
+
     public HttpServer2 build() throws IOException {
       Preconditions.checkNotNull(name, "name is not set");
       Preconditions.checkState(!endpoints.isEmpty(), "No endpoints specified");
@@ -315,6 +321,12 @@ public final class HttpServer2 implements FilterContainer {
             c.setTruststoreType(trustStoreType);
             c.setTrustPassword(trustStorePassword);
           }
+
+          if(null != excludeCiphers && !excludeCiphers.isEmpty()) {
+            c.setExcludeCipherSuites(excludeCiphers.split(","));
+            LOG.info("Excluded Cipher List:" + excludeCiphers);
+          }
+
           listener = c;
 
         } else {

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ssl/FileBasedKeyStoresFactory.java

@@ -67,6 +67,8 @@ public class FileBasedKeyStoresFactory implements KeyStoresFactory {
     "ssl.{0}.truststore.password";
   public static final String SSL_TRUSTSTORE_TYPE_TPL_KEY =
     "ssl.{0}.truststore.type";
+  public static final String SSL_EXCLUDE_CIPHER_LIST =
+      "ssl.{0}.exclude.cipher.list";
 
   /**
    * Default format of the keystore files.

hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/TestHttpCookieFlag.java

@@ -107,7 +107,10 @@ public class TestHttpCookieFlag {
                     sslConf.get("ssl.server.keystore.type", "jks"))
             .trustStore(sslConf.get("ssl.server.truststore.location"),
                     sslConf.get("ssl.server.truststore.password"),
-                    sslConf.get("ssl.server.truststore.type", "jks")).build();
+                    sslConf.get("ssl.server.truststore.type", "jks"))
+            .excludeCiphers(
+                    sslConf.get("ssl.server.exclude.cipher.list"))
+            .build();
     server.addServlet("echo", "/echo", TestHttpServer.EchoServlet.class);
     server.start();
   }

hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/TestSSLHttpServer.java

@@ -19,11 +19,18 @@ package org.apache.hadoop.http;
 
 import java.io.ByteArrayOutputStream;
 import java.io.File;
+import java.io.IOException;
 import java.io.InputStream;
+import java.net.InetAddress;
+import java.net.Socket;
 import java.net.URI;
 import java.net.URL;
+import java.net.UnknownHostException;
 
 import javax.net.ssl.HttpsURLConnection;
+import javax.net.ssl.SSLHandshakeException;
+import javax.net.ssl.SSLSocket;
+import javax.net.ssl.SSLSocketFactory;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
@@ -43,6 +50,7 @@ import org.junit.Test;
  * corresponding HTTPS URL.
  */
 public class TestSSLHttpServer extends HttpServerFunctionalTest {
+
   private static final String BASEDIR = System.getProperty("test.build.dir",
       "target/test-dir") + "/" + TestSSLHttpServer.class.getSimpleName();
 
@@ -52,6 +60,23 @@ public class TestSSLHttpServer extends HttpServerFunctionalTest {
   private static String keystoresDir;
   private static String sslConfDir;
   private static SSLFactory clientSslFactory;
+  private static final String excludeCiphers = "TLS_ECDHE_RSA_WITH_RC4_128_SHA,"
+      + "SSL_DHE_RSA_EXPORT_WITH_DES40_CBC_SHA,"
+      + "SSL_RSA_WITH_DES_CBC_SHA,"
+      + "SSL_DHE_RSA_WITH_DES_CBC_SHA,"
+      + "SSL_RSA_EXPORT_WITH_RC4_40_MD5,"
+      + "SSL_RSA_EXPORT_WITH_DES40_CBC_SHA,"
+      + "SSL_RSA_WITH_RC4_128_MD5";
+  private static final String oneEnabledCiphers = excludeCiphers
+      + ",TLS_RSA_WITH_AES_128_CBC_SHA";
+  private static final String exclusiveEnabledCiphers
+      = "TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA,"
+      + "TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA,"
+      + "TLS_RSA_WITH_AES_128_CBC_SHA,"
+      + "TLS_ECDH_ECDSA_WITH_AES_128_CBC_SHA,"
+      + "TLS_ECDH_RSA_WITH_AES_128_CBC_SHA,"
+      + "TLS_DHE_RSA_WITH_AES_128_CBC_SHA,"
+      + "TLS_DHE_DSS_WITH_AES_128_CBC_SHA";
 
   @BeforeClass
   public static void setup() throws Exception {
@@ -64,7 +89,8 @@ public class TestSSLHttpServer extends HttpServerFunctionalTest {
     keystoresDir = new File(BASEDIR).getAbsolutePath();
     sslConfDir = KeyStoreTestUtil.getClasspathDir(TestSSLHttpServer.class);
 
-    KeyStoreTestUtil.setupSSLConfig(keystoresDir, sslConfDir, conf, false);
+    KeyStoreTestUtil.setupSSLConfig(keystoresDir, sslConfDir, conf, false, true,
+        excludeCiphers);
     Configuration sslConf = new Configuration(false);
     sslConf.addResource("ssl-server.xml");
     sslConf.addResource("ssl-client.xml");
@@ -82,7 +108,9 @@ public class TestSSLHttpServer extends HttpServerFunctionalTest {
             sslConf.get("ssl.server.keystore.type", "jks"))
         .trustStore(sslConf.get("ssl.server.truststore.location"),
             sslConf.get("ssl.server.truststore.password"),
-            sslConf.get("ssl.server.truststore.type", "jks")).build();
+            sslConf.get("ssl.server.truststore.type", "jks"))
+        .excludeCiphers(
+            sslConf.get("ssl.server.exclude.cipher.list")).build();
     server.addServlet("echo", "/echo", TestHttpServer.EchoServlet.class);
     server.addServlet("longheader", "/longheader", LongHeaderServlet.class);
     server.start();
@@ -107,10 +135,10 @@ public class TestSSLHttpServer extends HttpServerFunctionalTest {
   }
 
   /**
-   *  Test that verifies headers can be up to 64K long.
+   * Test that verifies headers can be up to 64K long. The test adds a 63K
-   *  The test adds a 63K header leaving 1K for other headers.
+   * header leaving 1K for other headers. This is because the header buffer
-   *  This is because the header buffer setting is for ALL headers,
+   * setting is for ALL headers, names and values included.
-   *  names and values included. */
+   */
   @Test
   public void testLongHeader() throws Exception {
     URL url = new URL(baseUrl, "/longheader");
@@ -128,4 +156,162 @@ public class TestSSLHttpServer extends HttpServerFunctionalTest {
     return out.toString();
   }
 
+  /**
+   * Test that verifies that excluded ciphers (SSL_RSA_WITH_RC4_128_SHA,
+   * TLS_ECDH_ECDSA_WITH_RC4_128_SHA,TLS_ECDH_RSA_WITH_RC4_128_SHA,
+   * TLS_ECDHE_ECDSA_WITH_RC4_128_SHA,TLS_ECDHE_RSA_WITH_RC4_128_SHA) are not
+   * available for negotiation during SSL connection.
+   */
+  @Test
+  public void testExcludedCiphers() throws Exception {
+    URL url = new URL(baseUrl, "/echo?a=b&c=d");
+    HttpsURLConnection conn = (HttpsURLConnection) url.openConnection();
+    SSLSocketFactory sslSocketF = clientSslFactory.createSSLSocketFactory();
+    PrefferedCipherSSLSocketFactory testPreferredCipherSSLSocketF
+        = new PrefferedCipherSSLSocketFactory(sslSocketF,
+            excludeCiphers.split(","));
+    conn.setSSLSocketFactory(testPreferredCipherSSLSocketF);
+    assertFalse("excludedCipher list is empty", excludeCiphers.isEmpty());
+    try {
+      InputStream in = conn.getInputStream();
+      ByteArrayOutputStream out = new ByteArrayOutputStream();
+      IOUtils.copyBytes(in, out, 1024);
+      fail("No Ciphers in common, SSLHandshake must fail.");
+    } catch (SSLHandshakeException ex) {
+      LOG.info("No Ciphers in common, expected succesful test result.", ex);
+    }
+  }
+
+  /** Test that verified that additionally included cipher
+   * TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA is only available cipher for working
+   * TLS connection from client to server disabled for all other common ciphers.
+   */
+  @Test
+  public void testOneEnabledCiphers() throws Exception {
+    URL url = new URL(baseUrl, "/echo?a=b&c=d");
+    HttpsURLConnection conn = (HttpsURLConnection) url.openConnection();
+    SSLSocketFactory sslSocketF = clientSslFactory.createSSLSocketFactory();
+    PrefferedCipherSSLSocketFactory testPreferredCipherSSLSocketF
+        = new PrefferedCipherSSLSocketFactory(sslSocketF,
+            oneEnabledCiphers.split(","));
+    conn.setSSLSocketFactory(testPreferredCipherSSLSocketF);
+    assertFalse("excludedCipher list is empty", oneEnabledCiphers.isEmpty());
+    try {
+      InputStream in = conn.getInputStream();
+      ByteArrayOutputStream out = new ByteArrayOutputStream();
+      IOUtils.copyBytes(in, out, 1024);
+      assertEquals(out.toString(), "a:b\nc:d\n");
+      LOG.info("Atleast one additional enabled cipher than excluded ciphers,"
+          + " expected successful test result.");
+    } catch (SSLHandshakeException ex) {
+      fail("Atleast one additional cipher available for successful handshake."
+          + " Unexpected test failure: " + ex);
+    }
+  }
+
+  /** Test verifies that mutually exclusive server's disabled cipher suites and
+   * client's enabled cipher suites can successfully establish TLS connection.
+   */
+  @Test
+  public void testExclusiveEnabledCiphers() throws Exception {
+    URL url = new URL(baseUrl, "/echo?a=b&c=d");
+    HttpsURLConnection conn = (HttpsURLConnection) url.openConnection();
+    SSLSocketFactory sslSocketF = clientSslFactory.createSSLSocketFactory();
+    PrefferedCipherSSLSocketFactory testPreferredCipherSSLSocketF
+        = new PrefferedCipherSSLSocketFactory(sslSocketF,
+            exclusiveEnabledCiphers.split(","));
+    conn.setSSLSocketFactory(testPreferredCipherSSLSocketF);
+    assertFalse("excludedCipher list is empty",
+        exclusiveEnabledCiphers.isEmpty());
+    try {
+      InputStream in = conn.getInputStream();
+      ByteArrayOutputStream out = new ByteArrayOutputStream();
+      IOUtils.copyBytes(in, out, 1024);
+      assertEquals(out.toString(), "a:b\nc:d\n");
+      LOG.info("Atleast one additional enabled cipher than excluded ciphers,"
+          + " expected successful test result.");
+    } catch (SSLHandshakeException ex) {
+      fail("Atleast one additional cipher available for successful handshake."
+          + " Unexpected test failure: " + ex);
+    }
+  }
+
+  private class PrefferedCipherSSLSocketFactory extends SSLSocketFactory {
+    private final SSLSocketFactory delegateSocketFactory;
+    private final String[] enabledCipherSuites;
+
+    public PrefferedCipherSSLSocketFactory(SSLSocketFactory sslSocketFactory,
+        String[] pEnabledCipherSuites) {
+      delegateSocketFactory = sslSocketFactory;
+      if (null != pEnabledCipherSuites && pEnabledCipherSuites.length > 0) {
+        enabledCipherSuites = pEnabledCipherSuites;
+      } else {
+        enabledCipherSuites = null;
+      }
+    }
+
+    @Override
+    public String[] getDefaultCipherSuites() {
+      return delegateSocketFactory.getDefaultCipherSuites();
+    }
+
+    @Override
+    public String[] getSupportedCipherSuites() {
+      return delegateSocketFactory.getSupportedCipherSuites();
+    }
+
+    @Override
+    public Socket createSocket(Socket socket, String string, int i, boolean bln)
+        throws IOException {
+      SSLSocket sslSocket = (SSLSocket) delegateSocketFactory.createSocket(
+          socket, string, i, bln);
+      if (null != enabledCipherSuites) {
+        sslSocket.setEnabledCipherSuites(enabledCipherSuites);
+      }
+      return sslSocket;
+    }
+
+    @Override
+    public Socket createSocket(String string, int i) throws IOException,
+        UnknownHostException {
+      SSLSocket sslSocket = (SSLSocket) delegateSocketFactory.createSocket(
+          string, i);
+      if (null != enabledCipherSuites) {
+        sslSocket.setEnabledCipherSuites(enabledCipherSuites);
+      }
+      return sslSocket;
+    }
+
+    @Override
+    public Socket createSocket(String string, int i, InetAddress ia, int i1)
+        throws IOException, UnknownHostException {
+      SSLSocket sslSocket = (SSLSocket) delegateSocketFactory.createSocket(
+          string, i, ia, i1);
+      if (null != enabledCipherSuites) {
+        sslSocket.setEnabledCipherSuites(enabledCipherSuites);
+      }
+      return sslSocket;
+    }
+
+    @Override
+    public Socket createSocket(InetAddress ia, int i) throws IOException {
+      SSLSocket sslSocket = (SSLSocket) delegateSocketFactory.createSocket(ia,
+          i);
+      if (null != enabledCipherSuites) {
+        sslSocket.setEnabledCipherSuites(enabledCipherSuites);
+      }
+      return sslSocket;
+    }
+
+    @Override
+    public Socket createSocket(InetAddress ia, int i, InetAddress ia1, int i1)
+        throws IOException {
+      SSLSocket sslSocket = (SSLSocket) delegateSocketFactory.createSocket(ia,
+          i, ia1, i1);
+      if (null != enabledCipherSuites) {
+        sslSocket.setEnabledCipherSuites(enabledCipherSuites);
+      }
+      return sslSocket;
+    }
+  }
 }

hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/ssl/KeyStoreTestUtil.java

@@ -221,11 +221,33 @@ public class KeyStoreTestUtil {
    * @param useClientCert boolean true to make the client present a cert in the
    * SSL handshake
    * @param trustStore boolean true to create truststore, false not to create it
+   * @throws java.lang.Exception
    */
   public static void setupSSLConfig(String keystoresDir, String sslConfDir,
                                     Configuration conf, boolean useClientCert,
       boolean trustStore)
     throws Exception {
+    setupSSLConfig(keystoresDir, sslConfDir, conf, useClientCert, true,"");
+  }
+
+    /**
+     * Performs complete setup of SSL configuration in preparation for testing an
+     * SSLFactory.  This includes keys, certs, keystores, truststores, the server
+     * SSL configuration file, the client SSL configuration file, and the master
+     * configuration file read by the SSLFactory.
+     *
+     * @param keystoresDir
+     * @param sslConfDir
+     * @param conf
+     * @param useClientCert
+     * @param trustStore
+     * @param excludeCiphers
+     * @throws Exception
+     */
+    public static void setupSSLConfig(String keystoresDir, String sslConfDir,
+                                    Configuration conf, boolean useClientCert,
+      boolean trustStore, String excludeCiphers)
+    throws Exception {
     String clientKS = keystoresDir + "/clientKS.jks";
     String clientPassword = "clientP";
     String serverKS = keystoresDir + "/serverKS.jks";
@@ -262,9 +284,9 @@ public class KeyStoreTestUtil {
     }
 
     Configuration clientSSLConf = createClientSSLConfig(clientKS, clientPassword,
-      clientPassword, trustKS);
+      clientPassword, trustKS, excludeCiphers);
     Configuration serverSSLConf = createServerSSLConfig(serverKS, serverPassword,
-      serverPassword, trustKS);
+      serverPassword, trustKS, excludeCiphers);
 
     saveConfig(sslClientConfFile, clientSSLConf);
     saveConfig(sslServerConfFile, serverSSLConf);
@@ -288,9 +310,26 @@ public class KeyStoreTestUtil {
    */
   public static Configuration createClientSSLConfig(String clientKS,
       String password, String keyPassword, String trustKS) {
-    Configuration clientSSLConf = createSSLConfig(SSLFactory.Mode.CLIENT,
+    return createSSLConfig(SSLFactory.Mode.CLIENT,
-      clientKS, password, keyPassword, trustKS);
+      clientKS, password, keyPassword, trustKS, "");
-    return clientSSLConf;
+  }
+
+  /**
+   * Creates SSL configuration for a client.
+   *
+   * @param clientKS String client keystore file
+   * @param password String store password, or null to avoid setting store
+   *   password
+   * @param keyPassword String key password, or null to avoid setting key
+   *   password
+   * @param trustKS String truststore file
+   * @param excludeCiphers String comma separated ciphers to exclude
+   * @return Configuration for client SSL
+   */
+    public static Configuration createClientSSLConfig(String clientKS,
+      String password, String keyPassword, String trustKS, String excludeCiphers) {
+    return createSSLConfig(SSLFactory.Mode.CLIENT,
+      clientKS, password, keyPassword, trustKS, excludeCiphers);
   }
 
   /**
@@ -303,12 +342,31 @@ public class KeyStoreTestUtil {
    *   password
    * @param trustKS String truststore file
    * @return Configuration for server SSL
+   * @throws java.io.IOException
    */
   public static Configuration createServerSSLConfig(String serverKS,
       String password, String keyPassword, String trustKS) throws IOException {
-    Configuration serverSSLConf = createSSLConfig(SSLFactory.Mode.SERVER,
+    return createSSLConfig(SSLFactory.Mode.SERVER,
-      serverKS, password, keyPassword, trustKS);
+      serverKS, password, keyPassword, trustKS, "");
-    return serverSSLConf;
+  }
+
+  /**
+   * Creates SSL configuration for a server.
+   *
+   * @param serverKS String server keystore file
+   * @param password String store password, or null to avoid setting store
+   * password
+   * @param keyPassword String key password, or null to avoid setting key
+   * password
+   * @param trustKS String truststore file
+   * @param excludeCiphers String comma separated ciphers to exclude
+   * @return
+   * @throws IOException
+   */
+    public static Configuration createServerSSLConfig(String serverKS,
+      String password, String keyPassword, String trustKS, String excludeCiphers) throws IOException {
+    return createSSLConfig(SSLFactory.Mode.SERVER,
+      serverKS, password, keyPassword, trustKS, excludeCiphers);
   }
 
   /**
@@ -324,7 +382,7 @@ public class KeyStoreTestUtil {
    * @return Configuration for SSL
    */
   private static Configuration createSSLConfig(SSLFactory.Mode mode,
-      String keystore, String password, String keyPassword, String trustKS) {
+    String keystore, String password, String keyPassword, String trustKS, String excludeCiphers) {
     String trustPassword = "trustP";
 
     Configuration sslConf = new Configuration(false);
@@ -350,6 +408,11 @@ public class KeyStoreTestUtil {
         FileBasedKeyStoresFactory.SSL_TRUSTSTORE_PASSWORD_TPL_KEY),
         trustPassword);
     }
+    if(null != excludeCiphers && !excludeCiphers.isEmpty()) {
+      sslConf.set(FileBasedKeyStoresFactory.resolvePropertyName(mode,
+      FileBasedKeyStoresFactory.SSL_EXCLUDE_CIPHER_LIST),
+        excludeCiphers);
+    }
     sslConf.set(FileBasedKeyStoresFactory.resolvePropertyName(mode,
       FileBasedKeyStoresFactory.SSL_TRUSTSTORE_RELOAD_INTERVAL_TPL_KEY), "1000");
 

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java

@@ -1682,7 +1682,9 @@ public class DFSUtil {
             sslConf.get("ssl.server.keystore.type", "jks"))
         .trustStore(sslConf.get("ssl.server.truststore.location"),
             getPassword(sslConf, DFS_SERVER_HTTPS_TRUSTSTORE_PASSWORD_KEY),
-            sslConf.get("ssl.server.truststore.type", "jks"));
+            sslConf.get("ssl.server.truststore.type", "jks"))
+        .excludeCiphers(
+            sslConf.get("ssl.server.exclude.cipher.list"));
   }
 
   /**

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/util/WebAppUtils.java

@@ -352,7 +352,9 @@ public class WebAppUtils {
             sslConf.get("ssl.server.keystore.type", "jks"))
         .trustStore(sslConf.get("ssl.server.truststore.location"),
             getPassword(sslConf, WEB_APP_TRUSTSTORE_PASSWORD_KEY),
-            sslConf.get("ssl.server.truststore.type", "jks"));
+            sslConf.get("ssl.server.truststore.type", "jks"))
+        .excludeCiphers(
+            sslConf.get("ssl.server.exclude.cipher.list"));
   }
 
   /**