Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HADOOP-12668. Support excluding weak Ciphers in HttpServer2 through ssl-server.conf. Contributed by Vijay Singh. 

Change-Id: Ie46a5427d29188935427f67e55203c19fcd83335
This commit is contained in:
Zhe Zhang 2016-02-22 14:12:33 -08:00
parent d4f5fc23b2
commit a2fdfff02d
9 changed files with 302 additions and 18 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
hadoop-common-project/hadoop-common/CHANGES.txt
View File

@ -1140,6 +1140,9 @@ Release 2.8.0 - UNRELEASED
HADOOP-11613. Remove commons-httpclient dependency from hadoop-azure.
(Masatake Iwasaki via cnauroth)
HADOOP-12668. Support excluding weak Ciphers in HttpServer2 through
ssl-server.conf. (Vijay Singh via zhz)
OPTIMIZATIONS
HADOOP-11785. Reduce the number of listStatus operation in distcp

10
hadoop-common-project/hadoop-common/src/main/conf/ssl-server.xml.example
View File

@ -75,4 +75,14 @@
</description>
</property>
<property>
<name>ssl.server.exclude.cipher.list</name>
<value>TLS_ECDHE_RSA_WITH_RC4_128_SHA,SSL_DHE_RSA_EXPORT_WITH_DES40_CBC_SHA,
SSL_RSA_WITH_DES_CBC_SHA,SSL_DHE_RSA_WITH_DES_CBC_SHA,
SSL_RSA_EXPORT_WITH_RC4_40_MD5,SSL_RSA_EXPORT_WITH_DES40_CBC_SHA,
SSL_RSA_WITH_RC4_128_MD5</value>
<description>Optional. The weak security cipher suites that you want excluded
from SSL communication.</description>
</property>
</configuration>

12
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpServer2.java
View File

@ -172,6 +172,7 @@ public final class HttpServer2 implements FilterContainer {
private String hostName;
private boolean disallowFallbackToRandomSignerSecretProvider;
private String authFilterConfigurationPrefix = "hadoop.http.authentication.";
private String excludeCiphers;
public Builder setName(String name){
this.name = name;
@ -276,6 +277,11 @@ public final class HttpServer2 implements FilterContainer {
return this;
}
public Builder excludeCiphers(String pExcludeCiphers) {
this.excludeCiphers = pExcludeCiphers;
return this;
}
public HttpServer2 build() throws IOException {
Preconditions.checkNotNull(name, "name is not set");
Preconditions.checkState(!endpoints.isEmpty(), "No endpoints specified");
@ -316,6 +322,12 @@ public final class HttpServer2 implements FilterContainer {
c.setTruststoreType(trustStoreType);
c.setTrustPassword(trustStorePassword);
}
if(null != excludeCiphers && !excludeCiphers.isEmpty()) {
c.setExcludeCipherSuites(excludeCiphers.split(","));
LOG.info("Excluded Cipher List:" + excludeCiphers);
}
listener = c;
} else {

2
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ssl/FileBasedKeyStoresFactory.java
View File

@ -67,6 +67,8 @@ public class FileBasedKeyStoresFactory implements KeyStoresFactory {
"ssl.{0}.truststore.password";
public static final String SSL_TRUSTSTORE_TYPE_TPL_KEY =
"ssl.{0}.truststore.type";
public static final String SSL_EXCLUDE_CIPHER_LIST =
"ssl.{0}.exclude.cipher.list";
/**
* Default format of the keystore files.

5
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/TestHttpCookieFlag.java
View File

@ -102,7 +102,10 @@ public class TestHttpCookieFlag {
sslConf.get("ssl.server.keystore.type", "jks"))
.trustStore(sslConf.get("ssl.server.truststore.location"),
sslConf.get("ssl.server.truststore.password"),
sslConf.get("ssl.server.truststore.type", "jks")).build();
sslConf.get("ssl.server.truststore.type", "jks"))
.excludeCiphers(
sslConf.get("ssl.server.exclude.cipher.list"))
.build();
server.addServlet("echo", "/echo", TestHttpServer.EchoServlet.class);
server.start();
}

199
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/TestSSLHttpServer.java
View File

@ -19,11 +19,18 @@ package org.apache.hadoop.http;
import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.net.InetAddress;
import java.net.Socket;
import java.net.URI;
import java.net.URL;
import java.net.UnknownHostException;
import javax.net.ssl.HttpsURLConnection;
import javax.net.ssl.SSLHandshakeException;
import javax.net.ssl.SSLSocket;
import javax.net.ssl.SSLSocketFactory;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@ -43,6 +50,7 @@ import org.junit.Test;
* corresponding HTTPS URL.
*/
public class TestSSLHttpServer extends HttpServerFunctionalTest {
private static final String BASEDIR = System.getProperty("test.build.dir",
"target/test-dir") + "/" + TestSSLHttpServer.class.getSimpleName();
@ -52,6 +60,23 @@ public class TestSSLHttpServer extends HttpServerFunctionalTest {
private static String keystoresDir;
private static String sslConfDir;
private static SSLFactory clientSslFactory;
private static final String excludeCiphers = "TLS_ECDHE_RSA_WITH_RC4_128_SHA,"
+ "SSL_DHE_RSA_EXPORT_WITH_DES40_CBC_SHA,"
+ "SSL_RSA_WITH_DES_CBC_SHA,"
+ "SSL_DHE_RSA_WITH_DES_CBC_SHA,"
+ "SSL_RSA_EXPORT_WITH_RC4_40_MD5,"
+ "SSL_RSA_EXPORT_WITH_DES40_CBC_SHA,"
+ "SSL_RSA_WITH_RC4_128_MD5";
private static final String oneEnabledCiphers = excludeCiphers
+ ",TLS_RSA_WITH_AES_128_CBC_SHA";
private static final String exclusiveEnabledCiphers
= "TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA,"
+ "TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA,"
+ "TLS_RSA_WITH_AES_128_CBC_SHA,"
+ "TLS_ECDH_ECDSA_WITH_AES_128_CBC_SHA,"
+ "TLS_ECDH_RSA_WITH_AES_128_CBC_SHA,"
+ "TLS_DHE_RSA_WITH_AES_128_CBC_SHA,"
+ "TLS_DHE_DSS_WITH_AES_128_CBC_SHA";
@BeforeClass
public static void setup() throws Exception {
@ -64,7 +89,9 @@ public class TestSSLHttpServer extends HttpServerFunctionalTest {
keystoresDir = new File(BASEDIR).getAbsolutePath();
sslConfDir = KeyStoreTestUtil.getClasspathDir(TestSSLHttpServer.class);
KeyStoreTestUtil.setupSSLConfig(keystoresDir, sslConfDir, conf, false);
KeyStoreTestUtil.setupSSLConfig(keystoresDir, sslConfDir, conf, false, true,
excludeCiphers);
Configuration sslConf = KeyStoreTestUtil.getSslConfig();
clientSslFactory = new SSLFactory(SSLFactory.Mode.CLIENT, sslConf);
@ -80,7 +107,9 @@ public class TestSSLHttpServer extends HttpServerFunctionalTest {
sslConf.get("ssl.server.keystore.type", "jks"))
.trustStore(sslConf.get("ssl.server.truststore.location"),
sslConf.get("ssl.server.truststore.password"),
sslConf.get("ssl.server.truststore.type", "jks")).build();
sslConf.get("ssl.server.truststore.type", "jks"))
.excludeCiphers(
sslConf.get("ssl.server.exclude.cipher.list")).build();
server.addServlet("echo", "/echo", TestHttpServer.EchoServlet.class);
server.addServlet("longheader", "/longheader", LongHeaderServlet.class);
server.start();
@ -105,10 +134,10 @@ public class TestSSLHttpServer extends HttpServerFunctionalTest {
}
/**
* Test that verifies headers can be up to 64K long.
* The test adds a 63K header leaving 1K for other headers.
* This is because the header buffer setting is for ALL headers,
* names and values included. */
* Test that verifies headers can be up to 64K long. The test adds a 63K
* header leaving 1K for other headers. This is because the header buffer
* setting is for ALL headers, names and values included.
*/
@Test
public void testLongHeader() throws Exception {
URL url = new URL(baseUrl, "/longheader");
@ -126,4 +155,162 @@ public class TestSSLHttpServer extends HttpServerFunctionalTest {
return out.toString();
}
/**
* Test that verifies that excluded ciphers (SSL_RSA_WITH_RC4_128_SHA,
* TLS_ECDH_ECDSA_WITH_RC4_128_SHA,TLS_ECDH_RSA_WITH_RC4_128_SHA,
* TLS_ECDHE_ECDSA_WITH_RC4_128_SHA,TLS_ECDHE_RSA_WITH_RC4_128_SHA) are not
* available for negotiation during SSL connection.
*/
@Test
public void testExcludedCiphers() throws Exception {
URL url = new URL(baseUrl, "/echo?a=b&c=d");
HttpsURLConnection conn = (HttpsURLConnection) url.openConnection();
SSLSocketFactory sslSocketF = clientSslFactory.createSSLSocketFactory();
PrefferedCipherSSLSocketFactory testPreferredCipherSSLSocketF
= new PrefferedCipherSSLSocketFactory(sslSocketF,
excludeCiphers.split(","));
conn.setSSLSocketFactory(testPreferredCipherSSLSocketF);
assertFalse("excludedCipher list is empty", excludeCiphers.isEmpty());
try {
InputStream in = conn.getInputStream();
ByteArrayOutputStream out = new ByteArrayOutputStream();
IOUtils.copyBytes(in, out, 1024);
fail("No Ciphers in common, SSLHandshake must fail.");
} catch (SSLHandshakeException ex) {
LOG.info("No Ciphers in common, expected succesful test result.", ex);
}
}
/** Test that verified that additionally included cipher
* TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA is only available cipher for working
* TLS connection from client to server disabled for all other common ciphers.
*/
@Test
public void testOneEnabledCiphers() throws Exception {
URL url = new URL(baseUrl, "/echo?a=b&c=d");
HttpsURLConnection conn = (HttpsURLConnection) url.openConnection();
SSLSocketFactory sslSocketF = clientSslFactory.createSSLSocketFactory();
PrefferedCipherSSLSocketFactory testPreferredCipherSSLSocketF
= new PrefferedCipherSSLSocketFactory(sslSocketF,
oneEnabledCiphers.split(","));
conn.setSSLSocketFactory(testPreferredCipherSSLSocketF);
assertFalse("excludedCipher list is empty", oneEnabledCiphers.isEmpty());
try {
InputStream in = conn.getInputStream();
ByteArrayOutputStream out = new ByteArrayOutputStream();
IOUtils.copyBytes(in, out, 1024);
assertEquals(out.toString(), "a:b\nc:d\n");
LOG.info("Atleast one additional enabled cipher than excluded ciphers,"
+ " expected successful test result.");
} catch (SSLHandshakeException ex) {
fail("Atleast one additional cipher available for successful handshake."
+ " Unexpected test failure: " + ex);
}
}
/** Test verifies that mutually exclusive server's disabled cipher suites and
* client's enabled cipher suites can successfully establish TLS connection.
*/
@Test
public void testExclusiveEnabledCiphers() throws Exception {
URL url = new URL(baseUrl, "/echo?a=b&c=d");
HttpsURLConnection conn = (HttpsURLConnection) url.openConnection();
SSLSocketFactory sslSocketF = clientSslFactory.createSSLSocketFactory();
PrefferedCipherSSLSocketFactory testPreferredCipherSSLSocketF
= new PrefferedCipherSSLSocketFactory(sslSocketF,
exclusiveEnabledCiphers.split(","));
conn.setSSLSocketFactory(testPreferredCipherSSLSocketF);
assertFalse("excludedCipher list is empty",
exclusiveEnabledCiphers.isEmpty());
try {
InputStream in = conn.getInputStream();
ByteArrayOutputStream out = new ByteArrayOutputStream();
IOUtils.copyBytes(in, out, 1024);
assertEquals(out.toString(), "a:b\nc:d\n");
LOG.info("Atleast one additional enabled cipher than excluded ciphers,"
+ " expected successful test result.");
} catch (SSLHandshakeException ex) {
fail("Atleast one additional cipher available for successful handshake."
+ " Unexpected test failure: " + ex);
}
}
private class PrefferedCipherSSLSocketFactory extends SSLSocketFactory {
private final SSLSocketFactory delegateSocketFactory;
private final String[] enabledCipherSuites;
public PrefferedCipherSSLSocketFactory(SSLSocketFactory sslSocketFactory,
String[] pEnabledCipherSuites) {
delegateSocketFactory = sslSocketFactory;
if (null != pEnabledCipherSuites && pEnabledCipherSuites.length > 0) {
enabledCipherSuites = pEnabledCipherSuites;
} else {
enabledCipherSuites = null;
}
}
@Override
public String[] getDefaultCipherSuites() {
return delegateSocketFactory.getDefaultCipherSuites();
}
@Override
public String[] getSupportedCipherSuites() {
return delegateSocketFactory.getSupportedCipherSuites();
}
@Override
public Socket createSocket(Socket socket, String string, int i, boolean bln)
throws IOException {
SSLSocket sslSocket = (SSLSocket) delegateSocketFactory.createSocket(
socket, string, i, bln);
if (null != enabledCipherSuites) {
sslSocket.setEnabledCipherSuites(enabledCipherSuites);
}
return sslSocket;
}
@Override
public Socket createSocket(String string, int i) throws IOException,
UnknownHostException {
SSLSocket sslSocket = (SSLSocket) delegateSocketFactory.createSocket(
string, i);
if (null != enabledCipherSuites) {
sslSocket.setEnabledCipherSuites(enabledCipherSuites);
}
return sslSocket;
}
@Override
public Socket createSocket(String string, int i, InetAddress ia, int i1)
throws IOException, UnknownHostException {
SSLSocket sslSocket = (SSLSocket) delegateSocketFactory.createSocket(
string, i, ia, i1);
if (null != enabledCipherSuites) {
sslSocket.setEnabledCipherSuites(enabledCipherSuites);
}
return sslSocket;
}
@Override
public Socket createSocket(InetAddress ia, int i) throws IOException {
SSLSocket sslSocket = (SSLSocket) delegateSocketFactory.createSocket(ia,
i);
if (null != enabledCipherSuites) {
sslSocket.setEnabledCipherSuites(enabledCipherSuites);
}
return sslSocket;
}
@Override
public Socket createSocket(InetAddress ia, int i, InetAddress ia1, int i1)
throws IOException {
SSLSocket sslSocket = (SSLSocket) delegateSocketFactory.createSocket(ia,
i, ia1, i1);
if (null != enabledCipherSuites) {
sslSocket.setEnabledCipherSuites(enabledCipherSuites);
}
return sslSocket;
}
}
}

81
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/ssl/KeyStoreTestUtil.java
View File

@ -218,11 +218,33 @@ public class KeyStoreTestUtil {
* @param useClientCert boolean true to make the client present a cert in the
* SSL handshake
* @param trustStore boolean true to create truststore, false not to create it
* @throws java.lang.Exception
*/
public static void setupSSLConfig(String keystoresDir, String sslConfDir,
Configuration conf, boolean useClientCert,
boolean trustStore)
throws Exception {
setupSSLConfig(keystoresDir, sslConfDir, conf, useClientCert, true,"");
}
/**
* Performs complete setup of SSL configuration in preparation for testing an
* SSLFactory. This includes keys, certs, keystores, truststores, the server
* SSL configuration file, the client SSL configuration file, and the master
* configuration file read by the SSLFactory.
*
* @param keystoresDir
* @param sslConfDir
* @param conf
* @param useClientCert
* @param trustStore
* @param excludeCiphers
* @throws Exception
*/
public static void setupSSLConfig(String keystoresDir, String sslConfDir,
Configuration conf, boolean useClientCert,
boolean trustStore, String excludeCiphers)
throws Exception {
String clientKS = keystoresDir + "/clientKS.jks";
String clientPassword = "clientP";
String serverKS = keystoresDir + "/serverKS.jks";
@ -259,9 +281,9 @@ public class KeyStoreTestUtil {
}
Configuration clientSSLConf = createClientSSLConfig(clientKS, clientPassword,
clientPassword, trustKS);
clientPassword, trustKS, excludeCiphers);
Configuration serverSSLConf = createServerSSLConfig(serverKS, serverPassword,
serverPassword, trustKS);
serverPassword, trustKS, excludeCiphers);
saveConfig(sslClientConfFile, clientSSLConf);
saveConfig(sslServerConfFile, serverSSLConf);
@ -285,9 +307,26 @@ public class KeyStoreTestUtil {
*/
public static Configuration createClientSSLConfig(String clientKS,
String password, String keyPassword, String trustKS) {
Configuration clientSSLConf = createSSLConfig(SSLFactory.Mode.CLIENT,
clientKS, password, keyPassword, trustKS);
return clientSSLConf;
return createSSLConfig(SSLFactory.Mode.CLIENT,
clientKS, password, keyPassword, trustKS, "");
}
/**
* Creates SSL configuration for a client.
*
* @param clientKS String client keystore file
* @param password String store password, or null to avoid setting store
* password
* @param keyPassword String key password, or null to avoid setting key
* password
* @param trustKS String truststore file
* @param excludeCiphers String comma separated ciphers to exclude
* @return Configuration for client SSL
*/
public static Configuration createClientSSLConfig(String clientKS,
String password, String keyPassword, String trustKS, String excludeCiphers) {
return createSSLConfig(SSLFactory.Mode.CLIENT,
clientKS, password, keyPassword, trustKS, excludeCiphers);
}
/**
@ -300,12 +339,31 @@ public class KeyStoreTestUtil {
* password
* @param trustKS String truststore file
* @return Configuration for server SSL
* @throws java.io.IOException
*/
public static Configuration createServerSSLConfig(String serverKS,
String password, String keyPassword, String trustKS) throws IOException {
Configuration serverSSLConf = createSSLConfig(SSLFactory.Mode.SERVER,
serverKS, password, keyPassword, trustKS);
return serverSSLConf;
return createSSLConfig(SSLFactory.Mode.SERVER,
serverKS, password, keyPassword, trustKS, "");
}
/**
* Creates SSL configuration for a server.
*
* @param serverKS String server keystore file
* @param password String store password, or null to avoid setting store
* password
* @param keyPassword String key password, or null to avoid setting key
* password
* @param trustKS String truststore file
* @param excludeCiphers String comma separated ciphers to exclude
* @return
* @throws IOException
*/
public static Configuration createServerSSLConfig(String serverKS,
String password, String keyPassword, String trustKS, String excludeCiphers) throws IOException {
return createSSLConfig(SSLFactory.Mode.SERVER,
serverKS, password, keyPassword, trustKS, excludeCiphers);
}
/**
@ -357,7 +415,7 @@ public class KeyStoreTestUtil {
* @return Configuration for SSL
*/
private static Configuration createSSLConfig(SSLFactory.Mode mode,
String keystore, String password, String keyPassword, String trustKS) {
String keystore, String password, String keyPassword, String trustKS, String excludeCiphers) {
String trustPassword = "trustP";
Configuration sslConf = new Configuration(false);
@ -383,6 +441,11 @@ public class KeyStoreTestUtil {
FileBasedKeyStoresFactory.SSL_TRUSTSTORE_PASSWORD_TPL_KEY),
trustPassword);
}
if(null != excludeCiphers && !excludeCiphers.isEmpty()) {
sslConf.set(FileBasedKeyStoresFactory.resolvePropertyName(mode,
FileBasedKeyStoresFactory.SSL_EXCLUDE_CIPHER_LIST),
excludeCiphers);
}
sslConf.set(FileBasedKeyStoresFactory.resolvePropertyName(mode,
FileBasedKeyStoresFactory.SSL_TRUSTSTORE_RELOAD_INTERVAL_TPL_KEY), "1000");

4
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java
View File

@ -1295,7 +1295,9 @@ public class DFSUtil {
sslConf.get("ssl.server.keystore.type", "jks"))
.trustStore(sslConf.get("ssl.server.truststore.location"),
getPassword(sslConf, DFS_SERVER_HTTPS_TRUSTSTORE_PASSWORD_KEY),
sslConf.get("ssl.server.truststore.type", "jks"));
sslConf.get("ssl.server.truststore.type", "jks"))
.excludeCiphers(
sslConf.get("ssl.server.exclude.cipher.list"));
}
/**

4
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/util/WebAppUtils.java
View File

@ -352,7 +352,9 @@ public class WebAppUtils {
sslConf.get("ssl.server.keystore.type", "jks"))
.trustStore(sslConf.get("ssl.server.truststore.location"),
getPassword(sslConf, WEB_APP_TRUSTSTORE_PASSWORD_KEY),
sslConf.get("ssl.server.truststore.type", "jks"));
sslConf.get("ssl.server.truststore.type", "jks"))
.excludeCiphers(
sslConf.get("ssl.server.exclude.cipher.list"));
}
/**