HADOOP-9977. Merging change r1525657 from trunk to branch-2.

git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/branch-2@1525658 13f79535-47bb-0310-9956-ffa450edef68

hadoop-common-project/hadoop-common/CHANGES.txt

@@ -228,6 +228,9 @@ Release 2.1.1-beta - 2013-09-23
     HADOOP-9961. versions of a few transitive dependencies diverged between hadoop 
     subprojects. (rvs via tucu)
 
+    HADOOP-9977. Hadoop services won't start with different keypass and
+    keystorepass when https is enabled. (cnauroth)
+
 Release 2.1.0-beta - 2013-08-22
 
   INCOMPATIBLE CHANGES

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ssl/FileBasedKeyStoresFactory.java

@@ -53,6 +53,8 @@ public class FileBasedKeyStoresFactory implements KeyStoresFactory {
     "ssl.{0}.keystore.location";
   public static final String SSL_KEYSTORE_PASSWORD_TPL_KEY =
     "ssl.{0}.keystore.password";
+  public static final String SSL_KEYSTORE_KEYPASSWORD_TPL_KEY =
+    "ssl.{0}.keystore.keypassword";
   public static final String SSL_KEYSTORE_TYPE_TPL_KEY =
     "ssl.{0}.keystore.type";
 
@@ -136,7 +138,7 @@ public class FileBasedKeyStoresFactory implements KeyStoresFactory {
       conf.get(resolvePropertyName(mode, SSL_KEYSTORE_TYPE_TPL_KEY),
                DEFAULT_KEYSTORE_TYPE);
     KeyStore keystore = KeyStore.getInstance(keystoreType);
-    String keystorePassword = null;
+    String keystoreKeyPassword = null;
     if (requireClientCert || mode == SSLFactory.Mode.SERVER) {
       String locationProperty =
         resolvePropertyName(mode, SSL_KEYSTORE_LOCATION_TPL_KEY);
@@ -147,11 +149,17 @@ public class FileBasedKeyStoresFactory implements KeyStoresFactory {
       }
       String passwordProperty =
         resolvePropertyName(mode, SSL_KEYSTORE_PASSWORD_TPL_KEY);
-      keystorePassword = conf.get(passwordProperty, "");
+      String keystorePassword = conf.get(passwordProperty, "");
       if (keystorePassword.isEmpty()) {
         throw new GeneralSecurityException("The property '" + passwordProperty +
           "' has not been set in the ssl configuration file.");
       }
+      String keyPasswordProperty =
+        resolvePropertyName(mode, SSL_KEYSTORE_KEYPASSWORD_TPL_KEY);
+      // Key password defaults to the same value as store password for
+      // compatibility with legacy configurations that did not use a separate
+      // configuration property for key password.
+      keystoreKeyPassword = conf.get(keyPasswordProperty, keystorePassword);
       LOG.debug(mode.toString() + " KeyStore: " + keystoreLocation);
 
       InputStream is = new FileInputStream(keystoreLocation);
@@ -167,8 +175,8 @@ public class FileBasedKeyStoresFactory implements KeyStoresFactory {
     KeyManagerFactory keyMgrFactory = KeyManagerFactory
         .getInstance(SSLFactory.SSLCERTIFICATE);
       
-    keyMgrFactory.init(keystore, (keystorePassword != null) ?
+    keyMgrFactory.init(keystore, (keystoreKeyPassword != null) ?
-                                 keystorePassword.toCharArray() : null);
+                                 keystoreKeyPassword.toCharArray() : null);
     keyManagers = keyMgrFactory.getKeyManagers();
 
     //trust store

hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/ssl/KeyStoreTestUtil.java

@@ -145,6 +145,28 @@ public class KeyStoreTestUtil {
     saveKeyStore(ks, filename, password);
   }
 
+  /**
+   * Creates a keystore with a single key and saves it to a file.
+   * 
+   * @param filename String file to save
+   * @param password String store password to set on keystore
+   * @param keyPassword String key password to set on key
+   * @param alias String alias to use for the key
+   * @param privateKey Key to save in keystore
+   * @param cert Certificate to use as certificate chain associated to key
+   * @throws GeneralSecurityException for any error with the security APIs
+   * @throws IOException if there is an I/O error saving the file
+   */
+  public static void createKeyStore(String filename,
+                                    String password, String keyPassword, String alias,
+                                    Key privateKey, Certificate cert)
+    throws GeneralSecurityException, IOException {
+    KeyStore ks = createEmptyKeyStore();
+    ks.setKeyEntry(alias, privateKey, keyPassword.toCharArray(),
+                   new Certificate[]{cert});
+    saveKeyStore(ks, filename, password);
+  }
+
   public static void createTrustStore(String filename,
                                       String password, String alias,
                                       Certificate cert)
@@ -178,6 +200,19 @@ public class KeyStoreTestUtil {
     f.delete();
   }
 
+  /**
+   * Performs complete setup of SSL configuration in preparation for testing an
+   * SSLFactory.  This includes keys, certs, keystores, truststores, the server
+   * SSL configuration file, the client SSL configuration file, and the master
+   * configuration file read by the SSLFactory.
+   * 
+   * @param keystoresDir String directory to save keystores
+   * @param sslConfDir String directory to save SSL configuration files
+   * @param conf Configuration master configuration to be used by an SSLFactory,
+   *   which will be mutated by this method
+   * @param useClientCert boolean true to make the client present a cert in the
+   *   SSL handshake
+   */
   public static void setupSSLConfig(String keystoresDir, String sslConfDir,
                                     Configuration conf, boolean useClientCert)
     throws Exception {
@@ -213,53 +248,13 @@ public class KeyStoreTestUtil {
 
     KeyStoreTestUtil.createTrustStore(trustKS, trustPassword, certs);
 
-    Configuration clientSSLConf = new Configuration(false);
+    Configuration clientSSLConf = createClientSSLConfig(clientKS, clientPassword,
-    clientSSLConf.set(FileBasedKeyStoresFactory.resolvePropertyName(
+      clientPassword, trustKS);
-      SSLFactory.Mode.CLIENT,
+    Configuration serverSSLConf = createServerSSLConfig(serverKS, serverPassword,
-      FileBasedKeyStoresFactory.SSL_KEYSTORE_LOCATION_TPL_KEY), clientKS);
+      serverPassword, trustKS);
-    clientSSLConf.set(FileBasedKeyStoresFactory.resolvePropertyName(
-      SSLFactory.Mode.CLIENT,
-      FileBasedKeyStoresFactory.SSL_KEYSTORE_PASSWORD_TPL_KEY), clientPassword);
-    clientSSLConf.set(FileBasedKeyStoresFactory.resolvePropertyName(
-      SSLFactory.Mode.CLIENT,
-      FileBasedKeyStoresFactory.SSL_TRUSTSTORE_LOCATION_TPL_KEY), trustKS);
-    clientSSLConf.set(FileBasedKeyStoresFactory.resolvePropertyName(
-      SSLFactory.Mode.CLIENT,
-      FileBasedKeyStoresFactory.SSL_TRUSTSTORE_PASSWORD_TPL_KEY), trustPassword);
-    clientSSLConf.set(FileBasedKeyStoresFactory.resolvePropertyName(
-      SSLFactory.Mode.CLIENT,
-      FileBasedKeyStoresFactory.SSL_TRUSTSTORE_RELOAD_INTERVAL_TPL_KEY), "1000");
 
-    Configuration serverSSLConf = new Configuration(false);
+    saveConfig(sslClientConfFile, clientSSLConf);
-    serverSSLConf.set(FileBasedKeyStoresFactory.resolvePropertyName(
+    saveConfig(sslServerConfFile, serverSSLConf);
-      SSLFactory.Mode.SERVER,
-      FileBasedKeyStoresFactory.SSL_KEYSTORE_LOCATION_TPL_KEY), serverKS);
-    serverSSLConf.set(FileBasedKeyStoresFactory.resolvePropertyName(
-      SSLFactory.Mode.SERVER,
-      FileBasedKeyStoresFactory.SSL_KEYSTORE_PASSWORD_TPL_KEY), serverPassword);
-    serverSSLConf.set(FileBasedKeyStoresFactory.resolvePropertyName(
-      SSLFactory.Mode.SERVER,
-      FileBasedKeyStoresFactory.SSL_TRUSTSTORE_LOCATION_TPL_KEY), trustKS);
-    serverSSLConf.set(FileBasedKeyStoresFactory.resolvePropertyName(
-      SSLFactory.Mode.SERVER,
-      FileBasedKeyStoresFactory.SSL_TRUSTSTORE_PASSWORD_TPL_KEY), trustPassword);
-    serverSSLConf.set(FileBasedKeyStoresFactory.resolvePropertyName(
-      SSLFactory.Mode.SERVER,
-      FileBasedKeyStoresFactory.SSL_TRUSTSTORE_RELOAD_INTERVAL_TPL_KEY), "1000");
-
-    Writer writer = new FileWriter(sslClientConfFile);
-    try {
-      clientSSLConf.writeXml(writer);
-    } finally {
-      writer.close();
-    }
-
-    writer = new FileWriter(sslServerConfFile);
-    try {
-      serverSSLConf.writeXml(writer);
-    } finally {
-      writer.close();
-    }
 
     conf.set(SSLFactory.SSL_HOSTNAME_VERIFIER_KEY, "ALLOW_ALL");
     conf.set(SSLFactory.SSL_CLIENT_CONF_KEY, sslClientConfFile.getName());
@@ -267,4 +262,101 @@ public class KeyStoreTestUtil {
     conf.setBoolean(SSLFactory.SSL_REQUIRE_CLIENT_CERT_KEY, useClientCert);
   }
 
+  /**
+   * Creates SSL configuration for a client.
+   * 
+   * @param clientKS String client keystore file
+   * @param password String store password, or null to avoid setting store
+   *   password
+   * @param keyPassword String key password, or null to avoid setting key
+   *   password
+   * @param trustKS String truststore file
+   * @return Configuration for client SSL
+   */
+  public static Configuration createClientSSLConfig(String clientKS,
+      String password, String keyPassword, String trustKS) {
+    Configuration clientSSLConf = createSSLConfig(SSLFactory.Mode.CLIENT,
+      clientKS, password, keyPassword, trustKS);
+    return clientSSLConf;
+  }
+
+  /**
+   * Creates SSL configuration for a server.
+   * 
+   * @param serverKS String server keystore file
+   * @param password String store password, or null to avoid setting store
+   *   password
+   * @param keyPassword String key password, or null to avoid setting key
+   *   password
+   * @param trustKS String truststore file
+   * @return Configuration for server SSL
+   */
+  public static Configuration createServerSSLConfig(String serverKS,
+      String password, String keyPassword, String trustKS) throws IOException {
+    Configuration serverSSLConf = createSSLConfig(SSLFactory.Mode.SERVER,
+      serverKS, password, keyPassword, trustKS);
+    return serverSSLConf;
+  }
+
+  /**
+   * Creates SSL configuration.
+   * 
+   * @param mode SSLFactory.Mode mode to configure
+   * @param keystore String keystore file
+   * @param password String store password, or null to avoid setting store
+   *   password
+   * @param keyPassword String key password, or null to avoid setting key
+   *   password
+   * @param trustKS String truststore file
+   * @return Configuration for SSL
+   */
+  private static Configuration createSSLConfig(SSLFactory.Mode mode,
+      String keystore, String password, String keyPassword, String trustKS) {
+    String trustPassword = "trustP";
+
+    Configuration sslConf = new Configuration(false);
+    if (keystore != null) {
+      sslConf.set(FileBasedKeyStoresFactory.resolvePropertyName(mode,
+        FileBasedKeyStoresFactory.SSL_KEYSTORE_LOCATION_TPL_KEY), keystore);
+    }
+    if (password != null) {
+      sslConf.set(FileBasedKeyStoresFactory.resolvePropertyName(mode,
+        FileBasedKeyStoresFactory.SSL_KEYSTORE_PASSWORD_TPL_KEY), password);
+    }
+    if (keyPassword != null) {
+      sslConf.set(FileBasedKeyStoresFactory.resolvePropertyName(mode,
+        FileBasedKeyStoresFactory.SSL_KEYSTORE_KEYPASSWORD_TPL_KEY),
+        keyPassword);
+    }
+    if (trustKS != null) {
+      sslConf.set(FileBasedKeyStoresFactory.resolvePropertyName(mode,
+        FileBasedKeyStoresFactory.SSL_TRUSTSTORE_LOCATION_TPL_KEY), trustKS);
+    }
+    if (trustPassword != null) {
+      sslConf.set(FileBasedKeyStoresFactory.resolvePropertyName(mode,
+        FileBasedKeyStoresFactory.SSL_TRUSTSTORE_PASSWORD_TPL_KEY),
+        trustPassword);
+    }
+    sslConf.set(FileBasedKeyStoresFactory.resolvePropertyName(mode,
+      FileBasedKeyStoresFactory.SSL_TRUSTSTORE_RELOAD_INTERVAL_TPL_KEY), "1000");
+
+    return sslConf;
+  }
+
+  /**
+   * Saves configuration to a file.
+   * 
+   * @param file File to save
+   * @param conf Configuration contents to write to file
+   * @throws IOException if there is an I/O error saving the file
+   */
+  public static void saveConfig(File file, Configuration conf)
+      throws IOException {
+    Writer writer = new FileWriter(file);
+    try {
+      conf.writeXml(writer);
+    } finally {
+      writer.close();
+    }
+  }
 }

hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/ssl/TestSSLFactory.java

@@ -29,12 +29,19 @@ import javax.net.ssl.HttpsURLConnection;
 import java.io.File;
 import java.net.URL;
 import java.security.GeneralSecurityException;
+import java.security.KeyPair;
+import java.security.cert.X509Certificate;
+import java.util.Collections;
+import java.util.Map;
 
 public class TestSSLFactory {
 
   private static final String BASEDIR =
     System.getProperty("test.build.dir", "target/test-dir") + "/" +
     TestSSLFactory.class.getSimpleName();
+  private static final String KEYSTORES_DIR =
+    new File(BASEDIR).getAbsolutePath();
+  private String sslConfsDir;
 
   @BeforeClass
   public static void setUp() throws Exception {
@@ -46,18 +53,16 @@ public class TestSSLFactory {
   private Configuration createConfiguration(boolean clientCert)
     throws Exception {
     Configuration conf = new Configuration();
-    String keystoresDir = new File(BASEDIR).getAbsolutePath();
+    KeyStoreTestUtil.setupSSLConfig(KEYSTORES_DIR, sslConfsDir, conf,
-    String sslConfsDir = KeyStoreTestUtil.getClasspathDir(TestSSLFactory.class);
+      clientCert);
-    KeyStoreTestUtil.setupSSLConfig(keystoresDir, sslConfsDir, conf, clientCert);
     return conf;
   }
 
   @After
   @Before
   public void cleanUp() throws Exception {
-    String keystoresDir = new File(BASEDIR).getAbsolutePath();
+    sslConfsDir = KeyStoreTestUtil.getClasspathDir(TestSSLFactory.class);
-    String sslConfsDir = KeyStoreTestUtil.getClasspathDir(TestSSLFactory.class);
+    KeyStoreTestUtil.cleanupSSLConfig(KEYSTORES_DIR, sslConfsDir);
-    KeyStoreTestUtil.cleanupSSLConfig(keystoresDir, sslConfsDir);
   }
 
   @Test(expected = IllegalStateException.class)
@@ -181,4 +186,90 @@ public class TestSSLFactory {
     }
   }
 
+  @Test
+  public void testServerDifferentPasswordAndKeyPassword() throws Exception {
+    checkSSLFactoryInitWithPasswords(SSLFactory.Mode.SERVER, "password",
+      "keyPassword", "password", "keyPassword");
+  }
+
+  @Test
+  public void testServerKeyPasswordDefaultsToPassword() throws Exception {
+    checkSSLFactoryInitWithPasswords(SSLFactory.Mode.SERVER, "password",
+      "password", "password", null);
+  }
+
+  @Test
+  public void testClientDifferentPasswordAndKeyPassword() throws Exception {
+    checkSSLFactoryInitWithPasswords(SSLFactory.Mode.CLIENT, "password",
+      "keyPassword", "password", "keyPassword");
+  }
+
+  @Test
+  public void testClientKeyPasswordDefaultsToPassword() throws Exception {
+    checkSSLFactoryInitWithPasswords(SSLFactory.Mode.CLIENT, "password",
+      "password", "password", null);
+  }
+
+  /**
+   * Checks that SSLFactory initialization is successful with the given
+   * arguments.  This is a helper method for writing test cases that cover
+   * different combinations of settings for the store password and key password.
+   * It takes care of bootstrapping a keystore, a truststore, and SSL client or
+   * server configuration.  Then, it initializes an SSLFactory.  If no exception
+   * is thrown, then initialization was successful.
+   * 
+   * @param mode SSLFactory.Mode mode to test
+   * @param password String store password to set on keystore
+   * @param keyPassword String key password to set on keystore
+   * @param confPassword String store password to set in SSL config file, or null
+   *   to avoid setting in SSL config file
+   * @param confKeyPassword String key password to set in SSL config file, or
+   *   null to avoid setting in SSL config file
+   * @throws Exception for any error
+   */
+  private void checkSSLFactoryInitWithPasswords(SSLFactory.Mode mode,
+      String password, String keyPassword, String confPassword,
+      String confKeyPassword) throws Exception {
+    String keystore = new File(KEYSTORES_DIR, "keystore.jks").getAbsolutePath();
+    String truststore = new File(KEYSTORES_DIR, "truststore.jks")
+      .getAbsolutePath();
+    String trustPassword = "trustP";
+
+    // Create keys, certs, keystore, and truststore.
+    KeyPair keyPair = KeyStoreTestUtil.generateKeyPair("RSA");
+    X509Certificate cert = KeyStoreTestUtil.generateCertificate("CN=Test",
+      keyPair, 30, "SHA1withRSA");
+    KeyStoreTestUtil.createKeyStore(keystore, password, keyPassword, "Test",
+      keyPair.getPrivate(), cert);
+    Map<String, X509Certificate> certs = Collections.singletonMap("server",
+      cert);
+    KeyStoreTestUtil.createTrustStore(truststore, trustPassword, certs);
+
+    // Create SSL configuration file, for either server or client.
+    final String sslConfFileName;
+    final Configuration sslConf;
+    if (mode == SSLFactory.Mode.SERVER) {
+      sslConfFileName = "ssl-server.xml";
+      sslConf = KeyStoreTestUtil.createServerSSLConfig(keystore, confPassword,
+        confKeyPassword, truststore);
+    } else {
+      sslConfFileName = "ssl-client.xml";
+      sslConf = KeyStoreTestUtil.createClientSSLConfig(keystore, confPassword,
+        confKeyPassword, truststore);
+    }
+    KeyStoreTestUtil.saveConfig(new File(sslConfsDir, sslConfFileName), sslConf);
+
+    // Create the master configuration for use by the SSLFactory, which by
+    // default refers to the ssl-server.xml or ssl-client.xml created above.
+    Configuration conf = new Configuration();
+    conf.setBoolean(SSLFactory.SSL_REQUIRE_CLIENT_CERT_KEY, true);
+
+    // Try initializing an SSLFactory.
+    SSLFactory sslFactory = new SSLFactory(mode, conf);
+    try {
+      sslFactory.init();
+    } finally {
+      sslFactory.destroy();
+    }
+  }
 }