From 41d93323f25ef66bf78bc3fd44b97e632cd7e2ee Mon Sep 17 00:00:00 2001 From: Ashish Singhi Date: Wed, 18 Feb 2015 11:42:22 -0800 Subject: [PATCH] HBASE-13002 Make encryption cipher configurable Signed-off-by: Andrew Purtell Conflicts: hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java hbase-common/src/test/java/org/apache/hadoop/hbase/io/crypto/TestEncryption.java --- .../hadoop/hbase/security/EncryptionUtil.java | 20 ++++++------ .../hbase/security/TestEncryptionUtil.java | 4 ++- .../org/apache/hadoop/hbase/HConstants.java | 12 ++++++- .../hadoop/hbase/io/crypto/Encryption.java | 31 ++++++++++++++++--- .../hbase/io/crypto/TestCipherProvider.java | 8 +++-- .../hbase/io/crypto/TestEncryption.java | 7 +++-- .../hadoop/hbase/regionserver/HStore.java | 4 +-- .../wal/SecureProtobufLogWriter.java | 5 ++- .../hbase/io/hfile/TestHFileEncryption.java | 4 ++- .../TestEncryptionKeyRotation.java | 14 ++++++--- .../TestEncryptionRandomKeying.java | 4 ++- .../hadoop/hbase/util/TestEncryptionTest.java | 6 ++-- .../hbase/util/TestHBaseFsckEncryption.java | 6 ++-- 13 files changed, 90 insertions(+), 35 deletions(-) diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/security/EncryptionUtil.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/security/EncryptionUtil.java index f446c6649e3..485388e9c60 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/security/EncryptionUtil.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/security/EncryptionUtil.java @@ -63,8 +63,7 @@ public class EncryptionUtil { /** * Protect a key by encrypting it with the secret key of the given subject. - * The configuration must be set up correctly for key alias resolution. Keys - * are always wrapped using AES. + * The configuration must be set up correctly for key alias resolution. * @param conf configuration * @param subject subject key alias * @param key the key @@ -72,10 +71,12 @@ public class EncryptionUtil { */ public static byte[] wrapKey(Configuration conf, String subject, Key key) throws IOException { - // Wrap the key with AES - Cipher cipher = Encryption.getCipher(conf, "AES"); + // Wrap the key with the configured encryption algorithm. + String algorithm = + conf.get(HConstants.CRYPTO_KEY_ALGORITHM_CONF_KEY, HConstants.CIPHER_AES); + Cipher cipher = Encryption.getCipher(conf, algorithm); if (cipher == null) { - throw new RuntimeException("Cipher 'AES' not available"); + throw new RuntimeException("Cipher '" + algorithm + "' not available"); } EncryptionProtos.WrappedKey.Builder builder = EncryptionProtos.WrappedKey.newBuilder(); builder.setAlgorithm(key.getAlgorithm()); @@ -100,8 +101,7 @@ public class EncryptionUtil { /** * Unwrap a key by decrypting it with the secret key of the given subject. - * The configuration must be set up correctly for key alias resolution. Keys - * are always unwrapped using AES. + * The configuration must be set up correctly for key alias resolution. * @param conf configuration * @param subject subject key alias * @param value the encrypted key bytes @@ -113,9 +113,11 @@ public class EncryptionUtil { throws IOException, KeyException { EncryptionProtos.WrappedKey wrappedKey = EncryptionProtos.WrappedKey.PARSER .parseDelimitedFrom(new ByteArrayInputStream(value)); - Cipher cipher = Encryption.getCipher(conf, "AES"); + String algorithm = conf.get(HConstants.CRYPTO_KEY_ALGORITHM_CONF_KEY, + HConstants.CIPHER_AES); + Cipher cipher = Encryption.getCipher(conf, algorithm); if (cipher == null) { - throw new RuntimeException("Algorithm 'AES' not available"); + throw new RuntimeException("Cipher '" + algorithm + "' not available"); } ByteArrayOutputStream out = new ByteArrayOutputStream(); byte[] iv = wrappedKey.hasIv() ? wrappedKey.getIv().toByteArray() : null; diff --git a/hbase-client/src/test/java/org/apache/hadoop/hbase/security/TestEncryptionUtil.java b/hbase-client/src/test/java/org/apache/hadoop/hbase/security/TestEncryptionUtil.java index e5e7b78bf62..7aea5d9450c 100644 --- a/hbase-client/src/test/java/org/apache/hadoop/hbase/security/TestEncryptionUtil.java +++ b/hbase-client/src/test/java/org/apache/hadoop/hbase/security/TestEncryptionUtil.java @@ -47,7 +47,9 @@ public class TestEncryptionUtil { // generate a test key byte[] keyBytes = new byte[AES.KEY_LENGTH]; new SecureRandom().nextBytes(keyBytes); - Key key = new SecretKeySpec(keyBytes, "AES"); + String algorithm = + conf.get(HConstants.CRYPTO_KEY_ALGORITHM_CONF_KEY, HConstants.CIPHER_AES); + Key key = new SecretKeySpec(keyBytes, algorithm); // wrap the test key byte[] wrappedKeyBytes = EncryptionUtil.wrapKey(conf, "hbase", key); diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java index 42254c6fa62..d8e33991123 100644 --- a/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java @@ -931,7 +931,7 @@ public final class HConstants { * NONE: no preference in destination of replicas * ONE_SSD: place only one replica in SSD and the remaining in default storage * and ALL_SSD: place all replica on SSD - * + * * See http://hadoop.apache.org/docs/r2.6.0/hadoop-project-dist/hadoop-hdfs/ArchivalStorage.html*/ public static final String WAL_STORAGE_POLICY = "hbase.wal.storage.policy"; public static final String DEFAULT_WAL_STORAGE_POLICY = "NONE"; @@ -1043,6 +1043,9 @@ public final class HConstants { public static final long NO_NONCE = 0; + /** Default cipher for encryption */ + public static final String CIPHER_AES = "AES"; + /** Configuration key for the crypto algorithm provider, a class name */ public static final String CRYPTO_CIPHERPROVIDER_CONF_KEY = "hbase.crypto.cipherprovider"; @@ -1066,6 +1069,13 @@ public final class HConstants { /** Configuration key for the name of the master WAL encryption key for the cluster, a string */ public static final String CRYPTO_WAL_KEY_NAME_CONF_KEY = "hbase.crypto.wal.key.name"; + /** Configuration key for the algorithm used for creating jks key, a string */ + public static final String CRYPTO_KEY_ALGORITHM_CONF_KEY = "hbase.crypto.key.algorithm"; + + /** Configuration key for the name of the alternate cipher algorithm for the cluster, a string */ + public static final String CRYPTO_ALTERNATE_KEY_ALGORITHM_CONF_KEY = + "hbase.crypto.alternate.key.algorithm"; + /** Configuration key for enabling WAL encryption, a boolean */ public static final String ENABLE_WAL_ENCRYPTION = "hbase.regionserver.wal.encryption"; diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/crypto/Encryption.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/crypto/Encryption.java index 9c20f3b8cb4..2e6a7c9ab97 100644 --- a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/crypto/Encryption.java +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/crypto/Encryption.java @@ -469,9 +469,8 @@ public final class Encryption { * @param iv the initialization vector, can be null * @throws IOException */ - public static void decryptWithSubjectKey(OutputStream out, InputStream in, - int outLen, String subject, Configuration conf, Cipher cipher, - byte[] iv) throws IOException { + public static void decryptWithSubjectKey(OutputStream out, InputStream in, int outLen, + String subject, Configuration conf, Cipher cipher, byte[] iv) throws IOException { Key key = getSecretKeyForSubject(subject, conf); if (key == null) { throw new IOException("No key found for subject '" + subject + "'"); @@ -479,7 +478,31 @@ public final class Encryption { Decryptor d = cipher.getDecryptor(); d.setKey(key); d.setIv(iv); // can be null - decrypt(out, in, outLen, d); + try { + decrypt(out, in, outLen, d); + } catch (IOException e) { + // If the current cipher algorithm fails to unwrap, try the alternate cipher algorithm, if one + // is configured + String alternateAlgorithm = conf.get(HConstants.CRYPTO_ALTERNATE_KEY_ALGORITHM_CONF_KEY); + if (alternateAlgorithm != null) { + if (LOG.isDebugEnabled()) { + LOG.debug("Unable to decrypt data with current cipher algorithm '" + + conf.get(HConstants.CRYPTO_KEY_ALGORITHM_CONF_KEY, HConstants.CIPHER_AES) + + "'. Trying with the alternate cipher algorithm '" + alternateAlgorithm + + "' configured."); + } + Cipher alterCipher = Encryption.getCipher(conf, alternateAlgorithm); + if (alterCipher == null) { + throw new RuntimeException("Cipher '" + alternateAlgorithm + "' not available"); + } + d = alterCipher.getDecryptor(); + d.setKey(key); + d.setIv(iv); // can be null + decrypt(out, in, outLen, d); + } else { + throw new IOException(e); + } + } } private static ClassLoader getClassLoaderForClass(Class c) { diff --git a/hbase-common/src/test/java/org/apache/hadoop/hbase/io/crypto/TestCipherProvider.java b/hbase-common/src/test/java/org/apache/hadoop/hbase/io/crypto/TestCipherProvider.java index 126d7f6d868..95f8ba1f354 100644 --- a/hbase-common/src/test/java/org/apache/hadoop/hbase/io/crypto/TestCipherProvider.java +++ b/hbase-common/src/test/java/org/apache/hadoop/hbase/io/crypto/TestCipherProvider.java @@ -140,11 +140,13 @@ public class TestCipherProvider { Configuration conf = HBaseConfiguration.create(); CipherProvider provider = Encryption.getCipherProvider(conf); assertTrue(provider instanceof DefaultCipherProvider); - assertTrue(Arrays.asList(provider.getSupportedCiphers()).contains("AES")); - Cipher a = Encryption.getCipher(conf, "AES"); + String algorithm = + conf.get(HConstants.CRYPTO_KEY_ALGORITHM_CONF_KEY, HConstants.CIPHER_AES); + assertTrue(Arrays.asList(provider.getSupportedCiphers()).contains(algorithm)); + Cipher a = Encryption.getCipher(conf, algorithm); assertNotNull(a); assertTrue(a.getProvider() instanceof DefaultCipherProvider); - assertEquals(a.getName(), "AES"); + assertEquals(a.getName(), algorithm); assertEquals(a.getKeyLength(), AES.KEY_LENGTH); } diff --git a/hbase-common/src/test/java/org/apache/hadoop/hbase/io/crypto/TestEncryption.java b/hbase-common/src/test/java/org/apache/hadoop/hbase/io/crypto/TestEncryption.java index d9e51c1b2de..e31ab49a10b 100644 --- a/hbase-common/src/test/java/org/apache/hadoop/hbase/io/crypto/TestEncryption.java +++ b/hbase-common/src/test/java/org/apache/hadoop/hbase/io/crypto/TestEncryption.java @@ -28,6 +28,7 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.HBaseConfiguration; +import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.testclassification.SmallTests; import org.apache.hadoop.hbase.util.Bytes; import org.junit.Test; @@ -87,8 +88,10 @@ public class TestEncryption { LOG.info("checkTransformSymmetry: AES, plaintext length = " + plaintext.length); Configuration conf = HBaseConfiguration.create(); - Cipher aes = Encryption.getCipher(conf, "AES"); - Key key = new SecretKeySpec(keyBytes, "AES"); + String algorithm = + conf.get(HConstants.CRYPTO_KEY_ALGORITHM_CONF_KEY, HConstants.CIPHER_AES); + Cipher aes = Encryption.getCipher(conf, algorithm); + Key key = new SecretKeySpec(keyBytes, algorithm); Encryptor e = aes.getEncryptor(); e.setKey(key); diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java index 04b4539a48f..2da1bb7d84d 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java @@ -318,7 +318,7 @@ public class HStore implements Store { // Use the algorithm the key wants cipher = Encryption.getCipher(conf, key.getAlgorithm()); if (cipher == null) { - throw new RuntimeException("Cipher '" + cipher + "' is not available"); + throw new RuntimeException("Cipher '" + key.getAlgorithm() + "' is not available"); } // Fail if misconfigured // We use the encryption type specified in the column schema as a sanity check on @@ -332,7 +332,7 @@ public class HStore implements Store { // Family does not provide key material, create a random key cipher = Encryption.getCipher(conf, cipherName); if (cipher == null) { - throw new RuntimeException("Cipher '" + cipher + "' is not available"); + throw new RuntimeException("Cipher '" + cipherName + "' is not available"); } key = cipher.getRandomKey(); } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/SecureProtobufLogWriter.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/SecureProtobufLogWriter.java index e850485bb95..c352770a1c8 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/SecureProtobufLogWriter.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/SecureProtobufLogWriter.java @@ -43,8 +43,6 @@ import org.apache.hadoop.hbase.security.User; public class SecureProtobufLogWriter extends ProtobufLogWriter { private static final Log LOG = LogFactory.getLog(SecureProtobufLogWriter.class); - private static final String DEFAULT_CIPHER = "AES"; - private Encryptor encryptor = null; @Override @@ -56,7 +54,8 @@ public class SecureProtobufLogWriter extends ProtobufLogWriter { EncryptionTest.testCipherProvider(conf); // Get an instance of our cipher - final String cipherName = conf.get(HConstants.CRYPTO_WAL_ALGORITHM_CONF_KEY, DEFAULT_CIPHER); + final String cipherName = + conf.get(HConstants.CRYPTO_WAL_ALGORITHM_CONF_KEY, HConstants.CIPHER_AES); Cipher cipher = Encryption.getCipher(conf, cipherName); if (cipher == null) { throw new RuntimeException("Cipher '" + cipherName + "' is not available"); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileEncryption.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileEncryption.java index bf6770b49fb..2379df5712b 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileEncryption.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileEncryption.java @@ -69,7 +69,9 @@ public class TestHFileEncryption { fs = FileSystem.get(conf); cryptoContext = Encryption.newContext(conf); - Cipher aes = Encryption.getCipher(conf, "AES"); + String algorithm = + conf.get(HConstants.CRYPTO_KEY_ALGORITHM_CONF_KEY, HConstants.CIPHER_AES); + Cipher aes = Encryption.getCipher(conf, algorithm); assertNotNull(aes); cryptoContext.setCipher(aes); byte[] key = new byte[aes.getKeyLength()]; diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestEncryptionKeyRotation.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestEncryptionKeyRotation.java index 44daaed46e6..a025c97a11c 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestEncryptionKeyRotation.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestEncryptionKeyRotation.java @@ -66,9 +66,11 @@ public class TestEncryptionKeyRotation { SecureRandom rng = new SecureRandom(); byte[] keyBytes = new byte[AES.KEY_LENGTH]; rng.nextBytes(keyBytes); - initialCFKey = new SecretKeySpec(keyBytes, "AES"); + String algorithm = + conf.get(HConstants.CRYPTO_KEY_ALGORITHM_CONF_KEY, HConstants.CIPHER_AES); + initialCFKey = new SecretKeySpec(keyBytes, algorithm); rng.nextBytes(keyBytes); - secondCFKey = new SecretKeySpec(keyBytes, "AES"); + secondCFKey = new SecretKeySpec(keyBytes, algorithm); } @BeforeClass @@ -94,7 +96,9 @@ public class TestEncryptionKeyRotation { HTableDescriptor htd = new HTableDescriptor(TableName.valueOf("default", "testCFKeyRotation")); HColumnDescriptor hcd = new HColumnDescriptor("cf"); - hcd.setEncryptionType("AES"); + String algorithm = + conf.get(HConstants.CRYPTO_KEY_ALGORITHM_CONF_KEY, HConstants.CIPHER_AES); + hcd.setEncryptionType(algorithm); hcd.setEncryptionKey(EncryptionUtil.wrapKey(conf, "hbase", initialCFKey)); htd.addFamily(hcd); @@ -153,7 +157,9 @@ public class TestEncryptionKeyRotation { HTableDescriptor htd = new HTableDescriptor(TableName.valueOf("default", "testMasterKeyRotation")); HColumnDescriptor hcd = new HColumnDescriptor("cf"); - hcd.setEncryptionType("AES"); + String algorithm = + conf.get(HConstants.CRYPTO_KEY_ALGORITHM_CONF_KEY, HConstants.CIPHER_AES); + hcd.setEncryptionType(algorithm); hcd.setEncryptionKey(EncryptionUtil.wrapKey(conf, "hbase", initialCFKey)); htd.addFamily(hcd); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestEncryptionRandomKeying.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestEncryptionRandomKeying.java index 46d05a898cf..2b2a13419a0 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestEncryptionRandomKeying.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestEncryptionRandomKeying.java @@ -91,7 +91,9 @@ public class TestEncryptionRandomKeying { // Specify an encryption algorithm without a key htd = new HTableDescriptor(TableName.valueOf("default", "TestEncryptionRandomKeying")); HColumnDescriptor hcd = new HColumnDescriptor("cf"); - hcd.setEncryptionType("AES"); + String algorithm = + conf.get(HConstants.CRYPTO_KEY_ALGORITHM_CONF_KEY, HConstants.CIPHER_AES); + hcd.setEncryptionType(algorithm); htd.addFamily(hcd); // Start the minicluster diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestEncryptionTest.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestEncryptionTest.java index f42bb2e03a9..cf9dbee6b37 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestEncryptionTest.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestEncryptionTest.java @@ -74,10 +74,12 @@ public class TestEncryptionTest { public void testTestCipher() { Configuration conf = HBaseConfiguration.create(); conf.set(HConstants.CRYPTO_KEYPROVIDER_CONF_KEY, KeyProviderForTesting.class.getName()); + String algorithm = + conf.get(HConstants.CRYPTO_KEY_ALGORITHM_CONF_KEY, HConstants.CIPHER_AES); try { - EncryptionTest.testEncryption(conf, "AES", null); + EncryptionTest.testEncryption(conf, algorithm, null); } catch (Exception e) { - fail("Test for cipher AES should have succeeded"); + fail("Test for cipher " + algorithm + " should have succeeded"); } try { EncryptionTest.testEncryption(conf, "foobar", null); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsckEncryption.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsckEncryption.java index cd8c8850334..3332c0f57fa 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsckEncryption.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsckEncryption.java @@ -77,7 +77,9 @@ public class TestHBaseFsckEncryption { SecureRandom rng = new SecureRandom(); byte[] keyBytes = new byte[AES.KEY_LENGTH]; rng.nextBytes(keyBytes); - cfKey = new SecretKeySpec(keyBytes, "AES"); + String algorithm = + conf.get(HConstants.CRYPTO_KEY_ALGORITHM_CONF_KEY, HConstants.CIPHER_AES); + cfKey = new SecretKeySpec(keyBytes,algorithm); // Start the minicluster TEST_UTIL.startMiniCluster(3); @@ -85,7 +87,7 @@ public class TestHBaseFsckEncryption { // Create the table htd = new HTableDescriptor(TableName.valueOf("default", "TestHBaseFsckEncryption")); HColumnDescriptor hcd = new HColumnDescriptor("cf"); - hcd.setEncryptionType("AES"); + hcd.setEncryptionType(algorithm); hcd.setEncryptionKey(EncryptionUtil.wrapKey(conf, conf.get(HConstants.CRYPTO_MASTERKEY_NAME_CONF_KEY, User.getCurrent().getShortName()), cfKey));