HBASE-25181 Add options for disabling column family encryption and choosing hash algorithm for wrapped encryption keys.
Prior to this patch hbase always used the MD5 hash algorithm to store a hash for encryption keys. This hash is needed to verify the secret key of the subject. (e.g. making sure that the same secrey key is used during encrypted HFile read and write). The MD5 algorithm is considered weak, and can not be used in some (e.g. FIPS compliant) clusters. In this patch we: - add a config parameter to globally enable/disable column family encryption (def enabled) - introduce a backward compatible way of specifying the hash algorithm. This enable us to use newer and more secure hash algorithms like SHA-384 or SHA-512 (which are FIPS compliant). - add a config parameter to fail if an hfile is encountered that uses a different hash algorithm than the one currently configured to ease validation after migrating key hash algorithms (def disabled) Closes #2539 Signed-off-by: Sean Busbey <busbey@apache.org> Signed-off-by: Esteban Gutierrez <esteban@apache.org>
This commit is contained in:
Mate Szalay-Beko
Sean Busbey
parent
58c974888f
commit
6a5c928539
|
|
@ -24,24 +24,23 @@ import java.security.Key;
|
|||
import java.security.KeyException;
|
||||
import java.security.SecureRandom;
|
||||
import java.util.Properties;
|
||||
|
||||
import javax.crypto.spec.SecretKeySpec;
|
||||
|
||||
import org.apache.commons.crypto.cipher.CryptoCipherFactory;
|
||||
import org.apache.hadoop.conf.Configuration;
|
||||
import org.apache.hadoop.hbase.HConstants;
|
||||
import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor;
|
||||
import org.apache.hadoop.hbase.io.crypto.Cipher;
|
||||
import org.apache.hadoop.hbase.io.crypto.Encryption;
|
||||
import org.apache.hadoop.hbase.io.crypto.aes.CryptoAES;
|
||||
import org.apache.hadoop.hbase.util.Bytes;
|
||||
import org.apache.yetus.audience.InterfaceAudience;
|
||||
import org.apache.yetus.audience.InterfaceStability;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor;
|
||||
import org.apache.hadoop.hbase.io.crypto.Cipher;
|
||||
import org.apache.hadoop.hbase.io.crypto.Encryption;
|
||||
|
||||
import org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations;
|
||||
import org.apache.hadoop.hbase.shaded.protobuf.generated.EncryptionProtos;
|
||||
import org.apache.hadoop.hbase.shaded.protobuf.generated.RPCProtos;
|
||||
import org.apache.hadoop.hbase.io.crypto.aes.CryptoAES;
|
||||
import org.apache.hadoop.hbase.util.Bytes;
|
||||
|
||||
/**
|
||||
* Some static utility methods for encryption uses in hbase-client.
|
||||
|
|
@ -102,7 +101,9 @@ public final class EncryptionUtil {
|
|||
}
|
||||
byte[] keyBytes = key.getEncoded();
|
||||
builder.setLength(keyBytes.length);
|
||||
builder.setHash(UnsafeByteOperations.unsafeWrap(Encryption.hash128(keyBytes)));
|
||||
builder.setHashAlgorithm(Encryption.getConfiguredHashAlgorithm(conf));
|
||||
builder.setHash(
|
||||
UnsafeByteOperations.unsafeWrap(Encryption.computeCryptoKeyHash(conf, keyBytes)));
|
||||
ByteArrayOutputStream out = new ByteArrayOutputStream();
|
||||
Encryption.encryptWithSubjectKey(out, new ByteArrayInputStream(keyBytes), subject,
|
||||
conf, cipher, iv);
|
||||
|
|
@ -138,13 +139,24 @@ public final class EncryptionUtil {
|
|||
|
||||
private static Key getUnwrapKey(Configuration conf, String subject,
|
||||
EncryptionProtos.WrappedKey wrappedKey, Cipher cipher) throws IOException, KeyException {
|
||||
String configuredHashAlgorithm = Encryption.getConfiguredHashAlgorithm(conf);
|
||||
String wrappedHashAlgorithm = wrappedKey.getHashAlgorithm().trim();
|
||||
if(!configuredHashAlgorithm.equalsIgnoreCase(wrappedHashAlgorithm)) {
|
||||
String msg = String.format("Unexpected encryption key hash algorithm: %s (expecting: %s)",
|
||||
wrappedHashAlgorithm, configuredHashAlgorithm);
|
||||
if(Encryption.failOnHashAlgorithmMismatch(conf)) {
|
||||
throw new KeyException(msg);
|
||||
}
|
||||
LOG.debug(msg);
|
||||
}
|
||||
ByteArrayOutputStream out = new ByteArrayOutputStream();
|
||||
byte[] iv = wrappedKey.hasIv() ? wrappedKey.getIv().toByteArray() : null;
|
||||
Encryption.decryptWithSubjectKey(out, wrappedKey.getData().newInput(),
|
||||
wrappedKey.getLength(), subject, conf, cipher, iv);
|
||||
byte[] keyBytes = out.toByteArray();
|
||||
if (wrappedKey.hasHash()) {
|
||||
if (!Bytes.equals(wrappedKey.getHash().toByteArray(), Encryption.hash128(keyBytes))) {
|
||||
if (!Bytes.equals(wrappedKey.getHash().toByteArray(),
|
||||
Encryption.hashWithAlg(wrappedHashAlgorithm, keyBytes))) {
|
||||
throw new KeyException("Key was not successfully unwrapped");
|
||||
}
|
||||
}
|
||||
|
|
@ -186,6 +198,10 @@ public final class EncryptionUtil {
|
|||
Encryption.Context cryptoContext = Encryption.Context.NONE;
|
||||
String cipherName = family.getEncryptionType();
|
||||
if (cipherName != null) {
|
||||
if(!Encryption.isEncryptionEnabled(conf)) {
|
||||
throw new RuntimeException("Encryption for family '" + family.getNameAsString()
|
||||
+ "' configured with type '" + cipherName + "' but the encryption feature is disabled");
|
||||
}
|
||||
Cipher cipher;
|
||||
Key key;
|
||||
byte[] keyBytes = family.getEncryptionKey();
|
||||
|
|
|
|||
|
|
@ -28,6 +28,7 @@ import javax.crypto.spec.SecretKeySpec;
|
|||
import org.apache.hadoop.conf.Configuration;
|
||||
import org.apache.hadoop.hbase.HBaseClassTestRule;
|
||||
import org.apache.hadoop.hbase.HConstants;
|
||||
import org.apache.hadoop.hbase.io.crypto.Encryption;
|
||||
import org.apache.hadoop.hbase.io.crypto.KeyProviderForTesting;
|
||||
import org.apache.hadoop.hbase.io.crypto.aes.AES;
|
||||
import org.apache.hadoop.hbase.testclassification.ClientTests;
|
||||
|
|
@ -40,6 +41,9 @@ import org.junit.experimental.categories.Category;
|
|||
@Category({ClientTests.class, SmallTests.class})
|
||||
public class TestEncryptionUtil {
|
||||
|
||||
private static final String INVALID_HASH_ALG = "this-hash-algorithm-not-exists hopefully... :)";
|
||||
private static final String DEFAULT_HASH_ALGORITHM = "use-default";
|
||||
|
||||
@ClassRule
|
||||
public static final HBaseClassTestRule CLASS_RULE =
|
||||
HBaseClassTestRule.forClass(TestEncryptionUtil.class);
|
||||
|
|
@ -49,64 +53,53 @@ public class TestEncryptionUtil {
|
|||
// untested. Not ideal!
|
||||
|
||||
@Test
|
||||
public void testKeyWrapping() throws Exception {
|
||||
// set up the key provider for testing to resolve a key for our test subject
|
||||
Configuration conf = new Configuration(); // we don't need HBaseConfiguration for this
|
||||
conf.set(HConstants.CRYPTO_KEYPROVIDER_CONF_KEY, KeyProviderForTesting.class.getName());
|
||||
|
||||
// generate a test key
|
||||
byte[] keyBytes = new byte[AES.KEY_LENGTH];
|
||||
new SecureRandom().nextBytes(keyBytes);
|
||||
String algorithm =
|
||||
conf.get(HConstants.CRYPTO_KEY_ALGORITHM_CONF_KEY, HConstants.CIPHER_AES);
|
||||
Key key = new SecretKeySpec(keyBytes, algorithm);
|
||||
|
||||
// wrap the test key
|
||||
byte[] wrappedKeyBytes = EncryptionUtil.wrapKey(conf, "hbase", key);
|
||||
assertNotNull(wrappedKeyBytes);
|
||||
|
||||
// unwrap
|
||||
Key unwrappedKey = EncryptionUtil.unwrapKey(conf, "hbase", wrappedKeyBytes);
|
||||
assertNotNull(unwrappedKey);
|
||||
// only secretkeyspec supported for now
|
||||
assertTrue(unwrappedKey instanceof SecretKeySpec);
|
||||
// did we get back what we wrapped?
|
||||
assertTrue("Unwrapped key bytes do not match original",
|
||||
Bytes.equals(keyBytes, unwrappedKey.getEncoded()));
|
||||
|
||||
// unwrap with an incorrect key
|
||||
try {
|
||||
EncryptionUtil.unwrapKey(conf, "other", wrappedKeyBytes);
|
||||
fail("Unwrap with incorrect key did not throw KeyException");
|
||||
} catch (KeyException e) {
|
||||
// expected
|
||||
}
|
||||
public void testKeyWrappingUsingHashAlgDefault() throws Exception {
|
||||
testKeyWrapping(DEFAULT_HASH_ALGORITHM);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testWALKeyWrapping() throws Exception {
|
||||
// set up the key provider for testing to resolve a key for our test subject
|
||||
Configuration conf = new Configuration(); // we don't need HBaseConfiguration for this
|
||||
conf.set(HConstants.CRYPTO_KEYPROVIDER_CONF_KEY, KeyProviderForTesting.class.getName());
|
||||
public void testKeyWrappingUsingHashAlgMD5() throws Exception {
|
||||
testKeyWrapping("MD5");
|
||||
}
|
||||
|
||||
// generate a test key
|
||||
byte[] keyBytes = new byte[AES.KEY_LENGTH];
|
||||
new SecureRandom().nextBytes(keyBytes);
|
||||
String algorithm = conf.get(HConstants.CRYPTO_WAL_ALGORITHM_CONF_KEY, HConstants.CIPHER_AES);
|
||||
Key key = new SecretKeySpec(keyBytes, algorithm);
|
||||
@Test
|
||||
public void testKeyWrappingUsingHashAlgSHA256() throws Exception {
|
||||
testKeyWrapping("SHA-256");
|
||||
}
|
||||
|
||||
// wrap the test key
|
||||
byte[] wrappedKeyBytes = EncryptionUtil.wrapKey(conf, "hbase", key);
|
||||
assertNotNull(wrappedKeyBytes);
|
||||
@Test
|
||||
public void testKeyWrappingUsingHashAlgSHA384() throws Exception {
|
||||
testKeyWrapping("SHA-384");
|
||||
}
|
||||
|
||||
// unwrap
|
||||
Key unwrappedKey = EncryptionUtil.unwrapWALKey(conf, "hbase", wrappedKeyBytes);
|
||||
assertNotNull(unwrappedKey);
|
||||
// only secretkeyspec supported for now
|
||||
assertTrue(unwrappedKey instanceof SecretKeySpec);
|
||||
// did we get back what we wrapped?
|
||||
assertTrue("Unwrapped key bytes do not match original",
|
||||
Bytes.equals(keyBytes, unwrappedKey.getEncoded()));
|
||||
@Test(expected = RuntimeException.class)
|
||||
public void testKeyWrappingWithInvalidHashAlg() throws Exception {
|
||||
testKeyWrapping(INVALID_HASH_ALG);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testWALKeyWrappingUsingHashAlgDefault() throws Exception {
|
||||
testWALKeyWrapping(DEFAULT_HASH_ALGORITHM);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testWALKeyWrappingUsingHashAlgMD5() throws Exception {
|
||||
testWALKeyWrapping("MD5");
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testWALKeyWrappingUsingHashAlgSHA256() throws Exception {
|
||||
testWALKeyWrapping("SHA-256");
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testWALKeyWrappingUsingHashAlgSHA384() throws Exception {
|
||||
testWALKeyWrapping("SHA-384");
|
||||
}
|
||||
|
||||
@Test(expected = RuntimeException.class)
|
||||
public void testWALKeyWrappingWithInvalidHashAlg() throws Exception {
|
||||
testWALKeyWrapping(INVALID_HASH_ALG);
|
||||
}
|
||||
|
||||
@Test(expected = KeyException.class)
|
||||
|
|
@ -128,4 +121,120 @@ public class TestEncryptionUtil {
|
|||
// unwrap with an incorrect key
|
||||
EncryptionUtil.unwrapWALKey(conf, "other", wrappedKeyBytes);
|
||||
}
|
||||
|
||||
@Test(expected = KeyException.class)
|
||||
public void testHashAlgorithmMismatchWhenFailExpected() throws Exception {
|
||||
Configuration conf = new Configuration(); // we don't need HBaseConfiguration for this
|
||||
conf.setBoolean(Encryption.CRYPTO_KEY_FAIL_ON_ALGORITHM_MISMATCH_CONF_KEY, true);
|
||||
testKeyWrappingWithMismatchingAlgorithms(conf);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testHashAlgorithmMismatchWhenFailNotExpected() throws Exception {
|
||||
Configuration conf = new Configuration(); // we don't need HBaseConfiguration for this
|
||||
conf.setBoolean(Encryption.CRYPTO_KEY_FAIL_ON_ALGORITHM_MISMATCH_CONF_KEY, false);
|
||||
testKeyWrappingWithMismatchingAlgorithms(conf);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testHashAlgorithmMismatchShouldNotFailWithDefaultConfig() throws Exception {
|
||||
Configuration conf = new Configuration(); // we don't need HBaseConfiguration for this
|
||||
testKeyWrappingWithMismatchingAlgorithms(conf);
|
||||
}
|
||||
|
||||
private void testKeyWrapping(String hashAlgorithm) throws Exception {
|
||||
// set up the key provider for testing to resolve a key for our test subject
|
||||
Configuration conf = new Configuration(); // we don't need HBaseConfiguration for this
|
||||
conf.set(HConstants.CRYPTO_KEYPROVIDER_CONF_KEY, KeyProviderForTesting.class.getName());
|
||||
if(!hashAlgorithm.equals(DEFAULT_HASH_ALGORITHM)) {
|
||||
conf.set(Encryption.CRYPTO_KEY_HASH_ALGORITHM_CONF_KEY, hashAlgorithm);
|
||||
}
|
||||
|
||||
// generate a test key
|
||||
byte[] keyBytes = new byte[AES.KEY_LENGTH];
|
||||
new SecureRandom().nextBytes(keyBytes);
|
||||
String algorithm =
|
||||
conf.get(HConstants.CRYPTO_KEY_ALGORITHM_CONF_KEY, HConstants.CIPHER_AES);
|
||||
Key key = new SecretKeySpec(keyBytes, algorithm);
|
||||
|
||||
// wrap the test key
|
||||
byte[] wrappedKeyBytes = EncryptionUtil.wrapKey(conf, "hbase", key);
|
||||
assertNotNull(wrappedKeyBytes);
|
||||
|
||||
// unwrap
|
||||
Key unwrappedKey = EncryptionUtil.unwrapKey(conf, "hbase", wrappedKeyBytes);
|
||||
assertNotNull(unwrappedKey);
|
||||
// only secretkeyspec supported for now
|
||||
assertTrue(unwrappedKey instanceof SecretKeySpec);
|
||||
// did we get back what we wrapped?
|
||||
assertTrue("Unwrapped key bytes do not match original",
|
||||
Bytes.equals(keyBytes, unwrappedKey.getEncoded()));
|
||||
|
||||
// unwrap with an incorrect key
|
||||
try {
|
||||
EncryptionUtil.unwrapKey(conf, "other", wrappedKeyBytes);
|
||||
fail("Unwrap with incorrect key did not throw KeyException");
|
||||
} catch (KeyException e) {
|
||||
// expected
|
||||
}
|
||||
}
|
||||
|
||||
private void testWALKeyWrapping(String hashAlgorithm) throws Exception {
|
||||
// set up the key provider for testing to resolve a key for our test subject
|
||||
Configuration conf = new Configuration(); // we don't need HBaseConfiguration for this
|
||||
conf.set(HConstants.CRYPTO_KEYPROVIDER_CONF_KEY, KeyProviderForTesting.class.getName());
|
||||
if(!hashAlgorithm.equals(DEFAULT_HASH_ALGORITHM)) {
|
||||
conf.set(Encryption.CRYPTO_KEY_HASH_ALGORITHM_CONF_KEY, hashAlgorithm);
|
||||
}
|
||||
|
||||
// generate a test key
|
||||
byte[] keyBytes = new byte[AES.KEY_LENGTH];
|
||||
new SecureRandom().nextBytes(keyBytes);
|
||||
String algorithm = conf.get(HConstants.CRYPTO_WAL_ALGORITHM_CONF_KEY, HConstants.CIPHER_AES);
|
||||
Key key = new SecretKeySpec(keyBytes, algorithm);
|
||||
|
||||
// wrap the test key
|
||||
byte[] wrappedKeyBytes = EncryptionUtil.wrapKey(conf, "hbase", key);
|
||||
assertNotNull(wrappedKeyBytes);
|
||||
|
||||
// unwrap
|
||||
Key unwrappedKey = EncryptionUtil.unwrapWALKey(conf, "hbase", wrappedKeyBytes);
|
||||
assertNotNull(unwrappedKey);
|
||||
// only secretkeyspec supported for now
|
||||
assertTrue(unwrappedKey instanceof SecretKeySpec);
|
||||
// did we get back what we wrapped?
|
||||
assertTrue("Unwrapped key bytes do not match original",
|
||||
Bytes.equals(keyBytes, unwrappedKey.getEncoded()));
|
||||
}
|
||||
|
||||
private void testKeyWrappingWithMismatchingAlgorithms(Configuration conf) throws Exception {
|
||||
// we use MD5 to hash the encryption key during wrapping
|
||||
conf.set(HConstants.CRYPTO_KEYPROVIDER_CONF_KEY, KeyProviderForTesting.class.getName());
|
||||
conf.set(Encryption.CRYPTO_KEY_HASH_ALGORITHM_CONF_KEY, "MD5");
|
||||
|
||||
// generate a test key
|
||||
byte[] keyBytes = new byte[AES.KEY_LENGTH];
|
||||
new SecureRandom().nextBytes(keyBytes);
|
||||
String algorithm =
|
||||
conf.get(HConstants.CRYPTO_KEY_ALGORITHM_CONF_KEY, HConstants.CIPHER_AES);
|
||||
Key key = new SecretKeySpec(keyBytes, algorithm);
|
||||
|
||||
// wrap the test key
|
||||
byte[] wrappedKeyBytes = EncryptionUtil.wrapKey(conf, "hbase", key);
|
||||
assertNotNull(wrappedKeyBytes);
|
||||
|
||||
// we set the default hash algorithm to SHA-384 during unwrapping
|
||||
conf.set(Encryption.CRYPTO_KEY_HASH_ALGORITHM_CONF_KEY, "SHA-384");
|
||||
|
||||
// unwrap
|
||||
// we expect to fail, if CRYPTO_KEY_FAIL_ON_ALGORITHM_MISMATCH_CONF_KEY == true
|
||||
// otherwise we will use the algorithm written during wrapping
|
||||
Key unwrappedKey = EncryptionUtil.unwrapKey(conf, "hbase", wrappedKeyBytes);
|
||||
assertNotNull(unwrappedKey);
|
||||
|
||||
// did we get back what we wrapped?
|
||||
assertTrue("Unwrapped key bytes do not match original",
|
||||
Bytes.equals(keyBytes, unwrappedKey.getEncoded()));
|
||||
}
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -18,10 +18,10 @@ package org.apache.hadoop.hbase.io.crypto;
|
|||
|
||||
import java.security.Key;
|
||||
|
||||
import org.apache.commons.codec.binary.Hex;
|
||||
import org.apache.hadoop.conf.Configurable;
|
||||
import org.apache.hadoop.conf.Configuration;
|
||||
import org.apache.hadoop.hbase.HBaseConfiguration;
|
||||
import org.apache.hadoop.hbase.util.MD5Hash;
|
||||
import org.apache.yetus.audience.InterfaceAudience;
|
||||
|
||||
import org.apache.hbase.thirdparty.com.google.common.base.Preconditions;
|
||||
|
|
@ -94,7 +94,7 @@ public class Context implements Configurable {
|
|||
", want=" + cipher.getKeyLength());
|
||||
}
|
||||
this.key = key;
|
||||
this.keyHash = MD5Hash.getMD5AsHex(encoded);
|
||||
this.keyHash = new String(Hex.encodeHex(Encryption.computeCryptoKeyHash(conf, encoded)));
|
||||
return this;
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -16,10 +16,11 @@
|
|||
*/
|
||||
package org.apache.hadoop.hbase.io.crypto;
|
||||
|
||||
import static java.lang.String.format;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.io.OutputStream;
|
||||
import java.security.DigestException;
|
||||
import java.security.Key;
|
||||
import java.security.MessageDigest;
|
||||
import java.security.NoSuchAlgorithmException;
|
||||
|
|
@ -51,6 +52,50 @@ public final class Encryption {
|
|||
|
||||
private static final Logger LOG = LoggerFactory.getLogger(Encryption.class);
|
||||
|
||||
|
||||
/**
|
||||
* Configuration key for globally enable / disable column family encryption
|
||||
*/
|
||||
public static final String CRYPTO_ENABLED_CONF_KEY = "hbase.crypto.enabled";
|
||||
|
||||
/**
|
||||
* Default value for globally enable / disable column family encryption
|
||||
* (set to "true" for backward compatibility)
|
||||
*/
|
||||
public static final boolean CRYPTO_ENABLED_CONF_DEFAULT = true;
|
||||
|
||||
/**
|
||||
* Configuration key for the hash algorithm used for generating key hash in encrypted HFiles.
|
||||
* This is a MessageDigest algorithm identifier string, like "MD5", "SHA-256" or "SHA-384".
|
||||
* (default: "MD5" for backward compatibility reasons)
|
||||
*/
|
||||
public static final String CRYPTO_KEY_HASH_ALGORITHM_CONF_KEY = "hbase.crypto.key.hash.algorithm";
|
||||
|
||||
/**
|
||||
* Default hash algorithm used for generating key hash in encrypted HFiles.
|
||||
* (we use "MD5" for backward compatibility reasons)
|
||||
*/
|
||||
public static final String CRYPTO_KEY_HASH_ALGORITHM_CONF_DEFAULT = "MD5";
|
||||
|
||||
/**
|
||||
* Configuration key for specifying the behaviour if the configured hash algorithm
|
||||
* differs from the one used for generating key hash in encrypted HFiles currently being read.
|
||||
*
|
||||
* - "false" (default): we won't fail but use the hash algorithm stored in the HFile
|
||||
* - "true": we throw an exception (this can be useful if regulations are enforcing the usage
|
||||
* of certain algorithms, e.g. on FIPS compliant clusters)
|
||||
*/
|
||||
public static final String CRYPTO_KEY_FAIL_ON_ALGORITHM_MISMATCH_CONF_KEY =
|
||||
"hbase.crypto.key.hash.algorithm.failOnMismatch";
|
||||
|
||||
/**
|
||||
* Default behaviour is not to fail if the hash algorithm configured differs from the one
|
||||
* used in the HFile. (this is the more fail-safe approach, allowing us to read
|
||||
* encrypted HFiles written using a different encryption key hash algorithm)
|
||||
*/
|
||||
public static final boolean CRYPTO_KEY_FAIL_ON_ALGORITHM_MISMATCH_CONF_DEFAULT = false;
|
||||
|
||||
|
||||
/**
|
||||
* Crypto context
|
||||
*/
|
||||
|
|
@ -99,6 +144,14 @@ public final class Encryption {
|
|||
super();
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Returns true if the column family encryption feature is enabled globally.
|
||||
*/
|
||||
public static boolean isEncryptionEnabled(Configuration conf) {
|
||||
return conf.getBoolean(CRYPTO_ENABLED_CONF_KEY, CRYPTO_ENABLED_CONF_DEFAULT);
|
||||
}
|
||||
|
||||
/**
|
||||
* Get an cipher given a name
|
||||
* @param name the cipher name
|
||||
|
|
@ -126,80 +179,64 @@ public final class Encryption {
|
|||
return getCipherProvider(conf).getSupportedCiphers();
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the Hash Algorithm defined in the crypto configuration.
|
||||
*/
|
||||
public static String getConfiguredHashAlgorithm(Configuration conf) {
|
||||
return conf.getTrimmed(CRYPTO_KEY_HASH_ALGORITHM_CONF_KEY,
|
||||
CRYPTO_KEY_HASH_ALGORITHM_CONF_DEFAULT);
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the Hash Algorithm mismatch behaviour defined in the crypto configuration.
|
||||
*/
|
||||
public static boolean failOnHashAlgorithmMismatch(Configuration conf) {
|
||||
return conf.getBoolean(CRYPTO_KEY_FAIL_ON_ALGORITHM_MISMATCH_CONF_KEY,
|
||||
CRYPTO_KEY_FAIL_ON_ALGORITHM_MISMATCH_CONF_DEFAULT);
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the hash of the supplied argument, using the hash algorithm
|
||||
* specified in the given config.
|
||||
*/
|
||||
public static byte[] computeCryptoKeyHash(Configuration conf, byte[] arg) {
|
||||
String algorithm = getConfiguredHashAlgorithm(conf);
|
||||
try {
|
||||
return hashWithAlg(algorithm, arg);
|
||||
} catch (RuntimeException e) {
|
||||
String message = format("Error in computeCryptoKeyHash (please check your configuration " +
|
||||
"parameter %s and the security provider configuration of the JVM)",
|
||||
CRYPTO_KEY_HASH_ALGORITHM_CONF_KEY);
|
||||
throw new RuntimeException(message, e);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Return the MD5 digest of the concatenation of the supplied arguments.
|
||||
*/
|
||||
public static byte[] hash128(String... args) {
|
||||
byte[] result = new byte[16];
|
||||
try {
|
||||
MessageDigest md = MessageDigest.getInstance("MD5");
|
||||
for (String arg: args) {
|
||||
md.update(Bytes.toBytes(arg));
|
||||
}
|
||||
md.digest(result, 0, result.length);
|
||||
return result;
|
||||
} catch (NoSuchAlgorithmException e) {
|
||||
throw new RuntimeException(e);
|
||||
} catch (DigestException e) {
|
||||
throw new RuntimeException(e);
|
||||
}
|
||||
return hashWithAlg("MD5", Bytes.toByteArrays(args));
|
||||
}
|
||||
|
||||
/**
|
||||
* Return the MD5 digest of the concatenation of the supplied arguments.
|
||||
*/
|
||||
public static byte[] hash128(byte[]... args) {
|
||||
byte[] result = new byte[16];
|
||||
try {
|
||||
MessageDigest md = MessageDigest.getInstance("MD5");
|
||||
for (byte[] arg: args) {
|
||||
md.update(arg);
|
||||
}
|
||||
md.digest(result, 0, result.length);
|
||||
return result;
|
||||
} catch (NoSuchAlgorithmException e) {
|
||||
throw new RuntimeException(e);
|
||||
} catch (DigestException e) {
|
||||
throw new RuntimeException(e);
|
||||
}
|
||||
return hashWithAlg("MD5", args);
|
||||
}
|
||||
|
||||
/**
|
||||
* Return the SHA-256 digest of the concatenation of the supplied arguments.
|
||||
*/
|
||||
public static byte[] hash256(String... args) {
|
||||
byte[] result = new byte[32];
|
||||
try {
|
||||
MessageDigest md = MessageDigest.getInstance("SHA-256");
|
||||
for (String arg: args) {
|
||||
md.update(Bytes.toBytes(arg));
|
||||
}
|
||||
md.digest(result, 0, result.length);
|
||||
return result;
|
||||
} catch (NoSuchAlgorithmException e) {
|
||||
throw new RuntimeException(e);
|
||||
} catch (DigestException e) {
|
||||
throw new RuntimeException(e);
|
||||
}
|
||||
return hashWithAlg("SHA-256", Bytes.toByteArrays(args));
|
||||
}
|
||||
|
||||
/**
|
||||
* Return the SHA-256 digest of the concatenation of the supplied arguments.
|
||||
*/
|
||||
public static byte[] hash256(byte[]... args) {
|
||||
byte[] result = new byte[32];
|
||||
try {
|
||||
MessageDigest md = MessageDigest.getInstance("SHA-256");
|
||||
for (byte[] arg: args) {
|
||||
md.update(arg);
|
||||
}
|
||||
md.digest(result, 0, result.length);
|
||||
return result;
|
||||
} catch (NoSuchAlgorithmException e) {
|
||||
throw new RuntimeException(e);
|
||||
} catch (DigestException e) {
|
||||
throw new RuntimeException(e);
|
||||
}
|
||||
return hashWithAlg("SHA-256", args);
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -578,4 +615,20 @@ public final class Encryption {
|
|||
} while (v > 0);
|
||||
}
|
||||
|
||||
/**
|
||||
* Return the hash of the concatenation of the supplied arguments, using the
|
||||
* hash algorithm provided.
|
||||
*/
|
||||
public static byte[] hashWithAlg(String algorithm, byte[]... args) {
|
||||
try {
|
||||
MessageDigest md = MessageDigest.getInstance(algorithm);
|
||||
for (byte[] arg: args) {
|
||||
md.update(arg);
|
||||
}
|
||||
return md.digest();
|
||||
} catch (NoSuchAlgorithmException e) {
|
||||
throw new RuntimeException("unable to use hash algorithm: " + algorithm, e);
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -31,4 +31,5 @@ message WrappedKey {
|
|||
required bytes data = 3;
|
||||
optional bytes iv = 4;
|
||||
optional bytes hash = 5;
|
||||
optional string hash_algorithm = 6 [default = "MD5"];
|
||||
}
|
||||
|
|
|
|||
|
|
@ -99,18 +99,23 @@ public class EncryptionTest {
|
|||
/**
|
||||
* Check that the specified cipher can be loaded and initialized, or throw
|
||||
* an exception. Verifies key and cipher provider configuration as a
|
||||
* prerequisite for cipher verification.
|
||||
* prerequisite for cipher verification. Also verifies if encryption is enabled globally.
|
||||
*
|
||||
* @param conf
|
||||
* @param cipher
|
||||
* @param key
|
||||
* @throws IOException
|
||||
* @param conf HBase configuration
|
||||
* @param cipher chiper algorith to use for the column family
|
||||
* @param key encryption key
|
||||
* @throws IOException in case of encryption configuration error
|
||||
*/
|
||||
public static void testEncryption(final Configuration conf, final String cipher,
|
||||
byte[] key) throws IOException {
|
||||
if (cipher == null) {
|
||||
return;
|
||||
}
|
||||
if(!Encryption.isEncryptionEnabled(conf)) {
|
||||
String message = String.format("Cipher %s failed test: encryption is disabled on the cluster",
|
||||
cipher);
|
||||
throw new IOException(message);
|
||||
}
|
||||
testKeyProvider(conf);
|
||||
testCipherProvider(conf);
|
||||
Boolean result = cipherResults.get(cipher);
|
||||
|
|
|
|||
|
|
@ -0,0 +1,110 @@
|
|||
/**
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.apache.hadoop.hbase.regionserver;
|
||||
|
||||
import static org.junit.Assert.assertTrue;
|
||||
import static org.junit.Assert.fail;
|
||||
|
||||
import org.apache.hadoop.conf.Configuration;
|
||||
import org.apache.hadoop.hbase.DoNotRetryIOException;
|
||||
import org.apache.hadoop.hbase.HBaseClassTestRule;
|
||||
import org.apache.hadoop.hbase.HBaseTestingUtility;
|
||||
import org.apache.hadoop.hbase.HConstants;
|
||||
import org.apache.hadoop.hbase.TableName;
|
||||
import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
|
||||
import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
|
||||
import org.apache.hadoop.hbase.io.crypto.Encryption;
|
||||
import org.apache.hadoop.hbase.io.crypto.KeyProviderForTesting;
|
||||
import org.apache.hadoop.hbase.testclassification.MasterTests;
|
||||
import org.apache.hadoop.hbase.testclassification.MediumTests;
|
||||
import org.apache.hadoop.hbase.util.Bytes;
|
||||
import org.apache.hadoop.hbase.util.TableDescriptorChecker;
|
||||
import org.junit.AfterClass;
|
||||
import org.junit.BeforeClass;
|
||||
import org.junit.ClassRule;
|
||||
import org.junit.Test;
|
||||
import org.junit.experimental.categories.Category;
|
||||
|
||||
@Category({MasterTests.class, MediumTests.class})
|
||||
public class TestEncryptionDisabled {
|
||||
|
||||
@ClassRule
|
||||
public static final HBaseClassTestRule CLASS_RULE =
|
||||
HBaseClassTestRule.forClass(TestEncryptionDisabled.class);
|
||||
|
||||
private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
|
||||
private static Configuration conf = TEST_UTIL.getConfiguration();
|
||||
private static TableDescriptorBuilder tdb;
|
||||
|
||||
|
||||
@BeforeClass
|
||||
public static void setUp() throws Exception {
|
||||
conf.setInt("hfile.format.version", 3);
|
||||
conf.set(HConstants.CRYPTO_KEYPROVIDER_CONF_KEY, KeyProviderForTesting.class.getName());
|
||||
conf.set(HConstants.CRYPTO_MASTERKEY_NAME_CONF_KEY, "hbase");
|
||||
conf.set(Encryption.CRYPTO_ENABLED_CONF_KEY, "false");
|
||||
conf.set(TableDescriptorChecker.TABLE_SANITY_CHECKS, "true");
|
||||
|
||||
// Start the minicluster
|
||||
TEST_UTIL.startMiniCluster(1);
|
||||
}
|
||||
|
||||
@AfterClass
|
||||
public static void tearDown() throws Exception {
|
||||
TEST_UTIL.shutdownMiniCluster();
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testEncryptedTableShouldNotBeCreatedWhenEncryptionDisabled() throws Exception {
|
||||
// Create the table schema
|
||||
// Specify an encryption algorithm without a key (normally HBase would generate a random key)
|
||||
tdb = TableDescriptorBuilder.newBuilder(TableName.valueOf("default",
|
||||
"TestEncryptionDisabledFail"));
|
||||
ColumnFamilyDescriptorBuilder columnFamilyDescriptorBuilder =
|
||||
ColumnFamilyDescriptorBuilder.newBuilder(Bytes.toBytes("cf"));
|
||||
String algorithm = conf.get(HConstants.CRYPTO_KEY_ALGORITHM_CONF_KEY, HConstants.CIPHER_AES);
|
||||
columnFamilyDescriptorBuilder.setEncryptionType(algorithm);
|
||||
tdb.setColumnFamily(columnFamilyDescriptorBuilder.build());
|
||||
|
||||
// Create the test table, we expect to get back an exception
|
||||
try {
|
||||
TEST_UTIL.getAdmin().createTable(tdb.build());
|
||||
} catch (DoNotRetryIOException e) {
|
||||
assertTrue(e.getMessage().contains("encryption is disabled on the cluster"));
|
||||
return;
|
||||
} catch (Exception e) {
|
||||
throw new RuntimeException("create table command failed for the wrong reason", e);
|
||||
}
|
||||
fail("create table command unexpectedly succeeded");
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testNonEncryptedTableShouldBeCreatedWhenEncryptionDisabled() throws Exception {
|
||||
// Create the table schema
|
||||
tdb = TableDescriptorBuilder.newBuilder(TableName.valueOf("default",
|
||||
"TestEncryptionDisabledSuccess"));
|
||||
ColumnFamilyDescriptorBuilder columnFamilyDescriptorBuilder =
|
||||
ColumnFamilyDescriptorBuilder.newBuilder(Bytes.toBytes("cf"));
|
||||
tdb.setColumnFamily(columnFamilyDescriptorBuilder.build());
|
||||
|
||||
// Create the test table, this should succeed, as we don't use encryption
|
||||
TEST_UTIL.getAdmin().createTable(tdb.build());
|
||||
TEST_UTIL.waitTableAvailable(tdb.build().getTableName(), 5000);
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -27,6 +27,7 @@ import org.apache.hadoop.hbase.HConstants;
|
|||
import org.apache.hadoop.hbase.io.crypto.Cipher;
|
||||
import org.apache.hadoop.hbase.io.crypto.CipherProvider;
|
||||
import org.apache.hadoop.hbase.io.crypto.DefaultCipherProvider;
|
||||
import org.apache.hadoop.hbase.io.crypto.Encryption;
|
||||
import org.apache.hadoop.hbase.io.crypto.KeyProvider;
|
||||
import org.apache.hadoop.hbase.io.crypto.KeyProviderForTesting;
|
||||
import org.apache.hadoop.hbase.testclassification.MiscTests;
|
||||
|
|
@ -91,6 +92,35 @@ public class TestEncryptionTest {
|
|||
} catch (Exception e) { }
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testTestEnabled() {
|
||||
Configuration conf = HBaseConfiguration.create();
|
||||
conf.set(HConstants.CRYPTO_KEYPROVIDER_CONF_KEY, KeyProviderForTesting.class.getName());
|
||||
String algorithm =
|
||||
conf.get(HConstants.CRYPTO_KEY_ALGORITHM_CONF_KEY, HConstants.CIPHER_AES);
|
||||
try {
|
||||
EncryptionTest.testEncryption(conf, algorithm, null);
|
||||
} catch (Exception e) {
|
||||
fail("Test for cipher " + algorithm + " should have succeeded, when " +
|
||||
Encryption.CRYPTO_ENABLED_CONF_KEY + " is not set");
|
||||
}
|
||||
|
||||
conf.setBoolean(Encryption.CRYPTO_ENABLED_CONF_KEY, true);
|
||||
try {
|
||||
EncryptionTest.testEncryption(conf, algorithm, null);
|
||||
} catch (Exception e) {
|
||||
fail("Test for cipher " + algorithm + " should have succeeded, when " +
|
||||
Encryption.CRYPTO_ENABLED_CONF_KEY + " is set to true");
|
||||
}
|
||||
|
||||
conf.setBoolean(Encryption.CRYPTO_ENABLED_CONF_KEY, false);
|
||||
try {
|
||||
EncryptionTest.testEncryption(conf, algorithm, null);
|
||||
fail("Test for cipher " + algorithm + " should have failed, when " +
|
||||
Encryption.CRYPTO_ENABLED_CONF_KEY + " is set to false");
|
||||
} catch (Exception e) { }
|
||||
}
|
||||
|
||||
public static class FailingKeyProvider implements KeyProvider {
|
||||
|
||||
@Override
|
||||
|
|
|
|||
Loading…
Reference in New Issue