NIFI-7669 Changed custom PBE AEAD algorithm to derive key once rather than on every encrypt/decrypt operation, leading to substantial performance gains.

Updated documentation.
Added unit tests.

NIFI-7669 Moved time-based encryption tests to integration tests to avoid running during CI builds.

NIFI-7669 Fixed failing test due to nifi.properties initialization.

Signed-off-by: Pierre Villard <pierre.villard.fr@gmail.com>

This closes #4435.
This commit is contained in:
Andy LoPresto 2020-07-27 14:59:35 -07:00 committed by Pierre Villard
parent 7f0416ee8b
commit 716ba992f5
No known key found for this signature in database
GPG Key ID: F92A93B30C07C6D5
6 changed files with 286 additions and 83 deletions

View File

@ -1592,7 +1592,7 @@ NiFi always stores all sensitive values (passwords, tokens, and other credential
Both options require a password (`nifi.sensitive.props.key` value) of *at least 12 characters*. This means the "default" value (if left empty, a hard-coded default is used) will not be sufficient.
These options provide a bridge solution to higher security without requiring a change to the structure of _nifi.properties_. Due to the implementation of flow synchronization, on every change to the flow definition, all sensitive properties are re-encrypted during flow serialization, and each encryption operation requires the derivation of the key. _As Argon2 is intentionally time-hard, this will introduce an approximately 1 second cost per sensitive value per flow modification._ This is determined to be an acceptable tradeoff for security at this time but will be remedied with an internal key caching mechanism in a future release. In addition, a more full-featured configuration process, allowing for arbitrary combinations of KDFs and encryption algorithms, will be added in a future release. See link:https://issues.apache.org/jira/browse/NIFI-7638[NIFI-7638^], link:https://issues.apache.org/jira/browse/NIFI-7668[NIFI-7668^], link:https://issues.apache.org/jira/browse/NIFI-7669[NIFI-7669^], and link:https://issues.apache.org/jira/browse/NIFI-7670[NIFI-7670^] for more details.
These options provide a bridge solution to higher security without requiring a change to the structure of _nifi.properties_. A more full-featured configuration process, allowing for arbitrary combinations of KDFs and encryption algorithms, will be added in a future release. See link:https://issues.apache.org/jira/browse/NIFI-7668[NIFI-7668^] and link:https://issues.apache.org/jira/browse/NIFI-7670[NIFI-7670^] for more details.
[[encrypt-config_tool]]
== Encrypted Passwords in Configuration Files

View File

@ -33,6 +33,7 @@ import org.apache.commons.lang3.StringUtils;
import org.apache.nifi.security.kms.CryptoUtils;
import org.apache.nifi.security.util.EncryptionMethod;
import org.apache.nifi.security.util.KeyDerivationFunction;
import org.apache.nifi.security.util.crypto.Argon2SecureHasher;
import org.apache.nifi.security.util.crypto.CipherProvider;
import org.apache.nifi.security.util.crypto.CipherProviderFactory;
import org.apache.nifi.security.util.crypto.CipherUtility;
@ -88,7 +89,7 @@ public class StringEncryptor {
private final String algorithm;
private final String provider;
private final PBEKeySpec password;
private final SecretKeySpec key;
private SecretKeySpec key;
private static final String HEX_ENCODING = "HEX";
private static final String B64_ENCODING = "BASE64";
@ -260,7 +261,15 @@ public class StringEncryptor {
if (paramsAreValid()) {
if (isCustomAlgorithm(algorithm)) {
// Handle the initialization for Argon2 + AES
cipherProvider = CipherProviderFactory.getCipherProvider(KeyDerivationFunction.ARGON2);
// Perform the Argon2 key derivation once and store the key
Argon2SecureHasher argon2SecureHasher = new Argon2SecureHasher();
byte[] passwordBytes = new String(password.getPassword()).getBytes(StandardCharsets.UTF_8);
byte[] derivedKey = argon2SecureHasher.hashRaw(passwordBytes);
key = new SecretKeySpec(derivedKey, "AES");
// Use an AES keyed cipher provider to avoid derivation every time
cipherProvider = CipherProviderFactory.getCipherProvider(KeyDerivationFunction.NONE);
} else if (CipherUtility.isPBECipher(algorithm)) {
cipherProvider = CipherProviderFactory.getCipherProvider(KeyDerivationFunction.NIFI_LEGACY);
} else {
@ -303,8 +312,7 @@ public class StringEncryptor {
private boolean customSecretIsValid(PBEKeySpec password, SecretKeySpec key, String algorithm) {
// Currently, the only custom algorithms use AES-G/CM with a password via Argon2
String rawPassword = new String(password.getPassword());
final boolean secretIsValid = StringUtils.isNotBlank(rawPassword) && rawPassword.trim().length() >= 12;
return secretIsValid;
return StringUtils.isNotBlank(rawPassword) && rawPassword.trim().length() >= 12;
}
private boolean keyIsValid(SecretKeySpec key, String algorithm) {
@ -340,10 +348,10 @@ public class StringEncryptor {
try {
if (isInitialized()) {
byte[] rawBytes;
// Currently all custom algorithms are PBE (Argon2)
if (CipherUtility.isPBECipher(algorithm) || isCustomAlgorithm(algorithm)) {
if (CipherUtility.isPBECipher(algorithm)) {
rawBytes = encryptPBE(clearText);
} else {
// Currently all custom algorithms are keyed (Argon2 KDF has already run in initialization)
rawBytes = encryptKeyed(clearText);
}
return encode(rawBytes);
@ -446,10 +454,10 @@ public class StringEncryptor {
if (isInitialized()) {
byte[] plainBytes;
byte[] cipherBytes = decode(cipherText);
// Currently all custom algorithms are PBE (Argon2)
if (CipherUtility.isPBECipher(algorithm) || isCustomAlgorithm(algorithm)) {
if (CipherUtility.isPBECipher(algorithm)) {
plainBytes = decryptPBE(cipherBytes);
} else {
// Currently all custom algorithms are keyed (Argon2 KDF has already run in initialization)
plainBytes = decryptKeyed(cipherBytes);
}
return new String(plainBytes, StandardCharsets.UTF_8);

View File

@ -0,0 +1,124 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License") you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.nifi.encrypt
import org.bouncycastle.jce.provider.BouncyCastleProvider
import org.junit.After
import org.junit.Before
import org.junit.BeforeClass
import org.junit.Test
import org.junit.runner.RunWith
import org.junit.runners.JUnit4
import org.slf4j.Logger
import org.slf4j.LoggerFactory
import java.security.Security
@RunWith(JUnit4.class)
class StringEncryptorIT {
private static final Logger logger = LoggerFactory.getLogger(StringEncryptorIT.class)
private static final String DEFAULT_PROVIDER = "BC"
@BeforeClass
static void setUpOnce() throws Exception {
Security.addProvider(new BouncyCastleProvider())
logger.metaClass.methodMissing = { String name, args ->
logger.info("[${name?.toUpperCase()}] ${(args as List).join(" ")}")
}
}
@Before
void setUp() throws Exception {
}
@After
void tearDown() throws Exception {
}
private static long time(Closure code) {
long startNanos = System.nanoTime()
code.run()
long stopNanos = System.nanoTime()
return stopNanos - startNanos
}
/**
* Checks the custom algorithm (Argon2+AES-G/CM) doesn't derive the key (a slow process) on every encrypt/decrypt operation.
*
* @throws Exception
*/
@Test
void testCustomAlgorithmShouldOnlyDeriveKeyOnce() throws Exception {
// Arrange
final String CUSTOM_ALGORITHM = "NIFI_ARGON2_AES_GCM_256"
final String PASSWORD = "nifiPassword123"
final String plaintext = "some sensitive flow value"
final long SLOW_DURATION_NANOS = 500 * 1000 * 1000 // 500 ms
final long FAST_DURATION_NANOS = 1 * 1000 * 1000 // 1 ms
int testIterations = 100 //_000
def results = []
def resultDurations = []
StringEncryptor encryptor
long createNanos = time {
encryptor = StringEncryptor.createEncryptor(CUSTOM_ALGORITHM, DEFAULT_PROVIDER, PASSWORD)
}
logger.info("Created encryptor: ${encryptor} in ${createNanos / 1_000_000} ms")
// Prime the cipher with one encrypt/decrypt cycle
String primeCT = encryptor.encrypt(plaintext)
encryptor.decrypt(primeCT)
// Act
testIterations.times { int i ->
// Encrypt the value
def ciphertext
long encryptNanos = time {
ciphertext = encryptor.encrypt(plaintext)
}
logger.info("Encrypted plaintext in ${encryptNanos / 1_000_000} ms")
def recovered
long durationNanos = time {
recovered = encryptor.decrypt(ciphertext)
}
logger.info("Recovered ciphertext to ${recovered} in ${durationNanos / 1_000_000} ms")
results << recovered
resultDurations << encryptNanos
resultDurations << durationNanos
}
def milliDurations = [resultDurations.min(), resultDurations.max(), resultDurations.sum() / resultDurations.size()].collect { it / 1_000_000 }
logger.info("Min/Max/Avg durations in ms: ${milliDurations}")
// Assert
// The initial creation (including key derivation) should be slow
assert createNanos > SLOW_DURATION_NANOS
// The encryption/decryption process (repeated) should be fast
assert resultDurations.max() <= FAST_DURATION_NANOS * 3
assert resultDurations.sum() / testIterations < FAST_DURATION_NANOS
}
}

View File

@ -21,7 +21,7 @@ import org.apache.nifi.properties.StandardNiFiProperties
import org.apache.nifi.security.kms.CryptoUtils
import org.apache.nifi.security.util.EncryptionMethod
import org.apache.nifi.security.util.crypto.AESKeyedCipherProvider
import org.apache.nifi.security.util.crypto.Argon2CipherProvider
import org.apache.nifi.security.util.crypto.Argon2SecureHasher
import org.apache.nifi.security.util.crypto.CipherUtility
import org.apache.nifi.security.util.crypto.KeyedCipherProvider
import org.apache.nifi.util.NiFiProperties
@ -614,16 +614,15 @@ class StringEncryptorTest {
logger.info("Encrypted plaintext to ${ciphertext}")
// Decrypt the ciphertext using a manually-constructed cipher to validate
byte[] saltIvAndCipherBytes = Hex.decodeHex(ciphertext)
int sl = StringEncryptor.CUSTOM_ALGORITHM_SALT_LENGTH
byte[] saltBytes = saltIvAndCipherBytes[0..<sl]
byte[] ivBytes = saltIvAndCipherBytes[sl..<sl + IV_LENGTH]
byte[] cipherBytes = saltIvAndCipherBytes[sl + IV_LENGTH..-1]
int keyLength = CipherUtility.parseKeyLengthFromAlgorithm(CUSTOM_ALGORITHM)
byte[] ivAndCipherBytes = Hex.decodeHex(ciphertext)
byte[] ivBytes = ivAndCipherBytes[0..<IV_LENGTH]
byte[] cipherBytes = ivAndCipherBytes[IV_LENGTH..-1]
// Construct the decryption cipher provider manually
Argon2CipherProvider a2cp = new Argon2CipherProvider()
Cipher decryptCipher = a2cp.getCipher(EncryptionMethod.AES_GCM, PASSWORD, saltBytes, ivBytes, keyLength, false)
Argon2SecureHasher a2sh = new Argon2SecureHasher()
SecretKeySpec secretKey = new SecretKeySpec(a2sh.hashRaw(PASSWORD.bytes), "AES")
AESKeyedCipherProvider cipherProvider = new AESKeyedCipherProvider()
Cipher decryptCipher = cipherProvider.getCipher(EncryptionMethod.AES_GCM, secretKey, ivBytes, false)
// Decrypt a known message with the cipher
byte[] recoveredBytes = decryptCipher.doFinal(cipherBytes)
@ -646,20 +645,18 @@ class StringEncryptorTest {
final String PASSWORD = "nifiPassword123"
final String plaintext = "some sensitive flow value"
int keyLength = CipherUtility.parseKeyLengthFromAlgorithm(CUSTOM_ALGORITHM)
// Construct the encryption cipher provider manually
Argon2SecureHasher a2sh = new Argon2SecureHasher()
SecretKeySpec secretKey = new SecretKeySpec(a2sh.hashRaw(PASSWORD.bytes), "AES")
AESKeyedCipherProvider cipherProvider = new AESKeyedCipherProvider()
// Manually construct a cipher provider with a key derived from the password using Argon2
Argon2CipherProvider a2cp = new Argon2CipherProvider()
// Generate salt and IV
byte[] ivBytes = new byte[16]
new SecureRandom().nextBytes(ivBytes)
byte[] saltBytes = a2cp.generateSalt()
Cipher encryptCipher = a2cp.getCipher(EncryptionMethod.AES_GCM, PASSWORD, saltBytes, ivBytes, keyLength, true)
Cipher encryptCipher = cipherProvider.getCipher(EncryptionMethod.AES_GCM, secretKey, ivBytes, true)
// Encrypt a known message with the cipher
byte[] cipherBytes = encryptCipher.doFinal(plaintext.getBytes(StandardCharsets.UTF_8))
byte[] concatenatedBytes = CryptoUtils.concatByteArrays(saltBytes, ivBytes, cipherBytes)
byte[] concatenatedBytes = CryptoUtils.concatByteArrays(ivBytes, cipherBytes)
def ciphertext = Hex.encodeHexString(concatenatedBytes)
logger.info("Encrypted plaintext to ${ciphertext}")

View File

@ -0,0 +1,130 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.nifi.fingerprint
import org.apache.nifi.encrypt.StringEncryptor
import org.apache.nifi.nar.ExtensionManager
import org.apache.nifi.nar.StandardExtensionDiscoveringManager
import org.apache.nifi.util.NiFiProperties
import org.bouncycastle.jce.provider.BouncyCastleProvider
import org.junit.After
import org.junit.AfterClass
import org.junit.Before
import org.junit.BeforeClass
import org.junit.Test
import org.junit.runner.RunWith
import org.junit.runners.JUnit4
import org.slf4j.Logger
import org.slf4j.LoggerFactory
import java.security.Security
@RunWith(JUnit4.class)
class FingerprintFactoryGroovyIT extends GroovyTestCase {
private static final Logger logger = LoggerFactory.getLogger(FingerprintFactoryGroovyIT.class)
private static StringEncryptor mockEncryptor = [
encrypt: { String plaintext -> plaintext.reverse() },
decrypt: { String cipherText -> cipherText.reverse() }] as StringEncryptor
private static ExtensionManager extensionManager = new StandardExtensionDiscoveringManager()
private static String originalPropertiesPath = System.getProperty(NiFiProperties.PROPERTIES_FILE_PATH)
private static final String NIFI_PROPERTIES_PATH = "src/test/resources/conf/nifi.properties"
@BeforeClass
static void setUpOnce() throws Exception {
Security.addProvider(new BouncyCastleProvider())
logger.metaClass.methodMissing = { String name, args ->
logger.info("[${name?.toUpperCase()}] ${(args as List).join(" ")}")
}
}
@Before
void setUp() throws Exception {
}
@After
void tearDown() throws Exception {
}
@AfterClass
static void tearDownOnce() {
if (originalPropertiesPath) {
System.setProperty(NiFiProperties.PROPERTIES_FILE_PATH, originalPropertiesPath)
}
}
/**
* The initial implementation derived the hashed value using a time/memory-hard algorithm (Argon2) every time.
* For large flow definitions, this blocked startup for minutes. Deriving a secure key with the Argon2
* algorithm once at startup (~1 second) and using this cached key for a simple HMAC/SHA-256 operation on every
* fingerprint should be much faster.
*/
@Test
void testCreateFingerprintShouldNotBeSlow() {
// Arrange
int testIterations = 100 //_000
// Set up test nifi.properties
System.setProperty(NiFiProperties.PROPERTIES_FILE_PATH, NIFI_PROPERTIES_PATH)
// Create flow
String initialFlowXML = new File("src/test/resources/nifi/fingerprint/initial.xml").text
logger.info("Read initial flow: ${initialFlowXML[0..<100]}...")
// Create the FingerprintFactory with collaborators
FingerprintFactory fingerprintFactory = new FingerprintFactory(mockEncryptor, extensionManager)
def results = []
def resultDurations = []
// Act
testIterations.times { int i ->
long startNanos = System.nanoTime()
// Create the fingerprint from the flow
String fingerprint = fingerprintFactory.createFingerprint(initialFlowXML.bytes)
long endNanos = System.nanoTime()
long durationNanos = endNanos - startNanos
logger.info("Generated flow fingerprint: ${fingerprint} in ${durationNanos} ns")
results << fingerprint
resultDurations << durationNanos
}
def milliDurations = [resultDurations.min(), resultDurations.max(), resultDurations.sum() / resultDurations.size()].collect { it / 1_000_000 }
logger.info("Min/Max/Avg durations in ms: ${milliDurations}")
// Assert
final long MAX_DURATION_NANOS = 1_000_000_000 // 1 second
assert resultDurations.max() <= MAX_DURATION_NANOS * 2
assert resultDurations.sum() / testIterations < MAX_DURATION_NANOS
// Assert the fingerprint does not contain the password
results.each { String fingerprint ->
assert !(fingerprint =~ "originalPlaintextPassword")
def maskedValue = (fingerprint =~ /\[MASKED\] \([\w\/\+=]+\)/)
assert maskedValue
logger.info("Masked value: ${maskedValue[0]}")
}
}
}

View File

@ -77,6 +77,7 @@ class FingerprintFactoryGroovyTest extends GroovyTestCase {
@Test
void testCreateFingerprintShouldNotDiscloseSensitivePropertyValues() {
// Arrange
System.setProperty(NiFiProperties.PROPERTIES_FILE_PATH, NIFI_PROPERTIES_PATH)
// Create flow
String initialFlowXML = new File("src/test/resources/nifi/fingerprint/initial.xml").text
@ -99,61 +100,4 @@ class FingerprintFactoryGroovyTest extends GroovyTestCase {
assert maskedValue
logger.info("Masked value: ${maskedValue[0]}")
}
/**
* The initial implementation derived the hashed value using a time/memory-hard algorithm (Argon2) every time.
* For large flow definitions, this blocked startup for minutes. Deriving a secure key with the Argon2
* algorithm once at startup (~1 second) and using this cached key for a simple HMAC/SHA-256 operation on every
* fingerprint should be much faster.
*/
@Test
void testCreateFingerprintShouldNotBeSlow() {
// Arrange
int testIterations = 100 //_000
// Set up test nifi.properties
System.setProperty(NiFiProperties.PROPERTIES_FILE_PATH, NIFI_PROPERTIES_PATH)
// Create flow
String initialFlowXML = new File("src/test/resources/nifi/fingerprint/initial.xml").text
logger.info("Read initial flow: ${initialFlowXML[0..<100]}...")
// Create the FingerprintFactory with collaborators
FingerprintFactory fingerprintFactory = new FingerprintFactory(mockEncryptor, extensionManager)
def results = []
def resultDurations = []
// Act
testIterations.times { int i ->
long startNanos = System.nanoTime()
// Create the fingerprint from the flow
String fingerprint = fingerprintFactory.createFingerprint(initialFlowXML.bytes)
long endNanos = System.nanoTime()
long durationNanos = endNanos - startNanos
logger.info("Generated flow fingerprint: ${fingerprint} in ${durationNanos} ns")
results << fingerprint
resultDurations << durationNanos
}
def milliDurations = [resultDurations.min(), resultDurations.max(), resultDurations.sum() / resultDurations.size()].collect { it / 1_000_000 }
logger.info("Min/Max/Avg durations in ms: ${milliDurations}")
// Assert
final long MAX_DURATION_NANOS = 1_000_000_000 // 1 second
assert resultDurations.max() <= MAX_DURATION_NANOS * 2
assert resultDurations.sum() / testIterations < MAX_DURATION_NANOS
// Assert the fingerprint does not contain the password
results.each { String fingerprint ->
assert !(fingerprint =~ "originalPlaintextPassword")
def maskedValue = (fingerprint =~ /\[MASKED\] \([\w\/\+=]+\)/)
assert maskedValue
logger.info("Masked value: ${maskedValue[0]}")
}
}
}