NIFI-4139

- Moved key provider interface and implementations from nifi-data-provenance-utils module to nifi-security-utils module.
- Refactored duplicate byte[] concatenation methods from utility classes and removed deprecation warnings from CipherUtility.
- Created KeyProviderFactory to encapsulate key provider instantiation logic.
- Added logic to handle legacy package configuration values for key providers.
- Added unit tests.
- Added resource files for un/limited strength cryptography scenarios.
- Added ASL to test resources.
- Moved legacy FQCN handling logic to CryptUtils.
- Added unit tests to ensure application startup logic handles legacy FQCNs.
- Moved master key extraction/provision out of FBKP.
- Removed nifi-security-utils dependency on nifi-properties-loader module.
- Added unit tests.

nifi-commons/nifi-data-provenance-utils/src/main/java/org/apache/nifi/provenance/AESProvenanceEventEncryptor.java

@@ -31,6 +31,8 @@ import javax.crypto.Cipher;
 import javax.crypto.IllegalBlockSizeException;
 import javax.crypto.SecretKey;
 import org.apache.commons.lang3.StringUtils;
+import org.apache.nifi.security.kms.CryptoUtils;
+import org.apache.nifi.security.kms.KeyProvider;
 import org.apache.nifi.security.util.EncryptionMethod;
 import org.apache.nifi.security.util.crypto.AESKeyedCipherProvider;
 import org.bouncycastle.jce.provider.BouncyCastleProvider;

nifi-commons/nifi-data-provenance-utils/src/main/java/org/apache/nifi/provenance/ProvenanceEventEncryptor.java

@@ -17,6 +17,7 @@
 package org.apache.nifi.provenance;
 
 import java.security.KeyManagementException;
+import org.apache.nifi.security.kms.KeyProvider;
 
 public interface ProvenanceEventEncryptor {
 

nifi-commons/nifi-data-provenance-utils/src/test/groovy/org/apache/nifi/provenance/AESProvenanceEventEncryptorTest.groovy

@@ -16,6 +16,8 @@
  */
 package org.apache.nifi.provenance
 
+import org.apache.nifi.security.kms.CryptoUtils
+import org.apache.nifi.security.kms.KeyProvider
 import org.apache.nifi.security.util.EncryptionMethod
 import org.apache.nifi.security.util.crypto.AESKeyedCipherProvider
 import org.bouncycastle.jce.provider.BouncyCastleProvider

nifi-commons/nifi-security-utils/src/main/java/org/apache/nifi/security/kms/CryptoUtils.java

@@ -14,7 +14,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.nifi.provenance;
+package org.apache.nifi.security.kms;
 
 import java.io.BufferedReader;
 import java.io.File;
@@ -43,8 +43,13 @@ import org.slf4j.LoggerFactory;
 
 public class CryptoUtils {
     private static final Logger logger = LoggerFactory.getLogger(StaticKeyProvider.class);
-    private static final String STATIC_KEY_PROVIDER_CLASS_NAME = "org.apache.nifi.provenance.StaticKeyProvider";
+    private static final String STATIC_KEY_PROVIDER_CLASS_NAME = "org.apache.nifi.security.kms.StaticKeyProvider";
-    private static final String FILE_BASED_KEY_PROVIDER_CLASS_NAME = "org.apache.nifi.provenance.FileBasedKeyProvider";
+    private static final String FILE_BASED_KEY_PROVIDER_CLASS_NAME = "org.apache.nifi.security.kms.FileBasedKeyProvider";
+
+    private static final String LEGACY_SKP_FQCN = "org.apache.nifi.provenance.StaticKeyProvider";
+    private static final String LEGACY_FBKP_FQCN = "org.apache.nifi.provenance.FileBasedKeyProvider";
+
+
     private static final Pattern HEX_PATTERN = Pattern.compile("(?i)^[0-9a-f]+$");
 
     private static final List<Integer> UNLIMITED_KEY_LENGTHS = Arrays.asList(32, 48, 64);
@@ -101,6 +106,24 @@ public class CryptoUtils {
      * @return true if the provided configuration is valid
      */
     public static boolean isValidKeyProvider(String keyProviderImplementation, String keyProviderLocation, String keyId, Map<String, String> encryptionKeys) {
+        logger.debug("Attempting to validate the key provider: keyProviderImplementation = "
+                + keyProviderImplementation + " , keyProviderLocation = "
+                + keyProviderLocation + " , keyId = "
+                + keyId + " , encryptionKeys = "
+                + ((encryptionKeys == null) ? "0" : encryptionKeys.size()));
+
+        try {
+            keyProviderImplementation = handleLegacyPackages(keyProviderImplementation);
+        } catch (KeyManagementException e) {
+            logger.error("The attempt to validate the key provider failed keyProviderImplementation = "
+                    + keyProviderImplementation + " , keyProviderLocation = "
+                    + keyProviderLocation + " , keyId = "
+                    + keyId + " , encryptionKeys = "
+                    + ((encryptionKeys == null) ? "0" : encryptionKeys.size()));
+
+            return false;
+        }
+
         if (STATIC_KEY_PROVIDER_CLASS_NAME.equals(keyProviderImplementation)) {
             // Ensure the keyId and key(s) are valid
             if (encryptionKeys == null) {
@@ -124,6 +147,19 @@ public class CryptoUtils {
         }
     }
 
+    static String handleLegacyPackages(String implementationClassName) throws KeyManagementException {
+        if (org.apache.nifi.util.StringUtils.isBlank(implementationClassName)) {
+            throw new KeyManagementException("Invalid key provider implementation provided: " + implementationClassName);
+        }
+        if (implementationClassName.equalsIgnoreCase(LEGACY_SKP_FQCN)) {
+            return StaticKeyProvider.class.getName();
+        } else if (implementationClassName.equalsIgnoreCase(LEGACY_FBKP_FQCN)) {
+            return FileBasedKeyProvider.class.getName();
+        } else {
+            return implementationClassName;
+        }
+    }
+
     /**
      * Returns true if the provided key is valid hex and is the correct length for the current system's JCE policies.
      *
@@ -177,6 +213,10 @@ public class CryptoUtils {
         if (StringUtils.isBlank(filepath)) {
             throw new KeyManagementException("The key provider file is not present and readable");
         }
+        if (masterKey == null) {
+            throw new KeyManagementException("The master key must be provided to decrypt the individual keys");
+        }
+
         File file = new File(filepath);
         if (!file.exists() || !file.canRead()) {
             throw new KeyManagementException("The key provider file is not present and readable");

nifi-commons/nifi-security-utils/src/main/java/org/apache/nifi/security/kms/FileBasedKeyProvider.java

@@ -14,15 +14,11 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.nifi.provenance;
+package org.apache.nifi.security.kms;
 
-import java.io.IOException;
 import java.security.KeyManagementException;
 import javax.crypto.SecretKey;
-import javax.crypto.spec.SecretKeySpec;
 import javax.naming.OperationNotSupportedException;
-import org.apache.nifi.properties.NiFiPropertiesLoader;
-import org.bouncycastle.util.encoders.Hex;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -31,26 +27,11 @@ public class FileBasedKeyProvider extends StaticKeyProvider {
 
     private String filepath;
 
-    FileBasedKeyProvider(String location) throws KeyManagementException {
+    public FileBasedKeyProvider(String location, SecretKey masterKey) throws KeyManagementException {
-        this(location, getMasterKey());
-    }
-
-    FileBasedKeyProvider(String location, SecretKey masterKey) throws KeyManagementException {
         super(CryptoUtils.readKeys(location, masterKey));
         this.filepath = location;
     }
 
-    private static SecretKey getMasterKey() throws KeyManagementException {
-        try {
-            // Get the master encryption key from bootstrap.conf
-            String masterKeyHex = NiFiPropertiesLoader.extractKeyFromBootstrapFile();
-            return new SecretKeySpec(Hex.decode(masterKeyHex), "AES");
-        } catch (IOException e) {
-            logger.error("Encountered an error: ", e);
-            throw new KeyManagementException(e);
-        }
-    }
-
     /**
      * Adds the key to the provider and associates it with the given ID. Some implementations may not allow this operation.
      *

nifi-commons/nifi-security-utils/src/main/java/org/apache/nifi/security/kms/KeyProvider.java

@@ -14,7 +14,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.nifi.provenance;
+package org.apache.nifi.security.kms;
 
 import java.security.KeyManagementException;
 import java.util.List;

nifi-commons/nifi-security-utils/src/main/java/org/apache/nifi/security/kms/KeyProviderFactory.java

@@ -0,0 +1,72 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.security.kms;
+
+import java.security.KeyManagementException;
+import java.util.Map;
+import java.util.stream.Collectors;
+import javax.crypto.SecretKey;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class KeyProviderFactory {
+    private static final Logger logger = LoggerFactory.getLogger(KeyProviderFactory.class);
+
+    public static KeyProvider buildKeyProvider(String implementationClassName, String keyProviderLocation, String keyId, Map<String, String> encryptionKeys,
+                                               SecretKey masterKey) throws KeyManagementException {
+        KeyProvider keyProvider;
+
+       implementationClassName = CryptoUtils.handleLegacyPackages(implementationClassName);
+
+        if (StaticKeyProvider.class.getName().equals(implementationClassName)) {
+            // Get all the keys (map) from config
+            if (CryptoUtils.isValidKeyProvider(implementationClassName, keyProviderLocation, keyId, encryptionKeys)) {
+                Map<String, SecretKey> formedKeys = encryptionKeys.entrySet().stream()
+                        .collect(Collectors.toMap(
+                                Map.Entry::getKey,
+                                e -> {
+                                    try {
+                                        return CryptoUtils.formKeyFromHex(e.getValue());
+                                    } catch (KeyManagementException e1) {
+                                        // This should never happen because the hex has already been validated
+                                        logger.error("Encountered an error: ", e1);
+                                        return null;
+                                    }
+                                }));
+                keyProvider = new StaticKeyProvider(formedKeys);
+            } else {
+                final String msg = "The StaticKeyProvider definition is not valid";
+                logger.error(msg);
+                throw new KeyManagementException(msg);
+            }
+        } else if (FileBasedKeyProvider.class.getName().equals(implementationClassName)) {
+            keyProvider = new FileBasedKeyProvider(keyProviderLocation, masterKey);
+            if (!keyProvider.keyExists(keyId)) {
+                throw new KeyManagementException("The specified key ID " + keyId + " is not in the key definition file");
+            }
+        } else {
+            throw new KeyManagementException("Invalid key provider implementation provided: " + implementationClassName);
+        }
+
+        return keyProvider;
+    }
+
+    public static boolean requiresMasterKey(String implementationClassName) throws KeyManagementException {
+        implementationClassName = CryptoUtils.handleLegacyPackages(implementationClassName);
+        return FileBasedKeyProvider.class.getName().equals(implementationClassName);
+    }
+}

nifi-commons/nifi-security-utils/src/main/java/org/apache/nifi/security/kms/StaticKeyProvider.java

@@ -14,7 +14,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.nifi.provenance;
+package org.apache.nifi.security.kms;
 
 import java.security.KeyManagementException;
 import java.util.ArrayList;
@@ -34,11 +34,11 @@ public class StaticKeyProvider implements KeyProvider {
 
     private Map<String, SecretKey> keys = new HashMap<>();
 
-    StaticKeyProvider(String keyId, String keyHex) throws KeyManagementException {
+    public StaticKeyProvider(String keyId, String keyHex) throws KeyManagementException {
         this.keys.put(keyId, CryptoUtils.formKeyFromHex(keyHex));
     }
 
-    StaticKeyProvider(Map<String, SecretKey> keys) throws KeyManagementException {
+    public StaticKeyProvider(Map<String, SecretKey> keys) throws KeyManagementException {
         this.keys.putAll(keys);
     }
 

nifi-commons/nifi-security-utils/src/main/java/org/apache/nifi/security/util/crypto/CipherUtility.java

@@ -16,6 +16,7 @@
  */
 package org.apache.nifi.security.util.crypto;
 
+import java.io.ByteArrayOutputStream;
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.OutputStream;
@@ -32,7 +33,6 @@ import org.apache.commons.codec.binary.Base64;
 import org.apache.commons.lang3.StringUtils;
 import org.apache.nifi.processor.exception.ProcessException;
 import org.apache.nifi.security.util.EncryptionMethod;
-import org.apache.nifi.stream.io.ByteArrayOutputStream;
 import org.apache.nifi.stream.io.StreamUtils;
 
 public class CipherUtility {
@@ -316,13 +316,4 @@ public class CipherUtility {
             return -1;
         }
     }
-
-    public static byte[] concatBytes(byte[]... arrays) throws IOException {
-        ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
-        for (byte[] bytes : arrays) {
-            outputStream.write(bytes);
-        }
-
-        return outputStream.toByteArray();
-    }
 }

nifi-commons/nifi-security-utils/src/test/groovy/org/apache/nifi/security/kms/CryptoUtilsTest.groovy

@@ -14,7 +14,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.nifi.provenance
+package org.apache.nifi.security.kms
 
 import org.apache.commons.lang3.SystemUtils
 import org.bouncycastle.jce.provider.BouncyCastleProvider
@@ -126,6 +126,20 @@ class CryptoUtilsTest {
         assert keyProviderIsValid
     }
 
+    @Test
+    void testShouldValidateLegacyStaticKeyProvider() {
+        // Arrange
+        String staticProvider = StaticKeyProvider.class.name.replaceFirst("security.kms", "provenance")
+        String providerLocation = null
+
+        // Act
+        boolean keyProviderIsValid = CryptoUtils.isValidKeyProvider(staticProvider, providerLocation, KEY_ID, [(KEY_ID): KEY_HEX])
+        logger.info("Key Provider ${staticProvider} with location ${providerLocation} and keyId ${KEY_ID} / ${KEY_HEX} is ${keyProviderIsValid ? "valid" : "invalid"}")
+
+        // Assert
+        assert keyProviderIsValid
+    }
+
     @Test
     void testShouldNotValidateStaticKeyProviderMissingKeyId() {
         // Arrange
@@ -184,6 +198,22 @@ class CryptoUtilsTest {
         assert keyProviderIsValid
     }
 
+    @Test
+    void testShouldValidateLegacyFileBasedKeyProvider() {
+        // Arrange
+        String fileBasedProvider = FileBasedKeyProvider.class.name.replaceFirst("security.kms", "provenance")
+        File fileBasedProviderFile = tempFolder.newFile("filebased.kp")
+        String providerLocation = fileBasedProviderFile.path
+        logger.info("Created temporary file based key provider: ${providerLocation}")
+
+        // Act
+        boolean keyProviderIsValid = CryptoUtils.isValidKeyProvider(fileBasedProvider, providerLocation, KEY_ID, null)
+        logger.info("Key Provider ${fileBasedProvider} with location ${providerLocation} and keyId ${KEY_ID} / ${null} is ${keyProviderIsValid ? "valid" : "invalid"}")
+
+        // Assert
+        assert keyProviderIsValid
+    }
+
     @Test
     void testShouldNotValidateMissingFileBasedKeyProvider() {
         // Arrange

nifi-commons/nifi-security-utils/src/test/groovy/org/apache/nifi/security/kms/KeyProviderFactoryTest.groovy

@@ -0,0 +1,229 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License") you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.security.kms
+
+import org.apache.nifi.util.NiFiProperties
+import org.bouncycastle.jce.provider.BouncyCastleProvider
+import org.bouncycastle.util.encoders.Hex
+import org.junit.After
+import org.junit.AfterClass
+import org.junit.Before
+import org.junit.BeforeClass
+import org.junit.ClassRule
+import org.junit.Test
+import org.junit.rules.TemporaryFolder
+import org.junit.runner.RunWith
+import org.junit.runners.JUnit4
+import org.slf4j.Logger
+import org.slf4j.LoggerFactory
+
+import javax.crypto.Cipher
+import javax.crypto.SecretKey
+import javax.crypto.spec.IvParameterSpec
+import javax.crypto.spec.SecretKeySpec
+import java.security.KeyManagementException
+import java.security.SecureRandom
+import java.security.Security
+
+import static groovy.test.GroovyAssert.shouldFail
+
+@RunWith(JUnit4.class)
+class KeyProviderFactoryTest {
+    private static final Logger logger = LoggerFactory.getLogger(KeyProviderFactoryTest.class)
+
+    private static final String KEY_ID = "K1"
+    private static final String KEY_HEX_128 = "0123456789ABCDEFFEDCBA9876543210"
+    private static final String KEY_HEX_256 = KEY_HEX_128 * 2
+    private static final String KEY_HEX = isUnlimitedStrengthCryptoAvailable() ? KEY_HEX_256 : KEY_HEX_128
+
+    private static final String LEGACY_SKP_FQCN = "org.apache.nifi.provenance.StaticKeyProvider"
+    private static final String LEGACY_FBKP_FQCN = "org.apache.nifi.provenance.FileBasedKeyProvider"
+
+    private static final String ORIGINAL_PROPERTIES_PATH = System.getProperty(NiFiProperties.PROPERTIES_FILE_PATH)
+
+    private static final SecretKey MASTER_KEY = new SecretKeySpec(Hex.decode(KEY_HEX), "AES")
+
+    @ClassRule
+    public static TemporaryFolder tempFolder = new TemporaryFolder()
+
+    @BeforeClass
+    static void setUpOnce() throws Exception {
+        Security.addProvider(new BouncyCastleProvider())
+
+        logger.metaClass.methodMissing = { String name, args ->
+            logger.info("[${name?.toUpperCase()}] ${(args as List).join(" ")}")
+        }
+
+        logger.info("Original \$PROPERTIES_FILE_PATH is ${ORIGINAL_PROPERTIES_PATH}")
+        String testPath = new File("src/test/resources/${isUnlimitedStrengthCryptoAvailable() ? "256" : "128"}/conf/.").getAbsolutePath()
+        logger.info("Temporarily setting to ${testPath}")
+        System.setProperty(NiFiProperties.PROPERTIES_FILE_PATH, testPath)
+    }
+
+    @Before
+    void setUp() throws Exception {
+        tempFolder.create()
+    }
+
+    @After
+    void tearDown() throws Exception {
+        tempFolder?.delete()
+    }
+
+    @AfterClass
+    static void tearDownOnce() throws Exception {
+        if (ORIGINAL_PROPERTIES_PATH) {
+            logger.info("Restored \$PROPERTIES_FILE_PATH to ${ORIGINAL_PROPERTIES_PATH}")
+            System.setProperty(NiFiProperties.PROPERTIES_FILE_PATH, ORIGINAL_PROPERTIES_PATH)
+        }
+    }
+
+    private static boolean isUnlimitedStrengthCryptoAvailable() {
+        Cipher.getMaxAllowedKeyLength("AES") > 128
+    }
+
+    private static void populateKeyDefinitionsFile(String path = "src/test/resources/conf/filebased.kp") {
+        String masterKeyHex = KEY_HEX
+        SecretKey masterKey = new SecretKeySpec(Hex.decode(masterKeyHex), "AES")
+
+        // Generate the file
+        File keyFile = new File(path)
+        final int KEY_COUNT = 1
+        List<String> lines = []
+        KEY_COUNT.times { int i ->
+            lines.add("K${i + 1}=${generateEncryptedKey(masterKey)}")
+        }
+
+        keyFile.text = lines.join("\n")
+    }
+
+    private static String generateEncryptedKey(SecretKey masterKey) {
+        byte[] ivBytes = new byte[16]
+        byte[] keyBytes = new byte[isUnlimitedStrengthCryptoAvailable() ? 32 : 16]
+
+        SecureRandom sr = new SecureRandom()
+        sr.nextBytes(ivBytes)
+        sr.nextBytes(keyBytes)
+
+        Cipher masterCipher = Cipher.getInstance("AES/GCM/NoPadding", "BC")
+        masterCipher.init(Cipher.ENCRYPT_MODE, masterKey, new IvParameterSpec(ivBytes))
+        byte[] cipherBytes = masterCipher.doFinal(keyBytes)
+
+        Base64.encoder.encodeToString(CryptoUtils.concatByteArrays(ivBytes, cipherBytes))
+    }
+
+    @Test
+    void testShouldBuildStaticKeyProvider() {
+        // Arrange
+        String staticProvider = StaticKeyProvider.class.name
+        String providerLocation = null
+
+        // Act
+        KeyProvider keyProvider = KeyProviderFactory.buildKeyProvider(staticProvider, providerLocation, KEY_ID, [(KEY_ID): KEY_HEX], null)
+        logger.info("Key Provider ${staticProvider} with location ${providerLocation} and keyId ${KEY_ID} / ${KEY_HEX} formed: ${keyProvider}")
+
+        // Assert
+        assert keyProvider instanceof StaticKeyProvider
+        assert keyProvider.getAvailableKeyIds() == [KEY_ID]
+    }
+
+    @Test
+    void testShouldBuildStaticKeyProviderWithLegacyPackage() {
+        // Arrange
+        String staticProvider = LEGACY_SKP_FQCN
+        String providerLocation = null
+
+        // Act
+        KeyProvider keyProvider = KeyProviderFactory.buildKeyProvider(staticProvider, providerLocation, KEY_ID, [(KEY_ID): KEY_HEX], null)
+        logger.info("Key Provider ${staticProvider} with location ${providerLocation} and keyId ${KEY_ID} / ${KEY_HEX} formed: ${keyProvider}")
+
+        // Assert
+        assert keyProvider instanceof StaticKeyProvider
+        assert keyProvider.getAvailableKeyIds() == [KEY_ID]
+    }
+
+    @Test
+    void testShouldBuildFileBasedKeyProvider() {
+        // Arrange
+        String fileBasedProvider = FileBasedKeyProvider.class.name
+        File fileBasedProviderFile = tempFolder.newFile("filebased.kp")
+        String providerLocation = fileBasedProviderFile.path
+        populateKeyDefinitionsFile(providerLocation)
+        logger.info("Created temporary file based key provider: ${providerLocation}")
+
+        // Act
+        KeyProvider keyProvider = KeyProviderFactory.buildKeyProvider(fileBasedProvider, providerLocation, KEY_ID, [(KEY_ID): KEY_HEX], MASTER_KEY)
+        logger.info("Key Provider ${fileBasedProvider} with location ${providerLocation} and keyId ${KEY_ID} / ${KEY_HEX} formed: ${keyProvider}")
+
+        // Assert
+        assert keyProvider instanceof FileBasedKeyProvider
+        assert keyProvider.getAvailableKeyIds() == [KEY_ID]
+    }
+
+    @Test
+    void testShouldBuildFileBasedKeyProviderWithLegacyPackage() {
+        // Arrange
+        String fileBasedProvider = LEGACY_FBKP_FQCN
+        File fileBasedProviderFile = tempFolder.newFile("filebased.kp")
+        String providerLocation = fileBasedProviderFile.path
+        populateKeyDefinitionsFile(providerLocation)
+        logger.info("Created temporary file based key provider: ${providerLocation}")
+
+        // Act
+        KeyProvider keyProvider = KeyProviderFactory.buildKeyProvider(fileBasedProvider, providerLocation, KEY_ID, [(KEY_ID): KEY_HEX], MASTER_KEY)
+        logger.info("Key Provider ${fileBasedProvider} with location ${providerLocation} and keyId ${KEY_ID} / ${KEY_HEX} formed: ${keyProvider}")
+
+        // Assert
+        assert keyProvider instanceof FileBasedKeyProvider
+        assert keyProvider.getAvailableKeyIds() == [KEY_ID]
+    }
+
+    @Test
+    void testShouldNotBuildFileBasedKeyProviderWithoutMasterKey() {
+        // Arrange
+        String fileBasedProvider = FileBasedKeyProvider.class.name
+        File fileBasedProviderFile = tempFolder.newFile("filebased.kp")
+        String providerLocation = fileBasedProviderFile.path
+        populateKeyDefinitionsFile(providerLocation)
+        logger.info("Created temporary file based key provider: ${providerLocation}")
+
+        // Act
+        def msg = shouldFail(KeyManagementException) {
+            KeyProvider keyProvider = KeyProviderFactory.buildKeyProvider(fileBasedProvider, providerLocation, KEY_ID, [(KEY_ID): KEY_HEX], null)
+            logger.info("Key Provider ${fileBasedProvider} with location ${providerLocation} and keyId ${KEY_ID} / ${KEY_HEX} formed: ${keyProvider}")
+        }
+
+        // Assert
+        assert msg =~ "The master key must be provided to decrypt the individual keys"
+    }
+
+    @Test
+    void testShouldNotBuildUnknownKeyProvider() {
+        // Arrange
+        String providerImplementation = "org.apache.nifi.provenance.ImaginaryKeyProvider"
+        String providerLocation = null
+
+        // Act
+        def msg = shouldFail(KeyManagementException) {
+            KeyProvider keyProvider = KeyProviderFactory.buildKeyProvider(providerImplementation, providerLocation, KEY_ID, [(KEY_ID): KEY_HEX], null)
+            logger.info("Key Provider ${providerImplementation} with location ${providerLocation} and keyId ${KEY_ID} / ${KEY_HEX} formed: ${keyProvider}")
+        }
+
+        // Assert
+        assert msg =~ "Invalid key provider implementation provided"
+    }
+}

nifi-commons/nifi-security-utils/src/test/resources/128/conf/bootstrap.conf

@@ -0,0 +1,19 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# Master key in hexadecimal format for encrypted sensitive configuration values
+nifi.bootstrap.sensitive.key=0123456789ABCDEFFEDCBA9876543210

nifi-commons/nifi-security-utils/src/test/resources/256/conf/bootstrap.conf

@@ -0,0 +1,19 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# Master key in hexadecimal format for encrypted sensitive configuration values
+nifi.bootstrap.sensitive.key=0123456789ABCDEFFEDCBA98765432100123456789ABCDEFFEDCBA9876543210

nifi-docs/src/main/asciidoc/administration-guide.adoc

@@ -2819,7 +2819,7 @@ The simplest configuration is below:
 ....
 nifi.provenance.repository.implementation=org.apache.nifi.provenance.EncryptedWriteAheadProvenanceRepository
 nifi.provenance.repository.debug.frequency=100
-nifi.provenance.repository.encryption.key.provider.implementation=org.apache.nifi.provenance.StaticKeyProvider
+nifi.provenance.repository.encryption.key.provider.implementation=org.apache.nifi.security.kms.StaticKeyProvider
 nifi.provenance.repository.encryption.key.provider.location=
 nifi.provenance.repository.encryption.key.id=Key1
 nifi.provenance.repository.encryption.key=0123456789ABCDEFFEDCBA98765432100123456789ABCDEFFEDCBA9876543210

nifi-docs/src/main/asciidoc/user-guide.adoc

@@ -1920,7 +1920,7 @@ The `StaticKeyProvider` implementation defines keys directly in `nifi.properties
 
 The following configuration section would result in a key provider with two available keys, "Key1" (active) and "AnotherKey".
 ....
-nifi.provenance.repository.encryption.key.provider.implementation=org.apache.nifi.provenance.StaticKeyProvider
+nifi.provenance.repository.encryption.key.provider.implementation=org.apache.nifi.security.kms.StaticKeyProvider
 nifi.provenance.repository.encryption.key.id=Key1
 nifi.provenance.repository.encryption.key=0123456789ABCDEFFEDCBA98765432100123456789ABCDEFFEDCBA9876543210
 nifi.provenance.repository.encryption.key.id.AnotherKey=0101010101010101010101010101010101010101010101010101010101010101

nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/EncryptedWriteAheadProvenanceRepository.java

@@ -18,11 +18,13 @@ package org.apache.nifi.provenance;
 
 import java.io.IOException;
 import java.security.KeyManagementException;
-import java.util.Map;
-import java.util.stream.Collectors;
 import javax.crypto.SecretKey;
+import javax.crypto.spec.SecretKeySpec;
+import org.apache.commons.codec.DecoderException;
+import org.apache.commons.codec.binary.Hex;
 import org.apache.nifi.authorization.Authorizer;
 import org.apache.nifi.events.EventReporter;
+import org.apache.nifi.properties.NiFiPropertiesLoader;
 import org.apache.nifi.provenance.serialization.RecordReaders;
 import org.apache.nifi.provenance.store.EventFileManager;
 import org.apache.nifi.provenance.store.RecordReaderFactory;
@@ -30,6 +32,8 @@ import org.apache.nifi.provenance.store.RecordWriterFactory;
 import org.apache.nifi.provenance.toc.StandardTocWriter;
 import org.apache.nifi.provenance.toc.TocUtil;
 import org.apache.nifi.provenance.toc.TocWriter;
+import org.apache.nifi.security.kms.KeyProvider;
+import org.apache.nifi.security.kms.KeyProviderFactory;
 import org.apache.nifi.util.NiFiProperties;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -70,7 +74,13 @@ public class EncryptedWriteAheadProvenanceRepository extends WriteAheadProvenanc
         ProvenanceEventEncryptor provenanceEventEncryptor;
         if (getConfig().supportsEncryption()) {
             try {
-                KeyProvider keyProvider = buildKeyProvider();
+                KeyProvider keyProvider;
+                if (KeyProviderFactory.requiresMasterKey(getConfig().getKeyProviderImplementation())) {
+                    SecretKey masterKey = getMasterKey();
+                    keyProvider = buildKeyProvider(masterKey);
+                } else {
+                    keyProvider = buildKeyProvider();
+                }
                 provenanceEventEncryptor = new AESProvenanceEventEncryptor();
                 provenanceEventEncryptor.initialize(keyProvider);
             } catch (KeyManagementException e) {
@@ -111,6 +121,10 @@ public class EncryptedWriteAheadProvenanceRepository extends WriteAheadProvenanc
     }
 
     private KeyProvider buildKeyProvider() throws KeyManagementException {
+        return buildKeyProvider(null);
+    }
+
+    private KeyProvider buildKeyProvider(SecretKey masterKey) throws KeyManagementException {
         RepositoryConfiguration config = super.getConfig();
         if (config == null) {
             throw new KeyManagementException("The repository configuration is missing");
@@ -122,38 +136,17 @@ public class EncryptedWriteAheadProvenanceRepository extends WriteAheadProvenanc
                     + NiFiProperties.PROVENANCE_REPO_ENCRYPTION_KEY_PROVIDER_IMPLEMENTATION_CLASS);
         }
 
-        // TODO: Extract to factory
+        return KeyProviderFactory.buildKeyProvider(implementationClassName, config.getKeyProviderLocation(), config.getKeyId(), config.getEncryptionKeys(), masterKey);
-        KeyProvider keyProvider;
+    }
-        if (StaticKeyProvider.class.getName().equals(implementationClassName)) {
-            // Get all the keys (map) from config
-            if (CryptoUtils.isValidKeyProvider(implementationClassName, config.getKeyProviderLocation(), config.getKeyId(), config.getEncryptionKeys())) {
-                Map<String, SecretKey> formedKeys = config.getEncryptionKeys().entrySet().stream()
-                        .collect(Collectors.toMap(
-                                Map.Entry::getKey,
-                                e -> {
-                                    try {
-                                        return CryptoUtils.formKeyFromHex(e.getValue());
-                                    } catch (KeyManagementException e1) {
-                                        // This should never happen because the hex has already been validated
-                                        logger.error("Encountered an error: ", e1);
-                                        return null;
-                                    }
-                                }));
-                keyProvider = new StaticKeyProvider(formedKeys);
-            } else {
-                final String msg = "The StaticKeyProvider definition is not valid";
-                logger.error(msg);
-                throw new KeyManagementException(msg);
-            }
-        } else if (FileBasedKeyProvider.class.getName().equals(implementationClassName)) {
-            keyProvider = new FileBasedKeyProvider(config.getKeyProviderLocation());
-            if (!keyProvider.keyExists(config.getKeyId())) {
-                throw new KeyManagementException("The specified key ID " + config.getKeyId() + " is not in the key definition file");
-            }
-        } else {
-            throw new KeyManagementException("Invalid key provider implementation provided: " + implementationClassName);
-        }
 
-        return keyProvider;
+    private static SecretKey getMasterKey() throws KeyManagementException {
+        try {
+            // Get the master encryption key from bootstrap.conf
+            String masterKeyHex = NiFiPropertiesLoader.extractKeyFromBootstrapFile();
+            return new SecretKeySpec(Hex.decodeHex(masterKeyHex.toCharArray()), "AES");
+        } catch (IOException | DecoderException e) {
+            logger.error("Encountered an error: ", e);
+            throw new KeyManagementException(e);
+        }
     }
 }

nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/RepositoryConfiguration.java

@@ -28,6 +28,7 @@ import java.util.Optional;
 import java.util.concurrent.TimeUnit;
 import org.apache.nifi.processor.DataUnit;
 import org.apache.nifi.provenance.search.SearchableField;
+import org.apache.nifi.security.kms.CryptoUtils;
 import org.apache.nifi.util.FormatUtils;
 import org.apache.nifi.util.NiFiProperties;
 import org.slf4j.Logger;
@@ -371,6 +372,8 @@ public class RepositoryConfiguration {
         return keyProviderIsConfigured;
     }
 
+    // TODO: Add verbose error output for encryption support failure if requested
+
     public Map<String, String> getEncryptionKeys() {
         return encryptionKeys;
     }

nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/serialization/RecordReaders.java

@@ -30,7 +30,6 @@ import java.util.zip.GZIPInputStream;
 import org.apache.nifi.properties.NiFiPropertiesLoader;
 import org.apache.nifi.provenance.ByteArraySchemaRecordReader;
 import org.apache.nifi.provenance.ByteArraySchemaRecordWriter;
-import org.apache.nifi.provenance.CryptoUtils;
 import org.apache.nifi.provenance.EncryptedSchemaRecordReader;
 import org.apache.nifi.provenance.EventIdFirstSchemaRecordReader;
 import org.apache.nifi.provenance.EventIdFirstSchemaRecordWriter;
@@ -39,6 +38,7 @@ import org.apache.nifi.provenance.lucene.LuceneUtil;
 import org.apache.nifi.provenance.toc.StandardTocReader;
 import org.apache.nifi.provenance.toc.TocReader;
 import org.apache.nifi.provenance.toc.TocUtil;
+import org.apache.nifi.security.kms.CryptoUtils;
 import org.apache.nifi.util.NiFiProperties;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;

nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/test/groovy/org/apache/nifi/provenance/EncryptedSchemaRecordReaderWriterTest.groovy

@@ -24,6 +24,7 @@ import org.apache.nifi.provenance.toc.StandardTocWriter
 import org.apache.nifi.provenance.toc.TocReader
 import org.apache.nifi.provenance.toc.TocUtil
 import org.apache.nifi.provenance.toc.TocWriter
+import org.apache.nifi.security.kms.KeyProvider
 import org.apache.nifi.util.file.FileUtils
 import org.bouncycastle.jce.provider.BouncyCastleProvider
 import org.bouncycastle.util.encoders.Hex

nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/test/groovy/org/apache/nifi/provenance/EncryptedWriteAheadProvenanceRepositoryTest.groovy

@@ -20,6 +20,7 @@ import org.apache.nifi.events.EventReporter
 import org.apache.nifi.flowfile.FlowFile
 import org.apache.nifi.provenance.serialization.RecordReaders
 import org.apache.nifi.reporting.Severity
+import org.apache.nifi.security.kms.StaticKeyProvider
 import org.apache.nifi.util.file.FileUtils
 import org.bouncycastle.jce.provider.BouncyCastleProvider
 import org.junit.After

nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/groovy/org/apache/nifi/security/util/crypto/PasswordBasedEncryptorGroovyTest.groovy

@@ -18,6 +18,7 @@ package org.apache.nifi.security.util.crypto
 
 import org.apache.commons.codec.binary.Hex
 import org.apache.nifi.processor.io.StreamCallback
+import org.apache.nifi.security.kms.CryptoUtils
 import org.apache.nifi.security.util.EncryptionMethod
 import org.apache.nifi.security.util.KeyDerivationFunction
 import org.apache.nifi.stream.io.ByteArrayOutputStream
@@ -33,7 +34,7 @@ import org.slf4j.LoggerFactory
 import javax.crypto.Cipher
 import java.security.Security
 
-public class PasswordBasedEncryptorGroovyTest {
+class PasswordBasedEncryptorGroovyTest {
     private static final Logger logger = LoggerFactory.getLogger(PasswordBasedEncryptorGroovyTest.class)
 
     private static final String TEST_RESOURCES_PREFIX = "src/test/resources/TestEncryptContent/"
@@ -44,7 +45,7 @@ public class PasswordBasedEncryptorGroovyTest {
     private static final String LEGACY_PASSWORD = "Hello, World!"
 
     @BeforeClass
-    public static void setUpOnce() throws Exception {
+    static void setUpOnce() throws Exception {
         Security.addProvider(new BouncyCastleProvider())
 
         logger.metaClass.methodMissing = { String name, args ->
@@ -53,15 +54,15 @@ public class PasswordBasedEncryptorGroovyTest {
     }
 
     @Before
-    public void setUp() throws Exception {
+    void setUp() throws Exception {
     }
 
     @After
-    public void tearDown() throws Exception {
+    void tearDown() throws Exception {
     }
 
     @Test
-    public void testShouldEncryptAndDecrypt() throws Exception {
+    void testShouldEncryptAndDecrypt() throws Exception {
         // Arrange
         final String PLAINTEXT = "This is a plaintext message."
         logger.info("Plaintext: {}", PLAINTEXT)
@@ -107,7 +108,7 @@ public class PasswordBasedEncryptorGroovyTest {
     }
 
     @Test
-    public void testShouldDecryptLegacyOpenSSLSaltedCipherText() throws Exception {
+    void testShouldDecryptLegacyOpenSSLSaltedCipherText() throws Exception {
         // Arrange
         Assume.assumeTrue("Skipping test because unlimited strength crypto policy not installed", PasswordBasedEncryptor.supportsUnlimitedStrength())
 
@@ -136,7 +137,7 @@ public class PasswordBasedEncryptorGroovyTest {
     }
 
     @Test
-    public void testShouldDecryptLegacyOpenSSLUnsaltedCipherText() throws Exception {
+    void testShouldDecryptLegacyOpenSSLUnsaltedCipherText() throws Exception {
         // Arrange
         Assume.assumeTrue("Skipping test because unlimited strength crypto policy not installed", PasswordBasedEncryptor.supportsUnlimitedStrength())
 
@@ -165,7 +166,7 @@ public class PasswordBasedEncryptorGroovyTest {
     }
 
     @Test
-    public void testShouldDecryptNiFiLegacySaltedCipherTextWithVariableSaltLength() throws Exception {
+    void testShouldDecryptNiFiLegacySaltedCipherTextWithVariableSaltLength() throws Exception {
         // Arrange
         final String PLAINTEXT = new File("${TEST_RESOURCES_PREFIX}/plain.txt").text
         logger.info("Plaintext: {}", PLAINTEXT)
@@ -201,7 +202,7 @@ public class PasswordBasedEncryptorGroovyTest {
                 byte[] cipherBytes = legacyCipher.doFinal(PLAINTEXT.bytes)
                 logger.info("Cipher bytes: ${Hex.encodeHexString(cipherBytes)}")
 
-                byte[] completeCipherStreamBytes = CipherUtility.concatBytes(legacySalt, cipherBytes)
+                byte[] completeCipherStreamBytes = CryptoUtils.concatByteArrays(legacySalt, cipherBytes)
                 logger.info("Complete cipher stream: ${Hex.encodeHexString(completeCipherStreamBytes)}")
 
                 InputStream cipherStream = new ByteArrayInputStream(completeCipherStreamBytes)