NIFI-3594 Implemented encrypted provenance repository.

Added src/test/resources/logback-test.xml files resetting log level from DEBUG (in nifi-data-provenance-utils) to WARN because later tests depend on MockComponentLog recording a certain number of messages and this number is different than expected if the log level is DEBUG.

This closes #1686.

Signed-off-by: Bryan Bende, Yolanda M. Davis, and Mark Payne

nifi-api/src/main/java/org/apache/nifi/provenance/ProvenanceEventRecord.java

@@ -188,28 +188,24 @@ public interface ProvenanceEventRecord {
      * {@link ProvenanceEventType#FORK}, {@link ProvenanceEventType#JOIN}, or
      * {@link ProvenanceEventType#CLONE}), or if the queue identifier is
      * unknown, then this method will return <code>null</code>
-     *
      */
     String getSourceQueueIdentifier();
 
     /**
      * @return the Section for the Content Claim that this Event refers to, if
      * any; otherwise, returns <code>null</code>
-     *
      */
     String getContentClaimSection();
 
     /**
      * @return the Section for the Content Claim that the FlowFile previously
      * referenced, if any; otherwise, returns <code>null</code>
-     *
      */
     String getPreviousContentClaimSection();
 
     /**
      * @return the Container for the Content Claim that this Event refers to, if
      * any; otherwise, returns <code>null</code>
-     *
      */
     String getContentClaimContainer();
 
@@ -222,28 +218,31 @@ public interface ProvenanceEventRecord {
     /**
      * @return the Identifier for the Content Claim that this Event refers to,
      * if any; otherwise, returns <code>null</code>
-     *
      */
     String getContentClaimIdentifier();
 
     /**
      * @return the Identifier for the Content Claim that the FlowFile previously
      * referenced, if any; otherwise, returns <code>null</code>
-     *
      */
     String getPreviousContentClaimIdentifier();
 
     /**
      * @return the offset into the Content Claim at which the FlowFile's content
      * begins, if any; otherwise, returns <code>null</code>
-     *
      */
     Long getContentClaimOffset();
 
     /**
      * @return the offset into the Content Claim at which the FlowFile's
      * previous content began, if any; otherwise, returns <code>null</code>
-     *
      */
     Long getPreviousContentClaimOffset();
+
+    /**
+     * Returns the best event identifier for this event (eventId if available, descriptive identifier if not yet persisted to allow for traceability).
+     *
+     * @return a descriptive event ID to allow tracing
+     */
+    String getBestEventIdentifier();
 }

nifi-commons/nifi-data-provenance-utils/pom.xml

@@ -34,5 +34,21 @@
             <groupId>org.apache.nifi</groupId>
             <artifactId>nifi-utils</artifactId>
         </dependency>
+        <dependency>
+            <groupId>org.apache.nifi</groupId>
+            <artifactId>nifi-security-utils</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>org.bouncycastle</groupId>
+            <artifactId>bcprov-jdk15on</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.nifi</groupId>
+            <artifactId>nifi-properties-loader</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.nifi</groupId>
+            <artifactId>nifi-properties</artifactId>
+        </dependency>
     </dependencies>
 </project>

nifi-commons/nifi-data-provenance-utils/src/main/java/org/apache/nifi/provenance/AESProvenanceEventEncryptor.java

@@ -0,0 +1,237 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.provenance;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.ObjectInputStream;
+import java.io.ObjectOutputStream;
+import java.security.KeyManagementException;
+import java.security.SecureRandom;
+import java.security.Security;
+import java.util.Arrays;
+import java.util.List;
+import javax.crypto.BadPaddingException;
+import javax.crypto.Cipher;
+import javax.crypto.IllegalBlockSizeException;
+import javax.crypto.SecretKey;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.nifi.security.util.EncryptionMethod;
+import org.apache.nifi.security.util.crypto.AESKeyedCipherProvider;
+import org.bouncycastle.jce.provider.BouncyCastleProvider;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class AESProvenanceEventEncryptor implements ProvenanceEventEncryptor {
+    private static final Logger logger = LoggerFactory.getLogger(AESProvenanceEventEncryptor.class);
+    private static final String ALGORITHM = "AES/GCM/NoPadding";
+    private static final int IV_LENGTH = 16;
+    private static final byte[] EMPTY_IV = new byte[IV_LENGTH];
+    private static final String VERSION = "v1";
+    private static final List<String> SUPPORTED_VERSIONS = Arrays.asList(VERSION);
+    private static final int MIN_METADATA_LENGTH = IV_LENGTH + 3 + 3; // 3 delimiters and 3 non-zero elements
+    private static final int METADATA_DEFAULT_LENGTH = (20 + ALGORITHM.length() + IV_LENGTH + VERSION.length()) * 2; // Default to twice the expected length
+    private static final byte[] SENTINEL = new byte[]{0x01};
+
+    private KeyProvider keyProvider;
+
+    private AESKeyedCipherProvider aesKeyedCipherProvider = new AESKeyedCipherProvider();
+
+    /**
+     * Initializes the encryptor with a {@link KeyProvider}.
+     *
+     * @param keyProvider the key provider which will be responsible for accessing keys
+     * @throws KeyManagementException if there is an issue configuring the key provider
+     */
+    @Override
+    public void initialize(KeyProvider keyProvider) throws KeyManagementException {
+        this.keyProvider = keyProvider;
+
+        if (this.aesKeyedCipherProvider == null) {
+            this.aesKeyedCipherProvider = new AESKeyedCipherProvider();
+        }
+
+        if (Security.getProvider("BC") == null) {
+            Security.addProvider(new BouncyCastleProvider());
+        }
+    }
+
+    /**
+     * Available for dependency injection to override the default {@link AESKeyedCipherProvider} if necessary.
+     *
+     * @param cipherProvider the AES cipher provider to use
+     */
+    void setCipherProvider(AESKeyedCipherProvider cipherProvider) {
+        this.aesKeyedCipherProvider = cipherProvider;
+    }
+
+    /**
+     * Encrypts the provided {@link ProvenanceEventRecord}, serialized to a byte[] by the RecordWriter.
+     *
+     * @param plainRecord the plain record, serialized to a byte[]
+     * @param recordId    an identifier for this record (eventId, generated, etc.)
+     * @param keyId       the ID of the key to use
+     * @return the encrypted record
+     * @throws EncryptionException if there is an issue encrypting this record
+     */
+    @Override
+    public byte[] encrypt(byte[] plainRecord, String recordId, String keyId) throws EncryptionException {
+        if (plainRecord == null || CryptoUtils.isEmpty(keyId)) {
+            throw new EncryptionException("The provenance record and key ID cannot be missing");
+        }
+
+        if (keyProvider == null || !keyProvider.keyExists(keyId)) {
+            throw new EncryptionException("The requested key ID is not available");
+        } else {
+            byte[] ivBytes = new byte[IV_LENGTH];
+            new SecureRandom().nextBytes(ivBytes);
+            try {
+                logger.debug("Encrypting provenance record " + recordId + " with key ID " + keyId);
+                Cipher cipher = initCipher(EncryptionMethod.AES_GCM, Cipher.ENCRYPT_MODE, keyProvider.getKey(keyId), ivBytes);
+                ivBytes = cipher.getIV();
+
+                // Perform the actual encryption
+                byte[] cipherBytes = cipher.doFinal(plainRecord);
+
+                // Serialize and concat encryption details fields (keyId, algo, IV, version, CB length) outside of encryption
+                EncryptionMetadata metadata = new EncryptionMetadata(keyId, ALGORITHM, ivBytes, VERSION, cipherBytes.length);
+                byte[] serializedEncryptionMetadata = serializeEncryptionMetadata(metadata);
+
+                // Add the sentinel byte of 0x01
+                logger.debug("Encrypted provenance event record " + recordId + " with key ID " + keyId);
+                return CryptoUtils.concatByteArrays(SENTINEL, serializedEncryptionMetadata, cipherBytes);
+            } catch (EncryptionException | BadPaddingException | IllegalBlockSizeException | IOException | KeyManagementException e) {
+                final String msg = "Encountered an exception encrypting provenance record " + recordId;
+                logger.error(msg, e);
+                throw new EncryptionException(msg, e);
+            }
+        }
+    }
+
+    private byte[] serializeEncryptionMetadata(EncryptionMetadata metadata) throws IOException {
+        final ByteArrayOutputStream baos = new ByteArrayOutputStream();
+        ObjectOutputStream outputStream = new ObjectOutputStream(baos);
+        outputStream.writeObject(metadata);
+        outputStream.close();
+        return baos.toByteArray();
+    }
+
+    private Cipher initCipher(EncryptionMethod method, int mode, SecretKey key, byte[] ivBytes) throws EncryptionException {
+        try {
+            if (method == null || key == null || ivBytes == null) {
+                throw new IllegalArgumentException("Missing critical information");
+            }
+            return aesKeyedCipherProvider.getCipher(method, key, ivBytes, mode == Cipher.ENCRYPT_MODE);
+        } catch (Exception e) {
+            logger.error("Encountered an exception initializing the cipher", e);
+            throw new EncryptionException(e);
+        }
+    }
+
+    /**
+     * Decrypts the provided byte[] (an encrypted record with accompanying metadata).
+     *
+     * @param encryptedRecord the encrypted record in byte[] form
+     * @param recordId        an identifier for this record (eventId, generated, etc.)
+     * @return the decrypted record
+     * @throws EncryptionException if there is an issue decrypting this record
+     */
+    @Override
+    public byte[] decrypt(byte[] encryptedRecord, String recordId) throws EncryptionException {
+        if (encryptedRecord == null) {
+            throw new EncryptionException("The encrypted provenance record cannot be missing");
+        }
+
+        EncryptionMetadata metadata;
+        try {
+            metadata = extractEncryptionMetadata(encryptedRecord);
+        } catch (IOException | ClassNotFoundException e) {
+            final String msg = "Encountered an error reading the encryption metadata: ";
+            logger.error(msg, e);
+            throw new EncryptionException(msg, e);
+        }
+
+        if (!SUPPORTED_VERSIONS.contains(metadata.version)) {
+            throw new EncryptionException("The event was encrypted with version " + metadata.version + " which is not in the list of supported versions " + StringUtils.join(SUPPORTED_VERSIONS, ","));
+        }
+
+        // TODO: Actually use the version to determine schema, etc.
+
+        if (keyProvider == null || !keyProvider.keyExists(metadata.keyId) || CryptoUtils.isEmpty(metadata.keyId)) {
+            throw new EncryptionException("The requested key ID " + metadata.keyId + " is not available");
+        } else {
+            try {
+                logger.debug("Decrypting provenance record " + recordId + " with key ID " + metadata.keyId);
+                EncryptionMethod method = EncryptionMethod.forAlgorithm(metadata.algorithm);
+                Cipher cipher = initCipher(method, Cipher.DECRYPT_MODE, keyProvider.getKey(metadata.keyId), metadata.ivBytes);
+
+                // Strip the metadata away to get just the cipher bytes
+                byte[] cipherBytes = extractCipherBytes(encryptedRecord, metadata);
+
+                // Perform the actual decryption
+                byte[] plainBytes = cipher.doFinal(cipherBytes);
+
+                logger.debug("Decrypted provenance event record " + recordId + " with key ID " + metadata.keyId);
+                return plainBytes;
+            } catch (EncryptionException | BadPaddingException | IllegalBlockSizeException | KeyManagementException e) {
+                final String msg = "Encountered an exception decrypting provenance record " + recordId;
+                logger.error(msg, e);
+                throw new EncryptionException(msg, e);
+            }
+        }
+    }
+
+    /**
+     * Returns a valid key identifier for this encryptor (valid for encryption and decryption) or throws an exception if none are available.
+     *
+     * @return the key ID
+     * @throws KeyManagementException if no available key IDs are valid for both operations
+     */
+    @Override
+    public String getNextKeyId() throws KeyManagementException {
+        if (keyProvider != null) {
+            List<String> availableKeyIds = keyProvider.getAvailableKeyIds();
+            if (!availableKeyIds.isEmpty()) {
+                return availableKeyIds.get(0);
+            }
+        }
+        throw new KeyManagementException("No available key IDs");
+    }
+
+    private EncryptionMetadata extractEncryptionMetadata(byte[] encryptedRecord) throws EncryptionException, IOException, ClassNotFoundException {
+        if (encryptedRecord == null || encryptedRecord.length < MIN_METADATA_LENGTH) {
+            throw new EncryptionException("The encrypted record is too short to contain the metadata");
+        }
+
+        // Skip the first byte (SENTINEL) and don't need to copy all the serialized record
+        ByteArrayInputStream bais = new ByteArrayInputStream(encryptedRecord);
+        bais.read();
+        try (ObjectInputStream ois = new ObjectInputStream(bais)) {
+            return (EncryptionMetadata) ois.readObject();
+        }
+    }
+
+    private byte[] extractCipherBytes(byte[] encryptedRecord, EncryptionMetadata metadata) {
+        return Arrays.copyOfRange(encryptedRecord, encryptedRecord.length - metadata.cipherByteLength, encryptedRecord.length);
+    }
+
+    @Override
+    public String toString() {
+        return "AES Provenance Event Encryptor with Key Provider: " + keyProvider.toString();
+    }
+}

nifi-commons/nifi-data-provenance-utils/src/main/java/org/apache/nifi/provenance/CryptoUtils.java

@@ -0,0 +1,248 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.provenance;
+
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.FileReader;
+import java.io.IOException;
+import java.security.KeyManagementException;
+import java.security.NoSuchAlgorithmException;
+import java.util.Arrays;
+import java.util.Base64;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.regex.Pattern;
+import javax.crypto.BadPaddingException;
+import javax.crypto.Cipher;
+import javax.crypto.IllegalBlockSizeException;
+import javax.crypto.SecretKey;
+import javax.crypto.spec.SecretKeySpec;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.nifi.security.util.EncryptionMethod;
+import org.apache.nifi.security.util.crypto.AESKeyedCipherProvider;
+import org.apache.nifi.util.NiFiProperties;
+import org.bouncycastle.util.encoders.Hex;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class CryptoUtils {
+    private static final Logger logger = LoggerFactory.getLogger(StaticKeyProvider.class);
+    private static final String STATIC_KEY_PROVIDER_CLASS_NAME = "org.apache.nifi.provenance.StaticKeyProvider";
+    private static final String FILE_BASED_KEY_PROVIDER_CLASS_NAME = "org.apache.nifi.provenance.FileBasedKeyProvider";
+    private static final Pattern HEX_PATTERN = Pattern.compile("(?i)^[0-9a-f]+$");
+
+    private static final List<Integer> UNLIMITED_KEY_LENGTHS = Arrays.asList(32, 48, 64);
+
+    public static final int IV_LENGTH = 16;
+
+    public static boolean isUnlimitedStrengthCryptoAvailable() {
+        try {
+            return Cipher.getMaxAllowedKeyLength("AES") > 128;
+        } catch (NoSuchAlgorithmException e) {
+            logger.warn("Tried to determine if unlimited strength crypto is available but the AES algorithm is not available");
+            return false;
+        }
+    }
+
+    /**
+     * Utility method which returns true if the string is null, empty, or entirely whitespace.
+     *
+     * @param src the string to evaluate
+     * @return true if empty
+     */
+    public static boolean isEmpty(String src) {
+        return src == null || src.trim().isEmpty();
+    }
+
+    /**
+     * Concatenates multiple byte[] into a single byte[].
+     *
+     * @param arrays the component byte[] in order
+     * @return a concatenated byte[]
+     * @throws IOException this should never be thrown
+     */
+    public static byte[] concatByteArrays(byte[]... arrays) throws IOException {
+        int totalByteLength = 0;
+        for (byte[] bytes : arrays) {
+            totalByteLength += bytes.length;
+        }
+        byte[] totalBytes = new byte[totalByteLength];
+        int currentLength = 0;
+        for (byte[] bytes : arrays) {
+            System.arraycopy(bytes, 0, totalBytes, currentLength, bytes.length);
+            currentLength += bytes.length;
+        }
+        return totalBytes;
+    }
+
+    /**
+     * Returns true if the provided configuration values successfully define the specified {@link KeyProvider}.
+     *
+     * @param keyProviderImplementation the FQ class name of the {@link KeyProvider} implementation
+     * @param keyProviderLocation       the location of the definition (for {@link FileBasedKeyProvider}, etc.)
+     * @param keyId                     the active key ID
+     * @param encryptionKeys            a map of key IDs to key material in hex format
+     * @return true if the provided configuration is valid
+     */
+    public static boolean isValidKeyProvider(String keyProviderImplementation, String keyProviderLocation, String keyId, Map<String, String> encryptionKeys) {
+        if (STATIC_KEY_PROVIDER_CLASS_NAME.equals(keyProviderImplementation)) {
+            // Ensure the keyId and key(s) are valid
+            if (encryptionKeys == null) {
+                return false;
+            } else {
+                boolean everyKeyValid = encryptionKeys.values().stream().allMatch(CryptoUtils::keyIsValid);
+                return everyKeyValid && StringUtils.isNotEmpty(keyId);
+            }
+        } else if (FILE_BASED_KEY_PROVIDER_CLASS_NAME.equals(keyProviderImplementation)) {
+            // Ensure the file can be read and the keyId is populated (does not read file to validate)
+            final File kpf = new File(keyProviderLocation);
+            return kpf.exists() && kpf.canRead() && StringUtils.isNotEmpty(keyId);
+        } else {
+            logger.error("The attempt to validate the key provider failed keyProviderImplementation = "
+                    + keyProviderImplementation + " , keyProviderLocation = "
+                    + keyProviderLocation + " , keyId = "
+                    + keyId + " , encryptionKeys = "
+                    + ((encryptionKeys == null) ? "0" : encryptionKeys.size()));
+
+            return false;
+        }
+    }
+
+    /**
+     * Returns true if the provided key is valid hex and is the correct length for the current system's JCE policies.
+     *
+     * @param encryptionKeyHex the key in hexadecimal
+     * @return true if this key is valid
+     */
+    public static boolean keyIsValid(String encryptionKeyHex) {
+        return isHexString(encryptionKeyHex)
+                && (isUnlimitedStrengthCryptoAvailable()
+                ? UNLIMITED_KEY_LENGTHS.contains(encryptionKeyHex.length())
+                : encryptionKeyHex.length() == 32);
+    }
+
+    /**
+     * Returns true if the input is valid hexadecimal (does not enforce length and is case-insensitive).
+     *
+     * @param hexString the string to evaluate
+     * @return true if the string is valid hex
+     */
+    public static boolean isHexString(String hexString) {
+        return StringUtils.isNotEmpty(hexString) && HEX_PATTERN.matcher(hexString).matches();
+    }
+
+    /**
+     * Returns a {@link SecretKey} formed from the hexadecimal key bytes (validity is checked).
+     *
+     * @param keyHex the key in hex form
+     * @return the SecretKey
+     */
+    public static SecretKey formKeyFromHex(String keyHex) throws KeyManagementException {
+        if (keyIsValid(keyHex)) {
+            return new SecretKeySpec(Hex.decode(keyHex), "AES");
+        } else {
+            throw new KeyManagementException("The provided key material is not valid");
+        }
+    }
+
+    /**
+     * Returns a map containing the key IDs and the parsed key from a key provider definition file.
+     * The values in the file are decrypted using the master key provided. If the file is missing or empty,
+     * cannot be read, or if no valid keys are read, a {@link KeyManagementException} will be thrown.
+     *
+     * @param filepath  the key definition file path
+     * @param masterKey the master key used to decrypt each key definition
+     * @return a Map of key IDs to SecretKeys
+     * @throws KeyManagementException if the file is missing or invalid
+     */
+    public static Map<String, SecretKey> readKeys(String filepath, SecretKey masterKey) throws KeyManagementException {
+        Map<String, SecretKey> keys = new HashMap<>();
+
+        if (StringUtils.isBlank(filepath)) {
+            throw new KeyManagementException("The key provider file is not present and readable");
+        }
+        File file = new File(filepath);
+        if (!file.exists() || !file.canRead()) {
+            throw new KeyManagementException("The key provider file is not present and readable");
+        }
+
+        try (BufferedReader br = new BufferedReader(new FileReader(file))) {
+            AESKeyedCipherProvider masterCipherProvider = new AESKeyedCipherProvider();
+
+            String line;
+            int l = 1;
+            while ((line = br.readLine()) != null) {
+                String[] components = line.split("=", 2);
+                if (components.length != 2 || StringUtils.isAnyEmpty(components)) {
+                    logger.warn("Line " + l + " is not properly formatted -- keyId=Base64EncodedKey...");
+                }
+                String keyId = components[0];
+                if (StringUtils.isNotEmpty(keyId)) {
+                    try {
+                        byte[] base64Bytes = Base64.getDecoder().decode(components[1]);
+                        byte[] ivBytes = Arrays.copyOfRange(base64Bytes, 0, IV_LENGTH);
+
+                        Cipher masterCipher = null;
+                        try {
+                            masterCipher = masterCipherProvider.getCipher(EncryptionMethod.AES_GCM, masterKey, ivBytes, false);
+                        } catch (Exception e) {
+                            throw new KeyManagementException("Error building cipher to decrypt FileBaseKeyProvider definition at " + filepath, e);
+                        }
+                        byte[] individualKeyBytes = masterCipher.doFinal(Arrays.copyOfRange(base64Bytes, IV_LENGTH, base64Bytes.length));
+
+                        SecretKey key = new SecretKeySpec(individualKeyBytes, "AES");
+                        logger.debug("Read and decrypted key for " + keyId);
+                        if (keys.containsKey(keyId)) {
+                            logger.warn("Multiple key values defined for " + keyId + " -- using most recent value");
+                        }
+                        keys.put(keyId, key);
+                    } catch (IllegalArgumentException e) {
+                        logger.error("Encountered an error decoding Base64 for " + keyId + ": " + e.getLocalizedMessage());
+                    } catch (BadPaddingException | IllegalBlockSizeException e) {
+                        logger.error("Encountered an error decrypting key for " + keyId + ": " + e.getLocalizedMessage());
+                    }
+                }
+                l++;
+            }
+
+            if (keys.isEmpty()) {
+                throw new KeyManagementException("The provided file contained no valid keys");
+            }
+
+            logger.info("Read " + keys.size() + " keys from FileBasedKeyProvider " + filepath);
+            return keys;
+        } catch (IOException e) {
+            throw new KeyManagementException("Error reading FileBasedKeyProvider definition at " + filepath, e);
+        }
+
+    }
+
+    public static boolean isProvenanceRepositoryEncryptionConfigured(NiFiProperties niFiProperties) {
+        final String implementationClassName = niFiProperties.getProperty(NiFiProperties.PROVENANCE_REPO_IMPLEMENTATION_CLASS);
+        // Referencing EWAPR.class.getName() would require a dependency on the module
+        boolean encryptedRepo = "org.apache.nifi.provenance.EncryptedWriteAheadProvenanceRepository".equals(implementationClassName);
+        boolean keyProviderConfigured = isValidKeyProvider(
+                niFiProperties.getProperty(NiFiProperties.PROVENANCE_REPO_ENCRYPTION_KEY_PROVIDER_IMPLEMENTATION_CLASS),
+                niFiProperties.getProperty(NiFiProperties.PROVENANCE_REPO_ENCRYPTION_KEY_PROVIDER_LOCATION),
+                niFiProperties.getProvenanceRepoEncryptionKeyId(),
+                niFiProperties.getProvenanceRepoEncryptionKeys());
+
+        return encryptedRepo && keyProviderConfigured;
+    }
+}

nifi-commons/nifi-data-provenance-utils/src/main/java/org/apache/nifi/provenance/EncryptionException.java

@@ -0,0 +1,92 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.provenance;
+
+public class EncryptionException extends Throwable {
+    /**
+     * Constructs a new EncryptionException with {@code null} as its detail message.
+     * The cause is not initialized, and may subsequently be initialized by a
+     * call to {@link #initCause}.
+     * <p>
+     * <p>The {@link #fillInStackTrace()} method is called to initialize
+     * the stack trace data in the newly created exception.
+     */
+    public EncryptionException() {
+        super();
+    }
+
+    /**
+     * Constructs a new EncryptionException with the specified detail message.  The
+     * cause is not initialized, and may subsequently be initialized by
+     * a call to {@link #initCause}.
+     * <p>
+     * <p>The {@link #fillInStackTrace()} method is called to initialize
+     * the stack trace data in the newly created exception.
+     *
+     * @param message the detail message. The detail message is saved for
+     *                later retrieval by the {@link #getMessage()} method.
+     */
+    public EncryptionException(String message) {
+        super(message);
+    }
+
+    /**
+     * Constructs a new EncryptionException with the specified detail message and
+     * cause.  <p>Note that the detail message associated with
+     * {@code cause} is <i>not</i> automatically incorporated in
+     * this exception's detail message.
+     * <p>
+     * <p>The {@link #fillInStackTrace()} method is called to initialize
+     * the stack trace data in the newly created throwable.
+     *
+     * @param message the detail message (which is saved for later retrieval
+     *                by the {@link #getMessage()} method).
+     * @param cause   the cause (which is saved for later retrieval by the
+     *                {@link #getCause()} method).  (A {@code null} value is
+     *                permitted, and indicates that the cause is nonexistent or
+     *                unknown.)
+     * @since 1.4
+     */
+    public EncryptionException(String message, Throwable cause) {
+        super(message, cause);
+    }
+
+    /**
+     * Constructs a new EncryptionException with the specified cause and a detail
+     * message of {@code (cause==null ? null : cause.toString())} (which
+     * typically contains the class and detail message of {@code cause}).
+     * This constructor is useful for exceptions that are little more than
+     * wrappers for other exceptions.
+     * <p>
+     * <p>The {@link #fillInStackTrace()} method is called to initialize
+     * the stack trace data in the newly created exception.
+     *
+     * @param cause the cause (which is saved for later retrieval by the
+     *              {@link #getCause()} method).  (A {@code null} value is
+     *              permitted, and indicates that the cause is nonexistent or
+     *              unknown.)
+     * @since 1.4
+     */
+    public EncryptionException(Throwable cause) {
+        super(cause);
+    }
+
+    @Override
+    public String toString() {
+        return "EncryptionException " + getLocalizedMessage();
+    }
+}

nifi-commons/nifi-data-provenance-utils/src/main/java/org/apache/nifi/provenance/EncryptionMetadata.java

@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.provenance;
+
+import java.io.Serializable;
+import org.apache.commons.codec.binary.Hex;
+
+public class EncryptionMetadata implements Serializable {
+    protected String keyId;
+    protected String algorithm;
+    protected byte[] ivBytes;
+    protected String version;
+    protected int cipherByteLength;
+
+    EncryptionMetadata() {
+    }
+
+    EncryptionMetadata(String keyId, String algorithm, byte[] ivBytes, String version, int cipherByteLength) {
+        this.keyId = keyId;
+        this.ivBytes = ivBytes;
+        this.algorithm = algorithm;
+        this.version = version;
+        this.cipherByteLength = cipherByteLength;
+    }
+
+    @Override
+    public String toString() {
+        String sb = "AES Provenance Record Encryption Metadata" +
+                " Key ID: " +
+                keyId +
+                " Algorithm: " +
+                algorithm +
+                " IV: " +
+                Hex.encodeHexString(ivBytes) +
+                " Version: " +
+                version +
+                " Cipher text length: " +
+                cipherByteLength;
+        return sb;
+    }
+}

nifi-commons/nifi-data-provenance-utils/src/main/java/org/apache/nifi/provenance/FileBasedKeyProvider.java

@@ -0,0 +1,67 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.provenance;
+
+import java.io.IOException;
+import java.security.KeyManagementException;
+import javax.crypto.SecretKey;
+import javax.crypto.spec.SecretKeySpec;
+import javax.naming.OperationNotSupportedException;
+import org.apache.nifi.properties.NiFiPropertiesLoader;
+import org.bouncycastle.util.encoders.Hex;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class FileBasedKeyProvider extends StaticKeyProvider {
+    private static final Logger logger = LoggerFactory.getLogger(FileBasedKeyProvider.class);
+
+    private String filepath;
+
+    FileBasedKeyProvider(String location) throws KeyManagementException {
+        this(location, getMasterKey());
+    }
+
+    FileBasedKeyProvider(String location, SecretKey masterKey) throws KeyManagementException {
+        super(CryptoUtils.readKeys(location, masterKey));
+        this.filepath = location;
+    }
+
+    private static SecretKey getMasterKey() throws KeyManagementException {
+        try {
+            // Get the master encryption key from bootstrap.conf
+            String masterKeyHex = NiFiPropertiesLoader.extractKeyFromBootstrapFile();
+            return new SecretKeySpec(Hex.decode(masterKeyHex), "AES");
+        } catch (IOException e) {
+            logger.error("Encountered an error: ", e);
+            throw new KeyManagementException(e);
+        }
+    }
+
+    /**
+     * Adds the key to the provider and associates it with the given ID. Some implementations may not allow this operation.
+     *
+     * @param keyId the key identifier
+     * @param key   the key
+     * @return true if the key was successfully added
+     * @throws OperationNotSupportedException if this implementation doesn't support adding keys
+     * @throws KeyManagementException         if the key is invalid, the ID conflicts, etc.
+     */
+    @Override
+    public boolean addKey(String keyId, SecretKey key) throws OperationNotSupportedException, KeyManagementException {
+        throw new OperationNotSupportedException("This implementation does not allow adding keys. Modify the file backing this provider at " + filepath);
+    }
+}

nifi-commons/nifi-data-provenance-utils/src/main/java/org/apache/nifi/provenance/KeyProvider.java

@@ -0,0 +1,60 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.provenance;
+
+import java.security.KeyManagementException;
+import java.util.List;
+import javax.crypto.SecretKey;
+import javax.naming.OperationNotSupportedException;
+
+public interface KeyProvider {
+
+    /**
+     * Returns the key identified by this ID or throws an exception if one is not available.
+     *
+     * @param keyId the key identifier
+     * @return the key
+     * @throws KeyManagementException if the key cannot be retrieved
+     */
+     SecretKey getKey(String keyId) throws KeyManagementException;
+
+    /**
+     * Returns true if the key exists and is available. Null or empty IDs will return false.
+     *
+     * @param keyId the key identifier
+     * @return true if the key can be used
+     */
+     boolean keyExists(String keyId);
+
+    /**
+     * Returns a list of available key identifiers (useful for encryption, as retired keys may not be listed here even if they are available for decryption for legacy/BC reasons).
+     *
+     * @return a List of keyIds (empty list if none are available)
+     */
+     List<String> getAvailableKeyIds();
+
+    /**
+     * Adds the key to the provider and associates it with the given ID. Some implementations may not allow this operation.
+     *
+     * @param keyId the key identifier
+     * @param key the key
+     * @return true if the key was successfully added
+     * @throws OperationNotSupportedException if this implementation doesn't support adding keys
+     * @throws KeyManagementException if the key is invalid, the ID conflicts, etc.
+     */
+     boolean addKey(String keyId, SecretKey key) throws OperationNotSupportedException, KeyManagementException;
+}

nifi-commons/nifi-data-provenance-utils/src/main/java/org/apache/nifi/provenance/PlaceholderProvenanceEvent.java

@@ -187,4 +187,14 @@ public class PlaceholderProvenanceEvent implements ProvenanceEventRecord {
     public Long getPreviousContentClaimOffset() {
         return null;
     }
+
+    /**
+     * Returns the best event identifier for this event (eventId if available, descriptive identifier if not yet persisted to allow for traceability).
+     *
+     * @return a descriptive event ID to allow tracing
+     */
+    @Override
+    public String getBestEventIdentifier() {
+        return Long.toString(getEventId());
+    }
 }

nifi-commons/nifi-data-provenance-utils/src/main/java/org/apache/nifi/provenance/ProvenanceEventEncryptor.java

@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.provenance;
+
+import java.security.KeyManagementException;
+
+public interface ProvenanceEventEncryptor {
+
+    /**
+     * Initializes the encryptor with a {@link KeyProvider}.
+     *
+     * @param keyProvider the key provider which will be responsible for accessing keys
+     * @throws KeyManagementException if there is an issue configuring the key provider
+     */
+    void initialize(KeyProvider keyProvider) throws KeyManagementException;
+
+    /**
+     * Encrypts the provided {@link ProvenanceEventRecord}, serialized to a byte[] by the RecordWriter.
+     *
+     * @param plainRecord the plain record, serialized to a byte[]
+     * @param recordId    an identifier for this record (eventId, generated, etc.)
+     * @param keyId       the ID of the key to use
+     * @return the encrypted record
+     * @throws EncryptionException if there is an issue encrypting this record
+     */
+    byte[] encrypt(byte[] plainRecord, String recordId, String keyId) throws EncryptionException;
+
+    /**
+     * Decrypts the provided byte[] (an encrypted record with accompanying metadata).
+     *
+     * @param encryptedRecord the encrypted record in byte[] form
+     * @param recordId        an identifier for this record (eventId, generated, etc.)
+     * @return the decrypted record
+     * @throws EncryptionException if there is an issue decrypting this record
+     */
+    byte[] decrypt(byte[] encryptedRecord, String recordId) throws EncryptionException;
+
+    /**
+     * Returns a valid key identifier for this encryptor (valid for encryption and decryption) or throws an exception if none are available.
+     *
+     * @return the key ID
+     * @throws KeyManagementException if no available key IDs are valid for both operations
+     */
+    String getNextKeyId() throws KeyManagementException;
+}

nifi-commons/nifi-data-provenance-utils/src/main/java/org/apache/nifi/provenance/StandardProvenanceEventRecord.java

@@ -22,7 +22,6 @@ import java.util.Date;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
-
 import org.apache.nifi.flowfile.FlowFile;
 import org.apache.nifi.flowfile.attributes.CoreAttributes;
 import org.apache.nifi.processor.Relationship;
@@ -30,7 +29,7 @@ import org.apache.nifi.processor.Relationship;
 /**
  * Holder for provenance relevant information
  */
-public final class StandardProvenanceEventRecord implements ProvenanceEventRecord {
+public class StandardProvenanceEventRecord implements ProvenanceEventRecord {
 
     private final long eventTime;
     private final long entryDate;
@@ -69,7 +68,7 @@ public final class StandardProvenanceEventRecord implements ProvenanceEventRecor
 
     private volatile long eventId = -1L;
 
-    private StandardProvenanceEventRecord(final Builder builder) {
+    StandardProvenanceEventRecord(final Builder builder) {
         this.eventTime = builder.eventTime;
         this.entryDate = builder.entryDate;
         this.eventType = builder.eventType;
@@ -100,8 +99,8 @@ public final class StandardProvenanceEventRecord implements ProvenanceEventRecor
         contentClaimOffset = builder.contentClaimOffset;
         contentSize = builder.contentSize;
 
-        previousAttributes = builder.previousAttributes == null ? Collections.<String, String>emptyMap() : Collections.unmodifiableMap(builder.previousAttributes);
-        updatedAttributes = builder.updatedAttributes == null ? Collections.<String, String>emptyMap() : Collections.unmodifiableMap(builder.updatedAttributes);
+        previousAttributes = builder.previousAttributes == null ? Collections.emptyMap() : Collections.unmodifiableMap(builder.previousAttributes);
+        updatedAttributes = builder.updatedAttributes == null ? Collections.emptyMap() : Collections.unmodifiableMap(builder.updatedAttributes);
 
         sourceQueueIdentifier = builder.sourceQueueIdentifier;
 
@@ -110,6 +109,11 @@ public final class StandardProvenanceEventRecord implements ProvenanceEventRecor
         }
     }
 
+    public static StandardProvenanceEventRecord copy(StandardProvenanceEventRecord other) {
+        Builder builder = new Builder().fromEvent(other);
+        return new StandardProvenanceEventRecord(builder);
+    }
+
     public String getStorageFilename() {
         return storageFilename;
     }
@@ -199,12 +203,12 @@ public final class StandardProvenanceEventRecord implements ProvenanceEventRecor
 
     @Override
     public List<String> getParentUuids() {
-        return parentUuids == null ? Collections.<String>emptyList() : parentUuids;
+        return parentUuids == null ? Collections.emptyList() : parentUuids;
     }
 
     @Override
     public List<String> getChildUuids() {
-        return childrenUuids == null ? Collections.<String>emptyList() : childrenUuids;
+        return childrenUuids == null ? Collections.emptyList() : childrenUuids;
     }
 
     @Override
@@ -299,8 +303,8 @@ public final class StandardProvenanceEventRecord implements ProvenanceEventRecor
         }
 
         return -37423 + 3 * componentId.hashCode() + (transitUri == null ? 0 : 41 * transitUri.hashCode())
-            + (relationship == null ? 0 : 47 * relationship.hashCode()) + 44 * eventTypeCode
-            + 47 * getChildUuids().hashCode() + 47 * getParentUuids().hashCode();
+                + (relationship == null ? 0 : 47 * relationship.hashCode()) + 44 * eventTypeCode
+                + 47 * getChildUuids().hashCode() + 47 * getParentUuids().hashCode();
     }
 
     @Override
@@ -419,6 +423,23 @@ public final class StandardProvenanceEventRecord implements ProvenanceEventRecor
                 + ", alternateIdentifierUri=" + alternateIdentifierUri + "]";
     }
 
+    /**
+     * Returns a unique identifier for the record. By default, it uses
+     * {@link ProvenanceEventRecord#getEventId()} but if it has not been persisted to the
+     * repository, this is {@code -1}, so it constructs a String of the format
+     * {@code <event type>_on_<flowfile UUID>_by_<component UUID>_at_<event time>}.
+     *
+     * @return a String identifying the record for later analysis
+     */
+    @Override
+    public String getBestEventIdentifier() {
+        if (getEventId() != -1) {
+            return Long.toString(getEventId());
+        } else {
+            return getEventType().name() + "_on_" + getFlowFileUuid() + "_by_" + getComponentId() + "_at_" + getEventTime();
+        }
+    }
+
     public static class Builder implements ProvenanceEventBuilder {
 
         private long eventTime = System.currentTimeMillis();
@@ -733,7 +754,7 @@ public final class StandardProvenanceEventRecord implements ProvenanceEventRecor
         public ProvenanceEventBuilder fromFlowFile(final FlowFile flowFile) {
             setFlowFileEntryDate(flowFile.getEntryDate());
             setLineageStartDate(flowFile.getLineageStartDate());
-            setAttributes(Collections.<String, String>emptyMap(), flowFile.getAttributes());
+            setAttributes(Collections.emptyMap(), flowFile.getAttributes());
             uuid = flowFile.getAttribute(CoreAttributes.UUID.key());
             this.contentSize = flowFile.getSize();
             return this;

nifi-commons/nifi-data-provenance-utils/src/main/java/org/apache/nifi/provenance/StaticKeyProvider.java

@@ -0,0 +1,96 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.provenance;
+
+import java.security.KeyManagementException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import javax.crypto.SecretKey;
+import javax.naming.OperationNotSupportedException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Reference implementation for static key provider (used during tests).
+ */
+public class StaticKeyProvider implements KeyProvider {
+    private static final Logger logger = LoggerFactory.getLogger(StaticKeyProvider.class);
+
+    private Map<String, SecretKey> keys = new HashMap<>();
+
+    StaticKeyProvider(String keyId, String keyHex) throws KeyManagementException {
+        this.keys.put(keyId, CryptoUtils.formKeyFromHex(keyHex));
+    }
+
+    StaticKeyProvider(Map<String, SecretKey> keys) throws KeyManagementException {
+        this.keys.putAll(keys);
+    }
+
+    /**
+     * Returns the key identified by this ID or throws an exception if one is not available.
+     *
+     * @param keyId the key identifier
+     * @return the key
+     * @throws KeyManagementException if the key cannot be retrieved
+     */
+    @Override
+    public SecretKey getKey(String keyId) throws KeyManagementException {
+        logger.debug("Attempting to get key: " + keyId);
+        if (keyExists(keyId)) {
+            return keys.get(keyId);
+        } else {
+            throw new KeyManagementException("No key available for " + keyId);
+        }
+    }
+
+    /**
+     * Returns true if the key exists and is available. Null or empty IDs will return false.
+     *
+     * @param keyId the key identifier
+     * @return true if the key can be used
+     */
+    @Override
+    public boolean keyExists(String keyId) {
+        return keys.containsKey(keyId);
+    }
+
+    /**
+     * Returns a singleton list of the available key identifier.
+     *
+     * @return a List containing the {@code KEY_ID}
+     */
+    @Override
+    public List<String> getAvailableKeyIds() {
+        return new ArrayList<>(keys.keySet());
+    }
+
+    /**
+     * Adds the key to the provider and associates it with the given ID. Some implementations may not allow this operation.
+     *
+     * @param keyId the key identifier
+     * @param key   the key
+     * @return true if the key was successfully added
+     * @throws OperationNotSupportedException if this implementation doesn't support adding keys
+     * @throws KeyManagementException         if the key is invalid, the ID conflicts, etc.
+     */
+    @Override
+    public boolean addKey(String keyId, SecretKey key) throws OperationNotSupportedException, KeyManagementException {
+        throw new OperationNotSupportedException("This implementation does not allow adding keys");
+    }
+}

nifi-commons/nifi-data-provenance-utils/src/test/groovy/org/apache/nifi/provenance/AESProvenanceEventEncryptorTest.groovy

@@ -0,0 +1,303 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.provenance
+
+import org.apache.nifi.security.util.EncryptionMethod
+import org.apache.nifi.security.util.crypto.AESKeyedCipherProvider
+import org.bouncycastle.jce.provider.BouncyCastleProvider
+import org.bouncycastle.util.encoders.Hex
+import org.junit.After
+import org.junit.AfterClass
+import org.junit.Before
+import org.junit.BeforeClass
+import org.junit.Test
+import org.junit.runner.RunWith
+import org.junit.runners.JUnit4
+import org.slf4j.Logger
+import org.slf4j.LoggerFactory
+
+import javax.crypto.Cipher
+import javax.crypto.SecretKey
+import javax.crypto.spec.IvParameterSpec
+import javax.crypto.spec.SecretKeySpec
+import java.nio.charset.StandardCharsets
+import java.security.KeyManagementException
+import java.security.SecureRandom
+import java.security.Security
+
+import static groovy.test.GroovyAssert.shouldFail
+
+@RunWith(JUnit4.class)
+class AESProvenanceEventEncryptorTest {
+    private static final Logger logger = LoggerFactory.getLogger(AESProvenanceEventEncryptorTest.class)
+
+    private static final String KEY_HEX_128 = "0123456789ABCDEFFEDCBA9876543210"
+    private static final String KEY_HEX_256 = KEY_HEX_128 * 2
+    private static final String KEY_HEX = isUnlimitedStrengthCryptoAvailable() ? KEY_HEX_256 : KEY_HEX_128
+
+    private static KeyProvider mockKeyProvider
+    private static AESKeyedCipherProvider mockCipherProvider
+
+    private static String ORIGINAL_LOG_LEVEL
+
+    private ProvenanceEventEncryptor encryptor
+
+    @BeforeClass
+    static void setUpOnce() throws Exception {
+        ORIGINAL_LOG_LEVEL = System.getProperty("org.slf4j.simpleLogger.log.org.apache.nifi.provenance")
+        System.setProperty("org.slf4j.simpleLogger.log.org.apache.nifi.provenance", "DEBUG")
+
+        Security.addProvider(new BouncyCastleProvider())
+
+        logger.metaClass.methodMissing = { String name, args ->
+            logger.info("[${name?.toUpperCase()}] ${(args as List).join(" ")}")
+        }
+
+        mockKeyProvider = [
+                getKey   : { String keyId ->
+                    logger.mock("Requesting key ID: ${keyId}")
+                    new SecretKeySpec(Hex.decode(KEY_HEX), "AES")
+                },
+                keyExists: { String keyId ->
+                    logger.mock("Checking existence of ${keyId}")
+                    true
+                }] as KeyProvider
+
+        mockCipherProvider = [
+                getCipher: { EncryptionMethod em, SecretKey key, byte[] ivBytes, boolean encryptMode ->
+                    logger.mock("Getting cipher for ${em} with IV ${Hex.toHexString(ivBytes)} encrypt ${encryptMode}")
+                    Cipher cipher = Cipher.getInstance("AES/GCM/NoPadding")
+                    cipher.init((encryptMode ? Cipher.ENCRYPT_MODE : Cipher.DECRYPT_MODE) as int, key, new IvParameterSpec(ivBytes))
+                    cipher
+                }
+        ] as AESKeyedCipherProvider
+    }
+
+    @Before
+    void setUp() throws Exception {
+
+    }
+
+    @After
+    void tearDown() throws Exception {
+
+    }
+
+    @AfterClass
+    static void tearDownOnce() throws Exception {
+        if (ORIGINAL_LOG_LEVEL) {
+            System.setProperty("org.slf4j.simpleLogger.log.org.apache.nifi.provenance", ORIGINAL_LOG_LEVEL)
+        }
+    }
+
+    private static boolean isUnlimitedStrengthCryptoAvailable() {
+        Cipher.getMaxAllowedKeyLength("AES") > 128
+    }
+
+    /**
+     * Given arbitrary bytes, encrypt them and persist with the encryption metadata, then recover
+     */
+    @Test
+    void testShouldEncryptAndDecryptArbitraryBytes() {
+        // Arrange
+        final byte[] SERIALIZED_BYTES = "This is a plaintext message.".getBytes(StandardCharsets.UTF_8)
+        logger.info("Serialized bytes (${SERIALIZED_BYTES.size()}): ${Hex.toHexString(SERIALIZED_BYTES)}")
+
+        encryptor = new AESProvenanceEventEncryptor()
+        encryptor.initialize(mockKeyProvider)
+        encryptor.setCipherProvider(mockCipherProvider)
+        logger.info("Created ${encryptor}")
+
+        String keyId = "K1"
+        String recordId = "R1"
+        logger.info("Using record ID ${recordId} and key ID ${keyId}")
+
+        // Act
+        byte[] metadataAndCipherBytes = encryptor.encrypt(SERIALIZED_BYTES, recordId, keyId)
+        logger.info("Encrypted data to: \n\t${Hex.toHexString(metadataAndCipherBytes)}")
+
+        byte[] recoveredBytes = encryptor.decrypt(metadataAndCipherBytes, recordId)
+        logger.info("Decrypted data to: \n\t${Hex.toHexString(recoveredBytes)}")
+
+        // Assert
+        assert recoveredBytes == SERIALIZED_BYTES
+        logger.info("Decoded (usually would be serialized schema record): ${new String(recoveredBytes, StandardCharsets.UTF_8)}")
+    }
+
+    @Test
+    void testShouldInitializeNullCipherProvider() {
+        // Arrange
+        encryptor = new AESProvenanceEventEncryptor()
+        encryptor.setCipherProvider(null)
+        assert !encryptor.aesKeyedCipherProvider
+        
+        // Act
+        encryptor.initialize(mockKeyProvider)
+        logger.info("Created ${encryptor}")
+
+        // Assert
+        assert encryptor.aesKeyedCipherProvider instanceof AESKeyedCipherProvider
+    }
+
+    @Test
+    void testShouldFailOnMissingKeyId() {
+        // Arrange
+        final byte[] SERIALIZED_BYTES = "This is a plaintext message.".getBytes(StandardCharsets.UTF_8)
+        logger.info("Serialized bytes (${SERIALIZED_BYTES.size()}): ${Hex.toHexString(SERIALIZED_BYTES)}")
+
+        KeyProvider emptyKeyProvider = [
+                getKey   : { String kid ->
+                    throw new KeyManagementException("No key found for ${kid}")
+                },
+                keyExists: { String kid -> false }
+        ] as KeyProvider
+
+        encryptor = new AESProvenanceEventEncryptor()
+        encryptor.initialize(emptyKeyProvider)
+        encryptor.setCipherProvider(mockCipherProvider)
+        logger.info("Created ${encryptor}")
+
+        String keyId = "K1"
+        String recordId = "R1"
+        logger.info("Using record ID ${recordId} and key ID ${keyId}")
+
+        // Act
+        def msg = shouldFail(EncryptionException) {
+            byte[] metadataAndCipherBytes = encryptor.encrypt(SERIALIZED_BYTES, recordId, keyId)
+            logger.info("Encrypted data to: \n\t${Hex.toHexString(metadataAndCipherBytes)}")
+        }
+        logger.expected(msg)
+
+        // Assert
+        assert msg.getMessage() == "The requested key ID is not available"
+    }
+
+    @Test
+    void testShouldUseDifferentIVsForSequentialEncryptions() {
+        // Arrange
+        final byte[] SERIALIZED_BYTES = "This is a plaintext message.".getBytes(StandardCharsets.UTF_8)
+        logger.info("Serialized bytes (${SERIALIZED_BYTES.size()}): ${Hex.toHexString(SERIALIZED_BYTES)}")
+
+        encryptor = new AESProvenanceEventEncryptor()
+        encryptor.initialize(mockKeyProvider)
+        logger.info("Created ${encryptor}")
+
+        String keyId = "K1"
+        String recordId1 = "R1"
+        logger.info("Using record ID ${recordId1} and key ID ${keyId}")
+
+        // Act
+        byte[] metadataAndCipherBytes1 = encryptor.encrypt(SERIALIZED_BYTES, recordId1, keyId)
+        logger.info("Encrypted data to: \n\t${Hex.toHexString(metadataAndCipherBytes1)}")
+        EncryptionMetadata metadata1 = encryptor.extractEncryptionMetadata(metadataAndCipherBytes1)
+        logger.info("Record ${recordId1} IV: ${Hex.toHexString(metadata1.ivBytes)}")
+
+        byte[] recoveredBytes1 = encryptor.decrypt(metadataAndCipherBytes1, recordId1)
+        logger.info("Decrypted data to: \n\t${Hex.toHexString(recoveredBytes1)}")
+
+        String recordId2 = "R2"
+        byte[] metadataAndCipherBytes2 = encryptor.encrypt(SERIALIZED_BYTES, recordId2, keyId)
+        logger.info("Encrypted data to: \n\t${Hex.toHexString(metadataAndCipherBytes2)}")
+        EncryptionMetadata metadata2 = encryptor.extractEncryptionMetadata(metadataAndCipherBytes2)
+        logger.info("Record ${recordId2} IV: ${Hex.toHexString(metadata2.ivBytes)}")
+
+        byte[] recoveredBytes2 = encryptor.decrypt(metadataAndCipherBytes2, recordId2)
+        logger.info("Decrypted data to: \n\t${Hex.toHexString(recoveredBytes2)}")
+
+        // Assert
+        assert metadata1.ivBytes != metadata2.ivBytes
+
+        assert recoveredBytes1 == SERIALIZED_BYTES
+        assert recoveredBytes2 == SERIALIZED_BYTES
+    }
+
+    @Test
+    void testShouldFailOnBadMetadata() {
+        // Arrange
+        final byte[] SERIALIZED_BYTES = "This is a plaintext message.".getBytes(StandardCharsets.UTF_8)
+        logger.info("Serialized bytes (${SERIALIZED_BYTES.size()}): ${Hex.toHexString(SERIALIZED_BYTES)}")
+
+        def strictMockKeyProvider = [
+                getKey   : { String keyId ->
+                    if (keyId != "K1") {
+                        throw new KeyManagementException("No such key")
+                    }
+                    new SecretKeySpec(Hex.decode(KEY_HEX), "AES")
+                },
+                keyExists: { String keyId ->
+                    keyId == "K1"
+                }] as KeyProvider
+
+        encryptor = new AESProvenanceEventEncryptor()
+        encryptor.initialize(strictMockKeyProvider)
+        encryptor.setCipherProvider(mockCipherProvider)
+        logger.info("Created ${encryptor}")
+
+        String keyId = "K1"
+        String recordId = "R1"
+        logger.info("Using record ID ${recordId} and key ID ${keyId}")
+
+        final String ALGORITHM = "AES/GCM/NoPadding"
+        final byte[] ivBytes = new byte[16]
+        new SecureRandom().nextBytes(ivBytes)
+        final String VERSION = "v1"
+
+        // Perform the encryption independently of the encryptor
+        SecretKey key = mockKeyProvider.getKey(keyId)
+        Cipher cipher = new AESKeyedCipherProvider().getCipher(EncryptionMethod.AES_GCM, key, ivBytes, true)
+        byte[] cipherBytes = cipher.doFinal(SERIALIZED_BYTES)
+
+        int cipherBytesLength = cipherBytes.size()
+
+        // Construct accurate metadata
+        EncryptionMetadata goodMetadata = new EncryptionMetadata(keyId, ALGORITHM, ivBytes, VERSION, cipherBytesLength)
+        logger.info("Created good encryption metadata: ${goodMetadata}")
+
+        // Construct bad metadata instances
+        EncryptionMetadata badKeyId = new EncryptionMetadata(keyId.reverse(), ALGORITHM, ivBytes, VERSION, cipherBytesLength)
+        EncryptionMetadata badAlgorithm = new EncryptionMetadata(keyId, "ASE/GDM/SomePadding", ivBytes, VERSION, cipherBytesLength)
+        EncryptionMetadata badIV = new EncryptionMetadata(keyId, ALGORITHM, new byte[16], VERSION, cipherBytesLength)
+        EncryptionMetadata badVersion = new EncryptionMetadata(keyId, ALGORITHM, ivBytes, VERSION.reverse(), cipherBytesLength)
+        EncryptionMetadata badCBLength = new EncryptionMetadata(keyId, ALGORITHM, ivBytes, VERSION, cipherBytesLength - 5)
+
+        List badMetadata = [badKeyId, badAlgorithm, badIV, badVersion, badCBLength]
+
+        // Form the proper cipherBytes
+        byte[] completeGoodBytes = CryptoUtils.concatByteArrays([0x01] as byte[], encryptor.serializeEncryptionMetadata(goodMetadata), cipherBytes)
+
+        byte[] recoveredGoodBytes = encryptor.decrypt(completeGoodBytes, recordId)
+        logger.info("Recovered good bytes: ${Hex.toHexString(recoveredGoodBytes)}")
+
+        final List EXPECTED_MESSAGES = ["The requested key ID (\\w+)? is not available",
+                                        "Encountered an exception decrypting provenance record",
+                                        "The event was encrypted with version ${VERSION.reverse()} which is not in the list of supported versions v1"]
+
+        // Act
+        badMetadata.eachWithIndex { EncryptionMetadata metadata, int i ->
+            byte[] completeBytes = CryptoUtils.concatByteArrays([0x01] as byte[], encryptor.serializeEncryptionMetadata(metadata), cipherBytes)
+
+            def msg = shouldFail(EncryptionException) {
+                byte[] recoveredBytes = encryptor.decrypt(completeBytes, "R${i + 2}")
+                logger.info("Recovered bad bytes: ${Hex.toHexString(recoveredBytes)}")
+            }
+            logger.expected(msg)
+
+            // Assert
+            assert EXPECTED_MESSAGES.any { msg.getMessage() =~ it }
+        }
+    }
+}

nifi-commons/nifi-data-provenance-utils/src/test/groovy/org/apache/nifi/provenance/CryptoUtilsTest.groovy

@@ -0,0 +1,436 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License") you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.provenance
+
+import org.bouncycastle.jce.provider.BouncyCastleProvider
+import org.bouncycastle.util.encoders.Hex
+import org.junit.After
+import org.junit.AfterClass
+import org.junit.Before
+import org.junit.BeforeClass
+import org.junit.ClassRule
+import org.junit.Test
+import org.junit.rules.TemporaryFolder
+import org.junit.runner.RunWith
+import org.junit.runners.JUnit4
+import org.slf4j.Logger
+import org.slf4j.LoggerFactory
+
+import javax.crypto.Cipher
+import javax.crypto.SecretKey
+import javax.crypto.spec.IvParameterSpec
+import javax.crypto.spec.SecretKeySpec
+import java.nio.charset.StandardCharsets
+import java.nio.file.Files
+import java.nio.file.attribute.PosixFilePermission
+import java.security.KeyManagementException
+import java.security.SecureRandom
+import java.security.Security
+
+import static groovy.test.GroovyAssert.shouldFail
+
+@RunWith(JUnit4.class)
+class CryptoUtilsTest {
+    private static final Logger logger = LoggerFactory.getLogger(CryptoUtilsTest.class)
+
+    private static final String KEY_ID = "K1"
+    private static final String KEY_HEX_128 = "0123456789ABCDEFFEDCBA9876543210"
+    private static final String KEY_HEX_256 = KEY_HEX_128 * 2
+    private static final String KEY_HEX = isUnlimitedStrengthCryptoAvailable() ? KEY_HEX_256 : KEY_HEX_128
+
+    private static
+    final Set<PosixFilePermission> ALL_POSIX_ATTRS = PosixFilePermission.values() as Set<PosixFilePermission>
+
+    @ClassRule
+    public static TemporaryFolder tempFolder = new TemporaryFolder()
+
+    @BeforeClass
+    static void setUpOnce() throws Exception {
+        Security.addProvider(new BouncyCastleProvider())
+
+        logger.metaClass.methodMissing = { String name, args ->
+            logger.info("[${name?.toUpperCase()}] ${(args as List).join(" ")}")
+        }
+    }
+
+    @Before
+    void setUp() throws Exception {
+        tempFolder.create()
+    }
+
+    @After
+    void tearDown() throws Exception {
+        tempFolder?.delete()
+    }
+
+    @AfterClass
+    static void tearDownOnce() throws Exception {
+
+    }
+
+    private static boolean isUnlimitedStrengthCryptoAvailable() {
+        Cipher.getMaxAllowedKeyLength("AES") > 128
+    }
+
+    @Test
+    void testShouldConcatenateByteArrays() {
+        // Arrange
+        byte[] bytes1 = "These are some bytes".getBytes(StandardCharsets.UTF_8)
+        byte[] bytes2 = "These are some other bytes".getBytes(StandardCharsets.UTF_8)
+        final byte[] EXPECTED_CONCATENATED_BYTES = ((bytes1 as List) << (bytes2 as List)).flatten() as byte[]
+        logger.info("Expected concatenated bytes: ${Hex.toHexString(EXPECTED_CONCATENATED_BYTES)}")
+
+        // Act
+        byte[] concat = CryptoUtils.concatByteArrays(bytes1, bytes2)
+        logger.info("  Actual concatenated bytes: ${Hex.toHexString(concat)}")
+
+        // Assert
+        assert concat == EXPECTED_CONCATENATED_BYTES
+    }
+
+    @Test
+    void testShouldValidateStaticKeyProvider() {
+        // Arrange
+        String staticProvider = StaticKeyProvider.class.name
+        String providerLocation = null
+
+        // Act
+        boolean keyProviderIsValid = CryptoUtils.isValidKeyProvider(staticProvider, providerLocation, KEY_ID, [(KEY_ID): KEY_HEX])
+        logger.info("Key Provider ${staticProvider} with location ${providerLocation} and keyId ${KEY_ID} / ${KEY_HEX} is ${keyProviderIsValid ? "valid" : "invalid"}")
+
+        // Assert
+        assert keyProviderIsValid
+    }
+
+    @Test
+    void testShouldNotValidateStaticKeyProviderMissingKeyId() {
+        // Arrange
+        String staticProvider = StaticKeyProvider.class.name
+        String providerLocation = null
+
+        // Act
+        boolean keyProviderIsValid = CryptoUtils.isValidKeyProvider(staticProvider, providerLocation, null, [(KEY_ID): KEY_HEX])
+        logger.info("Key Provider ${staticProvider} with location ${providerLocation} and keyId ${null} / ${KEY_HEX} is ${keyProviderIsValid ? "valid" : "invalid"}")
+
+        // Assert
+        assert !keyProviderIsValid
+    }
+
+    @Test
+    void testShouldNotValidateStaticKeyProviderMissingKey() {
+        // Arrange
+        String staticProvider = StaticKeyProvider.class.name
+        String providerLocation = null
+
+        // Act
+        boolean keyProviderIsValid = CryptoUtils.isValidKeyProvider(staticProvider, providerLocation, KEY_ID, null)
+        logger.info("Key Provider ${staticProvider} with location ${providerLocation} and keyId ${KEY_ID} / ${null} is ${keyProviderIsValid ? "valid" : "invalid"}")
+
+        // Assert
+        assert !keyProviderIsValid
+    }
+
+    @Test
+    void testShouldNotValidateStaticKeyProviderWithInvalidKey() {
+        // Arrange
+        String staticProvider = StaticKeyProvider.class.name
+        String providerLocation = null
+
+        // Act
+        boolean keyProviderIsValid = CryptoUtils.isValidKeyProvider(staticProvider, providerLocation, KEY_ID, [(KEY_ID): KEY_HEX[0..<-2]])
+        logger.info("Key Provider ${staticProvider} with location ${providerLocation} and keyId ${KEY_ID} / ${KEY_HEX[0..<-2]} is ${keyProviderIsValid ? "valid" : "invalid"}")
+
+        // Assert
+        assert !keyProviderIsValid
+    }
+
+    @Test
+    void testShouldValidateFileBasedKeyProvider() {
+        // Arrange
+        String fileBasedProvider = FileBasedKeyProvider.class.name
+        File fileBasedProviderFile = tempFolder.newFile("filebased.kp")
+        String providerLocation = fileBasedProviderFile.path
+        logger.info("Created temporary file based key provider: ${providerLocation}")
+
+        // Act
+        boolean keyProviderIsValid = CryptoUtils.isValidKeyProvider(fileBasedProvider, providerLocation, KEY_ID, null)
+        logger.info("Key Provider ${fileBasedProvider} with location ${providerLocation} and keyId ${KEY_ID} / ${null} is ${keyProviderIsValid ? "valid" : "invalid"}")
+
+        // Assert
+        assert keyProviderIsValid
+    }
+
+    @Test
+    void testShouldNotValidateUnreadableOrMissingFileBasedKeyProvider() {
+        // Arrange
+        String fileBasedProvider = FileBasedKeyProvider.class.name
+        File fileBasedProviderFile = tempFolder.newFile("filebased.kp")
+        String providerLocation = fileBasedProviderFile.path
+        logger.info("Created temporary file based key provider: ${providerLocation}")
+
+        // Make it unreadable
+        fileBasedProviderFile.setReadable(false, false)
+        Files.setPosixFilePermissions(fileBasedProviderFile.toPath(), [] as Set<PosixFilePermission>)
+
+        // Act
+        boolean unreadableKeyProviderIsValid = CryptoUtils.isValidKeyProvider(fileBasedProvider, providerLocation, KEY_ID, null)
+        logger.info("Key Provider ${fileBasedProvider} with location ${providerLocation} and keyId ${KEY_ID} / ${null} is ${unreadableKeyProviderIsValid ? "valid" : "invalid"}")
+
+        String missingLocation = providerLocation + "_missing"
+        boolean missingKeyProviderIsValid = CryptoUtils.isValidKeyProvider(fileBasedProvider, missingLocation, KEY_ID, null)
+        logger.info("Key Provider ${fileBasedProvider} with location ${missingLocation} and keyId ${KEY_ID} / ${null} is ${missingKeyProviderIsValid ? "valid" : "invalid"}")
+
+        // Assert
+        assert !unreadableKeyProviderIsValid
+        assert !missingKeyProviderIsValid
+
+        // Make the file deletable so cleanup can occur
+        fileBasedProviderFile.setReadable(true, false)
+        Files.setPosixFilePermissions(fileBasedProviderFile.toPath(), ALL_POSIX_ATTRS)
+    }
+
+    @Test
+    void testShouldNotValidateFileBasedKeyProviderMissingKeyId() {
+        // Arrange
+        String fileBasedProvider = FileBasedKeyProvider.class.name
+        File fileBasedProviderFile = tempFolder.newFile("missing_key_id.kp")
+        String providerLocation = fileBasedProviderFile.path
+        logger.info("Created temporary file based key provider: ${providerLocation}")
+
+        // Act
+        boolean keyProviderIsValid = CryptoUtils.isValidKeyProvider(fileBasedProvider, providerLocation, null, null)
+        logger.info("Key Provider ${fileBasedProvider} with location ${providerLocation} and keyId ${null} / ${null} is ${keyProviderIsValid ? "valid" : "invalid"}")
+
+        // Assert
+        assert !keyProviderIsValid
+    }
+
+    @Test
+    void testShouldNotValidateUnknownKeyProvider() {
+        // Arrange
+        String providerImplementation = "org.apache.nifi.provenance.ImaginaryKeyProvider"
+        String providerLocation = null
+
+        // Act
+        boolean keyProviderIsValid = CryptoUtils.isValidKeyProvider(providerImplementation, providerLocation, KEY_ID, null)
+        logger.info("Key Provider ${providerImplementation} with location ${providerLocation} and keyId ${KEY_ID} / ${null} is ${keyProviderIsValid ? "valid" : "invalid"}")
+
+        // Assert
+        assert !keyProviderIsValid
+    }
+
+    @Test
+    void testShouldValidateKey() {
+        // Arrange
+        String validKey = KEY_HEX
+        String validLowercaseKey = KEY_HEX.toLowerCase()
+
+        String tooShortKey = KEY_HEX[0..<-2]
+        String tooLongKey = KEY_HEX + KEY_HEX // Guaranteed to be 2x the max valid key length
+        String nonHexKey = KEY_HEX.replaceFirst(/A/, "X")
+
+        def validKeys = [validKey, validLowercaseKey]
+        def invalidKeys = [tooShortKey, tooLongKey, nonHexKey]
+
+        // If unlimited strength is available, also validate 128 and 196 bit keys
+        if (isUnlimitedStrengthCryptoAvailable()) {
+            validKeys << KEY_HEX_128
+            validKeys << KEY_HEX_256[0..<48]
+        } else {
+            invalidKeys << KEY_HEX_256[0..<48]
+            invalidKeys << KEY_HEX_256
+        }
+
+        // Act
+        def validResults = validKeys.collect { String key ->
+            logger.info("Validating ${key}")
+            CryptoUtils.keyIsValid(key)
+        }
+
+        def invalidResults = invalidKeys.collect { String key ->
+            logger.info("Validating ${key}")
+            CryptoUtils.keyIsValid(key)
+        }
+
+        // Assert
+        assert validResults.every()
+        assert invalidResults.every { !it }
+    }
+
+    @Test
+    void testShouldReadKeys() {
+        // Arrange
+        String masterKeyHex = KEY_HEX
+        SecretKey masterKey = new SecretKeySpec(Hex.decode(masterKeyHex), "AES")
+
+        // Generate the file
+        String keyFileName = "keys.nkp"
+        File keyFile = tempFolder.newFile(keyFileName)
+        final int KEY_COUNT = 5
+        List<String> lines = []
+        KEY_COUNT.times { int i ->
+            lines.add("key${i + 1}=${generateEncryptedKey(masterKey)}")
+        }
+
+        keyFile.text = lines.join("\n")
+
+        logger.info("File contents: \n${keyFile.text}")
+
+        // Act
+        def readKeys = CryptoUtils.readKeys(keyFile.path, masterKey)
+        logger.info("Read ${readKeys.size()} keys from ${keyFile.path}")
+
+        // Assert
+        assert readKeys.size() == KEY_COUNT
+    }
+
+    @Test
+    void testShouldReadKeysWithDuplicates() {
+        // Arrange
+        String masterKeyHex = KEY_HEX
+        SecretKey masterKey = new SecretKeySpec(Hex.decode(masterKeyHex), "AES")
+
+        // Generate the file
+        String keyFileName = "keys.nkp"
+        File keyFile = tempFolder.newFile(keyFileName)
+        final int KEY_COUNT = 3
+        List<String> lines = []
+        KEY_COUNT.times { int i ->
+            lines.add("key${i + 1}=${generateEncryptedKey(masterKey)}")
+        }
+
+        lines.add("key3=${generateEncryptedKey(masterKey)}")
+
+        keyFile.text = lines.join("\n")
+
+        logger.info("File contents: \n${keyFile.text}")
+
+        // Act
+        def readKeys = CryptoUtils.readKeys(keyFile.path, masterKey)
+        logger.info("Read ${readKeys.size()} keys from ${keyFile.path}")
+
+        // Assert
+        assert readKeys.size() == KEY_COUNT
+    }
+
+    @Test
+    void testShouldReadKeysWithSomeMalformed() {
+        // Arrange
+        String masterKeyHex = KEY_HEX
+        SecretKey masterKey = new SecretKeySpec(Hex.decode(masterKeyHex), "AES")
+
+        // Generate the file
+        String keyFileName = "keys.nkp"
+        File keyFile = tempFolder.newFile(keyFileName)
+        final int KEY_COUNT = 5
+        List<String> lines = []
+        KEY_COUNT.times { int i ->
+            lines.add("key${i + 1}=${generateEncryptedKey(masterKey)}")
+        }
+
+        // Insert the malformed keys in the middle
+        lines.add(2, "keyX1==${generateEncryptedKey(masterKey)}")
+        lines.add(4, "=${generateEncryptedKey(masterKey)}")
+        lines.add(6, "keyX3=non Base64-encoded data")
+
+        keyFile.text = lines.join("\n")
+
+        logger.info("File contents: \n${keyFile.text}")
+
+        // Act
+        def readKeys = CryptoUtils.readKeys(keyFile.path, masterKey)
+        logger.info("Read ${readKeys.size()} keys from ${keyFile.path}")
+
+        // Assert
+        assert readKeys.size() == KEY_COUNT
+    }
+
+    @Test
+    void testShouldNotReadKeysIfAllMalformed() {
+        // Arrange
+        String masterKeyHex = KEY_HEX
+        SecretKey masterKey = new SecretKeySpec(Hex.decode(masterKeyHex), "AES")
+
+        // Generate the file
+        String keyFileName = "keys.nkp"
+        File keyFile = tempFolder.newFile(keyFileName)
+        final int KEY_COUNT = 5
+        List<String> lines = []
+
+        // All of these keys are malformed
+        KEY_COUNT.times { int i ->
+            lines.add("key${i + 1}=${generateEncryptedKey(masterKey)[0..<-4]}")
+        }
+
+        keyFile.text = lines.join("\n")
+
+        logger.info("File contents: \n${keyFile.text}")
+
+        // Act
+        def msg = shouldFail(KeyManagementException) {
+            def readKeys = CryptoUtils.readKeys(keyFile.path, masterKey)
+            logger.info("Read ${readKeys.size()} keys from ${keyFile.path}")
+        }
+
+        // Assert
+        assert msg.getMessage() == "The provided file contained no valid keys"
+    }
+
+    @Test
+    void testShouldNotReadKeysIfEmptyOrMissing() {
+        // Arrange
+        String masterKeyHex = KEY_HEX
+        SecretKey masterKey = new SecretKeySpec(Hex.decode(masterKeyHex), "AES")
+
+        // Generate the file
+        String keyFileName = "empty.nkp"
+        File keyFile = tempFolder.newFile(keyFileName)
+        logger.info("File contents: \n${keyFile.text}")
+
+        // Act
+        def missingMsg = shouldFail(KeyManagementException) {
+            def readKeys = CryptoUtils.readKeys(keyFile.path, masterKey)
+            logger.info("Read ${readKeys.size()} keys from ${keyFile.path}")
+        }
+        logger.expected("Missing file: ${missingMsg}")
+
+        def emptyMsg = shouldFail(KeyManagementException) {
+            def readKeys = CryptoUtils.readKeys(null, masterKey)
+            logger.info("Read ${readKeys.size()} keys from ${null}")
+        }
+        logger.expected("Empty file: ${emptyMsg}")
+
+        // Assert
+        assert missingMsg.getMessage() == "The provided file contained no valid keys"
+        assert emptyMsg.getMessage() == "The key provider file is not present and readable"
+    }
+
+    private static String generateEncryptedKey(SecretKey masterKey) {
+        byte[] ivBytes = new byte[16]
+        byte[] keyBytes = new byte[isUnlimitedStrengthCryptoAvailable() ? 32 : 16]
+
+        SecureRandom sr = new SecureRandom()
+        sr.nextBytes(ivBytes)
+        sr.nextBytes(keyBytes)
+
+        Cipher masterCipher = Cipher.getInstance("AES/GCM/NoPadding", "BC")
+        masterCipher.init(Cipher.ENCRYPT_MODE, masterKey, new IvParameterSpec(ivBytes))
+        byte[] cipherBytes = masterCipher.doFinal(keyBytes)
+
+        Base64.encoder.encodeToString(CryptoUtils.concatByteArrays(ivBytes, cipherBytes))
+    }
+}

nifi-commons/nifi-data-provenance-utils/src/test/java/org/apache/nifi/provenance/EncryptionExceptionTest.java

@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.provenance;
+
+import org.junit.Test;
+
+public class EncryptionExceptionTest {
+
+    @Test
+    public void testShouldTriggerGroovyTestExecution() {
+        // This method does nothing but tell Maven to run the groovy tests
+    }
+}

nifi-commons/nifi-data-provenance-utils/src/test/resources/logback-test.xml

@@ -0,0 +1,32 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+  Licensed to the Apache Software Foundation (ASF) under one or more
+  contributor license agreements.  See the NOTICE file distributed with
+  this work for additional information regarding copyright ownership.
+  The ASF licenses this file to You under the Apache License, Version 2.0
+  (the "License"); you may not use this file except in compliance with
+  the License.  You may obtain a copy of the License at
+      http://www.apache.org/licenses/LICENSE-2.0
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+-->
+
+<configuration scan="true" scanPeriod="30 seconds">
+    <appender name="CONSOLE" class="ch.qos.logback.core.ConsoleAppender">
+        <encoder class="ch.qos.logback.classic.encoder.PatternLayoutEncoder">
+            <pattern>%-4r [%t] %-5p %c - %m%n</pattern>
+        </encoder>
+    </appender>
+
+    <!-- valid logging levels: TRACE, DEBUG, INFO, WARN, ERROR -->
+    <logger name="org.apache.nifi" level="DEBUG"/>
+
+    <root level="INFO">
+        <appender-ref ref="CONSOLE"/>
+    </root>
+
+</configuration>
+

nifi-commons/nifi-properties/src/main/java/org/apache/nifi/util/NiFiProperties.java

@@ -32,6 +32,7 @@ import java.util.List;
 import java.util.Map;
 import java.util.Properties;
 import java.util.Set;
+import java.util.stream.Collectors;
 
 /**
  * The NiFiProperties class holds all properties which are needed for various
@@ -116,6 +117,11 @@ public abstract class NiFiProperties {
     public static final String PROVENANCE_INDEXED_ATTRIBUTES = "nifi.provenance.repository.indexed.attributes";
     public static final String PROVENANCE_INDEX_SHARD_SIZE = "nifi.provenance.repository.index.shard.size";
     public static final String PROVENANCE_JOURNAL_COUNT = "nifi.provenance.repository.journal.count";
+    public static final String PROVENANCE_REPO_ENCRYPTION_KEY = "nifi.provenance.repository.encryption.key";
+    public static final String PROVENANCE_REPO_ENCRYPTION_KEY_ID = "nifi.provenance.repository.encryption.key.id";
+    public static final String PROVENANCE_REPO_ENCRYPTION_KEY_PROVIDER_IMPLEMENTATION_CLASS = "nifi.provenance.repository.encryption.key.provider.implementation";
+    public static final String PROVENANCE_REPO_ENCRYPTION_KEY_PROVIDER_LOCATION = "nifi.provenance.repository.encryption.key.provider.location";
+    public static final String PROVENANCE_REPO_DEBUG_FREQUENCY = "nifi.provenance.repository.debug.frequency";
 
     // component status repository properties
     public static final String COMPONENT_STATUS_REPOSITORY_IMPLEMENTATION = "nifi.components.status.repository.implementation";
@@ -769,7 +775,7 @@ public abstract class NiFiProperties {
     /**
      * Returns true if client certificates are required for REST API. Determined
      * if the following conditions are all true:
-     *
+     * <p>
      * - login identity provider is not populated
      * - Kerberos service support is not enabled
      *
@@ -1034,6 +1040,61 @@ public abstract class NiFiProperties {
         return getPropertyKeys().size();
     }
 
+    public String getProvenanceRepoEncryptionKeyId() {
+        return getProperty(PROVENANCE_REPO_ENCRYPTION_KEY_ID);
+    }
+
+    /**
+     * Returns the active provenance repository encryption key if a {@code StaticKeyProvider} is in use.
+     * If no key ID is specified in the properties file, the default
+     * {@code nifi.provenance.repository.encryption.key} value is returned. If a key ID is specified in
+     * {@code nifi.provenance.repository.encryption.key.id}, it will attempt to read from
+     * {@code nifi.provenance.repository.encryption.key.id.XYZ} where {@code XYZ} is the provided key
+     * ID. If that value is empty, it will use the default property
+     * {@code nifi.provenance.repository.encryption.key}.
+     *
+     * @return the provenance repository encryption key in hex form
+     */
+    public String getProvenanceRepoEncryptionKey() {
+        String keyId = getProvenanceRepoEncryptionKeyId();
+        String keyKey = StringUtils.isBlank(keyId) ? PROVENANCE_REPO_ENCRYPTION_KEY : PROVENANCE_REPO_ENCRYPTION_KEY + ".id." + keyId;
+        return getProperty(keyKey, getProperty(PROVENANCE_REPO_ENCRYPTION_KEY));
+    }
+
+    /**
+     * Returns a map of keyId -> key in hex loaded from the {@code nifi.properties} file if a
+     * {@code StaticKeyProvider} is defined. If {@code FileBasedKeyProvider} is defined, use
+     * {@code CryptoUtils#readKeys()} instead -- this method will return an empty map.
+     *
+     * @return a Map of the keys identified by key ID
+     */
+    public Map<String, String> getProvenanceRepoEncryptionKeys() {
+        Map<String, String> keys = new HashMap<>();
+        List<String> keyProperties = getProvenanceRepositoryEncryptionKeyProperties();
+
+        // Retrieve the actual key values and store non-empty values in the map
+        for (String prop : keyProperties) {
+            final String value = getProperty(prop);
+            if (!StringUtils.isBlank(value)) {
+                if (prop.equalsIgnoreCase(PROVENANCE_REPO_ENCRYPTION_KEY)) {
+                    prop = getProvenanceRepoEncryptionKeyId();
+                } else {
+                    // Extract nifi.provenance.repository.encryption.key.id.key1 -> key1
+                    prop = prop.substring(prop.lastIndexOf(".") + 1);
+                }
+                keys.put(prop, value);
+            }
+        }
+        return keys;
+    }
+
+    private List<String> getProvenanceRepositoryEncryptionKeyProperties() {
+        // Filter all the property keys that define a key
+        return getPropertyKeys().stream().filter(k ->
+                k.startsWith(PROVENANCE_REPO_ENCRYPTION_KEY_ID + ".") || k.equalsIgnoreCase(PROVENANCE_REPO_ENCRYPTION_KEY)
+        ).collect(Collectors.toList());
+    }
+
     /**
      * Creates an instance of NiFiProperties. This should likely not be called
      * by any classes outside of the NiFi framework but can be useful by the
@@ -1042,11 +1103,11 @@ public abstract class NiFiProperties {
      * file specified cannot be found/read a runtime exception will be thrown.
      * If one is not specified no properties will be loaded by default.
      *
-     * @param propertiesFilePath if provided properties will be loaded from
-     * given file; else will be loaded from System property. Can be null.
+     * @param propertiesFilePath   if provided properties will be loaded from
+     *                             given file; else will be loaded from System property. Can be null.
      * @param additionalProperties allows overriding of properties with the
-     * supplied values. these will be applied after loading from any properties
-     * file. Can be null or empty.
+     *                             supplied values. these will be applied after loading from any properties
+     *                             file. Can be null or empty.
      * @return NiFiProperties
      */
     public static NiFiProperties createBasicNiFiProperties(final String propertiesFilePath, final Map<String, String> additionalProperties) {
@@ -1108,10 +1169,9 @@ public abstract class NiFiProperties {
     public void validate() {
         // REMOTE_INPUT_HOST should be a valid hostname
         String remoteInputHost = getProperty(REMOTE_INPUT_HOST);
-        if(!StringUtils.isBlank(remoteInputHost) && remoteInputHost.split(":").length > 1) { // no scheme/port needed here (http://)
+        if (!StringUtils.isBlank(remoteInputHost) && remoteInputHost.split(":").length > 1) { // no scheme/port needed here (http://)
             throw new IllegalArgumentException(remoteInputHost + " is not a correct value for " + REMOTE_INPUT_HOST + ". It should be a valid hostname without protocol or port.");
         }
         // Other properties to validate...
     }
-
 }

nifi-commons/nifi-security-utils/pom.xml

@@ -43,6 +43,10 @@
             <groupId>org.apache.commons</groupId>
             <artifactId>commons-lang3</artifactId>
         </dependency>
+        <dependency>
+            <groupId>commons-codec</groupId>
+            <artifactId>commons-codec</artifactId>
+        </dependency>
         <dependency>
             <groupId>org.bouncycastle</groupId>
             <artifactId>bcprov-jdk15on</artifactId>

nifi-commons/nifi-security-utils/src/main/java/org/apache/nifi/security/util/EncryptionMethod.java

@@ -16,13 +16,13 @@
  */
 package org.apache.nifi.security.util;
 
+import org.apache.commons.lang3.StringUtils;
 import org.apache.commons.lang3.builder.ToStringBuilder;
 import org.apache.commons.lang3.builder.ToStringStyle;
 
 /**
  * Enumeration capturing essential information about the various encryption
  * methods that might be supported.
- *
  */
 public enum EncryptionMethod {
 
@@ -105,4 +105,15 @@ public enum EncryptionMethod {
         builder.append("Keyed cipher", isKeyedCipher());
         return builder.toString();
     }
+
+    public static EncryptionMethod forAlgorithm(String algorithm) {
+        if (StringUtils.isNotBlank(algorithm)) {
+            for (EncryptionMethod em : EncryptionMethod.values()) {
+                if (em.algorithm.equalsIgnoreCase(algorithm)) {
+                    return em;
+                }
+            }
+        }
+        return null;
+    }
 }

nifi-commons/nifi-security-utils/src/main/java/org/apache/nifi/security/util/crypto/AESKeyedCipherProvider.java

@@ -14,18 +14,8 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.nifi.processors.standard.util.crypto;
+package org.apache.nifi.security.util.crypto;
 
-import org.apache.commons.lang3.StringUtils;
-import org.apache.nifi.processor.exception.ProcessException;
-import org.apache.nifi.security.util.EncryptionMethod;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import javax.crypto.Cipher;
-import javax.crypto.NoSuchPaddingException;
-import javax.crypto.SecretKey;
-import javax.crypto.spec.IvParameterSpec;
 import java.io.UnsupportedEncodingException;
 import java.security.InvalidAlgorithmParameterException;
 import java.security.InvalidKeyException;
@@ -35,6 +25,15 @@ import java.security.SecureRandom;
 import java.security.spec.InvalidKeySpecException;
 import java.util.Arrays;
 import java.util.List;
+import javax.crypto.Cipher;
+import javax.crypto.NoSuchPaddingException;
+import javax.crypto.SecretKey;
+import javax.crypto.spec.IvParameterSpec;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.nifi.processor.exception.ProcessException;
+import org.apache.nifi.security.util.EncryptionMethod;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 /**
  * This is a standard implementation of {@link KeyedCipherProvider} which supports {@code AES} cipher families with arbitrary modes of operation (currently only {@code CBC}, {@code CTR}, and {@code

nifi-commons/nifi-security-utils/src/main/java/org/apache/nifi/security/util/crypto/CipherProvider.java

@@ -14,7 +14,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.nifi.processors.standard.util.crypto;
+package org.apache.nifi.security.util.crypto;
 
 /**
  * Marker interface for cipher providers.

nifi-commons/nifi-security-utils/src/main/java/org/apache/nifi/security/util/crypto/CipherUtility.java

@@ -14,16 +14,8 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.nifi.processors.standard.util.crypto;
+package org.apache.nifi.security.util.crypto;
 
-import org.apache.commons.codec.binary.Base64;
-import org.apache.commons.lang3.StringUtils;
-import org.apache.nifi.processor.exception.ProcessException;
-import org.apache.nifi.security.util.EncryptionMethod;
-import org.apache.nifi.stream.io.ByteArrayOutputStream;
-import org.apache.nifi.stream.io.StreamUtils;
-
-import javax.crypto.Cipher;
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.OutputStream;
@@ -35,6 +27,13 @@ import java.util.List;
 import java.util.Map;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
+import javax.crypto.Cipher;
+import org.apache.commons.codec.binary.Base64;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.nifi.processor.exception.ProcessException;
+import org.apache.nifi.security.util.EncryptionMethod;
+import org.apache.nifi.stream.io.ByteArrayOutputStream;
+import org.apache.nifi.stream.io.StreamUtils;
 
 public class CipherUtility {
 

nifi-commons/nifi-security-utils/src/main/java/org/apache/nifi/security/util/crypto/KeyedCipherProvider.java

@@ -14,17 +14,16 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.nifi.processors.standard.util.crypto;
+package org.apache.nifi.security.util.crypto;
 
-import org.apache.nifi.processor.exception.ProcessException;
-import org.apache.nifi.security.util.EncryptionMethod;
-
-import javax.crypto.Cipher;
-import javax.crypto.SecretKey;
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.OutputStream;
 import java.nio.charset.StandardCharsets;
+import javax.crypto.Cipher;
+import javax.crypto.SecretKey;
+import org.apache.nifi.processor.exception.ProcessException;
+import org.apache.nifi.security.util.EncryptionMethod;
 
 public abstract class KeyedCipherProvider implements CipherProvider {
     static final byte[] IV_DELIMITER = "NiFiIV".getBytes(StandardCharsets.UTF_8);

nifi-commons/nifi-security-utils/src/test/groovy/org/apache/nifi/security/util/crypto/AESKeyedCipherProviderGroovyTest.groovy

@@ -14,12 +14,16 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.nifi.processors.standard.util.crypto
+package org.apache.nifi.security.util.crypto
 
 import org.apache.commons.codec.binary.Hex
 import org.apache.nifi.security.util.EncryptionMethod
 import org.bouncycastle.jce.provider.BouncyCastleProvider
-import org.junit.*
+import org.junit.After
+import org.junit.Assume
+import org.junit.Before
+import org.junit.BeforeClass
+import org.junit.Test
 import org.junit.runner.RunWith
 import org.junit.runners.JUnit4
 import org.slf4j.Logger
@@ -34,7 +38,7 @@ import java.security.Security
 import static groovy.test.GroovyAssert.shouldFail
 
 @RunWith(JUnit4.class)
-public class AESKeyedCipherProviderGroovyTest {
+class AESKeyedCipherProviderGroovyTest {
     private static final Logger logger = LoggerFactory.getLogger(AESKeyedCipherProviderGroovyTest.class)
 
     private static final String KEY_HEX = "0123456789ABCDEFFEDCBA9876543210"
@@ -44,7 +48,7 @@ public class AESKeyedCipherProviderGroovyTest {
     private static final SecretKey key = new SecretKeySpec(Hex.decodeHex(KEY_HEX as char[]), "AES")
 
     @BeforeClass
-    public static void setUpOnce() throws Exception {
+    static void setUpOnce() throws Exception {
         Security.addProvider(new BouncyCastleProvider())
 
         logger.metaClass.methodMissing = { String name, args ->
@@ -53,15 +57,19 @@ public class AESKeyedCipherProviderGroovyTest {
     }
 
     @Before
-    public void setUp() throws Exception {
+    void setUp() throws Exception {
     }
 
     @After
-    public void tearDown() throws Exception {
+    void tearDown() throws Exception {
+    }
+
+    private static boolean isUnlimitedStrengthCryptoAvailable() {
+        Cipher.getMaxAllowedKeyLength("AES") > 128
     }
 
     @Test
-    public void testGetCipherShouldBeInternallyConsistent() throws Exception {
+    void testGetCipherShouldBeInternallyConsistent() throws Exception {
         // Arrange
         KeyedCipherProvider cipherProvider = new AESKeyedCipherProvider()
 
@@ -90,7 +98,7 @@ public class AESKeyedCipherProviderGroovyTest {
     }
 
     @Test
-    public void testGetCipherWithExternalIVShouldBeInternallyConsistent() throws Exception {
+    void testGetCipherWithExternalIVShouldBeInternallyConsistent() throws Exception {
         // Arrange
         KeyedCipherProvider cipherProvider = new AESKeyedCipherProvider()
 
@@ -119,10 +127,9 @@ public class AESKeyedCipherProviderGroovyTest {
     }
 
     @Test
-    public void testGetCipherWithUnlimitedStrengthShouldBeInternallyConsistent() throws Exception {
+    void testGetCipherWithUnlimitedStrengthShouldBeInternallyConsistent() throws Exception {
         // Arrange
-        Assume.assumeTrue("Test is being skipped due to this JVM lacking JCE Unlimited Strength Jurisdiction Policy file.",
-                PasswordBasedEncryptor.supportsUnlimitedStrength())
+        Assume.assumeTrue("Test is being skipped due to this JVM lacking JCE Unlimited Strength Jurisdiction Policy file.", isUnlimitedStrengthCryptoAvailable())
 
         KeyedCipherProvider cipherProvider = new AESKeyedCipherProvider()
         final List<Integer> LONG_KEY_LENGTHS = [192, 256]
@@ -164,7 +171,7 @@ public class AESKeyedCipherProviderGroovyTest {
     }
 
     @Test
-    public void testShouldRejectEmptyKey() throws Exception {
+    void testShouldRejectEmptyKey() throws Exception {
         // Arrange
         KeyedCipherProvider cipherProvider = new AESKeyedCipherProvider()
 
@@ -180,7 +187,7 @@ public class AESKeyedCipherProviderGroovyTest {
     }
 
     @Test
-    public void testShouldRejectIncorrectLengthKey() throws Exception {
+    void testShouldRejectIncorrectLengthKey() throws Exception {
         // Arrange
         KeyedCipherProvider cipherProvider = new AESKeyedCipherProvider()
 
@@ -199,7 +206,7 @@ public class AESKeyedCipherProviderGroovyTest {
     }
 
     @Test
-    public void testShouldRejectEmptyEncryptionMethod() throws Exception {
+    void testShouldRejectEmptyEncryptionMethod() throws Exception {
         // Arrange
         KeyedCipherProvider cipherProvider = new AESKeyedCipherProvider()
 
@@ -213,7 +220,7 @@ public class AESKeyedCipherProviderGroovyTest {
     }
 
     @Test
-    public void testShouldRejectUnsupportedEncryptionMethod() throws Exception {
+    void testShouldRejectUnsupportedEncryptionMethod() throws Exception {
         // Arrange
         KeyedCipherProvider cipherProvider = new AESKeyedCipherProvider()
 
@@ -229,7 +236,7 @@ public class AESKeyedCipherProviderGroovyTest {
     }
 
     @Test
-    public void testGetCipherShouldSupportExternalCompatibility() throws Exception {
+    void testGetCipherShouldSupportExternalCompatibility() throws Exception {
         // Arrange
         KeyedCipherProvider cipherProvider = new AESKeyedCipherProvider()
 
@@ -258,7 +265,7 @@ public class AESKeyedCipherProviderGroovyTest {
     }
 
     @Test
-    public void testGetCipherForDecryptShouldRequireIV() throws Exception {
+    void testGetCipherForDecryptShouldRequireIV() throws Exception {
         // Arrange
         KeyedCipherProvider cipherProvider = new AESKeyedCipherProvider()
 
@@ -286,7 +293,7 @@ public class AESKeyedCipherProviderGroovyTest {
     }
 
     @Test
-    public void testGetCipherShouldRejectInvalidIVLengths() throws Exception {
+    void testGetCipherShouldRejectInvalidIVLengths() throws Exception {
         // Arrange
         KeyedCipherProvider cipherProvider = new AESKeyedCipherProvider()
 
@@ -313,7 +320,7 @@ public class AESKeyedCipherProviderGroovyTest {
     }
 
     @Test
-    public void testGetCipherShouldRejectEmptyIV() throws Exception {
+    void testGetCipherShouldRejectEmptyIV() throws Exception {
         // Arrange
         KeyedCipherProvider cipherProvider = new AESKeyedCipherProvider()
 

nifi-commons/nifi-security-utils/src/test/groovy/org/apache/nifi/security/util/crypto/CipherUtilityGroovyTest.groovy

@@ -14,7 +14,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.nifi.processors.standard.util.crypto
+package org.apache.nifi.security.util.crypto
 
 import org.apache.nifi.security.util.EncryptionMethod
 import org.bouncycastle.jce.provider.BouncyCastleProvider

nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/FlowController.java

@@ -16,7 +16,38 @@
  */
 package org.apache.nifi.controller;
 
+import static java.util.Objects.requireNonNull;
+
 import com.sun.jersey.api.client.ClientHandlerException;
+import java.io.ByteArrayInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.net.URL;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.LinkedHashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.UUID;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.Future;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.ScheduledFuture;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicReference;
+import java.util.concurrent.locks.Lock;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+import java.util.stream.Collectors;
+import javax.net.ssl.SSLContext;
 import org.apache.commons.collections4.Predicate;
 import org.apache.commons.lang3.StringUtils;
 import org.apache.nifi.action.Action;
@@ -85,7 +116,6 @@ import org.apache.nifi.controller.repository.FlowFileRecord;
 import org.apache.nifi.controller.repository.FlowFileRepository;
 import org.apache.nifi.controller.repository.FlowFileSwapManager;
 import org.apache.nifi.controller.repository.QueueProvider;
-import org.apache.nifi.controller.repository.RepositoryRecord;
 import org.apache.nifi.controller.repository.RepositoryStatusReport;
 import org.apache.nifi.controller.repository.StandardCounterRepository;
 import org.apache.nifi.controller.repository.StandardFlowFileRecord;
@@ -217,38 +247,6 @@ import org.apache.zookeeper.server.quorum.QuorumPeerConfig.ConfigException;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import javax.net.ssl.SSLContext;
-import java.io.ByteArrayInputStream;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.OutputStream;
-import java.net.URL;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.Date;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.LinkedHashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.UUID;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.ConcurrentMap;
-import java.util.concurrent.Future;
-import java.util.concurrent.ScheduledExecutorService;
-import java.util.concurrent.ScheduledFuture;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicBoolean;
-import java.util.concurrent.atomic.AtomicInteger;
-import java.util.concurrent.atomic.AtomicReference;
-import java.util.concurrent.locks.Lock;
-import java.util.concurrent.locks.ReentrantReadWriteLock;
-import java.util.stream.Collectors;
-
-import static java.util.Objects.requireNonNull;
-
 public class FlowController implements EventAccess, ControllerServiceProvider, ReportingTaskProvider,
     QueueProvider, Authorizable, ProvenanceAuthorizableFactory, NodeTypeProvider, IdentifierLookup, ReloadComponent {
 
@@ -3841,7 +3839,7 @@ public class FlowController implements EventAccess, ControllerServiceProvider, R
         final ProvenanceEventRecord sendEvent = new StandardProvenanceEventRecord.Builder()
                 .setEventType(ProvenanceEventType.DOWNLOAD)
                 .setFlowFileUUID(provEvent.getFlowFileUuid())
-                .setAttributes(provEvent.getAttributes(), Collections.<String, String>emptyMap())
+                .setAttributes(provEvent.getAttributes(), Collections.emptyMap())
                 .setCurrentContentClaim(resourceClaim.getContainer(), resourceClaim.getSection(), resourceClaim.getId(), offset, size)
                 .setTransitUri(requestUri)
                 .setEventTime(System.currentTimeMillis())
@@ -3883,7 +3881,7 @@ public class FlowController implements EventAccess, ControllerServiceProvider, R
         final StandardProvenanceEventRecord.Builder sendEventBuilder = new StandardProvenanceEventRecord.Builder()
                 .setEventType(ProvenanceEventType.DOWNLOAD)
                 .setFlowFileUUID(flowFile.getAttribute(CoreAttributes.UUID.key()))
-                .setAttributes(flowFile.getAttributes(), Collections.<String, String>emptyMap())
+                .setAttributes(flowFile.getAttributes(), Collections.emptyMap())
                 .setTransitUri(requestUri)
                 .setEventTime(System.currentTimeMillis())
                 .setFlowFileEntryDate(flowFile.getEntryDate())
@@ -4062,7 +4060,7 @@ public class FlowController implements EventAccess, ControllerServiceProvider, R
                 .addChildUuid(newFlowFileUUID)
                 .addParentUuid(parentUUID)
                 .setFlowFileUUID(parentUUID)
-                .setAttributes(Collections.<String, String>emptyMap(), flowFileRecord.getAttributes())
+                .setAttributes(Collections.emptyMap(), flowFileRecord.getAttributes())
                 .setCurrentContentClaim(event.getContentClaimContainer(), event.getContentClaimSection(), event.getContentClaimIdentifier(), event.getContentClaimOffset(), event.getFileSize())
                 .setDetails("Replay requested by " + user.getIdentity())
                 .setEventTime(System.currentTimeMillis())
@@ -4077,7 +4075,7 @@ public class FlowController implements EventAccess, ControllerServiceProvider, R
         final StandardRepositoryRecord record = new StandardRepositoryRecord(queue);
         record.setWorking(flowFileRecord);
         record.setDestination(queue);
-        flowFileRepository.updateRepository(Collections.<RepositoryRecord>singleton(record));
+        flowFileRepository.updateRepository(Collections.singleton(record));
 
         // Enqueue the data
         queue.put(flowFileRecord);

nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/main/java/org/apache/nifi/properties/ProtectedNiFiProperties.java

@@ -26,6 +26,7 @@ import java.util.List;
 import java.util.Map;
 import java.util.Properties;
 import java.util.Set;
+import java.util.stream.Collectors;
 import org.apache.commons.lang3.StringUtils;
 import org.apache.nifi.util.NiFiProperties;
 import org.slf4j.Logger;
@@ -50,7 +51,7 @@ class ProtectedNiFiProperties extends StandardNiFiProperties {
 
     // Default list of "sensitive" property keys
     public static final List<String> DEFAULT_SENSITIVE_PROPERTIES = new ArrayList<>(asList(SECURITY_KEY_PASSWD,
-            SECURITY_KEYSTORE_PASSWD, SECURITY_TRUSTSTORE_PASSWD, SENSITIVE_PROPS_KEY));
+            SECURITY_KEYSTORE_PASSWD, SECURITY_TRUSTSTORE_PASSWD, SENSITIVE_PROPS_KEY, PROVENANCE_REPO_ENCRYPTION_KEY));
 
     public ProtectedNiFiProperties() {
         this(new StandardNiFiProperties());
@@ -183,6 +184,17 @@ class ProtectedNiFiProperties extends StandardNiFiProperties {
         }
     }
 
+    /**
+     * Returns a list of the keys identifying "sensitive" properties. There is a default list,
+     * and additional keys can be provided in the {@code nifi.sensitive.props.additional.keys} property in {@code nifi.properties}.
+     *
+     * @return the list of sensitive property keys
+     */
+    public List<String> getPopulatedSensitivePropertyKeys() {
+        List<String> allSensitiveKeys = getSensitivePropertyKeys();
+        return allSensitiveKeys.stream().filter(k -> StringUtils.isNotBlank(getProperty(k))).collect(Collectors.toList());
+    }
+
     /**
      * Returns true if any sensitive keys are protected.
      *
@@ -219,7 +231,7 @@ class ProtectedNiFiProperties extends StandardNiFiProperties {
         Map<String, String> traditionalProtectedProperties = new HashMap<>();
         for (String key : sensitiveKeys) {
             String protection = getProperty(getProtectionKey(key));
-            if (!StringUtils.isBlank(protection)) {
+            if (StringUtils.isNotBlank(protection) && StringUtils.isNotBlank(getProperty(key))) {
                 traditionalProtectedProperties.put(key, protection);
             }
         }
@@ -237,12 +249,12 @@ class ProtectedNiFiProperties extends StandardNiFiProperties {
     }
 
     /**
-     * Returns a percentage of the total number of properties marked as sensitive that are currently protected.
+     * Returns a percentage of the total number of populated properties marked as sensitive that are currently protected.
      *
      * @return the percent of sensitive properties marked as protected
      */
     public int getPercentOfSensitivePropertiesProtected() {
-        return (int) Math.round(getProtectedPropertyKeys().size() / ((double) getSensitivePropertyKeys().size()) * 100);
+        return (int) Math.round(getProtectedPropertyKeys().size() / ((double) getPopulatedSensitivePropertyKeys().size()) * 100);
     }
 
     /**
@@ -421,9 +433,7 @@ class ProtectedNiFiProperties extends StandardNiFiProperties {
         // Add the protected keys and the protection schemes
         for (String key : getSensitivePropertyKeys()) {
             final String plainValue = getInternalNiFiProperties().getProperty(key);
-            if (plainValue == null || plainValue.trim().isEmpty()) {
-                protectedProperties.setProperty(key, plainValue);
-            } else {
+            if (plainValue != null && !plainValue.trim().isEmpty()) {
                 final String protectedValue = spp.protect(plainValue);
                 protectedProperties.setProperty(key, protectedValue);
                 protectedProperties.setProperty(getProtectionKey(key), protectionScheme);

nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/groovy/org/apache/nifi/properties/AESSensitivePropertyProviderTest.groovy

@@ -52,7 +52,7 @@ class AESSensitivePropertyProviderTest extends GroovyTestCase {
     private static final Base64.Decoder decoder = Base64.decoder
 
     @BeforeClass
-    public static void setUpOnce() throws Exception {
+    static void setUpOnce() throws Exception {
         Security.addProvider(new BouncyCastleProvider())
 
         logger.metaClass.methodMissing = { String name, args ->
@@ -61,12 +61,12 @@ class AESSensitivePropertyProviderTest extends GroovyTestCase {
     }
 
     @Before
-    public void setUp() throws Exception {
+    void setUp() throws Exception {
 
     }
 
     @After
-    public void tearDown() throws Exception {
+    void tearDown() throws Exception {
 
     }
 
@@ -112,7 +112,7 @@ class AESSensitivePropertyProviderTest extends GroovyTestCase {
     }
 
     @Test
-    public void testShouldThrowExceptionOnInitializationWithoutBouncyCastle() throws Exception {
+    void testShouldThrowExceptionOnInitializationWithoutBouncyCastle() throws Exception {
         // Arrange
         try {
             Security.removeProvider(new BouncyCastleProvider().getName())
@@ -133,7 +133,7 @@ class AESSensitivePropertyProviderTest extends GroovyTestCase {
     // TODO: testShouldGetName()
 
     @Test
-    public void testShouldProtectValue() throws Exception {
+    void testShouldProtectValue() throws Exception {
         final String PLAINTEXT = "This is a plaintext value"
 
         // Act
@@ -163,7 +163,7 @@ class AESSensitivePropertyProviderTest extends GroovyTestCase {
     }
 
     @Test
-    public void testShouldHandleProtectEmptyValue() throws Exception {
+    void testShouldHandleProtectEmptyValue() throws Exception {
         final List<String> EMPTY_PLAINTEXTS = ["", "    ", null]
 
         // Act
@@ -183,7 +183,7 @@ class AESSensitivePropertyProviderTest extends GroovyTestCase {
     }
 
     @Test
-    public void testShouldUnprotectValue() throws Exception {
+    void testShouldUnprotectValue() throws Exception {
         // Arrange
         final String PLAINTEXT = "This is a plaintext value"
 
@@ -218,7 +218,7 @@ class AESSensitivePropertyProviderTest extends GroovyTestCase {
      * @throws Exception
      */
     @Test
-    public void testShouldHandleUnprotectEmptyValue() throws Exception {
+    void testShouldHandleUnprotectEmptyValue() throws Exception {
         // Arrange
         final List<String> EMPTY_CIPHER_TEXTS = ["", "    ", null]
 
@@ -239,7 +239,7 @@ class AESSensitivePropertyProviderTest extends GroovyTestCase {
     }
 
     @Test
-    public void testShouldUnprotectValueWithWhitespace() throws Exception {
+    void testShouldUnprotectValueWithWhitespace() throws Exception {
         // Arrange
         final String PLAINTEXT = "This is a plaintext value"
 
@@ -269,7 +269,7 @@ class AESSensitivePropertyProviderTest extends GroovyTestCase {
     }
 
     @Test
-    public void testShouldHandleUnprotectMalformedValue() throws Exception {
+    void testShouldHandleUnprotectMalformedValue() throws Exception {
         // Arrange
         final String PLAINTEXT = "This is a plaintext value"
 
@@ -293,7 +293,7 @@ class AESSensitivePropertyProviderTest extends GroovyTestCase {
     }
 
     @Test
-    public void testShouldHandleUnprotectMissingIV() throws Exception {
+    void testShouldHandleUnprotectMissingIV() throws Exception {
         // Arrange
         final String PLAINTEXT = "This is a plaintext value"
 
@@ -334,7 +334,7 @@ class AESSensitivePropertyProviderTest extends GroovyTestCase {
      * @throws Exception
      */
     @Test
-    public void testShouldHandleUnprotectEmptyCipherText() throws Exception {
+    void testShouldHandleUnprotectEmptyCipherText() throws Exception {
         // Arrange
         final String IV_AND_DELIMITER = "${encoder.encodeToString("Bad IV value".getBytes(StandardCharsets.UTF_8))}||"
         logger.info("IV and delimiter: ${IV_AND_DELIMITER}")
@@ -358,7 +358,7 @@ class AESSensitivePropertyProviderTest extends GroovyTestCase {
     }
 
     @Test
-    public void testShouldHandleUnprotectMalformedIV() throws Exception {
+    void testShouldHandleUnprotectMalformedIV() throws Exception {
         // Arrange
         final String PLAINTEXT = "This is a plaintext value"
 
@@ -382,7 +382,7 @@ class AESSensitivePropertyProviderTest extends GroovyTestCase {
     }
 
     @Test
-    public void testShouldGetIdentifierKeyWithDifferentMaxKeyLengths() throws Exception {
+    void testShouldGetIdentifierKeyWithDifferentMaxKeyLengths() throws Exception {
         // Arrange
         def keys = getAvailableKeySizes().collectEntries { int keySize ->
             [(keySize): getKeyOfSize(keySize)]
@@ -400,7 +400,7 @@ class AESSensitivePropertyProviderTest extends GroovyTestCase {
     }
 
     @Test
-    public void testShouldNotAllowEmptyKey() throws Exception {
+    void testShouldNotAllowEmptyKey() throws Exception {
         // Arrange
         final String INVALID_KEY = ""
 
@@ -414,7 +414,7 @@ class AESSensitivePropertyProviderTest extends GroovyTestCase {
     }
 
     @Test
-    public void testShouldNotAllowIncorrectlySizedKey() throws Exception {
+    void testShouldNotAllowIncorrectlySizedKey() throws Exception {
         // Arrange
         final String INVALID_KEY = "Z" * 31
 
@@ -428,7 +428,7 @@ class AESSensitivePropertyProviderTest extends GroovyTestCase {
     }
 
     @Test
-    public void testShouldNotAllowInvalidKey() throws Exception {
+    void testShouldNotAllowInvalidKey() throws Exception {
         // Arrange
         final String INVALID_KEY = "Z" * 32
 
@@ -445,7 +445,7 @@ class AESSensitivePropertyProviderTest extends GroovyTestCase {
      * This test is to ensure internal consistency and allow for encrypting value for various property files
      */
     @Test
-    public void testShouldEncryptArbitraryValues() {
+    void testShouldEncryptArbitraryValues() {
         // Arrange
         def values = ["thisIsABadPassword", "thisIsABadSensitiveKeyPassword", "thisIsABadKeystorePassword", "thisIsABadKeyPassword", "thisIsABadTruststorePassword", "This is an encrypted banner message", "nififtw!"]
 
@@ -471,15 +471,15 @@ class AESSensitivePropertyProviderTest extends GroovyTestCase {
      * This test is to ensure external compatibility in case someone encodes the encrypted value with Base64 and does not remove the padding
      */
     @Test
-    public void testShouldDecryptPaddedValue() {
+    void testShouldDecryptPaddedValue() {
         // Arrange
         Assume.assumeTrue("JCE unlimited strength crypto policy must be installed for this test", Cipher.getMaxAllowedKeyLength("AES") > 128)
 
-        final String EXPECTED_VALUE = "thisIsABadKeyPassword"
-        String cipherText = "ac/BaE35SL/esLiJ||+ULRvRLYdIDA2VqpE0eQXDEMjaLBMG2kbKOdOwBk/hGebDKlVg=="
+        final String EXPECTED_VALUE = getKeyOfSize(256) // "thisIsABadKeyPassword"
+        String cipherText = "aYDkDKys1ENr3gp+||sTBPpMlIvHcOLTGZlfWct8r9RY8BuDlDkoaYmGJ/9m9af9tZIVzcnDwvYQAaIKxRGF7vI2yrY7Xd6x9GTDnWGiGiRXlaP458BBMMgfzH2O8"
         String unpaddedCipherText = cipherText.replaceAll("=", "")
 
-        String key = getKeyOfSize(256)
+        String key = "AAAABBBBCCCCDDDDEEEEFFFF00001111" * 2 // getKeyOfSize(256)
 
         SensitivePropertyProvider spp = new AESSensitivePropertyProvider(key)
 

nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/groovy/org/apache/nifi/properties/ProtectedNiFiPropertiesGroovyTest.groovy

@@ -38,7 +38,8 @@ class ProtectedNiFiPropertiesGroovyTest extends GroovyTestCase {
             "nifi.sensitive.props.key",
             "nifi.security.keystorePasswd",
             "nifi.security.keyPasswd",
-            "nifi.security.truststorePasswd"
+            "nifi.security.truststorePasswd",
+            "nifi.provenance.repository.encryption.key"
     ]
 
     final def COMMON_ADDITIONAL_SENSITIVE_PROPERTIES = [
@@ -53,7 +54,7 @@ class ProtectedNiFiPropertiesGroovyTest extends GroovyTestCase {
     private static String originalPropertiesPath = System.getProperty(NiFiProperties.PROPERTIES_FILE_PATH)
 
     @BeforeClass
-    public static void setUpOnce() throws Exception {
+    static void setUpOnce() throws Exception {
         Security.addProvider(new BouncyCastleProvider())
 
         logger.metaClass.methodMissing = { String name, args ->
@@ -62,15 +63,15 @@ class ProtectedNiFiPropertiesGroovyTest extends GroovyTestCase {
     }
 
     @Before
-    public void setUp() throws Exception {
+    void setUp() throws Exception {
     }
 
     @After
-    public void tearDown() throws Exception {
+    void tearDown() throws Exception {
     }
 
     @AfterClass
-    public static void tearDownOnce() {
+    static void tearDownOnce() {
         if (originalPropertiesPath) {
             System.setProperty(NiFiProperties.PROPERTIES_FILE_PATH, originalPropertiesPath)
         }
@@ -127,7 +128,7 @@ class ProtectedNiFiPropertiesGroovyTest extends GroovyTestCase {
     }
 
     @Test
-    public void testConstructorShouldCreateNewInstance() throws Exception {
+    void testConstructorShouldCreateNewInstance() throws Exception {
         // Arrange
 
         // Act
@@ -140,7 +141,7 @@ class ProtectedNiFiPropertiesGroovyTest extends GroovyTestCase {
     }
 
     @Test
-    public void testConstructorShouldAcceptRawProperties() throws Exception {
+    void testConstructorShouldAcceptRawProperties() throws Exception {
         // Arrange
         Properties rawProperties = new Properties()
         rawProperties.setProperty("key", "value")
@@ -157,7 +158,7 @@ class ProtectedNiFiPropertiesGroovyTest extends GroovyTestCase {
     }
 
     @Test
-    public void testConstructorShouldAcceptNiFiProperties() throws Exception {
+    void testConstructorShouldAcceptNiFiProperties() throws Exception {
         // Arrange
         Properties rawProperties = new Properties()
         rawProperties.setProperty("key", "value")
@@ -178,7 +179,7 @@ class ProtectedNiFiPropertiesGroovyTest extends GroovyTestCase {
     }
 
     @Test
-    public void testShouldAllowMultipleInstances() throws Exception {
+    void testShouldAllowMultipleInstances() throws Exception {
         // Arrange
         Properties rawProperties = new Properties()
         rawProperties.setProperty("key", "value")
@@ -200,7 +201,7 @@ class ProtectedNiFiPropertiesGroovyTest extends GroovyTestCase {
     }
 
     @Test
-    public void testShouldDetectIfPropertyIsSensitive() throws Exception {
+    void testShouldDetectIfPropertyIsSensitive() throws Exception {
         // Arrange
         final String INSENSITIVE_PROPERTY_KEY = "nifi.ui.banner.text"
         final String SENSITIVE_PROPERTY_KEY = "nifi.security.keystorePasswd"
@@ -219,7 +220,7 @@ class ProtectedNiFiPropertiesGroovyTest extends GroovyTestCase {
     }
 
     @Test
-    public void testShouldGetDefaultSensitiveProperties() throws Exception {
+    void testShouldGetDefaultSensitiveProperties() throws Exception {
         // Arrange
         logger.expected("${DEFAULT_SENSITIVE_PROPERTIES.size()} default sensitive properties: ${DEFAULT_SENSITIVE_PROPERTIES.join(", ")}")
 
@@ -235,9 +236,9 @@ class ProtectedNiFiPropertiesGroovyTest extends GroovyTestCase {
     }
 
     @Test
-    public void testShouldGetAdditionalSensitiveProperties() throws Exception {
+    void testShouldGetAdditionalSensitiveProperties() throws Exception {
         // Arrange
-        def completeSensitiveProperties = DEFAULT_SENSITIVE_PROPERTIES + ["nifi.ui.banner.text"]
+        def completeSensitiveProperties = DEFAULT_SENSITIVE_PROPERTIES + ["nifi.ui.banner.text", "nifi.version"]
         logger.expected("${completeSensitiveProperties.size()} total sensitive properties: ${completeSensitiveProperties.join(", ")}")
 
         ProtectedNiFiProperties properties = loadFromFile("/conf/nifi_with_additional_sensitive_keys.properties")
@@ -254,7 +255,7 @@ class ProtectedNiFiPropertiesGroovyTest extends GroovyTestCase {
     // TODO: Add negative tests (fuzz additional.keys property, etc.)
 
     @Test
-    public void testGetAdditionalSensitivePropertiesShouldNotIncludeSelf() throws Exception {
+    void testGetAdditionalSensitivePropertiesShouldNotIncludeSelf() throws Exception {
         // Arrange
         def completeSensitiveProperties = DEFAULT_SENSITIVE_PROPERTIES + ["nifi.ui.banner.text", "nifi.version"]
         logger.expected("${completeSensitiveProperties.size()} total sensitive properties: ${completeSensitiveProperties.join(", ")}")
@@ -275,7 +276,7 @@ class ProtectedNiFiPropertiesGroovyTest extends GroovyTestCase {
      * @throws Exception
      */
     @Test
-    public void testShouldGetUnprotectedValueOfSensitiveProperty() throws Exception {
+    void testShouldGetUnprotectedValueOfSensitiveProperty() throws Exception {
         // Arrange
         final String KEYSTORE_PASSWORD_KEY = "nifi.security.keystorePasswd"
         final String EXPECTED_KEYSTORE_PASSWORD = "thisIsABadKeystorePassword"
@@ -301,7 +302,7 @@ class ProtectedNiFiPropertiesGroovyTest extends GroovyTestCase {
      * @throws Exception
      */
     @Test
-    public void testShouldGetEmptyUnprotectedValueOfSensitiveProperty() throws Exception {
+    void testShouldGetEmptyUnprotectedValueOfSensitiveProperty() throws Exception {
         // Arrange
         final String TRUSTSTORE_PASSWORD_KEY = "nifi.security.truststorePasswd"
         final String EXPECTED_TRUSTSTORE_PASSWORD = ""
@@ -329,7 +330,7 @@ class ProtectedNiFiPropertiesGroovyTest extends GroovyTestCase {
      * @throws Exception
      */
     @Test
-    public void testShouldGetUnprotectedValueOfSensitivePropertyWhenProtected() throws Exception {
+    void testShouldGetUnprotectedValueOfSensitivePropertyWhenProtected() throws Exception {
         // Arrange
         final String KEYSTORE_PASSWORD_KEY = "nifi.security.keystorePasswd"
         final String EXPECTED_KEYSTORE_PASSWORD = "thisIsABadKeystorePassword"
@@ -356,7 +357,7 @@ class ProtectedNiFiPropertiesGroovyTest extends GroovyTestCase {
      * @throws Exception
      */
     @Test
-    public void testGetValueOfSensitivePropertyShouldHandleUnknownProtectionScheme() throws Exception {
+    void testGetValueOfSensitivePropertyShouldHandleUnknownProtectionScheme() throws Exception {
         // Arrange
         final String KEYSTORE_PASSWORD_KEY = "nifi.security.keystorePasswd"
 
@@ -390,7 +391,7 @@ class ProtectedNiFiPropertiesGroovyTest extends GroovyTestCase {
      * @throws Exception
      */
     @Test
-    public void testGetValueOfSensitivePropertyShouldHandleSingleMalformedValue() throws Exception {
+    void testGetValueOfSensitivePropertyShouldHandleSingleMalformedValue() throws Exception {
         // Arrange
         final String KEYSTORE_PASSWORD_KEY = "nifi.security.keystorePasswd"
 
@@ -425,7 +426,7 @@ class ProtectedNiFiPropertiesGroovyTest extends GroovyTestCase {
      * @throws Exception
      */
     @Test
-    public void testGetValueOfSensitivePropertyShouldHandleMultipleMalformedValues() throws Exception {
+    void testGetValueOfSensitivePropertyShouldHandleMultipleMalformedValues() throws Exception {
         // Arrange
 
         // Raw properties
@@ -468,7 +469,7 @@ class ProtectedNiFiPropertiesGroovyTest extends GroovyTestCase {
      * @throws Exception
      */
     @Test
-    public void testShouldGetEmptyUnprotectedValueOfSensitivePropertyWithDefault() throws Exception {
+    void testShouldGetEmptyUnprotectedValueOfSensitivePropertyWithDefault() throws Exception {
         // Arrange
         final String TRUSTSTORE_PASSWORD_KEY = "nifi.security.truststorePasswd"
         final String EXPECTED_TRUSTSTORE_PASSWORD = ""
@@ -502,7 +503,7 @@ class ProtectedNiFiPropertiesGroovyTest extends GroovyTestCase {
      * @throws Exception
      */
     @Test
-    public void testShouldGetUnprotectedValueOfSensitivePropertyWhenProtectedWithDefault() throws Exception {
+    void testShouldGetUnprotectedValueOfSensitivePropertyWhenProtectedWithDefault() throws Exception {
         // Arrange
         final String KEYSTORE_PASSWORD_KEY = "nifi.security.keystorePasswd"
         final String EXPECTED_KEYSTORE_PASSWORD = "thisIsABadKeystorePassword"
@@ -538,7 +539,7 @@ class ProtectedNiFiPropertiesGroovyTest extends GroovyTestCase {
      * @throws Exception
      */
     @Test
-    public void testGetValueOfSensitivePropertyShouldHandleInvalidatedInternalCache() throws Exception {
+    void testGetValueOfSensitivePropertyShouldHandleInvalidatedInternalCache() throws Exception {
         // Arrange
         final String KEYSTORE_PASSWORD_KEY = "nifi.security.keystorePasswd"
         final String EXPECTED_KEYSTORE_PASSWORD = "thisIsABadKeystorePassword"
@@ -567,7 +568,7 @@ class ProtectedNiFiPropertiesGroovyTest extends GroovyTestCase {
     }
 
     @Test
-    public void testShouldDetectIfPropertyIsProtected() throws Exception {
+    void testShouldDetectIfPropertyIsProtected() throws Exception {
         // Arrange
         final String UNPROTECTED_PROPERTY_KEY = "nifi.security.truststorePasswd"
         final String PROTECTED_PROPERTY_KEY = "nifi.security.keystorePasswd"
@@ -593,7 +594,7 @@ class ProtectedNiFiPropertiesGroovyTest extends GroovyTestCase {
     }
 
     @Test
-    public void testShouldDetectIfPropertyWithEmptyProtectionSchemeIsProtected() throws Exception {
+    void testShouldDetectIfPropertyWithEmptyProtectionSchemeIsProtected() throws Exception {
         // Arrange
         final String UNPROTECTED_PROPERTY_KEY = "nifi.sensitive.props.key"
 
@@ -611,7 +612,7 @@ class ProtectedNiFiPropertiesGroovyTest extends GroovyTestCase {
     }
 
     @Test
-    public void testShouldGetPercentageOfSensitivePropertiesProtected_0() throws Exception {
+    void testShouldGetPercentageOfSensitivePropertiesProtected_0() throws Exception {
         // Arrange
         ProtectedNiFiProperties properties = loadFromFile("/conf/nifi.properties")
 
@@ -620,14 +621,14 @@ class ProtectedNiFiPropertiesGroovyTest extends GroovyTestCase {
 
         // Act
         double percentProtected = properties.getPercentOfSensitivePropertiesProtected()
-        logger.info("${percentProtected}% (${properties.getProtectedPropertyKeys().size()} of ${properties.getSensitivePropertyKeys().size()}) protected")
+        logger.info("${percentProtected}% (${properties.getProtectedPropertyKeys().size()} of ${properties.getPopulatedSensitivePropertyKeys().size()}) protected")
 
         // Assert
         assert percentProtected == 0.0
     }
 
     @Test
-    public void testShouldGetPercentageOfSensitivePropertiesProtected_50() throws Exception {
+    void testShouldGetPercentageOfSensitivePropertiesProtected_75() throws Exception {
         // Arrange
         ProtectedNiFiProperties properties = loadFromFile("/conf/nifi_with_sensitive_properties_protected_aes.properties")
 
@@ -636,14 +637,14 @@ class ProtectedNiFiPropertiesGroovyTest extends GroovyTestCase {
 
         // Act
         double percentProtected = properties.getPercentOfSensitivePropertiesProtected()
-        logger.info("${percentProtected}% (${properties.getProtectedPropertyKeys().size()} of ${properties.getSensitivePropertyKeys().size()}) protected")
+        logger.info("${percentProtected}% (${properties.getProtectedPropertyKeys().size()} of ${properties.getPopulatedSensitivePropertyKeys().size()}) protected")
 
         // Assert
-        assert percentProtected == 50.0
+        assert percentProtected == 75.0
     }
 
     @Test
-    public void testShouldGetPercentageOfSensitivePropertiesProtected_100() throws Exception {
+    void testShouldGetPercentageOfSensitivePropertiesProtected_100() throws Exception {
         // Arrange
         ProtectedNiFiProperties properties = loadFromFile("/conf/nifi_with_all_sensitive_properties_protected_aes.properties")
 
@@ -652,14 +653,14 @@ class ProtectedNiFiPropertiesGroovyTest extends GroovyTestCase {
 
         // Act
         double percentProtected = properties.getPercentOfSensitivePropertiesProtected()
-        logger.info("${percentProtected}% (${properties.getProtectedPropertyKeys().size()} of ${properties.getSensitivePropertyKeys().size()}) protected")
+        logger.info("${percentProtected}% (${properties.getProtectedPropertyKeys().size()} of ${properties.getPopulatedSensitivePropertyKeys().size()}) protected")
 
         // Assert
         assert percentProtected == 100.0
     }
 
     @Test
-    public void testInstanceWithNoProtectedPropertiesShouldNotLoadSPP() throws Exception {
+    void testInstanceWithNoProtectedPropertiesShouldNotLoadSPP() throws Exception {
         // Arrange
         ProtectedNiFiProperties properties = loadFromFile("/conf/nifi.properties")
         assert properties.@localProviderCache?.isEmpty()
@@ -676,7 +677,7 @@ class ProtectedNiFiPropertiesGroovyTest extends GroovyTestCase {
     }
 
     @Test
-    public void testShouldAddSensitivePropertyProvider() throws Exception {
+    void testShouldAddSensitivePropertyProvider() throws Exception {
         // Arrange
         ProtectedNiFiProperties properties = new ProtectedNiFiProperties()
         assert properties.getSensitivePropertyProviders().isEmpty()
@@ -696,7 +697,7 @@ class ProtectedNiFiPropertiesGroovyTest extends GroovyTestCase {
     }
 
     @Test
-    public void testShouldNotAddNullSensitivePropertyProvider() throws Exception {
+    void testShouldNotAddNullSensitivePropertyProvider() throws Exception {
         // Arrange
         ProtectedNiFiProperties properties = new ProtectedNiFiProperties()
         assert properties.getSensitivePropertyProviders().isEmpty()
@@ -713,7 +714,7 @@ class ProtectedNiFiPropertiesGroovyTest extends GroovyTestCase {
     }
 
     @Test
-    public void testShouldNotAllowOverwriteOfProvider() throws Exception {
+    void testShouldNotAllowOverwriteOfProvider() throws Exception {
         // Arrange
         ProtectedNiFiProperties properties = new ProtectedNiFiProperties()
         assert properties.getSensitivePropertyProviders().isEmpty()

nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/groovy/org/apache/nifi/properties/StandardNiFiPropertiesGroovyTest.groovy

@@ -32,58 +32,60 @@ class StandardNiFiPropertiesGroovyTest extends GroovyTestCase {
     private static final Logger logger = LoggerFactory.getLogger(StandardNiFiPropertiesGroovyTest.class)
 
     private static String originalPropertiesPath = System.getProperty(NiFiProperties.PROPERTIES_FILE_PATH)
+    private static final String PREK = NiFiProperties.PROVENANCE_REPO_ENCRYPTION_KEY
+    private static final String PREKID = NiFiProperties.PROVENANCE_REPO_ENCRYPTION_KEY_ID
 
     @BeforeClass
-    public static void setUpOnce() throws Exception {
+    static void setUpOnce() throws Exception {
         logger.metaClass.methodMissing = { String name, args ->
             logger.info("[${name?.toUpperCase()}] ${(args as List).join(" ")}")
         }
     }
 
     @Before
-    public void setUp() throws Exception {
+    void setUp() throws Exception {
     }
 
     @After
-    public void tearDown() throws Exception {
+    void tearDown() throws Exception {
     }
 
     @AfterClass
-    public static void tearDownOnce() {
+    static void tearDownOnce() {
         if (originalPropertiesPath) {
             System.setProperty(NiFiProperties.PROPERTIES_FILE_PATH, originalPropertiesPath)
         }
     }
 
     private static StandardNiFiProperties loadFromFile(String propertiesFilePath) {
-        String filePath;
+        String filePath
         try {
-            filePath = StandardNiFiPropertiesGroovyTest.class.getResource(propertiesFilePath).toURI().getPath();
+            filePath = StandardNiFiPropertiesGroovyTest.class.getResource(propertiesFilePath).toURI().getPath()
         } catch (URISyntaxException ex) {
             throw new RuntimeException("Cannot load properties file due to "
-                    + ex.getLocalizedMessage(), ex);
+                    + ex.getLocalizedMessage(), ex)
         }
 
-        System.setProperty(NiFiProperties.PROPERTIES_FILE_PATH, filePath);
+        System.setProperty(NiFiProperties.PROPERTIES_FILE_PATH, filePath)
 
-        StandardNiFiProperties properties = new StandardNiFiProperties();
+        StandardNiFiProperties properties = new StandardNiFiProperties()
 
         // clear out existing properties
         for (String prop : properties.stringPropertyNames()) {
-            properties.remove(prop);
+            properties.remove(prop)
         }
 
-        InputStream inStream = null;
+        InputStream inStream = null
         try {
-            inStream = new BufferedInputStream(new FileInputStream(filePath));
-            properties.load(inStream);
+            inStream = new BufferedInputStream(new FileInputStream(filePath))
+            properties.load(inStream)
         } catch (final Exception ex) {
             throw new RuntimeException("Cannot load properties file due to "
-                    + ex.getLocalizedMessage(), ex);
+                    + ex.getLocalizedMessage(), ex)
         } finally {
             if (null != inStream) {
                 try {
-                    inStream.close();
+                    inStream.close()
                 } catch (Exception ex) {
                     /**
                      * do nothing *
@@ -92,11 +94,11 @@ class StandardNiFiPropertiesGroovyTest extends GroovyTestCase {
             }
         }
 
-        return properties;
+        return properties
     }
 
     @Test
-    public void testConstructorShouldCreateNewInstance() throws Exception {
+    void testConstructorShouldCreateNewInstance() throws Exception {
         // Arrange
 
         // Act
@@ -109,7 +111,7 @@ class StandardNiFiPropertiesGroovyTest extends GroovyTestCase {
     }
 
     @Test
-    public void testConstructorShouldAcceptRawProperties() throws Exception {
+    void testConstructorShouldAcceptRawProperties() throws Exception {
         // Arrange
         Properties rawProperties = new Properties()
         rawProperties.setProperty("key", "value")
@@ -126,7 +128,7 @@ class StandardNiFiPropertiesGroovyTest extends GroovyTestCase {
     }
 
     @Test
-    public void testShouldAllowMultipleInstances() throws Exception {
+    void testShouldAllowMultipleInstances() throws Exception {
         // Arrange
         Properties rawProperties = new Properties()
         rawProperties.setProperty("key", "value")
@@ -139,7 +141,6 @@ class StandardNiFiPropertiesGroovyTest extends GroovyTestCase {
         NiFiProperties emptyProperties = new StandardNiFiProperties()
         logger.info("emptyProperties has ${emptyProperties.size()} properties: ${emptyProperties.getPropertyKeys()}")
 
-
         // Assert
         assert niFiProperties.size() == 1
         assert niFiProperties.getPropertyKeys() == ["key"] as Set
@@ -147,4 +148,178 @@ class StandardNiFiPropertiesGroovyTest extends GroovyTestCase {
         assert emptyProperties.size() == 0
         assert emptyProperties.getPropertyKeys() == [] as Set
     }
+
+    @Test
+    void testShouldGetProvenanceRepoEncryptionKeyFromDefaultProperty() throws Exception {
+        // Arrange
+        Properties rawProperties = new Properties()
+        final String KEY_ID = "arbitraryKeyId"
+        final String KEY_HEX = "0123456789ABCDEFFEDCBA9876543210"
+        rawProperties.setProperty(PREKID, KEY_ID)
+        rawProperties.setProperty(PREK, KEY_HEX)
+        NiFiProperties niFiProperties = new StandardNiFiProperties(rawProperties)
+        logger.info("niFiProperties has ${niFiProperties.size()} properties: ${niFiProperties.getPropertyKeys()}")
+
+        // Act
+        def keyId = niFiProperties.getProvenanceRepoEncryptionKeyId()
+        def key = niFiProperties.getProvenanceRepoEncryptionKey()
+        def keys = niFiProperties.getProvenanceRepoEncryptionKeys()
+
+        logger.info("Retrieved key ID: ${keyId}")
+        logger.info("Retrieved key: ${key}")
+        logger.info("Retrieved keys: ${keys}")
+
+        // Assert
+        assert keyId == KEY_ID
+        assert key == KEY_HEX
+        assert keys == [(KEY_ID): KEY_HEX]
+    }
+
+    @Test
+    void testShouldGetProvenanceRepoEncryptionKeysFromMultipleProperties() throws Exception {
+        // Arrange
+        Properties rawProperties = new Properties()
+        final String KEY_ID = "arbitraryKeyId"
+        final String KEY_HEX = "0123456789ABCDEFFEDCBA9876543210"
+        final String KEY_ID_2 = "arbitraryKeyId2"
+        final String KEY_HEX_2 = "AAAABBBBCCCCDDDDEEEEFFFF00001111"
+        final String KEY_ID_3 = "arbitraryKeyId3"
+        final String KEY_HEX_3 = "01010101010101010101010101010101"
+
+        rawProperties.setProperty(PREKID, KEY_ID)
+        rawProperties.setProperty(PREK, KEY_HEX)
+        rawProperties.setProperty("${PREK}.id.${KEY_ID_2}", KEY_HEX_2)
+        rawProperties.setProperty("${PREK}.id.${KEY_ID_3}", KEY_HEX_3)
+        NiFiProperties niFiProperties = new StandardNiFiProperties(rawProperties)
+        logger.info("niFiProperties has ${niFiProperties.size()} properties: ${niFiProperties.getPropertyKeys()}")
+
+        // Act
+        def keyId = niFiProperties.getProvenanceRepoEncryptionKeyId()
+        def key = niFiProperties.getProvenanceRepoEncryptionKey()
+        def keys = niFiProperties.getProvenanceRepoEncryptionKeys()
+
+        logger.info("Retrieved key ID: ${keyId}")
+        logger.info("Retrieved key: ${key}")
+        logger.info("Retrieved keys: ${keys}")
+
+        // Assert
+        assert keyId == KEY_ID
+        assert key == KEY_HEX
+        assert keys == [(KEY_ID): KEY_HEX, (KEY_ID_2): KEY_HEX_2, (KEY_ID_3): KEY_HEX_3]
+    }
+
+    @Test
+    void testShouldGetProvenanceRepoEncryptionKeysWithNoDefaultDefined() throws Exception {
+        // Arrange
+        Properties rawProperties = new Properties()
+        final String KEY_ID = "arbitraryKeyId"
+        final String KEY_HEX = "0123456789ABCDEFFEDCBA9876543210"
+        final String KEY_ID_2 = "arbitraryKeyId2"
+        final String KEY_HEX_2 = "AAAABBBBCCCCDDDDEEEEFFFF00001111"
+        final String KEY_ID_3 = "arbitraryKeyId3"
+        final String KEY_HEX_3 = "01010101010101010101010101010101"
+
+        rawProperties.setProperty(PREKID, KEY_ID)
+        rawProperties.setProperty("${PREK}.id.${KEY_ID}", KEY_HEX)
+        rawProperties.setProperty("${PREK}.id.${KEY_ID_2}", KEY_HEX_2)
+        rawProperties.setProperty("${PREK}.id.${KEY_ID_3}", KEY_HEX_3)
+        NiFiProperties niFiProperties = new StandardNiFiProperties(rawProperties)
+        logger.info("niFiProperties has ${niFiProperties.size()} properties: ${niFiProperties.getPropertyKeys()}")
+
+        // Act
+        def keyId = niFiProperties.getProvenanceRepoEncryptionKeyId()
+        def key = niFiProperties.getProvenanceRepoEncryptionKey()
+        def keys = niFiProperties.getProvenanceRepoEncryptionKeys()
+
+        logger.info("Retrieved key ID: ${keyId}")
+        logger.info("Retrieved key: ${key}")
+        logger.info("Retrieved keys: ${keys}")
+
+        // Assert
+        assert keyId == KEY_ID
+        assert key == KEY_HEX
+        assert keys == [(KEY_ID): KEY_HEX, (KEY_ID_2): KEY_HEX_2, (KEY_ID_3): KEY_HEX_3]
+    }
+
+    @Test
+    void testShouldGetProvenanceRepoEncryptionKeysWithNoneDefined() throws Exception {
+        // Arrange
+        Properties rawProperties = new Properties()
+        NiFiProperties niFiProperties = new StandardNiFiProperties(rawProperties)
+        logger.info("niFiProperties has ${niFiProperties.size()} properties: ${niFiProperties.getPropertyKeys()}")
+
+        // Act
+        def keyId = niFiProperties.getProvenanceRepoEncryptionKeyId()
+        def key = niFiProperties.getProvenanceRepoEncryptionKey()
+        def keys = niFiProperties.getProvenanceRepoEncryptionKeys()
+
+        logger.info("Retrieved key ID: ${keyId}")
+        logger.info("Retrieved key: ${key}")
+        logger.info("Retrieved keys: ${keys}")
+
+        // Assert
+        assert keyId == null
+        assert key == null
+        assert keys == [:]
+    }
+
+    @Test
+    void testShouldNotGetProvenanceRepoEncryptionKeysIfFileBasedKeyProvider() throws Exception {
+        // Arrange
+        Properties rawProperties = new Properties()
+        final String KEY_ID = "arbitraryKeyId"
+
+        rawProperties.setProperty(PREKID, KEY_ID)
+        NiFiProperties niFiProperties = new StandardNiFiProperties(rawProperties)
+        logger.info("niFiProperties has ${niFiProperties.size()} properties: ${niFiProperties.getPropertyKeys()}")
+
+        // Act
+        def keyId = niFiProperties.getProvenanceRepoEncryptionKeyId()
+        def key = niFiProperties.getProvenanceRepoEncryptionKey()
+        def keys = niFiProperties.getProvenanceRepoEncryptionKeys()
+
+        logger.info("Retrieved key ID: ${keyId}")
+        logger.info("Retrieved key: ${key}")
+        logger.info("Retrieved keys: ${keys}")
+
+        // Assert
+        assert keyId == KEY_ID
+        assert key == null
+        assert keys == [:]
+    }
+
+    @Test
+    void testGetProvenanceRepoEncryptionKeysShouldFilterOtherProperties() throws Exception {
+        // Arrange
+        Properties rawProperties = new Properties()
+        final String KEY_ID = "arbitraryKeyId"
+        final String KEY_HEX = "0123456789ABCDEFFEDCBA9876543210"
+        final String KEY_ID_2 = "arbitraryKeyId2"
+        final String KEY_HEX_2 = "AAAABBBBCCCCDDDDEEEEFFFF00001111"
+        final String KEY_ID_3 = "arbitraryKeyId3"
+        final String KEY_HEX_3 = "01010101010101010101010101010101"
+
+        rawProperties.setProperty(PREKID, KEY_ID)
+        rawProperties.setProperty("${PREK}.id.${KEY_ID}", KEY_HEX)
+        rawProperties.setProperty("${PREK}.id.${KEY_ID_2}", KEY_HEX_2)
+        rawProperties.setProperty("${PREK}.id.${KEY_ID_3}", KEY_HEX_3)
+        rawProperties.setProperty(NiFiProperties.PROVENANCE_REPO_ENCRYPTION_KEY_PROVIDER_IMPLEMENTATION_CLASS, "some.class.provider")
+        rawProperties.setProperty(NiFiProperties.PROVENANCE_REPO_ENCRYPTION_KEY_PROVIDER_LOCATION, "some://url")
+        NiFiProperties niFiProperties = new StandardNiFiProperties(rawProperties)
+        logger.info("niFiProperties has ${niFiProperties.size()} properties: ${niFiProperties.getPropertyKeys()}")
+
+        // Act
+        def keyId = niFiProperties.getProvenanceRepoEncryptionKeyId()
+        def key = niFiProperties.getProvenanceRepoEncryptionKey()
+        def keys = niFiProperties.getProvenanceRepoEncryptionKeys()
+
+        logger.info("Retrieved key ID: ${keyId}")
+        logger.info("Retrieved key: ${key}")
+        logger.info("Retrieved keys: ${keys}")
+
+        // Assert
+        assert keyId == KEY_ID
+        assert key == KEY_HEX
+        assert keys == [(KEY_ID): KEY_HEX, (KEY_ID_2): KEY_HEX_2, (KEY_ID_3): KEY_HEX_3]
+    }
 }

nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/resources/conf/nifi_with_additional_sensitive_keys.properties

@@ -73,7 +73,7 @@ nifi.web.jetty.working.directory=./target/work/jetty
 nifi.sensitive.props.key=key
 nifi.sensitive.props.algorithm=PBEWITHMD5AND256BITAES-CBC-OPENSSL
 nifi.sensitive.props.provider=BC
-nifi.sensitive.props.additional.keys=nifi.ui.banner.text
+nifi.sensitive.props.additional.keys=nifi.ui.banner.text, nifi.version, nifi.sensitive.props.additional.keys
 
 nifi.security.keystore=
 nifi.security.keystoreType=

nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-resources/pom.xml

@@ -91,6 +91,11 @@
 
         <!-- persistent provenance repository properties -->
         <nifi.provenance.repository.implementation>org.apache.nifi.provenance.PersistentProvenanceRepository</nifi.provenance.repository.implementation>
+        <nifi.provenance.repository.debug.frequency>1_000_000</nifi.provenance.repository.debug.frequency>
+        <nifi.provenance.repository.encryption.key.provider.implementation/>
+        <nifi.provenance.repository.encryption.key.provider.location/>
+        <nifi.provenance.repository.encryption.key.id/>
+        <nifi.provenance.repository.encryption.key/>
         <nifi.provenance.repository.directory.default>./provenance_repository</nifi.provenance.repository.directory.default>
         <nifi.provenance.repository.max.storage.time>24 hours</nifi.provenance.repository.max.storage.time>
         <nifi.provenance.repository.max.storage.size>1 GB</nifi.provenance.repository.max.storage.size>

nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-resources/src/main/resources/conf/nifi.properties

@@ -81,6 +81,11 @@ nifi.content.viewer.url=${nifi.content.viewer.url}
 
 # Provenance Repository Properties
 nifi.provenance.repository.implementation=${nifi.provenance.repository.implementation}
+nifi.provenance.repository.debug.frequency=${nifi.provenance.repository.debug.frequency}
+nifi.provenance.repository.encryption.key.provider.implementation=${nifi.provenance.repository.encryption.key.provider.implementation}
+nifi.provenance.repository.encryption.key.provider.location=${nifi.provenance.repository.encryption.key.provider.location}
+nifi.provenance.repository.encryption.key.id=${nifi.provenance.repository.encryption.key.id}
+nifi.provenance.repository.encryption.key=${nifi.provenance.repository.encryption.key}
 
 # Persistent Provenance Repository Properties
 nifi.provenance.repository.directory.default=${nifi.provenance.repository.directory.default}

nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-processors/src/test/resources/logback-test.xml

@@ -0,0 +1,32 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+  Licensed to the Apache Software Foundation (ASF) under one or more
+  contributor license agreements.  See the NOTICE file distributed with
+  this work for additional information regarding copyright ownership.
+  The ASF licenses this file to You under the Apache License, Version 2.0
+  (the "License"); you may not use this file except in compliance with
+  the License.  You may obtain a copy of the License at
+      http://www.apache.org/licenses/LICENSE-2.0
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+-->
+
+<configuration scan="true" scanPeriod="30 seconds">
+    <appender name="CONSOLE" class="ch.qos.logback.core.ConsoleAppender">
+        <encoder class="ch.qos.logback.classic.encoder.PatternLayoutEncoder">
+            <pattern>%-4r [%t] %-5p %c - %m%n</pattern>
+        </encoder>
+    </appender>
+
+    <!-- valid logging levels: TRACE, DEBUG, INFO, WARN, ERROR -->
+    <logger name="org.apache.nifi" level="INFO"/>
+
+    <root level="INFO">
+        <appender-ref ref="CONSOLE"/>
+    </root>
+
+</configuration>
+

nifi-nar-bundles/nifi-poi-bundle/nifi-poi-processors/src/test/resources/logback-test.xml

@@ -0,0 +1,32 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+  Licensed to the Apache Software Foundation (ASF) under one or more
+  contributor license agreements.  See the NOTICE file distributed with
+  this work for additional information regarding copyright ownership.
+  The ASF licenses this file to You under the Apache License, Version 2.0
+  (the "License"); you may not use this file except in compliance with
+  the License.  You may obtain a copy of the License at
+      http://www.apache.org/licenses/LICENSE-2.0
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+-->
+
+<configuration scan="true" scanPeriod="30 seconds">
+    <appender name="CONSOLE" class="ch.qos.logback.core.ConsoleAppender">
+        <encoder class="ch.qos.logback.classic.encoder.PatternLayoutEncoder">
+            <pattern>%-4r [%t] %-5p %c - %m%n</pattern>
+        </encoder>
+    </appender>
+
+    <!-- valid logging levels: TRACE, DEBUG, INFO, WARN, ERROR -->
+    <logger name="org.apache.nifi" level="WARN"/>
+
+    <root level="INFO">
+        <appender-ref ref="CONSOLE"/>
+    </root>
+
+</configuration>
+

nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/pom.xml

@@ -63,5 +63,10 @@
             <artifactId>commons-lang3</artifactId>
             <scope>test</scope>
         </dependency>
+        <dependency>
+            <groupId>org.bouncycastle</groupId>
+            <artifactId>bcprov-jdk15on</artifactId>
+            <scope>test</scope>
+        </dependency>
     </dependencies>
 </project>

nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/EncryptedSchemaRecordReader.java

@@ -0,0 +1,154 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.provenance;
+
+import java.io.ByteArrayInputStream;
+import java.io.DataInputStream;
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.Collection;
+import java.util.Optional;
+import java.util.concurrent.TimeUnit;
+import org.apache.nifi.provenance.schema.LookupTableEventRecord;
+import org.apache.nifi.provenance.toc.TocReader;
+import org.apache.nifi.repository.schema.Record;
+import org.apache.nifi.stream.io.LimitingInputStream;
+import org.apache.nifi.stream.io.StreamUtils;
+import org.apache.nifi.util.timebuffer.LongEntityAccess;
+import org.apache.nifi.util.timebuffer.TimedBuffer;
+import org.apache.nifi.util.timebuffer.TimestampedLong;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class EncryptedSchemaRecordReader extends EventIdFirstSchemaRecordReader {
+    private static final Logger logger = LoggerFactory.getLogger(EncryptedSchemaRecordReader.class);
+
+    private static final int DEFAULT_DEBUG_FREQUENCY = 1_000_000;
+
+    private ProvenanceEventEncryptor provenanceEventEncryptor;
+
+    private static final TimedBuffer<TimestampedLong> decryptTimes = new TimedBuffer<>(TimeUnit.SECONDS, 60, new LongEntityAccess());
+
+    private int debugFrequency = DEFAULT_DEBUG_FREQUENCY;
+    public static final int SERIALIZATION_VERSION = 1;
+
+    public static final String SERIALIZATION_NAME = "EncryptedSchemaRecordWriter";
+
+    public EncryptedSchemaRecordReader(final InputStream inputStream, final String filename, final TocReader tocReader, final int maxAttributeChars,
+                                       ProvenanceEventEncryptor provenanceEventEncryptor) throws IOException {
+        this(inputStream, filename, tocReader, maxAttributeChars, provenanceEventEncryptor, DEFAULT_DEBUG_FREQUENCY);
+    }
+
+    public EncryptedSchemaRecordReader(final InputStream inputStream, final String filename, final TocReader tocReader, final int maxAttributeChars,
+                                       ProvenanceEventEncryptor provenanceEventEncryptor, int debugFrequency) throws IOException {
+        super(inputStream, filename, tocReader, maxAttributeChars);
+        this.provenanceEventEncryptor = provenanceEventEncryptor;
+        this.debugFrequency = debugFrequency;
+    }
+
+    @Override
+    protected StandardProvenanceEventRecord nextRecord(final DataInputStream in, final int serializationVersion) throws IOException {
+        verifySerializationVersion(serializationVersion);
+
+        final long byteOffset = getBytesConsumed();
+        final long eventId = in.readInt() + getFirstEventId();
+        final int recordLength = in.readInt();
+
+        return readRecord(in, eventId, byteOffset, recordLength);
+    }
+
+    private StandardProvenanceEventRecord readRecord(final DataInputStream inputStream, final long eventId, final long startOffset, final int recordLength) throws IOException {
+        try {
+            final InputStream limitedIn = new LimitingInputStream(inputStream, recordLength);
+
+            byte[] encryptedSerializedBytes = new byte[recordLength];
+            DataInputStream encryptedInputStream = new DataInputStream(limitedIn);
+            encryptedInputStream.readFully(encryptedSerializedBytes);
+
+            byte[] plainSerializedBytes = decrypt(encryptedSerializedBytes, Long.toString(eventId));
+            InputStream plainStream = new ByteArrayInputStream(plainSerializedBytes);
+
+            final Record eventRecord = getRecordReader().readRecord(plainStream);
+            if (eventRecord == null) {
+                return null;
+            }
+
+            final StandardProvenanceEventRecord deserializedEvent = LookupTableEventRecord.getEvent(eventRecord, getFilename(), startOffset, getMaxAttributeLength(),
+                    getFirstEventId(), getSystemTimeOffset(), getComponentIds(), getComponentTypes(), getQueueIds(), getEventTypes());
+            deserializedEvent.setEventId(eventId);
+            return deserializedEvent;
+        } catch (EncryptionException e) {
+            logger.error("Encountered an error reading the record: ", e);
+            throw new IOException(e);
+        }
+    }
+
+    // TODO: Copied from EventIdFirstSchemaRecordReader to force local/overridden readRecord()
+    @Override
+    protected Optional<StandardProvenanceEventRecord> readToEvent(final long eventId, final DataInputStream dis, final int serializationVersion) throws IOException {
+        verifySerializationVersion(serializationVersion);
+
+        while (isData(dis)) {
+            final long startOffset = getBytesConsumed();
+            final long id = dis.readInt() + getFirstEventId();
+            final int recordLength = dis.readInt();
+
+            if (id >= eventId) {
+                final StandardProvenanceEventRecord event = readRecord(dis, id, startOffset, recordLength);
+                return Optional.ofNullable(event);
+            } else {
+                // This is not the record we want. Skip over it instead of deserializing it.
+                StreamUtils.skip(dis, recordLength);
+            }
+        }
+
+        return Optional.empty();
+    }
+
+    private byte[] decrypt(byte[] encryptedBytes, String eventId) throws IOException, EncryptionException {
+        try {
+            return provenanceEventEncryptor.decrypt(encryptedBytes, eventId);
+        } catch (Exception e) {
+            logger.error("Encountered an error: ", e);
+            throw new EncryptionException(e);
+        }
+    }
+
+    @Override
+    public String toString() {
+        return getDescription();
+    }
+
+    private String getDescription() {
+        try {
+            return "EncryptedSchemaRecordReader, toc: " + getTocReader().getFile().getAbsolutePath() + ", journal: " + getFilename();
+        } catch (Exception e) {
+            return "EncryptedSchemaRecordReader@" + Integer.toHexString(this.hashCode());
+        }
+    }
+
+    /**
+     * Sets the encryptor to use (necessary because the
+     * {@link org.apache.nifi.provenance.serialization.RecordReaders#newRecordReader(File, Collection, int)} method doesn't accept the encryptor.
+     *
+     * @param provenanceEventEncryptor the encryptor
+     */
+    void setProvenanceEventEncryptor(ProvenanceEventEncryptor provenanceEventEncryptor) {
+        this.provenanceEventEncryptor = provenanceEventEncryptor;
+    }
+}

nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/EncryptedSchemaRecordWriter.java

@@ -0,0 +1,199 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.provenance;
+
+import java.io.ByteArrayOutputStream;
+import java.io.DataOutputStream;
+import java.io.File;
+import java.io.IOException;
+import java.security.KeyManagementException;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicLong;
+import org.apache.nifi.provenance.serialization.StorageSummary;
+import org.apache.nifi.provenance.toc.TocWriter;
+import org.apache.nifi.util.timebuffer.LongEntityAccess;
+import org.apache.nifi.util.timebuffer.TimedBuffer;
+import org.apache.nifi.util.timebuffer.TimestampedLong;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class EncryptedSchemaRecordWriter extends EventIdFirstSchemaRecordWriter {
+    private static final Logger logger = LoggerFactory.getLogger(EncryptedSchemaRecordWriter.class);
+
+    private static final int DEFAULT_DEBUG_FREQUENCY = 1_000_000;
+
+    private ProvenanceEventEncryptor provenanceEventEncryptor;
+
+    private static final TimedBuffer<TimestampedLong> encryptTimes = new TimedBuffer<>(TimeUnit.SECONDS, 60, new LongEntityAccess());
+
+    private String keyId;
+
+    private int debugFrequency;
+    public static final int SERIALIZATION_VERSION = 1;
+
+    public static final String SERIALIZATION_NAME = "EncryptedSchemaRecordWriter";
+
+    public EncryptedSchemaRecordWriter(final File file, final AtomicLong idGenerator, final TocWriter writer, final boolean compressed,
+                                       final int uncompressedBlockSize, final IdentifierLookup idLookup,
+                                       ProvenanceEventEncryptor provenanceEventEncryptor) throws IOException, EncryptionException {
+        this(file, idGenerator, writer, compressed, uncompressedBlockSize, idLookup, provenanceEventEncryptor, DEFAULT_DEBUG_FREQUENCY);
+    }
+
+    public EncryptedSchemaRecordWriter(final File file, final AtomicLong idGenerator, final TocWriter writer, final boolean compressed,
+                                       final int uncompressedBlockSize, final IdentifierLookup idLookup,
+                                       ProvenanceEventEncryptor provenanceEventEncryptor, int debugFrequency) throws IOException, EncryptionException {
+        super(file, idGenerator, writer, compressed, uncompressedBlockSize, idLookup);
+        this.provenanceEventEncryptor = provenanceEventEncryptor;
+        this.debugFrequency = debugFrequency;
+
+        try {
+            this.keyId = getNextAvailableKeyId();
+        } catch (KeyManagementException e) {
+            logger.error("Encountered an error initializing the encrypted schema record writer because the provided encryptor has no valid keys available: ", e);
+            throw new EncryptionException("No valid keys in the provenance event encryptor", e);
+        }
+    }
+
+    @Override
+    public StorageSummary writeRecord(final ProvenanceEventRecord record) throws IOException {
+        final long encryptStart = System.nanoTime();
+        byte[] cipherBytes;
+        try {
+            byte[] serialized;
+            try (final ByteArrayOutputStream baos = new ByteArrayOutputStream(256);
+                 final DataOutputStream dos = new DataOutputStream(baos)) {
+                writeRecord(record, 0L, dos);
+                serialized = baos.toByteArray();
+            }
+            String eventId = record.getBestEventIdentifier();
+            cipherBytes = encrypt(serialized, eventId);
+        } catch (EncryptionException e) {
+            logger.error("Encountered an error: ", e);
+            throw new IOException("Error encrypting the provenance record", e);
+        }
+        final long encryptStop = System.nanoTime();
+
+        final long lockStart = System.nanoTime();
+        final long writeStart;
+        final long startBytes;
+        final long endBytes;
+        final long recordIdentifier;
+        synchronized (this) {
+            writeStart = System.nanoTime();
+            try {
+                recordIdentifier = record.getEventId() == -1L ? getIdGenerator().getAndIncrement() : record.getEventId();
+                startBytes = getBytesWritten();
+
+                ensureStreamState(recordIdentifier, startBytes);
+
+                final DataOutputStream out = getBufferedOutputStream();
+                final int recordIdOffset = (int) (recordIdentifier - getFirstEventId());
+                out.writeInt(recordIdOffset);
+                out.writeInt(cipherBytes.length);
+                out.write(cipherBytes);
+
+                getRecordCount().incrementAndGet();
+                endBytes = getBytesWritten();
+            } catch (final IOException ioe) {
+                markDirty();
+                throw ioe;
+            }
+        }
+
+        if (logger.isDebugEnabled()) {
+            // Collect stats and periodically dump them if log level is set to at least info.
+            final long writeNanos = System.nanoTime() - writeStart;
+            getWriteTimes().add(new TimestampedLong(writeNanos));
+
+            final long serializeNanos = lockStart - encryptStart;
+            getSerializeTimes().add(new TimestampedLong(serializeNanos));
+
+            final long encryptNanos = encryptStop - encryptStart;
+            getEncryptTimes().add(new TimestampedLong(encryptNanos));
+
+            final long lockNanos = writeStart - lockStart;
+            getLockTimes().add(new TimestampedLong(lockNanos));
+            getBytesWrittenBuffer().add(new TimestampedLong(endBytes - startBytes));
+
+            final long recordCount = getTotalRecordCount().incrementAndGet();
+            if (recordCount % debugFrequency == 0) {
+                printStats();
+            }
+        }
+
+        final long serializedLength = endBytes - startBytes;
+        final TocWriter tocWriter = getTocWriter();
+        final Integer blockIndex = tocWriter == null ? null : tocWriter.getCurrentBlockIndex();
+        final File file = getFile();
+        final String storageLocation = file.getParentFile().getName() + "/" + file.getName();
+        return new StorageSummary(recordIdentifier, storageLocation, blockIndex, serializedLength, endBytes);
+    }
+
+    private void printStats() {
+        final long sixtySecondsAgo = System.currentTimeMillis() - 60000L;
+        final Long writeNanosLast60 = getWriteTimes().getAggregateValue(sixtySecondsAgo).getValue();
+        final Long lockNanosLast60 = getLockTimes().getAggregateValue(sixtySecondsAgo).getValue();
+        final Long serializeNanosLast60 = getSerializeTimes().getAggregateValue(sixtySecondsAgo).getValue();
+        final Long encryptNanosLast60 = getEncryptTimes().getAggregateValue(sixtySecondsAgo).getValue();
+        final Long bytesWrittenLast60 = getBytesWrittenBuffer().getAggregateValue(sixtySecondsAgo).getValue();
+        logger.debug("In the last 60 seconds, have spent {} millis writing to file ({} MB), {} millis waiting on synchronize block, {} millis serializing events, {} millis encrypting events",
+                TimeUnit.NANOSECONDS.toMillis(writeNanosLast60),
+                bytesWrittenLast60 / 1024 / 1024,
+                TimeUnit.NANOSECONDS.toMillis(lockNanosLast60),
+                TimeUnit.NANOSECONDS.toMillis(serializeNanosLast60),
+                TimeUnit.NANOSECONDS.toMillis(encryptNanosLast60));
+    }
+
+    static TimedBuffer<TimestampedLong> getEncryptTimes() {
+        return encryptTimes;
+    }
+
+    private byte[] encrypt(byte[] serialized, String eventId) throws IOException, EncryptionException {
+        String keyId = getKeyId();
+        try {
+            return provenanceEventEncryptor.encrypt(serialized, eventId, keyId);
+        } catch (Exception e) {
+            logger.error("Encountered an error: ", e);
+            throw new EncryptionException(e);
+        }
+    }
+
+    private String getNextAvailableKeyId() throws KeyManagementException {
+        return provenanceEventEncryptor.getNextKeyId();
+    }
+
+    @Override
+    protected int getSerializationVersion() {
+        return SERIALIZATION_VERSION;
+    }
+
+    @Override
+    protected String getSerializationName() {
+        return SERIALIZATION_NAME;
+    }
+
+    public String getKeyId() {
+        return keyId;
+    }
+
+    @Override
+    public String toString() {
+        return "EncryptedSchemaRecordWriter" +
+                " using " + provenanceEventEncryptor +
+                " and current keyId " + keyId;
+    }
+}

nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/EncryptedWriteAheadProvenanceRepository.java

@@ -0,0 +1,159 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.provenance;
+
+import java.io.IOException;
+import java.security.KeyManagementException;
+import java.util.Map;
+import java.util.stream.Collectors;
+import javax.crypto.SecretKey;
+import org.apache.nifi.authorization.Authorizer;
+import org.apache.nifi.events.EventReporter;
+import org.apache.nifi.provenance.serialization.RecordReaders;
+import org.apache.nifi.provenance.store.EventFileManager;
+import org.apache.nifi.provenance.store.RecordReaderFactory;
+import org.apache.nifi.provenance.store.RecordWriterFactory;
+import org.apache.nifi.provenance.toc.StandardTocWriter;
+import org.apache.nifi.provenance.toc.TocUtil;
+import org.apache.nifi.provenance.toc.TocWriter;
+import org.apache.nifi.util.NiFiProperties;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class EncryptedWriteAheadProvenanceRepository extends WriteAheadProvenanceRepository {
+    private static final Logger logger = LoggerFactory.getLogger(EncryptedWriteAheadProvenanceRepository.class);
+
+    /**
+     * This constructor exists solely for the use of the Java Service Loader mechanism and should not be used.
+     */
+    public EncryptedWriteAheadProvenanceRepository() {
+        super();
+    }
+
+    public EncryptedWriteAheadProvenanceRepository(final NiFiProperties nifiProperties) {
+        super(RepositoryConfiguration.create(nifiProperties));
+    }
+
+    public EncryptedWriteAheadProvenanceRepository(final RepositoryConfiguration config) {
+        super(config);
+    }
+
+    /**
+     * This method initializes the repository. It first builds the key provider and event encryptor
+     * from the config values, then creates the encrypted record writer and reader, then delegates
+     * back to the superclass for the common implementation.
+     *
+     * @param eventReporter   the event reporter
+     * @param authorizer      the authorizer
+     * @param resourceFactory the authorizable factory
+     * @param idLookup        the lookup provider
+     * @throws IOException if there is an error initializing this repository
+     */
+    @Override
+    public synchronized void initialize(final EventReporter eventReporter, final Authorizer authorizer, final ProvenanceAuthorizableFactory resourceFactory,
+                                        final IdentifierLookup idLookup) throws IOException {
+        // Initialize the encryption-specific fields
+        ProvenanceEventEncryptor provenanceEventEncryptor;
+        if (getConfig().supportsEncryption()) {
+            try {
+                KeyProvider keyProvider = buildKeyProvider();
+                provenanceEventEncryptor = new AESProvenanceEventEncryptor();
+                provenanceEventEncryptor.initialize(keyProvider);
+            } catch (KeyManagementException e) {
+                String msg = "Encountered an error building the key provider";
+                logger.error(msg, e);
+                throw new IOException(msg, e);
+            }
+        } else {
+            throw new IOException("The provided configuration does not support a encrypted repository");
+        }
+
+        // Build a factory using lambda which injects the encryptor
+        final RecordWriterFactory recordWriterFactory = (file, idGenerator, compressed, createToc) -> {
+            try {
+                final TocWriter tocWriter = createToc ? new StandardTocWriter(TocUtil.getTocFile(file), false, false) : null;
+                return new EncryptedSchemaRecordWriter(file, idGenerator, tocWriter, compressed, BLOCK_SIZE, idLookup, provenanceEventEncryptor, getConfig().getDebugFrequency());
+            } catch (EncryptionException e) {
+                logger.error("Encountered an error building the schema record writer factory: ", e);
+                throw new IOException(e);
+            }
+        };
+
+        // Build a factory using lambda which injects the encryptor
+        final EventFileManager fileManager = new EventFileManager();
+        final RecordReaderFactory recordReaderFactory = (file, logs, maxChars) -> {
+            fileManager.obtainReadLock(file);
+            try {
+                EncryptedSchemaRecordReader tempReader = (EncryptedSchemaRecordReader) RecordReaders.newRecordReader(file, logs, maxChars);
+                tempReader.setProvenanceEventEncryptor(provenanceEventEncryptor);
+                return tempReader;
+            } finally {
+                fileManager.releaseReadLock(file);
+            }
+        };
+
+        // Delegate the init to the parent impl
+        super.init(recordWriterFactory, recordReaderFactory, eventReporter, authorizer, resourceFactory);
+    }
+
+    private KeyProvider buildKeyProvider() throws KeyManagementException {
+        RepositoryConfiguration config = super.getConfig();
+        if (config == null) {
+            throw new KeyManagementException("The repository configuration is missing");
+        }
+
+        final String implementationClassName = config.getKeyProviderImplementation();
+        if (implementationClassName == null) {
+            throw new KeyManagementException("Cannot create Key Provider because the NiFi Properties is missing the following property: "
+                    + NiFiProperties.PROVENANCE_REPO_ENCRYPTION_KEY_PROVIDER_IMPLEMENTATION_CLASS);
+        }
+
+        // TODO: Extract to factory
+        KeyProvider keyProvider;
+        if (StaticKeyProvider.class.getName().equals(implementationClassName)) {
+            // Get all the keys (map) from config
+            if (CryptoUtils.isValidKeyProvider(implementationClassName, config.getKeyProviderLocation(), config.getKeyId(), config.getEncryptionKeys())) {
+                Map<String, SecretKey> formedKeys = config.getEncryptionKeys().entrySet().stream()
+                        .collect(Collectors.toMap(
+                                Map.Entry::getKey,
+                                e -> {
+                                    try {
+                                        return CryptoUtils.formKeyFromHex(e.getValue());
+                                    } catch (KeyManagementException e1) {
+                                        // This should never happen because the hex has already been validated
+                                        logger.error("Encountered an error: ", e1);
+                                        return null;
+                                    }
+                                }));
+                keyProvider = new StaticKeyProvider(formedKeys);
+            } else {
+                final String msg = "The StaticKeyProvider definition is not valid";
+                logger.error(msg);
+                throw new KeyManagementException(msg);
+            }
+        } else if (FileBasedKeyProvider.class.getName().equals(implementationClassName)) {
+            keyProvider = new FileBasedKeyProvider(config.getKeyProviderLocation());
+            if (!keyProvider.keyExists(config.getKeyId())) {
+                throw new KeyManagementException("The specified key ID " + config.getKeyId() + " is not in the key definition file");
+            }
+        } else {
+            throw new KeyManagementException("Invalid key provider implementation provided: " + implementationClassName);
+        }
+
+        return keyProvider;
+    }
+}

nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/EventIdFirstSchemaRecordReader.java

@@ -23,7 +23,6 @@ import java.io.IOException;
 import java.io.InputStream;
 import java.util.List;
 import java.util.Optional;
-
 import org.apache.nifi.provenance.schema.EventIdFirstHeaderSchema;
 import org.apache.nifi.provenance.schema.LookupTableEventRecord;
 import org.apache.nifi.provenance.serialization.CompressableRecordReader;
@@ -35,6 +34,14 @@ import org.apache.nifi.stream.io.LimitingInputStream;
 import org.apache.nifi.stream.io.StreamUtils;
 
 public class EventIdFirstSchemaRecordReader extends CompressableRecordReader {
+    RecordSchema getSchema() {
+        return schema;
+    }
+
+    SchemaRecordReader getRecordReader() {
+        return recordReader;
+    }
+
     private RecordSchema schema; // effectively final
     private SchemaRecordReader recordReader;  // effectively final
 
@@ -43,16 +50,41 @@ public class EventIdFirstSchemaRecordReader extends CompressableRecordReader {
     private List<String> queueIds;
     private List<String> eventTypes;
     private long firstEventId;
+
+    List<String> getComponentIds() {
+        return componentIds;
+    }
+
+    List<String> getComponentTypes() {
+        return componentTypes;
+    }
+
+    List<String> getQueueIds() {
+        return queueIds;
+    }
+
+    List<String> getEventTypes() {
+        return eventTypes;
+    }
+
+    long getFirstEventId() {
+        return firstEventId;
+    }
+
+    long getSystemTimeOffset() {
+        return systemTimeOffset;
+    }
+
     private long systemTimeOffset;
 
     public EventIdFirstSchemaRecordReader(final InputStream in, final String filename, final TocReader tocReader, final int maxAttributeChars) throws IOException {
         super(in, filename, tocReader, maxAttributeChars);
     }
 
-    private void verifySerializationVersion(final int serializationVersion) {
+    protected void verifySerializationVersion(final int serializationVersion) {
         if (serializationVersion > EventIdFirstSchemaRecordWriter.SERIALIZATION_VERSION) {
             throw new IllegalArgumentException("Unable to deserialize record because the version is " + serializationVersion
-                + " and supported versions are 1-" + EventIdFirstSchemaRecordWriter.SERIALIZATION_VERSION);
+                    + " and supported versions are 1-" + EventIdFirstSchemaRecordWriter.SERIALIZATION_VERSION);
         }
     }
 
@@ -109,12 +141,12 @@ public class EventIdFirstSchemaRecordReader extends CompressableRecordReader {
         }
 
         final StandardProvenanceEventRecord deserializedEvent = LookupTableEventRecord.getEvent(eventRecord, getFilename(), startOffset, getMaxAttributeLength(),
-            firstEventId, systemTimeOffset, componentIds, componentTypes, queueIds, eventTypes);
+                firstEventId, systemTimeOffset, componentIds, componentTypes, queueIds, eventTypes);
         deserializedEvent.setEventId(eventId);
         return deserializedEvent;
     }
 
-    private boolean isData(final InputStream in) throws IOException {
+    protected boolean isData(final InputStream in) throws IOException {
         in.mark(1);
         final int nextByte = in.read();
         in.reset();
@@ -142,4 +174,17 @@ public class EventIdFirstSchemaRecordReader extends CompressableRecordReader {
 
         return Optional.empty();
     }
+
+    @Override
+    public String toString() {
+        return getDescription();
+    }
+
+    private String getDescription() {
+        try {
+            return "EventIdFirstSchemaRecordReader, toc: " + getTocReader().getFile().getAbsolutePath() + ", journal: " + getFilename();
+        } catch (Exception e) {
+            return "EventIdFirstSchemaRecordReader@" + Integer.toHexString(this.hashCode());
+        }
+    }
 }

nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/EventIdFirstSchemaRecordWriter.java

@@ -29,7 +29,6 @@ import java.util.Map;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.concurrent.atomic.AtomicLong;
-
 import org.apache.nifi.provenance.schema.EventFieldNames;
 import org.apache.nifi.provenance.schema.EventIdFirstHeaderSchema;
 import org.apache.nifi.provenance.schema.LookupTableEventRecord;
@@ -238,4 +237,46 @@ public class EventIdFirstSchemaRecordWriter extends CompressableRecordWriter {
         return SERIALIZATION_NAME;
     }
 
+    /* Getters for internal state written to by subclass EncryptedSchemaRecordWriter */
+
+    IdentifierLookup getIdLookup() {
+        return idLookup;
+    }
+
+    SchemaRecordWriter getSchemaRecordWriter() {
+        return schemaRecordWriter;
+    }
+
+    AtomicInteger getRecordCount() {
+        return recordCount;
+    }
+
+    static TimedBuffer<TimestampedLong> getSerializeTimes() {
+        return serializeTimes;
+    }
+
+    static TimedBuffer<TimestampedLong> getLockTimes() {
+        return lockTimes;
+    }
+
+    static TimedBuffer<TimestampedLong> getWriteTimes() {
+        return writeTimes;
+    }
+
+    static TimedBuffer<TimestampedLong> getBytesWrittenBuffer() {
+        return bytesWritten;
+    }
+
+    static AtomicLong getTotalRecordCount() {
+        return totalRecordCount;
+    }
+
+    long getFirstEventId() {
+        return firstEventId;
+    }
+
+    long getSystemTimeOffset() {
+        return systemTimeOffset;
+    }
+
 }

nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/RepositoryConfiguration.java

@@ -26,7 +26,6 @@ import java.util.List;
 import java.util.Map;
 import java.util.Optional;
 import java.util.concurrent.TimeUnit;
-
 import org.apache.nifi.processor.DataUnit;
 import org.apache.nifi.provenance.search.SearchableField;
 import org.apache.nifi.util.FormatUtils;
@@ -50,6 +49,12 @@ public class RepositoryConfiguration {
     private int journalCount = 16;
     private int compressionBlockBytes = 1024 * 1024;
     private int maxAttributeChars = 65536;
+    private int debugFrequency = 1_000_000;
+
+    private Map<String, String> encryptionKeys;
+    private String keyId;
+    private String keyProviderImplementation;
+    private String keyProviderLocation;
 
     private List<SearchableField> searchableFields = new ArrayList<>();
     private List<SearchableField> searchableAttributes = new ArrayList<>();
@@ -360,6 +365,54 @@ public class RepositoryConfiguration {
         return Optional.ofNullable(warmCacheFrequencyMinutes);
     }
 
+    public boolean supportsEncryption() {
+        boolean keyProviderIsConfigured = CryptoUtils.isValidKeyProvider(keyProviderImplementation, keyProviderLocation, keyId, encryptionKeys);
+
+        return keyProviderIsConfigured;
+    }
+
+    public Map<String, String> getEncryptionKeys() {
+        return encryptionKeys;
+    }
+
+    public void setEncryptionKeys(Map<String, String> encryptionKeys) {
+        this.encryptionKeys = encryptionKeys;
+    }
+
+    public String getKeyId() {
+        return keyId;
+    }
+
+    public void setKeyId(String keyId) {
+        this.keyId = keyId;
+    }
+
+    public String getKeyProviderImplementation() {
+        return keyProviderImplementation;
+    }
+
+    public void setKeyProviderImplementation(String keyProviderImplementation) {
+        this.keyProviderImplementation = keyProviderImplementation;
+    }
+
+    public String getKeyProviderLocation() {
+        return keyProviderLocation;
+    }
+
+    public void setKeyProviderLocation(String keyProviderLocation) {
+        this.keyProviderLocation = keyProviderLocation;
+    }
+
+
+    public int getDebugFrequency() {
+        return debugFrequency;
+    }
+
+    public void setDebugFrequency(int debugFrequency) {
+        this.debugFrequency = debugFrequency;
+    }
+
+
     public static RepositoryConfiguration create(final NiFiProperties nifiProperties) {
         final Map<String, Path> storageDirectories = nifiProperties.getProvenanceRepositoryPaths();
         if (storageDirectories.isEmpty()) {
@@ -436,6 +489,17 @@ public class RepositoryConfiguration {
 
         config.setAlwaysSync(alwaysSync);
 
+        config.setDebugFrequency(nifiProperties.getIntegerProperty(NiFiProperties.PROVENANCE_REPO_DEBUG_FREQUENCY, config.getDebugFrequency()));
+
+        // Encryption values may not be present but are only required for EncryptedWriteAheadProvenanceRepository
+        final String implementationClassName = nifiProperties.getProperty(NiFiProperties.PROVENANCE_REPO_IMPLEMENTATION_CLASS);
+        if (EncryptedWriteAheadProvenanceRepository.class.getName().equals(implementationClassName)) {
+            config.setEncryptionKeys(nifiProperties.getProvenanceRepoEncryptionKeys());
+            config.setKeyId(nifiProperties.getProperty(NiFiProperties.PROVENANCE_REPO_ENCRYPTION_KEY_ID));
+            config.setKeyProviderImplementation(nifiProperties.getProperty(NiFiProperties.PROVENANCE_REPO_ENCRYPTION_KEY_PROVIDER_IMPLEMENTATION_CLASS));
+            config.setKeyProviderLocation(nifiProperties.getProperty(NiFiProperties.PROVENANCE_REPO_ENCRYPTION_KEY_PROVIDER_LOCATION));
+        }
+
         return config;
     }
 }

nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/WriteAheadProvenanceRepository.java

@@ -21,7 +21,6 @@ import java.io.IOException;
 import java.util.Collections;
 import java.util.List;
 import java.util.Map;
-
 import org.apache.nifi.authorization.Authorizer;
 import org.apache.nifi.authorization.RequestAction;
 import org.apache.nifi.authorization.resource.Authorizable;
@@ -84,7 +83,7 @@ import org.slf4j.LoggerFactory;
  */
 public class WriteAheadProvenanceRepository implements ProvenanceRepository {
     private static final Logger logger = LoggerFactory.getLogger(WriteAheadProvenanceRepository.class);
-    private static final int BLOCK_SIZE = 1024 * 32;
+    static final int BLOCK_SIZE = 1024 * 32;
     public static final String EVENT_CATEGORY = "Provenance Repository";
 
     private final RepositoryConfiguration config;
@@ -129,6 +128,14 @@ public class WriteAheadProvenanceRepository implements ProvenanceRepository {
             }
         };
 
+       init(recordWriterFactory, recordReaderFactory, eventReporter, authorizer, resourceFactory);
+    }
+
+    synchronized void init(RecordWriterFactory recordWriterFactory, RecordReaderFactory recordReaderFactory,
+                           final EventReporter eventReporter, final Authorizer authorizer,
+                           final ProvenanceAuthorizableFactory resourceFactory) throws IOException {
+        final EventFileManager fileManager = new EventFileManager();
+
         eventStore = new PartitionedWriteAheadEventStore(config, recordWriterFactory, recordReaderFactory, eventReporter, fileManager);
 
         final IndexManager indexManager = new SimpleIndexManager(config);
@@ -145,7 +152,7 @@ public class WriteAheadProvenanceRepository implements ProvenanceRepository {
             eventStore.reindexLatestEvents(eventIndex);
         } catch (final Exception e) {
             logger.error("Failed to re-index some of the Provenance Events. It is possible that some of the latest "
-                + "events will not be available from the Provenance Repository when a query is issued.", e);
+                    + "events will not be available from the Provenance Repository when a query is issued.", e);
         }
     }
 
@@ -282,4 +289,8 @@ public class WriteAheadProvenanceRepository implements ProvenanceRepository {
     public List<SearchableField> getSearchableAttributes() {
         return Collections.unmodifiableList(config.getSearchableAttributes());
     }
+
+    RepositoryConfiguration getConfig() {
+        return this.config;
+    }
 }

nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/index/lucene/LuceneEventIndex.java

@@ -36,7 +36,6 @@ import java.util.concurrent.LinkedBlockingQueue;
 import java.util.concurrent.ScheduledExecutorService;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicLong;
-
 import org.apache.lucene.document.Document;
 import org.apache.lucene.index.IndexReader;
 import org.apache.lucene.index.Term;
@@ -246,7 +245,7 @@ public class LuceneEventIndex implements EventIndex {
 
             final Document document = eventConverter.convert(event, summary);
             if (document == null) {
-                logger.debug("Received Provenance Event {} to index but it contained no information that should be indexed, so skipping it", event);
+                logger.debug("Received Provenance Event {} to index but it contained no information that should be indexed, so skipping it", event.getEventId());
             } else {
                 final File indexDir;
                 if (event.getEventTime() == lastEventTime) {
@@ -291,7 +290,7 @@ public class LuceneEventIndex implements EventIndex {
 
         final Document document = eventConverter.convert(event, location);
         if (document == null) {
-            logger.debug("Received Provenance Event {} to index but it contained no information that should be indexed, so skipping it", event);
+            logger.debug("Received Provenance Event {} to index but it contained no information that should be indexed, so skipping it", event.getEventId());
         } else {
             final StoredDocument doc = new StoredDocument(document, location);
             boolean added = false;
@@ -357,13 +356,13 @@ public class LuceneEventIndex implements EventIndex {
             eventOption = eventStore.getEvent(eventId);
         } catch (final Exception e) {
             logger.error("Failed to retrieve Provenance Event with ID " + eventId + " to calculate data lineage due to: " + e, e);
-            final AsyncLineageSubmission result = new AsyncLineageSubmission(LineageComputationType.FLOWFILE_LINEAGE, eventId, Collections.<String> emptySet(), 1, user.getIdentity());
+            final AsyncLineageSubmission result = new AsyncLineageSubmission(LineageComputationType.FLOWFILE_LINEAGE, eventId, Collections.emptySet(), 1, user.getIdentity());
             result.getResult().setError("Failed to retrieve Provenance Event with ID " + eventId + ". See logs for more information.");
             return result;
         }
 
         if (!eventOption.isPresent()) {
-            final AsyncLineageSubmission result = new AsyncLineageSubmission(LineageComputationType.FLOWFILE_LINEAGE, eventId, Collections.<String> emptySet(), 1, user.getIdentity());
+            final AsyncLineageSubmission result = new AsyncLineageSubmission(LineageComputationType.FLOWFILE_LINEAGE, eventId, Collections.emptySet(), 1, user.getIdentity());
             result.getResult().setError("Could not find Provenance Event with ID " + eventId);
             lineageSubmissionMap.put(result.getLineageIdentifier(), result);
             return result;
@@ -524,7 +523,7 @@ public class LuceneEventIndex implements EventIndex {
                 }
                 default: {
                     final AsyncLineageSubmission submission = new AsyncLineageSubmission(LineageComputationType.EXPAND_CHILDREN,
-                        eventId, Collections.<String> emptyList(), 1, userId);
+                        eventId, Collections.emptyList(), 1, userId);
 
                     lineageSubmissionMap.put(submission.getLineageIdentifier(), submission);
                     submission.getResult().setError("Event ID " + eventId + " indicates an event of type " + event.getEventType() + " so its children cannot be expanded");
@@ -533,7 +532,7 @@ public class LuceneEventIndex implements EventIndex {
             }
         } catch (final Exception e) {
             final AsyncLineageSubmission submission = new AsyncLineageSubmission(LineageComputationType.EXPAND_CHILDREN,
-                eventId, Collections.<String> emptyList(), 1, userId);
+                eventId, Collections.emptyList(), 1, userId);
             lineageSubmissionMap.put(submission.getLineageIdentifier(), submission);
             submission.getResult().setError("Failed to expand children for lineage of event with ID " + eventId + " due to: " + e);
             return submission;
@@ -564,7 +563,7 @@ public class LuceneEventIndex implements EventIndex {
                 }
                 default: {
                     final AsyncLineageSubmission submission = new AsyncLineageSubmission(LineageComputationType.EXPAND_PARENTS,
-                        eventId, Collections.<String> emptyList(), 1, userId);
+                        eventId, Collections.emptyList(), 1, userId);
 
                     lineageSubmissionMap.put(submission.getLineageIdentifier(), submission);
                     submission.getResult().setError("Event ID " + eventId + " indicates an event of type " + event.getEventType() + " so its parents cannot be expanded");
@@ -573,7 +572,7 @@ public class LuceneEventIndex implements EventIndex {
             }
         } catch (final Exception e) {
             final AsyncLineageSubmission submission = new AsyncLineageSubmission(LineageComputationType.EXPAND_PARENTS,
-                eventId, Collections.<String> emptyList(), 1, userId);
+                eventId, Collections.emptyList(), 1, userId);
             lineageSubmissionMap.put(submission.getLineageIdentifier(), submission);
 
             submission.getResult().setError("Failed to expand parents for lineage of event with ID " + eventId + " due to: " + e);

nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/schema/EventFieldNames.java

@@ -56,4 +56,12 @@ public class EventFieldNames {
     public static final String EXPLICIT_VALUE = "Explicit Value";
     public static final String LOOKUP_VALUE = "Lookup Value";
     public static final String UNCHANGED_VALUE = "Unchanged";
+
+    // For encrypted records
+    public static final String IS_ENCRYPTED = "Encrypted Record";
+    public static final String KEY_ID = "Encryption Key ID";
+    public static final String VERSION = "Encryption Version";
+    public static final String ALGORITHM = "Encryption Algorithm";
+    public static final String IV = "Initialization Vector";
+    public static final String ENCRYPTION_DETAILS = "Encryption Details";
 }

nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/schema/LookupTableEventRecordFields.java

@@ -66,12 +66,12 @@ public class LookupTableEventRecordFields {
     public static final RecordField CONTENT_CLAIM_SIZE = new SimpleRecordField(EventFieldNames.CONTENT_CLAIM_SIZE, FieldType.LONG, EXACTLY_ONE);
 
     public static final RecordField PREVIOUS_CONTENT_CLAIM = new ComplexRecordField(EventFieldNames.PREVIOUS_CONTENT_CLAIM, ZERO_OR_ONE,
-        CONTENT_CLAIM_CONTAINER, CONTENT_CLAIM_SECTION, CONTENT_CLAIM_IDENTIFIER, CONTENT_CLAIM_OFFSET, CONTENT_CLAIM_SIZE);
+            CONTENT_CLAIM_CONTAINER, CONTENT_CLAIM_SECTION, CONTENT_CLAIM_IDENTIFIER, CONTENT_CLAIM_OFFSET, CONTENT_CLAIM_SIZE);
 
     public static final RecordField CURRENT_CONTENT_CLAIM_EXPLICIT = new ComplexRecordField(EventFieldNames.EXPLICIT_VALUE, EXACTLY_ONE,
-        CONTENT_CLAIM_CONTAINER, CONTENT_CLAIM_SECTION, CONTENT_CLAIM_IDENTIFIER, CONTENT_CLAIM_OFFSET, CONTENT_CLAIM_SIZE);
+            CONTENT_CLAIM_CONTAINER, CONTENT_CLAIM_SECTION, CONTENT_CLAIM_IDENTIFIER, CONTENT_CLAIM_OFFSET, CONTENT_CLAIM_SIZE);
     public static final RecordField CURRENT_CONTENT_CLAIM = new UnionRecordField(EventFieldNames.CONTENT_CLAIM,
-        Repetition.EXACTLY_ONE, NO_VALUE, UNCHANGED_VALUE, CURRENT_CONTENT_CLAIM_EXPLICIT);
+            Repetition.EXACTLY_ONE, NO_VALUE, UNCHANGED_VALUE, CURRENT_CONTENT_CLAIM_EXPLICIT);
 
 
     // EventType-Specific fields

nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/schema/LookupTableEventSchema.java

@@ -45,7 +45,6 @@ import static org.apache.nifi.provenance.schema.LookupTableEventRecordFields.UPD
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
-
 import org.apache.nifi.repository.schema.RecordField;
 import org.apache.nifi.repository.schema.RecordSchema;
 
@@ -90,5 +89,4 @@ public class LookupTableEventSchema {
         final RecordSchema schema = new RecordSchema(fields);
         return schema;
     }
-
 }

nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/serialization/CompressableRecordReader.java

@@ -25,7 +25,6 @@ import java.io.InputStream;
 import java.nio.charset.StandardCharsets;
 import java.util.Optional;
 import java.util.zip.GZIPInputStream;
-
 import org.apache.nifi.provenance.ProvenanceEventRecord;
 import org.apache.nifi.provenance.StandardProvenanceEventRecord;
 import org.apache.nifi.provenance.toc.TocReader;
@@ -333,7 +332,7 @@ public abstract class CompressableRecordReader implements RecordReader {
         try {
             boolean read = true;
             while (read) {
-                final Optional<StandardProvenanceEventRecord> eventOptional = readToEvent(eventId, dis, serializationVersion);
+                final Optional<StandardProvenanceEventRecord> eventOptional = this.readToEvent(eventId, dis, serializationVersion);
                 if (eventOptional.isPresent()) {
                     pushbackEvent = eventOptional.get();
                     return Optional.of(pushbackEvent);

nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/serialization/RecordReaders.java

@@ -27,9 +27,11 @@ import java.io.InputStream;
 import java.nio.file.Path;
 import java.util.Collection;
 import java.util.zip.GZIPInputStream;
-
+import org.apache.nifi.properties.NiFiPropertiesLoader;
 import org.apache.nifi.provenance.ByteArraySchemaRecordReader;
 import org.apache.nifi.provenance.ByteArraySchemaRecordWriter;
+import org.apache.nifi.provenance.CryptoUtils;
+import org.apache.nifi.provenance.EncryptedSchemaRecordReader;
 import org.apache.nifi.provenance.EventIdFirstSchemaRecordReader;
 import org.apache.nifi.provenance.EventIdFirstSchemaRecordWriter;
 import org.apache.nifi.provenance.StandardRecordReader;
@@ -37,17 +39,25 @@ import org.apache.nifi.provenance.lucene.LuceneUtil;
 import org.apache.nifi.provenance.toc.StandardTocReader;
 import org.apache.nifi.provenance.toc.TocReader;
 import org.apache.nifi.provenance.toc.TocUtil;
+import org.apache.nifi.util.NiFiProperties;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 public class RecordReaders {
 
+    private static Logger logger = LoggerFactory.getLogger(RecordReaders.class);
+
+    private static boolean isEncryptionAvailable = false;
+    private static boolean encryptionPropertiesRead = false;
+
     /**
      * Creates a new Record Reader that is capable of reading Provenance Event Journals
      *
-     * @param file the Provenance Event Journal to read data from
+     * @param file               the Provenance Event Journal to read data from
      * @param provenanceLogFiles collection of all provenance journal files
-     * @param maxAttributeChars the maximum number of characters to retrieve for any one attribute. This allows us to avoid
-     *            issues where a FlowFile has an extremely large attribute and reading events
-     *            for that FlowFile results in loading that attribute into memory many times, exhausting the Java Heap
+     * @param maxAttributeChars  the maximum number of characters to retrieve for any one attribute. This allows us to avoid
+     *                           issues where a FlowFile has an extremely large attribute and reading events
+     *                           for that FlowFile results in loading that attribute into memory many times, exhausting the Java Heap
      * @return a Record Reader capable of reading Provenance Event Journals
      * @throws IOException if unable to create a Record Reader for the given file
      */
@@ -68,7 +78,7 @@ public class RecordReaders {
                 }
             }
 
-            if ( file.exists() ) {
+            if (file.exists()) {
                 try {
                     fis = new FileInputStream(file);
                 } catch (final FileNotFoundException fnfe) {
@@ -77,7 +87,8 @@ public class RecordReaders {
             }
 
             String filename = file.getName();
-            openStream: while ( fis == null ) {
+            openStream:
+            while (fis == null) {
                 final File dir = file.getParentFile();
                 final String baseName = LuceneUtil.substringBefore(file.getName(), ".prov");
 
@@ -85,9 +96,9 @@ public class RecordReaders {
                 // filename that we need. The majority of the time, we will use the extension ".prov.gz"
                 // because most often we are compressing on rollover and most often we have already finished
                 // compressing by the time that we are querying the data.
-                for ( final String extension : new String[] {".prov.gz", ".prov"} ) {
+                for (final String extension : new String[]{".prov.gz", ".prov"}) {
                     file = new File(dir, baseName + extension);
-                    if ( file.exists() ) {
+                    if (file.exists()) {
                         try {
                             fis = new FileInputStream(file);
                             filename = baseName + extension;
@@ -104,7 +115,7 @@ public class RecordReaders {
                 break;
             }
 
-            if ( fis == null ) {
+            if (fis == null) {
                 throw new FileNotFoundException("Unable to locate file " + originalFile);
             }
 
@@ -148,12 +159,25 @@ public class RecordReaders {
                     final TocReader tocReader = new StandardTocReader(tocFile);
                     return new EventIdFirstSchemaRecordReader(bufferedInStream, filename, tocReader, maxAttributeChars);
                 }
+                case EncryptedSchemaRecordReader.SERIALIZATION_NAME: {
+                    if (!tocFile.exists()) {
+                        throw new FileNotFoundException("Cannot create TOC Reader because the file " + tocFile + " does not exist");
+                    }
+
+                    if (!isEncryptionAvailable()) {
+                        throw new IOException("Cannot read encrypted repository because this reader is not configured for encryption");
+                    }
+
+                    final TocReader tocReader = new StandardTocReader(tocFile);
+                    // Return a reader with no eventEncryptor because this method contract cannot change, then inject the encryptor from the writer in the calling method
+                    return new EncryptedSchemaRecordReader(bufferedInStream, filename, tocReader, maxAttributeChars, null);
+                }
                 default: {
                     throw new IOException("Unable to read data from file " + file + " because the file was written using an unknown Serializer: " + serializationName);
                 }
             }
         } catch (final IOException ioe) {
-            if ( fis != null ) {
+            if (fis != null) {
                 try {
                     fis.close();
                 } catch (final IOException inner) {
@@ -165,4 +189,20 @@ public class RecordReaders {
         }
     }
 
+    private static boolean isEncryptionAvailable() {
+        if (encryptionPropertiesRead) {
+            return isEncryptionAvailable;
+        } else {
+            try {
+                NiFiProperties niFiProperties = NiFiPropertiesLoader.loadDefaultWithKeyFromBootstrap();
+                isEncryptionAvailable = CryptoUtils.isProvenanceRepositoryEncryptionConfigured(niFiProperties);
+                encryptionPropertiesRead = true;
+            } catch (IOException e) {
+                logger.error("Encountered an error checking the provenance repository encryption configuration: ", e);
+                isEncryptionAvailable = false;
+            }
+            return isEncryptionAvailable;
+        }
+    }
+
 }

nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/util/StorageSummaryEvent.java

@@ -19,7 +19,6 @@ package org.apache.nifi.provenance.util;
 
 import java.util.List;
 import java.util.Map;
-
 import org.apache.nifi.provenance.ProvenanceEventRecord;
 import org.apache.nifi.provenance.ProvenanceEventType;
 import org.apache.nifi.provenance.serialization.StorageSummary;
@@ -182,4 +181,14 @@ public class StorageSummaryEvent implements ProvenanceEventRecord {
     public Long getPreviousContentClaimOffset() {
         return event.getPreviousContentClaimOffset();
     }
+
+    /**
+     * Returns the best event identifier for this event (eventId if available, descriptive identifier if not yet persisted to allow for traceability).
+     *
+     * @return a descriptive event ID to allow tracing
+     */
+    @Override
+    public String getBestEventIdentifier() {
+        return Long.toString(getEventId());
+    }
 }

nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/resources/META-INF/services/org.apache.nifi.provenance.ProvenanceRepository

@@ -13,4 +13,5 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 org.apache.nifi.provenance.PersistentProvenanceRepository
-org.apache.nifi.provenance.WriteAheadProvenanceRepository
+org.apache.nifi.provenance.WriteAheadProvenanceRepository
+org.apache.nifi.provenance.EncryptedWriteAheadProvenanceRepository

nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/test/groovy/org/apache/nifi/provenance/EncryptedSchemaRecordReaderWriterTest.groovy

@@ -0,0 +1,281 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.provenance
+
+import org.apache.nifi.flowfile.FlowFile
+import org.apache.nifi.provenance.serialization.RecordReader
+import org.apache.nifi.provenance.serialization.RecordWriter
+import org.apache.nifi.provenance.toc.StandardTocReader
+import org.apache.nifi.provenance.toc.StandardTocWriter
+import org.apache.nifi.provenance.toc.TocReader
+import org.apache.nifi.provenance.toc.TocUtil
+import org.apache.nifi.provenance.toc.TocWriter
+import org.apache.nifi.util.file.FileUtils
+import org.bouncycastle.jce.provider.BouncyCastleProvider
+import org.bouncycastle.util.encoders.Hex
+import org.junit.After
+import org.junit.AfterClass
+import org.junit.Before
+import org.junit.BeforeClass
+import org.junit.ClassRule
+import org.junit.Test
+import org.junit.rules.TemporaryFolder
+import org.junit.runner.RunWith
+import org.junit.runners.JUnit4
+import org.slf4j.Logger
+import org.slf4j.LoggerFactory
+
+import javax.crypto.Cipher
+import javax.crypto.spec.SecretKeySpec
+import java.security.KeyManagementException
+import java.security.Security
+import java.util.concurrent.atomic.AtomicLong
+
+import static groovy.test.GroovyAssert.shouldFail
+import static org.apache.nifi.provenance.TestUtil.createFlowFile
+
+@RunWith(JUnit4.class)
+class EncryptedSchemaRecordReaderWriterTest extends AbstractTestRecordReaderWriter {
+    private static final Logger logger = LoggerFactory.getLogger(EncryptedSchemaRecordReaderWriterTest.class)
+
+    private static final String KEY_HEX_128 = "0123456789ABCDEFFEDCBA9876543210"
+    private static final String KEY_HEX_256 = KEY_HEX_128 * 2
+    private static final String KEY_HEX = isUnlimitedStrengthCryptoAvailable() ? KEY_HEX_256 : KEY_HEX_128
+    private static final String KEY_ID = "K1"
+
+    private static final String TRANSIT_URI = "nifi://unit-test"
+    private static final String PROCESSOR_TYPE = "Mock Processor"
+    private static final String COMPONENT_ID = "1234"
+
+    private static final int UNCOMPRESSED_BLOCK_SIZE = 1024 * 32
+    private static final int MAX_ATTRIBUTE_SIZE = 2048
+
+    private static final AtomicLong idGenerator = new AtomicLong(0L)
+    private File journalFile
+    private File tocFile
+
+    private static KeyProvider mockKeyProvider
+    private static ProvenanceEventEncryptor provenanceEventEncryptor = new AESProvenanceEventEncryptor()
+
+    @ClassRule
+    public static TemporaryFolder tempFolder = new TemporaryFolder()
+
+    private static String ORIGINAL_LOG_LEVEL
+
+    @BeforeClass
+    static void setUpOnce() throws Exception {
+        ORIGINAL_LOG_LEVEL = System.getProperty("org.slf4j.simpleLogger.log.org.apache.nifi.provenance")
+        System.setProperty("org.slf4j.simpleLogger.log.org.apache.nifi.provenance", "DEBUG")
+
+        Security.addProvider(new BouncyCastleProvider())
+
+        logger.metaClass.methodMissing = { String name, args ->
+            logger.info("[${name?.toUpperCase()}] ${(args as List).join(" ")}")
+        }
+
+        mockKeyProvider = [
+                getKey            : { String keyId ->
+                    logger.mock("Requesting key ID: ${keyId}")
+                    if (keyId == KEY_ID) {
+                        new SecretKeySpec(Hex.decode(KEY_HEX), "AES")
+                    } else {
+                        throw new KeyManagementException("${keyId} is not available")
+                    }
+                },
+                getAvailableKeyIds: { ->
+                    logger.mock("Available key IDs: [${KEY_ID}]")
+                    [KEY_ID]
+                },
+                keyExists         : { String keyId ->
+                    logger.mock("Checking availability of key ID: ${keyId}")
+                    keyId == KEY_ID
+                }] as KeyProvider
+        provenanceEventEncryptor.initialize(mockKeyProvider)
+    }
+
+    @Before
+    void setUp() throws Exception {
+        journalFile = new File("target/storage/${UUID.randomUUID()}/testEventIdFirstSchemaRecordReaderWriter")
+        tocFile = TocUtil.getTocFile(journalFile)
+        idGenerator.set(0L)
+    }
+
+    @After
+    void tearDown() throws Exception {
+        try {
+            FileUtils.deleteFile(journalFile.getParentFile(), true)
+        } catch (Exception e) {
+            logger.error(e.getMessage())
+        }
+    }
+
+    @AfterClass
+    static void tearDownOnce() throws Exception {
+        if (ORIGINAL_LOG_LEVEL) {
+            System.setProperty("org.slf4j.simpleLogger.log.org.apache.nifi.provenance", ORIGINAL_LOG_LEVEL)
+        }
+        try {
+            FileUtils.deleteFile(new File("target/storage"), true)
+        } catch (Exception e) {
+            logger.error(e)
+        }
+    }
+
+    private static boolean isUnlimitedStrengthCryptoAvailable() {
+        Cipher.getMaxAllowedKeyLength("AES") > 128
+    }
+
+    private static
+    final FlowFile buildFlowFile(Map attributes = [:], long id = idGenerator.getAndIncrement(), long fileSize = 3000L) {
+        if (!attributes?.uuid) {
+            attributes.uuid = UUID.randomUUID().toString()
+        }
+        createFlowFile(id, fileSize, attributes)
+    }
+
+    private
+    static ProvenanceEventRecord buildEventRecord(FlowFile flowfile = buildFlowFile(), ProvenanceEventType eventType = ProvenanceEventType.RECEIVE, String transitUri = TRANSIT_URI, String componentId = COMPONENT_ID, String componentType = PROCESSOR_TYPE, long eventTime = System.currentTimeMillis()) {
+        final ProvenanceEventBuilder builder = new StandardProvenanceEventRecord.Builder()
+        builder.setEventTime(eventTime)
+        builder.setEventType(eventType)
+        builder.setTransitUri(transitUri)
+        builder.fromFlowFile(flowfile)
+        builder.setComponentId(componentId)
+        builder.setComponentType(componentType)
+        builder.build()
+    }
+
+    @Override
+    protected RecordWriter createWriter(
+            final File file,
+            final TocWriter tocWriter, final boolean compressed, final int uncompressedBlockSize) throws IOException {
+        createWriter(file, tocWriter, compressed, uncompressedBlockSize, provenanceEventEncryptor)
+    }
+
+    protected static RecordWriter createWriter(
+            final File file,
+            final TocWriter tocWriter,
+            final boolean compressed,
+            final int uncompressedBlockSize, ProvenanceEventEncryptor encryptor) throws IOException {
+        return new EncryptedSchemaRecordWriter(file, idGenerator, tocWriter, compressed, uncompressedBlockSize, IdentifierLookup.EMPTY, encryptor, 1)
+    }
+
+    @Override
+    protected RecordReader createReader(
+            final InputStream inputStream,
+            final String journalFilename, final TocReader tocReader, final int maxAttributeSize) throws IOException {
+        return new EncryptedSchemaRecordReader(inputStream, journalFilename, tocReader, maxAttributeSize, provenanceEventEncryptor)
+    }
+
+    /**
+     * Build a record and write it to the repository with the encrypted writer. Recover with the encrypted reader and verify.
+     */
+    @Test
+    void testShouldWriteAndReadEncryptedRecord() {
+        // Arrange
+        final ProvenanceEventRecord record = buildEventRecord()
+        logger.info("Built sample PER: ${record}")
+
+        TocWriter tocWriter = new StandardTocWriter(tocFile, false, false)
+
+        RecordWriter encryptedWriter = createWriter(journalFile, tocWriter, false, UNCOMPRESSED_BLOCK_SIZE)
+        logger.info("Generated encrypted writer: ${encryptedWriter}")
+
+        // Act
+        int encryptedRecordId = idGenerator.get()
+        encryptedWriter.writeHeader(encryptedRecordId)
+        encryptedWriter.writeRecord(record)
+        encryptedWriter.close()
+        logger.info("Wrote encrypted record ${encryptedRecordId} to journal")
+
+        // Assert
+        TocReader tocReader = new StandardTocReader(tocFile)
+        final FileInputStream fis = new FileInputStream(journalFile)
+        final RecordReader reader = createReader(fis, journalFile.getName(), tocReader, MAX_ATTRIBUTE_SIZE)
+        logger.info("Generated encrypted reader: ${reader}")
+
+        ProvenanceEventRecord encryptedEvent = reader.nextRecord()
+        assert encryptedEvent
+        assert encryptedRecordId as long == encryptedEvent.getEventId()
+        assert record.componentId == encryptedEvent.getComponentId()
+        assert record.componentType == encryptedEvent.getComponentType()
+        logger.info("Successfully read encrypted record: ${encryptedEvent}")
+
+        assert !reader.nextRecord()
+    }
+
+    /**
+     * Build a record and write it with a standard writer and the encrypted writer to different repositories. Recover with the standard reader and the contents of the encrypted record should be unreadable.
+     */
+    @Test
+    void testShouldWriteEncryptedRecordAndPlainRecord() {
+        // Arrange
+        final ProvenanceEventRecord record = buildEventRecord()
+        logger.info("Built sample PER: ${record}")
+
+        TocWriter tocWriter = new StandardTocWriter(tocFile, false, false)
+
+        RecordWriter standardWriter = new EventIdFirstSchemaRecordWriter(journalFile, idGenerator, tocWriter, false, UNCOMPRESSED_BLOCK_SIZE, IdentifierLookup.EMPTY)
+        logger.info("Generated standard writer: ${standardWriter}")
+
+        File encryptedJournalFile = new File(journalFile.absolutePath + "_encrypted")
+        File encryptedTocFile = TocUtil.getTocFile(encryptedJournalFile)
+        TocWriter encryptedTocWriter = new StandardTocWriter(encryptedTocFile, false, false)
+        RecordWriter encryptedWriter = createWriter(encryptedJournalFile, encryptedTocWriter, false, UNCOMPRESSED_BLOCK_SIZE)
+        logger.info("Generated encrypted writer: ${encryptedWriter}")
+
+        // Act
+        int standardRecordId = idGenerator.get()
+        standardWriter.writeHeader(standardRecordId)
+        standardWriter.writeRecord(record)
+        standardWriter.close()
+        logger.info("Wrote standard record ${standardRecordId} to journal")
+
+        int encryptedRecordId = idGenerator.get()
+        encryptedWriter.writeHeader(encryptedRecordId)
+        encryptedWriter.writeRecord(record)
+        encryptedWriter.close()
+        logger.info("Wrote encrypted record ${encryptedRecordId} to journal")
+
+        // Assert
+        TocReader tocReader = new StandardTocReader(tocFile)
+        final FileInputStream fis = new FileInputStream(journalFile)
+        final RecordReader reader = new EventIdFirstSchemaRecordReader(fis, journalFile.getName(), tocReader, MAX_ATTRIBUTE_SIZE)
+        logger.info("Generated standard reader: ${reader}")
+
+        ProvenanceEventRecord standardEvent = reader.nextRecord()
+        assert standardEvent
+        assert standardRecordId as long == standardEvent.getEventId()
+        assert record.componentId == standardEvent.getComponentId()
+        assert record.componentType == standardEvent.getComponentType()
+        logger.info("Successfully read standard record: ${standardEvent}")
+
+        assert !reader.nextRecord()
+
+        // Demonstrate unable to read from encrypted file with standard reader
+        TocReader incompatibleTocReader = new StandardTocReader(encryptedTocFile)
+        final FileInputStream efis = new FileInputStream(encryptedJournalFile)
+        RecordReader incompatibleReader = new EventIdFirstSchemaRecordReader(efis, encryptedJournalFile.getName(), incompatibleTocReader, MAX_ATTRIBUTE_SIZE)
+        logger.info("Generated standard reader (attempting to read encrypted file): ${incompatibleReader}")
+
+        def msg = shouldFail(EOFException) {
+            ProvenanceEventRecord encryptedEvent = incompatibleReader.nextRecord()
+        }
+        logger.expected(msg)
+        assert msg =~ "EOFException: Failed to read field"
+    }
+}

nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/test/groovy/org/apache/nifi/provenance/EncryptedWriteAheadProvenanceRepositoryTest.groovy

@@ -0,0 +1,391 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License") you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.provenance
+
+import org.apache.nifi.events.EventReporter
+import org.apache.nifi.flowfile.FlowFile
+import org.apache.nifi.provenance.serialization.RecordReaders
+import org.apache.nifi.reporting.Severity
+import org.apache.nifi.util.file.FileUtils
+import org.bouncycastle.jce.provider.BouncyCastleProvider
+import org.junit.After
+import org.junit.AfterClass
+import org.junit.Before
+import org.junit.BeforeClass
+import org.junit.ClassRule
+import org.junit.Test
+import org.junit.rules.TemporaryFolder
+import org.junit.runner.RunWith
+import org.junit.runners.JUnit4
+import org.slf4j.Logger
+import org.slf4j.LoggerFactory
+
+import javax.crypto.Cipher
+import java.security.Security
+import java.util.concurrent.TimeUnit
+import java.util.concurrent.atomic.AtomicLong
+
+import static org.apache.nifi.provenance.TestUtil.createFlowFile
+
+@RunWith(JUnit4.class)
+class EncryptedWriteAheadProvenanceRepositoryTest {
+    private static final Logger logger = LoggerFactory.getLogger(EncryptedWriteAheadProvenanceRepositoryTest.class)
+
+    private static final String KEY_HEX_128 = "0123456789ABCDEFFEDCBA9876543210"
+    private static final String KEY_HEX_256 = KEY_HEX_128 * 2
+    private static final String KEY_HEX = isUnlimitedStrengthCryptoAvailable() ? KEY_HEX_256 : KEY_HEX_128
+    private static final String KEY_ID = "K1"
+
+    private static final String TRANSIT_URI = "nifi://unit-test"
+    private static final String PROCESSOR_TYPE = "Mock Processor"
+    private static final String COMPONENT_ID = "1234"
+
+    private static final AtomicLong recordId = new AtomicLong()
+
+    @ClassRule
+    public static TemporaryFolder tempFolder = new TemporaryFolder()
+
+    private ProvenanceRepository repo
+    private static RepositoryConfiguration config
+
+    public static final int DEFAULT_ROLLOVER_MILLIS = 2000
+    private EventReporter eventReporter
+    private List<ReportedEvent> reportedEvents = Collections.synchronizedList(new ArrayList<ReportedEvent>())
+
+    private static String ORIGINAL_LOG_LEVEL
+
+    @BeforeClass
+    static void setUpOnce() throws Exception {
+        ORIGINAL_LOG_LEVEL = System.getProperty("org.slf4j.simpleLogger.log.org.apache.nifi.provenance")
+        System.setProperty("org.slf4j.simpleLogger.log.org.apache.nifi.provenance", "DEBUG")
+
+        Security.addProvider(new BouncyCastleProvider())
+
+        logger.metaClass.methodMissing = { String name, args ->
+            logger.info("[${name?.toUpperCase()}] ${(args as List).join(" ")}")
+        }
+    }
+
+    @Before
+    void setUp() throws Exception {
+        reportedEvents?.clear()
+        eventReporter = createMockEventReporter()
+    }
+
+    @After
+    void tearDown() throws Exception {
+        closeRepo(repo, config)
+
+        // Reset the boolean determiner
+        RecordReaders.encryptionPropertiesRead = false
+        RecordReaders.isEncryptionAvailable = false
+    }
+
+    @AfterClass
+    static void tearDownOnce() throws Exception {
+        if (ORIGINAL_LOG_LEVEL) {
+            System.setProperty("org.slf4j.simpleLogger.log.org.apache.nifi.provenance", ORIGINAL_LOG_LEVEL)
+        }
+    }
+
+    private static boolean isUnlimitedStrengthCryptoAvailable() {
+        Cipher.getMaxAllowedKeyLength("AES") > 128
+    }
+
+    private static RepositoryConfiguration createConfiguration() {
+        RepositoryConfiguration config = new RepositoryConfiguration()
+        config.addStorageDirectory("1", new File("target/storage/" + UUID.randomUUID().toString()))
+        config.setCompressOnRollover(true)
+        config.setMaxEventFileLife(2000L, TimeUnit.SECONDS)
+        config.setCompressionBlockBytes(100)
+        return config
+    }
+
+    private static RepositoryConfiguration createEncryptedConfiguration() {
+        RepositoryConfiguration config = createConfiguration()
+        config.setEncryptionKeys([(KEY_ID): KEY_HEX])
+        config.setKeyId(KEY_ID)
+        config.setKeyProviderImplementation(StaticKeyProvider.class.name)
+        config
+    }
+
+    private EventReporter createMockEventReporter() {
+        [reportEvent: { Severity s, String c, String m ->
+            ReportedEvent event = new ReportedEvent(s, c, m)
+            reportedEvents.add(event)
+            logger.mock("Added ${event}")
+        }] as EventReporter
+    }
+
+    private void closeRepo(ProvenanceRepository repo = this.repo, RepositoryConfiguration config = this.config) throws IOException {
+        if (repo == null) {
+            return
+        }
+
+        try {
+            repo.close()
+        } catch (IOException ioe) {
+        }
+
+        // Delete all of the storage files. We do this in order to clean up the tons of files that
+        // we create but also to ensure that we have closed all of the file handles. If we leave any
+        // streams open, for instance, this will throw an IOException, causing our unit test to fail.
+        if (config != null) {
+            for (final File storageDir : config.getStorageDirectories().values()) {
+                int i
+                for (i = 0; i < 3; i++) {
+                    try {
+                        FileUtils.deleteFile(storageDir, true)
+                        break
+                    } catch (IOException ioe) {
+                        // if there is a virus scanner, etc. running in the background we may not be able to
+                        // delete the file. Wait a sec and try again.
+                        if (i == 2) {
+                            throw ioe
+                        } else {
+                            try {
+                                System.out.println("file: " + storageDir.toString() + " exists=" + storageDir.exists())
+                                FileUtils.deleteFile(storageDir, true)
+                                break
+                            } catch (final IOException ioe2) {
+                                // if there is a virus scanner, etc. running in the background we may not be able to
+                                // delete the file. Wait a sec and try again.
+                                if (i == 2) {
+                                    throw ioe2
+                                } else {
+                                    try {
+                                        Thread.sleep(1000L)
+                                    } catch (final InterruptedException ie) {
+                                    }
+                                }
+                            }
+                        }
+                    }
+                }
+            }
+        }
+    }
+
+    private static
+    final FlowFile buildFlowFile(Map attributes = [:], long id = recordId.getAndIncrement(), long fileSize = 3000L) {
+        if (!attributes?.uuid) {
+            attributes.uuid = UUID.randomUUID().toString()
+        }
+        createFlowFile(id, fileSize, attributes)
+    }
+
+    private
+    static ProvenanceEventRecord buildEventRecord(FlowFile flowfile = buildFlowFile(), ProvenanceEventType eventType = ProvenanceEventType.RECEIVE, String transitUri = TRANSIT_URI, String componentId = COMPONENT_ID, String componentType = PROCESSOR_TYPE, long eventTime = System.currentTimeMillis()) {
+        final ProvenanceEventBuilder builder = new StandardProvenanceEventRecord.Builder()
+        builder.setEventTime(eventTime)
+        builder.setEventType(eventType)
+        builder.setTransitUri(transitUri)
+        builder.fromFlowFile(flowfile)
+        builder.setComponentId(componentId)
+        builder.setComponentType(componentType)
+        builder.build()
+    }
+
+    /**
+     * This test operates on {@link PersistentProvenanceRepository} to verify the normal operations of existing implementations.
+     *
+     * @throws IOException
+     * @throws InterruptedException
+     */
+    @Test
+    void testPersistentProvenanceRepositoryShouldRegisterAndRetrieveEvents() throws IOException, InterruptedException {
+        // Arrange
+        config = createConfiguration()
+        config.setMaxEventFileCapacity(1L)
+        config.setMaxEventFileLife(1, TimeUnit.SECONDS)
+        repo = new PersistentProvenanceRepository(config, DEFAULT_ROLLOVER_MILLIS)
+        repo.initialize(eventReporter, null, null, IdentifierLookup.EMPTY)
+
+        Map attributes = ["abc": "xyz",
+                          "123": "456"]
+        final ProvenanceEventRecord record = buildEventRecord(buildFlowFile(attributes))
+
+        final int RECORD_COUNT = 10
+
+        // Act
+        RECORD_COUNT.times {
+            repo.registerEvent(record)
+        }
+
+        // Sleep to let the journal merge occur
+        Thread.sleep(1000L)
+
+        final List<ProvenanceEventRecord> recoveredRecords = repo.getEvents(0L, RECORD_COUNT + 1)
+
+        logger.info("Recovered ${recoveredRecords.size()} events: ")
+        recoveredRecords.each { logger.info("\t${it}") }
+
+        // Assert
+        assert recoveredRecords.size() == RECORD_COUNT
+        recoveredRecords.eachWithIndex { ProvenanceEventRecord recovered, int i ->
+            assert recovered.getEventId() == (i as Long)
+            assert recovered.getTransitUri() == TRANSIT_URI
+            assert recovered.getEventType() == ProvenanceEventType.RECEIVE
+            // The UUID was added later but we care that all attributes we provided are still there
+            assert recovered.getAttributes().entrySet().containsAll(attributes.entrySet())
+        }
+    }
+
+    /**
+     * This test operates on {@link WriteAheadProvenanceRepository} to verify the normal operations of existing implementations.
+     *
+     * @throws IOException
+     * @throws InterruptedException
+     */
+    @Test
+    void testWriteAheadProvenanceRepositoryShouldRegisterAndRetrieveEvents() throws IOException, InterruptedException {
+        // Arrange
+        config = createConfiguration()
+        // Needed until NIFI-3605 is implemented
+//        config.setMaxEventFileCapacity(1L)
+        config.setMaxEventFileCount(1)
+        config.setMaxEventFileLife(1, TimeUnit.SECONDS)
+        repo = new WriteAheadProvenanceRepository(config)
+        repo.initialize(eventReporter, null, null, IdentifierLookup.EMPTY)
+
+        Map attributes = ["abc": "xyz",
+                          "123": "456"]
+        final ProvenanceEventRecord record = buildEventRecord(buildFlowFile(attributes))
+
+        final int RECORD_COUNT = 10
+
+        // Act
+        RECORD_COUNT.times {
+            repo.registerEvent(record)
+        }
+
+        final List<ProvenanceEventRecord> recoveredRecords = repo.getEvents(0L, RECORD_COUNT + 1)
+
+        logger.info("Recovered ${recoveredRecords.size()} events: ")
+        recoveredRecords.each { logger.info("\t${it}") }
+
+        // Assert
+        assert recoveredRecords.size() == RECORD_COUNT
+        recoveredRecords.eachWithIndex { ProvenanceEventRecord recovered, int i ->
+            assert recovered.getEventId() == (i as Long)
+            assert recovered.getTransitUri() == TRANSIT_URI
+            assert recovered.getEventType() == ProvenanceEventType.RECEIVE
+            // The UUID was added later but we care that all attributes we provided are still there
+            assert recovered.getAttributes().entrySet().containsAll(attributes.entrySet())
+        }
+    }
+
+    @Test
+    void testShouldRegisterAndGetEvent() {
+        // Arrange
+
+        // Override the boolean determiner
+        RecordReaders.encryptionPropertiesRead = true
+        RecordReaders.isEncryptionAvailable = true
+
+        config = createEncryptedConfiguration()
+        // Needed until NIFI-3605 is implemented
+//        config.setMaxEventFileCapacity(1L)
+        config.setMaxEventFileCount(1)
+        config.setMaxEventFileLife(1, TimeUnit.SECONDS)
+        repo = new EncryptedWriteAheadProvenanceRepository(config)
+        repo.initialize(eventReporter, null, null, IdentifierLookup.EMPTY)
+
+        Map attributes = ["abc": "This is a plaintext attribute.",
+                          "123": "This is another plaintext attribute."]
+        final ProvenanceEventRecord record = buildEventRecord(buildFlowFile(attributes))
+
+        final long LAST_RECORD_ID = repo.getMaxEventId()
+
+        // Act
+        repo.registerEvent(record)
+
+        // Retrieve the event through the interface
+        ProvenanceEventRecord recoveredRecord = repo.getEvent(LAST_RECORD_ID + 1)
+        logger.info("Recovered ${recoveredRecord}")
+
+        // Assert
+        assert recoveredRecord.getEventId() == LAST_RECORD_ID + 1
+        assert recoveredRecord.getTransitUri() == TRANSIT_URI
+        assert recoveredRecord.getEventType() == ProvenanceEventType.RECEIVE
+        // The UUID was added later but we care that all attributes we provided are still there
+        assert recoveredRecord.getAttributes().entrySet().containsAll(attributes.entrySet())
+    }
+
+    @Test
+    void testShouldRegisterAndGetEvents() {
+        // Arrange
+        final int RECORD_COUNT = 10
+
+        // Override the boolean determiner
+        RecordReaders.encryptionPropertiesRead = true
+        RecordReaders.isEncryptionAvailable = true
+
+        config = createEncryptedConfiguration()
+        // Needed until NIFI-3605 is implemented
+//        config.setMaxEventFileCapacity(1L)
+        config.setMaxEventFileCount(1)
+        config.setMaxEventFileLife(1, TimeUnit.SECONDS)
+        repo = new EncryptedWriteAheadProvenanceRepository(config)
+        repo.initialize(eventReporter, null, null, IdentifierLookup.EMPTY)
+
+        Map attributes = ["abc": "This is a plaintext attribute.",
+                          "123": "This is another plaintext attribute."]
+        final List<ProvenanceEventRecord> records = []
+        RECORD_COUNT.times { int i ->
+            records << buildEventRecord(buildFlowFile(attributes + [count: i as String]))
+        }
+        logger.info("Generated ${RECORD_COUNT} records")
+
+        final long LAST_RECORD_ID = repo.getMaxEventId()
+
+        // Act
+        repo.registerEvents(records)
+        logger.info("Registered events")
+
+        // Retrieve the events through the interface
+        List<ProvenanceEventRecord> recoveredRecords = repo.getEvents(LAST_RECORD_ID + 1, RECORD_COUNT * 2)
+        logger.info("Recovered ${recoveredRecords.size()} records")
+
+        // Assert
+        recoveredRecords.eachWithIndex { ProvenanceEventRecord recoveredRecord, int i ->
+            assert recoveredRecord.getEventId() == LAST_RECORD_ID + 1 + i
+            assert recoveredRecord.getTransitUri() == TRANSIT_URI
+            assert recoveredRecord.getEventType() == ProvenanceEventType.RECEIVE
+            // The UUID was added later but we care that all attributes we provided are still there
+            assert recoveredRecord.getAttributes().entrySet().containsAll(attributes.entrySet())
+            assert recoveredRecord.getAttribute("count") == i as String
+        }
+    }
+
+    private static class ReportedEvent {
+        final Severity severity
+        final String category
+        final String message
+
+        ReportedEvent(final Severity severity, final String category, final String message) {
+            this.severity = severity
+            this.category = category
+            this.message = message
+        }
+
+        @Override
+        String toString() {
+            "ReportedEvent [${severity}] ${category}: ${message}"
+        }
+    }
+}

nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/test/java/org/apache/nifi/provenance/AbstractTestRecordReaderWriter.java

@@ -32,7 +32,6 @@ import java.util.List;
 import java.util.Map;
 import java.util.Optional;
 import java.util.UUID;
-
 import org.apache.nifi.provenance.serialization.RecordReader;
 import org.apache.nifi.provenance.serialization.RecordWriter;
 import org.apache.nifi.provenance.toc.StandardTocReader;
@@ -67,19 +66,25 @@ public abstract class AbstractTestRecordReaderWriter {
         writer.close();
 
         final TocReader tocReader = new StandardTocReader(tocFile);
+        final String expectedTransitUri = "nifi://unit-test";
+            final int expectedBlockIndex = 0;
 
+        assertRecoveredRecord(journalFile, tocReader, expectedTransitUri, expectedBlockIndex);
+
+        FileUtils.deleteFile(journalFile.getParentFile(), true);
+    }
+
+    private void assertRecoveredRecord(File journalFile, TocReader tocReader, String expectedTransitUri, int expectedBlockIndex) throws IOException {
         try (final FileInputStream fis = new FileInputStream(journalFile);
-            final RecordReader reader = createReader(fis, journalFile.getName(), tocReader, 2048)) {
-            assertEquals(0, reader.getBlockIndex());
-            reader.skipToBlock(0);
+             final RecordReader reader = createReader(fis, journalFile.getName(), tocReader, 2048)) {
+            assertEquals(expectedBlockIndex, reader.getBlockIndex());
+            reader.skipToBlock(expectedBlockIndex);
             final StandardProvenanceEventRecord recovered = reader.nextRecord();
             assertNotNull(recovered);
 
-            assertEquals("nifi://unit-test", recovered.getTransitUri());
+            assertEquals(expectedTransitUri, recovered.getTransitUri());
             assertNull(reader.nextRecord());
         }
-
-        FileUtils.deleteFile(journalFile.getParentFile(), true);
     }
 
 
@@ -96,16 +101,7 @@ public abstract class AbstractTestRecordReaderWriter {
 
         final TocReader tocReader = new StandardTocReader(tocFile);
 
-        try (final FileInputStream fis = new FileInputStream(journalFile);
-            final RecordReader reader = createReader(fis, journalFile.getName(), tocReader, 2048)) {
-            assertEquals(0, reader.getBlockIndex());
-            reader.skipToBlock(0);
-            final StandardProvenanceEventRecord recovered = reader.nextRecord();
-            assertNotNull(recovered);
-
-            assertEquals("nifi://unit-test", recovered.getTransitUri());
-            assertNull(reader.nextRecord());
-        }
+        assertRecoveredRecord(journalFile, tocReader, "nifi://unit-test", 0);
 
         FileUtils.deleteFile(journalFile.getParentFile(), true);
     }

nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-volatile-provenance-repository/src/main/java/org/apache/nifi/provenance/VolatileProvenanceRepository.java

@@ -16,6 +16,25 @@
  */
 package org.apache.nifi.provenance;
 
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Date;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.ThreadFactory;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.regex.Pattern;
 import org.apache.nifi.authorization.AccessDeniedException;
 import org.apache.nifi.authorization.AuthorizationResult;
 import org.apache.nifi.authorization.AuthorizationResult.Result;
@@ -42,26 +61,6 @@ import org.apache.nifi.util.RingBuffer.ForEachEvaluator;
 import org.apache.nifi.util.RingBuffer.IterationDirection;
 import org.apache.nifi.web.ResourceNotFoundException;
 
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.Date;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.ConcurrentMap;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import java.util.concurrent.ScheduledExecutorService;
-import java.util.concurrent.ThreadFactory;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicBoolean;
-import java.util.concurrent.atomic.AtomicInteger;
-import java.util.concurrent.atomic.AtomicLong;
-import java.util.regex.Pattern;
-
 public class VolatileProvenanceRepository implements ProvenanceRepository {
 
     // properties
@@ -472,7 +471,7 @@ public class VolatileProvenanceRepository implements ProvenanceRepository {
     }
 
     public Lineage computeLineage(final String flowFileUUID, final NiFiUser user) throws IOException {
-        return computeLineage(Collections.<String>singleton(flowFileUUID), user, LineageComputationType.FLOWFILE_LINEAGE, null);
+        return computeLineage(Collections.singleton(flowFileUUID), user, LineageComputationType.FLOWFILE_LINEAGE, null);
     }
 
     private Lineage computeLineage(final Collection<String> flowFileUuids, final NiFiUser user, final LineageComputationType computationType, final Long eventId) throws IOException {
@@ -497,7 +496,7 @@ public class VolatileProvenanceRepository implements ProvenanceRepository {
         final ProvenanceEventRecord event = getEvent(eventId);
         if (event == null) {
             final String userId = user.getIdentity();
-            final AsyncLineageSubmission result = new AsyncLineageSubmission(LineageComputationType.FLOWFILE_LINEAGE, eventId, Collections.<String>emptySet(), 1, userId);
+            final AsyncLineageSubmission result = new AsyncLineageSubmission(LineageComputationType.FLOWFILE_LINEAGE, eventId, Collections.emptySet(), 1, userId);
             result.getResult().setError("Could not find event with ID " + eventId);
             lineageSubmissionMap.put(result.getLineageIdentifier(), result);
             return result;
@@ -541,9 +540,9 @@ public class VolatileProvenanceRepository implements ProvenanceRepository {
 
         final ProvenanceEventRecord event = getEvent(eventId, user);
         if (event == null) {
-            final AsyncLineageSubmission submission = new AsyncLineageSubmission(LineageComputationType.EXPAND_PARENTS, eventId, Collections.<String>emptyList(), 1, userId);
+            final AsyncLineageSubmission submission = new AsyncLineageSubmission(LineageComputationType.EXPAND_PARENTS, eventId, Collections.emptyList(), 1, userId);
             lineageSubmissionMap.put(submission.getLineageIdentifier(), submission);
-            submission.getResult().update(Collections.<ProvenanceEventRecord> emptyList(), 0L);
+            submission.getResult().update(Collections.emptyList(), 0L);
             return submission;
         }
 
@@ -554,7 +553,7 @@ public class VolatileProvenanceRepository implements ProvenanceRepository {
             case CLONE:
                 return submitLineageComputation(event.getParentUuids(), user, LineageComputationType.EXPAND_PARENTS, eventId);
             default: {
-                final AsyncLineageSubmission submission = new AsyncLineageSubmission(LineageComputationType.EXPAND_PARENTS, eventId, Collections.<String>emptyList(), 1, userId);
+                final AsyncLineageSubmission submission = new AsyncLineageSubmission(LineageComputationType.EXPAND_PARENTS, eventId, Collections.emptyList(), 1, userId);
                 lineageSubmissionMap.put(submission.getLineageIdentifier(), submission);
                 submission.getResult().setError("Event ID " + eventId + " indicates an event of type " + event.getEventType() + " so its parents cannot be expanded");
                 return submission;
@@ -572,9 +571,9 @@ public class VolatileProvenanceRepository implements ProvenanceRepository {
 
         final ProvenanceEventRecord event = getEvent(eventId, user);
         if (event == null) {
-            final AsyncLineageSubmission submission = new AsyncLineageSubmission(LineageComputationType.EXPAND_CHILDREN, eventId, Collections.<String>emptyList(), 1, userId);
+            final AsyncLineageSubmission submission = new AsyncLineageSubmission(LineageComputationType.EXPAND_CHILDREN, eventId, Collections.emptyList(), 1, userId);
             lineageSubmissionMap.put(submission.getLineageIdentifier(), submission);
-            submission.getResult().update(Collections.<ProvenanceEventRecord> emptyList(), 0L);
+            submission.getResult().update(Collections.emptyList(), 0L);
             return submission;
         }
 
@@ -585,7 +584,7 @@ public class VolatileProvenanceRepository implements ProvenanceRepository {
             case CLONE:
                 return submitLineageComputation(event.getChildUuids(), user, LineageComputationType.EXPAND_CHILDREN, eventId);
             default: {
-                final AsyncLineageSubmission submission = new AsyncLineageSubmission(LineageComputationType.EXPAND_CHILDREN, eventId, Collections.<String>emptyList(), 1, userId);
+                final AsyncLineageSubmission submission = new AsyncLineageSubmission(LineageComputationType.EXPAND_CHILDREN, eventId, Collections.emptyList(), 1, userId);
                 lineageSubmissionMap.put(submission.getLineageIdentifier(), submission);
                 submission.getResult().setError("Event ID " + eventId + " indicates an event of type " + event.getEventType() + " so its children cannot be expanded");
                 return submission;
@@ -873,5 +872,15 @@ public class VolatileProvenanceRepository implements ProvenanceRepository {
         public Long getPreviousContentClaimOffset() {
             return record.getPreviousContentClaimOffset();
         }
+
+        /**
+         * Returns the best event identifier for this event (eventId if available, descriptive identifier if not yet persisted to allow for traceability).
+         *
+         * @return a descriptive event ID to allow tracing
+         */
+        @Override
+        public String getBestEventIdentifier() {
+            return Long.toString(getEventId());
+        }
     }
 }

nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/pom.xml

@@ -437,7 +437,7 @@
                         <exclude>src/test/resources/TestExtractGrok/simple_text.log</exclude>
                         <exclude>src/test/resources/TestExtractGrok/patterns</exclude>
                         <!-- This file is copied from https://github.com/jeremyh/jBCrypt because the binary is compiled for Java 8 and we must support Java 7 -->
-                        <exclude>src/main/java/org/apache/nifi/processors/standard/util/crypto/bcrypt/BCrypt.java</exclude>
+                        <exclude>src/main/java/org/apache/nifi/security/util/crypto/bcrypt/BCrypt.java</exclude>
                     </excludes>
                 </configuration>
             </plugin>

nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/EncryptContent.java

@@ -16,6 +16,16 @@
  */
 package org.apache.nifi.processors.standard;
 
+import java.nio.charset.StandardCharsets;
+import java.security.Security;
+import java.text.Normalizer;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+import java.util.concurrent.TimeUnit;
 import org.apache.commons.codec.DecoderException;
 import org.apache.commons.codec.binary.Hex;
 import org.apache.commons.lang3.StringUtils;
@@ -41,27 +51,16 @@ import org.apache.nifi.processor.Relationship;
 import org.apache.nifi.processor.exception.ProcessException;
 import org.apache.nifi.processor.io.StreamCallback;
 import org.apache.nifi.processor.util.StandardValidators;
-import org.apache.nifi.processors.standard.util.crypto.CipherUtility;
-import org.apache.nifi.processors.standard.util.crypto.KeyedEncryptor;
-import org.apache.nifi.processors.standard.util.crypto.OpenPGPKeyBasedEncryptor;
-import org.apache.nifi.processors.standard.util.crypto.OpenPGPPasswordBasedEncryptor;
-import org.apache.nifi.processors.standard.util.crypto.PasswordBasedEncryptor;
 import org.apache.nifi.security.util.EncryptionMethod;
 import org.apache.nifi.security.util.KeyDerivationFunction;
+import org.apache.nifi.security.util.crypto.CipherUtility;
+import org.apache.nifi.security.util.crypto.KeyedEncryptor;
+import org.apache.nifi.security.util.crypto.OpenPGPKeyBasedEncryptor;
+import org.apache.nifi.security.util.crypto.OpenPGPPasswordBasedEncryptor;
+import org.apache.nifi.security.util.crypto.PasswordBasedEncryptor;
 import org.apache.nifi.util.StopWatch;
 import org.bouncycastle.jce.provider.BouncyCastleProvider;
 
-import java.nio.charset.StandardCharsets;
-import java.security.Security;
-import java.text.Normalizer;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Set;
-import java.util.concurrent.TimeUnit;
-
 @EventDriven
 @SideEffectFree
 @SupportsBatching

nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/security/util/crypto/BcryptCipherProvider.java

@@ -14,23 +14,22 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.nifi.processors.standard.util.crypto;
+package org.apache.nifi.security.util.crypto;
 
-import org.apache.commons.lang3.StringUtils;
-import org.apache.nifi.processor.exception.ProcessException;
-import org.apache.nifi.processors.standard.util.crypto.bcrypt.BCrypt;
-import org.apache.nifi.security.util.EncryptionMethod;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import javax.crypto.Cipher;
-import javax.crypto.SecretKey;
-import javax.crypto.spec.SecretKeySpec;
 import java.nio.charset.StandardCharsets;
 import java.security.MessageDigest;
 import java.util.Arrays;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
+import javax.crypto.Cipher;
+import javax.crypto.SecretKey;
+import javax.crypto.spec.SecretKeySpec;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.nifi.processor.exception.ProcessException;
+import org.apache.nifi.security.util.EncryptionMethod;
+import org.apache.nifi.security.util.crypto.bcrypt.BCrypt;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 public class BcryptCipherProvider extends RandomIVPBECipherProvider {
     private static final Logger logger = LoggerFactory.getLogger(BcryptCipherProvider.class);

nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/security/util/crypto/CipherProviderFactory.java

@@ -14,16 +14,15 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.nifi.processors.standard.util.crypto;
+package org.apache.nifi.security.util.crypto;
 
+import java.util.HashMap;
+import java.util.Map;
 import org.apache.nifi.processor.exception.ProcessException;
 import org.apache.nifi.security.util.KeyDerivationFunction;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import java.util.HashMap;
-import java.util.Map;
-
 public class CipherProviderFactory {
     private static final Logger logger = LoggerFactory.getLogger(CipherProviderFactory.class);
 

nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/security/util/crypto/KeyedEncryptor.java

@@ -14,8 +14,15 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.nifi.processors.standard.util.crypto;
+package org.apache.nifi.security.util.crypto;
 
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.security.NoSuchAlgorithmException;
+import javax.crypto.Cipher;
+import javax.crypto.SecretKey;
+import javax.crypto.spec.SecretKeySpec;
 import org.apache.commons.lang3.StringUtils;
 import org.apache.nifi.processor.exception.ProcessException;
 import org.apache.nifi.processor.io.StreamCallback;
@@ -23,14 +30,6 @@ import org.apache.nifi.processors.standard.EncryptContent.Encryptor;
 import org.apache.nifi.security.util.EncryptionMethod;
 import org.apache.nifi.security.util.KeyDerivationFunction;
 
-import javax.crypto.Cipher;
-import javax.crypto.SecretKey;
-import javax.crypto.spec.SecretKeySpec;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.OutputStream;
-import java.security.NoSuchAlgorithmException;
-
 public class KeyedEncryptor implements Encryptor {
 
     private EncryptionMethod encryptionMethod;

nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/security/util/crypto/NiFiLegacyCipherProvider.java

@@ -14,20 +14,19 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.nifi.processors.standard.util.crypto;
+package org.apache.nifi.security.util.crypto;
 
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.security.SecureRandom;
+import javax.crypto.Cipher;
 import org.apache.nifi.processor.exception.ProcessException;
 import org.apache.nifi.security.util.EncryptionMethod;
 import org.apache.nifi.stream.io.StreamUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import javax.crypto.Cipher;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.OutputStream;
-import java.security.SecureRandom;
-
 /**
  * Provides a cipher initialized with the original NiFi key derivation process for password-based encryption (MD5 @ 1000 iterations). This is not a secure
  * {@link org.apache.nifi.security.util.KeyDerivationFunction} (KDF) and should no longer be used.
@@ -45,7 +44,7 @@ public class NiFiLegacyCipherProvider extends OpenSSLPKCS5CipherProvider impleme
     private static final int ITERATION_COUNT = 1000;
 
     /**
-     * Returns an initialized cipher for the specified algorithm. The key (and IV if necessary) are derived using the NiFi legacy code, based on @see org.apache.nifi.processors.standard.util.crypto
+     * Returns an initialized cipher for the specified algorithm. The key (and IV if necessary) are derived using the NiFi legacy code, based on @see org.apache.nifi.crypto
      * .OpenSSLPKCS5CipherProvider#getCipher(java.lang.String, java.lang.String, java.lang.String, byte[], boolean) [essentially {@code MD5(password || salt) * 1000 }].
      *
      * @param encryptionMethod the {@link EncryptionMethod}

nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/security/util/crypto/OpenPGPKeyBasedEncryptor.java

@@ -14,8 +14,20 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.nifi.processors.standard.util.crypto;
+package org.apache.nifi.security.util.crypto;
 
+import static org.apache.nifi.processors.standard.util.PGPUtil.BLOCK_SIZE;
+import static org.apache.nifi.processors.standard.util.PGPUtil.BUFFER_SIZE;
+
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.security.NoSuchProviderException;
+import java.security.SecureRandom;
+import java.util.Date;
+import java.util.Iterator;
+import java.util.zip.Deflater;
 import org.apache.nifi.processor.exception.ProcessException;
 import org.apache.nifi.processor.io.StreamCallback;
 import org.apache.nifi.processors.standard.EncryptContent;
@@ -51,19 +63,6 @@ import org.bouncycastle.openpgp.operator.jcajce.JcePublicKeyKeyEncryptionMethodG
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import java.io.FileInputStream;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.OutputStream;
-import java.security.NoSuchProviderException;
-import java.security.SecureRandom;
-import java.util.Date;
-import java.util.Iterator;
-import java.util.zip.Deflater;
-
-import static org.apache.nifi.processors.standard.util.PGPUtil.BLOCK_SIZE;
-import static org.apache.nifi.processors.standard.util.PGPUtil.BUFFER_SIZE;
-
 public class OpenPGPKeyBasedEncryptor implements Encryptor {
     private static final Logger logger = LoggerFactory.getLogger(OpenPGPPasswordBasedEncryptor.class);
 

nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/security/util/crypto/OpenPGPPasswordBasedEncryptor.java

@@ -14,8 +14,13 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.nifi.processors.standard.util.crypto;
+package org.apache.nifi.security.util.crypto;
 
+import static org.bouncycastle.openpgp.PGPUtil.getDecoderStream;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
 import org.apache.nifi.processor.exception.ProcessException;
 import org.apache.nifi.processor.io.StreamCallback;
 import org.apache.nifi.processors.standard.EncryptContent.Encryptor;
@@ -35,12 +40,6 @@ import org.bouncycastle.openpgp.operator.jcajce.JcePBEKeyEncryptionMethodGenerat
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.OutputStream;
-
-import static org.bouncycastle.openpgp.PGPUtil.getDecoderStream;
-
 public class OpenPGPPasswordBasedEncryptor implements Encryptor {
     private static final Logger logger = LoggerFactory.getLogger(OpenPGPPasswordBasedEncryptor.class);
 

nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/security/util/crypto/OpenSSLPKCS5CipherProvider.java

@@ -14,21 +14,8 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.nifi.processors.standard.util.crypto;
+package org.apache.nifi.security.util.crypto;
 
-import org.apache.commons.lang3.StringUtils;
-import org.apache.nifi.processor.exception.ProcessException;
-import org.apache.nifi.security.util.EncryptionMethod;
-import org.apache.nifi.stream.io.StreamUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import javax.crypto.Cipher;
-import javax.crypto.NoSuchPaddingException;
-import javax.crypto.SecretKey;
-import javax.crypto.SecretKeyFactory;
-import javax.crypto.spec.PBEKeySpec;
-import javax.crypto.spec.PBEParameterSpec;
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.OutputStream;
@@ -40,6 +27,18 @@ import java.security.NoSuchProviderException;
 import java.security.SecureRandom;
 import java.security.spec.InvalidKeySpecException;
 import java.util.Arrays;
+import javax.crypto.Cipher;
+import javax.crypto.NoSuchPaddingException;
+import javax.crypto.SecretKey;
+import javax.crypto.SecretKeyFactory;
+import javax.crypto.spec.PBEKeySpec;
+import javax.crypto.spec.PBEParameterSpec;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.nifi.processor.exception.ProcessException;
+import org.apache.nifi.security.util.EncryptionMethod;
+import org.apache.nifi.stream.io.StreamUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 public class OpenSSLPKCS5CipherProvider implements PBECipherProvider {
     private static final Logger logger = LoggerFactory.getLogger(OpenSSLPKCS5CipherProvider.class);

nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/security/util/crypto/PBECipherProvider.java

@@ -14,14 +14,13 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.nifi.processors.standard.util.crypto;
+package org.apache.nifi.security.util.crypto;
 
-import org.apache.nifi.security.util.EncryptionMethod;
-
-import javax.crypto.Cipher;
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.OutputStream;
+import javax.crypto.Cipher;
+import org.apache.nifi.security.util.EncryptionMethod;
 
 public interface PBECipherProvider extends CipherProvider {
 

nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/security/util/crypto/PBKDF2CipherProvider.java

@@ -14,8 +14,13 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.nifi.processors.standard.util.crypto;
+package org.apache.nifi.security.util.crypto;
 
+import java.nio.charset.StandardCharsets;
+import java.security.SecureRandom;
+import javax.crypto.Cipher;
+import javax.crypto.SecretKey;
+import javax.crypto.spec.SecretKeySpec;
 import org.apache.commons.lang3.StringUtils;
 import org.apache.nifi.processor.exception.ProcessException;
 import org.apache.nifi.security.util.EncryptionMethod;
@@ -30,12 +35,6 @@ import org.bouncycastle.crypto.params.KeyParameter;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import javax.crypto.Cipher;
-import javax.crypto.SecretKey;
-import javax.crypto.spec.SecretKeySpec;
-import java.nio.charset.StandardCharsets;
-import java.security.SecureRandom;
-
 public class PBKDF2CipherProvider extends RandomIVPBECipherProvider {
     private static final Logger logger = LoggerFactory.getLogger(PBKDF2CipherProvider.class);
     private static final int DEFAULT_SALT_LENGTH = 16;

nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/security/util/crypto/PasswordBasedEncryptor.java

@@ -14,8 +14,15 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.nifi.processors.standard.util.crypto;
+package org.apache.nifi.security.util.crypto;
 
+import java.io.EOFException;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.security.NoSuchAlgorithmException;
+import javax.crypto.Cipher;
+import javax.crypto.spec.PBEKeySpec;
 import org.apache.commons.lang3.StringUtils;
 import org.apache.nifi.processor.exception.ProcessException;
 import org.apache.nifi.processor.io.StreamCallback;
@@ -23,14 +30,6 @@ import org.apache.nifi.processors.standard.EncryptContent.Encryptor;
 import org.apache.nifi.security.util.EncryptionMethod;
 import org.apache.nifi.security.util.KeyDerivationFunction;
 
-import javax.crypto.Cipher;
-import javax.crypto.spec.PBEKeySpec;
-import java.io.EOFException;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.OutputStream;
-import java.security.NoSuchAlgorithmException;
-
 public class PasswordBasedEncryptor implements Encryptor {
 
     private EncryptionMethod encryptionMethod;

nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/security/util/crypto/RandomIVPBECipherProvider.java

@@ -14,17 +14,16 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.nifi.processors.standard.util.crypto;
+package org.apache.nifi.security.util.crypto;
 
-import org.apache.nifi.processor.exception.ProcessException;
-import org.apache.nifi.security.util.EncryptionMethod;
-import org.slf4j.Logger;
-
-import javax.crypto.Cipher;
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.OutputStream;
 import java.nio.charset.StandardCharsets;
+import javax.crypto.Cipher;
+import org.apache.nifi.processor.exception.ProcessException;
+import org.apache.nifi.security.util.EncryptionMethod;
+import org.slf4j.Logger;
 
 public abstract class RandomIVPBECipherProvider implements PBECipherProvider {
     static final byte[] SALT_DELIMITER = "NiFiSALT".getBytes(StandardCharsets.UTF_8);

nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/security/util/crypto/ScryptCipherProvider.java

@@ -14,27 +14,26 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.nifi.processors.standard.util.crypto;
+package org.apache.nifi.security.util.crypto;
 
-import org.apache.commons.codec.DecoderException;
-import org.apache.commons.codec.binary.Base64;
-import org.apache.commons.codec.binary.Hex;
-import org.apache.commons.lang3.StringUtils;
-import org.apache.nifi.processor.exception.ProcessException;
-import org.apache.nifi.processors.standard.util.crypto.scrypt.Scrypt;
-import org.apache.nifi.security.util.EncryptionMethod;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import javax.crypto.Cipher;
-import javax.crypto.SecretKey;
-import javax.crypto.spec.SecretKeySpec;
 import java.nio.charset.StandardCharsets;
 import java.security.SecureRandom;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
+import javax.crypto.Cipher;
+import javax.crypto.SecretKey;
+import javax.crypto.spec.SecretKeySpec;
+import org.apache.commons.codec.DecoderException;
+import org.apache.commons.codec.binary.Base64;
+import org.apache.commons.codec.binary.Hex;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.nifi.processor.exception.ProcessException;
+import org.apache.nifi.security.util.EncryptionMethod;
+import org.apache.nifi.security.util.crypto.scrypt.Scrypt;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 public class ScryptCipherProvider extends RandomIVPBECipherProvider {
     private static final Logger logger = LoggerFactory.getLogger(ScryptCipherProvider.class);

nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/security/util/crypto/bcrypt/BCrypt.java

@@ -12,7 +12,7 @@
 // ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
 // OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
 
-package org.apache.nifi.processors.standard.util.crypto.bcrypt;
+package org.apache.nifi.security.util.crypto.bcrypt;
 
 import java.io.UnsupportedEncodingException;
 import java.security.SecureRandom;

nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/security/util/crypto/scrypt/Scrypt.java

@@ -14,24 +14,23 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.nifi.processors.standard.util.crypto.scrypt;
+package org.apache.nifi.security.util.crypto.scrypt;
 
-import org.apache.commons.codec.binary.Base64;
-import org.apache.commons.lang3.StringUtils;
-import org.apache.nifi.processors.standard.util.crypto.CipherUtility;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
+import static java.lang.Integer.MAX_VALUE;
+import static java.lang.System.arraycopy;
 
-import javax.crypto.Mac;
-import javax.crypto.spec.SecretKeySpec;
 import java.nio.charset.StandardCharsets;
 import java.security.GeneralSecurityException;
 import java.security.SecureRandom;
 import java.util.ArrayList;
 import java.util.List;
-
-import static java.lang.Integer.MAX_VALUE;
-import static java.lang.System.arraycopy;
+import javax.crypto.Mac;
+import javax.crypto.spec.SecretKeySpec;
+import org.apache.commons.codec.binary.Base64;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.nifi.security.util.crypto.CipherUtility;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 
 /**

nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/groovy/org/apache/nifi/processors/standard/TestEncryptContentGroovy.groovy

@@ -17,10 +17,10 @@
 package org.apache.nifi.processors.standard
 
 import org.apache.nifi.components.ValidationResult
-import org.apache.nifi.processors.standard.util.crypto.CipherUtility
-import org.apache.nifi.processors.standard.util.crypto.PasswordBasedEncryptor
 import org.apache.nifi.security.util.EncryptionMethod
 import org.apache.nifi.security.util.KeyDerivationFunction
+import org.apache.nifi.security.util.crypto.CipherUtility
+import org.apache.nifi.security.util.crypto.PasswordBasedEncryptor
 import org.apache.nifi.util.MockFlowFile
 import org.apache.nifi.util.MockProcessContext
 import org.apache.nifi.util.TestRunner

nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/groovy/org/apache/nifi/security/util/crypto/BcryptCipherProviderGroovyTest.groovy

@@ -14,12 +14,12 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.nifi.processors.standard.util.crypto
+package org.apache.nifi.security.util.crypto
 
 import org.apache.commons.codec.binary.Base64
 import org.apache.commons.codec.binary.Hex
-import org.apache.nifi.processors.standard.util.crypto.bcrypt.BCrypt
 import org.apache.nifi.security.util.EncryptionMethod
+import org.apache.nifi.security.util.crypto.bcrypt.BCrypt
 import org.bouncycastle.jce.provider.BouncyCastleProvider
 import org.junit.After
 import org.junit.Assume
@@ -30,7 +30,6 @@ import org.junit.Test
 import org.junit.runner.RunWith
 import org.junit.runners.JUnit4
 import org.slf4j.Logger
-
 import org.slf4j.LoggerFactory
 
 import javax.crypto.Cipher

nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/groovy/org/apache/nifi/security/util/crypto/CipherProviderFactoryGroovyTest.groovy

@@ -14,7 +14,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.nifi.processors.standard.util.crypto
+package org.apache.nifi.security.util.crypto
 
 import org.apache.nifi.security.util.KeyDerivationFunction
 import org.bouncycastle.jce.provider.BouncyCastleProvider

nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/groovy/org/apache/nifi/security/util/crypto/KeyedEncryptorGroovyTest.groovy

@@ -14,7 +14,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.nifi.processors.standard.util.crypto
+package org.apache.nifi.security.util.crypto
 
 import org.apache.commons.codec.binary.Hex
 import org.apache.nifi.processor.io.StreamCallback

nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/groovy/org/apache/nifi/security/util/crypto/NiFiLegacyCipherProviderGroovyTest.groovy

@@ -14,12 +14,17 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.nifi.processors.standard.util.crypto
+package org.apache.nifi.security.util.crypto
 
 import org.apache.commons.codec.binary.Hex
 import org.apache.nifi.security.util.EncryptionMethod
 import org.bouncycastle.jce.provider.BouncyCastleProvider
-import org.junit.*
+import org.junit.After
+import org.junit.Assume
+import org.junit.Before
+import org.junit.BeforeClass
+import org.junit.Ignore
+import org.junit.Test
 import org.junit.runner.RunWith
 import org.junit.runners.JUnit4
 import org.slf4j.Logger

nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/groovy/org/apache/nifi/security/util/crypto/OpenSSLPKCS5CipherProviderGroovyTest.groovy

@@ -14,12 +14,16 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.nifi.processors.standard.util.crypto
+package org.apache.nifi.security.util.crypto
 
 import org.apache.commons.codec.binary.Hex
 import org.apache.nifi.security.util.EncryptionMethod
 import org.bouncycastle.jce.provider.BouncyCastleProvider
-import org.junit.*
+import org.junit.After
+import org.junit.Assume
+import org.junit.Before
+import org.junit.BeforeClass
+import org.junit.Test
 import org.junit.runner.RunWith
 import org.junit.runners.JUnit4
 import org.slf4j.Logger

nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/groovy/org/apache/nifi/security/util/crypto/PBKDF2CipherProviderGroovyTest.groovy

@@ -14,12 +14,17 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.nifi.processors.standard.util.crypto
+package org.apache.nifi.security.util.crypto
 
 import org.apache.commons.codec.binary.Hex
 import org.apache.nifi.security.util.EncryptionMethod
 import org.bouncycastle.jce.provider.BouncyCastleProvider
-import org.junit.*
+import org.junit.After
+import org.junit.Assume
+import org.junit.Before
+import org.junit.BeforeClass
+import org.junit.Ignore
+import org.junit.Test
 import org.junit.runner.RunWith
 import org.junit.runners.JUnit4
 import org.slf4j.Logger

nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/groovy/org/apache/nifi/security/util/crypto/PasswordBasedEncryptorGroovyTest.groovy

@@ -14,7 +14,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.nifi.processors.standard.util.crypto
+package org.apache.nifi.security.util.crypto
 
 import org.apache.commons.codec.binary.Hex
 import org.apache.nifi.processor.io.StreamCallback

nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/groovy/org/apache/nifi/security/util/crypto/ScryptCipherProviderGroovyTest.groovy

@@ -14,12 +14,12 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.nifi.processors.standard.util.crypto
+package org.apache.nifi.security.util.crypto
 
 import org.apache.commons.codec.binary.Base64
 import org.apache.commons.codec.binary.Hex
-import org.apache.nifi.processors.standard.util.crypto.scrypt.Scrypt
 import org.apache.nifi.security.util.EncryptionMethod
+import org.apache.nifi.security.util.crypto.scrypt.Scrypt
 import org.bouncycastle.jce.provider.BouncyCastleProvider
 import org.junit.After
 import org.junit.Assume

nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/groovy/org/apache/nifi/security/util/crypto/scrypt/ScryptGroovyTest.groovy

@@ -14,7 +14,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.nifi.processors.standard.util.crypto.scrypt
+package org.apache.nifi.security.util.crypto.scrypt
 
 import org.apache.commons.codec.binary.Hex
 import org.bouncycastle.jce.provider.BouncyCastleProvider

nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestEncryptContent.java

@@ -16,12 +16,17 @@
  */
 package org.apache.nifi.processors.standard;
 
+import java.io.File;
+import java.io.IOException;
+import java.nio.file.Paths;
+import java.security.Security;
+import java.util.Collection;
 import org.apache.commons.codec.binary.Hex;
 import org.apache.nifi.components.ValidationResult;
-import org.apache.nifi.processors.standard.util.crypto.CipherUtility;
-import org.apache.nifi.processors.standard.util.crypto.PasswordBasedEncryptor;
 import org.apache.nifi.security.util.EncryptionMethod;
 import org.apache.nifi.security.util.KeyDerivationFunction;
+import org.apache.nifi.security.util.crypto.CipherUtility;
+import org.apache.nifi.security.util.crypto.PasswordBasedEncryptor;
 import org.apache.nifi.util.MockFlowFile;
 import org.apache.nifi.util.MockProcessContext;
 import org.apache.nifi.util.TestRunner;
@@ -34,12 +39,6 @@ import org.junit.Test;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import java.io.File;
-import java.io.IOException;
-import java.nio.file.Paths;
-import java.security.Security;
-import java.util.Collection;
-
 public class TestEncryptContent {
 
     private static final Logger logger = LoggerFactory.getLogger(TestEncryptContent.class);

nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/security/util/crypto/OpenPGPKeyBasedEncryptorTest.java

@@ -14,8 +14,17 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.nifi.processors.standard.util.crypto;
+package org.apache.nifi.security.util.crypto;
 
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.nio.file.Files;
+import java.nio.file.Paths;
+import java.security.Security;
 import org.apache.commons.codec.binary.Hex;
 import org.apache.nifi.processor.io.StreamCallback;
 import org.apache.nifi.security.util.EncryptionMethod;
@@ -28,16 +37,6 @@ import org.junit.Test;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import java.io.ByteArrayInputStream;
-import java.io.ByteArrayOutputStream;
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.InputStream;
-import java.io.OutputStream;
-import java.nio.file.Files;
-import java.nio.file.Paths;
-import java.security.Security;
-
 public class OpenPGPKeyBasedEncryptorTest {
     private static final Logger logger = LoggerFactory.getLogger(OpenPGPKeyBasedEncryptorTest.class);
 

nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/security/util/crypto/OpenPGPPasswordBasedEncryptorTest.java

@@ -14,8 +14,17 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.nifi.processors.standard.util.crypto;
+package org.apache.nifi.security.util.crypto;
 
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.nio.file.Files;
+import java.nio.file.Paths;
+import java.security.Security;
 import org.apache.commons.codec.binary.Hex;
 import org.apache.nifi.processor.io.StreamCallback;
 import org.apache.nifi.security.util.EncryptionMethod;
@@ -28,16 +37,6 @@ import org.junit.Test;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import java.io.ByteArrayInputStream;
-import java.io.ByteArrayOutputStream;
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.InputStream;
-import java.io.OutputStream;
-import java.nio.file.Files;
-import java.nio.file.Paths;
-import java.security.Security;
-
 public class OpenPGPPasswordBasedEncryptorTest {
     private static final Logger logger = LoggerFactory.getLogger(OpenPGPPasswordBasedEncryptorTest.class);
 

nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/resources/logback-test.xml

@@ -39,7 +39,7 @@
     </appender>
     <!-- valid logging levels: TRACE, DEBUG, INFO, WARN, ERROR -->
     <logger name="org.apache.nifi" level="INFO"/>
-    <logger name="org.apache.nifi.processors.standard.util.crypto" level="DEBUG"/>
+    <logger name="org.apache.nifi.crypto" level="DEBUG"/>
 
     <!-- Logger for managing logging statements for nifi clusters. -->
     <logger name="org.apache.nifi.cluster" level="INFO"/>