keys = new HashMap<>();
+
+ if (StringUtils.isBlank(filepath)) {
+ throw new KeyManagementException("The key provider file is not present and readable");
+ }
+ File file = new File(filepath);
+ if (!file.exists() || !file.canRead()) {
+ throw new KeyManagementException("The key provider file is not present and readable");
+ }
+
+ try (BufferedReader br = new BufferedReader(new FileReader(file))) {
+ AESKeyedCipherProvider masterCipherProvider = new AESKeyedCipherProvider();
+
+ String line;
+ int l = 1;
+ while ((line = br.readLine()) != null) {
+ String[] components = line.split("=", 2);
+ if (components.length != 2 || StringUtils.isAnyEmpty(components)) {
+ logger.warn("Line " + l + " is not properly formatted -- keyId=Base64EncodedKey...");
+ }
+ String keyId = components[0];
+ if (StringUtils.isNotEmpty(keyId)) {
+ try {
+ byte[] base64Bytes = Base64.getDecoder().decode(components[1]);
+ byte[] ivBytes = Arrays.copyOfRange(base64Bytes, 0, IV_LENGTH);
+
+ Cipher masterCipher = null;
+ try {
+ masterCipher = masterCipherProvider.getCipher(EncryptionMethod.AES_GCM, masterKey, ivBytes, false);
+ } catch (Exception e) {
+ throw new KeyManagementException("Error building cipher to decrypt FileBaseKeyProvider definition at " + filepath, e);
+ }
+ byte[] individualKeyBytes = masterCipher.doFinal(Arrays.copyOfRange(base64Bytes, IV_LENGTH, base64Bytes.length));
+
+ SecretKey key = new SecretKeySpec(individualKeyBytes, "AES");
+ logger.debug("Read and decrypted key for " + keyId);
+ if (keys.containsKey(keyId)) {
+ logger.warn("Multiple key values defined for " + keyId + " -- using most recent value");
+ }
+ keys.put(keyId, key);
+ } catch (IllegalArgumentException e) {
+ logger.error("Encountered an error decoding Base64 for " + keyId + ": " + e.getLocalizedMessage());
+ } catch (BadPaddingException | IllegalBlockSizeException e) {
+ logger.error("Encountered an error decrypting key for " + keyId + ": " + e.getLocalizedMessage());
+ }
+ }
+ l++;
+ }
+
+ if (keys.isEmpty()) {
+ throw new KeyManagementException("The provided file contained no valid keys");
+ }
+
+ logger.info("Read " + keys.size() + " keys from FileBasedKeyProvider " + filepath);
+ return keys;
+ } catch (IOException e) {
+ throw new KeyManagementException("Error reading FileBasedKeyProvider definition at " + filepath, e);
+ }
+
+ }
+
+ public static boolean isProvenanceRepositoryEncryptionConfigured(NiFiProperties niFiProperties) {
+ final String implementationClassName = niFiProperties.getProperty(NiFiProperties.PROVENANCE_REPO_IMPLEMENTATION_CLASS);
+ // Referencing EWAPR.class.getName() would require a dependency on the module
+ boolean encryptedRepo = "org.apache.nifi.provenance.EncryptedWriteAheadProvenanceRepository".equals(implementationClassName);
+ boolean keyProviderConfigured = isValidKeyProvider(
+ niFiProperties.getProperty(NiFiProperties.PROVENANCE_REPO_ENCRYPTION_KEY_PROVIDER_IMPLEMENTATION_CLASS),
+ niFiProperties.getProperty(NiFiProperties.PROVENANCE_REPO_ENCRYPTION_KEY_PROVIDER_LOCATION),
+ niFiProperties.getProvenanceRepoEncryptionKeyId(),
+ niFiProperties.getProvenanceRepoEncryptionKeys());
+
+ return encryptedRepo && keyProviderConfigured;
+ }
+}
diff --git a/nifi-commons/nifi-data-provenance-utils/src/main/java/org/apache/nifi/provenance/EncryptionException.java b/nifi-commons/nifi-data-provenance-utils/src/main/java/org/apache/nifi/provenance/EncryptionException.java
new file mode 100644
index 0000000000..05c52e5a21
--- /dev/null
+++ b/nifi-commons/nifi-data-provenance-utils/src/main/java/org/apache/nifi/provenance/EncryptionException.java
@@ -0,0 +1,92 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.provenance;
+
+public class EncryptionException extends Throwable {
+ /**
+ * Constructs a new EncryptionException with {@code null} as its detail message.
+ * The cause is not initialized, and may subsequently be initialized by a
+ * call to {@link #initCause}.
+ *
+ *
The {@link #fillInStackTrace()} method is called to initialize
+ * the stack trace data in the newly created exception.
+ */
+ public EncryptionException() {
+ super();
+ }
+
+ /**
+ * Constructs a new EncryptionException with the specified detail message. The
+ * cause is not initialized, and may subsequently be initialized by
+ * a call to {@link #initCause}.
+ *
+ *
The {@link #fillInStackTrace()} method is called to initialize
+ * the stack trace data in the newly created exception.
+ *
+ * @param message the detail message. The detail message is saved for
+ * later retrieval by the {@link #getMessage()} method.
+ */
+ public EncryptionException(String message) {
+ super(message);
+ }
+
+ /**
+ * Constructs a new EncryptionException with the specified detail message and
+ * cause.
Note that the detail message associated with
+ * {@code cause} is not automatically incorporated in
+ * this exception's detail message.
+ *
+ *
The {@link #fillInStackTrace()} method is called to initialize
+ * the stack trace data in the newly created throwable.
+ *
+ * @param message the detail message (which is saved for later retrieval
+ * by the {@link #getMessage()} method).
+ * @param cause the cause (which is saved for later retrieval by the
+ * {@link #getCause()} method). (A {@code null} value is
+ * permitted, and indicates that the cause is nonexistent or
+ * unknown.)
+ * @since 1.4
+ */
+ public EncryptionException(String message, Throwable cause) {
+ super(message, cause);
+ }
+
+ /**
+ * Constructs a new EncryptionException with the specified cause and a detail
+ * message of {@code (cause==null ? null : cause.toString())} (which
+ * typically contains the class and detail message of {@code cause}).
+ * This constructor is useful for exceptions that are little more than
+ * wrappers for other exceptions.
+ *
+ *
The {@link #fillInStackTrace()} method is called to initialize
+ * the stack trace data in the newly created exception.
+ *
+ * @param cause the cause (which is saved for later retrieval by the
+ * {@link #getCause()} method). (A {@code null} value is
+ * permitted, and indicates that the cause is nonexistent or
+ * unknown.)
+ * @since 1.4
+ */
+ public EncryptionException(Throwable cause) {
+ super(cause);
+ }
+
+ @Override
+ public String toString() {
+ return "EncryptionException " + getLocalizedMessage();
+ }
+}
diff --git a/nifi-commons/nifi-data-provenance-utils/src/main/java/org/apache/nifi/provenance/EncryptionMetadata.java b/nifi-commons/nifi-data-provenance-utils/src/main/java/org/apache/nifi/provenance/EncryptionMetadata.java
new file mode 100644
index 0000000000..0d969daeff
--- /dev/null
+++ b/nifi-commons/nifi-data-provenance-utils/src/main/java/org/apache/nifi/provenance/EncryptionMetadata.java
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.provenance;
+
+import java.io.Serializable;
+import org.apache.commons.codec.binary.Hex;
+
+public class EncryptionMetadata implements Serializable {
+ protected String keyId;
+ protected String algorithm;
+ protected byte[] ivBytes;
+ protected String version;
+ protected int cipherByteLength;
+
+ EncryptionMetadata() {
+ }
+
+ EncryptionMetadata(String keyId, String algorithm, byte[] ivBytes, String version, int cipherByteLength) {
+ this.keyId = keyId;
+ this.ivBytes = ivBytes;
+ this.algorithm = algorithm;
+ this.version = version;
+ this.cipherByteLength = cipherByteLength;
+ }
+
+ @Override
+ public String toString() {
+ String sb = "AES Provenance Record Encryption Metadata" +
+ " Key ID: " +
+ keyId +
+ " Algorithm: " +
+ algorithm +
+ " IV: " +
+ Hex.encodeHexString(ivBytes) +
+ " Version: " +
+ version +
+ " Cipher text length: " +
+ cipherByteLength;
+ return sb;
+ }
+}
\ No newline at end of file
diff --git a/nifi-commons/nifi-data-provenance-utils/src/main/java/org/apache/nifi/provenance/FileBasedKeyProvider.java b/nifi-commons/nifi-data-provenance-utils/src/main/java/org/apache/nifi/provenance/FileBasedKeyProvider.java
new file mode 100644
index 0000000000..b70b3e8fbd
--- /dev/null
+++ b/nifi-commons/nifi-data-provenance-utils/src/main/java/org/apache/nifi/provenance/FileBasedKeyProvider.java
@@ -0,0 +1,67 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.provenance;
+
+import java.io.IOException;
+import java.security.KeyManagementException;
+import javax.crypto.SecretKey;
+import javax.crypto.spec.SecretKeySpec;
+import javax.naming.OperationNotSupportedException;
+import org.apache.nifi.properties.NiFiPropertiesLoader;
+import org.bouncycastle.util.encoders.Hex;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class FileBasedKeyProvider extends StaticKeyProvider {
+ private static final Logger logger = LoggerFactory.getLogger(FileBasedKeyProvider.class);
+
+ private String filepath;
+
+ FileBasedKeyProvider(String location) throws KeyManagementException {
+ this(location, getMasterKey());
+ }
+
+ FileBasedKeyProvider(String location, SecretKey masterKey) throws KeyManagementException {
+ super(CryptoUtils.readKeys(location, masterKey));
+ this.filepath = location;
+ }
+
+ private static SecretKey getMasterKey() throws KeyManagementException {
+ try {
+ // Get the master encryption key from bootstrap.conf
+ String masterKeyHex = NiFiPropertiesLoader.extractKeyFromBootstrapFile();
+ return new SecretKeySpec(Hex.decode(masterKeyHex), "AES");
+ } catch (IOException e) {
+ logger.error("Encountered an error: ", e);
+ throw new KeyManagementException(e);
+ }
+ }
+
+ /**
+ * Adds the key to the provider and associates it with the given ID. Some implementations may not allow this operation.
+ *
+ * @param keyId the key identifier
+ * @param key the key
+ * @return true if the key was successfully added
+ * @throws OperationNotSupportedException if this implementation doesn't support adding keys
+ * @throws KeyManagementException if the key is invalid, the ID conflicts, etc.
+ */
+ @Override
+ public boolean addKey(String keyId, SecretKey key) throws OperationNotSupportedException, KeyManagementException {
+ throw new OperationNotSupportedException("This implementation does not allow adding keys. Modify the file backing this provider at " + filepath);
+ }
+}
diff --git a/nifi-commons/nifi-data-provenance-utils/src/main/java/org/apache/nifi/provenance/KeyProvider.java b/nifi-commons/nifi-data-provenance-utils/src/main/java/org/apache/nifi/provenance/KeyProvider.java
new file mode 100644
index 0000000000..39f6384e11
--- /dev/null
+++ b/nifi-commons/nifi-data-provenance-utils/src/main/java/org/apache/nifi/provenance/KeyProvider.java
@@ -0,0 +1,60 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.provenance;
+
+import java.security.KeyManagementException;
+import java.util.List;
+import javax.crypto.SecretKey;
+import javax.naming.OperationNotSupportedException;
+
+public interface KeyProvider {
+
+ /**
+ * Returns the key identified by this ID or throws an exception if one is not available.
+ *
+ * @param keyId the key identifier
+ * @return the key
+ * @throws KeyManagementException if the key cannot be retrieved
+ */
+ SecretKey getKey(String keyId) throws KeyManagementException;
+
+ /**
+ * Returns true if the key exists and is available. Null or empty IDs will return false.
+ *
+ * @param keyId the key identifier
+ * @return true if the key can be used
+ */
+ boolean keyExists(String keyId);
+
+ /**
+ * Returns a list of available key identifiers (useful for encryption, as retired keys may not be listed here even if they are available for decryption for legacy/BC reasons).
+ *
+ * @return a List of keyIds (empty list if none are available)
+ */
+ List getAvailableKeyIds();
+
+ /**
+ * Adds the key to the provider and associates it with the given ID. Some implementations may not allow this operation.
+ *
+ * @param keyId the key identifier
+ * @param key the key
+ * @return true if the key was successfully added
+ * @throws OperationNotSupportedException if this implementation doesn't support adding keys
+ * @throws KeyManagementException if the key is invalid, the ID conflicts, etc.
+ */
+ boolean addKey(String keyId, SecretKey key) throws OperationNotSupportedException, KeyManagementException;
+}
diff --git a/nifi-commons/nifi-data-provenance-utils/src/main/java/org/apache/nifi/provenance/PlaceholderProvenanceEvent.java b/nifi-commons/nifi-data-provenance-utils/src/main/java/org/apache/nifi/provenance/PlaceholderProvenanceEvent.java
index 26696c8b17..5644355b22 100644
--- a/nifi-commons/nifi-data-provenance-utils/src/main/java/org/apache/nifi/provenance/PlaceholderProvenanceEvent.java
+++ b/nifi-commons/nifi-data-provenance-utils/src/main/java/org/apache/nifi/provenance/PlaceholderProvenanceEvent.java
@@ -187,4 +187,14 @@ public class PlaceholderProvenanceEvent implements ProvenanceEventRecord {
public Long getPreviousContentClaimOffset() {
return null;
}
+
+ /**
+ * Returns the best event identifier for this event (eventId if available, descriptive identifier if not yet persisted to allow for traceability).
+ *
+ * @return a descriptive event ID to allow tracing
+ */
+ @Override
+ public String getBestEventIdentifier() {
+ return Long.toString(getEventId());
+ }
}
diff --git a/nifi-commons/nifi-data-provenance-utils/src/main/java/org/apache/nifi/provenance/ProvenanceEventEncryptor.java b/nifi-commons/nifi-data-provenance-utils/src/main/java/org/apache/nifi/provenance/ProvenanceEventEncryptor.java
new file mode 100644
index 0000000000..c7690e1703
--- /dev/null
+++ b/nifi-commons/nifi-data-provenance-utils/src/main/java/org/apache/nifi/provenance/ProvenanceEventEncryptor.java
@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.provenance;
+
+import java.security.KeyManagementException;
+
+public interface ProvenanceEventEncryptor {
+
+ /**
+ * Initializes the encryptor with a {@link KeyProvider}.
+ *
+ * @param keyProvider the key provider which will be responsible for accessing keys
+ * @throws KeyManagementException if there is an issue configuring the key provider
+ */
+ void initialize(KeyProvider keyProvider) throws KeyManagementException;
+
+ /**
+ * Encrypts the provided {@link ProvenanceEventRecord}, serialized to a byte[] by the RecordWriter.
+ *
+ * @param plainRecord the plain record, serialized to a byte[]
+ * @param recordId an identifier for this record (eventId, generated, etc.)
+ * @param keyId the ID of the key to use
+ * @return the encrypted record
+ * @throws EncryptionException if there is an issue encrypting this record
+ */
+ byte[] encrypt(byte[] plainRecord, String recordId, String keyId) throws EncryptionException;
+
+ /**
+ * Decrypts the provided byte[] (an encrypted record with accompanying metadata).
+ *
+ * @param encryptedRecord the encrypted record in byte[] form
+ * @param recordId an identifier for this record (eventId, generated, etc.)
+ * @return the decrypted record
+ * @throws EncryptionException if there is an issue decrypting this record
+ */
+ byte[] decrypt(byte[] encryptedRecord, String recordId) throws EncryptionException;
+
+ /**
+ * Returns a valid key identifier for this encryptor (valid for encryption and decryption) or throws an exception if none are available.
+ *
+ * @return the key ID
+ * @throws KeyManagementException if no available key IDs are valid for both operations
+ */
+ String getNextKeyId() throws KeyManagementException;
+}
diff --git a/nifi-commons/nifi-data-provenance-utils/src/main/java/org/apache/nifi/provenance/StandardProvenanceEventRecord.java b/nifi-commons/nifi-data-provenance-utils/src/main/java/org/apache/nifi/provenance/StandardProvenanceEventRecord.java
index ac60d4f680..84e7419a29 100644
--- a/nifi-commons/nifi-data-provenance-utils/src/main/java/org/apache/nifi/provenance/StandardProvenanceEventRecord.java
+++ b/nifi-commons/nifi-data-provenance-utils/src/main/java/org/apache/nifi/provenance/StandardProvenanceEventRecord.java
@@ -22,7 +22,6 @@ import java.util.Date;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
-
import org.apache.nifi.flowfile.FlowFile;
import org.apache.nifi.flowfile.attributes.CoreAttributes;
import org.apache.nifi.processor.Relationship;
@@ -30,7 +29,7 @@ import org.apache.nifi.processor.Relationship;
/**
* Holder for provenance relevant information
*/
-public final class StandardProvenanceEventRecord implements ProvenanceEventRecord {
+public class StandardProvenanceEventRecord implements ProvenanceEventRecord {
private final long eventTime;
private final long entryDate;
@@ -69,7 +68,7 @@ public final class StandardProvenanceEventRecord implements ProvenanceEventRecor
private volatile long eventId = -1L;
- private StandardProvenanceEventRecord(final Builder builder) {
+ StandardProvenanceEventRecord(final Builder builder) {
this.eventTime = builder.eventTime;
this.entryDate = builder.entryDate;
this.eventType = builder.eventType;
@@ -100,8 +99,8 @@ public final class StandardProvenanceEventRecord implements ProvenanceEventRecor
contentClaimOffset = builder.contentClaimOffset;
contentSize = builder.contentSize;
- previousAttributes = builder.previousAttributes == null ? Collections.emptyMap() : Collections.unmodifiableMap(builder.previousAttributes);
- updatedAttributes = builder.updatedAttributes == null ? Collections.emptyMap() : Collections.unmodifiableMap(builder.updatedAttributes);
+ previousAttributes = builder.previousAttributes == null ? Collections.emptyMap() : Collections.unmodifiableMap(builder.previousAttributes);
+ updatedAttributes = builder.updatedAttributes == null ? Collections.emptyMap() : Collections.unmodifiableMap(builder.updatedAttributes);
sourceQueueIdentifier = builder.sourceQueueIdentifier;
@@ -110,6 +109,11 @@ public final class StandardProvenanceEventRecord implements ProvenanceEventRecor
}
}
+ public static StandardProvenanceEventRecord copy(StandardProvenanceEventRecord other) {
+ Builder builder = new Builder().fromEvent(other);
+ return new StandardProvenanceEventRecord(builder);
+ }
+
public String getStorageFilename() {
return storageFilename;
}
@@ -199,12 +203,12 @@ public final class StandardProvenanceEventRecord implements ProvenanceEventRecor
@Override
public List getParentUuids() {
- return parentUuids == null ? Collections.emptyList() : parentUuids;
+ return parentUuids == null ? Collections.emptyList() : parentUuids;
}
@Override
public List getChildUuids() {
- return childrenUuids == null ? Collections.emptyList() : childrenUuids;
+ return childrenUuids == null ? Collections.emptyList() : childrenUuids;
}
@Override
@@ -299,8 +303,8 @@ public final class StandardProvenanceEventRecord implements ProvenanceEventRecor
}
return -37423 + 3 * componentId.hashCode() + (transitUri == null ? 0 : 41 * transitUri.hashCode())
- + (relationship == null ? 0 : 47 * relationship.hashCode()) + 44 * eventTypeCode
- + 47 * getChildUuids().hashCode() + 47 * getParentUuids().hashCode();
+ + (relationship == null ? 0 : 47 * relationship.hashCode()) + 44 * eventTypeCode
+ + 47 * getChildUuids().hashCode() + 47 * getParentUuids().hashCode();
}
@Override
@@ -419,6 +423,23 @@ public final class StandardProvenanceEventRecord implements ProvenanceEventRecor
+ ", alternateIdentifierUri=" + alternateIdentifierUri + "]";
}
+ /**
+ * Returns a unique identifier for the record. By default, it uses
+ * {@link ProvenanceEventRecord#getEventId()} but if it has not been persisted to the
+ * repository, this is {@code -1}, so it constructs a String of the format
+ * {@code _on__by__at_}.
+ *
+ * @return a String identifying the record for later analysis
+ */
+ @Override
+ public String getBestEventIdentifier() {
+ if (getEventId() != -1) {
+ return Long.toString(getEventId());
+ } else {
+ return getEventType().name() + "_on_" + getFlowFileUuid() + "_by_" + getComponentId() + "_at_" + getEventTime();
+ }
+ }
+
public static class Builder implements ProvenanceEventBuilder {
private long eventTime = System.currentTimeMillis();
@@ -733,7 +754,7 @@ public final class StandardProvenanceEventRecord implements ProvenanceEventRecor
public ProvenanceEventBuilder fromFlowFile(final FlowFile flowFile) {
setFlowFileEntryDate(flowFile.getEntryDate());
setLineageStartDate(flowFile.getLineageStartDate());
- setAttributes(Collections.emptyMap(), flowFile.getAttributes());
+ setAttributes(Collections.emptyMap(), flowFile.getAttributes());
uuid = flowFile.getAttribute(CoreAttributes.UUID.key());
this.contentSize = flowFile.getSize();
return this;
diff --git a/nifi-commons/nifi-data-provenance-utils/src/main/java/org/apache/nifi/provenance/StaticKeyProvider.java b/nifi-commons/nifi-data-provenance-utils/src/main/java/org/apache/nifi/provenance/StaticKeyProvider.java
new file mode 100644
index 0000000000..e0981dca6b
--- /dev/null
+++ b/nifi-commons/nifi-data-provenance-utils/src/main/java/org/apache/nifi/provenance/StaticKeyProvider.java
@@ -0,0 +1,96 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.provenance;
+
+import java.security.KeyManagementException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import javax.crypto.SecretKey;
+import javax.naming.OperationNotSupportedException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Reference implementation for static key provider (used during tests).
+ */
+public class StaticKeyProvider implements KeyProvider {
+ private static final Logger logger = LoggerFactory.getLogger(StaticKeyProvider.class);
+
+ private Map keys = new HashMap<>();
+
+ StaticKeyProvider(String keyId, String keyHex) throws KeyManagementException {
+ this.keys.put(keyId, CryptoUtils.formKeyFromHex(keyHex));
+ }
+
+ StaticKeyProvider(Map keys) throws KeyManagementException {
+ this.keys.putAll(keys);
+ }
+
+ /**
+ * Returns the key identified by this ID or throws an exception if one is not available.
+ *
+ * @param keyId the key identifier
+ * @return the key
+ * @throws KeyManagementException if the key cannot be retrieved
+ */
+ @Override
+ public SecretKey getKey(String keyId) throws KeyManagementException {
+ logger.debug("Attempting to get key: " + keyId);
+ if (keyExists(keyId)) {
+ return keys.get(keyId);
+ } else {
+ throw new KeyManagementException("No key available for " + keyId);
+ }
+ }
+
+ /**
+ * Returns true if the key exists and is available. Null or empty IDs will return false.
+ *
+ * @param keyId the key identifier
+ * @return true if the key can be used
+ */
+ @Override
+ public boolean keyExists(String keyId) {
+ return keys.containsKey(keyId);
+ }
+
+ /**
+ * Returns a singleton list of the available key identifier.
+ *
+ * @return a List containing the {@code KEY_ID}
+ */
+ @Override
+ public List getAvailableKeyIds() {
+ return new ArrayList<>(keys.keySet());
+ }
+
+ /**
+ * Adds the key to the provider and associates it with the given ID. Some implementations may not allow this operation.
+ *
+ * @param keyId the key identifier
+ * @param key the key
+ * @return true if the key was successfully added
+ * @throws OperationNotSupportedException if this implementation doesn't support adding keys
+ * @throws KeyManagementException if the key is invalid, the ID conflicts, etc.
+ */
+ @Override
+ public boolean addKey(String keyId, SecretKey key) throws OperationNotSupportedException, KeyManagementException {
+ throw new OperationNotSupportedException("This implementation does not allow adding keys");
+ }
+}
diff --git a/nifi-commons/nifi-data-provenance-utils/src/test/groovy/org/apache/nifi/provenance/AESProvenanceEventEncryptorTest.groovy b/nifi-commons/nifi-data-provenance-utils/src/test/groovy/org/apache/nifi/provenance/AESProvenanceEventEncryptorTest.groovy
new file mode 100644
index 0000000000..61b35d0d31
--- /dev/null
+++ b/nifi-commons/nifi-data-provenance-utils/src/test/groovy/org/apache/nifi/provenance/AESProvenanceEventEncryptorTest.groovy
@@ -0,0 +1,303 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.provenance
+
+import org.apache.nifi.security.util.EncryptionMethod
+import org.apache.nifi.security.util.crypto.AESKeyedCipherProvider
+import org.bouncycastle.jce.provider.BouncyCastleProvider
+import org.bouncycastle.util.encoders.Hex
+import org.junit.After
+import org.junit.AfterClass
+import org.junit.Before
+import org.junit.BeforeClass
+import org.junit.Test
+import org.junit.runner.RunWith
+import org.junit.runners.JUnit4
+import org.slf4j.Logger
+import org.slf4j.LoggerFactory
+
+import javax.crypto.Cipher
+import javax.crypto.SecretKey
+import javax.crypto.spec.IvParameterSpec
+import javax.crypto.spec.SecretKeySpec
+import java.nio.charset.StandardCharsets
+import java.security.KeyManagementException
+import java.security.SecureRandom
+import java.security.Security
+
+import static groovy.test.GroovyAssert.shouldFail
+
+@RunWith(JUnit4.class)
+class AESProvenanceEventEncryptorTest {
+ private static final Logger logger = LoggerFactory.getLogger(AESProvenanceEventEncryptorTest.class)
+
+ private static final String KEY_HEX_128 = "0123456789ABCDEFFEDCBA9876543210"
+ private static final String KEY_HEX_256 = KEY_HEX_128 * 2
+ private static final String KEY_HEX = isUnlimitedStrengthCryptoAvailable() ? KEY_HEX_256 : KEY_HEX_128
+
+ private static KeyProvider mockKeyProvider
+ private static AESKeyedCipherProvider mockCipherProvider
+
+ private static String ORIGINAL_LOG_LEVEL
+
+ private ProvenanceEventEncryptor encryptor
+
+ @BeforeClass
+ static void setUpOnce() throws Exception {
+ ORIGINAL_LOG_LEVEL = System.getProperty("org.slf4j.simpleLogger.log.org.apache.nifi.provenance")
+ System.setProperty("org.slf4j.simpleLogger.log.org.apache.nifi.provenance", "DEBUG")
+
+ Security.addProvider(new BouncyCastleProvider())
+
+ logger.metaClass.methodMissing = { String name, args ->
+ logger.info("[${name?.toUpperCase()}] ${(args as List).join(" ")}")
+ }
+
+ mockKeyProvider = [
+ getKey : { String keyId ->
+ logger.mock("Requesting key ID: ${keyId}")
+ new SecretKeySpec(Hex.decode(KEY_HEX), "AES")
+ },
+ keyExists: { String keyId ->
+ logger.mock("Checking existence of ${keyId}")
+ true
+ }] as KeyProvider
+
+ mockCipherProvider = [
+ getCipher: { EncryptionMethod em, SecretKey key, byte[] ivBytes, boolean encryptMode ->
+ logger.mock("Getting cipher for ${em} with IV ${Hex.toHexString(ivBytes)} encrypt ${encryptMode}")
+ Cipher cipher = Cipher.getInstance("AES/GCM/NoPadding")
+ cipher.init((encryptMode ? Cipher.ENCRYPT_MODE : Cipher.DECRYPT_MODE) as int, key, new IvParameterSpec(ivBytes))
+ cipher
+ }
+ ] as AESKeyedCipherProvider
+ }
+
+ @Before
+ void setUp() throws Exception {
+
+ }
+
+ @After
+ void tearDown() throws Exception {
+
+ }
+
+ @AfterClass
+ static void tearDownOnce() throws Exception {
+ if (ORIGINAL_LOG_LEVEL) {
+ System.setProperty("org.slf4j.simpleLogger.log.org.apache.nifi.provenance", ORIGINAL_LOG_LEVEL)
+ }
+ }
+
+ private static boolean isUnlimitedStrengthCryptoAvailable() {
+ Cipher.getMaxAllowedKeyLength("AES") > 128
+ }
+
+ /**
+ * Given arbitrary bytes, encrypt them and persist with the encryption metadata, then recover
+ */
+ @Test
+ void testShouldEncryptAndDecryptArbitraryBytes() {
+ // Arrange
+ final byte[] SERIALIZED_BYTES = "This is a plaintext message.".getBytes(StandardCharsets.UTF_8)
+ logger.info("Serialized bytes (${SERIALIZED_BYTES.size()}): ${Hex.toHexString(SERIALIZED_BYTES)}")
+
+ encryptor = new AESProvenanceEventEncryptor()
+ encryptor.initialize(mockKeyProvider)
+ encryptor.setCipherProvider(mockCipherProvider)
+ logger.info("Created ${encryptor}")
+
+ String keyId = "K1"
+ String recordId = "R1"
+ logger.info("Using record ID ${recordId} and key ID ${keyId}")
+
+ // Act
+ byte[] metadataAndCipherBytes = encryptor.encrypt(SERIALIZED_BYTES, recordId, keyId)
+ logger.info("Encrypted data to: \n\t${Hex.toHexString(metadataAndCipherBytes)}")
+
+ byte[] recoveredBytes = encryptor.decrypt(metadataAndCipherBytes, recordId)
+ logger.info("Decrypted data to: \n\t${Hex.toHexString(recoveredBytes)}")
+
+ // Assert
+ assert recoveredBytes == SERIALIZED_BYTES
+ logger.info("Decoded (usually would be serialized schema record): ${new String(recoveredBytes, StandardCharsets.UTF_8)}")
+ }
+
+ @Test
+ void testShouldInitializeNullCipherProvider() {
+ // Arrange
+ encryptor = new AESProvenanceEventEncryptor()
+ encryptor.setCipherProvider(null)
+ assert !encryptor.aesKeyedCipherProvider
+
+ // Act
+ encryptor.initialize(mockKeyProvider)
+ logger.info("Created ${encryptor}")
+
+ // Assert
+ assert encryptor.aesKeyedCipherProvider instanceof AESKeyedCipherProvider
+ }
+
+ @Test
+ void testShouldFailOnMissingKeyId() {
+ // Arrange
+ final byte[] SERIALIZED_BYTES = "This is a plaintext message.".getBytes(StandardCharsets.UTF_8)
+ logger.info("Serialized bytes (${SERIALIZED_BYTES.size()}): ${Hex.toHexString(SERIALIZED_BYTES)}")
+
+ KeyProvider emptyKeyProvider = [
+ getKey : { String kid ->
+ throw new KeyManagementException("No key found for ${kid}")
+ },
+ keyExists: { String kid -> false }
+ ] as KeyProvider
+
+ encryptor = new AESProvenanceEventEncryptor()
+ encryptor.initialize(emptyKeyProvider)
+ encryptor.setCipherProvider(mockCipherProvider)
+ logger.info("Created ${encryptor}")
+
+ String keyId = "K1"
+ String recordId = "R1"
+ logger.info("Using record ID ${recordId} and key ID ${keyId}")
+
+ // Act
+ def msg = shouldFail(EncryptionException) {
+ byte[] metadataAndCipherBytes = encryptor.encrypt(SERIALIZED_BYTES, recordId, keyId)
+ logger.info("Encrypted data to: \n\t${Hex.toHexString(metadataAndCipherBytes)}")
+ }
+ logger.expected(msg)
+
+ // Assert
+ assert msg.getMessage() == "The requested key ID is not available"
+ }
+
+ @Test
+ void testShouldUseDifferentIVsForSequentialEncryptions() {
+ // Arrange
+ final byte[] SERIALIZED_BYTES = "This is a plaintext message.".getBytes(StandardCharsets.UTF_8)
+ logger.info("Serialized bytes (${SERIALIZED_BYTES.size()}): ${Hex.toHexString(SERIALIZED_BYTES)}")
+
+ encryptor = new AESProvenanceEventEncryptor()
+ encryptor.initialize(mockKeyProvider)
+ logger.info("Created ${encryptor}")
+
+ String keyId = "K1"
+ String recordId1 = "R1"
+ logger.info("Using record ID ${recordId1} and key ID ${keyId}")
+
+ // Act
+ byte[] metadataAndCipherBytes1 = encryptor.encrypt(SERIALIZED_BYTES, recordId1, keyId)
+ logger.info("Encrypted data to: \n\t${Hex.toHexString(metadataAndCipherBytes1)}")
+ EncryptionMetadata metadata1 = encryptor.extractEncryptionMetadata(metadataAndCipherBytes1)
+ logger.info("Record ${recordId1} IV: ${Hex.toHexString(metadata1.ivBytes)}")
+
+ byte[] recoveredBytes1 = encryptor.decrypt(metadataAndCipherBytes1, recordId1)
+ logger.info("Decrypted data to: \n\t${Hex.toHexString(recoveredBytes1)}")
+
+ String recordId2 = "R2"
+ byte[] metadataAndCipherBytes2 = encryptor.encrypt(SERIALIZED_BYTES, recordId2, keyId)
+ logger.info("Encrypted data to: \n\t${Hex.toHexString(metadataAndCipherBytes2)}")
+ EncryptionMetadata metadata2 = encryptor.extractEncryptionMetadata(metadataAndCipherBytes2)
+ logger.info("Record ${recordId2} IV: ${Hex.toHexString(metadata2.ivBytes)}")
+
+ byte[] recoveredBytes2 = encryptor.decrypt(metadataAndCipherBytes2, recordId2)
+ logger.info("Decrypted data to: \n\t${Hex.toHexString(recoveredBytes2)}")
+
+ // Assert
+ assert metadata1.ivBytes != metadata2.ivBytes
+
+ assert recoveredBytes1 == SERIALIZED_BYTES
+ assert recoveredBytes2 == SERIALIZED_BYTES
+ }
+
+ @Test
+ void testShouldFailOnBadMetadata() {
+ // Arrange
+ final byte[] SERIALIZED_BYTES = "This is a plaintext message.".getBytes(StandardCharsets.UTF_8)
+ logger.info("Serialized bytes (${SERIALIZED_BYTES.size()}): ${Hex.toHexString(SERIALIZED_BYTES)}")
+
+ def strictMockKeyProvider = [
+ getKey : { String keyId ->
+ if (keyId != "K1") {
+ throw new KeyManagementException("No such key")
+ }
+ new SecretKeySpec(Hex.decode(KEY_HEX), "AES")
+ },
+ keyExists: { String keyId ->
+ keyId == "K1"
+ }] as KeyProvider
+
+ encryptor = new AESProvenanceEventEncryptor()
+ encryptor.initialize(strictMockKeyProvider)
+ encryptor.setCipherProvider(mockCipherProvider)
+ logger.info("Created ${encryptor}")
+
+ String keyId = "K1"
+ String recordId = "R1"
+ logger.info("Using record ID ${recordId} and key ID ${keyId}")
+
+ final String ALGORITHM = "AES/GCM/NoPadding"
+ final byte[] ivBytes = new byte[16]
+ new SecureRandom().nextBytes(ivBytes)
+ final String VERSION = "v1"
+
+ // Perform the encryption independently of the encryptor
+ SecretKey key = mockKeyProvider.getKey(keyId)
+ Cipher cipher = new AESKeyedCipherProvider().getCipher(EncryptionMethod.AES_GCM, key, ivBytes, true)
+ byte[] cipherBytes = cipher.doFinal(SERIALIZED_BYTES)
+
+ int cipherBytesLength = cipherBytes.size()
+
+ // Construct accurate metadata
+ EncryptionMetadata goodMetadata = new EncryptionMetadata(keyId, ALGORITHM, ivBytes, VERSION, cipherBytesLength)
+ logger.info("Created good encryption metadata: ${goodMetadata}")
+
+ // Construct bad metadata instances
+ EncryptionMetadata badKeyId = new EncryptionMetadata(keyId.reverse(), ALGORITHM, ivBytes, VERSION, cipherBytesLength)
+ EncryptionMetadata badAlgorithm = new EncryptionMetadata(keyId, "ASE/GDM/SomePadding", ivBytes, VERSION, cipherBytesLength)
+ EncryptionMetadata badIV = new EncryptionMetadata(keyId, ALGORITHM, new byte[16], VERSION, cipherBytesLength)
+ EncryptionMetadata badVersion = new EncryptionMetadata(keyId, ALGORITHM, ivBytes, VERSION.reverse(), cipherBytesLength)
+ EncryptionMetadata badCBLength = new EncryptionMetadata(keyId, ALGORITHM, ivBytes, VERSION, cipherBytesLength - 5)
+
+ List badMetadata = [badKeyId, badAlgorithm, badIV, badVersion, badCBLength]
+
+ // Form the proper cipherBytes
+ byte[] completeGoodBytes = CryptoUtils.concatByteArrays([0x01] as byte[], encryptor.serializeEncryptionMetadata(goodMetadata), cipherBytes)
+
+ byte[] recoveredGoodBytes = encryptor.decrypt(completeGoodBytes, recordId)
+ logger.info("Recovered good bytes: ${Hex.toHexString(recoveredGoodBytes)}")
+
+ final List EXPECTED_MESSAGES = ["The requested key ID (\\w+)? is not available",
+ "Encountered an exception decrypting provenance record",
+ "The event was encrypted with version ${VERSION.reverse()} which is not in the list of supported versions v1"]
+
+ // Act
+ badMetadata.eachWithIndex { EncryptionMetadata metadata, int i ->
+ byte[] completeBytes = CryptoUtils.concatByteArrays([0x01] as byte[], encryptor.serializeEncryptionMetadata(metadata), cipherBytes)
+
+ def msg = shouldFail(EncryptionException) {
+ byte[] recoveredBytes = encryptor.decrypt(completeBytes, "R${i + 2}")
+ logger.info("Recovered bad bytes: ${Hex.toHexString(recoveredBytes)}")
+ }
+ logger.expected(msg)
+
+ // Assert
+ assert EXPECTED_MESSAGES.any { msg.getMessage() =~ it }
+ }
+ }
+}
diff --git a/nifi-commons/nifi-data-provenance-utils/src/test/groovy/org/apache/nifi/provenance/CryptoUtilsTest.groovy b/nifi-commons/nifi-data-provenance-utils/src/test/groovy/org/apache/nifi/provenance/CryptoUtilsTest.groovy
new file mode 100644
index 0000000000..162896f83c
--- /dev/null
+++ b/nifi-commons/nifi-data-provenance-utils/src/test/groovy/org/apache/nifi/provenance/CryptoUtilsTest.groovy
@@ -0,0 +1,436 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License") you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.provenance
+
+import org.bouncycastle.jce.provider.BouncyCastleProvider
+import org.bouncycastle.util.encoders.Hex
+import org.junit.After
+import org.junit.AfterClass
+import org.junit.Before
+import org.junit.BeforeClass
+import org.junit.ClassRule
+import org.junit.Test
+import org.junit.rules.TemporaryFolder
+import org.junit.runner.RunWith
+import org.junit.runners.JUnit4
+import org.slf4j.Logger
+import org.slf4j.LoggerFactory
+
+import javax.crypto.Cipher
+import javax.crypto.SecretKey
+import javax.crypto.spec.IvParameterSpec
+import javax.crypto.spec.SecretKeySpec
+import java.nio.charset.StandardCharsets
+import java.nio.file.Files
+import java.nio.file.attribute.PosixFilePermission
+import java.security.KeyManagementException
+import java.security.SecureRandom
+import java.security.Security
+
+import static groovy.test.GroovyAssert.shouldFail
+
+@RunWith(JUnit4.class)
+class CryptoUtilsTest {
+ private static final Logger logger = LoggerFactory.getLogger(CryptoUtilsTest.class)
+
+ private static final String KEY_ID = "K1"
+ private static final String KEY_HEX_128 = "0123456789ABCDEFFEDCBA9876543210"
+ private static final String KEY_HEX_256 = KEY_HEX_128 * 2
+ private static final String KEY_HEX = isUnlimitedStrengthCryptoAvailable() ? KEY_HEX_256 : KEY_HEX_128
+
+ private static
+ final Set ALL_POSIX_ATTRS = PosixFilePermission.values() as Set
+
+ @ClassRule
+ public static TemporaryFolder tempFolder = new TemporaryFolder()
+
+ @BeforeClass
+ static void setUpOnce() throws Exception {
+ Security.addProvider(new BouncyCastleProvider())
+
+ logger.metaClass.methodMissing = { String name, args ->
+ logger.info("[${name?.toUpperCase()}] ${(args as List).join(" ")}")
+ }
+ }
+
+ @Before
+ void setUp() throws Exception {
+ tempFolder.create()
+ }
+
+ @After
+ void tearDown() throws Exception {
+ tempFolder?.delete()
+ }
+
+ @AfterClass
+ static void tearDownOnce() throws Exception {
+
+ }
+
+ private static boolean isUnlimitedStrengthCryptoAvailable() {
+ Cipher.getMaxAllowedKeyLength("AES") > 128
+ }
+
+ @Test
+ void testShouldConcatenateByteArrays() {
+ // Arrange
+ byte[] bytes1 = "These are some bytes".getBytes(StandardCharsets.UTF_8)
+ byte[] bytes2 = "These are some other bytes".getBytes(StandardCharsets.UTF_8)
+ final byte[] EXPECTED_CONCATENATED_BYTES = ((bytes1 as List) << (bytes2 as List)).flatten() as byte[]
+ logger.info("Expected concatenated bytes: ${Hex.toHexString(EXPECTED_CONCATENATED_BYTES)}")
+
+ // Act
+ byte[] concat = CryptoUtils.concatByteArrays(bytes1, bytes2)
+ logger.info(" Actual concatenated bytes: ${Hex.toHexString(concat)}")
+
+ // Assert
+ assert concat == EXPECTED_CONCATENATED_BYTES
+ }
+
+ @Test
+ void testShouldValidateStaticKeyProvider() {
+ // Arrange
+ String staticProvider = StaticKeyProvider.class.name
+ String providerLocation = null
+
+ // Act
+ boolean keyProviderIsValid = CryptoUtils.isValidKeyProvider(staticProvider, providerLocation, KEY_ID, [(KEY_ID): KEY_HEX])
+ logger.info("Key Provider ${staticProvider} with location ${providerLocation} and keyId ${KEY_ID} / ${KEY_HEX} is ${keyProviderIsValid ? "valid" : "invalid"}")
+
+ // Assert
+ assert keyProviderIsValid
+ }
+
+ @Test
+ void testShouldNotValidateStaticKeyProviderMissingKeyId() {
+ // Arrange
+ String staticProvider = StaticKeyProvider.class.name
+ String providerLocation = null
+
+ // Act
+ boolean keyProviderIsValid = CryptoUtils.isValidKeyProvider(staticProvider, providerLocation, null, [(KEY_ID): KEY_HEX])
+ logger.info("Key Provider ${staticProvider} with location ${providerLocation} and keyId ${null} / ${KEY_HEX} is ${keyProviderIsValid ? "valid" : "invalid"}")
+
+ // Assert
+ assert !keyProviderIsValid
+ }
+
+ @Test
+ void testShouldNotValidateStaticKeyProviderMissingKey() {
+ // Arrange
+ String staticProvider = StaticKeyProvider.class.name
+ String providerLocation = null
+
+ // Act
+ boolean keyProviderIsValid = CryptoUtils.isValidKeyProvider(staticProvider, providerLocation, KEY_ID, null)
+ logger.info("Key Provider ${staticProvider} with location ${providerLocation} and keyId ${KEY_ID} / ${null} is ${keyProviderIsValid ? "valid" : "invalid"}")
+
+ // Assert
+ assert !keyProviderIsValid
+ }
+
+ @Test
+ void testShouldNotValidateStaticKeyProviderWithInvalidKey() {
+ // Arrange
+ String staticProvider = StaticKeyProvider.class.name
+ String providerLocation = null
+
+ // Act
+ boolean keyProviderIsValid = CryptoUtils.isValidKeyProvider(staticProvider, providerLocation, KEY_ID, [(KEY_ID): KEY_HEX[0..<-2]])
+ logger.info("Key Provider ${staticProvider} with location ${providerLocation} and keyId ${KEY_ID} / ${KEY_HEX[0..<-2]} is ${keyProviderIsValid ? "valid" : "invalid"}")
+
+ // Assert
+ assert !keyProviderIsValid
+ }
+
+ @Test
+ void testShouldValidateFileBasedKeyProvider() {
+ // Arrange
+ String fileBasedProvider = FileBasedKeyProvider.class.name
+ File fileBasedProviderFile = tempFolder.newFile("filebased.kp")
+ String providerLocation = fileBasedProviderFile.path
+ logger.info("Created temporary file based key provider: ${providerLocation}")
+
+ // Act
+ boolean keyProviderIsValid = CryptoUtils.isValidKeyProvider(fileBasedProvider, providerLocation, KEY_ID, null)
+ logger.info("Key Provider ${fileBasedProvider} with location ${providerLocation} and keyId ${KEY_ID} / ${null} is ${keyProviderIsValid ? "valid" : "invalid"}")
+
+ // Assert
+ assert keyProviderIsValid
+ }
+
+ @Test
+ void testShouldNotValidateUnreadableOrMissingFileBasedKeyProvider() {
+ // Arrange
+ String fileBasedProvider = FileBasedKeyProvider.class.name
+ File fileBasedProviderFile = tempFolder.newFile("filebased.kp")
+ String providerLocation = fileBasedProviderFile.path
+ logger.info("Created temporary file based key provider: ${providerLocation}")
+
+ // Make it unreadable
+ fileBasedProviderFile.setReadable(false, false)
+ Files.setPosixFilePermissions(fileBasedProviderFile.toPath(), [] as Set)
+
+ // Act
+ boolean unreadableKeyProviderIsValid = CryptoUtils.isValidKeyProvider(fileBasedProvider, providerLocation, KEY_ID, null)
+ logger.info("Key Provider ${fileBasedProvider} with location ${providerLocation} and keyId ${KEY_ID} / ${null} is ${unreadableKeyProviderIsValid ? "valid" : "invalid"}")
+
+ String missingLocation = providerLocation + "_missing"
+ boolean missingKeyProviderIsValid = CryptoUtils.isValidKeyProvider(fileBasedProvider, missingLocation, KEY_ID, null)
+ logger.info("Key Provider ${fileBasedProvider} with location ${missingLocation} and keyId ${KEY_ID} / ${null} is ${missingKeyProviderIsValid ? "valid" : "invalid"}")
+
+ // Assert
+ assert !unreadableKeyProviderIsValid
+ assert !missingKeyProviderIsValid
+
+ // Make the file deletable so cleanup can occur
+ fileBasedProviderFile.setReadable(true, false)
+ Files.setPosixFilePermissions(fileBasedProviderFile.toPath(), ALL_POSIX_ATTRS)
+ }
+
+ @Test
+ void testShouldNotValidateFileBasedKeyProviderMissingKeyId() {
+ // Arrange
+ String fileBasedProvider = FileBasedKeyProvider.class.name
+ File fileBasedProviderFile = tempFolder.newFile("missing_key_id.kp")
+ String providerLocation = fileBasedProviderFile.path
+ logger.info("Created temporary file based key provider: ${providerLocation}")
+
+ // Act
+ boolean keyProviderIsValid = CryptoUtils.isValidKeyProvider(fileBasedProvider, providerLocation, null, null)
+ logger.info("Key Provider ${fileBasedProvider} with location ${providerLocation} and keyId ${null} / ${null} is ${keyProviderIsValid ? "valid" : "invalid"}")
+
+ // Assert
+ assert !keyProviderIsValid
+ }
+
+ @Test
+ void testShouldNotValidateUnknownKeyProvider() {
+ // Arrange
+ String providerImplementation = "org.apache.nifi.provenance.ImaginaryKeyProvider"
+ String providerLocation = null
+
+ // Act
+ boolean keyProviderIsValid = CryptoUtils.isValidKeyProvider(providerImplementation, providerLocation, KEY_ID, null)
+ logger.info("Key Provider ${providerImplementation} with location ${providerLocation} and keyId ${KEY_ID} / ${null} is ${keyProviderIsValid ? "valid" : "invalid"}")
+
+ // Assert
+ assert !keyProviderIsValid
+ }
+
+ @Test
+ void testShouldValidateKey() {
+ // Arrange
+ String validKey = KEY_HEX
+ String validLowercaseKey = KEY_HEX.toLowerCase()
+
+ String tooShortKey = KEY_HEX[0..<-2]
+ String tooLongKey = KEY_HEX + KEY_HEX // Guaranteed to be 2x the max valid key length
+ String nonHexKey = KEY_HEX.replaceFirst(/A/, "X")
+
+ def validKeys = [validKey, validLowercaseKey]
+ def invalidKeys = [tooShortKey, tooLongKey, nonHexKey]
+
+ // If unlimited strength is available, also validate 128 and 196 bit keys
+ if (isUnlimitedStrengthCryptoAvailable()) {
+ validKeys << KEY_HEX_128
+ validKeys << KEY_HEX_256[0..<48]
+ } else {
+ invalidKeys << KEY_HEX_256[0..<48]
+ invalidKeys << KEY_HEX_256
+ }
+
+ // Act
+ def validResults = validKeys.collect { String key ->
+ logger.info("Validating ${key}")
+ CryptoUtils.keyIsValid(key)
+ }
+
+ def invalidResults = invalidKeys.collect { String key ->
+ logger.info("Validating ${key}")
+ CryptoUtils.keyIsValid(key)
+ }
+
+ // Assert
+ assert validResults.every()
+ assert invalidResults.every { !it }
+ }
+
+ @Test
+ void testShouldReadKeys() {
+ // Arrange
+ String masterKeyHex = KEY_HEX
+ SecretKey masterKey = new SecretKeySpec(Hex.decode(masterKeyHex), "AES")
+
+ // Generate the file
+ String keyFileName = "keys.nkp"
+ File keyFile = tempFolder.newFile(keyFileName)
+ final int KEY_COUNT = 5
+ List lines = []
+ KEY_COUNT.times { int i ->
+ lines.add("key${i + 1}=${generateEncryptedKey(masterKey)}")
+ }
+
+ keyFile.text = lines.join("\n")
+
+ logger.info("File contents: \n${keyFile.text}")
+
+ // Act
+ def readKeys = CryptoUtils.readKeys(keyFile.path, masterKey)
+ logger.info("Read ${readKeys.size()} keys from ${keyFile.path}")
+
+ // Assert
+ assert readKeys.size() == KEY_COUNT
+ }
+
+ @Test
+ void testShouldReadKeysWithDuplicates() {
+ // Arrange
+ String masterKeyHex = KEY_HEX
+ SecretKey masterKey = new SecretKeySpec(Hex.decode(masterKeyHex), "AES")
+
+ // Generate the file
+ String keyFileName = "keys.nkp"
+ File keyFile = tempFolder.newFile(keyFileName)
+ final int KEY_COUNT = 3
+ List lines = []
+ KEY_COUNT.times { int i ->
+ lines.add("key${i + 1}=${generateEncryptedKey(masterKey)}")
+ }
+
+ lines.add("key3=${generateEncryptedKey(masterKey)}")
+
+ keyFile.text = lines.join("\n")
+
+ logger.info("File contents: \n${keyFile.text}")
+
+ // Act
+ def readKeys = CryptoUtils.readKeys(keyFile.path, masterKey)
+ logger.info("Read ${readKeys.size()} keys from ${keyFile.path}")
+
+ // Assert
+ assert readKeys.size() == KEY_COUNT
+ }
+
+ @Test
+ void testShouldReadKeysWithSomeMalformed() {
+ // Arrange
+ String masterKeyHex = KEY_HEX
+ SecretKey masterKey = new SecretKeySpec(Hex.decode(masterKeyHex), "AES")
+
+ // Generate the file
+ String keyFileName = "keys.nkp"
+ File keyFile = tempFolder.newFile(keyFileName)
+ final int KEY_COUNT = 5
+ List lines = []
+ KEY_COUNT.times { int i ->
+ lines.add("key${i + 1}=${generateEncryptedKey(masterKey)}")
+ }
+
+ // Insert the malformed keys in the middle
+ lines.add(2, "keyX1==${generateEncryptedKey(masterKey)}")
+ lines.add(4, "=${generateEncryptedKey(masterKey)}")
+ lines.add(6, "keyX3=non Base64-encoded data")
+
+ keyFile.text = lines.join("\n")
+
+ logger.info("File contents: \n${keyFile.text}")
+
+ // Act
+ def readKeys = CryptoUtils.readKeys(keyFile.path, masterKey)
+ logger.info("Read ${readKeys.size()} keys from ${keyFile.path}")
+
+ // Assert
+ assert readKeys.size() == KEY_COUNT
+ }
+
+ @Test
+ void testShouldNotReadKeysIfAllMalformed() {
+ // Arrange
+ String masterKeyHex = KEY_HEX
+ SecretKey masterKey = new SecretKeySpec(Hex.decode(masterKeyHex), "AES")
+
+ // Generate the file
+ String keyFileName = "keys.nkp"
+ File keyFile = tempFolder.newFile(keyFileName)
+ final int KEY_COUNT = 5
+ List lines = []
+
+ // All of these keys are malformed
+ KEY_COUNT.times { int i ->
+ lines.add("key${i + 1}=${generateEncryptedKey(masterKey)[0..<-4]}")
+ }
+
+ keyFile.text = lines.join("\n")
+
+ logger.info("File contents: \n${keyFile.text}")
+
+ // Act
+ def msg = shouldFail(KeyManagementException) {
+ def readKeys = CryptoUtils.readKeys(keyFile.path, masterKey)
+ logger.info("Read ${readKeys.size()} keys from ${keyFile.path}")
+ }
+
+ // Assert
+ assert msg.getMessage() == "The provided file contained no valid keys"
+ }
+
+ @Test
+ void testShouldNotReadKeysIfEmptyOrMissing() {
+ // Arrange
+ String masterKeyHex = KEY_HEX
+ SecretKey masterKey = new SecretKeySpec(Hex.decode(masterKeyHex), "AES")
+
+ // Generate the file
+ String keyFileName = "empty.nkp"
+ File keyFile = tempFolder.newFile(keyFileName)
+ logger.info("File contents: \n${keyFile.text}")
+
+ // Act
+ def missingMsg = shouldFail(KeyManagementException) {
+ def readKeys = CryptoUtils.readKeys(keyFile.path, masterKey)
+ logger.info("Read ${readKeys.size()} keys from ${keyFile.path}")
+ }
+ logger.expected("Missing file: ${missingMsg}")
+
+ def emptyMsg = shouldFail(KeyManagementException) {
+ def readKeys = CryptoUtils.readKeys(null, masterKey)
+ logger.info("Read ${readKeys.size()} keys from ${null}")
+ }
+ logger.expected("Empty file: ${emptyMsg}")
+
+ // Assert
+ assert missingMsg.getMessage() == "The provided file contained no valid keys"
+ assert emptyMsg.getMessage() == "The key provider file is not present and readable"
+ }
+
+ private static String generateEncryptedKey(SecretKey masterKey) {
+ byte[] ivBytes = new byte[16]
+ byte[] keyBytes = new byte[isUnlimitedStrengthCryptoAvailable() ? 32 : 16]
+
+ SecureRandom sr = new SecureRandom()
+ sr.nextBytes(ivBytes)
+ sr.nextBytes(keyBytes)
+
+ Cipher masterCipher = Cipher.getInstance("AES/GCM/NoPadding", "BC")
+ masterCipher.init(Cipher.ENCRYPT_MODE, masterKey, new IvParameterSpec(ivBytes))
+ byte[] cipherBytes = masterCipher.doFinal(keyBytes)
+
+ Base64.encoder.encodeToString(CryptoUtils.concatByteArrays(ivBytes, cipherBytes))
+ }
+}
diff --git a/nifi-commons/nifi-data-provenance-utils/src/test/java/org/apache/nifi/provenance/EncryptionExceptionTest.java b/nifi-commons/nifi-data-provenance-utils/src/test/java/org/apache/nifi/provenance/EncryptionExceptionTest.java
new file mode 100644
index 0000000000..e23d492c5a
--- /dev/null
+++ b/nifi-commons/nifi-data-provenance-utils/src/test/java/org/apache/nifi/provenance/EncryptionExceptionTest.java
@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.provenance;
+
+import org.junit.Test;
+
+public class EncryptionExceptionTest {
+
+ @Test
+ public void testShouldTriggerGroovyTestExecution() {
+ // This method does nothing but tell Maven to run the groovy tests
+ }
+}
diff --git a/nifi-commons/nifi-data-provenance-utils/src/test/resources/logback-test.xml b/nifi-commons/nifi-data-provenance-utils/src/test/resources/logback-test.xml
new file mode 100644
index 0000000000..c10508d6bb
--- /dev/null
+++ b/nifi-commons/nifi-data-provenance-utils/src/test/resources/logback-test.xml
@@ -0,0 +1,32 @@
+
+
+
+
+
+
+ %-4r [%t] %-5p %c - %m%n
+
+
+
+
+
+
+
+
+
+
+
+
diff --git a/nifi-commons/nifi-properties/src/main/java/org/apache/nifi/util/NiFiProperties.java b/nifi-commons/nifi-properties/src/main/java/org/apache/nifi/util/NiFiProperties.java
index 3d657fb46c..d69a280fd5 100644
--- a/nifi-commons/nifi-properties/src/main/java/org/apache/nifi/util/NiFiProperties.java
+++ b/nifi-commons/nifi-properties/src/main/java/org/apache/nifi/util/NiFiProperties.java
@@ -32,6 +32,7 @@ import java.util.List;
import java.util.Map;
import java.util.Properties;
import java.util.Set;
+import java.util.stream.Collectors;
/**
* The NiFiProperties class holds all properties which are needed for various
@@ -116,6 +117,11 @@ public abstract class NiFiProperties {
public static final String PROVENANCE_INDEXED_ATTRIBUTES = "nifi.provenance.repository.indexed.attributes";
public static final String PROVENANCE_INDEX_SHARD_SIZE = "nifi.provenance.repository.index.shard.size";
public static final String PROVENANCE_JOURNAL_COUNT = "nifi.provenance.repository.journal.count";
+ public static final String PROVENANCE_REPO_ENCRYPTION_KEY = "nifi.provenance.repository.encryption.key";
+ public static final String PROVENANCE_REPO_ENCRYPTION_KEY_ID = "nifi.provenance.repository.encryption.key.id";
+ public static final String PROVENANCE_REPO_ENCRYPTION_KEY_PROVIDER_IMPLEMENTATION_CLASS = "nifi.provenance.repository.encryption.key.provider.implementation";
+ public static final String PROVENANCE_REPO_ENCRYPTION_KEY_PROVIDER_LOCATION = "nifi.provenance.repository.encryption.key.provider.location";
+ public static final String PROVENANCE_REPO_DEBUG_FREQUENCY = "nifi.provenance.repository.debug.frequency";
// component status repository properties
public static final String COMPONENT_STATUS_REPOSITORY_IMPLEMENTATION = "nifi.components.status.repository.implementation";
@@ -769,7 +775,7 @@ public abstract class NiFiProperties {
/**
* Returns true if client certificates are required for REST API. Determined
* if the following conditions are all true:
- *
+ *
* - login identity provider is not populated
* - Kerberos service support is not enabled
*
@@ -1034,6 +1040,61 @@ public abstract class NiFiProperties {
return getPropertyKeys().size();
}
+ public String getProvenanceRepoEncryptionKeyId() {
+ return getProperty(PROVENANCE_REPO_ENCRYPTION_KEY_ID);
+ }
+
+ /**
+ * Returns the active provenance repository encryption key if a {@code StaticKeyProvider} is in use.
+ * If no key ID is specified in the properties file, the default
+ * {@code nifi.provenance.repository.encryption.key} value is returned. If a key ID is specified in
+ * {@code nifi.provenance.repository.encryption.key.id}, it will attempt to read from
+ * {@code nifi.provenance.repository.encryption.key.id.XYZ} where {@code XYZ} is the provided key
+ * ID. If that value is empty, it will use the default property
+ * {@code nifi.provenance.repository.encryption.key}.
+ *
+ * @return the provenance repository encryption key in hex form
+ */
+ public String getProvenanceRepoEncryptionKey() {
+ String keyId = getProvenanceRepoEncryptionKeyId();
+ String keyKey = StringUtils.isBlank(keyId) ? PROVENANCE_REPO_ENCRYPTION_KEY : PROVENANCE_REPO_ENCRYPTION_KEY + ".id." + keyId;
+ return getProperty(keyKey, getProperty(PROVENANCE_REPO_ENCRYPTION_KEY));
+ }
+
+ /**
+ * Returns a map of keyId -> key in hex loaded from the {@code nifi.properties} file if a
+ * {@code StaticKeyProvider} is defined. If {@code FileBasedKeyProvider} is defined, use
+ * {@code CryptoUtils#readKeys()} instead -- this method will return an empty map.
+ *
+ * @return a Map of the keys identified by key ID
+ */
+ public Map getProvenanceRepoEncryptionKeys() {
+ Map keys = new HashMap<>();
+ List keyProperties = getProvenanceRepositoryEncryptionKeyProperties();
+
+ // Retrieve the actual key values and store non-empty values in the map
+ for (String prop : keyProperties) {
+ final String value = getProperty(prop);
+ if (!StringUtils.isBlank(value)) {
+ if (prop.equalsIgnoreCase(PROVENANCE_REPO_ENCRYPTION_KEY)) {
+ prop = getProvenanceRepoEncryptionKeyId();
+ } else {
+ // Extract nifi.provenance.repository.encryption.key.id.key1 -> key1
+ prop = prop.substring(prop.lastIndexOf(".") + 1);
+ }
+ keys.put(prop, value);
+ }
+ }
+ return keys;
+ }
+
+ private List getProvenanceRepositoryEncryptionKeyProperties() {
+ // Filter all the property keys that define a key
+ return getPropertyKeys().stream().filter(k ->
+ k.startsWith(PROVENANCE_REPO_ENCRYPTION_KEY_ID + ".") || k.equalsIgnoreCase(PROVENANCE_REPO_ENCRYPTION_KEY)
+ ).collect(Collectors.toList());
+ }
+
/**
* Creates an instance of NiFiProperties. This should likely not be called
* by any classes outside of the NiFi framework but can be useful by the
@@ -1042,11 +1103,11 @@ public abstract class NiFiProperties {
* file specified cannot be found/read a runtime exception will be thrown.
* If one is not specified no properties will be loaded by default.
*
- * @param propertiesFilePath if provided properties will be loaded from
- * given file; else will be loaded from System property. Can be null.
+ * @param propertiesFilePath if provided properties will be loaded from
+ * given file; else will be loaded from System property. Can be null.
* @param additionalProperties allows overriding of properties with the
- * supplied values. these will be applied after loading from any properties
- * file. Can be null or empty.
+ * supplied values. these will be applied after loading from any properties
+ * file. Can be null or empty.
* @return NiFiProperties
*/
public static NiFiProperties createBasicNiFiProperties(final String propertiesFilePath, final Map additionalProperties) {
@@ -1108,10 +1169,9 @@ public abstract class NiFiProperties {
public void validate() {
// REMOTE_INPUT_HOST should be a valid hostname
String remoteInputHost = getProperty(REMOTE_INPUT_HOST);
- if(!StringUtils.isBlank(remoteInputHost) && remoteInputHost.split(":").length > 1) { // no scheme/port needed here (http://)
+ if (!StringUtils.isBlank(remoteInputHost) && remoteInputHost.split(":").length > 1) { // no scheme/port needed here (http://)
throw new IllegalArgumentException(remoteInputHost + " is not a correct value for " + REMOTE_INPUT_HOST + ". It should be a valid hostname without protocol or port.");
}
// Other properties to validate...
}
-
}
diff --git a/nifi-commons/nifi-security-utils/pom.xml b/nifi-commons/nifi-security-utils/pom.xml
index e2e5ee169a..b004c8b435 100644
--- a/nifi-commons/nifi-security-utils/pom.xml
+++ b/nifi-commons/nifi-security-utils/pom.xml
@@ -43,6 +43,10 @@
org.apache.commons
commons-lang3
+
+ commons-codec
+ commons-codec
+
org.bouncycastle
bcprov-jdk15on
diff --git a/nifi-commons/nifi-security-utils/src/main/java/org/apache/nifi/security/util/EncryptionMethod.java b/nifi-commons/nifi-security-utils/src/main/java/org/apache/nifi/security/util/EncryptionMethod.java
index 53664f1cde..a1ef2a4444 100644
--- a/nifi-commons/nifi-security-utils/src/main/java/org/apache/nifi/security/util/EncryptionMethod.java
+++ b/nifi-commons/nifi-security-utils/src/main/java/org/apache/nifi/security/util/EncryptionMethod.java
@@ -16,13 +16,13 @@
*/
package org.apache.nifi.security.util;
+import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.builder.ToStringBuilder;
import org.apache.commons.lang3.builder.ToStringStyle;
/**
* Enumeration capturing essential information about the various encryption
* methods that might be supported.
- *
*/
public enum EncryptionMethod {
@@ -105,4 +105,15 @@ public enum EncryptionMethod {
builder.append("Keyed cipher", isKeyedCipher());
return builder.toString();
}
+
+ public static EncryptionMethod forAlgorithm(String algorithm) {
+ if (StringUtils.isNotBlank(algorithm)) {
+ for (EncryptionMethod em : EncryptionMethod.values()) {
+ if (em.algorithm.equalsIgnoreCase(algorithm)) {
+ return em;
+ }
+ }
+ }
+ return null;
+ }
}
diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/util/crypto/AESKeyedCipherProvider.java b/nifi-commons/nifi-security-utils/src/main/java/org/apache/nifi/security/util/crypto/AESKeyedCipherProvider.java
similarity index 99%
rename from nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/util/crypto/AESKeyedCipherProvider.java
rename to nifi-commons/nifi-security-utils/src/main/java/org/apache/nifi/security/util/crypto/AESKeyedCipherProvider.java
index 907aed268e..617a3e5e74 100644
--- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/util/crypto/AESKeyedCipherProvider.java
+++ b/nifi-commons/nifi-security-utils/src/main/java/org/apache/nifi/security/util/crypto/AESKeyedCipherProvider.java
@@ -14,18 +14,8 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.nifi.processors.standard.util.crypto;
+package org.apache.nifi.security.util.crypto;
-import org.apache.commons.lang3.StringUtils;
-import org.apache.nifi.processor.exception.ProcessException;
-import org.apache.nifi.security.util.EncryptionMethod;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import javax.crypto.Cipher;
-import javax.crypto.NoSuchPaddingException;
-import javax.crypto.SecretKey;
-import javax.crypto.spec.IvParameterSpec;
import java.io.UnsupportedEncodingException;
import java.security.InvalidAlgorithmParameterException;
import java.security.InvalidKeyException;
@@ -35,6 +25,15 @@ import java.security.SecureRandom;
import java.security.spec.InvalidKeySpecException;
import java.util.Arrays;
import java.util.List;
+import javax.crypto.Cipher;
+import javax.crypto.NoSuchPaddingException;
+import javax.crypto.SecretKey;
+import javax.crypto.spec.IvParameterSpec;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.nifi.processor.exception.ProcessException;
+import org.apache.nifi.security.util.EncryptionMethod;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
* This is a standard implementation of {@link KeyedCipherProvider} which supports {@code AES} cipher families with arbitrary modes of operation (currently only {@code CBC}, {@code CTR}, and {@code
diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/util/crypto/CipherProvider.java b/nifi-commons/nifi-security-utils/src/main/java/org/apache/nifi/security/util/crypto/CipherProvider.java
similarity index 93%
rename from nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/util/crypto/CipherProvider.java
rename to nifi-commons/nifi-security-utils/src/main/java/org/apache/nifi/security/util/crypto/CipherProvider.java
index 46e815fcc9..e3632b2c1b 100644
--- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/util/crypto/CipherProvider.java
+++ b/nifi-commons/nifi-security-utils/src/main/java/org/apache/nifi/security/util/crypto/CipherProvider.java
@@ -14,7 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.nifi.processors.standard.util.crypto;
+package org.apache.nifi.security.util.crypto;
/**
* Marker interface for cipher providers.
diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/util/crypto/CipherUtility.java b/nifi-commons/nifi-security-utils/src/main/java/org/apache/nifi/security/util/crypto/CipherUtility.java
similarity index 99%
rename from nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/util/crypto/CipherUtility.java
rename to nifi-commons/nifi-security-utils/src/main/java/org/apache/nifi/security/util/crypto/CipherUtility.java
index 6295c1d11d..6ba8056a96 100644
--- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/util/crypto/CipherUtility.java
+++ b/nifi-commons/nifi-security-utils/src/main/java/org/apache/nifi/security/util/crypto/CipherUtility.java
@@ -14,16 +14,8 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.nifi.processors.standard.util.crypto;
+package org.apache.nifi.security.util.crypto;
-import org.apache.commons.codec.binary.Base64;
-import org.apache.commons.lang3.StringUtils;
-import org.apache.nifi.processor.exception.ProcessException;
-import org.apache.nifi.security.util.EncryptionMethod;
-import org.apache.nifi.stream.io.ByteArrayOutputStream;
-import org.apache.nifi.stream.io.StreamUtils;
-
-import javax.crypto.Cipher;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
@@ -35,6 +27,13 @@ import java.util.List;
import java.util.Map;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
+import javax.crypto.Cipher;
+import org.apache.commons.codec.binary.Base64;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.nifi.processor.exception.ProcessException;
+import org.apache.nifi.security.util.EncryptionMethod;
+import org.apache.nifi.stream.io.ByteArrayOutputStream;
+import org.apache.nifi.stream.io.StreamUtils;
public class CipherUtility {
diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/util/crypto/KeyedCipherProvider.java b/nifi-commons/nifi-security-utils/src/main/java/org/apache/nifi/security/util/crypto/KeyedCipherProvider.java
similarity index 98%
rename from nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/util/crypto/KeyedCipherProvider.java
rename to nifi-commons/nifi-security-utils/src/main/java/org/apache/nifi/security/util/crypto/KeyedCipherProvider.java
index f0fa4fc0f4..719150f98b 100644
--- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/util/crypto/KeyedCipherProvider.java
+++ b/nifi-commons/nifi-security-utils/src/main/java/org/apache/nifi/security/util/crypto/KeyedCipherProvider.java
@@ -14,17 +14,16 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.nifi.processors.standard.util.crypto;
+package org.apache.nifi.security.util.crypto;
-import org.apache.nifi.processor.exception.ProcessException;
-import org.apache.nifi.security.util.EncryptionMethod;
-
-import javax.crypto.Cipher;
-import javax.crypto.SecretKey;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.nio.charset.StandardCharsets;
+import javax.crypto.Cipher;
+import javax.crypto.SecretKey;
+import org.apache.nifi.processor.exception.ProcessException;
+import org.apache.nifi.security.util.EncryptionMethod;
public abstract class KeyedCipherProvider implements CipherProvider {
static final byte[] IV_DELIMITER = "NiFiIV".getBytes(StandardCharsets.UTF_8);
diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/groovy/org/apache/nifi/processors/standard/util/crypto/AESKeyedCipherProviderGroovyTest.groovy b/nifi-commons/nifi-security-utils/src/test/groovy/org/apache/nifi/security/util/crypto/AESKeyedCipherProviderGroovyTest.groovy
similarity index 89%
rename from nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/groovy/org/apache/nifi/processors/standard/util/crypto/AESKeyedCipherProviderGroovyTest.groovy
rename to nifi-commons/nifi-security-utils/src/test/groovy/org/apache/nifi/security/util/crypto/AESKeyedCipherProviderGroovyTest.groovy
index 0596d7d233..80821496ba 100644
--- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/groovy/org/apache/nifi/processors/standard/util/crypto/AESKeyedCipherProviderGroovyTest.groovy
+++ b/nifi-commons/nifi-security-utils/src/test/groovy/org/apache/nifi/security/util/crypto/AESKeyedCipherProviderGroovyTest.groovy
@@ -14,12 +14,16 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.nifi.processors.standard.util.crypto
+package org.apache.nifi.security.util.crypto
import org.apache.commons.codec.binary.Hex
import org.apache.nifi.security.util.EncryptionMethod
import org.bouncycastle.jce.provider.BouncyCastleProvider
-import org.junit.*
+import org.junit.After
+import org.junit.Assume
+import org.junit.Before
+import org.junit.BeforeClass
+import org.junit.Test
import org.junit.runner.RunWith
import org.junit.runners.JUnit4
import org.slf4j.Logger
@@ -34,7 +38,7 @@ import java.security.Security
import static groovy.test.GroovyAssert.shouldFail
@RunWith(JUnit4.class)
-public class AESKeyedCipherProviderGroovyTest {
+class AESKeyedCipherProviderGroovyTest {
private static final Logger logger = LoggerFactory.getLogger(AESKeyedCipherProviderGroovyTest.class)
private static final String KEY_HEX = "0123456789ABCDEFFEDCBA9876543210"
@@ -44,7 +48,7 @@ public class AESKeyedCipherProviderGroovyTest {
private static final SecretKey key = new SecretKeySpec(Hex.decodeHex(KEY_HEX as char[]), "AES")
@BeforeClass
- public static void setUpOnce() throws Exception {
+ static void setUpOnce() throws Exception {
Security.addProvider(new BouncyCastleProvider())
logger.metaClass.methodMissing = { String name, args ->
@@ -53,15 +57,19 @@ public class AESKeyedCipherProviderGroovyTest {
}
@Before
- public void setUp() throws Exception {
+ void setUp() throws Exception {
}
@After
- public void tearDown() throws Exception {
+ void tearDown() throws Exception {
+ }
+
+ private static boolean isUnlimitedStrengthCryptoAvailable() {
+ Cipher.getMaxAllowedKeyLength("AES") > 128
}
@Test
- public void testGetCipherShouldBeInternallyConsistent() throws Exception {
+ void testGetCipherShouldBeInternallyConsistent() throws Exception {
// Arrange
KeyedCipherProvider cipherProvider = new AESKeyedCipherProvider()
@@ -90,7 +98,7 @@ public class AESKeyedCipherProviderGroovyTest {
}
@Test
- public void testGetCipherWithExternalIVShouldBeInternallyConsistent() throws Exception {
+ void testGetCipherWithExternalIVShouldBeInternallyConsistent() throws Exception {
// Arrange
KeyedCipherProvider cipherProvider = new AESKeyedCipherProvider()
@@ -119,10 +127,9 @@ public class AESKeyedCipherProviderGroovyTest {
}
@Test
- public void testGetCipherWithUnlimitedStrengthShouldBeInternallyConsistent() throws Exception {
+ void testGetCipherWithUnlimitedStrengthShouldBeInternallyConsistent() throws Exception {
// Arrange
- Assume.assumeTrue("Test is being skipped due to this JVM lacking JCE Unlimited Strength Jurisdiction Policy file.",
- PasswordBasedEncryptor.supportsUnlimitedStrength())
+ Assume.assumeTrue("Test is being skipped due to this JVM lacking JCE Unlimited Strength Jurisdiction Policy file.", isUnlimitedStrengthCryptoAvailable())
KeyedCipherProvider cipherProvider = new AESKeyedCipherProvider()
final List LONG_KEY_LENGTHS = [192, 256]
@@ -164,7 +171,7 @@ public class AESKeyedCipherProviderGroovyTest {
}
@Test
- public void testShouldRejectEmptyKey() throws Exception {
+ void testShouldRejectEmptyKey() throws Exception {
// Arrange
KeyedCipherProvider cipherProvider = new AESKeyedCipherProvider()
@@ -180,7 +187,7 @@ public class AESKeyedCipherProviderGroovyTest {
}
@Test
- public void testShouldRejectIncorrectLengthKey() throws Exception {
+ void testShouldRejectIncorrectLengthKey() throws Exception {
// Arrange
KeyedCipherProvider cipherProvider = new AESKeyedCipherProvider()
@@ -199,7 +206,7 @@ public class AESKeyedCipherProviderGroovyTest {
}
@Test
- public void testShouldRejectEmptyEncryptionMethod() throws Exception {
+ void testShouldRejectEmptyEncryptionMethod() throws Exception {
// Arrange
KeyedCipherProvider cipherProvider = new AESKeyedCipherProvider()
@@ -213,7 +220,7 @@ public class AESKeyedCipherProviderGroovyTest {
}
@Test
- public void testShouldRejectUnsupportedEncryptionMethod() throws Exception {
+ void testShouldRejectUnsupportedEncryptionMethod() throws Exception {
// Arrange
KeyedCipherProvider cipherProvider = new AESKeyedCipherProvider()
@@ -229,7 +236,7 @@ public class AESKeyedCipherProviderGroovyTest {
}
@Test
- public void testGetCipherShouldSupportExternalCompatibility() throws Exception {
+ void testGetCipherShouldSupportExternalCompatibility() throws Exception {
// Arrange
KeyedCipherProvider cipherProvider = new AESKeyedCipherProvider()
@@ -258,7 +265,7 @@ public class AESKeyedCipherProviderGroovyTest {
}
@Test
- public void testGetCipherForDecryptShouldRequireIV() throws Exception {
+ void testGetCipherForDecryptShouldRequireIV() throws Exception {
// Arrange
KeyedCipherProvider cipherProvider = new AESKeyedCipherProvider()
@@ -286,7 +293,7 @@ public class AESKeyedCipherProviderGroovyTest {
}
@Test
- public void testGetCipherShouldRejectInvalidIVLengths() throws Exception {
+ void testGetCipherShouldRejectInvalidIVLengths() throws Exception {
// Arrange
KeyedCipherProvider cipherProvider = new AESKeyedCipherProvider()
@@ -313,7 +320,7 @@ public class AESKeyedCipherProviderGroovyTest {
}
@Test
- public void testGetCipherShouldRejectEmptyIV() throws Exception {
+ void testGetCipherShouldRejectEmptyIV() throws Exception {
// Arrange
KeyedCipherProvider cipherProvider = new AESKeyedCipherProvider()
diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/groovy/org/apache/nifi/processors/standard/util/crypto/CipherUtilityGroovyTest.groovy b/nifi-commons/nifi-security-utils/src/test/groovy/org/apache/nifi/security/util/crypto/CipherUtilityGroovyTest.groovy
similarity index 99%
rename from nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/groovy/org/apache/nifi/processors/standard/util/crypto/CipherUtilityGroovyTest.groovy
rename to nifi-commons/nifi-security-utils/src/test/groovy/org/apache/nifi/security/util/crypto/CipherUtilityGroovyTest.groovy
index 6a6a95859d..8f092f3ff5 100644
--- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/groovy/org/apache/nifi/processors/standard/util/crypto/CipherUtilityGroovyTest.groovy
+++ b/nifi-commons/nifi-security-utils/src/test/groovy/org/apache/nifi/security/util/crypto/CipherUtilityGroovyTest.groovy
@@ -14,7 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.nifi.processors.standard.util.crypto
+package org.apache.nifi.security.util.crypto
import org.apache.nifi.security.util.EncryptionMethod
import org.bouncycastle.jce.provider.BouncyCastleProvider
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/FlowController.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/FlowController.java
index 7853591a83..5b2ea41976 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/FlowController.java
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/FlowController.java
@@ -16,7 +16,38 @@
*/
package org.apache.nifi.controller;
+import static java.util.Objects.requireNonNull;
+
import com.sun.jersey.api.client.ClientHandlerException;
+import java.io.ByteArrayInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.net.URL;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.LinkedHashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.UUID;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.Future;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.ScheduledFuture;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicReference;
+import java.util.concurrent.locks.Lock;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+import java.util.stream.Collectors;
+import javax.net.ssl.SSLContext;
import org.apache.commons.collections4.Predicate;
import org.apache.commons.lang3.StringUtils;
import org.apache.nifi.action.Action;
@@ -85,7 +116,6 @@ import org.apache.nifi.controller.repository.FlowFileRecord;
import org.apache.nifi.controller.repository.FlowFileRepository;
import org.apache.nifi.controller.repository.FlowFileSwapManager;
import org.apache.nifi.controller.repository.QueueProvider;
-import org.apache.nifi.controller.repository.RepositoryRecord;
import org.apache.nifi.controller.repository.RepositoryStatusReport;
import org.apache.nifi.controller.repository.StandardCounterRepository;
import org.apache.nifi.controller.repository.StandardFlowFileRecord;
@@ -217,38 +247,6 @@ import org.apache.zookeeper.server.quorum.QuorumPeerConfig.ConfigException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import javax.net.ssl.SSLContext;
-import java.io.ByteArrayInputStream;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.OutputStream;
-import java.net.URL;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.Date;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.LinkedHashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.UUID;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.ConcurrentMap;
-import java.util.concurrent.Future;
-import java.util.concurrent.ScheduledExecutorService;
-import java.util.concurrent.ScheduledFuture;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicBoolean;
-import java.util.concurrent.atomic.AtomicInteger;
-import java.util.concurrent.atomic.AtomicReference;
-import java.util.concurrent.locks.Lock;
-import java.util.concurrent.locks.ReentrantReadWriteLock;
-import java.util.stream.Collectors;
-
-import static java.util.Objects.requireNonNull;
-
public class FlowController implements EventAccess, ControllerServiceProvider, ReportingTaskProvider,
QueueProvider, Authorizable, ProvenanceAuthorizableFactory, NodeTypeProvider, IdentifierLookup, ReloadComponent {
@@ -3841,7 +3839,7 @@ public class FlowController implements EventAccess, ControllerServiceProvider, R
final ProvenanceEventRecord sendEvent = new StandardProvenanceEventRecord.Builder()
.setEventType(ProvenanceEventType.DOWNLOAD)
.setFlowFileUUID(provEvent.getFlowFileUuid())
- .setAttributes(provEvent.getAttributes(), Collections.emptyMap())
+ .setAttributes(provEvent.getAttributes(), Collections.emptyMap())
.setCurrentContentClaim(resourceClaim.getContainer(), resourceClaim.getSection(), resourceClaim.getId(), offset, size)
.setTransitUri(requestUri)
.setEventTime(System.currentTimeMillis())
@@ -3883,7 +3881,7 @@ public class FlowController implements EventAccess, ControllerServiceProvider, R
final StandardProvenanceEventRecord.Builder sendEventBuilder = new StandardProvenanceEventRecord.Builder()
.setEventType(ProvenanceEventType.DOWNLOAD)
.setFlowFileUUID(flowFile.getAttribute(CoreAttributes.UUID.key()))
- .setAttributes(flowFile.getAttributes(), Collections.emptyMap())
+ .setAttributes(flowFile.getAttributes(), Collections.emptyMap())
.setTransitUri(requestUri)
.setEventTime(System.currentTimeMillis())
.setFlowFileEntryDate(flowFile.getEntryDate())
@@ -4062,7 +4060,7 @@ public class FlowController implements EventAccess, ControllerServiceProvider, R
.addChildUuid(newFlowFileUUID)
.addParentUuid(parentUUID)
.setFlowFileUUID(parentUUID)
- .setAttributes(Collections.emptyMap(), flowFileRecord.getAttributes())
+ .setAttributes(Collections.emptyMap(), flowFileRecord.getAttributes())
.setCurrentContentClaim(event.getContentClaimContainer(), event.getContentClaimSection(), event.getContentClaimIdentifier(), event.getContentClaimOffset(), event.getFileSize())
.setDetails("Replay requested by " + user.getIdentity())
.setEventTime(System.currentTimeMillis())
@@ -4077,7 +4075,7 @@ public class FlowController implements EventAccess, ControllerServiceProvider, R
final StandardRepositoryRecord record = new StandardRepositoryRecord(queue);
record.setWorking(flowFileRecord);
record.setDestination(queue);
- flowFileRepository.updateRepository(Collections.singleton(record));
+ flowFileRepository.updateRepository(Collections.singleton(record));
// Enqueue the data
queue.put(flowFileRecord);
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/main/java/org/apache/nifi/properties/ProtectedNiFiProperties.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/main/java/org/apache/nifi/properties/ProtectedNiFiProperties.java
index 4774dc75df..fc1d72231f 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/main/java/org/apache/nifi/properties/ProtectedNiFiProperties.java
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/main/java/org/apache/nifi/properties/ProtectedNiFiProperties.java
@@ -26,6 +26,7 @@ import java.util.List;
import java.util.Map;
import java.util.Properties;
import java.util.Set;
+import java.util.stream.Collectors;
import org.apache.commons.lang3.StringUtils;
import org.apache.nifi.util.NiFiProperties;
import org.slf4j.Logger;
@@ -50,7 +51,7 @@ class ProtectedNiFiProperties extends StandardNiFiProperties {
// Default list of "sensitive" property keys
public static final List DEFAULT_SENSITIVE_PROPERTIES = new ArrayList<>(asList(SECURITY_KEY_PASSWD,
- SECURITY_KEYSTORE_PASSWD, SECURITY_TRUSTSTORE_PASSWD, SENSITIVE_PROPS_KEY));
+ SECURITY_KEYSTORE_PASSWD, SECURITY_TRUSTSTORE_PASSWD, SENSITIVE_PROPS_KEY, PROVENANCE_REPO_ENCRYPTION_KEY));
public ProtectedNiFiProperties() {
this(new StandardNiFiProperties());
@@ -183,6 +184,17 @@ class ProtectedNiFiProperties extends StandardNiFiProperties {
}
}
+ /**
+ * Returns a list of the keys identifying "sensitive" properties. There is a default list,
+ * and additional keys can be provided in the {@code nifi.sensitive.props.additional.keys} property in {@code nifi.properties}.
+ *
+ * @return the list of sensitive property keys
+ */
+ public List getPopulatedSensitivePropertyKeys() {
+ List allSensitiveKeys = getSensitivePropertyKeys();
+ return allSensitiveKeys.stream().filter(k -> StringUtils.isNotBlank(getProperty(k))).collect(Collectors.toList());
+ }
+
/**
* Returns true if any sensitive keys are protected.
*
@@ -219,7 +231,7 @@ class ProtectedNiFiProperties extends StandardNiFiProperties {
Map traditionalProtectedProperties = new HashMap<>();
for (String key : sensitiveKeys) {
String protection = getProperty(getProtectionKey(key));
- if (!StringUtils.isBlank(protection)) {
+ if (StringUtils.isNotBlank(protection) && StringUtils.isNotBlank(getProperty(key))) {
traditionalProtectedProperties.put(key, protection);
}
}
@@ -237,12 +249,12 @@ class ProtectedNiFiProperties extends StandardNiFiProperties {
}
/**
- * Returns a percentage of the total number of properties marked as sensitive that are currently protected.
+ * Returns a percentage of the total number of populated properties marked as sensitive that are currently protected.
*
* @return the percent of sensitive properties marked as protected
*/
public int getPercentOfSensitivePropertiesProtected() {
- return (int) Math.round(getProtectedPropertyKeys().size() / ((double) getSensitivePropertyKeys().size()) * 100);
+ return (int) Math.round(getProtectedPropertyKeys().size() / ((double) getPopulatedSensitivePropertyKeys().size()) * 100);
}
/**
@@ -421,9 +433,7 @@ class ProtectedNiFiProperties extends StandardNiFiProperties {
// Add the protected keys and the protection schemes
for (String key : getSensitivePropertyKeys()) {
final String plainValue = getInternalNiFiProperties().getProperty(key);
- if (plainValue == null || plainValue.trim().isEmpty()) {
- protectedProperties.setProperty(key, plainValue);
- } else {
+ if (plainValue != null && !plainValue.trim().isEmpty()) {
final String protectedValue = spp.protect(plainValue);
protectedProperties.setProperty(key, protectedValue);
protectedProperties.setProperty(getProtectionKey(key), protectionScheme);
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/groovy/org/apache/nifi/properties/AESSensitivePropertyProviderTest.groovy b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/groovy/org/apache/nifi/properties/AESSensitivePropertyProviderTest.groovy
index 7896afe9dc..73ae55a8e0 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/groovy/org/apache/nifi/properties/AESSensitivePropertyProviderTest.groovy
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/groovy/org/apache/nifi/properties/AESSensitivePropertyProviderTest.groovy
@@ -52,7 +52,7 @@ class AESSensitivePropertyProviderTest extends GroovyTestCase {
private static final Base64.Decoder decoder = Base64.decoder
@BeforeClass
- public static void setUpOnce() throws Exception {
+ static void setUpOnce() throws Exception {
Security.addProvider(new BouncyCastleProvider())
logger.metaClass.methodMissing = { String name, args ->
@@ -61,12 +61,12 @@ class AESSensitivePropertyProviderTest extends GroovyTestCase {
}
@Before
- public void setUp() throws Exception {
+ void setUp() throws Exception {
}
@After
- public void tearDown() throws Exception {
+ void tearDown() throws Exception {
}
@@ -112,7 +112,7 @@ class AESSensitivePropertyProviderTest extends GroovyTestCase {
}
@Test
- public void testShouldThrowExceptionOnInitializationWithoutBouncyCastle() throws Exception {
+ void testShouldThrowExceptionOnInitializationWithoutBouncyCastle() throws Exception {
// Arrange
try {
Security.removeProvider(new BouncyCastleProvider().getName())
@@ -133,7 +133,7 @@ class AESSensitivePropertyProviderTest extends GroovyTestCase {
// TODO: testShouldGetName()
@Test
- public void testShouldProtectValue() throws Exception {
+ void testShouldProtectValue() throws Exception {
final String PLAINTEXT = "This is a plaintext value"
// Act
@@ -163,7 +163,7 @@ class AESSensitivePropertyProviderTest extends GroovyTestCase {
}
@Test
- public void testShouldHandleProtectEmptyValue() throws Exception {
+ void testShouldHandleProtectEmptyValue() throws Exception {
final List EMPTY_PLAINTEXTS = ["", " ", null]
// Act
@@ -183,7 +183,7 @@ class AESSensitivePropertyProviderTest extends GroovyTestCase {
}
@Test
- public void testShouldUnprotectValue() throws Exception {
+ void testShouldUnprotectValue() throws Exception {
// Arrange
final String PLAINTEXT = "This is a plaintext value"
@@ -218,7 +218,7 @@ class AESSensitivePropertyProviderTest extends GroovyTestCase {
* @throws Exception
*/
@Test
- public void testShouldHandleUnprotectEmptyValue() throws Exception {
+ void testShouldHandleUnprotectEmptyValue() throws Exception {
// Arrange
final List EMPTY_CIPHER_TEXTS = ["", " ", null]
@@ -239,7 +239,7 @@ class AESSensitivePropertyProviderTest extends GroovyTestCase {
}
@Test
- public void testShouldUnprotectValueWithWhitespace() throws Exception {
+ void testShouldUnprotectValueWithWhitespace() throws Exception {
// Arrange
final String PLAINTEXT = "This is a plaintext value"
@@ -269,7 +269,7 @@ class AESSensitivePropertyProviderTest extends GroovyTestCase {
}
@Test
- public void testShouldHandleUnprotectMalformedValue() throws Exception {
+ void testShouldHandleUnprotectMalformedValue() throws Exception {
// Arrange
final String PLAINTEXT = "This is a plaintext value"
@@ -293,7 +293,7 @@ class AESSensitivePropertyProviderTest extends GroovyTestCase {
}
@Test
- public void testShouldHandleUnprotectMissingIV() throws Exception {
+ void testShouldHandleUnprotectMissingIV() throws Exception {
// Arrange
final String PLAINTEXT = "This is a plaintext value"
@@ -334,7 +334,7 @@ class AESSensitivePropertyProviderTest extends GroovyTestCase {
* @throws Exception
*/
@Test
- public void testShouldHandleUnprotectEmptyCipherText() throws Exception {
+ void testShouldHandleUnprotectEmptyCipherText() throws Exception {
// Arrange
final String IV_AND_DELIMITER = "${encoder.encodeToString("Bad IV value".getBytes(StandardCharsets.UTF_8))}||"
logger.info("IV and delimiter: ${IV_AND_DELIMITER}")
@@ -358,7 +358,7 @@ class AESSensitivePropertyProviderTest extends GroovyTestCase {
}
@Test
- public void testShouldHandleUnprotectMalformedIV() throws Exception {
+ void testShouldHandleUnprotectMalformedIV() throws Exception {
// Arrange
final String PLAINTEXT = "This is a plaintext value"
@@ -382,7 +382,7 @@ class AESSensitivePropertyProviderTest extends GroovyTestCase {
}
@Test
- public void testShouldGetIdentifierKeyWithDifferentMaxKeyLengths() throws Exception {
+ void testShouldGetIdentifierKeyWithDifferentMaxKeyLengths() throws Exception {
// Arrange
def keys = getAvailableKeySizes().collectEntries { int keySize ->
[(keySize): getKeyOfSize(keySize)]
@@ -400,7 +400,7 @@ class AESSensitivePropertyProviderTest extends GroovyTestCase {
}
@Test
- public void testShouldNotAllowEmptyKey() throws Exception {
+ void testShouldNotAllowEmptyKey() throws Exception {
// Arrange
final String INVALID_KEY = ""
@@ -414,7 +414,7 @@ class AESSensitivePropertyProviderTest extends GroovyTestCase {
}
@Test
- public void testShouldNotAllowIncorrectlySizedKey() throws Exception {
+ void testShouldNotAllowIncorrectlySizedKey() throws Exception {
// Arrange
final String INVALID_KEY = "Z" * 31
@@ -428,7 +428,7 @@ class AESSensitivePropertyProviderTest extends GroovyTestCase {
}
@Test
- public void testShouldNotAllowInvalidKey() throws Exception {
+ void testShouldNotAllowInvalidKey() throws Exception {
// Arrange
final String INVALID_KEY = "Z" * 32
@@ -445,7 +445,7 @@ class AESSensitivePropertyProviderTest extends GroovyTestCase {
* This test is to ensure internal consistency and allow for encrypting value for various property files
*/
@Test
- public void testShouldEncryptArbitraryValues() {
+ void testShouldEncryptArbitraryValues() {
// Arrange
def values = ["thisIsABadPassword", "thisIsABadSensitiveKeyPassword", "thisIsABadKeystorePassword", "thisIsABadKeyPassword", "thisIsABadTruststorePassword", "This is an encrypted banner message", "nififtw!"]
@@ -471,15 +471,15 @@ class AESSensitivePropertyProviderTest extends GroovyTestCase {
* This test is to ensure external compatibility in case someone encodes the encrypted value with Base64 and does not remove the padding
*/
@Test
- public void testShouldDecryptPaddedValue() {
+ void testShouldDecryptPaddedValue() {
// Arrange
Assume.assumeTrue("JCE unlimited strength crypto policy must be installed for this test", Cipher.getMaxAllowedKeyLength("AES") > 128)
- final String EXPECTED_VALUE = "thisIsABadKeyPassword"
- String cipherText = "ac/BaE35SL/esLiJ||+ULRvRLYdIDA2VqpE0eQXDEMjaLBMG2kbKOdOwBk/hGebDKlVg=="
+ final String EXPECTED_VALUE = getKeyOfSize(256) // "thisIsABadKeyPassword"
+ String cipherText = "aYDkDKys1ENr3gp+||sTBPpMlIvHcOLTGZlfWct8r9RY8BuDlDkoaYmGJ/9m9af9tZIVzcnDwvYQAaIKxRGF7vI2yrY7Xd6x9GTDnWGiGiRXlaP458BBMMgfzH2O8"
String unpaddedCipherText = cipherText.replaceAll("=", "")
- String key = getKeyOfSize(256)
+ String key = "AAAABBBBCCCCDDDDEEEEFFFF00001111" * 2 // getKeyOfSize(256)
SensitivePropertyProvider spp = new AESSensitivePropertyProvider(key)
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/groovy/org/apache/nifi/properties/ProtectedNiFiPropertiesGroovyTest.groovy b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/groovy/org/apache/nifi/properties/ProtectedNiFiPropertiesGroovyTest.groovy
index 0d5c976df4..6656867f9f 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/groovy/org/apache/nifi/properties/ProtectedNiFiPropertiesGroovyTest.groovy
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/groovy/org/apache/nifi/properties/ProtectedNiFiPropertiesGroovyTest.groovy
@@ -38,7 +38,8 @@ class ProtectedNiFiPropertiesGroovyTest extends GroovyTestCase {
"nifi.sensitive.props.key",
"nifi.security.keystorePasswd",
"nifi.security.keyPasswd",
- "nifi.security.truststorePasswd"
+ "nifi.security.truststorePasswd",
+ "nifi.provenance.repository.encryption.key"
]
final def COMMON_ADDITIONAL_SENSITIVE_PROPERTIES = [
@@ -53,7 +54,7 @@ class ProtectedNiFiPropertiesGroovyTest extends GroovyTestCase {
private static String originalPropertiesPath = System.getProperty(NiFiProperties.PROPERTIES_FILE_PATH)
@BeforeClass
- public static void setUpOnce() throws Exception {
+ static void setUpOnce() throws Exception {
Security.addProvider(new BouncyCastleProvider())
logger.metaClass.methodMissing = { String name, args ->
@@ -62,15 +63,15 @@ class ProtectedNiFiPropertiesGroovyTest extends GroovyTestCase {
}
@Before
- public void setUp() throws Exception {
+ void setUp() throws Exception {
}
@After
- public void tearDown() throws Exception {
+ void tearDown() throws Exception {
}
@AfterClass
- public static void tearDownOnce() {
+ static void tearDownOnce() {
if (originalPropertiesPath) {
System.setProperty(NiFiProperties.PROPERTIES_FILE_PATH, originalPropertiesPath)
}
@@ -127,7 +128,7 @@ class ProtectedNiFiPropertiesGroovyTest extends GroovyTestCase {
}
@Test
- public void testConstructorShouldCreateNewInstance() throws Exception {
+ void testConstructorShouldCreateNewInstance() throws Exception {
// Arrange
// Act
@@ -140,7 +141,7 @@ class ProtectedNiFiPropertiesGroovyTest extends GroovyTestCase {
}
@Test
- public void testConstructorShouldAcceptRawProperties() throws Exception {
+ void testConstructorShouldAcceptRawProperties() throws Exception {
// Arrange
Properties rawProperties = new Properties()
rawProperties.setProperty("key", "value")
@@ -157,7 +158,7 @@ class ProtectedNiFiPropertiesGroovyTest extends GroovyTestCase {
}
@Test
- public void testConstructorShouldAcceptNiFiProperties() throws Exception {
+ void testConstructorShouldAcceptNiFiProperties() throws Exception {
// Arrange
Properties rawProperties = new Properties()
rawProperties.setProperty("key", "value")
@@ -178,7 +179,7 @@ class ProtectedNiFiPropertiesGroovyTest extends GroovyTestCase {
}
@Test
- public void testShouldAllowMultipleInstances() throws Exception {
+ void testShouldAllowMultipleInstances() throws Exception {
// Arrange
Properties rawProperties = new Properties()
rawProperties.setProperty("key", "value")
@@ -200,7 +201,7 @@ class ProtectedNiFiPropertiesGroovyTest extends GroovyTestCase {
}
@Test
- public void testShouldDetectIfPropertyIsSensitive() throws Exception {
+ void testShouldDetectIfPropertyIsSensitive() throws Exception {
// Arrange
final String INSENSITIVE_PROPERTY_KEY = "nifi.ui.banner.text"
final String SENSITIVE_PROPERTY_KEY = "nifi.security.keystorePasswd"
@@ -219,7 +220,7 @@ class ProtectedNiFiPropertiesGroovyTest extends GroovyTestCase {
}
@Test
- public void testShouldGetDefaultSensitiveProperties() throws Exception {
+ void testShouldGetDefaultSensitiveProperties() throws Exception {
// Arrange
logger.expected("${DEFAULT_SENSITIVE_PROPERTIES.size()} default sensitive properties: ${DEFAULT_SENSITIVE_PROPERTIES.join(", ")}")
@@ -235,9 +236,9 @@ class ProtectedNiFiPropertiesGroovyTest extends GroovyTestCase {
}
@Test
- public void testShouldGetAdditionalSensitiveProperties() throws Exception {
+ void testShouldGetAdditionalSensitiveProperties() throws Exception {
// Arrange
- def completeSensitiveProperties = DEFAULT_SENSITIVE_PROPERTIES + ["nifi.ui.banner.text"]
+ def completeSensitiveProperties = DEFAULT_SENSITIVE_PROPERTIES + ["nifi.ui.banner.text", "nifi.version"]
logger.expected("${completeSensitiveProperties.size()} total sensitive properties: ${completeSensitiveProperties.join(", ")}")
ProtectedNiFiProperties properties = loadFromFile("/conf/nifi_with_additional_sensitive_keys.properties")
@@ -254,7 +255,7 @@ class ProtectedNiFiPropertiesGroovyTest extends GroovyTestCase {
// TODO: Add negative tests (fuzz additional.keys property, etc.)
@Test
- public void testGetAdditionalSensitivePropertiesShouldNotIncludeSelf() throws Exception {
+ void testGetAdditionalSensitivePropertiesShouldNotIncludeSelf() throws Exception {
// Arrange
def completeSensitiveProperties = DEFAULT_SENSITIVE_PROPERTIES + ["nifi.ui.banner.text", "nifi.version"]
logger.expected("${completeSensitiveProperties.size()} total sensitive properties: ${completeSensitiveProperties.join(", ")}")
@@ -275,7 +276,7 @@ class ProtectedNiFiPropertiesGroovyTest extends GroovyTestCase {
* @throws Exception
*/
@Test
- public void testShouldGetUnprotectedValueOfSensitiveProperty() throws Exception {
+ void testShouldGetUnprotectedValueOfSensitiveProperty() throws Exception {
// Arrange
final String KEYSTORE_PASSWORD_KEY = "nifi.security.keystorePasswd"
final String EXPECTED_KEYSTORE_PASSWORD = "thisIsABadKeystorePassword"
@@ -301,7 +302,7 @@ class ProtectedNiFiPropertiesGroovyTest extends GroovyTestCase {
* @throws Exception
*/
@Test
- public void testShouldGetEmptyUnprotectedValueOfSensitiveProperty() throws Exception {
+ void testShouldGetEmptyUnprotectedValueOfSensitiveProperty() throws Exception {
// Arrange
final String TRUSTSTORE_PASSWORD_KEY = "nifi.security.truststorePasswd"
final String EXPECTED_TRUSTSTORE_PASSWORD = ""
@@ -329,7 +330,7 @@ class ProtectedNiFiPropertiesGroovyTest extends GroovyTestCase {
* @throws Exception
*/
@Test
- public void testShouldGetUnprotectedValueOfSensitivePropertyWhenProtected() throws Exception {
+ void testShouldGetUnprotectedValueOfSensitivePropertyWhenProtected() throws Exception {
// Arrange
final String KEYSTORE_PASSWORD_KEY = "nifi.security.keystorePasswd"
final String EXPECTED_KEYSTORE_PASSWORD = "thisIsABadKeystorePassword"
@@ -356,7 +357,7 @@ class ProtectedNiFiPropertiesGroovyTest extends GroovyTestCase {
* @throws Exception
*/
@Test
- public void testGetValueOfSensitivePropertyShouldHandleUnknownProtectionScheme() throws Exception {
+ void testGetValueOfSensitivePropertyShouldHandleUnknownProtectionScheme() throws Exception {
// Arrange
final String KEYSTORE_PASSWORD_KEY = "nifi.security.keystorePasswd"
@@ -390,7 +391,7 @@ class ProtectedNiFiPropertiesGroovyTest extends GroovyTestCase {
* @throws Exception
*/
@Test
- public void testGetValueOfSensitivePropertyShouldHandleSingleMalformedValue() throws Exception {
+ void testGetValueOfSensitivePropertyShouldHandleSingleMalformedValue() throws Exception {
// Arrange
final String KEYSTORE_PASSWORD_KEY = "nifi.security.keystorePasswd"
@@ -425,7 +426,7 @@ class ProtectedNiFiPropertiesGroovyTest extends GroovyTestCase {
* @throws Exception
*/
@Test
- public void testGetValueOfSensitivePropertyShouldHandleMultipleMalformedValues() throws Exception {
+ void testGetValueOfSensitivePropertyShouldHandleMultipleMalformedValues() throws Exception {
// Arrange
// Raw properties
@@ -468,7 +469,7 @@ class ProtectedNiFiPropertiesGroovyTest extends GroovyTestCase {
* @throws Exception
*/
@Test
- public void testShouldGetEmptyUnprotectedValueOfSensitivePropertyWithDefault() throws Exception {
+ void testShouldGetEmptyUnprotectedValueOfSensitivePropertyWithDefault() throws Exception {
// Arrange
final String TRUSTSTORE_PASSWORD_KEY = "nifi.security.truststorePasswd"
final String EXPECTED_TRUSTSTORE_PASSWORD = ""
@@ -502,7 +503,7 @@ class ProtectedNiFiPropertiesGroovyTest extends GroovyTestCase {
* @throws Exception
*/
@Test
- public void testShouldGetUnprotectedValueOfSensitivePropertyWhenProtectedWithDefault() throws Exception {
+ void testShouldGetUnprotectedValueOfSensitivePropertyWhenProtectedWithDefault() throws Exception {
// Arrange
final String KEYSTORE_PASSWORD_KEY = "nifi.security.keystorePasswd"
final String EXPECTED_KEYSTORE_PASSWORD = "thisIsABadKeystorePassword"
@@ -538,7 +539,7 @@ class ProtectedNiFiPropertiesGroovyTest extends GroovyTestCase {
* @throws Exception
*/
@Test
- public void testGetValueOfSensitivePropertyShouldHandleInvalidatedInternalCache() throws Exception {
+ void testGetValueOfSensitivePropertyShouldHandleInvalidatedInternalCache() throws Exception {
// Arrange
final String KEYSTORE_PASSWORD_KEY = "nifi.security.keystorePasswd"
final String EXPECTED_KEYSTORE_PASSWORD = "thisIsABadKeystorePassword"
@@ -567,7 +568,7 @@ class ProtectedNiFiPropertiesGroovyTest extends GroovyTestCase {
}
@Test
- public void testShouldDetectIfPropertyIsProtected() throws Exception {
+ void testShouldDetectIfPropertyIsProtected() throws Exception {
// Arrange
final String UNPROTECTED_PROPERTY_KEY = "nifi.security.truststorePasswd"
final String PROTECTED_PROPERTY_KEY = "nifi.security.keystorePasswd"
@@ -593,7 +594,7 @@ class ProtectedNiFiPropertiesGroovyTest extends GroovyTestCase {
}
@Test
- public void testShouldDetectIfPropertyWithEmptyProtectionSchemeIsProtected() throws Exception {
+ void testShouldDetectIfPropertyWithEmptyProtectionSchemeIsProtected() throws Exception {
// Arrange
final String UNPROTECTED_PROPERTY_KEY = "nifi.sensitive.props.key"
@@ -611,7 +612,7 @@ class ProtectedNiFiPropertiesGroovyTest extends GroovyTestCase {
}
@Test
- public void testShouldGetPercentageOfSensitivePropertiesProtected_0() throws Exception {
+ void testShouldGetPercentageOfSensitivePropertiesProtected_0() throws Exception {
// Arrange
ProtectedNiFiProperties properties = loadFromFile("/conf/nifi.properties")
@@ -620,14 +621,14 @@ class ProtectedNiFiPropertiesGroovyTest extends GroovyTestCase {
// Act
double percentProtected = properties.getPercentOfSensitivePropertiesProtected()
- logger.info("${percentProtected}% (${properties.getProtectedPropertyKeys().size()} of ${properties.getSensitivePropertyKeys().size()}) protected")
+ logger.info("${percentProtected}% (${properties.getProtectedPropertyKeys().size()} of ${properties.getPopulatedSensitivePropertyKeys().size()}) protected")
// Assert
assert percentProtected == 0.0
}
@Test
- public void testShouldGetPercentageOfSensitivePropertiesProtected_50() throws Exception {
+ void testShouldGetPercentageOfSensitivePropertiesProtected_75() throws Exception {
// Arrange
ProtectedNiFiProperties properties = loadFromFile("/conf/nifi_with_sensitive_properties_protected_aes.properties")
@@ -636,14 +637,14 @@ class ProtectedNiFiPropertiesGroovyTest extends GroovyTestCase {
// Act
double percentProtected = properties.getPercentOfSensitivePropertiesProtected()
- logger.info("${percentProtected}% (${properties.getProtectedPropertyKeys().size()} of ${properties.getSensitivePropertyKeys().size()}) protected")
+ logger.info("${percentProtected}% (${properties.getProtectedPropertyKeys().size()} of ${properties.getPopulatedSensitivePropertyKeys().size()}) protected")
// Assert
- assert percentProtected == 50.0
+ assert percentProtected == 75.0
}
@Test
- public void testShouldGetPercentageOfSensitivePropertiesProtected_100() throws Exception {
+ void testShouldGetPercentageOfSensitivePropertiesProtected_100() throws Exception {
// Arrange
ProtectedNiFiProperties properties = loadFromFile("/conf/nifi_with_all_sensitive_properties_protected_aes.properties")
@@ -652,14 +653,14 @@ class ProtectedNiFiPropertiesGroovyTest extends GroovyTestCase {
// Act
double percentProtected = properties.getPercentOfSensitivePropertiesProtected()
- logger.info("${percentProtected}% (${properties.getProtectedPropertyKeys().size()} of ${properties.getSensitivePropertyKeys().size()}) protected")
+ logger.info("${percentProtected}% (${properties.getProtectedPropertyKeys().size()} of ${properties.getPopulatedSensitivePropertyKeys().size()}) protected")
// Assert
assert percentProtected == 100.0
}
@Test
- public void testInstanceWithNoProtectedPropertiesShouldNotLoadSPP() throws Exception {
+ void testInstanceWithNoProtectedPropertiesShouldNotLoadSPP() throws Exception {
// Arrange
ProtectedNiFiProperties properties = loadFromFile("/conf/nifi.properties")
assert properties.@localProviderCache?.isEmpty()
@@ -676,7 +677,7 @@ class ProtectedNiFiPropertiesGroovyTest extends GroovyTestCase {
}
@Test
- public void testShouldAddSensitivePropertyProvider() throws Exception {
+ void testShouldAddSensitivePropertyProvider() throws Exception {
// Arrange
ProtectedNiFiProperties properties = new ProtectedNiFiProperties()
assert properties.getSensitivePropertyProviders().isEmpty()
@@ -696,7 +697,7 @@ class ProtectedNiFiPropertiesGroovyTest extends GroovyTestCase {
}
@Test
- public void testShouldNotAddNullSensitivePropertyProvider() throws Exception {
+ void testShouldNotAddNullSensitivePropertyProvider() throws Exception {
// Arrange
ProtectedNiFiProperties properties = new ProtectedNiFiProperties()
assert properties.getSensitivePropertyProviders().isEmpty()
@@ -713,7 +714,7 @@ class ProtectedNiFiPropertiesGroovyTest extends GroovyTestCase {
}
@Test
- public void testShouldNotAllowOverwriteOfProvider() throws Exception {
+ void testShouldNotAllowOverwriteOfProvider() throws Exception {
// Arrange
ProtectedNiFiProperties properties = new ProtectedNiFiProperties()
assert properties.getSensitivePropertyProviders().isEmpty()
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/groovy/org/apache/nifi/properties/StandardNiFiPropertiesGroovyTest.groovy b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/groovy/org/apache/nifi/properties/StandardNiFiPropertiesGroovyTest.groovy
index c9492fb90f..ae43a3d6df 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/groovy/org/apache/nifi/properties/StandardNiFiPropertiesGroovyTest.groovy
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/groovy/org/apache/nifi/properties/StandardNiFiPropertiesGroovyTest.groovy
@@ -32,58 +32,60 @@ class StandardNiFiPropertiesGroovyTest extends GroovyTestCase {
private static final Logger logger = LoggerFactory.getLogger(StandardNiFiPropertiesGroovyTest.class)
private static String originalPropertiesPath = System.getProperty(NiFiProperties.PROPERTIES_FILE_PATH)
+ private static final String PREK = NiFiProperties.PROVENANCE_REPO_ENCRYPTION_KEY
+ private static final String PREKID = NiFiProperties.PROVENANCE_REPO_ENCRYPTION_KEY_ID
@BeforeClass
- public static void setUpOnce() throws Exception {
+ static void setUpOnce() throws Exception {
logger.metaClass.methodMissing = { String name, args ->
logger.info("[${name?.toUpperCase()}] ${(args as List).join(" ")}")
}
}
@Before
- public void setUp() throws Exception {
+ void setUp() throws Exception {
}
@After
- public void tearDown() throws Exception {
+ void tearDown() throws Exception {
}
@AfterClass
- public static void tearDownOnce() {
+ static void tearDownOnce() {
if (originalPropertiesPath) {
System.setProperty(NiFiProperties.PROPERTIES_FILE_PATH, originalPropertiesPath)
}
}
private static StandardNiFiProperties loadFromFile(String propertiesFilePath) {
- String filePath;
+ String filePath
try {
- filePath = StandardNiFiPropertiesGroovyTest.class.getResource(propertiesFilePath).toURI().getPath();
+ filePath = StandardNiFiPropertiesGroovyTest.class.getResource(propertiesFilePath).toURI().getPath()
} catch (URISyntaxException ex) {
throw new RuntimeException("Cannot load properties file due to "
- + ex.getLocalizedMessage(), ex);
+ + ex.getLocalizedMessage(), ex)
}
- System.setProperty(NiFiProperties.PROPERTIES_FILE_PATH, filePath);
+ System.setProperty(NiFiProperties.PROPERTIES_FILE_PATH, filePath)
- StandardNiFiProperties properties = new StandardNiFiProperties();
+ StandardNiFiProperties properties = new StandardNiFiProperties()
// clear out existing properties
for (String prop : properties.stringPropertyNames()) {
- properties.remove(prop);
+ properties.remove(prop)
}
- InputStream inStream = null;
+ InputStream inStream = null
try {
- inStream = new BufferedInputStream(new FileInputStream(filePath));
- properties.load(inStream);
+ inStream = new BufferedInputStream(new FileInputStream(filePath))
+ properties.load(inStream)
} catch (final Exception ex) {
throw new RuntimeException("Cannot load properties file due to "
- + ex.getLocalizedMessage(), ex);
+ + ex.getLocalizedMessage(), ex)
} finally {
if (null != inStream) {
try {
- inStream.close();
+ inStream.close()
} catch (Exception ex) {
/**
* do nothing *
@@ -92,11 +94,11 @@ class StandardNiFiPropertiesGroovyTest extends GroovyTestCase {
}
}
- return properties;
+ return properties
}
@Test
- public void testConstructorShouldCreateNewInstance() throws Exception {
+ void testConstructorShouldCreateNewInstance() throws Exception {
// Arrange
// Act
@@ -109,7 +111,7 @@ class StandardNiFiPropertiesGroovyTest extends GroovyTestCase {
}
@Test
- public void testConstructorShouldAcceptRawProperties() throws Exception {
+ void testConstructorShouldAcceptRawProperties() throws Exception {
// Arrange
Properties rawProperties = new Properties()
rawProperties.setProperty("key", "value")
@@ -126,7 +128,7 @@ class StandardNiFiPropertiesGroovyTest extends GroovyTestCase {
}
@Test
- public void testShouldAllowMultipleInstances() throws Exception {
+ void testShouldAllowMultipleInstances() throws Exception {
// Arrange
Properties rawProperties = new Properties()
rawProperties.setProperty("key", "value")
@@ -139,7 +141,6 @@ class StandardNiFiPropertiesGroovyTest extends GroovyTestCase {
NiFiProperties emptyProperties = new StandardNiFiProperties()
logger.info("emptyProperties has ${emptyProperties.size()} properties: ${emptyProperties.getPropertyKeys()}")
-
// Assert
assert niFiProperties.size() == 1
assert niFiProperties.getPropertyKeys() == ["key"] as Set
@@ -147,4 +148,178 @@ class StandardNiFiPropertiesGroovyTest extends GroovyTestCase {
assert emptyProperties.size() == 0
assert emptyProperties.getPropertyKeys() == [] as Set
}
+
+ @Test
+ void testShouldGetProvenanceRepoEncryptionKeyFromDefaultProperty() throws Exception {
+ // Arrange
+ Properties rawProperties = new Properties()
+ final String KEY_ID = "arbitraryKeyId"
+ final String KEY_HEX = "0123456789ABCDEFFEDCBA9876543210"
+ rawProperties.setProperty(PREKID, KEY_ID)
+ rawProperties.setProperty(PREK, KEY_HEX)
+ NiFiProperties niFiProperties = new StandardNiFiProperties(rawProperties)
+ logger.info("niFiProperties has ${niFiProperties.size()} properties: ${niFiProperties.getPropertyKeys()}")
+
+ // Act
+ def keyId = niFiProperties.getProvenanceRepoEncryptionKeyId()
+ def key = niFiProperties.getProvenanceRepoEncryptionKey()
+ def keys = niFiProperties.getProvenanceRepoEncryptionKeys()
+
+ logger.info("Retrieved key ID: ${keyId}")
+ logger.info("Retrieved key: ${key}")
+ logger.info("Retrieved keys: ${keys}")
+
+ // Assert
+ assert keyId == KEY_ID
+ assert key == KEY_HEX
+ assert keys == [(KEY_ID): KEY_HEX]
+ }
+
+ @Test
+ void testShouldGetProvenanceRepoEncryptionKeysFromMultipleProperties() throws Exception {
+ // Arrange
+ Properties rawProperties = new Properties()
+ final String KEY_ID = "arbitraryKeyId"
+ final String KEY_HEX = "0123456789ABCDEFFEDCBA9876543210"
+ final String KEY_ID_2 = "arbitraryKeyId2"
+ final String KEY_HEX_2 = "AAAABBBBCCCCDDDDEEEEFFFF00001111"
+ final String KEY_ID_3 = "arbitraryKeyId3"
+ final String KEY_HEX_3 = "01010101010101010101010101010101"
+
+ rawProperties.setProperty(PREKID, KEY_ID)
+ rawProperties.setProperty(PREK, KEY_HEX)
+ rawProperties.setProperty("${PREK}.id.${KEY_ID_2}", KEY_HEX_2)
+ rawProperties.setProperty("${PREK}.id.${KEY_ID_3}", KEY_HEX_3)
+ NiFiProperties niFiProperties = new StandardNiFiProperties(rawProperties)
+ logger.info("niFiProperties has ${niFiProperties.size()} properties: ${niFiProperties.getPropertyKeys()}")
+
+ // Act
+ def keyId = niFiProperties.getProvenanceRepoEncryptionKeyId()
+ def key = niFiProperties.getProvenanceRepoEncryptionKey()
+ def keys = niFiProperties.getProvenanceRepoEncryptionKeys()
+
+ logger.info("Retrieved key ID: ${keyId}")
+ logger.info("Retrieved key: ${key}")
+ logger.info("Retrieved keys: ${keys}")
+
+ // Assert
+ assert keyId == KEY_ID
+ assert key == KEY_HEX
+ assert keys == [(KEY_ID): KEY_HEX, (KEY_ID_2): KEY_HEX_2, (KEY_ID_3): KEY_HEX_3]
+ }
+
+ @Test
+ void testShouldGetProvenanceRepoEncryptionKeysWithNoDefaultDefined() throws Exception {
+ // Arrange
+ Properties rawProperties = new Properties()
+ final String KEY_ID = "arbitraryKeyId"
+ final String KEY_HEX = "0123456789ABCDEFFEDCBA9876543210"
+ final String KEY_ID_2 = "arbitraryKeyId2"
+ final String KEY_HEX_2 = "AAAABBBBCCCCDDDDEEEEFFFF00001111"
+ final String KEY_ID_3 = "arbitraryKeyId3"
+ final String KEY_HEX_3 = "01010101010101010101010101010101"
+
+ rawProperties.setProperty(PREKID, KEY_ID)
+ rawProperties.setProperty("${PREK}.id.${KEY_ID}", KEY_HEX)
+ rawProperties.setProperty("${PREK}.id.${KEY_ID_2}", KEY_HEX_2)
+ rawProperties.setProperty("${PREK}.id.${KEY_ID_3}", KEY_HEX_3)
+ NiFiProperties niFiProperties = new StandardNiFiProperties(rawProperties)
+ logger.info("niFiProperties has ${niFiProperties.size()} properties: ${niFiProperties.getPropertyKeys()}")
+
+ // Act
+ def keyId = niFiProperties.getProvenanceRepoEncryptionKeyId()
+ def key = niFiProperties.getProvenanceRepoEncryptionKey()
+ def keys = niFiProperties.getProvenanceRepoEncryptionKeys()
+
+ logger.info("Retrieved key ID: ${keyId}")
+ logger.info("Retrieved key: ${key}")
+ logger.info("Retrieved keys: ${keys}")
+
+ // Assert
+ assert keyId == KEY_ID
+ assert key == KEY_HEX
+ assert keys == [(KEY_ID): KEY_HEX, (KEY_ID_2): KEY_HEX_2, (KEY_ID_3): KEY_HEX_3]
+ }
+
+ @Test
+ void testShouldGetProvenanceRepoEncryptionKeysWithNoneDefined() throws Exception {
+ // Arrange
+ Properties rawProperties = new Properties()
+ NiFiProperties niFiProperties = new StandardNiFiProperties(rawProperties)
+ logger.info("niFiProperties has ${niFiProperties.size()} properties: ${niFiProperties.getPropertyKeys()}")
+
+ // Act
+ def keyId = niFiProperties.getProvenanceRepoEncryptionKeyId()
+ def key = niFiProperties.getProvenanceRepoEncryptionKey()
+ def keys = niFiProperties.getProvenanceRepoEncryptionKeys()
+
+ logger.info("Retrieved key ID: ${keyId}")
+ logger.info("Retrieved key: ${key}")
+ logger.info("Retrieved keys: ${keys}")
+
+ // Assert
+ assert keyId == null
+ assert key == null
+ assert keys == [:]
+ }
+
+ @Test
+ void testShouldNotGetProvenanceRepoEncryptionKeysIfFileBasedKeyProvider() throws Exception {
+ // Arrange
+ Properties rawProperties = new Properties()
+ final String KEY_ID = "arbitraryKeyId"
+
+ rawProperties.setProperty(PREKID, KEY_ID)
+ NiFiProperties niFiProperties = new StandardNiFiProperties(rawProperties)
+ logger.info("niFiProperties has ${niFiProperties.size()} properties: ${niFiProperties.getPropertyKeys()}")
+
+ // Act
+ def keyId = niFiProperties.getProvenanceRepoEncryptionKeyId()
+ def key = niFiProperties.getProvenanceRepoEncryptionKey()
+ def keys = niFiProperties.getProvenanceRepoEncryptionKeys()
+
+ logger.info("Retrieved key ID: ${keyId}")
+ logger.info("Retrieved key: ${key}")
+ logger.info("Retrieved keys: ${keys}")
+
+ // Assert
+ assert keyId == KEY_ID
+ assert key == null
+ assert keys == [:]
+ }
+
+ @Test
+ void testGetProvenanceRepoEncryptionKeysShouldFilterOtherProperties() throws Exception {
+ // Arrange
+ Properties rawProperties = new Properties()
+ final String KEY_ID = "arbitraryKeyId"
+ final String KEY_HEX = "0123456789ABCDEFFEDCBA9876543210"
+ final String KEY_ID_2 = "arbitraryKeyId2"
+ final String KEY_HEX_2 = "AAAABBBBCCCCDDDDEEEEFFFF00001111"
+ final String KEY_ID_3 = "arbitraryKeyId3"
+ final String KEY_HEX_3 = "01010101010101010101010101010101"
+
+ rawProperties.setProperty(PREKID, KEY_ID)
+ rawProperties.setProperty("${PREK}.id.${KEY_ID}", KEY_HEX)
+ rawProperties.setProperty("${PREK}.id.${KEY_ID_2}", KEY_HEX_2)
+ rawProperties.setProperty("${PREK}.id.${KEY_ID_3}", KEY_HEX_3)
+ rawProperties.setProperty(NiFiProperties.PROVENANCE_REPO_ENCRYPTION_KEY_PROVIDER_IMPLEMENTATION_CLASS, "some.class.provider")
+ rawProperties.setProperty(NiFiProperties.PROVENANCE_REPO_ENCRYPTION_KEY_PROVIDER_LOCATION, "some://url")
+ NiFiProperties niFiProperties = new StandardNiFiProperties(rawProperties)
+ logger.info("niFiProperties has ${niFiProperties.size()} properties: ${niFiProperties.getPropertyKeys()}")
+
+ // Act
+ def keyId = niFiProperties.getProvenanceRepoEncryptionKeyId()
+ def key = niFiProperties.getProvenanceRepoEncryptionKey()
+ def keys = niFiProperties.getProvenanceRepoEncryptionKeys()
+
+ logger.info("Retrieved key ID: ${keyId}")
+ logger.info("Retrieved key: ${key}")
+ logger.info("Retrieved keys: ${keys}")
+
+ // Assert
+ assert keyId == KEY_ID
+ assert key == KEY_HEX
+ assert keys == [(KEY_ID): KEY_HEX, (KEY_ID_2): KEY_HEX_2, (KEY_ID_3): KEY_HEX_3]
+ }
}
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/resources/conf/nifi_with_additional_sensitive_keys.properties b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/resources/conf/nifi_with_additional_sensitive_keys.properties
index f775d83cec..6a88c25569 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/resources/conf/nifi_with_additional_sensitive_keys.properties
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/resources/conf/nifi_with_additional_sensitive_keys.properties
@@ -73,7 +73,7 @@ nifi.web.jetty.working.directory=./target/work/jetty
nifi.sensitive.props.key=key
nifi.sensitive.props.algorithm=PBEWITHMD5AND256BITAES-CBC-OPENSSL
nifi.sensitive.props.provider=BC
-nifi.sensitive.props.additional.keys=nifi.ui.banner.text
+nifi.sensitive.props.additional.keys=nifi.ui.banner.text, nifi.version, nifi.sensitive.props.additional.keys
nifi.security.keystore=
nifi.security.keystoreType=
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-resources/pom.xml b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-resources/pom.xml
index d6b1aafbfe..0baaed756c 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-resources/pom.xml
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-resources/pom.xml
@@ -91,6 +91,11 @@
org.apache.nifi.provenance.PersistentProvenanceRepository
+ 1_000_000
+
+
+
+
./provenance_repository
24 hours
1 GB
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-resources/src/main/resources/conf/nifi.properties b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-resources/src/main/resources/conf/nifi.properties
index 62b4c8f85d..dadc5e6dd1 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-resources/src/main/resources/conf/nifi.properties
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-resources/src/main/resources/conf/nifi.properties
@@ -81,6 +81,11 @@ nifi.content.viewer.url=${nifi.content.viewer.url}
# Provenance Repository Properties
nifi.provenance.repository.implementation=${nifi.provenance.repository.implementation}
+nifi.provenance.repository.debug.frequency=${nifi.provenance.repository.debug.frequency}
+nifi.provenance.repository.encryption.key.provider.implementation=${nifi.provenance.repository.encryption.key.provider.implementation}
+nifi.provenance.repository.encryption.key.provider.location=${nifi.provenance.repository.encryption.key.provider.location}
+nifi.provenance.repository.encryption.key.id=${nifi.provenance.repository.encryption.key.id}
+nifi.provenance.repository.encryption.key=${nifi.provenance.repository.encryption.key}
# Persistent Provenance Repository Properties
nifi.provenance.repository.directory.default=${nifi.provenance.repository.directory.default}
diff --git a/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-processors/src/test/resources/logback-test.xml b/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-processors/src/test/resources/logback-test.xml
new file mode 100644
index 0000000000..80b8b49c70
--- /dev/null
+++ b/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-processors/src/test/resources/logback-test.xml
@@ -0,0 +1,32 @@
+
+
+
+
+
+
+ %-4r [%t] %-5p %c - %m%n
+
+
+
+
+
+
+
+
+
+
+
+
diff --git a/nifi-nar-bundles/nifi-poi-bundle/nifi-poi-processors/src/test/resources/logback-test.xml b/nifi-nar-bundles/nifi-poi-bundle/nifi-poi-processors/src/test/resources/logback-test.xml
new file mode 100644
index 0000000000..5afbc8ea75
--- /dev/null
+++ b/nifi-nar-bundles/nifi-poi-bundle/nifi-poi-processors/src/test/resources/logback-test.xml
@@ -0,0 +1,32 @@
+
+
+
+
+
+
+ %-4r [%t] %-5p %c - %m%n
+
+
+
+
+
+
+
+
+
+
+
+
diff --git a/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/pom.xml b/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/pom.xml
index 4db4169ff0..8fe5dbfcef 100644
--- a/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/pom.xml
+++ b/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/pom.xml
@@ -63,5 +63,10 @@
commons-lang3
test
+
+ org.bouncycastle
+ bcprov-jdk15on
+ test
+
diff --git a/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/EncryptedSchemaRecordReader.java b/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/EncryptedSchemaRecordReader.java
new file mode 100644
index 0000000000..fcd7fee53e
--- /dev/null
+++ b/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/EncryptedSchemaRecordReader.java
@@ -0,0 +1,154 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.provenance;
+
+import java.io.ByteArrayInputStream;
+import java.io.DataInputStream;
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.Collection;
+import java.util.Optional;
+import java.util.concurrent.TimeUnit;
+import org.apache.nifi.provenance.schema.LookupTableEventRecord;
+import org.apache.nifi.provenance.toc.TocReader;
+import org.apache.nifi.repository.schema.Record;
+import org.apache.nifi.stream.io.LimitingInputStream;
+import org.apache.nifi.stream.io.StreamUtils;
+import org.apache.nifi.util.timebuffer.LongEntityAccess;
+import org.apache.nifi.util.timebuffer.TimedBuffer;
+import org.apache.nifi.util.timebuffer.TimestampedLong;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class EncryptedSchemaRecordReader extends EventIdFirstSchemaRecordReader {
+ private static final Logger logger = LoggerFactory.getLogger(EncryptedSchemaRecordReader.class);
+
+ private static final int DEFAULT_DEBUG_FREQUENCY = 1_000_000;
+
+ private ProvenanceEventEncryptor provenanceEventEncryptor;
+
+ private static final TimedBuffer decryptTimes = new TimedBuffer<>(TimeUnit.SECONDS, 60, new LongEntityAccess());
+
+ private int debugFrequency = DEFAULT_DEBUG_FREQUENCY;
+ public static final int SERIALIZATION_VERSION = 1;
+
+ public static final String SERIALIZATION_NAME = "EncryptedSchemaRecordWriter";
+
+ public EncryptedSchemaRecordReader(final InputStream inputStream, final String filename, final TocReader tocReader, final int maxAttributeChars,
+ ProvenanceEventEncryptor provenanceEventEncryptor) throws IOException {
+ this(inputStream, filename, tocReader, maxAttributeChars, provenanceEventEncryptor, DEFAULT_DEBUG_FREQUENCY);
+ }
+
+ public EncryptedSchemaRecordReader(final InputStream inputStream, final String filename, final TocReader tocReader, final int maxAttributeChars,
+ ProvenanceEventEncryptor provenanceEventEncryptor, int debugFrequency) throws IOException {
+ super(inputStream, filename, tocReader, maxAttributeChars);
+ this.provenanceEventEncryptor = provenanceEventEncryptor;
+ this.debugFrequency = debugFrequency;
+ }
+
+ @Override
+ protected StandardProvenanceEventRecord nextRecord(final DataInputStream in, final int serializationVersion) throws IOException {
+ verifySerializationVersion(serializationVersion);
+
+ final long byteOffset = getBytesConsumed();
+ final long eventId = in.readInt() + getFirstEventId();
+ final int recordLength = in.readInt();
+
+ return readRecord(in, eventId, byteOffset, recordLength);
+ }
+
+ private StandardProvenanceEventRecord readRecord(final DataInputStream inputStream, final long eventId, final long startOffset, final int recordLength) throws IOException {
+ try {
+ final InputStream limitedIn = new LimitingInputStream(inputStream, recordLength);
+
+ byte[] encryptedSerializedBytes = new byte[recordLength];
+ DataInputStream encryptedInputStream = new DataInputStream(limitedIn);
+ encryptedInputStream.readFully(encryptedSerializedBytes);
+
+ byte[] plainSerializedBytes = decrypt(encryptedSerializedBytes, Long.toString(eventId));
+ InputStream plainStream = new ByteArrayInputStream(plainSerializedBytes);
+
+ final Record eventRecord = getRecordReader().readRecord(plainStream);
+ if (eventRecord == null) {
+ return null;
+ }
+
+ final StandardProvenanceEventRecord deserializedEvent = LookupTableEventRecord.getEvent(eventRecord, getFilename(), startOffset, getMaxAttributeLength(),
+ getFirstEventId(), getSystemTimeOffset(), getComponentIds(), getComponentTypes(), getQueueIds(), getEventTypes());
+ deserializedEvent.setEventId(eventId);
+ return deserializedEvent;
+ } catch (EncryptionException e) {
+ logger.error("Encountered an error reading the record: ", e);
+ throw new IOException(e);
+ }
+ }
+
+ // TODO: Copied from EventIdFirstSchemaRecordReader to force local/overridden readRecord()
+ @Override
+ protected Optional readToEvent(final long eventId, final DataInputStream dis, final int serializationVersion) throws IOException {
+ verifySerializationVersion(serializationVersion);
+
+ while (isData(dis)) {
+ final long startOffset = getBytesConsumed();
+ final long id = dis.readInt() + getFirstEventId();
+ final int recordLength = dis.readInt();
+
+ if (id >= eventId) {
+ final StandardProvenanceEventRecord event = readRecord(dis, id, startOffset, recordLength);
+ return Optional.ofNullable(event);
+ } else {
+ // This is not the record we want. Skip over it instead of deserializing it.
+ StreamUtils.skip(dis, recordLength);
+ }
+ }
+
+ return Optional.empty();
+ }
+
+ private byte[] decrypt(byte[] encryptedBytes, String eventId) throws IOException, EncryptionException {
+ try {
+ return provenanceEventEncryptor.decrypt(encryptedBytes, eventId);
+ } catch (Exception e) {
+ logger.error("Encountered an error: ", e);
+ throw new EncryptionException(e);
+ }
+ }
+
+ @Override
+ public String toString() {
+ return getDescription();
+ }
+
+ private String getDescription() {
+ try {
+ return "EncryptedSchemaRecordReader, toc: " + getTocReader().getFile().getAbsolutePath() + ", journal: " + getFilename();
+ } catch (Exception e) {
+ return "EncryptedSchemaRecordReader@" + Integer.toHexString(this.hashCode());
+ }
+ }
+
+ /**
+ * Sets the encryptor to use (necessary because the
+ * {@link org.apache.nifi.provenance.serialization.RecordReaders#newRecordReader(File, Collection, int)} method doesn't accept the encryptor.
+ *
+ * @param provenanceEventEncryptor the encryptor
+ */
+ void setProvenanceEventEncryptor(ProvenanceEventEncryptor provenanceEventEncryptor) {
+ this.provenanceEventEncryptor = provenanceEventEncryptor;
+ }
+}
diff --git a/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/EncryptedSchemaRecordWriter.java b/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/EncryptedSchemaRecordWriter.java
new file mode 100644
index 0000000000..f84ca4891d
--- /dev/null
+++ b/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/EncryptedSchemaRecordWriter.java
@@ -0,0 +1,199 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.provenance;
+
+import java.io.ByteArrayOutputStream;
+import java.io.DataOutputStream;
+import java.io.File;
+import java.io.IOException;
+import java.security.KeyManagementException;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicLong;
+import org.apache.nifi.provenance.serialization.StorageSummary;
+import org.apache.nifi.provenance.toc.TocWriter;
+import org.apache.nifi.util.timebuffer.LongEntityAccess;
+import org.apache.nifi.util.timebuffer.TimedBuffer;
+import org.apache.nifi.util.timebuffer.TimestampedLong;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class EncryptedSchemaRecordWriter extends EventIdFirstSchemaRecordWriter {
+ private static final Logger logger = LoggerFactory.getLogger(EncryptedSchemaRecordWriter.class);
+
+ private static final int DEFAULT_DEBUG_FREQUENCY = 1_000_000;
+
+ private ProvenanceEventEncryptor provenanceEventEncryptor;
+
+ private static final TimedBuffer encryptTimes = new TimedBuffer<>(TimeUnit.SECONDS, 60, new LongEntityAccess());
+
+ private String keyId;
+
+ private int debugFrequency;
+ public static final int SERIALIZATION_VERSION = 1;
+
+ public static final String SERIALIZATION_NAME = "EncryptedSchemaRecordWriter";
+
+ public EncryptedSchemaRecordWriter(final File file, final AtomicLong idGenerator, final TocWriter writer, final boolean compressed,
+ final int uncompressedBlockSize, final IdentifierLookup idLookup,
+ ProvenanceEventEncryptor provenanceEventEncryptor) throws IOException, EncryptionException {
+ this(file, idGenerator, writer, compressed, uncompressedBlockSize, idLookup, provenanceEventEncryptor, DEFAULT_DEBUG_FREQUENCY);
+ }
+
+ public EncryptedSchemaRecordWriter(final File file, final AtomicLong idGenerator, final TocWriter writer, final boolean compressed,
+ final int uncompressedBlockSize, final IdentifierLookup idLookup,
+ ProvenanceEventEncryptor provenanceEventEncryptor, int debugFrequency) throws IOException, EncryptionException {
+ super(file, idGenerator, writer, compressed, uncompressedBlockSize, idLookup);
+ this.provenanceEventEncryptor = provenanceEventEncryptor;
+ this.debugFrequency = debugFrequency;
+
+ try {
+ this.keyId = getNextAvailableKeyId();
+ } catch (KeyManagementException e) {
+ logger.error("Encountered an error initializing the encrypted schema record writer because the provided encryptor has no valid keys available: ", e);
+ throw new EncryptionException("No valid keys in the provenance event encryptor", e);
+ }
+ }
+
+ @Override
+ public StorageSummary writeRecord(final ProvenanceEventRecord record) throws IOException {
+ final long encryptStart = System.nanoTime();
+ byte[] cipherBytes;
+ try {
+ byte[] serialized;
+ try (final ByteArrayOutputStream baos = new ByteArrayOutputStream(256);
+ final DataOutputStream dos = new DataOutputStream(baos)) {
+ writeRecord(record, 0L, dos);
+ serialized = baos.toByteArray();
+ }
+ String eventId = record.getBestEventIdentifier();
+ cipherBytes = encrypt(serialized, eventId);
+ } catch (EncryptionException e) {
+ logger.error("Encountered an error: ", e);
+ throw new IOException("Error encrypting the provenance record", e);
+ }
+ final long encryptStop = System.nanoTime();
+
+ final long lockStart = System.nanoTime();
+ final long writeStart;
+ final long startBytes;
+ final long endBytes;
+ final long recordIdentifier;
+ synchronized (this) {
+ writeStart = System.nanoTime();
+ try {
+ recordIdentifier = record.getEventId() == -1L ? getIdGenerator().getAndIncrement() : record.getEventId();
+ startBytes = getBytesWritten();
+
+ ensureStreamState(recordIdentifier, startBytes);
+
+ final DataOutputStream out = getBufferedOutputStream();
+ final int recordIdOffset = (int) (recordIdentifier - getFirstEventId());
+ out.writeInt(recordIdOffset);
+ out.writeInt(cipherBytes.length);
+ out.write(cipherBytes);
+
+ getRecordCount().incrementAndGet();
+ endBytes = getBytesWritten();
+ } catch (final IOException ioe) {
+ markDirty();
+ throw ioe;
+ }
+ }
+
+ if (logger.isDebugEnabled()) {
+ // Collect stats and periodically dump them if log level is set to at least info.
+ final long writeNanos = System.nanoTime() - writeStart;
+ getWriteTimes().add(new TimestampedLong(writeNanos));
+
+ final long serializeNanos = lockStart - encryptStart;
+ getSerializeTimes().add(new TimestampedLong(serializeNanos));
+
+ final long encryptNanos = encryptStop - encryptStart;
+ getEncryptTimes().add(new TimestampedLong(encryptNanos));
+
+ final long lockNanos = writeStart - lockStart;
+ getLockTimes().add(new TimestampedLong(lockNanos));
+ getBytesWrittenBuffer().add(new TimestampedLong(endBytes - startBytes));
+
+ final long recordCount = getTotalRecordCount().incrementAndGet();
+ if (recordCount % debugFrequency == 0) {
+ printStats();
+ }
+ }
+
+ final long serializedLength = endBytes - startBytes;
+ final TocWriter tocWriter = getTocWriter();
+ final Integer blockIndex = tocWriter == null ? null : tocWriter.getCurrentBlockIndex();
+ final File file = getFile();
+ final String storageLocation = file.getParentFile().getName() + "/" + file.getName();
+ return new StorageSummary(recordIdentifier, storageLocation, blockIndex, serializedLength, endBytes);
+ }
+
+ private void printStats() {
+ final long sixtySecondsAgo = System.currentTimeMillis() - 60000L;
+ final Long writeNanosLast60 = getWriteTimes().getAggregateValue(sixtySecondsAgo).getValue();
+ final Long lockNanosLast60 = getLockTimes().getAggregateValue(sixtySecondsAgo).getValue();
+ final Long serializeNanosLast60 = getSerializeTimes().getAggregateValue(sixtySecondsAgo).getValue();
+ final Long encryptNanosLast60 = getEncryptTimes().getAggregateValue(sixtySecondsAgo).getValue();
+ final Long bytesWrittenLast60 = getBytesWrittenBuffer().getAggregateValue(sixtySecondsAgo).getValue();
+ logger.debug("In the last 60 seconds, have spent {} millis writing to file ({} MB), {} millis waiting on synchronize block, {} millis serializing events, {} millis encrypting events",
+ TimeUnit.NANOSECONDS.toMillis(writeNanosLast60),
+ bytesWrittenLast60 / 1024 / 1024,
+ TimeUnit.NANOSECONDS.toMillis(lockNanosLast60),
+ TimeUnit.NANOSECONDS.toMillis(serializeNanosLast60),
+ TimeUnit.NANOSECONDS.toMillis(encryptNanosLast60));
+ }
+
+ static TimedBuffer getEncryptTimes() {
+ return encryptTimes;
+ }
+
+ private byte[] encrypt(byte[] serialized, String eventId) throws IOException, EncryptionException {
+ String keyId = getKeyId();
+ try {
+ return provenanceEventEncryptor.encrypt(serialized, eventId, keyId);
+ } catch (Exception e) {
+ logger.error("Encountered an error: ", e);
+ throw new EncryptionException(e);
+ }
+ }
+
+ private String getNextAvailableKeyId() throws KeyManagementException {
+ return provenanceEventEncryptor.getNextKeyId();
+ }
+
+ @Override
+ protected int getSerializationVersion() {
+ return SERIALIZATION_VERSION;
+ }
+
+ @Override
+ protected String getSerializationName() {
+ return SERIALIZATION_NAME;
+ }
+
+ public String getKeyId() {
+ return keyId;
+ }
+
+ @Override
+ public String toString() {
+ return "EncryptedSchemaRecordWriter" +
+ " using " + provenanceEventEncryptor +
+ " and current keyId " + keyId;
+ }
+}
diff --git a/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/EncryptedWriteAheadProvenanceRepository.java b/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/EncryptedWriteAheadProvenanceRepository.java
new file mode 100644
index 0000000000..a2d455bc53
--- /dev/null
+++ b/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/EncryptedWriteAheadProvenanceRepository.java
@@ -0,0 +1,159 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.provenance;
+
+import java.io.IOException;
+import java.security.KeyManagementException;
+import java.util.Map;
+import java.util.stream.Collectors;
+import javax.crypto.SecretKey;
+import org.apache.nifi.authorization.Authorizer;
+import org.apache.nifi.events.EventReporter;
+import org.apache.nifi.provenance.serialization.RecordReaders;
+import org.apache.nifi.provenance.store.EventFileManager;
+import org.apache.nifi.provenance.store.RecordReaderFactory;
+import org.apache.nifi.provenance.store.RecordWriterFactory;
+import org.apache.nifi.provenance.toc.StandardTocWriter;
+import org.apache.nifi.provenance.toc.TocUtil;
+import org.apache.nifi.provenance.toc.TocWriter;
+import org.apache.nifi.util.NiFiProperties;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class EncryptedWriteAheadProvenanceRepository extends WriteAheadProvenanceRepository {
+ private static final Logger logger = LoggerFactory.getLogger(EncryptedWriteAheadProvenanceRepository.class);
+
+ /**
+ * This constructor exists solely for the use of the Java Service Loader mechanism and should not be used.
+ */
+ public EncryptedWriteAheadProvenanceRepository() {
+ super();
+ }
+
+ public EncryptedWriteAheadProvenanceRepository(final NiFiProperties nifiProperties) {
+ super(RepositoryConfiguration.create(nifiProperties));
+ }
+
+ public EncryptedWriteAheadProvenanceRepository(final RepositoryConfiguration config) {
+ super(config);
+ }
+
+ /**
+ * This method initializes the repository. It first builds the key provider and event encryptor
+ * from the config values, then creates the encrypted record writer and reader, then delegates
+ * back to the superclass for the common implementation.
+ *
+ * @param eventReporter the event reporter
+ * @param authorizer the authorizer
+ * @param resourceFactory the authorizable factory
+ * @param idLookup the lookup provider
+ * @throws IOException if there is an error initializing this repository
+ */
+ @Override
+ public synchronized void initialize(final EventReporter eventReporter, final Authorizer authorizer, final ProvenanceAuthorizableFactory resourceFactory,
+ final IdentifierLookup idLookup) throws IOException {
+ // Initialize the encryption-specific fields
+ ProvenanceEventEncryptor provenanceEventEncryptor;
+ if (getConfig().supportsEncryption()) {
+ try {
+ KeyProvider keyProvider = buildKeyProvider();
+ provenanceEventEncryptor = new AESProvenanceEventEncryptor();
+ provenanceEventEncryptor.initialize(keyProvider);
+ } catch (KeyManagementException e) {
+ String msg = "Encountered an error building the key provider";
+ logger.error(msg, e);
+ throw new IOException(msg, e);
+ }
+ } else {
+ throw new IOException("The provided configuration does not support a encrypted repository");
+ }
+
+ // Build a factory using lambda which injects the encryptor
+ final RecordWriterFactory recordWriterFactory = (file, idGenerator, compressed, createToc) -> {
+ try {
+ final TocWriter tocWriter = createToc ? new StandardTocWriter(TocUtil.getTocFile(file), false, false) : null;
+ return new EncryptedSchemaRecordWriter(file, idGenerator, tocWriter, compressed, BLOCK_SIZE, idLookup, provenanceEventEncryptor, getConfig().getDebugFrequency());
+ } catch (EncryptionException e) {
+ logger.error("Encountered an error building the schema record writer factory: ", e);
+ throw new IOException(e);
+ }
+ };
+
+ // Build a factory using lambda which injects the encryptor
+ final EventFileManager fileManager = new EventFileManager();
+ final RecordReaderFactory recordReaderFactory = (file, logs, maxChars) -> {
+ fileManager.obtainReadLock(file);
+ try {
+ EncryptedSchemaRecordReader tempReader = (EncryptedSchemaRecordReader) RecordReaders.newRecordReader(file, logs, maxChars);
+ tempReader.setProvenanceEventEncryptor(provenanceEventEncryptor);
+ return tempReader;
+ } finally {
+ fileManager.releaseReadLock(file);
+ }
+ };
+
+ // Delegate the init to the parent impl
+ super.init(recordWriterFactory, recordReaderFactory, eventReporter, authorizer, resourceFactory);
+ }
+
+ private KeyProvider buildKeyProvider() throws KeyManagementException {
+ RepositoryConfiguration config = super.getConfig();
+ if (config == null) {
+ throw new KeyManagementException("The repository configuration is missing");
+ }
+
+ final String implementationClassName = config.getKeyProviderImplementation();
+ if (implementationClassName == null) {
+ throw new KeyManagementException("Cannot create Key Provider because the NiFi Properties is missing the following property: "
+ + NiFiProperties.PROVENANCE_REPO_ENCRYPTION_KEY_PROVIDER_IMPLEMENTATION_CLASS);
+ }
+
+ // TODO: Extract to factory
+ KeyProvider keyProvider;
+ if (StaticKeyProvider.class.getName().equals(implementationClassName)) {
+ // Get all the keys (map) from config
+ if (CryptoUtils.isValidKeyProvider(implementationClassName, config.getKeyProviderLocation(), config.getKeyId(), config.getEncryptionKeys())) {
+ Map formedKeys = config.getEncryptionKeys().entrySet().stream()
+ .collect(Collectors.toMap(
+ Map.Entry::getKey,
+ e -> {
+ try {
+ return CryptoUtils.formKeyFromHex(e.getValue());
+ } catch (KeyManagementException e1) {
+ // This should never happen because the hex has already been validated
+ logger.error("Encountered an error: ", e1);
+ return null;
+ }
+ }));
+ keyProvider = new StaticKeyProvider(formedKeys);
+ } else {
+ final String msg = "The StaticKeyProvider definition is not valid";
+ logger.error(msg);
+ throw new KeyManagementException(msg);
+ }
+ } else if (FileBasedKeyProvider.class.getName().equals(implementationClassName)) {
+ keyProvider = new FileBasedKeyProvider(config.getKeyProviderLocation());
+ if (!keyProvider.keyExists(config.getKeyId())) {
+ throw new KeyManagementException("The specified key ID " + config.getKeyId() + " is not in the key definition file");
+ }
+ } else {
+ throw new KeyManagementException("Invalid key provider implementation provided: " + implementationClassName);
+ }
+
+ return keyProvider;
+ }
+}
diff --git a/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/EventIdFirstSchemaRecordReader.java b/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/EventIdFirstSchemaRecordReader.java
index 612b6c8c37..bd85846517 100644
--- a/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/EventIdFirstSchemaRecordReader.java
+++ b/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/EventIdFirstSchemaRecordReader.java
@@ -23,7 +23,6 @@ import java.io.IOException;
import java.io.InputStream;
import java.util.List;
import java.util.Optional;
-
import org.apache.nifi.provenance.schema.EventIdFirstHeaderSchema;
import org.apache.nifi.provenance.schema.LookupTableEventRecord;
import org.apache.nifi.provenance.serialization.CompressableRecordReader;
@@ -35,6 +34,14 @@ import org.apache.nifi.stream.io.LimitingInputStream;
import org.apache.nifi.stream.io.StreamUtils;
public class EventIdFirstSchemaRecordReader extends CompressableRecordReader {
+ RecordSchema getSchema() {
+ return schema;
+ }
+
+ SchemaRecordReader getRecordReader() {
+ return recordReader;
+ }
+
private RecordSchema schema; // effectively final
private SchemaRecordReader recordReader; // effectively final
@@ -43,16 +50,41 @@ public class EventIdFirstSchemaRecordReader extends CompressableRecordReader {
private List queueIds;
private List eventTypes;
private long firstEventId;
+
+ List getComponentIds() {
+ return componentIds;
+ }
+
+ List getComponentTypes() {
+ return componentTypes;
+ }
+
+ List getQueueIds() {
+ return queueIds;
+ }
+
+ List getEventTypes() {
+ return eventTypes;
+ }
+
+ long getFirstEventId() {
+ return firstEventId;
+ }
+
+ long getSystemTimeOffset() {
+ return systemTimeOffset;
+ }
+
private long systemTimeOffset;
public EventIdFirstSchemaRecordReader(final InputStream in, final String filename, final TocReader tocReader, final int maxAttributeChars) throws IOException {
super(in, filename, tocReader, maxAttributeChars);
}
- private void verifySerializationVersion(final int serializationVersion) {
+ protected void verifySerializationVersion(final int serializationVersion) {
if (serializationVersion > EventIdFirstSchemaRecordWriter.SERIALIZATION_VERSION) {
throw new IllegalArgumentException("Unable to deserialize record because the version is " + serializationVersion
- + " and supported versions are 1-" + EventIdFirstSchemaRecordWriter.SERIALIZATION_VERSION);
+ + " and supported versions are 1-" + EventIdFirstSchemaRecordWriter.SERIALIZATION_VERSION);
}
}
@@ -109,12 +141,12 @@ public class EventIdFirstSchemaRecordReader extends CompressableRecordReader {
}
final StandardProvenanceEventRecord deserializedEvent = LookupTableEventRecord.getEvent(eventRecord, getFilename(), startOffset, getMaxAttributeLength(),
- firstEventId, systemTimeOffset, componentIds, componentTypes, queueIds, eventTypes);
+ firstEventId, systemTimeOffset, componentIds, componentTypes, queueIds, eventTypes);
deserializedEvent.setEventId(eventId);
return deserializedEvent;
}
- private boolean isData(final InputStream in) throws IOException {
+ protected boolean isData(final InputStream in) throws IOException {
in.mark(1);
final int nextByte = in.read();
in.reset();
@@ -142,4 +174,17 @@ public class EventIdFirstSchemaRecordReader extends CompressableRecordReader {
return Optional.empty();
}
+
+ @Override
+ public String toString() {
+ return getDescription();
+ }
+
+ private String getDescription() {
+ try {
+ return "EventIdFirstSchemaRecordReader, toc: " + getTocReader().getFile().getAbsolutePath() + ", journal: " + getFilename();
+ } catch (Exception e) {
+ return "EventIdFirstSchemaRecordReader@" + Integer.toHexString(this.hashCode());
+ }
+ }
}
diff --git a/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/EventIdFirstSchemaRecordWriter.java b/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/EventIdFirstSchemaRecordWriter.java
index bb8d52fef3..8f5b2b2103 100644
--- a/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/EventIdFirstSchemaRecordWriter.java
+++ b/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/EventIdFirstSchemaRecordWriter.java
@@ -29,7 +29,6 @@ import java.util.Map;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicLong;
-
import org.apache.nifi.provenance.schema.EventFieldNames;
import org.apache.nifi.provenance.schema.EventIdFirstHeaderSchema;
import org.apache.nifi.provenance.schema.LookupTableEventRecord;
@@ -238,4 +237,46 @@ public class EventIdFirstSchemaRecordWriter extends CompressableRecordWriter {
return SERIALIZATION_NAME;
}
+ /* Getters for internal state written to by subclass EncryptedSchemaRecordWriter */
+
+ IdentifierLookup getIdLookup() {
+ return idLookup;
+ }
+
+ SchemaRecordWriter getSchemaRecordWriter() {
+ return schemaRecordWriter;
+ }
+
+ AtomicInteger getRecordCount() {
+ return recordCount;
+ }
+
+ static TimedBuffer getSerializeTimes() {
+ return serializeTimes;
+ }
+
+ static TimedBuffer getLockTimes() {
+ return lockTimes;
+ }
+
+ static TimedBuffer getWriteTimes() {
+ return writeTimes;
+ }
+
+ static TimedBuffer getBytesWrittenBuffer() {
+ return bytesWritten;
+ }
+
+ static AtomicLong getTotalRecordCount() {
+ return totalRecordCount;
+ }
+
+ long getFirstEventId() {
+ return firstEventId;
+ }
+
+ long getSystemTimeOffset() {
+ return systemTimeOffset;
+ }
+
}
diff --git a/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/RepositoryConfiguration.java b/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/RepositoryConfiguration.java
index 7a2f57e37a..5a75172552 100644
--- a/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/RepositoryConfiguration.java
+++ b/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/RepositoryConfiguration.java
@@ -26,7 +26,6 @@ import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.concurrent.TimeUnit;
-
import org.apache.nifi.processor.DataUnit;
import org.apache.nifi.provenance.search.SearchableField;
import org.apache.nifi.util.FormatUtils;
@@ -50,6 +49,12 @@ public class RepositoryConfiguration {
private int journalCount = 16;
private int compressionBlockBytes = 1024 * 1024;
private int maxAttributeChars = 65536;
+ private int debugFrequency = 1_000_000;
+
+ private Map encryptionKeys;
+ private String keyId;
+ private String keyProviderImplementation;
+ private String keyProviderLocation;
private List searchableFields = new ArrayList<>();
private List searchableAttributes = new ArrayList<>();
@@ -360,6 +365,54 @@ public class RepositoryConfiguration {
return Optional.ofNullable(warmCacheFrequencyMinutes);
}
+ public boolean supportsEncryption() {
+ boolean keyProviderIsConfigured = CryptoUtils.isValidKeyProvider(keyProviderImplementation, keyProviderLocation, keyId, encryptionKeys);
+
+ return keyProviderIsConfigured;
+ }
+
+ public Map getEncryptionKeys() {
+ return encryptionKeys;
+ }
+
+ public void setEncryptionKeys(Map encryptionKeys) {
+ this.encryptionKeys = encryptionKeys;
+ }
+
+ public String getKeyId() {
+ return keyId;
+ }
+
+ public void setKeyId(String keyId) {
+ this.keyId = keyId;
+ }
+
+ public String getKeyProviderImplementation() {
+ return keyProviderImplementation;
+ }
+
+ public void setKeyProviderImplementation(String keyProviderImplementation) {
+ this.keyProviderImplementation = keyProviderImplementation;
+ }
+
+ public String getKeyProviderLocation() {
+ return keyProviderLocation;
+ }
+
+ public void setKeyProviderLocation(String keyProviderLocation) {
+ this.keyProviderLocation = keyProviderLocation;
+ }
+
+
+ public int getDebugFrequency() {
+ return debugFrequency;
+ }
+
+ public void setDebugFrequency(int debugFrequency) {
+ this.debugFrequency = debugFrequency;
+ }
+
+
public static RepositoryConfiguration create(final NiFiProperties nifiProperties) {
final Map storageDirectories = nifiProperties.getProvenanceRepositoryPaths();
if (storageDirectories.isEmpty()) {
@@ -436,6 +489,17 @@ public class RepositoryConfiguration {
config.setAlwaysSync(alwaysSync);
+ config.setDebugFrequency(nifiProperties.getIntegerProperty(NiFiProperties.PROVENANCE_REPO_DEBUG_FREQUENCY, config.getDebugFrequency()));
+
+ // Encryption values may not be present but are only required for EncryptedWriteAheadProvenanceRepository
+ final String implementationClassName = nifiProperties.getProperty(NiFiProperties.PROVENANCE_REPO_IMPLEMENTATION_CLASS);
+ if (EncryptedWriteAheadProvenanceRepository.class.getName().equals(implementationClassName)) {
+ config.setEncryptionKeys(nifiProperties.getProvenanceRepoEncryptionKeys());
+ config.setKeyId(nifiProperties.getProperty(NiFiProperties.PROVENANCE_REPO_ENCRYPTION_KEY_ID));
+ config.setKeyProviderImplementation(nifiProperties.getProperty(NiFiProperties.PROVENANCE_REPO_ENCRYPTION_KEY_PROVIDER_IMPLEMENTATION_CLASS));
+ config.setKeyProviderLocation(nifiProperties.getProperty(NiFiProperties.PROVENANCE_REPO_ENCRYPTION_KEY_PROVIDER_LOCATION));
+ }
+
return config;
}
}
diff --git a/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/WriteAheadProvenanceRepository.java b/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/WriteAheadProvenanceRepository.java
index 89750282de..4782dbe60a 100644
--- a/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/WriteAheadProvenanceRepository.java
+++ b/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/WriteAheadProvenanceRepository.java
@@ -21,7 +21,6 @@ import java.io.IOException;
import java.util.Collections;
import java.util.List;
import java.util.Map;
-
import org.apache.nifi.authorization.Authorizer;
import org.apache.nifi.authorization.RequestAction;
import org.apache.nifi.authorization.resource.Authorizable;
@@ -84,7 +83,7 @@ import org.slf4j.LoggerFactory;
*/
public class WriteAheadProvenanceRepository implements ProvenanceRepository {
private static final Logger logger = LoggerFactory.getLogger(WriteAheadProvenanceRepository.class);
- private static final int BLOCK_SIZE = 1024 * 32;
+ static final int BLOCK_SIZE = 1024 * 32;
public static final String EVENT_CATEGORY = "Provenance Repository";
private final RepositoryConfiguration config;
@@ -129,6 +128,14 @@ public class WriteAheadProvenanceRepository implements ProvenanceRepository {
}
};
+ init(recordWriterFactory, recordReaderFactory, eventReporter, authorizer, resourceFactory);
+ }
+
+ synchronized void init(RecordWriterFactory recordWriterFactory, RecordReaderFactory recordReaderFactory,
+ final EventReporter eventReporter, final Authorizer authorizer,
+ final ProvenanceAuthorizableFactory resourceFactory) throws IOException {
+ final EventFileManager fileManager = new EventFileManager();
+
eventStore = new PartitionedWriteAheadEventStore(config, recordWriterFactory, recordReaderFactory, eventReporter, fileManager);
final IndexManager indexManager = new SimpleIndexManager(config);
@@ -145,7 +152,7 @@ public class WriteAheadProvenanceRepository implements ProvenanceRepository {
eventStore.reindexLatestEvents(eventIndex);
} catch (final Exception e) {
logger.error("Failed to re-index some of the Provenance Events. It is possible that some of the latest "
- + "events will not be available from the Provenance Repository when a query is issued.", e);
+ + "events will not be available from the Provenance Repository when a query is issued.", e);
}
}
@@ -282,4 +289,8 @@ public class WriteAheadProvenanceRepository implements ProvenanceRepository {
public List getSearchableAttributes() {
return Collections.unmodifiableList(config.getSearchableAttributes());
}
+
+ RepositoryConfiguration getConfig() {
+ return this.config;
+ }
}
diff --git a/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/index/lucene/LuceneEventIndex.java b/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/index/lucene/LuceneEventIndex.java
index a58340343b..f4b47d3dce 100644
--- a/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/index/lucene/LuceneEventIndex.java
+++ b/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/index/lucene/LuceneEventIndex.java
@@ -36,7 +36,6 @@ import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicLong;
-
import org.apache.lucene.document.Document;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.Term;
@@ -246,7 +245,7 @@ public class LuceneEventIndex implements EventIndex {
final Document document = eventConverter.convert(event, summary);
if (document == null) {
- logger.debug("Received Provenance Event {} to index but it contained no information that should be indexed, so skipping it", event);
+ logger.debug("Received Provenance Event {} to index but it contained no information that should be indexed, so skipping it", event.getEventId());
} else {
final File indexDir;
if (event.getEventTime() == lastEventTime) {
@@ -291,7 +290,7 @@ public class LuceneEventIndex implements EventIndex {
final Document document = eventConverter.convert(event, location);
if (document == null) {
- logger.debug("Received Provenance Event {} to index but it contained no information that should be indexed, so skipping it", event);
+ logger.debug("Received Provenance Event {} to index but it contained no information that should be indexed, so skipping it", event.getEventId());
} else {
final StoredDocument doc = new StoredDocument(document, location);
boolean added = false;
@@ -357,13 +356,13 @@ public class LuceneEventIndex implements EventIndex {
eventOption = eventStore.getEvent(eventId);
} catch (final Exception e) {
logger.error("Failed to retrieve Provenance Event with ID " + eventId + " to calculate data lineage due to: " + e, e);
- final AsyncLineageSubmission result = new AsyncLineageSubmission(LineageComputationType.FLOWFILE_LINEAGE, eventId, Collections. emptySet(), 1, user.getIdentity());
+ final AsyncLineageSubmission result = new AsyncLineageSubmission(LineageComputationType.FLOWFILE_LINEAGE, eventId, Collections.emptySet(), 1, user.getIdentity());
result.getResult().setError("Failed to retrieve Provenance Event with ID " + eventId + ". See logs for more information.");
return result;
}
if (!eventOption.isPresent()) {
- final AsyncLineageSubmission result = new AsyncLineageSubmission(LineageComputationType.FLOWFILE_LINEAGE, eventId, Collections. emptySet(), 1, user.getIdentity());
+ final AsyncLineageSubmission result = new AsyncLineageSubmission(LineageComputationType.FLOWFILE_LINEAGE, eventId, Collections.emptySet(), 1, user.getIdentity());
result.getResult().setError("Could not find Provenance Event with ID " + eventId);
lineageSubmissionMap.put(result.getLineageIdentifier(), result);
return result;
@@ -524,7 +523,7 @@ public class LuceneEventIndex implements EventIndex {
}
default: {
final AsyncLineageSubmission submission = new AsyncLineageSubmission(LineageComputationType.EXPAND_CHILDREN,
- eventId, Collections. emptyList(), 1, userId);
+ eventId, Collections.emptyList(), 1, userId);
lineageSubmissionMap.put(submission.getLineageIdentifier(), submission);
submission.getResult().setError("Event ID " + eventId + " indicates an event of type " + event.getEventType() + " so its children cannot be expanded");
@@ -533,7 +532,7 @@ public class LuceneEventIndex implements EventIndex {
}
} catch (final Exception e) {
final AsyncLineageSubmission submission = new AsyncLineageSubmission(LineageComputationType.EXPAND_CHILDREN,
- eventId, Collections. emptyList(), 1, userId);
+ eventId, Collections.emptyList(), 1, userId);
lineageSubmissionMap.put(submission.getLineageIdentifier(), submission);
submission.getResult().setError("Failed to expand children for lineage of event with ID " + eventId + " due to: " + e);
return submission;
@@ -564,7 +563,7 @@ public class LuceneEventIndex implements EventIndex {
}
default: {
final AsyncLineageSubmission submission = new AsyncLineageSubmission(LineageComputationType.EXPAND_PARENTS,
- eventId, Collections. emptyList(), 1, userId);
+ eventId, Collections.emptyList(), 1, userId);
lineageSubmissionMap.put(submission.getLineageIdentifier(), submission);
submission.getResult().setError("Event ID " + eventId + " indicates an event of type " + event.getEventType() + " so its parents cannot be expanded");
@@ -573,7 +572,7 @@ public class LuceneEventIndex implements EventIndex {
}
} catch (final Exception e) {
final AsyncLineageSubmission submission = new AsyncLineageSubmission(LineageComputationType.EXPAND_PARENTS,
- eventId, Collections. emptyList(), 1, userId);
+ eventId, Collections.emptyList(), 1, userId);
lineageSubmissionMap.put(submission.getLineageIdentifier(), submission);
submission.getResult().setError("Failed to expand parents for lineage of event with ID " + eventId + " due to: " + e);
diff --git a/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/schema/EventFieldNames.java b/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/schema/EventFieldNames.java
index d6f50dda2e..2bc7fbe9bd 100644
--- a/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/schema/EventFieldNames.java
+++ b/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/schema/EventFieldNames.java
@@ -56,4 +56,12 @@ public class EventFieldNames {
public static final String EXPLICIT_VALUE = "Explicit Value";
public static final String LOOKUP_VALUE = "Lookup Value";
public static final String UNCHANGED_VALUE = "Unchanged";
+
+ // For encrypted records
+ public static final String IS_ENCRYPTED = "Encrypted Record";
+ public static final String KEY_ID = "Encryption Key ID";
+ public static final String VERSION = "Encryption Version";
+ public static final String ALGORITHM = "Encryption Algorithm";
+ public static final String IV = "Initialization Vector";
+ public static final String ENCRYPTION_DETAILS = "Encryption Details";
}
diff --git a/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/schema/LookupTableEventRecordFields.java b/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/schema/LookupTableEventRecordFields.java
index 7b33ded095..057763656d 100644
--- a/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/schema/LookupTableEventRecordFields.java
+++ b/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/schema/LookupTableEventRecordFields.java
@@ -66,12 +66,12 @@ public class LookupTableEventRecordFields {
public static final RecordField CONTENT_CLAIM_SIZE = new SimpleRecordField(EventFieldNames.CONTENT_CLAIM_SIZE, FieldType.LONG, EXACTLY_ONE);
public static final RecordField PREVIOUS_CONTENT_CLAIM = new ComplexRecordField(EventFieldNames.PREVIOUS_CONTENT_CLAIM, ZERO_OR_ONE,
- CONTENT_CLAIM_CONTAINER, CONTENT_CLAIM_SECTION, CONTENT_CLAIM_IDENTIFIER, CONTENT_CLAIM_OFFSET, CONTENT_CLAIM_SIZE);
+ CONTENT_CLAIM_CONTAINER, CONTENT_CLAIM_SECTION, CONTENT_CLAIM_IDENTIFIER, CONTENT_CLAIM_OFFSET, CONTENT_CLAIM_SIZE);
public static final RecordField CURRENT_CONTENT_CLAIM_EXPLICIT = new ComplexRecordField(EventFieldNames.EXPLICIT_VALUE, EXACTLY_ONE,
- CONTENT_CLAIM_CONTAINER, CONTENT_CLAIM_SECTION, CONTENT_CLAIM_IDENTIFIER, CONTENT_CLAIM_OFFSET, CONTENT_CLAIM_SIZE);
+ CONTENT_CLAIM_CONTAINER, CONTENT_CLAIM_SECTION, CONTENT_CLAIM_IDENTIFIER, CONTENT_CLAIM_OFFSET, CONTENT_CLAIM_SIZE);
public static final RecordField CURRENT_CONTENT_CLAIM = new UnionRecordField(EventFieldNames.CONTENT_CLAIM,
- Repetition.EXACTLY_ONE, NO_VALUE, UNCHANGED_VALUE, CURRENT_CONTENT_CLAIM_EXPLICIT);
+ Repetition.EXACTLY_ONE, NO_VALUE, UNCHANGED_VALUE, CURRENT_CONTENT_CLAIM_EXPLICIT);
// EventType-Specific fields
diff --git a/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/schema/LookupTableEventSchema.java b/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/schema/LookupTableEventSchema.java
index 7110336422..d596c8e800 100644
--- a/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/schema/LookupTableEventSchema.java
+++ b/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/schema/LookupTableEventSchema.java
@@ -45,7 +45,6 @@ import static org.apache.nifi.provenance.schema.LookupTableEventRecordFields.UPD
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
-
import org.apache.nifi.repository.schema.RecordField;
import org.apache.nifi.repository.schema.RecordSchema;
@@ -90,5 +89,4 @@ public class LookupTableEventSchema {
final RecordSchema schema = new RecordSchema(fields);
return schema;
}
-
}
diff --git a/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/serialization/CompressableRecordReader.java b/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/serialization/CompressableRecordReader.java
index 93c066963e..dfbcd2b6ad 100644
--- a/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/serialization/CompressableRecordReader.java
+++ b/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/serialization/CompressableRecordReader.java
@@ -25,7 +25,6 @@ import java.io.InputStream;
import java.nio.charset.StandardCharsets;
import java.util.Optional;
import java.util.zip.GZIPInputStream;
-
import org.apache.nifi.provenance.ProvenanceEventRecord;
import org.apache.nifi.provenance.StandardProvenanceEventRecord;
import org.apache.nifi.provenance.toc.TocReader;
@@ -333,7 +332,7 @@ public abstract class CompressableRecordReader implements RecordReader {
try {
boolean read = true;
while (read) {
- final Optional eventOptional = readToEvent(eventId, dis, serializationVersion);
+ final Optional eventOptional = this.readToEvent(eventId, dis, serializationVersion);
if (eventOptional.isPresent()) {
pushbackEvent = eventOptional.get();
return Optional.of(pushbackEvent);
diff --git a/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/serialization/RecordReaders.java b/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/serialization/RecordReaders.java
index 8e79ddd7fc..7ae4adccc0 100644
--- a/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/serialization/RecordReaders.java
+++ b/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/serialization/RecordReaders.java
@@ -27,9 +27,11 @@ import java.io.InputStream;
import java.nio.file.Path;
import java.util.Collection;
import java.util.zip.GZIPInputStream;
-
+import org.apache.nifi.properties.NiFiPropertiesLoader;
import org.apache.nifi.provenance.ByteArraySchemaRecordReader;
import org.apache.nifi.provenance.ByteArraySchemaRecordWriter;
+import org.apache.nifi.provenance.CryptoUtils;
+import org.apache.nifi.provenance.EncryptedSchemaRecordReader;
import org.apache.nifi.provenance.EventIdFirstSchemaRecordReader;
import org.apache.nifi.provenance.EventIdFirstSchemaRecordWriter;
import org.apache.nifi.provenance.StandardRecordReader;
@@ -37,17 +39,25 @@ import org.apache.nifi.provenance.lucene.LuceneUtil;
import org.apache.nifi.provenance.toc.StandardTocReader;
import org.apache.nifi.provenance.toc.TocReader;
import org.apache.nifi.provenance.toc.TocUtil;
+import org.apache.nifi.util.NiFiProperties;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
public class RecordReaders {
+ private static Logger logger = LoggerFactory.getLogger(RecordReaders.class);
+
+ private static boolean isEncryptionAvailable = false;
+ private static boolean encryptionPropertiesRead = false;
+
/**
* Creates a new Record Reader that is capable of reading Provenance Event Journals
*
- * @param file the Provenance Event Journal to read data from
+ * @param file the Provenance Event Journal to read data from
* @param provenanceLogFiles collection of all provenance journal files
- * @param maxAttributeChars the maximum number of characters to retrieve for any one attribute. This allows us to avoid
- * issues where a FlowFile has an extremely large attribute and reading events
- * for that FlowFile results in loading that attribute into memory many times, exhausting the Java Heap
+ * @param maxAttributeChars the maximum number of characters to retrieve for any one attribute. This allows us to avoid
+ * issues where a FlowFile has an extremely large attribute and reading events
+ * for that FlowFile results in loading that attribute into memory many times, exhausting the Java Heap
* @return a Record Reader capable of reading Provenance Event Journals
* @throws IOException if unable to create a Record Reader for the given file
*/
@@ -68,7 +78,7 @@ public class RecordReaders {
}
}
- if ( file.exists() ) {
+ if (file.exists()) {
try {
fis = new FileInputStream(file);
} catch (final FileNotFoundException fnfe) {
@@ -77,7 +87,8 @@ public class RecordReaders {
}
String filename = file.getName();
- openStream: while ( fis == null ) {
+ openStream:
+ while (fis == null) {
final File dir = file.getParentFile();
final String baseName = LuceneUtil.substringBefore(file.getName(), ".prov");
@@ -85,9 +96,9 @@ public class RecordReaders {
// filename that we need. The majority of the time, we will use the extension ".prov.gz"
// because most often we are compressing on rollover and most often we have already finished
// compressing by the time that we are querying the data.
- for ( final String extension : new String[] {".prov.gz", ".prov"} ) {
+ for (final String extension : new String[]{".prov.gz", ".prov"}) {
file = new File(dir, baseName + extension);
- if ( file.exists() ) {
+ if (file.exists()) {
try {
fis = new FileInputStream(file);
filename = baseName + extension;
@@ -104,7 +115,7 @@ public class RecordReaders {
break;
}
- if ( fis == null ) {
+ if (fis == null) {
throw new FileNotFoundException("Unable to locate file " + originalFile);
}
@@ -148,12 +159,25 @@ public class RecordReaders {
final TocReader tocReader = new StandardTocReader(tocFile);
return new EventIdFirstSchemaRecordReader(bufferedInStream, filename, tocReader, maxAttributeChars);
}
+ case EncryptedSchemaRecordReader.SERIALIZATION_NAME: {
+ if (!tocFile.exists()) {
+ throw new FileNotFoundException("Cannot create TOC Reader because the file " + tocFile + " does not exist");
+ }
+
+ if (!isEncryptionAvailable()) {
+ throw new IOException("Cannot read encrypted repository because this reader is not configured for encryption");
+ }
+
+ final TocReader tocReader = new StandardTocReader(tocFile);
+ // Return a reader with no eventEncryptor because this method contract cannot change, then inject the encryptor from the writer in the calling method
+ return new EncryptedSchemaRecordReader(bufferedInStream, filename, tocReader, maxAttributeChars, null);
+ }
default: {
throw new IOException("Unable to read data from file " + file + " because the file was written using an unknown Serializer: " + serializationName);
}
}
} catch (final IOException ioe) {
- if ( fis != null ) {
+ if (fis != null) {
try {
fis.close();
} catch (final IOException inner) {
@@ -165,4 +189,20 @@ public class RecordReaders {
}
}
+ private static boolean isEncryptionAvailable() {
+ if (encryptionPropertiesRead) {
+ return isEncryptionAvailable;
+ } else {
+ try {
+ NiFiProperties niFiProperties = NiFiPropertiesLoader.loadDefaultWithKeyFromBootstrap();
+ isEncryptionAvailable = CryptoUtils.isProvenanceRepositoryEncryptionConfigured(niFiProperties);
+ encryptionPropertiesRead = true;
+ } catch (IOException e) {
+ logger.error("Encountered an error checking the provenance repository encryption configuration: ", e);
+ isEncryptionAvailable = false;
+ }
+ return isEncryptionAvailable;
+ }
+ }
+
}
diff --git a/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/util/StorageSummaryEvent.java b/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/util/StorageSummaryEvent.java
index 41d5ade7b6..766278ae21 100644
--- a/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/util/StorageSummaryEvent.java
+++ b/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/util/StorageSummaryEvent.java
@@ -19,7 +19,6 @@ package org.apache.nifi.provenance.util;
import java.util.List;
import java.util.Map;
-
import org.apache.nifi.provenance.ProvenanceEventRecord;
import org.apache.nifi.provenance.ProvenanceEventType;
import org.apache.nifi.provenance.serialization.StorageSummary;
@@ -182,4 +181,14 @@ public class StorageSummaryEvent implements ProvenanceEventRecord {
public Long getPreviousContentClaimOffset() {
return event.getPreviousContentClaimOffset();
}
+
+ /**
+ * Returns the best event identifier for this event (eventId if available, descriptive identifier if not yet persisted to allow for traceability).
+ *
+ * @return a descriptive event ID to allow tracing
+ */
+ @Override
+ public String getBestEventIdentifier() {
+ return Long.toString(getEventId());
+ }
}
diff --git a/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/resources/META-INF/services/org.apache.nifi.provenance.ProvenanceRepository b/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/resources/META-INF/services/org.apache.nifi.provenance.ProvenanceRepository
index 6a353d2af8..94cc70cd0f 100644
--- a/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/resources/META-INF/services/org.apache.nifi.provenance.ProvenanceRepository
+++ b/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/resources/META-INF/services/org.apache.nifi.provenance.ProvenanceRepository
@@ -13,4 +13,5 @@
# See the License for the specific language governing permissions and
# limitations under the License.
org.apache.nifi.provenance.PersistentProvenanceRepository
-org.apache.nifi.provenance.WriteAheadProvenanceRepository
\ No newline at end of file
+org.apache.nifi.provenance.WriteAheadProvenanceRepository
+org.apache.nifi.provenance.EncryptedWriteAheadProvenanceRepository
\ No newline at end of file
diff --git a/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/test/groovy/org/apache/nifi/provenance/EncryptedSchemaRecordReaderWriterTest.groovy b/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/test/groovy/org/apache/nifi/provenance/EncryptedSchemaRecordReaderWriterTest.groovy
new file mode 100644
index 0000000000..ec8c225e06
--- /dev/null
+++ b/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/test/groovy/org/apache/nifi/provenance/EncryptedSchemaRecordReaderWriterTest.groovy
@@ -0,0 +1,281 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.provenance
+
+import org.apache.nifi.flowfile.FlowFile
+import org.apache.nifi.provenance.serialization.RecordReader
+import org.apache.nifi.provenance.serialization.RecordWriter
+import org.apache.nifi.provenance.toc.StandardTocReader
+import org.apache.nifi.provenance.toc.StandardTocWriter
+import org.apache.nifi.provenance.toc.TocReader
+import org.apache.nifi.provenance.toc.TocUtil
+import org.apache.nifi.provenance.toc.TocWriter
+import org.apache.nifi.util.file.FileUtils
+import org.bouncycastle.jce.provider.BouncyCastleProvider
+import org.bouncycastle.util.encoders.Hex
+import org.junit.After
+import org.junit.AfterClass
+import org.junit.Before
+import org.junit.BeforeClass
+import org.junit.ClassRule
+import org.junit.Test
+import org.junit.rules.TemporaryFolder
+import org.junit.runner.RunWith
+import org.junit.runners.JUnit4
+import org.slf4j.Logger
+import org.slf4j.LoggerFactory
+
+import javax.crypto.Cipher
+import javax.crypto.spec.SecretKeySpec
+import java.security.KeyManagementException
+import java.security.Security
+import java.util.concurrent.atomic.AtomicLong
+
+import static groovy.test.GroovyAssert.shouldFail
+import static org.apache.nifi.provenance.TestUtil.createFlowFile
+
+@RunWith(JUnit4.class)
+class EncryptedSchemaRecordReaderWriterTest extends AbstractTestRecordReaderWriter {
+ private static final Logger logger = LoggerFactory.getLogger(EncryptedSchemaRecordReaderWriterTest.class)
+
+ private static final String KEY_HEX_128 = "0123456789ABCDEFFEDCBA9876543210"
+ private static final String KEY_HEX_256 = KEY_HEX_128 * 2
+ private static final String KEY_HEX = isUnlimitedStrengthCryptoAvailable() ? KEY_HEX_256 : KEY_HEX_128
+ private static final String KEY_ID = "K1"
+
+ private static final String TRANSIT_URI = "nifi://unit-test"
+ private static final String PROCESSOR_TYPE = "Mock Processor"
+ private static final String COMPONENT_ID = "1234"
+
+ private static final int UNCOMPRESSED_BLOCK_SIZE = 1024 * 32
+ private static final int MAX_ATTRIBUTE_SIZE = 2048
+
+ private static final AtomicLong idGenerator = new AtomicLong(0L)
+ private File journalFile
+ private File tocFile
+
+ private static KeyProvider mockKeyProvider
+ private static ProvenanceEventEncryptor provenanceEventEncryptor = new AESProvenanceEventEncryptor()
+
+ @ClassRule
+ public static TemporaryFolder tempFolder = new TemporaryFolder()
+
+ private static String ORIGINAL_LOG_LEVEL
+
+ @BeforeClass
+ static void setUpOnce() throws Exception {
+ ORIGINAL_LOG_LEVEL = System.getProperty("org.slf4j.simpleLogger.log.org.apache.nifi.provenance")
+ System.setProperty("org.slf4j.simpleLogger.log.org.apache.nifi.provenance", "DEBUG")
+
+ Security.addProvider(new BouncyCastleProvider())
+
+ logger.metaClass.methodMissing = { String name, args ->
+ logger.info("[${name?.toUpperCase()}] ${(args as List).join(" ")}")
+ }
+
+ mockKeyProvider = [
+ getKey : { String keyId ->
+ logger.mock("Requesting key ID: ${keyId}")
+ if (keyId == KEY_ID) {
+ new SecretKeySpec(Hex.decode(KEY_HEX), "AES")
+ } else {
+ throw new KeyManagementException("${keyId} is not available")
+ }
+ },
+ getAvailableKeyIds: { ->
+ logger.mock("Available key IDs: [${KEY_ID}]")
+ [KEY_ID]
+ },
+ keyExists : { String keyId ->
+ logger.mock("Checking availability of key ID: ${keyId}")
+ keyId == KEY_ID
+ }] as KeyProvider
+ provenanceEventEncryptor.initialize(mockKeyProvider)
+ }
+
+ @Before
+ void setUp() throws Exception {
+ journalFile = new File("target/storage/${UUID.randomUUID()}/testEventIdFirstSchemaRecordReaderWriter")
+ tocFile = TocUtil.getTocFile(journalFile)
+ idGenerator.set(0L)
+ }
+
+ @After
+ void tearDown() throws Exception {
+ try {
+ FileUtils.deleteFile(journalFile.getParentFile(), true)
+ } catch (Exception e) {
+ logger.error(e.getMessage())
+ }
+ }
+
+ @AfterClass
+ static void tearDownOnce() throws Exception {
+ if (ORIGINAL_LOG_LEVEL) {
+ System.setProperty("org.slf4j.simpleLogger.log.org.apache.nifi.provenance", ORIGINAL_LOG_LEVEL)
+ }
+ try {
+ FileUtils.deleteFile(new File("target/storage"), true)
+ } catch (Exception e) {
+ logger.error(e)
+ }
+ }
+
+ private static boolean isUnlimitedStrengthCryptoAvailable() {
+ Cipher.getMaxAllowedKeyLength("AES") > 128
+ }
+
+ private static
+ final FlowFile buildFlowFile(Map attributes = [:], long id = idGenerator.getAndIncrement(), long fileSize = 3000L) {
+ if (!attributes?.uuid) {
+ attributes.uuid = UUID.randomUUID().toString()
+ }
+ createFlowFile(id, fileSize, attributes)
+ }
+
+ private
+ static ProvenanceEventRecord buildEventRecord(FlowFile flowfile = buildFlowFile(), ProvenanceEventType eventType = ProvenanceEventType.RECEIVE, String transitUri = TRANSIT_URI, String componentId = COMPONENT_ID, String componentType = PROCESSOR_TYPE, long eventTime = System.currentTimeMillis()) {
+ final ProvenanceEventBuilder builder = new StandardProvenanceEventRecord.Builder()
+ builder.setEventTime(eventTime)
+ builder.setEventType(eventType)
+ builder.setTransitUri(transitUri)
+ builder.fromFlowFile(flowfile)
+ builder.setComponentId(componentId)
+ builder.setComponentType(componentType)
+ builder.build()
+ }
+
+ @Override
+ protected RecordWriter createWriter(
+ final File file,
+ final TocWriter tocWriter, final boolean compressed, final int uncompressedBlockSize) throws IOException {
+ createWriter(file, tocWriter, compressed, uncompressedBlockSize, provenanceEventEncryptor)
+ }
+
+ protected static RecordWriter createWriter(
+ final File file,
+ final TocWriter tocWriter,
+ final boolean compressed,
+ final int uncompressedBlockSize, ProvenanceEventEncryptor encryptor) throws IOException {
+ return new EncryptedSchemaRecordWriter(file, idGenerator, tocWriter, compressed, uncompressedBlockSize, IdentifierLookup.EMPTY, encryptor, 1)
+ }
+
+ @Override
+ protected RecordReader createReader(
+ final InputStream inputStream,
+ final String journalFilename, final TocReader tocReader, final int maxAttributeSize) throws IOException {
+ return new EncryptedSchemaRecordReader(inputStream, journalFilename, tocReader, maxAttributeSize, provenanceEventEncryptor)
+ }
+
+ /**
+ * Build a record and write it to the repository with the encrypted writer. Recover with the encrypted reader and verify.
+ */
+ @Test
+ void testShouldWriteAndReadEncryptedRecord() {
+ // Arrange
+ final ProvenanceEventRecord record = buildEventRecord()
+ logger.info("Built sample PER: ${record}")
+
+ TocWriter tocWriter = new StandardTocWriter(tocFile, false, false)
+
+ RecordWriter encryptedWriter = createWriter(journalFile, tocWriter, false, UNCOMPRESSED_BLOCK_SIZE)
+ logger.info("Generated encrypted writer: ${encryptedWriter}")
+
+ // Act
+ int encryptedRecordId = idGenerator.get()
+ encryptedWriter.writeHeader(encryptedRecordId)
+ encryptedWriter.writeRecord(record)
+ encryptedWriter.close()
+ logger.info("Wrote encrypted record ${encryptedRecordId} to journal")
+
+ // Assert
+ TocReader tocReader = new StandardTocReader(tocFile)
+ final FileInputStream fis = new FileInputStream(journalFile)
+ final RecordReader reader = createReader(fis, journalFile.getName(), tocReader, MAX_ATTRIBUTE_SIZE)
+ logger.info("Generated encrypted reader: ${reader}")
+
+ ProvenanceEventRecord encryptedEvent = reader.nextRecord()
+ assert encryptedEvent
+ assert encryptedRecordId as long == encryptedEvent.getEventId()
+ assert record.componentId == encryptedEvent.getComponentId()
+ assert record.componentType == encryptedEvent.getComponentType()
+ logger.info("Successfully read encrypted record: ${encryptedEvent}")
+
+ assert !reader.nextRecord()
+ }
+
+ /**
+ * Build a record and write it with a standard writer and the encrypted writer to different repositories. Recover with the standard reader and the contents of the encrypted record should be unreadable.
+ */
+ @Test
+ void testShouldWriteEncryptedRecordAndPlainRecord() {
+ // Arrange
+ final ProvenanceEventRecord record = buildEventRecord()
+ logger.info("Built sample PER: ${record}")
+
+ TocWriter tocWriter = new StandardTocWriter(tocFile, false, false)
+
+ RecordWriter standardWriter = new EventIdFirstSchemaRecordWriter(journalFile, idGenerator, tocWriter, false, UNCOMPRESSED_BLOCK_SIZE, IdentifierLookup.EMPTY)
+ logger.info("Generated standard writer: ${standardWriter}")
+
+ File encryptedJournalFile = new File(journalFile.absolutePath + "_encrypted")
+ File encryptedTocFile = TocUtil.getTocFile(encryptedJournalFile)
+ TocWriter encryptedTocWriter = new StandardTocWriter(encryptedTocFile, false, false)
+ RecordWriter encryptedWriter = createWriter(encryptedJournalFile, encryptedTocWriter, false, UNCOMPRESSED_BLOCK_SIZE)
+ logger.info("Generated encrypted writer: ${encryptedWriter}")
+
+ // Act
+ int standardRecordId = idGenerator.get()
+ standardWriter.writeHeader(standardRecordId)
+ standardWriter.writeRecord(record)
+ standardWriter.close()
+ logger.info("Wrote standard record ${standardRecordId} to journal")
+
+ int encryptedRecordId = idGenerator.get()
+ encryptedWriter.writeHeader(encryptedRecordId)
+ encryptedWriter.writeRecord(record)
+ encryptedWriter.close()
+ logger.info("Wrote encrypted record ${encryptedRecordId} to journal")
+
+ // Assert
+ TocReader tocReader = new StandardTocReader(tocFile)
+ final FileInputStream fis = new FileInputStream(journalFile)
+ final RecordReader reader = new EventIdFirstSchemaRecordReader(fis, journalFile.getName(), tocReader, MAX_ATTRIBUTE_SIZE)
+ logger.info("Generated standard reader: ${reader}")
+
+ ProvenanceEventRecord standardEvent = reader.nextRecord()
+ assert standardEvent
+ assert standardRecordId as long == standardEvent.getEventId()
+ assert record.componentId == standardEvent.getComponentId()
+ assert record.componentType == standardEvent.getComponentType()
+ logger.info("Successfully read standard record: ${standardEvent}")
+
+ assert !reader.nextRecord()
+
+ // Demonstrate unable to read from encrypted file with standard reader
+ TocReader incompatibleTocReader = new StandardTocReader(encryptedTocFile)
+ final FileInputStream efis = new FileInputStream(encryptedJournalFile)
+ RecordReader incompatibleReader = new EventIdFirstSchemaRecordReader(efis, encryptedJournalFile.getName(), incompatibleTocReader, MAX_ATTRIBUTE_SIZE)
+ logger.info("Generated standard reader (attempting to read encrypted file): ${incompatibleReader}")
+
+ def msg = shouldFail(EOFException) {
+ ProvenanceEventRecord encryptedEvent = incompatibleReader.nextRecord()
+ }
+ logger.expected(msg)
+ assert msg =~ "EOFException: Failed to read field"
+ }
+}
diff --git a/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/test/groovy/org/apache/nifi/provenance/EncryptedWriteAheadProvenanceRepositoryTest.groovy b/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/test/groovy/org/apache/nifi/provenance/EncryptedWriteAheadProvenanceRepositoryTest.groovy
new file mode 100644
index 0000000000..42cc8819be
--- /dev/null
+++ b/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/test/groovy/org/apache/nifi/provenance/EncryptedWriteAheadProvenanceRepositoryTest.groovy
@@ -0,0 +1,391 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License") you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.provenance
+
+import org.apache.nifi.events.EventReporter
+import org.apache.nifi.flowfile.FlowFile
+import org.apache.nifi.provenance.serialization.RecordReaders
+import org.apache.nifi.reporting.Severity
+import org.apache.nifi.util.file.FileUtils
+import org.bouncycastle.jce.provider.BouncyCastleProvider
+import org.junit.After
+import org.junit.AfterClass
+import org.junit.Before
+import org.junit.BeforeClass
+import org.junit.ClassRule
+import org.junit.Test
+import org.junit.rules.TemporaryFolder
+import org.junit.runner.RunWith
+import org.junit.runners.JUnit4
+import org.slf4j.Logger
+import org.slf4j.LoggerFactory
+
+import javax.crypto.Cipher
+import java.security.Security
+import java.util.concurrent.TimeUnit
+import java.util.concurrent.atomic.AtomicLong
+
+import static org.apache.nifi.provenance.TestUtil.createFlowFile
+
+@RunWith(JUnit4.class)
+class EncryptedWriteAheadProvenanceRepositoryTest {
+ private static final Logger logger = LoggerFactory.getLogger(EncryptedWriteAheadProvenanceRepositoryTest.class)
+
+ private static final String KEY_HEX_128 = "0123456789ABCDEFFEDCBA9876543210"
+ private static final String KEY_HEX_256 = KEY_HEX_128 * 2
+ private static final String KEY_HEX = isUnlimitedStrengthCryptoAvailable() ? KEY_HEX_256 : KEY_HEX_128
+ private static final String KEY_ID = "K1"
+
+ private static final String TRANSIT_URI = "nifi://unit-test"
+ private static final String PROCESSOR_TYPE = "Mock Processor"
+ private static final String COMPONENT_ID = "1234"
+
+ private static final AtomicLong recordId = new AtomicLong()
+
+ @ClassRule
+ public static TemporaryFolder tempFolder = new TemporaryFolder()
+
+ private ProvenanceRepository repo
+ private static RepositoryConfiguration config
+
+ public static final int DEFAULT_ROLLOVER_MILLIS = 2000
+ private EventReporter eventReporter
+ private List reportedEvents = Collections.synchronizedList(new ArrayList())
+
+ private static String ORIGINAL_LOG_LEVEL
+
+ @BeforeClass
+ static void setUpOnce() throws Exception {
+ ORIGINAL_LOG_LEVEL = System.getProperty("org.slf4j.simpleLogger.log.org.apache.nifi.provenance")
+ System.setProperty("org.slf4j.simpleLogger.log.org.apache.nifi.provenance", "DEBUG")
+
+ Security.addProvider(new BouncyCastleProvider())
+
+ logger.metaClass.methodMissing = { String name, args ->
+ logger.info("[${name?.toUpperCase()}] ${(args as List).join(" ")}")
+ }
+ }
+
+ @Before
+ void setUp() throws Exception {
+ reportedEvents?.clear()
+ eventReporter = createMockEventReporter()
+ }
+
+ @After
+ void tearDown() throws Exception {
+ closeRepo(repo, config)
+
+ // Reset the boolean determiner
+ RecordReaders.encryptionPropertiesRead = false
+ RecordReaders.isEncryptionAvailable = false
+ }
+
+ @AfterClass
+ static void tearDownOnce() throws Exception {
+ if (ORIGINAL_LOG_LEVEL) {
+ System.setProperty("org.slf4j.simpleLogger.log.org.apache.nifi.provenance", ORIGINAL_LOG_LEVEL)
+ }
+ }
+
+ private static boolean isUnlimitedStrengthCryptoAvailable() {
+ Cipher.getMaxAllowedKeyLength("AES") > 128
+ }
+
+ private static RepositoryConfiguration createConfiguration() {
+ RepositoryConfiguration config = new RepositoryConfiguration()
+ config.addStorageDirectory("1", new File("target/storage/" + UUID.randomUUID().toString()))
+ config.setCompressOnRollover(true)
+ config.setMaxEventFileLife(2000L, TimeUnit.SECONDS)
+ config.setCompressionBlockBytes(100)
+ return config
+ }
+
+ private static RepositoryConfiguration createEncryptedConfiguration() {
+ RepositoryConfiguration config = createConfiguration()
+ config.setEncryptionKeys([(KEY_ID): KEY_HEX])
+ config.setKeyId(KEY_ID)
+ config.setKeyProviderImplementation(StaticKeyProvider.class.name)
+ config
+ }
+
+ private EventReporter createMockEventReporter() {
+ [reportEvent: { Severity s, String c, String m ->
+ ReportedEvent event = new ReportedEvent(s, c, m)
+ reportedEvents.add(event)
+ logger.mock("Added ${event}")
+ }] as EventReporter
+ }
+
+ private void closeRepo(ProvenanceRepository repo = this.repo, RepositoryConfiguration config = this.config) throws IOException {
+ if (repo == null) {
+ return
+ }
+
+ try {
+ repo.close()
+ } catch (IOException ioe) {
+ }
+
+ // Delete all of the storage files. We do this in order to clean up the tons of files that
+ // we create but also to ensure that we have closed all of the file handles. If we leave any
+ // streams open, for instance, this will throw an IOException, causing our unit test to fail.
+ if (config != null) {
+ for (final File storageDir : config.getStorageDirectories().values()) {
+ int i
+ for (i = 0; i < 3; i++) {
+ try {
+ FileUtils.deleteFile(storageDir, true)
+ break
+ } catch (IOException ioe) {
+ // if there is a virus scanner, etc. running in the background we may not be able to
+ // delete the file. Wait a sec and try again.
+ if (i == 2) {
+ throw ioe
+ } else {
+ try {
+ System.out.println("file: " + storageDir.toString() + " exists=" + storageDir.exists())
+ FileUtils.deleteFile(storageDir, true)
+ break
+ } catch (final IOException ioe2) {
+ // if there is a virus scanner, etc. running in the background we may not be able to
+ // delete the file. Wait a sec and try again.
+ if (i == 2) {
+ throw ioe2
+ } else {
+ try {
+ Thread.sleep(1000L)
+ } catch (final InterruptedException ie) {
+ }
+ }
+ }
+ }
+ }
+ }
+ }
+ }
+ }
+
+ private static
+ final FlowFile buildFlowFile(Map attributes = [:], long id = recordId.getAndIncrement(), long fileSize = 3000L) {
+ if (!attributes?.uuid) {
+ attributes.uuid = UUID.randomUUID().toString()
+ }
+ createFlowFile(id, fileSize, attributes)
+ }
+
+ private
+ static ProvenanceEventRecord buildEventRecord(FlowFile flowfile = buildFlowFile(), ProvenanceEventType eventType = ProvenanceEventType.RECEIVE, String transitUri = TRANSIT_URI, String componentId = COMPONENT_ID, String componentType = PROCESSOR_TYPE, long eventTime = System.currentTimeMillis()) {
+ final ProvenanceEventBuilder builder = new StandardProvenanceEventRecord.Builder()
+ builder.setEventTime(eventTime)
+ builder.setEventType(eventType)
+ builder.setTransitUri(transitUri)
+ builder.fromFlowFile(flowfile)
+ builder.setComponentId(componentId)
+ builder.setComponentType(componentType)
+ builder.build()
+ }
+
+ /**
+ * This test operates on {@link PersistentProvenanceRepository} to verify the normal operations of existing implementations.
+ *
+ * @throws IOException
+ * @throws InterruptedException
+ */
+ @Test
+ void testPersistentProvenanceRepositoryShouldRegisterAndRetrieveEvents() throws IOException, InterruptedException {
+ // Arrange
+ config = createConfiguration()
+ config.setMaxEventFileCapacity(1L)
+ config.setMaxEventFileLife(1, TimeUnit.SECONDS)
+ repo = new PersistentProvenanceRepository(config, DEFAULT_ROLLOVER_MILLIS)
+ repo.initialize(eventReporter, null, null, IdentifierLookup.EMPTY)
+
+ Map attributes = ["abc": "xyz",
+ "123": "456"]
+ final ProvenanceEventRecord record = buildEventRecord(buildFlowFile(attributes))
+
+ final int RECORD_COUNT = 10
+
+ // Act
+ RECORD_COUNT.times {
+ repo.registerEvent(record)
+ }
+
+ // Sleep to let the journal merge occur
+ Thread.sleep(1000L)
+
+ final List recoveredRecords = repo.getEvents(0L, RECORD_COUNT + 1)
+
+ logger.info("Recovered ${recoveredRecords.size()} events: ")
+ recoveredRecords.each { logger.info("\t${it}") }
+
+ // Assert
+ assert recoveredRecords.size() == RECORD_COUNT
+ recoveredRecords.eachWithIndex { ProvenanceEventRecord recovered, int i ->
+ assert recovered.getEventId() == (i as Long)
+ assert recovered.getTransitUri() == TRANSIT_URI
+ assert recovered.getEventType() == ProvenanceEventType.RECEIVE
+ // The UUID was added later but we care that all attributes we provided are still there
+ assert recovered.getAttributes().entrySet().containsAll(attributes.entrySet())
+ }
+ }
+
+ /**
+ * This test operates on {@link WriteAheadProvenanceRepository} to verify the normal operations of existing implementations.
+ *
+ * @throws IOException
+ * @throws InterruptedException
+ */
+ @Test
+ void testWriteAheadProvenanceRepositoryShouldRegisterAndRetrieveEvents() throws IOException, InterruptedException {
+ // Arrange
+ config = createConfiguration()
+ // Needed until NIFI-3605 is implemented
+// config.setMaxEventFileCapacity(1L)
+ config.setMaxEventFileCount(1)
+ config.setMaxEventFileLife(1, TimeUnit.SECONDS)
+ repo = new WriteAheadProvenanceRepository(config)
+ repo.initialize(eventReporter, null, null, IdentifierLookup.EMPTY)
+
+ Map attributes = ["abc": "xyz",
+ "123": "456"]
+ final ProvenanceEventRecord record = buildEventRecord(buildFlowFile(attributes))
+
+ final int RECORD_COUNT = 10
+
+ // Act
+ RECORD_COUNT.times {
+ repo.registerEvent(record)
+ }
+
+ final List recoveredRecords = repo.getEvents(0L, RECORD_COUNT + 1)
+
+ logger.info("Recovered ${recoveredRecords.size()} events: ")
+ recoveredRecords.each { logger.info("\t${it}") }
+
+ // Assert
+ assert recoveredRecords.size() == RECORD_COUNT
+ recoveredRecords.eachWithIndex { ProvenanceEventRecord recovered, int i ->
+ assert recovered.getEventId() == (i as Long)
+ assert recovered.getTransitUri() == TRANSIT_URI
+ assert recovered.getEventType() == ProvenanceEventType.RECEIVE
+ // The UUID was added later but we care that all attributes we provided are still there
+ assert recovered.getAttributes().entrySet().containsAll(attributes.entrySet())
+ }
+ }
+
+ @Test
+ void testShouldRegisterAndGetEvent() {
+ // Arrange
+
+ // Override the boolean determiner
+ RecordReaders.encryptionPropertiesRead = true
+ RecordReaders.isEncryptionAvailable = true
+
+ config = createEncryptedConfiguration()
+ // Needed until NIFI-3605 is implemented
+// config.setMaxEventFileCapacity(1L)
+ config.setMaxEventFileCount(1)
+ config.setMaxEventFileLife(1, TimeUnit.SECONDS)
+ repo = new EncryptedWriteAheadProvenanceRepository(config)
+ repo.initialize(eventReporter, null, null, IdentifierLookup.EMPTY)
+
+ Map attributes = ["abc": "This is a plaintext attribute.",
+ "123": "This is another plaintext attribute."]
+ final ProvenanceEventRecord record = buildEventRecord(buildFlowFile(attributes))
+
+ final long LAST_RECORD_ID = repo.getMaxEventId()
+
+ // Act
+ repo.registerEvent(record)
+
+ // Retrieve the event through the interface
+ ProvenanceEventRecord recoveredRecord = repo.getEvent(LAST_RECORD_ID + 1)
+ logger.info("Recovered ${recoveredRecord}")
+
+ // Assert
+ assert recoveredRecord.getEventId() == LAST_RECORD_ID + 1
+ assert recoveredRecord.getTransitUri() == TRANSIT_URI
+ assert recoveredRecord.getEventType() == ProvenanceEventType.RECEIVE
+ // The UUID was added later but we care that all attributes we provided are still there
+ assert recoveredRecord.getAttributes().entrySet().containsAll(attributes.entrySet())
+ }
+
+ @Test
+ void testShouldRegisterAndGetEvents() {
+ // Arrange
+ final int RECORD_COUNT = 10
+
+ // Override the boolean determiner
+ RecordReaders.encryptionPropertiesRead = true
+ RecordReaders.isEncryptionAvailable = true
+
+ config = createEncryptedConfiguration()
+ // Needed until NIFI-3605 is implemented
+// config.setMaxEventFileCapacity(1L)
+ config.setMaxEventFileCount(1)
+ config.setMaxEventFileLife(1, TimeUnit.SECONDS)
+ repo = new EncryptedWriteAheadProvenanceRepository(config)
+ repo.initialize(eventReporter, null, null, IdentifierLookup.EMPTY)
+
+ Map attributes = ["abc": "This is a plaintext attribute.",
+ "123": "This is another plaintext attribute."]
+ final List records = []
+ RECORD_COUNT.times { int i ->
+ records << buildEventRecord(buildFlowFile(attributes + [count: i as String]))
+ }
+ logger.info("Generated ${RECORD_COUNT} records")
+
+ final long LAST_RECORD_ID = repo.getMaxEventId()
+
+ // Act
+ repo.registerEvents(records)
+ logger.info("Registered events")
+
+ // Retrieve the events through the interface
+ List recoveredRecords = repo.getEvents(LAST_RECORD_ID + 1, RECORD_COUNT * 2)
+ logger.info("Recovered ${recoveredRecords.size()} records")
+
+ // Assert
+ recoveredRecords.eachWithIndex { ProvenanceEventRecord recoveredRecord, int i ->
+ assert recoveredRecord.getEventId() == LAST_RECORD_ID + 1 + i
+ assert recoveredRecord.getTransitUri() == TRANSIT_URI
+ assert recoveredRecord.getEventType() == ProvenanceEventType.RECEIVE
+ // The UUID was added later but we care that all attributes we provided are still there
+ assert recoveredRecord.getAttributes().entrySet().containsAll(attributes.entrySet())
+ assert recoveredRecord.getAttribute("count") == i as String
+ }
+ }
+
+ private static class ReportedEvent {
+ final Severity severity
+ final String category
+ final String message
+
+ ReportedEvent(final Severity severity, final String category, final String message) {
+ this.severity = severity
+ this.category = category
+ this.message = message
+ }
+
+ @Override
+ String toString() {
+ "ReportedEvent [${severity}] ${category}: ${message}"
+ }
+ }
+}
diff --git a/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/test/java/org/apache/nifi/provenance/AbstractTestRecordReaderWriter.java b/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/test/java/org/apache/nifi/provenance/AbstractTestRecordReaderWriter.java
index 36397c4744..4b2ca50bd0 100644
--- a/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/test/java/org/apache/nifi/provenance/AbstractTestRecordReaderWriter.java
+++ b/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/test/java/org/apache/nifi/provenance/AbstractTestRecordReaderWriter.java
@@ -32,7 +32,6 @@ import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.UUID;
-
import org.apache.nifi.provenance.serialization.RecordReader;
import org.apache.nifi.provenance.serialization.RecordWriter;
import org.apache.nifi.provenance.toc.StandardTocReader;
@@ -67,19 +66,25 @@ public abstract class AbstractTestRecordReaderWriter {
writer.close();
final TocReader tocReader = new StandardTocReader(tocFile);
+ final String expectedTransitUri = "nifi://unit-test";
+ final int expectedBlockIndex = 0;
+ assertRecoveredRecord(journalFile, tocReader, expectedTransitUri, expectedBlockIndex);
+
+ FileUtils.deleteFile(journalFile.getParentFile(), true);
+ }
+
+ private void assertRecoveredRecord(File journalFile, TocReader tocReader, String expectedTransitUri, int expectedBlockIndex) throws IOException {
try (final FileInputStream fis = new FileInputStream(journalFile);
- final RecordReader reader = createReader(fis, journalFile.getName(), tocReader, 2048)) {
- assertEquals(0, reader.getBlockIndex());
- reader.skipToBlock(0);
+ final RecordReader reader = createReader(fis, journalFile.getName(), tocReader, 2048)) {
+ assertEquals(expectedBlockIndex, reader.getBlockIndex());
+ reader.skipToBlock(expectedBlockIndex);
final StandardProvenanceEventRecord recovered = reader.nextRecord();
assertNotNull(recovered);
- assertEquals("nifi://unit-test", recovered.getTransitUri());
+ assertEquals(expectedTransitUri, recovered.getTransitUri());
assertNull(reader.nextRecord());
}
-
- FileUtils.deleteFile(journalFile.getParentFile(), true);
}
@@ -96,16 +101,7 @@ public abstract class AbstractTestRecordReaderWriter {
final TocReader tocReader = new StandardTocReader(tocFile);
- try (final FileInputStream fis = new FileInputStream(journalFile);
- final RecordReader reader = createReader(fis, journalFile.getName(), tocReader, 2048)) {
- assertEquals(0, reader.getBlockIndex());
- reader.skipToBlock(0);
- final StandardProvenanceEventRecord recovered = reader.nextRecord();
- assertNotNull(recovered);
-
- assertEquals("nifi://unit-test", recovered.getTransitUri());
- assertNull(reader.nextRecord());
- }
+ assertRecoveredRecord(journalFile, tocReader, "nifi://unit-test", 0);
FileUtils.deleteFile(journalFile.getParentFile(), true);
}
diff --git a/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-volatile-provenance-repository/src/main/java/org/apache/nifi/provenance/VolatileProvenanceRepository.java b/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-volatile-provenance-repository/src/main/java/org/apache/nifi/provenance/VolatileProvenanceRepository.java
index f08fed4559..c3fbf426ea 100644
--- a/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-volatile-provenance-repository/src/main/java/org/apache/nifi/provenance/VolatileProvenanceRepository.java
+++ b/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-volatile-provenance-repository/src/main/java/org/apache/nifi/provenance/VolatileProvenanceRepository.java
@@ -16,6 +16,25 @@
*/
package org.apache.nifi.provenance;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Date;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.ThreadFactory;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.regex.Pattern;
import org.apache.nifi.authorization.AccessDeniedException;
import org.apache.nifi.authorization.AuthorizationResult;
import org.apache.nifi.authorization.AuthorizationResult.Result;
@@ -42,26 +61,6 @@ import org.apache.nifi.util.RingBuffer.ForEachEvaluator;
import org.apache.nifi.util.RingBuffer.IterationDirection;
import org.apache.nifi.web.ResourceNotFoundException;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.Date;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.ConcurrentMap;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import java.util.concurrent.ScheduledExecutorService;
-import java.util.concurrent.ThreadFactory;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicBoolean;
-import java.util.concurrent.atomic.AtomicInteger;
-import java.util.concurrent.atomic.AtomicLong;
-import java.util.regex.Pattern;
-
public class VolatileProvenanceRepository implements ProvenanceRepository {
// properties
@@ -472,7 +471,7 @@ public class VolatileProvenanceRepository implements ProvenanceRepository {
}
public Lineage computeLineage(final String flowFileUUID, final NiFiUser user) throws IOException {
- return computeLineage(Collections.singleton(flowFileUUID), user, LineageComputationType.FLOWFILE_LINEAGE, null);
+ return computeLineage(Collections.singleton(flowFileUUID), user, LineageComputationType.FLOWFILE_LINEAGE, null);
}
private Lineage computeLineage(final Collection flowFileUuids, final NiFiUser user, final LineageComputationType computationType, final Long eventId) throws IOException {
@@ -497,7 +496,7 @@ public class VolatileProvenanceRepository implements ProvenanceRepository {
final ProvenanceEventRecord event = getEvent(eventId);
if (event == null) {
final String userId = user.getIdentity();
- final AsyncLineageSubmission result = new AsyncLineageSubmission(LineageComputationType.FLOWFILE_LINEAGE, eventId, Collections.emptySet(), 1, userId);
+ final AsyncLineageSubmission result = new AsyncLineageSubmission(LineageComputationType.FLOWFILE_LINEAGE, eventId, Collections.emptySet(), 1, userId);
result.getResult().setError("Could not find event with ID " + eventId);
lineageSubmissionMap.put(result.getLineageIdentifier(), result);
return result;
@@ -541,9 +540,9 @@ public class VolatileProvenanceRepository implements ProvenanceRepository {
final ProvenanceEventRecord event = getEvent(eventId, user);
if (event == null) {
- final AsyncLineageSubmission submission = new AsyncLineageSubmission(LineageComputationType.EXPAND_PARENTS, eventId, Collections.emptyList(), 1, userId);
+ final AsyncLineageSubmission submission = new AsyncLineageSubmission(LineageComputationType.EXPAND_PARENTS, eventId, Collections.emptyList(), 1, userId);
lineageSubmissionMap.put(submission.getLineageIdentifier(), submission);
- submission.getResult().update(Collections. emptyList(), 0L);
+ submission.getResult().update(Collections.emptyList(), 0L);
return submission;
}
@@ -554,7 +553,7 @@ public class VolatileProvenanceRepository implements ProvenanceRepository {
case CLONE:
return submitLineageComputation(event.getParentUuids(), user, LineageComputationType.EXPAND_PARENTS, eventId);
default: {
- final AsyncLineageSubmission submission = new AsyncLineageSubmission(LineageComputationType.EXPAND_PARENTS, eventId, Collections.emptyList(), 1, userId);
+ final AsyncLineageSubmission submission = new AsyncLineageSubmission(LineageComputationType.EXPAND_PARENTS, eventId, Collections.emptyList(), 1, userId);
lineageSubmissionMap.put(submission.getLineageIdentifier(), submission);
submission.getResult().setError("Event ID " + eventId + " indicates an event of type " + event.getEventType() + " so its parents cannot be expanded");
return submission;
@@ -572,9 +571,9 @@ public class VolatileProvenanceRepository implements ProvenanceRepository {
final ProvenanceEventRecord event = getEvent(eventId, user);
if (event == null) {
- final AsyncLineageSubmission submission = new AsyncLineageSubmission(LineageComputationType.EXPAND_CHILDREN, eventId, Collections.emptyList(), 1, userId);
+ final AsyncLineageSubmission submission = new AsyncLineageSubmission(LineageComputationType.EXPAND_CHILDREN, eventId, Collections.emptyList(), 1, userId);
lineageSubmissionMap.put(submission.getLineageIdentifier(), submission);
- submission.getResult().update(Collections. emptyList(), 0L);
+ submission.getResult().update(Collections.emptyList(), 0L);
return submission;
}
@@ -585,7 +584,7 @@ public class VolatileProvenanceRepository implements ProvenanceRepository {
case CLONE:
return submitLineageComputation(event.getChildUuids(), user, LineageComputationType.EXPAND_CHILDREN, eventId);
default: {
- final AsyncLineageSubmission submission = new AsyncLineageSubmission(LineageComputationType.EXPAND_CHILDREN, eventId, Collections.emptyList(), 1, userId);
+ final AsyncLineageSubmission submission = new AsyncLineageSubmission(LineageComputationType.EXPAND_CHILDREN, eventId, Collections.emptyList(), 1, userId);
lineageSubmissionMap.put(submission.getLineageIdentifier(), submission);
submission.getResult().setError("Event ID " + eventId + " indicates an event of type " + event.getEventType() + " so its children cannot be expanded");
return submission;
@@ -873,5 +872,15 @@ public class VolatileProvenanceRepository implements ProvenanceRepository {
public Long getPreviousContentClaimOffset() {
return record.getPreviousContentClaimOffset();
}
+
+ /**
+ * Returns the best event identifier for this event (eventId if available, descriptive identifier if not yet persisted to allow for traceability).
+ *
+ * @return a descriptive event ID to allow tracing
+ */
+ @Override
+ public String getBestEventIdentifier() {
+ return Long.toString(getEventId());
+ }
}
}
diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/pom.xml b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/pom.xml
index d0a1f1ca0c..0b206e27c8 100644
--- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/pom.xml
+++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/pom.xml
@@ -437,7 +437,7 @@
src/test/resources/TestExtractGrok/simple_text.log
src/test/resources/TestExtractGrok/patterns
- src/main/java/org/apache/nifi/processors/standard/util/crypto/bcrypt/BCrypt.java
+ src/main/java/org/apache/nifi/security/util/crypto/bcrypt/BCrypt.java
diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/EncryptContent.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/EncryptContent.java
index 103790eb39..db6d9bad45 100644
--- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/EncryptContent.java
+++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/EncryptContent.java
@@ -16,6 +16,16 @@
*/
package org.apache.nifi.processors.standard;
+import java.nio.charset.StandardCharsets;
+import java.security.Security;
+import java.text.Normalizer;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+import java.util.concurrent.TimeUnit;
import org.apache.commons.codec.DecoderException;
import org.apache.commons.codec.binary.Hex;
import org.apache.commons.lang3.StringUtils;
@@ -41,27 +51,16 @@ import org.apache.nifi.processor.Relationship;
import org.apache.nifi.processor.exception.ProcessException;
import org.apache.nifi.processor.io.StreamCallback;
import org.apache.nifi.processor.util.StandardValidators;
-import org.apache.nifi.processors.standard.util.crypto.CipherUtility;
-import org.apache.nifi.processors.standard.util.crypto.KeyedEncryptor;
-import org.apache.nifi.processors.standard.util.crypto.OpenPGPKeyBasedEncryptor;
-import org.apache.nifi.processors.standard.util.crypto.OpenPGPPasswordBasedEncryptor;
-import org.apache.nifi.processors.standard.util.crypto.PasswordBasedEncryptor;
import org.apache.nifi.security.util.EncryptionMethod;
import org.apache.nifi.security.util.KeyDerivationFunction;
+import org.apache.nifi.security.util.crypto.CipherUtility;
+import org.apache.nifi.security.util.crypto.KeyedEncryptor;
+import org.apache.nifi.security.util.crypto.OpenPGPKeyBasedEncryptor;
+import org.apache.nifi.security.util.crypto.OpenPGPPasswordBasedEncryptor;
+import org.apache.nifi.security.util.crypto.PasswordBasedEncryptor;
import org.apache.nifi.util.StopWatch;
import org.bouncycastle.jce.provider.BouncyCastleProvider;
-import java.nio.charset.StandardCharsets;
-import java.security.Security;
-import java.text.Normalizer;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Set;
-import java.util.concurrent.TimeUnit;
-
@EventDriven
@SideEffectFree
@SupportsBatching
diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/util/crypto/BcryptCipherProvider.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/security/util/crypto/BcryptCipherProvider.java
similarity index 98%
rename from nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/util/crypto/BcryptCipherProvider.java
rename to nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/security/util/crypto/BcryptCipherProvider.java
index 125aea7e46..f28cde94ad 100644
--- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/util/crypto/BcryptCipherProvider.java
+++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/security/util/crypto/BcryptCipherProvider.java
@@ -14,23 +14,22 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.nifi.processors.standard.util.crypto;
+package org.apache.nifi.security.util.crypto;
-import org.apache.commons.lang3.StringUtils;
-import org.apache.nifi.processor.exception.ProcessException;
-import org.apache.nifi.processors.standard.util.crypto.bcrypt.BCrypt;
-import org.apache.nifi.security.util.EncryptionMethod;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import javax.crypto.Cipher;
-import javax.crypto.SecretKey;
-import javax.crypto.spec.SecretKeySpec;
import java.nio.charset.StandardCharsets;
import java.security.MessageDigest;
import java.util.Arrays;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
+import javax.crypto.Cipher;
+import javax.crypto.SecretKey;
+import javax.crypto.spec.SecretKeySpec;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.nifi.processor.exception.ProcessException;
+import org.apache.nifi.security.util.EncryptionMethod;
+import org.apache.nifi.security.util.crypto.bcrypt.BCrypt;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
public class BcryptCipherProvider extends RandomIVPBECipherProvider {
private static final Logger logger = LoggerFactory.getLogger(BcryptCipherProvider.class);
diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/util/crypto/CipherProviderFactory.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/security/util/crypto/CipherProviderFactory.java
similarity index 97%
rename from nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/util/crypto/CipherProviderFactory.java
rename to nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/security/util/crypto/CipherProviderFactory.java
index e04a7b4956..09004bfadd 100644
--- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/util/crypto/CipherProviderFactory.java
+++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/security/util/crypto/CipherProviderFactory.java
@@ -14,16 +14,15 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.nifi.processors.standard.util.crypto;
+package org.apache.nifi.security.util.crypto;
+import java.util.HashMap;
+import java.util.Map;
import org.apache.nifi.processor.exception.ProcessException;
import org.apache.nifi.security.util.KeyDerivationFunction;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.util.HashMap;
-import java.util.Map;
-
public class CipherProviderFactory {
private static final Logger logger = LoggerFactory.getLogger(CipherProviderFactory.class);
diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/util/crypto/KeyedEncryptor.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/security/util/crypto/KeyedEncryptor.java
similarity index 99%
rename from nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/util/crypto/KeyedEncryptor.java
rename to nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/security/util/crypto/KeyedEncryptor.java
index c573d3d4ba..011eb1f0bd 100644
--- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/util/crypto/KeyedEncryptor.java
+++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/security/util/crypto/KeyedEncryptor.java
@@ -14,8 +14,15 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.nifi.processors.standard.util.crypto;
+package org.apache.nifi.security.util.crypto;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.security.NoSuchAlgorithmException;
+import javax.crypto.Cipher;
+import javax.crypto.SecretKey;
+import javax.crypto.spec.SecretKeySpec;
import org.apache.commons.lang3.StringUtils;
import org.apache.nifi.processor.exception.ProcessException;
import org.apache.nifi.processor.io.StreamCallback;
@@ -23,14 +30,6 @@ import org.apache.nifi.processors.standard.EncryptContent.Encryptor;
import org.apache.nifi.security.util.EncryptionMethod;
import org.apache.nifi.security.util.KeyDerivationFunction;
-import javax.crypto.Cipher;
-import javax.crypto.SecretKey;
-import javax.crypto.spec.SecretKeySpec;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.OutputStream;
-import java.security.NoSuchAlgorithmException;
-
public class KeyedEncryptor implements Encryptor {
private EncryptionMethod encryptionMethod;
diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/util/crypto/NiFiLegacyCipherProvider.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/security/util/crypto/NiFiLegacyCipherProvider.java
similarity index 97%
rename from nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/util/crypto/NiFiLegacyCipherProvider.java
rename to nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/security/util/crypto/NiFiLegacyCipherProvider.java
index c4626294ee..df8a54d89d 100644
--- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/util/crypto/NiFiLegacyCipherProvider.java
+++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/security/util/crypto/NiFiLegacyCipherProvider.java
@@ -14,20 +14,19 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.nifi.processors.standard.util.crypto;
+package org.apache.nifi.security.util.crypto;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.security.SecureRandom;
+import javax.crypto.Cipher;
import org.apache.nifi.processor.exception.ProcessException;
import org.apache.nifi.security.util.EncryptionMethod;
import org.apache.nifi.stream.io.StreamUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import javax.crypto.Cipher;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.OutputStream;
-import java.security.SecureRandom;
-
/**
* Provides a cipher initialized with the original NiFi key derivation process for password-based encryption (MD5 @ 1000 iterations). This is not a secure
* {@link org.apache.nifi.security.util.KeyDerivationFunction} (KDF) and should no longer be used.
@@ -45,7 +44,7 @@ public class NiFiLegacyCipherProvider extends OpenSSLPKCS5CipherProvider impleme
private static final int ITERATION_COUNT = 1000;
/**
- * Returns an initialized cipher for the specified algorithm. The key (and IV if necessary) are derived using the NiFi legacy code, based on @see org.apache.nifi.processors.standard.util.crypto
+ * Returns an initialized cipher for the specified algorithm. The key (and IV if necessary) are derived using the NiFi legacy code, based on @see org.apache.nifi.crypto
* .OpenSSLPKCS5CipherProvider#getCipher(java.lang.String, java.lang.String, java.lang.String, byte[], boolean) [essentially {@code MD5(password || salt) * 1000 }].
*
* @param encryptionMethod the {@link EncryptionMethod}
diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/util/crypto/OpenPGPKeyBasedEncryptor.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/security/util/crypto/OpenPGPKeyBasedEncryptor.java
similarity index 99%
rename from nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/util/crypto/OpenPGPKeyBasedEncryptor.java
rename to nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/security/util/crypto/OpenPGPKeyBasedEncryptor.java
index f0f8631e33..6b6c2fc955 100644
--- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/util/crypto/OpenPGPKeyBasedEncryptor.java
+++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/security/util/crypto/OpenPGPKeyBasedEncryptor.java
@@ -14,8 +14,20 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.nifi.processors.standard.util.crypto;
+package org.apache.nifi.security.util.crypto;
+import static org.apache.nifi.processors.standard.util.PGPUtil.BLOCK_SIZE;
+import static org.apache.nifi.processors.standard.util.PGPUtil.BUFFER_SIZE;
+
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.security.NoSuchProviderException;
+import java.security.SecureRandom;
+import java.util.Date;
+import java.util.Iterator;
+import java.util.zip.Deflater;
import org.apache.nifi.processor.exception.ProcessException;
import org.apache.nifi.processor.io.StreamCallback;
import org.apache.nifi.processors.standard.EncryptContent;
@@ -51,19 +63,6 @@ import org.bouncycastle.openpgp.operator.jcajce.JcePublicKeyKeyEncryptionMethodG
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.io.FileInputStream;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.OutputStream;
-import java.security.NoSuchProviderException;
-import java.security.SecureRandom;
-import java.util.Date;
-import java.util.Iterator;
-import java.util.zip.Deflater;
-
-import static org.apache.nifi.processors.standard.util.PGPUtil.BLOCK_SIZE;
-import static org.apache.nifi.processors.standard.util.PGPUtil.BUFFER_SIZE;
-
public class OpenPGPKeyBasedEncryptor implements Encryptor {
private static final Logger logger = LoggerFactory.getLogger(OpenPGPPasswordBasedEncryptor.class);
diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/util/crypto/OpenPGPPasswordBasedEncryptor.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/security/util/crypto/OpenPGPPasswordBasedEncryptor.java
similarity index 99%
rename from nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/util/crypto/OpenPGPPasswordBasedEncryptor.java
rename to nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/security/util/crypto/OpenPGPPasswordBasedEncryptor.java
index 93e565aaea..6d5bb6d3c2 100644
--- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/util/crypto/OpenPGPPasswordBasedEncryptor.java
+++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/security/util/crypto/OpenPGPPasswordBasedEncryptor.java
@@ -14,8 +14,13 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.nifi.processors.standard.util.crypto;
+package org.apache.nifi.security.util.crypto;
+import static org.bouncycastle.openpgp.PGPUtil.getDecoderStream;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
import org.apache.nifi.processor.exception.ProcessException;
import org.apache.nifi.processor.io.StreamCallback;
import org.apache.nifi.processors.standard.EncryptContent.Encryptor;
@@ -35,12 +40,6 @@ import org.bouncycastle.openpgp.operator.jcajce.JcePBEKeyEncryptionMethodGenerat
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.OutputStream;
-
-import static org.bouncycastle.openpgp.PGPUtil.getDecoderStream;
-
public class OpenPGPPasswordBasedEncryptor implements Encryptor {
private static final Logger logger = LoggerFactory.getLogger(OpenPGPPasswordBasedEncryptor.class);
diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/util/crypto/OpenSSLPKCS5CipherProvider.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/security/util/crypto/OpenSSLPKCS5CipherProvider.java
similarity index 99%
rename from nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/util/crypto/OpenSSLPKCS5CipherProvider.java
rename to nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/security/util/crypto/OpenSSLPKCS5CipherProvider.java
index 4253f9b48d..597e516c76 100644
--- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/util/crypto/OpenSSLPKCS5CipherProvider.java
+++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/security/util/crypto/OpenSSLPKCS5CipherProvider.java
@@ -14,21 +14,8 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.nifi.processors.standard.util.crypto;
+package org.apache.nifi.security.util.crypto;
-import org.apache.commons.lang3.StringUtils;
-import org.apache.nifi.processor.exception.ProcessException;
-import org.apache.nifi.security.util.EncryptionMethod;
-import org.apache.nifi.stream.io.StreamUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import javax.crypto.Cipher;
-import javax.crypto.NoSuchPaddingException;
-import javax.crypto.SecretKey;
-import javax.crypto.SecretKeyFactory;
-import javax.crypto.spec.PBEKeySpec;
-import javax.crypto.spec.PBEParameterSpec;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
@@ -40,6 +27,18 @@ import java.security.NoSuchProviderException;
import java.security.SecureRandom;
import java.security.spec.InvalidKeySpecException;
import java.util.Arrays;
+import javax.crypto.Cipher;
+import javax.crypto.NoSuchPaddingException;
+import javax.crypto.SecretKey;
+import javax.crypto.SecretKeyFactory;
+import javax.crypto.spec.PBEKeySpec;
+import javax.crypto.spec.PBEParameterSpec;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.nifi.processor.exception.ProcessException;
+import org.apache.nifi.security.util.EncryptionMethod;
+import org.apache.nifi.stream.io.StreamUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
public class OpenSSLPKCS5CipherProvider implements PBECipherProvider {
private static final Logger logger = LoggerFactory.getLogger(OpenSSLPKCS5CipherProvider.class);
diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/util/crypto/PBECipherProvider.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/security/util/crypto/PBECipherProvider.java
similarity index 97%
rename from nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/util/crypto/PBECipherProvider.java
rename to nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/security/util/crypto/PBECipherProvider.java
index 4d9fcfc18e..235af00126 100644
--- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/util/crypto/PBECipherProvider.java
+++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/security/util/crypto/PBECipherProvider.java
@@ -14,14 +14,13 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.nifi.processors.standard.util.crypto;
+package org.apache.nifi.security.util.crypto;
-import org.apache.nifi.security.util.EncryptionMethod;
-
-import javax.crypto.Cipher;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
+import javax.crypto.Cipher;
+import org.apache.nifi.security.util.EncryptionMethod;
public interface PBECipherProvider extends CipherProvider {
diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/util/crypto/PBKDF2CipherProvider.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/security/util/crypto/PBKDF2CipherProvider.java
similarity index 99%
rename from nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/util/crypto/PBKDF2CipherProvider.java
rename to nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/security/util/crypto/PBKDF2CipherProvider.java
index ee8d5d31e4..fbad3b6b75 100644
--- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/util/crypto/PBKDF2CipherProvider.java
+++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/security/util/crypto/PBKDF2CipherProvider.java
@@ -14,8 +14,13 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.nifi.processors.standard.util.crypto;
+package org.apache.nifi.security.util.crypto;
+import java.nio.charset.StandardCharsets;
+import java.security.SecureRandom;
+import javax.crypto.Cipher;
+import javax.crypto.SecretKey;
+import javax.crypto.spec.SecretKeySpec;
import org.apache.commons.lang3.StringUtils;
import org.apache.nifi.processor.exception.ProcessException;
import org.apache.nifi.security.util.EncryptionMethod;
@@ -30,12 +35,6 @@ import org.bouncycastle.crypto.params.KeyParameter;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import javax.crypto.Cipher;
-import javax.crypto.SecretKey;
-import javax.crypto.spec.SecretKeySpec;
-import java.nio.charset.StandardCharsets;
-import java.security.SecureRandom;
-
public class PBKDF2CipherProvider extends RandomIVPBECipherProvider {
private static final Logger logger = LoggerFactory.getLogger(PBKDF2CipherProvider.class);
private static final int DEFAULT_SALT_LENGTH = 16;
diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/util/crypto/PasswordBasedEncryptor.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/security/util/crypto/PasswordBasedEncryptor.java
similarity index 99%
rename from nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/util/crypto/PasswordBasedEncryptor.java
rename to nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/security/util/crypto/PasswordBasedEncryptor.java
index eed5925dbd..8cdfaafe04 100644
--- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/util/crypto/PasswordBasedEncryptor.java
+++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/security/util/crypto/PasswordBasedEncryptor.java
@@ -14,8 +14,15 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.nifi.processors.standard.util.crypto;
+package org.apache.nifi.security.util.crypto;
+import java.io.EOFException;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.security.NoSuchAlgorithmException;
+import javax.crypto.Cipher;
+import javax.crypto.spec.PBEKeySpec;
import org.apache.commons.lang3.StringUtils;
import org.apache.nifi.processor.exception.ProcessException;
import org.apache.nifi.processor.io.StreamCallback;
@@ -23,14 +30,6 @@ import org.apache.nifi.processors.standard.EncryptContent.Encryptor;
import org.apache.nifi.security.util.EncryptionMethod;
import org.apache.nifi.security.util.KeyDerivationFunction;
-import javax.crypto.Cipher;
-import javax.crypto.spec.PBEKeySpec;
-import java.io.EOFException;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.OutputStream;
-import java.security.NoSuchAlgorithmException;
-
public class PasswordBasedEncryptor implements Encryptor {
private EncryptionMethod encryptionMethod;
diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/util/crypto/RandomIVPBECipherProvider.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/security/util/crypto/RandomIVPBECipherProvider.java
similarity index 98%
rename from nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/util/crypto/RandomIVPBECipherProvider.java
rename to nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/security/util/crypto/RandomIVPBECipherProvider.java
index 903dfacf9b..66fd1ba070 100644
--- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/util/crypto/RandomIVPBECipherProvider.java
+++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/security/util/crypto/RandomIVPBECipherProvider.java
@@ -14,17 +14,16 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.nifi.processors.standard.util.crypto;
+package org.apache.nifi.security.util.crypto;
-import org.apache.nifi.processor.exception.ProcessException;
-import org.apache.nifi.security.util.EncryptionMethod;
-import org.slf4j.Logger;
-
-import javax.crypto.Cipher;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.nio.charset.StandardCharsets;
+import javax.crypto.Cipher;
+import org.apache.nifi.processor.exception.ProcessException;
+import org.apache.nifi.security.util.EncryptionMethod;
+import org.slf4j.Logger;
public abstract class RandomIVPBECipherProvider implements PBECipherProvider {
static final byte[] SALT_DELIMITER = "NiFiSALT".getBytes(StandardCharsets.UTF_8);
diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/util/crypto/ScryptCipherProvider.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/security/util/crypto/ScryptCipherProvider.java
similarity index 98%
rename from nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/util/crypto/ScryptCipherProvider.java
rename to nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/security/util/crypto/ScryptCipherProvider.java
index 15a29e25e2..b532d8e7a5 100644
--- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/util/crypto/ScryptCipherProvider.java
+++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/security/util/crypto/ScryptCipherProvider.java
@@ -14,27 +14,26 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.nifi.processors.standard.util.crypto;
+package org.apache.nifi.security.util.crypto;
-import org.apache.commons.codec.DecoderException;
-import org.apache.commons.codec.binary.Base64;
-import org.apache.commons.codec.binary.Hex;
-import org.apache.commons.lang3.StringUtils;
-import org.apache.nifi.processor.exception.ProcessException;
-import org.apache.nifi.processors.standard.util.crypto.scrypt.Scrypt;
-import org.apache.nifi.security.util.EncryptionMethod;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import javax.crypto.Cipher;
-import javax.crypto.SecretKey;
-import javax.crypto.spec.SecretKeySpec;
import java.nio.charset.StandardCharsets;
import java.security.SecureRandom;
import java.util.ArrayList;
import java.util.List;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
+import javax.crypto.Cipher;
+import javax.crypto.SecretKey;
+import javax.crypto.spec.SecretKeySpec;
+import org.apache.commons.codec.DecoderException;
+import org.apache.commons.codec.binary.Base64;
+import org.apache.commons.codec.binary.Hex;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.nifi.processor.exception.ProcessException;
+import org.apache.nifi.security.util.EncryptionMethod;
+import org.apache.nifi.security.util.crypto.scrypt.Scrypt;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
public class ScryptCipherProvider extends RandomIVPBECipherProvider {
private static final Logger logger = LoggerFactory.getLogger(ScryptCipherProvider.class);
diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/util/crypto/bcrypt/BCrypt.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/security/util/crypto/bcrypt/BCrypt.java
similarity index 99%
rename from nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/util/crypto/bcrypt/BCrypt.java
rename to nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/security/util/crypto/bcrypt/BCrypt.java
index a57891758e..323fe8f630 100644
--- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/util/crypto/bcrypt/BCrypt.java
+++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/security/util/crypto/bcrypt/BCrypt.java
@@ -12,7 +12,7 @@
// ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
// OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
-package org.apache.nifi.processors.standard.util.crypto.bcrypt;
+package org.apache.nifi.security.util.crypto.bcrypt;
import java.io.UnsupportedEncodingException;
import java.security.SecureRandom;
diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/util/crypto/scrypt/Scrypt.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/security/util/crypto/scrypt/Scrypt.java
similarity index 99%
rename from nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/util/crypto/scrypt/Scrypt.java
rename to nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/security/util/crypto/scrypt/Scrypt.java
index 7785e9eca7..2aeae3deb5 100644
--- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/util/crypto/scrypt/Scrypt.java
+++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/security/util/crypto/scrypt/Scrypt.java
@@ -14,24 +14,23 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.nifi.processors.standard.util.crypto.scrypt;
+package org.apache.nifi.security.util.crypto.scrypt;
-import org.apache.commons.codec.binary.Base64;
-import org.apache.commons.lang3.StringUtils;
-import org.apache.nifi.processors.standard.util.crypto.CipherUtility;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
+import static java.lang.Integer.MAX_VALUE;
+import static java.lang.System.arraycopy;
-import javax.crypto.Mac;
-import javax.crypto.spec.SecretKeySpec;
import java.nio.charset.StandardCharsets;
import java.security.GeneralSecurityException;
import java.security.SecureRandom;
import java.util.ArrayList;
import java.util.List;
-
-import static java.lang.Integer.MAX_VALUE;
-import static java.lang.System.arraycopy;
+import javax.crypto.Mac;
+import javax.crypto.spec.SecretKeySpec;
+import org.apache.commons.codec.binary.Base64;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.nifi.security.util.crypto.CipherUtility;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/groovy/org/apache/nifi/processors/standard/TestEncryptContentGroovy.groovy b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/groovy/org/apache/nifi/processors/standard/TestEncryptContentGroovy.groovy
index e3de6b03d8..f940640fd8 100644
--- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/groovy/org/apache/nifi/processors/standard/TestEncryptContentGroovy.groovy
+++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/groovy/org/apache/nifi/processors/standard/TestEncryptContentGroovy.groovy
@@ -17,10 +17,10 @@
package org.apache.nifi.processors.standard
import org.apache.nifi.components.ValidationResult
-import org.apache.nifi.processors.standard.util.crypto.CipherUtility
-import org.apache.nifi.processors.standard.util.crypto.PasswordBasedEncryptor
import org.apache.nifi.security.util.EncryptionMethod
import org.apache.nifi.security.util.KeyDerivationFunction
+import org.apache.nifi.security.util.crypto.CipherUtility
+import org.apache.nifi.security.util.crypto.PasswordBasedEncryptor
import org.apache.nifi.util.MockFlowFile
import org.apache.nifi.util.MockProcessContext
import org.apache.nifi.util.TestRunner
diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/groovy/org/apache/nifi/processors/standard/util/crypto/BcryptCipherProviderGroovyTest.groovy b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/groovy/org/apache/nifi/security/util/crypto/BcryptCipherProviderGroovyTest.groovy
similarity index 99%
rename from nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/groovy/org/apache/nifi/processors/standard/util/crypto/BcryptCipherProviderGroovyTest.groovy
rename to nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/groovy/org/apache/nifi/security/util/crypto/BcryptCipherProviderGroovyTest.groovy
index 396e3b266e..e5e001fbc6 100644
--- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/groovy/org/apache/nifi/processors/standard/util/crypto/BcryptCipherProviderGroovyTest.groovy
+++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/groovy/org/apache/nifi/security/util/crypto/BcryptCipherProviderGroovyTest.groovy
@@ -14,12 +14,12 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.nifi.processors.standard.util.crypto
+package org.apache.nifi.security.util.crypto
import org.apache.commons.codec.binary.Base64
import org.apache.commons.codec.binary.Hex
-import org.apache.nifi.processors.standard.util.crypto.bcrypt.BCrypt
import org.apache.nifi.security.util.EncryptionMethod
+import org.apache.nifi.security.util.crypto.bcrypt.BCrypt
import org.bouncycastle.jce.provider.BouncyCastleProvider
import org.junit.After
import org.junit.Assume
@@ -30,7 +30,6 @@ import org.junit.Test
import org.junit.runner.RunWith
import org.junit.runners.JUnit4
import org.slf4j.Logger
-
import org.slf4j.LoggerFactory
import javax.crypto.Cipher
diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/groovy/org/apache/nifi/processors/standard/util/crypto/CipherProviderFactoryGroovyTest.groovy b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/groovy/org/apache/nifi/security/util/crypto/CipherProviderFactoryGroovyTest.groovy
similarity index 98%
rename from nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/groovy/org/apache/nifi/processors/standard/util/crypto/CipherProviderFactoryGroovyTest.groovy
rename to nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/groovy/org/apache/nifi/security/util/crypto/CipherProviderFactoryGroovyTest.groovy
index be8d5f48b4..28da9d1095 100644
--- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/groovy/org/apache/nifi/processors/standard/util/crypto/CipherProviderFactoryGroovyTest.groovy
+++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/groovy/org/apache/nifi/security/util/crypto/CipherProviderFactoryGroovyTest.groovy
@@ -14,7 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.nifi.processors.standard.util.crypto
+package org.apache.nifi.security.util.crypto
import org.apache.nifi.security.util.KeyDerivationFunction
import org.bouncycastle.jce.provider.BouncyCastleProvider
diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/groovy/org/apache/nifi/processors/standard/util/crypto/KeyedEncryptorGroovyTest.groovy b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/groovy/org/apache/nifi/security/util/crypto/KeyedEncryptorGroovyTest.groovy
similarity index 98%
rename from nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/groovy/org/apache/nifi/processors/standard/util/crypto/KeyedEncryptorGroovyTest.groovy
rename to nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/groovy/org/apache/nifi/security/util/crypto/KeyedEncryptorGroovyTest.groovy
index 8e78778f15..0487af3b91 100644
--- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/groovy/org/apache/nifi/processors/standard/util/crypto/KeyedEncryptorGroovyTest.groovy
+++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/groovy/org/apache/nifi/security/util/crypto/KeyedEncryptorGroovyTest.groovy
@@ -14,7 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.nifi.processors.standard.util.crypto
+package org.apache.nifi.security.util.crypto
import org.apache.commons.codec.binary.Hex
import org.apache.nifi.processor.io.StreamCallback
diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/groovy/org/apache/nifi/processors/standard/util/crypto/NiFiLegacyCipherProviderGroovyTest.groovy b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/groovy/org/apache/nifi/security/util/crypto/NiFiLegacyCipherProviderGroovyTest.groovy
similarity index 98%
rename from nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/groovy/org/apache/nifi/processors/standard/util/crypto/NiFiLegacyCipherProviderGroovyTest.groovy
rename to nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/groovy/org/apache/nifi/security/util/crypto/NiFiLegacyCipherProviderGroovyTest.groovy
index 176e61bf07..2a3d456cd8 100644
--- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/groovy/org/apache/nifi/processors/standard/util/crypto/NiFiLegacyCipherProviderGroovyTest.groovy
+++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/groovy/org/apache/nifi/security/util/crypto/NiFiLegacyCipherProviderGroovyTest.groovy
@@ -14,12 +14,17 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.nifi.processors.standard.util.crypto
+package org.apache.nifi.security.util.crypto
import org.apache.commons.codec.binary.Hex
import org.apache.nifi.security.util.EncryptionMethod
import org.bouncycastle.jce.provider.BouncyCastleProvider
-import org.junit.*
+import org.junit.After
+import org.junit.Assume
+import org.junit.Before
+import org.junit.BeforeClass
+import org.junit.Ignore
+import org.junit.Test
import org.junit.runner.RunWith
import org.junit.runners.JUnit4
import org.slf4j.Logger
diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/groovy/org/apache/nifi/processors/standard/util/crypto/OpenSSLPKCS5CipherProviderGroovyTest.groovy b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/groovy/org/apache/nifi/security/util/crypto/OpenSSLPKCS5CipherProviderGroovyTest.groovy
similarity index 98%
rename from nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/groovy/org/apache/nifi/processors/standard/util/crypto/OpenSSLPKCS5CipherProviderGroovyTest.groovy
rename to nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/groovy/org/apache/nifi/security/util/crypto/OpenSSLPKCS5CipherProviderGroovyTest.groovy
index 31cbd5a6b5..62b7970896 100644
--- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/groovy/org/apache/nifi/processors/standard/util/crypto/OpenSSLPKCS5CipherProviderGroovyTest.groovy
+++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/groovy/org/apache/nifi/security/util/crypto/OpenSSLPKCS5CipherProviderGroovyTest.groovy
@@ -14,12 +14,16 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.nifi.processors.standard.util.crypto
+package org.apache.nifi.security.util.crypto
import org.apache.commons.codec.binary.Hex
import org.apache.nifi.security.util.EncryptionMethod
import org.bouncycastle.jce.provider.BouncyCastleProvider
-import org.junit.*
+import org.junit.After
+import org.junit.Assume
+import org.junit.Before
+import org.junit.BeforeClass
+import org.junit.Test
import org.junit.runner.RunWith
import org.junit.runners.JUnit4
import org.slf4j.Logger
diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/groovy/org/apache/nifi/processors/standard/util/crypto/PBKDF2CipherProviderGroovyTest.groovy b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/groovy/org/apache/nifi/security/util/crypto/PBKDF2CipherProviderGroovyTest.groovy
similarity index 99%
rename from nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/groovy/org/apache/nifi/processors/standard/util/crypto/PBKDF2CipherProviderGroovyTest.groovy
rename to nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/groovy/org/apache/nifi/security/util/crypto/PBKDF2CipherProviderGroovyTest.groovy
index dfcd0dafea..51a81772a5 100644
--- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/groovy/org/apache/nifi/processors/standard/util/crypto/PBKDF2CipherProviderGroovyTest.groovy
+++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/groovy/org/apache/nifi/security/util/crypto/PBKDF2CipherProviderGroovyTest.groovy
@@ -14,12 +14,17 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.nifi.processors.standard.util.crypto
+package org.apache.nifi.security.util.crypto
import org.apache.commons.codec.binary.Hex
import org.apache.nifi.security.util.EncryptionMethod
import org.bouncycastle.jce.provider.BouncyCastleProvider
-import org.junit.*
+import org.junit.After
+import org.junit.Assume
+import org.junit.Before
+import org.junit.BeforeClass
+import org.junit.Ignore
+import org.junit.Test
import org.junit.runner.RunWith
import org.junit.runners.JUnit4
import org.slf4j.Logger
diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/groovy/org/apache/nifi/processors/standard/util/crypto/PasswordBasedEncryptorGroovyTest.groovy b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/groovy/org/apache/nifi/security/util/crypto/PasswordBasedEncryptorGroovyTest.groovy
similarity index 99%
rename from nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/groovy/org/apache/nifi/processors/standard/util/crypto/PasswordBasedEncryptorGroovyTest.groovy
rename to nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/groovy/org/apache/nifi/security/util/crypto/PasswordBasedEncryptorGroovyTest.groovy
index 35c20e58c3..7008381f6c 100644
--- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/groovy/org/apache/nifi/processors/standard/util/crypto/PasswordBasedEncryptorGroovyTest.groovy
+++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/groovy/org/apache/nifi/security/util/crypto/PasswordBasedEncryptorGroovyTest.groovy
@@ -14,7 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.nifi.processors.standard.util.crypto
+package org.apache.nifi.security.util.crypto
import org.apache.commons.codec.binary.Hex
import org.apache.nifi.processor.io.StreamCallback
diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/groovy/org/apache/nifi/processors/standard/util/crypto/ScryptCipherProviderGroovyTest.groovy b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/groovy/org/apache/nifi/security/util/crypto/ScryptCipherProviderGroovyTest.groovy
similarity index 99%
rename from nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/groovy/org/apache/nifi/processors/standard/util/crypto/ScryptCipherProviderGroovyTest.groovy
rename to nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/groovy/org/apache/nifi/security/util/crypto/ScryptCipherProviderGroovyTest.groovy
index 8fd04559b5..8fce9f2ca7 100644
--- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/groovy/org/apache/nifi/processors/standard/util/crypto/ScryptCipherProviderGroovyTest.groovy
+++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/groovy/org/apache/nifi/security/util/crypto/ScryptCipherProviderGroovyTest.groovy
@@ -14,12 +14,12 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.nifi.processors.standard.util.crypto
+package org.apache.nifi.security.util.crypto
import org.apache.commons.codec.binary.Base64
import org.apache.commons.codec.binary.Hex
-import org.apache.nifi.processors.standard.util.crypto.scrypt.Scrypt
import org.apache.nifi.security.util.EncryptionMethod
+import org.apache.nifi.security.util.crypto.scrypt.Scrypt
import org.bouncycastle.jce.provider.BouncyCastleProvider
import org.junit.After
import org.junit.Assume
diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/groovy/org/apache/nifi/processors/standard/util/crypto/scrypt/ScryptGroovyTest.groovy b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/groovy/org/apache/nifi/security/util/crypto/scrypt/ScryptGroovyTest.groovy
similarity index 99%
rename from nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/groovy/org/apache/nifi/processors/standard/util/crypto/scrypt/ScryptGroovyTest.groovy
rename to nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/groovy/org/apache/nifi/security/util/crypto/scrypt/ScryptGroovyTest.groovy
index c154a1fff4..da34c494fb 100644
--- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/groovy/org/apache/nifi/processors/standard/util/crypto/scrypt/ScryptGroovyTest.groovy
+++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/groovy/org/apache/nifi/security/util/crypto/scrypt/ScryptGroovyTest.groovy
@@ -14,7 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.nifi.processors.standard.util.crypto.scrypt
+package org.apache.nifi.security.util.crypto.scrypt
import org.apache.commons.codec.binary.Hex
import org.bouncycastle.jce.provider.BouncyCastleProvider
diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestEncryptContent.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestEncryptContent.java
index 3166bd0714..063652b8c3 100644
--- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestEncryptContent.java
+++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestEncryptContent.java
@@ -16,12 +16,17 @@
*/
package org.apache.nifi.processors.standard;
+import java.io.File;
+import java.io.IOException;
+import java.nio.file.Paths;
+import java.security.Security;
+import java.util.Collection;
import org.apache.commons.codec.binary.Hex;
import org.apache.nifi.components.ValidationResult;
-import org.apache.nifi.processors.standard.util.crypto.CipherUtility;
-import org.apache.nifi.processors.standard.util.crypto.PasswordBasedEncryptor;
import org.apache.nifi.security.util.EncryptionMethod;
import org.apache.nifi.security.util.KeyDerivationFunction;
+import org.apache.nifi.security.util.crypto.CipherUtility;
+import org.apache.nifi.security.util.crypto.PasswordBasedEncryptor;
import org.apache.nifi.util.MockFlowFile;
import org.apache.nifi.util.MockProcessContext;
import org.apache.nifi.util.TestRunner;
@@ -34,12 +39,6 @@ import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.io.File;
-import java.io.IOException;
-import java.nio.file.Paths;
-import java.security.Security;
-import java.util.Collection;
-
public class TestEncryptContent {
private static final Logger logger = LoggerFactory.getLogger(TestEncryptContent.class);
diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/util/crypto/OpenPGPKeyBasedEncryptorTest.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/security/util/crypto/OpenPGPKeyBasedEncryptorTest.java
similarity index 98%
rename from nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/util/crypto/OpenPGPKeyBasedEncryptorTest.java
rename to nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/security/util/crypto/OpenPGPKeyBasedEncryptorTest.java
index b4cd2e3972..c823e616d4 100644
--- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/util/crypto/OpenPGPKeyBasedEncryptorTest.java
+++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/security/util/crypto/OpenPGPKeyBasedEncryptorTest.java
@@ -14,8 +14,17 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.nifi.processors.standard.util.crypto;
+package org.apache.nifi.security.util.crypto;
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.nio.file.Files;
+import java.nio.file.Paths;
+import java.security.Security;
import org.apache.commons.codec.binary.Hex;
import org.apache.nifi.processor.io.StreamCallback;
import org.apache.nifi.security.util.EncryptionMethod;
@@ -28,16 +37,6 @@ import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.io.ByteArrayInputStream;
-import java.io.ByteArrayOutputStream;
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.InputStream;
-import java.io.OutputStream;
-import java.nio.file.Files;
-import java.nio.file.Paths;
-import java.security.Security;
-
public class OpenPGPKeyBasedEncryptorTest {
private static final Logger logger = LoggerFactory.getLogger(OpenPGPKeyBasedEncryptorTest.class);
diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/util/crypto/OpenPGPPasswordBasedEncryptorTest.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/security/util/crypto/OpenPGPPasswordBasedEncryptorTest.java
similarity index 98%
rename from nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/util/crypto/OpenPGPPasswordBasedEncryptorTest.java
rename to nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/security/util/crypto/OpenPGPPasswordBasedEncryptorTest.java
index 5698ea9f7e..2e1cd5f367 100644
--- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/util/crypto/OpenPGPPasswordBasedEncryptorTest.java
+++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/security/util/crypto/OpenPGPPasswordBasedEncryptorTest.java
@@ -14,8 +14,17 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.nifi.processors.standard.util.crypto;
+package org.apache.nifi.security.util.crypto;
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.nio.file.Files;
+import java.nio.file.Paths;
+import java.security.Security;
import org.apache.commons.codec.binary.Hex;
import org.apache.nifi.processor.io.StreamCallback;
import org.apache.nifi.security.util.EncryptionMethod;
@@ -28,16 +37,6 @@ import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.io.ByteArrayInputStream;
-import java.io.ByteArrayOutputStream;
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.InputStream;
-import java.io.OutputStream;
-import java.nio.file.Files;
-import java.nio.file.Paths;
-import java.security.Security;
-
public class OpenPGPPasswordBasedEncryptorTest {
private static final Logger logger = LoggerFactory.getLogger(OpenPGPPasswordBasedEncryptorTest.class);
diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/resources/logback-test.xml b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/resources/logback-test.xml
index 15e9255884..fad7cb3611 100644
--- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/resources/logback-test.xml
+++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/resources/logback-test.xml
@@ -39,7 +39,7 @@
-
+