From 7d242076ce1d0a555e2ea68d88f29b7461d37b9a Mon Sep 17 00:00:00 2001 From: Andy LoPresto Date: Mon, 13 Mar 2017 21:53:00 -0700 Subject: [PATCH] NIFI-3594 Implemented encrypted provenance repository. Added src/test/resources/logback-test.xml files resetting log level from DEBUG (in nifi-data-provenance-utils) to WARN because later tests depend on MockComponentLog recording a certain number of messages and this number is different than expected if the log level is DEBUG. This closes #1686. Signed-off-by: Bryan Bende, Yolanda M. Davis, and Mark Payne --- .../provenance/ProvenanceEventRecord.java | 15 +- .../nifi-data-provenance-utils/pom.xml | 16 + .../AESProvenanceEventEncryptor.java | 237 ++++++++++ .../apache/nifi/provenance/CryptoUtils.java | 248 ++++++++++ .../nifi/provenance/EncryptionException.java | 92 ++++ .../nifi/provenance/EncryptionMetadata.java | 55 +++ .../nifi/provenance/FileBasedKeyProvider.java | 67 +++ .../apache/nifi/provenance/KeyProvider.java | 60 +++ .../PlaceholderProvenanceEvent.java | 10 + .../provenance/ProvenanceEventEncryptor.java | 59 +++ .../StandardProvenanceEventRecord.java | 41 +- .../nifi/provenance/StaticKeyProvider.java | 96 ++++ .../AESProvenanceEventEncryptorTest.groovy | 303 ++++++++++++ .../nifi/provenance/CryptoUtilsTest.groovy | 436 ++++++++++++++++++ .../provenance/EncryptionExceptionTest.java | 27 ++ .../src/test/resources/logback-test.xml | 32 ++ .../org/apache/nifi/util/NiFiProperties.java | 74 ++- nifi-commons/nifi-security-utils/pom.xml | 4 + .../nifi/security/util/EncryptionMethod.java | 13 +- .../util/crypto/AESKeyedCipherProvider.java | 21 +- .../security}/util/crypto/CipherProvider.java | 2 +- .../security}/util/crypto/CipherUtility.java | 17 +- .../util/crypto/KeyedCipherProvider.java | 11 +- .../AESKeyedCipherProviderGroovyTest.groovy | 45 +- .../crypto/CipherUtilityGroovyTest.groovy | 2 +- .../nifi/controller/FlowController.java | 72 ++- .../properties/ProtectedNiFiProperties.java | 24 +- .../AESSensitivePropertyProviderTest.groovy | 44 +- .../ProtectedNiFiPropertiesGroovyTest.groovy | 73 +-- .../StandardNiFiPropertiesGroovyTest.groovy | 215 ++++++++- ..._with_additional_sensitive_keys.properties | 2 +- .../nifi-framework/nifi-resources/pom.xml | 5 + .../src/main/resources/conf/nifi.properties | 5 + .../src/test/resources/logback-test.xml | 32 ++ .../src/test/resources/logback-test.xml | 32 ++ .../pom.xml | 5 + .../EncryptedSchemaRecordReader.java | 154 +++++++ .../EncryptedSchemaRecordWriter.java | 199 ++++++++ ...cryptedWriteAheadProvenanceRepository.java | 159 +++++++ .../EventIdFirstSchemaRecordReader.java | 55 ++- .../EventIdFirstSchemaRecordWriter.java | 43 +- .../provenance/RepositoryConfiguration.java | 66 ++- .../WriteAheadProvenanceRepository.java | 17 +- .../index/lucene/LuceneEventIndex.java | 17 +- .../provenance/schema/EventFieldNames.java | 8 + .../schema/LookupTableEventRecordFields.java | 6 +- .../schema/LookupTableEventSchema.java | 2 - .../CompressableRecordReader.java | 3 +- .../serialization/RecordReaders.java | 62 ++- .../provenance/util/StorageSummaryEvent.java | 11 +- ...pache.nifi.provenance.ProvenanceRepository | 3 +- ...cryptedSchemaRecordReaderWriterTest.groovy | 281 +++++++++++ ...dWriteAheadProvenanceRepositoryTest.groovy | 391 ++++++++++++++++ .../AbstractTestRecordReaderWriter.java | 30 +- .../VolatileProvenanceRepository.java | 65 +-- .../nifi-standard-processors/pom.xml | 2 +- .../processors/standard/EncryptContent.java | 31 +- .../util/crypto/BcryptCipherProvider.java | 21 +- .../util/crypto/CipherProviderFactory.java | 7 +- .../util/crypto/KeyedEncryptor.java | 17 +- .../util/crypto/NiFiLegacyCipherProvider.java | 15 +- .../util/crypto/OpenPGPKeyBasedEncryptor.java | 27 +- .../crypto/OpenPGPPasswordBasedEncryptor.java | 13 +- .../crypto/OpenSSLPKCS5CipherProvider.java | 27 +- .../util/crypto/PBECipherProvider.java | 7 +- .../util/crypto/PBKDF2CipherProvider.java | 13 +- .../util/crypto/PasswordBasedEncryptor.java | 17 +- .../crypto/RandomIVPBECipherProvider.java | 11 +- .../util/crypto/ScryptCipherProvider.java | 27 +- .../util/crypto/bcrypt/BCrypt.java | 2 +- .../util/crypto/scrypt/Scrypt.java | 21 +- .../standard/TestEncryptContentGroovy.groovy | 4 +- .../BcryptCipherProviderGroovyTest.groovy | 5 +- .../CipherProviderFactoryGroovyTest.groovy | 2 +- .../crypto/KeyedEncryptorGroovyTest.groovy | 2 +- .../NiFiLegacyCipherProviderGroovyTest.groovy | 9 +- ...penSSLPKCS5CipherProviderGroovyTest.groovy | 8 +- .../PBKDF2CipherProviderGroovyTest.groovy | 9 +- .../PasswordBasedEncryptorGroovyTest.groovy | 2 +- .../ScryptCipherProviderGroovyTest.groovy | 4 +- .../crypto/scrypt/ScryptGroovyTest.groovy | 2 +- .../standard/TestEncryptContent.java | 15 +- .../crypto/OpenPGPKeyBasedEncryptorTest.java | 21 +- .../OpenPGPPasswordBasedEncryptorTest.java | 21 +- .../src/test/resources/logback-test.xml | 2 +- 85 files changed, 3964 insertions(+), 464 deletions(-) create mode 100644 nifi-commons/nifi-data-provenance-utils/src/main/java/org/apache/nifi/provenance/AESProvenanceEventEncryptor.java create mode 100644 nifi-commons/nifi-data-provenance-utils/src/main/java/org/apache/nifi/provenance/CryptoUtils.java create mode 100644 nifi-commons/nifi-data-provenance-utils/src/main/java/org/apache/nifi/provenance/EncryptionException.java create mode 100644 nifi-commons/nifi-data-provenance-utils/src/main/java/org/apache/nifi/provenance/EncryptionMetadata.java create mode 100644 nifi-commons/nifi-data-provenance-utils/src/main/java/org/apache/nifi/provenance/FileBasedKeyProvider.java create mode 100644 nifi-commons/nifi-data-provenance-utils/src/main/java/org/apache/nifi/provenance/KeyProvider.java create mode 100644 nifi-commons/nifi-data-provenance-utils/src/main/java/org/apache/nifi/provenance/ProvenanceEventEncryptor.java create mode 100644 nifi-commons/nifi-data-provenance-utils/src/main/java/org/apache/nifi/provenance/StaticKeyProvider.java create mode 100644 nifi-commons/nifi-data-provenance-utils/src/test/groovy/org/apache/nifi/provenance/AESProvenanceEventEncryptorTest.groovy create mode 100644 nifi-commons/nifi-data-provenance-utils/src/test/groovy/org/apache/nifi/provenance/CryptoUtilsTest.groovy create mode 100644 nifi-commons/nifi-data-provenance-utils/src/test/java/org/apache/nifi/provenance/EncryptionExceptionTest.java create mode 100644 nifi-commons/nifi-data-provenance-utils/src/test/resources/logback-test.xml rename {nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard => nifi-commons/nifi-security-utils/src/main/java/org/apache/nifi/security}/util/crypto/AESKeyedCipherProvider.java (99%) rename {nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard => nifi-commons/nifi-security-utils/src/main/java/org/apache/nifi/security}/util/crypto/CipherProvider.java (93%) rename {nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard => nifi-commons/nifi-security-utils/src/main/java/org/apache/nifi/security}/util/crypto/CipherUtility.java (99%) rename {nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard => nifi-commons/nifi-security-utils/src/main/java/org/apache/nifi/security}/util/crypto/KeyedCipherProvider.java (98%) rename {nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/groovy/org/apache/nifi/processors/standard => nifi-commons/nifi-security-utils/src/test/groovy/org/apache/nifi/security}/util/crypto/AESKeyedCipherProviderGroovyTest.groovy (89%) rename {nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/groovy/org/apache/nifi/processors/standard => nifi-commons/nifi-security-utils/src/test/groovy/org/apache/nifi/security}/util/crypto/CipherUtilityGroovyTest.groovy (99%) create mode 100644 nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-processors/src/test/resources/logback-test.xml create mode 100644 nifi-nar-bundles/nifi-poi-bundle/nifi-poi-processors/src/test/resources/logback-test.xml create mode 100644 nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/EncryptedSchemaRecordReader.java create mode 100644 nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/EncryptedSchemaRecordWriter.java create mode 100644 nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/EncryptedWriteAheadProvenanceRepository.java create mode 100644 nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/test/groovy/org/apache/nifi/provenance/EncryptedSchemaRecordReaderWriterTest.groovy create mode 100644 nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/test/groovy/org/apache/nifi/provenance/EncryptedWriteAheadProvenanceRepositoryTest.groovy rename nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/{processors/standard => security}/util/crypto/BcryptCipherProvider.java (98%) rename nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/{processors/standard => security}/util/crypto/CipherProviderFactory.java (97%) rename nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/{processors/standard => security}/util/crypto/KeyedEncryptor.java (99%) rename nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/{processors/standard => security}/util/crypto/NiFiLegacyCipherProvider.java (97%) rename nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/{processors/standard => security}/util/crypto/OpenPGPKeyBasedEncryptor.java (99%) rename nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/{processors/standard => security}/util/crypto/OpenPGPPasswordBasedEncryptor.java (99%) rename nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/{processors/standard => security}/util/crypto/OpenSSLPKCS5CipherProvider.java (99%) rename nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/{processors/standard => security}/util/crypto/PBECipherProvider.java (97%) rename nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/{processors/standard => security}/util/crypto/PBKDF2CipherProvider.java (99%) rename nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/{processors/standard => security}/util/crypto/PasswordBasedEncryptor.java (99%) rename nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/{processors/standard => security}/util/crypto/RandomIVPBECipherProvider.java (98%) rename nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/{processors/standard => security}/util/crypto/ScryptCipherProvider.java (98%) rename nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/{processors/standard => security}/util/crypto/bcrypt/BCrypt.java (99%) rename nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/{processors/standard => security}/util/crypto/scrypt/Scrypt.java (99%) rename nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/groovy/org/apache/nifi/{processors/standard => security}/util/crypto/BcryptCipherProviderGroovyTest.groovy (99%) rename nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/groovy/org/apache/nifi/{processors/standard => security}/util/crypto/CipherProviderFactoryGroovyTest.groovy (98%) rename nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/groovy/org/apache/nifi/{processors/standard => security}/util/crypto/KeyedEncryptorGroovyTest.groovy (98%) rename nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/groovy/org/apache/nifi/{processors/standard => security}/util/crypto/NiFiLegacyCipherProviderGroovyTest.groovy (98%) rename nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/groovy/org/apache/nifi/{processors/standard => security}/util/crypto/OpenSSLPKCS5CipherProviderGroovyTest.groovy (98%) rename nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/groovy/org/apache/nifi/{processors/standard => security}/util/crypto/PBKDF2CipherProviderGroovyTest.groovy (99%) rename nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/groovy/org/apache/nifi/{processors/standard => security}/util/crypto/PasswordBasedEncryptorGroovyTest.groovy (99%) rename nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/groovy/org/apache/nifi/{processors/standard => security}/util/crypto/ScryptCipherProviderGroovyTest.groovy (99%) rename nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/groovy/org/apache/nifi/{processors/standard => security}/util/crypto/scrypt/ScryptGroovyTest.groovy (99%) rename nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/{processors/standard => security}/util/crypto/OpenPGPKeyBasedEncryptorTest.java (98%) rename nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/{processors/standard => security}/util/crypto/OpenPGPPasswordBasedEncryptorTest.java (98%) diff --git a/nifi-api/src/main/java/org/apache/nifi/provenance/ProvenanceEventRecord.java b/nifi-api/src/main/java/org/apache/nifi/provenance/ProvenanceEventRecord.java index b05bd85932..51bb76f659 100644 --- a/nifi-api/src/main/java/org/apache/nifi/provenance/ProvenanceEventRecord.java +++ b/nifi-api/src/main/java/org/apache/nifi/provenance/ProvenanceEventRecord.java @@ -188,28 +188,24 @@ public interface ProvenanceEventRecord { * {@link ProvenanceEventType#FORK}, {@link ProvenanceEventType#JOIN}, or * {@link ProvenanceEventType#CLONE}), or if the queue identifier is * unknown, then this method will return null - * */ String getSourceQueueIdentifier(); /** * @return the Section for the Content Claim that this Event refers to, if * any; otherwise, returns null - * */ String getContentClaimSection(); /** * @return the Section for the Content Claim that the FlowFile previously * referenced, if any; otherwise, returns null - * */ String getPreviousContentClaimSection(); /** * @return the Container for the Content Claim that this Event refers to, if * any; otherwise, returns null - * */ String getContentClaimContainer(); @@ -222,28 +218,31 @@ public interface ProvenanceEventRecord { /** * @return the Identifier for the Content Claim that this Event refers to, * if any; otherwise, returns null - * */ String getContentClaimIdentifier(); /** * @return the Identifier for the Content Claim that the FlowFile previously * referenced, if any; otherwise, returns null - * */ String getPreviousContentClaimIdentifier(); /** * @return the offset into the Content Claim at which the FlowFile's content * begins, if any; otherwise, returns null - * */ Long getContentClaimOffset(); /** * @return the offset into the Content Claim at which the FlowFile's * previous content began, if any; otherwise, returns null - * */ Long getPreviousContentClaimOffset(); + + /** + * Returns the best event identifier for this event (eventId if available, descriptive identifier if not yet persisted to allow for traceability). + * + * @return a descriptive event ID to allow tracing + */ + String getBestEventIdentifier(); } diff --git a/nifi-commons/nifi-data-provenance-utils/pom.xml b/nifi-commons/nifi-data-provenance-utils/pom.xml index 6207646b33..86b25b0d78 100644 --- a/nifi-commons/nifi-data-provenance-utils/pom.xml +++ b/nifi-commons/nifi-data-provenance-utils/pom.xml @@ -34,5 +34,21 @@ org.apache.nifi nifi-utils + + org.apache.nifi + nifi-security-utils + + + org.bouncycastle + bcprov-jdk15on + + + org.apache.nifi + nifi-properties-loader + + + org.apache.nifi + nifi-properties + diff --git a/nifi-commons/nifi-data-provenance-utils/src/main/java/org/apache/nifi/provenance/AESProvenanceEventEncryptor.java b/nifi-commons/nifi-data-provenance-utils/src/main/java/org/apache/nifi/provenance/AESProvenanceEventEncryptor.java new file mode 100644 index 0000000000..d2cc9cad86 --- /dev/null +++ b/nifi-commons/nifi-data-provenance-utils/src/main/java/org/apache/nifi/provenance/AESProvenanceEventEncryptor.java @@ -0,0 +1,237 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.provenance; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.ObjectInputStream; +import java.io.ObjectOutputStream; +import java.security.KeyManagementException; +import java.security.SecureRandom; +import java.security.Security; +import java.util.Arrays; +import java.util.List; +import javax.crypto.BadPaddingException; +import javax.crypto.Cipher; +import javax.crypto.IllegalBlockSizeException; +import javax.crypto.SecretKey; +import org.apache.commons.lang3.StringUtils; +import org.apache.nifi.security.util.EncryptionMethod; +import org.apache.nifi.security.util.crypto.AESKeyedCipherProvider; +import org.bouncycastle.jce.provider.BouncyCastleProvider; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class AESProvenanceEventEncryptor implements ProvenanceEventEncryptor { + private static final Logger logger = LoggerFactory.getLogger(AESProvenanceEventEncryptor.class); + private static final String ALGORITHM = "AES/GCM/NoPadding"; + private static final int IV_LENGTH = 16; + private static final byte[] EMPTY_IV = new byte[IV_LENGTH]; + private static final String VERSION = "v1"; + private static final List SUPPORTED_VERSIONS = Arrays.asList(VERSION); + private static final int MIN_METADATA_LENGTH = IV_LENGTH + 3 + 3; // 3 delimiters and 3 non-zero elements + private static final int METADATA_DEFAULT_LENGTH = (20 + ALGORITHM.length() + IV_LENGTH + VERSION.length()) * 2; // Default to twice the expected length + private static final byte[] SENTINEL = new byte[]{0x01}; + + private KeyProvider keyProvider; + + private AESKeyedCipherProvider aesKeyedCipherProvider = new AESKeyedCipherProvider(); + + /** + * Initializes the encryptor with a {@link KeyProvider}. + * + * @param keyProvider the key provider which will be responsible for accessing keys + * @throws KeyManagementException if there is an issue configuring the key provider + */ + @Override + public void initialize(KeyProvider keyProvider) throws KeyManagementException { + this.keyProvider = keyProvider; + + if (this.aesKeyedCipherProvider == null) { + this.aesKeyedCipherProvider = new AESKeyedCipherProvider(); + } + + if (Security.getProvider("BC") == null) { + Security.addProvider(new BouncyCastleProvider()); + } + } + + /** + * Available for dependency injection to override the default {@link AESKeyedCipherProvider} if necessary. + * + * @param cipherProvider the AES cipher provider to use + */ + void setCipherProvider(AESKeyedCipherProvider cipherProvider) { + this.aesKeyedCipherProvider = cipherProvider; + } + + /** + * Encrypts the provided {@link ProvenanceEventRecord}, serialized to a byte[] by the RecordWriter. + * + * @param plainRecord the plain record, serialized to a byte[] + * @param recordId an identifier for this record (eventId, generated, etc.) + * @param keyId the ID of the key to use + * @return the encrypted record + * @throws EncryptionException if there is an issue encrypting this record + */ + @Override + public byte[] encrypt(byte[] plainRecord, String recordId, String keyId) throws EncryptionException { + if (plainRecord == null || CryptoUtils.isEmpty(keyId)) { + throw new EncryptionException("The provenance record and key ID cannot be missing"); + } + + if (keyProvider == null || !keyProvider.keyExists(keyId)) { + throw new EncryptionException("The requested key ID is not available"); + } else { + byte[] ivBytes = new byte[IV_LENGTH]; + new SecureRandom().nextBytes(ivBytes); + try { + logger.debug("Encrypting provenance record " + recordId + " with key ID " + keyId); + Cipher cipher = initCipher(EncryptionMethod.AES_GCM, Cipher.ENCRYPT_MODE, keyProvider.getKey(keyId), ivBytes); + ivBytes = cipher.getIV(); + + // Perform the actual encryption + byte[] cipherBytes = cipher.doFinal(plainRecord); + + // Serialize and concat encryption details fields (keyId, algo, IV, version, CB length) outside of encryption + EncryptionMetadata metadata = new EncryptionMetadata(keyId, ALGORITHM, ivBytes, VERSION, cipherBytes.length); + byte[] serializedEncryptionMetadata = serializeEncryptionMetadata(metadata); + + // Add the sentinel byte of 0x01 + logger.debug("Encrypted provenance event record " + recordId + " with key ID " + keyId); + return CryptoUtils.concatByteArrays(SENTINEL, serializedEncryptionMetadata, cipherBytes); + } catch (EncryptionException | BadPaddingException | IllegalBlockSizeException | IOException | KeyManagementException e) { + final String msg = "Encountered an exception encrypting provenance record " + recordId; + logger.error(msg, e); + throw new EncryptionException(msg, e); + } + } + } + + private byte[] serializeEncryptionMetadata(EncryptionMetadata metadata) throws IOException { + final ByteArrayOutputStream baos = new ByteArrayOutputStream(); + ObjectOutputStream outputStream = new ObjectOutputStream(baos); + outputStream.writeObject(metadata); + outputStream.close(); + return baos.toByteArray(); + } + + private Cipher initCipher(EncryptionMethod method, int mode, SecretKey key, byte[] ivBytes) throws EncryptionException { + try { + if (method == null || key == null || ivBytes == null) { + throw new IllegalArgumentException("Missing critical information"); + } + return aesKeyedCipherProvider.getCipher(method, key, ivBytes, mode == Cipher.ENCRYPT_MODE); + } catch (Exception e) { + logger.error("Encountered an exception initializing the cipher", e); + throw new EncryptionException(e); + } + } + + /** + * Decrypts the provided byte[] (an encrypted record with accompanying metadata). + * + * @param encryptedRecord the encrypted record in byte[] form + * @param recordId an identifier for this record (eventId, generated, etc.) + * @return the decrypted record + * @throws EncryptionException if there is an issue decrypting this record + */ + @Override + public byte[] decrypt(byte[] encryptedRecord, String recordId) throws EncryptionException { + if (encryptedRecord == null) { + throw new EncryptionException("The encrypted provenance record cannot be missing"); + } + + EncryptionMetadata metadata; + try { + metadata = extractEncryptionMetadata(encryptedRecord); + } catch (IOException | ClassNotFoundException e) { + final String msg = "Encountered an error reading the encryption metadata: "; + logger.error(msg, e); + throw new EncryptionException(msg, e); + } + + if (!SUPPORTED_VERSIONS.contains(metadata.version)) { + throw new EncryptionException("The event was encrypted with version " + metadata.version + " which is not in the list of supported versions " + StringUtils.join(SUPPORTED_VERSIONS, ",")); + } + + // TODO: Actually use the version to determine schema, etc. + + if (keyProvider == null || !keyProvider.keyExists(metadata.keyId) || CryptoUtils.isEmpty(metadata.keyId)) { + throw new EncryptionException("The requested key ID " + metadata.keyId + " is not available"); + } else { + try { + logger.debug("Decrypting provenance record " + recordId + " with key ID " + metadata.keyId); + EncryptionMethod method = EncryptionMethod.forAlgorithm(metadata.algorithm); + Cipher cipher = initCipher(method, Cipher.DECRYPT_MODE, keyProvider.getKey(metadata.keyId), metadata.ivBytes); + + // Strip the metadata away to get just the cipher bytes + byte[] cipherBytes = extractCipherBytes(encryptedRecord, metadata); + + // Perform the actual decryption + byte[] plainBytes = cipher.doFinal(cipherBytes); + + logger.debug("Decrypted provenance event record " + recordId + " with key ID " + metadata.keyId); + return plainBytes; + } catch (EncryptionException | BadPaddingException | IllegalBlockSizeException | KeyManagementException e) { + final String msg = "Encountered an exception decrypting provenance record " + recordId; + logger.error(msg, e); + throw new EncryptionException(msg, e); + } + } + } + + /** + * Returns a valid key identifier for this encryptor (valid for encryption and decryption) or throws an exception if none are available. + * + * @return the key ID + * @throws KeyManagementException if no available key IDs are valid for both operations + */ + @Override + public String getNextKeyId() throws KeyManagementException { + if (keyProvider != null) { + List availableKeyIds = keyProvider.getAvailableKeyIds(); + if (!availableKeyIds.isEmpty()) { + return availableKeyIds.get(0); + } + } + throw new KeyManagementException("No available key IDs"); + } + + private EncryptionMetadata extractEncryptionMetadata(byte[] encryptedRecord) throws EncryptionException, IOException, ClassNotFoundException { + if (encryptedRecord == null || encryptedRecord.length < MIN_METADATA_LENGTH) { + throw new EncryptionException("The encrypted record is too short to contain the metadata"); + } + + // Skip the first byte (SENTINEL) and don't need to copy all the serialized record + ByteArrayInputStream bais = new ByteArrayInputStream(encryptedRecord); + bais.read(); + try (ObjectInputStream ois = new ObjectInputStream(bais)) { + return (EncryptionMetadata) ois.readObject(); + } + } + + private byte[] extractCipherBytes(byte[] encryptedRecord, EncryptionMetadata metadata) { + return Arrays.copyOfRange(encryptedRecord, encryptedRecord.length - metadata.cipherByteLength, encryptedRecord.length); + } + + @Override + public String toString() { + return "AES Provenance Event Encryptor with Key Provider: " + keyProvider.toString(); + } +} diff --git a/nifi-commons/nifi-data-provenance-utils/src/main/java/org/apache/nifi/provenance/CryptoUtils.java b/nifi-commons/nifi-data-provenance-utils/src/main/java/org/apache/nifi/provenance/CryptoUtils.java new file mode 100644 index 0000000000..1b8f11ac9f --- /dev/null +++ b/nifi-commons/nifi-data-provenance-utils/src/main/java/org/apache/nifi/provenance/CryptoUtils.java @@ -0,0 +1,248 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.provenance; + +import java.io.BufferedReader; +import java.io.File; +import java.io.FileReader; +import java.io.IOException; +import java.security.KeyManagementException; +import java.security.NoSuchAlgorithmException; +import java.util.Arrays; +import java.util.Base64; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.regex.Pattern; +import javax.crypto.BadPaddingException; +import javax.crypto.Cipher; +import javax.crypto.IllegalBlockSizeException; +import javax.crypto.SecretKey; +import javax.crypto.spec.SecretKeySpec; +import org.apache.commons.lang3.StringUtils; +import org.apache.nifi.security.util.EncryptionMethod; +import org.apache.nifi.security.util.crypto.AESKeyedCipherProvider; +import org.apache.nifi.util.NiFiProperties; +import org.bouncycastle.util.encoders.Hex; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class CryptoUtils { + private static final Logger logger = LoggerFactory.getLogger(StaticKeyProvider.class); + private static final String STATIC_KEY_PROVIDER_CLASS_NAME = "org.apache.nifi.provenance.StaticKeyProvider"; + private static final String FILE_BASED_KEY_PROVIDER_CLASS_NAME = "org.apache.nifi.provenance.FileBasedKeyProvider"; + private static final Pattern HEX_PATTERN = Pattern.compile("(?i)^[0-9a-f]+$"); + + private static final List UNLIMITED_KEY_LENGTHS = Arrays.asList(32, 48, 64); + + public static final int IV_LENGTH = 16; + + public static boolean isUnlimitedStrengthCryptoAvailable() { + try { + return Cipher.getMaxAllowedKeyLength("AES") > 128; + } catch (NoSuchAlgorithmException e) { + logger.warn("Tried to determine if unlimited strength crypto is available but the AES algorithm is not available"); + return false; + } + } + + /** + * Utility method which returns true if the string is null, empty, or entirely whitespace. + * + * @param src the string to evaluate + * @return true if empty + */ + public static boolean isEmpty(String src) { + return src == null || src.trim().isEmpty(); + } + + /** + * Concatenates multiple byte[] into a single byte[]. + * + * @param arrays the component byte[] in order + * @return a concatenated byte[] + * @throws IOException this should never be thrown + */ + public static byte[] concatByteArrays(byte[]... arrays) throws IOException { + int totalByteLength = 0; + for (byte[] bytes : arrays) { + totalByteLength += bytes.length; + } + byte[] totalBytes = new byte[totalByteLength]; + int currentLength = 0; + for (byte[] bytes : arrays) { + System.arraycopy(bytes, 0, totalBytes, currentLength, bytes.length); + currentLength += bytes.length; + } + return totalBytes; + } + + /** + * Returns true if the provided configuration values successfully define the specified {@link KeyProvider}. + * + * @param keyProviderImplementation the FQ class name of the {@link KeyProvider} implementation + * @param keyProviderLocation the location of the definition (for {@link FileBasedKeyProvider}, etc.) + * @param keyId the active key ID + * @param encryptionKeys a map of key IDs to key material in hex format + * @return true if the provided configuration is valid + */ + public static boolean isValidKeyProvider(String keyProviderImplementation, String keyProviderLocation, String keyId, Map encryptionKeys) { + if (STATIC_KEY_PROVIDER_CLASS_NAME.equals(keyProviderImplementation)) { + // Ensure the keyId and key(s) are valid + if (encryptionKeys == null) { + return false; + } else { + boolean everyKeyValid = encryptionKeys.values().stream().allMatch(CryptoUtils::keyIsValid); + return everyKeyValid && StringUtils.isNotEmpty(keyId); + } + } else if (FILE_BASED_KEY_PROVIDER_CLASS_NAME.equals(keyProviderImplementation)) { + // Ensure the file can be read and the keyId is populated (does not read file to validate) + final File kpf = new File(keyProviderLocation); + return kpf.exists() && kpf.canRead() && StringUtils.isNotEmpty(keyId); + } else { + logger.error("The attempt to validate the key provider failed keyProviderImplementation = " + + keyProviderImplementation + " , keyProviderLocation = " + + keyProviderLocation + " , keyId = " + + keyId + " , encryptionKeys = " + + ((encryptionKeys == null) ? "0" : encryptionKeys.size())); + + return false; + } + } + + /** + * Returns true if the provided key is valid hex and is the correct length for the current system's JCE policies. + * + * @param encryptionKeyHex the key in hexadecimal + * @return true if this key is valid + */ + public static boolean keyIsValid(String encryptionKeyHex) { + return isHexString(encryptionKeyHex) + && (isUnlimitedStrengthCryptoAvailable() + ? UNLIMITED_KEY_LENGTHS.contains(encryptionKeyHex.length()) + : encryptionKeyHex.length() == 32); + } + + /** + * Returns true if the input is valid hexadecimal (does not enforce length and is case-insensitive). + * + * @param hexString the string to evaluate + * @return true if the string is valid hex + */ + public static boolean isHexString(String hexString) { + return StringUtils.isNotEmpty(hexString) && HEX_PATTERN.matcher(hexString).matches(); + } + + /** + * Returns a {@link SecretKey} formed from the hexadecimal key bytes (validity is checked). + * + * @param keyHex the key in hex form + * @return the SecretKey + */ + public static SecretKey formKeyFromHex(String keyHex) throws KeyManagementException { + if (keyIsValid(keyHex)) { + return new SecretKeySpec(Hex.decode(keyHex), "AES"); + } else { + throw new KeyManagementException("The provided key material is not valid"); + } + } + + /** + * Returns a map containing the key IDs and the parsed key from a key provider definition file. + * The values in the file are decrypted using the master key provided. If the file is missing or empty, + * cannot be read, or if no valid keys are read, a {@link KeyManagementException} will be thrown. + * + * @param filepath the key definition file path + * @param masterKey the master key used to decrypt each key definition + * @return a Map of key IDs to SecretKeys + * @throws KeyManagementException if the file is missing or invalid + */ + public static Map readKeys(String filepath, SecretKey masterKey) throws KeyManagementException { + Map keys = new HashMap<>(); + + if (StringUtils.isBlank(filepath)) { + throw new KeyManagementException("The key provider file is not present and readable"); + } + File file = new File(filepath); + if (!file.exists() || !file.canRead()) { + throw new KeyManagementException("The key provider file is not present and readable"); + } + + try (BufferedReader br = new BufferedReader(new FileReader(file))) { + AESKeyedCipherProvider masterCipherProvider = new AESKeyedCipherProvider(); + + String line; + int l = 1; + while ((line = br.readLine()) != null) { + String[] components = line.split("=", 2); + if (components.length != 2 || StringUtils.isAnyEmpty(components)) { + logger.warn("Line " + l + " is not properly formatted -- keyId=Base64EncodedKey..."); + } + String keyId = components[0]; + if (StringUtils.isNotEmpty(keyId)) { + try { + byte[] base64Bytes = Base64.getDecoder().decode(components[1]); + byte[] ivBytes = Arrays.copyOfRange(base64Bytes, 0, IV_LENGTH); + + Cipher masterCipher = null; + try { + masterCipher = masterCipherProvider.getCipher(EncryptionMethod.AES_GCM, masterKey, ivBytes, false); + } catch (Exception e) { + throw new KeyManagementException("Error building cipher to decrypt FileBaseKeyProvider definition at " + filepath, e); + } + byte[] individualKeyBytes = masterCipher.doFinal(Arrays.copyOfRange(base64Bytes, IV_LENGTH, base64Bytes.length)); + + SecretKey key = new SecretKeySpec(individualKeyBytes, "AES"); + logger.debug("Read and decrypted key for " + keyId); + if (keys.containsKey(keyId)) { + logger.warn("Multiple key values defined for " + keyId + " -- using most recent value"); + } + keys.put(keyId, key); + } catch (IllegalArgumentException e) { + logger.error("Encountered an error decoding Base64 for " + keyId + ": " + e.getLocalizedMessage()); + } catch (BadPaddingException | IllegalBlockSizeException e) { + logger.error("Encountered an error decrypting key for " + keyId + ": " + e.getLocalizedMessage()); + } + } + l++; + } + + if (keys.isEmpty()) { + throw new KeyManagementException("The provided file contained no valid keys"); + } + + logger.info("Read " + keys.size() + " keys from FileBasedKeyProvider " + filepath); + return keys; + } catch (IOException e) { + throw new KeyManagementException("Error reading FileBasedKeyProvider definition at " + filepath, e); + } + + } + + public static boolean isProvenanceRepositoryEncryptionConfigured(NiFiProperties niFiProperties) { + final String implementationClassName = niFiProperties.getProperty(NiFiProperties.PROVENANCE_REPO_IMPLEMENTATION_CLASS); + // Referencing EWAPR.class.getName() would require a dependency on the module + boolean encryptedRepo = "org.apache.nifi.provenance.EncryptedWriteAheadProvenanceRepository".equals(implementationClassName); + boolean keyProviderConfigured = isValidKeyProvider( + niFiProperties.getProperty(NiFiProperties.PROVENANCE_REPO_ENCRYPTION_KEY_PROVIDER_IMPLEMENTATION_CLASS), + niFiProperties.getProperty(NiFiProperties.PROVENANCE_REPO_ENCRYPTION_KEY_PROVIDER_LOCATION), + niFiProperties.getProvenanceRepoEncryptionKeyId(), + niFiProperties.getProvenanceRepoEncryptionKeys()); + + return encryptedRepo && keyProviderConfigured; + } +} diff --git a/nifi-commons/nifi-data-provenance-utils/src/main/java/org/apache/nifi/provenance/EncryptionException.java b/nifi-commons/nifi-data-provenance-utils/src/main/java/org/apache/nifi/provenance/EncryptionException.java new file mode 100644 index 0000000000..05c52e5a21 --- /dev/null +++ b/nifi-commons/nifi-data-provenance-utils/src/main/java/org/apache/nifi/provenance/EncryptionException.java @@ -0,0 +1,92 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.provenance; + +public class EncryptionException extends Throwable { + /** + * Constructs a new EncryptionException with {@code null} as its detail message. + * The cause is not initialized, and may subsequently be initialized by a + * call to {@link #initCause}. + *

+ *

The {@link #fillInStackTrace()} method is called to initialize + * the stack trace data in the newly created exception. + */ + public EncryptionException() { + super(); + } + + /** + * Constructs a new EncryptionException with the specified detail message. The + * cause is not initialized, and may subsequently be initialized by + * a call to {@link #initCause}. + *

+ *

The {@link #fillInStackTrace()} method is called to initialize + * the stack trace data in the newly created exception. + * + * @param message the detail message. The detail message is saved for + * later retrieval by the {@link #getMessage()} method. + */ + public EncryptionException(String message) { + super(message); + } + + /** + * Constructs a new EncryptionException with the specified detail message and + * cause.

Note that the detail message associated with + * {@code cause} is not automatically incorporated in + * this exception's detail message. + *

+ *

The {@link #fillInStackTrace()} method is called to initialize + * the stack trace data in the newly created throwable. + * + * @param message the detail message (which is saved for later retrieval + * by the {@link #getMessage()} method). + * @param cause the cause (which is saved for later retrieval by the + * {@link #getCause()} method). (A {@code null} value is + * permitted, and indicates that the cause is nonexistent or + * unknown.) + * @since 1.4 + */ + public EncryptionException(String message, Throwable cause) { + super(message, cause); + } + + /** + * Constructs a new EncryptionException with the specified cause and a detail + * message of {@code (cause==null ? null : cause.toString())} (which + * typically contains the class and detail message of {@code cause}). + * This constructor is useful for exceptions that are little more than + * wrappers for other exceptions. + *

+ *

The {@link #fillInStackTrace()} method is called to initialize + * the stack trace data in the newly created exception. + * + * @param cause the cause (which is saved for later retrieval by the + * {@link #getCause()} method). (A {@code null} value is + * permitted, and indicates that the cause is nonexistent or + * unknown.) + * @since 1.4 + */ + public EncryptionException(Throwable cause) { + super(cause); + } + + @Override + public String toString() { + return "EncryptionException " + getLocalizedMessage(); + } +} diff --git a/nifi-commons/nifi-data-provenance-utils/src/main/java/org/apache/nifi/provenance/EncryptionMetadata.java b/nifi-commons/nifi-data-provenance-utils/src/main/java/org/apache/nifi/provenance/EncryptionMetadata.java new file mode 100644 index 0000000000..0d969daeff --- /dev/null +++ b/nifi-commons/nifi-data-provenance-utils/src/main/java/org/apache/nifi/provenance/EncryptionMetadata.java @@ -0,0 +1,55 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.provenance; + +import java.io.Serializable; +import org.apache.commons.codec.binary.Hex; + +public class EncryptionMetadata implements Serializable { + protected String keyId; + protected String algorithm; + protected byte[] ivBytes; + protected String version; + protected int cipherByteLength; + + EncryptionMetadata() { + } + + EncryptionMetadata(String keyId, String algorithm, byte[] ivBytes, String version, int cipherByteLength) { + this.keyId = keyId; + this.ivBytes = ivBytes; + this.algorithm = algorithm; + this.version = version; + this.cipherByteLength = cipherByteLength; + } + + @Override + public String toString() { + String sb = "AES Provenance Record Encryption Metadata" + + " Key ID: " + + keyId + + " Algorithm: " + + algorithm + + " IV: " + + Hex.encodeHexString(ivBytes) + + " Version: " + + version + + " Cipher text length: " + + cipherByteLength; + return sb; + } +} \ No newline at end of file diff --git a/nifi-commons/nifi-data-provenance-utils/src/main/java/org/apache/nifi/provenance/FileBasedKeyProvider.java b/nifi-commons/nifi-data-provenance-utils/src/main/java/org/apache/nifi/provenance/FileBasedKeyProvider.java new file mode 100644 index 0000000000..b70b3e8fbd --- /dev/null +++ b/nifi-commons/nifi-data-provenance-utils/src/main/java/org/apache/nifi/provenance/FileBasedKeyProvider.java @@ -0,0 +1,67 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.provenance; + +import java.io.IOException; +import java.security.KeyManagementException; +import javax.crypto.SecretKey; +import javax.crypto.spec.SecretKeySpec; +import javax.naming.OperationNotSupportedException; +import org.apache.nifi.properties.NiFiPropertiesLoader; +import org.bouncycastle.util.encoders.Hex; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class FileBasedKeyProvider extends StaticKeyProvider { + private static final Logger logger = LoggerFactory.getLogger(FileBasedKeyProvider.class); + + private String filepath; + + FileBasedKeyProvider(String location) throws KeyManagementException { + this(location, getMasterKey()); + } + + FileBasedKeyProvider(String location, SecretKey masterKey) throws KeyManagementException { + super(CryptoUtils.readKeys(location, masterKey)); + this.filepath = location; + } + + private static SecretKey getMasterKey() throws KeyManagementException { + try { + // Get the master encryption key from bootstrap.conf + String masterKeyHex = NiFiPropertiesLoader.extractKeyFromBootstrapFile(); + return new SecretKeySpec(Hex.decode(masterKeyHex), "AES"); + } catch (IOException e) { + logger.error("Encountered an error: ", e); + throw new KeyManagementException(e); + } + } + + /** + * Adds the key to the provider and associates it with the given ID. Some implementations may not allow this operation. + * + * @param keyId the key identifier + * @param key the key + * @return true if the key was successfully added + * @throws OperationNotSupportedException if this implementation doesn't support adding keys + * @throws KeyManagementException if the key is invalid, the ID conflicts, etc. + */ + @Override + public boolean addKey(String keyId, SecretKey key) throws OperationNotSupportedException, KeyManagementException { + throw new OperationNotSupportedException("This implementation does not allow adding keys. Modify the file backing this provider at " + filepath); + } +} diff --git a/nifi-commons/nifi-data-provenance-utils/src/main/java/org/apache/nifi/provenance/KeyProvider.java b/nifi-commons/nifi-data-provenance-utils/src/main/java/org/apache/nifi/provenance/KeyProvider.java new file mode 100644 index 0000000000..39f6384e11 --- /dev/null +++ b/nifi-commons/nifi-data-provenance-utils/src/main/java/org/apache/nifi/provenance/KeyProvider.java @@ -0,0 +1,60 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.provenance; + +import java.security.KeyManagementException; +import java.util.List; +import javax.crypto.SecretKey; +import javax.naming.OperationNotSupportedException; + +public interface KeyProvider { + + /** + * Returns the key identified by this ID or throws an exception if one is not available. + * + * @param keyId the key identifier + * @return the key + * @throws KeyManagementException if the key cannot be retrieved + */ + SecretKey getKey(String keyId) throws KeyManagementException; + + /** + * Returns true if the key exists and is available. Null or empty IDs will return false. + * + * @param keyId the key identifier + * @return true if the key can be used + */ + boolean keyExists(String keyId); + + /** + * Returns a list of available key identifiers (useful for encryption, as retired keys may not be listed here even if they are available for decryption for legacy/BC reasons). + * + * @return a List of keyIds (empty list if none are available) + */ + List getAvailableKeyIds(); + + /** + * Adds the key to the provider and associates it with the given ID. Some implementations may not allow this operation. + * + * @param keyId the key identifier + * @param key the key + * @return true if the key was successfully added + * @throws OperationNotSupportedException if this implementation doesn't support adding keys + * @throws KeyManagementException if the key is invalid, the ID conflicts, etc. + */ + boolean addKey(String keyId, SecretKey key) throws OperationNotSupportedException, KeyManagementException; +} diff --git a/nifi-commons/nifi-data-provenance-utils/src/main/java/org/apache/nifi/provenance/PlaceholderProvenanceEvent.java b/nifi-commons/nifi-data-provenance-utils/src/main/java/org/apache/nifi/provenance/PlaceholderProvenanceEvent.java index 26696c8b17..5644355b22 100644 --- a/nifi-commons/nifi-data-provenance-utils/src/main/java/org/apache/nifi/provenance/PlaceholderProvenanceEvent.java +++ b/nifi-commons/nifi-data-provenance-utils/src/main/java/org/apache/nifi/provenance/PlaceholderProvenanceEvent.java @@ -187,4 +187,14 @@ public class PlaceholderProvenanceEvent implements ProvenanceEventRecord { public Long getPreviousContentClaimOffset() { return null; } + + /** + * Returns the best event identifier for this event (eventId if available, descriptive identifier if not yet persisted to allow for traceability). + * + * @return a descriptive event ID to allow tracing + */ + @Override + public String getBestEventIdentifier() { + return Long.toString(getEventId()); + } } diff --git a/nifi-commons/nifi-data-provenance-utils/src/main/java/org/apache/nifi/provenance/ProvenanceEventEncryptor.java b/nifi-commons/nifi-data-provenance-utils/src/main/java/org/apache/nifi/provenance/ProvenanceEventEncryptor.java new file mode 100644 index 0000000000..c7690e1703 --- /dev/null +++ b/nifi-commons/nifi-data-provenance-utils/src/main/java/org/apache/nifi/provenance/ProvenanceEventEncryptor.java @@ -0,0 +1,59 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.provenance; + +import java.security.KeyManagementException; + +public interface ProvenanceEventEncryptor { + + /** + * Initializes the encryptor with a {@link KeyProvider}. + * + * @param keyProvider the key provider which will be responsible for accessing keys + * @throws KeyManagementException if there is an issue configuring the key provider + */ + void initialize(KeyProvider keyProvider) throws KeyManagementException; + + /** + * Encrypts the provided {@link ProvenanceEventRecord}, serialized to a byte[] by the RecordWriter. + * + * @param plainRecord the plain record, serialized to a byte[] + * @param recordId an identifier for this record (eventId, generated, etc.) + * @param keyId the ID of the key to use + * @return the encrypted record + * @throws EncryptionException if there is an issue encrypting this record + */ + byte[] encrypt(byte[] plainRecord, String recordId, String keyId) throws EncryptionException; + + /** + * Decrypts the provided byte[] (an encrypted record with accompanying metadata). + * + * @param encryptedRecord the encrypted record in byte[] form + * @param recordId an identifier for this record (eventId, generated, etc.) + * @return the decrypted record + * @throws EncryptionException if there is an issue decrypting this record + */ + byte[] decrypt(byte[] encryptedRecord, String recordId) throws EncryptionException; + + /** + * Returns a valid key identifier for this encryptor (valid for encryption and decryption) or throws an exception if none are available. + * + * @return the key ID + * @throws KeyManagementException if no available key IDs are valid for both operations + */ + String getNextKeyId() throws KeyManagementException; +} diff --git a/nifi-commons/nifi-data-provenance-utils/src/main/java/org/apache/nifi/provenance/StandardProvenanceEventRecord.java b/nifi-commons/nifi-data-provenance-utils/src/main/java/org/apache/nifi/provenance/StandardProvenanceEventRecord.java index ac60d4f680..84e7419a29 100644 --- a/nifi-commons/nifi-data-provenance-utils/src/main/java/org/apache/nifi/provenance/StandardProvenanceEventRecord.java +++ b/nifi-commons/nifi-data-provenance-utils/src/main/java/org/apache/nifi/provenance/StandardProvenanceEventRecord.java @@ -22,7 +22,6 @@ import java.util.Date; import java.util.HashMap; import java.util.List; import java.util.Map; - import org.apache.nifi.flowfile.FlowFile; import org.apache.nifi.flowfile.attributes.CoreAttributes; import org.apache.nifi.processor.Relationship; @@ -30,7 +29,7 @@ import org.apache.nifi.processor.Relationship; /** * Holder for provenance relevant information */ -public final class StandardProvenanceEventRecord implements ProvenanceEventRecord { +public class StandardProvenanceEventRecord implements ProvenanceEventRecord { private final long eventTime; private final long entryDate; @@ -69,7 +68,7 @@ public final class StandardProvenanceEventRecord implements ProvenanceEventRecor private volatile long eventId = -1L; - private StandardProvenanceEventRecord(final Builder builder) { + StandardProvenanceEventRecord(final Builder builder) { this.eventTime = builder.eventTime; this.entryDate = builder.entryDate; this.eventType = builder.eventType; @@ -100,8 +99,8 @@ public final class StandardProvenanceEventRecord implements ProvenanceEventRecor contentClaimOffset = builder.contentClaimOffset; contentSize = builder.contentSize; - previousAttributes = builder.previousAttributes == null ? Collections.emptyMap() : Collections.unmodifiableMap(builder.previousAttributes); - updatedAttributes = builder.updatedAttributes == null ? Collections.emptyMap() : Collections.unmodifiableMap(builder.updatedAttributes); + previousAttributes = builder.previousAttributes == null ? Collections.emptyMap() : Collections.unmodifiableMap(builder.previousAttributes); + updatedAttributes = builder.updatedAttributes == null ? Collections.emptyMap() : Collections.unmodifiableMap(builder.updatedAttributes); sourceQueueIdentifier = builder.sourceQueueIdentifier; @@ -110,6 +109,11 @@ public final class StandardProvenanceEventRecord implements ProvenanceEventRecor } } + public static StandardProvenanceEventRecord copy(StandardProvenanceEventRecord other) { + Builder builder = new Builder().fromEvent(other); + return new StandardProvenanceEventRecord(builder); + } + public String getStorageFilename() { return storageFilename; } @@ -199,12 +203,12 @@ public final class StandardProvenanceEventRecord implements ProvenanceEventRecor @Override public List getParentUuids() { - return parentUuids == null ? Collections.emptyList() : parentUuids; + return parentUuids == null ? Collections.emptyList() : parentUuids; } @Override public List getChildUuids() { - return childrenUuids == null ? Collections.emptyList() : childrenUuids; + return childrenUuids == null ? Collections.emptyList() : childrenUuids; } @Override @@ -299,8 +303,8 @@ public final class StandardProvenanceEventRecord implements ProvenanceEventRecor } return -37423 + 3 * componentId.hashCode() + (transitUri == null ? 0 : 41 * transitUri.hashCode()) - + (relationship == null ? 0 : 47 * relationship.hashCode()) + 44 * eventTypeCode - + 47 * getChildUuids().hashCode() + 47 * getParentUuids().hashCode(); + + (relationship == null ? 0 : 47 * relationship.hashCode()) + 44 * eventTypeCode + + 47 * getChildUuids().hashCode() + 47 * getParentUuids().hashCode(); } @Override @@ -419,6 +423,23 @@ public final class StandardProvenanceEventRecord implements ProvenanceEventRecor + ", alternateIdentifierUri=" + alternateIdentifierUri + "]"; } + /** + * Returns a unique identifier for the record. By default, it uses + * {@link ProvenanceEventRecord#getEventId()} but if it has not been persisted to the + * repository, this is {@code -1}, so it constructs a String of the format + * {@code _on__by__at_}. + * + * @return a String identifying the record for later analysis + */ + @Override + public String getBestEventIdentifier() { + if (getEventId() != -1) { + return Long.toString(getEventId()); + } else { + return getEventType().name() + "_on_" + getFlowFileUuid() + "_by_" + getComponentId() + "_at_" + getEventTime(); + } + } + public static class Builder implements ProvenanceEventBuilder { private long eventTime = System.currentTimeMillis(); @@ -733,7 +754,7 @@ public final class StandardProvenanceEventRecord implements ProvenanceEventRecor public ProvenanceEventBuilder fromFlowFile(final FlowFile flowFile) { setFlowFileEntryDate(flowFile.getEntryDate()); setLineageStartDate(flowFile.getLineageStartDate()); - setAttributes(Collections.emptyMap(), flowFile.getAttributes()); + setAttributes(Collections.emptyMap(), flowFile.getAttributes()); uuid = flowFile.getAttribute(CoreAttributes.UUID.key()); this.contentSize = flowFile.getSize(); return this; diff --git a/nifi-commons/nifi-data-provenance-utils/src/main/java/org/apache/nifi/provenance/StaticKeyProvider.java b/nifi-commons/nifi-data-provenance-utils/src/main/java/org/apache/nifi/provenance/StaticKeyProvider.java new file mode 100644 index 0000000000..e0981dca6b --- /dev/null +++ b/nifi-commons/nifi-data-provenance-utils/src/main/java/org/apache/nifi/provenance/StaticKeyProvider.java @@ -0,0 +1,96 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.provenance; + +import java.security.KeyManagementException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import javax.crypto.SecretKey; +import javax.naming.OperationNotSupportedException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Reference implementation for static key provider (used during tests). + */ +public class StaticKeyProvider implements KeyProvider { + private static final Logger logger = LoggerFactory.getLogger(StaticKeyProvider.class); + + private Map keys = new HashMap<>(); + + StaticKeyProvider(String keyId, String keyHex) throws KeyManagementException { + this.keys.put(keyId, CryptoUtils.formKeyFromHex(keyHex)); + } + + StaticKeyProvider(Map keys) throws KeyManagementException { + this.keys.putAll(keys); + } + + /** + * Returns the key identified by this ID or throws an exception if one is not available. + * + * @param keyId the key identifier + * @return the key + * @throws KeyManagementException if the key cannot be retrieved + */ + @Override + public SecretKey getKey(String keyId) throws KeyManagementException { + logger.debug("Attempting to get key: " + keyId); + if (keyExists(keyId)) { + return keys.get(keyId); + } else { + throw new KeyManagementException("No key available for " + keyId); + } + } + + /** + * Returns true if the key exists and is available. Null or empty IDs will return false. + * + * @param keyId the key identifier + * @return true if the key can be used + */ + @Override + public boolean keyExists(String keyId) { + return keys.containsKey(keyId); + } + + /** + * Returns a singleton list of the available key identifier. + * + * @return a List containing the {@code KEY_ID} + */ + @Override + public List getAvailableKeyIds() { + return new ArrayList<>(keys.keySet()); + } + + /** + * Adds the key to the provider and associates it with the given ID. Some implementations may not allow this operation. + * + * @param keyId the key identifier + * @param key the key + * @return true if the key was successfully added + * @throws OperationNotSupportedException if this implementation doesn't support adding keys + * @throws KeyManagementException if the key is invalid, the ID conflicts, etc. + */ + @Override + public boolean addKey(String keyId, SecretKey key) throws OperationNotSupportedException, KeyManagementException { + throw new OperationNotSupportedException("This implementation does not allow adding keys"); + } +} diff --git a/nifi-commons/nifi-data-provenance-utils/src/test/groovy/org/apache/nifi/provenance/AESProvenanceEventEncryptorTest.groovy b/nifi-commons/nifi-data-provenance-utils/src/test/groovy/org/apache/nifi/provenance/AESProvenanceEventEncryptorTest.groovy new file mode 100644 index 0000000000..61b35d0d31 --- /dev/null +++ b/nifi-commons/nifi-data-provenance-utils/src/test/groovy/org/apache/nifi/provenance/AESProvenanceEventEncryptorTest.groovy @@ -0,0 +1,303 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.provenance + +import org.apache.nifi.security.util.EncryptionMethod +import org.apache.nifi.security.util.crypto.AESKeyedCipherProvider +import org.bouncycastle.jce.provider.BouncyCastleProvider +import org.bouncycastle.util.encoders.Hex +import org.junit.After +import org.junit.AfterClass +import org.junit.Before +import org.junit.BeforeClass +import org.junit.Test +import org.junit.runner.RunWith +import org.junit.runners.JUnit4 +import org.slf4j.Logger +import org.slf4j.LoggerFactory + +import javax.crypto.Cipher +import javax.crypto.SecretKey +import javax.crypto.spec.IvParameterSpec +import javax.crypto.spec.SecretKeySpec +import java.nio.charset.StandardCharsets +import java.security.KeyManagementException +import java.security.SecureRandom +import java.security.Security + +import static groovy.test.GroovyAssert.shouldFail + +@RunWith(JUnit4.class) +class AESProvenanceEventEncryptorTest { + private static final Logger logger = LoggerFactory.getLogger(AESProvenanceEventEncryptorTest.class) + + private static final String KEY_HEX_128 = "0123456789ABCDEFFEDCBA9876543210" + private static final String KEY_HEX_256 = KEY_HEX_128 * 2 + private static final String KEY_HEX = isUnlimitedStrengthCryptoAvailable() ? KEY_HEX_256 : KEY_HEX_128 + + private static KeyProvider mockKeyProvider + private static AESKeyedCipherProvider mockCipherProvider + + private static String ORIGINAL_LOG_LEVEL + + private ProvenanceEventEncryptor encryptor + + @BeforeClass + static void setUpOnce() throws Exception { + ORIGINAL_LOG_LEVEL = System.getProperty("org.slf4j.simpleLogger.log.org.apache.nifi.provenance") + System.setProperty("org.slf4j.simpleLogger.log.org.apache.nifi.provenance", "DEBUG") + + Security.addProvider(new BouncyCastleProvider()) + + logger.metaClass.methodMissing = { String name, args -> + logger.info("[${name?.toUpperCase()}] ${(args as List).join(" ")}") + } + + mockKeyProvider = [ + getKey : { String keyId -> + logger.mock("Requesting key ID: ${keyId}") + new SecretKeySpec(Hex.decode(KEY_HEX), "AES") + }, + keyExists: { String keyId -> + logger.mock("Checking existence of ${keyId}") + true + }] as KeyProvider + + mockCipherProvider = [ + getCipher: { EncryptionMethod em, SecretKey key, byte[] ivBytes, boolean encryptMode -> + logger.mock("Getting cipher for ${em} with IV ${Hex.toHexString(ivBytes)} encrypt ${encryptMode}") + Cipher cipher = Cipher.getInstance("AES/GCM/NoPadding") + cipher.init((encryptMode ? Cipher.ENCRYPT_MODE : Cipher.DECRYPT_MODE) as int, key, new IvParameterSpec(ivBytes)) + cipher + } + ] as AESKeyedCipherProvider + } + + @Before + void setUp() throws Exception { + + } + + @After + void tearDown() throws Exception { + + } + + @AfterClass + static void tearDownOnce() throws Exception { + if (ORIGINAL_LOG_LEVEL) { + System.setProperty("org.slf4j.simpleLogger.log.org.apache.nifi.provenance", ORIGINAL_LOG_LEVEL) + } + } + + private static boolean isUnlimitedStrengthCryptoAvailable() { + Cipher.getMaxAllowedKeyLength("AES") > 128 + } + + /** + * Given arbitrary bytes, encrypt them and persist with the encryption metadata, then recover + */ + @Test + void testShouldEncryptAndDecryptArbitraryBytes() { + // Arrange + final byte[] SERIALIZED_BYTES = "This is a plaintext message.".getBytes(StandardCharsets.UTF_8) + logger.info("Serialized bytes (${SERIALIZED_BYTES.size()}): ${Hex.toHexString(SERIALIZED_BYTES)}") + + encryptor = new AESProvenanceEventEncryptor() + encryptor.initialize(mockKeyProvider) + encryptor.setCipherProvider(mockCipherProvider) + logger.info("Created ${encryptor}") + + String keyId = "K1" + String recordId = "R1" + logger.info("Using record ID ${recordId} and key ID ${keyId}") + + // Act + byte[] metadataAndCipherBytes = encryptor.encrypt(SERIALIZED_BYTES, recordId, keyId) + logger.info("Encrypted data to: \n\t${Hex.toHexString(metadataAndCipherBytes)}") + + byte[] recoveredBytes = encryptor.decrypt(metadataAndCipherBytes, recordId) + logger.info("Decrypted data to: \n\t${Hex.toHexString(recoveredBytes)}") + + // Assert + assert recoveredBytes == SERIALIZED_BYTES + logger.info("Decoded (usually would be serialized schema record): ${new String(recoveredBytes, StandardCharsets.UTF_8)}") + } + + @Test + void testShouldInitializeNullCipherProvider() { + // Arrange + encryptor = new AESProvenanceEventEncryptor() + encryptor.setCipherProvider(null) + assert !encryptor.aesKeyedCipherProvider + + // Act + encryptor.initialize(mockKeyProvider) + logger.info("Created ${encryptor}") + + // Assert + assert encryptor.aesKeyedCipherProvider instanceof AESKeyedCipherProvider + } + + @Test + void testShouldFailOnMissingKeyId() { + // Arrange + final byte[] SERIALIZED_BYTES = "This is a plaintext message.".getBytes(StandardCharsets.UTF_8) + logger.info("Serialized bytes (${SERIALIZED_BYTES.size()}): ${Hex.toHexString(SERIALIZED_BYTES)}") + + KeyProvider emptyKeyProvider = [ + getKey : { String kid -> + throw new KeyManagementException("No key found for ${kid}") + }, + keyExists: { String kid -> false } + ] as KeyProvider + + encryptor = new AESProvenanceEventEncryptor() + encryptor.initialize(emptyKeyProvider) + encryptor.setCipherProvider(mockCipherProvider) + logger.info("Created ${encryptor}") + + String keyId = "K1" + String recordId = "R1" + logger.info("Using record ID ${recordId} and key ID ${keyId}") + + // Act + def msg = shouldFail(EncryptionException) { + byte[] metadataAndCipherBytes = encryptor.encrypt(SERIALIZED_BYTES, recordId, keyId) + logger.info("Encrypted data to: \n\t${Hex.toHexString(metadataAndCipherBytes)}") + } + logger.expected(msg) + + // Assert + assert msg.getMessage() == "The requested key ID is not available" + } + + @Test + void testShouldUseDifferentIVsForSequentialEncryptions() { + // Arrange + final byte[] SERIALIZED_BYTES = "This is a plaintext message.".getBytes(StandardCharsets.UTF_8) + logger.info("Serialized bytes (${SERIALIZED_BYTES.size()}): ${Hex.toHexString(SERIALIZED_BYTES)}") + + encryptor = new AESProvenanceEventEncryptor() + encryptor.initialize(mockKeyProvider) + logger.info("Created ${encryptor}") + + String keyId = "K1" + String recordId1 = "R1" + logger.info("Using record ID ${recordId1} and key ID ${keyId}") + + // Act + byte[] metadataAndCipherBytes1 = encryptor.encrypt(SERIALIZED_BYTES, recordId1, keyId) + logger.info("Encrypted data to: \n\t${Hex.toHexString(metadataAndCipherBytes1)}") + EncryptionMetadata metadata1 = encryptor.extractEncryptionMetadata(metadataAndCipherBytes1) + logger.info("Record ${recordId1} IV: ${Hex.toHexString(metadata1.ivBytes)}") + + byte[] recoveredBytes1 = encryptor.decrypt(metadataAndCipherBytes1, recordId1) + logger.info("Decrypted data to: \n\t${Hex.toHexString(recoveredBytes1)}") + + String recordId2 = "R2" + byte[] metadataAndCipherBytes2 = encryptor.encrypt(SERIALIZED_BYTES, recordId2, keyId) + logger.info("Encrypted data to: \n\t${Hex.toHexString(metadataAndCipherBytes2)}") + EncryptionMetadata metadata2 = encryptor.extractEncryptionMetadata(metadataAndCipherBytes2) + logger.info("Record ${recordId2} IV: ${Hex.toHexString(metadata2.ivBytes)}") + + byte[] recoveredBytes2 = encryptor.decrypt(metadataAndCipherBytes2, recordId2) + logger.info("Decrypted data to: \n\t${Hex.toHexString(recoveredBytes2)}") + + // Assert + assert metadata1.ivBytes != metadata2.ivBytes + + assert recoveredBytes1 == SERIALIZED_BYTES + assert recoveredBytes2 == SERIALIZED_BYTES + } + + @Test + void testShouldFailOnBadMetadata() { + // Arrange + final byte[] SERIALIZED_BYTES = "This is a plaintext message.".getBytes(StandardCharsets.UTF_8) + logger.info("Serialized bytes (${SERIALIZED_BYTES.size()}): ${Hex.toHexString(SERIALIZED_BYTES)}") + + def strictMockKeyProvider = [ + getKey : { String keyId -> + if (keyId != "K1") { + throw new KeyManagementException("No such key") + } + new SecretKeySpec(Hex.decode(KEY_HEX), "AES") + }, + keyExists: { String keyId -> + keyId == "K1" + }] as KeyProvider + + encryptor = new AESProvenanceEventEncryptor() + encryptor.initialize(strictMockKeyProvider) + encryptor.setCipherProvider(mockCipherProvider) + logger.info("Created ${encryptor}") + + String keyId = "K1" + String recordId = "R1" + logger.info("Using record ID ${recordId} and key ID ${keyId}") + + final String ALGORITHM = "AES/GCM/NoPadding" + final byte[] ivBytes = new byte[16] + new SecureRandom().nextBytes(ivBytes) + final String VERSION = "v1" + + // Perform the encryption independently of the encryptor + SecretKey key = mockKeyProvider.getKey(keyId) + Cipher cipher = new AESKeyedCipherProvider().getCipher(EncryptionMethod.AES_GCM, key, ivBytes, true) + byte[] cipherBytes = cipher.doFinal(SERIALIZED_BYTES) + + int cipherBytesLength = cipherBytes.size() + + // Construct accurate metadata + EncryptionMetadata goodMetadata = new EncryptionMetadata(keyId, ALGORITHM, ivBytes, VERSION, cipherBytesLength) + logger.info("Created good encryption metadata: ${goodMetadata}") + + // Construct bad metadata instances + EncryptionMetadata badKeyId = new EncryptionMetadata(keyId.reverse(), ALGORITHM, ivBytes, VERSION, cipherBytesLength) + EncryptionMetadata badAlgorithm = new EncryptionMetadata(keyId, "ASE/GDM/SomePadding", ivBytes, VERSION, cipherBytesLength) + EncryptionMetadata badIV = new EncryptionMetadata(keyId, ALGORITHM, new byte[16], VERSION, cipherBytesLength) + EncryptionMetadata badVersion = new EncryptionMetadata(keyId, ALGORITHM, ivBytes, VERSION.reverse(), cipherBytesLength) + EncryptionMetadata badCBLength = new EncryptionMetadata(keyId, ALGORITHM, ivBytes, VERSION, cipherBytesLength - 5) + + List badMetadata = [badKeyId, badAlgorithm, badIV, badVersion, badCBLength] + + // Form the proper cipherBytes + byte[] completeGoodBytes = CryptoUtils.concatByteArrays([0x01] as byte[], encryptor.serializeEncryptionMetadata(goodMetadata), cipherBytes) + + byte[] recoveredGoodBytes = encryptor.decrypt(completeGoodBytes, recordId) + logger.info("Recovered good bytes: ${Hex.toHexString(recoveredGoodBytes)}") + + final List EXPECTED_MESSAGES = ["The requested key ID (\\w+)? is not available", + "Encountered an exception decrypting provenance record", + "The event was encrypted with version ${VERSION.reverse()} which is not in the list of supported versions v1"] + + // Act + badMetadata.eachWithIndex { EncryptionMetadata metadata, int i -> + byte[] completeBytes = CryptoUtils.concatByteArrays([0x01] as byte[], encryptor.serializeEncryptionMetadata(metadata), cipherBytes) + + def msg = shouldFail(EncryptionException) { + byte[] recoveredBytes = encryptor.decrypt(completeBytes, "R${i + 2}") + logger.info("Recovered bad bytes: ${Hex.toHexString(recoveredBytes)}") + } + logger.expected(msg) + + // Assert + assert EXPECTED_MESSAGES.any { msg.getMessage() =~ it } + } + } +} diff --git a/nifi-commons/nifi-data-provenance-utils/src/test/groovy/org/apache/nifi/provenance/CryptoUtilsTest.groovy b/nifi-commons/nifi-data-provenance-utils/src/test/groovy/org/apache/nifi/provenance/CryptoUtilsTest.groovy new file mode 100644 index 0000000000..162896f83c --- /dev/null +++ b/nifi-commons/nifi-data-provenance-utils/src/test/groovy/org/apache/nifi/provenance/CryptoUtilsTest.groovy @@ -0,0 +1,436 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License") you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.provenance + +import org.bouncycastle.jce.provider.BouncyCastleProvider +import org.bouncycastle.util.encoders.Hex +import org.junit.After +import org.junit.AfterClass +import org.junit.Before +import org.junit.BeforeClass +import org.junit.ClassRule +import org.junit.Test +import org.junit.rules.TemporaryFolder +import org.junit.runner.RunWith +import org.junit.runners.JUnit4 +import org.slf4j.Logger +import org.slf4j.LoggerFactory + +import javax.crypto.Cipher +import javax.crypto.SecretKey +import javax.crypto.spec.IvParameterSpec +import javax.crypto.spec.SecretKeySpec +import java.nio.charset.StandardCharsets +import java.nio.file.Files +import java.nio.file.attribute.PosixFilePermission +import java.security.KeyManagementException +import java.security.SecureRandom +import java.security.Security + +import static groovy.test.GroovyAssert.shouldFail + +@RunWith(JUnit4.class) +class CryptoUtilsTest { + private static final Logger logger = LoggerFactory.getLogger(CryptoUtilsTest.class) + + private static final String KEY_ID = "K1" + private static final String KEY_HEX_128 = "0123456789ABCDEFFEDCBA9876543210" + private static final String KEY_HEX_256 = KEY_HEX_128 * 2 + private static final String KEY_HEX = isUnlimitedStrengthCryptoAvailable() ? KEY_HEX_256 : KEY_HEX_128 + + private static + final Set ALL_POSIX_ATTRS = PosixFilePermission.values() as Set + + @ClassRule + public static TemporaryFolder tempFolder = new TemporaryFolder() + + @BeforeClass + static void setUpOnce() throws Exception { + Security.addProvider(new BouncyCastleProvider()) + + logger.metaClass.methodMissing = { String name, args -> + logger.info("[${name?.toUpperCase()}] ${(args as List).join(" ")}") + } + } + + @Before + void setUp() throws Exception { + tempFolder.create() + } + + @After + void tearDown() throws Exception { + tempFolder?.delete() + } + + @AfterClass + static void tearDownOnce() throws Exception { + + } + + private static boolean isUnlimitedStrengthCryptoAvailable() { + Cipher.getMaxAllowedKeyLength("AES") > 128 + } + + @Test + void testShouldConcatenateByteArrays() { + // Arrange + byte[] bytes1 = "These are some bytes".getBytes(StandardCharsets.UTF_8) + byte[] bytes2 = "These are some other bytes".getBytes(StandardCharsets.UTF_8) + final byte[] EXPECTED_CONCATENATED_BYTES = ((bytes1 as List) << (bytes2 as List)).flatten() as byte[] + logger.info("Expected concatenated bytes: ${Hex.toHexString(EXPECTED_CONCATENATED_BYTES)}") + + // Act + byte[] concat = CryptoUtils.concatByteArrays(bytes1, bytes2) + logger.info(" Actual concatenated bytes: ${Hex.toHexString(concat)}") + + // Assert + assert concat == EXPECTED_CONCATENATED_BYTES + } + + @Test + void testShouldValidateStaticKeyProvider() { + // Arrange + String staticProvider = StaticKeyProvider.class.name + String providerLocation = null + + // Act + boolean keyProviderIsValid = CryptoUtils.isValidKeyProvider(staticProvider, providerLocation, KEY_ID, [(KEY_ID): KEY_HEX]) + logger.info("Key Provider ${staticProvider} with location ${providerLocation} and keyId ${KEY_ID} / ${KEY_HEX} is ${keyProviderIsValid ? "valid" : "invalid"}") + + // Assert + assert keyProviderIsValid + } + + @Test + void testShouldNotValidateStaticKeyProviderMissingKeyId() { + // Arrange + String staticProvider = StaticKeyProvider.class.name + String providerLocation = null + + // Act + boolean keyProviderIsValid = CryptoUtils.isValidKeyProvider(staticProvider, providerLocation, null, [(KEY_ID): KEY_HEX]) + logger.info("Key Provider ${staticProvider} with location ${providerLocation} and keyId ${null} / ${KEY_HEX} is ${keyProviderIsValid ? "valid" : "invalid"}") + + // Assert + assert !keyProviderIsValid + } + + @Test + void testShouldNotValidateStaticKeyProviderMissingKey() { + // Arrange + String staticProvider = StaticKeyProvider.class.name + String providerLocation = null + + // Act + boolean keyProviderIsValid = CryptoUtils.isValidKeyProvider(staticProvider, providerLocation, KEY_ID, null) + logger.info("Key Provider ${staticProvider} with location ${providerLocation} and keyId ${KEY_ID} / ${null} is ${keyProviderIsValid ? "valid" : "invalid"}") + + // Assert + assert !keyProviderIsValid + } + + @Test + void testShouldNotValidateStaticKeyProviderWithInvalidKey() { + // Arrange + String staticProvider = StaticKeyProvider.class.name + String providerLocation = null + + // Act + boolean keyProviderIsValid = CryptoUtils.isValidKeyProvider(staticProvider, providerLocation, KEY_ID, [(KEY_ID): KEY_HEX[0..<-2]]) + logger.info("Key Provider ${staticProvider} with location ${providerLocation} and keyId ${KEY_ID} / ${KEY_HEX[0..<-2]} is ${keyProviderIsValid ? "valid" : "invalid"}") + + // Assert + assert !keyProviderIsValid + } + + @Test + void testShouldValidateFileBasedKeyProvider() { + // Arrange + String fileBasedProvider = FileBasedKeyProvider.class.name + File fileBasedProviderFile = tempFolder.newFile("filebased.kp") + String providerLocation = fileBasedProviderFile.path + logger.info("Created temporary file based key provider: ${providerLocation}") + + // Act + boolean keyProviderIsValid = CryptoUtils.isValidKeyProvider(fileBasedProvider, providerLocation, KEY_ID, null) + logger.info("Key Provider ${fileBasedProvider} with location ${providerLocation} and keyId ${KEY_ID} / ${null} is ${keyProviderIsValid ? "valid" : "invalid"}") + + // Assert + assert keyProviderIsValid + } + + @Test + void testShouldNotValidateUnreadableOrMissingFileBasedKeyProvider() { + // Arrange + String fileBasedProvider = FileBasedKeyProvider.class.name + File fileBasedProviderFile = tempFolder.newFile("filebased.kp") + String providerLocation = fileBasedProviderFile.path + logger.info("Created temporary file based key provider: ${providerLocation}") + + // Make it unreadable + fileBasedProviderFile.setReadable(false, false) + Files.setPosixFilePermissions(fileBasedProviderFile.toPath(), [] as Set) + + // Act + boolean unreadableKeyProviderIsValid = CryptoUtils.isValidKeyProvider(fileBasedProvider, providerLocation, KEY_ID, null) + logger.info("Key Provider ${fileBasedProvider} with location ${providerLocation} and keyId ${KEY_ID} / ${null} is ${unreadableKeyProviderIsValid ? "valid" : "invalid"}") + + String missingLocation = providerLocation + "_missing" + boolean missingKeyProviderIsValid = CryptoUtils.isValidKeyProvider(fileBasedProvider, missingLocation, KEY_ID, null) + logger.info("Key Provider ${fileBasedProvider} with location ${missingLocation} and keyId ${KEY_ID} / ${null} is ${missingKeyProviderIsValid ? "valid" : "invalid"}") + + // Assert + assert !unreadableKeyProviderIsValid + assert !missingKeyProviderIsValid + + // Make the file deletable so cleanup can occur + fileBasedProviderFile.setReadable(true, false) + Files.setPosixFilePermissions(fileBasedProviderFile.toPath(), ALL_POSIX_ATTRS) + } + + @Test + void testShouldNotValidateFileBasedKeyProviderMissingKeyId() { + // Arrange + String fileBasedProvider = FileBasedKeyProvider.class.name + File fileBasedProviderFile = tempFolder.newFile("missing_key_id.kp") + String providerLocation = fileBasedProviderFile.path + logger.info("Created temporary file based key provider: ${providerLocation}") + + // Act + boolean keyProviderIsValid = CryptoUtils.isValidKeyProvider(fileBasedProvider, providerLocation, null, null) + logger.info("Key Provider ${fileBasedProvider} with location ${providerLocation} and keyId ${null} / ${null} is ${keyProviderIsValid ? "valid" : "invalid"}") + + // Assert + assert !keyProviderIsValid + } + + @Test + void testShouldNotValidateUnknownKeyProvider() { + // Arrange + String providerImplementation = "org.apache.nifi.provenance.ImaginaryKeyProvider" + String providerLocation = null + + // Act + boolean keyProviderIsValid = CryptoUtils.isValidKeyProvider(providerImplementation, providerLocation, KEY_ID, null) + logger.info("Key Provider ${providerImplementation} with location ${providerLocation} and keyId ${KEY_ID} / ${null} is ${keyProviderIsValid ? "valid" : "invalid"}") + + // Assert + assert !keyProviderIsValid + } + + @Test + void testShouldValidateKey() { + // Arrange + String validKey = KEY_HEX + String validLowercaseKey = KEY_HEX.toLowerCase() + + String tooShortKey = KEY_HEX[0..<-2] + String tooLongKey = KEY_HEX + KEY_HEX // Guaranteed to be 2x the max valid key length + String nonHexKey = KEY_HEX.replaceFirst(/A/, "X") + + def validKeys = [validKey, validLowercaseKey] + def invalidKeys = [tooShortKey, tooLongKey, nonHexKey] + + // If unlimited strength is available, also validate 128 and 196 bit keys + if (isUnlimitedStrengthCryptoAvailable()) { + validKeys << KEY_HEX_128 + validKeys << KEY_HEX_256[0..<48] + } else { + invalidKeys << KEY_HEX_256[0..<48] + invalidKeys << KEY_HEX_256 + } + + // Act + def validResults = validKeys.collect { String key -> + logger.info("Validating ${key}") + CryptoUtils.keyIsValid(key) + } + + def invalidResults = invalidKeys.collect { String key -> + logger.info("Validating ${key}") + CryptoUtils.keyIsValid(key) + } + + // Assert + assert validResults.every() + assert invalidResults.every { !it } + } + + @Test + void testShouldReadKeys() { + // Arrange + String masterKeyHex = KEY_HEX + SecretKey masterKey = new SecretKeySpec(Hex.decode(masterKeyHex), "AES") + + // Generate the file + String keyFileName = "keys.nkp" + File keyFile = tempFolder.newFile(keyFileName) + final int KEY_COUNT = 5 + List lines = [] + KEY_COUNT.times { int i -> + lines.add("key${i + 1}=${generateEncryptedKey(masterKey)}") + } + + keyFile.text = lines.join("\n") + + logger.info("File contents: \n${keyFile.text}") + + // Act + def readKeys = CryptoUtils.readKeys(keyFile.path, masterKey) + logger.info("Read ${readKeys.size()} keys from ${keyFile.path}") + + // Assert + assert readKeys.size() == KEY_COUNT + } + + @Test + void testShouldReadKeysWithDuplicates() { + // Arrange + String masterKeyHex = KEY_HEX + SecretKey masterKey = new SecretKeySpec(Hex.decode(masterKeyHex), "AES") + + // Generate the file + String keyFileName = "keys.nkp" + File keyFile = tempFolder.newFile(keyFileName) + final int KEY_COUNT = 3 + List lines = [] + KEY_COUNT.times { int i -> + lines.add("key${i + 1}=${generateEncryptedKey(masterKey)}") + } + + lines.add("key3=${generateEncryptedKey(masterKey)}") + + keyFile.text = lines.join("\n") + + logger.info("File contents: \n${keyFile.text}") + + // Act + def readKeys = CryptoUtils.readKeys(keyFile.path, masterKey) + logger.info("Read ${readKeys.size()} keys from ${keyFile.path}") + + // Assert + assert readKeys.size() == KEY_COUNT + } + + @Test + void testShouldReadKeysWithSomeMalformed() { + // Arrange + String masterKeyHex = KEY_HEX + SecretKey masterKey = new SecretKeySpec(Hex.decode(masterKeyHex), "AES") + + // Generate the file + String keyFileName = "keys.nkp" + File keyFile = tempFolder.newFile(keyFileName) + final int KEY_COUNT = 5 + List lines = [] + KEY_COUNT.times { int i -> + lines.add("key${i + 1}=${generateEncryptedKey(masterKey)}") + } + + // Insert the malformed keys in the middle + lines.add(2, "keyX1==${generateEncryptedKey(masterKey)}") + lines.add(4, "=${generateEncryptedKey(masterKey)}") + lines.add(6, "keyX3=non Base64-encoded data") + + keyFile.text = lines.join("\n") + + logger.info("File contents: \n${keyFile.text}") + + // Act + def readKeys = CryptoUtils.readKeys(keyFile.path, masterKey) + logger.info("Read ${readKeys.size()} keys from ${keyFile.path}") + + // Assert + assert readKeys.size() == KEY_COUNT + } + + @Test + void testShouldNotReadKeysIfAllMalformed() { + // Arrange + String masterKeyHex = KEY_HEX + SecretKey masterKey = new SecretKeySpec(Hex.decode(masterKeyHex), "AES") + + // Generate the file + String keyFileName = "keys.nkp" + File keyFile = tempFolder.newFile(keyFileName) + final int KEY_COUNT = 5 + List lines = [] + + // All of these keys are malformed + KEY_COUNT.times { int i -> + lines.add("key${i + 1}=${generateEncryptedKey(masterKey)[0..<-4]}") + } + + keyFile.text = lines.join("\n") + + logger.info("File contents: \n${keyFile.text}") + + // Act + def msg = shouldFail(KeyManagementException) { + def readKeys = CryptoUtils.readKeys(keyFile.path, masterKey) + logger.info("Read ${readKeys.size()} keys from ${keyFile.path}") + } + + // Assert + assert msg.getMessage() == "The provided file contained no valid keys" + } + + @Test + void testShouldNotReadKeysIfEmptyOrMissing() { + // Arrange + String masterKeyHex = KEY_HEX + SecretKey masterKey = new SecretKeySpec(Hex.decode(masterKeyHex), "AES") + + // Generate the file + String keyFileName = "empty.nkp" + File keyFile = tempFolder.newFile(keyFileName) + logger.info("File contents: \n${keyFile.text}") + + // Act + def missingMsg = shouldFail(KeyManagementException) { + def readKeys = CryptoUtils.readKeys(keyFile.path, masterKey) + logger.info("Read ${readKeys.size()} keys from ${keyFile.path}") + } + logger.expected("Missing file: ${missingMsg}") + + def emptyMsg = shouldFail(KeyManagementException) { + def readKeys = CryptoUtils.readKeys(null, masterKey) + logger.info("Read ${readKeys.size()} keys from ${null}") + } + logger.expected("Empty file: ${emptyMsg}") + + // Assert + assert missingMsg.getMessage() == "The provided file contained no valid keys" + assert emptyMsg.getMessage() == "The key provider file is not present and readable" + } + + private static String generateEncryptedKey(SecretKey masterKey) { + byte[] ivBytes = new byte[16] + byte[] keyBytes = new byte[isUnlimitedStrengthCryptoAvailable() ? 32 : 16] + + SecureRandom sr = new SecureRandom() + sr.nextBytes(ivBytes) + sr.nextBytes(keyBytes) + + Cipher masterCipher = Cipher.getInstance("AES/GCM/NoPadding", "BC") + masterCipher.init(Cipher.ENCRYPT_MODE, masterKey, new IvParameterSpec(ivBytes)) + byte[] cipherBytes = masterCipher.doFinal(keyBytes) + + Base64.encoder.encodeToString(CryptoUtils.concatByteArrays(ivBytes, cipherBytes)) + } +} diff --git a/nifi-commons/nifi-data-provenance-utils/src/test/java/org/apache/nifi/provenance/EncryptionExceptionTest.java b/nifi-commons/nifi-data-provenance-utils/src/test/java/org/apache/nifi/provenance/EncryptionExceptionTest.java new file mode 100644 index 0000000000..e23d492c5a --- /dev/null +++ b/nifi-commons/nifi-data-provenance-utils/src/test/java/org/apache/nifi/provenance/EncryptionExceptionTest.java @@ -0,0 +1,27 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.provenance; + +import org.junit.Test; + +public class EncryptionExceptionTest { + + @Test + public void testShouldTriggerGroovyTestExecution() { + // This method does nothing but tell Maven to run the groovy tests + } +} diff --git a/nifi-commons/nifi-data-provenance-utils/src/test/resources/logback-test.xml b/nifi-commons/nifi-data-provenance-utils/src/test/resources/logback-test.xml new file mode 100644 index 0000000000..c10508d6bb --- /dev/null +++ b/nifi-commons/nifi-data-provenance-utils/src/test/resources/logback-test.xml @@ -0,0 +1,32 @@ + + + + + + + %-4r [%t] %-5p %c - %m%n + + + + + + + + + + + + diff --git a/nifi-commons/nifi-properties/src/main/java/org/apache/nifi/util/NiFiProperties.java b/nifi-commons/nifi-properties/src/main/java/org/apache/nifi/util/NiFiProperties.java index 3d657fb46c..d69a280fd5 100644 --- a/nifi-commons/nifi-properties/src/main/java/org/apache/nifi/util/NiFiProperties.java +++ b/nifi-commons/nifi-properties/src/main/java/org/apache/nifi/util/NiFiProperties.java @@ -32,6 +32,7 @@ import java.util.List; import java.util.Map; import java.util.Properties; import java.util.Set; +import java.util.stream.Collectors; /** * The NiFiProperties class holds all properties which are needed for various @@ -116,6 +117,11 @@ public abstract class NiFiProperties { public static final String PROVENANCE_INDEXED_ATTRIBUTES = "nifi.provenance.repository.indexed.attributes"; public static final String PROVENANCE_INDEX_SHARD_SIZE = "nifi.provenance.repository.index.shard.size"; public static final String PROVENANCE_JOURNAL_COUNT = "nifi.provenance.repository.journal.count"; + public static final String PROVENANCE_REPO_ENCRYPTION_KEY = "nifi.provenance.repository.encryption.key"; + public static final String PROVENANCE_REPO_ENCRYPTION_KEY_ID = "nifi.provenance.repository.encryption.key.id"; + public static final String PROVENANCE_REPO_ENCRYPTION_KEY_PROVIDER_IMPLEMENTATION_CLASS = "nifi.provenance.repository.encryption.key.provider.implementation"; + public static final String PROVENANCE_REPO_ENCRYPTION_KEY_PROVIDER_LOCATION = "nifi.provenance.repository.encryption.key.provider.location"; + public static final String PROVENANCE_REPO_DEBUG_FREQUENCY = "nifi.provenance.repository.debug.frequency"; // component status repository properties public static final String COMPONENT_STATUS_REPOSITORY_IMPLEMENTATION = "nifi.components.status.repository.implementation"; @@ -769,7 +775,7 @@ public abstract class NiFiProperties { /** * Returns true if client certificates are required for REST API. Determined * if the following conditions are all true: - * + *

* - login identity provider is not populated * - Kerberos service support is not enabled * @@ -1034,6 +1040,61 @@ public abstract class NiFiProperties { return getPropertyKeys().size(); } + public String getProvenanceRepoEncryptionKeyId() { + return getProperty(PROVENANCE_REPO_ENCRYPTION_KEY_ID); + } + + /** + * Returns the active provenance repository encryption key if a {@code StaticKeyProvider} is in use. + * If no key ID is specified in the properties file, the default + * {@code nifi.provenance.repository.encryption.key} value is returned. If a key ID is specified in + * {@code nifi.provenance.repository.encryption.key.id}, it will attempt to read from + * {@code nifi.provenance.repository.encryption.key.id.XYZ} where {@code XYZ} is the provided key + * ID. If that value is empty, it will use the default property + * {@code nifi.provenance.repository.encryption.key}. + * + * @return the provenance repository encryption key in hex form + */ + public String getProvenanceRepoEncryptionKey() { + String keyId = getProvenanceRepoEncryptionKeyId(); + String keyKey = StringUtils.isBlank(keyId) ? PROVENANCE_REPO_ENCRYPTION_KEY : PROVENANCE_REPO_ENCRYPTION_KEY + ".id." + keyId; + return getProperty(keyKey, getProperty(PROVENANCE_REPO_ENCRYPTION_KEY)); + } + + /** + * Returns a map of keyId -> key in hex loaded from the {@code nifi.properties} file if a + * {@code StaticKeyProvider} is defined. If {@code FileBasedKeyProvider} is defined, use + * {@code CryptoUtils#readKeys()} instead -- this method will return an empty map. + * + * @return a Map of the keys identified by key ID + */ + public Map getProvenanceRepoEncryptionKeys() { + Map keys = new HashMap<>(); + List keyProperties = getProvenanceRepositoryEncryptionKeyProperties(); + + // Retrieve the actual key values and store non-empty values in the map + for (String prop : keyProperties) { + final String value = getProperty(prop); + if (!StringUtils.isBlank(value)) { + if (prop.equalsIgnoreCase(PROVENANCE_REPO_ENCRYPTION_KEY)) { + prop = getProvenanceRepoEncryptionKeyId(); + } else { + // Extract nifi.provenance.repository.encryption.key.id.key1 -> key1 + prop = prop.substring(prop.lastIndexOf(".") + 1); + } + keys.put(prop, value); + } + } + return keys; + } + + private List getProvenanceRepositoryEncryptionKeyProperties() { + // Filter all the property keys that define a key + return getPropertyKeys().stream().filter(k -> + k.startsWith(PROVENANCE_REPO_ENCRYPTION_KEY_ID + ".") || k.equalsIgnoreCase(PROVENANCE_REPO_ENCRYPTION_KEY) + ).collect(Collectors.toList()); + } + /** * Creates an instance of NiFiProperties. This should likely not be called * by any classes outside of the NiFi framework but can be useful by the @@ -1042,11 +1103,11 @@ public abstract class NiFiProperties { * file specified cannot be found/read a runtime exception will be thrown. * If one is not specified no properties will be loaded by default. * - * @param propertiesFilePath if provided properties will be loaded from - * given file; else will be loaded from System property. Can be null. + * @param propertiesFilePath if provided properties will be loaded from + * given file; else will be loaded from System property. Can be null. * @param additionalProperties allows overriding of properties with the - * supplied values. these will be applied after loading from any properties - * file. Can be null or empty. + * supplied values. these will be applied after loading from any properties + * file. Can be null or empty. * @return NiFiProperties */ public static NiFiProperties createBasicNiFiProperties(final String propertiesFilePath, final Map additionalProperties) { @@ -1108,10 +1169,9 @@ public abstract class NiFiProperties { public void validate() { // REMOTE_INPUT_HOST should be a valid hostname String remoteInputHost = getProperty(REMOTE_INPUT_HOST); - if(!StringUtils.isBlank(remoteInputHost) && remoteInputHost.split(":").length > 1) { // no scheme/port needed here (http://) + if (!StringUtils.isBlank(remoteInputHost) && remoteInputHost.split(":").length > 1) { // no scheme/port needed here (http://) throw new IllegalArgumentException(remoteInputHost + " is not a correct value for " + REMOTE_INPUT_HOST + ". It should be a valid hostname without protocol or port."); } // Other properties to validate... } - } diff --git a/nifi-commons/nifi-security-utils/pom.xml b/nifi-commons/nifi-security-utils/pom.xml index e2e5ee169a..b004c8b435 100644 --- a/nifi-commons/nifi-security-utils/pom.xml +++ b/nifi-commons/nifi-security-utils/pom.xml @@ -43,6 +43,10 @@ org.apache.commons commons-lang3 + + commons-codec + commons-codec + org.bouncycastle bcprov-jdk15on diff --git a/nifi-commons/nifi-security-utils/src/main/java/org/apache/nifi/security/util/EncryptionMethod.java b/nifi-commons/nifi-security-utils/src/main/java/org/apache/nifi/security/util/EncryptionMethod.java index 53664f1cde..a1ef2a4444 100644 --- a/nifi-commons/nifi-security-utils/src/main/java/org/apache/nifi/security/util/EncryptionMethod.java +++ b/nifi-commons/nifi-security-utils/src/main/java/org/apache/nifi/security/util/EncryptionMethod.java @@ -16,13 +16,13 @@ */ package org.apache.nifi.security.util; +import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.builder.ToStringBuilder; import org.apache.commons.lang3.builder.ToStringStyle; /** * Enumeration capturing essential information about the various encryption * methods that might be supported. - * */ public enum EncryptionMethod { @@ -105,4 +105,15 @@ public enum EncryptionMethod { builder.append("Keyed cipher", isKeyedCipher()); return builder.toString(); } + + public static EncryptionMethod forAlgorithm(String algorithm) { + if (StringUtils.isNotBlank(algorithm)) { + for (EncryptionMethod em : EncryptionMethod.values()) { + if (em.algorithm.equalsIgnoreCase(algorithm)) { + return em; + } + } + } + return null; + } } diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/util/crypto/AESKeyedCipherProvider.java b/nifi-commons/nifi-security-utils/src/main/java/org/apache/nifi/security/util/crypto/AESKeyedCipherProvider.java similarity index 99% rename from nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/util/crypto/AESKeyedCipherProvider.java rename to nifi-commons/nifi-security-utils/src/main/java/org/apache/nifi/security/util/crypto/AESKeyedCipherProvider.java index 907aed268e..617a3e5e74 100644 --- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/util/crypto/AESKeyedCipherProvider.java +++ b/nifi-commons/nifi-security-utils/src/main/java/org/apache/nifi/security/util/crypto/AESKeyedCipherProvider.java @@ -14,18 +14,8 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.nifi.processors.standard.util.crypto; +package org.apache.nifi.security.util.crypto; -import org.apache.commons.lang3.StringUtils; -import org.apache.nifi.processor.exception.ProcessException; -import org.apache.nifi.security.util.EncryptionMethod; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import javax.crypto.Cipher; -import javax.crypto.NoSuchPaddingException; -import javax.crypto.SecretKey; -import javax.crypto.spec.IvParameterSpec; import java.io.UnsupportedEncodingException; import java.security.InvalidAlgorithmParameterException; import java.security.InvalidKeyException; @@ -35,6 +25,15 @@ import java.security.SecureRandom; import java.security.spec.InvalidKeySpecException; import java.util.Arrays; import java.util.List; +import javax.crypto.Cipher; +import javax.crypto.NoSuchPaddingException; +import javax.crypto.SecretKey; +import javax.crypto.spec.IvParameterSpec; +import org.apache.commons.lang3.StringUtils; +import org.apache.nifi.processor.exception.ProcessException; +import org.apache.nifi.security.util.EncryptionMethod; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * This is a standard implementation of {@link KeyedCipherProvider} which supports {@code AES} cipher families with arbitrary modes of operation (currently only {@code CBC}, {@code CTR}, and {@code diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/util/crypto/CipherProvider.java b/nifi-commons/nifi-security-utils/src/main/java/org/apache/nifi/security/util/crypto/CipherProvider.java similarity index 93% rename from nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/util/crypto/CipherProvider.java rename to nifi-commons/nifi-security-utils/src/main/java/org/apache/nifi/security/util/crypto/CipherProvider.java index 46e815fcc9..e3632b2c1b 100644 --- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/util/crypto/CipherProvider.java +++ b/nifi-commons/nifi-security-utils/src/main/java/org/apache/nifi/security/util/crypto/CipherProvider.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.nifi.processors.standard.util.crypto; +package org.apache.nifi.security.util.crypto; /** * Marker interface for cipher providers. diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/util/crypto/CipherUtility.java b/nifi-commons/nifi-security-utils/src/main/java/org/apache/nifi/security/util/crypto/CipherUtility.java similarity index 99% rename from nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/util/crypto/CipherUtility.java rename to nifi-commons/nifi-security-utils/src/main/java/org/apache/nifi/security/util/crypto/CipherUtility.java index 6295c1d11d..6ba8056a96 100644 --- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/util/crypto/CipherUtility.java +++ b/nifi-commons/nifi-security-utils/src/main/java/org/apache/nifi/security/util/crypto/CipherUtility.java @@ -14,16 +14,8 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.nifi.processors.standard.util.crypto; +package org.apache.nifi.security.util.crypto; -import org.apache.commons.codec.binary.Base64; -import org.apache.commons.lang3.StringUtils; -import org.apache.nifi.processor.exception.ProcessException; -import org.apache.nifi.security.util.EncryptionMethod; -import org.apache.nifi.stream.io.ByteArrayOutputStream; -import org.apache.nifi.stream.io.StreamUtils; - -import javax.crypto.Cipher; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; @@ -35,6 +27,13 @@ import java.util.List; import java.util.Map; import java.util.regex.Matcher; import java.util.regex.Pattern; +import javax.crypto.Cipher; +import org.apache.commons.codec.binary.Base64; +import org.apache.commons.lang3.StringUtils; +import org.apache.nifi.processor.exception.ProcessException; +import org.apache.nifi.security.util.EncryptionMethod; +import org.apache.nifi.stream.io.ByteArrayOutputStream; +import org.apache.nifi.stream.io.StreamUtils; public class CipherUtility { diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/util/crypto/KeyedCipherProvider.java b/nifi-commons/nifi-security-utils/src/main/java/org/apache/nifi/security/util/crypto/KeyedCipherProvider.java similarity index 98% rename from nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/util/crypto/KeyedCipherProvider.java rename to nifi-commons/nifi-security-utils/src/main/java/org/apache/nifi/security/util/crypto/KeyedCipherProvider.java index f0fa4fc0f4..719150f98b 100644 --- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/util/crypto/KeyedCipherProvider.java +++ b/nifi-commons/nifi-security-utils/src/main/java/org/apache/nifi/security/util/crypto/KeyedCipherProvider.java @@ -14,17 +14,16 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.nifi.processors.standard.util.crypto; +package org.apache.nifi.security.util.crypto; -import org.apache.nifi.processor.exception.ProcessException; -import org.apache.nifi.security.util.EncryptionMethod; - -import javax.crypto.Cipher; -import javax.crypto.SecretKey; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; import java.nio.charset.StandardCharsets; +import javax.crypto.Cipher; +import javax.crypto.SecretKey; +import org.apache.nifi.processor.exception.ProcessException; +import org.apache.nifi.security.util.EncryptionMethod; public abstract class KeyedCipherProvider implements CipherProvider { static final byte[] IV_DELIMITER = "NiFiIV".getBytes(StandardCharsets.UTF_8); diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/groovy/org/apache/nifi/processors/standard/util/crypto/AESKeyedCipherProviderGroovyTest.groovy b/nifi-commons/nifi-security-utils/src/test/groovy/org/apache/nifi/security/util/crypto/AESKeyedCipherProviderGroovyTest.groovy similarity index 89% rename from nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/groovy/org/apache/nifi/processors/standard/util/crypto/AESKeyedCipherProviderGroovyTest.groovy rename to nifi-commons/nifi-security-utils/src/test/groovy/org/apache/nifi/security/util/crypto/AESKeyedCipherProviderGroovyTest.groovy index 0596d7d233..80821496ba 100644 --- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/groovy/org/apache/nifi/processors/standard/util/crypto/AESKeyedCipherProviderGroovyTest.groovy +++ b/nifi-commons/nifi-security-utils/src/test/groovy/org/apache/nifi/security/util/crypto/AESKeyedCipherProviderGroovyTest.groovy @@ -14,12 +14,16 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.nifi.processors.standard.util.crypto +package org.apache.nifi.security.util.crypto import org.apache.commons.codec.binary.Hex import org.apache.nifi.security.util.EncryptionMethod import org.bouncycastle.jce.provider.BouncyCastleProvider -import org.junit.* +import org.junit.After +import org.junit.Assume +import org.junit.Before +import org.junit.BeforeClass +import org.junit.Test import org.junit.runner.RunWith import org.junit.runners.JUnit4 import org.slf4j.Logger @@ -34,7 +38,7 @@ import java.security.Security import static groovy.test.GroovyAssert.shouldFail @RunWith(JUnit4.class) -public class AESKeyedCipherProviderGroovyTest { +class AESKeyedCipherProviderGroovyTest { private static final Logger logger = LoggerFactory.getLogger(AESKeyedCipherProviderGroovyTest.class) private static final String KEY_HEX = "0123456789ABCDEFFEDCBA9876543210" @@ -44,7 +48,7 @@ public class AESKeyedCipherProviderGroovyTest { private static final SecretKey key = new SecretKeySpec(Hex.decodeHex(KEY_HEX as char[]), "AES") @BeforeClass - public static void setUpOnce() throws Exception { + static void setUpOnce() throws Exception { Security.addProvider(new BouncyCastleProvider()) logger.metaClass.methodMissing = { String name, args -> @@ -53,15 +57,19 @@ public class AESKeyedCipherProviderGroovyTest { } @Before - public void setUp() throws Exception { + void setUp() throws Exception { } @After - public void tearDown() throws Exception { + void tearDown() throws Exception { + } + + private static boolean isUnlimitedStrengthCryptoAvailable() { + Cipher.getMaxAllowedKeyLength("AES") > 128 } @Test - public void testGetCipherShouldBeInternallyConsistent() throws Exception { + void testGetCipherShouldBeInternallyConsistent() throws Exception { // Arrange KeyedCipherProvider cipherProvider = new AESKeyedCipherProvider() @@ -90,7 +98,7 @@ public class AESKeyedCipherProviderGroovyTest { } @Test - public void testGetCipherWithExternalIVShouldBeInternallyConsistent() throws Exception { + void testGetCipherWithExternalIVShouldBeInternallyConsistent() throws Exception { // Arrange KeyedCipherProvider cipherProvider = new AESKeyedCipherProvider() @@ -119,10 +127,9 @@ public class AESKeyedCipherProviderGroovyTest { } @Test - public void testGetCipherWithUnlimitedStrengthShouldBeInternallyConsistent() throws Exception { + void testGetCipherWithUnlimitedStrengthShouldBeInternallyConsistent() throws Exception { // Arrange - Assume.assumeTrue("Test is being skipped due to this JVM lacking JCE Unlimited Strength Jurisdiction Policy file.", - PasswordBasedEncryptor.supportsUnlimitedStrength()) + Assume.assumeTrue("Test is being skipped due to this JVM lacking JCE Unlimited Strength Jurisdiction Policy file.", isUnlimitedStrengthCryptoAvailable()) KeyedCipherProvider cipherProvider = new AESKeyedCipherProvider() final List LONG_KEY_LENGTHS = [192, 256] @@ -164,7 +171,7 @@ public class AESKeyedCipherProviderGroovyTest { } @Test - public void testShouldRejectEmptyKey() throws Exception { + void testShouldRejectEmptyKey() throws Exception { // Arrange KeyedCipherProvider cipherProvider = new AESKeyedCipherProvider() @@ -180,7 +187,7 @@ public class AESKeyedCipherProviderGroovyTest { } @Test - public void testShouldRejectIncorrectLengthKey() throws Exception { + void testShouldRejectIncorrectLengthKey() throws Exception { // Arrange KeyedCipherProvider cipherProvider = new AESKeyedCipherProvider() @@ -199,7 +206,7 @@ public class AESKeyedCipherProviderGroovyTest { } @Test - public void testShouldRejectEmptyEncryptionMethod() throws Exception { + void testShouldRejectEmptyEncryptionMethod() throws Exception { // Arrange KeyedCipherProvider cipherProvider = new AESKeyedCipherProvider() @@ -213,7 +220,7 @@ public class AESKeyedCipherProviderGroovyTest { } @Test - public void testShouldRejectUnsupportedEncryptionMethod() throws Exception { + void testShouldRejectUnsupportedEncryptionMethod() throws Exception { // Arrange KeyedCipherProvider cipherProvider = new AESKeyedCipherProvider() @@ -229,7 +236,7 @@ public class AESKeyedCipherProviderGroovyTest { } @Test - public void testGetCipherShouldSupportExternalCompatibility() throws Exception { + void testGetCipherShouldSupportExternalCompatibility() throws Exception { // Arrange KeyedCipherProvider cipherProvider = new AESKeyedCipherProvider() @@ -258,7 +265,7 @@ public class AESKeyedCipherProviderGroovyTest { } @Test - public void testGetCipherForDecryptShouldRequireIV() throws Exception { + void testGetCipherForDecryptShouldRequireIV() throws Exception { // Arrange KeyedCipherProvider cipherProvider = new AESKeyedCipherProvider() @@ -286,7 +293,7 @@ public class AESKeyedCipherProviderGroovyTest { } @Test - public void testGetCipherShouldRejectInvalidIVLengths() throws Exception { + void testGetCipherShouldRejectInvalidIVLengths() throws Exception { // Arrange KeyedCipherProvider cipherProvider = new AESKeyedCipherProvider() @@ -313,7 +320,7 @@ public class AESKeyedCipherProviderGroovyTest { } @Test - public void testGetCipherShouldRejectEmptyIV() throws Exception { + void testGetCipherShouldRejectEmptyIV() throws Exception { // Arrange KeyedCipherProvider cipherProvider = new AESKeyedCipherProvider() diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/groovy/org/apache/nifi/processors/standard/util/crypto/CipherUtilityGroovyTest.groovy b/nifi-commons/nifi-security-utils/src/test/groovy/org/apache/nifi/security/util/crypto/CipherUtilityGroovyTest.groovy similarity index 99% rename from nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/groovy/org/apache/nifi/processors/standard/util/crypto/CipherUtilityGroovyTest.groovy rename to nifi-commons/nifi-security-utils/src/test/groovy/org/apache/nifi/security/util/crypto/CipherUtilityGroovyTest.groovy index 6a6a95859d..8f092f3ff5 100644 --- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/groovy/org/apache/nifi/processors/standard/util/crypto/CipherUtilityGroovyTest.groovy +++ b/nifi-commons/nifi-security-utils/src/test/groovy/org/apache/nifi/security/util/crypto/CipherUtilityGroovyTest.groovy @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.nifi.processors.standard.util.crypto +package org.apache.nifi.security.util.crypto import org.apache.nifi.security.util.EncryptionMethod import org.bouncycastle.jce.provider.BouncyCastleProvider diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/FlowController.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/FlowController.java index 7853591a83..5b2ea41976 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/FlowController.java +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/FlowController.java @@ -16,7 +16,38 @@ */ package org.apache.nifi.controller; +import static java.util.Objects.requireNonNull; + import com.sun.jersey.api.client.ClientHandlerException; +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.net.URL; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.Date; +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.UUID; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.Future; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ScheduledFuture; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; +import java.util.concurrent.locks.Lock; +import java.util.concurrent.locks.ReentrantReadWriteLock; +import java.util.stream.Collectors; +import javax.net.ssl.SSLContext; import org.apache.commons.collections4.Predicate; import org.apache.commons.lang3.StringUtils; import org.apache.nifi.action.Action; @@ -85,7 +116,6 @@ import org.apache.nifi.controller.repository.FlowFileRecord; import org.apache.nifi.controller.repository.FlowFileRepository; import org.apache.nifi.controller.repository.FlowFileSwapManager; import org.apache.nifi.controller.repository.QueueProvider; -import org.apache.nifi.controller.repository.RepositoryRecord; import org.apache.nifi.controller.repository.RepositoryStatusReport; import org.apache.nifi.controller.repository.StandardCounterRepository; import org.apache.nifi.controller.repository.StandardFlowFileRecord; @@ -217,38 +247,6 @@ import org.apache.zookeeper.server.quorum.QuorumPeerConfig.ConfigException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import javax.net.ssl.SSLContext; -import java.io.ByteArrayInputStream; -import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; -import java.net.URL; -import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; -import java.util.Date; -import java.util.HashMap; -import java.util.HashSet; -import java.util.LinkedHashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.UUID; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ConcurrentMap; -import java.util.concurrent.Future; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.ScheduledFuture; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicBoolean; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.concurrent.atomic.AtomicReference; -import java.util.concurrent.locks.Lock; -import java.util.concurrent.locks.ReentrantReadWriteLock; -import java.util.stream.Collectors; - -import static java.util.Objects.requireNonNull; - public class FlowController implements EventAccess, ControllerServiceProvider, ReportingTaskProvider, QueueProvider, Authorizable, ProvenanceAuthorizableFactory, NodeTypeProvider, IdentifierLookup, ReloadComponent { @@ -3841,7 +3839,7 @@ public class FlowController implements EventAccess, ControllerServiceProvider, R final ProvenanceEventRecord sendEvent = new StandardProvenanceEventRecord.Builder() .setEventType(ProvenanceEventType.DOWNLOAD) .setFlowFileUUID(provEvent.getFlowFileUuid()) - .setAttributes(provEvent.getAttributes(), Collections.emptyMap()) + .setAttributes(provEvent.getAttributes(), Collections.emptyMap()) .setCurrentContentClaim(resourceClaim.getContainer(), resourceClaim.getSection(), resourceClaim.getId(), offset, size) .setTransitUri(requestUri) .setEventTime(System.currentTimeMillis()) @@ -3883,7 +3881,7 @@ public class FlowController implements EventAccess, ControllerServiceProvider, R final StandardProvenanceEventRecord.Builder sendEventBuilder = new StandardProvenanceEventRecord.Builder() .setEventType(ProvenanceEventType.DOWNLOAD) .setFlowFileUUID(flowFile.getAttribute(CoreAttributes.UUID.key())) - .setAttributes(flowFile.getAttributes(), Collections.emptyMap()) + .setAttributes(flowFile.getAttributes(), Collections.emptyMap()) .setTransitUri(requestUri) .setEventTime(System.currentTimeMillis()) .setFlowFileEntryDate(flowFile.getEntryDate()) @@ -4062,7 +4060,7 @@ public class FlowController implements EventAccess, ControllerServiceProvider, R .addChildUuid(newFlowFileUUID) .addParentUuid(parentUUID) .setFlowFileUUID(parentUUID) - .setAttributes(Collections.emptyMap(), flowFileRecord.getAttributes()) + .setAttributes(Collections.emptyMap(), flowFileRecord.getAttributes()) .setCurrentContentClaim(event.getContentClaimContainer(), event.getContentClaimSection(), event.getContentClaimIdentifier(), event.getContentClaimOffset(), event.getFileSize()) .setDetails("Replay requested by " + user.getIdentity()) .setEventTime(System.currentTimeMillis()) @@ -4077,7 +4075,7 @@ public class FlowController implements EventAccess, ControllerServiceProvider, R final StandardRepositoryRecord record = new StandardRepositoryRecord(queue); record.setWorking(flowFileRecord); record.setDestination(queue); - flowFileRepository.updateRepository(Collections.singleton(record)); + flowFileRepository.updateRepository(Collections.singleton(record)); // Enqueue the data queue.put(flowFileRecord); diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/main/java/org/apache/nifi/properties/ProtectedNiFiProperties.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/main/java/org/apache/nifi/properties/ProtectedNiFiProperties.java index 4774dc75df..fc1d72231f 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/main/java/org/apache/nifi/properties/ProtectedNiFiProperties.java +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/main/java/org/apache/nifi/properties/ProtectedNiFiProperties.java @@ -26,6 +26,7 @@ import java.util.List; import java.util.Map; import java.util.Properties; import java.util.Set; +import java.util.stream.Collectors; import org.apache.commons.lang3.StringUtils; import org.apache.nifi.util.NiFiProperties; import org.slf4j.Logger; @@ -50,7 +51,7 @@ class ProtectedNiFiProperties extends StandardNiFiProperties { // Default list of "sensitive" property keys public static final List DEFAULT_SENSITIVE_PROPERTIES = new ArrayList<>(asList(SECURITY_KEY_PASSWD, - SECURITY_KEYSTORE_PASSWD, SECURITY_TRUSTSTORE_PASSWD, SENSITIVE_PROPS_KEY)); + SECURITY_KEYSTORE_PASSWD, SECURITY_TRUSTSTORE_PASSWD, SENSITIVE_PROPS_KEY, PROVENANCE_REPO_ENCRYPTION_KEY)); public ProtectedNiFiProperties() { this(new StandardNiFiProperties()); @@ -183,6 +184,17 @@ class ProtectedNiFiProperties extends StandardNiFiProperties { } } + /** + * Returns a list of the keys identifying "sensitive" properties. There is a default list, + * and additional keys can be provided in the {@code nifi.sensitive.props.additional.keys} property in {@code nifi.properties}. + * + * @return the list of sensitive property keys + */ + public List getPopulatedSensitivePropertyKeys() { + List allSensitiveKeys = getSensitivePropertyKeys(); + return allSensitiveKeys.stream().filter(k -> StringUtils.isNotBlank(getProperty(k))).collect(Collectors.toList()); + } + /** * Returns true if any sensitive keys are protected. * @@ -219,7 +231,7 @@ class ProtectedNiFiProperties extends StandardNiFiProperties { Map traditionalProtectedProperties = new HashMap<>(); for (String key : sensitiveKeys) { String protection = getProperty(getProtectionKey(key)); - if (!StringUtils.isBlank(protection)) { + if (StringUtils.isNotBlank(protection) && StringUtils.isNotBlank(getProperty(key))) { traditionalProtectedProperties.put(key, protection); } } @@ -237,12 +249,12 @@ class ProtectedNiFiProperties extends StandardNiFiProperties { } /** - * Returns a percentage of the total number of properties marked as sensitive that are currently protected. + * Returns a percentage of the total number of populated properties marked as sensitive that are currently protected. * * @return the percent of sensitive properties marked as protected */ public int getPercentOfSensitivePropertiesProtected() { - return (int) Math.round(getProtectedPropertyKeys().size() / ((double) getSensitivePropertyKeys().size()) * 100); + return (int) Math.round(getProtectedPropertyKeys().size() / ((double) getPopulatedSensitivePropertyKeys().size()) * 100); } /** @@ -421,9 +433,7 @@ class ProtectedNiFiProperties extends StandardNiFiProperties { // Add the protected keys and the protection schemes for (String key : getSensitivePropertyKeys()) { final String plainValue = getInternalNiFiProperties().getProperty(key); - if (plainValue == null || plainValue.trim().isEmpty()) { - protectedProperties.setProperty(key, plainValue); - } else { + if (plainValue != null && !plainValue.trim().isEmpty()) { final String protectedValue = spp.protect(plainValue); protectedProperties.setProperty(key, protectedValue); protectedProperties.setProperty(getProtectionKey(key), protectionScheme); diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/groovy/org/apache/nifi/properties/AESSensitivePropertyProviderTest.groovy b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/groovy/org/apache/nifi/properties/AESSensitivePropertyProviderTest.groovy index 7896afe9dc..73ae55a8e0 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/groovy/org/apache/nifi/properties/AESSensitivePropertyProviderTest.groovy +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/groovy/org/apache/nifi/properties/AESSensitivePropertyProviderTest.groovy @@ -52,7 +52,7 @@ class AESSensitivePropertyProviderTest extends GroovyTestCase { private static final Base64.Decoder decoder = Base64.decoder @BeforeClass - public static void setUpOnce() throws Exception { + static void setUpOnce() throws Exception { Security.addProvider(new BouncyCastleProvider()) logger.metaClass.methodMissing = { String name, args -> @@ -61,12 +61,12 @@ class AESSensitivePropertyProviderTest extends GroovyTestCase { } @Before - public void setUp() throws Exception { + void setUp() throws Exception { } @After - public void tearDown() throws Exception { + void tearDown() throws Exception { } @@ -112,7 +112,7 @@ class AESSensitivePropertyProviderTest extends GroovyTestCase { } @Test - public void testShouldThrowExceptionOnInitializationWithoutBouncyCastle() throws Exception { + void testShouldThrowExceptionOnInitializationWithoutBouncyCastle() throws Exception { // Arrange try { Security.removeProvider(new BouncyCastleProvider().getName()) @@ -133,7 +133,7 @@ class AESSensitivePropertyProviderTest extends GroovyTestCase { // TODO: testShouldGetName() @Test - public void testShouldProtectValue() throws Exception { + void testShouldProtectValue() throws Exception { final String PLAINTEXT = "This is a plaintext value" // Act @@ -163,7 +163,7 @@ class AESSensitivePropertyProviderTest extends GroovyTestCase { } @Test - public void testShouldHandleProtectEmptyValue() throws Exception { + void testShouldHandleProtectEmptyValue() throws Exception { final List EMPTY_PLAINTEXTS = ["", " ", null] // Act @@ -183,7 +183,7 @@ class AESSensitivePropertyProviderTest extends GroovyTestCase { } @Test - public void testShouldUnprotectValue() throws Exception { + void testShouldUnprotectValue() throws Exception { // Arrange final String PLAINTEXT = "This is a plaintext value" @@ -218,7 +218,7 @@ class AESSensitivePropertyProviderTest extends GroovyTestCase { * @throws Exception */ @Test - public void testShouldHandleUnprotectEmptyValue() throws Exception { + void testShouldHandleUnprotectEmptyValue() throws Exception { // Arrange final List EMPTY_CIPHER_TEXTS = ["", " ", null] @@ -239,7 +239,7 @@ class AESSensitivePropertyProviderTest extends GroovyTestCase { } @Test - public void testShouldUnprotectValueWithWhitespace() throws Exception { + void testShouldUnprotectValueWithWhitespace() throws Exception { // Arrange final String PLAINTEXT = "This is a plaintext value" @@ -269,7 +269,7 @@ class AESSensitivePropertyProviderTest extends GroovyTestCase { } @Test - public void testShouldHandleUnprotectMalformedValue() throws Exception { + void testShouldHandleUnprotectMalformedValue() throws Exception { // Arrange final String PLAINTEXT = "This is a plaintext value" @@ -293,7 +293,7 @@ class AESSensitivePropertyProviderTest extends GroovyTestCase { } @Test - public void testShouldHandleUnprotectMissingIV() throws Exception { + void testShouldHandleUnprotectMissingIV() throws Exception { // Arrange final String PLAINTEXT = "This is a plaintext value" @@ -334,7 +334,7 @@ class AESSensitivePropertyProviderTest extends GroovyTestCase { * @throws Exception */ @Test - public void testShouldHandleUnprotectEmptyCipherText() throws Exception { + void testShouldHandleUnprotectEmptyCipherText() throws Exception { // Arrange final String IV_AND_DELIMITER = "${encoder.encodeToString("Bad IV value".getBytes(StandardCharsets.UTF_8))}||" logger.info("IV and delimiter: ${IV_AND_DELIMITER}") @@ -358,7 +358,7 @@ class AESSensitivePropertyProviderTest extends GroovyTestCase { } @Test - public void testShouldHandleUnprotectMalformedIV() throws Exception { + void testShouldHandleUnprotectMalformedIV() throws Exception { // Arrange final String PLAINTEXT = "This is a plaintext value" @@ -382,7 +382,7 @@ class AESSensitivePropertyProviderTest extends GroovyTestCase { } @Test - public void testShouldGetIdentifierKeyWithDifferentMaxKeyLengths() throws Exception { + void testShouldGetIdentifierKeyWithDifferentMaxKeyLengths() throws Exception { // Arrange def keys = getAvailableKeySizes().collectEntries { int keySize -> [(keySize): getKeyOfSize(keySize)] @@ -400,7 +400,7 @@ class AESSensitivePropertyProviderTest extends GroovyTestCase { } @Test - public void testShouldNotAllowEmptyKey() throws Exception { + void testShouldNotAllowEmptyKey() throws Exception { // Arrange final String INVALID_KEY = "" @@ -414,7 +414,7 @@ class AESSensitivePropertyProviderTest extends GroovyTestCase { } @Test - public void testShouldNotAllowIncorrectlySizedKey() throws Exception { + void testShouldNotAllowIncorrectlySizedKey() throws Exception { // Arrange final String INVALID_KEY = "Z" * 31 @@ -428,7 +428,7 @@ class AESSensitivePropertyProviderTest extends GroovyTestCase { } @Test - public void testShouldNotAllowInvalidKey() throws Exception { + void testShouldNotAllowInvalidKey() throws Exception { // Arrange final String INVALID_KEY = "Z" * 32 @@ -445,7 +445,7 @@ class AESSensitivePropertyProviderTest extends GroovyTestCase { * This test is to ensure internal consistency and allow for encrypting value for various property files */ @Test - public void testShouldEncryptArbitraryValues() { + void testShouldEncryptArbitraryValues() { // Arrange def values = ["thisIsABadPassword", "thisIsABadSensitiveKeyPassword", "thisIsABadKeystorePassword", "thisIsABadKeyPassword", "thisIsABadTruststorePassword", "This is an encrypted banner message", "nififtw!"] @@ -471,15 +471,15 @@ class AESSensitivePropertyProviderTest extends GroovyTestCase { * This test is to ensure external compatibility in case someone encodes the encrypted value with Base64 and does not remove the padding */ @Test - public void testShouldDecryptPaddedValue() { + void testShouldDecryptPaddedValue() { // Arrange Assume.assumeTrue("JCE unlimited strength crypto policy must be installed for this test", Cipher.getMaxAllowedKeyLength("AES") > 128) - final String EXPECTED_VALUE = "thisIsABadKeyPassword" - String cipherText = "ac/BaE35SL/esLiJ||+ULRvRLYdIDA2VqpE0eQXDEMjaLBMG2kbKOdOwBk/hGebDKlVg==" + final String EXPECTED_VALUE = getKeyOfSize(256) // "thisIsABadKeyPassword" + String cipherText = "aYDkDKys1ENr3gp+||sTBPpMlIvHcOLTGZlfWct8r9RY8BuDlDkoaYmGJ/9m9af9tZIVzcnDwvYQAaIKxRGF7vI2yrY7Xd6x9GTDnWGiGiRXlaP458BBMMgfzH2O8" String unpaddedCipherText = cipherText.replaceAll("=", "") - String key = getKeyOfSize(256) + String key = "AAAABBBBCCCCDDDDEEEEFFFF00001111" * 2 // getKeyOfSize(256) SensitivePropertyProvider spp = new AESSensitivePropertyProvider(key) diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/groovy/org/apache/nifi/properties/ProtectedNiFiPropertiesGroovyTest.groovy b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/groovy/org/apache/nifi/properties/ProtectedNiFiPropertiesGroovyTest.groovy index 0d5c976df4..6656867f9f 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/groovy/org/apache/nifi/properties/ProtectedNiFiPropertiesGroovyTest.groovy +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/groovy/org/apache/nifi/properties/ProtectedNiFiPropertiesGroovyTest.groovy @@ -38,7 +38,8 @@ class ProtectedNiFiPropertiesGroovyTest extends GroovyTestCase { "nifi.sensitive.props.key", "nifi.security.keystorePasswd", "nifi.security.keyPasswd", - "nifi.security.truststorePasswd" + "nifi.security.truststorePasswd", + "nifi.provenance.repository.encryption.key" ] final def COMMON_ADDITIONAL_SENSITIVE_PROPERTIES = [ @@ -53,7 +54,7 @@ class ProtectedNiFiPropertiesGroovyTest extends GroovyTestCase { private static String originalPropertiesPath = System.getProperty(NiFiProperties.PROPERTIES_FILE_PATH) @BeforeClass - public static void setUpOnce() throws Exception { + static void setUpOnce() throws Exception { Security.addProvider(new BouncyCastleProvider()) logger.metaClass.methodMissing = { String name, args -> @@ -62,15 +63,15 @@ class ProtectedNiFiPropertiesGroovyTest extends GroovyTestCase { } @Before - public void setUp() throws Exception { + void setUp() throws Exception { } @After - public void tearDown() throws Exception { + void tearDown() throws Exception { } @AfterClass - public static void tearDownOnce() { + static void tearDownOnce() { if (originalPropertiesPath) { System.setProperty(NiFiProperties.PROPERTIES_FILE_PATH, originalPropertiesPath) } @@ -127,7 +128,7 @@ class ProtectedNiFiPropertiesGroovyTest extends GroovyTestCase { } @Test - public void testConstructorShouldCreateNewInstance() throws Exception { + void testConstructorShouldCreateNewInstance() throws Exception { // Arrange // Act @@ -140,7 +141,7 @@ class ProtectedNiFiPropertiesGroovyTest extends GroovyTestCase { } @Test - public void testConstructorShouldAcceptRawProperties() throws Exception { + void testConstructorShouldAcceptRawProperties() throws Exception { // Arrange Properties rawProperties = new Properties() rawProperties.setProperty("key", "value") @@ -157,7 +158,7 @@ class ProtectedNiFiPropertiesGroovyTest extends GroovyTestCase { } @Test - public void testConstructorShouldAcceptNiFiProperties() throws Exception { + void testConstructorShouldAcceptNiFiProperties() throws Exception { // Arrange Properties rawProperties = new Properties() rawProperties.setProperty("key", "value") @@ -178,7 +179,7 @@ class ProtectedNiFiPropertiesGroovyTest extends GroovyTestCase { } @Test - public void testShouldAllowMultipleInstances() throws Exception { + void testShouldAllowMultipleInstances() throws Exception { // Arrange Properties rawProperties = new Properties() rawProperties.setProperty("key", "value") @@ -200,7 +201,7 @@ class ProtectedNiFiPropertiesGroovyTest extends GroovyTestCase { } @Test - public void testShouldDetectIfPropertyIsSensitive() throws Exception { + void testShouldDetectIfPropertyIsSensitive() throws Exception { // Arrange final String INSENSITIVE_PROPERTY_KEY = "nifi.ui.banner.text" final String SENSITIVE_PROPERTY_KEY = "nifi.security.keystorePasswd" @@ -219,7 +220,7 @@ class ProtectedNiFiPropertiesGroovyTest extends GroovyTestCase { } @Test - public void testShouldGetDefaultSensitiveProperties() throws Exception { + void testShouldGetDefaultSensitiveProperties() throws Exception { // Arrange logger.expected("${DEFAULT_SENSITIVE_PROPERTIES.size()} default sensitive properties: ${DEFAULT_SENSITIVE_PROPERTIES.join(", ")}") @@ -235,9 +236,9 @@ class ProtectedNiFiPropertiesGroovyTest extends GroovyTestCase { } @Test - public void testShouldGetAdditionalSensitiveProperties() throws Exception { + void testShouldGetAdditionalSensitiveProperties() throws Exception { // Arrange - def completeSensitiveProperties = DEFAULT_SENSITIVE_PROPERTIES + ["nifi.ui.banner.text"] + def completeSensitiveProperties = DEFAULT_SENSITIVE_PROPERTIES + ["nifi.ui.banner.text", "nifi.version"] logger.expected("${completeSensitiveProperties.size()} total sensitive properties: ${completeSensitiveProperties.join(", ")}") ProtectedNiFiProperties properties = loadFromFile("/conf/nifi_with_additional_sensitive_keys.properties") @@ -254,7 +255,7 @@ class ProtectedNiFiPropertiesGroovyTest extends GroovyTestCase { // TODO: Add negative tests (fuzz additional.keys property, etc.) @Test - public void testGetAdditionalSensitivePropertiesShouldNotIncludeSelf() throws Exception { + void testGetAdditionalSensitivePropertiesShouldNotIncludeSelf() throws Exception { // Arrange def completeSensitiveProperties = DEFAULT_SENSITIVE_PROPERTIES + ["nifi.ui.banner.text", "nifi.version"] logger.expected("${completeSensitiveProperties.size()} total sensitive properties: ${completeSensitiveProperties.join(", ")}") @@ -275,7 +276,7 @@ class ProtectedNiFiPropertiesGroovyTest extends GroovyTestCase { * @throws Exception */ @Test - public void testShouldGetUnprotectedValueOfSensitiveProperty() throws Exception { + void testShouldGetUnprotectedValueOfSensitiveProperty() throws Exception { // Arrange final String KEYSTORE_PASSWORD_KEY = "nifi.security.keystorePasswd" final String EXPECTED_KEYSTORE_PASSWORD = "thisIsABadKeystorePassword" @@ -301,7 +302,7 @@ class ProtectedNiFiPropertiesGroovyTest extends GroovyTestCase { * @throws Exception */ @Test - public void testShouldGetEmptyUnprotectedValueOfSensitiveProperty() throws Exception { + void testShouldGetEmptyUnprotectedValueOfSensitiveProperty() throws Exception { // Arrange final String TRUSTSTORE_PASSWORD_KEY = "nifi.security.truststorePasswd" final String EXPECTED_TRUSTSTORE_PASSWORD = "" @@ -329,7 +330,7 @@ class ProtectedNiFiPropertiesGroovyTest extends GroovyTestCase { * @throws Exception */ @Test - public void testShouldGetUnprotectedValueOfSensitivePropertyWhenProtected() throws Exception { + void testShouldGetUnprotectedValueOfSensitivePropertyWhenProtected() throws Exception { // Arrange final String KEYSTORE_PASSWORD_KEY = "nifi.security.keystorePasswd" final String EXPECTED_KEYSTORE_PASSWORD = "thisIsABadKeystorePassword" @@ -356,7 +357,7 @@ class ProtectedNiFiPropertiesGroovyTest extends GroovyTestCase { * @throws Exception */ @Test - public void testGetValueOfSensitivePropertyShouldHandleUnknownProtectionScheme() throws Exception { + void testGetValueOfSensitivePropertyShouldHandleUnknownProtectionScheme() throws Exception { // Arrange final String KEYSTORE_PASSWORD_KEY = "nifi.security.keystorePasswd" @@ -390,7 +391,7 @@ class ProtectedNiFiPropertiesGroovyTest extends GroovyTestCase { * @throws Exception */ @Test - public void testGetValueOfSensitivePropertyShouldHandleSingleMalformedValue() throws Exception { + void testGetValueOfSensitivePropertyShouldHandleSingleMalformedValue() throws Exception { // Arrange final String KEYSTORE_PASSWORD_KEY = "nifi.security.keystorePasswd" @@ -425,7 +426,7 @@ class ProtectedNiFiPropertiesGroovyTest extends GroovyTestCase { * @throws Exception */ @Test - public void testGetValueOfSensitivePropertyShouldHandleMultipleMalformedValues() throws Exception { + void testGetValueOfSensitivePropertyShouldHandleMultipleMalformedValues() throws Exception { // Arrange // Raw properties @@ -468,7 +469,7 @@ class ProtectedNiFiPropertiesGroovyTest extends GroovyTestCase { * @throws Exception */ @Test - public void testShouldGetEmptyUnprotectedValueOfSensitivePropertyWithDefault() throws Exception { + void testShouldGetEmptyUnprotectedValueOfSensitivePropertyWithDefault() throws Exception { // Arrange final String TRUSTSTORE_PASSWORD_KEY = "nifi.security.truststorePasswd" final String EXPECTED_TRUSTSTORE_PASSWORD = "" @@ -502,7 +503,7 @@ class ProtectedNiFiPropertiesGroovyTest extends GroovyTestCase { * @throws Exception */ @Test - public void testShouldGetUnprotectedValueOfSensitivePropertyWhenProtectedWithDefault() throws Exception { + void testShouldGetUnprotectedValueOfSensitivePropertyWhenProtectedWithDefault() throws Exception { // Arrange final String KEYSTORE_PASSWORD_KEY = "nifi.security.keystorePasswd" final String EXPECTED_KEYSTORE_PASSWORD = "thisIsABadKeystorePassword" @@ -538,7 +539,7 @@ class ProtectedNiFiPropertiesGroovyTest extends GroovyTestCase { * @throws Exception */ @Test - public void testGetValueOfSensitivePropertyShouldHandleInvalidatedInternalCache() throws Exception { + void testGetValueOfSensitivePropertyShouldHandleInvalidatedInternalCache() throws Exception { // Arrange final String KEYSTORE_PASSWORD_KEY = "nifi.security.keystorePasswd" final String EXPECTED_KEYSTORE_PASSWORD = "thisIsABadKeystorePassword" @@ -567,7 +568,7 @@ class ProtectedNiFiPropertiesGroovyTest extends GroovyTestCase { } @Test - public void testShouldDetectIfPropertyIsProtected() throws Exception { + void testShouldDetectIfPropertyIsProtected() throws Exception { // Arrange final String UNPROTECTED_PROPERTY_KEY = "nifi.security.truststorePasswd" final String PROTECTED_PROPERTY_KEY = "nifi.security.keystorePasswd" @@ -593,7 +594,7 @@ class ProtectedNiFiPropertiesGroovyTest extends GroovyTestCase { } @Test - public void testShouldDetectIfPropertyWithEmptyProtectionSchemeIsProtected() throws Exception { + void testShouldDetectIfPropertyWithEmptyProtectionSchemeIsProtected() throws Exception { // Arrange final String UNPROTECTED_PROPERTY_KEY = "nifi.sensitive.props.key" @@ -611,7 +612,7 @@ class ProtectedNiFiPropertiesGroovyTest extends GroovyTestCase { } @Test - public void testShouldGetPercentageOfSensitivePropertiesProtected_0() throws Exception { + void testShouldGetPercentageOfSensitivePropertiesProtected_0() throws Exception { // Arrange ProtectedNiFiProperties properties = loadFromFile("/conf/nifi.properties") @@ -620,14 +621,14 @@ class ProtectedNiFiPropertiesGroovyTest extends GroovyTestCase { // Act double percentProtected = properties.getPercentOfSensitivePropertiesProtected() - logger.info("${percentProtected}% (${properties.getProtectedPropertyKeys().size()} of ${properties.getSensitivePropertyKeys().size()}) protected") + logger.info("${percentProtected}% (${properties.getProtectedPropertyKeys().size()} of ${properties.getPopulatedSensitivePropertyKeys().size()}) protected") // Assert assert percentProtected == 0.0 } @Test - public void testShouldGetPercentageOfSensitivePropertiesProtected_50() throws Exception { + void testShouldGetPercentageOfSensitivePropertiesProtected_75() throws Exception { // Arrange ProtectedNiFiProperties properties = loadFromFile("/conf/nifi_with_sensitive_properties_protected_aes.properties") @@ -636,14 +637,14 @@ class ProtectedNiFiPropertiesGroovyTest extends GroovyTestCase { // Act double percentProtected = properties.getPercentOfSensitivePropertiesProtected() - logger.info("${percentProtected}% (${properties.getProtectedPropertyKeys().size()} of ${properties.getSensitivePropertyKeys().size()}) protected") + logger.info("${percentProtected}% (${properties.getProtectedPropertyKeys().size()} of ${properties.getPopulatedSensitivePropertyKeys().size()}) protected") // Assert - assert percentProtected == 50.0 + assert percentProtected == 75.0 } @Test - public void testShouldGetPercentageOfSensitivePropertiesProtected_100() throws Exception { + void testShouldGetPercentageOfSensitivePropertiesProtected_100() throws Exception { // Arrange ProtectedNiFiProperties properties = loadFromFile("/conf/nifi_with_all_sensitive_properties_protected_aes.properties") @@ -652,14 +653,14 @@ class ProtectedNiFiPropertiesGroovyTest extends GroovyTestCase { // Act double percentProtected = properties.getPercentOfSensitivePropertiesProtected() - logger.info("${percentProtected}% (${properties.getProtectedPropertyKeys().size()} of ${properties.getSensitivePropertyKeys().size()}) protected") + logger.info("${percentProtected}% (${properties.getProtectedPropertyKeys().size()} of ${properties.getPopulatedSensitivePropertyKeys().size()}) protected") // Assert assert percentProtected == 100.0 } @Test - public void testInstanceWithNoProtectedPropertiesShouldNotLoadSPP() throws Exception { + void testInstanceWithNoProtectedPropertiesShouldNotLoadSPP() throws Exception { // Arrange ProtectedNiFiProperties properties = loadFromFile("/conf/nifi.properties") assert properties.@localProviderCache?.isEmpty() @@ -676,7 +677,7 @@ class ProtectedNiFiPropertiesGroovyTest extends GroovyTestCase { } @Test - public void testShouldAddSensitivePropertyProvider() throws Exception { + void testShouldAddSensitivePropertyProvider() throws Exception { // Arrange ProtectedNiFiProperties properties = new ProtectedNiFiProperties() assert properties.getSensitivePropertyProviders().isEmpty() @@ -696,7 +697,7 @@ class ProtectedNiFiPropertiesGroovyTest extends GroovyTestCase { } @Test - public void testShouldNotAddNullSensitivePropertyProvider() throws Exception { + void testShouldNotAddNullSensitivePropertyProvider() throws Exception { // Arrange ProtectedNiFiProperties properties = new ProtectedNiFiProperties() assert properties.getSensitivePropertyProviders().isEmpty() @@ -713,7 +714,7 @@ class ProtectedNiFiPropertiesGroovyTest extends GroovyTestCase { } @Test - public void testShouldNotAllowOverwriteOfProvider() throws Exception { + void testShouldNotAllowOverwriteOfProvider() throws Exception { // Arrange ProtectedNiFiProperties properties = new ProtectedNiFiProperties() assert properties.getSensitivePropertyProviders().isEmpty() diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/groovy/org/apache/nifi/properties/StandardNiFiPropertiesGroovyTest.groovy b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/groovy/org/apache/nifi/properties/StandardNiFiPropertiesGroovyTest.groovy index c9492fb90f..ae43a3d6df 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/groovy/org/apache/nifi/properties/StandardNiFiPropertiesGroovyTest.groovy +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/groovy/org/apache/nifi/properties/StandardNiFiPropertiesGroovyTest.groovy @@ -32,58 +32,60 @@ class StandardNiFiPropertiesGroovyTest extends GroovyTestCase { private static final Logger logger = LoggerFactory.getLogger(StandardNiFiPropertiesGroovyTest.class) private static String originalPropertiesPath = System.getProperty(NiFiProperties.PROPERTIES_FILE_PATH) + private static final String PREK = NiFiProperties.PROVENANCE_REPO_ENCRYPTION_KEY + private static final String PREKID = NiFiProperties.PROVENANCE_REPO_ENCRYPTION_KEY_ID @BeforeClass - public static void setUpOnce() throws Exception { + static void setUpOnce() throws Exception { logger.metaClass.methodMissing = { String name, args -> logger.info("[${name?.toUpperCase()}] ${(args as List).join(" ")}") } } @Before - public void setUp() throws Exception { + void setUp() throws Exception { } @After - public void tearDown() throws Exception { + void tearDown() throws Exception { } @AfterClass - public static void tearDownOnce() { + static void tearDownOnce() { if (originalPropertiesPath) { System.setProperty(NiFiProperties.PROPERTIES_FILE_PATH, originalPropertiesPath) } } private static StandardNiFiProperties loadFromFile(String propertiesFilePath) { - String filePath; + String filePath try { - filePath = StandardNiFiPropertiesGroovyTest.class.getResource(propertiesFilePath).toURI().getPath(); + filePath = StandardNiFiPropertiesGroovyTest.class.getResource(propertiesFilePath).toURI().getPath() } catch (URISyntaxException ex) { throw new RuntimeException("Cannot load properties file due to " - + ex.getLocalizedMessage(), ex); + + ex.getLocalizedMessage(), ex) } - System.setProperty(NiFiProperties.PROPERTIES_FILE_PATH, filePath); + System.setProperty(NiFiProperties.PROPERTIES_FILE_PATH, filePath) - StandardNiFiProperties properties = new StandardNiFiProperties(); + StandardNiFiProperties properties = new StandardNiFiProperties() // clear out existing properties for (String prop : properties.stringPropertyNames()) { - properties.remove(prop); + properties.remove(prop) } - InputStream inStream = null; + InputStream inStream = null try { - inStream = new BufferedInputStream(new FileInputStream(filePath)); - properties.load(inStream); + inStream = new BufferedInputStream(new FileInputStream(filePath)) + properties.load(inStream) } catch (final Exception ex) { throw new RuntimeException("Cannot load properties file due to " - + ex.getLocalizedMessage(), ex); + + ex.getLocalizedMessage(), ex) } finally { if (null != inStream) { try { - inStream.close(); + inStream.close() } catch (Exception ex) { /** * do nothing * @@ -92,11 +94,11 @@ class StandardNiFiPropertiesGroovyTest extends GroovyTestCase { } } - return properties; + return properties } @Test - public void testConstructorShouldCreateNewInstance() throws Exception { + void testConstructorShouldCreateNewInstance() throws Exception { // Arrange // Act @@ -109,7 +111,7 @@ class StandardNiFiPropertiesGroovyTest extends GroovyTestCase { } @Test - public void testConstructorShouldAcceptRawProperties() throws Exception { + void testConstructorShouldAcceptRawProperties() throws Exception { // Arrange Properties rawProperties = new Properties() rawProperties.setProperty("key", "value") @@ -126,7 +128,7 @@ class StandardNiFiPropertiesGroovyTest extends GroovyTestCase { } @Test - public void testShouldAllowMultipleInstances() throws Exception { + void testShouldAllowMultipleInstances() throws Exception { // Arrange Properties rawProperties = new Properties() rawProperties.setProperty("key", "value") @@ -139,7 +141,6 @@ class StandardNiFiPropertiesGroovyTest extends GroovyTestCase { NiFiProperties emptyProperties = new StandardNiFiProperties() logger.info("emptyProperties has ${emptyProperties.size()} properties: ${emptyProperties.getPropertyKeys()}") - // Assert assert niFiProperties.size() == 1 assert niFiProperties.getPropertyKeys() == ["key"] as Set @@ -147,4 +148,178 @@ class StandardNiFiPropertiesGroovyTest extends GroovyTestCase { assert emptyProperties.size() == 0 assert emptyProperties.getPropertyKeys() == [] as Set } + + @Test + void testShouldGetProvenanceRepoEncryptionKeyFromDefaultProperty() throws Exception { + // Arrange + Properties rawProperties = new Properties() + final String KEY_ID = "arbitraryKeyId" + final String KEY_HEX = "0123456789ABCDEFFEDCBA9876543210" + rawProperties.setProperty(PREKID, KEY_ID) + rawProperties.setProperty(PREK, KEY_HEX) + NiFiProperties niFiProperties = new StandardNiFiProperties(rawProperties) + logger.info("niFiProperties has ${niFiProperties.size()} properties: ${niFiProperties.getPropertyKeys()}") + + // Act + def keyId = niFiProperties.getProvenanceRepoEncryptionKeyId() + def key = niFiProperties.getProvenanceRepoEncryptionKey() + def keys = niFiProperties.getProvenanceRepoEncryptionKeys() + + logger.info("Retrieved key ID: ${keyId}") + logger.info("Retrieved key: ${key}") + logger.info("Retrieved keys: ${keys}") + + // Assert + assert keyId == KEY_ID + assert key == KEY_HEX + assert keys == [(KEY_ID): KEY_HEX] + } + + @Test + void testShouldGetProvenanceRepoEncryptionKeysFromMultipleProperties() throws Exception { + // Arrange + Properties rawProperties = new Properties() + final String KEY_ID = "arbitraryKeyId" + final String KEY_HEX = "0123456789ABCDEFFEDCBA9876543210" + final String KEY_ID_2 = "arbitraryKeyId2" + final String KEY_HEX_2 = "AAAABBBBCCCCDDDDEEEEFFFF00001111" + final String KEY_ID_3 = "arbitraryKeyId3" + final String KEY_HEX_3 = "01010101010101010101010101010101" + + rawProperties.setProperty(PREKID, KEY_ID) + rawProperties.setProperty(PREK, KEY_HEX) + rawProperties.setProperty("${PREK}.id.${KEY_ID_2}", KEY_HEX_2) + rawProperties.setProperty("${PREK}.id.${KEY_ID_3}", KEY_HEX_3) + NiFiProperties niFiProperties = new StandardNiFiProperties(rawProperties) + logger.info("niFiProperties has ${niFiProperties.size()} properties: ${niFiProperties.getPropertyKeys()}") + + // Act + def keyId = niFiProperties.getProvenanceRepoEncryptionKeyId() + def key = niFiProperties.getProvenanceRepoEncryptionKey() + def keys = niFiProperties.getProvenanceRepoEncryptionKeys() + + logger.info("Retrieved key ID: ${keyId}") + logger.info("Retrieved key: ${key}") + logger.info("Retrieved keys: ${keys}") + + // Assert + assert keyId == KEY_ID + assert key == KEY_HEX + assert keys == [(KEY_ID): KEY_HEX, (KEY_ID_2): KEY_HEX_2, (KEY_ID_3): KEY_HEX_3] + } + + @Test + void testShouldGetProvenanceRepoEncryptionKeysWithNoDefaultDefined() throws Exception { + // Arrange + Properties rawProperties = new Properties() + final String KEY_ID = "arbitraryKeyId" + final String KEY_HEX = "0123456789ABCDEFFEDCBA9876543210" + final String KEY_ID_2 = "arbitraryKeyId2" + final String KEY_HEX_2 = "AAAABBBBCCCCDDDDEEEEFFFF00001111" + final String KEY_ID_3 = "arbitraryKeyId3" + final String KEY_HEX_3 = "01010101010101010101010101010101" + + rawProperties.setProperty(PREKID, KEY_ID) + rawProperties.setProperty("${PREK}.id.${KEY_ID}", KEY_HEX) + rawProperties.setProperty("${PREK}.id.${KEY_ID_2}", KEY_HEX_2) + rawProperties.setProperty("${PREK}.id.${KEY_ID_3}", KEY_HEX_3) + NiFiProperties niFiProperties = new StandardNiFiProperties(rawProperties) + logger.info("niFiProperties has ${niFiProperties.size()} properties: ${niFiProperties.getPropertyKeys()}") + + // Act + def keyId = niFiProperties.getProvenanceRepoEncryptionKeyId() + def key = niFiProperties.getProvenanceRepoEncryptionKey() + def keys = niFiProperties.getProvenanceRepoEncryptionKeys() + + logger.info("Retrieved key ID: ${keyId}") + logger.info("Retrieved key: ${key}") + logger.info("Retrieved keys: ${keys}") + + // Assert + assert keyId == KEY_ID + assert key == KEY_HEX + assert keys == [(KEY_ID): KEY_HEX, (KEY_ID_2): KEY_HEX_2, (KEY_ID_3): KEY_HEX_3] + } + + @Test + void testShouldGetProvenanceRepoEncryptionKeysWithNoneDefined() throws Exception { + // Arrange + Properties rawProperties = new Properties() + NiFiProperties niFiProperties = new StandardNiFiProperties(rawProperties) + logger.info("niFiProperties has ${niFiProperties.size()} properties: ${niFiProperties.getPropertyKeys()}") + + // Act + def keyId = niFiProperties.getProvenanceRepoEncryptionKeyId() + def key = niFiProperties.getProvenanceRepoEncryptionKey() + def keys = niFiProperties.getProvenanceRepoEncryptionKeys() + + logger.info("Retrieved key ID: ${keyId}") + logger.info("Retrieved key: ${key}") + logger.info("Retrieved keys: ${keys}") + + // Assert + assert keyId == null + assert key == null + assert keys == [:] + } + + @Test + void testShouldNotGetProvenanceRepoEncryptionKeysIfFileBasedKeyProvider() throws Exception { + // Arrange + Properties rawProperties = new Properties() + final String KEY_ID = "arbitraryKeyId" + + rawProperties.setProperty(PREKID, KEY_ID) + NiFiProperties niFiProperties = new StandardNiFiProperties(rawProperties) + logger.info("niFiProperties has ${niFiProperties.size()} properties: ${niFiProperties.getPropertyKeys()}") + + // Act + def keyId = niFiProperties.getProvenanceRepoEncryptionKeyId() + def key = niFiProperties.getProvenanceRepoEncryptionKey() + def keys = niFiProperties.getProvenanceRepoEncryptionKeys() + + logger.info("Retrieved key ID: ${keyId}") + logger.info("Retrieved key: ${key}") + logger.info("Retrieved keys: ${keys}") + + // Assert + assert keyId == KEY_ID + assert key == null + assert keys == [:] + } + + @Test + void testGetProvenanceRepoEncryptionKeysShouldFilterOtherProperties() throws Exception { + // Arrange + Properties rawProperties = new Properties() + final String KEY_ID = "arbitraryKeyId" + final String KEY_HEX = "0123456789ABCDEFFEDCBA9876543210" + final String KEY_ID_2 = "arbitraryKeyId2" + final String KEY_HEX_2 = "AAAABBBBCCCCDDDDEEEEFFFF00001111" + final String KEY_ID_3 = "arbitraryKeyId3" + final String KEY_HEX_3 = "01010101010101010101010101010101" + + rawProperties.setProperty(PREKID, KEY_ID) + rawProperties.setProperty("${PREK}.id.${KEY_ID}", KEY_HEX) + rawProperties.setProperty("${PREK}.id.${KEY_ID_2}", KEY_HEX_2) + rawProperties.setProperty("${PREK}.id.${KEY_ID_3}", KEY_HEX_3) + rawProperties.setProperty(NiFiProperties.PROVENANCE_REPO_ENCRYPTION_KEY_PROVIDER_IMPLEMENTATION_CLASS, "some.class.provider") + rawProperties.setProperty(NiFiProperties.PROVENANCE_REPO_ENCRYPTION_KEY_PROVIDER_LOCATION, "some://url") + NiFiProperties niFiProperties = new StandardNiFiProperties(rawProperties) + logger.info("niFiProperties has ${niFiProperties.size()} properties: ${niFiProperties.getPropertyKeys()}") + + // Act + def keyId = niFiProperties.getProvenanceRepoEncryptionKeyId() + def key = niFiProperties.getProvenanceRepoEncryptionKey() + def keys = niFiProperties.getProvenanceRepoEncryptionKeys() + + logger.info("Retrieved key ID: ${keyId}") + logger.info("Retrieved key: ${key}") + logger.info("Retrieved keys: ${keys}") + + // Assert + assert keyId == KEY_ID + assert key == KEY_HEX + assert keys == [(KEY_ID): KEY_HEX, (KEY_ID_2): KEY_HEX_2, (KEY_ID_3): KEY_HEX_3] + } } diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/resources/conf/nifi_with_additional_sensitive_keys.properties b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/resources/conf/nifi_with_additional_sensitive_keys.properties index f775d83cec..6a88c25569 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/resources/conf/nifi_with_additional_sensitive_keys.properties +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/resources/conf/nifi_with_additional_sensitive_keys.properties @@ -73,7 +73,7 @@ nifi.web.jetty.working.directory=./target/work/jetty nifi.sensitive.props.key=key nifi.sensitive.props.algorithm=PBEWITHMD5AND256BITAES-CBC-OPENSSL nifi.sensitive.props.provider=BC -nifi.sensitive.props.additional.keys=nifi.ui.banner.text +nifi.sensitive.props.additional.keys=nifi.ui.banner.text, nifi.version, nifi.sensitive.props.additional.keys nifi.security.keystore= nifi.security.keystoreType= diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-resources/pom.xml b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-resources/pom.xml index d6b1aafbfe..0baaed756c 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-resources/pom.xml +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-resources/pom.xml @@ -91,6 +91,11 @@ org.apache.nifi.provenance.PersistentProvenanceRepository + 1_000_000 + + + + ./provenance_repository 24 hours 1 GB diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-resources/src/main/resources/conf/nifi.properties b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-resources/src/main/resources/conf/nifi.properties index 62b4c8f85d..dadc5e6dd1 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-resources/src/main/resources/conf/nifi.properties +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-resources/src/main/resources/conf/nifi.properties @@ -81,6 +81,11 @@ nifi.content.viewer.url=${nifi.content.viewer.url} # Provenance Repository Properties nifi.provenance.repository.implementation=${nifi.provenance.repository.implementation} +nifi.provenance.repository.debug.frequency=${nifi.provenance.repository.debug.frequency} +nifi.provenance.repository.encryption.key.provider.implementation=${nifi.provenance.repository.encryption.key.provider.implementation} +nifi.provenance.repository.encryption.key.provider.location=${nifi.provenance.repository.encryption.key.provider.location} +nifi.provenance.repository.encryption.key.id=${nifi.provenance.repository.encryption.key.id} +nifi.provenance.repository.encryption.key=${nifi.provenance.repository.encryption.key} # Persistent Provenance Repository Properties nifi.provenance.repository.directory.default=${nifi.provenance.repository.directory.default} diff --git a/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-processors/src/test/resources/logback-test.xml b/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-processors/src/test/resources/logback-test.xml new file mode 100644 index 0000000000..80b8b49c70 --- /dev/null +++ b/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-processors/src/test/resources/logback-test.xml @@ -0,0 +1,32 @@ + + + + + + + %-4r [%t] %-5p %c - %m%n + + + + + + + + + + + + diff --git a/nifi-nar-bundles/nifi-poi-bundle/nifi-poi-processors/src/test/resources/logback-test.xml b/nifi-nar-bundles/nifi-poi-bundle/nifi-poi-processors/src/test/resources/logback-test.xml new file mode 100644 index 0000000000..5afbc8ea75 --- /dev/null +++ b/nifi-nar-bundles/nifi-poi-bundle/nifi-poi-processors/src/test/resources/logback-test.xml @@ -0,0 +1,32 @@ + + + + + + + %-4r [%t] %-5p %c - %m%n + + + + + + + + + + + + diff --git a/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/pom.xml b/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/pom.xml index 4db4169ff0..8fe5dbfcef 100644 --- a/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/pom.xml +++ b/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/pom.xml @@ -63,5 +63,10 @@ commons-lang3 test + + org.bouncycastle + bcprov-jdk15on + test + diff --git a/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/EncryptedSchemaRecordReader.java b/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/EncryptedSchemaRecordReader.java new file mode 100644 index 0000000000..fcd7fee53e --- /dev/null +++ b/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/EncryptedSchemaRecordReader.java @@ -0,0 +1,154 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.provenance; + +import java.io.ByteArrayInputStream; +import java.io.DataInputStream; +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.util.Collection; +import java.util.Optional; +import java.util.concurrent.TimeUnit; +import org.apache.nifi.provenance.schema.LookupTableEventRecord; +import org.apache.nifi.provenance.toc.TocReader; +import org.apache.nifi.repository.schema.Record; +import org.apache.nifi.stream.io.LimitingInputStream; +import org.apache.nifi.stream.io.StreamUtils; +import org.apache.nifi.util.timebuffer.LongEntityAccess; +import org.apache.nifi.util.timebuffer.TimedBuffer; +import org.apache.nifi.util.timebuffer.TimestampedLong; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class EncryptedSchemaRecordReader extends EventIdFirstSchemaRecordReader { + private static final Logger logger = LoggerFactory.getLogger(EncryptedSchemaRecordReader.class); + + private static final int DEFAULT_DEBUG_FREQUENCY = 1_000_000; + + private ProvenanceEventEncryptor provenanceEventEncryptor; + + private static final TimedBuffer decryptTimes = new TimedBuffer<>(TimeUnit.SECONDS, 60, new LongEntityAccess()); + + private int debugFrequency = DEFAULT_DEBUG_FREQUENCY; + public static final int SERIALIZATION_VERSION = 1; + + public static final String SERIALIZATION_NAME = "EncryptedSchemaRecordWriter"; + + public EncryptedSchemaRecordReader(final InputStream inputStream, final String filename, final TocReader tocReader, final int maxAttributeChars, + ProvenanceEventEncryptor provenanceEventEncryptor) throws IOException { + this(inputStream, filename, tocReader, maxAttributeChars, provenanceEventEncryptor, DEFAULT_DEBUG_FREQUENCY); + } + + public EncryptedSchemaRecordReader(final InputStream inputStream, final String filename, final TocReader tocReader, final int maxAttributeChars, + ProvenanceEventEncryptor provenanceEventEncryptor, int debugFrequency) throws IOException { + super(inputStream, filename, tocReader, maxAttributeChars); + this.provenanceEventEncryptor = provenanceEventEncryptor; + this.debugFrequency = debugFrequency; + } + + @Override + protected StandardProvenanceEventRecord nextRecord(final DataInputStream in, final int serializationVersion) throws IOException { + verifySerializationVersion(serializationVersion); + + final long byteOffset = getBytesConsumed(); + final long eventId = in.readInt() + getFirstEventId(); + final int recordLength = in.readInt(); + + return readRecord(in, eventId, byteOffset, recordLength); + } + + private StandardProvenanceEventRecord readRecord(final DataInputStream inputStream, final long eventId, final long startOffset, final int recordLength) throws IOException { + try { + final InputStream limitedIn = new LimitingInputStream(inputStream, recordLength); + + byte[] encryptedSerializedBytes = new byte[recordLength]; + DataInputStream encryptedInputStream = new DataInputStream(limitedIn); + encryptedInputStream.readFully(encryptedSerializedBytes); + + byte[] plainSerializedBytes = decrypt(encryptedSerializedBytes, Long.toString(eventId)); + InputStream plainStream = new ByteArrayInputStream(plainSerializedBytes); + + final Record eventRecord = getRecordReader().readRecord(plainStream); + if (eventRecord == null) { + return null; + } + + final StandardProvenanceEventRecord deserializedEvent = LookupTableEventRecord.getEvent(eventRecord, getFilename(), startOffset, getMaxAttributeLength(), + getFirstEventId(), getSystemTimeOffset(), getComponentIds(), getComponentTypes(), getQueueIds(), getEventTypes()); + deserializedEvent.setEventId(eventId); + return deserializedEvent; + } catch (EncryptionException e) { + logger.error("Encountered an error reading the record: ", e); + throw new IOException(e); + } + } + + // TODO: Copied from EventIdFirstSchemaRecordReader to force local/overridden readRecord() + @Override + protected Optional readToEvent(final long eventId, final DataInputStream dis, final int serializationVersion) throws IOException { + verifySerializationVersion(serializationVersion); + + while (isData(dis)) { + final long startOffset = getBytesConsumed(); + final long id = dis.readInt() + getFirstEventId(); + final int recordLength = dis.readInt(); + + if (id >= eventId) { + final StandardProvenanceEventRecord event = readRecord(dis, id, startOffset, recordLength); + return Optional.ofNullable(event); + } else { + // This is not the record we want. Skip over it instead of deserializing it. + StreamUtils.skip(dis, recordLength); + } + } + + return Optional.empty(); + } + + private byte[] decrypt(byte[] encryptedBytes, String eventId) throws IOException, EncryptionException { + try { + return provenanceEventEncryptor.decrypt(encryptedBytes, eventId); + } catch (Exception e) { + logger.error("Encountered an error: ", e); + throw new EncryptionException(e); + } + } + + @Override + public String toString() { + return getDescription(); + } + + private String getDescription() { + try { + return "EncryptedSchemaRecordReader, toc: " + getTocReader().getFile().getAbsolutePath() + ", journal: " + getFilename(); + } catch (Exception e) { + return "EncryptedSchemaRecordReader@" + Integer.toHexString(this.hashCode()); + } + } + + /** + * Sets the encryptor to use (necessary because the + * {@link org.apache.nifi.provenance.serialization.RecordReaders#newRecordReader(File, Collection, int)} method doesn't accept the encryptor. + * + * @param provenanceEventEncryptor the encryptor + */ + void setProvenanceEventEncryptor(ProvenanceEventEncryptor provenanceEventEncryptor) { + this.provenanceEventEncryptor = provenanceEventEncryptor; + } +} diff --git a/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/EncryptedSchemaRecordWriter.java b/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/EncryptedSchemaRecordWriter.java new file mode 100644 index 0000000000..f84ca4891d --- /dev/null +++ b/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/EncryptedSchemaRecordWriter.java @@ -0,0 +1,199 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.provenance; + +import java.io.ByteArrayOutputStream; +import java.io.DataOutputStream; +import java.io.File; +import java.io.IOException; +import java.security.KeyManagementException; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicLong; +import org.apache.nifi.provenance.serialization.StorageSummary; +import org.apache.nifi.provenance.toc.TocWriter; +import org.apache.nifi.util.timebuffer.LongEntityAccess; +import org.apache.nifi.util.timebuffer.TimedBuffer; +import org.apache.nifi.util.timebuffer.TimestampedLong; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class EncryptedSchemaRecordWriter extends EventIdFirstSchemaRecordWriter { + private static final Logger logger = LoggerFactory.getLogger(EncryptedSchemaRecordWriter.class); + + private static final int DEFAULT_DEBUG_FREQUENCY = 1_000_000; + + private ProvenanceEventEncryptor provenanceEventEncryptor; + + private static final TimedBuffer encryptTimes = new TimedBuffer<>(TimeUnit.SECONDS, 60, new LongEntityAccess()); + + private String keyId; + + private int debugFrequency; + public static final int SERIALIZATION_VERSION = 1; + + public static final String SERIALIZATION_NAME = "EncryptedSchemaRecordWriter"; + + public EncryptedSchemaRecordWriter(final File file, final AtomicLong idGenerator, final TocWriter writer, final boolean compressed, + final int uncompressedBlockSize, final IdentifierLookup idLookup, + ProvenanceEventEncryptor provenanceEventEncryptor) throws IOException, EncryptionException { + this(file, idGenerator, writer, compressed, uncompressedBlockSize, idLookup, provenanceEventEncryptor, DEFAULT_DEBUG_FREQUENCY); + } + + public EncryptedSchemaRecordWriter(final File file, final AtomicLong idGenerator, final TocWriter writer, final boolean compressed, + final int uncompressedBlockSize, final IdentifierLookup idLookup, + ProvenanceEventEncryptor provenanceEventEncryptor, int debugFrequency) throws IOException, EncryptionException { + super(file, idGenerator, writer, compressed, uncompressedBlockSize, idLookup); + this.provenanceEventEncryptor = provenanceEventEncryptor; + this.debugFrequency = debugFrequency; + + try { + this.keyId = getNextAvailableKeyId(); + } catch (KeyManagementException e) { + logger.error("Encountered an error initializing the encrypted schema record writer because the provided encryptor has no valid keys available: ", e); + throw new EncryptionException("No valid keys in the provenance event encryptor", e); + } + } + + @Override + public StorageSummary writeRecord(final ProvenanceEventRecord record) throws IOException { + final long encryptStart = System.nanoTime(); + byte[] cipherBytes; + try { + byte[] serialized; + try (final ByteArrayOutputStream baos = new ByteArrayOutputStream(256); + final DataOutputStream dos = new DataOutputStream(baos)) { + writeRecord(record, 0L, dos); + serialized = baos.toByteArray(); + } + String eventId = record.getBestEventIdentifier(); + cipherBytes = encrypt(serialized, eventId); + } catch (EncryptionException e) { + logger.error("Encountered an error: ", e); + throw new IOException("Error encrypting the provenance record", e); + } + final long encryptStop = System.nanoTime(); + + final long lockStart = System.nanoTime(); + final long writeStart; + final long startBytes; + final long endBytes; + final long recordIdentifier; + synchronized (this) { + writeStart = System.nanoTime(); + try { + recordIdentifier = record.getEventId() == -1L ? getIdGenerator().getAndIncrement() : record.getEventId(); + startBytes = getBytesWritten(); + + ensureStreamState(recordIdentifier, startBytes); + + final DataOutputStream out = getBufferedOutputStream(); + final int recordIdOffset = (int) (recordIdentifier - getFirstEventId()); + out.writeInt(recordIdOffset); + out.writeInt(cipherBytes.length); + out.write(cipherBytes); + + getRecordCount().incrementAndGet(); + endBytes = getBytesWritten(); + } catch (final IOException ioe) { + markDirty(); + throw ioe; + } + } + + if (logger.isDebugEnabled()) { + // Collect stats and periodically dump them if log level is set to at least info. + final long writeNanos = System.nanoTime() - writeStart; + getWriteTimes().add(new TimestampedLong(writeNanos)); + + final long serializeNanos = lockStart - encryptStart; + getSerializeTimes().add(new TimestampedLong(serializeNanos)); + + final long encryptNanos = encryptStop - encryptStart; + getEncryptTimes().add(new TimestampedLong(encryptNanos)); + + final long lockNanos = writeStart - lockStart; + getLockTimes().add(new TimestampedLong(lockNanos)); + getBytesWrittenBuffer().add(new TimestampedLong(endBytes - startBytes)); + + final long recordCount = getTotalRecordCount().incrementAndGet(); + if (recordCount % debugFrequency == 0) { + printStats(); + } + } + + final long serializedLength = endBytes - startBytes; + final TocWriter tocWriter = getTocWriter(); + final Integer blockIndex = tocWriter == null ? null : tocWriter.getCurrentBlockIndex(); + final File file = getFile(); + final String storageLocation = file.getParentFile().getName() + "/" + file.getName(); + return new StorageSummary(recordIdentifier, storageLocation, blockIndex, serializedLength, endBytes); + } + + private void printStats() { + final long sixtySecondsAgo = System.currentTimeMillis() - 60000L; + final Long writeNanosLast60 = getWriteTimes().getAggregateValue(sixtySecondsAgo).getValue(); + final Long lockNanosLast60 = getLockTimes().getAggregateValue(sixtySecondsAgo).getValue(); + final Long serializeNanosLast60 = getSerializeTimes().getAggregateValue(sixtySecondsAgo).getValue(); + final Long encryptNanosLast60 = getEncryptTimes().getAggregateValue(sixtySecondsAgo).getValue(); + final Long bytesWrittenLast60 = getBytesWrittenBuffer().getAggregateValue(sixtySecondsAgo).getValue(); + logger.debug("In the last 60 seconds, have spent {} millis writing to file ({} MB), {} millis waiting on synchronize block, {} millis serializing events, {} millis encrypting events", + TimeUnit.NANOSECONDS.toMillis(writeNanosLast60), + bytesWrittenLast60 / 1024 / 1024, + TimeUnit.NANOSECONDS.toMillis(lockNanosLast60), + TimeUnit.NANOSECONDS.toMillis(serializeNanosLast60), + TimeUnit.NANOSECONDS.toMillis(encryptNanosLast60)); + } + + static TimedBuffer getEncryptTimes() { + return encryptTimes; + } + + private byte[] encrypt(byte[] serialized, String eventId) throws IOException, EncryptionException { + String keyId = getKeyId(); + try { + return provenanceEventEncryptor.encrypt(serialized, eventId, keyId); + } catch (Exception e) { + logger.error("Encountered an error: ", e); + throw new EncryptionException(e); + } + } + + private String getNextAvailableKeyId() throws KeyManagementException { + return provenanceEventEncryptor.getNextKeyId(); + } + + @Override + protected int getSerializationVersion() { + return SERIALIZATION_VERSION; + } + + @Override + protected String getSerializationName() { + return SERIALIZATION_NAME; + } + + public String getKeyId() { + return keyId; + } + + @Override + public String toString() { + return "EncryptedSchemaRecordWriter" + + " using " + provenanceEventEncryptor + + " and current keyId " + keyId; + } +} diff --git a/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/EncryptedWriteAheadProvenanceRepository.java b/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/EncryptedWriteAheadProvenanceRepository.java new file mode 100644 index 0000000000..a2d455bc53 --- /dev/null +++ b/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/EncryptedWriteAheadProvenanceRepository.java @@ -0,0 +1,159 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.provenance; + +import java.io.IOException; +import java.security.KeyManagementException; +import java.util.Map; +import java.util.stream.Collectors; +import javax.crypto.SecretKey; +import org.apache.nifi.authorization.Authorizer; +import org.apache.nifi.events.EventReporter; +import org.apache.nifi.provenance.serialization.RecordReaders; +import org.apache.nifi.provenance.store.EventFileManager; +import org.apache.nifi.provenance.store.RecordReaderFactory; +import org.apache.nifi.provenance.store.RecordWriterFactory; +import org.apache.nifi.provenance.toc.StandardTocWriter; +import org.apache.nifi.provenance.toc.TocUtil; +import org.apache.nifi.provenance.toc.TocWriter; +import org.apache.nifi.util.NiFiProperties; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class EncryptedWriteAheadProvenanceRepository extends WriteAheadProvenanceRepository { + private static final Logger logger = LoggerFactory.getLogger(EncryptedWriteAheadProvenanceRepository.class); + + /** + * This constructor exists solely for the use of the Java Service Loader mechanism and should not be used. + */ + public EncryptedWriteAheadProvenanceRepository() { + super(); + } + + public EncryptedWriteAheadProvenanceRepository(final NiFiProperties nifiProperties) { + super(RepositoryConfiguration.create(nifiProperties)); + } + + public EncryptedWriteAheadProvenanceRepository(final RepositoryConfiguration config) { + super(config); + } + + /** + * This method initializes the repository. It first builds the key provider and event encryptor + * from the config values, then creates the encrypted record writer and reader, then delegates + * back to the superclass for the common implementation. + * + * @param eventReporter the event reporter + * @param authorizer the authorizer + * @param resourceFactory the authorizable factory + * @param idLookup the lookup provider + * @throws IOException if there is an error initializing this repository + */ + @Override + public synchronized void initialize(final EventReporter eventReporter, final Authorizer authorizer, final ProvenanceAuthorizableFactory resourceFactory, + final IdentifierLookup idLookup) throws IOException { + // Initialize the encryption-specific fields + ProvenanceEventEncryptor provenanceEventEncryptor; + if (getConfig().supportsEncryption()) { + try { + KeyProvider keyProvider = buildKeyProvider(); + provenanceEventEncryptor = new AESProvenanceEventEncryptor(); + provenanceEventEncryptor.initialize(keyProvider); + } catch (KeyManagementException e) { + String msg = "Encountered an error building the key provider"; + logger.error(msg, e); + throw new IOException(msg, e); + } + } else { + throw new IOException("The provided configuration does not support a encrypted repository"); + } + + // Build a factory using lambda which injects the encryptor + final RecordWriterFactory recordWriterFactory = (file, idGenerator, compressed, createToc) -> { + try { + final TocWriter tocWriter = createToc ? new StandardTocWriter(TocUtil.getTocFile(file), false, false) : null; + return new EncryptedSchemaRecordWriter(file, idGenerator, tocWriter, compressed, BLOCK_SIZE, idLookup, provenanceEventEncryptor, getConfig().getDebugFrequency()); + } catch (EncryptionException e) { + logger.error("Encountered an error building the schema record writer factory: ", e); + throw new IOException(e); + } + }; + + // Build a factory using lambda which injects the encryptor + final EventFileManager fileManager = new EventFileManager(); + final RecordReaderFactory recordReaderFactory = (file, logs, maxChars) -> { + fileManager.obtainReadLock(file); + try { + EncryptedSchemaRecordReader tempReader = (EncryptedSchemaRecordReader) RecordReaders.newRecordReader(file, logs, maxChars); + tempReader.setProvenanceEventEncryptor(provenanceEventEncryptor); + return tempReader; + } finally { + fileManager.releaseReadLock(file); + } + }; + + // Delegate the init to the parent impl + super.init(recordWriterFactory, recordReaderFactory, eventReporter, authorizer, resourceFactory); + } + + private KeyProvider buildKeyProvider() throws KeyManagementException { + RepositoryConfiguration config = super.getConfig(); + if (config == null) { + throw new KeyManagementException("The repository configuration is missing"); + } + + final String implementationClassName = config.getKeyProviderImplementation(); + if (implementationClassName == null) { + throw new KeyManagementException("Cannot create Key Provider because the NiFi Properties is missing the following property: " + + NiFiProperties.PROVENANCE_REPO_ENCRYPTION_KEY_PROVIDER_IMPLEMENTATION_CLASS); + } + + // TODO: Extract to factory + KeyProvider keyProvider; + if (StaticKeyProvider.class.getName().equals(implementationClassName)) { + // Get all the keys (map) from config + if (CryptoUtils.isValidKeyProvider(implementationClassName, config.getKeyProviderLocation(), config.getKeyId(), config.getEncryptionKeys())) { + Map formedKeys = config.getEncryptionKeys().entrySet().stream() + .collect(Collectors.toMap( + Map.Entry::getKey, + e -> { + try { + return CryptoUtils.formKeyFromHex(e.getValue()); + } catch (KeyManagementException e1) { + // This should never happen because the hex has already been validated + logger.error("Encountered an error: ", e1); + return null; + } + })); + keyProvider = new StaticKeyProvider(formedKeys); + } else { + final String msg = "The StaticKeyProvider definition is not valid"; + logger.error(msg); + throw new KeyManagementException(msg); + } + } else if (FileBasedKeyProvider.class.getName().equals(implementationClassName)) { + keyProvider = new FileBasedKeyProvider(config.getKeyProviderLocation()); + if (!keyProvider.keyExists(config.getKeyId())) { + throw new KeyManagementException("The specified key ID " + config.getKeyId() + " is not in the key definition file"); + } + } else { + throw new KeyManagementException("Invalid key provider implementation provided: " + implementationClassName); + } + + return keyProvider; + } +} diff --git a/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/EventIdFirstSchemaRecordReader.java b/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/EventIdFirstSchemaRecordReader.java index 612b6c8c37..bd85846517 100644 --- a/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/EventIdFirstSchemaRecordReader.java +++ b/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/EventIdFirstSchemaRecordReader.java @@ -23,7 +23,6 @@ import java.io.IOException; import java.io.InputStream; import java.util.List; import java.util.Optional; - import org.apache.nifi.provenance.schema.EventIdFirstHeaderSchema; import org.apache.nifi.provenance.schema.LookupTableEventRecord; import org.apache.nifi.provenance.serialization.CompressableRecordReader; @@ -35,6 +34,14 @@ import org.apache.nifi.stream.io.LimitingInputStream; import org.apache.nifi.stream.io.StreamUtils; public class EventIdFirstSchemaRecordReader extends CompressableRecordReader { + RecordSchema getSchema() { + return schema; + } + + SchemaRecordReader getRecordReader() { + return recordReader; + } + private RecordSchema schema; // effectively final private SchemaRecordReader recordReader; // effectively final @@ -43,16 +50,41 @@ public class EventIdFirstSchemaRecordReader extends CompressableRecordReader { private List queueIds; private List eventTypes; private long firstEventId; + + List getComponentIds() { + return componentIds; + } + + List getComponentTypes() { + return componentTypes; + } + + List getQueueIds() { + return queueIds; + } + + List getEventTypes() { + return eventTypes; + } + + long getFirstEventId() { + return firstEventId; + } + + long getSystemTimeOffset() { + return systemTimeOffset; + } + private long systemTimeOffset; public EventIdFirstSchemaRecordReader(final InputStream in, final String filename, final TocReader tocReader, final int maxAttributeChars) throws IOException { super(in, filename, tocReader, maxAttributeChars); } - private void verifySerializationVersion(final int serializationVersion) { + protected void verifySerializationVersion(final int serializationVersion) { if (serializationVersion > EventIdFirstSchemaRecordWriter.SERIALIZATION_VERSION) { throw new IllegalArgumentException("Unable to deserialize record because the version is " + serializationVersion - + " and supported versions are 1-" + EventIdFirstSchemaRecordWriter.SERIALIZATION_VERSION); + + " and supported versions are 1-" + EventIdFirstSchemaRecordWriter.SERIALIZATION_VERSION); } } @@ -109,12 +141,12 @@ public class EventIdFirstSchemaRecordReader extends CompressableRecordReader { } final StandardProvenanceEventRecord deserializedEvent = LookupTableEventRecord.getEvent(eventRecord, getFilename(), startOffset, getMaxAttributeLength(), - firstEventId, systemTimeOffset, componentIds, componentTypes, queueIds, eventTypes); + firstEventId, systemTimeOffset, componentIds, componentTypes, queueIds, eventTypes); deserializedEvent.setEventId(eventId); return deserializedEvent; } - private boolean isData(final InputStream in) throws IOException { + protected boolean isData(final InputStream in) throws IOException { in.mark(1); final int nextByte = in.read(); in.reset(); @@ -142,4 +174,17 @@ public class EventIdFirstSchemaRecordReader extends CompressableRecordReader { return Optional.empty(); } + + @Override + public String toString() { + return getDescription(); + } + + private String getDescription() { + try { + return "EventIdFirstSchemaRecordReader, toc: " + getTocReader().getFile().getAbsolutePath() + ", journal: " + getFilename(); + } catch (Exception e) { + return "EventIdFirstSchemaRecordReader@" + Integer.toHexString(this.hashCode()); + } + } } diff --git a/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/EventIdFirstSchemaRecordWriter.java b/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/EventIdFirstSchemaRecordWriter.java index bb8d52fef3..8f5b2b2103 100644 --- a/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/EventIdFirstSchemaRecordWriter.java +++ b/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/EventIdFirstSchemaRecordWriter.java @@ -29,7 +29,6 @@ import java.util.Map; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicLong; - import org.apache.nifi.provenance.schema.EventFieldNames; import org.apache.nifi.provenance.schema.EventIdFirstHeaderSchema; import org.apache.nifi.provenance.schema.LookupTableEventRecord; @@ -238,4 +237,46 @@ public class EventIdFirstSchemaRecordWriter extends CompressableRecordWriter { return SERIALIZATION_NAME; } + /* Getters for internal state written to by subclass EncryptedSchemaRecordWriter */ + + IdentifierLookup getIdLookup() { + return idLookup; + } + + SchemaRecordWriter getSchemaRecordWriter() { + return schemaRecordWriter; + } + + AtomicInteger getRecordCount() { + return recordCount; + } + + static TimedBuffer getSerializeTimes() { + return serializeTimes; + } + + static TimedBuffer getLockTimes() { + return lockTimes; + } + + static TimedBuffer getWriteTimes() { + return writeTimes; + } + + static TimedBuffer getBytesWrittenBuffer() { + return bytesWritten; + } + + static AtomicLong getTotalRecordCount() { + return totalRecordCount; + } + + long getFirstEventId() { + return firstEventId; + } + + long getSystemTimeOffset() { + return systemTimeOffset; + } + } diff --git a/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/RepositoryConfiguration.java b/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/RepositoryConfiguration.java index 7a2f57e37a..5a75172552 100644 --- a/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/RepositoryConfiguration.java +++ b/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/RepositoryConfiguration.java @@ -26,7 +26,6 @@ import java.util.List; import java.util.Map; import java.util.Optional; import java.util.concurrent.TimeUnit; - import org.apache.nifi.processor.DataUnit; import org.apache.nifi.provenance.search.SearchableField; import org.apache.nifi.util.FormatUtils; @@ -50,6 +49,12 @@ public class RepositoryConfiguration { private int journalCount = 16; private int compressionBlockBytes = 1024 * 1024; private int maxAttributeChars = 65536; + private int debugFrequency = 1_000_000; + + private Map encryptionKeys; + private String keyId; + private String keyProviderImplementation; + private String keyProviderLocation; private List searchableFields = new ArrayList<>(); private List searchableAttributes = new ArrayList<>(); @@ -360,6 +365,54 @@ public class RepositoryConfiguration { return Optional.ofNullable(warmCacheFrequencyMinutes); } + public boolean supportsEncryption() { + boolean keyProviderIsConfigured = CryptoUtils.isValidKeyProvider(keyProviderImplementation, keyProviderLocation, keyId, encryptionKeys); + + return keyProviderIsConfigured; + } + + public Map getEncryptionKeys() { + return encryptionKeys; + } + + public void setEncryptionKeys(Map encryptionKeys) { + this.encryptionKeys = encryptionKeys; + } + + public String getKeyId() { + return keyId; + } + + public void setKeyId(String keyId) { + this.keyId = keyId; + } + + public String getKeyProviderImplementation() { + return keyProviderImplementation; + } + + public void setKeyProviderImplementation(String keyProviderImplementation) { + this.keyProviderImplementation = keyProviderImplementation; + } + + public String getKeyProviderLocation() { + return keyProviderLocation; + } + + public void setKeyProviderLocation(String keyProviderLocation) { + this.keyProviderLocation = keyProviderLocation; + } + + + public int getDebugFrequency() { + return debugFrequency; + } + + public void setDebugFrequency(int debugFrequency) { + this.debugFrequency = debugFrequency; + } + + public static RepositoryConfiguration create(final NiFiProperties nifiProperties) { final Map storageDirectories = nifiProperties.getProvenanceRepositoryPaths(); if (storageDirectories.isEmpty()) { @@ -436,6 +489,17 @@ public class RepositoryConfiguration { config.setAlwaysSync(alwaysSync); + config.setDebugFrequency(nifiProperties.getIntegerProperty(NiFiProperties.PROVENANCE_REPO_DEBUG_FREQUENCY, config.getDebugFrequency())); + + // Encryption values may not be present but are only required for EncryptedWriteAheadProvenanceRepository + final String implementationClassName = nifiProperties.getProperty(NiFiProperties.PROVENANCE_REPO_IMPLEMENTATION_CLASS); + if (EncryptedWriteAheadProvenanceRepository.class.getName().equals(implementationClassName)) { + config.setEncryptionKeys(nifiProperties.getProvenanceRepoEncryptionKeys()); + config.setKeyId(nifiProperties.getProperty(NiFiProperties.PROVENANCE_REPO_ENCRYPTION_KEY_ID)); + config.setKeyProviderImplementation(nifiProperties.getProperty(NiFiProperties.PROVENANCE_REPO_ENCRYPTION_KEY_PROVIDER_IMPLEMENTATION_CLASS)); + config.setKeyProviderLocation(nifiProperties.getProperty(NiFiProperties.PROVENANCE_REPO_ENCRYPTION_KEY_PROVIDER_LOCATION)); + } + return config; } } diff --git a/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/WriteAheadProvenanceRepository.java b/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/WriteAheadProvenanceRepository.java index 89750282de..4782dbe60a 100644 --- a/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/WriteAheadProvenanceRepository.java +++ b/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/WriteAheadProvenanceRepository.java @@ -21,7 +21,6 @@ import java.io.IOException; import java.util.Collections; import java.util.List; import java.util.Map; - import org.apache.nifi.authorization.Authorizer; import org.apache.nifi.authorization.RequestAction; import org.apache.nifi.authorization.resource.Authorizable; @@ -84,7 +83,7 @@ import org.slf4j.LoggerFactory; */ public class WriteAheadProvenanceRepository implements ProvenanceRepository { private static final Logger logger = LoggerFactory.getLogger(WriteAheadProvenanceRepository.class); - private static final int BLOCK_SIZE = 1024 * 32; + static final int BLOCK_SIZE = 1024 * 32; public static final String EVENT_CATEGORY = "Provenance Repository"; private final RepositoryConfiguration config; @@ -129,6 +128,14 @@ public class WriteAheadProvenanceRepository implements ProvenanceRepository { } }; + init(recordWriterFactory, recordReaderFactory, eventReporter, authorizer, resourceFactory); + } + + synchronized void init(RecordWriterFactory recordWriterFactory, RecordReaderFactory recordReaderFactory, + final EventReporter eventReporter, final Authorizer authorizer, + final ProvenanceAuthorizableFactory resourceFactory) throws IOException { + final EventFileManager fileManager = new EventFileManager(); + eventStore = new PartitionedWriteAheadEventStore(config, recordWriterFactory, recordReaderFactory, eventReporter, fileManager); final IndexManager indexManager = new SimpleIndexManager(config); @@ -145,7 +152,7 @@ public class WriteAheadProvenanceRepository implements ProvenanceRepository { eventStore.reindexLatestEvents(eventIndex); } catch (final Exception e) { logger.error("Failed to re-index some of the Provenance Events. It is possible that some of the latest " - + "events will not be available from the Provenance Repository when a query is issued.", e); + + "events will not be available from the Provenance Repository when a query is issued.", e); } } @@ -282,4 +289,8 @@ public class WriteAheadProvenanceRepository implements ProvenanceRepository { public List getSearchableAttributes() { return Collections.unmodifiableList(config.getSearchableAttributes()); } + + RepositoryConfiguration getConfig() { + return this.config; + } } diff --git a/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/index/lucene/LuceneEventIndex.java b/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/index/lucene/LuceneEventIndex.java index a58340343b..f4b47d3dce 100644 --- a/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/index/lucene/LuceneEventIndex.java +++ b/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/index/lucene/LuceneEventIndex.java @@ -36,7 +36,6 @@ import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicLong; - import org.apache.lucene.document.Document; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.Term; @@ -246,7 +245,7 @@ public class LuceneEventIndex implements EventIndex { final Document document = eventConverter.convert(event, summary); if (document == null) { - logger.debug("Received Provenance Event {} to index but it contained no information that should be indexed, so skipping it", event); + logger.debug("Received Provenance Event {} to index but it contained no information that should be indexed, so skipping it", event.getEventId()); } else { final File indexDir; if (event.getEventTime() == lastEventTime) { @@ -291,7 +290,7 @@ public class LuceneEventIndex implements EventIndex { final Document document = eventConverter.convert(event, location); if (document == null) { - logger.debug("Received Provenance Event {} to index but it contained no information that should be indexed, so skipping it", event); + logger.debug("Received Provenance Event {} to index but it contained no information that should be indexed, so skipping it", event.getEventId()); } else { final StoredDocument doc = new StoredDocument(document, location); boolean added = false; @@ -357,13 +356,13 @@ public class LuceneEventIndex implements EventIndex { eventOption = eventStore.getEvent(eventId); } catch (final Exception e) { logger.error("Failed to retrieve Provenance Event with ID " + eventId + " to calculate data lineage due to: " + e, e); - final AsyncLineageSubmission result = new AsyncLineageSubmission(LineageComputationType.FLOWFILE_LINEAGE, eventId, Collections. emptySet(), 1, user.getIdentity()); + final AsyncLineageSubmission result = new AsyncLineageSubmission(LineageComputationType.FLOWFILE_LINEAGE, eventId, Collections.emptySet(), 1, user.getIdentity()); result.getResult().setError("Failed to retrieve Provenance Event with ID " + eventId + ". See logs for more information."); return result; } if (!eventOption.isPresent()) { - final AsyncLineageSubmission result = new AsyncLineageSubmission(LineageComputationType.FLOWFILE_LINEAGE, eventId, Collections. emptySet(), 1, user.getIdentity()); + final AsyncLineageSubmission result = new AsyncLineageSubmission(LineageComputationType.FLOWFILE_LINEAGE, eventId, Collections.emptySet(), 1, user.getIdentity()); result.getResult().setError("Could not find Provenance Event with ID " + eventId); lineageSubmissionMap.put(result.getLineageIdentifier(), result); return result; @@ -524,7 +523,7 @@ public class LuceneEventIndex implements EventIndex { } default: { final AsyncLineageSubmission submission = new AsyncLineageSubmission(LineageComputationType.EXPAND_CHILDREN, - eventId, Collections. emptyList(), 1, userId); + eventId, Collections.emptyList(), 1, userId); lineageSubmissionMap.put(submission.getLineageIdentifier(), submission); submission.getResult().setError("Event ID " + eventId + " indicates an event of type " + event.getEventType() + " so its children cannot be expanded"); @@ -533,7 +532,7 @@ public class LuceneEventIndex implements EventIndex { } } catch (final Exception e) { final AsyncLineageSubmission submission = new AsyncLineageSubmission(LineageComputationType.EXPAND_CHILDREN, - eventId, Collections. emptyList(), 1, userId); + eventId, Collections.emptyList(), 1, userId); lineageSubmissionMap.put(submission.getLineageIdentifier(), submission); submission.getResult().setError("Failed to expand children for lineage of event with ID " + eventId + " due to: " + e); return submission; @@ -564,7 +563,7 @@ public class LuceneEventIndex implements EventIndex { } default: { final AsyncLineageSubmission submission = new AsyncLineageSubmission(LineageComputationType.EXPAND_PARENTS, - eventId, Collections. emptyList(), 1, userId); + eventId, Collections.emptyList(), 1, userId); lineageSubmissionMap.put(submission.getLineageIdentifier(), submission); submission.getResult().setError("Event ID " + eventId + " indicates an event of type " + event.getEventType() + " so its parents cannot be expanded"); @@ -573,7 +572,7 @@ public class LuceneEventIndex implements EventIndex { } } catch (final Exception e) { final AsyncLineageSubmission submission = new AsyncLineageSubmission(LineageComputationType.EXPAND_PARENTS, - eventId, Collections. emptyList(), 1, userId); + eventId, Collections.emptyList(), 1, userId); lineageSubmissionMap.put(submission.getLineageIdentifier(), submission); submission.getResult().setError("Failed to expand parents for lineage of event with ID " + eventId + " due to: " + e); diff --git a/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/schema/EventFieldNames.java b/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/schema/EventFieldNames.java index d6f50dda2e..2bc7fbe9bd 100644 --- a/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/schema/EventFieldNames.java +++ b/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/schema/EventFieldNames.java @@ -56,4 +56,12 @@ public class EventFieldNames { public static final String EXPLICIT_VALUE = "Explicit Value"; public static final String LOOKUP_VALUE = "Lookup Value"; public static final String UNCHANGED_VALUE = "Unchanged"; + + // For encrypted records + public static final String IS_ENCRYPTED = "Encrypted Record"; + public static final String KEY_ID = "Encryption Key ID"; + public static final String VERSION = "Encryption Version"; + public static final String ALGORITHM = "Encryption Algorithm"; + public static final String IV = "Initialization Vector"; + public static final String ENCRYPTION_DETAILS = "Encryption Details"; } diff --git a/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/schema/LookupTableEventRecordFields.java b/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/schema/LookupTableEventRecordFields.java index 7b33ded095..057763656d 100644 --- a/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/schema/LookupTableEventRecordFields.java +++ b/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/schema/LookupTableEventRecordFields.java @@ -66,12 +66,12 @@ public class LookupTableEventRecordFields { public static final RecordField CONTENT_CLAIM_SIZE = new SimpleRecordField(EventFieldNames.CONTENT_CLAIM_SIZE, FieldType.LONG, EXACTLY_ONE); public static final RecordField PREVIOUS_CONTENT_CLAIM = new ComplexRecordField(EventFieldNames.PREVIOUS_CONTENT_CLAIM, ZERO_OR_ONE, - CONTENT_CLAIM_CONTAINER, CONTENT_CLAIM_SECTION, CONTENT_CLAIM_IDENTIFIER, CONTENT_CLAIM_OFFSET, CONTENT_CLAIM_SIZE); + CONTENT_CLAIM_CONTAINER, CONTENT_CLAIM_SECTION, CONTENT_CLAIM_IDENTIFIER, CONTENT_CLAIM_OFFSET, CONTENT_CLAIM_SIZE); public static final RecordField CURRENT_CONTENT_CLAIM_EXPLICIT = new ComplexRecordField(EventFieldNames.EXPLICIT_VALUE, EXACTLY_ONE, - CONTENT_CLAIM_CONTAINER, CONTENT_CLAIM_SECTION, CONTENT_CLAIM_IDENTIFIER, CONTENT_CLAIM_OFFSET, CONTENT_CLAIM_SIZE); + CONTENT_CLAIM_CONTAINER, CONTENT_CLAIM_SECTION, CONTENT_CLAIM_IDENTIFIER, CONTENT_CLAIM_OFFSET, CONTENT_CLAIM_SIZE); public static final RecordField CURRENT_CONTENT_CLAIM = new UnionRecordField(EventFieldNames.CONTENT_CLAIM, - Repetition.EXACTLY_ONE, NO_VALUE, UNCHANGED_VALUE, CURRENT_CONTENT_CLAIM_EXPLICIT); + Repetition.EXACTLY_ONE, NO_VALUE, UNCHANGED_VALUE, CURRENT_CONTENT_CLAIM_EXPLICIT); // EventType-Specific fields diff --git a/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/schema/LookupTableEventSchema.java b/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/schema/LookupTableEventSchema.java index 7110336422..d596c8e800 100644 --- a/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/schema/LookupTableEventSchema.java +++ b/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/schema/LookupTableEventSchema.java @@ -45,7 +45,6 @@ import static org.apache.nifi.provenance.schema.LookupTableEventRecordFields.UPD import java.util.ArrayList; import java.util.Collections; import java.util.List; - import org.apache.nifi.repository.schema.RecordField; import org.apache.nifi.repository.schema.RecordSchema; @@ -90,5 +89,4 @@ public class LookupTableEventSchema { final RecordSchema schema = new RecordSchema(fields); return schema; } - } diff --git a/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/serialization/CompressableRecordReader.java b/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/serialization/CompressableRecordReader.java index 93c066963e..dfbcd2b6ad 100644 --- a/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/serialization/CompressableRecordReader.java +++ b/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/serialization/CompressableRecordReader.java @@ -25,7 +25,6 @@ import java.io.InputStream; import java.nio.charset.StandardCharsets; import java.util.Optional; import java.util.zip.GZIPInputStream; - import org.apache.nifi.provenance.ProvenanceEventRecord; import org.apache.nifi.provenance.StandardProvenanceEventRecord; import org.apache.nifi.provenance.toc.TocReader; @@ -333,7 +332,7 @@ public abstract class CompressableRecordReader implements RecordReader { try { boolean read = true; while (read) { - final Optional eventOptional = readToEvent(eventId, dis, serializationVersion); + final Optional eventOptional = this.readToEvent(eventId, dis, serializationVersion); if (eventOptional.isPresent()) { pushbackEvent = eventOptional.get(); return Optional.of(pushbackEvent); diff --git a/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/serialization/RecordReaders.java b/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/serialization/RecordReaders.java index 8e79ddd7fc..7ae4adccc0 100644 --- a/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/serialization/RecordReaders.java +++ b/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/serialization/RecordReaders.java @@ -27,9 +27,11 @@ import java.io.InputStream; import java.nio.file.Path; import java.util.Collection; import java.util.zip.GZIPInputStream; - +import org.apache.nifi.properties.NiFiPropertiesLoader; import org.apache.nifi.provenance.ByteArraySchemaRecordReader; import org.apache.nifi.provenance.ByteArraySchemaRecordWriter; +import org.apache.nifi.provenance.CryptoUtils; +import org.apache.nifi.provenance.EncryptedSchemaRecordReader; import org.apache.nifi.provenance.EventIdFirstSchemaRecordReader; import org.apache.nifi.provenance.EventIdFirstSchemaRecordWriter; import org.apache.nifi.provenance.StandardRecordReader; @@ -37,17 +39,25 @@ import org.apache.nifi.provenance.lucene.LuceneUtil; import org.apache.nifi.provenance.toc.StandardTocReader; import org.apache.nifi.provenance.toc.TocReader; import org.apache.nifi.provenance.toc.TocUtil; +import org.apache.nifi.util.NiFiProperties; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public class RecordReaders { + private static Logger logger = LoggerFactory.getLogger(RecordReaders.class); + + private static boolean isEncryptionAvailable = false; + private static boolean encryptionPropertiesRead = false; + /** * Creates a new Record Reader that is capable of reading Provenance Event Journals * - * @param file the Provenance Event Journal to read data from + * @param file the Provenance Event Journal to read data from * @param provenanceLogFiles collection of all provenance journal files - * @param maxAttributeChars the maximum number of characters to retrieve for any one attribute. This allows us to avoid - * issues where a FlowFile has an extremely large attribute and reading events - * for that FlowFile results in loading that attribute into memory many times, exhausting the Java Heap + * @param maxAttributeChars the maximum number of characters to retrieve for any one attribute. This allows us to avoid + * issues where a FlowFile has an extremely large attribute and reading events + * for that FlowFile results in loading that attribute into memory many times, exhausting the Java Heap * @return a Record Reader capable of reading Provenance Event Journals * @throws IOException if unable to create a Record Reader for the given file */ @@ -68,7 +78,7 @@ public class RecordReaders { } } - if ( file.exists() ) { + if (file.exists()) { try { fis = new FileInputStream(file); } catch (final FileNotFoundException fnfe) { @@ -77,7 +87,8 @@ public class RecordReaders { } String filename = file.getName(); - openStream: while ( fis == null ) { + openStream: + while (fis == null) { final File dir = file.getParentFile(); final String baseName = LuceneUtil.substringBefore(file.getName(), ".prov"); @@ -85,9 +96,9 @@ public class RecordReaders { // filename that we need. The majority of the time, we will use the extension ".prov.gz" // because most often we are compressing on rollover and most often we have already finished // compressing by the time that we are querying the data. - for ( final String extension : new String[] {".prov.gz", ".prov"} ) { + for (final String extension : new String[]{".prov.gz", ".prov"}) { file = new File(dir, baseName + extension); - if ( file.exists() ) { + if (file.exists()) { try { fis = new FileInputStream(file); filename = baseName + extension; @@ -104,7 +115,7 @@ public class RecordReaders { break; } - if ( fis == null ) { + if (fis == null) { throw new FileNotFoundException("Unable to locate file " + originalFile); } @@ -148,12 +159,25 @@ public class RecordReaders { final TocReader tocReader = new StandardTocReader(tocFile); return new EventIdFirstSchemaRecordReader(bufferedInStream, filename, tocReader, maxAttributeChars); } + case EncryptedSchemaRecordReader.SERIALIZATION_NAME: { + if (!tocFile.exists()) { + throw new FileNotFoundException("Cannot create TOC Reader because the file " + tocFile + " does not exist"); + } + + if (!isEncryptionAvailable()) { + throw new IOException("Cannot read encrypted repository because this reader is not configured for encryption"); + } + + final TocReader tocReader = new StandardTocReader(tocFile); + // Return a reader with no eventEncryptor because this method contract cannot change, then inject the encryptor from the writer in the calling method + return new EncryptedSchemaRecordReader(bufferedInStream, filename, tocReader, maxAttributeChars, null); + } default: { throw new IOException("Unable to read data from file " + file + " because the file was written using an unknown Serializer: " + serializationName); } } } catch (final IOException ioe) { - if ( fis != null ) { + if (fis != null) { try { fis.close(); } catch (final IOException inner) { @@ -165,4 +189,20 @@ public class RecordReaders { } } + private static boolean isEncryptionAvailable() { + if (encryptionPropertiesRead) { + return isEncryptionAvailable; + } else { + try { + NiFiProperties niFiProperties = NiFiPropertiesLoader.loadDefaultWithKeyFromBootstrap(); + isEncryptionAvailable = CryptoUtils.isProvenanceRepositoryEncryptionConfigured(niFiProperties); + encryptionPropertiesRead = true; + } catch (IOException e) { + logger.error("Encountered an error checking the provenance repository encryption configuration: ", e); + isEncryptionAvailable = false; + } + return isEncryptionAvailable; + } + } + } diff --git a/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/util/StorageSummaryEvent.java b/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/util/StorageSummaryEvent.java index 41d5ade7b6..766278ae21 100644 --- a/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/util/StorageSummaryEvent.java +++ b/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/util/StorageSummaryEvent.java @@ -19,7 +19,6 @@ package org.apache.nifi.provenance.util; import java.util.List; import java.util.Map; - import org.apache.nifi.provenance.ProvenanceEventRecord; import org.apache.nifi.provenance.ProvenanceEventType; import org.apache.nifi.provenance.serialization.StorageSummary; @@ -182,4 +181,14 @@ public class StorageSummaryEvent implements ProvenanceEventRecord { public Long getPreviousContentClaimOffset() { return event.getPreviousContentClaimOffset(); } + + /** + * Returns the best event identifier for this event (eventId if available, descriptive identifier if not yet persisted to allow for traceability). + * + * @return a descriptive event ID to allow tracing + */ + @Override + public String getBestEventIdentifier() { + return Long.toString(getEventId()); + } } diff --git a/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/resources/META-INF/services/org.apache.nifi.provenance.ProvenanceRepository b/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/resources/META-INF/services/org.apache.nifi.provenance.ProvenanceRepository index 6a353d2af8..94cc70cd0f 100644 --- a/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/resources/META-INF/services/org.apache.nifi.provenance.ProvenanceRepository +++ b/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/resources/META-INF/services/org.apache.nifi.provenance.ProvenanceRepository @@ -13,4 +13,5 @@ # See the License for the specific language governing permissions and # limitations under the License. org.apache.nifi.provenance.PersistentProvenanceRepository -org.apache.nifi.provenance.WriteAheadProvenanceRepository \ No newline at end of file +org.apache.nifi.provenance.WriteAheadProvenanceRepository +org.apache.nifi.provenance.EncryptedWriteAheadProvenanceRepository \ No newline at end of file diff --git a/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/test/groovy/org/apache/nifi/provenance/EncryptedSchemaRecordReaderWriterTest.groovy b/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/test/groovy/org/apache/nifi/provenance/EncryptedSchemaRecordReaderWriterTest.groovy new file mode 100644 index 0000000000..ec8c225e06 --- /dev/null +++ b/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/test/groovy/org/apache/nifi/provenance/EncryptedSchemaRecordReaderWriterTest.groovy @@ -0,0 +1,281 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.provenance + +import org.apache.nifi.flowfile.FlowFile +import org.apache.nifi.provenance.serialization.RecordReader +import org.apache.nifi.provenance.serialization.RecordWriter +import org.apache.nifi.provenance.toc.StandardTocReader +import org.apache.nifi.provenance.toc.StandardTocWriter +import org.apache.nifi.provenance.toc.TocReader +import org.apache.nifi.provenance.toc.TocUtil +import org.apache.nifi.provenance.toc.TocWriter +import org.apache.nifi.util.file.FileUtils +import org.bouncycastle.jce.provider.BouncyCastleProvider +import org.bouncycastle.util.encoders.Hex +import org.junit.After +import org.junit.AfterClass +import org.junit.Before +import org.junit.BeforeClass +import org.junit.ClassRule +import org.junit.Test +import org.junit.rules.TemporaryFolder +import org.junit.runner.RunWith +import org.junit.runners.JUnit4 +import org.slf4j.Logger +import org.slf4j.LoggerFactory + +import javax.crypto.Cipher +import javax.crypto.spec.SecretKeySpec +import java.security.KeyManagementException +import java.security.Security +import java.util.concurrent.atomic.AtomicLong + +import static groovy.test.GroovyAssert.shouldFail +import static org.apache.nifi.provenance.TestUtil.createFlowFile + +@RunWith(JUnit4.class) +class EncryptedSchemaRecordReaderWriterTest extends AbstractTestRecordReaderWriter { + private static final Logger logger = LoggerFactory.getLogger(EncryptedSchemaRecordReaderWriterTest.class) + + private static final String KEY_HEX_128 = "0123456789ABCDEFFEDCBA9876543210" + private static final String KEY_HEX_256 = KEY_HEX_128 * 2 + private static final String KEY_HEX = isUnlimitedStrengthCryptoAvailable() ? KEY_HEX_256 : KEY_HEX_128 + private static final String KEY_ID = "K1" + + private static final String TRANSIT_URI = "nifi://unit-test" + private static final String PROCESSOR_TYPE = "Mock Processor" + private static final String COMPONENT_ID = "1234" + + private static final int UNCOMPRESSED_BLOCK_SIZE = 1024 * 32 + private static final int MAX_ATTRIBUTE_SIZE = 2048 + + private static final AtomicLong idGenerator = new AtomicLong(0L) + private File journalFile + private File tocFile + + private static KeyProvider mockKeyProvider + private static ProvenanceEventEncryptor provenanceEventEncryptor = new AESProvenanceEventEncryptor() + + @ClassRule + public static TemporaryFolder tempFolder = new TemporaryFolder() + + private static String ORIGINAL_LOG_LEVEL + + @BeforeClass + static void setUpOnce() throws Exception { + ORIGINAL_LOG_LEVEL = System.getProperty("org.slf4j.simpleLogger.log.org.apache.nifi.provenance") + System.setProperty("org.slf4j.simpleLogger.log.org.apache.nifi.provenance", "DEBUG") + + Security.addProvider(new BouncyCastleProvider()) + + logger.metaClass.methodMissing = { String name, args -> + logger.info("[${name?.toUpperCase()}] ${(args as List).join(" ")}") + } + + mockKeyProvider = [ + getKey : { String keyId -> + logger.mock("Requesting key ID: ${keyId}") + if (keyId == KEY_ID) { + new SecretKeySpec(Hex.decode(KEY_HEX), "AES") + } else { + throw new KeyManagementException("${keyId} is not available") + } + }, + getAvailableKeyIds: { -> + logger.mock("Available key IDs: [${KEY_ID}]") + [KEY_ID] + }, + keyExists : { String keyId -> + logger.mock("Checking availability of key ID: ${keyId}") + keyId == KEY_ID + }] as KeyProvider + provenanceEventEncryptor.initialize(mockKeyProvider) + } + + @Before + void setUp() throws Exception { + journalFile = new File("target/storage/${UUID.randomUUID()}/testEventIdFirstSchemaRecordReaderWriter") + tocFile = TocUtil.getTocFile(journalFile) + idGenerator.set(0L) + } + + @After + void tearDown() throws Exception { + try { + FileUtils.deleteFile(journalFile.getParentFile(), true) + } catch (Exception e) { + logger.error(e.getMessage()) + } + } + + @AfterClass + static void tearDownOnce() throws Exception { + if (ORIGINAL_LOG_LEVEL) { + System.setProperty("org.slf4j.simpleLogger.log.org.apache.nifi.provenance", ORIGINAL_LOG_LEVEL) + } + try { + FileUtils.deleteFile(new File("target/storage"), true) + } catch (Exception e) { + logger.error(e) + } + } + + private static boolean isUnlimitedStrengthCryptoAvailable() { + Cipher.getMaxAllowedKeyLength("AES") > 128 + } + + private static + final FlowFile buildFlowFile(Map attributes = [:], long id = idGenerator.getAndIncrement(), long fileSize = 3000L) { + if (!attributes?.uuid) { + attributes.uuid = UUID.randomUUID().toString() + } + createFlowFile(id, fileSize, attributes) + } + + private + static ProvenanceEventRecord buildEventRecord(FlowFile flowfile = buildFlowFile(), ProvenanceEventType eventType = ProvenanceEventType.RECEIVE, String transitUri = TRANSIT_URI, String componentId = COMPONENT_ID, String componentType = PROCESSOR_TYPE, long eventTime = System.currentTimeMillis()) { + final ProvenanceEventBuilder builder = new StandardProvenanceEventRecord.Builder() + builder.setEventTime(eventTime) + builder.setEventType(eventType) + builder.setTransitUri(transitUri) + builder.fromFlowFile(flowfile) + builder.setComponentId(componentId) + builder.setComponentType(componentType) + builder.build() + } + + @Override + protected RecordWriter createWriter( + final File file, + final TocWriter tocWriter, final boolean compressed, final int uncompressedBlockSize) throws IOException { + createWriter(file, tocWriter, compressed, uncompressedBlockSize, provenanceEventEncryptor) + } + + protected static RecordWriter createWriter( + final File file, + final TocWriter tocWriter, + final boolean compressed, + final int uncompressedBlockSize, ProvenanceEventEncryptor encryptor) throws IOException { + return new EncryptedSchemaRecordWriter(file, idGenerator, tocWriter, compressed, uncompressedBlockSize, IdentifierLookup.EMPTY, encryptor, 1) + } + + @Override + protected RecordReader createReader( + final InputStream inputStream, + final String journalFilename, final TocReader tocReader, final int maxAttributeSize) throws IOException { + return new EncryptedSchemaRecordReader(inputStream, journalFilename, tocReader, maxAttributeSize, provenanceEventEncryptor) + } + + /** + * Build a record and write it to the repository with the encrypted writer. Recover with the encrypted reader and verify. + */ + @Test + void testShouldWriteAndReadEncryptedRecord() { + // Arrange + final ProvenanceEventRecord record = buildEventRecord() + logger.info("Built sample PER: ${record}") + + TocWriter tocWriter = new StandardTocWriter(tocFile, false, false) + + RecordWriter encryptedWriter = createWriter(journalFile, tocWriter, false, UNCOMPRESSED_BLOCK_SIZE) + logger.info("Generated encrypted writer: ${encryptedWriter}") + + // Act + int encryptedRecordId = idGenerator.get() + encryptedWriter.writeHeader(encryptedRecordId) + encryptedWriter.writeRecord(record) + encryptedWriter.close() + logger.info("Wrote encrypted record ${encryptedRecordId} to journal") + + // Assert + TocReader tocReader = new StandardTocReader(tocFile) + final FileInputStream fis = new FileInputStream(journalFile) + final RecordReader reader = createReader(fis, journalFile.getName(), tocReader, MAX_ATTRIBUTE_SIZE) + logger.info("Generated encrypted reader: ${reader}") + + ProvenanceEventRecord encryptedEvent = reader.nextRecord() + assert encryptedEvent + assert encryptedRecordId as long == encryptedEvent.getEventId() + assert record.componentId == encryptedEvent.getComponentId() + assert record.componentType == encryptedEvent.getComponentType() + logger.info("Successfully read encrypted record: ${encryptedEvent}") + + assert !reader.nextRecord() + } + + /** + * Build a record and write it with a standard writer and the encrypted writer to different repositories. Recover with the standard reader and the contents of the encrypted record should be unreadable. + */ + @Test + void testShouldWriteEncryptedRecordAndPlainRecord() { + // Arrange + final ProvenanceEventRecord record = buildEventRecord() + logger.info("Built sample PER: ${record}") + + TocWriter tocWriter = new StandardTocWriter(tocFile, false, false) + + RecordWriter standardWriter = new EventIdFirstSchemaRecordWriter(journalFile, idGenerator, tocWriter, false, UNCOMPRESSED_BLOCK_SIZE, IdentifierLookup.EMPTY) + logger.info("Generated standard writer: ${standardWriter}") + + File encryptedJournalFile = new File(journalFile.absolutePath + "_encrypted") + File encryptedTocFile = TocUtil.getTocFile(encryptedJournalFile) + TocWriter encryptedTocWriter = new StandardTocWriter(encryptedTocFile, false, false) + RecordWriter encryptedWriter = createWriter(encryptedJournalFile, encryptedTocWriter, false, UNCOMPRESSED_BLOCK_SIZE) + logger.info("Generated encrypted writer: ${encryptedWriter}") + + // Act + int standardRecordId = idGenerator.get() + standardWriter.writeHeader(standardRecordId) + standardWriter.writeRecord(record) + standardWriter.close() + logger.info("Wrote standard record ${standardRecordId} to journal") + + int encryptedRecordId = idGenerator.get() + encryptedWriter.writeHeader(encryptedRecordId) + encryptedWriter.writeRecord(record) + encryptedWriter.close() + logger.info("Wrote encrypted record ${encryptedRecordId} to journal") + + // Assert + TocReader tocReader = new StandardTocReader(tocFile) + final FileInputStream fis = new FileInputStream(journalFile) + final RecordReader reader = new EventIdFirstSchemaRecordReader(fis, journalFile.getName(), tocReader, MAX_ATTRIBUTE_SIZE) + logger.info("Generated standard reader: ${reader}") + + ProvenanceEventRecord standardEvent = reader.nextRecord() + assert standardEvent + assert standardRecordId as long == standardEvent.getEventId() + assert record.componentId == standardEvent.getComponentId() + assert record.componentType == standardEvent.getComponentType() + logger.info("Successfully read standard record: ${standardEvent}") + + assert !reader.nextRecord() + + // Demonstrate unable to read from encrypted file with standard reader + TocReader incompatibleTocReader = new StandardTocReader(encryptedTocFile) + final FileInputStream efis = new FileInputStream(encryptedJournalFile) + RecordReader incompatibleReader = new EventIdFirstSchemaRecordReader(efis, encryptedJournalFile.getName(), incompatibleTocReader, MAX_ATTRIBUTE_SIZE) + logger.info("Generated standard reader (attempting to read encrypted file): ${incompatibleReader}") + + def msg = shouldFail(EOFException) { + ProvenanceEventRecord encryptedEvent = incompatibleReader.nextRecord() + } + logger.expected(msg) + assert msg =~ "EOFException: Failed to read field" + } +} diff --git a/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/test/groovy/org/apache/nifi/provenance/EncryptedWriteAheadProvenanceRepositoryTest.groovy b/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/test/groovy/org/apache/nifi/provenance/EncryptedWriteAheadProvenanceRepositoryTest.groovy new file mode 100644 index 0000000000..42cc8819be --- /dev/null +++ b/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/test/groovy/org/apache/nifi/provenance/EncryptedWriteAheadProvenanceRepositoryTest.groovy @@ -0,0 +1,391 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License") you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.provenance + +import org.apache.nifi.events.EventReporter +import org.apache.nifi.flowfile.FlowFile +import org.apache.nifi.provenance.serialization.RecordReaders +import org.apache.nifi.reporting.Severity +import org.apache.nifi.util.file.FileUtils +import org.bouncycastle.jce.provider.BouncyCastleProvider +import org.junit.After +import org.junit.AfterClass +import org.junit.Before +import org.junit.BeforeClass +import org.junit.ClassRule +import org.junit.Test +import org.junit.rules.TemporaryFolder +import org.junit.runner.RunWith +import org.junit.runners.JUnit4 +import org.slf4j.Logger +import org.slf4j.LoggerFactory + +import javax.crypto.Cipher +import java.security.Security +import java.util.concurrent.TimeUnit +import java.util.concurrent.atomic.AtomicLong + +import static org.apache.nifi.provenance.TestUtil.createFlowFile + +@RunWith(JUnit4.class) +class EncryptedWriteAheadProvenanceRepositoryTest { + private static final Logger logger = LoggerFactory.getLogger(EncryptedWriteAheadProvenanceRepositoryTest.class) + + private static final String KEY_HEX_128 = "0123456789ABCDEFFEDCBA9876543210" + private static final String KEY_HEX_256 = KEY_HEX_128 * 2 + private static final String KEY_HEX = isUnlimitedStrengthCryptoAvailable() ? KEY_HEX_256 : KEY_HEX_128 + private static final String KEY_ID = "K1" + + private static final String TRANSIT_URI = "nifi://unit-test" + private static final String PROCESSOR_TYPE = "Mock Processor" + private static final String COMPONENT_ID = "1234" + + private static final AtomicLong recordId = new AtomicLong() + + @ClassRule + public static TemporaryFolder tempFolder = new TemporaryFolder() + + private ProvenanceRepository repo + private static RepositoryConfiguration config + + public static final int DEFAULT_ROLLOVER_MILLIS = 2000 + private EventReporter eventReporter + private List reportedEvents = Collections.synchronizedList(new ArrayList()) + + private static String ORIGINAL_LOG_LEVEL + + @BeforeClass + static void setUpOnce() throws Exception { + ORIGINAL_LOG_LEVEL = System.getProperty("org.slf4j.simpleLogger.log.org.apache.nifi.provenance") + System.setProperty("org.slf4j.simpleLogger.log.org.apache.nifi.provenance", "DEBUG") + + Security.addProvider(new BouncyCastleProvider()) + + logger.metaClass.methodMissing = { String name, args -> + logger.info("[${name?.toUpperCase()}] ${(args as List).join(" ")}") + } + } + + @Before + void setUp() throws Exception { + reportedEvents?.clear() + eventReporter = createMockEventReporter() + } + + @After + void tearDown() throws Exception { + closeRepo(repo, config) + + // Reset the boolean determiner + RecordReaders.encryptionPropertiesRead = false + RecordReaders.isEncryptionAvailable = false + } + + @AfterClass + static void tearDownOnce() throws Exception { + if (ORIGINAL_LOG_LEVEL) { + System.setProperty("org.slf4j.simpleLogger.log.org.apache.nifi.provenance", ORIGINAL_LOG_LEVEL) + } + } + + private static boolean isUnlimitedStrengthCryptoAvailable() { + Cipher.getMaxAllowedKeyLength("AES") > 128 + } + + private static RepositoryConfiguration createConfiguration() { + RepositoryConfiguration config = new RepositoryConfiguration() + config.addStorageDirectory("1", new File("target/storage/" + UUID.randomUUID().toString())) + config.setCompressOnRollover(true) + config.setMaxEventFileLife(2000L, TimeUnit.SECONDS) + config.setCompressionBlockBytes(100) + return config + } + + private static RepositoryConfiguration createEncryptedConfiguration() { + RepositoryConfiguration config = createConfiguration() + config.setEncryptionKeys([(KEY_ID): KEY_HEX]) + config.setKeyId(KEY_ID) + config.setKeyProviderImplementation(StaticKeyProvider.class.name) + config + } + + private EventReporter createMockEventReporter() { + [reportEvent: { Severity s, String c, String m -> + ReportedEvent event = new ReportedEvent(s, c, m) + reportedEvents.add(event) + logger.mock("Added ${event}") + }] as EventReporter + } + + private void closeRepo(ProvenanceRepository repo = this.repo, RepositoryConfiguration config = this.config) throws IOException { + if (repo == null) { + return + } + + try { + repo.close() + } catch (IOException ioe) { + } + + // Delete all of the storage files. We do this in order to clean up the tons of files that + // we create but also to ensure that we have closed all of the file handles. If we leave any + // streams open, for instance, this will throw an IOException, causing our unit test to fail. + if (config != null) { + for (final File storageDir : config.getStorageDirectories().values()) { + int i + for (i = 0; i < 3; i++) { + try { + FileUtils.deleteFile(storageDir, true) + break + } catch (IOException ioe) { + // if there is a virus scanner, etc. running in the background we may not be able to + // delete the file. Wait a sec and try again. + if (i == 2) { + throw ioe + } else { + try { + System.out.println("file: " + storageDir.toString() + " exists=" + storageDir.exists()) + FileUtils.deleteFile(storageDir, true) + break + } catch (final IOException ioe2) { + // if there is a virus scanner, etc. running in the background we may not be able to + // delete the file. Wait a sec and try again. + if (i == 2) { + throw ioe2 + } else { + try { + Thread.sleep(1000L) + } catch (final InterruptedException ie) { + } + } + } + } + } + } + } + } + } + + private static + final FlowFile buildFlowFile(Map attributes = [:], long id = recordId.getAndIncrement(), long fileSize = 3000L) { + if (!attributes?.uuid) { + attributes.uuid = UUID.randomUUID().toString() + } + createFlowFile(id, fileSize, attributes) + } + + private + static ProvenanceEventRecord buildEventRecord(FlowFile flowfile = buildFlowFile(), ProvenanceEventType eventType = ProvenanceEventType.RECEIVE, String transitUri = TRANSIT_URI, String componentId = COMPONENT_ID, String componentType = PROCESSOR_TYPE, long eventTime = System.currentTimeMillis()) { + final ProvenanceEventBuilder builder = new StandardProvenanceEventRecord.Builder() + builder.setEventTime(eventTime) + builder.setEventType(eventType) + builder.setTransitUri(transitUri) + builder.fromFlowFile(flowfile) + builder.setComponentId(componentId) + builder.setComponentType(componentType) + builder.build() + } + + /** + * This test operates on {@link PersistentProvenanceRepository} to verify the normal operations of existing implementations. + * + * @throws IOException + * @throws InterruptedException + */ + @Test + void testPersistentProvenanceRepositoryShouldRegisterAndRetrieveEvents() throws IOException, InterruptedException { + // Arrange + config = createConfiguration() + config.setMaxEventFileCapacity(1L) + config.setMaxEventFileLife(1, TimeUnit.SECONDS) + repo = new PersistentProvenanceRepository(config, DEFAULT_ROLLOVER_MILLIS) + repo.initialize(eventReporter, null, null, IdentifierLookup.EMPTY) + + Map attributes = ["abc": "xyz", + "123": "456"] + final ProvenanceEventRecord record = buildEventRecord(buildFlowFile(attributes)) + + final int RECORD_COUNT = 10 + + // Act + RECORD_COUNT.times { + repo.registerEvent(record) + } + + // Sleep to let the journal merge occur + Thread.sleep(1000L) + + final List recoveredRecords = repo.getEvents(0L, RECORD_COUNT + 1) + + logger.info("Recovered ${recoveredRecords.size()} events: ") + recoveredRecords.each { logger.info("\t${it}") } + + // Assert + assert recoveredRecords.size() == RECORD_COUNT + recoveredRecords.eachWithIndex { ProvenanceEventRecord recovered, int i -> + assert recovered.getEventId() == (i as Long) + assert recovered.getTransitUri() == TRANSIT_URI + assert recovered.getEventType() == ProvenanceEventType.RECEIVE + // The UUID was added later but we care that all attributes we provided are still there + assert recovered.getAttributes().entrySet().containsAll(attributes.entrySet()) + } + } + + /** + * This test operates on {@link WriteAheadProvenanceRepository} to verify the normal operations of existing implementations. + * + * @throws IOException + * @throws InterruptedException + */ + @Test + void testWriteAheadProvenanceRepositoryShouldRegisterAndRetrieveEvents() throws IOException, InterruptedException { + // Arrange + config = createConfiguration() + // Needed until NIFI-3605 is implemented +// config.setMaxEventFileCapacity(1L) + config.setMaxEventFileCount(1) + config.setMaxEventFileLife(1, TimeUnit.SECONDS) + repo = new WriteAheadProvenanceRepository(config) + repo.initialize(eventReporter, null, null, IdentifierLookup.EMPTY) + + Map attributes = ["abc": "xyz", + "123": "456"] + final ProvenanceEventRecord record = buildEventRecord(buildFlowFile(attributes)) + + final int RECORD_COUNT = 10 + + // Act + RECORD_COUNT.times { + repo.registerEvent(record) + } + + final List recoveredRecords = repo.getEvents(0L, RECORD_COUNT + 1) + + logger.info("Recovered ${recoveredRecords.size()} events: ") + recoveredRecords.each { logger.info("\t${it}") } + + // Assert + assert recoveredRecords.size() == RECORD_COUNT + recoveredRecords.eachWithIndex { ProvenanceEventRecord recovered, int i -> + assert recovered.getEventId() == (i as Long) + assert recovered.getTransitUri() == TRANSIT_URI + assert recovered.getEventType() == ProvenanceEventType.RECEIVE + // The UUID was added later but we care that all attributes we provided are still there + assert recovered.getAttributes().entrySet().containsAll(attributes.entrySet()) + } + } + + @Test + void testShouldRegisterAndGetEvent() { + // Arrange + + // Override the boolean determiner + RecordReaders.encryptionPropertiesRead = true + RecordReaders.isEncryptionAvailable = true + + config = createEncryptedConfiguration() + // Needed until NIFI-3605 is implemented +// config.setMaxEventFileCapacity(1L) + config.setMaxEventFileCount(1) + config.setMaxEventFileLife(1, TimeUnit.SECONDS) + repo = new EncryptedWriteAheadProvenanceRepository(config) + repo.initialize(eventReporter, null, null, IdentifierLookup.EMPTY) + + Map attributes = ["abc": "This is a plaintext attribute.", + "123": "This is another plaintext attribute."] + final ProvenanceEventRecord record = buildEventRecord(buildFlowFile(attributes)) + + final long LAST_RECORD_ID = repo.getMaxEventId() + + // Act + repo.registerEvent(record) + + // Retrieve the event through the interface + ProvenanceEventRecord recoveredRecord = repo.getEvent(LAST_RECORD_ID + 1) + logger.info("Recovered ${recoveredRecord}") + + // Assert + assert recoveredRecord.getEventId() == LAST_RECORD_ID + 1 + assert recoveredRecord.getTransitUri() == TRANSIT_URI + assert recoveredRecord.getEventType() == ProvenanceEventType.RECEIVE + // The UUID was added later but we care that all attributes we provided are still there + assert recoveredRecord.getAttributes().entrySet().containsAll(attributes.entrySet()) + } + + @Test + void testShouldRegisterAndGetEvents() { + // Arrange + final int RECORD_COUNT = 10 + + // Override the boolean determiner + RecordReaders.encryptionPropertiesRead = true + RecordReaders.isEncryptionAvailable = true + + config = createEncryptedConfiguration() + // Needed until NIFI-3605 is implemented +// config.setMaxEventFileCapacity(1L) + config.setMaxEventFileCount(1) + config.setMaxEventFileLife(1, TimeUnit.SECONDS) + repo = new EncryptedWriteAheadProvenanceRepository(config) + repo.initialize(eventReporter, null, null, IdentifierLookup.EMPTY) + + Map attributes = ["abc": "This is a plaintext attribute.", + "123": "This is another plaintext attribute."] + final List records = [] + RECORD_COUNT.times { int i -> + records << buildEventRecord(buildFlowFile(attributes + [count: i as String])) + } + logger.info("Generated ${RECORD_COUNT} records") + + final long LAST_RECORD_ID = repo.getMaxEventId() + + // Act + repo.registerEvents(records) + logger.info("Registered events") + + // Retrieve the events through the interface + List recoveredRecords = repo.getEvents(LAST_RECORD_ID + 1, RECORD_COUNT * 2) + logger.info("Recovered ${recoveredRecords.size()} records") + + // Assert + recoveredRecords.eachWithIndex { ProvenanceEventRecord recoveredRecord, int i -> + assert recoveredRecord.getEventId() == LAST_RECORD_ID + 1 + i + assert recoveredRecord.getTransitUri() == TRANSIT_URI + assert recoveredRecord.getEventType() == ProvenanceEventType.RECEIVE + // The UUID was added later but we care that all attributes we provided are still there + assert recoveredRecord.getAttributes().entrySet().containsAll(attributes.entrySet()) + assert recoveredRecord.getAttribute("count") == i as String + } + } + + private static class ReportedEvent { + final Severity severity + final String category + final String message + + ReportedEvent(final Severity severity, final String category, final String message) { + this.severity = severity + this.category = category + this.message = message + } + + @Override + String toString() { + "ReportedEvent [${severity}] ${category}: ${message}" + } + } +} diff --git a/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/test/java/org/apache/nifi/provenance/AbstractTestRecordReaderWriter.java b/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/test/java/org/apache/nifi/provenance/AbstractTestRecordReaderWriter.java index 36397c4744..4b2ca50bd0 100644 --- a/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/test/java/org/apache/nifi/provenance/AbstractTestRecordReaderWriter.java +++ b/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/test/java/org/apache/nifi/provenance/AbstractTestRecordReaderWriter.java @@ -32,7 +32,6 @@ import java.util.List; import java.util.Map; import java.util.Optional; import java.util.UUID; - import org.apache.nifi.provenance.serialization.RecordReader; import org.apache.nifi.provenance.serialization.RecordWriter; import org.apache.nifi.provenance.toc.StandardTocReader; @@ -67,19 +66,25 @@ public abstract class AbstractTestRecordReaderWriter { writer.close(); final TocReader tocReader = new StandardTocReader(tocFile); + final String expectedTransitUri = "nifi://unit-test"; + final int expectedBlockIndex = 0; + assertRecoveredRecord(journalFile, tocReader, expectedTransitUri, expectedBlockIndex); + + FileUtils.deleteFile(journalFile.getParentFile(), true); + } + + private void assertRecoveredRecord(File journalFile, TocReader tocReader, String expectedTransitUri, int expectedBlockIndex) throws IOException { try (final FileInputStream fis = new FileInputStream(journalFile); - final RecordReader reader = createReader(fis, journalFile.getName(), tocReader, 2048)) { - assertEquals(0, reader.getBlockIndex()); - reader.skipToBlock(0); + final RecordReader reader = createReader(fis, journalFile.getName(), tocReader, 2048)) { + assertEquals(expectedBlockIndex, reader.getBlockIndex()); + reader.skipToBlock(expectedBlockIndex); final StandardProvenanceEventRecord recovered = reader.nextRecord(); assertNotNull(recovered); - assertEquals("nifi://unit-test", recovered.getTransitUri()); + assertEquals(expectedTransitUri, recovered.getTransitUri()); assertNull(reader.nextRecord()); } - - FileUtils.deleteFile(journalFile.getParentFile(), true); } @@ -96,16 +101,7 @@ public abstract class AbstractTestRecordReaderWriter { final TocReader tocReader = new StandardTocReader(tocFile); - try (final FileInputStream fis = new FileInputStream(journalFile); - final RecordReader reader = createReader(fis, journalFile.getName(), tocReader, 2048)) { - assertEquals(0, reader.getBlockIndex()); - reader.skipToBlock(0); - final StandardProvenanceEventRecord recovered = reader.nextRecord(); - assertNotNull(recovered); - - assertEquals("nifi://unit-test", recovered.getTransitUri()); - assertNull(reader.nextRecord()); - } + assertRecoveredRecord(journalFile, tocReader, "nifi://unit-test", 0); FileUtils.deleteFile(journalFile.getParentFile(), true); } diff --git a/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-volatile-provenance-repository/src/main/java/org/apache/nifi/provenance/VolatileProvenanceRepository.java b/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-volatile-provenance-repository/src/main/java/org/apache/nifi/provenance/VolatileProvenanceRepository.java index f08fed4559..c3fbf426ea 100644 --- a/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-volatile-provenance-repository/src/main/java/org/apache/nifi/provenance/VolatileProvenanceRepository.java +++ b/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-volatile-provenance-repository/src/main/java/org/apache/nifi/provenance/VolatileProvenanceRepository.java @@ -16,6 +16,25 @@ */ package org.apache.nifi.provenance; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.Date; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ThreadFactory; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; +import java.util.regex.Pattern; import org.apache.nifi.authorization.AccessDeniedException; import org.apache.nifi.authorization.AuthorizationResult; import org.apache.nifi.authorization.AuthorizationResult.Result; @@ -42,26 +61,6 @@ import org.apache.nifi.util.RingBuffer.ForEachEvaluator; import org.apache.nifi.util.RingBuffer.IterationDirection; import org.apache.nifi.web.ResourceNotFoundException; -import java.io.IOException; -import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; -import java.util.Date; -import java.util.Iterator; -import java.util.List; -import java.util.Map; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ConcurrentMap; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.ThreadFactory; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicBoolean; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.concurrent.atomic.AtomicLong; -import java.util.regex.Pattern; - public class VolatileProvenanceRepository implements ProvenanceRepository { // properties @@ -472,7 +471,7 @@ public class VolatileProvenanceRepository implements ProvenanceRepository { } public Lineage computeLineage(final String flowFileUUID, final NiFiUser user) throws IOException { - return computeLineage(Collections.singleton(flowFileUUID), user, LineageComputationType.FLOWFILE_LINEAGE, null); + return computeLineage(Collections.singleton(flowFileUUID), user, LineageComputationType.FLOWFILE_LINEAGE, null); } private Lineage computeLineage(final Collection flowFileUuids, final NiFiUser user, final LineageComputationType computationType, final Long eventId) throws IOException { @@ -497,7 +496,7 @@ public class VolatileProvenanceRepository implements ProvenanceRepository { final ProvenanceEventRecord event = getEvent(eventId); if (event == null) { final String userId = user.getIdentity(); - final AsyncLineageSubmission result = new AsyncLineageSubmission(LineageComputationType.FLOWFILE_LINEAGE, eventId, Collections.emptySet(), 1, userId); + final AsyncLineageSubmission result = new AsyncLineageSubmission(LineageComputationType.FLOWFILE_LINEAGE, eventId, Collections.emptySet(), 1, userId); result.getResult().setError("Could not find event with ID " + eventId); lineageSubmissionMap.put(result.getLineageIdentifier(), result); return result; @@ -541,9 +540,9 @@ public class VolatileProvenanceRepository implements ProvenanceRepository { final ProvenanceEventRecord event = getEvent(eventId, user); if (event == null) { - final AsyncLineageSubmission submission = new AsyncLineageSubmission(LineageComputationType.EXPAND_PARENTS, eventId, Collections.emptyList(), 1, userId); + final AsyncLineageSubmission submission = new AsyncLineageSubmission(LineageComputationType.EXPAND_PARENTS, eventId, Collections.emptyList(), 1, userId); lineageSubmissionMap.put(submission.getLineageIdentifier(), submission); - submission.getResult().update(Collections. emptyList(), 0L); + submission.getResult().update(Collections.emptyList(), 0L); return submission; } @@ -554,7 +553,7 @@ public class VolatileProvenanceRepository implements ProvenanceRepository { case CLONE: return submitLineageComputation(event.getParentUuids(), user, LineageComputationType.EXPAND_PARENTS, eventId); default: { - final AsyncLineageSubmission submission = new AsyncLineageSubmission(LineageComputationType.EXPAND_PARENTS, eventId, Collections.emptyList(), 1, userId); + final AsyncLineageSubmission submission = new AsyncLineageSubmission(LineageComputationType.EXPAND_PARENTS, eventId, Collections.emptyList(), 1, userId); lineageSubmissionMap.put(submission.getLineageIdentifier(), submission); submission.getResult().setError("Event ID " + eventId + " indicates an event of type " + event.getEventType() + " so its parents cannot be expanded"); return submission; @@ -572,9 +571,9 @@ public class VolatileProvenanceRepository implements ProvenanceRepository { final ProvenanceEventRecord event = getEvent(eventId, user); if (event == null) { - final AsyncLineageSubmission submission = new AsyncLineageSubmission(LineageComputationType.EXPAND_CHILDREN, eventId, Collections.emptyList(), 1, userId); + final AsyncLineageSubmission submission = new AsyncLineageSubmission(LineageComputationType.EXPAND_CHILDREN, eventId, Collections.emptyList(), 1, userId); lineageSubmissionMap.put(submission.getLineageIdentifier(), submission); - submission.getResult().update(Collections. emptyList(), 0L); + submission.getResult().update(Collections.emptyList(), 0L); return submission; } @@ -585,7 +584,7 @@ public class VolatileProvenanceRepository implements ProvenanceRepository { case CLONE: return submitLineageComputation(event.getChildUuids(), user, LineageComputationType.EXPAND_CHILDREN, eventId); default: { - final AsyncLineageSubmission submission = new AsyncLineageSubmission(LineageComputationType.EXPAND_CHILDREN, eventId, Collections.emptyList(), 1, userId); + final AsyncLineageSubmission submission = new AsyncLineageSubmission(LineageComputationType.EXPAND_CHILDREN, eventId, Collections.emptyList(), 1, userId); lineageSubmissionMap.put(submission.getLineageIdentifier(), submission); submission.getResult().setError("Event ID " + eventId + " indicates an event of type " + event.getEventType() + " so its children cannot be expanded"); return submission; @@ -873,5 +872,15 @@ public class VolatileProvenanceRepository implements ProvenanceRepository { public Long getPreviousContentClaimOffset() { return record.getPreviousContentClaimOffset(); } + + /** + * Returns the best event identifier for this event (eventId if available, descriptive identifier if not yet persisted to allow for traceability). + * + * @return a descriptive event ID to allow tracing + */ + @Override + public String getBestEventIdentifier() { + return Long.toString(getEventId()); + } } } diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/pom.xml b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/pom.xml index d0a1f1ca0c..0b206e27c8 100644 --- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/pom.xml +++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/pom.xml @@ -437,7 +437,7 @@ src/test/resources/TestExtractGrok/simple_text.log src/test/resources/TestExtractGrok/patterns - src/main/java/org/apache/nifi/processors/standard/util/crypto/bcrypt/BCrypt.java + src/main/java/org/apache/nifi/security/util/crypto/bcrypt/BCrypt.java diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/EncryptContent.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/EncryptContent.java index 103790eb39..db6d9bad45 100644 --- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/EncryptContent.java +++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/EncryptContent.java @@ -16,6 +16,16 @@ */ package org.apache.nifi.processors.standard; +import java.nio.charset.StandardCharsets; +import java.security.Security; +import java.text.Normalizer; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import java.util.concurrent.TimeUnit; import org.apache.commons.codec.DecoderException; import org.apache.commons.codec.binary.Hex; import org.apache.commons.lang3.StringUtils; @@ -41,27 +51,16 @@ import org.apache.nifi.processor.Relationship; import org.apache.nifi.processor.exception.ProcessException; import org.apache.nifi.processor.io.StreamCallback; import org.apache.nifi.processor.util.StandardValidators; -import org.apache.nifi.processors.standard.util.crypto.CipherUtility; -import org.apache.nifi.processors.standard.util.crypto.KeyedEncryptor; -import org.apache.nifi.processors.standard.util.crypto.OpenPGPKeyBasedEncryptor; -import org.apache.nifi.processors.standard.util.crypto.OpenPGPPasswordBasedEncryptor; -import org.apache.nifi.processors.standard.util.crypto.PasswordBasedEncryptor; import org.apache.nifi.security.util.EncryptionMethod; import org.apache.nifi.security.util.KeyDerivationFunction; +import org.apache.nifi.security.util.crypto.CipherUtility; +import org.apache.nifi.security.util.crypto.KeyedEncryptor; +import org.apache.nifi.security.util.crypto.OpenPGPKeyBasedEncryptor; +import org.apache.nifi.security.util.crypto.OpenPGPPasswordBasedEncryptor; +import org.apache.nifi.security.util.crypto.PasswordBasedEncryptor; import org.apache.nifi.util.StopWatch; import org.bouncycastle.jce.provider.BouncyCastleProvider; -import java.nio.charset.StandardCharsets; -import java.security.Security; -import java.text.Normalizer; -import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; -import java.util.HashSet; -import java.util.List; -import java.util.Set; -import java.util.concurrent.TimeUnit; - @EventDriven @SideEffectFree @SupportsBatching diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/util/crypto/BcryptCipherProvider.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/security/util/crypto/BcryptCipherProvider.java similarity index 98% rename from nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/util/crypto/BcryptCipherProvider.java rename to nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/security/util/crypto/BcryptCipherProvider.java index 125aea7e46..f28cde94ad 100644 --- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/util/crypto/BcryptCipherProvider.java +++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/security/util/crypto/BcryptCipherProvider.java @@ -14,23 +14,22 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.nifi.processors.standard.util.crypto; +package org.apache.nifi.security.util.crypto; -import org.apache.commons.lang3.StringUtils; -import org.apache.nifi.processor.exception.ProcessException; -import org.apache.nifi.processors.standard.util.crypto.bcrypt.BCrypt; -import org.apache.nifi.security.util.EncryptionMethod; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import javax.crypto.Cipher; -import javax.crypto.SecretKey; -import javax.crypto.spec.SecretKeySpec; import java.nio.charset.StandardCharsets; import java.security.MessageDigest; import java.util.Arrays; import java.util.regex.Matcher; import java.util.regex.Pattern; +import javax.crypto.Cipher; +import javax.crypto.SecretKey; +import javax.crypto.spec.SecretKeySpec; +import org.apache.commons.lang3.StringUtils; +import org.apache.nifi.processor.exception.ProcessException; +import org.apache.nifi.security.util.EncryptionMethod; +import org.apache.nifi.security.util.crypto.bcrypt.BCrypt; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public class BcryptCipherProvider extends RandomIVPBECipherProvider { private static final Logger logger = LoggerFactory.getLogger(BcryptCipherProvider.class); diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/util/crypto/CipherProviderFactory.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/security/util/crypto/CipherProviderFactory.java similarity index 97% rename from nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/util/crypto/CipherProviderFactory.java rename to nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/security/util/crypto/CipherProviderFactory.java index e04a7b4956..09004bfadd 100644 --- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/util/crypto/CipherProviderFactory.java +++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/security/util/crypto/CipherProviderFactory.java @@ -14,16 +14,15 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.nifi.processors.standard.util.crypto; +package org.apache.nifi.security.util.crypto; +import java.util.HashMap; +import java.util.Map; import org.apache.nifi.processor.exception.ProcessException; import org.apache.nifi.security.util.KeyDerivationFunction; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.util.HashMap; -import java.util.Map; - public class CipherProviderFactory { private static final Logger logger = LoggerFactory.getLogger(CipherProviderFactory.class); diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/util/crypto/KeyedEncryptor.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/security/util/crypto/KeyedEncryptor.java similarity index 99% rename from nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/util/crypto/KeyedEncryptor.java rename to nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/security/util/crypto/KeyedEncryptor.java index c573d3d4ba..011eb1f0bd 100644 --- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/util/crypto/KeyedEncryptor.java +++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/security/util/crypto/KeyedEncryptor.java @@ -14,8 +14,15 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.nifi.processors.standard.util.crypto; +package org.apache.nifi.security.util.crypto; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.security.NoSuchAlgorithmException; +import javax.crypto.Cipher; +import javax.crypto.SecretKey; +import javax.crypto.spec.SecretKeySpec; import org.apache.commons.lang3.StringUtils; import org.apache.nifi.processor.exception.ProcessException; import org.apache.nifi.processor.io.StreamCallback; @@ -23,14 +30,6 @@ import org.apache.nifi.processors.standard.EncryptContent.Encryptor; import org.apache.nifi.security.util.EncryptionMethod; import org.apache.nifi.security.util.KeyDerivationFunction; -import javax.crypto.Cipher; -import javax.crypto.SecretKey; -import javax.crypto.spec.SecretKeySpec; -import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; -import java.security.NoSuchAlgorithmException; - public class KeyedEncryptor implements Encryptor { private EncryptionMethod encryptionMethod; diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/util/crypto/NiFiLegacyCipherProvider.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/security/util/crypto/NiFiLegacyCipherProvider.java similarity index 97% rename from nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/util/crypto/NiFiLegacyCipherProvider.java rename to nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/security/util/crypto/NiFiLegacyCipherProvider.java index c4626294ee..df8a54d89d 100644 --- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/util/crypto/NiFiLegacyCipherProvider.java +++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/security/util/crypto/NiFiLegacyCipherProvider.java @@ -14,20 +14,19 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.nifi.processors.standard.util.crypto; +package org.apache.nifi.security.util.crypto; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.security.SecureRandom; +import javax.crypto.Cipher; import org.apache.nifi.processor.exception.ProcessException; import org.apache.nifi.security.util.EncryptionMethod; import org.apache.nifi.stream.io.StreamUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import javax.crypto.Cipher; -import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; -import java.security.SecureRandom; - /** * Provides a cipher initialized with the original NiFi key derivation process for password-based encryption (MD5 @ 1000 iterations). This is not a secure * {@link org.apache.nifi.security.util.KeyDerivationFunction} (KDF) and should no longer be used. @@ -45,7 +44,7 @@ public class NiFiLegacyCipherProvider extends OpenSSLPKCS5CipherProvider impleme private static final int ITERATION_COUNT = 1000; /** - * Returns an initialized cipher for the specified algorithm. The key (and IV if necessary) are derived using the NiFi legacy code, based on @see org.apache.nifi.processors.standard.util.crypto + * Returns an initialized cipher for the specified algorithm. The key (and IV if necessary) are derived using the NiFi legacy code, based on @see org.apache.nifi.crypto * .OpenSSLPKCS5CipherProvider#getCipher(java.lang.String, java.lang.String, java.lang.String, byte[], boolean) [essentially {@code MD5(password || salt) * 1000 }]. * * @param encryptionMethod the {@link EncryptionMethod} diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/util/crypto/OpenPGPKeyBasedEncryptor.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/security/util/crypto/OpenPGPKeyBasedEncryptor.java similarity index 99% rename from nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/util/crypto/OpenPGPKeyBasedEncryptor.java rename to nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/security/util/crypto/OpenPGPKeyBasedEncryptor.java index f0f8631e33..6b6c2fc955 100644 --- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/util/crypto/OpenPGPKeyBasedEncryptor.java +++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/security/util/crypto/OpenPGPKeyBasedEncryptor.java @@ -14,8 +14,20 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.nifi.processors.standard.util.crypto; +package org.apache.nifi.security.util.crypto; +import static org.apache.nifi.processors.standard.util.PGPUtil.BLOCK_SIZE; +import static org.apache.nifi.processors.standard.util.PGPUtil.BUFFER_SIZE; + +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.security.NoSuchProviderException; +import java.security.SecureRandom; +import java.util.Date; +import java.util.Iterator; +import java.util.zip.Deflater; import org.apache.nifi.processor.exception.ProcessException; import org.apache.nifi.processor.io.StreamCallback; import org.apache.nifi.processors.standard.EncryptContent; @@ -51,19 +63,6 @@ import org.bouncycastle.openpgp.operator.jcajce.JcePublicKeyKeyEncryptionMethodG import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.FileInputStream; -import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; -import java.security.NoSuchProviderException; -import java.security.SecureRandom; -import java.util.Date; -import java.util.Iterator; -import java.util.zip.Deflater; - -import static org.apache.nifi.processors.standard.util.PGPUtil.BLOCK_SIZE; -import static org.apache.nifi.processors.standard.util.PGPUtil.BUFFER_SIZE; - public class OpenPGPKeyBasedEncryptor implements Encryptor { private static final Logger logger = LoggerFactory.getLogger(OpenPGPPasswordBasedEncryptor.class); diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/util/crypto/OpenPGPPasswordBasedEncryptor.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/security/util/crypto/OpenPGPPasswordBasedEncryptor.java similarity index 99% rename from nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/util/crypto/OpenPGPPasswordBasedEncryptor.java rename to nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/security/util/crypto/OpenPGPPasswordBasedEncryptor.java index 93e565aaea..6d5bb6d3c2 100644 --- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/util/crypto/OpenPGPPasswordBasedEncryptor.java +++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/security/util/crypto/OpenPGPPasswordBasedEncryptor.java @@ -14,8 +14,13 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.nifi.processors.standard.util.crypto; +package org.apache.nifi.security.util.crypto; +import static org.bouncycastle.openpgp.PGPUtil.getDecoderStream; + +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; import org.apache.nifi.processor.exception.ProcessException; import org.apache.nifi.processor.io.StreamCallback; import org.apache.nifi.processors.standard.EncryptContent.Encryptor; @@ -35,12 +40,6 @@ import org.bouncycastle.openpgp.operator.jcajce.JcePBEKeyEncryptionMethodGenerat import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; - -import static org.bouncycastle.openpgp.PGPUtil.getDecoderStream; - public class OpenPGPPasswordBasedEncryptor implements Encryptor { private static final Logger logger = LoggerFactory.getLogger(OpenPGPPasswordBasedEncryptor.class); diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/util/crypto/OpenSSLPKCS5CipherProvider.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/security/util/crypto/OpenSSLPKCS5CipherProvider.java similarity index 99% rename from nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/util/crypto/OpenSSLPKCS5CipherProvider.java rename to nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/security/util/crypto/OpenSSLPKCS5CipherProvider.java index 4253f9b48d..597e516c76 100644 --- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/util/crypto/OpenSSLPKCS5CipherProvider.java +++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/security/util/crypto/OpenSSLPKCS5CipherProvider.java @@ -14,21 +14,8 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.nifi.processors.standard.util.crypto; +package org.apache.nifi.security.util.crypto; -import org.apache.commons.lang3.StringUtils; -import org.apache.nifi.processor.exception.ProcessException; -import org.apache.nifi.security.util.EncryptionMethod; -import org.apache.nifi.stream.io.StreamUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import javax.crypto.Cipher; -import javax.crypto.NoSuchPaddingException; -import javax.crypto.SecretKey; -import javax.crypto.SecretKeyFactory; -import javax.crypto.spec.PBEKeySpec; -import javax.crypto.spec.PBEParameterSpec; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; @@ -40,6 +27,18 @@ import java.security.NoSuchProviderException; import java.security.SecureRandom; import java.security.spec.InvalidKeySpecException; import java.util.Arrays; +import javax.crypto.Cipher; +import javax.crypto.NoSuchPaddingException; +import javax.crypto.SecretKey; +import javax.crypto.SecretKeyFactory; +import javax.crypto.spec.PBEKeySpec; +import javax.crypto.spec.PBEParameterSpec; +import org.apache.commons.lang3.StringUtils; +import org.apache.nifi.processor.exception.ProcessException; +import org.apache.nifi.security.util.EncryptionMethod; +import org.apache.nifi.stream.io.StreamUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public class OpenSSLPKCS5CipherProvider implements PBECipherProvider { private static final Logger logger = LoggerFactory.getLogger(OpenSSLPKCS5CipherProvider.class); diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/util/crypto/PBECipherProvider.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/security/util/crypto/PBECipherProvider.java similarity index 97% rename from nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/util/crypto/PBECipherProvider.java rename to nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/security/util/crypto/PBECipherProvider.java index 4d9fcfc18e..235af00126 100644 --- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/util/crypto/PBECipherProvider.java +++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/security/util/crypto/PBECipherProvider.java @@ -14,14 +14,13 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.nifi.processors.standard.util.crypto; +package org.apache.nifi.security.util.crypto; -import org.apache.nifi.security.util.EncryptionMethod; - -import javax.crypto.Cipher; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; +import javax.crypto.Cipher; +import org.apache.nifi.security.util.EncryptionMethod; public interface PBECipherProvider extends CipherProvider { diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/util/crypto/PBKDF2CipherProvider.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/security/util/crypto/PBKDF2CipherProvider.java similarity index 99% rename from nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/util/crypto/PBKDF2CipherProvider.java rename to nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/security/util/crypto/PBKDF2CipherProvider.java index ee8d5d31e4..fbad3b6b75 100644 --- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/util/crypto/PBKDF2CipherProvider.java +++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/security/util/crypto/PBKDF2CipherProvider.java @@ -14,8 +14,13 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.nifi.processors.standard.util.crypto; +package org.apache.nifi.security.util.crypto; +import java.nio.charset.StandardCharsets; +import java.security.SecureRandom; +import javax.crypto.Cipher; +import javax.crypto.SecretKey; +import javax.crypto.spec.SecretKeySpec; import org.apache.commons.lang3.StringUtils; import org.apache.nifi.processor.exception.ProcessException; import org.apache.nifi.security.util.EncryptionMethod; @@ -30,12 +35,6 @@ import org.bouncycastle.crypto.params.KeyParameter; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import javax.crypto.Cipher; -import javax.crypto.SecretKey; -import javax.crypto.spec.SecretKeySpec; -import java.nio.charset.StandardCharsets; -import java.security.SecureRandom; - public class PBKDF2CipherProvider extends RandomIVPBECipherProvider { private static final Logger logger = LoggerFactory.getLogger(PBKDF2CipherProvider.class); private static final int DEFAULT_SALT_LENGTH = 16; diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/util/crypto/PasswordBasedEncryptor.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/security/util/crypto/PasswordBasedEncryptor.java similarity index 99% rename from nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/util/crypto/PasswordBasedEncryptor.java rename to nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/security/util/crypto/PasswordBasedEncryptor.java index eed5925dbd..8cdfaafe04 100644 --- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/util/crypto/PasswordBasedEncryptor.java +++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/security/util/crypto/PasswordBasedEncryptor.java @@ -14,8 +14,15 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.nifi.processors.standard.util.crypto; +package org.apache.nifi.security.util.crypto; +import java.io.EOFException; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.security.NoSuchAlgorithmException; +import javax.crypto.Cipher; +import javax.crypto.spec.PBEKeySpec; import org.apache.commons.lang3.StringUtils; import org.apache.nifi.processor.exception.ProcessException; import org.apache.nifi.processor.io.StreamCallback; @@ -23,14 +30,6 @@ import org.apache.nifi.processors.standard.EncryptContent.Encryptor; import org.apache.nifi.security.util.EncryptionMethod; import org.apache.nifi.security.util.KeyDerivationFunction; -import javax.crypto.Cipher; -import javax.crypto.spec.PBEKeySpec; -import java.io.EOFException; -import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; -import java.security.NoSuchAlgorithmException; - public class PasswordBasedEncryptor implements Encryptor { private EncryptionMethod encryptionMethod; diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/util/crypto/RandomIVPBECipherProvider.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/security/util/crypto/RandomIVPBECipherProvider.java similarity index 98% rename from nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/util/crypto/RandomIVPBECipherProvider.java rename to nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/security/util/crypto/RandomIVPBECipherProvider.java index 903dfacf9b..66fd1ba070 100644 --- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/util/crypto/RandomIVPBECipherProvider.java +++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/security/util/crypto/RandomIVPBECipherProvider.java @@ -14,17 +14,16 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.nifi.processors.standard.util.crypto; +package org.apache.nifi.security.util.crypto; -import org.apache.nifi.processor.exception.ProcessException; -import org.apache.nifi.security.util.EncryptionMethod; -import org.slf4j.Logger; - -import javax.crypto.Cipher; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; import java.nio.charset.StandardCharsets; +import javax.crypto.Cipher; +import org.apache.nifi.processor.exception.ProcessException; +import org.apache.nifi.security.util.EncryptionMethod; +import org.slf4j.Logger; public abstract class RandomIVPBECipherProvider implements PBECipherProvider { static final byte[] SALT_DELIMITER = "NiFiSALT".getBytes(StandardCharsets.UTF_8); diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/util/crypto/ScryptCipherProvider.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/security/util/crypto/ScryptCipherProvider.java similarity index 98% rename from nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/util/crypto/ScryptCipherProvider.java rename to nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/security/util/crypto/ScryptCipherProvider.java index 15a29e25e2..b532d8e7a5 100644 --- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/util/crypto/ScryptCipherProvider.java +++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/security/util/crypto/ScryptCipherProvider.java @@ -14,27 +14,26 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.nifi.processors.standard.util.crypto; +package org.apache.nifi.security.util.crypto; -import org.apache.commons.codec.DecoderException; -import org.apache.commons.codec.binary.Base64; -import org.apache.commons.codec.binary.Hex; -import org.apache.commons.lang3.StringUtils; -import org.apache.nifi.processor.exception.ProcessException; -import org.apache.nifi.processors.standard.util.crypto.scrypt.Scrypt; -import org.apache.nifi.security.util.EncryptionMethod; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import javax.crypto.Cipher; -import javax.crypto.SecretKey; -import javax.crypto.spec.SecretKeySpec; import java.nio.charset.StandardCharsets; import java.security.SecureRandom; import java.util.ArrayList; import java.util.List; import java.util.regex.Matcher; import java.util.regex.Pattern; +import javax.crypto.Cipher; +import javax.crypto.SecretKey; +import javax.crypto.spec.SecretKeySpec; +import org.apache.commons.codec.DecoderException; +import org.apache.commons.codec.binary.Base64; +import org.apache.commons.codec.binary.Hex; +import org.apache.commons.lang3.StringUtils; +import org.apache.nifi.processor.exception.ProcessException; +import org.apache.nifi.security.util.EncryptionMethod; +import org.apache.nifi.security.util.crypto.scrypt.Scrypt; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public class ScryptCipherProvider extends RandomIVPBECipherProvider { private static final Logger logger = LoggerFactory.getLogger(ScryptCipherProvider.class); diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/util/crypto/bcrypt/BCrypt.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/security/util/crypto/bcrypt/BCrypt.java similarity index 99% rename from nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/util/crypto/bcrypt/BCrypt.java rename to nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/security/util/crypto/bcrypt/BCrypt.java index a57891758e..323fe8f630 100644 --- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/util/crypto/bcrypt/BCrypt.java +++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/security/util/crypto/bcrypt/BCrypt.java @@ -12,7 +12,7 @@ // ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF // OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. -package org.apache.nifi.processors.standard.util.crypto.bcrypt; +package org.apache.nifi.security.util.crypto.bcrypt; import java.io.UnsupportedEncodingException; import java.security.SecureRandom; diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/util/crypto/scrypt/Scrypt.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/security/util/crypto/scrypt/Scrypt.java similarity index 99% rename from nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/util/crypto/scrypt/Scrypt.java rename to nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/security/util/crypto/scrypt/Scrypt.java index 7785e9eca7..2aeae3deb5 100644 --- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/util/crypto/scrypt/Scrypt.java +++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/security/util/crypto/scrypt/Scrypt.java @@ -14,24 +14,23 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.nifi.processors.standard.util.crypto.scrypt; +package org.apache.nifi.security.util.crypto.scrypt; -import org.apache.commons.codec.binary.Base64; -import org.apache.commons.lang3.StringUtils; -import org.apache.nifi.processors.standard.util.crypto.CipherUtility; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; +import static java.lang.Integer.MAX_VALUE; +import static java.lang.System.arraycopy; -import javax.crypto.Mac; -import javax.crypto.spec.SecretKeySpec; import java.nio.charset.StandardCharsets; import java.security.GeneralSecurityException; import java.security.SecureRandom; import java.util.ArrayList; import java.util.List; - -import static java.lang.Integer.MAX_VALUE; -import static java.lang.System.arraycopy; +import javax.crypto.Mac; +import javax.crypto.spec.SecretKeySpec; +import org.apache.commons.codec.binary.Base64; +import org.apache.commons.lang3.StringUtils; +import org.apache.nifi.security.util.crypto.CipherUtility; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/groovy/org/apache/nifi/processors/standard/TestEncryptContentGroovy.groovy b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/groovy/org/apache/nifi/processors/standard/TestEncryptContentGroovy.groovy index e3de6b03d8..f940640fd8 100644 --- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/groovy/org/apache/nifi/processors/standard/TestEncryptContentGroovy.groovy +++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/groovy/org/apache/nifi/processors/standard/TestEncryptContentGroovy.groovy @@ -17,10 +17,10 @@ package org.apache.nifi.processors.standard import org.apache.nifi.components.ValidationResult -import org.apache.nifi.processors.standard.util.crypto.CipherUtility -import org.apache.nifi.processors.standard.util.crypto.PasswordBasedEncryptor import org.apache.nifi.security.util.EncryptionMethod import org.apache.nifi.security.util.KeyDerivationFunction +import org.apache.nifi.security.util.crypto.CipherUtility +import org.apache.nifi.security.util.crypto.PasswordBasedEncryptor import org.apache.nifi.util.MockFlowFile import org.apache.nifi.util.MockProcessContext import org.apache.nifi.util.TestRunner diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/groovy/org/apache/nifi/processors/standard/util/crypto/BcryptCipherProviderGroovyTest.groovy b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/groovy/org/apache/nifi/security/util/crypto/BcryptCipherProviderGroovyTest.groovy similarity index 99% rename from nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/groovy/org/apache/nifi/processors/standard/util/crypto/BcryptCipherProviderGroovyTest.groovy rename to nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/groovy/org/apache/nifi/security/util/crypto/BcryptCipherProviderGroovyTest.groovy index 396e3b266e..e5e001fbc6 100644 --- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/groovy/org/apache/nifi/processors/standard/util/crypto/BcryptCipherProviderGroovyTest.groovy +++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/groovy/org/apache/nifi/security/util/crypto/BcryptCipherProviderGroovyTest.groovy @@ -14,12 +14,12 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.nifi.processors.standard.util.crypto +package org.apache.nifi.security.util.crypto import org.apache.commons.codec.binary.Base64 import org.apache.commons.codec.binary.Hex -import org.apache.nifi.processors.standard.util.crypto.bcrypt.BCrypt import org.apache.nifi.security.util.EncryptionMethod +import org.apache.nifi.security.util.crypto.bcrypt.BCrypt import org.bouncycastle.jce.provider.BouncyCastleProvider import org.junit.After import org.junit.Assume @@ -30,7 +30,6 @@ import org.junit.Test import org.junit.runner.RunWith import org.junit.runners.JUnit4 import org.slf4j.Logger - import org.slf4j.LoggerFactory import javax.crypto.Cipher diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/groovy/org/apache/nifi/processors/standard/util/crypto/CipherProviderFactoryGroovyTest.groovy b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/groovy/org/apache/nifi/security/util/crypto/CipherProviderFactoryGroovyTest.groovy similarity index 98% rename from nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/groovy/org/apache/nifi/processors/standard/util/crypto/CipherProviderFactoryGroovyTest.groovy rename to nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/groovy/org/apache/nifi/security/util/crypto/CipherProviderFactoryGroovyTest.groovy index be8d5f48b4..28da9d1095 100644 --- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/groovy/org/apache/nifi/processors/standard/util/crypto/CipherProviderFactoryGroovyTest.groovy +++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/groovy/org/apache/nifi/security/util/crypto/CipherProviderFactoryGroovyTest.groovy @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.nifi.processors.standard.util.crypto +package org.apache.nifi.security.util.crypto import org.apache.nifi.security.util.KeyDerivationFunction import org.bouncycastle.jce.provider.BouncyCastleProvider diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/groovy/org/apache/nifi/processors/standard/util/crypto/KeyedEncryptorGroovyTest.groovy b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/groovy/org/apache/nifi/security/util/crypto/KeyedEncryptorGroovyTest.groovy similarity index 98% rename from nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/groovy/org/apache/nifi/processors/standard/util/crypto/KeyedEncryptorGroovyTest.groovy rename to nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/groovy/org/apache/nifi/security/util/crypto/KeyedEncryptorGroovyTest.groovy index 8e78778f15..0487af3b91 100644 --- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/groovy/org/apache/nifi/processors/standard/util/crypto/KeyedEncryptorGroovyTest.groovy +++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/groovy/org/apache/nifi/security/util/crypto/KeyedEncryptorGroovyTest.groovy @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.nifi.processors.standard.util.crypto +package org.apache.nifi.security.util.crypto import org.apache.commons.codec.binary.Hex import org.apache.nifi.processor.io.StreamCallback diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/groovy/org/apache/nifi/processors/standard/util/crypto/NiFiLegacyCipherProviderGroovyTest.groovy b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/groovy/org/apache/nifi/security/util/crypto/NiFiLegacyCipherProviderGroovyTest.groovy similarity index 98% rename from nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/groovy/org/apache/nifi/processors/standard/util/crypto/NiFiLegacyCipherProviderGroovyTest.groovy rename to nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/groovy/org/apache/nifi/security/util/crypto/NiFiLegacyCipherProviderGroovyTest.groovy index 176e61bf07..2a3d456cd8 100644 --- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/groovy/org/apache/nifi/processors/standard/util/crypto/NiFiLegacyCipherProviderGroovyTest.groovy +++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/groovy/org/apache/nifi/security/util/crypto/NiFiLegacyCipherProviderGroovyTest.groovy @@ -14,12 +14,17 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.nifi.processors.standard.util.crypto +package org.apache.nifi.security.util.crypto import org.apache.commons.codec.binary.Hex import org.apache.nifi.security.util.EncryptionMethod import org.bouncycastle.jce.provider.BouncyCastleProvider -import org.junit.* +import org.junit.After +import org.junit.Assume +import org.junit.Before +import org.junit.BeforeClass +import org.junit.Ignore +import org.junit.Test import org.junit.runner.RunWith import org.junit.runners.JUnit4 import org.slf4j.Logger diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/groovy/org/apache/nifi/processors/standard/util/crypto/OpenSSLPKCS5CipherProviderGroovyTest.groovy b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/groovy/org/apache/nifi/security/util/crypto/OpenSSLPKCS5CipherProviderGroovyTest.groovy similarity index 98% rename from nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/groovy/org/apache/nifi/processors/standard/util/crypto/OpenSSLPKCS5CipherProviderGroovyTest.groovy rename to nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/groovy/org/apache/nifi/security/util/crypto/OpenSSLPKCS5CipherProviderGroovyTest.groovy index 31cbd5a6b5..62b7970896 100644 --- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/groovy/org/apache/nifi/processors/standard/util/crypto/OpenSSLPKCS5CipherProviderGroovyTest.groovy +++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/groovy/org/apache/nifi/security/util/crypto/OpenSSLPKCS5CipherProviderGroovyTest.groovy @@ -14,12 +14,16 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.nifi.processors.standard.util.crypto +package org.apache.nifi.security.util.crypto import org.apache.commons.codec.binary.Hex import org.apache.nifi.security.util.EncryptionMethod import org.bouncycastle.jce.provider.BouncyCastleProvider -import org.junit.* +import org.junit.After +import org.junit.Assume +import org.junit.Before +import org.junit.BeforeClass +import org.junit.Test import org.junit.runner.RunWith import org.junit.runners.JUnit4 import org.slf4j.Logger diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/groovy/org/apache/nifi/processors/standard/util/crypto/PBKDF2CipherProviderGroovyTest.groovy b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/groovy/org/apache/nifi/security/util/crypto/PBKDF2CipherProviderGroovyTest.groovy similarity index 99% rename from nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/groovy/org/apache/nifi/processors/standard/util/crypto/PBKDF2CipherProviderGroovyTest.groovy rename to nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/groovy/org/apache/nifi/security/util/crypto/PBKDF2CipherProviderGroovyTest.groovy index dfcd0dafea..51a81772a5 100644 --- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/groovy/org/apache/nifi/processors/standard/util/crypto/PBKDF2CipherProviderGroovyTest.groovy +++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/groovy/org/apache/nifi/security/util/crypto/PBKDF2CipherProviderGroovyTest.groovy @@ -14,12 +14,17 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.nifi.processors.standard.util.crypto +package org.apache.nifi.security.util.crypto import org.apache.commons.codec.binary.Hex import org.apache.nifi.security.util.EncryptionMethod import org.bouncycastle.jce.provider.BouncyCastleProvider -import org.junit.* +import org.junit.After +import org.junit.Assume +import org.junit.Before +import org.junit.BeforeClass +import org.junit.Ignore +import org.junit.Test import org.junit.runner.RunWith import org.junit.runners.JUnit4 import org.slf4j.Logger diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/groovy/org/apache/nifi/processors/standard/util/crypto/PasswordBasedEncryptorGroovyTest.groovy b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/groovy/org/apache/nifi/security/util/crypto/PasswordBasedEncryptorGroovyTest.groovy similarity index 99% rename from nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/groovy/org/apache/nifi/processors/standard/util/crypto/PasswordBasedEncryptorGroovyTest.groovy rename to nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/groovy/org/apache/nifi/security/util/crypto/PasswordBasedEncryptorGroovyTest.groovy index 35c20e58c3..7008381f6c 100644 --- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/groovy/org/apache/nifi/processors/standard/util/crypto/PasswordBasedEncryptorGroovyTest.groovy +++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/groovy/org/apache/nifi/security/util/crypto/PasswordBasedEncryptorGroovyTest.groovy @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.nifi.processors.standard.util.crypto +package org.apache.nifi.security.util.crypto import org.apache.commons.codec.binary.Hex import org.apache.nifi.processor.io.StreamCallback diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/groovy/org/apache/nifi/processors/standard/util/crypto/ScryptCipherProviderGroovyTest.groovy b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/groovy/org/apache/nifi/security/util/crypto/ScryptCipherProviderGroovyTest.groovy similarity index 99% rename from nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/groovy/org/apache/nifi/processors/standard/util/crypto/ScryptCipherProviderGroovyTest.groovy rename to nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/groovy/org/apache/nifi/security/util/crypto/ScryptCipherProviderGroovyTest.groovy index 8fd04559b5..8fce9f2ca7 100644 --- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/groovy/org/apache/nifi/processors/standard/util/crypto/ScryptCipherProviderGroovyTest.groovy +++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/groovy/org/apache/nifi/security/util/crypto/ScryptCipherProviderGroovyTest.groovy @@ -14,12 +14,12 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.nifi.processors.standard.util.crypto +package org.apache.nifi.security.util.crypto import org.apache.commons.codec.binary.Base64 import org.apache.commons.codec.binary.Hex -import org.apache.nifi.processors.standard.util.crypto.scrypt.Scrypt import org.apache.nifi.security.util.EncryptionMethod +import org.apache.nifi.security.util.crypto.scrypt.Scrypt import org.bouncycastle.jce.provider.BouncyCastleProvider import org.junit.After import org.junit.Assume diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/groovy/org/apache/nifi/processors/standard/util/crypto/scrypt/ScryptGroovyTest.groovy b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/groovy/org/apache/nifi/security/util/crypto/scrypt/ScryptGroovyTest.groovy similarity index 99% rename from nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/groovy/org/apache/nifi/processors/standard/util/crypto/scrypt/ScryptGroovyTest.groovy rename to nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/groovy/org/apache/nifi/security/util/crypto/scrypt/ScryptGroovyTest.groovy index c154a1fff4..da34c494fb 100644 --- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/groovy/org/apache/nifi/processors/standard/util/crypto/scrypt/ScryptGroovyTest.groovy +++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/groovy/org/apache/nifi/security/util/crypto/scrypt/ScryptGroovyTest.groovy @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.nifi.processors.standard.util.crypto.scrypt +package org.apache.nifi.security.util.crypto.scrypt import org.apache.commons.codec.binary.Hex import org.bouncycastle.jce.provider.BouncyCastleProvider diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestEncryptContent.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestEncryptContent.java index 3166bd0714..063652b8c3 100644 --- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestEncryptContent.java +++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestEncryptContent.java @@ -16,12 +16,17 @@ */ package org.apache.nifi.processors.standard; +import java.io.File; +import java.io.IOException; +import java.nio.file.Paths; +import java.security.Security; +import java.util.Collection; import org.apache.commons.codec.binary.Hex; import org.apache.nifi.components.ValidationResult; -import org.apache.nifi.processors.standard.util.crypto.CipherUtility; -import org.apache.nifi.processors.standard.util.crypto.PasswordBasedEncryptor; import org.apache.nifi.security.util.EncryptionMethod; import org.apache.nifi.security.util.KeyDerivationFunction; +import org.apache.nifi.security.util.crypto.CipherUtility; +import org.apache.nifi.security.util.crypto.PasswordBasedEncryptor; import org.apache.nifi.util.MockFlowFile; import org.apache.nifi.util.MockProcessContext; import org.apache.nifi.util.TestRunner; @@ -34,12 +39,6 @@ import org.junit.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.File; -import java.io.IOException; -import java.nio.file.Paths; -import java.security.Security; -import java.util.Collection; - public class TestEncryptContent { private static final Logger logger = LoggerFactory.getLogger(TestEncryptContent.class); diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/util/crypto/OpenPGPKeyBasedEncryptorTest.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/security/util/crypto/OpenPGPKeyBasedEncryptorTest.java similarity index 98% rename from nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/util/crypto/OpenPGPKeyBasedEncryptorTest.java rename to nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/security/util/crypto/OpenPGPKeyBasedEncryptorTest.java index b4cd2e3972..c823e616d4 100644 --- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/util/crypto/OpenPGPKeyBasedEncryptorTest.java +++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/security/util/crypto/OpenPGPKeyBasedEncryptorTest.java @@ -14,8 +14,17 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.nifi.processors.standard.util.crypto; +package org.apache.nifi.security.util.crypto; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.File; +import java.io.FileInputStream; +import java.io.InputStream; +import java.io.OutputStream; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.security.Security; import org.apache.commons.codec.binary.Hex; import org.apache.nifi.processor.io.StreamCallback; import org.apache.nifi.security.util.EncryptionMethod; @@ -28,16 +37,6 @@ import org.junit.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.ByteArrayInputStream; -import java.io.ByteArrayOutputStream; -import java.io.File; -import java.io.FileInputStream; -import java.io.InputStream; -import java.io.OutputStream; -import java.nio.file.Files; -import java.nio.file.Paths; -import java.security.Security; - public class OpenPGPKeyBasedEncryptorTest { private static final Logger logger = LoggerFactory.getLogger(OpenPGPKeyBasedEncryptorTest.class); diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/util/crypto/OpenPGPPasswordBasedEncryptorTest.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/security/util/crypto/OpenPGPPasswordBasedEncryptorTest.java similarity index 98% rename from nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/util/crypto/OpenPGPPasswordBasedEncryptorTest.java rename to nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/security/util/crypto/OpenPGPPasswordBasedEncryptorTest.java index 5698ea9f7e..2e1cd5f367 100644 --- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/util/crypto/OpenPGPPasswordBasedEncryptorTest.java +++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/security/util/crypto/OpenPGPPasswordBasedEncryptorTest.java @@ -14,8 +14,17 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.nifi.processors.standard.util.crypto; +package org.apache.nifi.security.util.crypto; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.File; +import java.io.FileInputStream; +import java.io.InputStream; +import java.io.OutputStream; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.security.Security; import org.apache.commons.codec.binary.Hex; import org.apache.nifi.processor.io.StreamCallback; import org.apache.nifi.security.util.EncryptionMethod; @@ -28,16 +37,6 @@ import org.junit.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.ByteArrayInputStream; -import java.io.ByteArrayOutputStream; -import java.io.File; -import java.io.FileInputStream; -import java.io.InputStream; -import java.io.OutputStream; -import java.nio.file.Files; -import java.nio.file.Paths; -import java.security.Security; - public class OpenPGPPasswordBasedEncryptorTest { private static final Logger logger = LoggerFactory.getLogger(OpenPGPPasswordBasedEncryptorTest.class); diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/resources/logback-test.xml b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/resources/logback-test.xml index 15e9255884..fad7cb3611 100644 --- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/resources/logback-test.xml +++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/resources/logback-test.xml @@ -39,7 +39,7 @@ - +