Apache/nifi
1
0
Fork 0
mirror of https://github.com/apache/nifi.git synced 2025-03-01 06:59:08 +00:00
Code Issues Packages Projects Releases Wiki Activity

NIFI-3834 This closes #3821. Added encrypted content repository implementation.

Browse Source 
Added skeleton implementation of EncryptedFileSystemRepository.
Added new impl to META-INF registry.
Added investigation comments to FileSystemRepository.
Implemented RepositoryObject block and stream encryptors.
Added passing unit test for encryption and decryption of multiple content writes (large buffered file) for AES-CTR encryptor.
Refactored shared logic from AES CTR and G/CM encryptors to abstract parent.
Added working unit test for writing/reading via encrypted file system repository.
Added stream wrappers.
Added encryptor.
Added working unit test for writing/reading multiple pieces of content via encrypted file system repository.
Added unit test skeleton for writing/reading multiple pieces of content with different keys via encrypted file system repository.
Implemented key management skeleton for encrypted content repository.
Multiple content claims can now be encrypted with different keys on the same resource claim and retrieved.
Implemented validation on setting active key id.
Added content repository encryption properties to NiFiProperties.
Implemented configuration of encryption services from NiFiProperties.
Refactored NiFiPropertiesLoader functionality to CryptoUtils for availability in other modules.
Added RepositoryEncryptionConfiguration and repo-specific subclasses for data containers.
Continued refactoring of CryptoUtils and RepositoryEncryptorUtils library methods.
Exposed some internal state of FileSystemRepository via protected getters so encrypted implementation could access.
Refactored EncryptedFileSystemRepository to extend rather than duplicate FSR.
Refactored EFSR to use ECROS which now extends extracted ContentRepositoryOutputStream protected inner class in FSR.
Added unit test to encrypt & decrypt image resource.
Added smaller image resource for easier unit test debugging.
Added importFrom method to resolve issue where GetFile would not encrypt content persisted to repository.
Added text test resource for tests around exporting claim subsets.
Added exportTo methods to handle decrypting encrypted content.
Performed large unit test refactoring, moving shared logic to helper methods.
Added unit test for merged content claim with header/footer/demarcator.
Added unit test for merging content claims each encrypted with a different key.
Ignored non-deterministically failing firewall DNS test.
Added documentation to User and Admin Guide for Encrypted Content Repository.
Added image.
Added refactored utility method for shared ROEM extraction and validation logic in AbstractAESEncryptor.
Replaced ad-hoc generation of ciphertext stream and byte[] for testing with static initialization from pre-generated serialized form for performance.
Cleaned up unused test code.
Cleaned up Javadoc and code comments.
Refactored shared logic.
Fixed checkstyle issue.
Fixed test failure due to error message change.
Added experimental warning to repository implementation classes and User Guide documentation.

Signed-off-by: Joe Witt <joewitt@apache.org>
This commit is contained in:
Andy LoPresto 2019-01-03 14:20:03 -08:00 committed by Joe Witt
parent 7c9d34f820
commit d148fb1854
No known key found for this signature in database
GPG Key ID: 9093BF854F811A1A
40 changed files with 4382 additions and 251 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
nifi-commons/nifi-data-provenance-utils/src/main/java/org/apache/nifi/provenance/EncryptionMetadata.java
View File

@ -16,16 +16,9 @@
*/
package org.apache.nifi.provenance;
import java.io.Serializable;
import org.apache.commons.codec.binary.Hex;
public class EncryptionMetadata implements Serializable {
protected String keyId;
protected String algorithm;
protected byte[] ivBytes;
protected String version;
protected int cipherByteLength;
import org.apache.nifi.security.repository.RepositoryObjectEncryptionMetadata;
public class EncryptionMetadata extends RepositoryObjectEncryptionMetadata {
EncryptionMetadata() {
}
@ -39,17 +32,8 @@ public class EncryptionMetadata implements Serializable {
@Override
public String toString() {
String sb = "AES Provenance Record Encryption Metadata" +
" Key ID: " +
keyId +
" Algorithm: " +
algorithm +
" IV: " +
Hex.encodeHexString(ivBytes) +
" Version: " +
version +
" Cipher text length: " +
cipherByteLength;
String sb = "Provenance Record Encryption Metadata: " +
super.toString();
return sb;
}
}

62
nifi-commons/nifi-properties/src/main/java/org/apache/nifi/util/NiFiProperties.java
View File

@ -93,6 +93,10 @@ public abstract class NiFiProperties {
public static final String CONTENT_ARCHIVE_ENABLED = "nifi.content.repository.archive.enabled";
public static final String CONTENT_ARCHIVE_CLEANUP_FREQUENCY = "nifi.content.repository.archive.cleanup.frequency";
public static final String CONTENT_VIEWER_URL = "nifi.content.viewer.url";
public static final String CONTENT_REPOSITORY_ENCRYPTION_KEY = "nifi.content.repository.encryption.key";
public static final String CONTENT_REPOSITORY_ENCRYPTION_KEY_ID = "nifi.content.repository.encryption.key.id";
public static final String CONTENT_REPOSITORY_ENCRYPTION_KEY_PROVIDER_IMPLEMENTATION_CLASS = "nifi.content.repository.encryption.key.provider.implementation";
public static final String CONTENT_REPOSITORY_ENCRYPTION_KEY_PROVIDER_LOCATION = "nifi.content.repository.encryption.key.provider.location";
// flowfile repository properties
public static final String FLOWFILE_REPOSITORY_IMPLEMENTATION = "nifi.flowfile.repository.implementation";
@ -1254,7 +1258,7 @@ public abstract class NiFiProperties {
final String vrPropertiesFiles = getVariableRegistryProperties();
if (!StringUtils.isEmpty(vrPropertiesFiles)) {
final List<String> vrPropertiesFileList = Arrays.asList(vrPropertiesFiles.split(","));
final String[] vrPropertiesFileList = vrPropertiesFiles.split(",");
for (String propertiesFile : vrPropertiesFileList) {
vrPropertiesPaths.add(Paths.get(propertiesFile));
@ -1362,6 +1366,55 @@ public abstract class NiFiProperties {
return keys;
}
public String getContentRepositoryEncryptionKeyId() {
return getProperty(CONTENT_REPOSITORY_ENCRYPTION_KEY_ID);
}
/**
* Returns the active content repository encryption key if a {@code StaticKeyProvider} is in use.
* If no key ID is specified in the properties file, the default
* {@code nifi.content.repository.encryption.key} value is returned. If a key ID is specified in
* {@code nifi.content.repository.encryption.key.id}, it will attempt to read from
* {@code nifi.content.repository.encryption.key.id.XYZ} where {@code XYZ} is the provided key
* ID. If that value is empty, it will use the default property
* {@code nifi.content.repository.encryption.key}.
*
* @return the content repository encryption key in hex form
*/
public String getContentRepositoryEncryptionKey() {
String keyId = getContentRepositoryEncryptionKeyId();
String keyKey = StringUtils.isBlank(keyId) ? CONTENT_REPOSITORY_ENCRYPTION_KEY : CONTENT_REPOSITORY_ENCRYPTION_KEY + ".id." + keyId;
return getProperty(keyKey, getProperty(CONTENT_REPOSITORY_ENCRYPTION_KEY));
}
/**
* Returns a map of keyId -> key in hex loaded from the {@code nifi.properties} file if a
* {@code StaticKeyProvider} is defined. If {@code FileBasedKeyProvider} is defined, use
* {@code CryptoUtils#readKeys()} instead -- this method will return an empty map.
*
* @return a Map of the keys identified by key ID
*/
public Map<String, String> getContentRepositoryEncryptionKeys() {
// TODO: Duplicate logic with different constants as provenance should be refactored to helper method
Map<String, String> keys = new HashMap<>();
List<String> keyProperties = getContentRepositoryEncryptionKeyProperties();
// Retrieve the actual key values and store non-empty values in the map
for (String prop : keyProperties) {
final String value = getProperty(prop);
if (!StringUtils.isBlank(value)) {
if (prop.equalsIgnoreCase(CONTENT_REPOSITORY_ENCRYPTION_KEY)) {
prop = getContentRepositoryEncryptionKeyId();
} else {
// Extract nifi.content.repository.encryption.key.id.key1 -> key1
prop = prop.substring(prop.lastIndexOf(".") + 1);
}
keys.put(prop, value);
}
}
return keys;
}
/**
* Returns the whitelisted proxy hostnames (and IP addresses) as a comma-delimited string.
* The hosts have been normalized to the form {@code somehost.com}, {@code somehost.com:port}, or {@code 127.0.0.1}.
@ -1437,6 +1490,13 @@ public abstract class NiFiProperties {
).collect(Collectors.toList());
}
private List<String> getContentRepositoryEncryptionKeyProperties() {
// Filter all the property keys that define a key
return getPropertyKeys().stream().filter(k ->
k.startsWith(CONTENT_REPOSITORY_ENCRYPTION_KEY_ID + ".") || k.equalsIgnoreCase(CONTENT_REPOSITORY_ENCRYPTION_KEY)
).collect(Collectors.toList());
}
public Long getDefaultBackPressureObjectThreshold() {
long backPressureCount;
try {

145
nifi-commons/nifi-security-utils/src/main/java/org/apache/nifi/security/kms/CryptoUtils.java
View File

@ -24,6 +24,8 @@ import java.nio.ByteBuffer;
import java.nio.CharBuffer;
import java.nio.charset.Charset;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Paths;
import java.security.KeyManagementException;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
@ -32,7 +34,9 @@ import java.util.Base64;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.regex.Pattern;
import java.util.stream.Stream;
import javax.crypto.BadPaddingException;
import javax.crypto.Cipher;
import javax.crypto.IllegalBlockSizeException;
@ -47,19 +51,23 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public class CryptoUtils {
private static final Logger logger = LoggerFactory.getLogger(StaticKeyProvider.class);
private static final Logger logger = LoggerFactory.getLogger(CryptoUtils.class);
private static final String STATIC_KEY_PROVIDER_CLASS_NAME = "org.apache.nifi.security.kms.StaticKeyProvider";
private static final String FILE_BASED_KEY_PROVIDER_CLASS_NAME = "org.apache.nifi.security.kms.FileBasedKeyProvider";
private static final String LEGACY_SKP_FQCN = "org.apache.nifi.provenance.StaticKeyProvider";
private static final String LEGACY_FBKP_FQCN = "org.apache.nifi.provenance.FileBasedKeyProvider";
private static final String RELATIVE_NIFI_PROPS_PATH = "conf/nifi.properties";
private static final String BOOTSTRAP_KEY_PREFIX = "nifi.bootstrap.sensitive.key=";
// TODO: Enforce even length
private static final Pattern HEX_PATTERN = Pattern.compile("(?i)^[0-9a-f]+$");
private static final List<Integer> UNLIMITED_KEY_LENGTHS = Arrays.asList(32, 48, 64);
public static final int IV_LENGTH = 16;
private static final String ENCRYPTED_FSR_CLASS_NAME = "org.apache.nifi.controller.repository.crypto.EncryptedFileSystemRepository";
public static boolean isUnlimitedStrengthCryptoAvailable() {
try {
@ -278,17 +286,140 @@ public class CryptoUtils {
}
/**
* Returns {@code true} if the provenance repository is correctly configured for an
* encrypted implementation. Requires the repository implementation to support encryption
* and at least one valid key to be configured.
*
* @param niFiProperties the {@link NiFiProperties} instance to validate
* @return true if encryption is successfully configured for the provenance repository
*/
public static boolean isProvenanceRepositoryEncryptionConfigured(NiFiProperties niFiProperties) {
final String implementationClassName = niFiProperties.getProperty(NiFiProperties.PROVENANCE_REPO_IMPLEMENTATION_CLASS);
// Referencing EWAPR.class.getName() would require a dependency on the module
boolean encryptedRepo = "org.apache.nifi.provenance.EncryptedWriteAheadProvenanceRepository".equals(implementationClassName);
boolean keyProviderConfigured = isValidKeyProvider(
niFiProperties.getProperty(NiFiProperties.PROVENANCE_REPO_ENCRYPTION_KEY_PROVIDER_IMPLEMENTATION_CLASS),
niFiProperties.getProperty(NiFiProperties.PROVENANCE_REPO_ENCRYPTION_KEY_PROVIDER_LOCATION),
niFiProperties.getProvenanceRepoEncryptionKeyId(),
niFiProperties.getProvenanceRepoEncryptionKeys());
if (encryptedRepo) {
return isValidKeyProvider(
niFiProperties.getProperty(NiFiProperties.PROVENANCE_REPO_ENCRYPTION_KEY_PROVIDER_IMPLEMENTATION_CLASS),
niFiProperties.getProperty(NiFiProperties.PROVENANCE_REPO_ENCRYPTION_KEY_PROVIDER_LOCATION),
niFiProperties.getProvenanceRepoEncryptionKeyId(),
niFiProperties.getProvenanceRepoEncryptionKeys());
} else {
return false;
}
}
return encryptedRepo && keyProviderConfigured;
/**
* Returns {@code true} if the content repository is correctly configured for an encrypted
* implementation. Requires the repository implementation to support encryption and at least
* one valid key to be configured.
*
* @param niFiProperties the {@link NiFiProperties} instance to validate
* @return true if encryption is successfully configured for the content repository
*/
public static boolean isContentRepositoryEncryptionConfigured(NiFiProperties niFiProperties) {
final String implementationClassName = niFiProperties.getProperty(NiFiProperties.CONTENT_REPOSITORY_IMPLEMENTATION);
// Referencing EFSR.class.getName() would require a dependency on the module
boolean encryptedRepo = ENCRYPTED_FSR_CLASS_NAME.equals(implementationClassName);
if (encryptedRepo) {
return isValidKeyProvider(
niFiProperties.getProperty(NiFiProperties.CONTENT_REPOSITORY_ENCRYPTION_KEY_PROVIDER_IMPLEMENTATION_CLASS),
niFiProperties.getProperty(NiFiProperties.CONTENT_REPOSITORY_ENCRYPTION_KEY_PROVIDER_LOCATION),
niFiProperties.getContentRepositoryEncryptionKeyId(),
niFiProperties.getContentRepositoryEncryptionKeys());
} else {
return false;
}
}
/**
* Returns the master key from the {@code bootstrap.conf} file used to encrypt various sensitive properties and data encryption keys.
*
* @return the master key
* @throws KeyManagementException if the key cannot be read
*/
public static SecretKey getMasterKey() throws KeyManagementException {
try {
// Get the master encryption key from bootstrap.conf
String masterKeyHex = extractKeyFromBootstrapFile();
return new SecretKeySpec(Hex.decode(masterKeyHex), "AES");
} catch (IOException e) {
logger.error("Encountered an error: ", e);
throw new KeyManagementException(e);
}
}
/**
* Returns the key (if any) used to encrypt sensitive properties, extracted from {@code $NIFI_HOME/conf/bootstrap.conf}.
*
* @return the key in hexadecimal format
* @throws IOException if the file is not readable
*/
public static String extractKeyFromBootstrapFile() throws IOException {
return extractKeyFromBootstrapFile("");
}
/**
* Returns the key (if any) used to encrypt sensitive properties, extracted from {@code $NIFI_HOME/conf/bootstrap.conf}.
*
* @param bootstrapPath the path to the bootstrap file
* @return the key in hexadecimal format
* @throws IOException if the file is not readable
*/
public static String extractKeyFromBootstrapFile(String bootstrapPath) throws IOException {
File expectedBootstrapFile;
if (StringUtils.isBlank(bootstrapPath)) {
// Guess at location of bootstrap.conf file from nifi.properties file
String defaultNiFiPropertiesPath = getDefaultFilePath();
File propertiesFile = new File(defaultNiFiPropertiesPath);
File confDir = new File(propertiesFile.getParent());
if (confDir.exists() && confDir.canRead()) {
expectedBootstrapFile = new File(confDir, "bootstrap.conf");
} else {
logger.error("Cannot read from bootstrap.conf file at {} to extract encryption key -- conf/ directory is missing or permissions are incorrect", confDir.getAbsolutePath());
throw new IOException("Cannot read from bootstrap.conf");
}
} else {
expectedBootstrapFile = new File(bootstrapPath);
}
if (expectedBootstrapFile.exists() && expectedBootstrapFile.canRead()) {
try (Stream<String> stream = Files.lines(Paths.get(expectedBootstrapFile.getAbsolutePath()))) {
Optional<String> keyLine = stream.filter(l -> l.startsWith(BOOTSTRAP_KEY_PREFIX)).findFirst();
if (keyLine.isPresent()) {
return keyLine.get().split("=", 2)[1];
} else {
logger.warn("No encryption key present in the bootstrap.conf file at {}", expectedBootstrapFile.getAbsolutePath());
return "";
}
} catch (IOException e) {
logger.error("Cannot read from bootstrap.conf file at {} to extract encryption key", expectedBootstrapFile.getAbsolutePath());
throw new IOException("Cannot read from bootstrap.conf", e);
}
} else {
logger.error("Cannot read from bootstrap.conf file at {} to extract encryption key -- file is missing or permissions are incorrect", expectedBootstrapFile.getAbsolutePath());
throw new IOException("Cannot read from bootstrap.conf");
}
}
/**
* Returns the default file path to {@code $NIFI_HOME/conf/nifi.properties}. If the system
* property {@code nifi.properties.file.path} is not set, it will be set to the relative
* path {@code conf/nifi.properties}.
*
* @return the path to the nifi.properties file
*/
public static String getDefaultFilePath() {
String systemPath = System.getProperty(NiFiProperties.PROPERTIES_FILE_PATH);
if (systemPath == null || systemPath.trim().isEmpty()) {
logger.warn("The system variable {} is not set, so it is being set to '{}'", NiFiProperties.PROPERTIES_FILE_PATH, RELATIVE_NIFI_PROPS_PATH);
System.setProperty(NiFiProperties.PROPERTIES_FILE_PATH, RELATIVE_NIFI_PROPS_PATH);
systemPath = RELATIVE_NIFI_PROPS_PATH;
}
logger.info("Determined default nifi.properties path to be '{}'", systemPath);
return systemPath;
}
/**

87
nifi-commons/nifi-security-utils/src/main/java/org/apache/nifi/security/kms/EncryptionException.java Normal file
View File

@ -0,0 +1,87 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.nifi.security.kms;
import java.security.PrivilegedActionException;
public class EncryptionException extends Exception {
/**
* Constructs a new exception with the specified detail message. The
* cause is not initialized, and may subsequently be initialized by
* a call to {@link #initCause}.
*
* @param message the detail message. The detail message is saved for
* later retrieval by the {@link #getMessage()} method.
*/
public EncryptionException(String message) {
super(message);
}
/**
* Constructs a new exception with the specified detail message and
* cause. <p>Note that the detail message associated with
* {@code cause} is <i>not</i> automatically incorporated in
* this exception's detail message.
*
* @param message the detail message (which is saved for later retrieval
* by the {@link #getMessage()} method).
* @param cause the cause (which is saved for later retrieval by the
* {@link #getCause()} method). (A <tt>null</tt> value is
* permitted, and indicates that the cause is nonexistent or
* unknown.)
* @since 1.4
*/
public EncryptionException(String message, Throwable cause) {
super(message, cause);
}
/**
* Constructs a new exception with the specified cause and a detail
* message of <tt>(cause==null ? null : cause.toString())</tt> (which
* typically contains the class and detail message of <tt>cause</tt>).
* This constructor is useful for exceptions that are little more than
* wrappers for other throwables (for example, {@link
* PrivilegedActionException}).
*
* @param cause the cause (which is saved for later retrieval by the
* {@link #getCause()} method). (A <tt>null</tt> value is
* permitted, and indicates that the cause is nonexistent or
* unknown.)
* @since 1.4
*/
public EncryptionException(Throwable cause) {
super(cause);
}
/**
* Constructs a new exception with the specified detail message,
* cause, suppression enabled or disabled, and writable stack
* trace enabled or disabled.
*
* @param message the detail message.
* @param cause the cause. (A {@code null} value is permitted,
* and indicates that the cause is nonexistent or unknown.)
* @param enableSuppression whether or not suppression is enabled
* or disabled
* @param writableStackTrace whether or not the stack trace should
* be writable
* @since 1.7
*/
protected EncryptionException(String message, Throwable cause, boolean enableSuppression, boolean writableStackTrace) {
super(message, cause, enableSuppression, writableStackTrace);
}
}

2
nifi-commons/nifi-security-utils/src/main/java/org/apache/nifi/security/kms/KeyProvider.java
View File

@ -57,4 +57,6 @@ public interface KeyProvider {
* @throws KeyManagementException if the key is invalid, the ID conflicts, etc.
*/
boolean addKey(String keyId, SecretKey key) throws OperationNotSupportedException, KeyManagementException;
// TODO: Add #getActiveKeyId() method
}

37
nifi-commons/nifi-security-utils/src/main/java/org/apache/nifi/security/kms/KeyProviderFactory.java
View File

@ -20,12 +20,42 @@ import java.security.KeyManagementException;
import java.util.Map;
import java.util.stream.Collectors;
import javax.crypto.SecretKey;
import org.apache.nifi.security.repository.config.RepositoryEncryptionConfiguration;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* Factory class to build {@link KeyProvider} instances. Currently supports {@link StaticKeyProvider} and {@link FileBasedKeyProvider}.
*/
public class KeyProviderFactory {
private static final Logger logger = LoggerFactory.getLogger(KeyProviderFactory.class);
/**
* Returns a key provider instantiated from the configuration values in a {@link RepositoryEncryptionConfiguration} object.
*
* @param rec the data container for config values (usually extracted from {@link org.apache.nifi.util.NiFiProperties})
* @param masterKey the master key used to decrypt wrapped keys
* @return the configured key provider
* @throws KeyManagementException if the key provider cannot be instantiated
*/
public static KeyProvider buildKeyProvider(RepositoryEncryptionConfiguration rec, SecretKey masterKey) throws KeyManagementException {
if (rec == null) {
throw new KeyManagementException("The repository encryption configuration values are required to build a key provider");
}
return buildKeyProvider(rec.getKeyProviderImplementation(), rec.getKeyProviderLocation(), rec.getEncryptionKeyId(), rec.getEncryptionKeys(), masterKey);
}
/**
* Returns a key provider instantiated from the configuration values in a {@link RepositoryEncryptionConfiguration} object.
*
* @param implementationClassName the key provider class name
* @param keyProviderLocation the filepath/URL of the stored keys
* @param keyId the active key id
* @param encryptionKeys the available encryption keys
* @param masterKey the master key used to decrypt wrapped keys
* @return the configured key provider
* @throws KeyManagementException if the key provider cannot be instantiated
*/
public static KeyProvider buildKeyProvider(String implementationClassName, String keyProviderLocation, String keyId, Map<String, String> encryptionKeys,
SecretKey masterKey) throws KeyManagementException {
KeyProvider keyProvider;
@ -65,6 +95,13 @@ public class KeyProviderFactory {
return keyProvider;
}
/**
* Returns true if this {@link KeyProvider} implementation requires the presence of the {@code master key} in order to decrypt the available data encryption keys.
*
* @param implementationClassName the key provider implementation class
* @return true if this implementation requires the master key to operate
* @throws KeyManagementException if the provided class name is not a valid key provider implementation
*/
public static boolean requiresMasterKey(String implementationClassName) throws KeyManagementException {
implementationClassName = CryptoUtils.handleLegacyPackages(implementationClassName);
return FileBasedKeyProvider.class.getName().equals(implementationClassName);

124
nifi-commons/nifi-security-utils/src/main/java/org/apache/nifi/security/repository/AbstractAESEncryptor.java Normal file
View File

@ -0,0 +1,124 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.nifi.security.repository;
import java.io.IOException;
import java.io.InputStream;
import java.security.KeyManagementException;
import java.security.Security;
import java.util.List;
import org.apache.commons.lang3.StringUtils;
import org.apache.nifi.security.kms.EncryptionException;
import org.apache.nifi.security.kms.KeyProvider;
import org.apache.nifi.security.util.crypto.AESKeyedCipherProvider;
import org.bouncycastle.jce.provider.BouncyCastleProvider;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public abstract class AbstractAESEncryptor implements RepositoryObjectEncryptor {
private static final Logger logger = LoggerFactory.getLogger(AbstractAESEncryptor.class);
private static final byte[] EM_START_SENTINEL = new byte[]{0x00, 0x00};
private static final byte[] EM_END_SENTINEL = new byte[]{(byte) 0xFF, (byte) 0xFF};
private static String ALGORITHM = "AES/CTR/NoPadding";
protected static final int IV_LENGTH = 16;
protected static final byte[] EMPTY_IV = new byte[IV_LENGTH];
// private static final String VERSION = "v1";
// private static final List<String> SUPPORTED_VERSIONS = Arrays.asList(VERSION);
protected KeyProvider keyProvider;
protected AESKeyedCipherProvider aesKeyedCipherProvider = new AESKeyedCipherProvider();
/**
* Initializes the encryptor with a {@link KeyProvider}.
*
* @param keyProvider the key provider which will be responsible for accessing keys
* @throws KeyManagementException if there is an issue configuring the key provider
*/
@Override
public void initialize(KeyProvider keyProvider) throws KeyManagementException {
this.keyProvider = keyProvider;
if (this.aesKeyedCipherProvider == null) {
this.aesKeyedCipherProvider = new AESKeyedCipherProvider();
}
if (Security.getProvider("BC") == null) {
Security.addProvider(new BouncyCastleProvider());
}
}
/**
* Available for dependency injection to override the default {@link AESKeyedCipherProvider} if necessary.
*
* @param cipherProvider the AES cipher provider to use
*/
void setCipherProvider(AESKeyedCipherProvider cipherProvider) {
this.aesKeyedCipherProvider = cipherProvider;
}
/**
* Utility method which extracts the {@link RepositoryObjectEncryptionMetadata} object from the {@code byte[]} or
* {@link InputStream} provided and verifies common validation across both streaming and block decryption. Returns
* the extracted metadata object.
*
* @param ciphertextSource the encrypted source -- can be {@code byte[]} or {@code InputStream}
* @param identifier the unique identifier for this source
* @param descriptor the generic name for this source type for logging/error messages
* @param supportedVersions the list of supported versions for the particular encryptor calling this method (see
* {@link org.apache.nifi.security.repository.stream.aes.RepositoryObjectAESCTREncryptor} and
* {@link org.apache.nifi.security.repository.block.aes.RepositoryObjectAESGCMEncryptor} for
* {@code SUPPORTED_VERSIONS})
* @return the extracted {@link RepositoryObjectEncryptionMetadata} object
* @throws EncryptionException if there is an exception parsing or validating the source
*/
public static RepositoryObjectEncryptionMetadata prepareObjectForDecryption(Object ciphertextSource,
String identifier, String descriptor, List<String> supportedVersions) throws EncryptionException {
if (ciphertextSource == null) {
throw new EncryptionException("The encrypted " + descriptor + " cannot be missing");
}
RepositoryObjectEncryptionMetadata metadata;
try {
if (ciphertextSource instanceof InputStream) {
logger.debug("Detected encrypted input stream for {} with ID {}", descriptor, identifier);
InputStream ciphertextStream = (InputStream) ciphertextSource;
metadata = RepositoryEncryptorUtils.extractEncryptionMetadata(ciphertextStream);
} else if (ciphertextSource instanceof byte[]) {
logger.debug("Detected byte[] for {} with ID {}", descriptor, identifier);
byte[] ciphertextBytes = (byte[]) ciphertextSource;
metadata = RepositoryEncryptorUtils.extractEncryptionMetadata(ciphertextBytes);
} else {
String errorMsg = "The " + descriptor + " with ID " + identifier + " was detected as " + ciphertextSource.getClass().getSimpleName() + "; this is not a supported source of ciphertext";
logger.error(errorMsg);
throw new EncryptionException(errorMsg);
}
} catch (IOException | ClassNotFoundException e) {
final String msg = "Encountered an error reading the encryption metadata: ";
logger.error(msg, e);
throw new EncryptionException(msg, e);
}
if (!supportedVersions.contains(metadata.version)) {
throw new EncryptionException("The " + descriptor + " with ID " + identifier
+ " was encrypted with version " + metadata.version
+ " which is not in the list of supported versions " + StringUtils.join(supportedVersions, ","));
}
return metadata;
}
}

121
nifi-commons/nifi-security-utils/src/main/java/org/apache/nifi/security/repository/RepositoryEncryptorUtils.java Normal file
View File

@ -0,0 +1,121 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.nifi.security.repository;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.ObjectInputStream;
import java.io.ObjectOutputStream;
import java.security.KeyManagementException;
import java.util.Arrays;
import java.util.List;
import javax.crypto.Cipher;
import javax.crypto.SecretKey;
import org.apache.nifi.security.kms.EncryptionException;
import org.apache.nifi.security.kms.KeyProvider;
import org.apache.nifi.security.kms.KeyProviderFactory;
import org.apache.nifi.security.repository.config.RepositoryEncryptionConfiguration;
import org.apache.nifi.security.util.EncryptionMethod;
import org.apache.nifi.security.util.crypto.AESKeyedCipherProvider;
import org.apache.nifi.stream.io.NonCloseableInputStream;
import org.apache.nifi.util.NiFiProperties;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public class RepositoryEncryptorUtils {
private static final Logger logger = LoggerFactory.getLogger(RepositoryEncryptorUtils.class);
private static final int CONTENT_HEADER_SIZE = 2;
private static final int IV_LENGTH = 16;
private static final byte[] EMPTY_IV = new byte[IV_LENGTH];
private static final String VERSION = "v1";
private static final List<String> SUPPORTED_VERSIONS = Arrays.asList(VERSION);
private static final int MIN_METADATA_LENGTH = IV_LENGTH + 3 + 3; // 3 delimiters and 3 non-zero elements
private static final int METADATA_DEFAULT_LENGTH = (20 + 17 + IV_LENGTH + VERSION.length()) * 2; // Default to twice the expected length
// TODO: Add Javadoc
public static byte[] serializeEncryptionMetadata(RepositoryObjectEncryptionMetadata metadata) throws IOException {
final ByteArrayOutputStream baos = new ByteArrayOutputStream();
ObjectOutputStream outputStream = new ObjectOutputStream(baos);
outputStream.writeObject(metadata);
outputStream.close();
return baos.toByteArray();
}
public static Cipher initCipher(AESKeyedCipherProvider aesKeyedCipherProvider, EncryptionMethod method, int mode, SecretKey key, byte[] ivBytes) throws EncryptionException {
try {
if (method == null || key == null || ivBytes == null) {
throw new IllegalArgumentException("Missing critical information");
}
return aesKeyedCipherProvider.getCipher(method, key, ivBytes, mode == Cipher.ENCRYPT_MODE);
} catch (Exception e) {
logger.error("Encountered an exception initializing the cipher", e);
throw new EncryptionException(e);
}
}
public static RepositoryObjectEncryptionMetadata extractEncryptionMetadata(byte[] encryptedRecord) throws EncryptionException, IOException, ClassNotFoundException {
// TODO: Inject parser for min metadata length
if (encryptedRecord == null || encryptedRecord.length < MIN_METADATA_LENGTH) {
throw new EncryptionException("The encrypted record is too short to contain the metadata");
}
// TODO: Inject parser for SENTINEL vs non-SENTINEL
// Skip the first byte (SENTINEL) and don't need to copy all the serialized record
ByteArrayInputStream bais = new ByteArrayInputStream(encryptedRecord);
// bais.read();
try (ObjectInputStream ois = new ObjectInputStream(bais)) {
return (RepositoryObjectEncryptionMetadata) ois.readObject();
}
}
public static RepositoryObjectEncryptionMetadata extractEncryptionMetadata(InputStream encryptedRecord) throws EncryptionException, IOException, ClassNotFoundException {
// TODO: Inject parser for min metadata length
if (encryptedRecord == null) {
throw new EncryptionException("The encrypted record is too short to contain the metadata");
}
// TODO: Inject parser for SENTINEL vs non-SENTINEL
// Skip the first two bytes (EM_START_SENTINEL) and don't need to copy all the serialized record
// TODO: May need to seek for EM_START_SENTINEL segment first
encryptedRecord.read(new byte[CONTENT_HEADER_SIZE]);
try (ObjectInputStream ois = new ObjectInputStream(new NonCloseableInputStream(encryptedRecord))) {
return (RepositoryObjectEncryptionMetadata) ois.readObject();
}
}
public static byte[] extractCipherBytes(byte[] encryptedRecord, RepositoryObjectEncryptionMetadata metadata) {
// If the length is known, there is no header, start from total length - cipher length
// If the length is unknown (streaming/content), calculate the metadata length + header length and start from there
int cipherBytesStart = metadata.cipherByteLength > 0 ? encryptedRecord.length - metadata.cipherByteLength : metadata.length() + CONTENT_HEADER_SIZE;
return Arrays.copyOfRange(encryptedRecord, cipherBytesStart, encryptedRecord.length);
}
public static KeyProvider buildKeyProvider(NiFiProperties niFiProperties, SecretKey masterKey, RepositoryType repositoryType) throws KeyManagementException {
RepositoryEncryptionConfiguration rec = RepositoryEncryptionConfiguration.fromNiFiProperties(niFiProperties, repositoryType);
if (rec.getKeyProviderImplementation() == null) {
throw new KeyManagementException("Cannot create key provider because the NiFi properties are missing the following property: "
+ NiFiProperties.CONTENT_REPOSITORY_ENCRYPTION_KEY_PROVIDER_IMPLEMENTATION_CLASS);
}
return KeyProviderFactory.buildKeyProvider(rec, masterKey);
}
}

64
nifi-commons/nifi-security-utils/src/main/java/org/apache/nifi/security/repository/RepositoryObjectEncryptionMetadata.java Normal file
View File

@ -0,0 +1,64 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.nifi.security.repository;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.ObjectOutputStream;
import java.io.Serializable;
import org.apache.commons.codec.binary.Hex;
public abstract class RepositoryObjectEncryptionMetadata implements Serializable {
public String keyId;
public String algorithm;
public byte[] ivBytes;
public String version;
public int cipherByteLength;
private transient int length = 0;
@Override
public String toString() {
String sb = "Repository Object Encryption Metadata" +
" Key ID: " +
keyId +
" Algorithm: " +
algorithm +
" IV: " +
Hex.encodeHexString(ivBytes) +
" Version: " +
version +
" Cipher text length: " +
cipherByteLength +
" Serialized byte length: " +
length();
return sb;
}
public int length() {
if (length == 0) {
try {
final ByteArrayOutputStream temp = new ByteArrayOutputStream(512);
ObjectOutputStream oos = new ObjectOutputStream(temp);
oos.writeObject(this);
length = temp.size();
} catch (IOException e) {
throw new AssertionError("This is unreachable code");
}
}
return length;
}
}

25
nifi-commons/nifi-security-utils/src/main/java/org/apache/nifi/security/repository/RepositoryObjectEncryptor.java Normal file
View File

@ -0,0 +1,25 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.nifi.security.repository;
import java.security.KeyManagementException;
import org.apache.nifi.security.kms.KeyProvider;
public interface RepositoryObjectEncryptor {
void initialize(KeyProvider keyProvider) throws KeyManagementException;
}

91
nifi-commons/nifi-security-utils/src/main/java/org/apache/nifi/security/repository/RepositoryType.java Normal file
View File

@ -0,0 +1,91 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.nifi.security.repository;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.builder.ToStringBuilder;
import org.apache.commons.lang3.builder.ToStringStyle;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public enum RepositoryType {
CONTENT("Content repository", "content.repository", "stream"),
PROVENANCE("Provenance repository", "provenance.repository", "block"),
FLOWFILE("Flowfile repository", "flowfile.repository", "stream");
private static final Logger logger = LoggerFactory.getLogger(RepositoryType.class);
private final String name;
private final String packagePath;
private final String encryptionProcess;
RepositoryType(String name, String packagePath, String encryptionProcess) {
this.name = name;
this.packagePath = packagePath;
this.encryptionProcess = encryptionProcess;
}
public String getName() {
return name;
}
public String getPackagePath() {
return packagePath;
}
public String getEncryptionProcess() {
return encryptionProcess;
}
@Override
public String toString() {
final ToStringBuilder builder = new ToStringBuilder(this);
ToStringBuilder.setDefaultStyle(ToStringStyle.SHORT_PREFIX_STYLE);
builder.append("Repository", name);
builder.append("Package path", packagePath);
builder.append("Encryption process", encryptionProcess);
return builder.toString();
}
/**
* Uses loose string matching to determine the repository type from input. Matches to
* content, provenance, or flowfile repository, or throws an {@link IllegalArgumentException}
* on input which cannot be reasonably matched to any known repository type.
*
* @param input a string containing some description of the repository type
* @return a matching instance of the RT enum
*/
public static RepositoryType determineType(String input) {
if (StringUtils.isBlank(input)) {
throw new IllegalArgumentException("The input cannot be null or empty");
}
String lowercaseInput = input.toLowerCase().trim();
// Use loose matching to handle prov[enance] and flow[ ][file]
if (lowercaseInput.contains("content")) {
return RepositoryType.CONTENT;
} else if (lowercaseInput.contains("prov")) {
return RepositoryType.PROVENANCE;
} else if (lowercaseInput.contains("flow")) {
return RepositoryType.FLOWFILE;
} else {
final String msg = "Could not determine repository type from '" + input + "'";
logger.error(msg);
throw new IllegalArgumentException(msg);
}
}
}

38
nifi-commons/nifi-security-utils/src/main/java/org/apache/nifi/security/repository/StreamingEncryptionMetadata.java Normal file
View File

@ -0,0 +1,38 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.nifi.security.repository;
public class StreamingEncryptionMetadata extends RepositoryObjectEncryptionMetadata {
public StreamingEncryptionMetadata() {
}
public StreamingEncryptionMetadata(String keyId, String algorithm, byte[] ivBytes, String version) {
this.keyId = keyId;
this.ivBytes = ivBytes;
this.algorithm = algorithm;
this.version = version;
this.cipherByteLength = -1;
}
@Override
public String toString() {
return "Streaming Encryption Metadata: " +
super.toString();
}
}

38
nifi-commons/nifi-security-utils/src/main/java/org/apache/nifi/security/repository/block/BlockEncryptionMetadata.java Normal file
View File

@ -0,0 +1,38 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.nifi.security.repository.block;
import org.apache.nifi.security.repository.RepositoryObjectEncryptionMetadata;
public class BlockEncryptionMetadata extends RepositoryObjectEncryptionMetadata {
public BlockEncryptionMetadata() {
}
public BlockEncryptionMetadata(String keyId, String algorithm, byte[] ivBytes, String version, int cipherByteLength) {
this.keyId = keyId;
this.ivBytes = ivBytes;
this.algorithm = algorithm;
this.version = version;
this.cipherByteLength = cipherByteLength;
}
@Override
public String toString() {
return "Block Encryption Metadata: " +
super.toString();
}
}

68
nifi-commons/nifi-security-utils/src/main/java/org/apache/nifi/security/repository/block/RepositoryObjectBlockEncryptor.java Normal file
View File

@ -0,0 +1,68 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.nifi.security.repository.block;
import java.security.KeyManagementException;
import org.apache.nifi.security.kms.EncryptionException;
import org.apache.nifi.security.kms.KeyProvider;
import org.apache.nifi.security.repository.RepositoryObjectEncryptor;
/**
* Provides an interface for encrypting and decrypting repository objects using a block cipher.
* This is suited for small objects like flowfile and provenance events. For larger objects (like
* content claims), see
* {@link org.apache.nifi.security.repository.stream.RepositoryObjectStreamEncryptor}.
*/
public interface RepositoryObjectBlockEncryptor extends RepositoryObjectEncryptor {
/**
* Initializes the encryptor with a {@link KeyProvider}.
*
* @param keyProvider the key provider which will be responsible for accessing keys
* @throws KeyManagementException if there is an issue configuring the key provider
*/
void initialize(KeyProvider keyProvider) throws KeyManagementException;
/**
* Encrypts the serialized byte[].
*
* @param plainRecord the plain record, serialized to a byte[]
* @param recordId an identifier for this record (eventId, generated, etc.)
* @param keyId the ID of the key to use
* @return the encrypted record
* @throws EncryptionException if there is an issue encrypting this record
*/
byte[] encrypt(byte[] plainRecord, String recordId, String keyId) throws EncryptionException;
/**
* Decrypts the provided byte[] (an encrypted record with accompanying metadata).
*
* @param encryptedRecord the encrypted record in byte[] form
* @param recordId an identifier for this record (eventId, generated, etc.)
* @return the decrypted record
* @throws EncryptionException if there is an issue decrypting this record
*/
byte[] decrypt(byte[] encryptedRecord, String recordId) throws EncryptionException;
/**
* Returns a valid key identifier for this encryptor (valid for encryption and decryption) or throws an exception if none are available.
*
* @return the key ID
* @throws KeyManagementException if no available key IDs are valid for both operations
*/
String getNextKeyId() throws KeyManagementException;
}

170
nifi-commons/nifi-security-utils/src/main/java/org/apache/nifi/security/repository/block/aes/RepositoryObjectAESGCMEncryptor.java Normal file
View File

@ -0,0 +1,170 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.nifi.security.repository.block.aes;
import java.io.IOException;
import java.security.KeyManagementException;
import java.security.SecureRandom;
import java.util.Arrays;
import java.util.List;
import javax.crypto.BadPaddingException;
import javax.crypto.Cipher;
import javax.crypto.IllegalBlockSizeException;
import org.apache.nifi.security.kms.CryptoUtils;
import org.apache.nifi.security.kms.EncryptionException;
import org.apache.nifi.security.repository.AbstractAESEncryptor;
import org.apache.nifi.security.repository.RepositoryEncryptorUtils;
import org.apache.nifi.security.repository.RepositoryObjectEncryptionMetadata;
import org.apache.nifi.security.repository.block.BlockEncryptionMetadata;
import org.apache.nifi.security.repository.block.RepositoryObjectBlockEncryptor;
import org.apache.nifi.security.util.EncryptionMethod;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* This implementation of the {@link RepositoryObjectBlockEncryptor} handles block data by accepting
* {@code byte[]} parameters and returning {@code byte[]} which contain the encrypted/decrypted content. This class
* should be used when a repository needs to persist and retrieve block data with the length known a priori (i.e.
* provenance records or flowfile attribute maps). For repositories handling streams of data with unknown or large
* lengths (i.e. content claims), use the
* {@link org.apache.nifi.security.repository.stream.aes.RepositoryObjectAESCTREncryptor} which does not provide
* authenticated encryption but performs much better with large data.
*/
public class RepositoryObjectAESGCMEncryptor extends AbstractAESEncryptor implements RepositoryObjectBlockEncryptor {
private static final Logger logger = LoggerFactory.getLogger(RepositoryObjectAESGCMEncryptor.class);
private static final String ALGORITHM = "AES/GCM/NoPadding";
// TODO: Increment the version for new implementations?
private static final String VERSION = "v1";
private static final List<String> SUPPORTED_VERSIONS = Arrays.asList(VERSION);
private static final int MIN_METADATA_LENGTH = IV_LENGTH + 3 + 3; // 3 delimiters and 3 non-zero elements
private static final int METADATA_DEFAULT_LENGTH = (20 + ALGORITHM.length() + IV_LENGTH + VERSION.length()) * 2; // Default to twice the expected length
private static final byte[] SENTINEL = new byte[]{0x01};
// TODO: Dependency injection for record parser (provenance SENTINEL vs. flowfile)?
/**
* Encrypts the serialized byte[].
*
* @param plainRecord the plain record, serialized to a byte[]
* @param recordId an identifier for this record (eventId, generated, etc.)
* @param keyId the ID of the key to use
* @return the encrypted record
* @throws EncryptionException if there is an issue encrypting this record
*/
@Override
public byte[] encrypt(byte[] plainRecord, String recordId, String keyId) throws EncryptionException {
if (plainRecord == null || CryptoUtils.isEmpty(keyId)) {
throw new EncryptionException("The repository object and key ID cannot be missing");
}
if (keyProvider == null || !keyProvider.keyExists(keyId)) {
throw new EncryptionException("The requested key ID is not available");
} else {
byte[] ivBytes = new byte[IV_LENGTH];
new SecureRandom().nextBytes(ivBytes);
try {
// TODO: Add object type to description
logger.debug("Encrypting repository object " + recordId + " with key ID " + keyId);
Cipher cipher = RepositoryEncryptorUtils.initCipher(aesKeyedCipherProvider, EncryptionMethod.AES_GCM, Cipher.ENCRYPT_MODE, keyProvider.getKey(keyId), ivBytes);
ivBytes = cipher.getIV();
// Perform the actual encryption
byte[] cipherBytes = cipher.doFinal(plainRecord);
// Serialize and concat encryption details fields (keyId, algo, IV, version, CB length) outside of encryption
RepositoryObjectEncryptionMetadata metadata = new BlockEncryptionMetadata(keyId, ALGORITHM, ivBytes, VERSION, cipherBytes.length);
byte[] serializedEncryptionMetadata = RepositoryEncryptorUtils.serializeEncryptionMetadata(metadata);
// Add the sentinel byte of 0x01
// TODO: Remove (required for prov repo but not FF repo)
logger.debug("Encrypted provenance event record " + recordId + " with key ID " + keyId);
// return CryptoUtils.concatByteArrays(SENTINEL, serializedEncryptionMetadata, cipherBytes);
return CryptoUtils.concatByteArrays(serializedEncryptionMetadata, cipherBytes);
} catch (EncryptionException | BadPaddingException | IllegalBlockSizeException | IOException | KeyManagementException e) {
final String msg = "Encountered an exception encrypting provenance record " + recordId;
logger.error(msg, e);
throw new EncryptionException(msg, e);
}
}
}
/**
* Decrypts the provided byte[] (an encrypted record with accompanying metadata).
*
* @param encryptedRecord the encrypted record in byte[] form
* @param recordId an identifier for this record (eventId, generated, etc.)
* @return the decrypted record
* @throws EncryptionException if there is an issue decrypting this record
*/
@Override
public byte[] decrypt(byte[] encryptedRecord, String recordId) throws EncryptionException {
RepositoryObjectEncryptionMetadata metadata = prepareObjectForDecryption(encryptedRecord, recordId, "provenance record", SUPPORTED_VERSIONS);
// TODO: Actually use the version to determine schema, etc.
if (keyProvider == null || !keyProvider.keyExists(metadata.keyId) || CryptoUtils.isEmpty(metadata.keyId)) {
throw new EncryptionException("The requested key ID " + metadata.keyId + " is not available");
} else {
try {
logger.debug("Decrypting provenance record " + recordId + " with key ID " + metadata.keyId);
EncryptionMethod method = EncryptionMethod.forAlgorithm(metadata.algorithm);
Cipher cipher = RepositoryEncryptorUtils.initCipher(aesKeyedCipherProvider, method, Cipher.DECRYPT_MODE, keyProvider.getKey(metadata.keyId), metadata.ivBytes);
// Strip the metadata away to get just the cipher bytes
byte[] cipherBytes = extractCipherBytes(encryptedRecord, metadata);
// Perform the actual decryption
byte[] plainBytes = cipher.doFinal(cipherBytes);
logger.debug("Decrypted provenance event record " + recordId + " with key ID " + metadata.keyId);
return plainBytes;
} catch (EncryptionException | BadPaddingException | IllegalBlockSizeException | KeyManagementException e) {
final String msg = "Encountered an exception decrypting provenance record " + recordId;
logger.error(msg, e);
throw new EncryptionException(msg, e);
}
}
}
/**
* Returns a valid key identifier for this encryptor (valid for encryption and decryption) or throws an exception if none are available.
*
* @return the key ID
* @throws KeyManagementException if no available key IDs are valid for both operations
*/
@Override
public String getNextKeyId() throws KeyManagementException {
if (keyProvider != null) {
List<String> availableKeyIds = keyProvider.getAvailableKeyIds();
if (!availableKeyIds.isEmpty()) {
return availableKeyIds.get(0);
}
}
throw new KeyManagementException("No available key IDs");
}
private byte[] extractCipherBytes(byte[] encryptedRecord, RepositoryObjectEncryptionMetadata metadata) {
return Arrays.copyOfRange(encryptedRecord, encryptedRecord.length - metadata.cipherByteLength, encryptedRecord.length);
}
@Override
public String toString() {
return "Repository Object Block Encryptor using AES G/CM with Key Provider: " + keyProvider.toString();
}
}

64
nifi-commons/nifi-security-utils/src/main/java/org/apache/nifi/security/repository/config/ContentRepositoryEncryptionConfiguration.java Normal file
View File

@ -0,0 +1,64 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.nifi.security.repository.config;
import java.util.Map;
import org.apache.nifi.security.repository.RepositoryType;
import org.apache.nifi.util.NiFiProperties;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public class ContentRepositoryEncryptionConfiguration extends RepositoryEncryptionConfiguration {
private static final Logger logger = LoggerFactory.getLogger(ContentRepositoryEncryptionConfiguration.class);
/**
* Contructor which accepts a {@link NiFiProperties} object and extracts the relevant
* property values directly.
*
* @param niFiProperties the NiFi properties
*/
public ContentRepositoryEncryptionConfiguration(NiFiProperties niFiProperties) {
this(niFiProperties.getProperty(NiFiProperties.CONTENT_REPOSITORY_ENCRYPTION_KEY_PROVIDER_IMPLEMENTATION_CLASS),
niFiProperties.getProperty(NiFiProperties.CONTENT_REPOSITORY_ENCRYPTION_KEY_PROVIDER_LOCATION),
niFiProperties.getContentRepositoryEncryptionKeyId(),
niFiProperties.getContentRepositoryEncryptionKeys(),
niFiProperties.getProperty(NiFiProperties.CONTENT_REPOSITORY_IMPLEMENTATION)
);
}
/**
* Constructor which accepts explicit values for each configuration value.
*
* @param keyProviderImplementation the key provider implementation class
* @param keyProviderLocation the key provider location
* @param encryptionKeyId the active encryption key id
* @param encryptionKeys the map of available keys
* @param repositoryImplementation the repository implementation class
*/
public ContentRepositoryEncryptionConfiguration(String keyProviderImplementation,
String keyProviderLocation,
String encryptionKeyId,
Map<String, String> encryptionKeys,
String repositoryImplementation) {
this.keyProviderImplementation = keyProviderImplementation;
this.keyProviderLocation = keyProviderLocation;
this.encryptionKeyId = encryptionKeyId;
this.encryptionKeys = encryptionKeys;
this.repositoryImplementation = repositoryImplementation;
this.repositoryType = RepositoryType.CONTENT;
}
}

64
nifi-commons/nifi-security-utils/src/main/java/org/apache/nifi/security/repository/config/ProvenanceRepositoryEncryptionConfiguration.java Normal file
View File

@ -0,0 +1,64 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.nifi.security.repository.config;
import java.util.Map;
import org.apache.nifi.security.repository.RepositoryType;
import org.apache.nifi.util.NiFiProperties;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public class ProvenanceRepositoryEncryptionConfiguration extends RepositoryEncryptionConfiguration {
private static final Logger logger = LoggerFactory.getLogger(ProvenanceRepositoryEncryptionConfiguration.class);
/**
* Constructor which accepts a {@link NiFiProperties} object and extracts the relevant
* property values directly.
*
* @param niFiProperties the NiFi properties
*/
public ProvenanceRepositoryEncryptionConfiguration(NiFiProperties niFiProperties) {
this(niFiProperties.getProperty(NiFiProperties.PROVENANCE_REPO_ENCRYPTION_KEY_PROVIDER_IMPLEMENTATION_CLASS),
niFiProperties.getProperty(NiFiProperties.PROVENANCE_REPO_ENCRYPTION_KEY_PROVIDER_LOCATION),
niFiProperties.getProvenanceRepoEncryptionKeyId(),
niFiProperties.getProvenanceRepoEncryptionKeys(),
niFiProperties.getProperty(NiFiProperties.PROVENANCE_REPO_IMPLEMENTATION_CLASS)
);
}
/**
* Constructor which accepts explicit values for each configuration value.
*
* @param keyProviderImplementation the key provider implementation class
* @param keyProviderLocation the key provider location
* @param encryptionKeyId the active encryption key id
* @param encryptionKeys the map of available keys
* @param repositoryImplementation the repository implementation class
*/
public ProvenanceRepositoryEncryptionConfiguration(String keyProviderImplementation,
String keyProviderLocation,
String encryptionKeyId,
Map<String, String> encryptionKeys,
String repositoryImplementation) {
this.keyProviderImplementation = keyProviderImplementation;
this.keyProviderLocation = keyProviderLocation;
this.encryptionKeyId = encryptionKeyId;
this.encryptionKeys = encryptionKeys;
this.repositoryImplementation = repositoryImplementation;
this.repositoryType = RepositoryType.CONTENT;
}
}

114
nifi-commons/nifi-security-utils/src/main/java/org/apache/nifi/security/repository/config/RepositoryEncryptionConfiguration.java Normal file
View File

@ -0,0 +1,114 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.nifi.security.repository.config;
import java.util.Map;
import javax.crypto.SecretKey;
import org.apache.nifi.security.kms.CryptoUtils;
import org.apache.nifi.security.kms.FileBasedKeyProvider;
import org.apache.nifi.security.kms.KeyProvider;
import org.apache.nifi.security.kms.StaticKeyProvider;
import org.apache.nifi.security.repository.RepositoryType;
import org.apache.nifi.util.NiFiProperties;
/**
* Abstract class which defines the method contracts for various repository encryption configuration
* values. The implementing classes will act as data containers for the encryption configs when
* initializing the repositories.
*/
public abstract class RepositoryEncryptionConfiguration {
String keyProviderImplementation;
String keyProviderLocation;
String encryptionKeyId;
Map<String, String> encryptionKeys;
String repositoryImplementation;
RepositoryType repositoryType;
/**
* Returns the class name of the {@link KeyProvider} implementation used.
*
* @return the class of the key provider
*/
public String getKeyProviderImplementation() {
return keyProviderImplementation;
}
/**
* Returns the location of the key provider. For a
* {@link StaticKeyProvider} this will be null; for all
* others, it will be the location (file path/URL/etc.) to access the key definitions.
*
* @return the file, URL, etc. where the keys are defined
*/
public String getKeyProviderLocation() {
return keyProviderLocation;
}
/**
* Returns the "active" encryption key id.
*
* @return the key id
*/
public String getEncryptionKeyId() {
return encryptionKeyId;
}
/**
* Returns a map of all available encryption keys indexed by the key id if using
* {@link StaticKeyProvider}. For
* {@link FileBasedKeyProvider}, this method will return an
* empty map because the keys must be loaded using the {@code master key} to decrypt them
* via {@link CryptoUtils#readKeys(String, SecretKey)}.
*
* @return a map of key ids & keys
* @see NiFiProperties#getContentRepositoryEncryptionKeys()
*/
public Map<String, String> getEncryptionKeys() {
return encryptionKeys;
}
/**
* Returns the class name for the repository implementation.
*
* @return the repository class
*/
public String getRepositoryImplementation() {
return repositoryImplementation;
}
/**
* Returns the {@link RepositoryType} enum identifying this repository. Useful for
* programmatically determining the kind of repository being configured.
*
* @return the repository type
*/
public RepositoryType getRepositoryType() {
return repositoryType;
}
public static RepositoryEncryptionConfiguration fromNiFiProperties(NiFiProperties niFiProperties, RepositoryType repositoryType) {
switch (repositoryType) {
case CONTENT:
return new ContentRepositoryEncryptionConfiguration(niFiProperties);
case PROVENANCE:
return new ProvenanceRepositoryEncryptionConfiguration(niFiProperties);
case FLOWFILE:
default:
throw new IllegalArgumentException("The specified repository does not support encryption");
}
}
}

71
nifi-commons/nifi-security-utils/src/main/java/org/apache/nifi/security/repository/stream/RepositoryObjectStreamEncryptor.java Normal file
View File

@ -0,0 +1,71 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.nifi.security.repository.stream;
import java.io.InputStream;
import java.io.OutputStream;
import java.security.KeyManagementException;
import org.apache.nifi.security.kms.EncryptionException;
import org.apache.nifi.security.kms.KeyProvider;
import org.apache.nifi.security.repository.RepositoryObjectEncryptor;
/**
* Provides an interface for encrypting and decrypting repository objects using a stream cipher.
* This is suited for large objects like content claims. For small objects (like flowfile and
* provenance events), see
* {@link org.apache.nifi.security.repository.block.RepositoryObjectBlockEncryptor}.
*/
public interface RepositoryObjectStreamEncryptor extends RepositoryObjectEncryptor {
/**
* Initializes the encryptor with a {@link KeyProvider}.
*
* @param keyProvider the key provider which will be responsible for accessing keys
* @throws KeyManagementException if there is an issue configuring the key provider
*/
void initialize(KeyProvider keyProvider) throws KeyManagementException;
/**
* Encrypts the serialized byte[].
*
* @param plainRecord the plain record, serialized to a byte[]
* @param recordId an identifier for this record (eventId, generated, etc.)
* @param keyId the ID of the key to use
* @return the encrypted record
* @throws EncryptionException if there is an issue encrypting this record
*/
OutputStream encrypt(OutputStream plainRecord, String recordId, String keyId) throws EncryptionException;
/**
* Decrypts the provided byte[] (an encrypted record with accompanying metadata).
*
* @param encryptedRecord the encrypted record in byte[] form
* @param recordId an identifier for this record (eventId, generated, etc.)
* @return the decrypted record
* @throws EncryptionException if there is an issue decrypting this record
*/
InputStream decrypt(InputStream encryptedRecord, String recordId) throws EncryptionException;
/**
* Returns a valid key identifier for this encryptor (valid for encryption and decryption) or throws an exception if none are available.
*
* @return the key ID
* @throws KeyManagementException if no available key IDs are valid for both operations
*/
String getNextKeyId() throws KeyManagementException;
}

151
nifi-commons/nifi-security-utils/src/main/java/org/apache/nifi/security/repository/stream/aes/RepositoryObjectAESCTREncryptor.java Normal file
View File

@ -0,0 +1,151 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.nifi.security.repository.stream.aes;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.security.KeyManagementException;
import java.security.SecureRandom;
import java.util.Arrays;
import java.util.List;
import javax.crypto.Cipher;
import javax.crypto.CipherInputStream;
import javax.crypto.CipherOutputStream;
import org.apache.nifi.security.kms.CryptoUtils;
import org.apache.nifi.security.kms.EncryptionException;
import org.apache.nifi.security.repository.AbstractAESEncryptor;
import org.apache.nifi.security.repository.RepositoryEncryptorUtils;
import org.apache.nifi.security.repository.RepositoryObjectEncryptionMetadata;
import org.apache.nifi.security.repository.StreamingEncryptionMetadata;
import org.apache.nifi.security.repository.stream.RepositoryObjectStreamEncryptor;
import org.apache.nifi.security.util.EncryptionMethod;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* This implementation of the {@link RepositoryObjectStreamEncryptor} handles streaming data by accepting
* {@link OutputStream} and {@link InputStream} parameters and returning custom implementations which wrap the normal
* behavior with encryption/decryption logic transparently. This class should be used when a repository needs to persist
* and retrieve streaming data (i.e. content claims). For repositories handling limited blocks of data with the length
* known a priori (i.e. provenance records or flowfile attribute maps), use the
* {@link org.apache.nifi.security.repository.block.aes.RepositoryObjectAESGCMEncryptor} which will provide
* authenticated encryption.
*/
public class RepositoryObjectAESCTREncryptor extends AbstractAESEncryptor implements RepositoryObjectStreamEncryptor {
private static final Logger logger = LoggerFactory.getLogger(RepositoryObjectAESCTREncryptor.class);
private static final byte[] EM_START_SENTINEL = new byte[]{0x00, 0x00};
private static String ALGORITHM = "AES/CTR/NoPadding";
private static final String VERSION = "v1";
private static final List<String> SUPPORTED_VERSIONS = Arrays.asList(VERSION);
/**
* Returns an {@link OutputStream} which encrypts the content of the provided OutputStream. This method works on
* streams to allow for streaming data rather than blocks of bytes of a known length. It is recommended to use this
* for data like flowfile content claims, rather than provenance records or flowfile attribute maps.
*
* @param plainStream the plain OutputStream which is being written to
* @param streamId an identifier for this stream (eventId, generated, etc.)
* @param keyId the ID of the key to use
* @return a stream which will encrypt the data and include the {@link RepositoryObjectEncryptionMetadata}
* @throws EncryptionException if there is an issue encrypting this streaming repository object
*/
@Override
public OutputStream encrypt(OutputStream plainStream, String streamId, String keyId) throws EncryptionException {
if (plainStream == null || CryptoUtils.isEmpty(keyId)) {
throw new EncryptionException("The streaming repository object and key ID cannot be missing");
}
if (keyProvider == null || !keyProvider.keyExists(keyId)) {
throw new EncryptionException("The requested key ID is not available");
} else {
byte[] ivBytes = new byte[IV_LENGTH];
new SecureRandom().nextBytes(ivBytes);
try {
logger.debug("Encrypting streaming repository object " + streamId + " with key ID " + keyId);
Cipher cipher = RepositoryEncryptorUtils.initCipher(aesKeyedCipherProvider, EncryptionMethod.forAlgorithm(ALGORITHM), Cipher.ENCRYPT_MODE, keyProvider.getKey(keyId), ivBytes);
ivBytes = cipher.getIV();
// Prepare the output stream for the actual encryption
CipherOutputStream cipherOutputStream = new CipherOutputStream(plainStream, cipher);
// Serialize and concat encryption details fields (keyId, algo, IV, version, CB length) outside of encryption
RepositoryObjectEncryptionMetadata metadata = new StreamingEncryptionMetadata(keyId, ALGORITHM, ivBytes, VERSION);
byte[] serializedEncryptionMetadata = RepositoryEncryptorUtils.serializeEncryptionMetadata(metadata);
// Write the SENTINEL bytes and the encryption metadata to the raw output stream
plainStream.write(EM_START_SENTINEL);
plainStream.write(serializedEncryptionMetadata);
plainStream.flush();
logger.debug("Encrypted streaming repository object " + streamId + " with key ID " + keyId);
return cipherOutputStream;
} catch (EncryptionException | IOException | KeyManagementException e) {
final String msg = "Encountered an exception encrypting streaming repository object " + streamId;
logger.error(msg, e);
throw new EncryptionException(msg, e);
}
}
}
/**
* Returns an {@link InputStream} which decrypts the content of the provided InputStream. The provided InputStream
* must contain a valid {@link RepositoryObjectEncryptionMetadata} object at the start of the stream. This method
* works on streams to allow for streaming data rather than blocks of bytes of a known length. It is recommended to
* use this for data like flowfile content claims, rather than provenance records or flowfile attribute maps.
*
* @param encryptedInputStream the encrypted InputStream (starting with the plaintext ROEM) which is being read from
* @param streamId an identifier for this stream (eventId, generated, etc.)
* @return a stream which will decrypt the data based on the data in the {@link RepositoryObjectEncryptionMetadata}
* @throws EncryptionException if there is an issue decrypting this streaming repository object
*/
@Override
public InputStream decrypt(InputStream encryptedInputStream, String streamId) throws EncryptionException {
RepositoryObjectEncryptionMetadata metadata = prepareObjectForDecryption(encryptedInputStream, streamId, "streaming repository object", SUPPORTED_VERSIONS);
if (keyProvider == null || !keyProvider.keyExists(metadata.keyId) || CryptoUtils.isEmpty(metadata.keyId)) {
throw new EncryptionException("The requested key ID " + metadata.keyId + " is not available");
} else {
try {
logger.debug("Decrypting streaming repository object with ID " + streamId + " with key ID " + metadata.keyId);
EncryptionMethod method = EncryptionMethod.forAlgorithm(metadata.algorithm);
Cipher cipher = RepositoryEncryptorUtils.initCipher(aesKeyedCipherProvider, method, Cipher.DECRYPT_MODE, keyProvider.getKey(metadata.keyId), metadata.ivBytes);
// Return a new CipherInputStream wrapping the encrypted stream at the present location
CipherInputStream cipherInputStream = new CipherInputStream(encryptedInputStream, cipher);
logger.debug("Decrypted streaming repository object with ID " + streamId + " with key ID " + metadata.keyId);
return cipherInputStream;
} catch (EncryptionException | KeyManagementException e) {
final String msg = "Encountered an exception decrypting streaming repository object with ID " + streamId;
logger.error(msg, e);
throw new EncryptionException(msg, e);
}
}
}
/**
* Returns a valid key identifier for this encryptor (valid for encryption and decryption) or throws an exception if none are available.
*
* @return the key ID
* @throws KeyManagementException if no available key IDs are valid for both operations
*/
@Override
public String getNextKeyId() throws KeyManagementException {
return null;
}
}

160
nifi-commons/nifi-security-utils/src/test/groovy/org/apache/nifi/security/repository/AbstractAESEncryptorTest.groovy Normal file
View File

@ -0,0 +1,160 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.nifi.security.repository
import org.apache.nifi.security.kms.EncryptionException
import org.apache.nifi.security.util.EncryptionMethod
import org.bouncycastle.jce.provider.BouncyCastleProvider
import org.bouncycastle.util.encoders.Hex
import org.junit.After
import org.junit.AfterClass
import org.junit.Before
import org.junit.BeforeClass
import org.junit.Test
import org.junit.runner.RunWith
import org.junit.runners.JUnit4
import org.slf4j.Logger
import org.slf4j.LoggerFactory
import javax.crypto.Cipher
import java.security.Security
@RunWith(JUnit4.class)
class AbstractAESEncryptorTest extends GroovyTestCase {
private static final Logger logger = LoggerFactory.getLogger(AbstractAESEncryptor.class)
private static final String KEY_HEX_128 = "0123456789ABCDEFFEDCBA9876543210"
private static final String KEY_HEX_256 = KEY_HEX_128 * 2
private static final String KEY_HEX = isUnlimitedStrengthCryptoAvailable() ? KEY_HEX_256 : KEY_HEX_128
private static final String KEY_ID = "K1"
private static final String LOG_PACKAGE = "org.slf4j.simpleLogger.log.org.apache.nifi.security.repository"
private static String ORIGINAL_LOG_LEVEL
@BeforeClass
static void setUpOnce() throws Exception {
ORIGINAL_LOG_LEVEL = System.getProperty(LOG_PACKAGE)
System.setProperty(LOG_PACKAGE, "DEBUG")
Security.addProvider(new BouncyCastleProvider())
logger.metaClass.methodMissing = { String name, args ->
logger.info("[${name?.toUpperCase()}] ${(args as List).join(" ")}")
}
}
@Before
void setUp() throws Exception {
}
@After
void tearDown() throws Exception {
}
@AfterClass
static void tearDownOnce() throws Exception {
if (ORIGINAL_LOG_LEVEL) {
System.setProperty(LOG_PACKAGE, ORIGINAL_LOG_LEVEL)
}
}
private static boolean isUnlimitedStrengthCryptoAvailable() {
Cipher.getMaxAllowedKeyLength("AES") > 128
}
/**
* This serialized input was generated by code available in commit 094fea6d1c1f798c4e54cfaf0998416e6c59e41a.
*
* @return a valid {@link InputStream} containing a {@link RepositoryObjectEncryptionMetadata} and ciphertext
*/
private static InputStream formCiphertextStream() {
byte[] encryptedBytes = Hex.decode("0000aced00057372003f6f72672e6170616368652e6e6966692e73656375726974792e7265706" +
"f7369746f72792e53747265616d696e67456e6372797074696f6e4d6574616461746118466982894d442c020000787200466f726" +
"72e6170616368652e6e6966692e73656375726974792e7265706f7369746f72792e5265706f7369746f72794f626a656374456e6" +
"372797074696f6e4d657461646174619f4328584edfdf08020005490010636970686572427974654c656e6774684c0009616c676" +
"f726974686d7400124c6a6176612f6c616e672f537472696e673b5b0007697642797465737400025b424c00056b6579496471007" +
"e00024c000776657273696f6e71007e00027870ffffffff7400114145532f4354522f4e6f50616464696e67757200025b42acf31" +
"7f8060854e00200007870000000109a796446562404a917b9b06479be0f2f7400024b3174000276312356e626790d1b188345a3f" +
"4b5e52cfa4641ed18647caec833ff6a26")
new ByteArrayInputStream(encryptedBytes)
}
/**
* This serialized input was generated by code available in commit 094fea6d1c1f798c4e54cfaf0998416e6c59e41a.
*
* @return a valid {@code byte[]} containing a {@link RepositoryObjectEncryptionMetadata} and ciphertext
*/
private static byte[] formCiphertextBytes() {
byte[] encryptedBytes = Hex.decode("aced0005737200416f72672e6170616368652e6e6966692e73656375726974792e7265706f736" +
"9746f72792e626c6f636b2e426c6f636b456e6372797074696f6e4d6574616461746136c69c49d597a81f020000787200466f726" +
"72e6170616368652e6e6966692e73656375726974792e7265706f7369746f72792e5265706f7369746f72794f626a656374456e6" +
"372797074696f6e4d657461646174619f4328584edfdf08020005490010636970686572427974654c656e6774684c0009616c676" +
"f726974686d7400124c6a6176612f6c616e672f537472696e673b5b0007697642797465737400025b424c00056b6579496471007" +
"e00024c000776657273696f6e71007e000278700000002c7400114145532f47434d2f4e6f50616464696e67757200025b42acf31" +
"7f8060854e002000078700000001054d7d66e359a194854af6d8211def7a47400024b31740002763145c7cfddb413ad677c5e5e4" +
"ba59993db96df90cfff386a62bd9db094c62f752386017d28e267a3eb903090ca")
encryptedBytes
}
@Test
void testShouldPrepareInputStreamForDecryption() {
// Arrange
InputStream ciphertextStream = formCiphertextStream()
// Act
RepositoryObjectEncryptionMetadata metadata = AbstractAESEncryptor.prepareObjectForDecryption(ciphertextStream, "S1", "streaming repository object", ["v1"])
logger.info("Extracted ROEM: ${metadata}")
// Assert
assert metadata.keyId == KEY_ID
assert metadata.algorithm == EncryptionMethod.AES_CTR.algorithm
assert metadata.version == "v1"
}
@Test
void testShouldPrepareByteArrayForDecryption() {
// Arrange
byte[] ciphertextBytes = formCiphertextBytes()
// Act
RepositoryObjectEncryptionMetadata metadata = AbstractAESEncryptor.prepareObjectForDecryption(ciphertextBytes, "R1", "block repository object", ["v1"])
logger.info("Extracted ROEM: ${metadata}")
// Assert
assert metadata.keyId == KEY_ID
assert metadata.algorithm == EncryptionMethod.AES_GCM.algorithm
assert metadata.version == "v1"
}
@Test
void testPrepareObjectForDecryptionShouldFailOnUnsupportedSourceType() {
// Arrange
String ciphertextString = "This is not a valid ciphertext source"
// Act
def msg = shouldFail(EncryptionException) {
RepositoryObjectEncryptionMetadata metadata = AbstractAESEncryptor.prepareObjectForDecryption(ciphertextString, "X1", "unsupported repository object", ["v1"])
logger.info("Extracted ROEM: ${metadata}")
}
// Assert
assert msg == "The unsupported repository object with ID X1 was detected as String; this is not a supported source of ciphertext"
}
}

267
nifi-commons/nifi-security-utils/src/test/groovy/org/apache/nifi/security/repository/block/aes/RepositoryObjectAESGCMEncryptorTest.groovy Normal file
View File

@ -0,0 +1,267 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.nifi.security.repository.block.aes
import org.apache.nifi.security.kms.KeyProvider
import org.apache.nifi.security.util.EncryptionMethod
import org.apache.nifi.security.util.crypto.AESKeyedCipherProvider
import org.bouncycastle.jce.provider.BouncyCastleProvider
import org.bouncycastle.util.encoders.Hex
import org.junit.After
import org.junit.AfterClass
import org.junit.Before
import org.junit.BeforeClass
import org.junit.Test
import org.junit.runner.RunWith
import org.junit.runners.JUnit4
import org.slf4j.Logger
import org.slf4j.LoggerFactory
import javax.crypto.Cipher
import javax.crypto.SecretKey
import javax.crypto.spec.IvParameterSpec
import javax.crypto.spec.SecretKeySpec
import java.nio.charset.StandardCharsets
import java.security.Security
@RunWith(JUnit4.class)
class RepositoryObjectAESGCMEncryptorTest extends GroovyTestCase {
private static final Logger logger = LoggerFactory.getLogger(RepositoryObjectAESGCMEncryptorTest.class)
private static final String KEY_HEX_128 = "0123456789ABCDEFFEDCBA9876543210"
private static final String KEY_HEX_256 = KEY_HEX_128 * 2
private static final String KEY_HEX = isUnlimitedStrengthCryptoAvailable() ? KEY_HEX_256 : KEY_HEX_128
private static final String LOG_PACKAGE = "org.slf4j.simpleLogger.log.org.apache.nifi.security.repository.block.aes"
private static KeyProvider mockKeyProvider
private static AESKeyedCipherProvider mockCipherProvider
private static String ORIGINAL_LOG_LEVEL
private RepositoryObjectAESGCMEncryptor encryptor
@BeforeClass
static void setUpOnce() throws Exception {
ORIGINAL_LOG_LEVEL = System.getProperty(LOG_PACKAGE)
System.setProperty(LOG_PACKAGE, "DEBUG")
Security.addProvider(new BouncyCastleProvider())
logger.metaClass.methodMissing = { String name, args ->
logger.info("[${name?.toUpperCase()}] ${(args as List).join(" ")}")
}
mockKeyProvider = [
getKey : { String keyId ->
logger.mock("Requesting key ID: ${keyId}")
new SecretKeySpec(Hex.decode(KEY_HEX), "AES")
},
keyExists: { String keyId ->
logger.mock("Checking existence of ${keyId}")
true
}] as KeyProvider
mockCipherProvider = [
getCipher: { EncryptionMethod em, SecretKey key, byte[] ivBytes, boolean encryptMode ->
logger.mock("Getting cipher for ${em} with IV ${Hex.toHexString(ivBytes)} encrypt ${encryptMode}")
Cipher cipher = Cipher.getInstance("AES/GCM/NoPadding")
cipher.init((encryptMode ? Cipher.ENCRYPT_MODE : Cipher.DECRYPT_MODE) as int, key, new IvParameterSpec(ivBytes))
cipher
}] as AESKeyedCipherProvider
}
@Before
void setUp() throws Exception {
}
@After
void tearDown() throws Exception {
}
@AfterClass
static void tearDownOnce() throws Exception {
if (ORIGINAL_LOG_LEVEL) {
System.setProperty(LOG_PACKAGE, ORIGINAL_LOG_LEVEL)
}
}
private static boolean isUnlimitedStrengthCryptoAvailable() {
Cipher.getMaxAllowedKeyLength("AES") > 128
}
/**
* Given arbitrary bytes, create an OutputStream, encrypt them, and persist with the (plaintext) encryption metadata, then recover
*/
@Test
void testShouldEncryptAndDecryptArbitraryBytes() {
// Arrange
final byte[] SERIALIZED_BYTES = "This is a plaintext message.".getBytes(StandardCharsets.UTF_8)
logger.info("Serialized bytes (${SERIALIZED_BYTES.size()}): ${Hex.toHexString(SERIALIZED_BYTES)}")
encryptor = new RepositoryObjectAESGCMEncryptor()
encryptor.initialize(mockKeyProvider)
encryptor.setCipherProvider(mockCipherProvider)
logger.info("Created ${encryptor}")
String keyId = "K1"
String recordId = "R1"
logger.info("Using record ID ${recordId} and key ID ${keyId}")
// Act
byte[] encryptedBytes = encryptor.encrypt(SERIALIZED_BYTES, recordId, keyId)
logger.info("Encrypted bytes: ${Hex.toHexString(encryptedBytes)}".toString())
byte[] decryptedBytes = encryptor.decrypt(encryptedBytes, recordId)
logger.info("Decrypted data to: \n\t${Hex.toHexString(decryptedBytes)}")
// Assert
assert decryptedBytes == SERIALIZED_BYTES
logger.info("Decoded: ${new String(decryptedBytes, StandardCharsets.UTF_8)}")
}
/**
* Test which demonstrates that multiple encryption and decryption calls each receive their own independent {@link RepositoryObjectEncryptionMetadata} instance.
*/
@Test
void testShouldEncryptAndDecryptMultiplePiecesOfContent() {
// Arrange
final byte[] SERIALIZED_BYTES_1 = "This is plaintext content 1.".getBytes(StandardCharsets.UTF_8)
final byte[] SERIALIZED_BYTES_2 = "This is plaintext content 2.".getBytes(StandardCharsets.UTF_8)
logger.info("Serialized bytes 1 (${SERIALIZED_BYTES_1.size()}): ${Hex.toHexString(SERIALIZED_BYTES_1)}")
logger.info("Serialized bytes 2 (${SERIALIZED_BYTES_2.size()}): ${Hex.toHexString(SERIALIZED_BYTES_2)}")
encryptor = new RepositoryObjectAESGCMEncryptor()
encryptor.initialize(mockKeyProvider)
encryptor.setCipherProvider(mockCipherProvider)
logger.info("Created ${encryptor}")
String keyId = "K1"
String recordId1 = "R1"
String recordId2 = "R2"
// Act
logger.info("Using record ID ${recordId1} and key ID ${keyId}")
byte[] encryptedBytes1 = encryptor.encrypt(SERIALIZED_BYTES_1, recordId1, keyId)
logger.info("Encrypted bytes 1: ${Hex.toHexString(encryptedBytes1)}".toString())
logger.info("Using record ID ${recordId2} and key ID ${keyId}")
byte[] encryptedBytes2 = encryptor.encrypt(SERIALIZED_BYTES_2, recordId2, keyId)
logger.info("Encrypted bytes 2: ${Hex.toHexString(encryptedBytes2)}".toString())
byte[] decryptedBytes1 = encryptor.decrypt(encryptedBytes1, recordId1)
logger.info("Decrypted data 1 to: \n\t${Hex.toHexString(decryptedBytes1)}")
byte[] decryptedBytes2 = encryptor.decrypt(encryptedBytes2, recordId2)
logger.info("Decrypted data 2 to: \n\t${Hex.toHexString(decryptedBytes2)}")
// Assert
assert decryptedBytes1 == SERIALIZED_BYTES_1
logger.info("Decoded 1: ${new String(decryptedBytes1, StandardCharsets.UTF_8)}")
assert decryptedBytes2 == SERIALIZED_BYTES_2
logger.info("Decoded 2: ${new String(decryptedBytes2, StandardCharsets.UTF_8)}")
}
/**
* Test which demonstrates that encrypting and decrypting large blocks of content (~6 KB) works via block mechanism
*/
@Test
void testShouldEncryptAndDecryptLargeContent() {
// Arrange
final byte[] IMAGE_BYTES = new File("src/test/resources/nifi.png").readBytes()
logger.info("Image bytes (${IMAGE_BYTES.size()}): src/test/resources/nifi.png")
encryptor = new RepositoryObjectAESGCMEncryptor()
encryptor.initialize(mockKeyProvider)
encryptor.setCipherProvider(mockCipherProvider)
logger.info("Created ${encryptor}")
String keyId = "K1"
String recordId = "R1"
// Act
logger.info("Using record ID ${recordId} and key ID ${keyId}")
byte[] encryptedBytes = encryptor.encrypt(IMAGE_BYTES, recordId, keyId)
logger.info("Encrypted bytes (${encryptedBytes.size()}): ${Hex.toHexString(encryptedBytes)[0..<32]}...".toString())
byte[] decryptedBytes = encryptor.decrypt(encryptedBytes, recordId)
logger.info("Decrypted data to (${decryptedBytes.size()}): \n\t${Hex.toHexString(decryptedBytes)[0..<32]}...")
// Assert
assert decryptedBytes == IMAGE_BYTES
logger.info("Decoded (binary PNG header): ${new String(decryptedBytes[0..<16] as byte[], StandardCharsets.UTF_8)}...")
}
/**
* Test which demonstrates that multiple encryption and decryption calls each receive their own independent {@code RepositoryObjectEncryptionMetadata} instance and use independent keys.
*/
@Test
void testShouldEncryptAndDecryptMultiplePiecesOfContentWithDifferentKeys() {
// Arrange
final byte[] SERIALIZED_BYTES_1 = "This is plaintext content 1.".getBytes(StandardCharsets.UTF_8)
final byte[] SERIALIZED_BYTES_2 = "This is plaintext content 2.".getBytes(StandardCharsets.UTF_8)
logger.info("Serialized bytes 1 (${SERIALIZED_BYTES_1.size()}): ${Hex.toHexString(SERIALIZED_BYTES_1)}")
logger.info("Serialized bytes 2 (${SERIALIZED_BYTES_2.size()}): ${Hex.toHexString(SERIALIZED_BYTES_2)}")
// Set up a mock that can provide multiple keys
KeyProvider mockMultipleKeyProvider = [
getKey : { String keyId ->
logger.mock("Requesting key ID: ${keyId}")
def keyHex = keyId == "K1" ? KEY_HEX : "AB" * 16
new SecretKeySpec(Hex.decode(keyHex), "AES")
},
keyExists: { String keyId ->
logger.mock("Checking existence of ${keyId}")
true
}] as KeyProvider
encryptor = new RepositoryObjectAESGCMEncryptor()
encryptor.initialize(mockMultipleKeyProvider)
encryptor.setCipherProvider(mockCipherProvider)
logger.info("Created ${encryptor}")
String keyId1 = "K1"
String keyId2 = "K2"
String recordId1 = "R1"
String recordId2 = "R2"
// Act
logger.info("Using record ID ${recordId1} and key ID ${keyId1}")
byte[] encryptedBytes1 = encryptor.encrypt(SERIALIZED_BYTES_1, recordId1, keyId1)
logger.info("Encrypted bytes 1: ${Hex.toHexString(encryptedBytes1)}".toString())
logger.info("Using record ID ${recordId2} and key ID ${keyId2}")
byte[] encryptedBytes2 = encryptor.encrypt(SERIALIZED_BYTES_2, recordId2, keyId2)
logger.info("Encrypted bytes 2: ${Hex.toHexString(encryptedBytes2)}".toString())
byte[] decryptedBytes1 = encryptor.decrypt(encryptedBytes1, recordId1)
logger.info("Decrypted data 1 to: \n\t${Hex.toHexString(decryptedBytes1)}")
byte[] decryptedBytes2 = encryptor.decrypt(encryptedBytes2, recordId2)
logger.info("Decrypted data 2 to: \n\t${Hex.toHexString(decryptedBytes2)}")
// Assert
assert decryptedBytes1 == SERIALIZED_BYTES_1
logger.info("Decoded 1: ${new String(decryptedBytes1, StandardCharsets.UTF_8)}")
assert decryptedBytes2 == SERIALIZED_BYTES_2
logger.info("Decoded 2: ${new String(decryptedBytes2, StandardCharsets.UTF_8)}")
}
}

389
nifi-commons/nifi-security-utils/src/test/groovy/org/apache/nifi/security/repository/stream/aes/RepositoryObjectAESCTREncryptorTest.groovy Normal file
View File

@ -0,0 +1,389 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.nifi.security.repository.stream.aes
import org.apache.nifi.security.kms.KeyProvider
import org.apache.nifi.security.util.EncryptionMethod
import org.apache.nifi.security.util.crypto.AESKeyedCipherProvider
import org.bouncycastle.jce.provider.BouncyCastleProvider
import org.bouncycastle.util.encoders.Hex
import org.junit.After
import org.junit.AfterClass
import org.junit.Before
import org.junit.BeforeClass
import org.junit.Test
import org.junit.runner.RunWith
import org.junit.runners.JUnit4
import org.slf4j.Logger
import org.slf4j.LoggerFactory
import javax.crypto.Cipher
import javax.crypto.SecretKey
import javax.crypto.spec.IvParameterSpec
import javax.crypto.spec.SecretKeySpec
import java.nio.charset.StandardCharsets
import java.security.Security
@RunWith(JUnit4.class)
class RepositoryObjectAESCTREncryptorTest extends GroovyTestCase {
private static final Logger logger = LoggerFactory.getLogger(RepositoryObjectAESCTREncryptorTest.class)
private static final String KEY_HEX_128 = "0123456789ABCDEFFEDCBA9876543210"
private static final String KEY_HEX_256 = KEY_HEX_128 * 2
private static final String KEY_HEX = isUnlimitedStrengthCryptoAvailable() ? KEY_HEX_256 : KEY_HEX_128
private static final String LOG_PACKAGE = "org.slf4j.simpleLogger.log.org.apache.nifi.security.repository.stream.aes"
private static String ORIGINAL_LOG_LEVEL
private static KeyProvider mockKeyProvider
private static AESKeyedCipherProvider mockCipherProvider
private RepositoryObjectAESCTREncryptor encryptor
@BeforeClass
static void setUpOnce() throws Exception {
ORIGINAL_LOG_LEVEL = System.getProperty(LOG_PACKAGE)
System.setProperty(LOG_PACKAGE, "DEBUG")
Security.addProvider(new BouncyCastleProvider())
logger.metaClass.methodMissing = { String name, args ->
logger.info("[${name?.toUpperCase()}] ${(args as List).join(" ")}")
}
mockKeyProvider = [
getKey : { String keyId ->
logger.mock("Requesting key ID: ${keyId}")
new SecretKeySpec(Hex.decode(KEY_HEX), "AES")
},
keyExists: { String keyId ->
logger.mock("Checking existence of ${keyId}")
true
}] as KeyProvider
mockCipherProvider = [
getCipher: { EncryptionMethod em, SecretKey key, byte[] ivBytes, boolean encryptMode ->
logger.mock("Getting cipher for ${em} with IV ${Hex.toHexString(ivBytes)} encrypt ${encryptMode}")
Cipher cipher = Cipher.getInstance("AES/CTR/NoPadding")
cipher.init((encryptMode ? Cipher.ENCRYPT_MODE : Cipher.DECRYPT_MODE) as int, key, new IvParameterSpec(ivBytes))
cipher
}] as AESKeyedCipherProvider
}
@Before
void setUp() throws Exception {
}
@After
void tearDown() throws Exception {
}
@AfterClass
static void tearDownOnce() throws Exception {
if (ORIGINAL_LOG_LEVEL) {
System.setProperty(LOG_PACKAGE, ORIGINAL_LOG_LEVEL)
}
}
private static boolean isUnlimitedStrengthCryptoAvailable() {
Cipher.getMaxAllowedKeyLength("AES") > 128
}
/**
* Given arbitrary bytes, create an OutputStream, encrypt them, and persist with the (plaintext) encryption metadata, then recover
*/
@Test
void testShouldEncryptAndDecryptArbitraryBytes() {
// Arrange
final byte[] SERIALIZED_BYTES = "This is a plaintext message.".getBytes(StandardCharsets.UTF_8)
logger.info("Serialized bytes (${SERIALIZED_BYTES.size()}): ${Hex.toHexString(SERIALIZED_BYTES)}")
encryptor = new RepositoryObjectAESCTREncryptor()
encryptor.initialize(mockKeyProvider)
encryptor.setCipherProvider(mockCipherProvider)
logger.info("Created ${encryptor}")
String keyId = "K1"
String recordId = "R1"
logger.info("Using record ID ${recordId} and key ID ${keyId}")
OutputStream encryptDestination = new ByteArrayOutputStream(256)
// Act
OutputStream encryptedOutputStream = encryptor.encrypt(encryptDestination, recordId, keyId)
encryptedOutputStream.write(SERIALIZED_BYTES)
encryptedOutputStream.flush()
encryptedOutputStream.close()
byte[] encryptedBytes = encryptDestination.toByteArray()
logger.info("Encrypted bytes: ${Hex.toHexString(encryptedBytes)}".toString())
InputStream encryptedInputStream = new ByteArrayInputStream(encryptedBytes)
InputStream decryptedInputStream = encryptor.decrypt(encryptedInputStream, recordId)
byte[] recoveredBytes = new byte[SERIALIZED_BYTES.length]
decryptedInputStream.read(recoveredBytes)
logger.info("Decrypted data to: \n\t${Hex.toHexString(recoveredBytes)}")
// Assert
assert recoveredBytes == SERIALIZED_BYTES
logger.info("Decoded: ${new String(recoveredBytes, StandardCharsets.UTF_8)}")
}
/**
* Test which demonstrates that normal mechanism of {@code OutputStream os = repository.write(contentClaim); os.write(content1); os.write(content2);} works because only one encryption metadata record is written (before {@code content1}). {@code content2} is written with the same recordId and keyId because the output stream is written to by the same {@code session.write()}
*/
@Test
void testShouldEncryptAndDecryptMultiplePiecesOfContent() {
// Arrange
final byte[] SERIALIZED_BYTES_1 = "This is plaintext content 1.".getBytes(StandardCharsets.UTF_8)
final byte[] SERIALIZED_BYTES_2 = "This is plaintext content 2.".getBytes(StandardCharsets.UTF_8)
logger.info("Serialized bytes 1 (${SERIALIZED_BYTES_1.size()}): ${Hex.toHexString(SERIALIZED_BYTES_1)}")
logger.info("Serialized bytes 2 (${SERIALIZED_BYTES_2.size()}): ${Hex.toHexString(SERIALIZED_BYTES_2)}")
encryptor = new RepositoryObjectAESCTREncryptor()
encryptor.initialize(mockKeyProvider)
encryptor.setCipherProvider(mockCipherProvider)
logger.info("Created ${encryptor}")
String keyId = "K1"
String recordId = "R1"
OutputStream encryptDestination = new ByteArrayOutputStream(512)
// Act
logger.info("Using record ID ${recordId} and key ID ${keyId}")
OutputStream encryptedOutputStream = encryptor.encrypt(encryptDestination, recordId, keyId)
encryptedOutputStream.write(SERIALIZED_BYTES_1)
encryptedOutputStream.write(SERIALIZED_BYTES_2)
encryptedOutputStream.flush()
encryptedOutputStream.close()
byte[] encryptedBytes = encryptDestination.toByteArray()
logger.info("Encrypted bytes: ${Hex.toHexString(encryptedBytes)}".toString())
InputStream encryptedInputStream = new ByteArrayInputStream(encryptedBytes)
InputStream decryptedInputStream = encryptor.decrypt(encryptedInputStream, recordId)
byte[] recoveredBytes1 = new byte[SERIALIZED_BYTES_1.length]
decryptedInputStream.read(recoveredBytes1)
logger.info("Decrypted data 1 to: \n\t${Hex.toHexString(recoveredBytes1)}")
byte[] recoveredBytes2 = new byte[SERIALIZED_BYTES_2.length]
decryptedInputStream.read(recoveredBytes2)
logger.info("Decrypted data 2 to: \n\t${Hex.toHexString(recoveredBytes2)}")
// Assert
assert recoveredBytes1 == SERIALIZED_BYTES_1
logger.info("Decoded 1: ${new String(recoveredBytes1, StandardCharsets.UTF_8)}")
assert recoveredBytes2 == SERIALIZED_BYTES_2
logger.info("Decoded 2: ${new String(recoveredBytes2, StandardCharsets.UTF_8)}")
}
/**
* Test which demonstrates that encrypting and decrypting large blocks of content (~6 KB) works via streaming mechanism
*/
@Test
void testShouldEncryptAndDecryptLargeContent() {
// Arrange
final byte[] IMAGE_BYTES = new File("src/test/resources/nifi.png").readBytes()
logger.info("Image bytes (${IMAGE_BYTES.size()}): src/test/resources/nifi.png")
// Arbitrary buffer size to force multiple writes
final int BUFFER_SIZE = 256
encryptor = new RepositoryObjectAESCTREncryptor()
encryptor.initialize(mockKeyProvider)
encryptor.setCipherProvider(mockCipherProvider)
logger.info("Created ${encryptor}")
String keyId = "K1"
String recordId = "R1"
// Create a stream with enough room for the content and some header & encryption overhead
OutputStream encryptDestination = new ByteArrayOutputStream(6 * 1024 + 512)
// Act
logger.info("Using record ID ${recordId} and key ID ${keyId}")
OutputStream encryptedOutputStream = encryptor.encrypt(encryptDestination, recordId, keyId)
// Buffer the byte[] writing to the stream in chunks of BUFFER_SIZE
for (int i = 0; i < IMAGE_BYTES.length; i+= BUFFER_SIZE) {
int buf = Math.min(i+BUFFER_SIZE, IMAGE_BYTES.length)
encryptedOutputStream.write((byte[]) (IMAGE_BYTES[i..<buf]))
encryptedOutputStream.flush()
}
encryptedOutputStream.close()
byte[] encryptedBytes = encryptDestination.toByteArray()
logger.info("Encrypted bytes (${encryptedBytes.size()}): ${Hex.toHexString(encryptedBytes)}".toString())
InputStream encryptedInputStream = new ByteArrayInputStream(encryptedBytes)
InputStream decryptedInputStream = encryptor.decrypt(encryptedInputStream, recordId)
byte[] recoveredBytes = decryptedInputStream.getBytes()
logger.info("Decrypted data to (${recoveredBytes.size()}): \n\t${Hex.toHexString(recoveredBytes)}")
// Assert
assert recoveredBytes == IMAGE_BYTES
logger.info("Decoded (binary PNG header): ${new String(recoveredBytes[0..<16] as byte[], StandardCharsets.UTF_8)}...")
}
/**
* Test which demonstrates that if two {@code #encrypt()} calls are made, each piece of content is encrypted and decrypted independently, and each has its own encryption metadata persisted
*/
@Test
void testShouldEncryptAndDecryptMultiplePiecesOfContentIndependently() {
// Arrange
final byte[] SERIALIZED_BYTES_1 = "This is plaintext content 1.".getBytes(StandardCharsets.UTF_8)
final byte[] SERIALIZED_BYTES_2 = "This is plaintext content 2.".getBytes(StandardCharsets.UTF_8)
logger.info("Serialized bytes 1 (${SERIALIZED_BYTES_1.size()}): ${Hex.toHexString(SERIALIZED_BYTES_1)}")
logger.info("Serialized bytes 2 (${SERIALIZED_BYTES_2.size()}): ${Hex.toHexString(SERIALIZED_BYTES_2)}")
encryptor = new RepositoryObjectAESCTREncryptor()
encryptor.initialize(mockKeyProvider)
encryptor.setCipherProvider(mockCipherProvider)
logger.info("Created ${encryptor}")
String keyId = "K1"
String recordId1 = "R1"
String recordId2 = "R2"
OutputStream encryptDestination1 = new ByteArrayOutputStream(512)
OutputStream encryptDestination2 = new ByteArrayOutputStream(512)
// Act
logger.info("Using record ID ${recordId1} and key ID ${keyId}")
OutputStream encryptedOutputStream1 = encryptor.encrypt(encryptDestination1, recordId1, keyId)
encryptedOutputStream1.write(SERIALIZED_BYTES_1)
encryptedOutputStream1.flush()
encryptedOutputStream1.close()
logger.info("Using record ID ${recordId2} and key ID ${keyId}")
OutputStream encryptedOutputStream2 = encryptor.encrypt(encryptDestination2, recordId2, keyId)
encryptedOutputStream2.write(SERIALIZED_BYTES_2)
encryptedOutputStream2.flush()
encryptedOutputStream2.close()
byte[] encryptedBytes1 = encryptDestination1.toByteArray()
logger.info("Encrypted bytes 1: ${Hex.toHexString(encryptedBytes1)}".toString())
byte[] encryptedBytes2 = encryptDestination2.toByteArray()
logger.info("Encrypted bytes 2: ${Hex.toHexString(encryptedBytes2)}".toString())
InputStream encryptedInputStream1 = new ByteArrayInputStream(encryptedBytes1)
InputStream encryptedInputStream2 = new ByteArrayInputStream(encryptedBytes2)
InputStream decryptedInputStream1 = encryptor.decrypt(encryptedInputStream1, recordId1)
byte[] recoveredBytes1 = new byte[SERIALIZED_BYTES_1.length]
decryptedInputStream1.read(recoveredBytes1)
logger.info("Decrypted data 1 to: \n\t${Hex.toHexString(recoveredBytes1)}")
InputStream decryptedInputStream2 = encryptor.decrypt(encryptedInputStream2, recordId2)
byte[] recoveredBytes2 = new byte[SERIALIZED_BYTES_2.length]
decryptedInputStream2.read(recoveredBytes2)
logger.info("Decrypted data 2 to: \n\t${Hex.toHexString(recoveredBytes2)}")
// Assert
assert recoveredBytes1 == SERIALIZED_BYTES_1
logger.info("Decoded 1: ${new String(recoveredBytes1, StandardCharsets.UTF_8)}")
assert recoveredBytes2 == SERIALIZED_BYTES_2
logger.info("Decoded 2: ${new String(recoveredBytes2, StandardCharsets.UTF_8)}")
}
/**
* Test which demonstrates that if two {@code #encrypt()} calls are made *with different keys*, each piece of content is encrypted and decrypted independently, and each has its own encryption metadata persisted (including the key ID)
*/
@Test
void testShouldEncryptAndDecryptMultiplePiecesOfContentWithDifferentKeys() {
// Arrange
final byte[] SERIALIZED_BYTES_1 = "This is plaintext content 1.".getBytes(StandardCharsets.UTF_8)
final byte[] SERIALIZED_BYTES_2 = "This is plaintext content 2.".getBytes(StandardCharsets.UTF_8)
logger.info("Serialized bytes 1 (${SERIALIZED_BYTES_1.size()}): ${Hex.toHexString(SERIALIZED_BYTES_1)}")
logger.info("Serialized bytes 2 (${SERIALIZED_BYTES_2.size()}): ${Hex.toHexString(SERIALIZED_BYTES_2)}")
// Set up a mock that can provide multiple keys
KeyProvider mockMultipleKeyProvider = [
getKey : { String keyId ->
logger.mock("Requesting key ID: ${keyId}")
def keyHex = keyId == "K1" ? KEY_HEX : "AB" * 16
new SecretKeySpec(Hex.decode(keyHex), "AES")
},
keyExists: { String keyId ->
logger.mock("Checking existence of ${keyId}")
true
}] as KeyProvider
encryptor = new RepositoryObjectAESCTREncryptor()
encryptor.initialize(mockMultipleKeyProvider)
encryptor.setCipherProvider(mockCipherProvider)
logger.info("Created ${encryptor}")
String keyId1 = "K1"
String keyId2 = "K1"
String recordId1 = "R1"
String recordId2 = "R2"
OutputStream encryptDestination1 = new ByteArrayOutputStream(512)
OutputStream encryptDestination2 = new ByteArrayOutputStream(512)
// Act
logger.info("Using record ID ${recordId1} and key ID ${keyId1}")
OutputStream encryptedOutputStream1 = encryptor.encrypt(encryptDestination1, recordId1, keyId1)
encryptedOutputStream1.write(SERIALIZED_BYTES_1)
encryptedOutputStream1.flush()
encryptedOutputStream1.close()
logger.info("Using record ID ${recordId2} and key ID ${keyId2}")
OutputStream encryptedOutputStream2 = encryptor.encrypt(encryptDestination2, recordId2, keyId2)
encryptedOutputStream2.write(SERIALIZED_BYTES_2)
encryptedOutputStream2.flush()
encryptedOutputStream2.close()
byte[] encryptedBytes1 = encryptDestination1.toByteArray()
logger.info("Encrypted bytes 1: ${Hex.toHexString(encryptedBytes1)}".toString())
byte[] encryptedBytes2 = encryptDestination2.toByteArray()
logger.info("Encrypted bytes 2: ${Hex.toHexString(encryptedBytes2)}".toString())
InputStream encryptedInputStream1 = new ByteArrayInputStream(encryptedBytes1)
InputStream encryptedInputStream2 = new ByteArrayInputStream(encryptedBytes2)
InputStream decryptedInputStream1 = encryptor.decrypt(encryptedInputStream1, recordId1)
byte[] recoveredBytes1 = new byte[SERIALIZED_BYTES_1.length]
decryptedInputStream1.read(recoveredBytes1)
logger.info("Decrypted data 1 to: \n\t${Hex.toHexString(recoveredBytes1)}")
InputStream decryptedInputStream2 = encryptor.decrypt(encryptedInputStream2, recordId2)
byte[] recoveredBytes2 = new byte[SERIALIZED_BYTES_2.length]
decryptedInputStream2.read(recoveredBytes2)
logger.info("Decrypted data 2 to: \n\t${Hex.toHexString(recoveredBytes2)}")
// Assert
assert recoveredBytes1 == SERIALIZED_BYTES_1
logger.info("Decoded 1: ${new String(recoveredBytes1, StandardCharsets.UTF_8)}")
assert recoveredBytes2 == SERIALIZED_BYTES_2
logger.info("Decoded 2: ${new String(recoveredBytes2, StandardCharsets.UTF_8)}")
}
}

BIN
nifi-commons/nifi-security-utils/src/test/resources/nifi.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 5.5 KiB

24
nifi-docs/src/main/asciidoc/administration-guide.adoc
View File

@ -2688,6 +2688,30 @@ this property specifies the maximum amount of time to keep the archived data. Th
|`nifi.content.viewer.url`|The URL for a web-based content viewer if one is available. It is blank by default.
|====
[[encrypted-file-system-content-repository-properties]]
=== Encrypted File System Content Repository Properties
All of the properties defined above (see <<file-system-content-repository-properties,File System Content Repository Properties>>) still apply. Only encryption-specific properties are listed here. See <<user-guide.adoc#encrypted-content,Encrypted Content Repository in the User Guide>> for more information.
|====
|*Property*|*Description*
|`nifi.content.repository.encryption.key.provider.implementation`|This is the fully-qualified class name of the **key provider**. A key provider is the datastore interface for accessing the encryption key to protect the content claims. There are currently two implementations -- `StaticKeyProvider` which reads a key directly from _nifi.properties_, and `FileBasedKeyProvider` which reads *n* many keys from an encrypted file. The interface is extensible, and HSM-backed or other providers are expected in the future.
|`nifi.content.repository.encryption.key.provider.location`|The path to the key definition resource (empty for `StaticKeyProvider`, `./keys.nkp` or similar path for `FileBasedKeyProvider`). For future providers like an HSM, this may be a connection string or URL.
|`nifi.content.repository.encryption.key.id`|The active key ID to use for encryption (e.g. `Key1`).
|`nifi.content.repository.encryption.key`|The key to use for `StaticKeyProvider`. The key format is hex-encoded (`0123456789ABCDEFFEDCBA98765432100123456789ABCDEFFEDCBA9876543210`) but can also be encrypted using the `./encrypt-config.sh` tool in NiFi Toolkit (see the <<toolkit-guide.adoc#encrypt_config_tool,Encrypt-Config Tool>> section in the link:toolkit-guide.html[NiFi Toolkit Guide] for more information).
|`nifi.content.repository.encryption.key.id.`*|Allows for additional keys to be specified for the `StaticKeyProvider`. For example, the line `nifi.content.repository.encryption.key.id.Key2=012...210` would provide an available key `Key2`.
|====
The simplest configuration is below:
....
nifi.content.repository.implementation=org.apache.nifi.controller.repository.crypto.EncryptedFileSystemRepository
nifi.content.repository.encryption.key.provider.implementation=org.apache.nifi.security.kms.StaticKeyProvider
nifi.content.repository.encryption.key.provider.location=
nifi.content.repository.encryption.key.id=K1
nifi.content.repository.encryption.key=0123456789ABCDEFFEDCBA98765432100123456789ABCDEFFEDCBA9876543210
....
=== Volatile Content Repository Properties
|====

BIN
nifi-docs/src/main/asciidoc/images/encrypted-content-hex.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 2.0 MiB

112
nifi-docs/src/main/asciidoc/user-guide.adoc
View File

@ -575,6 +575,12 @@ The first configuration option is the Scheduling Strategy. There are three possi
option can be set to 0. In this case, the number of threads is limited only by the size of the Event-Driven Thread Pool that
the administrator has configured.
[WARNING]
.Experimental
============
This implementation is marked <<experimental_warning, *experimental*>> as of Apache NiFi 1.10.0 (October 2019). The API, configuration, and internal behavior may change without warning, and such changes may occur during a minor release. Use at your own risk.
============
*CRON driven*: When using the CRON driven scheduling mode, the Processor is scheduled to run periodically, similar to the
Timer driven scheduling mode. However, the CRON driven mode provides significantly more flexibility at the expense of
increasing the complexity of the configuration. The CRON driven scheduling value is a string of six required fields and one
@ -2606,6 +2612,12 @@ The next section has more information about implementing an Encrypted Provenance
=== Encrypted Provenance Repository
While OS-level access control can offer some security over the provenance data written to the disk in a repository, there are scenarios where the data may be sensitive, compliance and regulatory requirements exist, or NiFi is running on hardware not under the direct control of the organization (cloud, etc.). In this case, the provenance repository allows for all data to be encrypted before being persisted to the disk.
[WARNING]
.Experimental
============
This implementation is marked <<experimental_warning, *experimental*>> as of Apache NiFi 1.10.0 (October 2019). The API, configuration, and internal behavior may change without warning, and such changes may occur during a minor release. Use at your own risk.
============
[WARNING]
.Performance
============
@ -2674,6 +2686,106 @@ When switching between implementation "families" (i.e. `VolatileProvenanceReposi
* Multiple repositories -- No additional effort or testing has been applied to multiple repositories at this time. It is possible/likely issues will occur with repositories on different physical devices. There is no option to provide a heterogenous environment (i.e. one encrypted, one plaintext repository).
* Corruption -- when a disk is filled or corrupted, there have been reported issues with the repository becoming corrupted and recovery steps are necessary. This is likely to continue to be an issue with the encrypted repository, although still limited in scope to individual records (i.e. an entire repository file won't be irrecoverable due to the encryption).
[[encrypted-content]]
=== Encrypted Content Repository
While OS-level access control can offer some security over the flowfile content data written to the disk in a repository, there are scenarios where the data may be sensitive, compliance and regulatory requirements exist, or NiFi is running on hardware not under the direct control of the organization (cloud, etc.). In this case, the content repository allows for all data to be encrypted before being persisted to the disk.
[WARNING]
.Experimental
============
This implementation is marked <<experimental_warning, *experimental*>> as of Apache NiFi 1.10.0 (October 2019). The API, configuration, and internal behavior may change without warning, and such changes may occur during a minor release. Use at your own risk.
============
[WARNING]
.Performance
============
The current implementation of the encrypted content repository intercepts the serialization of content data via the `EncryptedContentRepositoryOutputStream` and uses the `AES/CTR` algorithm, which is fairly performant on commodity hardware. This use of a stream cipher (because the content is operated on in a streaming manner for performance) differs from the use of an authenticated encryption algorithm (AEAD) like `AES/GCM` in the <<encrypted-provenance,Encrypted Provenance Repository>>. In most scenarios, the added cost will not be significant (unnoticable on a flow with hundreds of content read/write events per second, moderately noticable on a flow with thousands - tens of thousands of events per second). However, administrators should perform their own risk assessment and performance analysis and decide how to move forward. Switching back and forth between encrypted/unencrypted implementations is not recommended at this time.
============
==== What is it?
The `EncryptedFileSystemRepository` is a new implementation of the content repository which encrypts all content data before it is written to the repository. This allows for storage on systems where OS-level access controls are not sufficient to protect the data while still allowing querying and access to the data through the NiFi UI/API.
==== How does it work?
The `FileSystemRepository` was introduced in NiFi 0.2.1 and provided the only persistent content repository implementation. The encrypted version wraps that implementation with functionality to return to the `Session` (usually `StandardProcessSession`) a special `OutputStream`/`InputStream` which encrypt and decrypt the serialized bytes respectively. This allows all components to continue interacting with the content repository interface in the same way as before and continue operating on content data in a streaming manner, without requiring any changes to handle the data protection.
The fully qualified class `org.apache.nifi.content.EncryptedFileSystemRepository` is specified as the content repository implementation in _nifi.properties_ as the value of `nifi.content.repository.implementation`. In addition, <<administration-guide.adoc#encrypted-file-system-content-repository-properties,new properties>> must be populated to allow successful initialization.
===== StaticKeyProvider
The `StaticKeyProvider` implementation defines keys directly in _nifi.properties_. Individual keys are provided in hexadecimal encoding. The keys can also be encrypted like any other sensitive property in _nifi.properties_ using the <<administration-guide.adoc#encrypt-config_tool,`./encrypt-config.sh`>> tool in the NiFi Toolkit.
The following configuration section would result in a key provider with two available keys, "Key1" (active) and "AnotherKey".
....
nifi.content.repository.encryption.key.provider.implementation=org.apache.nifi.security.kms.StaticKeyProvider
nifi.content.repository.encryption.key.id=Key1
nifi.content.repository.encryption.key=0123456789ABCDEFFEDCBA98765432100123456789ABCDEFFEDCBA9876543210
nifi.content.repository.encryption.key.id.AnotherKey=0101010101010101010101010101010101010101010101010101010101010101
....
===== FileBasedKeyProvider
The `FileBasedKeyProvider` implementation reads from an encrypted definition file of the format:
....
key1=NGCpDpxBZNN0DBodz0p1SDbTjC2FG5kp1pCmdUKJlxxtcMSo6GC4fMlTyy1mPeKOxzLut3DRX+51j6PCO5SznA==
key2=GYxPbMMDbnraXs09eGJudAM5jTvVYp05XtImkAg4JY4rIbmHOiVUUI6OeOf7ZW+hH42jtPgNW9pSkkQ9HWY/vQ==
key3=SFe11xuz7J89Y/IQ7YbJPOL0/YKZRFL/VUxJgEHxxlXpd/8ELA7wwN59K1KTr3BURCcFP5YGmwrSKfr4OE4Vlg==
key4=kZprfcTSTH69UuOU3jMkZfrtiVR/eqWmmbdku3bQcUJ/+UToecNB5lzOVEMBChyEXppyXXC35Wa6GEXFK6PMKw==
key5=c6FzfnKm7UR7xqI2NFpZ+fEKBfSU7+1NvRw+XWQ9U39MONWqk5gvoyOCdFR1kUgeg46jrN5dGXk13sRqE0GETQ==
....
Each line defines a key ID and then the Base64-encoded cipher text of a 16 byte IV and wrapped AES-128, AES-192, or AES-256 key depending on the JCE policies available. The individual keys are wrapped by AES/GCM encryption using the **master key** defined by `nifi.bootstrap.sensitive.key` in _conf/bootstrap.conf_.
.Data Protection vs. Key Protection
****
Even though the flowfile content is encrypted using `AES/CTR` to handle streaming data, if using the _Config Encrypt
Tool_ or `FileBasedKeyProvider`, those _keys_ will be protected using `AES/GCM` to provide authenticated encryption
over the key material.
****
===== Key Rotation
Simply update _nifi.properties_ to reference a new key ID in `nifi.content.repository.encryption.key.id`. Previously-encrypted content claims can still be decrypted as long as that key is still available in the key definition file or `nifi.content.repository.encryption.key.id.<OldKeyID>` as the key ID is serialized alongside the encrypted content.
==== Writing and Reading Content Claims
Once the repository is initialized, all content claim write operations are serialized using `RepositoryObjectStreamEncryptor` (the only currently existing implementation is `RepositoryObjectAESCTREncryptor`) to an `OutputStream`. The actual implementation is `EncryptedContentRepositoryOutputStream`, which encrypts the data written by the component via `StandardProcessSession` inline and the encryption metadata (`keyId`, `algorithm`, `version`, `IV`) is serialized and prepended. The complete `OutputStream` is then written to the repository on disk as normal.
image:encrypted-content-hex.png["Encrypted content repository file on disk"]
On content claim read, the process is reversed. The encryption metadata (`RepositoryObjectEncryptionMetadata`) is parsed and used to decrypt the serialized bytes, which are then deserialized into a `CipherInputStream` object. The delegation to the normal repository file system interaction allows for "random-access" (i.e. immediate seek without decryption of unnecessary content claims).
Within the NiFi UI/API, there is no detectable difference between an encrypted and unencrypted content repository. The Provenance Query operations to view content work as expected with no change to the process.
==== Potential Issues
[WARNING]
.Switching Implementations
============
When switching between implementation "families" (i.e. `VolatileContentRepository` or `FileSystemRepository` to `EncryptedFileSystemRepository`), the existing repository must be cleared from the file system before starting NiFi. A terminal command like `localhost:$NIFI_HOME $ rm -rf content_repository/` is sufficient.
============
* Switching between unencrypted and encrypted repositories
** If a user has an existing repository (`FileSystemRepository`) that is not encrypted and switches their configuration to use an encrypted repository, the application writes an error to the log but starts up. However, previous content claims are not accessible through the provenance query interface and new content claims will overwrite the existing claims. The same behavior occurs if a user switches from an encrypted repository to an unencrypted repository. Automatic roll-over is a future effort (link:https://issues.apache.org/jira/browse/NIFI-6783[NIFI-6783^]) but NiFi is not intended for long-term storage of content claims so the impact should be minimal. There are two scenarios for roll-over:
*** Encrypted -> unencrypted -- if the previous repository implementation was encrypted, these claims should be handled seamlessly as long as the key provider available still has the keys used to encrypt the claims (see **Key Rotation**)
*** Unencrypted -> encrypted -- if the previous repository implementation was unencrypted, these claims should be handled seamlessly as the previously written claims simply need to be read with a plaintext `InputStream` and then be written back with the `EncryptedContentRepositoryOutputStream`
** There is also a future effort to provide a standalone tool in NiFi Toolkit to encrypt/decrypt an existing content repository to make the transition easier. The translation process could take a long time depending on the size of the existing repository, and being able to perform this task outside of application startup would be valuable (link:https://issues.apache.org/jira/browse/NIFI-6783[NIFI-6783^]).
* Multiple repositories -- No additional effort or testing has been applied to multiple repositories at this time. It is possible/likely issues will occur with repositories on different physical devices. There is no option to provide a heterogenous environment (i.e. one encrypted, one plaintext repository).
* Corruption -- when a disk is filled or corrupted, there have been reported issues with the repository becoming corrupted and recovery steps are necessary. This is likely to continue to be an issue with the encrypted repository, although still limited in scope to individual claims (i.e. an entire repository file won't be irrecoverable due to the encryption). Some testing has been performed on scenarios where disk space is exhausted. While the flow can no longer write additional content claims to the repository in that case, the NiFi application continues to function properly, and successfully written content claims are still available via the Provenance Query operations. Stopping NiFi and removing the content repository (or moving it to a larger disk) resolves the issue.
[[experimental_warning]]
== Experimental Warning
While all Apache licensed code is provided "on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied" (see https://www.apache.org/licenses/LICENSE-2.0[Apache License, Version 2.0]), some features of Apache NiFi may be marked *experimental*. Experimental features may:
* have undergone less extensive testing than is normal for standard NiFi features
* interact with unstable external dependencies
* be subject to change (any exposed APIs should *not* be considered covered under the minor release backward compatibility guidelines of https://semver.org[Semantic Versioning])
* potentially cause data loss
* not be directly supported by the community in the event issues arise
Every attempt is made to provide more detailed and specific information around the nature of the experimental warning on a per-feature basis. Questions around specific experimental features should be directed to the mailto:dev@nifi.apache.org[Apache NiFi Developer Mailing List].
[[other_management_features]]
== Other Management Features

3
nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-cluster/src/test/java/org/apache/nifi/cluster/firewall/impl/FileBasedClusterNodeFirewallTest.java
View File

@ -24,9 +24,9 @@ import static org.junit.Assume.assumeTrue;
import java.io.File;
import java.net.InetAddress;
import java.net.UnknownHostException;
import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.Ignore;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.TemporaryFolder;
@ -80,6 +80,7 @@ public class FileBasedClusterNodeFirewallTest {
/**
* We have two garbage lines in our test config file, ensure they didn't get turned into hosts.
*/
@Ignore("This does not run consistently on different environments")
@Test
public void ensureBadDataWasIgnored() {
assumeTrue(badHostsDoNotResolve);

339
nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/repository/FileSystemRepository.java
View File

@ -57,7 +57,6 @@ import java.util.concurrent.locks.Condition;
import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReentrantLock;
import java.util.regex.Pattern;
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.nifi.controller.repository.claim.ContentClaim;
@ -910,7 +909,26 @@ public class FileSystemRepository implements ContentRepository {
return write(claim, false);
}
private OutputStream write(final ContentClaim claim, final boolean append) throws IOException {
private OutputStream write(final ContentClaim claim, final boolean append) {
StandardContentClaim scc = validateContentClaimForWriting(claim);
ByteCountingOutputStream claimStream = writableClaimStreams.get(scc.getResourceClaim());
final int initialLength = append ? (int) Math.max(0, scc.getLength()) : 0;
final ByteCountingOutputStream bcos = claimStream;
// TODO: Refactor OS implementation out (deduplicate methods, etc.)
final OutputStream out = new ContentRepositoryOutputStream(scc, bcos, initialLength);
LOG.debug("Writing to {}", out);
if (LOG.isTraceEnabled()) {
LOG.trace("Stack trace: ", new RuntimeException("Stack Trace for writing to " + out));
}
return out;
}
public static StandardContentClaim validateContentClaimForWriting(ContentClaim claim) {
if (claim == null) {
throw new NullPointerException("ContentClaim cannot be null");
}
@ -921,154 +939,11 @@ public class FileSystemRepository implements ContentRepository {
throw new IllegalArgumentException("Cannot write to " + claim + " because that Content Claim does belong to this Content Repository");
}
final StandardContentClaim scc = (StandardContentClaim) claim;
if (claim.getLength() > 0) {
throw new IllegalArgumentException("Cannot write to " + claim + " because it has already been written to.");
}
ByteCountingOutputStream claimStream = writableClaimStreams.get(scc.getResourceClaim());
final int initialLength = append ? (int) Math.max(0, scc.getLength()) : 0;
final ByteCountingOutputStream bcos = claimStream;
final OutputStream out = new OutputStream() {
private long bytesWritten = 0L;
private boolean recycle = true;
private boolean closed = false;
@Override
public String toString() {
return "FileSystemRepository Stream [" + scc + "]";
}
@Override
public synchronized void write(final int b) throws IOException {
if (closed) {
throw new IOException("Stream is closed");
}
try {
bcos.write(b);
} catch (final IOException ioe) {
recycle = false;
throw new IOException("Failed to write to " + this, ioe);
}
bytesWritten++;
scc.setLength(bytesWritten + initialLength);
}
@Override
public synchronized void write(final byte[] b) throws IOException {
if (closed) {
throw new IOException("Stream is closed");
}
try {
bcos.write(b);
} catch (final IOException ioe) {
recycle = false;
throw new IOException("Failed to write to " + this, ioe);
}
bytesWritten += b.length;
scc.setLength(bytesWritten + initialLength);
}
@Override
public synchronized void write(final byte[] b, final int off, final int len) throws IOException {
if (closed) {
throw new IOException("Stream is closed");
}
try {
bcos.write(b, off, len);
} catch (final IOException ioe) {
recycle = false;
throw new IOException("Failed to write to " + this, ioe);
}
bytesWritten += len;
scc.setLength(bytesWritten + initialLength);
}
@Override
public synchronized void flush() throws IOException {
if (closed) {
throw new IOException("Stream is closed");
}
bcos.flush();
}
@Override
public synchronized void close() throws IOException {
closed = true;
if (alwaysSync) {
((FileOutputStream) bcos.getWrappedStream()).getFD().sync();
}
if (scc.getLength() < 0) {
// If claim was not written to, set length to 0
scc.setLength(0L);
}
// if we've not yet hit the threshold for appending to a resource claim, add the claim
// to the writableClaimQueue so that the Resource Claim can be used again when create()
// is called. In this case, we don't have to actually close the file stream. Instead, we
// can just add it onto the queue and continue to use it for the next content claim.
final long resourceClaimLength = scc.getOffset() + scc.getLength();
if (recycle && resourceClaimLength < maxAppendableClaimLength) {
final ClaimLengthPair pair = new ClaimLengthPair(scc.getResourceClaim(), resourceClaimLength);
// We are checking that writableClaimStreams contains the resource claim as a key, as a sanity check.
// It should always be there. However, we have encountered a bug before where we archived content before
// we should have. As a result, the Resource Claim and the associated OutputStream were removed from the
// writableClaimStreams map, and this caused a NullPointerException. Worse, the call here to
// writableClaimQueue.offer() means that the ResourceClaim was then reused, which resulted in an endless
// loop of NullPointerException's being thrown. As a result, we simply ensure that the Resource Claim does
// in fact have an OutputStream associated with it before adding it back to the writableClaimQueue.
final boolean enqueued = writableClaimStreams.get(scc.getResourceClaim()) != null && writableClaimQueue.offer(pair);
if (enqueued) {
LOG.debug("Claim length less than max; Adding {} back to Writable Claim Queue", this);
} else {
writableClaimStreams.remove(scc.getResourceClaim());
resourceClaimManager.freeze(scc.getResourceClaim());
bcos.close();
LOG.debug("Claim length less than max; Closing {} because could not add back to queue", this);
if (LOG.isTraceEnabled()) {
LOG.trace("Stack trace: ", new RuntimeException("Stack Trace for closing " + this));
}
}
} else {
// we've reached the limit for this claim. Don't add it back to our queue.
// Instead, just remove it and move on.
// Mark the claim as no longer being able to be written to
resourceClaimManager.freeze(scc.getResourceClaim());
// ensure that the claim is no longer on the queue
writableClaimQueue.remove(new ClaimLengthPair(scc.getResourceClaim(), resourceClaimLength));
bcos.close();
LOG.debug("Claim lenth >= max; Closing {}", this);
if (LOG.isTraceEnabled()) {
LOG.trace("Stack trace: ", new RuntimeException("Stack Trace for closing " + this));
}
}
}
};
LOG.debug("Writing to {}", out);
if (LOG.isTraceEnabled()) {
LOG.trace("Stack trace: ", new RuntimeException("Stack Trace for writing to " + out));
}
return out;
return (StandardContentClaim) claim;
}
@Override
@ -1214,6 +1089,30 @@ public class FileSystemRepository implements ContentRepository {
return writableClaimStreams.size();
}
protected ConcurrentMap<ResourceClaim, ByteCountingOutputStream> getWritableClaimStreams() {
return writableClaimStreams;
}
protected ByteCountingOutputStream getWritableClaimStreamByResourceClaim(ResourceClaim rc) {
return writableClaimStreams.get(rc);
}
protected ResourceClaimManager getResourceClaimManager() {
return resourceClaimManager;
}
protected BlockingQueue<ClaimLengthPair> getWritableClaimQueue() {
return writableClaimQueue;
}
protected long getMaxAppendableClaimLength() {
return maxAppendableClaimLength;
}
protected boolean isAlwaysSync() {
return alwaysSync;
}
// marked protected for visibility and ability to override for unit tests.
protected boolean archive(final Path curPath) throws IOException {
// check if already archived
@ -1739,7 +1638,7 @@ public class FileSystemRepository implements ContentRepository {
}
}
private static class ClaimLengthPair {
protected static class ClaimLengthPair {
private final ResourceClaim claim;
private final Long length;
@ -1892,4 +1791,150 @@ public class FileSystemRepository implements ContentRepository {
}
}
protected class ContentRepositoryOutputStream extends OutputStream {
protected final StandardContentClaim scc;
protected final ByteCountingOutputStream bcos;
protected final int initialLength;
protected long bytesWritten;
protected boolean recycle;
protected boolean closed;
public ContentRepositoryOutputStream(StandardContentClaim scc, ByteCountingOutputStream bcos, int initialLength) {
this.scc = scc;
this.bcos = bcos;
this.initialLength = initialLength;
bytesWritten = 0L;
recycle = true;
closed = false;
}
@Override
public String toString() {
return "FileSystemRepository Stream [" + scc + "]";
}
@Override
public synchronized void write(final int b) throws IOException {
if (closed) {
throw new IOException("Stream is closed");
}
try {
bcos.write(b);
} catch (final IOException ioe) {
recycle = false;
throw new IOException("Failed to write to " + this, ioe);
}
bytesWritten++;
scc.setLength(bytesWritten + initialLength);
}
@Override
public synchronized void write(final byte[] b) throws IOException {
if (closed) {
throw new IOException("Stream is closed");
}
try {
bcos.write(b);
} catch (final IOException ioe) {
recycle = false;
throw new IOException("Failed to write to " + this, ioe);
}
bytesWritten += b.length;
scc.setLength(bytesWritten + initialLength);
}
@Override
public synchronized void write(final byte[] b, final int off, final int len) throws IOException {
if (closed) {
throw new IOException("Stream is closed");
}
try {
bcos.write(b, off, len);
} catch (final IOException ioe) {
recycle = false;
throw new IOException("Failed to write to " + this, ioe);
}
bytesWritten += len;
scc.setLength(bytesWritten + initialLength);
}
@Override
public synchronized void flush() throws IOException {
if (closed) {
throw new IOException("Stream is closed");
}
bcos.flush();
}
@Override
public synchronized void close() throws IOException {
closed = true;
if (alwaysSync) {
((FileOutputStream) bcos.getWrappedStream()).getFD().sync();
}
if (scc.getLength() < 0) {
// If claim was not written to, set length to 0
scc.setLength(0L);
}
// if we've not yet hit the threshold for appending to a resource claim, add the claim
// to the writableClaimQueue so that the Resource Claim can be used again when create()
// is called. In this case, we don't have to actually close the file stream. Instead, we
// can just add it onto the queue and continue to use it for the next content claim.
final long resourceClaimLength = scc.getOffset() + scc.getLength();
if (recycle && resourceClaimLength < maxAppendableClaimLength) {
final ClaimLengthPair pair = new ClaimLengthPair(scc.getResourceClaim(), resourceClaimLength);
// We are checking that writableClaimStreams contains the resource claim as a key, as a sanity check.
// It should always be there. However, we have encountered a bug before where we archived content before
// we should have. As a result, the Resource Claim and the associated OutputStream were removed from the
// writableClaimStreams map, and this caused a NullPointerException. Worse, the call here to
// writableClaimQueue.offer() means that the ResourceClaim was then reused, which resulted in an endless
// loop of NullPointerException's being thrown. As a result, we simply ensure that the Resource Claim does
// in fact have an OutputStream associated with it before adding it back to the writableClaimQueue.
final boolean enqueued = writableClaimStreams.get(scc.getResourceClaim()) != null && writableClaimQueue.offer(pair);
if (enqueued) {
LOG.debug("Claim length less than max; Adding {} back to Writable Claim Queue", this);
} else {
writableClaimStreams.remove(scc.getResourceClaim());
resourceClaimManager.freeze(scc.getResourceClaim());
bcos.close();
LOG.debug("Claim length less than max; Closing {} because could not add back to queue", this);
if (LOG.isTraceEnabled()) {
LOG.trace("Stack trace: ", new RuntimeException("Stack Trace for closing " + this));
}
}
} else {
// we've reached the limit for this claim. Don't add it back to our queue.
// Instead, just remove it and move on.
// Mark the claim as no longer being able to be written to
resourceClaimManager.freeze(scc.getResourceClaim());
// ensure that the claim is no longer on the queue
writableClaimQueue.remove(new ClaimLengthPair(scc.getResourceClaim(), resourceClaimLength));
bcos.close();
LOG.debug("Claim lenth >= max; Closing {}", this);
if (LOG.isTraceEnabled()) {
LOG.trace("Stack trace: ", new RuntimeException("Stack Trace for closing " + this));
}
}
}
}
}

420
nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/repository/crypto/EncryptedFileSystemRepository.java Normal file
View File

@ -0,0 +1,420 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.nifi.controller.repository.crypto;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.nio.ByteBuffer;
import java.nio.file.Path;
import java.security.KeyManagementException;
import javax.crypto.CipherOutputStream;
import javax.crypto.SecretKey;
import org.apache.commons.lang3.StringUtils;
import org.apache.nifi.controller.repository.FileSystemRepository;
import org.apache.nifi.controller.repository.claim.ContentClaim;
import org.apache.nifi.controller.repository.claim.StandardContentClaim;
import org.apache.nifi.security.kms.CryptoUtils;
import org.apache.nifi.security.kms.EncryptionException;
import org.apache.nifi.security.kms.KeyProvider;
import org.apache.nifi.security.kms.KeyProviderFactory;
import org.apache.nifi.security.repository.RepositoryType;
import org.apache.nifi.security.repository.config.RepositoryEncryptionConfiguration;
import org.apache.nifi.security.repository.stream.RepositoryObjectStreamEncryptor;
import org.apache.nifi.security.repository.stream.aes.RepositoryObjectAESCTREncryptor;
import org.apache.nifi.stream.io.ByteCountingOutputStream;
import org.apache.nifi.stream.io.NonCloseableOutputStream;
import org.apache.nifi.stream.io.StreamUtils;
import org.apache.nifi.util.NiFiProperties;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* This class is an implementation of the {@link FileSystemRepository} content repository which provides transparent
* streaming encryption/decryption of content claim data during file system interaction. As of Apache NiFi 1.10.0
* (October 2019), this implementation is considered <a href="https://nifi.apache.org/docs/nifi-docs/html/user-guide.html#experimental-warning">*experimental*</a>. For further details, review the
* <a href="https://nifi.apache.org/docs/nifi-docs/html/user-guide.html#encrypted-content">Apache NiFi User Guide -
* Encrypted Content Repository</a> and
* <a href="https://nifi.apache.org/docs/nifi-docs/html/administration-guide.html#encrypted-file-system-content-repository-properties">Apache NiFi Admin Guide - Encrypted File System Content
* Repository Properties</a>.
*/
public class EncryptedFileSystemRepository extends FileSystemRepository {
private static final Logger logger = LoggerFactory.getLogger(EncryptedFileSystemRepository.class);
private String activeKeyId;
private KeyProvider keyProvider;
/**
* Default no args constructor for service loading only
*/
public EncryptedFileSystemRepository() {
super();
keyProvider = null;
}
public EncryptedFileSystemRepository(final NiFiProperties niFiProperties) throws IOException {
super(niFiProperties);
// Initialize key provider
initializeEncryptionServices(niFiProperties);
}
private void initializeEncryptionServices(NiFiProperties niFiProperties) throws IOException {
// Initialize the encryption-specific fields
if (CryptoUtils.isContentRepositoryEncryptionConfigured(niFiProperties)) {
try {
KeyProvider keyProvider;
final String keyProviderImplementation = niFiProperties.getProperty(NiFiProperties.CONTENT_REPOSITORY_ENCRYPTION_KEY_PROVIDER_IMPLEMENTATION_CLASS);
if (KeyProviderFactory.requiresMasterKey(keyProviderImplementation)) {
SecretKey masterKey = CryptoUtils.getMasterKey();
keyProvider = buildKeyProvider(niFiProperties, masterKey);
} else {
keyProvider = buildKeyProvider(niFiProperties);
}
this.keyProvider = keyProvider;
} catch (KeyManagementException e) {
String msg = "Encountered an error building the key provider";
logger.error(msg, e);
throw new IOException(msg, e);
}
} else {
throw new IOException("The provided configuration does not support a encrypted repository");
}
// Set active key ID
setActiveKeyId(niFiProperties.getContentRepositoryEncryptionKeyId());
}
/**
* Returns a configured {@link KeyProvider} instance that does not require a {@code master key} to use (usually a {@link org.apache.nifi.security.kms.StaticKeyProvider}).
*
* @param niFiProperties the {@link NiFiProperties} object
* @return the configured KeyProvider
* @throws KeyManagementException if there is a problem with the configuration
*/
private static KeyProvider buildKeyProvider(NiFiProperties niFiProperties) throws KeyManagementException {
return buildKeyProvider(niFiProperties, null);
}
/**
* Returns a configured {@link KeyProvider} instance that requires a {@code master key} to use
* (usually a {@link org.apache.nifi.security.kms.FileBasedKeyProvider} or an encrypted
* {@link org.apache.nifi.security.kms.StaticKeyProvider}).
*
* @param niFiProperties the {@link NiFiProperties} object
* @param masterKey the master encryption key used to encrypt the data encryption keys in the key provider configuration
* @return the configured KeyProvider
* @throws KeyManagementException if there is a problem with the configuration
*/
private static KeyProvider buildKeyProvider(NiFiProperties niFiProperties, SecretKey masterKey) throws KeyManagementException {
return KeyProviderFactory.buildKeyProvider(RepositoryEncryptionConfiguration.fromNiFiProperties(niFiProperties, RepositoryType.CONTENT), masterKey);
}
/**
* Returns the number of bytes read after importing content from the provided
* {@link InputStream} into the {@link ContentClaim}. This method has the same logic as
* the parent method, but must be overridden to use the subclass's
* {@link #write(ContentClaim)} method which performs the encryption. The
* overloaded method {@link super#importFrom(Path, ContentClaim)} does not need to be
* overridden because it delegates to this one.
*
* @param content the InputStream containing the desired content
* @param claim the ContentClaim to put the content into
* @return the number of bytes read
* @throws IOException if there is a problem reading from the stream
*/
@Override
public long importFrom(final InputStream content, final ContentClaim claim) throws IOException {
try (final OutputStream out = write(claim)) {
return StreamUtils.copy(content, out);
}
}
/**
* Exports the content of the given claim to the given destination. Returns the number of bytes written. <strong>This method decrypts the encrypted content and writes it in plaintext.</strong>
*
* @param claim to export from
* @param destination where to export data
* @return the size of the claim in bytes
* @throws IOException if an IO error occurs
*/
@Override
public long exportTo(final ContentClaim claim, final OutputStream destination) throws IOException {
logger.warn("Exporting content from {} to output stream {}. This content will be decrypted", claim.getResourceClaim().getId(), destination);
return super.exportTo(claim, destination);
}
/**
* Exports a subset of the content of the given claim, starting at offset
* and copying length bytes, to the given destination. Returns the number of bytes written. <strong>This method decrypts the encrypted content and writes it in plaintext.</strong>
*
* @param claim to export from
* @param destination where to export data
* @param offset the offset into the claim at which the copy should begin
* @param length the number of bytes to copy
* @return the size of the claim in bytes
* @throws IOException if an IO error occurs
*/
@Override
public long exportTo(final ContentClaim claim, final OutputStream destination, final long offset, final long length) throws IOException {
logger.warn("Exporting content from {} (offset: {}, length: {}) to output stream {}. This content will be decrypted", claim.getResourceClaim().getId(), offset, length, destination);
return super.exportTo(claim, destination, offset, length);
}
/**
* Exports the content of the given claim to the given destination. Returns the number of bytes written. <strong>This method decrypts the encrypted content and writes it in plaintext.</strong>
*
* @param claim to export from
* @param destination where to export data
* @return the size of the claim in bytes
* @throws IOException if an IO error occurs
*/
@Override
public long exportTo(final ContentClaim claim, final Path destination, final boolean append) throws IOException {
logger.warn("Exporting content from {} to path {}. This content will be decrypted", claim.getResourceClaim().getId(), destination);
return super.exportTo(claim, destination, append);
}
/**
* Exports a subset of the content of the given claim, starting at offset
* and copying length bytes, to the given destination. <strong>This method decrypts the encrypted content and writes it in plaintext.</strong>
*
* @param claim to export from
* @param destination where to export data
* @param offset the offset into the claim at which the copy should begin
* @param length the number of bytes to copy
* @return the number of bytes copied
* @throws IOException if an IO error occurs
*/
@Override
public long exportTo(final ContentClaim claim, final Path destination, final boolean append, final long offset, final long length) throws IOException {
logger.warn("Exporting content from {} (offset: {}, length: {}) to path {}. This content will be decrypted", claim.getResourceClaim().getId(), offset, length, destination);
return super.exportTo(claim, destination, append, offset, length);
}
/**
* Returns an InputStream (actually a {@link javax.crypto.CipherInputStream}) which wraps
* the {@link java.io.FileInputStream} from the content repository claim on disk. This
* allows a consuming caller to automatically decrypt the content as it is read.
*
* @param claim the content claim to read
* @return the decrypting input stream
* @throws IOException if there is a problem reading from disk or configuring the cipher
*/
@Override
public InputStream read(final ContentClaim claim) throws IOException {
InputStream inputStream = super.read(claim);
try {
String recordId = getRecordId(claim);
logger.debug("Creating decrypted input stream to read flowfile content with record ID: " + recordId);
final InputStream decryptingInputStream = getDecryptingInputStream(inputStream, recordId);
logger.debug("Reading from record ID {}", recordId);
if (logger.isTraceEnabled()) {
logger.trace("Stack trace: ", new RuntimeException("Stack Trace for reading from record ID " + recordId));
}
return decryptingInputStream;
} catch (EncryptionException | KeyManagementException e) {
logger.error("Encountered an error instantiating the encrypted content repository input stream: " + e.getMessage());
throw new IOException("Error creating encrypted content repository input stream", e);
}
}
private InputStream getDecryptingInputStream(InputStream inputStream, String recordId) throws KeyManagementException, EncryptionException {
RepositoryObjectStreamEncryptor encryptor = new RepositoryObjectAESCTREncryptor();
encryptor.initialize(keyProvider);
// ECROS wrapping COS wrapping BCOS wrapping FOS
return encryptor.decrypt(inputStream, recordId);
}
/**
* Returns an OutputStream (actually a {@link javax.crypto.CipherOutputStream}) which wraps
* the {@link ByteCountingOutputStream} to the content repository claim on disk. This
* allows a consuming caller to automatically encrypt the content as it is written.
*
* @param claim the content claim to write to
* @return the encrypting output stream
* @throws IOException if there is a problem writing to disk or configuring the cipher
*/
@Override
public OutputStream write(final ContentClaim claim) throws IOException {
StandardContentClaim scc = validateContentClaimForWriting(claim);
// BCOS wrapping FOS
ByteCountingOutputStream claimStream = getWritableClaimStreamByResourceClaim(scc.getResourceClaim());
final long startingOffset = claimStream.getBytesWritten();
try {
String keyId = getActiveKeyId();
String recordId = getRecordId(claim);
logger.debug("Creating encrypted output stream (keyId: " + keyId + ") to write flowfile content with record ID: " + recordId);
final OutputStream out = getEncryptedOutputStream(scc, claimStream, startingOffset, keyId, recordId);
logger.debug("Writing to {}", out);
if (logger.isTraceEnabled()) {
logger.trace("Stack trace: ", new RuntimeException("Stack Trace for writing to " + out));
}
return out;
} catch (EncryptionException | KeyManagementException e) {
logger.error("Encountered an error instantiating the encrypted content repository output stream: " + e.getMessage());
throw new IOException("Error creating encrypted content repository output stream", e);
}
}
String getActiveKeyId() {
return activeKeyId;
}
public void setActiveKeyId(String activeKeyId) {
// Key must not be blank and key provider must make key available
if (StringUtils.isNotBlank(activeKeyId) && keyProvider.keyExists(activeKeyId)) {
this.activeKeyId = activeKeyId;
logger.debug("Set active key ID to '" + activeKeyId + "'");
} else {
logger.warn("Attempted to set active key ID to '" + activeKeyId + "' but that is not a valid or available key ID. Keeping active key ID as '" + this.activeKeyId + "'");
}
}
/**
* Returns an identifier for this {@link ContentClaim} to be used when serializing/retrieving the encrypted content.
* For version 1, the identifier is {@code "nifi-ecr-rc-" + the resource claim ID + offset}. If any piece of the
* CC -> RC -> ID chain is null or empty, the current system time in nanoseconds is used with a different
* prefix ({@code "nifi-ecr-ts-"}).
*
* @param claim the content claim
* @return the string identifier
*/
public static String getRecordId(ContentClaim claim) {
// For version 1, use the content claim's resource claim ID as the record ID rather than introducing a new field in the metadata
if (claim != null && claim.getResourceClaim() != null
&& !StringUtils.isBlank(claim.getResourceClaim().getId())) {
return "nifi-ecr-rc-" + claim.getResourceClaim().getId() + "+" + claim.getOffset();
} else {
String tempId = "nifi-ecr-ts-" + System.nanoTime();
logger.error("Cannot determine record ID from null content claim or claim with missing/empty resource claim ID; using timestamp-generated ID: " + tempId + "+0");
return tempId;
}
}
private OutputStream getEncryptedOutputStream(StandardContentClaim scc,
ByteCountingOutputStream claimStream,
long startingOffset,
String keyId,
String recordId) throws KeyManagementException,
EncryptionException {
RepositoryObjectStreamEncryptor encryptor = new RepositoryObjectAESCTREncryptor();
encryptor.initialize(keyProvider);
// ECROS wrapping COS wrapping BCOS wrapping FOS
return new EncryptedContentRepositoryOutputStream(scc, claimStream, encryptor, recordId, keyId, startingOffset);
}
/**
* Private class which wraps the {@link org.apache.nifi.controller.repository.FileSystemRepository.ContentRepositoryOutputStream}'s
* internal {@link ByteCountingOutputStream} with a {@link CipherOutputStream}
* to handle streaming encryption operations.
*/
private class EncryptedContentRepositoryOutputStream extends ContentRepositoryOutputStream {
private final CipherOutputStream cipherOutputStream;
private final long startingOffset;
EncryptedContentRepositoryOutputStream(StandardContentClaim scc,
ByteCountingOutputStream byteCountingOutputStream,
RepositoryObjectStreamEncryptor encryptor, String recordId, String keyId, long startingOffset) throws EncryptionException {
super(scc, byteCountingOutputStream, 0);
this.startingOffset = startingOffset;
// Set up cipher stream
this.cipherOutputStream = (CipherOutputStream) encryptor.encrypt(new NonCloseableOutputStream(byteCountingOutputStream), recordId, keyId);
}
@Override
public String toString() {
return "EncryptedFileSystemRepository Stream [" + scc + "]";
}
@Override
public synchronized void write(final int b) throws IOException {
ByteBuffer bb = ByteBuffer.allocate(4);
bb.putInt(b);
writeBytes(bb.array(), 0, 4);
scc.setLength(bcos.getBytesWritten() - startingOffset);
}
@Override
public synchronized void write(final byte[] b) throws IOException {
writeBytes(b, 0, b.length);
scc.setLength(bcos.getBytesWritten() - startingOffset);
}
@Override
public synchronized void write(final byte[] b, final int off, final int len) throws IOException {
writeBytes(b, off, len);
scc.setLength(bcos.getBytesWritten() - startingOffset);
}
/**
* Internal method used to reduce duplication throughout code.
*
* @param b the byte array to write
* @param off the offset in bytes
* @param len the length in bytes to write
* @throws IOException if there is a problem writing the output
*/
private void writeBytes(byte[] b, int off, int len) throws IOException {
if (closed) {
throw new IOException("Stream is closed");
}
try {
cipherOutputStream.write(b, off, len);
} catch (final IOException ioe) {
recycle = false;
throw new IOException("Failed to write to " + this, ioe);
}
}
@Override
public synchronized void flush() throws IOException {
if (closed) {
throw new IOException("Stream is closed");
}
cipherOutputStream.flush();
}
@Override
public synchronized void close() throws IOException {
closed = true;
// Always flush and close (close triggers cipher.doFinal())
cipherOutputStream.flush();
cipherOutputStream.close();
// Add the additional bytes written to the scc.length
scc.setLength(bcos.getBytesWritten() - startingOffset);
super.close();
}
}
}

33
nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/resources/META-INF/services/org.apache.nifi.controller.repository.ContentRepository
View File

@ -1,16 +1,17 @@
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
org.apache.nifi.controller.repository.FileSystemRepository
org.apache.nifi.controller.repository.VolatileContentRepository
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
org.apache.nifi.controller.repository.FileSystemRepository
org.apache.nifi.controller.repository.VolatileContentRepository
org.apache.nifi.controller.repository.crypto.EncryptedFileSystemRepository

1040
nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/groovy/org/apache/nifi/controller/repository/crypto/EncryptedFileSystemRepositoryTest.groovy Normal file
View File

File diff suppressed because it is too large Load Diff

BIN
nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/resources/bgBannerFoot.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 189 B

BIN
nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/resources/encrypted_content_repo.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 810 KiB

4
nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/resources/logback-test.xml
View File

@ -17,7 +17,7 @@
<configuration>
<appender name="CONSOLE" class="ch.qos.logback.core.ConsoleAppender">
<encoder class="ch.qos.logback.classic.encoder.PatternLayoutEncoder">
<pattern>%-4r [%t] %-5p %c - %m%n</pattern>
<pattern>%-4r [%t] %-5p %c{3} - %m%n</pattern>
</encoder>
</appender>
@ -33,6 +33,8 @@
<logger name="org.apache.nifi.controller.tasks" level="DEBUG" />"
<logger name="org.apache.nifi.controller.service" level="DEBUG"/>
<logger name="org.apache.nifi.encrypt" level="DEBUG"/>
<logger name="org.apache.nifi.controller.repository.crypto" level="DEBUG"/>
<logger name="org.apache.nifi.security.repository" level="DEBUG"/>
<logger name="org.apache.nifi.controller.service.mock" level="ERROR"/>
<logger name="StandardProcessSession.claims" level="INFO" />

16
nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/resources/longtext.txt Normal file
View File

@ -0,0 +1,16 @@
This file is a plain text file used as a test resource for various content repository unit tests. It contains a large amount of plain (ASCII) text, with various whitespace and on multiple lines.
The content includes the Apache Software License copy.
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with
this work for additional information regarding copyright ownership.
The ASF licenses this file to You under the Apache License, Version 2.0
(the "License"); you may not use this file except in compliance with
the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.

8
nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/pom.xml
View File

@ -43,6 +43,14 @@
<groupId>org.apache.commons</groupId>
<artifactId>commons-lang3</artifactId>
</dependency>
<dependency>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-security-utils</artifactId>
</dependency>
<dependency>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-security-utils</artifactId>
</dependency>
</dependencies>
<build>
<!-- Required to run Groovy tests without any Java tests -->

73
nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/main/java/org/apache/nifi/properties/NiFiPropertiesLoader.java
View File

@ -21,15 +21,11 @@ import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.nio.file.Files;
import java.nio.file.Paths;
import java.security.NoSuchAlgorithmException;
import java.security.Security;
import java.util.Optional;
import java.util.Properties;
import java.util.stream.Stream;
import javax.crypto.Cipher;
import org.apache.commons.lang3.StringUtils;
import org.apache.nifi.security.kms.CryptoUtils;
import org.apache.nifi.util.NiFiProperties;
import org.bouncycastle.jce.provider.BouncyCastleProvider;
import org.slf4j.Logger;
@ -39,10 +35,6 @@ public class NiFiPropertiesLoader {
private static final Logger logger = LoggerFactory.getLogger(NiFiPropertiesLoader.class);
private static final String RELATIVE_PATH = "conf/nifi.properties";
private static final String BOOTSTRAP_KEY_PREFIX = "nifi.bootstrap.sensitive.key=";
private NiFiProperties instance;
private String keyHex;
@ -98,7 +90,7 @@ public class NiFiPropertiesLoader {
String keyHex = extractKeyFromBootstrapFile();
return NiFiPropertiesLoader.withKey(keyHex).loadDefault();
} catch (IOException e) {
logger.error("Encountered an exception loading the default nifi.properties file {} with the key provided in bootstrap.conf", getDefaultFilePath(), e);
logger.error("Encountered an exception loading the default nifi.properties file {} with the key provided in bootstrap.conf", CryptoUtils.getDefaultFilePath(), e);
throw e;
}
}
@ -106,74 +98,39 @@ public class NiFiPropertiesLoader {
/**
* Returns the key (if any) used to encrypt sensitive properties, extracted from {@code $NIFI_HOME/conf/bootstrap.conf}.
*
* @deprecated
* Use {@link CryptoUtils#extractKeyFromBootstrapFile()} instead.
*
* @return the key in hexadecimal format
* @throws IOException if the file is not readable
*/
@Deprecated
public static String extractKeyFromBootstrapFile() throws IOException {
// TODO: Replace all existing uses with direct reference to CryptoUtils
return extractKeyFromBootstrapFile("");
}
/**
* Returns the key (if any) used to encrypt sensitive properties, extracted from {@code $NIFI_HOME/conf/bootstrap.conf}.
*
* @deprecated
* Use {@link CryptoUtils#extractKeyFromBootstrapFile(String)} instead.
*
* @param bootstrapPath the path to the bootstrap file
* @return the key in hexadecimal format
* @throws IOException if the file is not readable
*/
@Deprecated
public static String extractKeyFromBootstrapFile(String bootstrapPath) throws IOException {
File expectedBootstrapFile;
if (StringUtils.isBlank(bootstrapPath)) {
// Guess at location of bootstrap.conf file from nifi.properties file
String defaultNiFiPropertiesPath = getDefaultFilePath();
File propertiesFile = new File(defaultNiFiPropertiesPath);
File confDir = new File(propertiesFile.getParent());
if (confDir.exists() && confDir.canRead()) {
expectedBootstrapFile = new File(confDir, "bootstrap.conf");
} else {
logger.error("Cannot read from bootstrap.conf file at {} to extract encryption key -- conf/ directory is missing or permissions are incorrect", confDir.getAbsolutePath());
throw new IOException("Cannot read from bootstrap.conf");
}
} else {
expectedBootstrapFile = new File(bootstrapPath);
}
if (expectedBootstrapFile.exists() && expectedBootstrapFile.canRead()) {
try (Stream<String> stream = Files.lines(Paths.get(expectedBootstrapFile.getAbsolutePath()))) {
Optional<String> keyLine = stream.filter(l -> l.startsWith(BOOTSTRAP_KEY_PREFIX)).findFirst();
if (keyLine.isPresent()) {
return keyLine.get().split("=", 2)[1];
} else {
logger.warn("No encryption key present in the bootstrap.conf file at {}", expectedBootstrapFile.getAbsolutePath());
return "";
}
} catch (IOException e) {
logger.error("Cannot read from bootstrap.conf file at {} to extract encryption key", expectedBootstrapFile.getAbsolutePath());
throw new IOException("Cannot read from bootstrap.conf", e);
}
} else {
logger.error("Cannot read from bootstrap.conf file at {} to extract encryption key -- file is missing or permissions are incorrect", expectedBootstrapFile.getAbsolutePath());
throw new IOException("Cannot read from bootstrap.conf");
}
}
private static String getDefaultFilePath() {
String systemPath = System.getProperty(NiFiProperties.PROPERTIES_FILE_PATH);
if (systemPath == null || systemPath.trim().isEmpty()) {
logger.warn("The system variable {} is not set, so it is being set to '{}'", NiFiProperties.PROPERTIES_FILE_PATH, RELATIVE_PATH);
System.setProperty(NiFiProperties.PROPERTIES_FILE_PATH, RELATIVE_PATH);
systemPath = RELATIVE_PATH;
}
logger.info("Determined default nifi.properties path to be '{}'", systemPath);
return systemPath;
// TODO: Replace all existing uses with direct reference to CryptoUtils
return CryptoUtils.extractKeyFromBootstrapFile(bootstrapPath);
}
private NiFiProperties loadDefault() {
return load(getDefaultFilePath());
return load(CryptoUtils.getDefaultFilePath());
}
private static String getDefaultProviderKey() {
static String getDefaultProviderKey() {
try {
return "aes/gcm/" + (Cipher.getMaxAllowedKeyLength("AES") > 128 ? "256" : "128");
} catch (NoSuchAlgorithmException e) {

176
nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/groovy/org/apache/nifi/properties/StandardNiFiPropertiesGroovyTest.groovy
View File

@ -35,6 +35,9 @@ class StandardNiFiPropertiesGroovyTest extends GroovyTestCase {
private static final String PREK = NiFiProperties.PROVENANCE_REPO_ENCRYPTION_KEY
private static final String PREKID = NiFiProperties.PROVENANCE_REPO_ENCRYPTION_KEY_ID
private static final String CREK = NiFiProperties.CONTENT_REPOSITORY_ENCRYPTION_KEY
private static final String CREKID = NiFiProperties.CONTENT_REPOSITORY_ENCRYPTION_KEY_ID
@BeforeClass
static void setUpOnce() throws Exception {
logger.metaClass.methodMissing = { String name, args ->
@ -323,6 +326,179 @@ class StandardNiFiPropertiesGroovyTest extends GroovyTestCase {
assert keys == [(KEY_ID): KEY_HEX, (KEY_ID_2): KEY_HEX_2, (KEY_ID_3): KEY_HEX_3]
}
@Test
void testShouldGetContentRepositoryEncryptionKeyFromDefaultProperty() throws Exception {
// Arrange
Properties rawProperties = new Properties()
final String KEY_ID = "arbitraryKeyId"
final String KEY_HEX = "0123456789ABCDEFFEDCBA9876543210"
rawProperties.setProperty(CREKID, KEY_ID)
rawProperties.setProperty(CREK, KEY_HEX)
NiFiProperties niFiProperties = new StandardNiFiProperties(rawProperties)
logger.info("niFiProperties has ${niFiProperties.size()} properties: ${niFiProperties.getPropertyKeys()}")
// Act
def keyId = niFiProperties.getContentRepositoryEncryptionKeyId()
def key = niFiProperties.getContentRepositoryEncryptionKey()
def keys = niFiProperties.getContentRepositoryEncryptionKeys()
logger.info("Retrieved key ID: ${keyId}")
logger.info("Retrieved key: ${key}")
logger.info("Retrieved keys: ${keys}")
// Assert
assert keyId == KEY_ID
assert key == KEY_HEX
assert keys == [(KEY_ID): KEY_HEX]
}
@Test
void testShouldGetContentRepositoryEncryptionKeysFromMultipleProperties() throws Exception {
// Arrange
Properties rawProperties = new Properties()
final String KEY_ID = "arbitraryKeyId"
final String KEY_HEX = "0123456789ABCDEFFEDCBA9876543210"
final String KEY_ID_2 = "arbitraryKeyId2"
final String KEY_HEX_2 = "AAAABBBBCCCCDDDDEEEEFFFF00001111"
final String KEY_ID_3 = "arbitraryKeyId3"
final String KEY_HEX_3 = "01010101010101010101010101010101"
rawProperties.setProperty(CREKID, KEY_ID)
rawProperties.setProperty(CREK, KEY_HEX)
rawProperties.setProperty("${CREK}.id.${KEY_ID_2}", KEY_HEX_2)
rawProperties.setProperty("${CREK}.id.${KEY_ID_3}", KEY_HEX_3)
NiFiProperties niFiProperties = new StandardNiFiProperties(rawProperties)
logger.info("niFiProperties has ${niFiProperties.size()} properties: ${niFiProperties.getPropertyKeys()}")
// Act
def keyId = niFiProperties.getContentRepositoryEncryptionKeyId()
def key = niFiProperties.getContentRepositoryEncryptionKey()
def keys = niFiProperties.getContentRepositoryEncryptionKeys()
logger.info("Retrieved key ID: ${keyId}")
logger.info("Retrieved key: ${key}")
logger.info("Retrieved keys: ${keys}")
// Assert
assert keyId == KEY_ID
assert key == KEY_HEX
assert keys == [(KEY_ID): KEY_HEX, (KEY_ID_2): KEY_HEX_2, (KEY_ID_3): KEY_HEX_3]
}
@Test
void testShouldGetContentRepositoryEncryptionKeysWithNoDefaultDefined() throws Exception {
// Arrange
Properties rawProperties = new Properties()
final String KEY_ID = "arbitraryKeyId"
final String KEY_HEX = "0123456789ABCDEFFEDCBA9876543210"
final String KEY_ID_2 = "arbitraryKeyId2"
final String KEY_HEX_2 = "AAAABBBBCCCCDDDDEEEEFFFF00001111"
final String KEY_ID_3 = "arbitraryKeyId3"
final String KEY_HEX_3 = "01010101010101010101010101010101"
rawProperties.setProperty(CREKID, KEY_ID)
rawProperties.setProperty("${CREK}.id.${KEY_ID}", KEY_HEX)
rawProperties.setProperty("${CREK}.id.${KEY_ID_2}", KEY_HEX_2)
rawProperties.setProperty("${CREK}.id.${KEY_ID_3}", KEY_HEX_3)
NiFiProperties niFiProperties = new StandardNiFiProperties(rawProperties)
logger.info("niFiProperties has ${niFiProperties.size()} properties: ${niFiProperties.getPropertyKeys()}")
// Act
def keyId = niFiProperties.getContentRepositoryEncryptionKeyId()
def key = niFiProperties.getContentRepositoryEncryptionKey()
def keys = niFiProperties.getContentRepositoryEncryptionKeys()
logger.info("Retrieved key ID: ${keyId}")
logger.info("Retrieved key: ${key}")
logger.info("Retrieved keys: ${keys}")
// Assert
assert keyId == KEY_ID
assert key == KEY_HEX
assert keys == [(KEY_ID): KEY_HEX, (KEY_ID_2): KEY_HEX_2, (KEY_ID_3): KEY_HEX_3]
}
@Test
void testShouldGetContentRepositoryEncryptionKeysWithNoneDefined() throws Exception {
// Arrange
Properties rawProperties = new Properties()
NiFiProperties niFiProperties = new StandardNiFiProperties(rawProperties)
logger.info("niFiProperties has ${niFiProperties.size()} properties: ${niFiProperties.getPropertyKeys()}")
// Act
def keyId = niFiProperties.getContentRepositoryEncryptionKeyId()
def key = niFiProperties.getContentRepositoryEncryptionKey()
def keys = niFiProperties.getContentRepositoryEncryptionKeys()
logger.info("Retrieved key ID: ${keyId}")
logger.info("Retrieved key: ${key}")
logger.info("Retrieved keys: ${keys}")
// Assert
assert keyId == null
assert key == null
assert keys == [:]
}
@Test
void testShouldNotGetContentRepositoryEncryptionKeysIfFileBasedKeyProvider() throws Exception {
// Arrange
Properties rawProperties = new Properties()
final String KEY_ID = "arbitraryKeyId"
rawProperties.setProperty(CREKID, KEY_ID)
NiFiProperties niFiProperties = new StandardNiFiProperties(rawProperties)
logger.info("niFiProperties has ${niFiProperties.size()} properties: ${niFiProperties.getPropertyKeys()}")
// Act
def keyId = niFiProperties.getContentRepositoryEncryptionKeyId()
def key = niFiProperties.getContentRepositoryEncryptionKey()
def keys = niFiProperties.getContentRepositoryEncryptionKeys()
logger.info("Retrieved key ID: ${keyId}")
logger.info("Retrieved key: ${key}")
logger.info("Retrieved keys: ${keys}")
// Assert
assert keyId == KEY_ID
assert key == null
assert keys == [:]
}
@Test
void testGetContentRepoEncryptionKeysShouldFilterOtherProperties() throws Exception {
// Arrange
Properties rawProperties = new Properties()
final String KEY_ID = "arbitraryKeyId"
final String KEY_HEX = "0123456789ABCDEFFEDCBA9876543210"
final String KEY_ID_2 = "arbitraryKeyId2"
final String KEY_HEX_2 = "AAAABBBBCCCCDDDDEEEEFFFF00001111"
final String KEY_ID_3 = "arbitraryKeyId3"
final String KEY_HEX_3 = "01010101010101010101010101010101"
rawProperties.setProperty(CREKID, KEY_ID)
rawProperties.setProperty("${CREK}.id.${KEY_ID}", KEY_HEX)
rawProperties.setProperty("${CREK}.id.${KEY_ID_2}", KEY_HEX_2)
rawProperties.setProperty("${CREK}.id.${KEY_ID_3}", KEY_HEX_3)
rawProperties.setProperty(NiFiProperties.CONTENT_REPOSITORY_ENCRYPTION_KEY_PROVIDER_IMPLEMENTATION_CLASS, "some.class.provider")
rawProperties.setProperty(NiFiProperties.CONTENT_REPOSITORY_ENCRYPTION_KEY_PROVIDER_LOCATION, "some://url")
NiFiProperties niFiProperties = new StandardNiFiProperties(rawProperties)
logger.info("niFiProperties has ${niFiProperties.size()} properties: ${niFiProperties.getPropertyKeys()}")
// Act
def keyId = niFiProperties.getContentRepositoryEncryptionKeyId()
def key = niFiProperties.getContentRepositoryEncryptionKey()
def keys = niFiProperties.getContentRepositoryEncryptionKeys()
logger.info("Retrieved key ID: ${keyId}")
logger.info("Retrieved key: ${key}")
logger.info("Retrieved keys: ${keys}")
// Assert
assert keyId == KEY_ID
assert key == KEY_HEX
assert keys == [(KEY_ID): KEY_HEX, (KEY_ID_2): KEY_HEX_2, (KEY_ID_3): KEY_HEX_3]
}
@Test
void testShouldNormalizeContextPathProperty() {

9
nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/EncryptedWriteAheadProvenanceRepository.java
View File

@ -38,6 +38,15 @@ import org.apache.nifi.util.NiFiProperties;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* This class is an implementation of the {@link WriteAheadProvenanceRepository} provenance repository which provides transparent
* block encryption/decryption of provenance event data during file system interaction. As of Apache NiFi 1.10.0
* (October 2019), this implementation is considered <a href="https://nifi.apache.org/docs/nifi-docs/html/user-guide.html#experimental-warning">*experimental*</a>. For further details, review the
* <a href="https://nifi.apache.org/docs/nifi-docs/html/user-guide.html#encrypted-provenance">Apache NiFi User Guide -
* Encrypted Provenance Repository</a> and
* <a href="https://nifi.apache.org/docs/nifi-docs/html/administration-guide.html#encrypted-write-ahead-provenance-repository-properties">Apache NiFi Admin Guide - Encrypted Write-Ahead Provenance
* Repository Properties</a>.
*/
public class EncryptedWriteAheadProvenanceRepository extends WriteAheadProvenanceRepository {
private static final Logger logger = LoggerFactory.getLogger(EncryptedWriteAheadProvenanceRepository.class);