Apache/nifi
1
0
Fork 0
mirror of https://github.com/apache/nifi.git synced 2025-02-06 01:58:32 +00:00
Code Issues Packages Projects Releases Wiki Activity

NIFI-9184 Refactored shared methods in Sensitive Property Providers

Browse Source 
- Added EncodedSensitivePropertiesProvider with Base64 encoding methods
- Added ClientBasedEncodedSensitivePropertiesProvider with validate method
- Abstracted client configuration to ClientProvider interface and implementations
- Added unit tests for AWS and Azure Property Providers

NIFI-9184 Adjusted abstract provider class names and updated documentation

Signed-off-by: Nathan Gough <thenatog@gmail.com>

This closes #5363.
This commit is contained in:
exceptionfactory 2021-08-31 23:02:37 -05:00 committed by Nathan Gough
parent ba775d28de
commit e78674ec59
23 changed files with 1139 additions and 915 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
nifi-commons/nifi-property-utils/src/main/java/org/apache/nifi/properties/AbstractBootstrapPropertiesLoader.java
View File

@ -136,7 +136,7 @@ public abstract class AbstractBootstrapPropertiesLoader {
if (confDir.exists() && confDir.canRead()) {
expectedBootstrapFile = new File(confDir, BOOTSTRAP_CONF);
} else {
throw new IOException(String.format("Cannot read %s directory for %s", confDir, bootstrapPath));
throw new IOException(String.format("Configuration Directory [%s] not found for Bootstrap Properties", confDir));
}
} else {
expectedBootstrapFile = new File(bootstrapPath);
@ -164,12 +164,12 @@ public abstract class AbstractBootstrapPropertiesLoader {
String systemPath = System.getProperty(systemPropertyName);
if (systemPath == null || systemPath.trim().isEmpty()) {
logger.warn("The system property {} is not set, so it is being set to '{}'", systemPropertyName, defaultRelativePath);
logger.warn("System Property [{}] not found: Using Relative Path [{}]", systemPropertyName, defaultRelativePath);
System.setProperty(systemPropertyName, defaultRelativePath);
systemPath = defaultRelativePath;
}
logger.info("Determined default application properties path to be '{}'", systemPath);
logger.debug("Default Application Properties Path [{}]", systemPath);
return systemPath;
}
}

338
nifi-commons/nifi-sensitive-property-provider/src/main/java/org/apache/nifi/properties/AWSKMSSensitivePropertyProvider.java
View File

@ -1,338 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.nifi.properties;
import org.apache.commons.lang3.StringUtils;
import org.apache.nifi.properties.BootstrapProperties.BootstrapPropertyKey;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import software.amazon.awssdk.auth.credentials.AwsBasicCredentials;
import software.amazon.awssdk.auth.credentials.DefaultCredentialsProvider;
import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider;
import software.amazon.awssdk.core.SdkBytes;
import software.amazon.awssdk.core.exception.SdkClientException;
import software.amazon.awssdk.regions.Region;
import software.amazon.awssdk.services.kms.KmsClient;
import software.amazon.awssdk.services.kms.model.DecryptRequest;
import software.amazon.awssdk.services.kms.model.DecryptResponse;
import software.amazon.awssdk.services.kms.model.DescribeKeyRequest;
import software.amazon.awssdk.services.kms.model.DescribeKeyResponse;
import software.amazon.awssdk.services.kms.model.EncryptRequest;
import software.amazon.awssdk.services.kms.model.EncryptResponse;
import software.amazon.awssdk.services.kms.model.KeyMetadata;
import software.amazon.awssdk.services.kms.model.KmsException;
import java.util.Base64;
import java.io.IOException;
import java.nio.charset.Charset;
import java.nio.charset.StandardCharsets;
import java.nio.file.Paths;
import java.util.Objects;
public class AWSKMSSensitivePropertyProvider extends AbstractSensitivePropertyProvider {
private static final Logger logger = LoggerFactory.getLogger(AWSKMSSensitivePropertyProvider.class);
private static final String AWS_PREFIX = "aws";
private static final String ACCESS_KEY_PROPS_NAME = "aws.access.key.id";
private static final String SECRET_KEY_PROPS_NAME = "aws.secret.access.key";
private static final String REGION_KEY_PROPS_NAME = "aws.region";
private static final String KMS_KEY_PROPS_NAME = "aws.kms.key.id";
private static final Charset PROPERTY_CHARSET = StandardCharsets.UTF_8;
private final BootstrapProperties awsBootstrapProperties;
private KmsClient client;
private String keyId;
AWSKMSSensitivePropertyProvider(final BootstrapProperties bootstrapProperties) throws SensitivePropertyProtectionException {
super(bootstrapProperties);
Objects.requireNonNull(bootstrapProperties, "The file bootstrap.conf provided to AWS SPP is null");
awsBootstrapProperties = getAWSBootstrapProperties(bootstrapProperties);
loadRequiredAWSProperties(awsBootstrapProperties);
}
/**
* Initializes the KMS Client to be used for encrypt, decrypt and other interactions with AWS KMS.
* First attempts to use credentials/configuration in bootstrap-aws.conf.
* If credentials/configuration in bootstrap-aws.conf is not fully configured,
* attempt to initialize credentials using default AWS credentials/configuration chain.
* Note: This does not verify if credentials are valid.
*/
private void initializeClient() {
if (awsBootstrapProperties == null) {
logger.warn("AWS Bootstrap Properties are required for KMS Client initialization");
return;
}
final String accessKey = awsBootstrapProperties.getProperty(ACCESS_KEY_PROPS_NAME);
final String secretKey = awsBootstrapProperties.getProperty(SECRET_KEY_PROPS_NAME);
final String region = awsBootstrapProperties.getProperty(REGION_KEY_PROPS_NAME);
if (StringUtils.isNoneBlank(accessKey, secretKey, region)) {
logger.debug("Using AWS credentials from bootstrap properties");
try {
final AwsBasicCredentials credentials = AwsBasicCredentials.create(accessKey, secretKey);
client = KmsClient.builder()
.region(Region.of(region))
.credentialsProvider(StaticCredentialsProvider.create(credentials))
.build();
} catch (final RuntimeException e) {
final String msg = "Valid configuration/credentials are required to initialize KMS client";
throw new SensitivePropertyProtectionException(msg, e);
}
} else {
logger.debug("Using AWS credentials from default credentials provider");
try {
final DefaultCredentialsProvider credentialsProvider = DefaultCredentialsProvider.builder()
.build();
credentialsProvider.resolveCredentials();
client = KmsClient.builder()
.credentialsProvider(credentialsProvider)
.build();
} catch (final SdkClientException e) {
final String msg = "Valid configuration/credentials are required to initialize KMS client";
throw new SensitivePropertyProtectionException(msg, e);
}
}
}
/**
* Validates the key ARN, credentials and configuration provided by the user.
* Note: This function performs checks on the key and indirectly also validates the credentials and
* configurations provided during the initialization of the client.
*/
private void validate() throws KmsException, SensitivePropertyProtectionException {
if (client == null) {
final String msg = "The AWS KMS Client failed to open, cannot validate key";
throw new SensitivePropertyProtectionException(msg);
}
if (StringUtils.isBlank(keyId)) {
final String msg = "The AWS KMS key provided is blank";
throw new SensitivePropertyProtectionException(msg);
}
// asking for a Key Description is the best way to check whether a key is valid
// because AWS KMS accepts various formats for its keys.
final DescribeKeyRequest request = DescribeKeyRequest.builder()
.keyId(keyId)
.build();
// using the KmsClient in a DescribeKey request indirectly also verifies if the credentials provided
// during the initialization of the key are valid
final DescribeKeyResponse response = client.describeKey(request);
final KeyMetadata metadata = response.keyMetadata();
if (!metadata.enabled()) {
final String msg = String.format("AWS KMS key [%s] is not enabled", keyId);
throw new SensitivePropertyProtectionException(msg);
}
}
/**
* Checks if we have a key ID from AWS KMS and loads it into {@link #keyId}. Will load null if key is not present.
* Note: This function does not verify if the key is correctly formatted/valid.
* @param props the properties representing bootstrap-aws.conf
*/
private void loadRequiredAWSProperties(final BootstrapProperties props) {
if (props != null) {
keyId = props.getProperty(KMS_KEY_PROPS_NAME);
}
}
/**
* Checks bootstrap.conf to check if BootstrapPropertyKey.AWS_KMS_SENSITIVE_PROPERTY_PROVIDER_CONF property is
* configured to the bootstrap-aws.conf file. Also will try to load bootstrap-aws.conf to {@link #awsBootstrapProperties}.
* @param bootstrapProperties BootstrapProperties object corresponding to bootstrap.conf.
* @return BootstrapProperties object corresponding to bootstrap-aws.conf, null otherwise.
*/
private BootstrapProperties getAWSBootstrapProperties(final BootstrapProperties bootstrapProperties) {
final BootstrapProperties cloudBootstrapProperties;
// Load the bootstrap-aws.conf file based on path specified in
// "nifi.bootstrap.protection.aws.kms.conf" property of bootstrap.conf
final String filePath = bootstrapProperties.getProperty(BootstrapPropertyKey.AWS_KMS_SENSITIVE_PROPERTY_PROVIDER_CONF).orElse(null);
if (StringUtils.isBlank(filePath)) {
logger.warn("AWS KMS properties file path not configured in bootstrap properties");
return null;
}
try {
cloudBootstrapProperties = AbstractBootstrapPropertiesLoader.loadBootstrapProperties(
Paths.get(filePath), AWS_PREFIX);
} catch (final IOException e) {
throw new SensitivePropertyProtectionException("Could not load " + filePath, e);
}
return cloudBootstrapProperties;
}
/**
* Checks bootstrap-aws.conf for the required configurations for AWS KMS encrypt/decrypt operations.
* Note: This does not check for credentials/region configurations.
* Credentials/configuration will be checked during the first protect/unprotect call during runtime.
* @return true if bootstrap-aws.conf contains the required properties for AWS SPP, false otherwise.
*/
private boolean hasRequiredAWSProperties() {
return awsBootstrapProperties != null && StringUtils.isNotBlank(keyId);
}
@Override
public boolean isSupported() {
return hasRequiredAWSProperties();
}
@Override
protected PropertyProtectionScheme getProtectionScheme() {
return PropertyProtectionScheme.AWS_KMS;
}
/**
* Returns the name of the underlying implementation.
*
* @return the name of this sensitive property provider.
*/
@Override
public String getName() {
return PropertyProtectionScheme.AWS_KMS.getName();
}
/**
* Returns the key used to identify the provider implementation in {@code nifi.properties}.
*
* @return the key to persist in the sibling property.
*/
@Override
public String getIdentifierKey() {
return PropertyProtectionScheme.AWS_KMS.getIdentifier();
}
/**
* Returns the ciphertext blob of this value encrypted using an AWS KMS CMK.
*
* @return the ciphertext blob to persist in the {@code nifi.properties} file.
*/
private byte[] encrypt(final byte[] input) {
final SdkBytes plainBytes = SdkBytes.fromByteArray(input);
final EncryptRequest encryptRequest = EncryptRequest.builder()
.keyId(keyId)
.plaintext(plainBytes)
.build();
final EncryptResponse response = client.encrypt(encryptRequest);
final SdkBytes encryptedData = response.ciphertextBlob();
return encryptedData.asByteArray();
}
/**
* Returns the value corresponding to a ciphertext blob decrypted using an AWS KMS CMK.
*
* @return the "unprotected" byte[] of this value, which could be used by the application.
*/
private byte[] decrypt(final byte[] input) {
final SdkBytes cipherBytes = SdkBytes.fromByteArray(input);
final DecryptRequest decryptRequest = DecryptRequest.builder()
.ciphertextBlob(cipherBytes)
.keyId(keyId)
.build();
final DecryptResponse response = client.decrypt(decryptRequest);
final SdkBytes decryptedData = response.plaintext();
return decryptedData.asByteArray();
}
/**
* Checks if the client is open and if not, initializes the client and validates the key required for AWS KMS.
*/
private void checkAndInitializeClient() throws SensitivePropertyProtectionException {
if (client == null) {
try {
initializeClient();
validate();
} catch (final SdkClientException | KmsException | SensitivePropertyProtectionException e) {
throw new SensitivePropertyProtectionException("Error initializing the AWS KMS Client", e);
}
}
}
/**
* Returns the "protected" form of this value. This is a form which can safely be persisted in the {@code nifi.properties} file without compromising the value.
* Encrypts a sensitive value using a key managed by AWS Key Management Service.
*
* @param unprotectedValue the sensitive value.
* @param context The context of the value (ignored in this implementation)
* @return the value to persist in the {@code nifi.properties} file.
*/
@Override
public String protect(final String unprotectedValue, final ProtectedPropertyContext context) throws SensitivePropertyProtectionException {
if (StringUtils.isBlank(unprotectedValue)) {
throw new IllegalArgumentException("Cannot encrypt a blank value");
}
checkAndInitializeClient();
try {
final byte[] plainBytes = unprotectedValue.getBytes(PROPERTY_CHARSET);
final byte[] cipherBytes = encrypt(plainBytes);
return Base64.getEncoder().encodeToString(cipherBytes);
} catch (final SdkClientException | KmsException e) {
throw new SensitivePropertyProtectionException("Encrypt failed", e);
}
}
/**
* Returns the "unprotected" form of this value. This is the raw sensitive value which is used by the application logic.
* Decrypts a secured value from a ciphertext using a key managed by AWS Key Management Service.
*
* @param protectedValue the protected value read from the {@code nifi.properties} file.
* @param context The context of the value (ignored in this implementation)
* @return the raw value to be used by the application.
*/
@Override
public String unprotect(final String protectedValue, final ProtectedPropertyContext context) throws SensitivePropertyProtectionException {
if (StringUtils.isBlank(protectedValue)) {
throw new IllegalArgumentException("Cannot decrypt a blank value");
}
checkAndInitializeClient();
try {
final byte[] cipherBytes = Base64.getDecoder().decode(protectedValue);
final byte[] plainBytes = decrypt(cipherBytes);
return new String(plainBytes, PROPERTY_CHARSET);
} catch (final SdkClientException | KmsException e) {
throw new SensitivePropertyProtectionException("Decrypt failed", e);
}
}
/**
* Closes AWS KMS client that may have been opened.
*/
@Override
public void cleanUp() {
if (client != null) {
client.close();
client = null;
}
}
}

121
nifi-commons/nifi-sensitive-property-provider/src/main/java/org/apache/nifi/properties/AwsKmsSensitivePropertyProvider.java Normal file
View File

@ -0,0 +1,121 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.nifi.properties;
import software.amazon.awssdk.core.SdkBytes;
import software.amazon.awssdk.services.kms.KmsClient;
import software.amazon.awssdk.services.kms.model.DecryptRequest;
import software.amazon.awssdk.services.kms.model.DecryptResponse;
import software.amazon.awssdk.services.kms.model.DescribeKeyRequest;
import software.amazon.awssdk.services.kms.model.DescribeKeyResponse;
import software.amazon.awssdk.services.kms.model.EncryptRequest;
import software.amazon.awssdk.services.kms.model.EncryptResponse;
import software.amazon.awssdk.services.kms.model.KeyMetadata;
import java.util.Properties;
/**
* Amazon Web Services Key Management Service Sensitive Property Provider
*/
public class AwsKmsSensitivePropertyProvider extends ClientBasedEncodedSensitivePropertyProvider<KmsClient> {
protected static final String KEY_ID_PROPERTY = "aws.kms.key.id";
AwsKmsSensitivePropertyProvider(final KmsClient kmsClient, final Properties properties) throws SensitivePropertyProtectionException {
super(PropertyProtectionScheme.AWS_KMS, kmsClient, properties);
}
/**
* Close KMS Client when configured
*/
@Override
public void cleanUp() {
final KmsClient kmsClient = getClient();
if (kmsClient == null) {
logger.debug("AWS KMS Client not configured");
} else {
kmsClient.close();
}
}
/**
* Validate Client and Key Identifier status when client is configured
*
* @param kmsClient KMS Client
*/
@Override
protected void validate(final KmsClient kmsClient) {
if (kmsClient == null) {
logger.debug("AWS KMS Client not configured");
} else {
final String keyId = getKeyId();
try {
final DescribeKeyRequest describeKeyRequest = DescribeKeyRequest.builder()
.keyId(keyId)
.build();
final DescribeKeyResponse describeKeyResponse = kmsClient.describeKey(describeKeyRequest);
final KeyMetadata keyMetadata = describeKeyResponse.keyMetadata();
if (keyMetadata.enabled()) {
logger.info("AWS KMS Key [{}] Enabled", keyId);
} else {
throw new SensitivePropertyProtectionException(String.format("AWS KMS Key [%s] Disabled", keyId));
}
} catch (final RuntimeException e) {
throw new SensitivePropertyProtectionException(String.format("AWS KMS Key [%s] Validation Failed", keyId), e);
}
}
}
/**
* Get encrypted bytes
*
* @param bytes Unprotected bytes
* @return Encrypted bytes
*/
@Override
protected byte[] getEncrypted(final byte[] bytes) {
final SdkBytes plainBytes = SdkBytes.fromByteArray(bytes);
final EncryptRequest encryptRequest = EncryptRequest.builder()
.keyId(getKeyId())
.plaintext(plainBytes)
.build();
final EncryptResponse response = getClient().encrypt(encryptRequest);
final SdkBytes encryptedData = response.ciphertextBlob();
return encryptedData.asByteArray();
}
/**
* Get decrypted bytes
*
* @param bytes Encrypted bytes
* @return Decrypted bytes
*/
@Override
protected byte[] getDecrypted(final byte[] bytes) {
final SdkBytes cipherBytes = SdkBytes.fromByteArray(bytes);
final DecryptRequest decryptRequest = DecryptRequest.builder()
.ciphertextBlob(cipherBytes)
.keyId(getKeyId())
.build();
final DecryptResponse response = getClient().decrypt(decryptRequest);
final SdkBytes decryptedData = response.plaintext();
return decryptedData.asByteArray();
}
private String getKeyId() {
return getProperties().getProperty(KEY_ID_PROPERTY);
}
}

300
nifi-commons/nifi-sensitive-property-provider/src/main/java/org/apache/nifi/properties/AzureKeyVaultKeySensitivePropertyProvider.java
View File

@ -16,282 +16,90 @@
*/
package org.apache.nifi.properties;
import org.apache.commons.lang3.StringUtils;
import org.apache.nifi.properties.BootstrapProperties.BootstrapPropertyKey;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.azure.core.exception.ResourceNotFoundException;
import com.azure.identity.DefaultAzureCredentialBuilder;
import com.azure.security.keyvault.keys.models.KeyVaultKey;
import com.azure.security.keyvault.keys.cryptography.CryptographyClient;
import com.azure.security.keyvault.keys.cryptography.CryptographyClientBuilder;
import com.azure.security.keyvault.keys.cryptography.models.DecryptResult;
import com.azure.security.keyvault.keys.cryptography.models.EncryptResult;
import com.azure.security.keyvault.keys.cryptography.models.EncryptionAlgorithm;
import com.azure.security.keyvault.keys.models.KeyOperation;
import com.azure.security.keyvault.keys.models.KeyProperties;
import java.util.Base64;
import java.io.IOException;
import java.nio.charset.Charset;
import java.nio.charset.StandardCharsets;
import java.nio.file.Paths;
import org.apache.nifi.util.StringUtils;
import java.util.Arrays;
import java.util.List;
import java.util.Objects;
import java.util.Properties;
public class AzureKeyVaultKeySensitivePropertyProvider extends AbstractSensitivePropertyProvider {
private static final Logger logger = LoggerFactory.getLogger(AzureKeyVaultKeySensitivePropertyProvider.class);
/**
* Microsoft Azure Key Vault Key Sensitive Property Provider using Cryptography Client for encryption operations
*/
public class AzureKeyVaultKeySensitivePropertyProvider extends ClientBasedEncodedSensitivePropertyProvider<CryptographyClient> {
protected static final String ENCRYPTION_ALGORITHM_PROPERTY = "azure.keyvault.encryption.algorithm";
private static final String AZURE_PREFIX = "azure";
private static final String KEYVAULT_KEY_PROPS_NAME = "azure.keyvault.key.id";
private static final String ENCRYPTION_ALGORITHM_PROPS_NAME = "azure.keyvault.encryption.algorithm";
protected static final List<KeyOperation> REQUIRED_OPERATIONS = Arrays.asList(KeyOperation.DECRYPT, KeyOperation.ENCRYPT);
private static final Charset PROPERTY_CHARSET = StandardCharsets.UTF_8;
private EncryptionAlgorithm encryptionAlgorithm;
private final BootstrapProperties azureBootstrapProperties;
private CryptographyClient client;
private String keyId;
private String algorithm;
AzureKeyVaultKeySensitivePropertyProvider(final BootstrapProperties bootstrapProperties) throws SensitivePropertyProtectionException {
super(bootstrapProperties);
Objects.requireNonNull(bootstrapProperties, "Bootstrap Properties required");
azureBootstrapProperties = getAzureBootstrapProperties(bootstrapProperties);
loadRequiredAzureProperties(azureBootstrapProperties);
AzureKeyVaultKeySensitivePropertyProvider(final CryptographyClient cryptographyClient, final Properties properties) {
super(PropertyProtectionScheme.AZURE_KEYVAULT_KEY, cryptographyClient, properties);
}
/**
* Initializes the Azure Key Vault Cryptography Client to be used for encrypt, decrypt and other interactions with Azure Key Vault.
* Uses the default Azure credentials provider chain.
* Validate Client and Key Operations with Encryption Algorithm when configured
*
* @param cryptographyClient Cryptography Client
*/
private void initializeClient() {
if (azureBootstrapProperties == null) {
logger.warn("Azure Bootstrap Properties are required for Key Vault Client initialization");
return;
}
if (StringUtils.isBlank(keyId)) {
logger.warn("Cannot initialize client if Azure Key Vault Key ID is blank");
return;
}
try {
client = new CryptographyClientBuilder()
.credential(new DefaultAzureCredentialBuilder().build())
.keyIdentifier(keyId)
.buildClient();
} catch (final RuntimeException e) {
throw new SensitivePropertyProtectionException("Azure Key Vault Client initialization failed", e);
}
}
/**
* Validates the key provided by the user.
* Note: This function performs checks on the key and indirectly also validates the credentials provided
* during the initialization of the client.
*/
private void validate() throws SensitivePropertyProtectionException {
if (client == null) {
throw new SensitivePropertyProtectionException("Azure Key Vault validation failed: Client not initialized");
}
if (StringUtils.isBlank(keyId)) {
throw new SensitivePropertyProtectionException("Azure Key Vault validation failed: Key not specified");
}
try {
final KeyProperties keyProps = client.getKey().getProperties();
if (!keyProps.isEnabled()) {
throw new SensitivePropertyProtectionException("Azure Key Vault validation failed: Key not enabled");
@Override
protected void validate(final CryptographyClient cryptographyClient) {
if (cryptographyClient == null) {
logger.debug("Azure Cryptography Client not configured");
} else {
try {
final KeyVaultKey keyVaultKey = cryptographyClient.getKey();
final String id = keyVaultKey.getId();
final KeyProperties keyProperties = keyVaultKey.getProperties();
if (keyProperties.isEnabled()) {
final List<KeyOperation> keyOperations = keyVaultKey.getKeyOperations();
if (keyOperations.containsAll(REQUIRED_OPERATIONS)) {
logger.info("Azure Key Vault Key [{}] Validated", id);
} else {
throw new SensitivePropertyProtectionException(String.format("Azure Key Vault Key [%s] Missing Operations %s", id, REQUIRED_OPERATIONS));
}
} else {
throw new SensitivePropertyProtectionException(String.format("Azure Key Vault Key [%s] Disabled", id));
}
} catch (final RuntimeException e) {
throw new SensitivePropertyProtectionException("Azure Key Vault Key Validation Failed", e);
}
final List<KeyOperation> keyOps = client.getKey().getKeyOperations();
if (!(keyOps.contains(KeyOperation.ENCRYPT) && keyOps.contains(KeyOperation.DECRYPT))) {
throw new SensitivePropertyProtectionException("Azure Key Vault validation failed: Encrypt and Decrypt not supported");
final String algorithm = getProperties().getProperty(ENCRYPTION_ALGORITHM_PROPERTY);
if (StringUtils.isBlank(algorithm)) {
throw new SensitivePropertyProtectionException("Azure Key Vault Key Algorithm not configured");
}
} catch (final ResourceNotFoundException e) {
throw new SensitivePropertyProtectionException("Azure Key Vault validation failed: Key not found", e);
} catch (final RuntimeException e) {
throw new SensitivePropertyProtectionException("Azure Key Vault validation failed", e);
encryptionAlgorithm = EncryptionAlgorithm.fromString(algorithm);
}
}
/**
* Checks if we have the required properties {@link #keyId} and {@link #algorithm} from bootstrap-azure.conf
* for Azure KeyVault and loads it into the appropriate variables, will load null if values don't exist.
* Note: This function does not verify if the properties are valid.
* @param props the properties representing bootstrap-azure.conf
*/
private void loadRequiredAzureProperties(final BootstrapProperties props) {
if (props != null) {
keyId = props.getProperty(KEYVAULT_KEY_PROPS_NAME);
algorithm = props.getProperty(ENCRYPTION_ALGORITHM_PROPS_NAME);
}
}
/**
* Checks bootstrap.conf to check if BootstrapPropertyKey.AZURE_KEYVAULT_SENSITIVE_PROPERTY_PROVIDER_CONF property is configured to the
* bootstrap-azure.conf file. Also will load bootstrap-azure.conf to {@link #azureBootstrapProperties} if possible
* @param bootstrapProperties BootstrapProperties object corresponding to bootstrap.conf
* @return BootstrapProperties object corresponding to bootstrap-azure.conf, null otherwise
*/
private BootstrapProperties getAzureBootstrapProperties(final BootstrapProperties bootstrapProperties) {
final BootstrapProperties cloudBootstrapProperties;
// Load the bootstrap-azure.conf file based on path specified in
// "nifi.bootstrap.protection.azure.keyvault.conf" property of bootstrap.conf
final String filePath = bootstrapProperties.getProperty(BootstrapPropertyKey.AZURE_KEYVAULT_SENSITIVE_PROPERTY_PROVIDER_CONF).orElse(null);
if (StringUtils.isBlank(filePath)) {
logger.warn("Azure Key Vault properties file path not configured in bootstrap properties");
return null;
}
try {
cloudBootstrapProperties = AbstractBootstrapPropertiesLoader.loadBootstrapProperties(
Paths.get(filePath), AZURE_PREFIX);
} catch (final IOException e) {
throw new SensitivePropertyProtectionException("Could not load " + filePath, e);
}
return cloudBootstrapProperties;
}
/**
* Checks the BootstrapProperties corresponding to bootstrap-azure.conf for the required configurations
* for Azure encrypt/decrypt operations.
* Note: This does not check for credentials/region configurations.
* Credentials/configuration will be checked during the first protect/unprotect call during runtime.
* @return True if bootstrap-azure.conf contains the required properties for Azure SPP, False otherwise
*/
private boolean hasRequiredAzureProperties() {
return azureBootstrapProperties != null && StringUtils.isNoneBlank(keyId, algorithm);
}
/**
* Return true if this SensitivePropertyProvider is supported, given the provided Bootstrap properties.
* @return True if this SensitivePropertyProvider is supported
* Get encrypted bytes
*
* @param bytes Unprotected bytes
* @return Encrypted bytes
*/
@Override
public boolean isSupported() {
return hasRequiredAzureProperties();
}
/**
* Return the appropriate PropertyProtectionScheme for this provider.
*
* @return The PropertyProtectionScheme
*/
@Override
protected PropertyProtectionScheme getProtectionScheme() {
return PropertyProtectionScheme.AZURE_KEYVAULT_KEY;
}
/**
* Returns the name of the underlying implementation.
*
* @return the name of this sensitive property provider
*/
@Override
public String getName() {
return PropertyProtectionScheme.AZURE_KEYVAULT_KEY.getName();
}
/**
* Returns the key used to identify the provider implementation in {@code nifi.properties}.
*
* @return the key to persist in the sibling property
*/
@Override
public String getIdentifierKey() {
return PropertyProtectionScheme.AZURE_KEYVAULT_KEY.getIdentifier();
}
/**
* Returns the ciphertext of this value encrypted using a key stored in Azure Key Vault.
*
* @return the ciphertext blob to persist in the {@code nifi.properties} file
*/
private byte[] encrypt(final byte[] input) {
EncryptResult encryptResult = client.encrypt(EncryptionAlgorithm.fromString(algorithm), input);
protected byte[] getEncrypted(final byte[] bytes) {
final EncryptResult encryptResult = getClient().encrypt(encryptionAlgorithm, bytes);
return encryptResult.getCipherText();
}
/**
* Returns the value corresponding to a ciphertext decrypted using a key stored in Azure Key Vault
* Get decrypted bytes
*
* @return the "unprotected" byte[] of this value, which could be used by the application
* @param bytes Encrypted bytes
* @return Decrypted bytes
*/
private byte[] decrypt(final byte[] input) {
DecryptResult decryptResult = client.decrypt(EncryptionAlgorithm.fromString(algorithm), input);
@Override
protected byte[] getDecrypted(final byte[] bytes) {
final DecryptResult decryptResult = getClient().decrypt(encryptionAlgorithm, bytes);
return decryptResult.getPlainText();
}
/**
* Checks if the client is open and if not, initializes the client and validates the configuration required for Azure Key Vault.
*/
private void checkAndInitializeClient() {
if (client == null) {
initializeClient();
validate();
}
}
/**
* Returns the "protected" form of this value. This is a form which can safely be persisted in the {@code nifi.properties} file without compromising the value.
* Encrypts a sensitive value using a key managed by Azure Key Vault.
*
* @param unprotectedValue the sensitive value
* @param context The context of the value (ignored in this implementation)
* @return the value to persist in the {@code nifi.properties} file
*/
@Override
public String protect(final String unprotectedValue, final ProtectedPropertyContext context) throws SensitivePropertyProtectionException {
if (StringUtils.isBlank(unprotectedValue)) {
throw new IllegalArgumentException("Cannot encrypt a blank value");
}
checkAndInitializeClient();
try {
final byte[] plainBytes = unprotectedValue.getBytes(PROPERTY_CHARSET);
final byte[] cipherBytes = encrypt(plainBytes);
return Base64.getEncoder().encodeToString(cipherBytes);
} catch (final RuntimeException e) {
throw new SensitivePropertyProtectionException("Encrypt failed", e);
}
}
/**
* Returns the "unprotected" form of this value. This is the raw sensitive value which is used by the application logic.
* Decrypts a secured value from a ciphertext using a key managed by Azure Key Vault.
*
* @param protectedValue the protected value read from the {@code nifi.properties} file
* @param context The context of the value (ignored in this implementation)
* @return the raw value to be used by the application
*/
@Override
public String unprotect(final String protectedValue, final ProtectedPropertyContext context) throws SensitivePropertyProtectionException {
if (StringUtils.isBlank(protectedValue)) {
throw new IllegalArgumentException("Cannot decrypt a blank value");
}
checkAndInitializeClient();
try {
final byte[] cipherBytes = Base64.getDecoder().decode(protectedValue);
final byte[] plainBytes = decrypt(cipherBytes);
return new String(plainBytes, PROPERTY_CHARSET);
} catch (final RuntimeException e) {
throw new SensitivePropertyProtectionException("Decrypt failed", e);
}
}
/**
* Nothing required to be done for Azure Client cleanUp function.
*/
@Override
public void cleanUp() {}
}

94
nifi-commons/nifi-sensitive-property-provider/src/main/java/org/apache/nifi/properties/ClientBasedEncodedSensitivePropertyProvider.java Normal file
View File

@ -0,0 +1,94 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.nifi.properties;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.Properties;
/**
* Client-Based extension of Encoded Sensitive Property Provider
*
* @param <T> Client Type
*/
public abstract class ClientBasedEncodedSensitivePropertyProvider<T> extends EncodedSensitivePropertyProvider {
protected final Logger logger = LoggerFactory.getLogger(getClass());
private final T client;
private final Properties properties;
public ClientBasedEncodedSensitivePropertyProvider(final PropertyProtectionScheme propertyProtectionScheme,
final T client,
final Properties properties) {
super(propertyProtectionScheme);
this.client = client;
this.properties = properties;
validate(client);
}
/**
* Is Provider supported based on client status
*
* @return Provider supported status
*/
@Override
public boolean isSupported() {
return client != null;
}
/**
* Clean up resources should be overridden when client requires shutdown
*/
@Override
public void cleanUp() {
logger.debug("Cleanup Started");
}
/**
* Get Client Properties
*
* @return Client Properties
*/
protected Properties getProperties() {
return properties;
}
/**
* Get Client
*
* @return Client can be null when not configured
*/
protected T getClient() {
if (client == null) {
throw new IllegalStateException("Client not configured");
}
return client;
}
/**
* Validate Provider and Client configuration
*
* @param configuredClient Configured Client
*/
protected void validate(final T configuredClient) {
if (configuredClient == null) {
logger.debug("Client not configured");
}
}
}

117
nifi-commons/nifi-sensitive-property-provider/src/main/java/org/apache/nifi/properties/EncodedSensitivePropertyProvider.java Normal file
View File

@ -0,0 +1,117 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.nifi.properties;
import java.nio.charset.Charset;
import java.nio.charset.StandardCharsets;
import java.util.Base64;
import java.util.Objects;
/**
* Encoded Sensitive Property Provider handles Base64 encoding and decoding of property values
*/
public abstract class EncodedSensitivePropertyProvider implements SensitivePropertyProvider {
private static final Charset VALUE_CHARACTER_SET = StandardCharsets.UTF_8;
private static final Base64.Encoder ENCODER = Base64.getEncoder().withoutPadding();
private static final Base64.Decoder DECODER = Base64.getDecoder();
private final PropertyProtectionScheme propertyProtectionScheme;
public EncodedSensitivePropertyProvider(final PropertyProtectionScheme propertyProtectionScheme) {
this.propertyProtectionScheme = Objects.requireNonNull(propertyProtectionScheme, "Scheme Required");
}
/**
* Get Property Protection Scheme Name
*
* @return Property Protection Scheme Name
*/
@Override
public String getName() {
return propertyProtectionScheme.getName();
}
/**
* Get Identifier Key based on Property Protection Scheme
*
* @return Property Protection Scheme Identifier
*/
@Override
public String getIdentifierKey() {
return propertyProtectionScheme.getIdentifier();
}
/**
* Protect property value and return Base64-encoded representation of encrypted bytes
*
* @param unprotectedValue Unprotected property value to be encrypted
* @param context Property Context
* @return Base64-encoded representation of encrypted bytes
*/
@Override
public String protect(final String unprotectedValue, final ProtectedPropertyContext context) {
Objects.requireNonNull(unprotectedValue, "Value required");
Objects.requireNonNull(context, "Context required");
try {
final byte[] bytes = unprotectedValue.getBytes(VALUE_CHARACTER_SET);
final byte[] encrypted = getEncrypted(bytes);
return ENCODER.encodeToString(encrypted);
} catch (final RuntimeException e) {
final String message = String.format("Property [%s] Encryption Failed", context.getContextKey());
throw new SensitivePropertyProtectionException(message, e);
}
}
/**
* Unprotect Base64-encoded representation of encrypted property value and return string
*
* @param protectedValue Base64-encoded representation of encrypted bytes
* @param context Property Context
* @return Decrypted property value string
*/
@Override
public String unprotect(final String protectedValue, final ProtectedPropertyContext context) {
Objects.requireNonNull(protectedValue, "Value required");
Objects.requireNonNull(context, "Context required");
try {
final byte[] decoded = DECODER.decode(protectedValue);
final byte[] decrypted = getDecrypted(decoded);
return new String(decrypted, VALUE_CHARACTER_SET);
} catch (final RuntimeException e) {
final String message = String.format("Property [%s] Decryption Failed", context.getContextKey());
throw new SensitivePropertyProtectionException(message, e);
}
}
/**
* Get encrypted byte array representation of bytes
*
* @param bytes Unprotected bytes
* @return Encrypted bytes
*/
protected abstract byte[] getEncrypted(byte[] bytes);
/**
* Get decrypted byte array representation of encrypted bytes
*
* @param bytes Encrypted bytes
* @return Decrypted bytes
*/
protected abstract byte[] getDecrypted(byte[] bytes);
}

288
nifi-commons/nifi-sensitive-property-provider/src/main/java/org/apache/nifi/properties/GCPKMSSensitivePropertyProvider.java
View File

@ -1,288 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.nifi.properties;
import org.apache.commons.lang3.StringUtils;
import org.apache.nifi.properties.BootstrapProperties.BootstrapPropertyKey;
import com.google.api.gax.rpc.ApiException;
import com.google.cloud.kms.v1.CryptoKey;
import com.google.cloud.kms.v1.CryptoKeyName;
import com.google.cloud.kms.v1.CryptoKeyVersion;
import com.google.cloud.kms.v1.DecryptResponse;
import com.google.cloud.kms.v1.EncryptResponse;
import com.google.cloud.kms.v1.KeyManagementServiceClient;
import com.google.protobuf.ByteString;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.Base64;
import java.io.IOException;
import java.nio.charset.Charset;
import java.nio.charset.StandardCharsets;
import java.nio.file.Paths;
import java.util.Objects;
public class GCPKMSSensitivePropertyProvider extends AbstractSensitivePropertyProvider {
private static final Logger logger = LoggerFactory.getLogger(GCPKMSSensitivePropertyProvider.class);
private static final String GCP_PREFIX = "gcp";
private static final String PROJECT_ID_PROPS_NAME = "gcp.kms.project";
private static final String LOCATION_ID_PROPS_NAME = "gcp.kms.location";
private static final String KEYRING_ID_PROPS_NAME = "gcp.kms.keyring";
private static final String KEY_ID_PROPS_NAME = "gcp.kms.key";
private static final Charset PROPERTY_CHARSET = StandardCharsets.UTF_8;
private final BootstrapProperties gcpBootstrapProperties;
private KeyManagementServiceClient client;
private CryptoKeyName keyName;
GCPKMSSensitivePropertyProvider(final BootstrapProperties bootstrapProperties) {
super(bootstrapProperties);
Objects.requireNonNull(bootstrapProperties, "The file bootstrap.conf provided to GCP SPP is null");
gcpBootstrapProperties = getGCPBootstrapProperties(bootstrapProperties);
loadRequiredGCPProperties(gcpBootstrapProperties);
}
/**
* Initializes the GCP KMS Client to be used for encrypt, decrypt and other interactions with GCP Cloud KMS.
* Note: This does not verify if credentials are valid.
*/
private void initializeClient() {
try {
client = KeyManagementServiceClient.create();
} catch (final IOException e) {
final String msg = "Encountered an error initializing GCP Cloud KMS client";
throw new SensitivePropertyProtectionException(msg, e);
}
}
/**
* Validates the key details provided by the user.
*/
private void validate() throws ApiException, SensitivePropertyProtectionException {
if (client == null) {
final String msg = "The GCP KMS client failed to open, cannot validate key";
throw new SensitivePropertyProtectionException(msg);
}
if (keyName == null) {
final String msg = "The GCP KMS key provided is not provided/complete";
throw new SensitivePropertyProtectionException(msg);
}
final CryptoKey key;
final CryptoKeyVersion keyVersion;
try {
key = client.getCryptoKey(keyName);
keyVersion = client.getCryptoKeyVersion(key.getPrimary().getName());
} catch (final ApiException e) {
throw new SensitivePropertyProtectionException("Encountered an error while fetching key details", e);
}
if (keyVersion.getState() != CryptoKeyVersion.CryptoKeyVersionState.ENABLED) {
throw new SensitivePropertyProtectionException("The key is not enabled");
}
}
/**
* Checks if we have the required key properties for GCP Cloud KMS and loads it into {@link #keyName}.
* Will load null if key is not present.
* Note: This function does not verify if the key is correctly formatted/valid.
* @param props the properties representing bootstrap-gcp.conf.
*/
private void loadRequiredGCPProperties(final BootstrapProperties props) {
if (props != null) {
final String projectId = props.getProperty(PROJECT_ID_PROPS_NAME);
final String locationId = props.getProperty(LOCATION_ID_PROPS_NAME);
final String keyRingId = props.getProperty(KEYRING_ID_PROPS_NAME);
final String keyId = props.getProperty(KEY_ID_PROPS_NAME);
if (StringUtils.isNoneBlank(projectId, locationId, keyRingId, keyId)) {
keyName = CryptoKeyName.of(projectId, locationId, keyRingId, keyId);
}
}
}
/**
* Checks bootstrap.conf to check if BootstrapPropertyKey.GCP_KMS_SENSITIVE_PROPERTY_PROVIDER_CONF property is
* configured to the bootstrap-gcp.conf file. Also will load bootstrap-gcp.conf to {@link #gcpBootstrapProperties}.
* @param bootstrapProperties BootstrapProperties object corresponding to bootstrap.conf.
* @return BootstrapProperties object corresponding to bootstrap-gcp.conf, null otherwise.
*/
private BootstrapProperties getGCPBootstrapProperties(final BootstrapProperties bootstrapProperties) {
final BootstrapProperties cloudBootstrapProperties;
// Load the bootstrap-gcp.conf file based on path specified in
// "nifi.bootstrap.protection.gcp.kms.conf" property of bootstrap.conf
final String filePath = bootstrapProperties.getProperty(BootstrapPropertyKey.GCP_KMS_SENSITIVE_PROPERTY_PROVIDER_CONF).orElse(null);
if (StringUtils.isBlank(filePath)) {
logger.warn("GCP KMS properties file path not configured in bootstrap properties");
return null;
}
try {
cloudBootstrapProperties = AbstractBootstrapPropertiesLoader.loadBootstrapProperties(
Paths.get(filePath), GCP_PREFIX);
} catch (final IOException e) {
throw new SensitivePropertyProtectionException("Could not load " + filePath, e);
}
return cloudBootstrapProperties;
}
/**
* Checks bootstrap-gcp.conf for the required configurations for Google Cloud KMS encrypt/decrypt operations.
* @return true if bootstrap-gcp.conf contains the required properties for GCP KMS SPP, false otherwise.
*/
private boolean hasRequiredGCPProperties() {
if (gcpBootstrapProperties == null) {
return false;
}
final String projectId = gcpBootstrapProperties.getProperty(PROJECT_ID_PROPS_NAME);
final String locationId = gcpBootstrapProperties.getProperty(LOCATION_ID_PROPS_NAME);
final String keyRingId = gcpBootstrapProperties.getProperty(KEYRING_ID_PROPS_NAME);
final String keyId = gcpBootstrapProperties.getProperty(KEY_ID_PROPS_NAME);
// Note: the following does not verify if the properties are valid properties, they only verify if
// the properties are configured in bootstrap-gcp.conf.
return StringUtils.isNoneBlank(projectId, locationId, keyRingId, keyId);
}
@Override
public boolean isSupported() {
return hasRequiredGCPProperties();
}
@Override
protected PropertyProtectionScheme getProtectionScheme() {
return PropertyProtectionScheme.GCP_KMS;
}
/**
* Returns the name of the underlying implementation.
*
* @return the name of this sensitive property provider.
*/
@Override
public String getName() {
return PropertyProtectionScheme.GCP_KMS.getName();
}
/**
* Returns the key used to identify the provider implementation in {@code nifi.properties}.
*
* @return the key to persist in the sibling property.
*/
@Override
public String getIdentifierKey() {
return PropertyProtectionScheme.GCP_KMS.getIdentifier();
}
/**
* Returns the ciphertext blob of this value encrypted using a key stored in GCP KMS.
* @return the ciphertext blob to persist in the {@code nifi.properties} file.
*/
private byte[] encrypt(final byte[] input) throws IOException {
final EncryptResponse response = client.encrypt(keyName, ByteString.copyFrom(input));
return response.getCiphertext().toByteArray();
}
/**
* Returns the value corresponding to a ciphertext blob decrypted using a key stored in GCP KMS.
* @return the "unprotected" byte[] of this value, which could be used by the application.
*/
private byte[] decrypt(final byte[] input) throws IOException {
final DecryptResponse response = client.decrypt(keyName, ByteString.copyFrom(input));
return response.getPlaintext().toByteArray();
}
/**
* Checks if the client is open and if not, initializes the client and validates the key required for GCP KMS.
*/
private void checkAndInitializeClient() throws SensitivePropertyProtectionException {
if (client == null) {
try {
initializeClient();
validate();
} catch (final SensitivePropertyProtectionException e) {
throw new SensitivePropertyProtectionException("Error initializing the GCP KMS client", e);
}
}
}
/**
* Returns the "protected" form of this value. This is a form which can safely be persisted in the {@code nifi.properties} file without compromising the value.
* Encrypts a sensitive value using a key managed by Google Cloud Key Management Service.
*
* @param unprotectedValue the sensitive value.
* @param context The context of the value (ignored in this implementation)
* @return the value to persist in the {@code nifi.properties} file.
*/
@Override
public String protect(final String unprotectedValue, final ProtectedPropertyContext context) throws SensitivePropertyProtectionException {
if (StringUtils.isBlank(unprotectedValue)) {
throw new IllegalArgumentException("Cannot encrypt a blank value");
}
checkAndInitializeClient();
try {
byte[] plainBytes = unprotectedValue.getBytes(PROPERTY_CHARSET);
byte[] cipherBytes = encrypt(plainBytes);
return Base64.getEncoder().encodeToString(cipherBytes);
} catch (final IOException | ApiException e) {
throw new SensitivePropertyProtectionException("Encrypt failed", e);
}
}
/**
* Returns the "unprotected" form of this value. This is the raw sensitive value which is used by the application logic.
* Decrypts a secured value from a ciphertext using a key managed by Google Cloud Key Management Service.
*
* @param protectedValue the protected value read from the {@code nifi.properties} file.
* @param context The context of the value (ignored in this implementation)
* @return the raw value to be used by the application.
*/
@Override
public String unprotect(final String protectedValue, final ProtectedPropertyContext context) throws SensitivePropertyProtectionException {
if (StringUtils.isBlank(protectedValue)) {
throw new IllegalArgumentException("Cannot decrypt a blank value");
}
checkAndInitializeClient();
try {
byte[] cipherBytes = Base64.getDecoder().decode(protectedValue);
byte[] plainBytes = decrypt(cipherBytes);
return new String(plainBytes, PROPERTY_CHARSET);
} catch (final IOException | ApiException e) {
throw new SensitivePropertyProtectionException("Decrypt failed", e);
}
}
/**
* Closes GCP KMS client that may have been opened.
*/
@Override
public void cleanUp() {
if (client != null) {
client.close();
client = null;
}
}
}

115
nifi-commons/nifi-sensitive-property-provider/src/main/java/org/apache/nifi/properties/GcpKmsSensitivePropertyProvider.java Normal file
View File

@ -0,0 +1,115 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.nifi.properties;
import com.google.api.gax.rpc.ApiException;
import com.google.cloud.kms.v1.CryptoKey;
import com.google.cloud.kms.v1.CryptoKeyName;
import com.google.cloud.kms.v1.CryptoKeyVersion;
import com.google.cloud.kms.v1.DecryptResponse;
import com.google.cloud.kms.v1.EncryptResponse;
import com.google.cloud.kms.v1.KeyManagementServiceClient;
import com.google.protobuf.ByteString;
import org.apache.commons.lang3.StringUtils;
import java.util.Properties;
/**
* Google Cloud Platform Key Management Service Sensitive Property Provider
*/
public class GcpKmsSensitivePropertyProvider extends ClientBasedEncodedSensitivePropertyProvider<KeyManagementServiceClient> {
protected static final String PROJECT_PROPERTY = "gcp.kms.project";
protected static final String LOCATION_PROPERTY = "gcp.kms.location";
protected static final String KEYRING_PROPERTY = "gcp.kms.keyring";
protected static final String KEY_PROPERTY = "gcp.kms.key";
private CryptoKeyName cryptoKeyName;
GcpKmsSensitivePropertyProvider(final KeyManagementServiceClient keyManagementServiceClient, final Properties properties) {
super(PropertyProtectionScheme.GCP_KMS, keyManagementServiceClient, properties);
}
/**
* Close Client when configured
*/
@Override
public void cleanUp() {
final KeyManagementServiceClient keyManagementServiceClient = getClient();
if (keyManagementServiceClient == null) {
logger.debug("GCP KMS Client not configured");
} else {
keyManagementServiceClient.close();
}
}
/**
* Validate Client and Key Operations with Encryption Algorithm when configured
*
* @param keyManagementServiceClient Key Management Service Client
*/
@Override
protected void validate(final KeyManagementServiceClient keyManagementServiceClient) {
if (keyManagementServiceClient == null) {
logger.debug("GCP KMS Client not configured");
} else {
final String project = getProperties().getProperty(PROJECT_PROPERTY);
final String location = getProperties().getProperty(LOCATION_PROPERTY);
final String keyring = getProperties().getProperty(KEYRING_PROPERTY);
final String key = getProperties().getProperty(KEY_PROPERTY);
if (StringUtils.isNoneBlank(project, location, keyring, key)) {
cryptoKeyName = CryptoKeyName.of(project, location, keyring, key);
try {
final CryptoKey cryptoKey = keyManagementServiceClient.getCryptoKey(cryptoKeyName);
final CryptoKeyVersion cryptoKeyVersion = cryptoKey.getPrimary();
if (CryptoKeyVersion.CryptoKeyVersionState.ENABLED == cryptoKeyVersion.getState()) {
logger.info("GCP KMS Crypto Key [{}] Validated", cryptoKeyName);
} else {
throw new SensitivePropertyProtectionException(String.format("GCP KMS Crypto Key [%s] Disabled", cryptoKeyName));
}
} catch (final ApiException e) {
throw new SensitivePropertyProtectionException(String.format("GCP KMS Crypto Key [%s] Validation Failed", cryptoKeyName), e);
}
} else {
throw new SensitivePropertyProtectionException("GCP KMS Missing Required Properties");
}
}
}
/**
* Get encrypted bytes
*
* @param bytes Unprotected bytes
* @return Encrypted bytes
*/
@Override
protected byte[] getEncrypted(final byte[] bytes) {
final EncryptResponse encryptResponse = getClient().encrypt(cryptoKeyName, ByteString.copyFrom(bytes));
return encryptResponse.getCiphertext().toByteArray();
}
/**
* Get decrypted bytes
*
* @param bytes Encrypted bytes
* @return Decrypted bytes
*/
@Override
protected byte[] getDecrypted(final byte[] bytes) {
final DecryptResponse decryptResponse = getClient().decrypt(cryptoKeyName, ByteString.copyFrom(bytes));
return decryptResponse.getPlaintext().toByteArray();
}
}

35
nifi-commons/nifi-sensitive-property-provider/src/main/java/org/apache/nifi/properties/StandardSensitivePropertyProviderFactory.java
View File

@ -16,11 +16,18 @@
*/
package org.apache.nifi.properties;
import com.azure.security.keyvault.keys.cryptography.CryptographyClient;
import com.google.cloud.kms.v1.KeyManagementServiceClient;
import org.apache.nifi.properties.BootstrapProperties.BootstrapPropertyKey;
import org.apache.nifi.properties.configuration.AwsKmsClientProvider;
import org.apache.nifi.properties.configuration.AzureCryptographyClientProvider;
import org.apache.nifi.properties.configuration.ClientProvider;
import org.apache.nifi.properties.configuration.GoogleKeyManagementServiceClientProvider;
import org.apache.nifi.util.NiFiBootstrapUtils;
import org.apache.nifi.util.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import software.amazon.awssdk.services.kms.KmsClient;
import java.io.IOException;
import java.util.Arrays;
@ -29,6 +36,7 @@ import java.util.HashMap;
import java.util.Map;
import java.util.Objects;
import java.util.Optional;
import java.util.Properties;
import java.util.function.Supplier;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
@ -106,7 +114,7 @@ public class StandardSensitivePropertyProviderFactory implements SensitiveProper
try {
return NiFiBootstrapUtils.loadBootstrapProperties();
} catch (final IOException e) {
logger.debug("Could not load bootstrap.conf from disk, so using empty bootstrap.conf", e);
logger.debug("Bootstrap Properties loading failed", e);
return BootstrapProperties.EMPTY;
}
});
@ -121,11 +129,26 @@ public class StandardSensitivePropertyProviderFactory implements SensitiveProper
case AES_GCM:
return providerMap.computeIfAbsent(protectionScheme, s -> new AESSensitivePropertyProvider(keyHex));
case AWS_KMS:
return providerMap.computeIfAbsent(protectionScheme, s -> new AWSKMSSensitivePropertyProvider(getBootstrapProperties()));
return providerMap.computeIfAbsent(protectionScheme, s -> {
final AwsKmsClientProvider clientProvider = new AwsKmsClientProvider();
final Properties clientProperties = getClientProperties(clientProvider);
final Optional<KmsClient> kmsClient = clientProvider.getClient(clientProperties);
return new AwsKmsSensitivePropertyProvider(kmsClient.orElse(null), clientProperties);
});
case AZURE_KEYVAULT_KEY:
return providerMap.computeIfAbsent(protectionScheme, s -> new AzureKeyVaultKeySensitivePropertyProvider(getBootstrapProperties()));
return providerMap.computeIfAbsent(protectionScheme, s -> {
final AzureCryptographyClientProvider clientProvider = new AzureCryptographyClientProvider();
final Properties clientProperties = getClientProperties(clientProvider);
final Optional<CryptographyClient> cryptographyClient = clientProvider.getClient(clientProperties);
return new AzureKeyVaultKeySensitivePropertyProvider(cryptographyClient.orElse(null), clientProperties);
});
case GCP_KMS:
return providerMap.computeIfAbsent(protectionScheme, s -> new GCPKMSSensitivePropertyProvider(getBootstrapProperties()));
return providerMap.computeIfAbsent(protectionScheme, s -> {
final GoogleKeyManagementServiceClientProvider clientProvider = new GoogleKeyManagementServiceClientProvider();
final Properties clientProperties = getClientProperties(clientProvider);
final Optional<KeyManagementServiceClient> keyManagementServiceClient = clientProvider.getClient(clientProperties);
return new GcpKmsSensitivePropertyProvider(keyManagementServiceClient.orElse(null), clientProperties);
});
case HASHICORP_VAULT_TRANSIT:
return providerMap.computeIfAbsent(protectionScheme, s -> new HashiCorpVaultTransitSensitivePropertyProvider(getBootstrapProperties()));
case HASHICORP_VAULT_KV:
@ -156,4 +179,8 @@ public class StandardSensitivePropertyProviderFactory implements SensitiveProper
return ProtectedPropertyContext.contextFor(propertyName, contextName);
}
private <T> Properties getClientProperties(final ClientProvider<T> clientProvider) {
final Optional<Properties> clientProperties = clientProvider.getClientProperties(getBootstrapProperties());
return clientProperties.orElse(null);
}
}

80
nifi-commons/nifi-sensitive-property-provider/src/main/java/org/apache/nifi/properties/configuration/AwsKmsClientProvider.java Normal file
View File

@ -0,0 +1,80 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.nifi.properties.configuration;
import org.apache.commons.lang3.StringUtils;
import org.apache.nifi.properties.BootstrapProperties;
import org.apache.nifi.properties.SensitivePropertyProtectionException;
import software.amazon.awssdk.auth.credentials.AwsBasicCredentials;
import software.amazon.awssdk.auth.credentials.DefaultCredentialsProvider;
import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider;
import software.amazon.awssdk.regions.Region;
import software.amazon.awssdk.services.kms.KmsClient;
import software.amazon.awssdk.services.kms.KmsClientBuilder;
import java.util.Properties;
/**
* Amazon Web Services Key Management Service Client Provider
*/
public class AwsKmsClientProvider extends BootstrapPropertiesClientProvider<KmsClient> {
private static final String ACCESS_KEY_PROPS_NAME = "aws.access.key.id";
private static final String SECRET_KEY_PROPS_NAME = "aws.secret.access.key";
private static final String REGION_KEY_PROPS_NAME = "aws.region";
public AwsKmsClientProvider() {
super(BootstrapProperties.BootstrapPropertyKey.AWS_KMS_SENSITIVE_PROPERTY_PROVIDER_CONF);
}
/**
* Get Configured Client using either Client Properties or AWS Default Credentials Provider
*
* @param clientProperties Client Properties
* @return KMS Client
*/
@Override
protected KmsClient getConfiguredClient(final Properties clientProperties) {
final String accessKey = clientProperties.getProperty(ACCESS_KEY_PROPS_NAME);
final String secretKey = clientProperties.getProperty(SECRET_KEY_PROPS_NAME);
final String region = clientProperties.getProperty(REGION_KEY_PROPS_NAME);
final KmsClientBuilder kmsClientBuilder = KmsClient.builder();
if (StringUtils.isNoneBlank(accessKey, secretKey, region)) {
logger.debug("AWS Credentials Location: Client Properties");
try {
final AwsBasicCredentials credentials = AwsBasicCredentials.create(accessKey, secretKey);
return kmsClientBuilder
.region(Region.of(region))
.credentialsProvider(StaticCredentialsProvider.create(credentials))
.build();
} catch (final RuntimeException e) {
throw new SensitivePropertyProtectionException("AWS KMS Client Builder Failed using Client Properties", e);
}
} else {
logger.debug("AWS Credentials Location: Default Credentials Provider");
try {
final DefaultCredentialsProvider credentialsProvider = DefaultCredentialsProvider.builder().build();
return kmsClientBuilder.credentialsProvider(credentialsProvider).build();
} catch (final RuntimeException e) {
throw new SensitivePropertyProtectionException("AWS KMS Client Builder Failed using Default Credentials Provider", e);
}
}
}
}

57
nifi-commons/nifi-sensitive-property-provider/src/main/java/org/apache/nifi/properties/configuration/AzureCryptographyClientProvider.java Normal file
View File

@ -0,0 +1,57 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.nifi.properties.configuration;
import com.azure.identity.DefaultAzureCredentialBuilder;
import com.azure.security.keyvault.keys.cryptography.CryptographyClient;
import com.azure.security.keyvault.keys.cryptography.CryptographyClientBuilder;
import org.apache.nifi.properties.BootstrapProperties;
import org.apache.nifi.properties.SensitivePropertyProtectionException;
import java.util.Properties;
/**
* Microsoft Azure Cryptography Client Provider
*/
public class AzureCryptographyClientProvider extends BootstrapPropertiesClientProvider<CryptographyClient> {
private static final String KEY_ID_PROPERTY = "azure.keyvault.key.id";
public AzureCryptographyClientProvider() {
super(BootstrapProperties.BootstrapPropertyKey.AZURE_KEYVAULT_SENSITIVE_PROPERTY_PROVIDER_CONF);
}
/**
* Get Configured Client using Default Azure Credentials Builder and configured Key Identifier
*
* @param clientProperties Client Properties
* @return Cryptography Client
*/
@Override
protected CryptographyClient getConfiguredClient(final Properties clientProperties) {
final String keyIdentifier = clientProperties.getProperty(KEY_ID_PROPERTY);
logger.debug("Azure Cryptography Client with Key Identifier [{}]", keyIdentifier);
try {
return new CryptographyClientBuilder()
.credential(new DefaultAzureCredentialBuilder().build())
.keyIdentifier(keyIdentifier)
.buildClient();
} catch (final RuntimeException e) {
throw new SensitivePropertyProtectionException("Azure Cryptography Builder Client Failed using Default Credentials", e);
}
}
}

99
nifi-commons/nifi-sensitive-property-provider/src/main/java/org/apache/nifi/properties/configuration/BootstrapPropertiesClientProvider.java Normal file
View File

@ -0,0 +1,99 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.nifi.properties.configuration;
import org.apache.commons.lang3.StringUtils;
import org.apache.nifi.properties.BootstrapProperties;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.io.InputStream;
import java.io.UncheckedIOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.Objects;
import java.util.Optional;
import java.util.Properties;
/**
* Shared Client Provider for reading Client Properties from file referenced in configured Bootstrap Property Key
*
* @param <T> Client Type
*/
public abstract class BootstrapPropertiesClientProvider<T> implements ClientProvider<T> {
protected final Logger logger = LoggerFactory.getLogger(getClass());
private final BootstrapProperties.BootstrapPropertyKey bootstrapPropertyKey;
public BootstrapPropertiesClientProvider(final BootstrapProperties.BootstrapPropertyKey bootstrapPropertyKey) {
this.bootstrapPropertyKey = Objects.requireNonNull(bootstrapPropertyKey, "Bootstrap Property Key required");
}
/**
* Get Client using Client Properties
*
* @param clientProperties Client Properties can be null
* @return Configured Client or empty when Client Properties object is null
*/
@Override
public Optional<T> getClient(final Properties clientProperties) {
return clientProperties == null ? Optional.empty() : Optional.of(getConfiguredClient(clientProperties));
}
/**
* Get Client Properties from file referenced in Bootstrap Properties
*
* @param bootstrapProperties Bootstrap Properties
* @return Client Properties or empty when not configured
*/
@Override
public Optional<Properties> getClientProperties(final BootstrapProperties bootstrapProperties) {
Objects.requireNonNull(bootstrapProperties, "Bootstrap Properties required");
final String clientBootstrapPropertiesPath = bootstrapProperties.getProperty(bootstrapPropertyKey).orElse(null);
if (StringUtils.isBlank(clientBootstrapPropertiesPath)) {
logger.debug("Client Properties [{}] not configured", bootstrapPropertyKey);
return Optional.empty();
} else {
final Path propertiesPath = Paths.get(clientBootstrapPropertiesPath);
if (Files.exists(propertiesPath)) {
try {
final Properties clientProperties = new Properties();
try (final InputStream inputStream = Files.newInputStream(propertiesPath)) {
clientProperties.load(inputStream);
}
return Optional.of(clientProperties);
} catch (final IOException e) {
final String message = String.format("Loading Client Properties Failed [%s]", propertiesPath);
throw new UncheckedIOException(message, e);
}
} else {
logger.debug("Client Properties [{}] Path [{}] not found", bootstrapPropertyKey, propertiesPath);
return Optional.empty();
}
}
}
/**
* Get Configured Client using Client Properties
*
* @param clientProperties Client Properties
* @return Configured Client
*/
protected abstract T getConfiguredClient(final Properties clientProperties);
}

45
nifi-commons/nifi-sensitive-property-provider/src/main/java/org/apache/nifi/properties/configuration/ClientProvider.java Normal file
View File

@ -0,0 +1,45 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.nifi.properties.configuration;
import org.apache.nifi.properties.BootstrapProperties;
import java.util.Optional;
import java.util.Properties;
/**
* Client Provider responsible for reading Client Properties and instantiating client services
*
* @param <T> Client Type
*/
public interface ClientProvider<T> {
/**
* Get Client Properties from Bootstrap Properties
*
* @param bootstrapProperties Bootstrap Properties
* @return Client Properties or empty when not configured
*/
Optional<Properties> getClientProperties(BootstrapProperties bootstrapProperties);
/**
* Get Client using Client Properties
*
* @param properties Client Properties
* @return Client or empty when not configured
*/
Optional<T> getClient(Properties properties);
}

49
nifi-commons/nifi-sensitive-property-provider/src/main/java/org/apache/nifi/properties/configuration/GoogleKeyManagementServiceClientProvider.java Normal file
View File

@ -0,0 +1,49 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.nifi.properties.configuration;
import com.google.cloud.kms.v1.KeyManagementServiceClient;
import org.apache.nifi.properties.BootstrapProperties;
import org.apache.nifi.properties.SensitivePropertyProtectionException;
import java.io.IOException;
import java.util.Properties;
/**
* Google Key Management Service Client Provider
*/
public class GoogleKeyManagementServiceClientProvider extends BootstrapPropertiesClientProvider<KeyManagementServiceClient> {
public GoogleKeyManagementServiceClientProvider() {
super(BootstrapProperties.BootstrapPropertyKey.GCP_KMS_SENSITIVE_PROPERTY_PROVIDER_CONF);
}
/**
* Get Configured Client using default Key Management Service Client settings
*
* @param clientProperties Client Properties
* @return Key Management Service Client
*/
@Override
protected KeyManagementServiceClient getConfiguredClient(final Properties clientProperties) {
try {
return KeyManagementServiceClient.create();
} catch (final IOException e) {
throw new SensitivePropertyProtectionException("Google Key Management Service Create Failed", e);
}
}
}

13
nifi-commons/nifi-sensitive-property-provider/src/test/java/org/apache/nifi/properties/AWSKMSSensitivePropertyProviderIT.java → nifi-commons/nifi-sensitive-property-provider/src/test/java/org/apache/nifi/properties/AwsKmsSensitivePropertyProviderIT.java
View File

@ -16,6 +16,7 @@
*/
package org.apache.nifi.properties;
import org.apache.nifi.properties.configuration.AwsKmsClientProvider;
import org.junit.AfterClass;
import org.junit.Assert;
import org.junit.BeforeClass;
@ -23,6 +24,7 @@ import org.junit.Test;
import org.mockito.internal.util.io.IOUtil;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import software.amazon.awssdk.services.kms.KmsClient;
import java.io.IOException;
import java.io.InputStream;
@ -52,7 +54,7 @@ import java.util.Properties;
*
*/
public class AWSKMSSensitivePropertyProviderIT {
public class AwsKmsSensitivePropertyProviderIT {
private static final String SAMPLE_PLAINTEXT = "AWSKMSSensitivePropertyProviderIT SAMPLE-PLAINTEXT";
private static final String ACCESS_KEY_PROPS_NAME = "aws.access.key.id";
private static final String SECRET_KEY_PROPS_NAME = "aws.secret.access.key";
@ -63,13 +65,13 @@ public class AWSKMSSensitivePropertyProviderIT {
private static final String EMPTY_PROPERTY = "";
private static AWSKMSSensitivePropertyProvider spp;
private static AwsKmsSensitivePropertyProvider spp;
private static BootstrapProperties props;
private static Path mockBootstrapConf, mockAWSBootstrapConf;
private static final Logger logger = LoggerFactory.getLogger(AWSKMSSensitivePropertyProviderIT.class);
private static final Logger logger = LoggerFactory.getLogger(AwsKmsSensitivePropertyProviderIT.class);
private static void initializeBootstrapProperties() throws IOException{
mockBootstrapConf = Files.createTempFile("bootstrap", ".conf").toAbsolutePath();
@ -101,7 +103,10 @@ public class AWSKMSSensitivePropertyProviderIT {
public static void initOnce() throws IOException {
initializeBootstrapProperties();
Assert.assertNotNull(props);
spp = new AWSKMSSensitivePropertyProvider(props);
final AwsKmsClientProvider provider = new AwsKmsClientProvider();
final Properties properties = provider.getClientProperties(props).orElse(null);
final KmsClient kmsClient = provider.getClient(properties).orElse(null);
spp = new AwsKmsSensitivePropertyProvider(kmsClient, properties);
Assert.assertNotNull(spp);
}

119
nifi-commons/nifi-sensitive-property-provider/src/test/java/org/apache/nifi/properties/AwsKmsSensitivePropertyProviderTest.java Normal file
View File

@ -0,0 +1,119 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.nifi.properties;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.mockito.Mock;
import org.mockito.junit.jupiter.MockitoExtension;
import software.amazon.awssdk.core.SdkBytes;
import software.amazon.awssdk.services.kms.KmsClient;
import software.amazon.awssdk.services.kms.model.DecryptRequest;
import software.amazon.awssdk.services.kms.model.DecryptResponse;
import software.amazon.awssdk.services.kms.model.DescribeKeyRequest;
import software.amazon.awssdk.services.kms.model.DescribeKeyResponse;
import software.amazon.awssdk.services.kms.model.EncryptRequest;
import software.amazon.awssdk.services.kms.model.EncryptResponse;
import software.amazon.awssdk.services.kms.model.KeyMetadata;
import java.nio.charset.StandardCharsets;
import java.util.Base64;
import java.util.Properties;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
import static org.mockito.ArgumentMatchers.any;
@ExtendWith(MockitoExtension.class)
public class AwsKmsSensitivePropertyProviderTest {
private static final String PROPERTY_NAME = String.class.getSimpleName();
private static final String PROPERTY = String.class.getName();
private static final String ENCRYPTED_PROPERTY = Integer.class.getName();
private static final byte[] ENCRYPTED_BYTES = ENCRYPTED_PROPERTY.getBytes(StandardCharsets.UTF_8);
private static final String PROTECTED_PROPERTY = Base64.getEncoder().withoutPadding().encodeToString(ENCRYPTED_BYTES);
private static final String KEY_ID = AwsKmsSensitivePropertyProvider.class.getSimpleName();
private static final Properties PROPERTIES = new Properties();
static {
PROPERTIES.setProperty(AwsKmsSensitivePropertyProvider.KEY_ID_PROPERTY, KEY_ID);
}
@Mock
private KmsClient kmsClient;
private AwsKmsSensitivePropertyProvider provider;
@BeforeEach
public void setProvider() {
final KeyMetadata keyMetadata = KeyMetadata.builder().enabled(true).build();
final DescribeKeyResponse describeKeyResponse = DescribeKeyResponse.builder().keyMetadata(keyMetadata).build();
when(kmsClient.describeKey(any(DescribeKeyRequest.class))).thenReturn(describeKeyResponse);
provider = new AwsKmsSensitivePropertyProvider(kmsClient, PROPERTIES);
}
@Test
public void testValidateClientNull() {
final AwsKmsSensitivePropertyProvider provider = new AwsKmsSensitivePropertyProvider(null, PROPERTIES);
assertNotNull(provider);
}
@Test
public void testValidateKeyDisabled() {
final KeyMetadata keyMetadata = KeyMetadata.builder().enabled(false).build();
final DescribeKeyResponse describeKeyResponse = DescribeKeyResponse.builder().keyMetadata(keyMetadata).build();
when(kmsClient.describeKey(any(DescribeKeyRequest.class))).thenReturn(describeKeyResponse);
assertThrows(SensitivePropertyProtectionException.class, () -> new AwsKmsSensitivePropertyProvider(kmsClient, PROPERTIES));
}
@Test
public void testCleanUp() {
provider.cleanUp();
verify(kmsClient).close();
}
@Test
public void testProtect() {
final SdkBytes blob = SdkBytes.fromUtf8String(ENCRYPTED_PROPERTY);
final EncryptResponse encryptResponse = EncryptResponse.builder().ciphertextBlob(blob).build();
when(kmsClient.encrypt(any(EncryptRequest.class))).thenReturn(encryptResponse);
final String protectedProperty = provider.protect(PROPERTY, ProtectedPropertyContext.defaultContext(PROPERTY_NAME));
assertEquals(PROTECTED_PROPERTY, protectedProperty);
}
@Test
public void testUnprotect() {
final SdkBytes blob = SdkBytes.fromUtf8String(PROPERTY);
final DecryptResponse decryptResponse = DecryptResponse.builder().plaintext(blob).build();
when(kmsClient.decrypt(any(DecryptRequest.class))).thenReturn(decryptResponse);
final String property = provider.unprotect(PROTECTED_PROPERTY, ProtectedPropertyContext.defaultContext(PROPERTY_NAME));
assertEquals(PROPERTY, property);
}
}

9
nifi-commons/nifi-sensitive-property-provider/src/test/java/org/apache/nifi/properties/AzureKeyVaultKeySensitivePropertyProviderIT.java
View File

@ -16,6 +16,8 @@
*/
package org.apache.nifi.properties;
import com.azure.security.keyvault.keys.cryptography.CryptographyClient;
import org.apache.nifi.properties.configuration.AzureCryptographyClientProvider;
import org.junit.AfterClass;
import org.junit.Assert;
import org.junit.BeforeClass;
@ -94,7 +96,10 @@ public class AzureKeyVaultKeySensitivePropertyProviderIT {
public static void initOnce() throws IOException {
initializeBootstrapProperties();
Assert.assertNotNull(props);
spp = new AzureKeyVaultKeySensitivePropertyProvider(props);
final AzureCryptographyClientProvider provider = new AzureCryptographyClientProvider();
final Properties properties = provider.getClientProperties(props).orElse(null);
final CryptographyClient cryptographyClient = provider.getClient(properties).orElse(null);
spp = new AzureKeyVaultKeySensitivePropertyProvider(cryptographyClient, properties);
Assert.assertNotNull(spp);
}
@ -109,7 +114,7 @@ public class AzureKeyVaultKeySensitivePropertyProviderIT {
@Test
public void testEncryptDecrypt() {
logger.info("Running testEncryptDecrypt of Azure Key Vault Key SPP integration test");
this.runEncryptDecryptTest();
runEncryptDecryptTest();
logger.info("testEncryptDecrypt of Azure Key Vault Key SPP integration test completed");
}

110
nifi-commons/nifi-sensitive-property-provider/src/test/java/org/apache/nifi/properties/AzureKeyVaultKeySensitivePropertyProviderTest.java Normal file
View File

@ -0,0 +1,110 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.nifi.properties;
import com.azure.security.keyvault.keys.cryptography.CryptographyClient;
import com.azure.security.keyvault.keys.cryptography.models.DecryptResult;
import com.azure.security.keyvault.keys.cryptography.models.EncryptResult;
import com.azure.security.keyvault.keys.cryptography.models.EncryptionAlgorithm;
import com.azure.security.keyvault.keys.models.KeyProperties;
import com.azure.security.keyvault.keys.models.KeyVaultKey;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.mockito.Mock;
import org.mockito.junit.jupiter.MockitoExtension;
import java.nio.charset.StandardCharsets;
import java.util.Base64;
import java.util.Properties;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.Mockito.when;
@ExtendWith(MockitoExtension.class)
public class AzureKeyVaultKeySensitivePropertyProviderTest {
private static final String PROPERTY_NAME = String.class.getSimpleName();
private static final String PROPERTY = String.class.getName();
private static final byte[] PROPERTY_BYTES = PROPERTY.getBytes(StandardCharsets.UTF_8);
private static final String ENCRYPTED_PROPERTY = Integer.class.getName();
private static final byte[] ENCRYPTED_BYTES = ENCRYPTED_PROPERTY.getBytes(StandardCharsets.UTF_8);
private static final String PROTECTED_PROPERTY = Base64.getEncoder().withoutPadding().encodeToString(ENCRYPTED_BYTES);
private static final String ID = KeyVaultKey.class.getSimpleName();
private static final Properties PROPERTIES = new Properties();
private static final EncryptionAlgorithm ALGORITHM = EncryptionAlgorithm.A256GCM;
static {
PROPERTIES.setProperty(AzureKeyVaultKeySensitivePropertyProvider.ENCRYPTION_ALGORITHM_PROPERTY, ALGORITHM.toString());
}
@Mock
private CryptographyClient cryptographyClient;
@Mock
private KeyVaultKey keyVaultKey;
@Mock
private KeyProperties keyProperties;
private AzureKeyVaultKeySensitivePropertyProvider provider;
@BeforeEach
public void setProvider() {
when(keyProperties.isEnabled()).thenReturn(true);
when(keyVaultKey.getId()).thenReturn(ID);
when(keyVaultKey.getProperties()).thenReturn(keyProperties);
when(keyVaultKey.getKeyOperations()).thenReturn(AzureKeyVaultKeySensitivePropertyProvider.REQUIRED_OPERATIONS);
when(cryptographyClient.getKey()).thenReturn(keyVaultKey);
provider = new AzureKeyVaultKeySensitivePropertyProvider(cryptographyClient, PROPERTIES);
}
@Test
public void testValidateClientNull() {
final AzureKeyVaultKeySensitivePropertyProvider provider = new AzureKeyVaultKeySensitivePropertyProvider(null, PROPERTIES);
assertNotNull(provider);
}
@Test
public void testProtect() {
final EncryptResult encryptResult = new EncryptResult(ENCRYPTED_BYTES, ALGORITHM, ID);
when(cryptographyClient.encrypt(eq(ALGORITHM), any(byte[].class))).thenReturn(encryptResult);
final String protectedProperty = provider.protect(PROPERTY, ProtectedPropertyContext.defaultContext(PROPERTY_NAME));
assertEquals(PROTECTED_PROPERTY, protectedProperty);
}
@Test
public void testUnprotect() {
final DecryptResult decryptResult = new DecryptResult(PROPERTY_BYTES, ALGORITHM, ID);
when(cryptographyClient.decrypt(eq(ALGORITHM), any(byte[].class))).thenReturn(decryptResult);
final String property = provider.unprotect(PROTECTED_PROPERTY, ProtectedPropertyContext.defaultContext(PROPERTY_NAME));
assertEquals(PROPERTY, property);
}
}

23
nifi-commons/nifi-sensitive-property-provider/src/test/java/org/apache/nifi/properties/GCPKMSSensitivePropertyProviderIT.java → nifi-commons/nifi-sensitive-property-provider/src/test/java/org/apache/nifi/properties/GcpKmsSensitivePropertyProviderIT.java
View File

@ -16,6 +16,8 @@
*/
package org.apache.nifi.properties;
import com.google.cloud.kms.v1.KeyManagementServiceClient;
import org.apache.nifi.properties.configuration.GoogleKeyManagementServiceClientProvider;
import org.junit.AfterClass;
import org.junit.Assert;
import org.junit.BeforeClass;
@ -46,7 +48,7 @@ import java.util.Properties;
* when running the integration tests
*/
public class GCPKMSSensitivePropertyProviderIT {
public class GcpKmsSensitivePropertyProviderIT {
private static final String SAMPLE_PLAINTEXT = "GCPKMSSensitivePropertyProviderIT SAMPLE-PLAINTEXT";
private static final String PROJECT_ID_PROPS_NAME = "gcp.kms.project";
private static final String LOCATION_ID_PROPS_NAME = "gcp.kms.location";
@ -56,13 +58,13 @@ public class GCPKMSSensitivePropertyProviderIT {
private static final String EMPTY_PROPERTY = "";
private static GCPKMSSensitivePropertyProvider spp;
private static GcpKmsSensitivePropertyProvider spp;
private static BootstrapProperties props;
private static Path mockBootstrapConf, mockGCPBootstrapConf;
private static final Logger logger = LoggerFactory.getLogger(GCPKMSSensitivePropertyProviderIT.class);
private static final Logger logger = LoggerFactory.getLogger(GcpKmsSensitivePropertyProviderIT.class);
private static void initializeBootstrapProperties() throws IOException{
mockBootstrapConf = Files.createTempFile("bootstrap", ".conf").toAbsolutePath();
@ -81,11 +83,11 @@ public class GCPKMSSensitivePropertyProviderIT {
String keyId = System.getProperty(KEY_ID_PROPS_NAME, EMPTY_PROPERTY);
StringBuilder bootstrapConfText = new StringBuilder();
String lineSeparator = System.getProperty("line.separator");
bootstrapConfText.append(PROJECT_ID_PROPS_NAME + "=" + projectId);
bootstrapConfText.append(lineSeparator + LOCATION_ID_PROPS_NAME + "=" + locationId);
bootstrapConfText.append(lineSeparator + KEYRING_ID_PROPS_NAME + "=" + keyringId);
bootstrapConfText.append(lineSeparator + KEY_ID_PROPS_NAME + "=" + keyId);
String lineSeparator = System.lineSeparator();
bootstrapConfText.append(PROJECT_ID_PROPS_NAME).append("=").append(projectId).append(lineSeparator);
bootstrapConfText.append(LOCATION_ID_PROPS_NAME).append("=").append(locationId).append(lineSeparator);
bootstrapConfText.append(KEYRING_ID_PROPS_NAME).append("=").append(keyringId).append(lineSeparator);
bootstrapConfText.append(KEY_ID_PROPS_NAME).append("=").append(keyId).append(lineSeparator);
IOUtil.writeText(bootstrapConfText.toString(), mockGCPBootstrapConf.toFile());
}
@ -93,7 +95,10 @@ public class GCPKMSSensitivePropertyProviderIT {
public static void initOnce() throws IOException {
initializeBootstrapProperties();
Assert.assertNotNull(props);
spp = new GCPKMSSensitivePropertyProvider(props);
final GoogleKeyManagementServiceClientProvider provider = new GoogleKeyManagementServiceClientProvider();
final Properties clientProperties = provider.getClientProperties(props).orElse(null);
final KeyManagementServiceClient client = provider.getClient(clientProperties).orElse(null);
spp = new GcpKmsSensitivePropertyProvider(client, clientProperties);
Assert.assertNotNull(spp);
}

18
nifi-commons/nifi-sensitive-property-provider/src/test/java/org/apache/nifi/properties/StandardSensitivePropertyProviderFactoryTest.java
View File

@ -56,8 +56,6 @@ public class StandardSensitivePropertyProviderFactoryTest {
private static Path nifiProperties;
private static String defaultBootstrapContents;
private static NiFiProperties niFiProperties;
@BeforeClass
public static void initOnce() throws IOException {
Security.addProvider(new BouncyCastleProvider());
@ -74,8 +72,6 @@ public class StandardSensitivePropertyProviderFactoryTest {
"nifi.bootstrap.protection.hashicorp.vault.conf", FilenameUtils.separatorsToUnix(hashicorpVaultBootstrapConf.toString()));
bootstrapConf = writeDefaultBootstrapConf();
System.setProperty(NiFiProperties.PROPERTIES_FILE_PATH, FilenameUtils.separatorsToUnix(nifiProperties.toString()));
niFiProperties = new NiFiProperties();
}
private static Path writeDefaultBootstrapConf() throws IOException {
@ -86,8 +82,7 @@ public class StandardSensitivePropertyProviderFactoryTest {
final Path tempBootstrapConf = Files.createTempFile("bootstrap", ".conf").toAbsolutePath();
final Path bootstrapConf = Files.move(tempBootstrapConf, tempConfDir.resolve("bootstrap.conf"), StandardCopyOption.REPLACE_EXISTING);
final String bootstrapConfText = String.format(contents);
IOUtil.writeText(bootstrapConfText, bootstrapConf.toFile());
IOUtil.writeText(contents, bootstrapConf.toFile());
return bootstrapConf;
}
@ -137,10 +132,9 @@ public class StandardSensitivePropertyProviderFactoryTest {
}
@Test
public void testGetPropertyContextUnconfigured() {
public void testGetPropertyContextNotConfigured() {
configureDefaultFactory();
assertEquals("default/prop", factory.getPropertyContext("ldap-provider", "prop").getContextKey());
}
@Test
@ -163,11 +157,11 @@ public class StandardSensitivePropertyProviderFactoryTest {
properties.put("vault.transit.path", "nifi-transit");
configureHashicorpVault(properties);
final SensitivePropertyProvider spp = factory.getProvider(PropertyProtectionScheme.HASHICORP_VAULT_TRANSIT);
factory.getProvider(PropertyProtectionScheme.HASHICORP_VAULT_TRANSIT);
}
@Test
public void testHashicorpVaultTransit_isSupported() throws IOException {
public void testHashicorpVaultTransitSupported() throws IOException {
configureDefaultFactory();
final Properties properties = new Properties();
properties.put("vault.transit.path", "nifi-transit");
@ -191,7 +185,7 @@ public class StandardSensitivePropertyProviderFactoryTest {
}
@Test
public void testHashicorpVaultTransit_invalidCharacters() throws IOException {
public void testHashicorpVaultTransitInvalidCharacters() throws IOException {
configureDefaultFactory();
final Properties properties = new Properties();
properties.put("vault.transit.path", "invalid/characters");
@ -201,7 +195,7 @@ public class StandardSensitivePropertyProviderFactoryTest {
}
@Test
public void testAES_GCM() throws IOException {
public void testAesGcm() throws IOException {
configureDefaultFactory();
final ProtectedPropertyContext context = ProtectedPropertyContext.defaultContext("propertyName");

2
nifi-docs/src/main/asciidoc/administration-guide.adoc
View File

@ -1795,7 +1795,7 @@ If no administrator action is taken, the configuration values remain unencrypted
For more information, see the <<toolkit-guide.adoc#encrypt_config_tool,Encrypt-Config Tool>> section in the link:toolkit-guide.html[NiFi Toolkit Guide].
In addition to the default AES encryption provider, other providers can be configured in their respective `bootstrap-*.conf` files. Following is a list of additional encryption providers and their configuration:
Configuring each Sensitive Property Provider requires including the appropriate file reference property in `bootstrap.conf`. The default `bootstrap.conf` includes commented file reference properties for available providers.
=== HashiCorp Vault providers
Two encryption providers are currently configurable in the `bootstrap-hashicorp-vault.conf` file:

8
nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-resources/src/main/resources/conf/bootstrap.conf
View File

@ -61,16 +61,16 @@ nifi.bootstrap.sensitive.key=
# Sensitive Property Provider configuration
# HashiCorp Vault Sensitive Property Providers
nifi.bootstrap.protection.hashicorp.vault.conf=./conf/bootstrap-hashicorp-vault.conf
#nifi.bootstrap.protection.hashicorp.vault.conf=./conf/bootstrap-hashicorp-vault.conf
# AWS KMS Sensitive Property Providers
nifi.bootstrap.protection.aws.kms.conf=./conf/bootstrap-aws.conf
#nifi.bootstrap.protection.aws.kms.conf=./conf/bootstrap-aws.conf
# Azure Key Vault Sensitive Property Providers
nifi.bootstrap.protection.azure.keyvault.conf=./conf/bootstrap-azure.conf
#nifi.bootstrap.protection.azure.keyvault.conf=./conf/bootstrap-azure.conf
# GCP KMS Sensitive Property Providers
nifi.bootstrap.protection.gcp.kms.conf=./conf/bootstrap-gcp.conf
#nifi.bootstrap.protection.gcp.kms.conf=./conf/bootstrap-gcp.conf
# Sets the provider of SecureRandom to /dev/urandom to prevent blocking on VMs
java.arg.15=-Djava.security.egd=file:/dev/urandom

8
nifi-registry/nifi-registry-core/nifi-registry-resources/src/main/resources/conf/bootstrap.conf
View File

@ -56,13 +56,13 @@ nifi.registry.bootstrap.sensitive.key=
# Sensitive Property Provider configuration
# HashiCorp Vault Sensitive Property Providers
nifi.registry.bootstrap.protection.hashicorp.vault.conf=./conf/bootstrap-hashicorp-vault.conf
#nifi.registry.bootstrap.protection.hashicorp.vault.conf=./conf/bootstrap-hashicorp-vault.conf
# AWS KMS Sensitive Property Providers
nifi.registry.bootstrap.protection.aws.kms.conf=./conf/bootstrap-aws.conf
#nifi.registry.bootstrap.protection.aws.kms.conf=./conf/bootstrap-aws.conf
# Azure Key Vault Sensitive Property Providers
nifi.registry.bootstrap.protection.azure.keyvault.conf=./conf/bootstrap-azure.conf
#nifi.registry.bootstrap.protection.azure.keyvault.conf=./conf/bootstrap-azure.conf
# GCP KMS Sensitive Property Providers
nifi.registry.bootstrap.protection.gcp.kms.conf=./conf/bootstrap-gcp.conf
#nifi.registry.bootstrap.protection.gcp.kms.conf=./conf/bootstrap-gcp.conf