mirror of https://github.com/apache/nifi.git
NIFI-9636: Adding AwsSecretsManagerParameterProvider
This closes #6392 Signed-off-by: David Handermann <exceptionfactory@apache.org>
This commit is contained in:
Joe Gresock
exceptionfactory
parent
fa193ca14b
commit
315e54a812
|
|
@ -41,6 +41,11 @@
|
||||||
<artifactId>nifi-aws-processors</artifactId>
|
<artifactId>nifi-aws-processors</artifactId>
|
||||||
<version>1.18.0-SNAPSHOT</version>
|
<version>1.18.0-SNAPSHOT</version>
|
||||||
</dependency>
|
</dependency>
|
||||||
|
<dependency>
|
||||||
|
<groupId>org.apache.nifi</groupId>
|
||||||
|
<artifactId>nifi-aws-parameter-providers</artifactId>
|
||||||
|
<version>1.18.0-SNAPSHOT</version>
|
||||||
|
</dependency>
|
||||||
<dependency>
|
<dependency>
|
||||||
<groupId>org.apache.nifi</groupId>
|
<groupId>org.apache.nifi</groupId>
|
||||||
<artifactId>nifi-aws-parameter-value-providers</artifactId>
|
<artifactId>nifi-aws-parameter-value-providers</artifactId>
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,75 @@
|
||||||
|
<?xml version="1.0" encoding="UTF-8"?>
|
||||||
|
<!--
|
||||||
|
Licensed to the Apache Software Foundation (ASF) under one or more
|
||||||
|
contributor license agreements. See the NOTICE file distributed with
|
||||||
|
this work for additional information regarding copyright ownership.
|
||||||
|
The ASF licenses this file to You under the Apache License, Version 2.0
|
||||||
|
(the "License"); you may not use this file except in compliance with
|
||||||
|
the License. You may obtain a copy of the License at
|
||||||
|
http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
Unless required by applicable law or agreed to in writing, software
|
||||||
|
distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
See the License for the specific language governing permissions and
|
||||||
|
limitations under the License.
|
||||||
|
-->
|
||||||
|
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
|
||||||
|
<modelVersion>4.0.0</modelVersion>
|
||||||
|
<parent>
|
||||||
|
<groupId>org.apache.nifi</groupId>
|
||||||
|
<artifactId>nifi-aws-bundle</artifactId>
|
||||||
|
<version>1.18.0-SNAPSHOT</version>
|
||||||
|
</parent>
|
||||||
|
|
||||||
|
<artifactId>nifi-aws-parameter-providers</artifactId>
|
||||||
|
<packaging>jar</packaging>
|
||||||
|
|
||||||
|
<dependencies>
|
||||||
|
<dependency>
|
||||||
|
<groupId>org.apache.nifi</groupId>
|
||||||
|
<artifactId>nifi-api</artifactId>
|
||||||
|
</dependency>
|
||||||
|
<dependency>
|
||||||
|
<groupId>org.apache.nifi</groupId>
|
||||||
|
<artifactId>nifi-aws-service-api</artifactId>
|
||||||
|
<version>1.18.0-SNAPSHOT</version>
|
||||||
|
<scope>provided</scope>
|
||||||
|
</dependency>
|
||||||
|
<dependency>
|
||||||
|
<groupId>org.apache.nifi</groupId>
|
||||||
|
<artifactId>nifi-ssl-context-service-api</artifactId>
|
||||||
|
<version>1.18.0-SNAPSHOT</version>
|
||||||
|
<scope>provided</scope>
|
||||||
|
</dependency>
|
||||||
|
<dependency>
|
||||||
|
<groupId>org.apache.nifi</groupId>
|
||||||
|
<artifactId>nifi-utils</artifactId>
|
||||||
|
<version>1.18.0-SNAPSHOT</version>
|
||||||
|
<scope>provided</scope>
|
||||||
|
</dependency>
|
||||||
|
<dependency>
|
||||||
|
<groupId>org.slf4j</groupId>
|
||||||
|
<artifactId>jcl-over-slf4j</artifactId>
|
||||||
|
</dependency>
|
||||||
|
<dependency>
|
||||||
|
<groupId>com.fasterxml.jackson.core</groupId>
|
||||||
|
<artifactId>jackson-databind</artifactId>
|
||||||
|
</dependency>
|
||||||
|
<dependency>
|
||||||
|
<groupId>com.amazonaws</groupId>
|
||||||
|
<artifactId>aws-java-sdk-secretsmanager</artifactId>
|
||||||
|
</dependency>
|
||||||
|
<dependency>
|
||||||
|
<groupId>org.apache.nifi</groupId>
|
||||||
|
<artifactId>nifi-expression-language</artifactId>
|
||||||
|
<version>1.18.0-SNAPSHOT</version>
|
||||||
|
<scope>test</scope>
|
||||||
|
</dependency>
|
||||||
|
<dependency>
|
||||||
|
<groupId>org.apache.nifi</groupId>
|
||||||
|
<artifactId>nifi-mock</artifactId>
|
||||||
|
<version>1.18.0-SNAPSHOT</version>
|
||||||
|
<scope>test</scope>
|
||||||
|
</dependency>
|
||||||
|
</dependencies>
|
||||||
|
</project>
|
||||||
|
|
@ -0,0 +1,297 @@
|
||||||
|
/*
|
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||||
|
* contributor license agreements. See the NOTICE file distributed with
|
||||||
|
* this work for additional information regarding copyright ownership.
|
||||||
|
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||||
|
* (the "License"); you may not use this file except in compliance with
|
||||||
|
* the License. You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
|
package org.apache.nifi.parameter.aws;
|
||||||
|
|
||||||
|
import com.amazonaws.ClientConfiguration;
|
||||||
|
import com.amazonaws.Protocol;
|
||||||
|
import com.amazonaws.auth.AWSCredentialsProvider;
|
||||||
|
import com.amazonaws.http.conn.ssl.SdkTLSSocketFactory;
|
||||||
|
import com.amazonaws.regions.Regions;
|
||||||
|
import com.amazonaws.services.secretsmanager.AWSSecretsManager;
|
||||||
|
import com.amazonaws.services.secretsmanager.AWSSecretsManagerClientBuilder;
|
||||||
|
import com.amazonaws.services.secretsmanager.model.AWSSecretsManagerException;
|
||||||
|
import com.amazonaws.services.secretsmanager.model.GetSecretValueRequest;
|
||||||
|
import com.amazonaws.services.secretsmanager.model.GetSecretValueResult;
|
||||||
|
import com.amazonaws.services.secretsmanager.model.ListSecretsRequest;
|
||||||
|
import com.amazonaws.services.secretsmanager.model.ListSecretsResult;
|
||||||
|
import com.amazonaws.services.secretsmanager.model.ResourceNotFoundException;
|
||||||
|
import com.amazonaws.services.secretsmanager.model.SecretListEntry;
|
||||||
|
import com.fasterxml.jackson.core.JsonProcessingException;
|
||||||
|
import com.fasterxml.jackson.databind.JsonNode;
|
||||||
|
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||||
|
import com.fasterxml.jackson.databind.node.ObjectNode;
|
||||||
|
import org.apache.nifi.annotation.documentation.CapabilityDescription;
|
||||||
|
import org.apache.nifi.annotation.documentation.Tags;
|
||||||
|
import org.apache.nifi.components.AllowableValue;
|
||||||
|
import org.apache.nifi.components.ConfigVerificationResult;
|
||||||
|
import org.apache.nifi.components.PropertyDescriptor;
|
||||||
|
import org.apache.nifi.controller.ConfigurationContext;
|
||||||
|
import org.apache.nifi.logging.ComponentLog;
|
||||||
|
import org.apache.nifi.parameter.AbstractParameterProvider;
|
||||||
|
import org.apache.nifi.parameter.Parameter;
|
||||||
|
import org.apache.nifi.parameter.ParameterDescriptor;
|
||||||
|
import org.apache.nifi.parameter.ParameterGroup;
|
||||||
|
import org.apache.nifi.parameter.VerifiableParameterProvider;
|
||||||
|
import org.apache.nifi.processor.util.StandardValidators;
|
||||||
|
import org.apache.nifi.processors.aws.credentials.provider.service.AWSCredentialsProviderService;
|
||||||
|
import org.apache.nifi.ssl.SSLContextService;
|
||||||
|
|
||||||
|
import javax.net.ssl.SSLContext;
|
||||||
|
import java.util.ArrayList;
|
||||||
|
import java.util.Arrays;
|
||||||
|
import java.util.Collections;
|
||||||
|
import java.util.Iterator;
|
||||||
|
import java.util.List;
|
||||||
|
import java.util.Map;
|
||||||
|
import java.util.concurrent.TimeUnit;
|
||||||
|
import java.util.regex.Pattern;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Reads secrets from AWS Secrets Manager to provide parameter values. Secrets must be created similar to the following AWS cli command: <br/><br/>
|
||||||
|
* <code>aws secretsmanager create-secret --name "[Context]" --secret-string '{ "[Param]": "[secretValue]", "[Param2]": "[secretValue2]" }'</code> <br/><br/>
|
||||||
|
*
|
||||||
|
*/
|
||||||
|
|
||||||
|
@Tags({"aws", "secretsmanager", "secrets", "manager"})
|
||||||
|
@CapabilityDescription("Fetches parameters from AWS SecretsManager. Each secret becomes a Parameter group, which can map to a Parameter Context, with " +
|
||||||
|
"key/value pairs in the secret mapping to Parameters in the group.")
|
||||||
|
public class AwsSecretsManagerParameterProvider extends AbstractParameterProvider implements VerifiableParameterProvider {
|
||||||
|
|
||||||
|
public static final PropertyDescriptor SECRET_NAME_PATTERN = new PropertyDescriptor.Builder()
|
||||||
|
.name("secret-name-pattern")
|
||||||
|
.displayName("Secret Name Pattern")
|
||||||
|
.description("A Regular Expression matching on Secret Name that identifies Secrets whose parameters should be fetched. " +
|
||||||
|
"Any secrets whose names do not match this pattern will not be fetched.")
|
||||||
|
.addValidator(StandardValidators.REGULAR_EXPRESSION_VALIDATOR)
|
||||||
|
.required(true)
|
||||||
|
.defaultValue(".*")
|
||||||
|
.build();
|
||||||
|
/**
|
||||||
|
* AWS credentials provider service
|
||||||
|
*
|
||||||
|
* @see <a href="http://docs.aws.amazon.com/AWSJavaSDK/latest/javadoc/com/amazonaws/auth/AWSCredentialsProvider.html">AWSCredentialsProvider</a>
|
||||||
|
*/
|
||||||
|
public static final PropertyDescriptor AWS_CREDENTIALS_PROVIDER_SERVICE = new PropertyDescriptor.Builder()
|
||||||
|
.name("aws-credentials-provider-service")
|
||||||
|
.displayName("AWS Credentials Provider Service")
|
||||||
|
.description("Service used to obtain an Amazon Web Services Credentials Provider")
|
||||||
|
.required(true)
|
||||||
|
.identifiesControllerService(AWSCredentialsProviderService.class)
|
||||||
|
.build();
|
||||||
|
|
||||||
|
public static final PropertyDescriptor REGION = new PropertyDescriptor.Builder()
|
||||||
|
.name("aws-region")
|
||||||
|
.displayName("Region")
|
||||||
|
.required(true)
|
||||||
|
.allowableValues(getAvailableRegions())
|
||||||
|
.defaultValue(createAllowableValue(Regions.DEFAULT_REGION).getValue())
|
||||||
|
.build();
|
||||||
|
|
||||||
|
public static final PropertyDescriptor TIMEOUT = new PropertyDescriptor.Builder()
|
||||||
|
.name("aws-communications-timeout")
|
||||||
|
.displayName("Communications Timeout")
|
||||||
|
.required(true)
|
||||||
|
.addValidator(StandardValidators.TIME_PERIOD_VALIDATOR)
|
||||||
|
.defaultValue("30 secs")
|
||||||
|
.build();
|
||||||
|
|
||||||
|
public static final PropertyDescriptor SSL_CONTEXT_SERVICE = new PropertyDescriptor.Builder()
|
||||||
|
.name("aws-ssl-context-service")
|
||||||
|
.displayName("SSL Context Service")
|
||||||
|
.description("Specifies an optional SSL Context Service that, if provided, will be used to create connections")
|
||||||
|
.required(false)
|
||||||
|
.identifiesControllerService(SSLContextService.class)
|
||||||
|
.build();
|
||||||
|
|
||||||
|
private static final String DEFAULT_USER_AGENT = "NiFi";
|
||||||
|
private static final Protocol DEFAULT_PROTOCOL = Protocol.HTTPS;
|
||||||
|
private static final List<PropertyDescriptor> PROPERTIES = Collections.unmodifiableList(Arrays.asList(
|
||||||
|
SECRET_NAME_PATTERN,
|
||||||
|
REGION,
|
||||||
|
AWS_CREDENTIALS_PROVIDER_SERVICE,
|
||||||
|
TIMEOUT,
|
||||||
|
SSL_CONTEXT_SERVICE
|
||||||
|
));
|
||||||
|
|
||||||
|
private final ObjectMapper objectMapper = new ObjectMapper();
|
||||||
|
|
||||||
|
@Override
|
||||||
|
protected List<PropertyDescriptor> getSupportedPropertyDescriptors() {
|
||||||
|
return PROPERTIES;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public List<ParameterGroup> fetchParameters(final ConfigurationContext context) {
|
||||||
|
AWSSecretsManager secretsManager = this.configureClient(context);
|
||||||
|
|
||||||
|
final List<ParameterGroup> groups = new ArrayList<>();
|
||||||
|
final ListSecretsRequest listSecretsRequest = new ListSecretsRequest();
|
||||||
|
final ListSecretsResult listSecretsResult = secretsManager.listSecrets(listSecretsRequest);
|
||||||
|
for (final SecretListEntry entry : listSecretsResult.getSecretList()) {
|
||||||
|
groups.addAll(fetchSecret(secretsManager, context, entry.getName()));
|
||||||
|
}
|
||||||
|
|
||||||
|
return groups;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public List<ConfigVerificationResult> verify(final ConfigurationContext context, final ComponentLog verificationLogger) {
|
||||||
|
final List<ConfigVerificationResult> results = new ArrayList<>();
|
||||||
|
|
||||||
|
try {
|
||||||
|
final List<ParameterGroup> parameterGroups = fetchParameters(context);
|
||||||
|
int parameterCount = 0;
|
||||||
|
for (final ParameterGroup group : parameterGroups) {
|
||||||
|
parameterCount += group.getParameters().size();
|
||||||
|
}
|
||||||
|
results.add(new ConfigVerificationResult.Builder()
|
||||||
|
.outcome(ConfigVerificationResult.Outcome.SUCCESSFUL)
|
||||||
|
.verificationStepName("Fetch Parameters")
|
||||||
|
.explanation(String.format("Fetched secret keys [%d] as parameters, across groups [%d]",
|
||||||
|
parameterCount, parameterGroups.size()))
|
||||||
|
.build());
|
||||||
|
} catch (final Exception e) {
|
||||||
|
verificationLogger.error("Failed to fetch parameters", e);
|
||||||
|
results.add(new ConfigVerificationResult.Builder()
|
||||||
|
.outcome(ConfigVerificationResult.Outcome.FAILED)
|
||||||
|
.verificationStepName("Fetch Parameters")
|
||||||
|
.explanation("Failed to fetch parameters: " + e.getMessage())
|
||||||
|
.build());
|
||||||
|
}
|
||||||
|
return results;
|
||||||
|
}
|
||||||
|
|
||||||
|
private List<ParameterGroup> fetchSecret(final AWSSecretsManager secretsManager, final ConfigurationContext context, final String secretName) {
|
||||||
|
final List<ParameterGroup> groups = new ArrayList<>();
|
||||||
|
final Pattern secretNamePattern = Pattern.compile(context.getProperty(SECRET_NAME_PATTERN).getValue());
|
||||||
|
|
||||||
|
final List<Parameter> parameters = new ArrayList<>();
|
||||||
|
|
||||||
|
if (!secretNamePattern.matcher(secretName).matches()) {
|
||||||
|
getLogger().debug("Secret [{}] does not match the secret name pattern {}", secretName, secretNamePattern);
|
||||||
|
return groups;
|
||||||
|
}
|
||||||
|
|
||||||
|
final GetSecretValueRequest getSecretValueRequest = new GetSecretValueRequest().withSecretId(secretName);
|
||||||
|
try {
|
||||||
|
final GetSecretValueResult getSecretValueResult = secretsManager.getSecretValue(getSecretValueRequest);
|
||||||
|
|
||||||
|
if (getSecretValueResult.getSecretString() == null) {
|
||||||
|
getLogger().debug("Secret [{}] is not configured", secretName);
|
||||||
|
return groups;
|
||||||
|
}
|
||||||
|
|
||||||
|
final ObjectNode secretObject = parseSecret(getSecretValueResult.getSecretString());
|
||||||
|
if (secretObject == null) {
|
||||||
|
getLogger().debug("Secret [{}] is not in the expected JSON key/value format", secretName);
|
||||||
|
return groups;
|
||||||
|
}
|
||||||
|
|
||||||
|
for (final Iterator<Map.Entry<String, JsonNode>> it = secretObject.fields(); it.hasNext(); ) {
|
||||||
|
final Map.Entry<String, JsonNode> field = it.next();
|
||||||
|
final String parameterName = field.getKey();
|
||||||
|
final String parameterValue = field.getValue().textValue();
|
||||||
|
if (parameterValue == null) {
|
||||||
|
getLogger().debug("Secret [{}] Parameter [{}] has no value", secretName, parameterName);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
parameters.add(createParameter(parameterName, parameterValue));
|
||||||
|
}
|
||||||
|
|
||||||
|
groups.add(new ParameterGroup(secretName, parameters));
|
||||||
|
|
||||||
|
return groups;
|
||||||
|
} catch (final ResourceNotFoundException e) {
|
||||||
|
throw new IllegalStateException(String.format("Secret %s not found", secretName), e);
|
||||||
|
} catch (final AWSSecretsManagerException e) {
|
||||||
|
throw new IllegalStateException("Error retrieving secret " + secretName, e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private Parameter createParameter(final String parameterName, final String parameterValue) {
|
||||||
|
final ParameterDescriptor parameterDescriptor = new ParameterDescriptor.Builder().name(parameterName).build();
|
||||||
|
return new Parameter(parameterDescriptor, parameterValue, null, true);
|
||||||
|
}
|
||||||
|
|
||||||
|
protected ClientConfiguration createConfiguration(final ConfigurationContext context) {
|
||||||
|
final ClientConfiguration config = new ClientConfiguration();
|
||||||
|
config.setMaxErrorRetry(0);
|
||||||
|
config.setUserAgentPrefix(DEFAULT_USER_AGENT);
|
||||||
|
config.setProtocol(DEFAULT_PROTOCOL);
|
||||||
|
final int commsTimeout = context.getProperty(TIMEOUT).asTimePeriod(TimeUnit.MILLISECONDS).intValue();
|
||||||
|
config.setConnectionTimeout(commsTimeout);
|
||||||
|
config.setSocketTimeout(commsTimeout);
|
||||||
|
|
||||||
|
final SSLContextService sslContextService = context.getProperty(SSL_CONTEXT_SERVICE).asControllerService(SSLContextService.class);
|
||||||
|
if (sslContextService != null) {
|
||||||
|
final SSLContext sslContext = sslContextService.createContext();
|
||||||
|
SdkTLSSocketFactory sdkTLSSocketFactory = new SdkTLSSocketFactory(sslContext, SdkTLSSocketFactory.BROWSER_COMPATIBLE_HOSTNAME_VERIFIER);
|
||||||
|
config.getApacheHttpClientConfig().setSslSocketFactory(sdkTLSSocketFactory);
|
||||||
|
}
|
||||||
|
|
||||||
|
return config;
|
||||||
|
}
|
||||||
|
|
||||||
|
private ObjectNode parseSecret(final String secretString) {
|
||||||
|
try {
|
||||||
|
final JsonNode root = objectMapper.readTree(secretString);
|
||||||
|
if (root instanceof ObjectNode) {
|
||||||
|
return (ObjectNode) root;
|
||||||
|
}
|
||||||
|
return null;
|
||||||
|
} catch (final JsonProcessingException e) {
|
||||||
|
getLogger().debug("Error parsing JSON", e);
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
AWSSecretsManager configureClient(final ConfigurationContext context) {
|
||||||
|
return AWSSecretsManagerClientBuilder.standard()
|
||||||
|
.withRegion(context.getProperty(REGION).getValue())
|
||||||
|
.withClientConfiguration(createConfiguration(context))
|
||||||
|
.withCredentials(getCredentialsProvider(context))
|
||||||
|
.build();
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Get credentials provider using the {@link AWSCredentialsProviderService}
|
||||||
|
* @param context the configuration context
|
||||||
|
* @return AWSCredentialsProvider the credential provider
|
||||||
|
* @see <a href="http://docs.aws.amazon.com/AWSJavaSDK/latest/javadoc/com/amazonaws/auth/AWSCredentialsProvider.html">AWSCredentialsProvider</a>
|
||||||
|
*/
|
||||||
|
protected AWSCredentialsProvider getCredentialsProvider(final ConfigurationContext context) {
|
||||||
|
|
||||||
|
final AWSCredentialsProviderService awsCredentialsProviderService =
|
||||||
|
context.getProperty(AWS_CREDENTIALS_PROVIDER_SERVICE).asControllerService(AWSCredentialsProviderService.class);
|
||||||
|
|
||||||
|
return awsCredentialsProviderService.getCredentialsProvider();
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
private static AllowableValue createAllowableValue(final Regions region) {
|
||||||
|
return new AllowableValue(region.getName(), region.getDescription(), "AWS Region Code : " + region.getName());
|
||||||
|
}
|
||||||
|
|
||||||
|
private static AllowableValue[] getAvailableRegions() {
|
||||||
|
final List<AllowableValue> values = new ArrayList<>();
|
||||||
|
for (final Regions region : Regions.values()) {
|
||||||
|
values.add(createAllowableValue(region));
|
||||||
|
}
|
||||||
|
return values.toArray(new AllowableValue[0]);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,16 @@
|
||||||
|
# Licensed to the Apache Software Foundation (ASF) under one or more
|
||||||
|
# contributor license agreements. See the NOTICE file distributed with
|
||||||
|
# this work for additional information regarding copyright ownership.
|
||||||
|
# The ASF licenses this file to You under the Apache License, Version 2.0
|
||||||
|
# (the "License"); you may not use this file except in compliance with
|
||||||
|
# the License. You may obtain a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
# See the License for the specific language governing permissions and
|
||||||
|
# limitations under the License.
|
||||||
|
org.apache.nifi.parameter.aws.AwsSecretsManagerParameterProvider
|
||||||
|
|
||||||
|
|
@ -0,0 +1,62 @@
|
||||||
|
<!DOCTYPE html>
|
||||||
|
<html lang="en">
|
||||||
|
<!--
|
||||||
|
Licensed to the Apache Software Foundation (ASF) under one or more
|
||||||
|
contributor license agreements. See the NOTICE file distributed with
|
||||||
|
this work for additional information regarding copyright ownership.
|
||||||
|
The ASF licenses this file to You under the Apache License, Version 2.0
|
||||||
|
(the "License"); you may not use this file except in compliance with
|
||||||
|
the License. You may obtain a copy of the License at
|
||||||
|
http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
Unless required by applicable law or agreed to in writing, software
|
||||||
|
distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
See the License for the specific language governing permissions and
|
||||||
|
limitations under the License.
|
||||||
|
-->
|
||||||
|
<head>
|
||||||
|
<meta charset="utf-8" />
|
||||||
|
<title>AWSSecretsManagerParameterProvider</title>
|
||||||
|
|
||||||
|
<link rel="stylesheet" href="../../../../../css/component-usage.css" type="text/css" />
|
||||||
|
</head>
|
||||||
|
<body>
|
||||||
|
<h3>Mapping AWS Secrets to Parameter Contexts</h3>
|
||||||
|
|
||||||
|
<p>
|
||||||
|
The AwsSecretsManagerParameterProvider maps a Secret to a Parameter Context, with key/value pairs in the Secret
|
||||||
|
mapping to parameters. To create a compatible secret from the AWS Console:
|
||||||
|
</p>
|
||||||
|
|
||||||
|
<ol>
|
||||||
|
<li>From the Secrets Manager service, click the "Store a new Secret" button</li>
|
||||||
|
<li>Select "Other type of secret"</li>
|
||||||
|
<li>Under "Key/value", enter your parameters, with the parameter names being the keys and the parameter values
|
||||||
|
being the values. Click Next.</li>
|
||||||
|
<li>Enter the Secret name. This will determine which Parameter Context receives the parameters. Continue
|
||||||
|
through the rest of the wizard and finally click the "Store" button.</li>
|
||||||
|
</ol>
|
||||||
|
|
||||||
|
<p>
|
||||||
|
Alternatively, from the command line, run a command like the following:
|
||||||
|
</p>
|
||||||
|
|
||||||
|
<pre>
|
||||||
|
aws secretsmanager create-secret --name "[Context]" --secret-string '{ "[Param]": "[secretValue]", "[Param2]": "[secretValue2]" }'
|
||||||
|
</pre>
|
||||||
|
|
||||||
|
<p>In this example, [Context] should be the intended name of the Parameter Context, [Param] and [Param2] should be
|
||||||
|
parameter names, and [secretValue] and [secretValue2] should be the values of each respective parameter.</p>
|
||||||
|
|
||||||
|
<h3>Configuring the Parameter Provider</h3>
|
||||||
|
|
||||||
|
<p>
|
||||||
|
AWS Secrets must be explicitly matched in the "Secret Name Pattern" property in order for them to be fetched. This
|
||||||
|
prevents more than the intended Secrets from being pulled into NiFi.
|
||||||
|
</p>
|
||||||
|
|
||||||
|
<p>
|
||||||
|
|
||||||
|
</p>
|
||||||
|
</body>
|
||||||
|
</html>
|
||||||
|
|
@ -0,0 +1,199 @@
|
||||||
|
/*
|
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||||
|
* contributor license agreements. See the NOTICE file distributed with
|
||||||
|
* this work for additional information regarding copyright ownership.
|
||||||
|
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||||
|
* (the "License"); you may not use this file except in compliance with
|
||||||
|
* the License. You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
|
package org.apache.nifi.parameter.aws;
|
||||||
|
|
||||||
|
import com.amazonaws.services.secretsmanager.AWSSecretsManager;
|
||||||
|
import com.amazonaws.services.secretsmanager.model.AWSSecretsManagerException;
|
||||||
|
import com.amazonaws.services.secretsmanager.model.GetSecretValueRequest;
|
||||||
|
import com.amazonaws.services.secretsmanager.model.GetSecretValueResult;
|
||||||
|
import com.amazonaws.services.secretsmanager.model.ListSecretsResult;
|
||||||
|
import com.amazonaws.services.secretsmanager.model.SecretListEntry;
|
||||||
|
import com.fasterxml.jackson.core.JsonProcessingException;
|
||||||
|
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||||
|
import org.apache.nifi.components.ConfigVerificationResult;
|
||||||
|
import org.apache.nifi.components.PropertyDescriptor;
|
||||||
|
import org.apache.nifi.parameter.Parameter;
|
||||||
|
import org.apache.nifi.parameter.ParameterDescriptor;
|
||||||
|
import org.apache.nifi.parameter.ParameterGroup;
|
||||||
|
import org.apache.nifi.parameter.VerifiableParameterProvider;
|
||||||
|
import org.apache.nifi.reporting.InitializationException;
|
||||||
|
import org.apache.nifi.util.MockComponentLog;
|
||||||
|
import org.apache.nifi.util.MockConfigurationContext;
|
||||||
|
import org.apache.nifi.util.MockParameterProviderInitializationContext;
|
||||||
|
import org.junit.jupiter.api.Test;
|
||||||
|
import org.junit.jupiter.api.extension.ExtendWith;
|
||||||
|
import org.mockito.ArgumentMatcher;
|
||||||
|
import org.mockito.Mock;
|
||||||
|
import org.mockito.junit.jupiter.MockitoExtension;
|
||||||
|
|
||||||
|
import java.util.ArrayList;
|
||||||
|
import java.util.Arrays;
|
||||||
|
import java.util.Collections;
|
||||||
|
import java.util.HashMap;
|
||||||
|
import java.util.List;
|
||||||
|
import java.util.Map;
|
||||||
|
import java.util.stream.Collectors;
|
||||||
|
|
||||||
|
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||||
|
import static org.junit.jupiter.api.Assertions.assertThrows;
|
||||||
|
import static org.mockito.ArgumentMatchers.any;
|
||||||
|
import static org.mockito.ArgumentMatchers.argThat;
|
||||||
|
import static org.mockito.Mockito.doReturn;
|
||||||
|
import static org.mockito.Mockito.mock;
|
||||||
|
import static org.mockito.Mockito.spy;
|
||||||
|
import static org.mockito.Mockito.when;
|
||||||
|
|
||||||
|
@ExtendWith(MockitoExtension.class)
|
||||||
|
public class TestAwsSecretsManagerParameterProvider {
|
||||||
|
|
||||||
|
@Mock
|
||||||
|
private AWSSecretsManager defaultSecretsManager;
|
||||||
|
|
||||||
|
@Mock
|
||||||
|
private ListSecretsResult listSecretsResult;
|
||||||
|
|
||||||
|
final ObjectMapper objectMapper = new ObjectMapper();
|
||||||
|
|
||||||
|
final List<Parameter> mySecretParameters = Arrays.asList(
|
||||||
|
parameter("paramA", "valueA"),
|
||||||
|
parameter("paramB", "valueB"),
|
||||||
|
parameter("otherC", "valueOther"),
|
||||||
|
parameter("paramD", "valueD"),
|
||||||
|
parameter("nonSensitiveE", "valueE"),
|
||||||
|
parameter("otherF", "valueF")
|
||||||
|
);
|
||||||
|
final List<Parameter> otherSecretParameters = Arrays.asList(
|
||||||
|
parameter("paramG", "valueG"),
|
||||||
|
parameter("otherH", "valueOther")
|
||||||
|
);
|
||||||
|
final List<ParameterGroup> mockParameterGroups = Arrays.asList(
|
||||||
|
new ParameterGroup("MySecret", mySecretParameters),
|
||||||
|
new ParameterGroup("OtherSecret", otherSecretParameters)
|
||||||
|
);
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testFetchParametersWithNoSecrets() throws InitializationException {
|
||||||
|
final List<ParameterGroup> expectedGroups = Collections.singletonList(new ParameterGroup("MySecret", Collections.emptyList()));
|
||||||
|
runProviderTest(mockSecretsManager(expectedGroups), 0, ConfigVerificationResult.Outcome.SUCCESSFUL);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testFetchParameters() throws InitializationException {
|
||||||
|
runProviderTest(mockSecretsManager(mockParameterGroups), 8, ConfigVerificationResult.Outcome.SUCCESSFUL);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testFetchParametersListFailure() throws InitializationException {
|
||||||
|
when(defaultSecretsManager.listSecrets(any())).thenThrow(new AWSSecretsManagerException("Fake exception"));
|
||||||
|
runProviderTest(defaultSecretsManager, 0, ConfigVerificationResult.Outcome.FAILED);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testFetchParametersGetSecretFailure() throws InitializationException {
|
||||||
|
final List<SecretListEntry> secretList = Collections.singletonList(new SecretListEntry().withName("MySecret"));
|
||||||
|
when(listSecretsResult.getSecretList()).thenReturn(secretList);
|
||||||
|
when(defaultSecretsManager.listSecrets(any())).thenReturn(listSecretsResult);
|
||||||
|
when(defaultSecretsManager.getSecretValue(argThat(matchesGetSecretValueRequest("MySecret")))).thenThrow(new AWSSecretsManagerException("Fake exception"));
|
||||||
|
runProviderTest(defaultSecretsManager, 0, ConfigVerificationResult.Outcome.FAILED);
|
||||||
|
}
|
||||||
|
|
||||||
|
private AwsSecretsManagerParameterProvider getParameterProvider() {
|
||||||
|
return spy(new AwsSecretsManagerParameterProvider());
|
||||||
|
}
|
||||||
|
|
||||||
|
private AWSSecretsManager mockSecretsManager(final List<ParameterGroup> mockGroup) {
|
||||||
|
final AWSSecretsManager secretsManager = mock(AWSSecretsManager.class);
|
||||||
|
|
||||||
|
final List<SecretListEntry> secretList = mockGroup.stream()
|
||||||
|
.map(group -> new SecretListEntry().withName(group.getGroupName()))
|
||||||
|
.collect(Collectors.toList());
|
||||||
|
when(listSecretsResult.getSecretList()).thenReturn(secretList);
|
||||||
|
when(secretsManager.listSecrets(any())).thenReturn(listSecretsResult);
|
||||||
|
|
||||||
|
mockGroup.forEach(group -> {
|
||||||
|
final String groupName = group.getGroupName();
|
||||||
|
final Map<String, String> keyValues = group.getParameters().stream().collect(Collectors.toMap(
|
||||||
|
param -> param.getDescriptor().getName(),
|
||||||
|
Parameter::getValue));
|
||||||
|
final String secretString;
|
||||||
|
try {
|
||||||
|
secretString = objectMapper.writeValueAsString(keyValues);
|
||||||
|
final GetSecretValueResult result = new GetSecretValueResult().withName(groupName).withSecretString(secretString);
|
||||||
|
when(secretsManager.getSecretValue(argThat(matchesGetSecretValueRequest(groupName))))
|
||||||
|
.thenReturn(result);
|
||||||
|
} catch (final JsonProcessingException e) {
|
||||||
|
throw new IllegalStateException(e);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
return secretsManager;
|
||||||
|
}
|
||||||
|
|
||||||
|
private List<ParameterGroup> runProviderTest(final AWSSecretsManager secretsManager, final int expectedCount,
|
||||||
|
final ConfigVerificationResult.Outcome expectedOutcome) throws InitializationException {
|
||||||
|
|
||||||
|
final AwsSecretsManagerParameterProvider parameterProvider = getParameterProvider();
|
||||||
|
doReturn(secretsManager).when(parameterProvider).configureClient(any());
|
||||||
|
final MockParameterProviderInitializationContext initContext = new MockParameterProviderInitializationContext("id", "name",
|
||||||
|
new MockComponentLog("providerId", parameterProvider));
|
||||||
|
parameterProvider.initialize(initContext);
|
||||||
|
|
||||||
|
final Map<PropertyDescriptor, String> properties = new HashMap<>();
|
||||||
|
final MockConfigurationContext mockConfigurationContext = new MockConfigurationContext(properties, null);
|
||||||
|
|
||||||
|
List<ParameterGroup> parameterGroups = new ArrayList<>();
|
||||||
|
// Verify parameter fetching
|
||||||
|
if (expectedOutcome == ConfigVerificationResult.Outcome.FAILED) {
|
||||||
|
assertThrows(RuntimeException.class, () -> parameterProvider.fetchParameters(mockConfigurationContext));
|
||||||
|
} else {
|
||||||
|
parameterGroups = parameterProvider.fetchParameters(mockConfigurationContext);
|
||||||
|
final int parameterCount = (int) parameterGroups.stream()
|
||||||
|
.flatMap(group -> group.getParameters().stream())
|
||||||
|
.count();
|
||||||
|
assertEquals(expectedCount, parameterCount);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Verify config verification
|
||||||
|
final List<ConfigVerificationResult> results = ((VerifiableParameterProvider) parameterProvider).verify(mockConfigurationContext, initContext.getLogger());
|
||||||
|
|
||||||
|
assertEquals(1, results.size());
|
||||||
|
assertEquals(expectedOutcome, results.get(0).getOutcome());
|
||||||
|
|
||||||
|
return parameterGroups;
|
||||||
|
}
|
||||||
|
|
||||||
|
private static Parameter parameter(final String name, final String value) {
|
||||||
|
return new Parameter(new ParameterDescriptor.Builder().name(name).build(), value);
|
||||||
|
}
|
||||||
|
|
||||||
|
private static ArgumentMatcher<GetSecretValueRequest> matchesGetSecretValueRequest(final String groupName) {
|
||||||
|
return new GetSecretValueRequestMatcher(groupName);
|
||||||
|
}
|
||||||
|
|
||||||
|
private static class GetSecretValueRequestMatcher implements ArgumentMatcher<GetSecretValueRequest> {
|
||||||
|
|
||||||
|
private final String secretId;
|
||||||
|
|
||||||
|
private GetSecretValueRequestMatcher(final String secretId) {
|
||||||
|
this.secretId = secretId;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public boolean matches(final GetSecretValueRequest argument) {
|
||||||
|
return argument != null && argument.getSecretId().equals(secretId);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -37,5 +37,6 @@
|
||||||
<module>nifi-aws-service-api-nar</module>
|
<module>nifi-aws-service-api-nar</module>
|
||||||
<module>nifi-aws-abstract-processors</module>
|
<module>nifi-aws-abstract-processors</module>
|
||||||
<module>nifi-aws-parameter-value-providers</module>
|
<module>nifi-aws-parameter-value-providers</module>
|
||||||
|
<module>nifi-aws-parameter-providers</module>
|
||||||
</modules>
|
</modules>
|
||||||
</project>
|
</project>
|
||||||
|
|
|
||||||
Loading…
Reference in New Issue