mirror of https://github.com/apache/nifi.git
NIFI-9636: Adding AwsSecretsManagerParameterProvider
This closes #6392 Signed-off-by: David Handermann <exceptionfactory@apache.org>
This commit is contained in:
Joe Gresock
exceptionfactory
parent
fa193ca14b
commit
315e54a812
|
|
@ -41,6 +41,11 @@
|
|||
<artifactId>nifi-aws-processors</artifactId>
|
||||
<version>1.18.0-SNAPSHOT</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.apache.nifi</groupId>
|
||||
<artifactId>nifi-aws-parameter-providers</artifactId>
|
||||
<version>1.18.0-SNAPSHOT</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.apache.nifi</groupId>
|
||||
<artifactId>nifi-aws-parameter-value-providers</artifactId>
|
||||
|
|
|
|||
|
|
@ -0,0 +1,75 @@
|
|||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<!--
|
||||
Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
contributor license agreements. See the NOTICE file distributed with
|
||||
this work for additional information regarding copyright ownership.
|
||||
The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
(the "License"); you may not use this file except in compliance with
|
||||
the License. You may obtain a copy of the License at
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
-->
|
||||
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
<parent>
|
||||
<groupId>org.apache.nifi</groupId>
|
||||
<artifactId>nifi-aws-bundle</artifactId>
|
||||
<version>1.18.0-SNAPSHOT</version>
|
||||
</parent>
|
||||
|
||||
<artifactId>nifi-aws-parameter-providers</artifactId>
|
||||
<packaging>jar</packaging>
|
||||
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>org.apache.nifi</groupId>
|
||||
<artifactId>nifi-api</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.apache.nifi</groupId>
|
||||
<artifactId>nifi-aws-service-api</artifactId>
|
||||
<version>1.18.0-SNAPSHOT</version>
|
||||
<scope>provided</scope>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.apache.nifi</groupId>
|
||||
<artifactId>nifi-ssl-context-service-api</artifactId>
|
||||
<version>1.18.0-SNAPSHOT</version>
|
||||
<scope>provided</scope>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.apache.nifi</groupId>
|
||||
<artifactId>nifi-utils</artifactId>
|
||||
<version>1.18.0-SNAPSHOT</version>
|
||||
<scope>provided</scope>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.slf4j</groupId>
|
||||
<artifactId>jcl-over-slf4j</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>com.fasterxml.jackson.core</groupId>
|
||||
<artifactId>jackson-databind</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>com.amazonaws</groupId>
|
||||
<artifactId>aws-java-sdk-secretsmanager</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.apache.nifi</groupId>
|
||||
<artifactId>nifi-expression-language</artifactId>
|
||||
<version>1.18.0-SNAPSHOT</version>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.apache.nifi</groupId>
|
||||
<artifactId>nifi-mock</artifactId>
|
||||
<version>1.18.0-SNAPSHOT</version>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
</project>
|
||||
|
|
@ -0,0 +1,297 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.apache.nifi.parameter.aws;
|
||||
|
||||
import com.amazonaws.ClientConfiguration;
|
||||
import com.amazonaws.Protocol;
|
||||
import com.amazonaws.auth.AWSCredentialsProvider;
|
||||
import com.amazonaws.http.conn.ssl.SdkTLSSocketFactory;
|
||||
import com.amazonaws.regions.Regions;
|
||||
import com.amazonaws.services.secretsmanager.AWSSecretsManager;
|
||||
import com.amazonaws.services.secretsmanager.AWSSecretsManagerClientBuilder;
|
||||
import com.amazonaws.services.secretsmanager.model.AWSSecretsManagerException;
|
||||
import com.amazonaws.services.secretsmanager.model.GetSecretValueRequest;
|
||||
import com.amazonaws.services.secretsmanager.model.GetSecretValueResult;
|
||||
import com.amazonaws.services.secretsmanager.model.ListSecretsRequest;
|
||||
import com.amazonaws.services.secretsmanager.model.ListSecretsResult;
|
||||
import com.amazonaws.services.secretsmanager.model.ResourceNotFoundException;
|
||||
import com.amazonaws.services.secretsmanager.model.SecretListEntry;
|
||||
import com.fasterxml.jackson.core.JsonProcessingException;
|
||||
import com.fasterxml.jackson.databind.JsonNode;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.fasterxml.jackson.databind.node.ObjectNode;
|
||||
import org.apache.nifi.annotation.documentation.CapabilityDescription;
|
||||
import org.apache.nifi.annotation.documentation.Tags;
|
||||
import org.apache.nifi.components.AllowableValue;
|
||||
import org.apache.nifi.components.ConfigVerificationResult;
|
||||
import org.apache.nifi.components.PropertyDescriptor;
|
||||
import org.apache.nifi.controller.ConfigurationContext;
|
||||
import org.apache.nifi.logging.ComponentLog;
|
||||
import org.apache.nifi.parameter.AbstractParameterProvider;
|
||||
import org.apache.nifi.parameter.Parameter;
|
||||
import org.apache.nifi.parameter.ParameterDescriptor;
|
||||
import org.apache.nifi.parameter.ParameterGroup;
|
||||
import org.apache.nifi.parameter.VerifiableParameterProvider;
|
||||
import org.apache.nifi.processor.util.StandardValidators;
|
||||
import org.apache.nifi.processors.aws.credentials.provider.service.AWSCredentialsProviderService;
|
||||
import org.apache.nifi.ssl.SSLContextService;
|
||||
|
||||
import javax.net.ssl.SSLContext;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Arrays;
|
||||
import java.util.Collections;
|
||||
import java.util.Iterator;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
/**
|
||||
* Reads secrets from AWS Secrets Manager to provide parameter values. Secrets must be created similar to the following AWS cli command: <br/><br/>
|
||||
* <code>aws secretsmanager create-secret --name "[Context]" --secret-string '{ "[Param]": "[secretValue]", "[Param2]": "[secretValue2]" }'</code> <br/><br/>
|
||||
*
|
||||
*/
|
||||
|
||||
@Tags({"aws", "secretsmanager", "secrets", "manager"})
|
||||
@CapabilityDescription("Fetches parameters from AWS SecretsManager. Each secret becomes a Parameter group, which can map to a Parameter Context, with " +
|
||||
"key/value pairs in the secret mapping to Parameters in the group.")
|
||||
public class AwsSecretsManagerParameterProvider extends AbstractParameterProvider implements VerifiableParameterProvider {
|
||||
|
||||
public static final PropertyDescriptor SECRET_NAME_PATTERN = new PropertyDescriptor.Builder()
|
||||
.name("secret-name-pattern")
|
||||
.displayName("Secret Name Pattern")
|
||||
.description("A Regular Expression matching on Secret Name that identifies Secrets whose parameters should be fetched. " +
|
||||
"Any secrets whose names do not match this pattern will not be fetched.")
|
||||
.addValidator(StandardValidators.REGULAR_EXPRESSION_VALIDATOR)
|
||||
.required(true)
|
||||
.defaultValue(".*")
|
||||
.build();
|
||||
/**
|
||||
* AWS credentials provider service
|
||||
*
|
||||
* @see <a href="http://docs.aws.amazon.com/AWSJavaSDK/latest/javadoc/com/amazonaws/auth/AWSCredentialsProvider.html">AWSCredentialsProvider</a>
|
||||
*/
|
||||
public static final PropertyDescriptor AWS_CREDENTIALS_PROVIDER_SERVICE = new PropertyDescriptor.Builder()
|
||||
.name("aws-credentials-provider-service")
|
||||
.displayName("AWS Credentials Provider Service")
|
||||
.description("Service used to obtain an Amazon Web Services Credentials Provider")
|
||||
.required(true)
|
||||
.identifiesControllerService(AWSCredentialsProviderService.class)
|
||||
.build();
|
||||
|
||||
public static final PropertyDescriptor REGION = new PropertyDescriptor.Builder()
|
||||
.name("aws-region")
|
||||
.displayName("Region")
|
||||
.required(true)
|
||||
.allowableValues(getAvailableRegions())
|
||||
.defaultValue(createAllowableValue(Regions.DEFAULT_REGION).getValue())
|
||||
.build();
|
||||
|
||||
public static final PropertyDescriptor TIMEOUT = new PropertyDescriptor.Builder()
|
||||
.name("aws-communications-timeout")
|
||||
.displayName("Communications Timeout")
|
||||
.required(true)
|
||||
.addValidator(StandardValidators.TIME_PERIOD_VALIDATOR)
|
||||
.defaultValue("30 secs")
|
||||
.build();
|
||||
|
||||
public static final PropertyDescriptor SSL_CONTEXT_SERVICE = new PropertyDescriptor.Builder()
|
||||
.name("aws-ssl-context-service")
|
||||
.displayName("SSL Context Service")
|
||||
.description("Specifies an optional SSL Context Service that, if provided, will be used to create connections")
|
||||
.required(false)
|
||||
.identifiesControllerService(SSLContextService.class)
|
||||
.build();
|
||||
|
||||
private static final String DEFAULT_USER_AGENT = "NiFi";
|
||||
private static final Protocol DEFAULT_PROTOCOL = Protocol.HTTPS;
|
||||
private static final List<PropertyDescriptor> PROPERTIES = Collections.unmodifiableList(Arrays.asList(
|
||||
SECRET_NAME_PATTERN,
|
||||
REGION,
|
||||
AWS_CREDENTIALS_PROVIDER_SERVICE,
|
||||
TIMEOUT,
|
||||
SSL_CONTEXT_SERVICE
|
||||
));
|
||||
|
||||
private final ObjectMapper objectMapper = new ObjectMapper();
|
||||
|
||||
@Override
|
||||
protected List<PropertyDescriptor> getSupportedPropertyDescriptors() {
|
||||
return PROPERTIES;
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<ParameterGroup> fetchParameters(final ConfigurationContext context) {
|
||||
AWSSecretsManager secretsManager = this.configureClient(context);
|
||||
|
||||
final List<ParameterGroup> groups = new ArrayList<>();
|
||||
final ListSecretsRequest listSecretsRequest = new ListSecretsRequest();
|
||||
final ListSecretsResult listSecretsResult = secretsManager.listSecrets(listSecretsRequest);
|
||||
for (final SecretListEntry entry : listSecretsResult.getSecretList()) {
|
||||
groups.addAll(fetchSecret(secretsManager, context, entry.getName()));
|
||||
}
|
||||
|
||||
return groups;
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<ConfigVerificationResult> verify(final ConfigurationContext context, final ComponentLog verificationLogger) {
|
||||
final List<ConfigVerificationResult> results = new ArrayList<>();
|
||||
|
||||
try {
|
||||
final List<ParameterGroup> parameterGroups = fetchParameters(context);
|
||||
int parameterCount = 0;
|
||||
for (final ParameterGroup group : parameterGroups) {
|
||||
parameterCount += group.getParameters().size();
|
||||
}
|
||||
results.add(new ConfigVerificationResult.Builder()
|
||||
.outcome(ConfigVerificationResult.Outcome.SUCCESSFUL)
|
||||
.verificationStepName("Fetch Parameters")
|
||||
.explanation(String.format("Fetched secret keys [%d] as parameters, across groups [%d]",
|
||||
parameterCount, parameterGroups.size()))
|
||||
.build());
|
||||
} catch (final Exception e) {
|
||||
verificationLogger.error("Failed to fetch parameters", e);
|
||||
results.add(new ConfigVerificationResult.Builder()
|
||||
.outcome(ConfigVerificationResult.Outcome.FAILED)
|
||||
.verificationStepName("Fetch Parameters")
|
||||
.explanation("Failed to fetch parameters: " + e.getMessage())
|
||||
.build());
|
||||
}
|
||||
return results;
|
||||
}
|
||||
|
||||
private List<ParameterGroup> fetchSecret(final AWSSecretsManager secretsManager, final ConfigurationContext context, final String secretName) {
|
||||
final List<ParameterGroup> groups = new ArrayList<>();
|
||||
final Pattern secretNamePattern = Pattern.compile(context.getProperty(SECRET_NAME_PATTERN).getValue());
|
||||
|
||||
final List<Parameter> parameters = new ArrayList<>();
|
||||
|
||||
if (!secretNamePattern.matcher(secretName).matches()) {
|
||||
getLogger().debug("Secret [{}] does not match the secret name pattern {}", secretName, secretNamePattern);
|
||||
return groups;
|
||||
}
|
||||
|
||||
final GetSecretValueRequest getSecretValueRequest = new GetSecretValueRequest().withSecretId(secretName);
|
||||
try {
|
||||
final GetSecretValueResult getSecretValueResult = secretsManager.getSecretValue(getSecretValueRequest);
|
||||
|
||||
if (getSecretValueResult.getSecretString() == null) {
|
||||
getLogger().debug("Secret [{}] is not configured", secretName);
|
||||
return groups;
|
||||
}
|
||||
|
||||
final ObjectNode secretObject = parseSecret(getSecretValueResult.getSecretString());
|
||||
if (secretObject == null) {
|
||||
getLogger().debug("Secret [{}] is not in the expected JSON key/value format", secretName);
|
||||
return groups;
|
||||
}
|
||||
|
||||
for (final Iterator<Map.Entry<String, JsonNode>> it = secretObject.fields(); it.hasNext(); ) {
|
||||
final Map.Entry<String, JsonNode> field = it.next();
|
||||
final String parameterName = field.getKey();
|
||||
final String parameterValue = field.getValue().textValue();
|
||||
if (parameterValue == null) {
|
||||
getLogger().debug("Secret [{}] Parameter [{}] has no value", secretName, parameterName);
|
||||
continue;
|
||||
}
|
||||
|
||||
parameters.add(createParameter(parameterName, parameterValue));
|
||||
}
|
||||
|
||||
groups.add(new ParameterGroup(secretName, parameters));
|
||||
|
||||
return groups;
|
||||
} catch (final ResourceNotFoundException e) {
|
||||
throw new IllegalStateException(String.format("Secret %s not found", secretName), e);
|
||||
} catch (final AWSSecretsManagerException e) {
|
||||
throw new IllegalStateException("Error retrieving secret " + secretName, e);
|
||||
}
|
||||
}
|
||||
|
||||
private Parameter createParameter(final String parameterName, final String parameterValue) {
|
||||
final ParameterDescriptor parameterDescriptor = new ParameterDescriptor.Builder().name(parameterName).build();
|
||||
return new Parameter(parameterDescriptor, parameterValue, null, true);
|
||||
}
|
||||
|
||||
protected ClientConfiguration createConfiguration(final ConfigurationContext context) {
|
||||
final ClientConfiguration config = new ClientConfiguration();
|
||||
config.setMaxErrorRetry(0);
|
||||
config.setUserAgentPrefix(DEFAULT_USER_AGENT);
|
||||
config.setProtocol(DEFAULT_PROTOCOL);
|
||||
final int commsTimeout = context.getProperty(TIMEOUT).asTimePeriod(TimeUnit.MILLISECONDS).intValue();
|
||||
config.setConnectionTimeout(commsTimeout);
|
||||
config.setSocketTimeout(commsTimeout);
|
||||
|
||||
final SSLContextService sslContextService = context.getProperty(SSL_CONTEXT_SERVICE).asControllerService(SSLContextService.class);
|
||||
if (sslContextService != null) {
|
||||
final SSLContext sslContext = sslContextService.createContext();
|
||||
SdkTLSSocketFactory sdkTLSSocketFactory = new SdkTLSSocketFactory(sslContext, SdkTLSSocketFactory.BROWSER_COMPATIBLE_HOSTNAME_VERIFIER);
|
||||
config.getApacheHttpClientConfig().setSslSocketFactory(sdkTLSSocketFactory);
|
||||
}
|
||||
|
||||
return config;
|
||||
}
|
||||
|
||||
private ObjectNode parseSecret(final String secretString) {
|
||||
try {
|
||||
final JsonNode root = objectMapper.readTree(secretString);
|
||||
if (root instanceof ObjectNode) {
|
||||
return (ObjectNode) root;
|
||||
}
|
||||
return null;
|
||||
} catch (final JsonProcessingException e) {
|
||||
getLogger().debug("Error parsing JSON", e);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
AWSSecretsManager configureClient(final ConfigurationContext context) {
|
||||
return AWSSecretsManagerClientBuilder.standard()
|
||||
.withRegion(context.getProperty(REGION).getValue())
|
||||
.withClientConfiguration(createConfiguration(context))
|
||||
.withCredentials(getCredentialsProvider(context))
|
||||
.build();
|
||||
}
|
||||
|
||||
/**
|
||||
* Get credentials provider using the {@link AWSCredentialsProviderService}
|
||||
* @param context the configuration context
|
||||
* @return AWSCredentialsProvider the credential provider
|
||||
* @see <a href="http://docs.aws.amazon.com/AWSJavaSDK/latest/javadoc/com/amazonaws/auth/AWSCredentialsProvider.html">AWSCredentialsProvider</a>
|
||||
*/
|
||||
protected AWSCredentialsProvider getCredentialsProvider(final ConfigurationContext context) {
|
||||
|
||||
final AWSCredentialsProviderService awsCredentialsProviderService =
|
||||
context.getProperty(AWS_CREDENTIALS_PROVIDER_SERVICE).asControllerService(AWSCredentialsProviderService.class);
|
||||
|
||||
return awsCredentialsProviderService.getCredentialsProvider();
|
||||
|
||||
}
|
||||
|
||||
private static AllowableValue createAllowableValue(final Regions region) {
|
||||
return new AllowableValue(region.getName(), region.getDescription(), "AWS Region Code : " + region.getName());
|
||||
}
|
||||
|
||||
private static AllowableValue[] getAvailableRegions() {
|
||||
final List<AllowableValue> values = new ArrayList<>();
|
||||
for (final Regions region : Regions.values()) {
|
||||
values.add(createAllowableValue(region));
|
||||
}
|
||||
return values.toArray(new AllowableValue[0]);
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,16 @@
|
|||
# Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
# contributor license agreements. See the NOTICE file distributed with
|
||||
# this work for additional information regarding copyright ownership.
|
||||
# The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
# (the "License"); you may not use this file except in compliance with
|
||||
# the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
org.apache.nifi.parameter.aws.AwsSecretsManagerParameterProvider
|
||||
|
||||
|
|
@ -0,0 +1,62 @@
|
|||
<!DOCTYPE html>
|
||||
<html lang="en">
|
||||
<!--
|
||||
Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
contributor license agreements. See the NOTICE file distributed with
|
||||
this work for additional information regarding copyright ownership.
|
||||
The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
(the "License"); you may not use this file except in compliance with
|
||||
the License. You may obtain a copy of the License at
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
-->
|
||||
<head>
|
||||
<meta charset="utf-8" />
|
||||
<title>AWSSecretsManagerParameterProvider</title>
|
||||
|
||||
<link rel="stylesheet" href="../../../../../css/component-usage.css" type="text/css" />
|
||||
</head>
|
||||
<body>
|
||||
<h3>Mapping AWS Secrets to Parameter Contexts</h3>
|
||||
|
||||
<p>
|
||||
The AwsSecretsManagerParameterProvider maps a Secret to a Parameter Context, with key/value pairs in the Secret
|
||||
mapping to parameters. To create a compatible secret from the AWS Console:
|
||||
</p>
|
||||
|
||||
<ol>
|
||||
<li>From the Secrets Manager service, click the "Store a new Secret" button</li>
|
||||
<li>Select "Other type of secret"</li>
|
||||
<li>Under "Key/value", enter your parameters, with the parameter names being the keys and the parameter values
|
||||
being the values. Click Next.</li>
|
||||
<li>Enter the Secret name. This will determine which Parameter Context receives the parameters. Continue
|
||||
through the rest of the wizard and finally click the "Store" button.</li>
|
||||
</ol>
|
||||
|
||||
<p>
|
||||
Alternatively, from the command line, run a command like the following:
|
||||
</p>
|
||||
|
||||
<pre>
|
||||
aws secretsmanager create-secret --name "[Context]" --secret-string '{ "[Param]": "[secretValue]", "[Param2]": "[secretValue2]" }'
|
||||
</pre>
|
||||
|
||||
<p>In this example, [Context] should be the intended name of the Parameter Context, [Param] and [Param2] should be
|
||||
parameter names, and [secretValue] and [secretValue2] should be the values of each respective parameter.</p>
|
||||
|
||||
<h3>Configuring the Parameter Provider</h3>
|
||||
|
||||
<p>
|
||||
AWS Secrets must be explicitly matched in the "Secret Name Pattern" property in order for them to be fetched. This
|
||||
prevents more than the intended Secrets from being pulled into NiFi.
|
||||
</p>
|
||||
|
||||
<p>
|
||||
|
||||
</p>
|
||||
</body>
|
||||
</html>
|
||||
|
|
@ -0,0 +1,199 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.apache.nifi.parameter.aws;
|
||||
|
||||
import com.amazonaws.services.secretsmanager.AWSSecretsManager;
|
||||
import com.amazonaws.services.secretsmanager.model.AWSSecretsManagerException;
|
||||
import com.amazonaws.services.secretsmanager.model.GetSecretValueRequest;
|
||||
import com.amazonaws.services.secretsmanager.model.GetSecretValueResult;
|
||||
import com.amazonaws.services.secretsmanager.model.ListSecretsResult;
|
||||
import com.amazonaws.services.secretsmanager.model.SecretListEntry;
|
||||
import com.fasterxml.jackson.core.JsonProcessingException;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import org.apache.nifi.components.ConfigVerificationResult;
|
||||
import org.apache.nifi.components.PropertyDescriptor;
|
||||
import org.apache.nifi.parameter.Parameter;
|
||||
import org.apache.nifi.parameter.ParameterDescriptor;
|
||||
import org.apache.nifi.parameter.ParameterGroup;
|
||||
import org.apache.nifi.parameter.VerifiableParameterProvider;
|
||||
import org.apache.nifi.reporting.InitializationException;
|
||||
import org.apache.nifi.util.MockComponentLog;
|
||||
import org.apache.nifi.util.MockConfigurationContext;
|
||||
import org.apache.nifi.util.MockParameterProviderInitializationContext;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.extension.ExtendWith;
|
||||
import org.mockito.ArgumentMatcher;
|
||||
import org.mockito.Mock;
|
||||
import org.mockito.junit.jupiter.MockitoExtension;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.Arrays;
|
||||
import java.util.Collections;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertThrows;
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.ArgumentMatchers.argThat;
|
||||
import static org.mockito.Mockito.doReturn;
|
||||
import static org.mockito.Mockito.mock;
|
||||
import static org.mockito.Mockito.spy;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
@ExtendWith(MockitoExtension.class)
|
||||
public class TestAwsSecretsManagerParameterProvider {
|
||||
|
||||
@Mock
|
||||
private AWSSecretsManager defaultSecretsManager;
|
||||
|
||||
@Mock
|
||||
private ListSecretsResult listSecretsResult;
|
||||
|
||||
final ObjectMapper objectMapper = new ObjectMapper();
|
||||
|
||||
final List<Parameter> mySecretParameters = Arrays.asList(
|
||||
parameter("paramA", "valueA"),
|
||||
parameter("paramB", "valueB"),
|
||||
parameter("otherC", "valueOther"),
|
||||
parameter("paramD", "valueD"),
|
||||
parameter("nonSensitiveE", "valueE"),
|
||||
parameter("otherF", "valueF")
|
||||
);
|
||||
final List<Parameter> otherSecretParameters = Arrays.asList(
|
||||
parameter("paramG", "valueG"),
|
||||
parameter("otherH", "valueOther")
|
||||
);
|
||||
final List<ParameterGroup> mockParameterGroups = Arrays.asList(
|
||||
new ParameterGroup("MySecret", mySecretParameters),
|
||||
new ParameterGroup("OtherSecret", otherSecretParameters)
|
||||
);
|
||||
|
||||
@Test
|
||||
public void testFetchParametersWithNoSecrets() throws InitializationException {
|
||||
final List<ParameterGroup> expectedGroups = Collections.singletonList(new ParameterGroup("MySecret", Collections.emptyList()));
|
||||
runProviderTest(mockSecretsManager(expectedGroups), 0, ConfigVerificationResult.Outcome.SUCCESSFUL);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testFetchParameters() throws InitializationException {
|
||||
runProviderTest(mockSecretsManager(mockParameterGroups), 8, ConfigVerificationResult.Outcome.SUCCESSFUL);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testFetchParametersListFailure() throws InitializationException {
|
||||
when(defaultSecretsManager.listSecrets(any())).thenThrow(new AWSSecretsManagerException("Fake exception"));
|
||||
runProviderTest(defaultSecretsManager, 0, ConfigVerificationResult.Outcome.FAILED);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testFetchParametersGetSecretFailure() throws InitializationException {
|
||||
final List<SecretListEntry> secretList = Collections.singletonList(new SecretListEntry().withName("MySecret"));
|
||||
when(listSecretsResult.getSecretList()).thenReturn(secretList);
|
||||
when(defaultSecretsManager.listSecrets(any())).thenReturn(listSecretsResult);
|
||||
when(defaultSecretsManager.getSecretValue(argThat(matchesGetSecretValueRequest("MySecret")))).thenThrow(new AWSSecretsManagerException("Fake exception"));
|
||||
runProviderTest(defaultSecretsManager, 0, ConfigVerificationResult.Outcome.FAILED);
|
||||
}
|
||||
|
||||
private AwsSecretsManagerParameterProvider getParameterProvider() {
|
||||
return spy(new AwsSecretsManagerParameterProvider());
|
||||
}
|
||||
|
||||
private AWSSecretsManager mockSecretsManager(final List<ParameterGroup> mockGroup) {
|
||||
final AWSSecretsManager secretsManager = mock(AWSSecretsManager.class);
|
||||
|
||||
final List<SecretListEntry> secretList = mockGroup.stream()
|
||||
.map(group -> new SecretListEntry().withName(group.getGroupName()))
|
||||
.collect(Collectors.toList());
|
||||
when(listSecretsResult.getSecretList()).thenReturn(secretList);
|
||||
when(secretsManager.listSecrets(any())).thenReturn(listSecretsResult);
|
||||
|
||||
mockGroup.forEach(group -> {
|
||||
final String groupName = group.getGroupName();
|
||||
final Map<String, String> keyValues = group.getParameters().stream().collect(Collectors.toMap(
|
||||
param -> param.getDescriptor().getName(),
|
||||
Parameter::getValue));
|
||||
final String secretString;
|
||||
try {
|
||||
secretString = objectMapper.writeValueAsString(keyValues);
|
||||
final GetSecretValueResult result = new GetSecretValueResult().withName(groupName).withSecretString(secretString);
|
||||
when(secretsManager.getSecretValue(argThat(matchesGetSecretValueRequest(groupName))))
|
||||
.thenReturn(result);
|
||||
} catch (final JsonProcessingException e) {
|
||||
throw new IllegalStateException(e);
|
||||
}
|
||||
});
|
||||
return secretsManager;
|
||||
}
|
||||
|
||||
private List<ParameterGroup> runProviderTest(final AWSSecretsManager secretsManager, final int expectedCount,
|
||||
final ConfigVerificationResult.Outcome expectedOutcome) throws InitializationException {
|
||||
|
||||
final AwsSecretsManagerParameterProvider parameterProvider = getParameterProvider();
|
||||
doReturn(secretsManager).when(parameterProvider).configureClient(any());
|
||||
final MockParameterProviderInitializationContext initContext = new MockParameterProviderInitializationContext("id", "name",
|
||||
new MockComponentLog("providerId", parameterProvider));
|
||||
parameterProvider.initialize(initContext);
|
||||
|
||||
final Map<PropertyDescriptor, String> properties = new HashMap<>();
|
||||
final MockConfigurationContext mockConfigurationContext = new MockConfigurationContext(properties, null);
|
||||
|
||||
List<ParameterGroup> parameterGroups = new ArrayList<>();
|
||||
// Verify parameter fetching
|
||||
if (expectedOutcome == ConfigVerificationResult.Outcome.FAILED) {
|
||||
assertThrows(RuntimeException.class, () -> parameterProvider.fetchParameters(mockConfigurationContext));
|
||||
} else {
|
||||
parameterGroups = parameterProvider.fetchParameters(mockConfigurationContext);
|
||||
final int parameterCount = (int) parameterGroups.stream()
|
||||
.flatMap(group -> group.getParameters().stream())
|
||||
.count();
|
||||
assertEquals(expectedCount, parameterCount);
|
||||
}
|
||||
|
||||
// Verify config verification
|
||||
final List<ConfigVerificationResult> results = ((VerifiableParameterProvider) parameterProvider).verify(mockConfigurationContext, initContext.getLogger());
|
||||
|
||||
assertEquals(1, results.size());
|
||||
assertEquals(expectedOutcome, results.get(0).getOutcome());
|
||||
|
||||
return parameterGroups;
|
||||
}
|
||||
|
||||
private static Parameter parameter(final String name, final String value) {
|
||||
return new Parameter(new ParameterDescriptor.Builder().name(name).build(), value);
|
||||
}
|
||||
|
||||
private static ArgumentMatcher<GetSecretValueRequest> matchesGetSecretValueRequest(final String groupName) {
|
||||
return new GetSecretValueRequestMatcher(groupName);
|
||||
}
|
||||
|
||||
private static class GetSecretValueRequestMatcher implements ArgumentMatcher<GetSecretValueRequest> {
|
||||
|
||||
private final String secretId;
|
||||
|
||||
private GetSecretValueRequestMatcher(final String secretId) {
|
||||
this.secretId = secretId;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean matches(final GetSecretValueRequest argument) {
|
||||
return argument != null && argument.getSecretId().equals(secretId);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -37,5 +37,6 @@
|
|||
<module>nifi-aws-service-api-nar</module>
|
||||
<module>nifi-aws-abstract-processors</module>
|
||||
<module>nifi-aws-parameter-value-providers</module>
|
||||
<module>nifi-aws-parameter-providers</module>
|
||||
</modules>
|
||||
</project>
|
||||
|
|
|
|||
Loading…
Reference in New Issue