mirror of https://github.com/apache/nifi.git
NIFI-9174: Adding AWS SecretsManager ParamValueProvider for Stateless (#5391)
NIFI-9174: Adding AWS SecretsManager ParamValueProvider for Stateless
This commit is contained in:
Joe Gresock
GitHub
parent
f3136f07eb
commit
05407476b4
|
|
@ -41,6 +41,11 @@
|
|||
<artifactId>nifi-aws-processors</artifactId>
|
||||
<version>1.15.0-SNAPSHOT</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.apache.nifi</groupId>
|
||||
<artifactId>nifi-aws-parameter-value-providers</artifactId>
|
||||
<version>1.15.0-SNAPSHOT</version>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
|
||||
</project>
|
||||
|
|
|
|||
|
|
@ -0,0 +1,60 @@
|
|||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<!--
|
||||
Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
contributor license agreements. See the NOTICE file distributed with
|
||||
this work for additional information regarding copyright ownership.
|
||||
The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
(the "License"); you may not use this file except in compliance with
|
||||
the License. You may obtain a copy of the License at
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
-->
|
||||
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
<parent>
|
||||
<groupId>org.apache.nifi</groupId>
|
||||
<artifactId>nifi-aws-bundle</artifactId>
|
||||
<version>1.15.0-SNAPSHOT</version>
|
||||
</parent>
|
||||
|
||||
<artifactId>nifi-aws-parameter-value-providers</artifactId>
|
||||
<packaging>jar</packaging>
|
||||
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>org.apache.nifi</groupId>
|
||||
<artifactId>nifi-api</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.apache.nifi</groupId>
|
||||
<artifactId>nifi-utils</artifactId>
|
||||
<version>1.15.0-SNAPSHOT</version>
|
||||
<scope>provided</scope>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.apache.nifi</groupId>
|
||||
<artifactId>nifi-stateless-api</artifactId>
|
||||
<version>1.15.0-SNAPSHOT</version>
|
||||
<scope>provided</scope>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>com.fasterxml.jackson.core</groupId>
|
||||
<artifactId>jackson-databind</artifactId>
|
||||
<version>${jackson.version}</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>com.amazonaws</groupId>
|
||||
<artifactId>aws-java-sdk-secretsmanager</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.apache.nifi</groupId>
|
||||
<artifactId>nifi-expression-language</artifactId>
|
||||
<version>1.15.0-SNAPSHOT</version>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
</project>
|
||||
|
|
@ -0,0 +1,164 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.apache.nifi.stateless.parameter;
|
||||
|
||||
import com.amazonaws.auth.AWSStaticCredentialsProvider;
|
||||
import com.amazonaws.auth.BasicAWSCredentials;
|
||||
import com.amazonaws.auth.DefaultAWSCredentialsProviderChain;
|
||||
import com.amazonaws.services.secretsmanager.AWSSecretsManager;
|
||||
import com.amazonaws.services.secretsmanager.AWSSecretsManagerClientBuilder;
|
||||
import com.amazonaws.services.secretsmanager.model.AWSSecretsManagerException;
|
||||
import com.amazonaws.services.secretsmanager.model.GetSecretValueRequest;
|
||||
import com.amazonaws.services.secretsmanager.model.GetSecretValueResult;
|
||||
import com.amazonaws.services.secretsmanager.model.ResourceNotFoundException;
|
||||
import com.fasterxml.jackson.core.JsonProcessingException;
|
||||
import com.fasterxml.jackson.databind.JsonNode;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import org.apache.nifi.components.PropertyDescriptor;
|
||||
import org.apache.nifi.processor.util.StandardValidators;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import java.io.FileInputStream;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.nio.file.Paths;
|
||||
import java.util.Collections;
|
||||
import java.util.List;
|
||||
import java.util.Properties;
|
||||
|
||||
/**
|
||||
* Reads secrets from AWS Secrets Manager to provide parameter values. Secrets must be created similar to the following AWS cli command: <br/><br/>
|
||||
* <code>aws secretsmanager create-secret --name "[Context]" --secret-string '{ "[Param]": "[secretValue]", "[Param2]": "[secretValue2]" }'</code> <br/><br/>
|
||||
*
|
||||
* A standard configuration for this provider would be: <br/><br/>
|
||||
*
|
||||
* <code>
|
||||
* nifi.stateless.parameter.provider.AWSSecretsManager.name=AWS Secrets Manager Value Provider
|
||||
* nifi.stateless.parameter.provider.AWSSecretsManager.type=org.apache.nifi.stateless.parameter.AwsSecretsManagerParameterValueProvider
|
||||
* nifi.stateless.parameter.provider.AWSSecretsManager.properties.aws-credentials-file=./conf/bootstrap-aws.conf
|
||||
* </code>
|
||||
*/
|
||||
public class AwsSecretsManagerParameterValueProvider extends AbstractSecretBasedParameterValueProvider implements ParameterValueProvider {
|
||||
private static final Logger logger = LoggerFactory.getLogger(AwsSecretsManagerParameterValueProvider.class);
|
||||
|
||||
private static final String ACCESS_KEY_PROPS_NAME = "aws.access.key.id";
|
||||
private static final String SECRET_KEY_PROPS_NAME = "aws.secret.access.key";
|
||||
private static final String REGION_KEY_PROPS_NAME = "aws.region";
|
||||
|
||||
public static final PropertyDescriptor AWS_CREDENTIALS_FILE = new PropertyDescriptor.Builder()
|
||||
.displayName("AWS Credentials File")
|
||||
.name("aws-credentials-file")
|
||||
.required(false)
|
||||
.description("Location of the configuration file (e.g., ./conf/bootstrap-aws.conf) that configures the AWS credentials. If not provided, the default AWS credentials will be used.")
|
||||
.addValidator(StandardValidators.FILE_EXISTS_VALIDATOR)
|
||||
.build();
|
||||
|
||||
private final ObjectMapper objectMapper = new ObjectMapper();
|
||||
|
||||
private AWSSecretsManager secretsManager;
|
||||
|
||||
@Override
|
||||
protected List<PropertyDescriptor> getAdditionalSupportedPropertyDescriptors() {
|
||||
return Collections.singletonList(AWS_CREDENTIALS_FILE);
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void additionalInit(final ParameterValueProviderInitializationContext context) {
|
||||
final String awsCredentialsFilename = context.getProperty(AWS_CREDENTIALS_FILE).getValue();
|
||||
try {
|
||||
this.secretsManager = this.configureClient(awsCredentialsFilename);
|
||||
} catch (final IOException e) {
|
||||
throw new IllegalStateException("Could not configure AWS Secrets Manager Client", e);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
protected String getSecretValue(final String secretName, final String keyName) {
|
||||
final GetSecretValueRequest getSecretValueRequest = new GetSecretValueRequest()
|
||||
.withSecretId(secretName);
|
||||
try {
|
||||
final GetSecretValueResult getSecretValueResult = secretsManager.getSecretValue(getSecretValueRequest);
|
||||
|
||||
if (getSecretValueResult.getSecretString() == null) {
|
||||
logger.debug("Secret [{}] not configured", secretName);
|
||||
return null;
|
||||
}
|
||||
|
||||
return parseParameterValue(getSecretValueResult.getSecretString(), keyName);
|
||||
} catch (final ResourceNotFoundException e) {
|
||||
logger.debug("Secret [{}] not found", secretName);
|
||||
return null;
|
||||
} catch (final AWSSecretsManagerException e) {
|
||||
logger.debug("Error retrieving secret [{}]", secretName);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
private String parseParameterValue(final String secretString, final String parameterName) {
|
||||
try {
|
||||
final JsonNode root = objectMapper.readTree(secretString);
|
||||
final JsonNode parameter = root.get(parameterName);
|
||||
if (parameter == null) {
|
||||
logger.debug("Parameter [{}] not found", parameterName);
|
||||
return null;
|
||||
}
|
||||
|
||||
return parameter.textValue();
|
||||
} catch (final JsonProcessingException e) {
|
||||
throw new IllegalArgumentException(String.format("Secret String for [%s] could not be parsed", parameterName), e);
|
||||
}
|
||||
}
|
||||
|
||||
private Properties loadProperties(final String propertiesFilename) throws IOException {
|
||||
final Properties properties = new Properties();
|
||||
|
||||
try (final InputStream in = new FileInputStream(Paths.get(propertiesFilename).toFile())) {
|
||||
properties.load(in);
|
||||
return properties;
|
||||
}
|
||||
}
|
||||
|
||||
AWSSecretsManager configureClient(final String awsCredentialsFilename) throws IOException {
|
||||
if (awsCredentialsFilename == null) {
|
||||
return getDefaultClient();
|
||||
}
|
||||
final Properties properties = loadProperties(awsCredentialsFilename);
|
||||
final String accessKey = properties.getProperty(ACCESS_KEY_PROPS_NAME);
|
||||
final String secretKey = properties.getProperty(SECRET_KEY_PROPS_NAME);
|
||||
final String region = properties.getProperty(REGION_KEY_PROPS_NAME);
|
||||
|
||||
if (isNotBlank(accessKey) && isNotBlank(secretKey) && isNotBlank(region)) {
|
||||
return AWSSecretsManagerClientBuilder.standard()
|
||||
.withRegion(region)
|
||||
.withCredentials(new AWSStaticCredentialsProvider(new BasicAWSCredentials(accessKey, secretKey)))
|
||||
.build();
|
||||
} else {
|
||||
return getDefaultClient();
|
||||
}
|
||||
}
|
||||
|
||||
private AWSSecretsManager getDefaultClient() {
|
||||
return AWSSecretsManagerClientBuilder.standard()
|
||||
.withCredentials(DefaultAWSCredentialsProviderChain.getInstance())
|
||||
.build();
|
||||
}
|
||||
|
||||
private static boolean isNotBlank(final String value) {
|
||||
return value != null && !value.trim().equals("");
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,16 @@
|
|||
# Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
# contributor license agreements. See the NOTICE file distributed with
|
||||
# this work for additional information regarding copyright ownership.
|
||||
# The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
# (the "License"); you may not use this file except in compliance with
|
||||
# the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
org.apache.nifi.stateless.parameter.AwsSecretsManagerParameterValueProvider
|
||||
|
||||
|
|
@ -0,0 +1,223 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.apache.nifi.stateless.parameter;
|
||||
|
||||
import com.amazonaws.services.secretsmanager.AWSSecretsManager;
|
||||
import com.amazonaws.services.secretsmanager.model.GetSecretValueRequest;
|
||||
import com.amazonaws.services.secretsmanager.model.GetSecretValueResult;
|
||||
import com.amazonaws.services.secretsmanager.model.ResourceNotFoundException;
|
||||
import com.fasterxml.jackson.core.JsonProcessingException;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.fasterxml.jackson.databind.node.ObjectNode;
|
||||
import org.apache.nifi.attribute.expression.language.StandardPropertyValue;
|
||||
import org.apache.nifi.components.PropertyDescriptor;
|
||||
import org.apache.nifi.components.PropertyValue;
|
||||
import org.junit.Before;
|
||||
import org.junit.Test;
|
||||
import org.junit.runner.RunWith;
|
||||
import org.mockito.ArgumentMatcher;
|
||||
import org.mockito.Mock;
|
||||
import org.mockito.Spy;
|
||||
import org.mockito.junit.MockitoJUnitRunner;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.Collections;
|
||||
import java.util.HashMap;
|
||||
import java.util.Map;
|
||||
|
||||
import static org.junit.Assert.assertEquals;
|
||||
import static org.junit.Assert.assertNull;
|
||||
import static org.junit.Assert.assertTrue;
|
||||
import static org.mockito.ArgumentMatchers.argThat;
|
||||
import static org.mockito.ArgumentMatchers.eq;
|
||||
import static org.mockito.ArgumentMatchers.isNull;
|
||||
import static org.mockito.Mockito.doReturn;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
@RunWith(MockitoJUnitRunner.class)
|
||||
public class TestSecretsManagerParameterValueProvider {
|
||||
private static final String CONTEXT = "context";
|
||||
private static final String PARAMETER = "param";
|
||||
private static final String VALUE = "secret";
|
||||
private static final String DEFAULT_SECRET_NAME = "Test";
|
||||
private static final String DEFAULT_VALUE = "DefaultValue";
|
||||
|
||||
private static final String CONFIG_FILE = "./conf/my-config.file";
|
||||
|
||||
private final ObjectMapper objectMapper = new ObjectMapper();
|
||||
|
||||
@Spy
|
||||
private AwsSecretsManagerParameterValueProvider provider;
|
||||
|
||||
@Mock
|
||||
private AWSSecretsManager secretsManager;
|
||||
|
||||
@Before
|
||||
public void init() throws IOException {
|
||||
doReturn(secretsManager).when(provider).configureClient(eq(CONFIG_FILE));
|
||||
doReturn(secretsManager).when(provider).configureClient(isNull());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testIsParameterDefined() throws JsonProcessingException {
|
||||
mockGetSecretValue();
|
||||
provider.init(createContext(CONFIG_FILE));
|
||||
|
||||
assertTrue(provider.isParameterDefined(CONTEXT, PARAMETER));
|
||||
|
||||
provider.init(createContext(null));
|
||||
assertTrue(provider.isParameterDefined(CONTEXT, PARAMETER));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testGetParameterValue() throws JsonProcessingException {
|
||||
mockGetSecretValue();
|
||||
|
||||
runGetParameterValueTest(CONFIG_FILE);
|
||||
|
||||
runGetParameterValueTest(null);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testGetParameterValueWithMissingSecretString() throws JsonProcessingException {
|
||||
mockGetSecretValue(CONTEXT, PARAMETER, "value", false, false);
|
||||
mockGetSecretValue(DEFAULT_SECRET_NAME, PARAMETER, DEFAULT_VALUE, false, false);
|
||||
|
||||
provider.init(createContext(CONFIG_FILE));
|
||||
assertNull(provider.getParameterValue(CONTEXT, PARAMETER));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testGetParameterValueWithSecretMapping() throws JsonProcessingException {
|
||||
final String mappedSecretName = "MyMappedSecretName";
|
||||
mockGetSecretValue(mappedSecretName, PARAMETER, VALUE, true, false);
|
||||
|
||||
final Map<String, String> dynamicProperties = new HashMap<>();
|
||||
dynamicProperties.put(CONTEXT, mappedSecretName);
|
||||
provider.init(createContext(CONFIG_FILE, null, dynamicProperties));
|
||||
assertEquals(VALUE, provider.getParameterValue(CONTEXT, PARAMETER));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testGetParameterValueWithNoDefault() throws JsonProcessingException {
|
||||
mockGetSecretValue("Does not exist", PARAMETER, null, false, true);
|
||||
|
||||
provider.init(createContext(CONFIG_FILE, null, Collections.emptyMap()));
|
||||
|
||||
// Nothing to fall back to here
|
||||
assertNull(provider.getParameterValue("Does not exist", PARAMETER));
|
||||
}
|
||||
|
||||
private void runGetParameterValueTest(final String configFileName) throws JsonProcessingException {
|
||||
runGetParameterValueTest(CONTEXT, PARAMETER, configFileName);
|
||||
}
|
||||
|
||||
private void runGetParameterValueTest(final String context, final String parameterName, final String configFileName) throws JsonProcessingException {
|
||||
mockGetSecretValue(DEFAULT_SECRET_NAME, PARAMETER, DEFAULT_VALUE, true, false);
|
||||
mockGetSecretValue("Does not exist", PARAMETER, null, false, true);
|
||||
|
||||
provider.init(createContext(configFileName));
|
||||
assertEquals(VALUE, provider.getParameterValue(context, parameterName));
|
||||
|
||||
// Should fall back to the default context, which does have the parameter
|
||||
assertEquals(DEFAULT_VALUE, provider.getParameterValue("Does not exist", PARAMETER));
|
||||
}
|
||||
|
||||
private void mockGetSecretValue() throws JsonProcessingException {
|
||||
mockGetSecretValue(CONTEXT, PARAMETER, VALUE, true, false);
|
||||
}
|
||||
|
||||
private void mockGetSecretValue(final String context, final String parameterName, final String secretValue, final boolean hasSecretString, final boolean resourceNotFound)
|
||||
throws JsonProcessingException {
|
||||
if (resourceNotFound) {
|
||||
when(secretsManager.getSecretValue(argThat(matchesGetSecretValueRequest(context)))).thenThrow(new ResourceNotFoundException("Not found"));
|
||||
} else {
|
||||
GetSecretValueResult result = new GetSecretValueResult();
|
||||
if (hasSecretString) {
|
||||
result = result.withSecretString(getSecretString(parameterName, secretValue));
|
||||
}
|
||||
when(secretsManager.getSecretValue(argThat(matchesGetSecretValueRequest(context)))).thenReturn(result);
|
||||
}
|
||||
}
|
||||
|
||||
private static String getSecretName(final String context) {
|
||||
return context == null ? DEFAULT_SECRET_NAME : context;
|
||||
}
|
||||
|
||||
private static ParameterValueProviderInitializationContext createContext(final String awsConfigFilename) {
|
||||
return createContext(awsConfigFilename, DEFAULT_SECRET_NAME, Collections.emptyMap());
|
||||
}
|
||||
|
||||
private static ParameterValueProviderInitializationContext createContext(final String awsConfigFilename, final String defaultSecretName, final Map<String, String> dynamicProperties) {
|
||||
return new ParameterValueProviderInitializationContext() {
|
||||
@Override
|
||||
public String getIdentifier() {
|
||||
return null;
|
||||
}
|
||||
|
||||
@Override
|
||||
public PropertyValue getProperty(final PropertyDescriptor descriptor) {
|
||||
if (descriptor.equals(AwsSecretsManagerParameterValueProvider.AWS_CREDENTIALS_FILE)) {
|
||||
return new StandardPropertyValue(awsConfigFilename, null, null);
|
||||
} else if (descriptor.equals(AwsSecretsManagerParameterValueProvider.DEFAULT_SECRET_NAME)) {
|
||||
return new StandardPropertyValue(defaultSecretName, null, null);
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Map<String, String> getAllProperties() {
|
||||
final Map<String, String> properties = new HashMap<>(dynamicProperties);
|
||||
properties.put(AwsSecretsManagerParameterValueProvider.AWS_CREDENTIALS_FILE.getName(), awsConfigFilename);
|
||||
properties.put(AwsSecretsManagerParameterValueProvider.DEFAULT_SECRET_NAME.getName(), defaultSecretName);
|
||||
return properties;
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
private String getSecretString(final String parameterName, final String parameterValue) throws JsonProcessingException {
|
||||
final Map<String, String> parameters = new HashMap<>();
|
||||
parameters.put(parameterName, parameterValue);
|
||||
return getSecretString(parameters);
|
||||
}
|
||||
|
||||
private String getSecretString(final Map<String, String> parameters) throws JsonProcessingException {
|
||||
final ObjectNode root = objectMapper.createObjectNode();
|
||||
for(final Map.Entry<String, String> entry : parameters.entrySet()) {
|
||||
root.put(entry.getKey(), entry.getValue());
|
||||
}
|
||||
return objectMapper.writeValueAsString(root);
|
||||
}
|
||||
|
||||
private static ArgumentMatcher<GetSecretValueRequest> matchesGetSecretValueRequest(final String context) {
|
||||
return new GetSecretValueRequestMatcher(getSecretName(context));
|
||||
}
|
||||
|
||||
private static class GetSecretValueRequestMatcher implements ArgumentMatcher<GetSecretValueRequest> {
|
||||
|
||||
private final String secretId;
|
||||
|
||||
private GetSecretValueRequestMatcher(final String secretId) {
|
||||
this.secretId = secretId;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean matches(final GetSecretValueRequest argument) {
|
||||
return argument != null && argument.getSecretId().equals(secretId);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -50,6 +50,7 @@
|
|||
<module>nifi-aws-service-api</module>
|
||||
<module>nifi-aws-service-api-nar</module>
|
||||
<module>nifi-aws-abstract-processors</module>
|
||||
<module>nifi-aws-parameter-value-providers</module>
|
||||
</modules>
|
||||
|
||||
</project>
|
||||
|
|
|
|||
|
|
@ -0,0 +1,119 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.apache.nifi.stateless.parameter;
|
||||
|
||||
import org.apache.nifi.components.PropertyDescriptor;
|
||||
import org.apache.nifi.components.ValidationResult;
|
||||
import org.apache.nifi.components.Validator;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collections;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
/**
|
||||
* A base class for secret-based <code>ParameterValueProvider</code>s, which map a ParameterContext to a named "Secret" with key/value pairs. This
|
||||
* class allows a default Secret name to be configured for parameters not found in specific ParameterContext Secrets, and uses dynamic user-added
|
||||
* properties to map ParameterContext names to different Secret names. Subclasses must provide the implementation for retrieving the actual
|
||||
* secret values.
|
||||
*/
|
||||
public abstract class AbstractSecretBasedParameterValueProvider extends AbstractParameterValueProvider implements ParameterValueProvider {
|
||||
private static final Validator NON_EMPTY_VALIDATOR = (subject, value, context) ->
|
||||
new ValidationResult.Builder().subject(subject).input(value).valid(value != null && !value.isEmpty()).explanation(subject + " cannot be empty").build();
|
||||
|
||||
public static final PropertyDescriptor DEFAULT_SECRET_NAME = new PropertyDescriptor.Builder()
|
||||
.displayName("Default Secret Name")
|
||||
.name("default-secret-name")
|
||||
.description("The default secret name to use. This secret represents a default Parameter Context if there is not a matching key within the mapped Parameter Context secret")
|
||||
.addValidator(NON_EMPTY_VALIDATOR)
|
||||
.build();
|
||||
|
||||
private List<PropertyDescriptor> descriptors;
|
||||
|
||||
private String defaultSecretName = null;
|
||||
|
||||
private Map<String, String> contextToSecretMapping;
|
||||
|
||||
/**
|
||||
* Define any additional properties.
|
||||
* @return Any additional property descriptors
|
||||
*/
|
||||
protected abstract List<PropertyDescriptor> getAdditionalSupportedPropertyDescriptors();
|
||||
|
||||
/**
|
||||
* Perform any additional initialization based on the context.
|
||||
* @param context The initialization context
|
||||
*/
|
||||
protected abstract void additionalInit(final ParameterValueProviderInitializationContext context);
|
||||
|
||||
/**
|
||||
* Extract the value for the given key from the secret with the given name.
|
||||
* @param secretName The name of a secret
|
||||
* @param keyName The key within the secret
|
||||
* @return The secret value, or null if either the secret or the key is not found
|
||||
*/
|
||||
protected abstract String getSecretValue(final String secretName, final String keyName);
|
||||
|
||||
@Override
|
||||
protected PropertyDescriptor getSupportedDynamicPropertyDescriptor(final String propertyDescriptorName) {
|
||||
return new PropertyDescriptor.Builder()
|
||||
.displayName(propertyDescriptorName)
|
||||
.name(propertyDescriptorName)
|
||||
.dynamic(true)
|
||||
.addValidator(NON_EMPTY_VALIDATOR)
|
||||
.build();
|
||||
}
|
||||
|
||||
@Override
|
||||
protected List<PropertyDescriptor> getSupportedPropertyDescriptors() {
|
||||
return descriptors;
|
||||
}
|
||||
|
||||
@Override
|
||||
protected final void init(final ParameterValueProviderInitializationContext context) {
|
||||
super.init(context);
|
||||
|
||||
final List<PropertyDescriptor> propertyDescriptors = new ArrayList<>(getAdditionalSupportedPropertyDescriptors());
|
||||
propertyDescriptors.add(DEFAULT_SECRET_NAME);
|
||||
this.descriptors = Collections.unmodifiableList(propertyDescriptors);
|
||||
|
||||
defaultSecretName = context.getProperty(DEFAULT_SECRET_NAME).getValue();
|
||||
contextToSecretMapping = new HashMap<>();
|
||||
for (final Map.Entry<String, String> entry : context.getAllProperties().entrySet()) {
|
||||
if (getPropertyDescriptor(entry.getKey()).isDynamic()) {
|
||||
contextToSecretMapping.put(entry.getKey(), entry.getValue());
|
||||
}
|
||||
}
|
||||
this.additionalInit(context);
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean isParameterDefined(final String contextName, final String parameterName) {
|
||||
return getParameterValue(contextName, parameterName) != null;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getParameterValue(final String contextName, final String parameterName) {
|
||||
final String contextBasedValue = getSecretValue(getSecretName(contextName), parameterName);
|
||||
return contextBasedValue != null || defaultSecretName == null ? contextBasedValue : getSecretValue(defaultSecretName, parameterName);
|
||||
}
|
||||
|
||||
private String getSecretName(final String contextName) {
|
||||
return contextToSecretMapping.getOrDefault(contextName, contextName);
|
||||
}
|
||||
}
|
||||
|
|
@ -515,3 +515,60 @@ nifi.stateless.parameter.provider.Vault.name=HashiCorp Vault Provider
|
|||
nifi.stateless.parameter.provider.Vault.type=org.apache.nifi.stateless.parameter.HashiCorpVaultParameterValueProvider
|
||||
nifi.stateless.parameter.provider.Vault.properties.vault-configuration-file=./conf/bootstrap-hashicorp-vault.conf
|
||||
```
|
||||
|
||||
**AWS SecretsManagerParameterValueProvider**
|
||||
|
||||
This provider reads parameter values from AWS SecretsManager. Each AWS secret is mapped to a Parameter Context, with
|
||||
the Secret name representing the Parameter Context name and the key/value pairs in the Secret representing the
|
||||
Parameter names and values.
|
||||
|
||||
The AWS credentials can be configured via the `./conf/bootstrap-aws.conf` file, which comes with NiFi.
|
||||
|
||||
Note: The provided AWS credentials must have the `secretsmanager:GetSecretValue` permission in order to use this provider.
|
||||
An example of creating a single secret in the correct format is:
|
||||
|
||||
```
|
||||
aws secretsmanager create-secret --name "Context" --secret-string '{ "Param": "secretValue", "Param2": "secretValue2" }'
|
||||
```
|
||||
|
||||
In this example, `Context` is the name of a Parameter Context, `Param` is the name of the parameter whose value
|
||||
should be retrieved from the Vault server, and `secretValue` is the actual value of the parameter. Notice that
|
||||
there are multiple parameters stored in this secret: a second parameter named `Param2` has the value of `secretValue2`.
|
||||
|
||||
Alternatively, if you use the AWS Console to create a secret, follow these steps:
|
||||
1. Select a secret type of "Other type of secrets (e.g. API key)"
|
||||
2. Enter one Secret key/value for each Parameter, where the key is the Parameter Name and the value is the Parameter value
|
||||
3. On the next page, enter the name of the Parameter Context as the Secret name. Save the Secret.
|
||||
|
||||
This Parameter Provider allows the following properties:
|
||||
|
||||
| Property Name | Description | Example Value |
|
||||
|---------------|-------------|---------------|
|
||||
| nifi.stateless.parameter.provider.\<key>.properties.aws-credentials-file | The filename of a configuration file optionally specifying the AWS credentials. If this property is not provided, or if the credentials are not provided in the file, the default AWS credentials chain will be followed. | `./conf/bootstrap-aws.conf` |
|
||||
| nifi.stateless.parameter.provider.\<key>.default-secret-name | The default AWS secret name to use. This secret represents a default Parameter Context if there is not a matching key within the mapped Parameter Context secret. | `Default` |
|
||||
|
||||
An example of configuring this provider in the dataflow configuration file is:
|
||||
|
||||
```
|
||||
nifi.stateless.parameter.provider.AWSSecretsManager.name=AWS SecretsManager Provider
|
||||
nifi.stateless.parameter.provider.AWSSecretsManager.type=org.apache.nifi.stateless.parameter.AwsSecretsManagerParameterValueProvider
|
||||
nifi.stateless.parameter.provider.AWSSecretsManager.properties.aws-credentials-file=./conf/bootstrap-aws.conf
|
||||
nifi.stateless.parameter.provider.AWSSecretsManager.properties.default-secret-name=Default
|
||||
nifi.stateless.parameter.provider.AWSSecretsManager.properties.MyContextName=MappedSecretName
|
||||
```
|
||||
|
||||
This provider will map each ParameterContext to a secret of the same name. In the above example, the Parameter Context named `MyContextName`
|
||||
will instead be mapped to a secret named `MappedSecretName`.
|
||||
|
||||
Additionally, the provider will assume there is a secret named `Default` that may contain any parameters not found in other mapped ParameterContexts.
|
||||
For example, assume the following dataflow and AWS SecretsManager configuration:
|
||||
|
||||
- Flow contains a ParameterContext named `ABC`, with parameters `foo` and `bar`.
|
||||
- Flow contains a ParameterContext named `MyContextName`, with parameter `baz`.
|
||||
- AWS SecretsManager contains a secret named `ABC`, with a key of `foo`.
|
||||
- AWS SecretsManager also contains a secret named `Default`, with keys `foo` and `bar`.
|
||||
- AWS SecretsManager also contains a secret named `MappedSecretName`, with a key of `baz`.
|
||||
|
||||
When executing the dataflow with the above provider configuration, the `foo` parameter will be pulled from the `ABC` secret, since it was found directly in the mapped secret.
|
||||
However, the `bar` parameter will be pulled from the `Default` secret, because it was not found in the `ABC` secret, but was found in the `Default` secret, which is indicated by the `default-secret-name` property.
|
||||
Additionally, Stateless will pull the `baz` parameter from the `MappedSecretName` secret because of the `MyContextName` mapping property.
|
||||
Loading…
Reference in New Issue