+ *
+ */
+
+@Tags({"aws", "secretsmanager", "secrets", "manager"})
+@CapabilityDescription("Fetches parameters from AWS SecretsManager. Each secret becomes a Parameter group, which can map to a Parameter Context, with " +
+ "key/value pairs in the secret mapping to Parameters in the group.")
+public class AwsSecretsManagerParameterProvider extends AbstractParameterProvider implements VerifiableParameterProvider {
+
+ public static final PropertyDescriptor SECRET_NAME_PATTERN = new PropertyDescriptor.Builder()
+ .name("secret-name-pattern")
+ .displayName("Secret Name Pattern")
+ .description("A Regular Expression matching on Secret Name that identifies Secrets whose parameters should be fetched. " +
+ "Any secrets whose names do not match this pattern will not be fetched.")
+ .addValidator(StandardValidators.REGULAR_EXPRESSION_VALIDATOR)
+ .required(true)
+ .defaultValue(".*")
+ .build();
+ /**
+ * AWS credentials provider service
+ *
+ * @see AWSCredentialsProvider
+ */
+ public static final PropertyDescriptor AWS_CREDENTIALS_PROVIDER_SERVICE = new PropertyDescriptor.Builder()
+ .name("aws-credentials-provider-service")
+ .displayName("AWS Credentials Provider Service")
+ .description("Service used to obtain an Amazon Web Services Credentials Provider")
+ .required(true)
+ .identifiesControllerService(AWSCredentialsProviderService.class)
+ .build();
+
+ public static final PropertyDescriptor REGION = new PropertyDescriptor.Builder()
+ .name("aws-region")
+ .displayName("Region")
+ .required(true)
+ .allowableValues(getAvailableRegions())
+ .defaultValue(createAllowableValue(Regions.DEFAULT_REGION).getValue())
+ .build();
+
+ public static final PropertyDescriptor TIMEOUT = new PropertyDescriptor.Builder()
+ .name("aws-communications-timeout")
+ .displayName("Communications Timeout")
+ .required(true)
+ .addValidator(StandardValidators.TIME_PERIOD_VALIDATOR)
+ .defaultValue("30 secs")
+ .build();
+
+ public static final PropertyDescriptor SSL_CONTEXT_SERVICE = new PropertyDescriptor.Builder()
+ .name("aws-ssl-context-service")
+ .displayName("SSL Context Service")
+ .description("Specifies an optional SSL Context Service that, if provided, will be used to create connections")
+ .required(false)
+ .identifiesControllerService(SSLContextService.class)
+ .build();
+
+ private static final String DEFAULT_USER_AGENT = "NiFi";
+ private static final Protocol DEFAULT_PROTOCOL = Protocol.HTTPS;
+ private static final List PROPERTIES = Collections.unmodifiableList(Arrays.asList(
+ SECRET_NAME_PATTERN,
+ REGION,
+ AWS_CREDENTIALS_PROVIDER_SERVICE,
+ TIMEOUT,
+ SSL_CONTEXT_SERVICE
+ ));
+
+ private final ObjectMapper objectMapper = new ObjectMapper();
+
+ @Override
+ protected List getSupportedPropertyDescriptors() {
+ return PROPERTIES;
+ }
+
+ @Override
+ public List fetchParameters(final ConfigurationContext context) {
+ AWSSecretsManager secretsManager = this.configureClient(context);
+
+ final List groups = new ArrayList<>();
+ final ListSecretsRequest listSecretsRequest = new ListSecretsRequest();
+ final ListSecretsResult listSecretsResult = secretsManager.listSecrets(listSecretsRequest);
+ for (final SecretListEntry entry : listSecretsResult.getSecretList()) {
+ groups.addAll(fetchSecret(secretsManager, context, entry.getName()));
+ }
+
+ return groups;
+ }
+
+ @Override
+ public List verify(final ConfigurationContext context, final ComponentLog verificationLogger) {
+ final List results = new ArrayList<>();
+
+ try {
+ final List parameterGroups = fetchParameters(context);
+ int parameterCount = 0;
+ for (final ParameterGroup group : parameterGroups) {
+ parameterCount += group.getParameters().size();
+ }
+ results.add(new ConfigVerificationResult.Builder()
+ .outcome(ConfigVerificationResult.Outcome.SUCCESSFUL)
+ .verificationStepName("Fetch Parameters")
+ .explanation(String.format("Fetched secret keys [%d] as parameters, across groups [%d]",
+ parameterCount, parameterGroups.size()))
+ .build());
+ } catch (final Exception e) {
+ verificationLogger.error("Failed to fetch parameters", e);
+ results.add(new ConfigVerificationResult.Builder()
+ .outcome(ConfigVerificationResult.Outcome.FAILED)
+ .verificationStepName("Fetch Parameters")
+ .explanation("Failed to fetch parameters: " + e.getMessage())
+ .build());
+ }
+ return results;
+ }
+
+ private List fetchSecret(final AWSSecretsManager secretsManager, final ConfigurationContext context, final String secretName) {
+ final List groups = new ArrayList<>();
+ final Pattern secretNamePattern = Pattern.compile(context.getProperty(SECRET_NAME_PATTERN).getValue());
+
+ final List parameters = new ArrayList<>();
+
+ if (!secretNamePattern.matcher(secretName).matches()) {
+ getLogger().debug("Secret [{}] does not match the secret name pattern {}", secretName, secretNamePattern);
+ return groups;
+ }
+
+ final GetSecretValueRequest getSecretValueRequest = new GetSecretValueRequest().withSecretId(secretName);
+ try {
+ final GetSecretValueResult getSecretValueResult = secretsManager.getSecretValue(getSecretValueRequest);
+
+ if (getSecretValueResult.getSecretString() == null) {
+ getLogger().debug("Secret [{}] is not configured", secretName);
+ return groups;
+ }
+
+ final ObjectNode secretObject = parseSecret(getSecretValueResult.getSecretString());
+ if (secretObject == null) {
+ getLogger().debug("Secret [{}] is not in the expected JSON key/value format", secretName);
+ return groups;
+ }
+
+ for (final Iterator> it = secretObject.fields(); it.hasNext(); ) {
+ final Map.Entry field = it.next();
+ final String parameterName = field.getKey();
+ final String parameterValue = field.getValue().textValue();
+ if (parameterValue == null) {
+ getLogger().debug("Secret [{}] Parameter [{}] has no value", secretName, parameterName);
+ continue;
+ }
+
+ parameters.add(createParameter(parameterName, parameterValue));
+ }
+
+ groups.add(new ParameterGroup(secretName, parameters));
+
+ return groups;
+ } catch (final ResourceNotFoundException e) {
+ throw new IllegalStateException(String.format("Secret %s not found", secretName), e);
+ } catch (final AWSSecretsManagerException e) {
+ throw new IllegalStateException("Error retrieving secret " + secretName, e);
+ }
+ }
+
+ private Parameter createParameter(final String parameterName, final String parameterValue) {
+ final ParameterDescriptor parameterDescriptor = new ParameterDescriptor.Builder().name(parameterName).build();
+ return new Parameter(parameterDescriptor, parameterValue, null, true);
+ }
+
+ protected ClientConfiguration createConfiguration(final ConfigurationContext context) {
+ final ClientConfiguration config = new ClientConfiguration();
+ config.setMaxErrorRetry(0);
+ config.setUserAgentPrefix(DEFAULT_USER_AGENT);
+ config.setProtocol(DEFAULT_PROTOCOL);
+ final int commsTimeout = context.getProperty(TIMEOUT).asTimePeriod(TimeUnit.MILLISECONDS).intValue();
+ config.setConnectionTimeout(commsTimeout);
+ config.setSocketTimeout(commsTimeout);
+
+ final SSLContextService sslContextService = context.getProperty(SSL_CONTEXT_SERVICE).asControllerService(SSLContextService.class);
+ if (sslContextService != null) {
+ final SSLContext sslContext = sslContextService.createContext();
+ SdkTLSSocketFactory sdkTLSSocketFactory = new SdkTLSSocketFactory(sslContext, SdkTLSSocketFactory.BROWSER_COMPATIBLE_HOSTNAME_VERIFIER);
+ config.getApacheHttpClientConfig().setSslSocketFactory(sdkTLSSocketFactory);
+ }
+
+ return config;
+ }
+
+ private ObjectNode parseSecret(final String secretString) {
+ try {
+ final JsonNode root = objectMapper.readTree(secretString);
+ if (root instanceof ObjectNode) {
+ return (ObjectNode) root;
+ }
+ return null;
+ } catch (final JsonProcessingException e) {
+ getLogger().debug("Error parsing JSON", e);
+ return null;
+ }
+ }
+
+ AWSSecretsManager configureClient(final ConfigurationContext context) {
+ return AWSSecretsManagerClientBuilder.standard()
+ .withRegion(context.getProperty(REGION).getValue())
+ .withClientConfiguration(createConfiguration(context))
+ .withCredentials(getCredentialsProvider(context))
+ .build();
+ }
+
+ /**
+ * Get credentials provider using the {@link AWSCredentialsProviderService}
+ * @param context the configuration context
+ * @return AWSCredentialsProvider the credential provider
+ * @see AWSCredentialsProvider
+ */
+ protected AWSCredentialsProvider getCredentialsProvider(final ConfigurationContext context) {
+
+ final AWSCredentialsProviderService awsCredentialsProviderService =
+ context.getProperty(AWS_CREDENTIALS_PROVIDER_SERVICE).asControllerService(AWSCredentialsProviderService.class);
+
+ return awsCredentialsProviderService.getCredentialsProvider();
+
+ }
+
+ private static AllowableValue createAllowableValue(final Regions region) {
+ return new AllowableValue(region.getName(), region.getDescription(), "AWS Region Code : " + region.getName());
+ }
+
+ private static AllowableValue[] getAvailableRegions() {
+ final List values = new ArrayList<>();
+ for (final Regions region : Regions.values()) {
+ values.add(createAllowableValue(region));
+ }
+ return values.toArray(new AllowableValue[0]);
+ }
+}
diff --git a/nifi-nar-bundles/nifi-aws-bundle/nifi-aws-parameter-providers/src/main/resources/META-INF/services/org.apache.nifi.parameter.ParameterProvider b/nifi-nar-bundles/nifi-aws-bundle/nifi-aws-parameter-providers/src/main/resources/META-INF/services/org.apache.nifi.parameter.ParameterProvider
new file mode 100644
index 0000000000..e4fad1ecbc
--- /dev/null
+++ b/nifi-nar-bundles/nifi-aws-bundle/nifi-aws-parameter-providers/src/main/resources/META-INF/services/org.apache.nifi.parameter.ParameterProvider
@@ -0,0 +1,16 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+org.apache.nifi.parameter.aws.AwsSecretsManagerParameterProvider
+
diff --git a/nifi-nar-bundles/nifi-aws-bundle/nifi-aws-parameter-providers/src/main/resources/docs/org.apache.nifi.parameter.aws.AwsSecretsManagerParameterProvider/additionalDetails.html b/nifi-nar-bundles/nifi-aws-bundle/nifi-aws-parameter-providers/src/main/resources/docs/org.apache.nifi.parameter.aws.AwsSecretsManagerParameterProvider/additionalDetails.html
new file mode 100644
index 0000000000..5974197a8f
--- /dev/null
+++ b/nifi-nar-bundles/nifi-aws-bundle/nifi-aws-parameter-providers/src/main/resources/docs/org.apache.nifi.parameter.aws.AwsSecretsManagerParameterProvider/additionalDetails.html
@@ -0,0 +1,62 @@
+
+
+
+
+
+ AWSSecretsManagerParameterProvider
+
+
+
+
+
Mapping AWS Secrets to Parameter Contexts
+
+
+ The AwsSecretsManagerParameterProvider maps a Secret to a Parameter Context, with key/value pairs in the Secret
+ mapping to parameters. To create a compatible secret from the AWS Console:
+
+
+
+
From the Secrets Manager service, click the "Store a new Secret" button
+
Select "Other type of secret"
+
Under "Key/value", enter your parameters, with the parameter names being the keys and the parameter values
+ being the values. Click Next.
+
Enter the Secret name. This will determine which Parameter Context receives the parameters. Continue
+ through the rest of the wizard and finally click the "Store" button.
+
+
+
+ Alternatively, from the command line, run a command like the following:
+
In this example, [Context] should be the intended name of the Parameter Context, [Param] and [Param2] should be
+ parameter names, and [secretValue] and [secretValue2] should be the values of each respective parameter.
+
+
Configuring the Parameter Provider
+
+
+ AWS Secrets must be explicitly matched in the "Secret Name Pattern" property in order for them to be fetched. This
+ prevents more than the intended Secrets from being pulled into NiFi.
+
+
+
+
+
+
+
diff --git a/nifi-nar-bundles/nifi-aws-bundle/nifi-aws-parameter-providers/src/test/java/org/apache/nifi/parameter/aws/TestAwsSecretsManagerParameterProvider.java b/nifi-nar-bundles/nifi-aws-bundle/nifi-aws-parameter-providers/src/test/java/org/apache/nifi/parameter/aws/TestAwsSecretsManagerParameterProvider.java
new file mode 100644
index 0000000000..8b96a6e7f0
--- /dev/null
+++ b/nifi-nar-bundles/nifi-aws-bundle/nifi-aws-parameter-providers/src/test/java/org/apache/nifi/parameter/aws/TestAwsSecretsManagerParameterProvider.java
@@ -0,0 +1,199 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.parameter.aws;
+
+import com.amazonaws.services.secretsmanager.AWSSecretsManager;
+import com.amazonaws.services.secretsmanager.model.AWSSecretsManagerException;
+import com.amazonaws.services.secretsmanager.model.GetSecretValueRequest;
+import com.amazonaws.services.secretsmanager.model.GetSecretValueResult;
+import com.amazonaws.services.secretsmanager.model.ListSecretsResult;
+import com.amazonaws.services.secretsmanager.model.SecretListEntry;
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import org.apache.nifi.components.ConfigVerificationResult;
+import org.apache.nifi.components.PropertyDescriptor;
+import org.apache.nifi.parameter.Parameter;
+import org.apache.nifi.parameter.ParameterDescriptor;
+import org.apache.nifi.parameter.ParameterGroup;
+import org.apache.nifi.parameter.VerifiableParameterProvider;
+import org.apache.nifi.reporting.InitializationException;
+import org.apache.nifi.util.MockComponentLog;
+import org.apache.nifi.util.MockConfigurationContext;
+import org.apache.nifi.util.MockParameterProviderInitializationContext;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.extension.ExtendWith;
+import org.mockito.ArgumentMatcher;
+import org.mockito.Mock;
+import org.mockito.junit.jupiter.MockitoExtension;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.argThat;
+import static org.mockito.Mockito.doReturn;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.spy;
+import static org.mockito.Mockito.when;
+
+@ExtendWith(MockitoExtension.class)
+public class TestAwsSecretsManagerParameterProvider {
+
+ @Mock
+ private AWSSecretsManager defaultSecretsManager;
+
+ @Mock
+ private ListSecretsResult listSecretsResult;
+
+ final ObjectMapper objectMapper = new ObjectMapper();
+
+ final List mySecretParameters = Arrays.asList(
+ parameter("paramA", "valueA"),
+ parameter("paramB", "valueB"),
+ parameter("otherC", "valueOther"),
+ parameter("paramD", "valueD"),
+ parameter("nonSensitiveE", "valueE"),
+ parameter("otherF", "valueF")
+ );
+ final List otherSecretParameters = Arrays.asList(
+ parameter("paramG", "valueG"),
+ parameter("otherH", "valueOther")
+ );
+ final List mockParameterGroups = Arrays.asList(
+ new ParameterGroup("MySecret", mySecretParameters),
+ new ParameterGroup("OtherSecret", otherSecretParameters)
+ );
+
+ @Test
+ public void testFetchParametersWithNoSecrets() throws InitializationException {
+ final List expectedGroups = Collections.singletonList(new ParameterGroup("MySecret", Collections.emptyList()));
+ runProviderTest(mockSecretsManager(expectedGroups), 0, ConfigVerificationResult.Outcome.SUCCESSFUL);
+ }
+
+ @Test
+ public void testFetchParameters() throws InitializationException {
+ runProviderTest(mockSecretsManager(mockParameterGroups), 8, ConfigVerificationResult.Outcome.SUCCESSFUL);
+ }
+
+ @Test
+ public void testFetchParametersListFailure() throws InitializationException {
+ when(defaultSecretsManager.listSecrets(any())).thenThrow(new AWSSecretsManagerException("Fake exception"));
+ runProviderTest(defaultSecretsManager, 0, ConfigVerificationResult.Outcome.FAILED);
+ }
+
+ @Test
+ public void testFetchParametersGetSecretFailure() throws InitializationException {
+ final List secretList = Collections.singletonList(new SecretListEntry().withName("MySecret"));
+ when(listSecretsResult.getSecretList()).thenReturn(secretList);
+ when(defaultSecretsManager.listSecrets(any())).thenReturn(listSecretsResult);
+ when(defaultSecretsManager.getSecretValue(argThat(matchesGetSecretValueRequest("MySecret")))).thenThrow(new AWSSecretsManagerException("Fake exception"));
+ runProviderTest(defaultSecretsManager, 0, ConfigVerificationResult.Outcome.FAILED);
+ }
+
+ private AwsSecretsManagerParameterProvider getParameterProvider() {
+ return spy(new AwsSecretsManagerParameterProvider());
+ }
+
+ private AWSSecretsManager mockSecretsManager(final List mockGroup) {
+ final AWSSecretsManager secretsManager = mock(AWSSecretsManager.class);
+
+ final List secretList = mockGroup.stream()
+ .map(group -> new SecretListEntry().withName(group.getGroupName()))
+ .collect(Collectors.toList());
+ when(listSecretsResult.getSecretList()).thenReturn(secretList);
+ when(secretsManager.listSecrets(any())).thenReturn(listSecretsResult);
+
+ mockGroup.forEach(group -> {
+ final String groupName = group.getGroupName();
+ final Map keyValues = group.getParameters().stream().collect(Collectors.toMap(
+ param -> param.getDescriptor().getName(),
+ Parameter::getValue));
+ final String secretString;
+ try {
+ secretString = objectMapper.writeValueAsString(keyValues);
+ final GetSecretValueResult result = new GetSecretValueResult().withName(groupName).withSecretString(secretString);
+ when(secretsManager.getSecretValue(argThat(matchesGetSecretValueRequest(groupName))))
+ .thenReturn(result);
+ } catch (final JsonProcessingException e) {
+ throw new IllegalStateException(e);
+ }
+ });
+ return secretsManager;
+ }
+
+ private List runProviderTest(final AWSSecretsManager secretsManager, final int expectedCount,
+ final ConfigVerificationResult.Outcome expectedOutcome) throws InitializationException {
+
+ final AwsSecretsManagerParameterProvider parameterProvider = getParameterProvider();
+ doReturn(secretsManager).when(parameterProvider).configureClient(any());
+ final MockParameterProviderInitializationContext initContext = new MockParameterProviderInitializationContext("id", "name",
+ new MockComponentLog("providerId", parameterProvider));
+ parameterProvider.initialize(initContext);
+
+ final Map properties = new HashMap<>();
+ final MockConfigurationContext mockConfigurationContext = new MockConfigurationContext(properties, null);
+
+ List parameterGroups = new ArrayList<>();
+ // Verify parameter fetching
+ if (expectedOutcome == ConfigVerificationResult.Outcome.FAILED) {
+ assertThrows(RuntimeException.class, () -> parameterProvider.fetchParameters(mockConfigurationContext));
+ } else {
+ parameterGroups = parameterProvider.fetchParameters(mockConfigurationContext);
+ final int parameterCount = (int) parameterGroups.stream()
+ .flatMap(group -> group.getParameters().stream())
+ .count();
+ assertEquals(expectedCount, parameterCount);
+ }
+
+ // Verify config verification
+ final List results = ((VerifiableParameterProvider) parameterProvider).verify(mockConfigurationContext, initContext.getLogger());
+
+ assertEquals(1, results.size());
+ assertEquals(expectedOutcome, results.get(0).getOutcome());
+
+ return parameterGroups;
+ }
+
+ private static Parameter parameter(final String name, final String value) {
+ return new Parameter(new ParameterDescriptor.Builder().name(name).build(), value);
+ }
+
+ private static ArgumentMatcher matchesGetSecretValueRequest(final String groupName) {
+ return new GetSecretValueRequestMatcher(groupName);
+ }
+
+ private static class GetSecretValueRequestMatcher implements ArgumentMatcher {
+
+ private final String secretId;
+
+ private GetSecretValueRequestMatcher(final String secretId) {
+ this.secretId = secretId;
+ }
+
+ @Override
+ public boolean matches(final GetSecretValueRequest argument) {
+ return argument != null && argument.getSecretId().equals(secretId);
+ }
+ }
+}
diff --git a/nifi-nar-bundles/nifi-aws-bundle/pom.xml b/nifi-nar-bundles/nifi-aws-bundle/pom.xml
index a6ab4fc31e..fc6a89e7e9 100644
--- a/nifi-nar-bundles/nifi-aws-bundle/pom.xml
+++ b/nifi-nar-bundles/nifi-aws-bundle/pom.xml
@@ -37,5 +37,6 @@
nifi-aws-service-api-narnifi-aws-abstract-processorsnifi-aws-parameter-value-providers
+ nifi-aws-parameter-providers