From 315e54a812947494f1fb32fb49997505f0339d3c Mon Sep 17 00:00:00 2001 From: Joe Gresock Date: Sat, 10 Sep 2022 10:01:58 -0400 Subject: [PATCH] NIFI-9636: Adding AwsSecretsManagerParameterProvider This closes #6392 Signed-off-by: David Handermann --- .../nifi-aws-bundle/nifi-aws-nar/pom.xml | 5 + .../nifi-aws-parameter-providers/pom.xml | 75 +++++ .../AwsSecretsManagerParameterProvider.java | 297 ++++++++++++++++++ ...rg.apache.nifi.parameter.ParameterProvider | 16 + .../additionalDetails.html | 62 ++++ ...estAwsSecretsManagerParameterProvider.java | 199 ++++++++++++ nifi-nar-bundles/nifi-aws-bundle/pom.xml | 1 + 7 files changed, 655 insertions(+) create mode 100644 nifi-nar-bundles/nifi-aws-bundle/nifi-aws-parameter-providers/pom.xml create mode 100644 nifi-nar-bundles/nifi-aws-bundle/nifi-aws-parameter-providers/src/main/java/org/apache/nifi/parameter/aws/AwsSecretsManagerParameterProvider.java create mode 100644 nifi-nar-bundles/nifi-aws-bundle/nifi-aws-parameter-providers/src/main/resources/META-INF/services/org.apache.nifi.parameter.ParameterProvider create mode 100644 nifi-nar-bundles/nifi-aws-bundle/nifi-aws-parameter-providers/src/main/resources/docs/org.apache.nifi.parameter.aws.AwsSecretsManagerParameterProvider/additionalDetails.html create mode 100644 nifi-nar-bundles/nifi-aws-bundle/nifi-aws-parameter-providers/src/test/java/org/apache/nifi/parameter/aws/TestAwsSecretsManagerParameterProvider.java diff --git a/nifi-nar-bundles/nifi-aws-bundle/nifi-aws-nar/pom.xml b/nifi-nar-bundles/nifi-aws-bundle/nifi-aws-nar/pom.xml index ea980b7490..984f1b179b 100644 --- a/nifi-nar-bundles/nifi-aws-bundle/nifi-aws-nar/pom.xml +++ b/nifi-nar-bundles/nifi-aws-bundle/nifi-aws-nar/pom.xml @@ -41,6 +41,11 @@ nifi-aws-processors 1.18.0-SNAPSHOT + + org.apache.nifi + nifi-aws-parameter-providers + 1.18.0-SNAPSHOT + org.apache.nifi nifi-aws-parameter-value-providers diff --git a/nifi-nar-bundles/nifi-aws-bundle/nifi-aws-parameter-providers/pom.xml b/nifi-nar-bundles/nifi-aws-bundle/nifi-aws-parameter-providers/pom.xml new file mode 100644 index 0000000000..282f89f576 --- /dev/null +++ b/nifi-nar-bundles/nifi-aws-bundle/nifi-aws-parameter-providers/pom.xml @@ -0,0 +1,75 @@ + + + + 4.0.0 + + org.apache.nifi + nifi-aws-bundle + 1.18.0-SNAPSHOT + + + nifi-aws-parameter-providers + jar + + + + org.apache.nifi + nifi-api + + + org.apache.nifi + nifi-aws-service-api + 1.18.0-SNAPSHOT + provided + + + org.apache.nifi + nifi-ssl-context-service-api + 1.18.0-SNAPSHOT + provided + + + org.apache.nifi + nifi-utils + 1.18.0-SNAPSHOT + provided + + + org.slf4j + jcl-over-slf4j + + + com.fasterxml.jackson.core + jackson-databind + + + com.amazonaws + aws-java-sdk-secretsmanager + + + org.apache.nifi + nifi-expression-language + 1.18.0-SNAPSHOT + test + + + org.apache.nifi + nifi-mock + 1.18.0-SNAPSHOT + test + + + diff --git a/nifi-nar-bundles/nifi-aws-bundle/nifi-aws-parameter-providers/src/main/java/org/apache/nifi/parameter/aws/AwsSecretsManagerParameterProvider.java b/nifi-nar-bundles/nifi-aws-bundle/nifi-aws-parameter-providers/src/main/java/org/apache/nifi/parameter/aws/AwsSecretsManagerParameterProvider.java new file mode 100644 index 0000000000..4cab6fa9a2 --- /dev/null +++ b/nifi-nar-bundles/nifi-aws-bundle/nifi-aws-parameter-providers/src/main/java/org/apache/nifi/parameter/aws/AwsSecretsManagerParameterProvider.java @@ -0,0 +1,297 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.parameter.aws; + +import com.amazonaws.ClientConfiguration; +import com.amazonaws.Protocol; +import com.amazonaws.auth.AWSCredentialsProvider; +import com.amazonaws.http.conn.ssl.SdkTLSSocketFactory; +import com.amazonaws.regions.Regions; +import com.amazonaws.services.secretsmanager.AWSSecretsManager; +import com.amazonaws.services.secretsmanager.AWSSecretsManagerClientBuilder; +import com.amazonaws.services.secretsmanager.model.AWSSecretsManagerException; +import com.amazonaws.services.secretsmanager.model.GetSecretValueRequest; +import com.amazonaws.services.secretsmanager.model.GetSecretValueResult; +import com.amazonaws.services.secretsmanager.model.ListSecretsRequest; +import com.amazonaws.services.secretsmanager.model.ListSecretsResult; +import com.amazonaws.services.secretsmanager.model.ResourceNotFoundException; +import com.amazonaws.services.secretsmanager.model.SecretListEntry; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ObjectNode; +import org.apache.nifi.annotation.documentation.CapabilityDescription; +import org.apache.nifi.annotation.documentation.Tags; +import org.apache.nifi.components.AllowableValue; +import org.apache.nifi.components.ConfigVerificationResult; +import org.apache.nifi.components.PropertyDescriptor; +import org.apache.nifi.controller.ConfigurationContext; +import org.apache.nifi.logging.ComponentLog; +import org.apache.nifi.parameter.AbstractParameterProvider; +import org.apache.nifi.parameter.Parameter; +import org.apache.nifi.parameter.ParameterDescriptor; +import org.apache.nifi.parameter.ParameterGroup; +import org.apache.nifi.parameter.VerifiableParameterProvider; +import org.apache.nifi.processor.util.StandardValidators; +import org.apache.nifi.processors.aws.credentials.provider.service.AWSCredentialsProviderService; +import org.apache.nifi.ssl.SSLContextService; + +import javax.net.ssl.SSLContext; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.concurrent.TimeUnit; +import java.util.regex.Pattern; + +/** + * Reads secrets from AWS Secrets Manager to provide parameter values. Secrets must be created similar to the following AWS cli command:

+ * aws secretsmanager create-secret --name "[Context]" --secret-string '{ "[Param]": "[secretValue]", "[Param2]": "[secretValue2]" }'

+ * + */ + +@Tags({"aws", "secretsmanager", "secrets", "manager"}) +@CapabilityDescription("Fetches parameters from AWS SecretsManager. Each secret becomes a Parameter group, which can map to a Parameter Context, with " + + "key/value pairs in the secret mapping to Parameters in the group.") +public class AwsSecretsManagerParameterProvider extends AbstractParameterProvider implements VerifiableParameterProvider { + + public static final PropertyDescriptor SECRET_NAME_PATTERN = new PropertyDescriptor.Builder() + .name("secret-name-pattern") + .displayName("Secret Name Pattern") + .description("A Regular Expression matching on Secret Name that identifies Secrets whose parameters should be fetched. " + + "Any secrets whose names do not match this pattern will not be fetched.") + .addValidator(StandardValidators.REGULAR_EXPRESSION_VALIDATOR) + .required(true) + .defaultValue(".*") + .build(); + /** + * AWS credentials provider service + * + * @see AWSCredentialsProvider + */ + public static final PropertyDescriptor AWS_CREDENTIALS_PROVIDER_SERVICE = new PropertyDescriptor.Builder() + .name("aws-credentials-provider-service") + .displayName("AWS Credentials Provider Service") + .description("Service used to obtain an Amazon Web Services Credentials Provider") + .required(true) + .identifiesControllerService(AWSCredentialsProviderService.class) + .build(); + + public static final PropertyDescriptor REGION = new PropertyDescriptor.Builder() + .name("aws-region") + .displayName("Region") + .required(true) + .allowableValues(getAvailableRegions()) + .defaultValue(createAllowableValue(Regions.DEFAULT_REGION).getValue()) + .build(); + + public static final PropertyDescriptor TIMEOUT = new PropertyDescriptor.Builder() + .name("aws-communications-timeout") + .displayName("Communications Timeout") + .required(true) + .addValidator(StandardValidators.TIME_PERIOD_VALIDATOR) + .defaultValue("30 secs") + .build(); + + public static final PropertyDescriptor SSL_CONTEXT_SERVICE = new PropertyDescriptor.Builder() + .name("aws-ssl-context-service") + .displayName("SSL Context Service") + .description("Specifies an optional SSL Context Service that, if provided, will be used to create connections") + .required(false) + .identifiesControllerService(SSLContextService.class) + .build(); + + private static final String DEFAULT_USER_AGENT = "NiFi"; + private static final Protocol DEFAULT_PROTOCOL = Protocol.HTTPS; + private static final List PROPERTIES = Collections.unmodifiableList(Arrays.asList( + SECRET_NAME_PATTERN, + REGION, + AWS_CREDENTIALS_PROVIDER_SERVICE, + TIMEOUT, + SSL_CONTEXT_SERVICE + )); + + private final ObjectMapper objectMapper = new ObjectMapper(); + + @Override + protected List getSupportedPropertyDescriptors() { + return PROPERTIES; + } + + @Override + public List fetchParameters(final ConfigurationContext context) { + AWSSecretsManager secretsManager = this.configureClient(context); + + final List groups = new ArrayList<>(); + final ListSecretsRequest listSecretsRequest = new ListSecretsRequest(); + final ListSecretsResult listSecretsResult = secretsManager.listSecrets(listSecretsRequest); + for (final SecretListEntry entry : listSecretsResult.getSecretList()) { + groups.addAll(fetchSecret(secretsManager, context, entry.getName())); + } + + return groups; + } + + @Override + public List verify(final ConfigurationContext context, final ComponentLog verificationLogger) { + final List results = new ArrayList<>(); + + try { + final List parameterGroups = fetchParameters(context); + int parameterCount = 0; + for (final ParameterGroup group : parameterGroups) { + parameterCount += group.getParameters().size(); + } + results.add(new ConfigVerificationResult.Builder() + .outcome(ConfigVerificationResult.Outcome.SUCCESSFUL) + .verificationStepName("Fetch Parameters") + .explanation(String.format("Fetched secret keys [%d] as parameters, across groups [%d]", + parameterCount, parameterGroups.size())) + .build()); + } catch (final Exception e) { + verificationLogger.error("Failed to fetch parameters", e); + results.add(new ConfigVerificationResult.Builder() + .outcome(ConfigVerificationResult.Outcome.FAILED) + .verificationStepName("Fetch Parameters") + .explanation("Failed to fetch parameters: " + e.getMessage()) + .build()); + } + return results; + } + + private List fetchSecret(final AWSSecretsManager secretsManager, final ConfigurationContext context, final String secretName) { + final List groups = new ArrayList<>(); + final Pattern secretNamePattern = Pattern.compile(context.getProperty(SECRET_NAME_PATTERN).getValue()); + + final List parameters = new ArrayList<>(); + + if (!secretNamePattern.matcher(secretName).matches()) { + getLogger().debug("Secret [{}] does not match the secret name pattern {}", secretName, secretNamePattern); + return groups; + } + + final GetSecretValueRequest getSecretValueRequest = new GetSecretValueRequest().withSecretId(secretName); + try { + final GetSecretValueResult getSecretValueResult = secretsManager.getSecretValue(getSecretValueRequest); + + if (getSecretValueResult.getSecretString() == null) { + getLogger().debug("Secret [{}] is not configured", secretName); + return groups; + } + + final ObjectNode secretObject = parseSecret(getSecretValueResult.getSecretString()); + if (secretObject == null) { + getLogger().debug("Secret [{}] is not in the expected JSON key/value format", secretName); + return groups; + } + + for (final Iterator> it = secretObject.fields(); it.hasNext(); ) { + final Map.Entry field = it.next(); + final String parameterName = field.getKey(); + final String parameterValue = field.getValue().textValue(); + if (parameterValue == null) { + getLogger().debug("Secret [{}] Parameter [{}] has no value", secretName, parameterName); + continue; + } + + parameters.add(createParameter(parameterName, parameterValue)); + } + + groups.add(new ParameterGroup(secretName, parameters)); + + return groups; + } catch (final ResourceNotFoundException e) { + throw new IllegalStateException(String.format("Secret %s not found", secretName), e); + } catch (final AWSSecretsManagerException e) { + throw new IllegalStateException("Error retrieving secret " + secretName, e); + } + } + + private Parameter createParameter(final String parameterName, final String parameterValue) { + final ParameterDescriptor parameterDescriptor = new ParameterDescriptor.Builder().name(parameterName).build(); + return new Parameter(parameterDescriptor, parameterValue, null, true); + } + + protected ClientConfiguration createConfiguration(final ConfigurationContext context) { + final ClientConfiguration config = new ClientConfiguration(); + config.setMaxErrorRetry(0); + config.setUserAgentPrefix(DEFAULT_USER_AGENT); + config.setProtocol(DEFAULT_PROTOCOL); + final int commsTimeout = context.getProperty(TIMEOUT).asTimePeriod(TimeUnit.MILLISECONDS).intValue(); + config.setConnectionTimeout(commsTimeout); + config.setSocketTimeout(commsTimeout); + + final SSLContextService sslContextService = context.getProperty(SSL_CONTEXT_SERVICE).asControllerService(SSLContextService.class); + if (sslContextService != null) { + final SSLContext sslContext = sslContextService.createContext(); + SdkTLSSocketFactory sdkTLSSocketFactory = new SdkTLSSocketFactory(sslContext, SdkTLSSocketFactory.BROWSER_COMPATIBLE_HOSTNAME_VERIFIER); + config.getApacheHttpClientConfig().setSslSocketFactory(sdkTLSSocketFactory); + } + + return config; + } + + private ObjectNode parseSecret(final String secretString) { + try { + final JsonNode root = objectMapper.readTree(secretString); + if (root instanceof ObjectNode) { + return (ObjectNode) root; + } + return null; + } catch (final JsonProcessingException e) { + getLogger().debug("Error parsing JSON", e); + return null; + } + } + + AWSSecretsManager configureClient(final ConfigurationContext context) { + return AWSSecretsManagerClientBuilder.standard() + .withRegion(context.getProperty(REGION).getValue()) + .withClientConfiguration(createConfiguration(context)) + .withCredentials(getCredentialsProvider(context)) + .build(); + } + + /** + * Get credentials provider using the {@link AWSCredentialsProviderService} + * @param context the configuration context + * @return AWSCredentialsProvider the credential provider + * @see AWSCredentialsProvider + */ + protected AWSCredentialsProvider getCredentialsProvider(final ConfigurationContext context) { + + final AWSCredentialsProviderService awsCredentialsProviderService = + context.getProperty(AWS_CREDENTIALS_PROVIDER_SERVICE).asControllerService(AWSCredentialsProviderService.class); + + return awsCredentialsProviderService.getCredentialsProvider(); + + } + + private static AllowableValue createAllowableValue(final Regions region) { + return new AllowableValue(region.getName(), region.getDescription(), "AWS Region Code : " + region.getName()); + } + + private static AllowableValue[] getAvailableRegions() { + final List values = new ArrayList<>(); + for (final Regions region : Regions.values()) { + values.add(createAllowableValue(region)); + } + return values.toArray(new AllowableValue[0]); + } +} diff --git a/nifi-nar-bundles/nifi-aws-bundle/nifi-aws-parameter-providers/src/main/resources/META-INF/services/org.apache.nifi.parameter.ParameterProvider b/nifi-nar-bundles/nifi-aws-bundle/nifi-aws-parameter-providers/src/main/resources/META-INF/services/org.apache.nifi.parameter.ParameterProvider new file mode 100644 index 0000000000..e4fad1ecbc --- /dev/null +++ b/nifi-nar-bundles/nifi-aws-bundle/nifi-aws-parameter-providers/src/main/resources/META-INF/services/org.apache.nifi.parameter.ParameterProvider @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +org.apache.nifi.parameter.aws.AwsSecretsManagerParameterProvider + diff --git a/nifi-nar-bundles/nifi-aws-bundle/nifi-aws-parameter-providers/src/main/resources/docs/org.apache.nifi.parameter.aws.AwsSecretsManagerParameterProvider/additionalDetails.html b/nifi-nar-bundles/nifi-aws-bundle/nifi-aws-parameter-providers/src/main/resources/docs/org.apache.nifi.parameter.aws.AwsSecretsManagerParameterProvider/additionalDetails.html new file mode 100644 index 0000000000..5974197a8f --- /dev/null +++ b/nifi-nar-bundles/nifi-aws-bundle/nifi-aws-parameter-providers/src/main/resources/docs/org.apache.nifi.parameter.aws.AwsSecretsManagerParameterProvider/additionalDetails.html @@ -0,0 +1,62 @@ + + + + + + AWSSecretsManagerParameterProvider + + + + +

Mapping AWS Secrets to Parameter Contexts

+ +

+ The AwsSecretsManagerParameterProvider maps a Secret to a Parameter Context, with key/value pairs in the Secret + mapping to parameters. To create a compatible secret from the AWS Console: +

+ +
    +
  1. From the Secrets Manager service, click the "Store a new Secret" button
    2. +
  2. Select "Other type of secret"
    3. +
  3. Under "Key/value", enter your parameters, with the parameter names being the keys and the parameter values + being the values. Click Next.
    4. +
  4. Enter the Secret name. This will determine which Parameter Context receives the parameters. Continue + through the rest of the wizard and finally click the "Store" button.
    5. +
+ +

+ Alternatively, from the command line, run a command like the following: +

+ + 
+aws secretsmanager create-secret --name "[Context]" --secret-string '{ "[Param]": "[secretValue]", "[Param2]": "[secretValue2]" }'
+
+ +

In this example, [Context] should be the intended name of the Parameter Context, [Param] and [Param2] should be + parameter names, and [secretValue] and [secretValue2] should be the values of each respective parameter.

+ +

Configuring the Parameter Provider

+ +

+ AWS Secrets must be explicitly matched in the "Secret Name Pattern" property in order for them to be fetched. This + prevents more than the intended Secrets from being pulled into NiFi. +

+ +

+ +

+ + diff --git a/nifi-nar-bundles/nifi-aws-bundle/nifi-aws-parameter-providers/src/test/java/org/apache/nifi/parameter/aws/TestAwsSecretsManagerParameterProvider.java b/nifi-nar-bundles/nifi-aws-bundle/nifi-aws-parameter-providers/src/test/java/org/apache/nifi/parameter/aws/TestAwsSecretsManagerParameterProvider.java new file mode 100644 index 0000000000..8b96a6e7f0 --- /dev/null +++ b/nifi-nar-bundles/nifi-aws-bundle/nifi-aws-parameter-providers/src/test/java/org/apache/nifi/parameter/aws/TestAwsSecretsManagerParameterProvider.java @@ -0,0 +1,199 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.parameter.aws; + +import com.amazonaws.services.secretsmanager.AWSSecretsManager; +import com.amazonaws.services.secretsmanager.model.AWSSecretsManagerException; +import com.amazonaws.services.secretsmanager.model.GetSecretValueRequest; +import com.amazonaws.services.secretsmanager.model.GetSecretValueResult; +import com.amazonaws.services.secretsmanager.model.ListSecretsResult; +import com.amazonaws.services.secretsmanager.model.SecretListEntry; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import org.apache.nifi.components.ConfigVerificationResult; +import org.apache.nifi.components.PropertyDescriptor; +import org.apache.nifi.parameter.Parameter; +import org.apache.nifi.parameter.ParameterDescriptor; +import org.apache.nifi.parameter.ParameterGroup; +import org.apache.nifi.parameter.VerifiableParameterProvider; +import org.apache.nifi.reporting.InitializationException; +import org.apache.nifi.util.MockComponentLog; +import org.apache.nifi.util.MockConfigurationContext; +import org.apache.nifi.util.MockParameterProviderInitializationContext; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.ArgumentMatcher; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.argThat; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.when; + +@ExtendWith(MockitoExtension.class) +public class TestAwsSecretsManagerParameterProvider { + + @Mock + private AWSSecretsManager defaultSecretsManager; + + @Mock + private ListSecretsResult listSecretsResult; + + final ObjectMapper objectMapper = new ObjectMapper(); + + final List mySecretParameters = Arrays.asList( + parameter("paramA", "valueA"), + parameter("paramB", "valueB"), + parameter("otherC", "valueOther"), + parameter("paramD", "valueD"), + parameter("nonSensitiveE", "valueE"), + parameter("otherF", "valueF") + ); + final List otherSecretParameters = Arrays.asList( + parameter("paramG", "valueG"), + parameter("otherH", "valueOther") + ); + final List mockParameterGroups = Arrays.asList( + new ParameterGroup("MySecret", mySecretParameters), + new ParameterGroup("OtherSecret", otherSecretParameters) + ); + + @Test + public void testFetchParametersWithNoSecrets() throws InitializationException { + final List expectedGroups = Collections.singletonList(new ParameterGroup("MySecret", Collections.emptyList())); + runProviderTest(mockSecretsManager(expectedGroups), 0, ConfigVerificationResult.Outcome.SUCCESSFUL); + } + + @Test + public void testFetchParameters() throws InitializationException { + runProviderTest(mockSecretsManager(mockParameterGroups), 8, ConfigVerificationResult.Outcome.SUCCESSFUL); + } + + @Test + public void testFetchParametersListFailure() throws InitializationException { + when(defaultSecretsManager.listSecrets(any())).thenThrow(new AWSSecretsManagerException("Fake exception")); + runProviderTest(defaultSecretsManager, 0, ConfigVerificationResult.Outcome.FAILED); + } + + @Test + public void testFetchParametersGetSecretFailure() throws InitializationException { + final List secretList = Collections.singletonList(new SecretListEntry().withName("MySecret")); + when(listSecretsResult.getSecretList()).thenReturn(secretList); + when(defaultSecretsManager.listSecrets(any())).thenReturn(listSecretsResult); + when(defaultSecretsManager.getSecretValue(argThat(matchesGetSecretValueRequest("MySecret")))).thenThrow(new AWSSecretsManagerException("Fake exception")); + runProviderTest(defaultSecretsManager, 0, ConfigVerificationResult.Outcome.FAILED); + } + + private AwsSecretsManagerParameterProvider getParameterProvider() { + return spy(new AwsSecretsManagerParameterProvider()); + } + + private AWSSecretsManager mockSecretsManager(final List mockGroup) { + final AWSSecretsManager secretsManager = mock(AWSSecretsManager.class); + + final List secretList = mockGroup.stream() + .map(group -> new SecretListEntry().withName(group.getGroupName())) + .collect(Collectors.toList()); + when(listSecretsResult.getSecretList()).thenReturn(secretList); + when(secretsManager.listSecrets(any())).thenReturn(listSecretsResult); + + mockGroup.forEach(group -> { + final String groupName = group.getGroupName(); + final Map keyValues = group.getParameters().stream().collect(Collectors.toMap( + param -> param.getDescriptor().getName(), + Parameter::getValue)); + final String secretString; + try { + secretString = objectMapper.writeValueAsString(keyValues); + final GetSecretValueResult result = new GetSecretValueResult().withName(groupName).withSecretString(secretString); + when(secretsManager.getSecretValue(argThat(matchesGetSecretValueRequest(groupName)))) + .thenReturn(result); + } catch (final JsonProcessingException e) { + throw new IllegalStateException(e); + } + }); + return secretsManager; + } + + private List runProviderTest(final AWSSecretsManager secretsManager, final int expectedCount, + final ConfigVerificationResult.Outcome expectedOutcome) throws InitializationException { + + final AwsSecretsManagerParameterProvider parameterProvider = getParameterProvider(); + doReturn(secretsManager).when(parameterProvider).configureClient(any()); + final MockParameterProviderInitializationContext initContext = new MockParameterProviderInitializationContext("id", "name", + new MockComponentLog("providerId", parameterProvider)); + parameterProvider.initialize(initContext); + + final Map properties = new HashMap<>(); + final MockConfigurationContext mockConfigurationContext = new MockConfigurationContext(properties, null); + + List parameterGroups = new ArrayList<>(); + // Verify parameter fetching + if (expectedOutcome == ConfigVerificationResult.Outcome.FAILED) { + assertThrows(RuntimeException.class, () -> parameterProvider.fetchParameters(mockConfigurationContext)); + } else { + parameterGroups = parameterProvider.fetchParameters(mockConfigurationContext); + final int parameterCount = (int) parameterGroups.stream() + .flatMap(group -> group.getParameters().stream()) + .count(); + assertEquals(expectedCount, parameterCount); + } + + // Verify config verification + final List results = ((VerifiableParameterProvider) parameterProvider).verify(mockConfigurationContext, initContext.getLogger()); + + assertEquals(1, results.size()); + assertEquals(expectedOutcome, results.get(0).getOutcome()); + + return parameterGroups; + } + + private static Parameter parameter(final String name, final String value) { + return new Parameter(new ParameterDescriptor.Builder().name(name).build(), value); + } + + private static ArgumentMatcher matchesGetSecretValueRequest(final String groupName) { + return new GetSecretValueRequestMatcher(groupName); + } + + private static class GetSecretValueRequestMatcher implements ArgumentMatcher { + + private final String secretId; + + private GetSecretValueRequestMatcher(final String secretId) { + this.secretId = secretId; + } + + @Override + public boolean matches(final GetSecretValueRequest argument) { + return argument != null && argument.getSecretId().equals(secretId); + } + } +} diff --git a/nifi-nar-bundles/nifi-aws-bundle/pom.xml b/nifi-nar-bundles/nifi-aws-bundle/pom.xml index a6ab4fc31e..fc6a89e7e9 100644 --- a/nifi-nar-bundles/nifi-aws-bundle/pom.xml +++ b/nifi-nar-bundles/nifi-aws-bundle/pom.xml @@ -37,5 +37,6 @@ nifi-aws-service-api-nar nifi-aws-abstract-processors nifi-aws-parameter-value-providers + nifi-aws-parameter-providers