mirror of https://github.com/apache/nifi.git
NIFI-9637: Adding GcpSecretManagerParameterProvider
Signed-off-by: Nathan Gough <thenatog@gmail.com> This closes #6394.
This commit is contained in:
Joe Gresock
Nathan Gough
parent
8148588c9c
commit
518f413d9f
|
|
@ -41,5 +41,10 @@
|
|||
<artifactId>nifi-gcp-processors</artifactId>
|
||||
<version>1.18.0-SNAPSHOT</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.apache.nifi</groupId>
|
||||
<artifactId>nifi-gcp-parameter-providers</artifactId>
|
||||
<version>1.18.0-SNAPSHOT</version>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
</project>
|
||||
|
|
|
|||
|
|
@ -0,0 +1,84 @@
|
|||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<!--
|
||||
Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
contributor license agreements. See the NOTICE file distributed with
|
||||
this work for additional information regarding copyright ownership.
|
||||
The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
(the "License"); you may not use this file except in compliance with
|
||||
the License. You may obtain a copy of the License at
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
-->
|
||||
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
<parent>
|
||||
<groupId>org.apache.nifi</groupId>
|
||||
<artifactId>nifi-gcp-bundle</artifactId>
|
||||
<version>1.18.0-SNAPSHOT</version>
|
||||
</parent>
|
||||
|
||||
<artifactId>nifi-gcp-parameter-providers</artifactId>
|
||||
<packaging>jar</packaging>
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>org.apache.nifi</groupId>
|
||||
<artifactId>nifi-api</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.apache.nifi</groupId>
|
||||
<artifactId>nifi-utils</artifactId>
|
||||
<version>1.18.0-SNAPSHOT</version>
|
||||
<scope>provided</scope>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.apache.nifi</groupId>
|
||||
<artifactId>nifi-gcp-services-api</artifactId>
|
||||
<version>1.18.0-SNAPSHOT</version>
|
||||
<scope>provided</scope>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.slf4j</groupId>
|
||||
<artifactId>jcl-over-slf4j</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>com.google.cloud</groupId>
|
||||
<artifactId>google-cloud-secretmanager</artifactId>
|
||||
<exclusions>
|
||||
<exclusion>
|
||||
<groupId>commons-logging</groupId>
|
||||
<artifactId>commons-logging</artifactId>
|
||||
</exclusion>
|
||||
</exclusions>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>com.google.auth</groupId>
|
||||
<artifactId>google-auth-library-oauth2-http</artifactId>
|
||||
<exclusions>
|
||||
<exclusion>
|
||||
<groupId>com.google.code.findbugs</groupId>
|
||||
<artifactId>jsr305</artifactId>
|
||||
</exclusion>
|
||||
<exclusion>
|
||||
<groupId>commons-logging</groupId>
|
||||
<artifactId>commons-logging</artifactId>
|
||||
</exclusion>
|
||||
</exclusions>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.apache.nifi</groupId>
|
||||
<artifactId>nifi-expression-language</artifactId>
|
||||
<version>1.18.0-SNAPSHOT</version>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.apache.nifi</groupId>
|
||||
<artifactId>nifi-mock</artifactId>
|
||||
<version>1.18.0-SNAPSHOT</version>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
</project>
|
||||
|
|
@ -0,0 +1,200 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.apache.nifi.parameter.gcp;
|
||||
|
||||
import com.google.cloud.secretmanager.v1.AccessSecretVersionResponse;
|
||||
import com.google.cloud.secretmanager.v1.ProjectName;
|
||||
import com.google.cloud.secretmanager.v1.Secret;
|
||||
import com.google.cloud.secretmanager.v1.SecretManagerServiceClient;
|
||||
import com.google.cloud.secretmanager.v1.SecretManagerServiceClient.ListSecretsPage;
|
||||
import com.google.cloud.secretmanager.v1.SecretManagerServiceSettings;
|
||||
import com.google.cloud.secretmanager.v1.SecretVersionName;
|
||||
import org.apache.nifi.annotation.documentation.CapabilityDescription;
|
||||
import org.apache.nifi.annotation.documentation.Tags;
|
||||
import org.apache.nifi.components.ConfigVerificationResult;
|
||||
import org.apache.nifi.components.PropertyDescriptor;
|
||||
import org.apache.nifi.controller.ConfigurationContext;
|
||||
import org.apache.nifi.gcp.credentials.service.GCPCredentialsService;
|
||||
import org.apache.nifi.logging.ComponentLog;
|
||||
import org.apache.nifi.parameter.AbstractParameterProvider;
|
||||
import org.apache.nifi.parameter.Parameter;
|
||||
import org.apache.nifi.parameter.ParameterDescriptor;
|
||||
import org.apache.nifi.parameter.ParameterGroup;
|
||||
import org.apache.nifi.parameter.VerifiableParameterProvider;
|
||||
import org.apache.nifi.processor.util.StandardValidators;
|
||||
import org.apache.nifi.util.StringUtils;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Arrays;
|
||||
import java.util.Collections;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
/**
|
||||
* Reads secrets from GCP Secret Manager to provide parameter values. Secrets must be created similar to the following GCP cli command: <br/><br/>
|
||||
* <code>gcp secretsmanager create-secret --name "[Context]" --secret-string '{ "[Param]": "[secretValue]", "[Param2]": "[secretValue2]" }'</code> <br/><br/>
|
||||
*
|
||||
*/
|
||||
|
||||
@Tags({"gcp", "secret", "manager"})
|
||||
@CapabilityDescription("Fetches parameters from GCP Secret Manager. Each secret becomes a Parameter, which can be mapped to a Parameter Group " +
|
||||
"by adding a GCP label named 'group-name'.")
|
||||
public class GcpSecretManagerParameterProvider extends AbstractParameterProvider implements VerifiableParameterProvider {
|
||||
private static final Logger logger = LoggerFactory.getLogger(GcpSecretManagerParameterProvider.class);
|
||||
|
||||
public static final PropertyDescriptor GROUP_NAME_PATTERN = new PropertyDescriptor.Builder()
|
||||
.name("group-name-pattern")
|
||||
.displayName("Group Name Pattern")
|
||||
.description("A Regular Expression matching on the 'group-name' label value that identifies Secrets whose parameters should be fetched. " +
|
||||
"Any secrets without a 'group-name' label value that matches this Regex will not be fetched.")
|
||||
.addValidator(StandardValidators.REGULAR_EXPRESSION_VALIDATOR)
|
||||
.required(true)
|
||||
.defaultValue(".*")
|
||||
.build();
|
||||
|
||||
public static final PropertyDescriptor PROJECT_ID = new PropertyDescriptor
|
||||
.Builder().name("gcp-project-id")
|
||||
.displayName("Project ID")
|
||||
.description("Google Cloud Project ID")
|
||||
.addValidator(StandardValidators.NON_EMPTY_VALIDATOR)
|
||||
.required(true)
|
||||
.build();
|
||||
|
||||
/**
|
||||
* Links to the {@link GCPCredentialsService} which provides credentials for this particular processor.
|
||||
*/
|
||||
public static final PropertyDescriptor GCP_CREDENTIALS_PROVIDER_SERVICE = new PropertyDescriptor.Builder()
|
||||
.name("gcp-credentials-provider-service")
|
||||
.displayName("GCP Credentials Provider Service")
|
||||
.description("The Controller Service used to obtain Google Cloud Platform credentials.")
|
||||
.required(true)
|
||||
.identifiesControllerService(GCPCredentialsService.class)
|
||||
.build();
|
||||
|
||||
private static final String GROUP_NAME_LABEL = "group-name";
|
||||
private static final String SECRETS_PATH = "secrets/";
|
||||
private static final List<PropertyDescriptor> PROPERTIES = Collections.unmodifiableList(Arrays.asList(
|
||||
GROUP_NAME_PATTERN,
|
||||
PROJECT_ID,
|
||||
GCP_CREDENTIALS_PROVIDER_SERVICE
|
||||
));
|
||||
|
||||
@Override
|
||||
protected List<PropertyDescriptor> getSupportedPropertyDescriptors() {
|
||||
return PROPERTIES;
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<ParameterGroup> fetchParameters(final ConfigurationContext context) throws IOException {
|
||||
final Map<String, ParameterGroup> providedParameterGroups = new HashMap<>();
|
||||
final SecretManagerServiceClient secretsManager = this.configureClient(context);
|
||||
final ProjectName projectName = ProjectName.of(context.getProperty(PROJECT_ID).getValue());
|
||||
|
||||
final SecretManagerServiceClient.ListSecretsPagedResponse pagedResponse = secretsManager.listSecrets(projectName);
|
||||
ListSecretsPage page = pagedResponse.getPage();
|
||||
do {
|
||||
for (final Secret secret : page.getValues()) {
|
||||
final String contextName = secret.getLabelsOrDefault(GROUP_NAME_LABEL, null);
|
||||
if (contextName == null) {
|
||||
getLogger().debug("Secret [{}] does not have the {} label, and will be skipped", secret.getName(), GROUP_NAME_LABEL);
|
||||
continue;
|
||||
}
|
||||
final String secretName = StringUtils.substringAfter(secret.getName(), SECRETS_PATH);
|
||||
|
||||
fetchSecret(secretsManager, context, secretName, contextName, providedParameterGroups);
|
||||
}
|
||||
if (page.hasNextPage()) {
|
||||
page = page.getNextPage();
|
||||
}
|
||||
} while (page.hasNextPage());
|
||||
return new ArrayList<>(providedParameterGroups.values());
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<ConfigVerificationResult> verify(final ConfigurationContext context, final ComponentLog verificationLogger) {
|
||||
final List<ConfigVerificationResult> results = new ArrayList<>();
|
||||
|
||||
try {
|
||||
final List<ParameterGroup> parameterGroups = fetchParameters(context);
|
||||
int count = 0;
|
||||
for (final ParameterGroup group : parameterGroups) {
|
||||
count += group.getParameters().size();
|
||||
}
|
||||
results.add(new ConfigVerificationResult.Builder()
|
||||
.outcome(ConfigVerificationResult.Outcome.SUCCESSFUL)
|
||||
.verificationStepName("Fetch Parameters")
|
||||
.explanation(String.format("Fetched secret keys [%d] as parameters within groups [%d]",
|
||||
count, parameterGroups.size()))
|
||||
.build());
|
||||
} catch (final Exception e) {
|
||||
verificationLogger.error("Failed to fetch parameters", e);
|
||||
results.add(new ConfigVerificationResult.Builder()
|
||||
.outcome(ConfigVerificationResult.Outcome.FAILED)
|
||||
.verificationStepName("Fetch Parameters")
|
||||
.explanation("Failed to fetch parameters: " + e.getMessage())
|
||||
.build());
|
||||
}
|
||||
return results;
|
||||
}
|
||||
|
||||
private void fetchSecret(final SecretManagerServiceClient secretsManager, final ConfigurationContext context, final String secretName,
|
||||
final String groupName, final Map<String, ParameterGroup> providedParameterGroups) {
|
||||
final Pattern groupNamePattern = Pattern.compile(context.getProperty(GROUP_NAME_PATTERN).getValue());
|
||||
final String projectId = context.getProperty(PROJECT_ID).getValue();
|
||||
|
||||
if (!groupNamePattern.matcher(groupName).matches()) {
|
||||
logger.debug("Secret [{}] label [{}] does not match the group name pattern {}", secretName, groupName, groupNamePattern);
|
||||
return;
|
||||
}
|
||||
|
||||
final SecretVersionName secretVersionName = SecretVersionName.of(projectId, secretName, "latest");
|
||||
|
||||
// Access the secret version.
|
||||
final AccessSecretVersionResponse response = secretsManager.accessSecretVersion(secretVersionName);
|
||||
final String parameterValue = response.getPayload().getData().toStringUtf8();
|
||||
|
||||
final Parameter parameter = createParameter(secretName, parameterValue);
|
||||
|
||||
if (parameter != null) {
|
||||
final ParameterGroup group = providedParameterGroups
|
||||
.computeIfAbsent(groupName, key -> new ParameterGroup(groupName, new ArrayList<>()));
|
||||
final List<Parameter> updatedParameters = new ArrayList<>(group.getParameters());
|
||||
updatedParameters.add(parameter);
|
||||
providedParameterGroups.put(groupName, new ParameterGroup(groupName, updatedParameters));
|
||||
}
|
||||
}
|
||||
|
||||
private Parameter createParameter(final String parameterName, final String parameterValue) {
|
||||
final ParameterDescriptor parameterDescriptor = new ParameterDescriptor.Builder().name(parameterName).build();
|
||||
return new Parameter(parameterDescriptor, parameterValue, null, true);
|
||||
}
|
||||
|
||||
SecretManagerServiceClient configureClient(final ConfigurationContext context) throws IOException {
|
||||
final GCPCredentialsService credentialsService = context.getProperty(GCP_CREDENTIALS_PROVIDER_SERVICE).asControllerService(GCPCredentialsService.class);
|
||||
|
||||
final SecretManagerServiceClient client = SecretManagerServiceClient.create(SecretManagerServiceSettings
|
||||
.newBuilder().setCredentialsProvider(() -> credentialsService.getGoogleCredentials())
|
||||
.build());
|
||||
|
||||
return client;
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,16 @@
|
|||
# Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
# contributor license agreements. See the NOTICE file distributed with
|
||||
# this work for additional information regarding copyright ownership.
|
||||
# The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
# (the "License"); you may not use this file except in compliance with
|
||||
# the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
org.apache.nifi.parameter.gcp.GcpSecretManagerParameterProvider
|
||||
|
||||
|
|
@ -0,0 +1,55 @@
|
|||
<!DOCTYPE html>
|
||||
<html lang="en">
|
||||
<!--
|
||||
Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
contributor license agreements. See the NOTICE file distributed with
|
||||
this work for additional information regarding copyright ownership.
|
||||
The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
(the "License"); you may not use this file except in compliance with
|
||||
the License. You may obtain a copy of the License at
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
-->
|
||||
<head>
|
||||
<meta charset="utf-8" />
|
||||
<title>GcpSecretManagerParameterProvider</title>
|
||||
|
||||
<link rel="stylesheet" href="../../../../../css/component-usage.css" type="text/css" />
|
||||
</head>
|
||||
<body>
|
||||
<h3>Mapping GCP Secrets to Parameter Contexts</h3>
|
||||
|
||||
<p>
|
||||
The GcpSecretManagerParameterProvider maps a Secret to a Parameter, which can be grouped by adding a "group-name" label.
|
||||
To create a compatible secret from the GCP Console:
|
||||
</p>
|
||||
|
||||
<ol>
|
||||
<li>From the Secret Manager service, click the "Create Secret" button</li>
|
||||
<li>Enter the Secret name. This is the name of a parameter. Enter a value.</li>
|
||||
<li>Under "Labels", add a label with a Key of "group-name" and a value of the intended Parameter Group name.</li>
|
||||
</ol>
|
||||
|
||||
<p>
|
||||
Alternatively, from the command line, run a command like the following:
|
||||
</p>
|
||||
|
||||
<pre>
|
||||
printf "[Parameter Value]" | gcloud secrets create --labels=group-name="[Parameter Group Name]" "[Parameter Name]" --data-file=-
|
||||
</pre>
|
||||
|
||||
<p>In this example, [Parameter Group Name] should be the intended name of the Parameter Group, [Parameter Name] should be
|
||||
the parameter name, and [Parameter Value] should be the value of the parameter.</p>
|
||||
|
||||
<h3>Configuring the Parameter Provider</h3>
|
||||
|
||||
<p>
|
||||
GCP Secrets must be explicitly matched in the "Group Name Pattern" property in order for them to be fetched. This
|
||||
prevents more than the intended Secrets from being pulled into NiFi.
|
||||
</p>
|
||||
</body>
|
||||
</html>
|
||||
|
|
@ -0,0 +1,208 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.apache.nifi.parameter.gcp;
|
||||
|
||||
import com.google.cloud.secretmanager.v1.AccessSecretVersionResponse;
|
||||
import com.google.cloud.secretmanager.v1.ProjectName;
|
||||
import com.google.cloud.secretmanager.v1.Secret;
|
||||
import com.google.cloud.secretmanager.v1.SecretManagerServiceClient;
|
||||
import com.google.cloud.secretmanager.v1.SecretManagerServiceClient.ListSecretsPage;
|
||||
import com.google.cloud.secretmanager.v1.SecretManagerServiceClient.ListSecretsPagedResponse;
|
||||
import com.google.cloud.secretmanager.v1.SecretPayload;
|
||||
import com.google.cloud.secretmanager.v1.SecretVersionName;
|
||||
import com.google.protobuf.ByteString;
|
||||
import org.apache.nifi.components.ConfigVerificationResult;
|
||||
import org.apache.nifi.components.PropertyDescriptor;
|
||||
import org.apache.nifi.parameter.Parameter;
|
||||
import org.apache.nifi.parameter.ParameterDescriptor;
|
||||
import org.apache.nifi.parameter.ParameterGroup;
|
||||
import org.apache.nifi.parameter.VerifiableParameterProvider;
|
||||
import org.apache.nifi.reporting.InitializationException;
|
||||
import org.apache.nifi.util.MockComponentLog;
|
||||
import org.apache.nifi.util.MockConfigurationContext;
|
||||
import org.apache.nifi.util.MockParameterProviderInitializationContext;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Arrays;
|
||||
import java.util.Collections;
|
||||
import java.util.HashMap;
|
||||
import java.util.Iterator;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertThrows;
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.ArgumentMatchers.argThat;
|
||||
import static org.mockito.Mockito.doReturn;
|
||||
import static org.mockito.Mockito.mock;
|
||||
import static org.mockito.Mockito.spy;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
public class TestGcpSecretManagerParameterProvider {
|
||||
|
||||
final List<Parameter> mySecretParameters = Arrays.asList(
|
||||
parameter("paramA", "valueA"),
|
||||
parameter("paramB", "valueB"),
|
||||
parameter("otherC", "valueOther"),
|
||||
parameter("paramD", "valueD"),
|
||||
parameter("nonSensitiveE", "valueE"),
|
||||
parameter("otherF", "valueF")
|
||||
);
|
||||
final List<Parameter> otherSecretParameters = Arrays.asList(
|
||||
parameter("paramG", "valueG"),
|
||||
parameter("otherH", "valueOther")
|
||||
);
|
||||
final List<Parameter> unrelatedParameters = Collections.singletonList(parameter("paramK", "unused"));
|
||||
final List<ParameterGroup> mockParameterGroups = Arrays.asList(
|
||||
new ParameterGroup("MySecret", mySecretParameters),
|
||||
new ParameterGroup("OtherSecret", otherSecretParameters),
|
||||
new ParameterGroup("Unrelated", unrelatedParameters) // will not be picked up
|
||||
);
|
||||
|
||||
@Test
|
||||
public void testFetchParametersWithNoSecrets() throws InitializationException, IOException {
|
||||
final List<ParameterGroup> expectedGroups = Collections.singletonList(new ParameterGroup("MySecret", Collections.emptyList()));
|
||||
runProviderTest(mockSecretManagerClient(expectedGroups), 0, ConfigVerificationResult.Outcome.SUCCESSFUL);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testFetchParameters() throws InitializationException, IOException {
|
||||
runProviderTest(mockSecretManagerClient(mockParameterGroups), 8, ConfigVerificationResult.Outcome.SUCCESSFUL);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testFetchParametersListFailure() throws InitializationException, IOException {
|
||||
final SecretManagerServiceClient mockSecretsManager = mock(SecretManagerServiceClient.class);
|
||||
when(mockSecretsManager.listSecrets(any(ProjectName.class))).thenThrow(new RuntimeException("Fake exception"));
|
||||
runProviderTest(mockSecretsManager, 0, ConfigVerificationResult.Outcome.FAILED);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testFetchParametersGetSecretFailure() throws InitializationException, IOException {
|
||||
final SecretManagerServiceClient mockSecretsManager = mock(SecretManagerServiceClient.class);
|
||||
final ListSecretsPagedResponse listSecretsPagedResponse = mock(ListSecretsPagedResponse.class);
|
||||
final ListSecretsPage page = mock(ListSecretsPage.class);
|
||||
when(listSecretsPagedResponse.getPage()).thenReturn(page);
|
||||
|
||||
when(page.hasNextPage()).thenReturn(false);
|
||||
final Secret secret = mock(Secret.class);
|
||||
when(secret.getName()).thenReturn("paramA");
|
||||
when(secret.getLabelsOrDefault("group-name", null)).thenReturn("Secret");
|
||||
when(page.getValues()).thenReturn(Collections.singletonList(secret));
|
||||
|
||||
when(mockSecretsManager.accessSecretVersion(any(SecretVersionName.class))).thenThrow(new RuntimeException("Fake exception"));
|
||||
runProviderTest(mockSecretsManager, 0, ConfigVerificationResult.Outcome.FAILED);
|
||||
}
|
||||
|
||||
private GcpSecretManagerParameterProvider getParameterProvider() {
|
||||
return spy(new GcpSecretManagerParameterProvider());
|
||||
}
|
||||
|
||||
private SecretManagerServiceClient mockSecretManagerClient(final List<ParameterGroup> mockGroups) {
|
||||
final SecretManagerServiceClient secretManager = mock(SecretManagerServiceClient.class);
|
||||
|
||||
final ListSecretsPagedResponse listSecretsPagedResponse = mock(ListSecretsPagedResponse.class);
|
||||
when(secretManager.listSecrets(any(ProjectName.class))).thenReturn(listSecretsPagedResponse);
|
||||
|
||||
boolean mockedFirstPage = false;
|
||||
ListSecretsPage currentPage;
|
||||
ListSecretsPage previousPage = null;
|
||||
for (final Iterator<ParameterGroup> it = mockGroups.iterator(); it.hasNext(); ) {
|
||||
final ParameterGroup group = it.next();
|
||||
currentPage = mock(ListSecretsPage.class);
|
||||
if (mockedFirstPage) {
|
||||
when(previousPage.getNextPage()).thenReturn(currentPage);
|
||||
} else {
|
||||
when(listSecretsPagedResponse.getPage()).thenReturn(currentPage);
|
||||
mockedFirstPage = true;
|
||||
}
|
||||
final List<Secret> values = new ArrayList<>();
|
||||
values.addAll(group.getParameters().stream()
|
||||
.map(parameter -> mockSecret(secretManager, group.getGroupName(), parameter))
|
||||
.collect(Collectors.toList()));
|
||||
when(currentPage.getValues()).thenReturn(values);
|
||||
if (it.hasNext()) {
|
||||
when(currentPage.hasNextPage()).thenReturn(true);
|
||||
previousPage = currentPage;
|
||||
} else {
|
||||
when(currentPage.hasNextPage()).thenReturn(false);
|
||||
}
|
||||
}
|
||||
|
||||
return secretManager;
|
||||
}
|
||||
|
||||
final Secret mockSecret(final SecretManagerServiceClient secretManager, final String groupName, final Parameter parameter) {
|
||||
final Secret secret = mock(Secret.class);
|
||||
|
||||
final String parameterName = parameter.getDescriptor().getName();
|
||||
when(secret.getName()).thenReturn("projects/project/secrets/" + parameterName);
|
||||
when(secret.getLabelsOrDefault("group-name", null)).thenReturn(groupName);
|
||||
|
||||
final AccessSecretVersionResponse response = mock(AccessSecretVersionResponse.class);
|
||||
doReturn(response).when(secretManager).accessSecretVersion(argThat((SecretVersionName secretVersionName) -> secretVersionName.getSecret().equals(parameterName)));
|
||||
final SecretPayload payload = mock(SecretPayload.class);
|
||||
final ByteString data = ByteString.copyFromUtf8(parameter.getValue());
|
||||
when(payload.getData()).thenReturn(data);
|
||||
when(response.getPayload()).thenReturn(payload);
|
||||
return secret;
|
||||
}
|
||||
|
||||
private List<ParameterGroup> runProviderTest(final SecretManagerServiceClient secretsManager, final int expectedCount,
|
||||
final ConfigVerificationResult.Outcome expectedOutcome) throws InitializationException, IOException {
|
||||
|
||||
final GcpSecretManagerParameterProvider parameterProvider = getParameterProvider();
|
||||
doReturn(secretsManager).when(parameterProvider).configureClient(any());
|
||||
final MockParameterProviderInitializationContext initContext = new MockParameterProviderInitializationContext("id", "name",
|
||||
new MockComponentLog("providerId", parameterProvider));
|
||||
parameterProvider.initialize(initContext);
|
||||
|
||||
final Map<PropertyDescriptor, String> properties = new HashMap<>();
|
||||
properties.put(GcpSecretManagerParameterProvider.GROUP_NAME_PATTERN, ".*Secret");
|
||||
properties.put(GcpSecretManagerParameterProvider.PROJECT_ID, "my-project");
|
||||
final MockConfigurationContext mockConfigurationContext = new MockConfigurationContext(properties, null);
|
||||
|
||||
List<ParameterGroup> parameterGroups = new ArrayList<>();
|
||||
// Verify parameter fetching
|
||||
if (expectedOutcome == ConfigVerificationResult.Outcome.FAILED) {
|
||||
assertThrows(RuntimeException.class, () -> parameterProvider.fetchParameters(mockConfigurationContext));
|
||||
} else {
|
||||
parameterGroups = parameterProvider.fetchParameters(mockConfigurationContext);
|
||||
final int count = (int) parameterGroups.stream()
|
||||
.flatMap(group -> group.getParameters().stream())
|
||||
.count();
|
||||
assertEquals(expectedCount, count);
|
||||
}
|
||||
|
||||
// Verify config verification
|
||||
final List<ConfigVerificationResult> results = ((VerifiableParameterProvider) parameterProvider).verify(mockConfigurationContext, initContext.getLogger());
|
||||
|
||||
assertEquals(1, results.size());
|
||||
assertEquals(expectedOutcome, results.get(0).getOutcome());
|
||||
|
||||
return parameterGroups;
|
||||
}
|
||||
|
||||
private static Parameter parameter(final String name, final String value) {
|
||||
return new Parameter(new ParameterDescriptor.Builder().name(name).build(), value);
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -0,0 +1,15 @@
|
|||
# Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
# contributor license agreements. See the NOTICE file distributed with
|
||||
# this work for additional information regarding copyright ownership.
|
||||
# The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
# (the "License"); you may not use this file except in compliance with
|
||||
# the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
mock-maker-inline
|
||||
|
|
@ -45,6 +45,7 @@
|
|||
<module>nifi-gcp-services-api</module>
|
||||
<module>nifi-gcp-services-api-nar</module>
|
||||
<module>nifi-gcp-processors</module>
|
||||
<module>nifi-gcp-parameter-providers</module>
|
||||
<module>nifi-gcp-nar</module>
|
||||
</modules>
|
||||
</project>
|
||||
|
|
|
|||
Loading…
Reference in New Issue