mirror of https://github.com/apache/nifi.git
NIFI-9637: Adding GcpSecretManagerParameterProvider
Signed-off-by: Nathan Gough <thenatog@gmail.com> This closes #6394.
This commit is contained in:
Joe Gresock
Nathan Gough
parent
8148588c9c
commit
518f413d9f
|
|
@ -41,5 +41,10 @@
|
||||||
<artifactId>nifi-gcp-processors</artifactId>
|
<artifactId>nifi-gcp-processors</artifactId>
|
||||||
<version>1.18.0-SNAPSHOT</version>
|
<version>1.18.0-SNAPSHOT</version>
|
||||||
</dependency>
|
</dependency>
|
||||||
|
<dependency>
|
||||||
|
<groupId>org.apache.nifi</groupId>
|
||||||
|
<artifactId>nifi-gcp-parameter-providers</artifactId>
|
||||||
|
<version>1.18.0-SNAPSHOT</version>
|
||||||
|
</dependency>
|
||||||
</dependencies>
|
</dependencies>
|
||||||
</project>
|
</project>
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,84 @@
|
||||||
|
<?xml version="1.0" encoding="UTF-8"?>
|
||||||
|
<!--
|
||||||
|
Licensed to the Apache Software Foundation (ASF) under one or more
|
||||||
|
contributor license agreements. See the NOTICE file distributed with
|
||||||
|
this work for additional information regarding copyright ownership.
|
||||||
|
The ASF licenses this file to You under the Apache License, Version 2.0
|
||||||
|
(the "License"); you may not use this file except in compliance with
|
||||||
|
the License. You may obtain a copy of the License at
|
||||||
|
http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
Unless required by applicable law or agreed to in writing, software
|
||||||
|
distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
See the License for the specific language governing permissions and
|
||||||
|
limitations under the License.
|
||||||
|
-->
|
||||||
|
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
|
||||||
|
<modelVersion>4.0.0</modelVersion>
|
||||||
|
<parent>
|
||||||
|
<groupId>org.apache.nifi</groupId>
|
||||||
|
<artifactId>nifi-gcp-bundle</artifactId>
|
||||||
|
<version>1.18.0-SNAPSHOT</version>
|
||||||
|
</parent>
|
||||||
|
|
||||||
|
<artifactId>nifi-gcp-parameter-providers</artifactId>
|
||||||
|
<packaging>jar</packaging>
|
||||||
|
<dependencies>
|
||||||
|
<dependency>
|
||||||
|
<groupId>org.apache.nifi</groupId>
|
||||||
|
<artifactId>nifi-api</artifactId>
|
||||||
|
</dependency>
|
||||||
|
<dependency>
|
||||||
|
<groupId>org.apache.nifi</groupId>
|
||||||
|
<artifactId>nifi-utils</artifactId>
|
||||||
|
<version>1.18.0-SNAPSHOT</version>
|
||||||
|
<scope>provided</scope>
|
||||||
|
</dependency>
|
||||||
|
<dependency>
|
||||||
|
<groupId>org.apache.nifi</groupId>
|
||||||
|
<artifactId>nifi-gcp-services-api</artifactId>
|
||||||
|
<version>1.18.0-SNAPSHOT</version>
|
||||||
|
<scope>provided</scope>
|
||||||
|
</dependency>
|
||||||
|
<dependency>
|
||||||
|
<groupId>org.slf4j</groupId>
|
||||||
|
<artifactId>jcl-over-slf4j</artifactId>
|
||||||
|
</dependency>
|
||||||
|
<dependency>
|
||||||
|
<groupId>com.google.cloud</groupId>
|
||||||
|
<artifactId>google-cloud-secretmanager</artifactId>
|
||||||
|
<exclusions>
|
||||||
|
<exclusion>
|
||||||
|
<groupId>commons-logging</groupId>
|
||||||
|
<artifactId>commons-logging</artifactId>
|
||||||
|
</exclusion>
|
||||||
|
</exclusions>
|
||||||
|
</dependency>
|
||||||
|
<dependency>
|
||||||
|
<groupId>com.google.auth</groupId>
|
||||||
|
<artifactId>google-auth-library-oauth2-http</artifactId>
|
||||||
|
<exclusions>
|
||||||
|
<exclusion>
|
||||||
|
<groupId>com.google.code.findbugs</groupId>
|
||||||
|
<artifactId>jsr305</artifactId>
|
||||||
|
</exclusion>
|
||||||
|
<exclusion>
|
||||||
|
<groupId>commons-logging</groupId>
|
||||||
|
<artifactId>commons-logging</artifactId>
|
||||||
|
</exclusion>
|
||||||
|
</exclusions>
|
||||||
|
</dependency>
|
||||||
|
<dependency>
|
||||||
|
<groupId>org.apache.nifi</groupId>
|
||||||
|
<artifactId>nifi-expression-language</artifactId>
|
||||||
|
<version>1.18.0-SNAPSHOT</version>
|
||||||
|
<scope>test</scope>
|
||||||
|
</dependency>
|
||||||
|
<dependency>
|
||||||
|
<groupId>org.apache.nifi</groupId>
|
||||||
|
<artifactId>nifi-mock</artifactId>
|
||||||
|
<version>1.18.0-SNAPSHOT</version>
|
||||||
|
<scope>test</scope>
|
||||||
|
</dependency>
|
||||||
|
</dependencies>
|
||||||
|
</project>
|
||||||
|
|
@ -0,0 +1,200 @@
|
||||||
|
/*
|
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||||
|
* contributor license agreements. See the NOTICE file distributed with
|
||||||
|
* this work for additional information regarding copyright ownership.
|
||||||
|
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||||
|
* (the "License"); you may not use this file except in compliance with
|
||||||
|
* the License. You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
|
package org.apache.nifi.parameter.gcp;
|
||||||
|
|
||||||
|
import com.google.cloud.secretmanager.v1.AccessSecretVersionResponse;
|
||||||
|
import com.google.cloud.secretmanager.v1.ProjectName;
|
||||||
|
import com.google.cloud.secretmanager.v1.Secret;
|
||||||
|
import com.google.cloud.secretmanager.v1.SecretManagerServiceClient;
|
||||||
|
import com.google.cloud.secretmanager.v1.SecretManagerServiceClient.ListSecretsPage;
|
||||||
|
import com.google.cloud.secretmanager.v1.SecretManagerServiceSettings;
|
||||||
|
import com.google.cloud.secretmanager.v1.SecretVersionName;
|
||||||
|
import org.apache.nifi.annotation.documentation.CapabilityDescription;
|
||||||
|
import org.apache.nifi.annotation.documentation.Tags;
|
||||||
|
import org.apache.nifi.components.ConfigVerificationResult;
|
||||||
|
import org.apache.nifi.components.PropertyDescriptor;
|
||||||
|
import org.apache.nifi.controller.ConfigurationContext;
|
||||||
|
import org.apache.nifi.gcp.credentials.service.GCPCredentialsService;
|
||||||
|
import org.apache.nifi.logging.ComponentLog;
|
||||||
|
import org.apache.nifi.parameter.AbstractParameterProvider;
|
||||||
|
import org.apache.nifi.parameter.Parameter;
|
||||||
|
import org.apache.nifi.parameter.ParameterDescriptor;
|
||||||
|
import org.apache.nifi.parameter.ParameterGroup;
|
||||||
|
import org.apache.nifi.parameter.VerifiableParameterProvider;
|
||||||
|
import org.apache.nifi.processor.util.StandardValidators;
|
||||||
|
import org.apache.nifi.util.StringUtils;
|
||||||
|
import org.slf4j.Logger;
|
||||||
|
import org.slf4j.LoggerFactory;
|
||||||
|
|
||||||
|
import java.io.IOException;
|
||||||
|
import java.util.ArrayList;
|
||||||
|
import java.util.Arrays;
|
||||||
|
import java.util.Collections;
|
||||||
|
import java.util.HashMap;
|
||||||
|
import java.util.List;
|
||||||
|
import java.util.Map;
|
||||||
|
import java.util.regex.Pattern;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Reads secrets from GCP Secret Manager to provide parameter values. Secrets must be created similar to the following GCP cli command: <br/><br/>
|
||||||
|
* <code>gcp secretsmanager create-secret --name "[Context]" --secret-string '{ "[Param]": "[secretValue]", "[Param2]": "[secretValue2]" }'</code> <br/><br/>
|
||||||
|
*
|
||||||
|
*/
|
||||||
|
|
||||||
|
@Tags({"gcp", "secret", "manager"})
|
||||||
|
@CapabilityDescription("Fetches parameters from GCP Secret Manager. Each secret becomes a Parameter, which can be mapped to a Parameter Group " +
|
||||||
|
"by adding a GCP label named 'group-name'.")
|
||||||
|
public class GcpSecretManagerParameterProvider extends AbstractParameterProvider implements VerifiableParameterProvider {
|
||||||
|
private static final Logger logger = LoggerFactory.getLogger(GcpSecretManagerParameterProvider.class);
|
||||||
|
|
||||||
|
public static final PropertyDescriptor GROUP_NAME_PATTERN = new PropertyDescriptor.Builder()
|
||||||
|
.name("group-name-pattern")
|
||||||
|
.displayName("Group Name Pattern")
|
||||||
|
.description("A Regular Expression matching on the 'group-name' label value that identifies Secrets whose parameters should be fetched. " +
|
||||||
|
"Any secrets without a 'group-name' label value that matches this Regex will not be fetched.")
|
||||||
|
.addValidator(StandardValidators.REGULAR_EXPRESSION_VALIDATOR)
|
||||||
|
.required(true)
|
||||||
|
.defaultValue(".*")
|
||||||
|
.build();
|
||||||
|
|
||||||
|
public static final PropertyDescriptor PROJECT_ID = new PropertyDescriptor
|
||||||
|
.Builder().name("gcp-project-id")
|
||||||
|
.displayName("Project ID")
|
||||||
|
.description("Google Cloud Project ID")
|
||||||
|
.addValidator(StandardValidators.NON_EMPTY_VALIDATOR)
|
||||||
|
.required(true)
|
||||||
|
.build();
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Links to the {@link GCPCredentialsService} which provides credentials for this particular processor.
|
||||||
|
*/
|
||||||
|
public static final PropertyDescriptor GCP_CREDENTIALS_PROVIDER_SERVICE = new PropertyDescriptor.Builder()
|
||||||
|
.name("gcp-credentials-provider-service")
|
||||||
|
.displayName("GCP Credentials Provider Service")
|
||||||
|
.description("The Controller Service used to obtain Google Cloud Platform credentials.")
|
||||||
|
.required(true)
|
||||||
|
.identifiesControllerService(GCPCredentialsService.class)
|
||||||
|
.build();
|
||||||
|
|
||||||
|
private static final String GROUP_NAME_LABEL = "group-name";
|
||||||
|
private static final String SECRETS_PATH = "secrets/";
|
||||||
|
private static final List<PropertyDescriptor> PROPERTIES = Collections.unmodifiableList(Arrays.asList(
|
||||||
|
GROUP_NAME_PATTERN,
|
||||||
|
PROJECT_ID,
|
||||||
|
GCP_CREDENTIALS_PROVIDER_SERVICE
|
||||||
|
));
|
||||||
|
|
||||||
|
@Override
|
||||||
|
protected List<PropertyDescriptor> getSupportedPropertyDescriptors() {
|
||||||
|
return PROPERTIES;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public List<ParameterGroup> fetchParameters(final ConfigurationContext context) throws IOException {
|
||||||
|
final Map<String, ParameterGroup> providedParameterGroups = new HashMap<>();
|
||||||
|
final SecretManagerServiceClient secretsManager = this.configureClient(context);
|
||||||
|
final ProjectName projectName = ProjectName.of(context.getProperty(PROJECT_ID).getValue());
|
||||||
|
|
||||||
|
final SecretManagerServiceClient.ListSecretsPagedResponse pagedResponse = secretsManager.listSecrets(projectName);
|
||||||
|
ListSecretsPage page = pagedResponse.getPage();
|
||||||
|
do {
|
||||||
|
for (final Secret secret : page.getValues()) {
|
||||||
|
final String contextName = secret.getLabelsOrDefault(GROUP_NAME_LABEL, null);
|
||||||
|
if (contextName == null) {
|
||||||
|
getLogger().debug("Secret [{}] does not have the {} label, and will be skipped", secret.getName(), GROUP_NAME_LABEL);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
final String secretName = StringUtils.substringAfter(secret.getName(), SECRETS_PATH);
|
||||||
|
|
||||||
|
fetchSecret(secretsManager, context, secretName, contextName, providedParameterGroups);
|
||||||
|
}
|
||||||
|
if (page.hasNextPage()) {
|
||||||
|
page = page.getNextPage();
|
||||||
|
}
|
||||||
|
} while (page.hasNextPage());
|
||||||
|
return new ArrayList<>(providedParameterGroups.values());
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public List<ConfigVerificationResult> verify(final ConfigurationContext context, final ComponentLog verificationLogger) {
|
||||||
|
final List<ConfigVerificationResult> results = new ArrayList<>();
|
||||||
|
|
||||||
|
try {
|
||||||
|
final List<ParameterGroup> parameterGroups = fetchParameters(context);
|
||||||
|
int count = 0;
|
||||||
|
for (final ParameterGroup group : parameterGroups) {
|
||||||
|
count += group.getParameters().size();
|
||||||
|
}
|
||||||
|
results.add(new ConfigVerificationResult.Builder()
|
||||||
|
.outcome(ConfigVerificationResult.Outcome.SUCCESSFUL)
|
||||||
|
.verificationStepName("Fetch Parameters")
|
||||||
|
.explanation(String.format("Fetched secret keys [%d] as parameters within groups [%d]",
|
||||||
|
count, parameterGroups.size()))
|
||||||
|
.build());
|
||||||
|
} catch (final Exception e) {
|
||||||
|
verificationLogger.error("Failed to fetch parameters", e);
|
||||||
|
results.add(new ConfigVerificationResult.Builder()
|
||||||
|
.outcome(ConfigVerificationResult.Outcome.FAILED)
|
||||||
|
.verificationStepName("Fetch Parameters")
|
||||||
|
.explanation("Failed to fetch parameters: " + e.getMessage())
|
||||||
|
.build());
|
||||||
|
}
|
||||||
|
return results;
|
||||||
|
}
|
||||||
|
|
||||||
|
private void fetchSecret(final SecretManagerServiceClient secretsManager, final ConfigurationContext context, final String secretName,
|
||||||
|
final String groupName, final Map<String, ParameterGroup> providedParameterGroups) {
|
||||||
|
final Pattern groupNamePattern = Pattern.compile(context.getProperty(GROUP_NAME_PATTERN).getValue());
|
||||||
|
final String projectId = context.getProperty(PROJECT_ID).getValue();
|
||||||
|
|
||||||
|
if (!groupNamePattern.matcher(groupName).matches()) {
|
||||||
|
logger.debug("Secret [{}] label [{}] does not match the group name pattern {}", secretName, groupName, groupNamePattern);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
final SecretVersionName secretVersionName = SecretVersionName.of(projectId, secretName, "latest");
|
||||||
|
|
||||||
|
// Access the secret version.
|
||||||
|
final AccessSecretVersionResponse response = secretsManager.accessSecretVersion(secretVersionName);
|
||||||
|
final String parameterValue = response.getPayload().getData().toStringUtf8();
|
||||||
|
|
||||||
|
final Parameter parameter = createParameter(secretName, parameterValue);
|
||||||
|
|
||||||
|
if (parameter != null) {
|
||||||
|
final ParameterGroup group = providedParameterGroups
|
||||||
|
.computeIfAbsent(groupName, key -> new ParameterGroup(groupName, new ArrayList<>()));
|
||||||
|
final List<Parameter> updatedParameters = new ArrayList<>(group.getParameters());
|
||||||
|
updatedParameters.add(parameter);
|
||||||
|
providedParameterGroups.put(groupName, new ParameterGroup(groupName, updatedParameters));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private Parameter createParameter(final String parameterName, final String parameterValue) {
|
||||||
|
final ParameterDescriptor parameterDescriptor = new ParameterDescriptor.Builder().name(parameterName).build();
|
||||||
|
return new Parameter(parameterDescriptor, parameterValue, null, true);
|
||||||
|
}
|
||||||
|
|
||||||
|
SecretManagerServiceClient configureClient(final ConfigurationContext context) throws IOException {
|
||||||
|
final GCPCredentialsService credentialsService = context.getProperty(GCP_CREDENTIALS_PROVIDER_SERVICE).asControllerService(GCPCredentialsService.class);
|
||||||
|
|
||||||
|
final SecretManagerServiceClient client = SecretManagerServiceClient.create(SecretManagerServiceSettings
|
||||||
|
.newBuilder().setCredentialsProvider(() -> credentialsService.getGoogleCredentials())
|
||||||
|
.build());
|
||||||
|
|
||||||
|
return client;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,16 @@
|
||||||
|
# Licensed to the Apache Software Foundation (ASF) under one or more
|
||||||
|
# contributor license agreements. See the NOTICE file distributed with
|
||||||
|
# this work for additional information regarding copyright ownership.
|
||||||
|
# The ASF licenses this file to You under the Apache License, Version 2.0
|
||||||
|
# (the "License"); you may not use this file except in compliance with
|
||||||
|
# the License. You may obtain a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
# See the License for the specific language governing permissions and
|
||||||
|
# limitations under the License.
|
||||||
|
org.apache.nifi.parameter.gcp.GcpSecretManagerParameterProvider
|
||||||
|
|
||||||
|
|
@ -0,0 +1,55 @@
|
||||||
|
<!DOCTYPE html>
|
||||||
|
<html lang="en">
|
||||||
|
<!--
|
||||||
|
Licensed to the Apache Software Foundation (ASF) under one or more
|
||||||
|
contributor license agreements. See the NOTICE file distributed with
|
||||||
|
this work for additional information regarding copyright ownership.
|
||||||
|
The ASF licenses this file to You under the Apache License, Version 2.0
|
||||||
|
(the "License"); you may not use this file except in compliance with
|
||||||
|
the License. You may obtain a copy of the License at
|
||||||
|
http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
Unless required by applicable law or agreed to in writing, software
|
||||||
|
distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
See the License for the specific language governing permissions and
|
||||||
|
limitations under the License.
|
||||||
|
-->
|
||||||
|
<head>
|
||||||
|
<meta charset="utf-8" />
|
||||||
|
<title>GcpSecretManagerParameterProvider</title>
|
||||||
|
|
||||||
|
<link rel="stylesheet" href="../../../../../css/component-usage.css" type="text/css" />
|
||||||
|
</head>
|
||||||
|
<body>
|
||||||
|
<h3>Mapping GCP Secrets to Parameter Contexts</h3>
|
||||||
|
|
||||||
|
<p>
|
||||||
|
The GcpSecretManagerParameterProvider maps a Secret to a Parameter, which can be grouped by adding a "group-name" label.
|
||||||
|
To create a compatible secret from the GCP Console:
|
||||||
|
</p>
|
||||||
|
|
||||||
|
<ol>
|
||||||
|
<li>From the Secret Manager service, click the "Create Secret" button</li>
|
||||||
|
<li>Enter the Secret name. This is the name of a parameter. Enter a value.</li>
|
||||||
|
<li>Under "Labels", add a label with a Key of "group-name" and a value of the intended Parameter Group name.</li>
|
||||||
|
</ol>
|
||||||
|
|
||||||
|
<p>
|
||||||
|
Alternatively, from the command line, run a command like the following:
|
||||||
|
</p>
|
||||||
|
|
||||||
|
<pre>
|
||||||
|
printf "[Parameter Value]" | gcloud secrets create --labels=group-name="[Parameter Group Name]" "[Parameter Name]" --data-file=-
|
||||||
|
</pre>
|
||||||
|
|
||||||
|
<p>In this example, [Parameter Group Name] should be the intended name of the Parameter Group, [Parameter Name] should be
|
||||||
|
the parameter name, and [Parameter Value] should be the value of the parameter.</p>
|
||||||
|
|
||||||
|
<h3>Configuring the Parameter Provider</h3>
|
||||||
|
|
||||||
|
<p>
|
||||||
|
GCP Secrets must be explicitly matched in the "Group Name Pattern" property in order for them to be fetched. This
|
||||||
|
prevents more than the intended Secrets from being pulled into NiFi.
|
||||||
|
</p>
|
||||||
|
</body>
|
||||||
|
</html>
|
||||||
|
|
@ -0,0 +1,208 @@
|
||||||
|
/*
|
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||||
|
* contributor license agreements. See the NOTICE file distributed with
|
||||||
|
* this work for additional information regarding copyright ownership.
|
||||||
|
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||||
|
* (the "License"); you may not use this file except in compliance with
|
||||||
|
* the License. You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
|
package org.apache.nifi.parameter.gcp;
|
||||||
|
|
||||||
|
import com.google.cloud.secretmanager.v1.AccessSecretVersionResponse;
|
||||||
|
import com.google.cloud.secretmanager.v1.ProjectName;
|
||||||
|
import com.google.cloud.secretmanager.v1.Secret;
|
||||||
|
import com.google.cloud.secretmanager.v1.SecretManagerServiceClient;
|
||||||
|
import com.google.cloud.secretmanager.v1.SecretManagerServiceClient.ListSecretsPage;
|
||||||
|
import com.google.cloud.secretmanager.v1.SecretManagerServiceClient.ListSecretsPagedResponse;
|
||||||
|
import com.google.cloud.secretmanager.v1.SecretPayload;
|
||||||
|
import com.google.cloud.secretmanager.v1.SecretVersionName;
|
||||||
|
import com.google.protobuf.ByteString;
|
||||||
|
import org.apache.nifi.components.ConfigVerificationResult;
|
||||||
|
import org.apache.nifi.components.PropertyDescriptor;
|
||||||
|
import org.apache.nifi.parameter.Parameter;
|
||||||
|
import org.apache.nifi.parameter.ParameterDescriptor;
|
||||||
|
import org.apache.nifi.parameter.ParameterGroup;
|
||||||
|
import org.apache.nifi.parameter.VerifiableParameterProvider;
|
||||||
|
import org.apache.nifi.reporting.InitializationException;
|
||||||
|
import org.apache.nifi.util.MockComponentLog;
|
||||||
|
import org.apache.nifi.util.MockConfigurationContext;
|
||||||
|
import org.apache.nifi.util.MockParameterProviderInitializationContext;
|
||||||
|
import org.junit.jupiter.api.Test;
|
||||||
|
|
||||||
|
import java.io.IOException;
|
||||||
|
import java.util.ArrayList;
|
||||||
|
import java.util.Arrays;
|
||||||
|
import java.util.Collections;
|
||||||
|
import java.util.HashMap;
|
||||||
|
import java.util.Iterator;
|
||||||
|
import java.util.List;
|
||||||
|
import java.util.Map;
|
||||||
|
import java.util.stream.Collectors;
|
||||||
|
|
||||||
|
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||||
|
import static org.junit.jupiter.api.Assertions.assertThrows;
|
||||||
|
import static org.mockito.ArgumentMatchers.any;
|
||||||
|
import static org.mockito.ArgumentMatchers.argThat;
|
||||||
|
import static org.mockito.Mockito.doReturn;
|
||||||
|
import static org.mockito.Mockito.mock;
|
||||||
|
import static org.mockito.Mockito.spy;
|
||||||
|
import static org.mockito.Mockito.when;
|
||||||
|
|
||||||
|
public class TestGcpSecretManagerParameterProvider {
|
||||||
|
|
||||||
|
final List<Parameter> mySecretParameters = Arrays.asList(
|
||||||
|
parameter("paramA", "valueA"),
|
||||||
|
parameter("paramB", "valueB"),
|
||||||
|
parameter("otherC", "valueOther"),
|
||||||
|
parameter("paramD", "valueD"),
|
||||||
|
parameter("nonSensitiveE", "valueE"),
|
||||||
|
parameter("otherF", "valueF")
|
||||||
|
);
|
||||||
|
final List<Parameter> otherSecretParameters = Arrays.asList(
|
||||||
|
parameter("paramG", "valueG"),
|
||||||
|
parameter("otherH", "valueOther")
|
||||||
|
);
|
||||||
|
final List<Parameter> unrelatedParameters = Collections.singletonList(parameter("paramK", "unused"));
|
||||||
|
final List<ParameterGroup> mockParameterGroups = Arrays.asList(
|
||||||
|
new ParameterGroup("MySecret", mySecretParameters),
|
||||||
|
new ParameterGroup("OtherSecret", otherSecretParameters),
|
||||||
|
new ParameterGroup("Unrelated", unrelatedParameters) // will not be picked up
|
||||||
|
);
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testFetchParametersWithNoSecrets() throws InitializationException, IOException {
|
||||||
|
final List<ParameterGroup> expectedGroups = Collections.singletonList(new ParameterGroup("MySecret", Collections.emptyList()));
|
||||||
|
runProviderTest(mockSecretManagerClient(expectedGroups), 0, ConfigVerificationResult.Outcome.SUCCESSFUL);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testFetchParameters() throws InitializationException, IOException {
|
||||||
|
runProviderTest(mockSecretManagerClient(mockParameterGroups), 8, ConfigVerificationResult.Outcome.SUCCESSFUL);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testFetchParametersListFailure() throws InitializationException, IOException {
|
||||||
|
final SecretManagerServiceClient mockSecretsManager = mock(SecretManagerServiceClient.class);
|
||||||
|
when(mockSecretsManager.listSecrets(any(ProjectName.class))).thenThrow(new RuntimeException("Fake exception"));
|
||||||
|
runProviderTest(mockSecretsManager, 0, ConfigVerificationResult.Outcome.FAILED);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testFetchParametersGetSecretFailure() throws InitializationException, IOException {
|
||||||
|
final SecretManagerServiceClient mockSecretsManager = mock(SecretManagerServiceClient.class);
|
||||||
|
final ListSecretsPagedResponse listSecretsPagedResponse = mock(ListSecretsPagedResponse.class);
|
||||||
|
final ListSecretsPage page = mock(ListSecretsPage.class);
|
||||||
|
when(listSecretsPagedResponse.getPage()).thenReturn(page);
|
||||||
|
|
||||||
|
when(page.hasNextPage()).thenReturn(false);
|
||||||
|
final Secret secret = mock(Secret.class);
|
||||||
|
when(secret.getName()).thenReturn("paramA");
|
||||||
|
when(secret.getLabelsOrDefault("group-name", null)).thenReturn("Secret");
|
||||||
|
when(page.getValues()).thenReturn(Collections.singletonList(secret));
|
||||||
|
|
||||||
|
when(mockSecretsManager.accessSecretVersion(any(SecretVersionName.class))).thenThrow(new RuntimeException("Fake exception"));
|
||||||
|
runProviderTest(mockSecretsManager, 0, ConfigVerificationResult.Outcome.FAILED);
|
||||||
|
}
|
||||||
|
|
||||||
|
private GcpSecretManagerParameterProvider getParameterProvider() {
|
||||||
|
return spy(new GcpSecretManagerParameterProvider());
|
||||||
|
}
|
||||||
|
|
||||||
|
private SecretManagerServiceClient mockSecretManagerClient(final List<ParameterGroup> mockGroups) {
|
||||||
|
final SecretManagerServiceClient secretManager = mock(SecretManagerServiceClient.class);
|
||||||
|
|
||||||
|
final ListSecretsPagedResponse listSecretsPagedResponse = mock(ListSecretsPagedResponse.class);
|
||||||
|
when(secretManager.listSecrets(any(ProjectName.class))).thenReturn(listSecretsPagedResponse);
|
||||||
|
|
||||||
|
boolean mockedFirstPage = false;
|
||||||
|
ListSecretsPage currentPage;
|
||||||
|
ListSecretsPage previousPage = null;
|
||||||
|
for (final Iterator<ParameterGroup> it = mockGroups.iterator(); it.hasNext(); ) {
|
||||||
|
final ParameterGroup group = it.next();
|
||||||
|
currentPage = mock(ListSecretsPage.class);
|
||||||
|
if (mockedFirstPage) {
|
||||||
|
when(previousPage.getNextPage()).thenReturn(currentPage);
|
||||||
|
} else {
|
||||||
|
when(listSecretsPagedResponse.getPage()).thenReturn(currentPage);
|
||||||
|
mockedFirstPage = true;
|
||||||
|
}
|
||||||
|
final List<Secret> values = new ArrayList<>();
|
||||||
|
values.addAll(group.getParameters().stream()
|
||||||
|
.map(parameter -> mockSecret(secretManager, group.getGroupName(), parameter))
|
||||||
|
.collect(Collectors.toList()));
|
||||||
|
when(currentPage.getValues()).thenReturn(values);
|
||||||
|
if (it.hasNext()) {
|
||||||
|
when(currentPage.hasNextPage()).thenReturn(true);
|
||||||
|
previousPage = currentPage;
|
||||||
|
} else {
|
||||||
|
when(currentPage.hasNextPage()).thenReturn(false);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return secretManager;
|
||||||
|
}
|
||||||
|
|
||||||
|
final Secret mockSecret(final SecretManagerServiceClient secretManager, final String groupName, final Parameter parameter) {
|
||||||
|
final Secret secret = mock(Secret.class);
|
||||||
|
|
||||||
|
final String parameterName = parameter.getDescriptor().getName();
|
||||||
|
when(secret.getName()).thenReturn("projects/project/secrets/" + parameterName);
|
||||||
|
when(secret.getLabelsOrDefault("group-name", null)).thenReturn(groupName);
|
||||||
|
|
||||||
|
final AccessSecretVersionResponse response = mock(AccessSecretVersionResponse.class);
|
||||||
|
doReturn(response).when(secretManager).accessSecretVersion(argThat((SecretVersionName secretVersionName) -> secretVersionName.getSecret().equals(parameterName)));
|
||||||
|
final SecretPayload payload = mock(SecretPayload.class);
|
||||||
|
final ByteString data = ByteString.copyFromUtf8(parameter.getValue());
|
||||||
|
when(payload.getData()).thenReturn(data);
|
||||||
|
when(response.getPayload()).thenReturn(payload);
|
||||||
|
return secret;
|
||||||
|
}
|
||||||
|
|
||||||
|
private List<ParameterGroup> runProviderTest(final SecretManagerServiceClient secretsManager, final int expectedCount,
|
||||||
|
final ConfigVerificationResult.Outcome expectedOutcome) throws InitializationException, IOException {
|
||||||
|
|
||||||
|
final GcpSecretManagerParameterProvider parameterProvider = getParameterProvider();
|
||||||
|
doReturn(secretsManager).when(parameterProvider).configureClient(any());
|
||||||
|
final MockParameterProviderInitializationContext initContext = new MockParameterProviderInitializationContext("id", "name",
|
||||||
|
new MockComponentLog("providerId", parameterProvider));
|
||||||
|
parameterProvider.initialize(initContext);
|
||||||
|
|
||||||
|
final Map<PropertyDescriptor, String> properties = new HashMap<>();
|
||||||
|
properties.put(GcpSecretManagerParameterProvider.GROUP_NAME_PATTERN, ".*Secret");
|
||||||
|
properties.put(GcpSecretManagerParameterProvider.PROJECT_ID, "my-project");
|
||||||
|
final MockConfigurationContext mockConfigurationContext = new MockConfigurationContext(properties, null);
|
||||||
|
|
||||||
|
List<ParameterGroup> parameterGroups = new ArrayList<>();
|
||||||
|
// Verify parameter fetching
|
||||||
|
if (expectedOutcome == ConfigVerificationResult.Outcome.FAILED) {
|
||||||
|
assertThrows(RuntimeException.class, () -> parameterProvider.fetchParameters(mockConfigurationContext));
|
||||||
|
} else {
|
||||||
|
parameterGroups = parameterProvider.fetchParameters(mockConfigurationContext);
|
||||||
|
final int count = (int) parameterGroups.stream()
|
||||||
|
.flatMap(group -> group.getParameters().stream())
|
||||||
|
.count();
|
||||||
|
assertEquals(expectedCount, count);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Verify config verification
|
||||||
|
final List<ConfigVerificationResult> results = ((VerifiableParameterProvider) parameterProvider).verify(mockConfigurationContext, initContext.getLogger());
|
||||||
|
|
||||||
|
assertEquals(1, results.size());
|
||||||
|
assertEquals(expectedOutcome, results.get(0).getOutcome());
|
||||||
|
|
||||||
|
return parameterGroups;
|
||||||
|
}
|
||||||
|
|
||||||
|
private static Parameter parameter(final String name, final String value) {
|
||||||
|
return new Parameter(new ParameterDescriptor.Builder().name(name).build(), value);
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,15 @@
|
||||||
|
# Licensed to the Apache Software Foundation (ASF) under one or more
|
||||||
|
# contributor license agreements. See the NOTICE file distributed with
|
||||||
|
# this work for additional information regarding copyright ownership.
|
||||||
|
# The ASF licenses this file to You under the Apache License, Version 2.0
|
||||||
|
# (the "License"); you may not use this file except in compliance with
|
||||||
|
# the License. You may obtain a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
# See the License for the specific language governing permissions and
|
||||||
|
# limitations under the License.
|
||||||
|
mock-maker-inline
|
||||||
|
|
@ -45,6 +45,7 @@
|
||||||
<module>nifi-gcp-services-api</module>
|
<module>nifi-gcp-services-api</module>
|
||||||
<module>nifi-gcp-services-api-nar</module>
|
<module>nifi-gcp-services-api-nar</module>
|
||||||
<module>nifi-gcp-processors</module>
|
<module>nifi-gcp-processors</module>
|
||||||
|
<module>nifi-gcp-parameter-providers</module>
|
||||||
<module>nifi-gcp-nar</module>
|
<module>nifi-gcp-nar</module>
|
||||||
</modules>
|
</modules>
|
||||||
</project>
|
</project>
|
||||||
|
|
|
||||||
Loading…
Reference in New Issue