Apache
/
nifi
mirror of https://github.com/apache/nifi.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

NIFI-12080 Added support for KV_2 in HashiCorp Parameter Provider 

This closes #9209

Signed-off-by: David Handermann <exceptionfactory@apache.org>
This commit is contained in:
Pierre Villard 2024-08-28 22:12:41 +02:00 committed by exceptionfactory
parent 31a3014696
commit f828907df5
No known key found for this signature in database
4 changed files with 63 additions and 16 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
nifi-commons/nifi-hashicorp-vault-api/src/main/java/org/apache/nifi/vault/hashicorp/HashiCorpVaultCommunicationService.java
View File

@ -89,10 +89,29 @@ public interface HashiCorpVaultCommunicationService {
*/
Map<String, String> readKeyValueSecretMap(String keyValuePath, String secretKey);
/**
* Reads a secret with multiple key/value pairs from Vault's Key/Value Secrets Engine.
*
* @see <a href="https://www.vaultproject.io/api-docs/secret/kv">https://www.vaultproject.io/api-docs/secret/kv</a>
* @param keyValuePath The Vault path to use for the configured Key/Value Secrets Engine
* @param secretKey The secret key
* @param version the Key/Vault Secrets engine version
* @return A map from key to value from the secret key/values, or an empty map if not found
*/
Map<String, String> readKeyValueSecretMap(String keyValuePath, String secretKey, String version);
/**
* Lists the secrets at the given Key/Value Version 1 Secrets Engine path.
* @param keyValuePath The Vault path to list
* @return The list of secret names
*/
List<String> listKeyValueSecrets(String keyValuePath);
/**
* Lists the secrets at the given Key/Value Secrets Engine path.
* @param keyValuePath The Vault path to list
* @param version the Key/Vault Secrets engine version
* @return The list of secret names
*/
List<String> listKeyValueSecrets(String keyValuePath, String version);
}

18
nifi-commons/nifi-hashicorp-vault/src/main/java/org/apache/nifi/vault/hashicorp/StandardHashiCorpVaultCommunicationService.java
View File

@ -136,17 +136,23 @@ public class StandardHashiCorpVaultCommunicationService implements HashiCorpVaul
@Override
public Map<String, String> readKeyValueSecretMap(final String keyValuePath, final String key) {
final VaultKeyValueOperations keyValueOperations = keyValueOperationsMap
.computeIfAbsent(keyValuePath, path -> vaultTemplate.opsForKeyValue(path, keyValueBackend));
final VaultResponseSupport<Map> response = keyValueOperations.get(key, Map.class);
return readKeyValueSecretMap(keyValuePath, key, keyValueBackend.name());
}
@Override
public Map<String, String> readKeyValueSecretMap(final String keyValuePath, final String key, final String version) {
final VaultResponseSupport<Map> response = vaultTemplate.opsForKeyValue(keyValuePath, KeyValueBackend.valueOf(version)).get(key, Map.class);
return response == null ? Collections.emptyMap() : (Map<String, String>) response.getRequiredData();
}
@Override
public List<String> listKeyValueSecrets(final String keyValuePath) {
final VaultKeyValueOperations keyValueOperations = keyValueOperationsMap
.computeIfAbsent(keyValuePath, path -> vaultTemplate.opsForKeyValue(path, KeyValueBackend.KV_1));
return keyValueOperations.list("/");
return listKeyValueSecrets(keyValuePath, KeyValueBackend.KV_1.name());
}
@Override
public List<String> listKeyValueSecrets(final String keyValuePath, final String version) {
return vaultTemplate.opsForKeyValue(keyValuePath, KeyValueBackend.valueOf(version)).list("/");
}
private static class SecretData {

24
nifi-extension-bundles/nifi-hashicorp-vault-bundle/nifi-hashicorp-vault-parameter-provider/src/main/java/org/apache/nifi/vault/hashicorp/HashiCorpVaultParameterProvider.java
View File

@ -18,6 +18,7 @@ package org.apache.nifi.vault.hashicorp;
import org.apache.nifi.annotation.documentation.CapabilityDescription;
import org.apache.nifi.annotation.documentation.Tags;
import org.apache.nifi.components.AllowableValue;
import org.apache.nifi.components.ConfigVerificationResult;
import org.apache.nifi.components.PropertyDescriptor;
import org.apache.nifi.components.Validator;
@ -37,11 +38,15 @@ import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
@CapabilityDescription("Provides parameters from HashiCorp Vault Key/Value Version 1 Secrets. Each Secret represents a parameter group, " +
@CapabilityDescription("Provides parameters from HashiCorp Vault Key/Value Version 1 and Version 2 Secrets. Each Secret represents a parameter group, " +
"which will map to a Parameter Context. The keys and values in the Secret map to Parameters.")
@Tags({"hashicorp", "vault", "secret"})
public class HashiCorpVaultParameterProvider extends AbstractParameterProvider implements ParameterProvider, VerifiableParameterProvider {
static final AllowableValue KV_1 = new AllowableValue("KV_1", "KV_1", "Key/Value Secret Engine Version 1.");
static final AllowableValue KV_2 = new AllowableValue("KV_2", "KV_2", "Key/Value Secret Engine Version 2. "
+ "If multiple versions of the secret exist, latest will be used.");
public static final PropertyDescriptor VAULT_CLIENT_SERVICE = new PropertyDescriptor.Builder()
.name("vault-client-service")
.displayName("HashiCorp Vault Client Service")
@ -53,11 +58,20 @@ public class HashiCorpVaultParameterProvider extends AbstractParameterProvider i
public static final PropertyDescriptor KV_PATH = new PropertyDescriptor.Builder()
.name("kv-path")
.displayName("Key/Value Path")
.description("The HashiCorp Vault path to the Key/Value Version 1 Secrets Engine")
.description("The HashiCorp Vault path to the Key/Value Secrets Engine")
.addValidator(StandardValidators.NON_EMPTY_VALIDATOR)
.required(true)
.defaultValue("kv")
.build();
public static final PropertyDescriptor KV_VERSION = new PropertyDescriptor.Builder()
.name("kv-version")
.displayName("Key/Value Version")
.description("The version of the Key/Value Secrets Engine")
.addValidator(StandardValidators.NON_EMPTY_VALIDATOR)
.required(true)
.allowableValues(KV_1, KV_2)
.defaultValue(KV_1)
.build();
public static final PropertyDescriptor SECRET_NAME_PATTERN = new PropertyDescriptor.Builder()
.name("secret-name-pattern")
.displayName("Secret Name Pattern")
@ -70,6 +84,7 @@ public class HashiCorpVaultParameterProvider extends AbstractParameterProvider i
private static final List<PropertyDescriptor> PROPERTIES = Collections.unmodifiableList(Arrays.asList(
VAULT_CLIENT_SERVICE,
KV_PATH,
KV_VERSION,
SECRET_NAME_PATTERN));
private HashiCorpVaultCommunicationService vaultCommunicationService;
@ -92,15 +107,16 @@ public class HashiCorpVaultParameterProvider extends AbstractParameterProvider i
private List<ParameterGroup> getParameterGroups(final HashiCorpVaultCommunicationService vaultCommunicationService,
final ConfigurationContext context) {
final String kvPath = context.getProperty(KV_PATH).getValue();
final String kvVersion = context.getProperty(KV_VERSION).getValue();
final String secretIncludeRegex = context.getProperty(SECRET_NAME_PATTERN).getValue();
final List<String> allSecretNames = vaultCommunicationService.listKeyValueSecrets(kvPath);
final List<String> allSecretNames = vaultCommunicationService.listKeyValueSecrets(kvPath, kvVersion);
final List<String> secretNames = allSecretNames.stream()
.filter(name -> name.matches(secretIncludeRegex))
.collect(Collectors.toList());
final List<ParameterGroup> parameterGroups = new ArrayList<>();
for (final String secretName : secretNames) {
final Map<String, String> keyValues = vaultCommunicationService.readKeyValueSecretMap(kvPath, secretName);
final Map<String, String> keyValues = vaultCommunicationService.readKeyValueSecretMap(kvPath, secretName, kvVersion);
final List<Parameter> parameters = new ArrayList<>();
keyValues.forEach( (key, value) -> {
parameters.add(new Parameter.Builder()

18
nifi-extension-bundles/nifi-hashicorp-vault-bundle/nifi-hashicorp-vault-parameter-provider/src/test/java/org/apache/nifi/vault/hashicorp/TestHashiCorpVaultParameterProvider.java
View File

@ -84,10 +84,12 @@ public class TestHashiCorpVaultParameterProvider {
@Test
public void testFetchParameters() {
mockSecrets("kv2", mockedGroups);
final String kvVersion = "KV_1";
mockSecrets("kv2", kvVersion, mockedGroups);
final Map<PropertyDescriptor, String> properties = new HashMap<>();
properties.put(HashiCorpVaultParameterProvider.KV_PATH, "kv2");
properties.put(HashiCorpVaultParameterProvider.KV_VERSION, kvVersion);
properties.put(HashiCorpVaultParameterProvider.VAULT_CLIENT_SERVICE, "service");
properties.put(HashiCorpVaultParameterProvider.SECRET_NAME_PATTERN, ".*");
final ConfigurationContext context = mockContext(properties);
@ -101,10 +103,12 @@ public class TestHashiCorpVaultParameterProvider {
@Test
public void testFetchParametersSecretRegex() {
mockSecrets("kv2", mockedGroups);
final String kvVersion = "KV_2";
mockSecrets("kv2", kvVersion, mockedGroups);
final Map<PropertyDescriptor, String> properties = new HashMap<>();
properties.put(HashiCorpVaultParameterProvider.KV_PATH, "kv2");
properties.put(HashiCorpVaultParameterProvider.KV_VERSION, kvVersion);
properties.put(HashiCorpVaultParameterProvider.VAULT_CLIENT_SERVICE, "service");
properties.put(HashiCorpVaultParameterProvider.SECRET_NAME_PATTERN, ".*A");
final ConfigurationContext context = mockContext(properties);
@ -118,10 +122,12 @@ public class TestHashiCorpVaultParameterProvider {
@Test
public void testVerifyParameters() {
mockSecrets("kv2", mockedGroups);
final String kvVersion = "KV_1";
mockSecrets("kv2", kvVersion, mockedGroups);
final Map<PropertyDescriptor, String> properties = new HashMap<>();
properties.put(HashiCorpVaultParameterProvider.KV_PATH, "kv2");
properties.put(HashiCorpVaultParameterProvider.KV_VERSION, kvVersion);
properties.put(HashiCorpVaultParameterProvider.VAULT_CLIENT_SERVICE, "service");
properties.put(HashiCorpVaultParameterProvider.SECRET_NAME_PATTERN, ".*");
final ConfigurationContext context = mockContext(properties);
@ -145,13 +151,13 @@ public class TestHashiCorpVaultParameterProvider {
lenient().when(context.getProperty(descriptor)).thenReturn(propertyValue);
}
private void mockSecrets(final String kvPath, final List<ParameterGroup> parameterGroups) {
when(vaultCommunicationService.listKeyValueSecrets(kvPath))
private void mockSecrets(final String kvPath, final String kvVersion, final List<ParameterGroup> parameterGroups) {
when(vaultCommunicationService.listKeyValueSecrets(kvPath, kvVersion))
.thenReturn(parameterGroups.stream().map(group -> group.getGroupName()).collect(Collectors.toList()));
for (final ParameterGroup parameterGroup : parameterGroups) {
final Map<String, String> keyValues = parameterGroup.getParameters().stream()
.collect(Collectors.toMap(parameter -> parameter.getDescriptor().getName(), parameter -> parameter.getValue()));
lenient().when(vaultCommunicationService.readKeyValueSecretMap(kvPath, parameterGroup.getGroupName())).thenReturn(keyValues);
lenient().when(vaultCommunicationService.readKeyValueSecretMap(kvPath, parameterGroup.getGroupName(), kvVersion)).thenReturn(keyValues);
}
}