Apache
/
nifi
mirror of https://github.com/apache/nifi.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

NIFI-9263: Using secret-per-context model for Stateless Vault PVP 

Rebasing after PVP refactor

Signed-off-by: Nathan Gough <thenatog@gmail.com>

This closes #5427.
This commit is contained in:
Joe Gresock 2021-10-01 10:29:37 -04:00 committed by Nathan Gough
parent d2f8f97b10
commit 207da405ee
6 changed files with 156 additions and 50 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

32
nifi-commons/nifi-vault-utils/src/main/java/org/apache/nifi/vault/hashicorp/HashiCorpVaultCommunicationService.java
View File

@ -16,6 +16,7 @@
*/
package org.apache.nifi.vault.hashicorp;
import java.util.Map;
import java.util.Optional;
/**
@ -45,22 +46,41 @@ public interface HashiCorpVaultCommunicationService {
byte[] decrypt(String transitPath, String cipherText);
/**
* Writes a secret using Vault's unversioned Key/Value Secrets Engine.
* Writes a single secret value using Vault's unversioned Key/Value Secrets Engine.
*
* @see <a href="https://www.vaultproject.io/api-docs/secret/kv/kv-v1">https://www.vaultproject.io/api-docs/secret/kv/kv-v1</a>
* @param keyValuePath The Vault path to use for the configured Key/Value v1 Secrets Engine
* @param key The secret key
* @param secretKey The secret key
* @param value The secret value
*/
void writeKeyValueSecret(String keyValuePath, String key, String value);
void writeKeyValueSecret(String keyValuePath, String secretKey, String value);
/**
* Reads a secret from Vault's unversioned Key/Value Secrets Engine.
* Reads a single secret value from Vault's unversioned Key/Value Secrets Engine.
*
* @see <a href="https://www.vaultproject.io/api-docs/secret/kv/kv-v1">https://www.vaultproject.io/api-docs/secret/kv/kv-v1</a>
* @param keyValuePath The Vault path to use for the configured Key/Value v1 Secrets Engine
* @param key The secret key
* @param secretKey The secret key
* @return The secret value, or empty if not found
*/
Optional<String> readKeyValueSecret(String keyValuePath, String key);
Optional<String> readKeyValueSecret(String keyValuePath, String secretKey);
/**
* Writes a secret with multiple key/value pairs using Vault's unversioned Key/Value Secrets Engine.
*
* @see <a href="https://www.vaultproject.io/api-docs/secret/kv/kv-v1">https://www.vaultproject.io/api-docs/secret/kv/kv-v1</a>
* @param keyValuePath The Vault path to use for the configured Key/Value v1 Secrets Engine
* @param keyValues A map from key to value for keys/values that should be stored in the secret
*/
void writeKeyValueSecretMap(String keyValuePath, String secretKey, Map<String, String> keyValues);
/**
* Reads a secret with multiple key/value pairs from Vault's unversioned Key/Value Secrets Engine.
*
* @see <a href="https://www.vaultproject.io/api-docs/secret/kv/kv-v1">https://www.vaultproject.io/api-docs/secret/kv/kv-v1</a>
* @param keyValuePath The Vault path to use for the configured Key/Value v1 Secrets Engine
* @param secretKey The secret key
* @return A map from key to value from the secret key/values, or an empty map if not found
*/
Map<String, String> readKeyValueSecretMap(String keyValuePath, String secretKey);
}

47
nifi-commons/nifi-vault-utils/src/main/java/org/apache/nifi/vault/hashicorp/StandardHashiCorpVaultCommunicationService.java
View File

@ -31,8 +31,10 @@ import org.springframework.vault.support.Ciphertext;
import org.springframework.vault.support.Plaintext;
import org.springframework.vault.support.VaultResponseSupport;
import java.util.Collections;
import java.util.HashMap;
import java.util.Map;
import java.util.Objects;
import java.util.Optional;
import static org.springframework.vault.core.VaultKeyValueOperationsSupport.KeyValueBackend.KV_1;
@ -41,7 +43,6 @@ import static org.springframework.vault.core.VaultKeyValueOperationsSupport.KeyV
* Implements the VaultCommunicationService using Spring Vault
*/
public class StandardHashiCorpVaultCommunicationService implements HashiCorpVaultCommunicationService {
private final HashiCorpVaultConfiguration vaultConfiguration;
private final VaultTemplate vaultTemplate;
private final VaultTransitOperations transitOperations;
private final Map<String, VaultKeyValueOperations> keyValueOperationsMap;
@ -52,7 +53,7 @@ public class StandardHashiCorpVaultCommunicationService implements HashiCorpVaul
* @throws HashiCorpVaultConfigurationException If the configuration was invalid
*/
public StandardHashiCorpVaultCommunicationService(final PropertySource<?>... propertySources) throws HashiCorpVaultConfigurationException {
vaultConfiguration = new HashiCorpVaultConfiguration(propertySources);
final HashiCorpVaultConfiguration vaultConfiguration = new HashiCorpVaultConfiguration(propertySources);
vaultTemplate = new VaultTemplate(vaultConfiguration.vaultEndpoint(),
ClientHttpRequestFactoryFactory.create(vaultConfiguration.clientOptions(), vaultConfiguration.sslConfiguration()),
@ -82,32 +83,58 @@ public class StandardHashiCorpVaultCommunicationService implements HashiCorpVaul
}
/**
* Writes the value to the "value" key of the secret with the path [keyValuePath]/[key].
* Writes the value to the "value" secretKey of the secret with the path [keyValuePath]/[secretKey].
* @param keyValuePath The Vault path to use for the configured Key/Value v1 Secrets Engine
* @param key The secret key
* @param secretKey The secret secretKey
* @param value The secret value
*/
@Override
public void writeKeyValueSecret(final String keyValuePath, final String key, final String value) {
public void writeKeyValueSecret(final String keyValuePath, final String secretKey, final String value) {
Objects.requireNonNull(keyValuePath, "Vault K/V path must be specified");
Objects.requireNonNull(secretKey, "Secret secretKey must be specified");
Objects.requireNonNull(value, "Secret value must be specified");
final VaultKeyValueOperations keyValueOperations = keyValueOperationsMap
.computeIfAbsent(keyValuePath, path -> vaultTemplate.opsForKeyValue(path, KV_1));
keyValueOperations.put(key, new SecretData(value));
keyValueOperations.put(secretKey, new SecretData(value));
}
/**
* Returns the value of the "value" key from the secret at the path [keyValuePath]/[key].
* Returns the value of the "value" secretKey from the secret at the path [keyValuePath]/[secretKey].
* @param keyValuePath The Vault path to use for the configured Key/Value v1 Secrets Engine
* @param key The secret key
* @param secretKey The secret secretKey
* @return The value of the secret
*/
@Override
public Optional<String> readKeyValueSecret(final String keyValuePath, final String key) {
public Optional<String> readKeyValueSecret(final String keyValuePath, final String secretKey) {
Objects.requireNonNull(keyValuePath, "Vault K/V path must be specified");
Objects.requireNonNull(secretKey, "Secret secretKey must be specified");
final VaultKeyValueOperations keyValueOperations = keyValueOperationsMap
.computeIfAbsent(keyValuePath, path -> vaultTemplate.opsForKeyValue(path, KV_1));
final VaultResponseSupport<SecretData> response = keyValueOperations.get(key, SecretData.class);
final VaultResponseSupport<SecretData> response = keyValueOperations.get(secretKey, SecretData.class);
return response == null ? Optional.empty() : Optional.ofNullable(response.getRequiredData().getValue());
}
@Override
public void writeKeyValueSecretMap(final String keyValuePath, final String secretKey, final Map<String, String> keyValues) {
Objects.requireNonNull(keyValuePath, "Vault K/V path must be specified");
Objects.requireNonNull(secretKey, "Secret secretKey must be specified");
Objects.requireNonNull(keyValues, "Key/values map must be specified");
if (keyValues.isEmpty()) {
return;
}
final VaultKeyValueOperations keyValueOperations = keyValueOperationsMap
.computeIfAbsent(keyValuePath, path -> vaultTemplate.opsForKeyValue(path, KV_1));
keyValueOperations.put(secretKey, keyValues);
}
@Override
public Map<String, String> readKeyValueSecretMap(final String keyValuePath, final String key) {
final VaultKeyValueOperations keyValueOperations = keyValueOperationsMap
.computeIfAbsent(keyValuePath, path -> vaultTemplate.opsForKeyValue(path, KV_1));
final VaultResponseSupport<Map> response = keyValueOperations.get(key, Map.class);
return response == null ? Collections.emptyMap() : (Map<String, String>) response.getRequiredData();
}
private static class SecretData {
private final String value;

26
nifi-commons/nifi-vault-utils/src/test/java/org/apache/nifi/vault/hashicorp/StandardHashiCorpVaultCommunicationServiceIT.java
View File

@ -21,6 +21,8 @@ import org.junit.Before;
import org.junit.Test;
import java.nio.charset.StandardCharsets;
import java.util.HashMap;
import java.util.Map;
import static org.junit.Assert.assertEquals;
@ -67,6 +69,9 @@ public class StandardHashiCorpVaultCommunicationServiceIT {
assertEquals(plaintext, new String(decrypted, StandardCharsets.UTF_8));
}
/**
* Run <code>vault kv get kv/key</code> to see the secret
*/
@Test
public void testReadWriteSecret() {
final String key = "key";
@ -77,4 +82,25 @@ public class StandardHashiCorpVaultCommunicationServiceIT {
final String resultValue = vcs.readKeyValueSecret("kv", key).orElseThrow(() -> new NullPointerException("Missing secret for kv/key"));
assertEquals(value, resultValue);
}
/**
* Run <code>vault kv get kv/secret</code> to see the secret
*/
@Test
public void testReadWriteSecretMap() {
final String secretKey = "secret";
final String key = "key";
final String value = "value";
final String key2 = "key2";
final String value2 = "value2";
final Map<String, String> keyValues = new HashMap<>();
keyValues.put(key, value);
keyValues.put(key2, value2);
vcs.writeKeyValueSecretMap("kv", secretKey, keyValues);
final Map<String, String> resultMap = vcs.readKeyValueSecretMap("kv", secretKey);
assertEquals(keyValues, resultMap);
}
}

35
nifi-nar-bundles/nifi-hashicorp-vault-bundle/nifi-hashicorp-vault-parameter-value-provider/src/main/java/org/apache/nifi/stateless/parameter/HashiCorpVaultParameterValueProvider.java
View File

@ -24,16 +24,16 @@ import org.apache.nifi.vault.hashicorp.config.HashiCorpVaultConfiguration;
import org.springframework.core.env.PropertySource;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.Objects;
/**
* Reads secrets from HashiCorp Vault to provide parameters. An example of setting one such secret parameter value
* using the Vault CLI would be:
*
* <code>vault kv put "${vault.kv.path}/[ParamContextName]/[ParamName]" value=[ParamValue]</code>
* <code>vault kv put "${vault.kv.path}/[ParamContextName]" [Param1]=[ParamValue1] [Param2]=[ParamValue2]</code>
*
* Here, vault.kv.path is supplied by the file specified by the "Vault Configuration File" property.
*
@ -45,9 +45,8 @@ import java.util.Objects;
* nifi.stateless.parameter.provider.Vault.properties.vault-configuration-file=./conf/bootstrap-hashicorp-vault.conf
* </code>
*/
public class HashiCorpVaultParameterValueProvider extends AbstractParameterValueProvider implements ParameterValueProvider {
public class HashiCorpVaultParameterValueProvider extends AbstractSecretBasedParameterValueProvider implements ParameterValueProvider {
private static final String KEY_VALUE_PATH = "vault.kv.path";
private static final String QUALIFIED_SECRET_FORMAT = "%s/%s";
public static final PropertyDescriptor VAULT_CONFIG_FILE = new PropertyDescriptor.Builder()
.displayName("Vault Configuration File")
.name("vault-configuration-file")
@ -57,24 +56,16 @@ public class HashiCorpVaultParameterValueProvider extends AbstractParameterValue
.addValidator(StandardValidators.FILE_EXISTS_VALIDATOR)
.build();
private List<PropertyDescriptor> descriptors;
private HashiCorpVaultCommunicationService vaultCommunicationService;
private String path;
@Override
protected List<PropertyDescriptor> getSupportedPropertyDescriptors() {
return descriptors;
protected List<PropertyDescriptor> getAdditionalSupportedPropertyDescriptors() {
return Collections.singletonList(VAULT_CONFIG_FILE);
}
@Override
protected void init(final ParameterValueProviderInitializationContext context) {
super.init(context);
final List<PropertyDescriptor> descriptors = new ArrayList<>();
descriptors.add(VAULT_CONFIG_FILE);
this.descriptors = Collections.unmodifiableList(descriptors);
protected void additionalInit(final ParameterValueProviderInitializationContext context) {
final String vaultBootstrapConfFilename = context.getProperty(VAULT_CONFIG_FILE).getValue();
this.configure(vaultBootstrapConfFilename);
}
@ -89,18 +80,10 @@ public class HashiCorpVaultParameterValueProvider extends AbstractParameterValue
}
}
private String getQualifiedSecretFormat(final String contextName, final String parameterName) {
return String.format(QUALIFIED_SECRET_FORMAT, contextName, parameterName);
}
@Override
public String getParameterValue(final String contextName, final String parameterName) {
return vaultCommunicationService.readKeyValueSecret(path, getQualifiedSecretFormat(contextName, parameterName)).orElse(null);
}
@Override
public boolean isParameterDefined(final String contextName, final String parameterName) {
return getParameterValue(contextName, parameterName) != null;
protected String getSecretValue(final String secretName, final String keyName) {
final Map<String, String> keyValues = vaultCommunicationService.readKeyValueSecretMap(path, secretName);
return keyValues.get(keyName);
}
void setVaultCommunicationService(final HashiCorpVaultCommunicationService vaultCommunicationService) {

61
nifi-nar-bundles/nifi-hashicorp-vault-bundle/nifi-hashicorp-vault-parameter-value-provider/src/test/java/org/apache/nifi/stateless/parameter/TestHashiCorpVaultParameterValueProvider.java
View File

@ -27,10 +27,14 @@ import org.mockito.Mock;
import org.mockito.Mockito;
import org.mockito.junit.MockitoJUnitRunner;
import java.util.Collections;
import java.util.HashMap;
import java.util.Map;
import java.util.Optional;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue;
import static org.mockito.Mockito.doAnswer;
import static org.mockito.Mockito.when;
@ -40,6 +44,7 @@ public class TestHashiCorpVaultParameterValueProvider {
private static final String PATH = "kv";
private static final String CONFIG_FILE = "bootstrap-hashicorp-vault.conf";
private static final String DEFAULT_SECRET_NAME = "Test";
@Mock
private HashiCorpVaultCommunicationService vaultCommunicationService;
@ -63,13 +68,19 @@ public class TestHashiCorpVaultParameterValueProvider {
}
@Override
public PropertyValue getProperty(PropertyDescriptor descriptor) {
return new StandardPropertyValue(CONFIG_FILE, null, null);
public PropertyValue getProperty(final PropertyDescriptor descriptor) {
if (descriptor.equals(HashiCorpVaultParameterValueProvider.VAULT_CONFIG_FILE)) {
return new StandardPropertyValue(CONFIG_FILE, null, null);
} else if (descriptor.equals(HashiCorpVaultParameterValueProvider.DEFAULT_SECRET_NAME)) {
return new StandardPropertyValue(DEFAULT_SECRET_NAME, null, null);
}
return null;
}
@Override
public Map<String, String> getAllProperties() {
return null;
return Stream.of(HashiCorpVaultParameterValueProvider.VAULT_CONFIG_FILE, HashiCorpVaultParameterValueProvider.DEFAULT_SECRET_NAME)
.collect(Collectors.toMap(PropertyDescriptor::getName, descriptor -> getProperty(descriptor).getValue()));
}
});
}
@ -77,15 +88,53 @@ public class TestHashiCorpVaultParameterValueProvider {
@Test
public void testGetParameterValue() {
final String value = "value";
when(vaultCommunicationService.readKeyValueSecret(PATH, "context/param")).thenReturn(Optional.of(value));
final Map<String, String> keyValues = new HashMap<>();
keyValues.put("param", value);
when(vaultCommunicationService.readKeyValueSecretMap(PATH, "context")).thenReturn(keyValues);
assertEquals(value, parameterProvider.getParameterValue("context", "param"));
}
@Test
public void testGetParameterValueNoSecret() {
when(vaultCommunicationService.readKeyValueSecretMap(PATH, "context")).thenReturn(Collections.emptyMap());
assertNull(parameterProvider.getParameterValue("context", "param"));
}
@Test
public void testGetParameterValueNoMatchingKey() {
final String value = "value";
final Map<String, String> keyValues = new HashMap<>();
keyValues.put("differentParam", value);
when(vaultCommunicationService.readKeyValueSecretMap(PATH, "context")).thenReturn(keyValues);
assertNull(parameterProvider.getParameterValue("context", "param"));
}
@Test
public void testGetParameterValueDefaultSecret() {
final String value = "value";
final String defaultValue = "defaultValue";
final Map<String, String> keyValues = new HashMap<>();
keyValues.put("differentParam", value);
final Map<String, String> defaultKeyValues = new HashMap<>();
defaultKeyValues.put("param", defaultValue);
// Although 'context' does not contain 'param', the default secret does, so it should fall back to that
when(vaultCommunicationService.readKeyValueSecretMap(PATH, "context")).thenReturn(keyValues);
when(vaultCommunicationService.readKeyValueSecretMap(PATH, DEFAULT_SECRET_NAME)).thenReturn(defaultKeyValues);
assertEquals(defaultValue, parameterProvider.getParameterValue("context", "param"));
}
@Test
public void testIsParameterDefined() {
final String value = "value";
when(vaultCommunicationService.readKeyValueSecret(PATH, "context/param")).thenReturn(Optional.of(value));
final Map<String, String> keyValues = new HashMap<>();
keyValues.put("param", value);
when(vaultCommunicationService.readKeyValueSecretMap(PATH, "context")).thenReturn(keyValues);
assertTrue(parameterProvider.isParameterDefined("context", "param"));
}

5
nifi-stateless/nifi-stateless-assembly/README.md
View File

@ -503,17 +503,18 @@ via the `./conf/bootstrap-hashicorp-vault.conf` file, which comes with NiFi.
An example of creating a single secret in the correct format is:
```
vault kv put "nifi-kv/Context/param" value=my-vault-value
vault kv put "nifi-kv/Context" param=value param2=value2
```
In this example, `nifi-kv` would be supplied by the `vault.kv.path` property in the `bootstrap-hashicorp-vault.conf` file,
`Context` is the name of a Parameter Context, and `param` is the name of the parameter whose value should be retrieved from the Vault server.
`Context` is the name of a mapped Parameter Context, and `param` and `param2` are the names of the parameters whose values should be retrieved from the Vault server.
This Parameter Provider requires the following properties:
| Property Name | Description | Example Value |
|---------------|-------------|---------------|
| nifi.stateless.parameter.provider.\<key>.properties.vault-configuration-file | The filename of a configuration file specifying the Vault settings | ./conf/bootstrap-hashicorp-vault.conf |
| nifi.stateless.parameter.provider.\<key>.properties.default-secret-name | The default K/V secret name to use. This secret represents a default Parameter Context if there is not a matching key within the mapped Parameter Context secret. | `Default` |
An example of configuring this provider in the dataflow configuration file is: