mirror of https://github.com/apache/nifi.git
NIFI-9263: Using secret-per-context model for Stateless Vault PVP
Rebasing after PVP refactor Signed-off-by: Nathan Gough <thenatog@gmail.com> This closes #5427.
This commit is contained in:
parent
d2f8f97b10
commit
207da405ee
|
@ -16,6 +16,7 @@
|
|||
*/
|
||||
package org.apache.nifi.vault.hashicorp;
|
||||
|
||||
import java.util.Map;
|
||||
import java.util.Optional;
|
||||
|
||||
/**
|
||||
|
@ -45,22 +46,41 @@ public interface HashiCorpVaultCommunicationService {
|
|||
byte[] decrypt(String transitPath, String cipherText);
|
||||
|
||||
/**
|
||||
* Writes a secret using Vault's unversioned Key/Value Secrets Engine.
|
||||
* Writes a single secret value using Vault's unversioned Key/Value Secrets Engine.
|
||||
*
|
||||
* @see <a href="https://www.vaultproject.io/api-docs/secret/kv/kv-v1">https://www.vaultproject.io/api-docs/secret/kv/kv-v1</a>
|
||||
* @param keyValuePath The Vault path to use for the configured Key/Value v1 Secrets Engine
|
||||
* @param key The secret key
|
||||
* @param secretKey The secret key
|
||||
* @param value The secret value
|
||||
*/
|
||||
void writeKeyValueSecret(String keyValuePath, String key, String value);
|
||||
void writeKeyValueSecret(String keyValuePath, String secretKey, String value);
|
||||
|
||||
/**
|
||||
* Reads a secret from Vault's unversioned Key/Value Secrets Engine.
|
||||
* Reads a single secret value from Vault's unversioned Key/Value Secrets Engine.
|
||||
*
|
||||
* @see <a href="https://www.vaultproject.io/api-docs/secret/kv/kv-v1">https://www.vaultproject.io/api-docs/secret/kv/kv-v1</a>
|
||||
* @param keyValuePath The Vault path to use for the configured Key/Value v1 Secrets Engine
|
||||
* @param key The secret key
|
||||
* @param secretKey The secret key
|
||||
* @return The secret value, or empty if not found
|
||||
*/
|
||||
Optional<String> readKeyValueSecret(String keyValuePath, String key);
|
||||
Optional<String> readKeyValueSecret(String keyValuePath, String secretKey);
|
||||
|
||||
/**
|
||||
* Writes a secret with multiple key/value pairs using Vault's unversioned Key/Value Secrets Engine.
|
||||
*
|
||||
* @see <a href="https://www.vaultproject.io/api-docs/secret/kv/kv-v1">https://www.vaultproject.io/api-docs/secret/kv/kv-v1</a>
|
||||
* @param keyValuePath The Vault path to use for the configured Key/Value v1 Secrets Engine
|
||||
* @param keyValues A map from key to value for keys/values that should be stored in the secret
|
||||
*/
|
||||
void writeKeyValueSecretMap(String keyValuePath, String secretKey, Map<String, String> keyValues);
|
||||
|
||||
/**
|
||||
* Reads a secret with multiple key/value pairs from Vault's unversioned Key/Value Secrets Engine.
|
||||
*
|
||||
* @see <a href="https://www.vaultproject.io/api-docs/secret/kv/kv-v1">https://www.vaultproject.io/api-docs/secret/kv/kv-v1</a>
|
||||
* @param keyValuePath The Vault path to use for the configured Key/Value v1 Secrets Engine
|
||||
* @param secretKey The secret key
|
||||
* @return A map from key to value from the secret key/values, or an empty map if not found
|
||||
*/
|
||||
Map<String, String> readKeyValueSecretMap(String keyValuePath, String secretKey);
|
||||
}
|
||||
|
|
|
@ -31,8 +31,10 @@ import org.springframework.vault.support.Ciphertext;
|
|||
import org.springframework.vault.support.Plaintext;
|
||||
import org.springframework.vault.support.VaultResponseSupport;
|
||||
|
||||
import java.util.Collections;
|
||||
import java.util.HashMap;
|
||||
import java.util.Map;
|
||||
import java.util.Objects;
|
||||
import java.util.Optional;
|
||||
|
||||
import static org.springframework.vault.core.VaultKeyValueOperationsSupport.KeyValueBackend.KV_1;
|
||||
|
@ -41,7 +43,6 @@ import static org.springframework.vault.core.VaultKeyValueOperationsSupport.KeyV
|
|||
* Implements the VaultCommunicationService using Spring Vault
|
||||
*/
|
||||
public class StandardHashiCorpVaultCommunicationService implements HashiCorpVaultCommunicationService {
|
||||
private final HashiCorpVaultConfiguration vaultConfiguration;
|
||||
private final VaultTemplate vaultTemplate;
|
||||
private final VaultTransitOperations transitOperations;
|
||||
private final Map<String, VaultKeyValueOperations> keyValueOperationsMap;
|
||||
|
@ -52,7 +53,7 @@ public class StandardHashiCorpVaultCommunicationService implements HashiCorpVaul
|
|||
* @throws HashiCorpVaultConfigurationException If the configuration was invalid
|
||||
*/
|
||||
public StandardHashiCorpVaultCommunicationService(final PropertySource<?>... propertySources) throws HashiCorpVaultConfigurationException {
|
||||
vaultConfiguration = new HashiCorpVaultConfiguration(propertySources);
|
||||
final HashiCorpVaultConfiguration vaultConfiguration = new HashiCorpVaultConfiguration(propertySources);
|
||||
|
||||
vaultTemplate = new VaultTemplate(vaultConfiguration.vaultEndpoint(),
|
||||
ClientHttpRequestFactoryFactory.create(vaultConfiguration.clientOptions(), vaultConfiguration.sslConfiguration()),
|
||||
|
@ -82,32 +83,58 @@ public class StandardHashiCorpVaultCommunicationService implements HashiCorpVaul
|
|||
}
|
||||
|
||||
/**
|
||||
* Writes the value to the "value" key of the secret with the path [keyValuePath]/[key].
|
||||
* Writes the value to the "value" secretKey of the secret with the path [keyValuePath]/[secretKey].
|
||||
* @param keyValuePath The Vault path to use for the configured Key/Value v1 Secrets Engine
|
||||
* @param key The secret key
|
||||
* @param secretKey The secret secretKey
|
||||
* @param value The secret value
|
||||
*/
|
||||
@Override
|
||||
public void writeKeyValueSecret(final String keyValuePath, final String key, final String value) {
|
||||
public void writeKeyValueSecret(final String keyValuePath, final String secretKey, final String value) {
|
||||
Objects.requireNonNull(keyValuePath, "Vault K/V path must be specified");
|
||||
Objects.requireNonNull(secretKey, "Secret secretKey must be specified");
|
||||
Objects.requireNonNull(value, "Secret value must be specified");
|
||||
final VaultKeyValueOperations keyValueOperations = keyValueOperationsMap
|
||||
.computeIfAbsent(keyValuePath, path -> vaultTemplate.opsForKeyValue(path, KV_1));
|
||||
keyValueOperations.put(key, new SecretData(value));
|
||||
keyValueOperations.put(secretKey, new SecretData(value));
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the value of the "value" key from the secret at the path [keyValuePath]/[key].
|
||||
* Returns the value of the "value" secretKey from the secret at the path [keyValuePath]/[secretKey].
|
||||
* @param keyValuePath The Vault path to use for the configured Key/Value v1 Secrets Engine
|
||||
* @param key The secret key
|
||||
* @param secretKey The secret secretKey
|
||||
* @return The value of the secret
|
||||
*/
|
||||
@Override
|
||||
public Optional<String> readKeyValueSecret(final String keyValuePath, final String key) {
|
||||
public Optional<String> readKeyValueSecret(final String keyValuePath, final String secretKey) {
|
||||
Objects.requireNonNull(keyValuePath, "Vault K/V path must be specified");
|
||||
Objects.requireNonNull(secretKey, "Secret secretKey must be specified");
|
||||
final VaultKeyValueOperations keyValueOperations = keyValueOperationsMap
|
||||
.computeIfAbsent(keyValuePath, path -> vaultTemplate.opsForKeyValue(path, KV_1));
|
||||
final VaultResponseSupport<SecretData> response = keyValueOperations.get(key, SecretData.class);
|
||||
final VaultResponseSupport<SecretData> response = keyValueOperations.get(secretKey, SecretData.class);
|
||||
return response == null ? Optional.empty() : Optional.ofNullable(response.getRequiredData().getValue());
|
||||
}
|
||||
|
||||
@Override
|
||||
public void writeKeyValueSecretMap(final String keyValuePath, final String secretKey, final Map<String, String> keyValues) {
|
||||
Objects.requireNonNull(keyValuePath, "Vault K/V path must be specified");
|
||||
Objects.requireNonNull(secretKey, "Secret secretKey must be specified");
|
||||
Objects.requireNonNull(keyValues, "Key/values map must be specified");
|
||||
if (keyValues.isEmpty()) {
|
||||
return;
|
||||
}
|
||||
final VaultKeyValueOperations keyValueOperations = keyValueOperationsMap
|
||||
.computeIfAbsent(keyValuePath, path -> vaultTemplate.opsForKeyValue(path, KV_1));
|
||||
keyValueOperations.put(secretKey, keyValues);
|
||||
}
|
||||
|
||||
@Override
|
||||
public Map<String, String> readKeyValueSecretMap(final String keyValuePath, final String key) {
|
||||
final VaultKeyValueOperations keyValueOperations = keyValueOperationsMap
|
||||
.computeIfAbsent(keyValuePath, path -> vaultTemplate.opsForKeyValue(path, KV_1));
|
||||
final VaultResponseSupport<Map> response = keyValueOperations.get(key, Map.class);
|
||||
return response == null ? Collections.emptyMap() : (Map<String, String>) response.getRequiredData();
|
||||
}
|
||||
|
||||
private static class SecretData {
|
||||
private final String value;
|
||||
|
||||
|
|
|
@ -21,6 +21,8 @@ import org.junit.Before;
|
|||
import org.junit.Test;
|
||||
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.util.HashMap;
|
||||
import java.util.Map;
|
||||
|
||||
import static org.junit.Assert.assertEquals;
|
||||
|
||||
|
@ -67,6 +69,9 @@ public class StandardHashiCorpVaultCommunicationServiceIT {
|
|||
assertEquals(plaintext, new String(decrypted, StandardCharsets.UTF_8));
|
||||
}
|
||||
|
||||
/**
|
||||
* Run <code>vault kv get kv/key</code> to see the secret
|
||||
*/
|
||||
@Test
|
||||
public void testReadWriteSecret() {
|
||||
final String key = "key";
|
||||
|
@ -77,4 +82,25 @@ public class StandardHashiCorpVaultCommunicationServiceIT {
|
|||
final String resultValue = vcs.readKeyValueSecret("kv", key).orElseThrow(() -> new NullPointerException("Missing secret for kv/key"));
|
||||
assertEquals(value, resultValue);
|
||||
}
|
||||
|
||||
/**
|
||||
* Run <code>vault kv get kv/secret</code> to see the secret
|
||||
*/
|
||||
@Test
|
||||
public void testReadWriteSecretMap() {
|
||||
final String secretKey = "secret";
|
||||
final String key = "key";
|
||||
final String value = "value";
|
||||
final String key2 = "key2";
|
||||
final String value2 = "value2";
|
||||
|
||||
final Map<String, String> keyValues = new HashMap<>();
|
||||
keyValues.put(key, value);
|
||||
keyValues.put(key2, value2);
|
||||
|
||||
vcs.writeKeyValueSecretMap("kv", secretKey, keyValues);
|
||||
|
||||
final Map<String, String> resultMap = vcs.readKeyValueSecretMap("kv", secretKey);
|
||||
assertEquals(keyValues, resultMap);
|
||||
}
|
||||
}
|
||||
|
|
|
@ -24,16 +24,16 @@ import org.apache.nifi.vault.hashicorp.config.HashiCorpVaultConfiguration;
|
|||
import org.springframework.core.env.PropertySource;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collections;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Objects;
|
||||
|
||||
/**
|
||||
* Reads secrets from HashiCorp Vault to provide parameters. An example of setting one such secret parameter value
|
||||
* using the Vault CLI would be:
|
||||
*
|
||||
* <code>vault kv put "${vault.kv.path}/[ParamContextName]/[ParamName]" value=[ParamValue]</code>
|
||||
* <code>vault kv put "${vault.kv.path}/[ParamContextName]" [Param1]=[ParamValue1] [Param2]=[ParamValue2]</code>
|
||||
*
|
||||
* Here, vault.kv.path is supplied by the file specified by the "Vault Configuration File" property.
|
||||
*
|
||||
|
@ -45,9 +45,8 @@ import java.util.Objects;
|
|||
* nifi.stateless.parameter.provider.Vault.properties.vault-configuration-file=./conf/bootstrap-hashicorp-vault.conf
|
||||
* </code>
|
||||
*/
|
||||
public class HashiCorpVaultParameterValueProvider extends AbstractParameterValueProvider implements ParameterValueProvider {
|
||||
public class HashiCorpVaultParameterValueProvider extends AbstractSecretBasedParameterValueProvider implements ParameterValueProvider {
|
||||
private static final String KEY_VALUE_PATH = "vault.kv.path";
|
||||
private static final String QUALIFIED_SECRET_FORMAT = "%s/%s";
|
||||
public static final PropertyDescriptor VAULT_CONFIG_FILE = new PropertyDescriptor.Builder()
|
||||
.displayName("Vault Configuration File")
|
||||
.name("vault-configuration-file")
|
||||
|
@ -57,24 +56,16 @@ public class HashiCorpVaultParameterValueProvider extends AbstractParameterValue
|
|||
.addValidator(StandardValidators.FILE_EXISTS_VALIDATOR)
|
||||
.build();
|
||||
|
||||
private List<PropertyDescriptor> descriptors;
|
||||
|
||||
private HashiCorpVaultCommunicationService vaultCommunicationService;
|
||||
private String path;
|
||||
|
||||
@Override
|
||||
protected List<PropertyDescriptor> getSupportedPropertyDescriptors() {
|
||||
return descriptors;
|
||||
protected List<PropertyDescriptor> getAdditionalSupportedPropertyDescriptors() {
|
||||
return Collections.singletonList(VAULT_CONFIG_FILE);
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void init(final ParameterValueProviderInitializationContext context) {
|
||||
super.init(context);
|
||||
|
||||
final List<PropertyDescriptor> descriptors = new ArrayList<>();
|
||||
descriptors.add(VAULT_CONFIG_FILE);
|
||||
this.descriptors = Collections.unmodifiableList(descriptors);
|
||||
|
||||
protected void additionalInit(final ParameterValueProviderInitializationContext context) {
|
||||
final String vaultBootstrapConfFilename = context.getProperty(VAULT_CONFIG_FILE).getValue();
|
||||
this.configure(vaultBootstrapConfFilename);
|
||||
}
|
||||
|
@ -89,18 +80,10 @@ public class HashiCorpVaultParameterValueProvider extends AbstractParameterValue
|
|||
}
|
||||
}
|
||||
|
||||
private String getQualifiedSecretFormat(final String contextName, final String parameterName) {
|
||||
return String.format(QUALIFIED_SECRET_FORMAT, contextName, parameterName);
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getParameterValue(final String contextName, final String parameterName) {
|
||||
return vaultCommunicationService.readKeyValueSecret(path, getQualifiedSecretFormat(contextName, parameterName)).orElse(null);
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean isParameterDefined(final String contextName, final String parameterName) {
|
||||
return getParameterValue(contextName, parameterName) != null;
|
||||
protected String getSecretValue(final String secretName, final String keyName) {
|
||||
final Map<String, String> keyValues = vaultCommunicationService.readKeyValueSecretMap(path, secretName);
|
||||
return keyValues.get(keyName);
|
||||
}
|
||||
|
||||
void setVaultCommunicationService(final HashiCorpVaultCommunicationService vaultCommunicationService) {
|
||||
|
|
|
@ -27,10 +27,14 @@ import org.mockito.Mock;
|
|||
import org.mockito.Mockito;
|
||||
import org.mockito.junit.MockitoJUnitRunner;
|
||||
|
||||
import java.util.Collections;
|
||||
import java.util.HashMap;
|
||||
import java.util.Map;
|
||||
import java.util.Optional;
|
||||
import java.util.stream.Collectors;
|
||||
import java.util.stream.Stream;
|
||||
|
||||
import static org.junit.Assert.assertEquals;
|
||||
import static org.junit.Assert.assertNull;
|
||||
import static org.junit.Assert.assertTrue;
|
||||
import static org.mockito.Mockito.doAnswer;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
@ -40,6 +44,7 @@ public class TestHashiCorpVaultParameterValueProvider {
|
|||
|
||||
private static final String PATH = "kv";
|
||||
private static final String CONFIG_FILE = "bootstrap-hashicorp-vault.conf";
|
||||
private static final String DEFAULT_SECRET_NAME = "Test";
|
||||
|
||||
@Mock
|
||||
private HashiCorpVaultCommunicationService vaultCommunicationService;
|
||||
|
@ -63,13 +68,19 @@ public class TestHashiCorpVaultParameterValueProvider {
|
|||
}
|
||||
|
||||
@Override
|
||||
public PropertyValue getProperty(PropertyDescriptor descriptor) {
|
||||
public PropertyValue getProperty(final PropertyDescriptor descriptor) {
|
||||
if (descriptor.equals(HashiCorpVaultParameterValueProvider.VAULT_CONFIG_FILE)) {
|
||||
return new StandardPropertyValue(CONFIG_FILE, null, null);
|
||||
} else if (descriptor.equals(HashiCorpVaultParameterValueProvider.DEFAULT_SECRET_NAME)) {
|
||||
return new StandardPropertyValue(DEFAULT_SECRET_NAME, null, null);
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Map<String, String> getAllProperties() {
|
||||
return null;
|
||||
return Stream.of(HashiCorpVaultParameterValueProvider.VAULT_CONFIG_FILE, HashiCorpVaultParameterValueProvider.DEFAULT_SECRET_NAME)
|
||||
.collect(Collectors.toMap(PropertyDescriptor::getName, descriptor -> getProperty(descriptor).getValue()));
|
||||
}
|
||||
});
|
||||
}
|
||||
|
@ -77,15 +88,53 @@ public class TestHashiCorpVaultParameterValueProvider {
|
|||
@Test
|
||||
public void testGetParameterValue() {
|
||||
final String value = "value";
|
||||
when(vaultCommunicationService.readKeyValueSecret(PATH, "context/param")).thenReturn(Optional.of(value));
|
||||
final Map<String, String> keyValues = new HashMap<>();
|
||||
keyValues.put("param", value);
|
||||
when(vaultCommunicationService.readKeyValueSecretMap(PATH, "context")).thenReturn(keyValues);
|
||||
|
||||
assertEquals(value, parameterProvider.getParameterValue("context", "param"));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testGetParameterValueNoSecret() {
|
||||
when(vaultCommunicationService.readKeyValueSecretMap(PATH, "context")).thenReturn(Collections.emptyMap());
|
||||
|
||||
assertNull(parameterProvider.getParameterValue("context", "param"));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testGetParameterValueNoMatchingKey() {
|
||||
final String value = "value";
|
||||
final Map<String, String> keyValues = new HashMap<>();
|
||||
keyValues.put("differentParam", value);
|
||||
when(vaultCommunicationService.readKeyValueSecretMap(PATH, "context")).thenReturn(keyValues);
|
||||
|
||||
assertNull(parameterProvider.getParameterValue("context", "param"));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testGetParameterValueDefaultSecret() {
|
||||
final String value = "value";
|
||||
final String defaultValue = "defaultValue";
|
||||
final Map<String, String> keyValues = new HashMap<>();
|
||||
keyValues.put("differentParam", value);
|
||||
|
||||
final Map<String, String> defaultKeyValues = new HashMap<>();
|
||||
defaultKeyValues.put("param", defaultValue);
|
||||
|
||||
// Although 'context' does not contain 'param', the default secret does, so it should fall back to that
|
||||
when(vaultCommunicationService.readKeyValueSecretMap(PATH, "context")).thenReturn(keyValues);
|
||||
when(vaultCommunicationService.readKeyValueSecretMap(PATH, DEFAULT_SECRET_NAME)).thenReturn(defaultKeyValues);
|
||||
|
||||
assertEquals(defaultValue, parameterProvider.getParameterValue("context", "param"));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testIsParameterDefined() {
|
||||
final String value = "value";
|
||||
when(vaultCommunicationService.readKeyValueSecret(PATH, "context/param")).thenReturn(Optional.of(value));
|
||||
final Map<String, String> keyValues = new HashMap<>();
|
||||
keyValues.put("param", value);
|
||||
when(vaultCommunicationService.readKeyValueSecretMap(PATH, "context")).thenReturn(keyValues);
|
||||
|
||||
assertTrue(parameterProvider.isParameterDefined("context", "param"));
|
||||
}
|
||||
|
|
|
@ -503,17 +503,18 @@ via the `./conf/bootstrap-hashicorp-vault.conf` file, which comes with NiFi.
|
|||
An example of creating a single secret in the correct format is:
|
||||
|
||||
```
|
||||
vault kv put "nifi-kv/Context/param" value=my-vault-value
|
||||
vault kv put "nifi-kv/Context" param=value param2=value2
|
||||
```
|
||||
|
||||
In this example, `nifi-kv` would be supplied by the `vault.kv.path` property in the `bootstrap-hashicorp-vault.conf` file,
|
||||
`Context` is the name of a Parameter Context, and `param` is the name of the parameter whose value should be retrieved from the Vault server.
|
||||
`Context` is the name of a mapped Parameter Context, and `param` and `param2` are the names of the parameters whose values should be retrieved from the Vault server.
|
||||
|
||||
This Parameter Provider requires the following properties:
|
||||
|
||||
| Property Name | Description | Example Value |
|
||||
|---------------|-------------|---------------|
|
||||
| nifi.stateless.parameter.provider.\<key>.properties.vault-configuration-file | The filename of a configuration file specifying the Vault settings | ./conf/bootstrap-hashicorp-vault.conf |
|
||||
| nifi.stateless.parameter.provider.\<key>.properties.default-secret-name | The default K/V secret name to use. This secret represents a default Parameter Context if there is not a matching key within the mapped Parameter Context secret. | `Default` |
|
||||
|
||||
An example of configuring this provider in the dataflow configuration file is:
|
||||
|
||||
|
|
Loading…
Reference in New Issue