NIFI-9263: Using secret-per-context model for Stateless Vault PVP

Rebasing after PVP refactor

Signed-off-by: Nathan Gough <thenatog@gmail.com>

This closes #5427.

nifi-commons/nifi-vault-utils/src/main/java/org/apache/nifi/vault/hashicorp/HashiCorpVaultCommunicationService.java

@@ -16,6 +16,7 @@
  */
 package org.apache.nifi.vault.hashicorp;
 
+import java.util.Map;
 import java.util.Optional;
 
 /**
@@ -45,22 +46,41 @@ public interface HashiCorpVaultCommunicationService {
     byte[] decrypt(String transitPath, String cipherText);
 
     /**
-     * Writes a secret using Vault's unversioned Key/Value Secrets Engine.
+     * Writes a single secret value using Vault's unversioned Key/Value Secrets Engine.
      *
      * @see <a href="https://www.vaultproject.io/api-docs/secret/kv/kv-v1">https://www.vaultproject.io/api-docs/secret/kv/kv-v1</a>
      * @param keyValuePath The Vault path to use for the configured Key/Value v1 Secrets Engine
-     * @param key The secret key
+     * @param secretKey The secret key
      * @param value The secret value
      */
-    void writeKeyValueSecret(String keyValuePath, String key, String value);
+    void writeKeyValueSecret(String keyValuePath, String secretKey, String value);
 
     /**
-     * Reads a secret from Vault's unversioned Key/Value Secrets Engine.
+     * Reads a single secret value from Vault's unversioned Key/Value Secrets Engine.
      *
      * @see <a href="https://www.vaultproject.io/api-docs/secret/kv/kv-v1">https://www.vaultproject.io/api-docs/secret/kv/kv-v1</a>
      * @param keyValuePath The Vault path to use for the configured Key/Value v1 Secrets Engine
-     * @param key The secret key
+     * @param secretKey The secret key
      * @return The secret value, or empty if not found
      */
-    Optional<String> readKeyValueSecret(String keyValuePath, String key);
+    Optional<String> readKeyValueSecret(String keyValuePath, String secretKey);
+
+    /**
+     * Writes a secret with multiple key/value pairs using Vault's unversioned Key/Value Secrets Engine.
+     *
+     * @see <a href="https://www.vaultproject.io/api-docs/secret/kv/kv-v1">https://www.vaultproject.io/api-docs/secret/kv/kv-v1</a>
+     * @param keyValuePath The Vault path to use for the configured Key/Value v1 Secrets Engine
+     * @param keyValues A map from key to value for keys/values that should be stored in the secret
+     */
+    void writeKeyValueSecretMap(String keyValuePath, String secretKey, Map<String, String> keyValues);
+
+    /**
+     * Reads a secret with multiple key/value pairs from Vault's unversioned Key/Value Secrets Engine.
+     *
+     * @see <a href="https://www.vaultproject.io/api-docs/secret/kv/kv-v1">https://www.vaultproject.io/api-docs/secret/kv/kv-v1</a>
+     * @param keyValuePath The Vault path to use for the configured Key/Value v1 Secrets Engine
+     * @param secretKey The secret key
+     * @return A map from key to value from the secret key/values, or an empty map if not found
+     */
+    Map<String, String> readKeyValueSecretMap(String keyValuePath, String secretKey);
 }

nifi-commons/nifi-vault-utils/src/main/java/org/apache/nifi/vault/hashicorp/StandardHashiCorpVaultCommunicationService.java

@@ -31,8 +31,10 @@ import org.springframework.vault.support.Ciphertext;
 import org.springframework.vault.support.Plaintext;
 import org.springframework.vault.support.VaultResponseSupport;
 
+import java.util.Collections;
 import java.util.HashMap;
 import java.util.Map;
+import java.util.Objects;
 import java.util.Optional;
 
 import static org.springframework.vault.core.VaultKeyValueOperationsSupport.KeyValueBackend.KV_1;
@@ -41,7 +43,6 @@ import static org.springframework.vault.core.VaultKeyValueOperationsSupport.KeyV
  * Implements the VaultCommunicationService using Spring Vault
  */
 public class StandardHashiCorpVaultCommunicationService implements HashiCorpVaultCommunicationService {
-    private final HashiCorpVaultConfiguration vaultConfiguration;
     private final VaultTemplate vaultTemplate;
     private final VaultTransitOperations transitOperations;
     private final Map<String, VaultKeyValueOperations> keyValueOperationsMap;
@@ -52,7 +53,7 @@ public class StandardHashiCorpVaultCommunicationService implements HashiCorpVaul
      * @throws HashiCorpVaultConfigurationException If the configuration was invalid
      */
     public StandardHashiCorpVaultCommunicationService(final PropertySource<?>... propertySources) throws HashiCorpVaultConfigurationException {
-        vaultConfiguration = new HashiCorpVaultConfiguration(propertySources);
+        final HashiCorpVaultConfiguration vaultConfiguration = new HashiCorpVaultConfiguration(propertySources);
 
         vaultTemplate = new VaultTemplate(vaultConfiguration.vaultEndpoint(),
                 ClientHttpRequestFactoryFactory.create(vaultConfiguration.clientOptions(), vaultConfiguration.sslConfiguration()),
@@ -82,32 +83,58 @@ public class StandardHashiCorpVaultCommunicationService implements HashiCorpVaul
     }
 
     /**
-     * Writes the value to the "value" key of the secret with the path [keyValuePath]/[key].
+     * Writes the value to the "value" secretKey of the secret with the path [keyValuePath]/[secretKey].
      * @param keyValuePath The Vault path to use for the configured Key/Value v1 Secrets Engine
-     * @param key The secret key
+     * @param secretKey The secret secretKey
      * @param value The secret value
      */
     @Override
-    public void writeKeyValueSecret(final String keyValuePath, final String key, final String value) {
+    public void writeKeyValueSecret(final String keyValuePath, final String secretKey, final String value) {
+        Objects.requireNonNull(keyValuePath, "Vault K/V path must be specified");
+        Objects.requireNonNull(secretKey, "Secret secretKey must be specified");
+        Objects.requireNonNull(value, "Secret value must be specified");
         final VaultKeyValueOperations keyValueOperations = keyValueOperationsMap
                 .computeIfAbsent(keyValuePath, path -> vaultTemplate.opsForKeyValue(path, KV_1));
-        keyValueOperations.put(key, new SecretData(value));
+        keyValueOperations.put(secretKey, new SecretData(value));
     }
 
     /**
-     * Returns the value of the "value" key from the secret at the path [keyValuePath]/[key].
+     * Returns the value of the "value" secretKey from the secret at the path [keyValuePath]/[secretKey].
      * @param keyValuePath The Vault path to use for the configured Key/Value v1 Secrets Engine
-     * @param key The secret key
+     * @param secretKey The secret secretKey
      * @return The value of the secret
      */
     @Override
-    public Optional<String> readKeyValueSecret(final String keyValuePath, final String key) {
+    public Optional<String> readKeyValueSecret(final String keyValuePath, final String secretKey) {
+        Objects.requireNonNull(keyValuePath, "Vault K/V path must be specified");
+        Objects.requireNonNull(secretKey, "Secret secretKey must be specified");
         final VaultKeyValueOperations keyValueOperations = keyValueOperationsMap
                 .computeIfAbsent(keyValuePath, path -> vaultTemplate.opsForKeyValue(path, KV_1));
-        final VaultResponseSupport<SecretData> response = keyValueOperations.get(key, SecretData.class);
+        final VaultResponseSupport<SecretData> response = keyValueOperations.get(secretKey, SecretData.class);
         return response == null ? Optional.empty() : Optional.ofNullable(response.getRequiredData().getValue());
     }
 
+    @Override
+    public void writeKeyValueSecretMap(final String keyValuePath, final String secretKey, final Map<String, String> keyValues) {
+        Objects.requireNonNull(keyValuePath, "Vault K/V path must be specified");
+        Objects.requireNonNull(secretKey, "Secret secretKey must be specified");
+        Objects.requireNonNull(keyValues, "Key/values map must be specified");
+        if (keyValues.isEmpty()) {
+            return;
+        }
+        final VaultKeyValueOperations keyValueOperations = keyValueOperationsMap
+                .computeIfAbsent(keyValuePath, path -> vaultTemplate.opsForKeyValue(path, KV_1));
+        keyValueOperations.put(secretKey, keyValues);
+    }
+
+    @Override
+    public Map<String, String> readKeyValueSecretMap(final String keyValuePath, final String key) {
+        final VaultKeyValueOperations keyValueOperations = keyValueOperationsMap
+                .computeIfAbsent(keyValuePath, path -> vaultTemplate.opsForKeyValue(path, KV_1));
+        final VaultResponseSupport<Map> response = keyValueOperations.get(key, Map.class);
+        return response == null ? Collections.emptyMap() : (Map<String, String>) response.getRequiredData();
+    }
+
     private static class SecretData {
         private final String value;
 

nifi-commons/nifi-vault-utils/src/test/java/org/apache/nifi/vault/hashicorp/StandardHashiCorpVaultCommunicationServiceIT.java

@@ -21,6 +21,8 @@ import org.junit.Before;
 import org.junit.Test;
 
 import java.nio.charset.StandardCharsets;
+import java.util.HashMap;
+import java.util.Map;
 
 import static org.junit.Assert.assertEquals;
 
@@ -67,6 +69,9 @@ public class StandardHashiCorpVaultCommunicationServiceIT {
         assertEquals(plaintext, new String(decrypted, StandardCharsets.UTF_8));
     }
 
+    /**
+     * Run <code>vault kv get kv/key</code> to see the secret
+     */
     @Test
     public void testReadWriteSecret() {
         final String key = "key";
@@ -77,4 +82,25 @@ public class StandardHashiCorpVaultCommunicationServiceIT {
         final String resultValue = vcs.readKeyValueSecret("kv", key).orElseThrow(() -> new NullPointerException("Missing secret for kv/key"));
         assertEquals(value, resultValue);
     }
+
+    /**
+     * Run <code>vault kv get kv/secret</code> to see the secret
+     */
+    @Test
+    public void testReadWriteSecretMap() {
+        final String secretKey = "secret";
+        final String key = "key";
+        final String value = "value";
+        final String key2 = "key2";
+        final String value2 = "value2";
+
+        final Map<String, String> keyValues = new HashMap<>();
+        keyValues.put(key, value);
+        keyValues.put(key2, value2);
+
+        vcs.writeKeyValueSecretMap("kv", secretKey, keyValues);
+
+        final Map<String, String> resultMap = vcs.readKeyValueSecretMap("kv", secretKey);
+        assertEquals(keyValues, resultMap);
+    }
 }

nifi-nar-bundles/nifi-hashicorp-vault-bundle/nifi-hashicorp-vault-parameter-value-provider/src/main/java/org/apache/nifi/stateless/parameter/HashiCorpVaultParameterValueProvider.java

@@ -24,16 +24,16 @@ import org.apache.nifi.vault.hashicorp.config.HashiCorpVaultConfiguration;
 import org.springframework.core.env.PropertySource;
 
 import java.io.IOException;
-import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
+import java.util.Map;
 import java.util.Objects;
 
 /**
  * Reads secrets from HashiCorp Vault to provide parameters.  An example of setting one such secret parameter value
  * using the Vault CLI would be:
  *
- * <code>vault kv put "${vault.kv.path}/[ParamContextName]/[ParamName]" value=[ParamValue]</code>
+ * <code>vault kv put "${vault.kv.path}/[ParamContextName]" [Param1]=[ParamValue1] [Param2]=[ParamValue2]</code>
  *
  * Here, vault.kv.path is supplied by the file specified by the "Vault Configuration File" property.
  *
@@ -45,9 +45,8 @@ import java.util.Objects;
  *      nifi.stateless.parameter.provider.Vault.properties.vault-configuration-file=./conf/bootstrap-hashicorp-vault.conf
  * </code>
  */
-public class HashiCorpVaultParameterValueProvider extends AbstractParameterValueProvider implements ParameterValueProvider {
+public class HashiCorpVaultParameterValueProvider extends AbstractSecretBasedParameterValueProvider implements ParameterValueProvider {
     private static final String KEY_VALUE_PATH = "vault.kv.path";
-    private static final String QUALIFIED_SECRET_FORMAT = "%s/%s";
     public static final PropertyDescriptor VAULT_CONFIG_FILE = new PropertyDescriptor.Builder()
             .displayName("Vault Configuration File")
             .name("vault-configuration-file")
@@ -57,24 +56,16 @@ public class HashiCorpVaultParameterValueProvider extends AbstractParameterValue
             .addValidator(StandardValidators.FILE_EXISTS_VALIDATOR)
             .build();
 
-    private List<PropertyDescriptor> descriptors;
-
     private HashiCorpVaultCommunicationService vaultCommunicationService;
     private String path;
 
     @Override
-    protected List<PropertyDescriptor> getSupportedPropertyDescriptors() {
+    protected List<PropertyDescriptor> getAdditionalSupportedPropertyDescriptors() {
-        return descriptors;
+        return Collections.singletonList(VAULT_CONFIG_FILE);
     }
 
     @Override
-    protected void init(final ParameterValueProviderInitializationContext context) {
+    protected void additionalInit(final ParameterValueProviderInitializationContext context) {
-        super.init(context);
-
-        final List<PropertyDescriptor> descriptors = new ArrayList<>();
-        descriptors.add(VAULT_CONFIG_FILE);
-        this.descriptors = Collections.unmodifiableList(descriptors);
-
         final String vaultBootstrapConfFilename = context.getProperty(VAULT_CONFIG_FILE).getValue();
         this.configure(vaultBootstrapConfFilename);
     }
@@ -89,18 +80,10 @@ public class HashiCorpVaultParameterValueProvider extends AbstractParameterValue
         }
     }
 
-    private String getQualifiedSecretFormat(final String contextName, final String parameterName) {
-        return String.format(QUALIFIED_SECRET_FORMAT, contextName, parameterName);
-    }
-
     @Override
-    public String getParameterValue(final String contextName, final String parameterName) {
+    protected String getSecretValue(final String secretName, final String keyName) {
-        return vaultCommunicationService.readKeyValueSecret(path, getQualifiedSecretFormat(contextName, parameterName)).orElse(null);
+        final Map<String, String> keyValues = vaultCommunicationService.readKeyValueSecretMap(path, secretName);
-    }
+        return keyValues.get(keyName);
-
-    @Override
-    public boolean isParameterDefined(final String contextName, final String parameterName) {
-        return getParameterValue(contextName, parameterName) != null;
     }
 
     void setVaultCommunicationService(final HashiCorpVaultCommunicationService vaultCommunicationService) {

nifi-nar-bundles/nifi-hashicorp-vault-bundle/nifi-hashicorp-vault-parameter-value-provider/src/test/java/org/apache/nifi/stateless/parameter/TestHashiCorpVaultParameterValueProvider.java

@@ -27,10 +27,14 @@ import org.mockito.Mock;
 import org.mockito.Mockito;
 import org.mockito.junit.MockitoJUnitRunner;
 
+import java.util.Collections;
+import java.util.HashMap;
 import java.util.Map;
-import java.util.Optional;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
 
 import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNull;
 import static org.junit.Assert.assertTrue;
 import static org.mockito.Mockito.doAnswer;
 import static org.mockito.Mockito.when;
@@ -40,6 +44,7 @@ public class TestHashiCorpVaultParameterValueProvider {
 
     private static final String PATH = "kv";
     private static final String CONFIG_FILE = "bootstrap-hashicorp-vault.conf";
+    private static final String DEFAULT_SECRET_NAME = "Test";
 
     @Mock
     private HashiCorpVaultCommunicationService vaultCommunicationService;
@@ -63,13 +68,19 @@ public class TestHashiCorpVaultParameterValueProvider {
             }
 
             @Override
-            public PropertyValue getProperty(PropertyDescriptor descriptor) {
+            public PropertyValue getProperty(final PropertyDescriptor descriptor) {
-                return new StandardPropertyValue(CONFIG_FILE, null, null);
+                if (descriptor.equals(HashiCorpVaultParameterValueProvider.VAULT_CONFIG_FILE)) {
+                    return new StandardPropertyValue(CONFIG_FILE, null, null);
+                } else if (descriptor.equals(HashiCorpVaultParameterValueProvider.DEFAULT_SECRET_NAME)) {
+                    return new StandardPropertyValue(DEFAULT_SECRET_NAME, null, null);
+                }
+                return null;
             }
 
             @Override
             public Map<String, String> getAllProperties() {
-                return null;
+                return Stream.of(HashiCorpVaultParameterValueProvider.VAULT_CONFIG_FILE, HashiCorpVaultParameterValueProvider.DEFAULT_SECRET_NAME)
+                        .collect(Collectors.toMap(PropertyDescriptor::getName, descriptor -> getProperty(descriptor).getValue()));
             }
         });
     }
@@ -77,15 +88,53 @@ public class TestHashiCorpVaultParameterValueProvider {
     @Test
     public void testGetParameterValue() {
         final String value = "value";
-        when(vaultCommunicationService.readKeyValueSecret(PATH, "context/param")).thenReturn(Optional.of(value));
+        final Map<String, String> keyValues = new HashMap<>();
+        keyValues.put("param", value);
+        when(vaultCommunicationService.readKeyValueSecretMap(PATH, "context")).thenReturn(keyValues);
 
         assertEquals(value, parameterProvider.getParameterValue("context", "param"));
     }
 
+    @Test
+    public void testGetParameterValueNoSecret() {
+        when(vaultCommunicationService.readKeyValueSecretMap(PATH, "context")).thenReturn(Collections.emptyMap());
+
+        assertNull(parameterProvider.getParameterValue("context", "param"));
+    }
+
+    @Test
+    public void testGetParameterValueNoMatchingKey() {
+        final String value = "value";
+        final Map<String, String> keyValues = new HashMap<>();
+        keyValues.put("differentParam", value);
+        when(vaultCommunicationService.readKeyValueSecretMap(PATH, "context")).thenReturn(keyValues);
+
+        assertNull(parameterProvider.getParameterValue("context", "param"));
+    }
+
+    @Test
+    public void testGetParameterValueDefaultSecret() {
+        final String value = "value";
+        final String defaultValue = "defaultValue";
+        final Map<String, String> keyValues = new HashMap<>();
+        keyValues.put("differentParam", value);
+
+        final Map<String, String> defaultKeyValues = new HashMap<>();
+        defaultKeyValues.put("param", defaultValue);
+
+        // Although 'context' does not contain 'param', the default secret does, so it should fall back to that
+        when(vaultCommunicationService.readKeyValueSecretMap(PATH, "context")).thenReturn(keyValues);
+        when(vaultCommunicationService.readKeyValueSecretMap(PATH, DEFAULT_SECRET_NAME)).thenReturn(defaultKeyValues);
+
+        assertEquals(defaultValue, parameterProvider.getParameterValue("context", "param"));
+    }
+
     @Test
     public void testIsParameterDefined() {
         final String value = "value";
-        when(vaultCommunicationService.readKeyValueSecret(PATH, "context/param")).thenReturn(Optional.of(value));
+        final Map<String, String> keyValues = new HashMap<>();
+        keyValues.put("param", value);
+        when(vaultCommunicationService.readKeyValueSecretMap(PATH, "context")).thenReturn(keyValues);
 
         assertTrue(parameterProvider.isParameterDefined("context", "param"));
     }

nifi-stateless/nifi-stateless-assembly/README.md

@@ -503,17 +503,18 @@ via the `./conf/bootstrap-hashicorp-vault.conf` file, which comes with NiFi.
 An example of creating a single secret in the correct format is:
 
 ```
-vault kv put "nifi-kv/Context/param" value=my-vault-value
+vault kv put "nifi-kv/Context" param=value param2=value2
 ```
 
 In this example, `nifi-kv` would be supplied by the `vault.kv.path` property in the `bootstrap-hashicorp-vault.conf` file, 
-`Context` is the name of a Parameter Context, and `param` is the name of the parameter whose value should be retrieved from the Vault server.
+`Context` is the name of a mapped Parameter Context, and `param` and `param2` are the names of the parameters whose values should be retrieved from the Vault server.
 
 This Parameter Provider requires the following properties:
 
 | Property Name | Description | Example Value |
 |---------------|-------------|---------------|
 | nifi.stateless.parameter.provider.\<key>.properties.vault-configuration-file | The filename of a configuration file specifying the Vault settings | ./conf/bootstrap-hashicorp-vault.conf |
+| nifi.stateless.parameter.provider.\<key>.properties.default-secret-name | The default K/V secret name to use.  This secret represents a default Parameter Context if there is not a matching key within the mapped Parameter Context secret. | `Default` |
 
 An example of configuring this provider in the dataflow configuration file is: