NIFI-8447 Added HashiCorp Vault Transit Sensitive Properties Provider

- Added default bootstrap-hashicorp-vault.conf
- Updated Toolkit Guide documentation with HashiCorp Vault properties

This closes #5154

Signed-off-by: David Handermann <exceptionfactory@apache.org>
This commit is contained in:
Joe Gresock 2021-06-14 06:53:37 -04:00 committed by exceptionfactory
parent 5e4f32663e
commit 726082ffa6
No known key found for this signature in database
GPG Key ID: 29B6A52D2AAE8DBA
30 changed files with 734 additions and 132 deletions

View File

@ -16,6 +16,7 @@
*/
package org.apache.nifi.properties;
import org.apache.nifi.properties.BootstrapProperties.BootstrapPropertyKey;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -24,6 +25,7 @@ import java.io.IOException;
import java.io.InputStream;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.Objects;
import java.util.Properties;
/**
@ -74,14 +76,27 @@ public abstract class AbstractBootstrapPropertiesLoader {
* @throws IOException If the file is not readable
*/
public BootstrapProperties loadBootstrapProperties(final String bootstrapPath) throws IOException {
final Properties properties = new Properties();
final Path bootstrapFilePath = getBootstrapFile(bootstrapPath).toPath();
try (final InputStream bootstrapInput = Files.newInputStream(bootstrapFilePath)) {
return loadBootstrapProperties(bootstrapFilePath, getApplicationPrefix());
}
/**
* Loads a properties file into a BootstrapProperties object.
* @param bootstrapPath The path to the properties file
* @param propertyPrefix The property prefix to enforce
* @return The BootstrapProperties
* @throws IOException If the properties file could not be read
*/
public static BootstrapProperties loadBootstrapProperties(final Path bootstrapPath, final String propertyPrefix) throws IOException {
Objects.requireNonNull(bootstrapPath, "Bootstrap path must be provided");
Objects.requireNonNull(propertyPrefix, "Property prefix must be provided");
final Properties properties = new Properties();
try (final InputStream bootstrapInput = Files.newInputStream(bootstrapPath)) {
properties.load(bootstrapInput);
return new BootstrapProperties(getApplicationPrefix(), properties, bootstrapFilePath);
return new BootstrapProperties(propertyPrefix, properties, bootstrapPath);
} catch (final IOException e) {
logger.error("Cannot read from bootstrap.conf file at {}", bootstrapFilePath);
throw new IOException("Cannot read from bootstrap.conf", e);
throw new IOException("Cannot read from " + bootstrapPath, e);
}
}
@ -97,7 +112,7 @@ public abstract class AbstractBootstrapPropertiesLoader {
public String extractKeyFromBootstrapFile(final String bootstrapPath) throws IOException {
final BootstrapProperties bootstrapProperties = loadBootstrapProperties(bootstrapPath);
return bootstrapProperties.getBootstrapSensitiveKey().orElseGet(() -> {
return bootstrapProperties.getProperty(BootstrapPropertyKey.SENSITIVE_KEY).orElseGet(() -> {
logger.warn("No encryption key present in the bootstrap.conf file at {}", bootstrapProperties.getConfigFilePath());
return "";
});
@ -121,8 +136,7 @@ public abstract class AbstractBootstrapPropertiesLoader {
if (confDir.exists() && confDir.canRead()) {
expectedBootstrapFile = new File(confDir, BOOTSTRAP_CONF);
} else {
logger.error("Cannot read from bootstrap.conf file at {} -- conf/ directory is missing or permissions are incorrect", confDir.getAbsolutePath());
throw new IOException("Cannot read from bootstrap.conf");
throw new IOException(String.format("Cannot read %s directory for %s", confDir, bootstrapPath));
}
} else {
expectedBootstrapFile = new File(bootstrapPath);
@ -131,8 +145,7 @@ public abstract class AbstractBootstrapPropertiesLoader {
if (expectedBootstrapFile.exists() && expectedBootstrapFile.canRead()) {
return expectedBootstrapFile;
} else {
logger.error("Cannot read from bootstrap.conf file at {} -- file is missing or permissions are incorrect", expectedBootstrapFile.getAbsolutePath());
throw new IOException("Cannot read from bootstrap.conf");
throw new IOException("Cannot read from " + expectedBootstrapFile.getAbsolutePath());
}
}

View File

@ -17,17 +17,29 @@
package org.apache.nifi.properties;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.Enumeration;
import java.util.Objects;
import java.util.Optional;
import java.util.Properties;
import java.util.Set;
/**
* Properties representing bootstrap.conf.
*/
public class BootstrapProperties extends StandardReadableProperties {
private static final String PROPERTY_KEY_FORMAT = "%s.%s";
private static final String BOOTSTRAP_SENSITIVE_KEY = "bootstrap.sensitive.key";
public enum BootstrapPropertyKey {
SENSITIVE_KEY("bootstrap.sensitive.key"),
HASHICORP_VAULT_SENSITIVE_PROPERTY_PROVIDER_CONF("bootstrap.protection.hashicorp.vault.conf");
private final String key;
BootstrapPropertyKey(final String key) {
this.key = key;
}
}
private final String propertyPrefix;
private final Path configFilePath;
@ -43,6 +55,29 @@ public class BootstrapProperties extends StandardReadableProperties {
}
/**
* Ensures that blank or empty properties are returned as null.
* @param key The property key
* @param defaultValue The default value to use if the value is null or empty
* @return The property value (null if empty or blank)
*/
@Override
public String getProperty(final String key, final String defaultValue) {
final String property = super.getProperty(key, defaultValue);
return isBlank(property) ? null : property;
}
/**
* Ensures that blank or empty properties are returned as null.
* @param key The property key
* @return The property value (null if empty or blank)
*/
@Override
public String getProperty(final String key) {
final String property = super.getProperty(key);
return isBlank(property) ? null : property;
}
/**
* Returns the path to the bootstrap config file.
* @return The path to the file
@ -72,15 +107,45 @@ public class BootstrapProperties extends StandardReadableProperties {
}
/**
* Returns the bootstrap sensitive key.
* @return The bootstrap sensitive key
* Returns the optional property value with the given BootstrapPropertyKey.
* @param key A BootstrapPropertyKey, representing properties in bootstrap.conf
* @return The property value
*/
public Optional<String> getBootstrapSensitiveKey() {
return Optional.ofNullable(getProperty(getPropertyKey(BOOTSTRAP_SENSITIVE_KEY)));
public Optional<String> getProperty(final BootstrapPropertyKey key) {
return Optional.ofNullable(getProperty(getPropertyKey(key.key)));
}
@Override
public String toString() {
return String.format("Bootstrap properties [%s] with prefix [%s]", configFilePath, propertyPrefix);
}
/**
* An empty instance of BootstrapProperties.
*/
public static final BootstrapProperties EMPTY = new BootstrapProperties("", new Properties(), Paths.get("conf/bootstrap.conf")) {
@Override
public Set<String> getPropertyKeys() {
return null;
}
@Override
public String getProperty(String key) {
return null;
}
@Override
public String getProperty(String key, String defaultValue) {
return null;
}
@Override
public int size() {
return 0;
}
};
private static boolean isBlank(final String string) {
return (string == null) || string.isEmpty() || string.trim().isEmpty();
}
}

View File

@ -42,6 +42,17 @@
<artifactId>nifi-security-utils</artifactId>
<version>1.14.0-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-vault-utils</artifactId>
<version>1.14.0-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>commons-io</groupId>
<artifactId>commons-io</artifactId>
<version>2.10.0</version>
<scope>test</scope>
</dependency>
</dependencies>
<build>
<!-- Required to run Groovy tests without any Java tests -->

View File

@ -81,7 +81,7 @@ public class AESSensitivePropertyProvider extends AbstractSensitivePropertyProvi
}
@Override
protected boolean isSupported(final BootstrapProperties bootstrapProperties) {
public boolean isSupported() {
return true; // AES protection is always supported
}

View File

@ -0,0 +1,132 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.nifi.properties;
import org.apache.nifi.properties.BootstrapProperties.BootstrapPropertyKey;
import org.apache.nifi.vault.hashicorp.HashiCorpVaultCommunicationService;
import org.apache.nifi.vault.hashicorp.StandardHashiCorpVaultCommunicationService;
import org.apache.nifi.vault.hashicorp.config.HashiCorpVaultConfiguration;
import org.apache.nifi.vault.hashicorp.config.HashiCorpVaultConfiguration.VaultConfigurationKey;
import org.springframework.core.env.PropertySource;
import java.io.IOException;
import java.nio.file.Paths;
public abstract class AbstractHashiCorpVaultSensitivePropertyProvider extends AbstractSensitivePropertyProvider {
private static final String VAULT_PREFIX = "vault";
private final String path;
private final HashiCorpVaultCommunicationService vaultCommunicationService;
private final BootstrapProperties vaultBootstrapProperties;
AbstractHashiCorpVaultSensitivePropertyProvider(final BootstrapProperties bootstrapProperties) {
super(bootstrapProperties);
final String vaultBootstrapConfFilename = bootstrapProperties
.getProperty(BootstrapPropertyKey.HASHICORP_VAULT_SENSITIVE_PROPERTY_PROVIDER_CONF).orElse(null);
vaultBootstrapProperties = getVaultBootstrapProperties(vaultBootstrapConfFilename);
path = getSecretsEnginePath(vaultBootstrapProperties);
if (hasRequiredVaultProperties()) {
try {
vaultCommunicationService = new StandardHashiCorpVaultCommunicationService(getVaultPropertySource(vaultBootstrapConfFilename));
} catch (IOException e) {
throw new SensitivePropertyProtectionException("Error configuring HashiCorpVaultCommunicationService", e);
}
} else {
vaultCommunicationService = null;
}
}
/**
* Return the configured Secrets Engine path for this sensitive property provider.
* @param vaultBootstrapProperties The Properties from the file located at bootstrap.protection.hashicorp.vault.conf
* @return The Secrets Engine path
*/
protected abstract String getSecretsEnginePath(final BootstrapProperties vaultBootstrapProperties);
private static BootstrapProperties getVaultBootstrapProperties(final String vaultBootstrapConfFilename) {
final BootstrapProperties vaultBootstrapProperties;
if (vaultBootstrapConfFilename != null) {
try {
vaultBootstrapProperties = AbstractBootstrapPropertiesLoader.loadBootstrapProperties(
Paths.get(vaultBootstrapConfFilename), VAULT_PREFIX);
} catch (IOException e) {
throw new SensitivePropertyProtectionException("Could not load " + vaultBootstrapConfFilename, e);
}
} else {
vaultBootstrapProperties = null;
}
return vaultBootstrapProperties;
}
private PropertySource<?> getVaultPropertySource(final String vaultBootstrapConfFilename) throws IOException {
return HashiCorpVaultConfiguration.createPropertiesFileSource(vaultBootstrapConfFilename);
}
/**
* Returns the Secrets Engine path.
* @return The Secrets Engine path
*/
protected String getPath() {
return path;
}
protected HashiCorpVaultCommunicationService getVaultCommunicationService() {
if (vaultCommunicationService == null) {
throw new SensitivePropertyProtectionException(getIdentifierKey() + " protection scheme is not fully configured in hashicorp-vault-bootstrap.conf");
}
return vaultCommunicationService;
}
@Override
public boolean isSupported() {
return hasRequiredVaultProperties();
}
/**
* Returns the Vault-specific bootstrap properties (e.g., bootstrap-vault.properties)
* @return The Vault-specific bootstrap properties
*/
protected BootstrapProperties getVaultBootstrapProperties() {
return vaultBootstrapProperties;
}
private boolean hasRequiredVaultProperties() {
return vaultBootstrapProperties != null
&& (vaultBootstrapProperties.getProperty(VaultConfigurationKey.URI.getKey()) != null)
&& hasRequiredSecretsEngineProperties(vaultBootstrapProperties);
}
/**
* Return true if the relevant Secrets Engine-specific properties are configured.
* @param vaultBootstrapProperties The Vault-specific bootstrap properties
* @return true if the relevant Secrets Engine-specific properties are configured
*/
protected abstract boolean hasRequiredSecretsEngineProperties(final BootstrapProperties vaultBootstrapProperties);
/**
* Returns the key used to identify the provider implementation in {@code nifi.properties},
* in the format 'vault/{secretsEngine}/{secretsEnginePath}'.
*
* @return the key to persist in the sibling property
*/
@Override
public String getIdentifierKey() {
return getProtectionScheme().getIdentifier(path);
}
}

View File

@ -33,14 +33,6 @@ public abstract class AbstractSensitivePropertyProvider implements SensitiveProp
*/
protected abstract PropertyProtectionScheme getProtectionScheme();
/**
* Return true if this SensitivePropertyProvider is supported, given the provided
* Bootstrap properties.
* @param bootstrapProperties The Bootstrap properties
* @return True if this SensitivePropertyProvider is supported
*/
protected abstract boolean isSupported(BootstrapProperties bootstrapProperties);
@Override
public String getName() {
return getProtectionScheme().getName();
@ -55,9 +47,4 @@ public abstract class AbstractSensitivePropertyProvider implements SensitiveProp
public String getIdentifierKey() {
return getProtectionScheme().getIdentifier();
}
@Override
public boolean isSupported() {
return isSupported(bootstrapProperties);
}
}

View File

@ -0,0 +1,92 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.nifi.properties;
import org.apache.commons.lang3.StringUtils;
import java.nio.charset.Charset;
import java.nio.charset.StandardCharsets;
/**
* Uses the HashiCorp Vault Transit Secrets Engine to encrypt sensitive values at rest.
*/
public class HashiCorpVaultTransitSensitivePropertyProvider extends AbstractHashiCorpVaultSensitivePropertyProvider {
private static final Charset PROPERTY_CHARSET = StandardCharsets.UTF_8;
private static final String TRANSIT_PATH = "vault.transit.path";
HashiCorpVaultTransitSensitivePropertyProvider(final BootstrapProperties bootstrapProperties) {
super(bootstrapProperties);
}
@Override
protected String getSecretsEnginePath(final BootstrapProperties vaultBootstrapProperties) {
if (vaultBootstrapProperties == null) {
return null;
}
final String transitPath = vaultBootstrapProperties.getProperty(TRANSIT_PATH);
// Validate transit path
try {
PropertyProtectionScheme.fromIdentifier(getProtectionScheme().getIdentifier(transitPath));
} catch (IllegalArgumentException e) {
throw new SensitivePropertyProtectionException(String.format("%s [%s] contains unsupported characters", TRANSIT_PATH, transitPath), e);
}
return transitPath;
}
@Override
protected PropertyProtectionScheme getProtectionScheme() {
return PropertyProtectionScheme.HASHICORP_VAULT_TRANSIT;
}
@Override
protected boolean hasRequiredSecretsEngineProperties(final BootstrapProperties vaultBootstrapProperties) {
return getSecretsEnginePath(vaultBootstrapProperties) != null;
}
/**
* Returns the encrypted cipher text.
*
* @param unprotectedValue the sensitive value
* @return the value to persist in the {@code nifi.properties} file
* @throws SensitivePropertyProtectionException if there is an exception encrypting the value
*/
@Override
public String protect(final String unprotectedValue) throws SensitivePropertyProtectionException {
if (StringUtils.isBlank(unprotectedValue)) {
throw new IllegalArgumentException("Cannot encrypt an empty value");
}
return getVaultCommunicationService().encrypt(getPath(), unprotectedValue.getBytes(PROPERTY_CHARSET));
}
/**
* Returns the decrypted plaintext.
*
* @param protectedValue the cipher text read from the {@code nifi.properties} file
* @return the raw value to be used by the application
* @throws SensitivePropertyProtectionException if there is an error decrypting the cipher text
*/
@Override
public String unprotect(final String protectedValue) throws SensitivePropertyProtectionException {
if (StringUtils.isBlank(protectedValue)) {
throw new IllegalArgumentException("Cannot decrypt an empty value");
}
return new String(getVaultCommunicationService().decrypt(getPath(), protectedValue), PROPERTY_CHARSET);
}
}

View File

@ -24,7 +24,8 @@ import java.util.Objects;
* SensitivePropertyProvider.
*/
public enum PropertyProtectionScheme {
AES_GCM("aes/gcm/(128|192|256)", "aes/gcm/%s", "AES Sensitive Property Provider", true);
AES_GCM("aes/gcm/(128|192|256)", "aes/gcm/%s", "AES Sensitive Property Provider", true),
HASHICORP_VAULT_TRANSIT("hashicorp/vault/transit/[a-zA-Z0-9_-]+", "hashicorp/vault/transit/%s", "HashiCorp Vault Transit Engine Sensitive Property Provider", false);
PropertyProtectionScheme(final String identifierPattern, final String identifierFormat, final String name, final boolean requiresSecretKey) {
this.identifierPattern = identifierPattern;

View File

@ -16,6 +16,7 @@
*/
package org.apache.nifi.properties;
import org.apache.nifi.properties.BootstrapProperties.BootstrapPropertyKey;
import org.apache.nifi.util.NiFiBootstrapUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -76,7 +77,7 @@ public class StandardSensitivePropertyProviderFactory implements SensitiveProper
}
private String getKeyHex() {
return keyHex.orElseGet(() -> getBootstrapProperties().getBootstrapSensitiveKey()
return keyHex.orElseGet(() -> getBootstrapProperties().getProperty(BootstrapPropertyKey.SENSITIVE_KEY)
.orElseThrow(() -> new SensitivePropertyProtectionException("Could not read root key from bootstrap.conf")));
}
@ -90,8 +91,8 @@ public class StandardSensitivePropertyProviderFactory implements SensitiveProper
try {
return NiFiBootstrapUtils.loadBootstrapProperties();
} catch (final IOException e) {
logger.error("Error extracting root key from bootstrap.conf for login identity provider decryption", e);
throw new SensitivePropertyProtectionException("Could not read root key from bootstrap.conf");
logger.debug("Could not load bootstrap.conf from disk, so using empty bootstrap.conf", e);
return BootstrapProperties.EMPTY;
}
});
}
@ -104,7 +105,8 @@ public class StandardSensitivePropertyProviderFactory implements SensitiveProper
switch (protectionScheme) {
case AES_GCM:
return providerMap.computeIfAbsent(protectionScheme, s -> new AESSensitivePropertyProvider(keyHex));
// Other providers may choose to pass getBootstrapProperties() into the constructor
case HASHICORP_VAULT_TRANSIT:
return providerMap.computeIfAbsent(protectionScheme, s -> new HashiCorpVaultTransitSensitivePropertyProvider(getBootstrapProperties()));
default:
throw new SensitivePropertyProtectionException("Unsupported protection scheme " + protectionScheme);
}

View File

@ -16,6 +16,7 @@
*/
package org.apache.nifi.properties;
import org.apache.commons.io.FilenameUtils;
import org.apache.nifi.util.NiFiProperties;
import org.bouncycastle.jce.provider.BouncyCastleProvider;
import org.junit.AfterClass;
@ -23,8 +24,10 @@ import org.junit.BeforeClass;
import org.junit.Test;
import org.mockito.internal.util.io.IOUtil;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.nio.file.Files;
import java.nio.file.Path;
import java.security.Security;
@ -32,8 +35,10 @@ import java.util.Properties;
import java.util.function.Supplier;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotEquals;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertThrows;
import static org.junit.Assert.assertTrue;
public class StandardSensitivePropertyProviderFactoryTest {
@ -45,8 +50,9 @@ public class StandardSensitivePropertyProviderFactoryTest {
private static final String AD_HOC_KEY_HEX = "123456789ABCDEFFEDCBA98765432101";
private static Path tempConfDir;
private static Path mockBootstrapConf;
private static Path mockNifiProperties;
private static Path bootstrapConf;
private static Path hashicorpVaultBootstrapConf;
private static Path nifiProperties;
private static NiFiProperties niFiProperties;
@ -54,23 +60,28 @@ public class StandardSensitivePropertyProviderFactoryTest {
public static void initOnce() throws IOException {
Security.addProvider(new BouncyCastleProvider());
tempConfDir = Files.createTempDirectory("conf");
mockBootstrapConf = Files.createTempFile("bootstrap", ".conf").toAbsolutePath();
bootstrapConf = Files.createTempFile("bootstrap", ".conf").toAbsolutePath();
hashicorpVaultBootstrapConf = Files.createTempFile("bootstrap-hashicorp-vault", ".conf").toAbsolutePath();
mockNifiProperties = Files.createTempFile("nifi", ".properties").toAbsolutePath();
nifiProperties = Files.createTempFile("nifi", ".properties").toAbsolutePath();
mockBootstrapConf = Files.move(mockBootstrapConf, tempConfDir.resolve("bootstrap.conf"));
mockNifiProperties = Files.move(mockNifiProperties, tempConfDir.resolve("nifi.properties"));
bootstrapConf = Files.move(bootstrapConf, tempConfDir.resolve("bootstrap.conf"));
nifiProperties = Files.move(nifiProperties, tempConfDir.resolve("nifi.properties"));
IOUtil.writeText("nifi.bootstrap.sensitive.key=" + BOOTSTRAP_KEY_HEX, mockBootstrapConf.toFile());
System.setProperty(NiFiProperties.PROPERTIES_FILE_PATH, mockNifiProperties.toString());
final String bootstrapConfText = String.format("%s=%s\n%s=%s",
"nifi.bootstrap.sensitive.key", BOOTSTRAP_KEY_HEX,
"nifi.bootstrap.protection.hashicorp.vault.conf", FilenameUtils.separatorsToUnix(hashicorpVaultBootstrapConf.toString()));
IOUtil.writeText(bootstrapConfText, bootstrapConf.toFile());
System.setProperty(NiFiProperties.PROPERTIES_FILE_PATH, FilenameUtils.separatorsToUnix(nifiProperties.toString()));
niFiProperties = new NiFiProperties();
}
@AfterClass
public static void tearDownOnce() throws IOException {
Files.deleteIfExists(mockBootstrapConf);
Files.deleteIfExists(mockNifiProperties);
Files.deleteIfExists(bootstrapConf);
Files.deleteIfExists(hashicorpVaultBootstrapConf);
Files.deleteIfExists(nifiProperties);
Files.deleteIfExists(tempConfDir);
System.clearProperty(NiFiProperties.PROPERTIES_FILE_PATH);
}
@ -99,12 +110,62 @@ public class StandardSensitivePropertyProviderFactoryTest {
private Supplier<BootstrapProperties> mockBootstrapProperties() throws IOException {
final Properties bootstrapProperties = new Properties();
try (final InputStream inputStream = Files.newInputStream(mockBootstrapConf)) {
try (final InputStream inputStream = Files.newInputStream(bootstrapConf)) {
bootstrapProperties.load(inputStream);
return () -> new BootstrapProperties("nifi", bootstrapProperties, mockBootstrapConf);
return () -> new BootstrapProperties("nifi", bootstrapProperties, bootstrapConf);
}
}
private void configureHashicorpVault(final Properties properties) throws IOException {
try (OutputStream out = new FileOutputStream(hashicorpVaultBootstrapConf.toFile())) {
properties.store(out, "HashiCorpVault test");
}
}
@Test
public void testHashicorpVaultTransit() throws IOException {
configureDefaultFactory();
final Properties properties = new Properties();
properties.put("vault.transit.path", "nifi-transit");
configureHashicorpVault(properties);
final SensitivePropertyProvider spp = factory.getProvider(PropertyProtectionScheme.HASHICORP_VAULT_TRANSIT);
}
@Test
public void testHashicorpVaultTransit_isSupported() throws IOException {
configureDefaultFactory();
final Properties properties = new Properties();
properties.put("vault.transit.path", "nifi-transit");
properties.put("vault.uri", "http://localhost:8200");
properties.put("vault.token", "test-token");
configureHashicorpVault(properties);
SensitivePropertyProvider spp = factory.getProvider(PropertyProtectionScheme.HASHICORP_VAULT_TRANSIT);
assertTrue(spp.isSupported());
properties.remove("vault.uri");
configureHashicorpVault(properties);
configureDefaultFactory();
spp = factory.getProvider(PropertyProtectionScheme.HASHICORP_VAULT_TRANSIT);
assertFalse(spp.isSupported());
properties.put("vault.uri", "http://localhost:8200");
properties.remove("vault.transit.path");
spp = factory.getProvider(PropertyProtectionScheme.HASHICORP_VAULT_TRANSIT);
assertFalse(spp.isSupported());
}
@Test
public void testHashicorpVaultTransit_invalidCharacters() throws IOException {
configureDefaultFactory();
final Properties properties = new Properties();
properties.put("vault.transit.path", "invalid/characters");
configureHashicorpVault(properties);
assertThrows(SensitivePropertyProtectionException.class, () -> factory.getProvider(PropertyProtectionScheme.HASHICORP_VAULT_TRANSIT));
}
@Test
public void testAES_GCM() throws IOException {
configureDefaultFactory();

View File

@ -16,27 +16,21 @@
*/
package org.apache.nifi.vault.hashicorp;
import org.apache.nifi.util.FormatUtils;
import org.apache.nifi.vault.hashicorp.config.HashiCorpVaultConfiguration;
import org.apache.nifi.vault.hashicorp.config.HashiCorpVaultProperties;
import org.apache.nifi.vault.hashicorp.config.HashiCorpVaultPropertySource;
import org.springframework.core.env.PropertySource;
import org.springframework.vault.authentication.SimpleSessionManager;
import org.springframework.vault.client.ClientHttpRequestFactoryFactory;
import org.springframework.vault.core.VaultTemplate;
import org.springframework.vault.core.VaultTransitOperations;
import org.springframework.vault.support.Ciphertext;
import org.springframework.vault.support.ClientOptions;
import org.springframework.vault.support.Plaintext;
import org.springframework.vault.support.SslConfiguration;
import java.time.Duration;
import java.util.Optional;
import java.util.concurrent.TimeUnit;
/**
* Implements the VaultCommunicationService using Spring Vault
*/
public class StandardHashiCorpVaultCommunicationService implements HashiCorpVaultCommunicationService {
private static final String HTTPS = "https";
private final HashiCorpVaultConfiguration vaultConfiguration;
private final VaultTemplate vaultTemplate;
@ -44,42 +38,26 @@ public class StandardHashiCorpVaultCommunicationService implements HashiCorpVaul
/**
* Creates a VaultCommunicationService that uses Spring Vault.
* @param vaultProperties Properties to configure the service
* @param propertySources Property sources to configure the service
* @throws HashiCorpVaultConfigurationException If the configuration was invalid
*/
public StandardHashiCorpVaultCommunicationService(final HashiCorpVaultProperties vaultProperties) throws HashiCorpVaultConfigurationException {
this.vaultConfiguration = new HashiCorpVaultConfiguration(vaultProperties);
final SslConfiguration sslConfiguration = vaultProperties.getUri().contains(HTTPS)
? vaultConfiguration.sslConfiguration() : SslConfiguration.unconfigured();
final ClientOptions clientOptions = getClientOptions(vaultProperties);
public StandardHashiCorpVaultCommunicationService(final PropertySource<?>... propertySources) throws HashiCorpVaultConfigurationException {
vaultConfiguration = new HashiCorpVaultConfiguration(propertySources);
vaultTemplate = new VaultTemplate(vaultConfiguration.vaultEndpoint(),
ClientHttpRequestFactoryFactory.create(clientOptions, sslConfiguration),
ClientHttpRequestFactoryFactory.create(vaultConfiguration.clientOptions(), vaultConfiguration.sslConfiguration()),
new SimpleSessionManager(vaultConfiguration.clientAuthentication()));
transitOperations = vaultTemplate.opsForTransit();
}
private static ClientOptions getClientOptions(HashiCorpVaultProperties vaultProperties) {
final ClientOptions clientOptions = new ClientOptions();
Duration readTimeoutDuration = clientOptions.getReadTimeout();
Duration connectionTimeoutDuration = clientOptions.getConnectionTimeout();
final Optional<String> configuredReadTimeout = vaultProperties.getReadTimeout();
if (configuredReadTimeout.isPresent()) {
readTimeoutDuration = getDuration(configuredReadTimeout.get());
}
final Optional<String> configuredConnectionTimeout = vaultProperties.getConnectionTimeout();
if (configuredConnectionTimeout.isPresent()) {
connectionTimeoutDuration = getDuration(configuredConnectionTimeout.get());
}
return new ClientOptions(connectionTimeoutDuration, readTimeoutDuration);
}
private static Duration getDuration(String formattedDuration) {
final double duration = FormatUtils.getPreciseTimeDuration(formattedDuration, TimeUnit.MILLISECONDS);
return Duration.ofMillis(Double.valueOf(duration).longValue());
/**
* Creates a VaultCommunicationService that uses Spring Vault.
* @param vaultProperties Properties to configure the service
* @throws HashiCorpVaultConfigurationException If the configuration was invalid
*/
public StandardHashiCorpVaultCommunicationService(final HashiCorpVaultProperties vaultProperties) throws HashiCorpVaultConfigurationException {
this(new HashiCorpVaultPropertySource(vaultProperties));
}
@Override

View File

@ -16,31 +16,124 @@
*/
package org.apache.nifi.vault.hashicorp.config;
import org.apache.nifi.util.FormatUtils;
import org.apache.nifi.vault.hashicorp.HashiCorpVaultConfigurationException;
import org.springframework.core.env.ConfigurableEnvironment;
import org.springframework.core.env.PropertySource;
import org.springframework.core.env.StandardEnvironment;
import org.springframework.core.io.FileSystemResource;
import org.springframework.core.io.support.ResourcePropertySource;
import org.springframework.vault.client.RestTemplateFactory;
import org.springframework.vault.config.EnvironmentVaultConfiguration;
import org.springframework.vault.support.ClientOptions;
import org.springframework.vault.support.SslConfiguration;
import java.io.IOException;
import java.nio.file.Paths;
import java.time.Duration;
import java.util.concurrent.TimeUnit;
/**
* A Vault configuration that uses the NiFiVaultEnvironment.
*/
public class HashiCorpVaultConfiguration extends EnvironmentVaultConfiguration {
public enum VaultConfigurationKey {
AUTHENTICATION_PROPERTIES_FILE("vault.authentication.properties.file"),
READ_TIMEOUT("vault.read.timeout"),
CONNECTION_TIMEOUT("vault.connection.timeout"),
URI("vault.uri");
public HashiCorpVaultConfiguration(final HashiCorpVaultProperties vaultProperties) throws HashiCorpVaultConfigurationException {
final ConfigurableEnvironment env = new StandardEnvironment();
private final String key;
try {
env.getPropertySources().addFirst(new ResourcePropertySource(new FileSystemResource(Paths.get(vaultProperties.getAuthPropertiesFilename()))));
} catch (IOException e) {
throw new HashiCorpVaultConfigurationException("Could not load auth properties", e);
VaultConfigurationKey(final String key) {
this.key = key;
}
/**
* Returns the property key.
* @return The property key
*/
public String getKey() {
return key;
}
}
private static final String HTTPS = "https";
private final SslConfiguration sslConfiguration;
private final ClientOptions clientOptions;
/**
* Creates a HashiCorpVaultConfiguration from property sources
* @param propertySources A series of Spring PropertySource objects
* @throws HashiCorpVaultConfigurationException If the authentication properties file could not be read
*/
public HashiCorpVaultConfiguration(final PropertySource<?>... propertySources) {
final ConfigurableEnvironment env = new StandardEnvironment();
for(final PropertySource<?> propertySource : propertySources) {
env.getPropertySources().addFirst(propertySource);
}
if (env.containsProperty(VaultConfigurationKey.AUTHENTICATION_PROPERTIES_FILE.key)) {
final String authPropertiesFilename = env.getProperty(VaultConfigurationKey.AUTHENTICATION_PROPERTIES_FILE.key);
try {
final PropertySource<?> authPropertiesSource = createPropertiesFileSource(authPropertiesFilename);
env.getPropertySources().addFirst(authPropertiesSource);
} catch (IOException e) {
throw new HashiCorpVaultConfigurationException("Could not load HashiCorp Vault authentication properties " + authPropertiesFilename, e);
}
}
env.getPropertySources().addFirst(new HashiCorpVaultPropertySource(vaultProperties));
this.setApplicationContext(new HashiCorpVaultApplicationContext(env));
sslConfiguration = env.getProperty(VaultConfigurationKey.URI.key).contains(HTTPS)
? super.sslConfiguration() : SslConfiguration.unconfigured();
clientOptions = getClientOptions();
}
/**
* A convenience method to create a PropertySource from a file on disk.
* @param filename The properties filename.
* @return A PropertySource containing the properties in the given file
* @throws IOException If the file could not be read
*/
public static PropertySource<?> createPropertiesFileSource(final String filename) throws IOException {
return new ResourcePropertySource(new FileSystemResource(Paths.get(filename)));
}
@Override
public ClientOptions clientOptions() {
return clientOptions;
}
@Override
protected RestTemplateFactory getRestTemplateFactory() {
return this.restTemplateFactory(clientHttpRequestFactoryWrapper());
}
@Override
public SslConfiguration sslConfiguration() {
return sslConfiguration;
}
private ClientOptions getClientOptions() {
final ClientOptions clientOptions = new ClientOptions();
Duration readTimeoutDuration = clientOptions.getReadTimeout();
Duration connectionTimeoutDuration = clientOptions.getConnectionTimeout();
final String configuredReadTimeout = getEnvironment().getProperty(VaultConfigurationKey.READ_TIMEOUT.key);
if (configuredReadTimeout != null) {
readTimeoutDuration = getDuration(configuredReadTimeout);
}
final String configuredConnectionTimeout = getEnvironment().getProperty(VaultConfigurationKey.CONNECTION_TIMEOUT.key);
if (configuredConnectionTimeout != null) {
connectionTimeoutDuration = getDuration(configuredConnectionTimeout);
}
return new ClientOptions(connectionTimeoutDuration, readTimeoutDuration);
}
private static Duration getDuration(String formattedDuration) {
final double duration = FormatUtils.getPreciseTimeDuration(formattedDuration, TimeUnit.MILLISECONDS);
return Duration.ofMillis(Double.valueOf(duration).longValue());
}
}

View File

@ -79,6 +79,7 @@ public class HashiCorpVaultProperties {
return ssl;
}
@HashiCorpVaultProperty(key = "authentication.properties.file")
public String getAuthPropertiesFilename() {
return authPropertiesFilename;
}

View File

@ -28,4 +28,5 @@ import java.lang.annotation.Target;
@Target(ElementType.METHOD)
@Retention(RetentionPolicy.RUNTIME)
public @interface HashiCorpVaultProperty {
String key() default "";
}

View File

@ -30,10 +30,10 @@ public class HashiCorpVaultPropertySource extends PropertySource<HashiCorpVaultP
private PropertyLookup propertyLookup;
public HashiCorpVaultPropertySource(HashiCorpVaultProperties source) {
public HashiCorpVaultPropertySource(final HashiCorpVaultProperties source) {
super(HashiCorpVaultPropertySource.class.getName(), source);
propertyLookup = new BeanPropertyLookup(PREFIX, HashiCorpVaultProperties.class, HashiCorpVaultProperty.class);
propertyLookup = new BeanPropertyLookup(PREFIX, HashiCorpVaultProperties.class);
}
@Override

View File

@ -17,10 +17,10 @@
package org.apache.nifi.vault.hashicorp.config.lookup;
import org.apache.nifi.vault.hashicorp.HashiCorpVaultConfigurationException;
import org.apache.nifi.vault.hashicorp.config.HashiCorpVaultProperty;
import org.springframework.beans.BeanUtils;
import java.beans.PropertyDescriptor;
import java.lang.annotation.Annotation;
import java.lang.reflect.InvocationTargetException;
import java.util.Arrays;
import java.util.Map;
@ -34,24 +34,25 @@ public class BeanPropertyLookup extends PropertyLookup {
private final Map<String, PropertyLookup> propertyLookupMap;
public BeanPropertyLookup(final String prefix, final Class<?> beanClass, final Class<? extends Annotation> propertyFilter) {
this(prefix, beanClass, propertyFilter, null);
public BeanPropertyLookup(final String prefix, final Class<?> beanClass) {
this(prefix, beanClass, null);
}
private BeanPropertyLookup(final String prefix, final Class<?> beanClass, final Class<? extends Annotation> propertyFilter,
final PropertyDescriptor propertyDescriptor) {
private BeanPropertyLookup(final String prefix, final Class<?> beanClass, final PropertyDescriptor propertyDescriptor) {
super(propertyDescriptor);
propertyLookupMap = Arrays.stream(BeanUtils.getPropertyDescriptors(beanClass))
.filter(pd -> pd.getReadMethod().getAnnotation(propertyFilter) != null)
.filter(pd -> pd.getReadMethod().getAnnotation(HashiCorpVaultProperty.class) != null)
.collect(Collectors.toMap(
pd -> getPropertyKey(prefix, pd),
pd -> pd.getReadMethod().getReturnType().equals(String.class) ? new ValuePropertyLookup(pd)
: new BeanPropertyLookup(getPropertyKey(prefix, pd), pd.getReadMethod().getReturnType(), propertyFilter, pd)
: new BeanPropertyLookup(getPropertyKey(prefix, pd), pd.getReadMethod().getReturnType(), pd)
));
}
private static String getPropertyKey(final String prefix, final PropertyDescriptor propertyDescriptor) {
return prefix == null ? propertyDescriptor.getDisplayName() : String.join(SEPARATOR, prefix, propertyDescriptor.getDisplayName());
final HashiCorpVaultProperty propertyAnnotation = propertyDescriptor.getReadMethod().getAnnotation(HashiCorpVaultProperty.class);
final String unqualifiedPropertyKey = !propertyAnnotation.key().isEmpty() ? propertyAnnotation.key() : propertyDescriptor.getDisplayName();
return prefix == null ? unqualifiedPropertyKey: String.join(SEPARATOR, prefix, unqualifiedPropertyKey);
}
@Override

View File

@ -18,6 +18,7 @@ package org.apache.nifi.vault.hashicorp;
import org.apache.nifi.vault.hashicorp.config.HashiCorpVaultConfiguration;
import org.apache.nifi.vault.hashicorp.config.HashiCorpVaultProperties;
import org.apache.nifi.vault.hashicorp.config.HashiCorpVaultPropertySource;
import org.junit.AfterClass;
import org.junit.Assert;
import org.junit.Before;
@ -113,13 +114,13 @@ public class TestHashiCorpVaultConfiguration {
}
}
public void runTest() {
config = new HashiCorpVaultConfiguration(propertiesBuilder.build());
public void runTest(final String expectedScheme) {
config = new HashiCorpVaultConfiguration(new HashiCorpVaultPropertySource(propertiesBuilder.build()));
VaultEndpoint endpoint = config.vaultEndpoint();
Assert.assertEquals("localhost", endpoint.getHost());
Assert.assertEquals(8200, endpoint.getPort());
Assert.assertEquals("http", endpoint.getScheme());
Assert.assertEquals(expectedScheme, endpoint.getScheme());
ClientAuthentication clientAuthentication = config.clientAuthentication();
Assert.assertNotNull(clientAuthentication);
@ -127,7 +128,7 @@ public class TestHashiCorpVaultConfiguration {
@Test
public void testBasicProperties() {
this.runTest();
this.runTest("http");
}
@Test
@ -140,8 +141,9 @@ public class TestHashiCorpVaultConfiguration {
propertiesBuilder.setTrustStoreType(TRUSTSTORE_TYPE_VALUE);
propertiesBuilder.setEnabledTlsProtocols(TLS_V_1_3_VALUE);
propertiesBuilder.setEnabledTlsCipherSuites(TEST_CIPHER_SUITE_VALUE);
propertiesBuilder.setUri(URI_VALUE.replace("http", "https"));
this.runTest();
this.runTest("https");
SslConfiguration sslConfiguration = config.sslConfiguration();
Assert.assertEquals(keystoreFile.toFile().getAbsolutePath(), sslConfiguration.getKeyStoreConfiguration().getResource().getFile().getAbsolutePath());
@ -157,7 +159,7 @@ public class TestHashiCorpVaultConfiguration {
@Test
public void testInvalidTLS() {
propertiesBuilder.setUri(URI_VALUE.replace("http", "https"));
Assert.assertThrows(NullPointerException.class, () -> this.runTest());
Assert.assertThrows(NullPointerException.class, () -> this.runTest("https"));
}
@Test
@ -169,7 +171,7 @@ public class TestHashiCorpVaultConfiguration {
authProperties = writeVaultAuthProperties(props);
propertiesBuilder.setAuthPropertiesFilename(authProperties.getAbsolutePath());
Assert.assertThrows(IllegalArgumentException.class, () -> this.runTest());
Assert.assertThrows(IllegalArgumentException.class, () -> this.runTest("http"));
} finally {
if (authProperties != null) {
Files.deleteIfExists(authProperties.toPath());
@ -185,7 +187,7 @@ public class TestHashiCorpVaultConfiguration {
authProperties = writeVaultAuthProperties(props);
propertiesBuilder.setAuthPropertiesFilename(authProperties.getAbsolutePath());
Assert.assertThrows(IllegalArgumentException.class, () -> this.runTest());
Assert.assertThrows(IllegalArgumentException.class, () -> this.runTest("http"));
} finally {
if (authProperties != null) {
Files.deleteIfExists(authProperties.toPath());

View File

@ -70,9 +70,8 @@ public class TestStandardHashiCorpVaultCommunicationService {
// Once to check if the URI is https, and once by VaultTemplate
Mockito.verify(properties, Mockito.times(2)).getUri();
// Once each to check if they are configured
Mockito.verify(properties, Mockito.times(1)).getConnectionTimeout();
Mockito.verify(properties, Mockito.times(1)).getReadTimeout();
// Once to check if the property is set, and once to retrieve the value
Mockito.verify(properties, Mockito.times(2)).getAuthPropertiesFilename();
// These should not be called because TLS is not configured
this.ensureTlsPropertiesAccessed(0);
@ -94,9 +93,6 @@ public class TestStandardHashiCorpVaultCommunicationService {
Mockito.when(properties.getConnectionTimeout()).thenReturn(Optional.of("20 secs"));
Mockito.when(properties.getReadTimeout()).thenReturn(Optional.of("40 secs"));
this.configureService();
Mockito.verify(properties, Mockito.times(1)).getConnectionTimeout();
Mockito.verify(properties, Mockito.times(1)).getReadTimeout();
}
@Test

View File

@ -1759,7 +1759,7 @@ All options require a password (`nifi.sensitive.props.key` value) of *at least 1
[[encrypt-config_tool]]
== Encrypted Passwords in Configuration Files
In order to facilitate the secure setup of NiFi, you can use the `encrypt-config` command line utility to encrypt raw configuration values that NiFi decrypts in memory on startup. This extensible protection scheme transparently allows NiFi to use raw values in operation, while protecting them at rest. In the future, hardware security modules (HSM) and external secure storage mechanisms will be integrated, but for now, an AES encryption provider is the default implementation.
In order to facilitate the secure setup of NiFi, you can use the `encrypt-config` command line utility to encrypt raw configuration values that NiFi decrypts in memory on startup. This extensible protection scheme transparently allows NiFi to use raw values in operation, while protecting them at rest. In addition to the default AES encryption provider, a HashiCorp Vault encryption provider can be configured in the `bootstrap-hashicorp-vault.properties` file.
This is a change in behavior; prior to 1.0, all configuration values were stored in plaintext on the file system. POSIX file permissions were recommended to limit unauthorized access to these files.

View File

@ -434,13 +434,15 @@ The following are available options when targeting NiFi:
* `-u`,`--outputAuthorizers <file>` The destination _authorizers.xml_ file containing protected config values (will not modify input _authorizers.xml_)
* `-f`,`--flowXml <file>` The _flow.xml.gz_ file currently protected with old password (will be overwritten unless `-g` is specified)
* `-g`,`--outputFlowXml <file>` The destination _flow.xml.gz_ file containing protected config values (will not modify input _flow.xml.gz_)
* `-b`,`--bootstrapConf <file>` The _bootstrap.conf_ file to persist root key
* `-b`,`--bootstrapConf <file>` The bootstrap.conf file to persist root key and to optionally provide any configuration for the protection scheme.
* `-S`,`--protectionScheme <protectionScheme>` Selects the protection scheme for encrypted properties. Valid values are: [AES_GCM, HASHICORP_VAULT_TRANSIT] (default is AES_GCM)
* `-k`,`--key <keyhex>` The raw hexadecimal key to use to encrypt the sensitive properties
* `-e`,`--oldKey <keyhex>` The old raw hexadecimal key to use during key migration
* `-H`,`--oldProtectionScheme <protectionScheme>` The old protection scheme to use during encryption migration (see --protectionScheme for possible values). Default is AES_GCM
* `-p`,`--password <password>` The password from which to derive the key to use to encrypt the sensitive properties
* `-w`,`--oldPassword <password>` The old password from which to derive the key during migration
* `-r`,`--useRawKey` If provided, the secure console will prompt for the raw key value in hexadecimal form
* `-m`,`--migrate` If provided, the _nifi.properties_ and/or _login-identity-providers.xml_ sensitive properties will be re-encrypted with a new key
* `-m`,`--migrate` If provided, the _nifi.properties_ and/or _login-identity-providers.xml_ sensitive properties will be re-encrypted with the new scheme
* `-x`,`--encryptFlowXmlOnly` If provided, the properties in _flow.xml.gz_ will be re-encrypted with a new key but the _nifi.properties_ and/or _login-identity-providers.xml_ files will not be modified
* `-s`,`--propsKey <password|keyhex>` The password or key to use to encrypt the sensitive processor properties in _flow.xml.gz_
* `-A`,`--newFlowAlgorithm <algorithm>` The algorithm to use to encrypt the sensitive processor properties in _flow.xml.gz_
@ -454,9 +456,11 @@ The following are available options when targeting NiFi Registry using the `--ni
* `-v`,`--verbose` Sets verbose mode (default false)
* `-p`,`--password <password>` Protect the files using a password-derived key. If an argument is not provided to this flag, interactive mode will be triggered to prompt the user to enter the password.
* `-k`,`--key <keyhex>` Protect the files using a raw hexadecimal key. If an argument is not provided to this flag, interactive mode will be triggered to prompt the user to enter the key.
* `-S`,`--protectionScheme <protectionScheme>` Selects the protection scheme for encrypted properties. Valid values are: [AES_GCM, HASHICORP_VAULT_TRANSIT] (default is AES_GCM)
* `--oldPassword <password>` If the input files are already protected using a password-derived key, this specifies the old password so that the files can be unprotected before re-protecting.
* `--oldKey <keyhex>` If the input files are already protected using a key, this specifies the raw hexadecimal key so that the files can be unprotected before re-protecting.
* `-b`,`--bootstrapConf <file>` The _bootstrap.conf_ file containing no root key or an existing root key. If a new password or key is specified (using `-p` or `-k`) and no output _bootstrap.conf_ file is specified, then this file will be overwritten to persist the new root key.
* `-H`,`--oldProtectionScheme <protectionScheme>`The old protection scheme to use during encryption migration (see --protectionScheme for possible values). Default is AES_GCM.
* `-b`,`--bootstrapConf <file>` The _bootstrap.conf_ file containing no root key or an existing root key, and any other protection scheme configuration properties. If a new password or key is specified (using -p or -k) and no output _bootstrap.conf_ file is specified, then this file will be overwritten to persist the new master key.
* `-B`,`--outputBootstrapConf <file>` The destination _bootstrap.conf_ file to persist root key. If specified, the input _bootstrap.conf_ will not be modified.
* `-r`,`--nifiRegistryProperties <file>` The _nifi-registry.properties_ file containing unprotected config values, overwritten if no output file specified.
* `-R`,`--outputNifiRegistryProperties <file>` The destination _nifi-registry.properties_ file containing protected config values.
@ -466,6 +470,40 @@ The following are available options when targeting NiFi Registry using the `--ni
* `-I`,`--outputIdentityProvidersXml <file>` The destination _identity-providers.xml_ file containing protected config values.
* `--decrypt` Can be used with `-r` to decrypt a previously encrypted NiFi Registry Properties file. Decrypted content is printed to STDOUT.
=== Protection Schemes
The protection scheme can be selected during encryption using the `--protectionScheme` flag. During migration, the former protection scheme is specified using the `--oldProtectionScheme` flag. This distinction allows a set of protected configuration files to be migrated not only to a new key, but to a completely different protection scheme.
==== AES_GCM
The default protection scheme, `AES-G/CM` simply encrypts sensitive properties and marks their protection as either `aes/gcm/256` or `aes/gcm/256` as appropriate. This protection is all done within NiFi itself.
==== HASHICORP_VAULT_TRANSIT
This protection scheme uses HashiCorp Vault's Transit Secrets Engine (https://www.vaultproject.io/docs/secrets/transit) to outsource encryption to a configured Vault server. All HashiCorp Vault configuration is stored in the `bootstrap-hashicorp-vault.conf` file, as referenced in the `bootstrap.conf` of a NiFi or NiFi Registry instance. Therefore, when using the HASHICORP_VAULT_TRANSIT protection scheme, the `nifi(.registry)?.bootstrap.protection.hashicorp.vault.conf` property in the `bootstrap.conf` specified using the `-b` flag must be available to the Encrypt Configuration Tool and must be configured as follows:
===== Required properties
[options="header,footer"]
|===
|Property Name|Description|Default
|`vault.uri`|The HashiCorp Vault URI (e.g., `https://vault-server:8200`). If not set, this provider will be disabled.|_none_
|`vault.authentication.properties.file`|Filename of a properties file containing Vault authentication properties. See the `Authentication-specific property keys` section of https://docs.spring.io/spring-vault/docs/2.3.x/reference/html/#vault.core.environment-vault-configuration for all authentication property keys. If not set, all Spring Vault authentication properties must be configured directly in bootstrap-hashicorp-vault.conf.|_none_
|`vault.transit.path`|The HashiCorp Vault `path` specifying the Transit Secrets Engine (e.g., `nifi-transit`). Valid characters include alphanumeric, dash, and underscore. If not set, this provider will be disabled.|_none_
|===
===== Optional properties
[options="header,footer"]
|===
|Property Name|Description|Default
|`vault.connection.timeout`|The connection timeout of the Vault client|`5 secs`
|`vault.read.timeout`|The read timeout of the Vault client|`15 secs`
|`vault.ssl.enabledCipherSuites`|A comma-separated list of the enabled TLS cipher suites|_none_
|`vault.ssl.enabledProtocols`|A comma-separated list of the enabled TLS protocols|_none_
|`vault.ssl.key-store`|Path to a keystore. Required if the Vault server is TLS-enabled|_none_
|`vault.ssl.key-store-type`|Keystore type (JKS, BCFKS or PKCS12). Required if the Vault server is TLS-enabled|_none_
|`vault.ssl.key-store-password`|Keystore password. Required if the Vault server is TLS-enabled|_none_
|`vault.ssl.trust-store`|Path to a truststore. Required if the Vault server is TLS-enabled|_none_
|`vault.ssl.trust-store-type`|Truststore type (JKS, BCFKS or PKCS12). Required if the Vault server is TLS-enabled|_none_
|`vault.ssl.trust-store-password`|Truststore password. Required if the Vault server is TLS-enabled|_none_
|===
=== Examples
==== NiFi
@ -658,6 +696,8 @@ for each phase (old vs. new), and any combination is sufficient:
* old password -> new key
* old password -> new password
In order to change the protection scheme (e.g., migrating from AES encryption to Vault encryption), specify the `--protectionScheme`
and `--oldProtectionScheme` in the migration command.
== File Manager
The File Manager utility (invoked as `./bin/file-manager.sh` or `bin\file-manager.bat`) allows system administrators to take a backup of an existing NiFi installation, install a new version of NiFi in a designated location (while migrating any previous configuration settings) or restore an installation from a previous backup. File Manager supports NiFi version 1.0.0 and higher.

View File

@ -290,6 +290,8 @@ class NiFiPropertiesLoaderGroovyTest extends GroovyTestCase {
void testShouldLoadUnprotectedPropertiesFromProtectedFile() throws Exception {
// Arrange
File protectedFile = new File("src/test/resources/conf/nifi_with_sensitive_properties_protected_aes.properties")
System.setProperty(NiFiProperties.PROPERTIES_FILE_PATH, protectedFile.path)
NiFiPropertiesLoader niFiPropertiesLoader = NiFiPropertiesLoader.withKey(KEY_HEX)
final def EXPECTED_PLAIN_VALUES = [
@ -378,7 +380,7 @@ class NiFiPropertiesLoaderGroovyTest extends GroovyTestCase {
logger.expected(msg)
// Assert
assert msg == "Cannot read from bootstrap.conf"
assert msg =~ "Cannot read from .*bootstrap.conf"
}
@Test
@ -399,7 +401,7 @@ class NiFiPropertiesLoaderGroovyTest extends GroovyTestCase {
logger.expected(msg)
// Assert
assert msg == "Cannot read from bootstrap.conf"
assert msg =~ "Cannot read from .*bootstrap.conf"
// Clean up to allow for indexing, etc.
Files.setPosixFilePermissions(unreadableFile.toPath(), originalPermissions)

View File

@ -0,0 +1,48 @@
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# HTTP or HTTPS URI for HashiCorp Vault is required to enable the Sensitive Properties Provider
vault.uri=
# Transit Path is required to enable the Sensitive Properties Provider Protection Scheme 'hashicorp/vault/transit/{path}'
vault.transit.path=
# Token Authentication example properties
# vault.authentication=TOKEN
# vault.token=<token value>
# Optional file supports authentication properties described in the Spring Vault Environment Configuration
# https://docs.spring.io/spring-vault/docs/2.3.x/reference/html/#vault.core.environment-vault-configuration
#
# All authentication properties must be included in bootstrap-hashicorp-vault.conf when this property is not specified.
# Properties in bootstrap-hashicorp-vault.conf take precedence when the same values are defined in both files.
# Token Authentication is the default when the 'vault.authentication' property is not specified.
vault.authentication.properties.file=
# Optional Timeout properties
vault.connection.timeout=5 secs
vault.read.timeout=15 secs
# Optional TLS properties
vault.ssl.enabledCipherSuites=
vault.ssl.enabledProtocols=
vault.ssl.key-store=
vault.ssl.key-store-type=
vault.ssl.key-store-password=
vault.ssl.trust-store=
vault.ssl.trust-store-type=
vault.ssl.trust-store-password=

View File

@ -58,6 +58,11 @@ java.arg.14=-Djava.awt.headless=true
# Root key in hexadecimal format for encrypted sensitive configuration values
nifi.bootstrap.sensitive.key=
# Sensitive Property Provider configuration
# HashiCorp Vault Sensitive Property Providers
nifi.bootstrap.protection.hashicorp.vault.conf=./conf/bootstrap-hashicorp-vault.conf
# Sets the provider of SecureRandom to /dev/urandom to prevent blocking on VMs
java.arg.15=-Djava.security.egd=file:/dev/urandom

View File

@ -16,7 +16,7 @@
*/
package org.apache.nifi.registry.properties;
import org.apache.nifi.properties.SensitivePropertyProtectionException;
import org.apache.nifi.properties.BootstrapProperties;
import org.apache.nifi.properties.SensitivePropertyProvider;
import org.apache.nifi.properties.SensitivePropertyProviderFactory;
import org.apache.nifi.properties.StandardSensitivePropertyProviderFactory;
@ -82,7 +82,8 @@ public class NiFiRegistryPropertiesLoader {
try {
return NiFiRegistryBootstrapUtils.loadBootstrapProperties();
} catch (IOException e) {
throw new SensitivePropertyProtectionException("Could not load bootstrap.conf for sensitive property provider configuration.", e);
logger.debug("Cannot read bootstrap.conf -- file is missing or not readable. Defaulting to empty bootstrap.conf");
return BootstrapProperties.EMPTY;
}
});
}

View File

@ -95,7 +95,7 @@ class NiFiRegistryBootstrapUtilsGroovyTest extends GroovyTestCase {
logger.info(msg)
// Assert
assert msg == "Cannot read from bootstrap.conf"
assert msg =~ "Cannot read from .*bootstrap.missing.conf"
}
@Test
@ -114,7 +114,7 @@ class NiFiRegistryBootstrapUtilsGroovyTest extends GroovyTestCase {
logger.info(msg)
// Assert
assert msg == "Cannot read from bootstrap.conf"
assert msg =~ "Cannot read from .*bootstrap.unreadable_file_permissions.conf"
} finally {
// Clean up to allow for indexing, etc.
Files.setPosixFilePermissions(unreadableFile.toPath(), originalPermissions)

View File

@ -0,0 +1,48 @@
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# HTTP or HTTPS URI for HashiCorp Vault is required to enable the Sensitive Properties Provider
vault.uri=
# Transit Path is required to enable the Sensitive Properties Provider Protection Scheme 'hashicorp/vault/transit/{path}'
vault.transit.path=
# Token Authentication example properties
# vault.authentication=TOKEN
# vault.token=<token value>
# Optional file supports authentication properties described in the Spring Vault Environment Configuration
# https://docs.spring.io/spring-vault/docs/2.3.x/reference/html/#vault.core.environment-vault-configuration
#
# All authentication properties must be included in bootstrap-hashicorp-vault.conf when this property is not specified.
# Properties in bootstrap-hashicorp-vault.conf take precedence when the same values are defined in both files.
# Token Authentication is the default when the 'vault.authentication' property is not specified.
vault.authentication.properties.file=
# Optional Timeout properties
vault.connection.timeout=5 secs
vault.read.timeout=15 secs
# Optional TLS properties
vault.ssl.enabledCipherSuites=
vault.ssl.enabledProtocols=
vault.ssl.key-store=
vault.ssl.key-store-type=
vault.ssl.key-store-password=
vault.ssl.trust-store=
vault.ssl.trust-store-type=
vault.ssl.trust-store-password=

View File

@ -51,4 +51,9 @@ java.arg.5=-Dsun.net.http.allowRestrictedHeaders=true
java.arg.6=-Djava.protocol.handler.pkgs=sun.net.www.protocol
# Master key in hexadecimal format for encrypted sensitive configuration values
nifi.registry.bootstrap.sensitive.key=
nifi.registry.bootstrap.sensitive.key=
# Sensitive Property Provider configuration
# HashiCorp Vault Sensitive Property Providers
nifi.registry.bootstrap.protection.hashicorp.vault.conf=./conf/bootstrap-hashicorp-vault.conf

View File

@ -31,7 +31,6 @@ import org.apache.nifi.encrypt.PropertyEncryptor
import org.apache.nifi.encrypt.PropertyEncryptorFactory
import org.apache.nifi.flow.encryptor.FlowEncryptor
import org.apache.nifi.flow.encryptor.StandardFlowEncryptor
import org.apache.nifi.registry.properties.util.NiFiRegistryBootstrapUtils
import org.apache.nifi.toolkit.tls.commandLine.CommandLineParseException
import org.apache.nifi.toolkit.tls.commandLine.ExitCode
import org.apache.nifi.util.NiFiBootstrapUtils
@ -599,7 +598,8 @@ class ConfigEncryptionTool {
}
private NiFiPropertiesLoader getNiFiPropertiesLoader(final String keyHex) {
return protectionScheme.requiresSecretKey() ? NiFiPropertiesLoader.withKey(keyHex) : new NiFiPropertiesLoader()
return protectionScheme.requiresSecretKey() || migrationProtectionScheme.requiresSecretKey()
? NiFiPropertiesLoader.withKey(keyHex) : new NiFiPropertiesLoader()
}
/**
@ -1566,7 +1566,7 @@ class ConfigEncryptionTool {
@Override
BootstrapProperties get() {
try {
NiFiRegistryBootstrapUtils.loadBootstrapProperties(bootstrapConfPath)
NiFiBootstrapUtils.loadBootstrapProperties(bootstrapConfPath)
} catch (final IOException e) {
throw new SensitivePropertyProtectionException(e.getCause(), e)
}

View File

@ -17,7 +17,6 @@
package org.apache.nifi.toolkit.encryptconfig
import groovy.cli.commons.CliBuilder
import org.apache.nifi.properties.ConfigEncryptionTool
import org.apache.nifi.properties.PropertyProtectionScheme
import org.apache.nifi.properties.StandardSensitivePropertyProviderFactory
import org.apache.nifi.toolkit.encryptconfig.util.BootstrapUtil
@ -117,7 +116,7 @@ class NiFiRegistryDecryptMode extends DecryptMode {
}
config.decryptionProvider = StandardSensitivePropertyProviderFactory
.withKeyAndBootstrapSupplier(config.key, ConfigEncryptionTool.getBootstrapSupplier(config.inputBootstrapPath))
.withKeyAndBootstrapSupplier(config.key, NiFiRegistryMode.getBootstrapSupplier(config.inputBootstrapPath))
.getProvider(config.protectionScheme)
run(config)

View File

@ -20,10 +20,13 @@ import groovy.cli.commons.CliBuilder
import groovy.cli.commons.OptionAccessor
import org.apache.commons.cli.HelpFormatter
import org.apache.commons.cli.Options
import org.apache.nifi.properties.BootstrapProperties
import org.apache.nifi.properties.ConfigEncryptionTool
import org.apache.nifi.properties.PropertyProtectionScheme
import org.apache.nifi.properties.SensitivePropertyProtectionException
import org.apache.nifi.properties.SensitivePropertyProvider
import org.apache.nifi.properties.StandardSensitivePropertyProviderFactory
import org.apache.nifi.registry.properties.util.NiFiRegistryBootstrapUtils
import org.apache.nifi.toolkit.encryptconfig.util.BootstrapUtil
import org.apache.nifi.toolkit.encryptconfig.util.NiFiRegistryAuthorizersXmlEncryptor
import org.apache.nifi.toolkit.encryptconfig.util.NiFiRegistryIdentityProvidersXmlEncryptor
@ -33,6 +36,8 @@ import org.apache.nifi.util.console.TextDevices
import org.slf4j.Logger
import org.slf4j.LoggerFactory
import java.util.function.Supplier
class NiFiRegistryMode implements ToolMode {
private static final Logger logger = LoggerFactory.getLogger(NiFiRegistryMode.class)
@ -45,6 +50,19 @@ class NiFiRegistryMode implements ToolMode {
verboseEnabled = false
}
static Supplier<BootstrapProperties> getBootstrapSupplier(final String bootstrapConfPath) {
new Supplier<BootstrapProperties>() {
@Override
BootstrapProperties get() {
try {
NiFiRegistryBootstrapUtils.loadBootstrapProperties(bootstrapConfPath)
} catch (final IOException e) {
throw new SensitivePropertyProtectionException(e.getCause(), e)
}
}
}
}
@Override
void run(String[] args) {
try {
@ -318,12 +336,12 @@ class NiFiRegistryMode implements ToolMode {
throw new RuntimeException("Failed to configure tool, could not determine encryption key. Must provide -p, -k, or -b. If using -b, bootstrap.conf argument must already contain root key.")
}
encryptionProvider = StandardSensitivePropertyProviderFactory
.withKeyAndBootstrapSupplier(encryptionKey, ConfigEncryptionTool.getBootstrapSupplier(inputBootstrapPath))
.getProvider(oldProtectionScheme)
.withKeyAndBootstrapSupplier(encryptionKey, getBootstrapSupplier(inputBootstrapPath))
.getProvider(protectionScheme)
decryptionProvider = decryptionKey ? StandardSensitivePropertyProviderFactory
.withKeyAndBootstrapSupplier(decryptionKey, ConfigEncryptionTool.getBootstrapSupplier(inputBootstrapPath))
.getProvider(protectionScheme) : null
.withKeyAndBootstrapSupplier(decryptionKey, getBootstrapSupplier(inputBootstrapPath))
.getProvider(oldProtectionScheme) : null
if (handlingNiFiRegistryProperties) {
propertiesEncryptor = new NiFiRegistryPropertiesEncryptor(encryptionProvider, decryptionProvider)