Apache
/
nifi
mirror of https://github.com/apache/nifi.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

NIFI-12501 Encrypt MiNiFi bootstrap properties 

Signed-off-by: Ferenc Kis <briansolo1985@gmail.com>

This closes #8151.
This commit is contained in:
Ferenc Erdei 2023-12-12 13:40:05 +01:00 committed by Ferenc Kis
parent f76d7884ba
commit 5b9897cd4b
No known key found for this signature in database
GPG Key ID: 5E1CCAC15A5958F2
71 changed files with 2533 additions and 275 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
minifi/minifi-bootstrap/pom.xml
View File

@ -86,6 +86,13 @@ limitations under the License.
<artifactId>minifi-commons-api</artifactId>
<version>2.0.0-SNAPSHOT</version>
</dependency>
<!-- This needs to be here because it is relied upon by the minifi-runtime which starts MiNiFi. It uses this bootstrap module
and the libs that get laid down in lib/bootstrap to create a child classloader with these bits in it -->
<dependency>
<groupId>org.apache.nifi.minifi</groupId>
<artifactId>minifi-properties-loader</artifactId>
<version>2.0.0-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>org.apache.nifi.minifi</groupId>
<artifactId>minifi-commons-framework</artifactId>

6
minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/RunMiNiFi.java
View File

@ -48,7 +48,7 @@ import org.apache.nifi.minifi.bootstrap.util.UnixProcessUtils;
import org.apache.nifi.minifi.commons.api.MiNiFiCommandState;
import org.apache.nifi.minifi.commons.service.StandardFlowEnrichService;
import org.apache.nifi.minifi.commons.service.StandardFlowSerDeService;
import org.apache.nifi.properties.ApplicationProperties;
import org.apache.nifi.minifi.properties.BootstrapProperties;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -103,7 +103,7 @@ public class RunMiNiFi implements ConfigurationFileHolder {
bootstrapFileProvider = new BootstrapFileProvider(bootstrapConfigFile);
objectMapper = getObjectMapper();
Properties properties = bootstrapFileProvider.getStatusProperties();
Properties bootstrapProperties = bootstrapFileProvider.getBootstrapProperties();
BootstrapProperties bootstrapProperties = bootstrapFileProvider.getBootstrapProperties();
miNiFiParameters = new MiNiFiParameters(
Optional.ofNullable(properties.getProperty(STATUS_FILE_PORT_KEY)).map(Integer::parseInt).orElse(UNINITIALIZED),
@ -118,7 +118,7 @@ public class RunMiNiFi implements ConfigurationFileHolder {
periodicStatusReporterManager =
new PeriodicStatusReporterManager(bootstrapProperties, miNiFiStatusProvider, miNiFiCommandSender, miNiFiParameters);
MiNiFiConfigurationChangeListener configurationChangeListener = new MiNiFiConfigurationChangeListener(this, DEFAULT_LOGGER, bootstrapFileProvider,
new StandardFlowEnrichService(new ApplicationProperties(bootstrapProperties)), StandardFlowSerDeService.defaultInstance());
new StandardFlowEnrichService(bootstrapProperties), StandardFlowSerDeService.defaultInstance());
configurationChangeCoordinator = new ConfigurationChangeCoordinator(bootstrapFileProvider, this, singleton(configurationChangeListener));
currentPortProvider = new CurrentPortProvider(miNiFiCommandSender, miNiFiParameters, processUtils);

9
minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/command/StartRunner.java
View File

@ -55,6 +55,7 @@ import org.apache.nifi.minifi.bootstrap.service.MiNiFiPropertiesGenerator;
import org.apache.nifi.minifi.bootstrap.service.MiNiFiStdLogHandler;
import org.apache.nifi.minifi.bootstrap.service.PeriodicStatusReporterManager;
import org.apache.nifi.minifi.commons.api.MiNiFiProperties;
import org.apache.nifi.minifi.properties.BootstrapProperties;
public class StartRunner implements CommandRunner {
@ -122,10 +123,10 @@ public class StartRunner implements CommandRunner {
CMD_LOGGER.warn("Failed to delete previous lock file {}; this file should be cleaned up manually", prevLockFile);
}
Properties bootstrapProperties = bootstrapFileProvider.getBootstrapProperties();
BootstrapProperties bootstrapProperties = bootstrapFileProvider.getBootstrapProperties();
String confDir = bootstrapProperties.getProperty(CONF_DIR_KEY);
generateMiNiFiProperties(bootstrapProperties, confDir);
generateMiNiFiProperties(bootstrapFileProvider.getProtectedBootstrapProperties(), confDir);
regenerateFlowConfiguration(bootstrapProperties.getProperty(MiNiFiProperties.NIFI_MINIFI_FLOW_CONFIG.getKey()));
Process process = startMiNiFi();
@ -259,7 +260,7 @@ public class StartRunner implements CommandRunner {
}
}
private void generateMiNiFiProperties(Properties bootstrapProperties, String confDir) {
private void generateMiNiFiProperties(BootstrapProperties bootstrapProperties, String confDir) {
DEFAULT_LOGGER.debug("Generating minifi.properties from bootstrap.conf");
try {
miNiFiPropertiesGenerator.generateMinifiProperties(confDir, bootstrapProperties);
@ -317,7 +318,7 @@ public class StartRunner implements CommandRunner {
}
private File getWorkingDir() throws IOException {
Properties props = bootstrapFileProvider.getBootstrapProperties();
BootstrapProperties props = bootstrapFileProvider.getBootstrapProperties();
File bootstrapConfigAbsoluteFile = bootstrapConfigFile.getAbsoluteFile();
File binDir = bootstrapConfigAbsoluteFile.getParentFile();

14
minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/configuration/ConfigurationChangeCoordinator.java
View File

@ -17,26 +17,26 @@
package org.apache.nifi.minifi.bootstrap.configuration;
import static java.util.Optional.ofNullable;
import static java.util.function.Predicate.not;
import static java.util.stream.Collectors.toList;
import java.io.Closeable;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.Collection;
import java.util.Collections;
import java.util.HashSet;
import java.util.Properties;
import java.util.Set;
import java.util.stream.Stream;
import org.apache.nifi.minifi.bootstrap.RunMiNiFi;
import org.apache.nifi.minifi.bootstrap.configuration.ingestors.interfaces.ChangeIngestor;
import org.apache.nifi.minifi.bootstrap.service.BootstrapFileProvider;
import org.apache.nifi.minifi.bootstrap.util.ByteBufferInputStream;
import org.apache.nifi.minifi.properties.BootstrapProperties;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import static java.util.Optional.ofNullable;
import static java.util.function.Predicate.not;
import static java.util.stream.Collectors.toList;
public class ConfigurationChangeCoordinator implements Closeable, ConfigurationChangeNotifier {
public static final String NOTIFIER_INGESTORS_KEY = "nifi.minifi.notifier.ingestors";
@ -94,7 +94,7 @@ public class ConfigurationChangeCoordinator implements Closeable, ConfigurationC
private void initialize() throws IOException {
closeIngestors();
Properties bootstrapProperties = bootstrapFileProvider.getBootstrapProperties();
BootstrapProperties bootstrapProperties = bootstrapFileProvider.getBootstrapProperties();
ofNullable(bootstrapProperties.getProperty(NOTIFIER_INGESTORS_KEY))
.filter(not(String::isBlank))
.map(ingestors -> ingestors.split(COMMA))
@ -115,7 +115,7 @@ public class ConfigurationChangeCoordinator implements Closeable, ConfigurationC
}
}
private void instantiateIngestor(Properties bootstrapProperties, String ingestorClassname) {
private void instantiateIngestor(BootstrapProperties bootstrapProperties, String ingestorClassname) {
try {
Class<?> ingestorClass = Class.forName(ingestorClassname);
ChangeIngestor changeIngestor = (ChangeIngestor) ingestorClass.getDeclaredConstructor().newInstance();

6
minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/configuration/ingestors/AbstractPullChangeIngestor.java
View File

@ -18,7 +18,6 @@
package org.apache.nifi.minifi.bootstrap.configuration.ingestors;
import java.io.IOException;
import java.util.Properties;
import java.util.concurrent.ScheduledThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
@ -26,20 +25,21 @@ import java.util.concurrent.atomic.AtomicReference;
import org.apache.nifi.minifi.bootstrap.ConfigurationFileHolder;
import org.apache.nifi.minifi.bootstrap.configuration.ConfigurationChangeNotifier;
import org.apache.nifi.minifi.bootstrap.configuration.ingestors.interfaces.ChangeIngestor;
import org.apache.nifi.minifi.properties.BootstrapProperties;
public abstract class AbstractPullChangeIngestor implements Runnable, ChangeIngestor {
protected static final String DEFAULT_POLLING_PERIOD_MILLISECONDS = "300000";
protected final AtomicInteger pollingPeriodMS = new AtomicInteger();
protected final AtomicReference<Properties> properties = new AtomicReference<>();
protected final AtomicReference<BootstrapProperties> properties = new AtomicReference<>();
private final ScheduledThreadPoolExecutor scheduledThreadPoolExecutor = new ScheduledThreadPoolExecutor(1);
protected volatile ConfigurationChangeNotifier configurationChangeNotifier;
@Override
public void initialize(Properties properties, ConfigurationFileHolder configurationFileHolder, ConfigurationChangeNotifier configurationChangeNotifier) {
public void initialize(BootstrapProperties properties, ConfigurationFileHolder configurationFileHolder, ConfigurationChangeNotifier configurationChangeNotifier) {
this.configurationChangeNotifier = configurationChangeNotifier;
this.properties.set(properties);
}

6
minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/configuration/ingestors/FileChangeIngestor.java
View File

@ -39,7 +39,6 @@ import java.nio.file.WatchKey;
import java.nio.file.WatchService;
import java.util.Map;
import java.util.Optional;
import java.util.Properties;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;
@ -50,6 +49,7 @@ import org.apache.nifi.minifi.bootstrap.configuration.differentiators.Differenti
import org.apache.nifi.minifi.bootstrap.configuration.differentiators.WholeConfigDifferentiator;
import org.apache.nifi.minifi.bootstrap.configuration.ingestors.interfaces.ChangeIngestor;
import org.apache.nifi.minifi.commons.api.MiNiFiProperties;
import org.apache.nifi.minifi.properties.BootstrapProperties;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -84,7 +84,7 @@ public class FileChangeIngestor implements Runnable, ChangeIngestor {
private long pollingSeconds;
@Override
public void initialize(Properties properties, ConfigurationFileHolder configurationFileHolder, ConfigurationChangeNotifier configurationChangeNotifier) {
public void initialize(BootstrapProperties properties, ConfigurationFileHolder configurationFileHolder, ConfigurationChangeNotifier configurationChangeNotifier) {
Path configFile = ofNullable(properties.getProperty(CONFIG_FILE_PATH_KEY))
.filter(not(String::isBlank))
.map(Path::of)
@ -190,7 +190,7 @@ public class FileChangeIngestor implements Runnable, ChangeIngestor {
+ " which does not correspond to any in the FileChangeIngestor Map:" + DIFFERENTIATOR_CONSTRUCTOR_MAP.keySet());
}
private void checkConfigFileLocationCorrectness(Properties properties, Path configFile) {
private void checkConfigFileLocationCorrectness(BootstrapProperties properties, Path configFile) {
Path flowConfigFile = Path.of(properties.getProperty(MiNiFiProperties.NIFI_MINIFI_FLOW_CONFIG.getKey())).toAbsolutePath();
Path rawFlowConfigFile = flowConfigFile.getParent().resolve(getBaseName(flowConfigFile.toString()) + RAW_EXTENSION);
if (flowConfigFile.equals(configFile) || rawFlowConfigFile.equals(configFile)) {

19
minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/configuration/ingestors/PullHttpChangeIngestor.java
View File

@ -41,7 +41,6 @@ import java.nio.ByteBuffer;
import java.security.KeyStore;
import java.util.Arrays;
import java.util.Map;
import java.util.Properties;
import java.util.concurrent.atomic.AtomicReference;
import java.util.function.Supplier;
import java.util.stream.Stream;
@ -59,6 +58,7 @@ import org.apache.nifi.minifi.bootstrap.ConfigurationFileHolder;
import org.apache.nifi.minifi.bootstrap.configuration.ConfigurationChangeNotifier;
import org.apache.nifi.minifi.bootstrap.configuration.differentiators.Differentiator;
import org.apache.nifi.minifi.bootstrap.configuration.differentiators.WholeConfigDifferentiator;
import org.apache.nifi.minifi.properties.BootstrapProperties;
import org.apache.nifi.security.ssl.StandardKeyStoreBuilder;
import org.apache.nifi.security.ssl.StandardSslContextBuilder;
import org.apache.nifi.security.ssl.StandardTrustManagerBuilder;
@ -91,18 +91,18 @@ public class PullHttpChangeIngestor extends AbstractPullChangeIngestor {
public static final String OVERRIDE_SECURITY = PULL_HTTP_BASE_KEY + ".override.security";
public static final String HTTP_HEADERS = PULL_HTTP_BASE_KEY + ".headers";
protected static final String DEFAULT_CONNECT_TIMEOUT_MS = "5000";
protected static final String DEFAULT_READ_TIMEOUT_MS = "15000";
protected static final String DEFAULT_PATH = "/";
private static final Logger logger = LoggerFactory.getLogger(PullHttpChangeIngestor.class);
private static final Map<String, Supplier<Differentiator<ByteBuffer>>> DIFFERENTIATOR_CONSTRUCTOR_MAP = Map.of(
WHOLE_CONFIG_KEY, WholeConfigDifferentiator::getByteBufferDifferentiator
);
private static final int NOT_MODIFIED_STATUS_CODE = 304;
private static final String DEFAULT_CONNECT_TIMEOUT_MS = "5000";
private static final String DEFAULT_READ_TIMEOUT_MS = "15000";
private static final String DOUBLE_QUOTES = "\"";
private static final String ETAG_HEADER = "ETag";
private static final String PROXY_AUTHORIZATION_HEADER = "Proxy-Authorization";
private static final String DEFAULT_PATH = "/";
private static final int BAD_REQUEST_STATUS_CODE = 400;
private static final String IF_NONE_MATCH_HEADER_KEY = "If-None-Match";
private static final String HTTP_HEADERS_SEPARATOR = ",";
@ -121,7 +121,7 @@ public class PullHttpChangeIngestor extends AbstractPullChangeIngestor {
private volatile boolean useEtag = false;
@Override
public void initialize(Properties properties, ConfigurationFileHolder configurationFileHolder, ConfigurationChangeNotifier configurationChangeNotifier) {
public void initialize(BootstrapProperties properties, ConfigurationFileHolder configurationFileHolder, ConfigurationChangeNotifier configurationChangeNotifier) {
super.initialize(properties, configurationFileHolder, configurationChangeNotifier);
pollingPeriodMS.set(Integer.parseInt(properties.getProperty(PULL_HTTP_POLLING_PERIOD_KEY, DEFAULT_POLLING_PERIOD_MILLISECONDS)));
@ -145,8 +145,7 @@ public class PullHttpChangeIngestor extends AbstractPullChangeIngestor {
.collect(toMap(split -> ofNullable(split[0]).map(String::trim).orElse(EMPTY), split -> ofNullable(split[1]).map(String::trim).orElse(EMPTY)));
logger.debug("Configured HTTP headers: {}", httpHeaders);
ofNullable(properties.get(PORT_KEY))
.map(String.class::cast)
ofNullable(properties.getProperty(PORT_KEY))
.map(Integer::parseInt)
.ifPresentOrElse(
portReference::set,
@ -157,7 +156,7 @@ public class PullHttpChangeIngestor extends AbstractPullChangeIngestor {
pathReference.set(path);
queryReference.set(query);
httpHeadersReference.set(httpHeaders);
useEtag = parseBoolean((String) properties.getOrDefault(USE_ETAG_KEY, FALSE.toString()));
useEtag = parseBoolean(properties.getProperty(USE_ETAG_KEY, FALSE.toString()));
httpClientReference.set(null);
@ -263,7 +262,7 @@ public class PullHttpChangeIngestor extends AbstractPullChangeIngestor {
}
}
private void setSslSocketFactory(OkHttpClient.Builder okHttpClientBuilder, Properties properties) {
private void setSslSocketFactory(OkHttpClient.Builder okHttpClientBuilder, BootstrapProperties properties) {
String keystorePass = properties.getProperty(KEYSTORE_PASSWORD_KEY);
KeyStore keyStore = buildKeyStore(properties, KEYSTORE_LOCATION_KEY, KEYSTORE_PASSWORD_KEY, KEYSTORE_TYPE_KEY);
KeyStore truststore = buildKeyStore(properties, TRUSTSTORE_LOCATION_KEY, TRUSTSTORE_PASSWORD_KEY, TRUSTSTORE_TYPE_KEY);
@ -279,7 +278,7 @@ public class PullHttpChangeIngestor extends AbstractPullChangeIngestor {
okHttpClientBuilder.sslSocketFactory(sslSocketFactory, trustManager);
}
private KeyStore buildKeyStore(Properties properties, String locationKey, String passKey, String typeKey) {
private KeyStore buildKeyStore(BootstrapProperties properties, String locationKey, String passKey, String typeKey) {
String keystoreLocation = ofNullable(properties.getProperty(locationKey))
.filter(StringUtils::isNotBlank)
.orElseThrow(() -> new IllegalArgumentException(locationKey + " is null or empty"));

8
minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/configuration/ingestors/RestChangeIngestor.java
View File

@ -32,7 +32,6 @@ import java.net.URI;
import java.nio.ByteBuffer;
import java.security.KeyStore;
import java.util.Map;
import java.util.Properties;
import java.util.function.Supplier;
import javax.net.ssl.SSLContext;
import org.apache.nifi.jetty.configuration.connector.StandardServerConnectorFactory;
@ -42,6 +41,7 @@ import org.apache.nifi.minifi.bootstrap.configuration.ListenerHandleResult;
import org.apache.nifi.minifi.bootstrap.configuration.differentiators.Differentiator;
import org.apache.nifi.minifi.bootstrap.configuration.differentiators.WholeConfigDifferentiator;
import org.apache.nifi.minifi.bootstrap.configuration.ingestors.interfaces.ChangeIngestor;
import org.apache.nifi.minifi.properties.BootstrapProperties;
import org.apache.nifi.security.ssl.StandardKeyStoreBuilder;
import org.apache.nifi.security.ssl.StandardSslContextBuilder;
import org.apache.nifi.security.util.KeystoreType;
@ -100,7 +100,7 @@ public class RestChangeIngestor implements ChangeIngestor {
}
@Override
public void initialize(Properties properties, ConfigurationFileHolder configurationFileHolder, ConfigurationChangeNotifier configurationChangeNotifier) {
public void initialize(BootstrapProperties properties, ConfigurationFileHolder configurationFileHolder, ConfigurationChangeNotifier configurationChangeNotifier) {
logger.info("Initializing RestChangeIngestor");
this.differentiator = ofNullable(properties.getProperty(DIFFERENTIATOR_KEY))
.filter(not(String::isBlank))
@ -154,7 +154,7 @@ public class RestChangeIngestor implements ChangeIngestor {
return ((ServerConnector) jetty.getConnectors()[0]).getLocalPort();
}
private void createConnector(Properties properties) {
private void createConnector(BootstrapProperties properties) {
ServerConnector http = new ServerConnector(jetty);
http.setPort(Integer.parseInt(properties.getProperty(PORT_KEY, "0")));
@ -167,7 +167,7 @@ public class RestChangeIngestor implements ChangeIngestor {
logger.info("Added an http connector on the host '{}' and port '{}'", http.getHost(), http.getPort());
}
private void createSecureConnector(Properties properties) {
private void createSecureConnector(BootstrapProperties properties) {
KeyStore keyStore;
KeyStore trustStore = null;

7
minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/configuration/ingestors/interfaces/ChangeIngestor.java
View File

@ -17,14 +17,13 @@
package org.apache.nifi.minifi.bootstrap.configuration.ingestors.interfaces;
import java.io.IOException;
import org.apache.nifi.minifi.bootstrap.ConfigurationFileHolder;
import org.apache.nifi.minifi.bootstrap.configuration.ConfigurationChangeNotifier;
import java.io.IOException;
import java.util.Properties;
import org.apache.nifi.minifi.properties.BootstrapProperties;
public interface ChangeIngestor {
void initialize(Properties properties, ConfigurationFileHolder configurationFileHolder, ConfigurationChangeNotifier configurationChangeNotifier);
void initialize(BootstrapProperties properties, ConfigurationFileHolder configurationFileHolder, ConfigurationChangeNotifier configurationChangeNotifier);
void start();

24
minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/service/BootstrapFileProvider.java
View File

@ -22,7 +22,6 @@ import static org.apache.nifi.minifi.commons.api.MiNiFiConstants.BOOTSTRAP_UPDAT
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
@ -35,6 +34,8 @@ import java.util.Properties;
import java.util.Set;
import java.util.stream.Collectors;
import org.apache.commons.lang3.StringUtils;
import org.apache.nifi.minifi.properties.BootstrapProperties;
import org.apache.nifi.minifi.properties.BootstrapPropertiesLoader;
import org.apache.nifi.util.file.FileUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -61,6 +62,10 @@ public class BootstrapFileProvider {
this.bootstrapConfigFile = bootstrapConfigFile;
}
public String getBootstrapFilePath() {
return bootstrapConfigFile.getAbsolutePath();
}
public static File getBootstrapConfFile() {
File bootstrapConfigFile = Optional.ofNullable(System.getProperty(BOOTSTRAP_CONFIG_FILE_SYSTEM_PROPERTY_KEY))
.map(File::new)
@ -110,19 +115,12 @@ public class BootstrapFileProvider {
return newFile;
}
public Properties getBootstrapProperties() throws IOException {
if (!bootstrapConfigFile.exists()) {
throw new FileNotFoundException(bootstrapConfigFile.getAbsolutePath());
}
public BootstrapProperties getBootstrapProperties() {
return BootstrapPropertiesLoader.load(bootstrapConfigFile);
}
Properties bootstrapProperties = BootstrapProperties.getInstance();
try (FileInputStream fis = new FileInputStream(bootstrapConfigFile)) {
bootstrapProperties.load(fis);
}
logProperties("Bootstrap", bootstrapProperties);
return bootstrapProperties;
public BootstrapProperties getProtectedBootstrapProperties() {
return BootstrapPropertiesLoader.loadProtectedProperties(bootstrapConfigFile).getApplicationProperties();
}
public Properties getStatusProperties() {

4
minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/service/GracefulShutdownParameterProvider.java
View File

@ -18,7 +18,7 @@
package org.apache.nifi.minifi.bootstrap.service;
import java.io.IOException;
import java.util.Properties;
import org.apache.nifi.minifi.properties.BootstrapProperties;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -36,7 +36,7 @@ public class GracefulShutdownParameterProvider {
}
public int getGracefulShutdownSeconds() throws IOException {
Properties bootstrapProperties = bootstrapFileProvider.getBootstrapProperties();
BootstrapProperties bootstrapProperties = bootstrapFileProvider.getBootstrapProperties();
String gracefulShutdown = bootstrapProperties.getProperty(GRACEFUL_SHUTDOWN_PROP, DEFAULT_GRACEFUL_SHUTDOWN_VALUE);

4
minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/service/MiNiFiConfigurationChangeListener.java
View File

@ -32,7 +32,6 @@ import java.io.IOException;
import java.io.InputStream;
import java.nio.ByteBuffer;
import java.nio.file.Path;
import java.util.Properties;
import java.util.concurrent.locks.ReentrantLock;
import org.apache.commons.io.FilenameUtils;
import org.apache.nifi.controller.flow.VersionedDataflow;
@ -42,6 +41,7 @@ import org.apache.nifi.minifi.bootstrap.configuration.ConfigurationChangeListene
import org.apache.nifi.minifi.commons.api.MiNiFiProperties;
import org.apache.nifi.minifi.commons.service.FlowEnrichService;
import org.apache.nifi.minifi.commons.service.FlowSerDeService;
import org.apache.nifi.minifi.properties.BootstrapProperties;
import org.slf4j.Logger;
public class MiNiFiConfigurationChangeListener implements ConfigurationChangeListener {
@ -76,7 +76,7 @@ public class MiNiFiConfigurationChangeListener implements ConfigurationChangeLis
Path currentRawFlowConfigFile = null;
Path backupRawFlowConfigFile = null;
try {
Properties bootstrapProperties = bootstrapFileProvider.getBootstrapProperties();
BootstrapProperties bootstrapProperties = bootstrapFileProvider.getBootstrapProperties();
currentFlowConfigFile = Path.of(bootstrapProperties.getProperty(MiNiFiProperties.NIFI_MINIFI_FLOW_CONFIG.getKey())).toAbsolutePath();
backupFlowConfigFile = Path.of(currentFlowConfigFile + BACKUP_EXTENSION);

22
minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/service/MiNiFiExecCommandProvider.java
View File

@ -28,12 +28,13 @@ import java.io.IOException;
import java.nio.file.Path;
import java.util.List;
import java.util.Optional;
import java.util.Properties;
import java.util.stream.Stream;
import org.apache.nifi.minifi.properties.BootstrapProperties;
public class MiNiFiExecCommandProvider {
public static final String LOG_DIR = "org.apache.nifi.minifi.bootstrap.config.log.dir";
public static final String MINIFI_BOOTSTRAP_CONF_FILE_PATH = "minifi.bootstrap.conf.file.path";
public static final String DEFAULT_LOG_DIR = "./logs";
public static final String APP_LOG_FILE_NAME = "org.apache.nifi.minifi.bootstrap.config.log.app.file.name";
@ -73,7 +74,7 @@ public class MiNiFiExecCommandProvider {
this.bootstrapFileProvider = bootstrapFileProvider;
}
public static String getMiNiFiPropertiesPath(Properties props, File confDir) {
public static String getMiNiFiPropertiesPath(BootstrapProperties props, File confDir) {
return ofNullable(props.getProperty(PROPERTIES_FILE_KEY))
.orElseGet(() -> ofNullable(confDir)
.filter(File::exists)
@ -92,7 +93,7 @@ public class MiNiFiExecCommandProvider {
* @throws IOException throws IOException if any of the configuration file read fails
*/
public List<String> getMiNiFiExecCommand(int listenPort, File workingDir) throws IOException {
Properties bootstrapProperties = bootstrapFileProvider.getBootstrapProperties();
BootstrapProperties bootstrapProperties = bootstrapFileProvider.getBootstrapProperties();
File confDir = getFile(bootstrapProperties.getProperty(CONF_DIR_KEY, DEFAULT_CONF_DIR).trim(), workingDir);
File libDir = getFile(bootstrapProperties.getProperty(LIB_DIR_KEY, DEFAULT_LIB_DIR).trim(), workingDir);
@ -107,6 +108,7 @@ public class MiNiFiExecCommandProvider {
List<String> javaAdditionalArgs = getJavaAdditionalArgs(bootstrapProperties);
List<String> systemProperties = List.of(
systemProperty(PROPERTIES_FILE_PATH, getMiNiFiPropertiesPath(bootstrapProperties, confDir)),
systemProperty(MINIFI_BOOTSTRAP_CONF_FILE_PATH, bootstrapFileProvider.getBootstrapFilePath()),
systemProperty(NIFI_BOOTSTRAP_LISTEN_PORT, listenPort),
systemProperty(APP, MINIFI_CLASS_NAME),
systemProperty(LOG_DIR, minifiLogDir),
@ -116,8 +118,7 @@ public class MiNiFiExecCommandProvider {
systemProperty(BOOTSTRAP_LOG_FILE_EXTENSION, minifiBootstrapLogFileExtension)
);
return List.of(javaCommand, javaAdditionalArgs, systemProperties, List.of(MINIFI_FULLY_QUALIFIED_CLASS_NAME))
.stream()
return Stream.of(javaCommand, javaAdditionalArgs, systemProperties, List.of(MINIFI_FULLY_QUALIFIED_CLASS_NAME))
.flatMap(List::stream)
.toList();
}
@ -127,7 +128,7 @@ public class MiNiFiExecCommandProvider {
return file.isAbsolute() ? file : new File(workingDir, filename).getAbsoluteFile();
}
private String getJavaCommand(Properties bootstrapProperties) {
private String getJavaCommand(BootstrapProperties bootstrapProperties) {
String javaCommand = bootstrapProperties.getProperty(JAVA_COMMAND_KEY, DEFAULT_JAVA_CMD);
return javaCommand.equals(DEFAULT_JAVA_CMD)
? ofNullable(System.getenv(JAVA_HOME_ENVIRONMENT_VARIABLE))
@ -159,15 +160,16 @@ public class MiNiFiExecCommandProvider {
.collect(joining(File.pathSeparator));
}
private List<String> getJavaAdditionalArgs(Properties props) {
return props.entrySet()
private List<String> getJavaAdditionalArgs(BootstrapProperties props) {
return props.getPropertyKeys()
.stream()
.filter(entry -> ((String) entry.getKey()).startsWith(JAVA_ARG_KEY_PREFIX))
.map(entry -> (String) entry.getValue())
.filter(key -> key.startsWith(JAVA_ARG_KEY_PREFIX))
.map(props::getProperty)
.toList();
}
private String systemProperty(String key, Object value) {
return String.format(SYSTEM_PROPERTY_TEMPLATE, key, value);
}
}

44
minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/service/MiNiFiPropertiesGenerator.java
View File

@ -19,6 +19,7 @@ package org.apache.nifi.minifi.bootstrap.service;
import static java.lang.String.join;
import static java.lang.System.getProperty;
import static java.util.Map.entry;
import static java.util.Optional.ofNullable;
import static org.apache.commons.lang3.StringUtils.EMPTY;
import static org.apache.commons.lang3.StringUtils.isNotBlank;
@ -59,6 +60,7 @@ import org.apache.commons.lang3.tuple.Triple;
import org.apache.nifi.minifi.bootstrap.configuration.ConfigurationChangeException;
import org.apache.nifi.minifi.bootstrap.util.OrderedProperties;
import org.apache.nifi.minifi.commons.api.MiNiFiProperties;
import org.apache.nifi.minifi.properties.BootstrapProperties;
import org.apache.nifi.util.NiFiProperties;
public class MiNiFiPropertiesGenerator {
@ -139,15 +141,19 @@ public class MiNiFiPropertiesGenerator {
Triple.of(NiFiProperties.FLOW_CONFIGURATION_FILE, "./conf/flow.json.gz", EMPTY)
);
static final Map<String, String> MINIFI_TO_NIFI_PROPERTY_MAPPING = Map.of(
MiNiFiProperties.NIFI_MINIFI_FLOW_CONFIG.getKey(), NiFiProperties.FLOW_CONFIGURATION_FILE,
MiNiFiProperties.NIFI_MINIFI_SECURITY_KEYSTORE.getKey(), NiFiProperties.SECURITY_KEYSTORE,
MiNiFiProperties.NIFI_MINIFI_SECURITY_KEYSTORE_TYPE.getKey(), NiFiProperties.SECURITY_KEYSTORE_TYPE,
MiNiFiProperties.NIFI_MINIFI_SECURITY_KEYSTORE_PASSWD.getKey(), NiFiProperties.SECURITY_KEYSTORE_PASSWD,
MiNiFiProperties.NIFI_MINIFI_SECURITY_KEY_PASSWD.getKey(), NiFiProperties.SECURITY_KEY_PASSWD,
MiNiFiProperties.NIFI_MINIFI_SECURITY_TRUSTSTORE.getKey(), NiFiProperties.SECURITY_TRUSTSTORE,
MiNiFiProperties.NIFI_MINIFI_SECURITY_TRUSTSTORE_TYPE.getKey(), NiFiProperties.SECURITY_TRUSTSTORE_TYPE,
MiNiFiProperties.NIFI_MINIFI_SECURITY_TRUSTSTORE_PASSWD.getKey(), NiFiProperties.SECURITY_TRUSTSTORE_PASSWD
static final String PROTECTED_POSTFIX = ".protected";
static final Map<String, String> MINIFI_TO_NIFI_PROPERTY_MAPPING = Map.ofEntries(
entry(MiNiFiProperties.NIFI_MINIFI_FLOW_CONFIG.getKey(), NiFiProperties.FLOW_CONFIGURATION_FILE),
entry(MiNiFiProperties.NIFI_MINIFI_SECURITY_KEYSTORE.getKey(), NiFiProperties.SECURITY_KEYSTORE),
entry(MiNiFiProperties.NIFI_MINIFI_SECURITY_KEYSTORE_TYPE.getKey(), NiFiProperties.SECURITY_KEYSTORE_TYPE),
entry(MiNiFiProperties.NIFI_MINIFI_SECURITY_KEYSTORE_PASSWD.getKey(), NiFiProperties.SECURITY_KEYSTORE_PASSWD),
entry(MiNiFiProperties.NIFI_MINIFI_SECURITY_KEYSTORE_PASSWD.getKey() + PROTECTED_POSTFIX, NiFiProperties.SECURITY_KEYSTORE_PASSWD + PROTECTED_POSTFIX),
entry(MiNiFiProperties.NIFI_MINIFI_SECURITY_KEY_PASSWD.getKey(), NiFiProperties.SECURITY_KEY_PASSWD),
entry(MiNiFiProperties.NIFI_MINIFI_SECURITY_KEY_PASSWD.getKey() + PROTECTED_POSTFIX, NiFiProperties.SECURITY_KEY_PASSWD + PROTECTED_POSTFIX),
entry(MiNiFiProperties.NIFI_MINIFI_SECURITY_TRUSTSTORE.getKey(), NiFiProperties.SECURITY_TRUSTSTORE),
entry(MiNiFiProperties.NIFI_MINIFI_SECURITY_TRUSTSTORE_TYPE.getKey(), NiFiProperties.SECURITY_TRUSTSTORE_TYPE),
entry(MiNiFiProperties.NIFI_MINIFI_SECURITY_TRUSTSTORE_PASSWD.getKey(), NiFiProperties.SECURITY_TRUSTSTORE_PASSWD),
entry(MiNiFiProperties.NIFI_MINIFI_SECURITY_TRUSTSTORE_PASSWD.getKey() + PROTECTED_POSTFIX, NiFiProperties.SECURITY_TRUSTSTORE_PASSWD + PROTECTED_POSTFIX)
);
static final String DEFAULT_SENSITIVE_PROPERTIES_ENCODING_ALGORITHM = "NIFI_PBKDF2_AES_GCM_256";
@ -160,7 +166,7 @@ public class MiNiFiPropertiesGenerator {
public static final String FILE_EXTENSION_DELIMITER = ".";
public void generateMinifiProperties(String configDirectory, Properties bootstrapProperties) throws ConfigurationChangeException {
public void generateMinifiProperties(String configDirectory, BootstrapProperties bootstrapProperties) throws ConfigurationChangeException {
String minifiPropertiesFileName = Path.of(getMiNiFiPropertiesPath(bootstrapProperties, new File(configDirectory))).getFileName().toString();
Path minifiPropertiesFile = Path.of(configDirectory, minifiPropertiesFileName);
@ -188,22 +194,22 @@ public class MiNiFiPropertiesGenerator {
);
}
private OrderedProperties prepareMinifiProperties(Properties bootstrapProperties, Map<String, String> existingSensitivePropertiesConfiguration) {
private OrderedProperties prepareMinifiProperties(BootstrapProperties bootstrapProperties, Map<String, String> existingSensitivePropertiesConfiguration) {
OrderedProperties minifiProperties = new OrderedProperties();
NIFI_PROPERTIES_WITH_DEFAULT_VALUES_AND_COMMENTS
.forEach(triple -> minifiProperties.setProperty(triple.getLeft(), triple.getMiddle(), triple.getRight()));
getNonBlankPropertiesWithPredicate(bootstrapProperties, entry -> MINIFI_TO_NIFI_PROPERTY_MAPPING.containsKey(entry.getKey()))
getNonBlankPropertiesWithPredicate(bootstrapProperties, MINIFI_TO_NIFI_PROPERTY_MAPPING::containsKey)
.forEach(entry -> minifiProperties.setProperty(MINIFI_TO_NIFI_PROPERTY_MAPPING.get(entry.getKey()), entry.getValue()));
getSensitiveProperties(bootstrapProperties, existingSensitivePropertiesConfiguration)
.forEach(entry -> minifiProperties.setProperty(entry.getKey(), entry.getValue()));
getNonBlankPropertiesWithPredicate(bootstrapProperties, entry -> ((String) entry.getKey()).startsWith(C2_PROPERTY_PREFIX))
getNonBlankPropertiesWithPredicate(bootstrapProperties, key -> key.startsWith(C2_PROPERTY_PREFIX))
.forEach(entry -> minifiProperties.setProperty(entry.getKey(), entry.getValue()));
getNonBlankPropertiesWithPredicate(bootstrapProperties, entry -> ((String) entry.getKey()).startsWith(NIFI_PREFIX))
getNonBlankPropertiesWithPredicate(bootstrapProperties, key -> key.startsWith(NIFI_PREFIX))
.forEach(entry -> minifiProperties.setProperty(entry.getKey(), entry.getValue()));
bootstrapFileAndLogProperties()
@ -212,19 +218,19 @@ public class MiNiFiPropertiesGenerator {
return minifiProperties;
}
private List<Pair<String, String>> getNonBlankPropertiesWithPredicate(Properties bootstrapProperties, Predicate<Map.Entry> predicate) {
private List<Pair<String, String>> getNonBlankPropertiesWithPredicate(BootstrapProperties bootstrapProperties, Predicate<String> predicate) {
return ofNullable(bootstrapProperties)
.map(Properties::entrySet)
.map(BootstrapProperties::getPropertyKeys)
.orElseGet(Set::of)
.stream()
.filter(predicate)
.filter(entry -> isNotBlank((String) entry.getValue()))
.map(entry -> Pair.of((String) entry.getKey(), (String) entry.getValue()))
.map(key -> Pair.of(key, bootstrapProperties.getProperty(key)))
.filter(pair -> isNotBlank(pair.getValue()))
.sorted((o1, o2) -> Comparator.<String>naturalOrder().compare(o1.getKey(), o2.getKey()))
.toList();
}
private List<Pair<String, String>> getSensitiveProperties(Properties bootstrapProperties, Map<String, String> existingSensitivePropertiesConfiguration) {
private List<Pair<String, String>> getSensitiveProperties(BootstrapProperties bootstrapProperties, Map<String, String> existingSensitivePropertiesConfiguration) {
return existingSensitivePropertiesConfiguration.isEmpty()
? List.of(
Pair.of(NiFiProperties.SENSITIVE_PROPS_KEY,

10
minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/service/PeriodicStatusReporterManager.java
View File

@ -17,35 +17,35 @@
package org.apache.nifi.minifi.bootstrap.service;
import static org.apache.nifi.minifi.commons.api.MiNiFiProperties.NIFI_MINIFI_STATUS_REPORTER_COMPONENTS;
import java.lang.reflect.InvocationTargetException;
import java.util.Collections;
import java.util.HashSet;
import java.util.LinkedList;
import java.util.List;
import java.util.Properties;
import java.util.Set;
import org.apache.nifi.minifi.bootstrap.MiNiFiParameters;
import org.apache.nifi.minifi.bootstrap.MiNiFiStatus;
import org.apache.nifi.minifi.bootstrap.QueryableStatusAggregator;
import org.apache.nifi.minifi.bootstrap.status.PeriodicStatusReporter;
import org.apache.nifi.minifi.commons.status.FlowStatusReport;
import org.apache.nifi.minifi.properties.BootstrapProperties;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import static org.apache.nifi.minifi.commons.api.MiNiFiProperties.NIFI_MINIFI_STATUS_REPORTER_COMPONENTS;
public class PeriodicStatusReporterManager implements QueryableStatusAggregator {
private static final Logger LOGGER = LoggerFactory.getLogger(PeriodicStatusReporterManager.class);
private static final String FLOW_STATUS_REPORT_CMD = "FLOW_STATUS_REPORT";
private final Properties bootstrapProperties;
private final BootstrapProperties bootstrapProperties;
private final MiNiFiStatusProvider miNiFiStatusProvider;
private final MiNiFiCommandSender miNiFiCommandSender;
private final MiNiFiParameters miNiFiParameters;
private Set<PeriodicStatusReporter> periodicStatusReporters = Collections.emptySet();
public PeriodicStatusReporterManager(Properties bootstrapProperties, MiNiFiStatusProvider miNiFiStatusProvider, MiNiFiCommandSender miNiFiCommandSender,
public PeriodicStatusReporterManager(BootstrapProperties bootstrapProperties, MiNiFiStatusProvider miNiFiStatusProvider, MiNiFiCommandSender miNiFiCommandSender,
MiNiFiParameters miNiFiParameters) {
this.bootstrapProperties = bootstrapProperties;
this.miNiFiStatusProvider = miNiFiStatusProvider;

6
minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/service/UpdatePropertiesService.java
View File

@ -25,10 +25,10 @@ import java.io.FileInputStream;
import java.io.IOException;
import java.nio.file.Files;
import java.util.Optional;
import java.util.Properties;
import org.apache.nifi.minifi.bootstrap.RunMiNiFi;
import org.apache.nifi.minifi.bootstrap.configuration.ConfigurationChangeException;
import org.apache.nifi.minifi.commons.api.MiNiFiCommandState;
import org.apache.nifi.minifi.properties.BootstrapProperties;
import org.slf4j.Logger;
public class UpdatePropertiesService {
@ -59,7 +59,7 @@ public class UpdatePropertiesService {
Files.copy(bootstrapFileProvider.getBootstrapConfNewFile().toPath(), bootstrapConfigFile.toPath(), REPLACE_EXISTING);
// already from new
commandState = generateConfigfilesBasedOnNewProperties(bootstrapConfigFile, bootstrapSwapConfigFile, bootstrapFileProvider.getBootstrapProperties());
commandState = generateConfigfilesBasedOnNewProperties(bootstrapConfigFile, bootstrapSwapConfigFile, bootstrapFileProvider.getProtectedBootstrapProperties());
} catch (Exception e) {
commandState = Optional.of(MiNiFiCommandState.NOT_APPLIED_WITHOUT_RESTART);
logger.error("Failed to load new bootstrap properties", e);
@ -67,7 +67,7 @@ public class UpdatePropertiesService {
return commandState;
}
private Optional<MiNiFiCommandState> generateConfigfilesBasedOnNewProperties(File bootstrapConfigFile, File bootstrapSwapConfigFile, Properties bootstrapProperties)
private Optional<MiNiFiCommandState> generateConfigfilesBasedOnNewProperties(File bootstrapConfigFile, File bootstrapSwapConfigFile, BootstrapProperties bootstrapProperties)
throws IOException, ConfigurationChangeException {
Optional<MiNiFiCommandState> commandState = Optional.empty();
try {

7
minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/status/PeriodicStatusReporter.java
View File

@ -17,12 +17,11 @@
package org.apache.nifi.minifi.bootstrap.status;
import org.apache.nifi.minifi.bootstrap.QueryableStatusAggregator;
import java.util.Properties;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.ScheduledThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
import org.apache.nifi.minifi.bootstrap.QueryableStatusAggregator;
import org.apache.nifi.minifi.properties.BootstrapProperties;
public abstract class PeriodicStatusReporter {
@ -38,7 +37,7 @@ public abstract class PeriodicStatusReporter {
*
* @param properties from the bootstrap configuration
*/
public abstract void initialize(Properties properties, QueryableStatusAggregator queryableStatusAggregator);
public abstract void initialize(BootstrapProperties properties, QueryableStatusAggregator queryableStatusAggregator);
/**
* Begins the associated reporting service provided by the given implementation. In most implementations, no action will occur until this method is invoked. The implementing class must have set

4
minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/status/reporters/StatusLogger.java
View File

@ -22,11 +22,11 @@ import static org.apache.nifi.minifi.commons.api.MiNiFiProperties.NIFI_MINIFI_ST
import static org.apache.nifi.minifi.commons.api.MiNiFiProperties.NIFI_MINIFI_STATUS_REPORTER_LOG_QUERY;
import java.io.IOException;
import java.util.Properties;
import org.apache.nifi.logging.LogLevel;
import org.apache.nifi.minifi.bootstrap.QueryableStatusAggregator;
import org.apache.nifi.minifi.bootstrap.status.PeriodicStatusReporter;
import org.apache.nifi.minifi.commons.status.FlowStatusReport;
import org.apache.nifi.minifi.properties.BootstrapProperties;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -41,7 +41,7 @@ public class StatusLogger extends PeriodicStatusReporter {
static final String ENCOUNTERED_IO_EXCEPTION = "Encountered an IO Exception while attempting to query the flow status.";
@Override
public void initialize(Properties properties, QueryableStatusAggregator queryableStatusAggregator) {
public void initialize(BootstrapProperties properties, QueryableStatusAggregator queryableStatusAggregator) {
this.queryableStatusAggregator = queryableStatusAggregator;
String periodString = properties.getProperty(NIFI_MINIFI_STATUS_REPORTER_LOG_PERIOD.getKey());

24
minifi/minifi-bootstrap/src/test/java/org/apache/nifi/minifi/bootstrap/configuration/ingestors/FileChangeIngestorTest.java
View File

@ -18,8 +18,11 @@
package org.apache.nifi.minifi.bootstrap.configuration.ingestors;
import static java.nio.file.StandardWatchEventKinds.ENTRY_MODIFY;
import static org.apache.nifi.minifi.bootstrap.configuration.ingestors.FileChangeIngestor.DEFAULT_POLLING_PERIOD_INTERVAL;
import static org.apache.nifi.minifi.bootstrap.configuration.ingestors.PullHttpChangeIngestor.PULL_HTTP_BASE_KEY;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
@ -31,12 +34,12 @@ import java.nio.file.WatchEvent;
import java.nio.file.WatchKey;
import java.nio.file.WatchService;
import java.util.Collections;
import java.util.Properties;
import java.util.concurrent.atomic.AtomicReference;
import org.apache.nifi.minifi.bootstrap.ConfigurationFileHolder;
import org.apache.nifi.minifi.bootstrap.configuration.ConfigurationChangeNotifier;
import org.apache.nifi.minifi.bootstrap.configuration.differentiators.Differentiator;
import org.apache.nifi.minifi.commons.api.MiNiFiProperties;
import org.apache.nifi.minifi.properties.BootstrapProperties;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
@ -49,7 +52,7 @@ public class FileChangeIngestorTest {
private FileChangeIngestor notifierSpy;
private WatchService mockWatchService;
private Properties testProperties;
private BootstrapProperties testProperties;
private Differentiator<ByteBuffer> mockDifferentiator;
private ConfigurationChangeNotifier testNotifier;
@ -60,15 +63,15 @@ public class FileChangeIngestorTest {
notifierSpy = Mockito.spy(new FileChangeIngestor());
mockDifferentiator = mock(Differentiator.class);
testNotifier = mock(ConfigurationChangeNotifier.class);
testProperties = mock(BootstrapProperties.class);
setMocks();
testProperties = new Properties();
testProperties.put(FileChangeIngestor.CONFIG_FILE_PATH_KEY, TEST_CONFIG_PATH);
testProperties.put(PullHttpChangeIngestor.OVERRIDE_SECURITY, "true");
testProperties.put(PULL_HTTP_BASE_KEY + ".override.core", "true");
testProperties.put(FileChangeIngestor.POLLING_PERIOD_INTERVAL_KEY, FileChangeIngestor.DEFAULT_POLLING_PERIOD_INTERVAL);
testProperties.put(MiNiFiProperties.NIFI_MINIFI_FLOW_CONFIG.getKey(), MiNiFiProperties.NIFI_MINIFI_FLOW_CONFIG.getDefaultValue());
when(testProperties.getProperty(FileChangeIngestor.CONFIG_FILE_PATH_KEY)).thenReturn(TEST_CONFIG_PATH);
when(testProperties.getProperty(PullHttpChangeIngestor.OVERRIDE_SECURITY)).thenReturn("true");
when(testProperties.getProperty(PULL_HTTP_BASE_KEY + ".override.core")).thenReturn("true");
when(testProperties.getProperty(eq(FileChangeIngestor.POLLING_PERIOD_INTERVAL_KEY), any())).thenReturn(String.valueOf(DEFAULT_POLLING_PERIOD_INTERVAL));
when(testProperties.getProperty(MiNiFiProperties.NIFI_MINIFI_FLOW_CONFIG.getKey())).thenReturn(MiNiFiProperties.NIFI_MINIFI_FLOW_CONFIG.getDefaultValue());
}
@AfterEach
@ -78,7 +81,7 @@ public class FileChangeIngestorTest {
@Test
public void testInitializeInvalidFile() {
testProperties.put(FileChangeIngestor.CONFIG_FILE_PATH_KEY, "/land/of/make/believe");
when(testProperties.getProperty(FileChangeIngestor.CONFIG_FILE_PATH_KEY)).thenReturn("/land/of/make/believe");
assertThrows(IllegalStateException.class, () -> notifierSpy.initialize(testProperties, mock(ConfigurationFileHolder.class), mock(ConfigurationChangeNotifier.class)));
}
@ -89,13 +92,12 @@ public class FileChangeIngestorTest {
@Test
public void testInitializeInvalidPollingPeriod() {
testProperties.put(FileChangeIngestor.POLLING_PERIOD_INTERVAL_KEY, "abc");
when(testProperties.getProperty(eq(FileChangeIngestor.POLLING_PERIOD_INTERVAL_KEY), any())).thenReturn("abc");
assertThrows(IllegalStateException.class, () -> notifierSpy.initialize(testProperties, mock(ConfigurationFileHolder.class), mock(ConfigurationChangeNotifier.class)));
}
@Test
public void testInitializeUseDefaultPolling() {
testProperties.remove(FileChangeIngestor.POLLING_PERIOD_INTERVAL_KEY);
notifierSpy.initialize(testProperties, mock(ConfigurationFileHolder.class), mock(ConfigurationChangeNotifier.class));
}

25
minifi/minifi-bootstrap/src/test/java/org/apache/nifi/minifi/bootstrap/configuration/ingestors/RestChangeIngestorSSLTest.java
View File

@ -25,7 +25,8 @@ import java.io.IOException;
import java.nio.ByteBuffer;
import java.security.KeyStore;
import java.util.Collections;
import java.util.Properties;
import java.util.HashMap;
import java.util.Map;
import java.util.concurrent.atomic.AtomicReference;
import javax.net.ssl.SSLContext;
import javax.net.ssl.SSLSocketFactory;
@ -35,6 +36,7 @@ import org.apache.nifi.minifi.bootstrap.ConfigurationFileHolder;
import org.apache.nifi.minifi.bootstrap.configuration.ConfigurationChangeListener;
import org.apache.nifi.minifi.bootstrap.configuration.ConfigurationChangeNotifier;
import org.apache.nifi.minifi.bootstrap.configuration.ListenerHandleResult;
import org.apache.nifi.minifi.properties.BootstrapProperties;
import org.apache.nifi.security.ssl.StandardKeyStoreBuilder;
import org.apache.nifi.security.ssl.StandardSslContextBuilder;
import org.apache.nifi.security.ssl.StandardTrustManagerBuilder;
@ -50,16 +52,17 @@ public class RestChangeIngestorSSLTest extends RestChangeIngestorCommonTest {
public static void setUpHttps() throws IOException, InterruptedException {
final TlsConfiguration tlsConfiguration = new TemporaryKeyStoreBuilder().trustStoreType("JKS").build();
Properties properties = new Properties();
properties.setProperty(RestChangeIngestor.TRUSTSTORE_LOCATION_KEY, tlsConfiguration.getTruststorePath());
properties.setProperty(RestChangeIngestor.TRUSTSTORE_PASSWORD_KEY, tlsConfiguration.getTruststorePassword());
properties.setProperty(RestChangeIngestor.TRUSTSTORE_TYPE_KEY, tlsConfiguration.getTruststoreType().getType());
properties.setProperty(RestChangeIngestor.KEYSTORE_LOCATION_KEY, tlsConfiguration.getKeystorePath());
properties.setProperty(RestChangeIngestor.KEYSTORE_PASSWORD_KEY, tlsConfiguration.getKeystorePassword());
properties.setProperty(RestChangeIngestor.KEYSTORE_TYPE_KEY, tlsConfiguration.getKeystoreType().getType());
properties.setProperty(RestChangeIngestor.NEED_CLIENT_AUTH_KEY, "false");
properties.put(PullHttpChangeIngestor.OVERRIDE_SECURITY, "true");
properties.put(PULL_HTTP_BASE_KEY + ".override.core", "true");
Map<String, String> bootstrapProperties = new HashMap<>();
bootstrapProperties.put(RestChangeIngestor.TRUSTSTORE_LOCATION_KEY, tlsConfiguration.getTruststorePath());
bootstrapProperties.put(RestChangeIngestor.TRUSTSTORE_PASSWORD_KEY, tlsConfiguration.getTruststorePassword());
bootstrapProperties.put(RestChangeIngestor.TRUSTSTORE_TYPE_KEY, tlsConfiguration.getTruststoreType().getType());
bootstrapProperties.put(RestChangeIngestor.KEYSTORE_LOCATION_KEY, tlsConfiguration.getKeystorePath());
bootstrapProperties.put(RestChangeIngestor.KEYSTORE_PASSWORD_KEY, tlsConfiguration.getKeystorePassword());
bootstrapProperties.put(RestChangeIngestor.KEYSTORE_TYPE_KEY, tlsConfiguration.getKeystoreType().getType());
bootstrapProperties.put(RestChangeIngestor.NEED_CLIENT_AUTH_KEY, "false");
bootstrapProperties.put(PullHttpChangeIngestor.OVERRIDE_SECURITY, "true");
bootstrapProperties.put(PULL_HTTP_BASE_KEY + ".override.core", "true");
BootstrapProperties properties = new BootstrapProperties(bootstrapProperties);
restChangeIngestor = new RestChangeIngestor();

15
minifi/minifi-bootstrap/src/test/java/org/apache/nifi/minifi/bootstrap/configuration/ingestors/RestChangeIngestorTest.java
View File

@ -18,15 +18,20 @@
package org.apache.nifi.minifi.bootstrap.configuration.ingestors;
import static org.apache.nifi.minifi.bootstrap.configuration.ingestors.PullHttpChangeIngestor.PULL_HTTP_BASE_KEY;
import static org.apache.nifi.minifi.bootstrap.configuration.ingestors.RestChangeIngestor.HOST_KEY;
import static org.apache.nifi.minifi.bootstrap.configuration.ingestors.RestChangeIngestor.PORT_KEY;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
import java.net.MalformedURLException;
import java.nio.ByteBuffer;
import java.util.Properties;
import java.util.concurrent.atomic.AtomicReference;
import okhttp3.OkHttpClient;
import org.apache.nifi.minifi.bootstrap.ConfigurationFileHolder;
import org.apache.nifi.minifi.bootstrap.configuration.ConfigurationChangeNotifier;
import org.apache.nifi.minifi.properties.BootstrapProperties;
import org.junit.jupiter.api.AfterAll;
import org.junit.jupiter.api.BeforeAll;
import org.mockito.Mockito;
@ -35,9 +40,11 @@ public class RestChangeIngestorTest extends RestChangeIngestorCommonTest {
@BeforeAll
public static void setUp() throws InterruptedException, MalformedURLException {
Properties properties = new Properties();
properties.put(PullHttpChangeIngestor.OVERRIDE_SECURITY, "true");
properties.put(PULL_HTTP_BASE_KEY + ".override.core", "true");
BootstrapProperties properties = mock(BootstrapProperties.class);
when(properties.getProperty(PullHttpChangeIngestor.OVERRIDE_SECURITY)).thenReturn("true");
when(properties.getProperty(PULL_HTTP_BASE_KEY + ".override.core")).thenReturn("true");
when(properties.getProperty(eq(PORT_KEY), any())).thenReturn("0");
when(properties.getProperty(eq(HOST_KEY), any())).thenReturn("localhost");
restChangeIngestor = new RestChangeIngestor();
testNotifier = Mockito.mock(ConfigurationChangeNotifier.class);

19
minifi/minifi-bootstrap/src/test/java/org/apache/nifi/minifi/bootstrap/service/GracefulShutdownParameterProviderTest.java
View File

@ -20,10 +20,14 @@ package org.apache.nifi.minifi.bootstrap.service;
import static org.apache.nifi.minifi.bootstrap.service.GracefulShutdownParameterProvider.DEFAULT_GRACEFUL_SHUTDOWN_VALUE;
import static org.apache.nifi.minifi.bootstrap.service.GracefulShutdownParameterProvider.GRACEFUL_SHUTDOWN_PROP;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
import java.io.IOException;
import java.util.Properties;
import java.util.Optional;
import org.apache.nifi.minifi.properties.BootstrapProperties;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.junit.jupiter.params.ParameterizedTest;
@ -46,10 +50,11 @@ class GracefulShutdownParameterProviderTest {
@NullSource
@ValueSource(strings = {"notAnInteger", "-1"})
void testGetBootstrapPropertiesShouldReturnDefaultShutdownPropertyValue(String shutdownProperty) throws IOException {
Properties properties = new Properties();
if (shutdownProperty != null) {
properties.setProperty(GRACEFUL_SHUTDOWN_PROP, shutdownProperty);
}
BootstrapProperties properties = mock(BootstrapProperties.class);
when(properties.getProperty(eq(GRACEFUL_SHUTDOWN_PROP), any()))
.thenReturn(Optional.ofNullable(shutdownProperty).orElse(DEFAULT_GRACEFUL_SHUTDOWN_VALUE));
when(bootstrapFileProvider.getBootstrapProperties()).thenReturn(properties);
assertEquals(Integer.parseInt(DEFAULT_GRACEFUL_SHUTDOWN_VALUE), gracefulShutdownParameterProvider.getGracefulShutdownSeconds());
@ -57,8 +62,8 @@ class GracefulShutdownParameterProviderTest {
@Test
void testGetBootstrapPropertiesShouldReturnShutdownPropertyValue() throws IOException {
Properties properties = new Properties();
properties.setProperty(GRACEFUL_SHUTDOWN_PROP, "1000");
BootstrapProperties properties = mock(BootstrapProperties.class);
when(properties.getProperty(eq(GRACEFUL_SHUTDOWN_PROP), any())).thenReturn("1000");
when(bootstrapFileProvider.getBootstrapProperties()).thenReturn(properties);
assertEquals(1000, gracefulShutdownParameterProvider.getGracefulShutdownSeconds());

22
minifi/minifi-bootstrap/src/test/java/org/apache/nifi/minifi/bootstrap/service/MiNiFiPropertiesGeneratorTest.java
View File

@ -53,6 +53,7 @@ import java.util.stream.Stream;
import org.apache.commons.lang3.StringUtils;
import org.apache.nifi.minifi.bootstrap.configuration.ConfigurationChangeException;
import org.apache.nifi.minifi.commons.api.MiNiFiProperties;
import org.apache.nifi.minifi.properties.BootstrapProperties;
import org.apache.nifi.util.NiFiProperties;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
@ -82,7 +83,7 @@ public class MiNiFiPropertiesGeneratorTest {
@Test
public void testGenerateDefaultNiFiProperties() throws ConfigurationChangeException {
// given
Properties bootstrapProperties = createBootstrapProperties(Map.of());
BootstrapProperties bootstrapProperties = createBootstrapProperties(Map.of());
// when
testPropertiesGenerator.generateMinifiProperties(configDirectory.toString(), bootstrapProperties);
@ -101,7 +102,7 @@ public class MiNiFiPropertiesGeneratorTest {
@Test
public void testMiNiFiPropertiesMappedToAppropriateNiFiProperties() throws ConfigurationChangeException {
// given
Properties bootstrapProperties = createBootstrapProperties(Stream.of(
BootstrapProperties bootstrapProperties = createBootstrapProperties(Stream.of(
MiNiFiProperties.NIFI_MINIFI_FLOW_CONFIG.getKey(),
MiNiFiProperties.NIFI_MINIFI_SECURITY_KEYSTORE.getKey(),
MiNiFiProperties.NIFI_MINIFI_SECURITY_KEYSTORE_TYPE.getKey(),
@ -126,7 +127,7 @@ public class MiNiFiPropertiesGeneratorTest {
@Test
public void testSensitivePropertiesAreGeneratedWhenNotProvidedInBootstrap() throws ConfigurationChangeException {
// given
Properties bootstrapProperties = createBootstrapProperties(Map.of());
BootstrapProperties bootstrapProperties = createBootstrapProperties(Map.of());
// when
testPropertiesGenerator.generateMinifiProperties(configDirectory.toString(), bootstrapProperties);
@ -142,7 +143,7 @@ public class MiNiFiPropertiesGeneratorTest {
// given
String sensitivePropertiesKey = "sensitive_properties_key";
String sensitivePropertiesAlgorithm = "sensitive_properties_algorithm";
Properties bootstrapProperties = createBootstrapProperties(Map.of(
BootstrapProperties bootstrapProperties = createBootstrapProperties(Map.of(
MiNiFiProperties.NIFI_MINIFI_SENSITIVE_PROPS_KEY.getKey(), sensitivePropertiesKey,
MiNiFiProperties.NIFI_MINIFI_SENSITIVE_PROPS_ALGORITHM.getKey(), sensitivePropertiesAlgorithm
));
@ -159,7 +160,7 @@ public class MiNiFiPropertiesGeneratorTest {
@Test
public void testNonBlankC2PropertiesAreCopiedToMiNiFiProperties() throws ConfigurationChangeException {
// given
Properties bootstrapProperties = createBootstrapProperties(Map.of(
BootstrapProperties bootstrapProperties = createBootstrapProperties(Map.of(
MiNiFiProperties.C2_ENABLE.getKey(), TRUE.toString(),
MiNiFiProperties.C2_AGENT_CLASS.getKey(), EMPTY,
MiNiFiProperties.C2_AGENT_IDENTIFIER.getKey(), SPACE
@ -180,7 +181,7 @@ public class MiNiFiPropertiesGeneratorTest {
public void testDefaultNiFiPropertiesAreOverridden() throws ConfigurationChangeException {
// given
String archiveDir = "/path/to";
Properties bootstrapProperties = createBootstrapProperties(Map.of(
BootstrapProperties bootstrapProperties = createBootstrapProperties(Map.of(
NiFiProperties.FLOW_CONFIGURATION_ARCHIVE_ENABLED, TRUE.toString(),
NiFiProperties.FLOW_CONFIGURATION_ARCHIVE_DIR, archiveDir
));
@ -211,7 +212,7 @@ public class MiNiFiPropertiesGeneratorTest {
@Test
public void testArbitraryNiFiPropertyCanBePassedViaBootstrapConf() throws ConfigurationChangeException {
// given
Properties bootstrapProperties = createBootstrapProperties(Map.of(
BootstrapProperties bootstrapProperties = createBootstrapProperties(Map.of(
"nifi.new.property", "new_property_value",
"nifi.other.new.property", "other_new_property_value"
));
@ -229,7 +230,7 @@ public class MiNiFiPropertiesGeneratorTest {
@Test
public void bootstrapFileAndLogPropertiesAreGeneratedIntoMiNiFiProperties() throws ConfigurationChangeException {
// given
Properties bootstrapProperties = createBootstrapProperties(Map.of());
BootstrapProperties bootstrapProperties = createBootstrapProperties(Map.of());
// when
testPropertiesGenerator.generateMinifiProperties(configDirectory.toString(), bootstrapProperties);
@ -243,12 +244,13 @@ public class MiNiFiPropertiesGeneratorTest {
);
}
private Properties createBootstrapProperties(Map<String, String> keyValues) {
private BootstrapProperties createBootstrapProperties(Map<String, String> keyValues) {
try (OutputStream outputStream = newOutputStream(bootstrapPropertiesFile)) {
Properties properties = new Properties();
BootstrapProperties bootstrapProperties = new BootstrapProperties(keyValues);
properties.putAll(keyValues);
properties.store(outputStream, EMPTY);
return properties;
return bootstrapProperties;
} catch (IOException e) {
throw new UncheckedIOException(e);
}

41
minifi/minifi-bootstrap/src/test/java/org/apache/nifi/minifi/bootstrap/status/reporters/StatusLoggerTest.java
View File

@ -22,16 +22,18 @@ import static org.apache.nifi.minifi.commons.api.MiNiFiProperties.NIFI_MINIFI_ST
import static org.apache.nifi.minifi.commons.api.MiNiFiProperties.NIFI_MINIFI_STATUS_REPORTER_LOG_PERIOD;
import static org.apache.nifi.minifi.commons.api.MiNiFiProperties.NIFI_MINIFI_STATUS_REPORTER_LOG_QUERY;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.mockito.BDDMockito.given;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.verify;
import java.io.IOException;
import java.util.Properties;
import java.util.concurrent.ScheduledFuture;
import java.util.concurrent.ScheduledThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
import org.apache.nifi.logging.LogLevel;
import org.apache.nifi.minifi.bootstrap.QueryableStatusAggregator;
import org.apache.nifi.minifi.commons.status.FlowStatusReport;
import org.apache.nifi.minifi.properties.BootstrapProperties;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.mockito.Mockito;
@ -68,28 +70,28 @@ public class StatusLoggerTest {
@Test
public void testFailedInitDueToFatalLogLevel() {
Properties properties = new Properties();
properties.setProperty(NIFI_MINIFI_STATUS_REPORTER_LOG_PERIOD.getKey(), "1");
properties.setProperty(NIFI_MINIFI_STATUS_REPORTER_LOG_LEVEL.getKey(), LogLevel.FATAL.name());
properties.setProperty(NIFI_MINIFI_STATUS_REPORTER_LOG_QUERY.getKey(), MOCK_QUERY);
BootstrapProperties properties = mock(BootstrapProperties.class);
given(properties.getProperty(NIFI_MINIFI_STATUS_REPORTER_LOG_PERIOD.getKey())).willReturn("1");
given(properties.getProperty(NIFI_MINIFI_STATUS_REPORTER_LOG_LEVEL.getKey())).willReturn(LogLevel.FATAL.name());
given(properties.getProperty(NIFI_MINIFI_STATUS_REPORTER_LOG_QUERY.getKey())).willReturn(MOCK_QUERY);
assertThrows(IllegalStateException.class, () -> statusLogger.initialize(properties, queryableStatusAggregator));
}
@Test
public void testFailedInitDueToNoPeriod() {
Properties properties = new Properties();
properties.setProperty(NIFI_MINIFI_STATUS_REPORTER_LOG_LEVEL.getKey(), LogLevel.INFO.name());
properties.setProperty(NIFI_MINIFI_STATUS_REPORTER_LOG_QUERY.getKey(), MOCK_QUERY);
BootstrapProperties properties = mock(BootstrapProperties.class);
given(properties.getProperty(NIFI_MINIFI_STATUS_REPORTER_LOG_LEVEL.getKey())).willReturn(LogLevel.INFO.name());
given(properties.getProperty(NIFI_MINIFI_STATUS_REPORTER_LOG_QUERY.getKey())).willReturn(MOCK_QUERY);
assertThrows(IllegalStateException.class, () -> statusLogger.initialize(properties, queryableStatusAggregator));
}
@Test
public void testFailedInitDueToNoQuery() {
Properties properties = new Properties();
properties.setProperty(NIFI_MINIFI_STATUS_REPORTER_LOG_PERIOD.getKey(), "1");
properties.setProperty(NIFI_MINIFI_STATUS_REPORTER_LOG_LEVEL.getKey(), LogLevel.INFO.name());
BootstrapProperties properties = mock(BootstrapProperties.class);
given(properties.getProperty(NIFI_MINIFI_STATUS_REPORTER_LOG_LEVEL.getKey())).willReturn(LogLevel.INFO.name());
given(properties.getProperty(NIFI_MINIFI_STATUS_REPORTER_LOG_QUERY.getKey())).willReturn(MOCK_QUERY);
assertThrows(IllegalStateException.class, () -> statusLogger.initialize(properties, queryableStatusAggregator));
}
@ -139,13 +141,9 @@ public class StatusLoggerTest {
verify(logger, Mockito.atLeastOnce()).error(MOCK_STATUS, (Throwable) null);
}
// Exception testing
@Test
public void testTraceException() throws IOException {
Properties properties = new Properties();
properties.setProperty(NIFI_MINIFI_STATUS_REPORTER_LOG_PERIOD.getKey(), "1");
properties.setProperty(NIFI_MINIFI_STATUS_REPORTER_LOG_LEVEL.getKey(), LogLevel.TRACE.name());
properties.setProperty(NIFI_MINIFI_STATUS_REPORTER_LOG_QUERY.getKey(), MOCK_QUERY);
BootstrapProperties properties = getProperties(LogLevel.TRACE);
IOException ioException = new IOException("This is an expected test exception");
Mockito.when(queryableStatusAggregator.statusReport(MOCK_QUERY)).thenThrow(ioException);
@ -205,11 +203,11 @@ public class StatusLoggerTest {
verify(logger, Mockito.atLeastOnce()).error(ENCOUNTERED_IO_EXCEPTION, ioException);
}
private static Properties getProperties(LogLevel logLevel) {
Properties properties = new Properties();
properties.setProperty(NIFI_MINIFI_STATUS_REPORTER_LOG_PERIOD.getKey(), "1");
properties.setProperty(NIFI_MINIFI_STATUS_REPORTER_LOG_LEVEL.getKey(), logLevel.name());
properties.setProperty(NIFI_MINIFI_STATUS_REPORTER_LOG_QUERY.getKey(), MOCK_QUERY);
private static BootstrapProperties getProperties(LogLevel logLevel) {
BootstrapProperties properties = mock(BootstrapProperties.class);
given(properties.getProperty(NIFI_MINIFI_STATUS_REPORTER_LOG_PERIOD.getKey())).willReturn("1");
given(properties.getProperty(NIFI_MINIFI_STATUS_REPORTER_LOG_LEVEL.getKey())).willReturn(logLevel.name());
given(properties.getProperty(NIFI_MINIFI_STATUS_REPORTER_LOG_QUERY.getKey())).willReturn(MOCK_QUERY);
return properties;
}
@ -222,7 +220,6 @@ public class StatusLoggerTest {
@Override
public ScheduledFuture<?> scheduleAtFixedRate(Runnable command, long initialDelay, long period, TimeUnit unit) {
command.run();
// Return value is not used
return null;
}
}

1
minifi/minifi-c2/minifi-c2-service/pom.xml
View File

@ -74,7 +74,6 @@ limitations under the License.
<dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>

3
minifi/minifi-commons/minifi-commons-api/src/main/java/org/apache/nifi/minifi/commons/api/MiNiFiProperties.java
View File

@ -63,7 +63,7 @@ public enum MiNiFiProperties {
NIFI_MINIFI_SECURITY_SSL_PROTOCOL("nifi.minifi.security.ssl.protocol", null, false, false, VALID),
NIFI_MINIFI_FLOW_USE_PARENT_SSL("nifi.minifi.flow.use.parent.ssl", null, false, true, VALID),
NIFI_MINIFI_SENSITIVE_PROPS_KEY("nifi.minifi.sensitive.props.key", null, true, false, VALID),
NIFI_MINIFI_SENSITIVE_PROPS_ALGORITHM("nifi.minifi.sensitive.props.algorithm", null, true, false, VALID),
NIFI_MINIFI_SENSITIVE_PROPS_ALGORITHM("nifi.minifi.sensitive.props.algorithm", null, false, false, VALID),
C2_ENABLE("c2.enable", "false", false, true, BOOLEAN_VALIDATOR),
C2_AGENT_HEARTBEAT_PERIOD("c2.agent.heartbeat.period", "1000", false, true, LONG_VALIDATOR),
C2_AGENT_CLASS("c2.agent.class", "", false, true, VALID),
@ -120,6 +120,7 @@ public enum MiNiFiProperties {
public static final String MINIFI_LOG_DIRECTORY = "nifi.minifi.log.directory";
public static final String MINIFI_APP_LOG_FILE = "nifi.minifi.app.log.file";
public static final String MINIFI_BOOTSTRAP_LOG_FILE = "nifi.minifi.bootstrap.log.file";
public static final String ADDITIONAL_SENSITIVE_PROPERTIES_KEY = "nifi.minifi.sensitive.props.additional.keys";
private final String key;
private final String defaultValue;

1
minifi/minifi-commons/minifi-commons-utils/src/main/java/org/apache/nifi/minifi/commons/utils/PropertyUtil.java
View File

@ -48,4 +48,5 @@ public abstract class PropertyUtil {
private static Stream<String> keyPermutations(String name) {
return Stream.of(name, name.replace(DOT, UNDERSCORE), name.replace(HYPHEN, UNDERSCORE), name.replace(DOT, UNDERSCORE).replace(HYPHEN, UNDERSCORE)).distinct();
}
}

83
minifi/minifi-commons/minifi-commons-utils/src/main/java/org/apache/nifi/minifi/commons/utils/SensitivePropertyUtils.java Normal file
View File

@ -0,0 +1,83 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.nifi.minifi.commons.utils;
import static java.lang.String.format;
import java.io.BufferedInputStream;
import java.io.FileInputStream;
import java.io.InputStream;
import java.util.Optional;
import java.util.Properties;
public class SensitivePropertyUtils {
public static final String MINIFI_BOOTSTRAP_SENSITIVE_KEY = "minifi.bootstrap.sensitive.key";
private static final String EMPTY = "";
private SensitivePropertyUtils() {
}
public static String getFormattedKey() {
String key = getKey(System.getProperty("minifi.bootstrap.conf.file.path"));
// Format the key (check hex validity and remove spaces)
return getFormattedKey(key);
}
public static String getFormattedKey(String unformattedKey) {
String key = formatHexKey(unformattedKey);
if (isNotEmpty(key) && !isHexKeyValid(key)) {
throw new IllegalArgumentException("The key was not provided in valid hex format and of the correct length");
} else {
return key;
}
}
private static String formatHexKey(String input) {
return Optional.ofNullable(input)
.map(String::trim)
.filter(SensitivePropertyUtils::isNotEmpty)
.map(str -> str.replaceAll("[^0-9a-fA-F]", EMPTY).toLowerCase())
.orElse(EMPTY);
}
private static boolean isHexKeyValid(String key) {
return Optional.ofNullable(key)
.map(String::trim)
.filter(SensitivePropertyUtils::isNotEmpty)
.filter(k -> k.matches("^[0-9a-fA-F]{64}$"))
.isPresent();
}
private static String getKey(String bootstrapConfigFilePath) {
Properties properties = new Properties();
try (InputStream inputStream = new BufferedInputStream(new FileInputStream(bootstrapConfigFilePath))) {
properties.load(inputStream);
} catch (Exception e) {
throw new RuntimeException(format("Loading Bootstrap Properties [%s] failed", bootstrapConfigFilePath), e);
}
return properties.getProperty(MINIFI_BOOTSTRAP_SENSITIVE_KEY);
}
private static boolean isNotEmpty(String keyFilePath) {
return keyFilePath != null && !keyFilePath.isBlank();
}
}

55
minifi/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-properties-loader/pom.xml Normal file
View File

@ -0,0 +1,55 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--
~ Licensed to the Apache Software Foundation (ASF) under one or more
~ contributor license agreements. See the NOTICE file distributed with
~ this work for additional information regarding copyright ownership.
~ The ASF licenses this file to You under the Apache License, Version 2.0
~ (the "License"); you may not use this file except in compliance with
~ the License. You may obtain a copy of the License at
~
~ http://www.apache.org/licenses/LICENSE-2.0
~
~ Unless required by applicable law or agreed to in writing, software
~ distributed under the License is distributed on an "AS IS" BASIS,
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
~ See the License for the specific language governing permissions and
~ limitations under the License.
-->
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.apache.nifi.minifi</groupId>
<artifactId>minifi-framework</artifactId>
<version>2.0.0-SNAPSHOT</version>
</parent>
<artifactId>minifi-properties-loader</artifactId>
<dependencies>
<dependency>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-property-protection-loader</artifactId>
</dependency>
<dependency>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-property-protection-cipher</artifactId>
</dependency>
<dependency>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-properties</artifactId>
</dependency>
<dependency>
<groupId>org.apache.nifi.minifi</groupId>
<artifactId>minifi-commons-utils</artifactId>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.nifi.minifi</groupId>
<artifactId>minifi-commons-api</artifactId>
</dependency>
</dependencies>
</project>

39
minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/service/BootstrapProperties.java → minifi/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-properties-loader/src/main/java/org/apache/nifi/minifi/properties/BootstrapProperties.java
View File

@ -14,24 +14,33 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.nifi.minifi.bootstrap.service;
package org.apache.nifi.minifi.properties;
import static org.apache.nifi.minifi.commons.utils.PropertyUtil.resolvePropertyValue;
import java.util.Collections;
import java.util.Map;
import java.util.Properties;
import java.util.Set;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import org.apache.nifi.properties.ApplicationProperties;
/**
* Extends Properties functionality with System and Environment property override possibility. The property resolution also works with
* dots and hyphens that are not supported in some shells.
*/
public class BootstrapProperties extends Properties {
public class BootstrapProperties extends ApplicationProperties {
private BootstrapProperties() {
super();
public BootstrapProperties() {
super(Collections.emptyMap());
}
public static BootstrapProperties getInstance() {
return new BootstrapProperties();
public BootstrapProperties(Properties properties) {
super(properties);
}
public BootstrapProperties(Map<String, String> properties) {
super(properties);
}
@Override
@ -41,4 +50,22 @@ public class BootstrapProperties extends Properties {
.orElseGet(() -> super.getProperty(key));
}
@Override
public String getProperty(String key, String defaultValue) {
return resolvePropertyValue(key, System.getProperties())
.or(() -> resolvePropertyValue(key, System.getenv()))
.orElseGet(() -> super.getProperty(key, defaultValue));
}
@Override
public Set<String> getPropertyKeys() {
Set<String> systemKeys = System.getProperties().keySet().stream().map(String::valueOf).collect(Collectors.toSet());
return Stream.of(systemKeys, System.getenv().keySet(), super.getPropertyKeys())
.flatMap(Set::stream)
.collect(Collectors.toSet());
}
public boolean containsKey(String key) {
return getPropertyKeys().contains(key);
}
}

49
minifi/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-properties-loader/src/main/java/org/apache/nifi/minifi/properties/BootstrapPropertiesLoader.java Normal file
View File

@ -0,0 +1,49 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.nifi.minifi.properties;
import static java.lang.String.format;
import static org.apache.nifi.minifi.commons.utils.SensitivePropertyUtils.MINIFI_BOOTSTRAP_SENSITIVE_KEY;
import static org.apache.nifi.minifi.commons.utils.SensitivePropertyUtils.getFormattedKey;
import java.io.File;
import org.apache.nifi.properties.AesGcmSensitivePropertyProvider;
public class BootstrapPropertiesLoader {
public static BootstrapProperties load(File file) {
ProtectedBootstrapProperties protectedProperties = loadProtectedProperties(file);
if (protectedProperties.hasProtectedKeys()) {
String sensitiveKey = protectedProperties.getApplicationProperties().getProperty(MINIFI_BOOTSTRAP_SENSITIVE_KEY);
validateSensitiveKeyProperty(sensitiveKey);
String keyHex = getFormattedKey(sensitiveKey);
protectedProperties.addSensitivePropertyProvider(new AesGcmSensitivePropertyProvider(keyHex));
}
return protectedProperties.getUnprotectedProperties();
}
public static ProtectedBootstrapProperties loadProtectedProperties(File file) {
return new ProtectedBootstrapProperties(PropertiesLoader.load(file, "Bootstrap"));
}
private static void validateSensitiveKeyProperty(String sensitiveKey) {
if (sensitiveKey == null || sensitiveKey.trim().isEmpty()) {
throw new IllegalArgumentException(format("bootstrap.conf contains protected properties but %s is not found", MINIFI_BOOTSTRAP_SENSITIVE_KEY));
}
}
}

53
minifi/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-properties-loader/src/main/java/org/apache/nifi/minifi/properties/DuplicateDetectingProperties.java Normal file
View File

@ -0,0 +1,53 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.nifi.minifi.properties;
import java.util.HashSet;
import java.util.Properties;
import java.util.Set;
class DuplicateDetectingProperties extends Properties {
// Only need to retain Properties key. This will help prevent possible inadvertent exposure of sensitive Properties value
private final Set<String> duplicateKeys = new HashSet<>();
private final Set<String> redundantKeys = new HashSet<>();
public DuplicateDetectingProperties() {
super();
}
public Set<String> duplicateKeySet() {
return duplicateKeys;
}
public Set<String> redundantKeySet() {
return redundantKeys;
}
@Override
public Object put(Object key, Object value) {
Object existingValue = super.put(key, value);
if (existingValue != null) {
if (existingValue.toString().equals(value.toString())) {
redundantKeys.add(key.toString());
} else {
duplicateKeys.add(key.toString());
}
}
return value;
}
}

104
minifi/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-properties-loader/src/main/java/org/apache/nifi/minifi/properties/MiNiFiPropertiesLoader.java Normal file
View File

@ -0,0 +1,104 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.nifi.minifi.properties;
import java.io.File;
import org.apache.nifi.properties.AesGcmSensitivePropertyProvider;
import org.apache.nifi.util.NiFiBootstrapUtils;
import org.apache.nifi.util.NiFiProperties;
public class MiNiFiPropertiesLoader {
private static final String DEFAULT_APPLICATION_PROPERTIES_FILE_PATH = NiFiBootstrapUtils.getDefaultApplicationPropertiesFilePath();
private NiFiProperties instance;
private String keyHex;
public MiNiFiPropertiesLoader(String keyHex) {
this.keyHex = keyHex;
}
/**
* Returns a {@link ProtectedMiNiFiProperties} instance loaded from the
* serialized form in the file. Responsible for actually reading from disk
* and deserializing the properties. Returns a protected instance to allow
* for decryption operations.
*
* @param file the file containing serialized properties
* @return the ProtectedMiNiFiProperties instance
*/
ProtectedMiNiFiProperties loadProtectedProperties(File file) {
return new ProtectedMiNiFiProperties(PropertiesLoader.load(file, "Application"));
}
/**
* Returns an instance of {@link NiFiProperties} loaded from the provided
* {@link File}. If any properties are protected, will attempt to use the
* {@link AesGcmSensitivePropertyProvider} to unprotect them
* transparently.
*
* @param file the File containing the serialized properties
* @return the NiFiProperties instance
*/
public NiFiProperties load(File file) {
ProtectedMiNiFiProperties protectedProperties = loadProtectedProperties(file);
if (protectedProperties.hasProtectedKeys()) {
protectedProperties.addSensitivePropertyProvider(new AesGcmSensitivePropertyProvider(keyHex));
}
return new MultiSourceMinifiProperties(protectedProperties.getUnprotectedPropertiesAsMap());
}
/**
* Returns an instance of {@link NiFiProperties}. If the path is empty, this
* will load the default properties file as specified by
* {@code NiFiProperties.PROPERTY_FILE_PATH}.
*
* @param path the path of the serialized properties file
* @return the NiFiProperties instance
* @see MiNiFiPropertiesLoader#load(File)
*/
public NiFiProperties load(String path) {
if (path != null && !path.trim().isEmpty()) {
return load(new File(path));
} else {
return loadDefault();
}
}
/**
* Returns the loaded {@link NiFiProperties} instance. If none is currently
* loaded, attempts to load the default instance.
* <p>
* <p>
* NOTE: This method is used reflectively by the process which starts MiNiFi
* so changes to it must be made in conjunction with that mechanism.</p>
*
* @return the current NiFiProperties instance
*/
public NiFiProperties get() {
if (instance == null) {
instance = loadDefault();
}
return instance;
}
private NiFiProperties loadDefault() {
return load(DEFAULT_APPLICATION_PROPERTIES_FILE_PATH);
}
}

55
minifi/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-properties-loader/src/main/java/org/apache/nifi/minifi/properties/MultiSourceMinifiProperties.java Normal file
View File

@ -0,0 +1,55 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.nifi.minifi.properties;
import java.util.Map;
import java.util.Set;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import org.apache.nifi.minifi.commons.utils.PropertyUtil;
import org.apache.nifi.util.NiFiProperties;
/**
* Extends NiFi properties functionality with System and Environment property override possibility. The property resolution also works with
* dots and hyphens that are not supported in some shells.
*/
public class MultiSourceMinifiProperties extends NiFiProperties {
public MultiSourceMinifiProperties(Map<String, String> props) {
super(props);
}
@Override
public Set<String> getPropertyKeys() {
return Stream.of(System.getProperties().stringPropertyNames(), System.getenv().keySet(), super.getPropertyKeys())
.flatMap(Set::stream)
.collect(Collectors.toSet());
}
@Override
public int size() {
return getPropertyKeys().size();
}
@Override
public String getProperty(String key) {
return PropertyUtil.resolvePropertyValue(key, System.getProperties())
.or(() -> PropertyUtil.resolvePropertyValue(key, System.getenv()))
.orElseGet(() -> super.getProperty(key));
}
}

61
minifi/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-properties-loader/src/main/java/org/apache/nifi/minifi/properties/PropertiesLoader.java Normal file
View File

@ -0,0 +1,61 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.nifi.minifi.properties;
import java.io.BufferedInputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.InputStream;
import java.util.Properties;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public interface PropertiesLoader {
static final Logger logger = LoggerFactory.getLogger(PropertiesLoader.class);
static Properties load(File file, String propertiesType) {
if (file == null || !file.exists() || !file.canRead()) {
throw new IllegalArgumentException(String.format("{} Properties [%s] not found", propertiesType, file));
}
logger.info("Loading {} Properties [{}]", propertiesType, file);
DuplicateDetectingProperties rawProperties = new DuplicateDetectingProperties();
try (InputStream inputStream = new BufferedInputStream(new FileInputStream(file))) {
rawProperties.load(inputStream);
} catch (Exception e) {
throw new RuntimeException(String.format("Loading {} Properties [%s] failed", propertiesType, file), e);
}
if (!rawProperties.redundantKeySet().isEmpty()) {
logger.warn("Duplicate property keys with the same value were detected in the properties file: {}", String.join(", ", rawProperties.redundantKeySet()));
}
if (!rawProperties.duplicateKeySet().isEmpty()) {
throw new IllegalArgumentException("Duplicate property keys with different values were detected in the properties file: " + String.join(", ", rawProperties.duplicateKeySet()));
}
Properties properties = new Properties();
rawProperties.stringPropertyNames()
.forEach(key -> {
String property = rawProperties.getProperty(key);
properties.setProperty(key, property.trim());
});
return properties;
}
}

134
minifi/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-properties-loader/src/main/java/org/apache/nifi/minifi/properties/ProtectedBootstrapProperties.java Normal file
View File

@ -0,0 +1,134 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.nifi.minifi.properties;
import static org.apache.nifi.minifi.commons.api.MiNiFiProperties.ADDITIONAL_SENSITIVE_PROPERTIES_KEY;
import static org.apache.nifi.minifi.properties.ProtectedMiNiFiProperties.DEFAULT_SENSITIVE_PROPERTIES;
import java.util.Arrays;
import java.util.List;
import java.util.Map;
import java.util.Properties;
import java.util.Set;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import org.apache.nifi.minifi.commons.api.MiNiFiProperties;
import org.apache.nifi.properties.ApplicationPropertiesProtector;
import org.apache.nifi.properties.ProtectedProperties;
import org.apache.nifi.properties.SensitivePropertyProtectionException;
import org.apache.nifi.properties.SensitivePropertyProtector;
import org.apache.nifi.properties.SensitivePropertyProvider;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public class ProtectedBootstrapProperties extends BootstrapProperties implements ProtectedProperties<BootstrapProperties>,
SensitivePropertyProtector<ProtectedBootstrapProperties, BootstrapProperties> {
private static final Logger logger = LoggerFactory.getLogger(ProtectedBootstrapProperties.class);
private BootstrapProperties bootstrapProperties;
private final SensitivePropertyProtector<ProtectedBootstrapProperties, BootstrapProperties> propertyProtectionDelegate;
public ProtectedBootstrapProperties(BootstrapProperties props) {
super();
this.bootstrapProperties = props;
this.propertyProtectionDelegate = new ApplicationPropertiesProtector<>(this);
logger.debug("Loaded {} properties (including {} protection schemes) into ProtectedBootstrapProperties", getApplicationProperties().getPropertyKeys().size(),
getProtectedPropertyKeys().size());
}
public ProtectedBootstrapProperties(Properties rawProps) {
this(new BootstrapProperties(rawProps));
}
@Override
public Set<String> getPropertyKeysIncludingProtectionSchemes() {
return propertyProtectionDelegate.getPropertyKeysIncludingProtectionSchemes();
}
@Override
public List<String> getSensitivePropertyKeys() {
return propertyProtectionDelegate.getSensitivePropertyKeys();
}
@Override
public List<String> getPopulatedSensitivePropertyKeys() {
return propertyProtectionDelegate.getPopulatedSensitivePropertyKeys();
}
@Override
public boolean hasProtectedKeys() {
return propertyProtectionDelegate.hasProtectedKeys();
}
@Override
public Map<String, String> getProtectedPropertyKeys() {
return propertyProtectionDelegate.getProtectedPropertyKeys();
}
@Override
public boolean isPropertySensitive(String key) {
return propertyProtectionDelegate.isPropertySensitive(key);
}
@Override
public boolean isPropertyProtected(String key) {
return propertyProtectionDelegate.isPropertyProtected(key);
}
@Override
public BootstrapProperties getUnprotectedProperties() throws SensitivePropertyProtectionException {
return propertyProtectionDelegate.getUnprotectedProperties();
}
@Override
public void addSensitivePropertyProvider(SensitivePropertyProvider sensitivePropertyProvider) {
propertyProtectionDelegate.addSensitivePropertyProvider(sensitivePropertyProvider);
}
@Override
public String getAdditionalSensitivePropertiesKeys() {
return getProperty(getAdditionalSensitivePropertiesKeysName());
}
@Override
public String getAdditionalSensitivePropertiesKeysName() {
return ADDITIONAL_SENSITIVE_PROPERTIES_KEY;
}
@Override
public List<String> getDefaultSensitiveProperties() {
return Stream.of(DEFAULT_SENSITIVE_PROPERTIES, Arrays.stream(MiNiFiProperties.values()).filter(MiNiFiProperties::isSensitive).map(MiNiFiProperties::getKey).collect(Collectors.toList()))
.flatMap(List::stream).distinct().collect(Collectors.toList());
}
@Override
public BootstrapProperties getApplicationProperties() {
if (this.bootstrapProperties == null) {
this.bootstrapProperties = new BootstrapProperties();
}
return this.bootstrapProperties;
}
@Override
public BootstrapProperties createApplicationProperties(Properties rawProperties) {
return new BootstrapProperties(rawProperties);
}
}

239
minifi/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-properties-loader/src/main/java/org/apache/nifi/minifi/properties/ProtectedMiNiFiProperties.java Normal file
View File

@ -0,0 +1,239 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.nifi.minifi.properties;
import static java.util.Arrays.asList;
import static org.apache.nifi.minifi.commons.api.MiNiFiProperties.ADDITIONAL_SENSITIVE_PROPERTIES_KEY;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.Map;
import java.util.Properties;
import java.util.Set;
import java.util.function.Function;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import org.apache.nifi.minifi.commons.api.MiNiFiProperties;
import org.apache.nifi.properties.ApplicationPropertiesProtector;
import org.apache.nifi.properties.ProtectedProperties;
import org.apache.nifi.properties.SensitivePropertyProtectionException;
import org.apache.nifi.properties.SensitivePropertyProtector;
import org.apache.nifi.properties.SensitivePropertyProvider;
import org.apache.nifi.util.NiFiProperties;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* Decorator class for intermediate phase when {@link MiNiFiPropertiesLoader} loads the
* raw properties file and performs unprotection activities before returning a clean
* implementation of {@link NiFiProperties}.
* This encapsulates the sensitive property access logic from external consumers
* of {@code NiFiProperties}.
*/
public class ProtectedMiNiFiProperties extends NiFiProperties implements ProtectedProperties<NiFiProperties>,
SensitivePropertyProtector<ProtectedMiNiFiProperties, NiFiProperties> {
private static final Logger logger = LoggerFactory.getLogger(ProtectedMiNiFiProperties.class);
public static final List<String> DEFAULT_SENSITIVE_PROPERTIES = new ArrayList<>(asList(
SECURITY_KEY_PASSWD,
SECURITY_KEYSTORE_PASSWD,
SECURITY_TRUSTSTORE_PASSWD,
SENSITIVE_PROPS_KEY
));
private final SensitivePropertyProtector<ProtectedMiNiFiProperties, NiFiProperties> propertyProtectionDelegate;
private NiFiProperties applicationProperties;
public ProtectedMiNiFiProperties() {
this(new NiFiProperties());
}
/**
* Creates an instance containing the provided {@link NiFiProperties}.
*
* @param props the NiFiProperties to contain
*/
public ProtectedMiNiFiProperties(NiFiProperties props) {
this.applicationProperties = props;
this.propertyProtectionDelegate = new ApplicationPropertiesProtector<>(this);
logger.debug("Loaded {} properties (including {} protection schemes) into ProtectedNiFiProperties", getApplicationProperties()
.getPropertyKeys().size(), getProtectedPropertyKeys().size());
}
/**
* Creates an instance containing the provided raw {@link Properties}.
*
* @param rawProps the Properties to contain
*/
public ProtectedMiNiFiProperties(Properties rawProps) {
this(new NiFiProperties(rawProps));
}
@Override
public String getAdditionalSensitivePropertiesKeys() {
return getProperty(getAdditionalSensitivePropertiesKeysName());
}
@Override
public String getAdditionalSensitivePropertiesKeysName() {
return ADDITIONAL_SENSITIVE_PROPERTIES_KEY;
}
@Override
public List<String> getDefaultSensitiveProperties() {
return Stream.of(DEFAULT_SENSITIVE_PROPERTIES,
Arrays.stream(MiNiFiProperties.values()).filter(MiNiFiProperties::isSensitive).map(MiNiFiProperties::getKey).toList()).flatMap(List::stream).distinct().toList();
}
/**
* Returns the internal representation of the {@link NiFiProperties} -- protected
* or not as determined by the current state. No guarantee is made to the
* protection state of these properties. If the internal reference is null, a new
* {@link NiFiProperties} instance is created.
*
* @return the internal properties
*/
public NiFiProperties getApplicationProperties() {
if (this.applicationProperties == null) {
this.applicationProperties = new NiFiProperties();
}
return this.applicationProperties;
}
@Override
public NiFiProperties createApplicationProperties(final Properties rawProperties) {
return new NiFiProperties(rawProperties);
}
/**
* Retrieves the property value for the given property key.
*
* @param key the key of property value to lookup
* @return value of property at given key or null if not found
*/
@Override
public String getProperty(String key) {
return getApplicationProperties().getProperty(key);
}
/**
* Retrieves all known property keys.
*
* @return all known property keys
*/
@Override
public Set<String> getPropertyKeys() {
return propertyProtectionDelegate.getPropertyKeys();
}
/**
* Returns the number of properties, excluding protection scheme properties.
* <p>
* Example:
* <p>
* key: E(value, key)
* key.protected: aes/gcm/256
* key2: value2
* <p>
* would return size 2
*
* @return the count of real properties
*/
@Override
public int size() {
return propertyProtectionDelegate.size();
}
@Override
public Set<String> getPropertyKeysIncludingProtectionSchemes() {
return propertyProtectionDelegate.getPropertyKeysIncludingProtectionSchemes();
}
@Override
public List<String> getSensitivePropertyKeys() {
return propertyProtectionDelegate.getSensitivePropertyKeys();
}
@Override
public List<String> getPopulatedSensitivePropertyKeys() {
return propertyProtectionDelegate.getPopulatedSensitivePropertyKeys();
}
@Override
public boolean hasProtectedKeys() {
return propertyProtectionDelegate.hasProtectedKeys();
}
@Override
public Map<String, String> getProtectedPropertyKeys() {
return propertyProtectionDelegate.getProtectedPropertyKeys();
}
@Override
public boolean isPropertySensitive(final String key) {
return propertyProtectionDelegate.isPropertySensitive(key);
}
@Override
public boolean isPropertyProtected(final String key) {
return propertyProtectionDelegate.isPropertyProtected(key);
}
@Override
public NiFiProperties getUnprotectedProperties() throws SensitivePropertyProtectionException {
return propertyProtectionDelegate.getUnprotectedProperties();
}
@Override
public void addSensitivePropertyProvider(final SensitivePropertyProvider sensitivePropertyProvider) {
propertyProtectionDelegate.addSensitivePropertyProvider(sensitivePropertyProvider);
}
public Map<String, String> getUnprotectedPropertiesAsMap() {
NiFiProperties niFiProperties = propertyProtectionDelegate.getUnprotectedProperties();
return niFiProperties.getPropertyKeys().stream().collect(Collectors.toMap(Function.identity(), niFiProperties::getProperty));
}
/**
* Returns the number of properties that are marked as protected in the provided {@link NiFiProperties} instance without requiring external creation of a
* {@link ProtectedMiNiFiProperties} instance.
*
* @param plainProperties the instance to count protected properties
* @return the number of protected properties
*/
public static int countProtectedProperties(final NiFiProperties plainProperties) {
return new ProtectedMiNiFiProperties(plainProperties).getProtectedPropertyKeys().size();
}
/**
* Returns the number of properties that are marked as sensitive in the provided {@link NiFiProperties} instance without requiring external creation of a
* {@link ProtectedMiNiFiProperties} instance.
*
* @param plainProperties the instance to count sensitive properties
* @return the number of sensitive properties
*/
public static int countSensitiveProperties(final NiFiProperties plainProperties) {
return new ProtectedMiNiFiProperties(plainProperties).getSensitivePropertyKeys().size();
}
@Override
public String toString() {
return String.format("%s Size [%d]", getClass().getSimpleName(), size());
}
}

82
minifi/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-properties-loader/src/test/java/org/apache/nifi/minifi/properties/BootstrapPropertiesLoaderTest.java Normal file
View File

@ -0,0 +1,82 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.nifi.minifi.properties;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertThrows;
import java.io.File;
import java.net.URL;
import java.util.Map;
import java.util.function.Function;
import java.util.stream.Collectors;
import org.junit.jupiter.api.Test;
class BootstrapPropertiesLoaderTest {
private static final String NON_EXISTING_FILE_PATH = "/conf/nonexisting.conf";
private static final String DUPLICATED_ITEMS_FILE_PATH = "/conf/bootstrap_duplicated_items.conf";
private static final String UNPROTECTED_ITEMS_FILE_PATH = "/conf/bootstrap_unprotected.conf";
private static final String PROTECTED_ITEMS_FILE_PATH = "/conf/bootstrap_protected.conf";
private static final String PROTECTED_ITEMS_WITHOUT_SENSITIVE_KEY_FILE_PATH = "/conf/bootstrap_protected_without_sensitive_key.conf";
private static final Map<String, String> UNPROTECTED_PROPERTIES = Map.of("nifi.minifi.security.keystorePasswd", "testPassword", "nifi.minifi.sensitive.props.key", "testSensitivePropsKey");
@Test
void shouldThrowIllegalArgumentExceptionIfFileIsNotProvided() {
assertThrows(IllegalArgumentException.class, () -> BootstrapPropertiesLoader.load(null));
}
@Test
void shouldThrowIllegalArgumentExceptionIfFileDoesNotExists() {
assertThrows(IllegalArgumentException.class, () -> BootstrapPropertiesLoader.load(new File(NON_EXISTING_FILE_PATH)));
}
@Test
void shouldThrowIllegalArgumentExceptionIfTheConfigFileContainsDuplicatedKeysWithDifferentValues() {
assertThrows(IllegalArgumentException.class, () -> BootstrapPropertiesLoader.load(getFile(DUPLICATED_ITEMS_FILE_PATH)));
}
@Test
void shouldReturnPropertiesIfConfigFileDoesNotContainProtectedProperties() {
BootstrapProperties bootstrapProperties = BootstrapPropertiesLoader.load(getFile(UNPROTECTED_ITEMS_FILE_PATH));
assertEquals(UNPROTECTED_PROPERTIES,
bootstrapProperties.getPropertyKeys().stream().filter(UNPROTECTED_PROPERTIES::containsKey).collect(Collectors.toMap(Function.identity(), bootstrapProperties::getProperty)));
}
@Test
void shouldReturnUnProtectedProperties() {
BootstrapProperties bootstrapProperties = BootstrapPropertiesLoader.load(getFile(PROTECTED_ITEMS_FILE_PATH));
assertEquals(UNPROTECTED_PROPERTIES,
bootstrapProperties.getPropertyKeys().stream().filter(UNPROTECTED_PROPERTIES::containsKey).collect(Collectors.toMap(Function.identity(), bootstrapProperties::getProperty)));
}
@Test
void shouldThrowIllegalArgumentExceptionIfFileContainsProtectedPropertiesButSensitiveKeyIsMissing() {
assertThrows(IllegalArgumentException.class, () -> BootstrapPropertiesLoader.load(getFile(PROTECTED_ITEMS_WITHOUT_SENSITIVE_KEY_FILE_PATH)));
}
private File getFile(String duplicatedItemsFilePath) {
URL resource = BootstrapPropertiesLoaderTest.class.getResource(duplicatedItemsFilePath);
assertNotNull(resource);
return new File(resource.getPath());
}
}

90
minifi/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-properties-loader/src/test/java/org/apache/nifi/minifi/properties/MiNiFiPropertiesLoaderTest.java Normal file
View File

@ -0,0 +1,90 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.nifi.minifi.properties;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertThrows;
import java.io.File;
import java.net.URL;
import java.util.Map;
import java.util.function.Function;
import java.util.stream.Collectors;
import org.apache.nifi.util.NiFiProperties;
import org.junit.jupiter.api.Test;
class MiNiFiPropertiesLoaderTest {
private static final String NON_EXISTING_FILE_PATH = "/conf/nonexisting.properties";
private static final String DUPLICATED_ITEMS_FILE_PATH = "/conf/bootstrap_duplicated_items.conf";
private static final String UNPROTECTED_ITEMS_FILE_PATH = "/conf/minifi_unprotected.properties";
private static final String PROTECTED_ITEMS_FILE_PATH = "/conf/minifi_protected.properties";
private static final Map<String, String> UNPROTECTED_PROPERTIES = Map.of("nifi.security.keystorePasswd", "testPassword", "nifi.security.keyPasswd",
"testSensitivePropsKey");
private static final String PROTECTION_KEY = "00714ae7a77b24cde1d36bd19472777e0d4ab02c38913b7f9bf41f3963147b4f";
@Test
void shouldThrowIllegalArgumentExceptionIfFileIsNotProvided() {
MiNiFiPropertiesLoader miNiFiPropertiesLoader = new MiNiFiPropertiesLoader("");
assertThrows(IllegalArgumentException.class, () -> miNiFiPropertiesLoader.load((String) null));
}
@Test
void shouldThrowIllegalArgumentExceptionIfFileDoesNotExists() {
MiNiFiPropertiesLoader miNiFiPropertiesLoader = new MiNiFiPropertiesLoader("");
assertThrows(IllegalArgumentException.class, () -> miNiFiPropertiesLoader.load(new File(NON_EXISTING_FILE_PATH)));
}
@Test
void shouldThrowIllegalArgumentExceptionIfTheConfigFileContainsDuplicatedKeysWithDifferentValues() {
MiNiFiPropertiesLoader miNiFiPropertiesLoader = new MiNiFiPropertiesLoader("");
assertThrows(IllegalArgumentException.class, () -> miNiFiPropertiesLoader.load(getFile(DUPLICATED_ITEMS_FILE_PATH)));
}
@Test
void shouldReturnPropertiesIfConfigFileDoesNotContainProtectedProperties() {
MiNiFiPropertiesLoader miNiFiPropertiesLoader = new MiNiFiPropertiesLoader("");
NiFiProperties niFiProperties = miNiFiPropertiesLoader.load(getFile(UNPROTECTED_ITEMS_FILE_PATH));
assertEquals(UNPROTECTED_PROPERTIES,
niFiProperties.getPropertyKeys().stream().filter(UNPROTECTED_PROPERTIES::containsKey).collect(Collectors.toMap(Function.identity(), niFiProperties::getProperty)));
}
@Test
void shouldReturnUnProtectedProperties() {
MiNiFiPropertiesLoader miNiFiPropertiesLoader = new MiNiFiPropertiesLoader(PROTECTION_KEY);
NiFiProperties niFiProperties = miNiFiPropertiesLoader.load(getUrl(PROTECTED_ITEMS_FILE_PATH).getPath());
assertEquals(UNPROTECTED_PROPERTIES,
niFiProperties.getPropertyKeys().stream().filter(UNPROTECTED_PROPERTIES::containsKey).collect(Collectors.toMap(Function.identity(), niFiProperties::getProperty)));
}
private File getFile(String propertiesFilePath) {
URL resource = getUrl(propertiesFilePath);
return new File(resource.getPath());
}
private static URL getUrl(String propertiesFilePath) {
URL resource = BootstrapPropertiesLoaderTest.class.getResource(propertiesFilePath);
assertNotNull(resource);
return resource;
}
}

18
minifi/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-properties-loader/src/test/resources/conf/bootstrap_duplicated_items.conf Normal file
View File

@ -0,0 +1,18 @@
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# property file that contains the same key with different values
property1=test
property1=test2

22
minifi/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-properties-loader/src/test/resources/conf/bootstrap_protected.conf Normal file
View File

@ -0,0 +1,22 @@
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# property file that contains the protected properties and a sensitive key to decrypt it
nifi.minifi.security.keystorePasswd=noBD32A0ANcz/BHv||nhtyNhlFgNNZcVhgxEOg2xUZ5UAihgyBit1drQ==
nifi.minifi.security.keystorePasswd.protected=aes/gcm/256
nifi.minifi.sensitive.props.key=JUMeAtpD1Q7CiXHt||BppiPMoWXxkKBl5mYsxP5vkVabsXZyLu2lxjyv9LMHc6RJEE9g==
nifi.minifi.sensitive.props.key.protected=aes/gcm/256
minifi.bootstrap.sensitive.key=00714ae7a77b24cde1d36bd19472777e0d4ab02c38913b7f9bf41f3963147b4f

20
minifi/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-properties-loader/src/test/resources/conf/bootstrap_protected_without_sensitive_key.conf Normal file
View File

@ -0,0 +1,20 @@
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# property file that contains the protected properties but without minifi.bootstrap.sensitive.key
nifi.minifi.security.keystorePasswd=noBD32A0ANcz/BHv||nhtyNhlFgNNZcVhgxEOg2xUZ5UAihgyBit1drQ==
nifi.minifi.security.keystorePasswd.protected=aes/gcm/256
nifi.minifi.sensitive.props.key=JUMeAtpD1Q7CiXHt||BppiPMoWXxkKBl5mYsxP5vkVabsXZyLu2lxjyv9LMHc6RJEE9g==
nifi.minifi.sensitive.props.key.protected=aes/gcm/256

18
minifi/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-properties-loader/src/test/resources/conf/bootstrap_unprotected.conf Normal file
View File

@ -0,0 +1,18 @@
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# property file that contains the unprotected properties
nifi.minifi.security.keystorePasswd=testPassword
nifi.minifi.sensitive.props.key=testSensitivePropsKey

20
minifi/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-properties-loader/src/test/resources/conf/minifi_duplicated.properties Normal file
View File

@ -0,0 +1,20 @@
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
nifi.security.keystorePasswd=s+jNt0t608+F0uq9||PWPtOCr5fmItmL8ZsgpheQxkkJJzXWqrNvqdCL/gcGE2cy1NTgGDhI1apuacNHjj
nifi.security.keystorePasswd=123
nifi.security.keystorePasswd.protected=aes/gcm/256
nifi.security.keyPasswd=JUMeAtpD1Q7CiXHt||BppiPMoWXxkKBl5mYsxP5vkVabsXZyLu2lxjyv9LMHc6RJEE9g==
nifi.security.keyPasswd.protected=aes/gcm/256

19
minifi/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-properties-loader/src/test/resources/conf/minifi_protected.properties Normal file
View File

@ -0,0 +1,19 @@
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
nifi.security.keystorePasswd=noBD32A0ANcz/BHv||nhtyNhlFgNNZcVhgxEOg2xUZ5UAihgyBit1drQ==
nifi.security.keystorePasswd.protected=aes/gcm/256
nifi.security.keyPasswd=JUMeAtpD1Q7CiXHt||BppiPMoWXxkKBl5mYsxP5vkVabsXZyLu2lxjyv9LMHc6RJEE9g==
nifi.security.keyPasswd.protected=aes/gcm/256

17
minifi/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-properties-loader/src/test/resources/conf/minifi_unprotected.properties Normal file
View File

@ -0,0 +1,17 @@
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
nifi.security.keystorePasswd=testPassword
nifi.security.keyPasswd=testSensitivePropsKey

6
minifi/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-runtime/pom.xml
View File

@ -58,8 +58,9 @@ limitations under the License.
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-properties-loader</artifactId>
<groupId>org.apache.nifi.minifi</groupId>
<artifactId>minifi-properties-loader</artifactId>
<version>2.0.0-SNAPSHOT</version>
<!-- The dependency is required to be made available by the binary build in the bootstrap folder -->
<scope>provided</scope>
</dependency>
@ -76,5 +77,6 @@ limitations under the License.
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-databind</artifactId>
</dependency>
</dependencies>
</project>

45
minifi/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-runtime/src/main/java/org/apache/nifi/minifi/MiNiFi.java
View File

@ -16,8 +16,14 @@
*/
package org.apache.nifi.minifi;
import static org.apache.nifi.minifi.commons.utils.SensitivePropertyUtils.getFormattedKey;
import static org.apache.nifi.minifi.util.BootstrapClassLoaderUtils.createBootstrapClassLoader;
import java.io.File;
import java.io.IOException;
import java.lang.reflect.Constructor;
import java.lang.reflect.InvocationTargetException;
import java.lang.reflect.Method;
import java.util.Set;
import java.util.Timer;
import java.util.TimerTask;
@ -30,7 +36,6 @@ import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicLong;
import org.apache.nifi.NiFiServer;
import org.apache.nifi.bundle.Bundle;
import org.apache.nifi.minifi.util.MultiSourceMinifiProperties;
import org.apache.nifi.nar.ExtensionMapping;
import org.apache.nifi.nar.NarClassLoaders;
import org.apache.nifi.nar.NarClassLoadersHolder;
@ -46,6 +51,7 @@ import org.slf4j.bridge.SLF4JBridgeHandler;
public class MiNiFi {
private static final Logger logger = LoggerFactory.getLogger(MiNiFi.class);
private final MiNiFiServer minifiServer;
private volatile boolean shutdown = false;
@ -155,7 +161,7 @@ public class MiNiFi {
logger.info("MiNiFi server shutdown completed (nicely or otherwise).");
} catch (final Throwable t) {
logger.warn("Problem occurred ensuring MiNiFi server was properly terminated due to " + t);
logger.warn("Problem occurred ensuring MiNiFi server was properly terminated due to {}", t.getMessage());
}
}
@ -218,10 +224,41 @@ public class MiNiFi {
public static void main(String[] args) {
logger.info("Launching MiNiFi...");
try {
NiFiProperties niFiProperties = MultiSourceMinifiProperties.getInstance();
new MiNiFi(niFiProperties);
NiFiProperties properties = getValidatedMiNifiProperties();
new MiNiFi(properties);
} catch (final Throwable t) {
logger.error("Failure to launch MiNiFi due to " + t, t);
}
}
protected static NiFiProperties getValidatedMiNifiProperties() {
NiFiProperties properties = initializeProperties(createBootstrapClassLoader());
properties.validate();
return properties;
}
private static NiFiProperties initializeProperties(ClassLoader boostrapLoader) {
ClassLoader contextClassLoader = Thread.currentThread().getContextClassLoader();
String key = getFormattedKey();
Thread.currentThread().setContextClassLoader(boostrapLoader);
try {
Class<?> propsLoaderClass = Class.forName("org.apache.nifi.minifi.properties.MiNiFiPropertiesLoader", true, boostrapLoader);
Constructor<?> constructor = propsLoaderClass.getDeclaredConstructor(String.class);
Object loaderInstance = constructor.newInstance(key);
Method getMethod = propsLoaderClass.getMethod("get");
NiFiProperties properties = (NiFiProperties) getMethod.invoke(loaderInstance);
logger.info("Application Properties loaded [{}]", properties.size());
return properties;
} catch (InvocationTargetException wrappedException) {
throw new IllegalArgumentException("There was an issue decrypting protected properties", wrappedException.getCause() == null ? wrappedException : wrappedException.getCause());
} catch (final IllegalAccessException | NoSuchMethodException | ClassNotFoundException | InstantiationException reex) {
throw new IllegalArgumentException("Unable to access properties loader in the expected manner - apparent classpath or build issue", reex);
} catch (final RuntimeException e) {
throw new IllegalArgumentException("There was an issue decrypting protected properties", e);
} finally {
Thread.currentThread().setContextClassLoader(contextClassLoader);
}
}
}

59
minifi/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-runtime/src/main/java/org/apache/nifi/minifi/util/BootstrapClassLoaderUtils.java Normal file
View File

@ -0,0 +1,59 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.nifi.minifi.util;
import java.io.IOException;
import java.net.MalformedURLException;
import java.net.URL;
import java.net.URLClassLoader;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.List;
import java.util.stream.Stream;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* Util for creating class loader with bootstrap libs.
*/
public final class BootstrapClassLoaderUtils {
private static final Logger LOGGER = LoggerFactory.getLogger(BootstrapClassLoaderUtils.class);
private static final String LIB_BOOTSTRAP_DIR = "lib/bootstrap";
private BootstrapClassLoaderUtils() {
}
public static ClassLoader createBootstrapClassLoader() {
List<URL> urls = new ArrayList<>();
try (Stream<Path> files = Files.list(Paths.get(LIB_BOOTSTRAP_DIR))) {
files.forEach(p -> {
try {
urls.add(p.toUri().toURL());
} catch (MalformedURLException mef) {
LOGGER.warn("Unable to load bootstrap library [{}]", p.getFileName(), mef);
}
});
} catch (IOException ioe) {
LOGGER.warn("Unable to access lib/bootstrap to create bootstrap classloader", ioe);
}
return new URLClassLoader(urls.toArray(new URL[0]), Thread.currentThread().getContextClassLoader());
}
}

87
minifi/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-runtime/src/main/java/org/apache/nifi/minifi/util/MultiSourceMinifiProperties.java
View File

@ -1,87 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.nifi.minifi.util;
import static org.apache.nifi.minifi.commons.utils.PropertyUtil.resolvePropertyValue;
import java.io.BufferedInputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.InputStream;
import java.util.Properties;
import java.util.Set;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import org.apache.nifi.util.NiFiProperties;
/**
* Extends NiFi properties functionality with System and Environment property override possibility. The property resolution also works with
* dots and hyphens that are not supported in some shells.
*/
public class MultiSourceMinifiProperties extends NiFiProperties {
private final Properties properties = new Properties();
public static MultiSourceMinifiProperties getInstance() {
return new MultiSourceMinifiProperties();
}
private MultiSourceMinifiProperties() {
readFromPropertiesFile();
}
@Override
public Set<String> getPropertyKeys() {
return Stream.of(System.getProperties().stringPropertyNames(), System.getenv().keySet(), properties.stringPropertyNames())
.flatMap(Set::stream)
.collect(Collectors.toSet());
}
@Override
public int size() {
return getPropertyKeys().size();
}
@Override
public String getProperty(String key) {
return resolvePropertyValue(key, System.getProperties())
.or(() -> resolvePropertyValue(key, System.getenv()))
.orElseGet(() -> properties.getProperty(key));
}
private void readFromPropertiesFile() {
String propertiesFilePath = System.getProperty(NiFiProperties.PROPERTIES_FILE_PATH);
if (propertiesFilePath != null) {
File propertiesFile = new File(propertiesFilePath.trim());
if (!propertiesFile.exists()) {
throw new RuntimeException("Properties file doesn't exist '" + propertiesFile.getAbsolutePath() + "'");
}
if (!propertiesFile.canRead()) {
throw new RuntimeException("Properties file exists but cannot be read '" + propertiesFile.getAbsolutePath() + "'");
}
try (InputStream inStream = new BufferedInputStream(new FileInputStream(propertiesFile))) {
properties.load(inStream);
} catch (Exception ex) {
throw new RuntimeException("Cannot load properties file due to " + ex.getLocalizedMessage(), ex);
}
}
}
}

1
minifi/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/pom.xml
View File

@ -30,6 +30,7 @@ limitations under the License.
<module>minifi-runtime</module>
<module>minifi-resources</module>
<module>minifi-server</module>
<module>minifi-properties-loader</module>
</modules>
<dependencies>
<dependency>

13
minifi/minifi-nar-bundles/minifi-framework-bundle/pom.xml
View File

@ -71,8 +71,7 @@ limitations under the License.
</dependency>
<dependency>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-properties-loader</artifactId>
<version>2.0.0-SNAPSHOT</version>
<artifactId>minifi-properties-loader</artifactId>
</dependency>
<dependency>
<groupId>org.apache.nifi</groupId>
@ -80,6 +79,16 @@ limitations under the License.
<version>2.0.0-SNAPSHOT</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-property-protection-loader</artifactId>
<version>2.0.0-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-property-protection-cipher</artifactId>
<version>2.0.0-SNAPSHOT</version>
</dependency>
</dependencies>
</dependencyManagement>
</project>

90
minifi/minifi-toolkit/minifi-toolkit-assembly/README.md
View File

@ -19,18 +19,19 @@ MiNiFi is a child project effort of Apache NiFi. The MiNiFi toolkit aids in cre
## Table of Contents
- [Requirements](#requirements)
- [Getting Started](#getting-started)
- [MiNiFi Toolkit Converter](#minifi-toolkit-converter)
- [Encrypting Sensitive Properties in bootstrap.conf](#encrypt-sensitive-properties-in-bootstrapconf)
- [Getting Help](#getting-help)
- [Documentation](#documentation)
- [License](#license)
- [Export Control](#export-control)
## Requirements
* JRE 1.8
* JRE 21
## Getting Started
The latest version of the MiNiFi Toolkit can be found at https://nifi.apache.org/minifi/download.html under the `MiNiFi Toolkit Binaries` section.
The latest version of the MiNiFi Toolkit Converter can be found at https://nifi.apache.org/minifi/download.html under the `MiNiFi Toolkit Binaries` section.
# <a id="minifi-toolkit-converter" href="#minifi-toolkit-converter">MiNiFi Toolkit Converter</a>
After downloading the binary and extracting it, to run the MiNiFi Toolkit Converter:
- Change directory to the location where you installed MiNiFi Toolkit and run it and view usage information
@ -60,6 +61,87 @@ After downloading the binary and extracting it, to run the MiNiFi Toolkit Conver
## Note
It's not guaranteed in all circumstances that the migration will result in a correct flow. For example if a processor's configuration has changed between version, the conversion tool won't be aware of this, and will use the deprecated property names. You will need to fix such issues manually.
# <a id="encrypt-sensitive-properties-in-bootstrapconf" href="#encrypt-sensitive-properties-in-bootstrapconf">Encrypting Sensitive Properties in bootstrap.conf</a>
## MiNiFi Encrypt-Config Tool
The encrypt-config command line tool (invoked in minifi-toolkit as ./bin/encrypt-config.sh or bin\encrypt-config.bat) reads from a bootstrap.conf file with plaintext sensitive configuration values and encrypts each value using a random encryption key. It replaces the plain values with the protected value in the same file, or writes to a new bootstrap.conf file if specified.
The supported encryption algorithm utilized is AES/GCM 256-bit.
### Usage
To show help:
```
./bin/encrypt-config.sh -h
```
The following are the available options:
* -b, --bootstrapConf <bootstrapConfPath> Path to file containing Bootstrap Configuration [bootstrap.conf]
* -B, --outputBootstrapConf <outputBootstrapConf> Path to output file for Bootstrap Configuration [bootstrap.conf] with root key configured
* -h, --help Show help message and exit.
### Example
As an example of how the tool works with the following existing values in the bootstrap.conf file:
```
nifi.sensitive.props.key=thisIsABadSensitiveKeyPassword
nifi.sensitive.props.algorithm=NIFI_PBKDF2_AES_GCM_256
nifi.sensitive.props.additional.keys=
nifi.security.keystore=/path/to/keystore.jks
nifi.security.keystoreType=JKS
nifi.security.keystorePasswd=thisIsABadKeystorePassword
nifi.security.keyPasswd=thisIsABadKeyPassword
nifi.security.truststore=
nifi.security.truststoreType=
nifi.security.truststorePasswd=
c2.security.truststore.location=
c2.security.truststore.password=thisIsABadTruststorePassword
c2.security.truststore.type=JKS
c2.security.keystore.location=
c2.security.keystore.password=thisIsABadKeystorePassword
c2.security.keystore.type=JKS
```
Enter the following arguments when using the tool:
```
encrypt-config.sh \
-b bootstrap.conf \
```
As a result, the bootstrap.conf file is overwritten with protected properties and sibling encryption identifiers (aes/gcm/256, the currently supported algorithm):
```
nifi.sensitive.props.key=4OjkrFywZb7BlGz4||Tm9pg0jV4TltvVKeiMlm9zBsqmtmYUA2QkzcLKQpspyggtQuhNAkAla5s2695A==
nifi.sensitive.props.key.protected=aes/gcm/256
nifi.sensitive.props.algorithm=NIFI_PBKDF2_AES_GCM_256
nifi.sensitive.props.additional.keys=
nifi.security.keystore=/path/to/keystore.jks
nifi.security.keystoreType=JKS
nifi.security.keystorePasswd=iXDmDCadoNJ3VotZ||WvOGbrii4Gk0vr3b6mDstZg+NE0BPZUPk6LVqQlf2Sx3G5XFbUbUYAUz
nifi.security.keystorePasswd.protected=aes/gcm/256
nifi.security.keyPasswd=199uUUgpPqB4Fuoo||KckbW7iu+HZf1r4KSMQAFn8NLJK+CnUuayqPsTsdM0Wxou1BHg==
nifi.security.keyPasswd.protected=aes/gcm/256
nifi.security.truststore=
nifi.security.truststoreType=
nifi.security.truststorePasswd=
c2.security.truststore.location=
c2.security.truststore.password=0pHpp+l/WHsDM/sm||fXBvDAQ1BXvNQ8b4EHKa1GspsLx+UD+2EDhph0HbsdmgpVhEv4qj0q5TDo0=
c2.security.truststore.password.protected=aes/gcm/256
c2.security.truststore.type=JKS
c2.security.keystore.location=
c2.security.keystore.password=j+80L7++RNDf9INQ||RX/QkdVFwRos6Y4XJ8YSUWoI3W5Wx50dyw7HrAA84719SvfxA9eUSDEA
c2.security.keystore.password.protected=aes/gcm/256
c2.security.keystore.type=JKS
```
Additionally, the bootstrap.conf file is updated with the encryption key as follows:
```
minifi.bootstrap.sensitive.key=c92623e798be949379d0d18f432a57f1b74732141be321cb4af9ed94aa0ae8ac
```
Sensitive configuration values are encrypted by the tool by default, however you can encrypt any additional properties, if desired. To encrypt additional properties, specify them as comma-separated values in the minifi.sensitive.props.additional.keys property.
If the bootstrap.conf file already has valid protected values, those property values are not modified by the tool.
## Getting Help
If you have questions, you can reach out to our mailing list: dev@nifi.apache.org
([archive](https://mail-archives.apache.org/mod_mbox/nifi-dev)).

10
minifi/minifi-toolkit/minifi-toolkit-assembly/pom.xml
View File

@ -62,6 +62,16 @@ limitations under the License.
<artifactId>minifi-toolkit-configuration</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>org.apache.nifi.minifi</groupId>
<artifactId>minifi-toolkit-encrypt-config</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
<scope>compile</scope>
</dependency>
</dependencies>
<profiles>
<profile>

41
minifi/minifi-toolkit/minifi-toolkit-assembly/src/main/resources/bin/encrypt-config.bat Normal file
View File

@ -0,0 +1,41 @@
@echo off
rem
rem Licensed to the Apache Software Foundation (ASF) under one or more
rem contributor license agreements. See the NOTICE file distributed with
rem this work for additional information regarding copyright ownership.
rem The ASF licenses this file to You under the Apache License, Version 2.0
rem (the "License"); you may not use this file except in compliance with
rem the License. You may obtain a copy of the License at
rem
rem http://www.apache.org/licenses/LICENSE-2.0
rem
rem Unless required by applicable law or agreed to in writing, software
rem distributed under the License is distributed on an "AS IS" BASIS,
rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
rem See the License for the specific language governing permissions and
rem limitations under the License.
rem
rem Use JAVA_HOME if it's set; otherwise, just use java
if "%JAVA_HOME%" == "" goto noJavaHome
if not exist "%JAVA_HOME%\bin\java.exe" goto noJavaHome
set JAVA_EXE=%JAVA_HOME%\bin\java.exe
goto startConfig
:noJavaHome
echo The JAVA_HOME environment variable is not defined correctly.
echo Instead the PATH will be used to find the java executable.
echo.
set JAVA_EXE=java
goto startConfig
:startConfig
set LIB_DIR=%~dp0..\classpath;%~dp0..\lib
if "%JAVA_OPTS%" == "" set JAVA_OPTS=-Xms128m -Xmx256m
SET JAVA_PARAMS=-cp %LIB_DIR%\* %JAVA_OPTS% org.apache.nifi.minifi.toolkit.config.EncryptConfigCommand
cmd.exe /C ""%JAVA_EXE%" %JAVA_PARAMS% %* ""

119
minifi/minifi-toolkit/minifi-toolkit-assembly/src/main/resources/bin/encrypt-config.sh Normal file
View File

@ -0,0 +1,119 @@
#!/bin/sh
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
#
# Script structure inspired from Apache Karaf and other Apache projects with similar startup approaches
SCRIPT_DIR=$(dirname "$0")
SCRIPT_NAME=$(basename "$0")
MINIFI_TOOLKIT_HOME=$(cd "${SCRIPT_DIR}" && cd .. && pwd)
PROGNAME=$(basename "$0")
warn() {
echo "${PROGNAME}: $*"
}
die() {
warn "$*"
exit 1
}
detectOS() {
# OS specific support (must be 'true' or 'false').
cygwin=false;
aix=false;
os400=false;
darwin=false;
case "$(uname)" in
CYGWIN*)
cygwin=true
;;
AIX*)
aix=true
;;
OS400*)
os400=true
;;
Darwin)
darwin=true
;;
esac
# For AIX, set an environment variable
if ${aix}; then
export LDR_CNTRL=MAXDATA=0xB0000000@DSA
echo ${LDR_CNTRL}
fi
}
locateJava() {
# Setup the Java Virtual Machine
if $cygwin ; then
[ -n "${JAVA}" ] && JAVA=$(cygpath --unix "${JAVA}")
[ -n "${JAVA_HOME}" ] && JAVA_HOME=$(cygpath --unix "${JAVA_HOME}")
fi
if [ "x${JAVA}" = "x" ] && [ -r /etc/gentoo-release ] ; then
JAVA_HOME=$(java-config --jre-home)
fi
if [ "x${JAVA}" = "x" ]; then
if [ "x${JAVA_HOME}" != "x" ]; then
if [ ! -d "${JAVA_HOME}" ]; then
die "JAVA_HOME is not valid: ${JAVA_HOME}"
fi
JAVA="${JAVA_HOME}/bin/java"
else
warn "JAVA_HOME not set; results may vary"
JAVA=$(type java)
JAVA=$(expr "${JAVA}" : '.* \(/.*\)$')
if [ "x${JAVA}" = "x" ]; then
die "java command not found"
fi
fi
fi
}
init() {
# Determine if there is special OS handling we must perform
detectOS
# Locate the Java VM to execute
locateJava
}
run() {
LIBS="${MINIFI_TOOLKIT_HOME}/lib/*"
sudo_cmd_prefix=""
if $cygwin; then
MINIFI_TOOLKIT_HOME=$(cygpath --path --windows "${MINIFI_TOOLKIT_HOME}")
CLASSPATH="$MINIFI_TOOLKIT_HOME/classpath;$(cygpath --path --windows "${LIBS}")"
else
CLASSPATH="$MINIFI_TOOLKIT_HOME/classpath:${LIBS}"
fi
export JAVA_HOME="$JAVA_HOME"
export MINIFI_TOOLKIT_HOME="$MINIFI_TOOLKIT_HOME"
umask 0077
exec "${JAVA}" -cp "${CLASSPATH}" ${JAVA_OPTS:--Xms128m -Xmx256m} org.apache.nifi.minifi.toolkit.config.EncryptConfigCommand "$@"
}
init
run "$@"

18
minifi/minifi-toolkit/minifi-toolkit-configuration/pom.xml
View File

@ -35,10 +35,6 @@ limitations under the License.
<artifactId>minifi-toolkit-schema</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-framework-core</artifactId>
</dependency>
<dependency>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-property-utils</artifactId>
@ -48,6 +44,20 @@ limitations under the License.
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-properties</artifactId>
</dependency>
<dependency>
<groupId>commons-io</groupId>
<artifactId>commons-io</artifactId>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.module</groupId>
<artifactId>jackson-module-jaxb-annotations</artifactId>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.module</groupId>
<artifactId>jackson-module-jakarta-xmlbind-annotations</artifactId>
</dependency>
</dependencies>
<build>

57
minifi/minifi-toolkit/minifi-toolkit-encrypt-config/pom.xml Normal file
View File

@ -0,0 +1,57 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with
this work for additional information regarding copyright ownership.
The ASF licenses this file to You under the Apache License, Version 2.0
(the "License"); you may not use this file except in compliance with
the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
<parent>
<groupId>org.apache.nifi.minifi</groupId>
<artifactId>minifi-toolkit</artifactId>
<version>2.0.0-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<artifactId>minifi-toolkit-encrypt-config</artifactId>
<description>Tool to encrypt sensitive configuration values</description>
<dependencies>
<dependency>
<groupId>info.picocli</groupId>
<artifactId>picocli</artifactId>
<version>4.7.5</version>
</dependency>
<dependency>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-property-protection-api</artifactId>
<version>2.0.0-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-property-protection-cipher</artifactId>
<version>2.0.0-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>org.apache.nifi.minifi</groupId>
<artifactId>minifi-properties-loader</artifactId>
<version>2.0.0-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
<scope>compile</scope>
</dependency>
<dependency>
<groupId>org.apache.nifi.minifi</groupId>
<artifactId>minifi-commons-utils</artifactId>
</dependency>
</dependencies>
</project>

42
minifi/minifi-toolkit/minifi-toolkit-encrypt-config/src/main/java/org/apache/nifi/minifi/toolkit/config/EncryptConfigCommand.java Normal file
View File

@ -0,0 +1,42 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.nifi.minifi.toolkit.config;
import org.apache.nifi.minifi.toolkit.config.command.MiNiFiEncryptConfig;
import picocli.CommandLine;
/**
* Encrypt Config Command launcher for Command Line implementation
*/
public class EncryptConfigCommand {
/**
* Main command method launches Picocli Command Line implementation of Encrypt Config
*
* @param arguments Command line arguments
*/
public static void main(String[] arguments) {
CommandLine commandLine = new CommandLine(new MiNiFiEncryptConfig());
if (arguments.length == 0) {
commandLine.usage(System.out);
} else {
int status = commandLine.execute(arguments);
System.exit(status);
}
}
}

146
minifi/minifi-toolkit/minifi-toolkit-encrypt-config/src/main/java/org/apache/nifi/minifi/toolkit/config/command/MiNiFiEncryptConfig.java Normal file
View File

@ -0,0 +1,146 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.nifi.minifi.toolkit.config.command;
import java.io.IOException;
import java.io.UncheckedIOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.StandardCopyOption;
import java.security.SecureRandom;
import java.util.HashSet;
import java.util.HexFormat;
import java.util.Optional;
import java.util.Set;
import org.apache.nifi.minifi.properties.BootstrapProperties;
import org.apache.nifi.minifi.properties.BootstrapPropertiesLoader;
import org.apache.nifi.minifi.properties.ProtectedBootstrapProperties;
import org.apache.nifi.minifi.toolkit.config.transformer.ApplicationPropertiesFileTransformer;
import org.apache.nifi.minifi.toolkit.config.transformer.BootstrapConfigurationFileTransformer;
import org.apache.nifi.minifi.toolkit.config.transformer.FileTransformer;
import org.apache.nifi.properties.AesGcmSensitivePropertyProvider;
import org.apache.nifi.properties.SensitivePropertyProvider;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import picocli.CommandLine.Command;
import picocli.CommandLine.Option;
/**
* Shared Encrypt Configuration for NiFi and NiFi Registry
*/
@Command(
name = "encrypt-config",
sortOptions = false,
mixinStandardHelpOptions = true,
usageHelpWidth = 160,
separator = " ",
version = {
"Java ${java.version} (${java.vendor} ${java.vm.name} ${java.vm.version})"
},
descriptionHeading = "Description: ",
description = {
"encrypt-config supports protection of sensitive values in Apache MiNiFi"
}
)
public class MiNiFiEncryptConfig implements Runnable{
static final String BOOTSTRAP_ROOT_KEY_PROPERTY = "minifi.bootstrap.sensitive.key";
private static final String WORKING_FILE_NAME_FORMAT = "%s.%d.working";
private static final int KEY_LENGTH = 32;
@Option(
names = {"-b", "--bootstrapConf"},
description = "Path to file containing Bootstrap Configuration [bootstrap.conf] for optional root key and property protection scheme settings"
)
Path bootstrapConfPath;
@Option(
names = {"-B", "--outputBootstrapConf"},
description = "Path to output file for Bootstrap Configuration [bootstrap.conf] with root key configured"
)
Path outputBootstrapConf;
protected final Logger logger = LoggerFactory.getLogger(getClass());
@Override
public void run() {
processBootstrapConf();
}
/**
* Process bootstrap.conf writing new Root Key to specified Root Key Property when bootstrap.conf is specified
*
*/
protected void processBootstrapConf() {
BootstrapProperties unprotectedProperties = BootstrapPropertiesLoader.load(bootstrapConfPath.toFile());
logger.info("Started processing Bootstrap Configuration [{}]", bootstrapConfPath);
String newRootKey = getRootKey();
Set<String> sensitivePropertyNames = new HashSet<>((new ProtectedBootstrapProperties(unprotectedProperties)).getSensitivePropertyKeys());
FileTransformer fileTransformer2 = new ApplicationPropertiesFileTransformer(unprotectedProperties, getInputSensitivePropertyProvider(newRootKey), sensitivePropertyNames);
runFileTransformer(fileTransformer2, bootstrapConfPath, outputBootstrapConf);
FileTransformer fileTransformer = new BootstrapConfigurationFileTransformer(BOOTSTRAP_ROOT_KEY_PROPERTY, newRootKey);
runFileTransformer(fileTransformer, Optional.ofNullable(outputBootstrapConf).orElse(bootstrapConfPath), outputBootstrapConf);
logger.info("Completed processing Bootstrap Configuration [{}]", bootstrapConfPath);
}
private String getRootKey() {
SecureRandom secureRandom = new SecureRandom();
byte[] sensitivePropertiesKeyBinary = new byte[KEY_LENGTH];
secureRandom.nextBytes(sensitivePropertiesKeyBinary);
return HexFormat.of().formatHex(sensitivePropertiesKeyBinary);
}
/**
* Run File Transformer using working path based on output path
*
* @param fileTransformer File Transformer to be invoked
* @param inputPath Input path of file to be transformed
* @param outputPath Output path for transformed file that defaults to the input path when not specified
*/
protected void runFileTransformer(FileTransformer fileTransformer, Path inputPath, Path outputPath) {
Path configuredOutputPath = outputPath == null ? inputPath : outputPath;
Path workingPath = getWorkingPath(configuredOutputPath);
try {
fileTransformer.transform(inputPath, workingPath);
Files.move(workingPath, configuredOutputPath, StandardCopyOption.REPLACE_EXISTING);
} catch (IOException e) {
String message = String.format("Processing Configuration [%s] failed", inputPath);
throw new UncheckedIOException(message, e);
}
}
/**
* Get Input Sensitive Property Provider for decrypting previous values
*
* @return Input Sensitive Property Provider
*/
protected SensitivePropertyProvider getInputSensitivePropertyProvider(String keyHex) {
return new AesGcmSensitivePropertyProvider(keyHex);
}
private Path getWorkingPath(Path resourcePath) {
Path fileName = resourcePath.getFileName();
String workingFileName = String.format(WORKING_FILE_NAME_FORMAT, fileName, System.currentTimeMillis());
return resourcePath.resolveSibling(workingFileName);
}
}

152
minifi/minifi-toolkit/minifi-toolkit-encrypt-config/src/main/java/org/apache/nifi/minifi/toolkit/config/transformer/ApplicationPropertiesFileTransformer.java Normal file
View File

@ -0,0 +1,152 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.nifi.minifi.toolkit.config.transformer;
import static org.apache.nifi.properties.ApplicationPropertiesProtector.PROTECTED_KEY_SUFFIX;
import java.io.BufferedReader;
import java.io.BufferedWriter;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.Objects;
import java.util.Set;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import org.apache.nifi.properties.ApplicationProperties;
import org.apache.nifi.properties.ApplicationPropertiesProtector;
import org.apache.nifi.properties.ProtectedPropertyContext;
import org.apache.nifi.properties.SensitivePropertyProvider;
/**
* File Transformer supporting transformation of Application Properties with sensitive property values
*/
public class ApplicationPropertiesFileTransformer implements FileTransformer {
private static final Pattern PROPERTY_VALUE_PATTERN = Pattern.compile("^([^#!][^=]+?)\\s*=\\s?(.+)");
private static final int NAME_GROUP = 1;
private static final int VALUE_GROUP = 2;
private static final char PROPERTY_VALUE_SEPARATOR = '=';
private final ApplicationProperties applicationProperties;
private final SensitivePropertyProvider outputSensitivePropertyProvider;
private final Set<String> sensitivePropertyNames;
/**
* Application Properties File Transformer uses provided Application Properties as the source of protected values
*
* @param applicationProperties Application Properties containing decrypted source property values
* @param outputSensitivePropertyProvider Sensitive Property Provider encrypts specified sensitive property values
* @param sensitivePropertyNames Sensitive Property Names marked for encryption
*/
public ApplicationPropertiesFileTransformer(
ApplicationProperties applicationProperties,
SensitivePropertyProvider outputSensitivePropertyProvider,
Set<String> sensitivePropertyNames
) {
this.applicationProperties = Objects.requireNonNull(applicationProperties, "Application Properties required");
this.outputSensitivePropertyProvider = Objects.requireNonNull(outputSensitivePropertyProvider, "Output Property Provider required");
this.sensitivePropertyNames = Objects.requireNonNull(sensitivePropertyNames, "Sensitive Property Names required");
}
/**
* Transform input application properties using configured Sensitive Property Provider and write output properties
*
* @param inputPath Input file path to be transformed containing source application properties
* @param outputPath Output file path for protected application properties
* @throws IOException Thrown on transformation failures
*/
@Override
public void transform(Path inputPath, Path outputPath) throws IOException {
Objects.requireNonNull(inputPath, "Input path required");
Objects.requireNonNull(outputPath, "Output path required");
try (BufferedReader reader = Files.newBufferedReader(inputPath);
BufferedWriter writer = Files.newBufferedWriter(outputPath)) {
transform(reader, writer);
}
}
private void transform(BufferedReader reader, BufferedWriter writer) throws IOException {
String line = reader.readLine();
while (line != null) {
Matcher matcher = PROPERTY_VALUE_PATTERN.matcher(line);
String nextLine = null;
if (matcher.matches()) {
nextLine = processPropertyLine(reader, writer, matcher, line);
} else {
writeNonSensitiveLine(writer, line);
}
line = nextLine == null ? reader.readLine() : nextLine;
}
}
private String processPropertyLine(BufferedReader reader, BufferedWriter writer, Matcher matcher, String line) throws IOException {
String actualPropertyName = matcher.group(NAME_GROUP);
String actualPropertyValue = matcher.group(VALUE_GROUP);
String nextLine = null;
if (!actualPropertyName.endsWith(PROTECTED_KEY_SUFFIX)) {
nextLine = reader.readLine();
if (sensitivePropertyNames.contains(actualPropertyName) || isNextPropertyProtected(actualPropertyName, nextLine)) {
String applicationProperty = applicationProperties.getProperty(actualPropertyName, actualPropertyValue);
writeProtectedProperty(writer, actualPropertyName, applicationProperty);
} else {
writeNonSensitiveLine(writer, line);
}
}
return nextLine;
}
private void writeNonSensitiveLine(BufferedWriter writer, String line) throws IOException {
writer.write(line);
writer.newLine();
}
private boolean isNextPropertyProtected(String actualPropertyName, String nextLine) {
if (nextLine != null) {
Matcher nextLineMatcher = PROPERTY_VALUE_PATTERN.matcher(nextLine);
if (nextLineMatcher.matches()) {
String protectedActualPropertyName = ApplicationPropertiesProtector.getProtectionKey(actualPropertyName);
String nextName = nextLineMatcher.group(NAME_GROUP);
return protectedActualPropertyName.equals(nextName);
}
}
return false;
}
private void writeProtectedProperty(BufferedWriter writer, String name, String value) throws IOException {
ProtectedPropertyContext propertyContext = ProtectedPropertyContext.defaultContext(name);
String protectedValue = outputSensitivePropertyProvider.protect(value, propertyContext);
writer.write(name);
writer.write(PROPERTY_VALUE_SEPARATOR);
writeNonSensitiveLine(writer, protectedValue);
String protectedName = ApplicationPropertiesProtector.getProtectionKey(name);
writer.write(protectedName);
writer.write(PROPERTY_VALUE_SEPARATOR);
String protectionIdentifierKey = outputSensitivePropertyProvider.getIdentifierKey();
writeNonSensitiveLine(writer, protectionIdentifierKey);
}
}

108
minifi/minifi-toolkit/minifi-toolkit-encrypt-config/src/main/java/org/apache/nifi/minifi/toolkit/config/transformer/BootstrapConfigurationFileTransformer.java Normal file
View File

@ -0,0 +1,108 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.nifi.minifi.toolkit.config.transformer;
import java.io.BufferedReader;
import java.io.BufferedWriter;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.Objects;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
/**
* File Transformer supporting Bootstrap Configuration with updated Root Key
*/
public class BootstrapConfigurationFileTransformer implements FileTransformer {
private static final Pattern PROPERTY_VALUE_PATTERN = Pattern.compile("^([^#!][^=]+?)\\s*=.*");
private static final int NAME_GROUP = 1;
private static final char PROPERTY_VALUE_SEPARATOR = '=';
private final String rootKeyPropertyName;
private final String rootKey;
/**
* Bootstrap Configuration File Transformer writes provided Root Key to output files
*
* @param rootKeyPropertyName Root Key property name to be written
* @param rootKey Root Key to be written
*/
public BootstrapConfigurationFileTransformer(String rootKeyPropertyName, String rootKey) {
this.rootKeyPropertyName = Objects.requireNonNull(rootKeyPropertyName, "Root Key Property Name required");
this.rootKey = Objects.requireNonNull(rootKey, "Root Key required");
}
/**
* Transform input configuration and write Root Key to output location
*
* @param inputPath Input file path to be transformed containing Bootstrap Configuration
* @param outputPath Output file path for updated configuration
* @throws IOException Thrown on transformation failures
*/
@Override
public void transform(Path inputPath, Path outputPath) throws IOException {
Objects.requireNonNull(inputPath, "Input path required");
Objects.requireNonNull(outputPath, "Output path required");
try (BufferedReader reader = Files.newBufferedReader(inputPath);
BufferedWriter writer = Files.newBufferedWriter(outputPath)) {
transform(reader, writer);
}
}
private void transform(BufferedReader reader, BufferedWriter writer) throws IOException {
boolean rootKeyPropertyNotFound = true;
String line = reader.readLine();
while (line != null) {
Matcher matcher = PROPERTY_VALUE_PATTERN.matcher(line);
if (matcher.matches()) {
String name = matcher.group(NAME_GROUP);
if (rootKeyPropertyName.equals(name)) {
writeRootKey(writer);
rootKeyPropertyNotFound = false;
} else {
writer.write(line);
writer.newLine();
}
} else {
writer.write(line);
writer.newLine();
}
line = reader.readLine();
}
if (rootKeyPropertyNotFound) {
writer.newLine();
writeRootKey(writer);
}
}
private void writeRootKey(BufferedWriter writer) throws IOException {
writer.write(rootKeyPropertyName);
writer.write(PROPERTY_VALUE_SEPARATOR);
writer.write(rootKey);
writer.newLine();
}
}

34
minifi/minifi-toolkit/minifi-toolkit-encrypt-config/src/main/java/org/apache/nifi/minifi/toolkit/config/transformer/FileTransformer.java Normal file
View File

@ -0,0 +1,34 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.nifi.minifi.toolkit.config.transformer;
import java.io.IOException;
import java.nio.file.Path;
/**
* Abstraction for transforming Files
*/
public interface FileTransformer {
/**
* Transform input file and write contents to output file path
*
* @param inputPath Input file path to be transformed
* @param outputPath Output file path
* @throws IOException Thrown on input or output processing failures
*/
void transform(Path inputPath, Path outputPath) throws IOException;
}

99
minifi/minifi-toolkit/minifi-toolkit-encrypt-config/src/test/java/org/apache/nifi/minifi/toolkit/config/transformer/ApplicationPropertiesFileTransformerTest.java Normal file
View File

@ -0,0 +1,99 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.nifi.minifi.toolkit.config.transformer;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.Mockito.when;
import java.io.IOException;
import java.io.InputStream;
import java.net.URISyntaxException;
import java.net.URL;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.Map;
import java.util.Objects;
import java.util.Properties;
import java.util.Set;
import org.apache.nifi.properties.ApplicationProperties;
import org.apache.nifi.properties.SensitivePropertyProvider;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.junit.jupiter.api.io.TempDir;
import org.mockito.Mock;
import org.mockito.junit.jupiter.MockitoExtension;
@ExtendWith(MockitoExtension.class)
class ApplicationPropertiesFileTransformerTest {
private static final String UNPROTECTED_BOOTSTRAP_CONF = "/transformer/unprotected.conf";
private static final String PROTECTED_BOOTSTRAP_PROPERTIES = "/transformer/protected.conf";
private static final String PROPERTIES_TRANSFORMED = "transformed.conf";
private static final String SENSITIVE_PROPERTY_NAME_1 = "property1";
private static final String SENSITIVE_PROPERTY_NAME_2 = "property2";
private static final String SENSITIVE_PROPERTY_NAME_3 = "property3";
private static final String PROVIDER_IDENTIFIER_KEY = "mocked-provider";
private static final String UNPROTECTED = "UNPROTECTED";
private static final String ENCRYPTED = "ENCRYPTED";
private static final Set<String> SENSITIVE_PROPERTY_NAMES = Set.of(SENSITIVE_PROPERTY_NAME_1, SENSITIVE_PROPERTY_NAME_2, SENSITIVE_PROPERTY_NAME_3);
@TempDir
private Path tempDir;
@Mock
private SensitivePropertyProvider sensitivePropertyProvider;
@Test
void shouldTransformProperties() throws URISyntaxException, IOException {
Path propertiesPath = getResourcePath(UNPROTECTED_BOOTSTRAP_CONF);
Path tempPropertiesPath = tempDir.resolve(propertiesPath.getFileName());
Files.copy(propertiesPath, tempPropertiesPath);
Path outputPropertiesPath = tempDir.resolve(PROPERTIES_TRANSFORMED);
ApplicationProperties applicationProperties = new ApplicationProperties(Map.of(SENSITIVE_PROPERTY_NAME_3, UNPROTECTED));
FileTransformer transformer = new ApplicationPropertiesFileTransformer(applicationProperties, sensitivePropertyProvider, SENSITIVE_PROPERTY_NAMES);
when(sensitivePropertyProvider.getIdentifierKey()).thenReturn(PROVIDER_IDENTIFIER_KEY);
when(sensitivePropertyProvider.protect(eq(UNPROTECTED), any())).thenReturn(ENCRYPTED);
transformer.transform(tempPropertiesPath, outputPropertiesPath);
Properties expectedProperties = loadProperties(getResourcePath(PROTECTED_BOOTSTRAP_PROPERTIES));
Properties resultProperties = loadProperties(outputPropertiesPath);
assertEquals(expectedProperties, resultProperties);
}
private Properties loadProperties(Path resourcePath) throws IOException {
Properties properties = new Properties();
try (InputStream propertiesStream = Files.newInputStream(resourcePath)) {
properties.load(propertiesStream);
}
return properties;
}
private Path getResourcePath(String resource) throws URISyntaxException {
final URL resourceUrl = Objects.requireNonNull(getClass().getResource(resource), String.format("Resource [%s] not found", resource));
return Paths.get(resourceUrl.toURI());
}
}

74
minifi/minifi-toolkit/minifi-toolkit-encrypt-config/src/test/java/org/apache/nifi/minifi/toolkit/config/transformer/BootstrapConfigurationFileTransformerTest.java Normal file
View File

@ -0,0 +1,74 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.nifi.minifi.toolkit.config.transformer;
import static org.junit.jupiter.api.Assertions.assertEquals;
import java.io.IOException;
import java.io.InputStream;
import java.net.URISyntaxException;
import java.net.URL;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.Objects;
import java.util.Properties;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.junit.jupiter.api.io.TempDir;
import org.mockito.junit.jupiter.MockitoExtension;
@ExtendWith(MockitoExtension.class)
class BootstrapConfigurationFileTransformerTest {
private static final String BOOTSTRAP_ROOT_KEY_PROPERTY = "minifi.bootstrap.sensitive.key";
private static final String MOCK_KEY = "mockKey";
private static final String BOOTSTRAP_CONF_FILE_WITHOUT_KEY = "/transformer/bootstrap_without_key.conf";
private static final String BOOTSTRAP_CONF_TRANSFORMED= "transformed.conf";
@TempDir
private Path tempDir;
@Test
void shouldWriteRootPropertyKeyIfItIsNotPresent() throws URISyntaxException, IOException {
Path propertiesPath = getResourcePath(BOOTSTRAP_CONF_FILE_WITHOUT_KEY);
Path tempPropertiesPath = tempDir.resolve(propertiesPath.getFileName());
Files.copy(propertiesPath, tempPropertiesPath);
Path outputPropertiesPath = tempDir.resolve(BOOTSTRAP_CONF_TRANSFORMED);
BootstrapConfigurationFileTransformer transformer = new BootstrapConfigurationFileTransformer(BOOTSTRAP_ROOT_KEY_PROPERTY, MOCK_KEY);
transformer.transform(tempPropertiesPath, outputPropertiesPath);
Properties properties = loadProperties(outputPropertiesPath);
assertEquals(MOCK_KEY, properties.get(BOOTSTRAP_ROOT_KEY_PROPERTY));
}
private Properties loadProperties(Path resourcePath) throws IOException {
Properties properties = new Properties();
try (InputStream propertiesStream = Files.newInputStream(resourcePath)) {
properties.load(propertiesStream);
}
return properties;
}
private Path getResourcePath(String resource) throws URISyntaxException {
final URL resourceUrl = Objects.requireNonNull(getClass().getResource(resource), String.format("Resource [%s] not found", resource));
return Paths.get(resourceUrl.toURI());
}
}

14
minifi/minifi-toolkit/minifi-toolkit-encrypt-config/src/test/resources/transformer/bootstrap_without_key.conf Normal file
View File

@ -0,0 +1,14 @@
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

22
minifi/minifi-toolkit/minifi-toolkit-encrypt-config/src/test/resources/transformer/protected.conf Normal file
View File

@ -0,0 +1,22 @@
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
property1=ENCRYPTED
property1.protected=mocked-provider
property2=ENCRYPTED
property2.protected=mocked-provider
nonsensitive.property=value
property3=ENCRYPTED
property3.protected=mocked-provider

20
minifi/minifi-toolkit/minifi-toolkit-encrypt-config/src/test/resources/transformer/unprotected.conf Normal file
View File

@ -0,0 +1,20 @@
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
property1=UNPROTECTED
property2=UNPROTECTED
nonsensitive.property=value
property3=ENCRYPTED
property3.protected=mocked-provider

8
minifi/minifi-toolkit/pom.xml
View File

@ -29,5 +29,13 @@ limitations under the License.
<module>minifi-toolkit-schema</module>
<module>minifi-toolkit-configuration</module>
<module>minifi-toolkit-assembly</module>
<module>minifi-toolkit-encrypt-config</module>
</modules>
<dependencies>
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-simple</artifactId>
</dependency>
</dependencies>
</project>

5
minifi/pom.xml
View File

@ -86,6 +86,11 @@ limitations under the License.
<artifactId>minifi-resources</artifactId>
<version>2.0.0-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>org.apache.nifi.minifi</groupId>
<artifactId>minifi-properties-loader</artifactId>
<version>2.0.0-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>org.apache.nifi.minifi</groupId>
<artifactId>minifi-provenance-repository-nar</artifactId>

2
nifi-commons/nifi-property-protection-cipher/src/main/java/org/apache/nifi/properties/AesGcmSensitivePropertyProvider.java
View File

@ -39,7 +39,7 @@ import java.util.Objects;
/**
* Sensitive Property Provider implementation using AES-GCM with configurable key sizes
*/
class AesGcmSensitivePropertyProvider implements SensitivePropertyProvider {
public class AesGcmSensitivePropertyProvider implements SensitivePropertyProvider {
private static final String ALGORITHM = "AES/GCM/NoPadding";
private static final String SECRET_KEY_ALGORITHM = "AES";
private static final String DELIMITER = "||"; // "|" is not a valid Base64 character, so ensured not to be present in cipher text