diff --git a/minifi/minifi-bootstrap/pom.xml b/minifi/minifi-bootstrap/pom.xml index bbab3d59aa..8f96c8f6be 100644 --- a/minifi/minifi-bootstrap/pom.xml +++ b/minifi/minifi-bootstrap/pom.xml @@ -86,6 +86,13 @@ limitations under the License. minifi-commons-api 2.0.0-SNAPSHOT + + + org.apache.nifi.minifi + minifi-properties-loader + 2.0.0-SNAPSHOT + org.apache.nifi.minifi minifi-commons-framework diff --git a/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/RunMiNiFi.java b/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/RunMiNiFi.java index cbd23dcfac..4d00feee1a 100644 --- a/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/RunMiNiFi.java +++ b/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/RunMiNiFi.java @@ -48,7 +48,7 @@ import org.apache.nifi.minifi.bootstrap.util.UnixProcessUtils; import org.apache.nifi.minifi.commons.api.MiNiFiCommandState; import org.apache.nifi.minifi.commons.service.StandardFlowEnrichService; import org.apache.nifi.minifi.commons.service.StandardFlowSerDeService; -import org.apache.nifi.properties.ApplicationProperties; +import org.apache.nifi.minifi.properties.BootstrapProperties; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -103,7 +103,7 @@ public class RunMiNiFi implements ConfigurationFileHolder { bootstrapFileProvider = new BootstrapFileProvider(bootstrapConfigFile); objectMapper = getObjectMapper(); Properties properties = bootstrapFileProvider.getStatusProperties(); - Properties bootstrapProperties = bootstrapFileProvider.getBootstrapProperties(); + BootstrapProperties bootstrapProperties = bootstrapFileProvider.getBootstrapProperties(); miNiFiParameters = new MiNiFiParameters( Optional.ofNullable(properties.getProperty(STATUS_FILE_PORT_KEY)).map(Integer::parseInt).orElse(UNINITIALIZED), @@ -118,7 +118,7 @@ public class RunMiNiFi implements ConfigurationFileHolder { periodicStatusReporterManager = new PeriodicStatusReporterManager(bootstrapProperties, miNiFiStatusProvider, miNiFiCommandSender, miNiFiParameters); MiNiFiConfigurationChangeListener configurationChangeListener = new MiNiFiConfigurationChangeListener(this, DEFAULT_LOGGER, bootstrapFileProvider, - new StandardFlowEnrichService(new ApplicationProperties(bootstrapProperties)), StandardFlowSerDeService.defaultInstance()); + new StandardFlowEnrichService(bootstrapProperties), StandardFlowSerDeService.defaultInstance()); configurationChangeCoordinator = new ConfigurationChangeCoordinator(bootstrapFileProvider, this, singleton(configurationChangeListener)); currentPortProvider = new CurrentPortProvider(miNiFiCommandSender, miNiFiParameters, processUtils); diff --git a/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/command/StartRunner.java b/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/command/StartRunner.java index 827f6ef171..af331fcc2b 100644 --- a/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/command/StartRunner.java +++ b/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/command/StartRunner.java @@ -55,6 +55,7 @@ import org.apache.nifi.minifi.bootstrap.service.MiNiFiPropertiesGenerator; import org.apache.nifi.minifi.bootstrap.service.MiNiFiStdLogHandler; import org.apache.nifi.minifi.bootstrap.service.PeriodicStatusReporterManager; import org.apache.nifi.minifi.commons.api.MiNiFiProperties; +import org.apache.nifi.minifi.properties.BootstrapProperties; public class StartRunner implements CommandRunner { @@ -122,10 +123,10 @@ public class StartRunner implements CommandRunner { CMD_LOGGER.warn("Failed to delete previous lock file {}; this file should be cleaned up manually", prevLockFile); } - Properties bootstrapProperties = bootstrapFileProvider.getBootstrapProperties(); + BootstrapProperties bootstrapProperties = bootstrapFileProvider.getBootstrapProperties(); String confDir = bootstrapProperties.getProperty(CONF_DIR_KEY); - generateMiNiFiProperties(bootstrapProperties, confDir); + generateMiNiFiProperties(bootstrapFileProvider.getProtectedBootstrapProperties(), confDir); regenerateFlowConfiguration(bootstrapProperties.getProperty(MiNiFiProperties.NIFI_MINIFI_FLOW_CONFIG.getKey())); Process process = startMiNiFi(); @@ -259,7 +260,7 @@ public class StartRunner implements CommandRunner { } } - private void generateMiNiFiProperties(Properties bootstrapProperties, String confDir) { + private void generateMiNiFiProperties(BootstrapProperties bootstrapProperties, String confDir) { DEFAULT_LOGGER.debug("Generating minifi.properties from bootstrap.conf"); try { miNiFiPropertiesGenerator.generateMinifiProperties(confDir, bootstrapProperties); @@ -317,7 +318,7 @@ public class StartRunner implements CommandRunner { } private File getWorkingDir() throws IOException { - Properties props = bootstrapFileProvider.getBootstrapProperties(); + BootstrapProperties props = bootstrapFileProvider.getBootstrapProperties(); File bootstrapConfigAbsoluteFile = bootstrapConfigFile.getAbsoluteFile(); File binDir = bootstrapConfigAbsoluteFile.getParentFile(); diff --git a/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/configuration/ConfigurationChangeCoordinator.java b/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/configuration/ConfigurationChangeCoordinator.java index b6a11d2564..fb187ae857 100644 --- a/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/configuration/ConfigurationChangeCoordinator.java +++ b/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/configuration/ConfigurationChangeCoordinator.java @@ -17,26 +17,26 @@ package org.apache.nifi.minifi.bootstrap.configuration; +import static java.util.Optional.ofNullable; +import static java.util.function.Predicate.not; +import static java.util.stream.Collectors.toList; + import java.io.Closeable; import java.io.IOException; import java.nio.ByteBuffer; import java.util.Collection; import java.util.Collections; import java.util.HashSet; -import java.util.Properties; import java.util.Set; import java.util.stream.Stream; import org.apache.nifi.minifi.bootstrap.RunMiNiFi; import org.apache.nifi.minifi.bootstrap.configuration.ingestors.interfaces.ChangeIngestor; import org.apache.nifi.minifi.bootstrap.service.BootstrapFileProvider; import org.apache.nifi.minifi.bootstrap.util.ByteBufferInputStream; +import org.apache.nifi.minifi.properties.BootstrapProperties; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import static java.util.Optional.ofNullable; -import static java.util.function.Predicate.not; -import static java.util.stream.Collectors.toList; - public class ConfigurationChangeCoordinator implements Closeable, ConfigurationChangeNotifier { public static final String NOTIFIER_INGESTORS_KEY = "nifi.minifi.notifier.ingestors"; @@ -94,7 +94,7 @@ public class ConfigurationChangeCoordinator implements Closeable, ConfigurationC private void initialize() throws IOException { closeIngestors(); - Properties bootstrapProperties = bootstrapFileProvider.getBootstrapProperties(); + BootstrapProperties bootstrapProperties = bootstrapFileProvider.getBootstrapProperties(); ofNullable(bootstrapProperties.getProperty(NOTIFIER_INGESTORS_KEY)) .filter(not(String::isBlank)) .map(ingestors -> ingestors.split(COMMA)) @@ -115,7 +115,7 @@ public class ConfigurationChangeCoordinator implements Closeable, ConfigurationC } } - private void instantiateIngestor(Properties bootstrapProperties, String ingestorClassname) { + private void instantiateIngestor(BootstrapProperties bootstrapProperties, String ingestorClassname) { try { Class ingestorClass = Class.forName(ingestorClassname); ChangeIngestor changeIngestor = (ChangeIngestor) ingestorClass.getDeclaredConstructor().newInstance(); diff --git a/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/configuration/ingestors/AbstractPullChangeIngestor.java b/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/configuration/ingestors/AbstractPullChangeIngestor.java index 8d3b225df7..14c2405a59 100644 --- a/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/configuration/ingestors/AbstractPullChangeIngestor.java +++ b/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/configuration/ingestors/AbstractPullChangeIngestor.java @@ -18,7 +18,6 @@ package org.apache.nifi.minifi.bootstrap.configuration.ingestors; import java.io.IOException; -import java.util.Properties; import java.util.concurrent.ScheduledThreadPoolExecutor; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; @@ -26,20 +25,21 @@ import java.util.concurrent.atomic.AtomicReference; import org.apache.nifi.minifi.bootstrap.ConfigurationFileHolder; import org.apache.nifi.minifi.bootstrap.configuration.ConfigurationChangeNotifier; import org.apache.nifi.minifi.bootstrap.configuration.ingestors.interfaces.ChangeIngestor; +import org.apache.nifi.minifi.properties.BootstrapProperties; public abstract class AbstractPullChangeIngestor implements Runnable, ChangeIngestor { protected static final String DEFAULT_POLLING_PERIOD_MILLISECONDS = "300000"; protected final AtomicInteger pollingPeriodMS = new AtomicInteger(); - protected final AtomicReference properties = new AtomicReference<>(); + protected final AtomicReference properties = new AtomicReference<>(); private final ScheduledThreadPoolExecutor scheduledThreadPoolExecutor = new ScheduledThreadPoolExecutor(1); protected volatile ConfigurationChangeNotifier configurationChangeNotifier; @Override - public void initialize(Properties properties, ConfigurationFileHolder configurationFileHolder, ConfigurationChangeNotifier configurationChangeNotifier) { + public void initialize(BootstrapProperties properties, ConfigurationFileHolder configurationFileHolder, ConfigurationChangeNotifier configurationChangeNotifier) { this.configurationChangeNotifier = configurationChangeNotifier; this.properties.set(properties); } diff --git a/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/configuration/ingestors/FileChangeIngestor.java b/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/configuration/ingestors/FileChangeIngestor.java index 95a49a0d99..a77b41f004 100644 --- a/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/configuration/ingestors/FileChangeIngestor.java +++ b/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/configuration/ingestors/FileChangeIngestor.java @@ -39,7 +39,6 @@ import java.nio.file.WatchKey; import java.nio.file.WatchService; import java.util.Map; import java.util.Optional; -import java.util.Properties; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; @@ -50,6 +49,7 @@ import org.apache.nifi.minifi.bootstrap.configuration.differentiators.Differenti import org.apache.nifi.minifi.bootstrap.configuration.differentiators.WholeConfigDifferentiator; import org.apache.nifi.minifi.bootstrap.configuration.ingestors.interfaces.ChangeIngestor; import org.apache.nifi.minifi.commons.api.MiNiFiProperties; +import org.apache.nifi.minifi.properties.BootstrapProperties; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -84,7 +84,7 @@ public class FileChangeIngestor implements Runnable, ChangeIngestor { private long pollingSeconds; @Override - public void initialize(Properties properties, ConfigurationFileHolder configurationFileHolder, ConfigurationChangeNotifier configurationChangeNotifier) { + public void initialize(BootstrapProperties properties, ConfigurationFileHolder configurationFileHolder, ConfigurationChangeNotifier configurationChangeNotifier) { Path configFile = ofNullable(properties.getProperty(CONFIG_FILE_PATH_KEY)) .filter(not(String::isBlank)) .map(Path::of) @@ -190,7 +190,7 @@ public class FileChangeIngestor implements Runnable, ChangeIngestor { + " which does not correspond to any in the FileChangeIngestor Map:" + DIFFERENTIATOR_CONSTRUCTOR_MAP.keySet()); } - private void checkConfigFileLocationCorrectness(Properties properties, Path configFile) { + private void checkConfigFileLocationCorrectness(BootstrapProperties properties, Path configFile) { Path flowConfigFile = Path.of(properties.getProperty(MiNiFiProperties.NIFI_MINIFI_FLOW_CONFIG.getKey())).toAbsolutePath(); Path rawFlowConfigFile = flowConfigFile.getParent().resolve(getBaseName(flowConfigFile.toString()) + RAW_EXTENSION); if (flowConfigFile.equals(configFile) || rawFlowConfigFile.equals(configFile)) { diff --git a/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/configuration/ingestors/PullHttpChangeIngestor.java b/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/configuration/ingestors/PullHttpChangeIngestor.java index 0b1238f1f4..21d666f02f 100644 --- a/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/configuration/ingestors/PullHttpChangeIngestor.java +++ b/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/configuration/ingestors/PullHttpChangeIngestor.java @@ -41,7 +41,6 @@ import java.nio.ByteBuffer; import java.security.KeyStore; import java.util.Arrays; import java.util.Map; -import java.util.Properties; import java.util.concurrent.atomic.AtomicReference; import java.util.function.Supplier; import java.util.stream.Stream; @@ -59,6 +58,7 @@ import org.apache.nifi.minifi.bootstrap.ConfigurationFileHolder; import org.apache.nifi.minifi.bootstrap.configuration.ConfigurationChangeNotifier; import org.apache.nifi.minifi.bootstrap.configuration.differentiators.Differentiator; import org.apache.nifi.minifi.bootstrap.configuration.differentiators.WholeConfigDifferentiator; +import org.apache.nifi.minifi.properties.BootstrapProperties; import org.apache.nifi.security.ssl.StandardKeyStoreBuilder; import org.apache.nifi.security.ssl.StandardSslContextBuilder; import org.apache.nifi.security.ssl.StandardTrustManagerBuilder; @@ -91,18 +91,18 @@ public class PullHttpChangeIngestor extends AbstractPullChangeIngestor { public static final String OVERRIDE_SECURITY = PULL_HTTP_BASE_KEY + ".override.security"; public static final String HTTP_HEADERS = PULL_HTTP_BASE_KEY + ".headers"; + protected static final String DEFAULT_CONNECT_TIMEOUT_MS = "5000"; + protected static final String DEFAULT_READ_TIMEOUT_MS = "15000"; + protected static final String DEFAULT_PATH = "/"; private static final Logger logger = LoggerFactory.getLogger(PullHttpChangeIngestor.class); private static final Map>> DIFFERENTIATOR_CONSTRUCTOR_MAP = Map.of( WHOLE_CONFIG_KEY, WholeConfigDifferentiator::getByteBufferDifferentiator ); private static final int NOT_MODIFIED_STATUS_CODE = 304; - private static final String DEFAULT_CONNECT_TIMEOUT_MS = "5000"; - private static final String DEFAULT_READ_TIMEOUT_MS = "15000"; private static final String DOUBLE_QUOTES = "\""; private static final String ETAG_HEADER = "ETag"; private static final String PROXY_AUTHORIZATION_HEADER = "Proxy-Authorization"; - private static final String DEFAULT_PATH = "/"; private static final int BAD_REQUEST_STATUS_CODE = 400; private static final String IF_NONE_MATCH_HEADER_KEY = "If-None-Match"; private static final String HTTP_HEADERS_SEPARATOR = ","; @@ -121,7 +121,7 @@ public class PullHttpChangeIngestor extends AbstractPullChangeIngestor { private volatile boolean useEtag = false; @Override - public void initialize(Properties properties, ConfigurationFileHolder configurationFileHolder, ConfigurationChangeNotifier configurationChangeNotifier) { + public void initialize(BootstrapProperties properties, ConfigurationFileHolder configurationFileHolder, ConfigurationChangeNotifier configurationChangeNotifier) { super.initialize(properties, configurationFileHolder, configurationChangeNotifier); pollingPeriodMS.set(Integer.parseInt(properties.getProperty(PULL_HTTP_POLLING_PERIOD_KEY, DEFAULT_POLLING_PERIOD_MILLISECONDS))); @@ -145,8 +145,7 @@ public class PullHttpChangeIngestor extends AbstractPullChangeIngestor { .collect(toMap(split -> ofNullable(split[0]).map(String::trim).orElse(EMPTY), split -> ofNullable(split[1]).map(String::trim).orElse(EMPTY))); logger.debug("Configured HTTP headers: {}", httpHeaders); - ofNullable(properties.get(PORT_KEY)) - .map(String.class::cast) + ofNullable(properties.getProperty(PORT_KEY)) .map(Integer::parseInt) .ifPresentOrElse( portReference::set, @@ -157,7 +156,7 @@ public class PullHttpChangeIngestor extends AbstractPullChangeIngestor { pathReference.set(path); queryReference.set(query); httpHeadersReference.set(httpHeaders); - useEtag = parseBoolean((String) properties.getOrDefault(USE_ETAG_KEY, FALSE.toString())); + useEtag = parseBoolean(properties.getProperty(USE_ETAG_KEY, FALSE.toString())); httpClientReference.set(null); @@ -263,7 +262,7 @@ public class PullHttpChangeIngestor extends AbstractPullChangeIngestor { } } - private void setSslSocketFactory(OkHttpClient.Builder okHttpClientBuilder, Properties properties) { + private void setSslSocketFactory(OkHttpClient.Builder okHttpClientBuilder, BootstrapProperties properties) { String keystorePass = properties.getProperty(KEYSTORE_PASSWORD_KEY); KeyStore keyStore = buildKeyStore(properties, KEYSTORE_LOCATION_KEY, KEYSTORE_PASSWORD_KEY, KEYSTORE_TYPE_KEY); KeyStore truststore = buildKeyStore(properties, TRUSTSTORE_LOCATION_KEY, TRUSTSTORE_PASSWORD_KEY, TRUSTSTORE_TYPE_KEY); @@ -279,7 +278,7 @@ public class PullHttpChangeIngestor extends AbstractPullChangeIngestor { okHttpClientBuilder.sslSocketFactory(sslSocketFactory, trustManager); } - private KeyStore buildKeyStore(Properties properties, String locationKey, String passKey, String typeKey) { + private KeyStore buildKeyStore(BootstrapProperties properties, String locationKey, String passKey, String typeKey) { String keystoreLocation = ofNullable(properties.getProperty(locationKey)) .filter(StringUtils::isNotBlank) .orElseThrow(() -> new IllegalArgumentException(locationKey + " is null or empty")); diff --git a/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/configuration/ingestors/RestChangeIngestor.java b/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/configuration/ingestors/RestChangeIngestor.java index 1b9c390125..d8c35984cf 100644 --- a/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/configuration/ingestors/RestChangeIngestor.java +++ b/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/configuration/ingestors/RestChangeIngestor.java @@ -32,7 +32,6 @@ import java.net.URI; import java.nio.ByteBuffer; import java.security.KeyStore; import java.util.Map; -import java.util.Properties; import java.util.function.Supplier; import javax.net.ssl.SSLContext; import org.apache.nifi.jetty.configuration.connector.StandardServerConnectorFactory; @@ -42,6 +41,7 @@ import org.apache.nifi.minifi.bootstrap.configuration.ListenerHandleResult; import org.apache.nifi.minifi.bootstrap.configuration.differentiators.Differentiator; import org.apache.nifi.minifi.bootstrap.configuration.differentiators.WholeConfigDifferentiator; import org.apache.nifi.minifi.bootstrap.configuration.ingestors.interfaces.ChangeIngestor; +import org.apache.nifi.minifi.properties.BootstrapProperties; import org.apache.nifi.security.ssl.StandardKeyStoreBuilder; import org.apache.nifi.security.ssl.StandardSslContextBuilder; import org.apache.nifi.security.util.KeystoreType; @@ -100,7 +100,7 @@ public class RestChangeIngestor implements ChangeIngestor { } @Override - public void initialize(Properties properties, ConfigurationFileHolder configurationFileHolder, ConfigurationChangeNotifier configurationChangeNotifier) { + public void initialize(BootstrapProperties properties, ConfigurationFileHolder configurationFileHolder, ConfigurationChangeNotifier configurationChangeNotifier) { logger.info("Initializing RestChangeIngestor"); this.differentiator = ofNullable(properties.getProperty(DIFFERENTIATOR_KEY)) .filter(not(String::isBlank)) @@ -154,7 +154,7 @@ public class RestChangeIngestor implements ChangeIngestor { return ((ServerConnector) jetty.getConnectors()[0]).getLocalPort(); } - private void createConnector(Properties properties) { + private void createConnector(BootstrapProperties properties) { ServerConnector http = new ServerConnector(jetty); http.setPort(Integer.parseInt(properties.getProperty(PORT_KEY, "0"))); @@ -167,7 +167,7 @@ public class RestChangeIngestor implements ChangeIngestor { logger.info("Added an http connector on the host '{}' and port '{}'", http.getHost(), http.getPort()); } - private void createSecureConnector(Properties properties) { + private void createSecureConnector(BootstrapProperties properties) { KeyStore keyStore; KeyStore trustStore = null; diff --git a/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/configuration/ingestors/interfaces/ChangeIngestor.java b/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/configuration/ingestors/interfaces/ChangeIngestor.java index dcd39cd6d1..96e5c4e50c 100644 --- a/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/configuration/ingestors/interfaces/ChangeIngestor.java +++ b/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/configuration/ingestors/interfaces/ChangeIngestor.java @@ -17,14 +17,13 @@ package org.apache.nifi.minifi.bootstrap.configuration.ingestors.interfaces; +import java.io.IOException; import org.apache.nifi.minifi.bootstrap.ConfigurationFileHolder; import org.apache.nifi.minifi.bootstrap.configuration.ConfigurationChangeNotifier; - -import java.io.IOException; -import java.util.Properties; +import org.apache.nifi.minifi.properties.BootstrapProperties; public interface ChangeIngestor { - void initialize(Properties properties, ConfigurationFileHolder configurationFileHolder, ConfigurationChangeNotifier configurationChangeNotifier); + void initialize(BootstrapProperties properties, ConfigurationFileHolder configurationFileHolder, ConfigurationChangeNotifier configurationChangeNotifier); void start(); diff --git a/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/service/BootstrapFileProvider.java b/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/service/BootstrapFileProvider.java index 1957b05aec..96cc04ad29 100644 --- a/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/service/BootstrapFileProvider.java +++ b/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/service/BootstrapFileProvider.java @@ -22,7 +22,6 @@ import static org.apache.nifi.minifi.commons.api.MiNiFiConstants.BOOTSTRAP_UPDAT import java.io.File; import java.io.FileInputStream; -import java.io.FileNotFoundException; import java.io.FileOutputStream; import java.io.IOException; import java.nio.charset.StandardCharsets; @@ -35,6 +34,8 @@ import java.util.Properties; import java.util.Set; import java.util.stream.Collectors; import org.apache.commons.lang3.StringUtils; +import org.apache.nifi.minifi.properties.BootstrapProperties; +import org.apache.nifi.minifi.properties.BootstrapPropertiesLoader; import org.apache.nifi.util.file.FileUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -61,6 +62,10 @@ public class BootstrapFileProvider { this.bootstrapConfigFile = bootstrapConfigFile; } + public String getBootstrapFilePath() { + return bootstrapConfigFile.getAbsolutePath(); + } + public static File getBootstrapConfFile() { File bootstrapConfigFile = Optional.ofNullable(System.getProperty(BOOTSTRAP_CONFIG_FILE_SYSTEM_PROPERTY_KEY)) .map(File::new) @@ -110,19 +115,12 @@ public class BootstrapFileProvider { return newFile; } - public Properties getBootstrapProperties() throws IOException { - if (!bootstrapConfigFile.exists()) { - throw new FileNotFoundException(bootstrapConfigFile.getAbsolutePath()); - } + public BootstrapProperties getBootstrapProperties() { + return BootstrapPropertiesLoader.load(bootstrapConfigFile); + } - Properties bootstrapProperties = BootstrapProperties.getInstance(); - try (FileInputStream fis = new FileInputStream(bootstrapConfigFile)) { - bootstrapProperties.load(fis); - } - - logProperties("Bootstrap", bootstrapProperties); - - return bootstrapProperties; + public BootstrapProperties getProtectedBootstrapProperties() { + return BootstrapPropertiesLoader.loadProtectedProperties(bootstrapConfigFile).getApplicationProperties(); } public Properties getStatusProperties() { diff --git a/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/service/GracefulShutdownParameterProvider.java b/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/service/GracefulShutdownParameterProvider.java index b75c0157e1..ebe6f36b31 100644 --- a/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/service/GracefulShutdownParameterProvider.java +++ b/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/service/GracefulShutdownParameterProvider.java @@ -18,7 +18,7 @@ package org.apache.nifi.minifi.bootstrap.service; import java.io.IOException; -import java.util.Properties; +import org.apache.nifi.minifi.properties.BootstrapProperties; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -36,7 +36,7 @@ public class GracefulShutdownParameterProvider { } public int getGracefulShutdownSeconds() throws IOException { - Properties bootstrapProperties = bootstrapFileProvider.getBootstrapProperties(); + BootstrapProperties bootstrapProperties = bootstrapFileProvider.getBootstrapProperties(); String gracefulShutdown = bootstrapProperties.getProperty(GRACEFUL_SHUTDOWN_PROP, DEFAULT_GRACEFUL_SHUTDOWN_VALUE); diff --git a/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/service/MiNiFiConfigurationChangeListener.java b/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/service/MiNiFiConfigurationChangeListener.java index ea0f04dd84..5925ac3b5e 100644 --- a/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/service/MiNiFiConfigurationChangeListener.java +++ b/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/service/MiNiFiConfigurationChangeListener.java @@ -32,7 +32,6 @@ import java.io.IOException; import java.io.InputStream; import java.nio.ByteBuffer; import java.nio.file.Path; -import java.util.Properties; import java.util.concurrent.locks.ReentrantLock; import org.apache.commons.io.FilenameUtils; import org.apache.nifi.controller.flow.VersionedDataflow; @@ -42,6 +41,7 @@ import org.apache.nifi.minifi.bootstrap.configuration.ConfigurationChangeListene import org.apache.nifi.minifi.commons.api.MiNiFiProperties; import org.apache.nifi.minifi.commons.service.FlowEnrichService; import org.apache.nifi.minifi.commons.service.FlowSerDeService; +import org.apache.nifi.minifi.properties.BootstrapProperties; import org.slf4j.Logger; public class MiNiFiConfigurationChangeListener implements ConfigurationChangeListener { @@ -76,7 +76,7 @@ public class MiNiFiConfigurationChangeListener implements ConfigurationChangeLis Path currentRawFlowConfigFile = null; Path backupRawFlowConfigFile = null; try { - Properties bootstrapProperties = bootstrapFileProvider.getBootstrapProperties(); + BootstrapProperties bootstrapProperties = bootstrapFileProvider.getBootstrapProperties(); currentFlowConfigFile = Path.of(bootstrapProperties.getProperty(MiNiFiProperties.NIFI_MINIFI_FLOW_CONFIG.getKey())).toAbsolutePath(); backupFlowConfigFile = Path.of(currentFlowConfigFile + BACKUP_EXTENSION); diff --git a/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/service/MiNiFiExecCommandProvider.java b/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/service/MiNiFiExecCommandProvider.java index 8f2b0fce08..0e99379a6d 100644 --- a/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/service/MiNiFiExecCommandProvider.java +++ b/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/service/MiNiFiExecCommandProvider.java @@ -28,12 +28,13 @@ import java.io.IOException; import java.nio.file.Path; import java.util.List; import java.util.Optional; -import java.util.Properties; import java.util.stream.Stream; +import org.apache.nifi.minifi.properties.BootstrapProperties; public class MiNiFiExecCommandProvider { public static final String LOG_DIR = "org.apache.nifi.minifi.bootstrap.config.log.dir"; + public static final String MINIFI_BOOTSTRAP_CONF_FILE_PATH = "minifi.bootstrap.conf.file.path"; public static final String DEFAULT_LOG_DIR = "./logs"; public static final String APP_LOG_FILE_NAME = "org.apache.nifi.minifi.bootstrap.config.log.app.file.name"; @@ -73,7 +74,7 @@ public class MiNiFiExecCommandProvider { this.bootstrapFileProvider = bootstrapFileProvider; } - public static String getMiNiFiPropertiesPath(Properties props, File confDir) { + public static String getMiNiFiPropertiesPath(BootstrapProperties props, File confDir) { return ofNullable(props.getProperty(PROPERTIES_FILE_KEY)) .orElseGet(() -> ofNullable(confDir) .filter(File::exists) @@ -92,7 +93,7 @@ public class MiNiFiExecCommandProvider { * @throws IOException throws IOException if any of the configuration file read fails */ public List getMiNiFiExecCommand(int listenPort, File workingDir) throws IOException { - Properties bootstrapProperties = bootstrapFileProvider.getBootstrapProperties(); + BootstrapProperties bootstrapProperties = bootstrapFileProvider.getBootstrapProperties(); File confDir = getFile(bootstrapProperties.getProperty(CONF_DIR_KEY, DEFAULT_CONF_DIR).trim(), workingDir); File libDir = getFile(bootstrapProperties.getProperty(LIB_DIR_KEY, DEFAULT_LIB_DIR).trim(), workingDir); @@ -107,6 +108,7 @@ public class MiNiFiExecCommandProvider { List javaAdditionalArgs = getJavaAdditionalArgs(bootstrapProperties); List systemProperties = List.of( systemProperty(PROPERTIES_FILE_PATH, getMiNiFiPropertiesPath(bootstrapProperties, confDir)), + systemProperty(MINIFI_BOOTSTRAP_CONF_FILE_PATH, bootstrapFileProvider.getBootstrapFilePath()), systemProperty(NIFI_BOOTSTRAP_LISTEN_PORT, listenPort), systemProperty(APP, MINIFI_CLASS_NAME), systemProperty(LOG_DIR, minifiLogDir), @@ -116,8 +118,7 @@ public class MiNiFiExecCommandProvider { systemProperty(BOOTSTRAP_LOG_FILE_EXTENSION, minifiBootstrapLogFileExtension) ); - return List.of(javaCommand, javaAdditionalArgs, systemProperties, List.of(MINIFI_FULLY_QUALIFIED_CLASS_NAME)) - .stream() + return Stream.of(javaCommand, javaAdditionalArgs, systemProperties, List.of(MINIFI_FULLY_QUALIFIED_CLASS_NAME)) .flatMap(List::stream) .toList(); } @@ -127,7 +128,7 @@ public class MiNiFiExecCommandProvider { return file.isAbsolute() ? file : new File(workingDir, filename).getAbsoluteFile(); } - private String getJavaCommand(Properties bootstrapProperties) { + private String getJavaCommand(BootstrapProperties bootstrapProperties) { String javaCommand = bootstrapProperties.getProperty(JAVA_COMMAND_KEY, DEFAULT_JAVA_CMD); return javaCommand.equals(DEFAULT_JAVA_CMD) ? ofNullable(System.getenv(JAVA_HOME_ENVIRONMENT_VARIABLE)) @@ -159,15 +160,16 @@ public class MiNiFiExecCommandProvider { .collect(joining(File.pathSeparator)); } - private List getJavaAdditionalArgs(Properties props) { - return props.entrySet() + private List getJavaAdditionalArgs(BootstrapProperties props) { + return props.getPropertyKeys() .stream() - .filter(entry -> ((String) entry.getKey()).startsWith(JAVA_ARG_KEY_PREFIX)) - .map(entry -> (String) entry.getValue()) + .filter(key -> key.startsWith(JAVA_ARG_KEY_PREFIX)) + .map(props::getProperty) .toList(); } private String systemProperty(String key, Object value) { return String.format(SYSTEM_PROPERTY_TEMPLATE, key, value); } + } \ No newline at end of file diff --git a/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/service/MiNiFiPropertiesGenerator.java b/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/service/MiNiFiPropertiesGenerator.java index 939ded36f7..42c55c8bf6 100644 --- a/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/service/MiNiFiPropertiesGenerator.java +++ b/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/service/MiNiFiPropertiesGenerator.java @@ -19,6 +19,7 @@ package org.apache.nifi.minifi.bootstrap.service; import static java.lang.String.join; import static java.lang.System.getProperty; +import static java.util.Map.entry; import static java.util.Optional.ofNullable; import static org.apache.commons.lang3.StringUtils.EMPTY; import static org.apache.commons.lang3.StringUtils.isNotBlank; @@ -59,6 +60,7 @@ import org.apache.commons.lang3.tuple.Triple; import org.apache.nifi.minifi.bootstrap.configuration.ConfigurationChangeException; import org.apache.nifi.minifi.bootstrap.util.OrderedProperties; import org.apache.nifi.minifi.commons.api.MiNiFiProperties; +import org.apache.nifi.minifi.properties.BootstrapProperties; import org.apache.nifi.util.NiFiProperties; public class MiNiFiPropertiesGenerator { @@ -139,15 +141,19 @@ public class MiNiFiPropertiesGenerator { Triple.of(NiFiProperties.FLOW_CONFIGURATION_FILE, "./conf/flow.json.gz", EMPTY) ); - static final Map MINIFI_TO_NIFI_PROPERTY_MAPPING = Map.of( - MiNiFiProperties.NIFI_MINIFI_FLOW_CONFIG.getKey(), NiFiProperties.FLOW_CONFIGURATION_FILE, - MiNiFiProperties.NIFI_MINIFI_SECURITY_KEYSTORE.getKey(), NiFiProperties.SECURITY_KEYSTORE, - MiNiFiProperties.NIFI_MINIFI_SECURITY_KEYSTORE_TYPE.getKey(), NiFiProperties.SECURITY_KEYSTORE_TYPE, - MiNiFiProperties.NIFI_MINIFI_SECURITY_KEYSTORE_PASSWD.getKey(), NiFiProperties.SECURITY_KEYSTORE_PASSWD, - MiNiFiProperties.NIFI_MINIFI_SECURITY_KEY_PASSWD.getKey(), NiFiProperties.SECURITY_KEY_PASSWD, - MiNiFiProperties.NIFI_MINIFI_SECURITY_TRUSTSTORE.getKey(), NiFiProperties.SECURITY_TRUSTSTORE, - MiNiFiProperties.NIFI_MINIFI_SECURITY_TRUSTSTORE_TYPE.getKey(), NiFiProperties.SECURITY_TRUSTSTORE_TYPE, - MiNiFiProperties.NIFI_MINIFI_SECURITY_TRUSTSTORE_PASSWD.getKey(), NiFiProperties.SECURITY_TRUSTSTORE_PASSWD + static final String PROTECTED_POSTFIX = ".protected"; + static final Map MINIFI_TO_NIFI_PROPERTY_MAPPING = Map.ofEntries( + entry(MiNiFiProperties.NIFI_MINIFI_FLOW_CONFIG.getKey(), NiFiProperties.FLOW_CONFIGURATION_FILE), + entry(MiNiFiProperties.NIFI_MINIFI_SECURITY_KEYSTORE.getKey(), NiFiProperties.SECURITY_KEYSTORE), + entry(MiNiFiProperties.NIFI_MINIFI_SECURITY_KEYSTORE_TYPE.getKey(), NiFiProperties.SECURITY_KEYSTORE_TYPE), + entry(MiNiFiProperties.NIFI_MINIFI_SECURITY_KEYSTORE_PASSWD.getKey(), NiFiProperties.SECURITY_KEYSTORE_PASSWD), + entry(MiNiFiProperties.NIFI_MINIFI_SECURITY_KEYSTORE_PASSWD.getKey() + PROTECTED_POSTFIX, NiFiProperties.SECURITY_KEYSTORE_PASSWD + PROTECTED_POSTFIX), + entry(MiNiFiProperties.NIFI_MINIFI_SECURITY_KEY_PASSWD.getKey(), NiFiProperties.SECURITY_KEY_PASSWD), + entry(MiNiFiProperties.NIFI_MINIFI_SECURITY_KEY_PASSWD.getKey() + PROTECTED_POSTFIX, NiFiProperties.SECURITY_KEY_PASSWD + PROTECTED_POSTFIX), + entry(MiNiFiProperties.NIFI_MINIFI_SECURITY_TRUSTSTORE.getKey(), NiFiProperties.SECURITY_TRUSTSTORE), + entry(MiNiFiProperties.NIFI_MINIFI_SECURITY_TRUSTSTORE_TYPE.getKey(), NiFiProperties.SECURITY_TRUSTSTORE_TYPE), + entry(MiNiFiProperties.NIFI_MINIFI_SECURITY_TRUSTSTORE_PASSWD.getKey(), NiFiProperties.SECURITY_TRUSTSTORE_PASSWD), + entry(MiNiFiProperties.NIFI_MINIFI_SECURITY_TRUSTSTORE_PASSWD.getKey() + PROTECTED_POSTFIX, NiFiProperties.SECURITY_TRUSTSTORE_PASSWD + PROTECTED_POSTFIX) ); static final String DEFAULT_SENSITIVE_PROPERTIES_ENCODING_ALGORITHM = "NIFI_PBKDF2_AES_GCM_256"; @@ -160,7 +166,7 @@ public class MiNiFiPropertiesGenerator { public static final String FILE_EXTENSION_DELIMITER = "."; - public void generateMinifiProperties(String configDirectory, Properties bootstrapProperties) throws ConfigurationChangeException { + public void generateMinifiProperties(String configDirectory, BootstrapProperties bootstrapProperties) throws ConfigurationChangeException { String minifiPropertiesFileName = Path.of(getMiNiFiPropertiesPath(bootstrapProperties, new File(configDirectory))).getFileName().toString(); Path minifiPropertiesFile = Path.of(configDirectory, minifiPropertiesFileName); @@ -188,22 +194,22 @@ public class MiNiFiPropertiesGenerator { ); } - private OrderedProperties prepareMinifiProperties(Properties bootstrapProperties, Map existingSensitivePropertiesConfiguration) { + private OrderedProperties prepareMinifiProperties(BootstrapProperties bootstrapProperties, Map existingSensitivePropertiesConfiguration) { OrderedProperties minifiProperties = new OrderedProperties(); NIFI_PROPERTIES_WITH_DEFAULT_VALUES_AND_COMMENTS .forEach(triple -> minifiProperties.setProperty(triple.getLeft(), triple.getMiddle(), triple.getRight())); - getNonBlankPropertiesWithPredicate(bootstrapProperties, entry -> MINIFI_TO_NIFI_PROPERTY_MAPPING.containsKey(entry.getKey())) + getNonBlankPropertiesWithPredicate(bootstrapProperties, MINIFI_TO_NIFI_PROPERTY_MAPPING::containsKey) .forEach(entry -> minifiProperties.setProperty(MINIFI_TO_NIFI_PROPERTY_MAPPING.get(entry.getKey()), entry.getValue())); getSensitiveProperties(bootstrapProperties, existingSensitivePropertiesConfiguration) .forEach(entry -> minifiProperties.setProperty(entry.getKey(), entry.getValue())); - getNonBlankPropertiesWithPredicate(bootstrapProperties, entry -> ((String) entry.getKey()).startsWith(C2_PROPERTY_PREFIX)) + getNonBlankPropertiesWithPredicate(bootstrapProperties, key -> key.startsWith(C2_PROPERTY_PREFIX)) .forEach(entry -> minifiProperties.setProperty(entry.getKey(), entry.getValue())); - getNonBlankPropertiesWithPredicate(bootstrapProperties, entry -> ((String) entry.getKey()).startsWith(NIFI_PREFIX)) + getNonBlankPropertiesWithPredicate(bootstrapProperties, key -> key.startsWith(NIFI_PREFIX)) .forEach(entry -> minifiProperties.setProperty(entry.getKey(), entry.getValue())); bootstrapFileAndLogProperties() @@ -212,19 +218,19 @@ public class MiNiFiPropertiesGenerator { return minifiProperties; } - private List> getNonBlankPropertiesWithPredicate(Properties bootstrapProperties, Predicate predicate) { + private List> getNonBlankPropertiesWithPredicate(BootstrapProperties bootstrapProperties, Predicate predicate) { return ofNullable(bootstrapProperties) - .map(Properties::entrySet) + .map(BootstrapProperties::getPropertyKeys) .orElseGet(Set::of) .stream() .filter(predicate) - .filter(entry -> isNotBlank((String) entry.getValue())) - .map(entry -> Pair.of((String) entry.getKey(), (String) entry.getValue())) + .map(key -> Pair.of(key, bootstrapProperties.getProperty(key))) + .filter(pair -> isNotBlank(pair.getValue())) .sorted((o1, o2) -> Comparator.naturalOrder().compare(o1.getKey(), o2.getKey())) .toList(); } - private List> getSensitiveProperties(Properties bootstrapProperties, Map existingSensitivePropertiesConfiguration) { + private List> getSensitiveProperties(BootstrapProperties bootstrapProperties, Map existingSensitivePropertiesConfiguration) { return existingSensitivePropertiesConfiguration.isEmpty() ? List.of( Pair.of(NiFiProperties.SENSITIVE_PROPS_KEY, diff --git a/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/service/PeriodicStatusReporterManager.java b/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/service/PeriodicStatusReporterManager.java index 6b9833774b..1db737e76b 100644 --- a/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/service/PeriodicStatusReporterManager.java +++ b/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/service/PeriodicStatusReporterManager.java @@ -17,35 +17,35 @@ package org.apache.nifi.minifi.bootstrap.service; +import static org.apache.nifi.minifi.commons.api.MiNiFiProperties.NIFI_MINIFI_STATUS_REPORTER_COMPONENTS; + import java.lang.reflect.InvocationTargetException; import java.util.Collections; import java.util.HashSet; import java.util.LinkedList; import java.util.List; -import java.util.Properties; import java.util.Set; import org.apache.nifi.minifi.bootstrap.MiNiFiParameters; import org.apache.nifi.minifi.bootstrap.MiNiFiStatus; import org.apache.nifi.minifi.bootstrap.QueryableStatusAggregator; import org.apache.nifi.minifi.bootstrap.status.PeriodicStatusReporter; import org.apache.nifi.minifi.commons.status.FlowStatusReport; +import org.apache.nifi.minifi.properties.BootstrapProperties; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import static org.apache.nifi.minifi.commons.api.MiNiFiProperties.NIFI_MINIFI_STATUS_REPORTER_COMPONENTS; - public class PeriodicStatusReporterManager implements QueryableStatusAggregator { private static final Logger LOGGER = LoggerFactory.getLogger(PeriodicStatusReporterManager.class); private static final String FLOW_STATUS_REPORT_CMD = "FLOW_STATUS_REPORT"; - private final Properties bootstrapProperties; + private final BootstrapProperties bootstrapProperties; private final MiNiFiStatusProvider miNiFiStatusProvider; private final MiNiFiCommandSender miNiFiCommandSender; private final MiNiFiParameters miNiFiParameters; private Set periodicStatusReporters = Collections.emptySet(); - public PeriodicStatusReporterManager(Properties bootstrapProperties, MiNiFiStatusProvider miNiFiStatusProvider, MiNiFiCommandSender miNiFiCommandSender, + public PeriodicStatusReporterManager(BootstrapProperties bootstrapProperties, MiNiFiStatusProvider miNiFiStatusProvider, MiNiFiCommandSender miNiFiCommandSender, MiNiFiParameters miNiFiParameters) { this.bootstrapProperties = bootstrapProperties; this.miNiFiStatusProvider = miNiFiStatusProvider; diff --git a/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/service/UpdatePropertiesService.java b/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/service/UpdatePropertiesService.java index 36e68f7124..a6058f953c 100644 --- a/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/service/UpdatePropertiesService.java +++ b/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/service/UpdatePropertiesService.java @@ -25,10 +25,10 @@ import java.io.FileInputStream; import java.io.IOException; import java.nio.file.Files; import java.util.Optional; -import java.util.Properties; import org.apache.nifi.minifi.bootstrap.RunMiNiFi; import org.apache.nifi.minifi.bootstrap.configuration.ConfigurationChangeException; import org.apache.nifi.minifi.commons.api.MiNiFiCommandState; +import org.apache.nifi.minifi.properties.BootstrapProperties; import org.slf4j.Logger; public class UpdatePropertiesService { @@ -59,7 +59,7 @@ public class UpdatePropertiesService { Files.copy(bootstrapFileProvider.getBootstrapConfNewFile().toPath(), bootstrapConfigFile.toPath(), REPLACE_EXISTING); // already from new - commandState = generateConfigfilesBasedOnNewProperties(bootstrapConfigFile, bootstrapSwapConfigFile, bootstrapFileProvider.getBootstrapProperties()); + commandState = generateConfigfilesBasedOnNewProperties(bootstrapConfigFile, bootstrapSwapConfigFile, bootstrapFileProvider.getProtectedBootstrapProperties()); } catch (Exception e) { commandState = Optional.of(MiNiFiCommandState.NOT_APPLIED_WITHOUT_RESTART); logger.error("Failed to load new bootstrap properties", e); @@ -67,7 +67,7 @@ public class UpdatePropertiesService { return commandState; } - private Optional generateConfigfilesBasedOnNewProperties(File bootstrapConfigFile, File bootstrapSwapConfigFile, Properties bootstrapProperties) + private Optional generateConfigfilesBasedOnNewProperties(File bootstrapConfigFile, File bootstrapSwapConfigFile, BootstrapProperties bootstrapProperties) throws IOException, ConfigurationChangeException { Optional commandState = Optional.empty(); try { diff --git a/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/status/PeriodicStatusReporter.java b/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/status/PeriodicStatusReporter.java index 0816aa79ea..6cb53617ff 100644 --- a/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/status/PeriodicStatusReporter.java +++ b/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/status/PeriodicStatusReporter.java @@ -17,12 +17,11 @@ package org.apache.nifi.minifi.bootstrap.status; -import org.apache.nifi.minifi.bootstrap.QueryableStatusAggregator; - -import java.util.Properties; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.ScheduledThreadPoolExecutor; import java.util.concurrent.TimeUnit; +import org.apache.nifi.minifi.bootstrap.QueryableStatusAggregator; +import org.apache.nifi.minifi.properties.BootstrapProperties; public abstract class PeriodicStatusReporter { @@ -38,7 +37,7 @@ public abstract class PeriodicStatusReporter { * * @param properties from the bootstrap configuration */ - public abstract void initialize(Properties properties, QueryableStatusAggregator queryableStatusAggregator); + public abstract void initialize(BootstrapProperties properties, QueryableStatusAggregator queryableStatusAggregator); /** * Begins the associated reporting service provided by the given implementation. In most implementations, no action will occur until this method is invoked. The implementing class must have set diff --git a/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/status/reporters/StatusLogger.java b/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/status/reporters/StatusLogger.java index 2d6f7fa5bb..0f58dbdcbf 100644 --- a/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/status/reporters/StatusLogger.java +++ b/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/status/reporters/StatusLogger.java @@ -22,11 +22,11 @@ import static org.apache.nifi.minifi.commons.api.MiNiFiProperties.NIFI_MINIFI_ST import static org.apache.nifi.minifi.commons.api.MiNiFiProperties.NIFI_MINIFI_STATUS_REPORTER_LOG_QUERY; import java.io.IOException; -import java.util.Properties; import org.apache.nifi.logging.LogLevel; import org.apache.nifi.minifi.bootstrap.QueryableStatusAggregator; import org.apache.nifi.minifi.bootstrap.status.PeriodicStatusReporter; import org.apache.nifi.minifi.commons.status.FlowStatusReport; +import org.apache.nifi.minifi.properties.BootstrapProperties; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -41,7 +41,7 @@ public class StatusLogger extends PeriodicStatusReporter { static final String ENCOUNTERED_IO_EXCEPTION = "Encountered an IO Exception while attempting to query the flow status."; @Override - public void initialize(Properties properties, QueryableStatusAggregator queryableStatusAggregator) { + public void initialize(BootstrapProperties properties, QueryableStatusAggregator queryableStatusAggregator) { this.queryableStatusAggregator = queryableStatusAggregator; String periodString = properties.getProperty(NIFI_MINIFI_STATUS_REPORTER_LOG_PERIOD.getKey()); diff --git a/minifi/minifi-bootstrap/src/test/java/org/apache/nifi/minifi/bootstrap/configuration/ingestors/FileChangeIngestorTest.java b/minifi/minifi-bootstrap/src/test/java/org/apache/nifi/minifi/bootstrap/configuration/ingestors/FileChangeIngestorTest.java index cc967e552c..08dbc4b836 100644 --- a/minifi/minifi-bootstrap/src/test/java/org/apache/nifi/minifi/bootstrap/configuration/ingestors/FileChangeIngestorTest.java +++ b/minifi/minifi-bootstrap/src/test/java/org/apache/nifi/minifi/bootstrap/configuration/ingestors/FileChangeIngestorTest.java @@ -18,8 +18,11 @@ package org.apache.nifi.minifi.bootstrap.configuration.ingestors; import static java.nio.file.StandardWatchEventKinds.ENTRY_MODIFY; +import static org.apache.nifi.minifi.bootstrap.configuration.ingestors.FileChangeIngestor.DEFAULT_POLLING_PERIOD_INTERVAL; import static org.apache.nifi.minifi.bootstrap.configuration.ingestors.PullHttpChangeIngestor.PULL_HTTP_BASE_KEY; import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; @@ -31,12 +34,12 @@ import java.nio.file.WatchEvent; import java.nio.file.WatchKey; import java.nio.file.WatchService; import java.util.Collections; -import java.util.Properties; import java.util.concurrent.atomic.AtomicReference; import org.apache.nifi.minifi.bootstrap.ConfigurationFileHolder; import org.apache.nifi.minifi.bootstrap.configuration.ConfigurationChangeNotifier; import org.apache.nifi.minifi.bootstrap.configuration.differentiators.Differentiator; import org.apache.nifi.minifi.commons.api.MiNiFiProperties; +import org.apache.nifi.minifi.properties.BootstrapProperties; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -49,7 +52,7 @@ public class FileChangeIngestorTest { private FileChangeIngestor notifierSpy; private WatchService mockWatchService; - private Properties testProperties; + private BootstrapProperties testProperties; private Differentiator mockDifferentiator; private ConfigurationChangeNotifier testNotifier; @@ -60,15 +63,15 @@ public class FileChangeIngestorTest { notifierSpy = Mockito.spy(new FileChangeIngestor()); mockDifferentiator = mock(Differentiator.class); testNotifier = mock(ConfigurationChangeNotifier.class); + testProperties = mock(BootstrapProperties.class); setMocks(); - testProperties = new Properties(); - testProperties.put(FileChangeIngestor.CONFIG_FILE_PATH_KEY, TEST_CONFIG_PATH); - testProperties.put(PullHttpChangeIngestor.OVERRIDE_SECURITY, "true"); - testProperties.put(PULL_HTTP_BASE_KEY + ".override.core", "true"); - testProperties.put(FileChangeIngestor.POLLING_PERIOD_INTERVAL_KEY, FileChangeIngestor.DEFAULT_POLLING_PERIOD_INTERVAL); - testProperties.put(MiNiFiProperties.NIFI_MINIFI_FLOW_CONFIG.getKey(), MiNiFiProperties.NIFI_MINIFI_FLOW_CONFIG.getDefaultValue()); + when(testProperties.getProperty(FileChangeIngestor.CONFIG_FILE_PATH_KEY)).thenReturn(TEST_CONFIG_PATH); + when(testProperties.getProperty(PullHttpChangeIngestor.OVERRIDE_SECURITY)).thenReturn("true"); + when(testProperties.getProperty(PULL_HTTP_BASE_KEY + ".override.core")).thenReturn("true"); + when(testProperties.getProperty(eq(FileChangeIngestor.POLLING_PERIOD_INTERVAL_KEY), any())).thenReturn(String.valueOf(DEFAULT_POLLING_PERIOD_INTERVAL)); + when(testProperties.getProperty(MiNiFiProperties.NIFI_MINIFI_FLOW_CONFIG.getKey())).thenReturn(MiNiFiProperties.NIFI_MINIFI_FLOW_CONFIG.getDefaultValue()); } @AfterEach @@ -78,7 +81,7 @@ public class FileChangeIngestorTest { @Test public void testInitializeInvalidFile() { - testProperties.put(FileChangeIngestor.CONFIG_FILE_PATH_KEY, "/land/of/make/believe"); + when(testProperties.getProperty(FileChangeIngestor.CONFIG_FILE_PATH_KEY)).thenReturn("/land/of/make/believe"); assertThrows(IllegalStateException.class, () -> notifierSpy.initialize(testProperties, mock(ConfigurationFileHolder.class), mock(ConfigurationChangeNotifier.class))); } @@ -89,13 +92,12 @@ public class FileChangeIngestorTest { @Test public void testInitializeInvalidPollingPeriod() { - testProperties.put(FileChangeIngestor.POLLING_PERIOD_INTERVAL_KEY, "abc"); + when(testProperties.getProperty(eq(FileChangeIngestor.POLLING_PERIOD_INTERVAL_KEY), any())).thenReturn("abc"); assertThrows(IllegalStateException.class, () -> notifierSpy.initialize(testProperties, mock(ConfigurationFileHolder.class), mock(ConfigurationChangeNotifier.class))); } @Test public void testInitializeUseDefaultPolling() { - testProperties.remove(FileChangeIngestor.POLLING_PERIOD_INTERVAL_KEY); notifierSpy.initialize(testProperties, mock(ConfigurationFileHolder.class), mock(ConfigurationChangeNotifier.class)); } diff --git a/minifi/minifi-bootstrap/src/test/java/org/apache/nifi/minifi/bootstrap/configuration/ingestors/RestChangeIngestorSSLTest.java b/minifi/minifi-bootstrap/src/test/java/org/apache/nifi/minifi/bootstrap/configuration/ingestors/RestChangeIngestorSSLTest.java index 2f7d644ef3..8597da3168 100644 --- a/minifi/minifi-bootstrap/src/test/java/org/apache/nifi/minifi/bootstrap/configuration/ingestors/RestChangeIngestorSSLTest.java +++ b/minifi/minifi-bootstrap/src/test/java/org/apache/nifi/minifi/bootstrap/configuration/ingestors/RestChangeIngestorSSLTest.java @@ -25,7 +25,8 @@ import java.io.IOException; import java.nio.ByteBuffer; import java.security.KeyStore; import java.util.Collections; -import java.util.Properties; +import java.util.HashMap; +import java.util.Map; import java.util.concurrent.atomic.AtomicReference; import javax.net.ssl.SSLContext; import javax.net.ssl.SSLSocketFactory; @@ -35,6 +36,7 @@ import org.apache.nifi.minifi.bootstrap.ConfigurationFileHolder; import org.apache.nifi.minifi.bootstrap.configuration.ConfigurationChangeListener; import org.apache.nifi.minifi.bootstrap.configuration.ConfigurationChangeNotifier; import org.apache.nifi.minifi.bootstrap.configuration.ListenerHandleResult; +import org.apache.nifi.minifi.properties.BootstrapProperties; import org.apache.nifi.security.ssl.StandardKeyStoreBuilder; import org.apache.nifi.security.ssl.StandardSslContextBuilder; import org.apache.nifi.security.ssl.StandardTrustManagerBuilder; @@ -50,16 +52,17 @@ public class RestChangeIngestorSSLTest extends RestChangeIngestorCommonTest { public static void setUpHttps() throws IOException, InterruptedException { final TlsConfiguration tlsConfiguration = new TemporaryKeyStoreBuilder().trustStoreType("JKS").build(); - Properties properties = new Properties(); - properties.setProperty(RestChangeIngestor.TRUSTSTORE_LOCATION_KEY, tlsConfiguration.getTruststorePath()); - properties.setProperty(RestChangeIngestor.TRUSTSTORE_PASSWORD_KEY, tlsConfiguration.getTruststorePassword()); - properties.setProperty(RestChangeIngestor.TRUSTSTORE_TYPE_KEY, tlsConfiguration.getTruststoreType().getType()); - properties.setProperty(RestChangeIngestor.KEYSTORE_LOCATION_KEY, tlsConfiguration.getKeystorePath()); - properties.setProperty(RestChangeIngestor.KEYSTORE_PASSWORD_KEY, tlsConfiguration.getKeystorePassword()); - properties.setProperty(RestChangeIngestor.KEYSTORE_TYPE_KEY, tlsConfiguration.getKeystoreType().getType()); - properties.setProperty(RestChangeIngestor.NEED_CLIENT_AUTH_KEY, "false"); - properties.put(PullHttpChangeIngestor.OVERRIDE_SECURITY, "true"); - properties.put(PULL_HTTP_BASE_KEY + ".override.core", "true"); + Map bootstrapProperties = new HashMap<>(); + bootstrapProperties.put(RestChangeIngestor.TRUSTSTORE_LOCATION_KEY, tlsConfiguration.getTruststorePath()); + bootstrapProperties.put(RestChangeIngestor.TRUSTSTORE_PASSWORD_KEY, tlsConfiguration.getTruststorePassword()); + bootstrapProperties.put(RestChangeIngestor.TRUSTSTORE_TYPE_KEY, tlsConfiguration.getTruststoreType().getType()); + bootstrapProperties.put(RestChangeIngestor.KEYSTORE_LOCATION_KEY, tlsConfiguration.getKeystorePath()); + bootstrapProperties.put(RestChangeIngestor.KEYSTORE_PASSWORD_KEY, tlsConfiguration.getKeystorePassword()); + bootstrapProperties.put(RestChangeIngestor.KEYSTORE_TYPE_KEY, tlsConfiguration.getKeystoreType().getType()); + bootstrapProperties.put(RestChangeIngestor.NEED_CLIENT_AUTH_KEY, "false"); + bootstrapProperties.put(PullHttpChangeIngestor.OVERRIDE_SECURITY, "true"); + bootstrapProperties.put(PULL_HTTP_BASE_KEY + ".override.core", "true"); + BootstrapProperties properties = new BootstrapProperties(bootstrapProperties); restChangeIngestor = new RestChangeIngestor(); diff --git a/minifi/minifi-bootstrap/src/test/java/org/apache/nifi/minifi/bootstrap/configuration/ingestors/RestChangeIngestorTest.java b/minifi/minifi-bootstrap/src/test/java/org/apache/nifi/minifi/bootstrap/configuration/ingestors/RestChangeIngestorTest.java index 64142eccbf..57c3959754 100644 --- a/minifi/minifi-bootstrap/src/test/java/org/apache/nifi/minifi/bootstrap/configuration/ingestors/RestChangeIngestorTest.java +++ b/minifi/minifi-bootstrap/src/test/java/org/apache/nifi/minifi/bootstrap/configuration/ingestors/RestChangeIngestorTest.java @@ -18,15 +18,20 @@ package org.apache.nifi.minifi.bootstrap.configuration.ingestors; import static org.apache.nifi.minifi.bootstrap.configuration.ingestors.PullHttpChangeIngestor.PULL_HTTP_BASE_KEY; +import static org.apache.nifi.minifi.bootstrap.configuration.ingestors.RestChangeIngestor.HOST_KEY; +import static org.apache.nifi.minifi.bootstrap.configuration.ingestors.RestChangeIngestor.PORT_KEY; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; import java.net.MalformedURLException; import java.nio.ByteBuffer; -import java.util.Properties; import java.util.concurrent.atomic.AtomicReference; import okhttp3.OkHttpClient; import org.apache.nifi.minifi.bootstrap.ConfigurationFileHolder; import org.apache.nifi.minifi.bootstrap.configuration.ConfigurationChangeNotifier; +import org.apache.nifi.minifi.properties.BootstrapProperties; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; import org.mockito.Mockito; @@ -35,9 +40,11 @@ public class RestChangeIngestorTest extends RestChangeIngestorCommonTest { @BeforeAll public static void setUp() throws InterruptedException, MalformedURLException { - Properties properties = new Properties(); - properties.put(PullHttpChangeIngestor.OVERRIDE_SECURITY, "true"); - properties.put(PULL_HTTP_BASE_KEY + ".override.core", "true"); + BootstrapProperties properties = mock(BootstrapProperties.class); + when(properties.getProperty(PullHttpChangeIngestor.OVERRIDE_SECURITY)).thenReturn("true"); + when(properties.getProperty(PULL_HTTP_BASE_KEY + ".override.core")).thenReturn("true"); + when(properties.getProperty(eq(PORT_KEY), any())).thenReturn("0"); + when(properties.getProperty(eq(HOST_KEY), any())).thenReturn("localhost"); restChangeIngestor = new RestChangeIngestor(); testNotifier = Mockito.mock(ConfigurationChangeNotifier.class); diff --git a/minifi/minifi-bootstrap/src/test/java/org/apache/nifi/minifi/bootstrap/service/GracefulShutdownParameterProviderTest.java b/minifi/minifi-bootstrap/src/test/java/org/apache/nifi/minifi/bootstrap/service/GracefulShutdownParameterProviderTest.java index 87f3ecd810..e580b1cadf 100644 --- a/minifi/minifi-bootstrap/src/test/java/org/apache/nifi/minifi/bootstrap/service/GracefulShutdownParameterProviderTest.java +++ b/minifi/minifi-bootstrap/src/test/java/org/apache/nifi/minifi/bootstrap/service/GracefulShutdownParameterProviderTest.java @@ -20,10 +20,14 @@ package org.apache.nifi.minifi.bootstrap.service; import static org.apache.nifi.minifi.bootstrap.service.GracefulShutdownParameterProvider.DEFAULT_GRACEFUL_SHUTDOWN_VALUE; import static org.apache.nifi.minifi.bootstrap.service.GracefulShutdownParameterProvider.GRACEFUL_SHUTDOWN_PROP; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; import java.io.IOException; -import java.util.Properties; +import java.util.Optional; +import org.apache.nifi.minifi.properties.BootstrapProperties; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.junit.jupiter.params.ParameterizedTest; @@ -46,10 +50,11 @@ class GracefulShutdownParameterProviderTest { @NullSource @ValueSource(strings = {"notAnInteger", "-1"}) void testGetBootstrapPropertiesShouldReturnDefaultShutdownPropertyValue(String shutdownProperty) throws IOException { - Properties properties = new Properties(); - if (shutdownProperty != null) { - properties.setProperty(GRACEFUL_SHUTDOWN_PROP, shutdownProperty); - } + BootstrapProperties properties = mock(BootstrapProperties.class); + + when(properties.getProperty(eq(GRACEFUL_SHUTDOWN_PROP), any())) + .thenReturn(Optional.ofNullable(shutdownProperty).orElse(DEFAULT_GRACEFUL_SHUTDOWN_VALUE)); + when(bootstrapFileProvider.getBootstrapProperties()).thenReturn(properties); assertEquals(Integer.parseInt(DEFAULT_GRACEFUL_SHUTDOWN_VALUE), gracefulShutdownParameterProvider.getGracefulShutdownSeconds()); @@ -57,8 +62,8 @@ class GracefulShutdownParameterProviderTest { @Test void testGetBootstrapPropertiesShouldReturnShutdownPropertyValue() throws IOException { - Properties properties = new Properties(); - properties.setProperty(GRACEFUL_SHUTDOWN_PROP, "1000"); + BootstrapProperties properties = mock(BootstrapProperties.class); + when(properties.getProperty(eq(GRACEFUL_SHUTDOWN_PROP), any())).thenReturn("1000"); when(bootstrapFileProvider.getBootstrapProperties()).thenReturn(properties); assertEquals(1000, gracefulShutdownParameterProvider.getGracefulShutdownSeconds()); diff --git a/minifi/minifi-bootstrap/src/test/java/org/apache/nifi/minifi/bootstrap/service/MiNiFiPropertiesGeneratorTest.java b/minifi/minifi-bootstrap/src/test/java/org/apache/nifi/minifi/bootstrap/service/MiNiFiPropertiesGeneratorTest.java index 1ee026b445..9034e99b2b 100644 --- a/minifi/minifi-bootstrap/src/test/java/org/apache/nifi/minifi/bootstrap/service/MiNiFiPropertiesGeneratorTest.java +++ b/minifi/minifi-bootstrap/src/test/java/org/apache/nifi/minifi/bootstrap/service/MiNiFiPropertiesGeneratorTest.java @@ -53,6 +53,7 @@ import java.util.stream.Stream; import org.apache.commons.lang3.StringUtils; import org.apache.nifi.minifi.bootstrap.configuration.ConfigurationChangeException; import org.apache.nifi.minifi.commons.api.MiNiFiProperties; +import org.apache.nifi.minifi.properties.BootstrapProperties; import org.apache.nifi.util.NiFiProperties; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -82,7 +83,7 @@ public class MiNiFiPropertiesGeneratorTest { @Test public void testGenerateDefaultNiFiProperties() throws ConfigurationChangeException { // given - Properties bootstrapProperties = createBootstrapProperties(Map.of()); + BootstrapProperties bootstrapProperties = createBootstrapProperties(Map.of()); // when testPropertiesGenerator.generateMinifiProperties(configDirectory.toString(), bootstrapProperties); @@ -101,7 +102,7 @@ public class MiNiFiPropertiesGeneratorTest { @Test public void testMiNiFiPropertiesMappedToAppropriateNiFiProperties() throws ConfigurationChangeException { // given - Properties bootstrapProperties = createBootstrapProperties(Stream.of( + BootstrapProperties bootstrapProperties = createBootstrapProperties(Stream.of( MiNiFiProperties.NIFI_MINIFI_FLOW_CONFIG.getKey(), MiNiFiProperties.NIFI_MINIFI_SECURITY_KEYSTORE.getKey(), MiNiFiProperties.NIFI_MINIFI_SECURITY_KEYSTORE_TYPE.getKey(), @@ -126,7 +127,7 @@ public class MiNiFiPropertiesGeneratorTest { @Test public void testSensitivePropertiesAreGeneratedWhenNotProvidedInBootstrap() throws ConfigurationChangeException { // given - Properties bootstrapProperties = createBootstrapProperties(Map.of()); + BootstrapProperties bootstrapProperties = createBootstrapProperties(Map.of()); // when testPropertiesGenerator.generateMinifiProperties(configDirectory.toString(), bootstrapProperties); @@ -142,7 +143,7 @@ public class MiNiFiPropertiesGeneratorTest { // given String sensitivePropertiesKey = "sensitive_properties_key"; String sensitivePropertiesAlgorithm = "sensitive_properties_algorithm"; - Properties bootstrapProperties = createBootstrapProperties(Map.of( + BootstrapProperties bootstrapProperties = createBootstrapProperties(Map.of( MiNiFiProperties.NIFI_MINIFI_SENSITIVE_PROPS_KEY.getKey(), sensitivePropertiesKey, MiNiFiProperties.NIFI_MINIFI_SENSITIVE_PROPS_ALGORITHM.getKey(), sensitivePropertiesAlgorithm )); @@ -159,7 +160,7 @@ public class MiNiFiPropertiesGeneratorTest { @Test public void testNonBlankC2PropertiesAreCopiedToMiNiFiProperties() throws ConfigurationChangeException { // given - Properties bootstrapProperties = createBootstrapProperties(Map.of( + BootstrapProperties bootstrapProperties = createBootstrapProperties(Map.of( MiNiFiProperties.C2_ENABLE.getKey(), TRUE.toString(), MiNiFiProperties.C2_AGENT_CLASS.getKey(), EMPTY, MiNiFiProperties.C2_AGENT_IDENTIFIER.getKey(), SPACE @@ -180,7 +181,7 @@ public class MiNiFiPropertiesGeneratorTest { public void testDefaultNiFiPropertiesAreOverridden() throws ConfigurationChangeException { // given String archiveDir = "/path/to"; - Properties bootstrapProperties = createBootstrapProperties(Map.of( + BootstrapProperties bootstrapProperties = createBootstrapProperties(Map.of( NiFiProperties.FLOW_CONFIGURATION_ARCHIVE_ENABLED, TRUE.toString(), NiFiProperties.FLOW_CONFIGURATION_ARCHIVE_DIR, archiveDir )); @@ -211,7 +212,7 @@ public class MiNiFiPropertiesGeneratorTest { @Test public void testArbitraryNiFiPropertyCanBePassedViaBootstrapConf() throws ConfigurationChangeException { // given - Properties bootstrapProperties = createBootstrapProperties(Map.of( + BootstrapProperties bootstrapProperties = createBootstrapProperties(Map.of( "nifi.new.property", "new_property_value", "nifi.other.new.property", "other_new_property_value" )); @@ -229,7 +230,7 @@ public class MiNiFiPropertiesGeneratorTest { @Test public void bootstrapFileAndLogPropertiesAreGeneratedIntoMiNiFiProperties() throws ConfigurationChangeException { // given - Properties bootstrapProperties = createBootstrapProperties(Map.of()); + BootstrapProperties bootstrapProperties = createBootstrapProperties(Map.of()); // when testPropertiesGenerator.generateMinifiProperties(configDirectory.toString(), bootstrapProperties); @@ -243,12 +244,13 @@ public class MiNiFiPropertiesGeneratorTest { ); } - private Properties createBootstrapProperties(Map keyValues) { + private BootstrapProperties createBootstrapProperties(Map keyValues) { try (OutputStream outputStream = newOutputStream(bootstrapPropertiesFile)) { Properties properties = new Properties(); + BootstrapProperties bootstrapProperties = new BootstrapProperties(keyValues); properties.putAll(keyValues); properties.store(outputStream, EMPTY); - return properties; + return bootstrapProperties; } catch (IOException e) { throw new UncheckedIOException(e); } diff --git a/minifi/minifi-bootstrap/src/test/java/org/apache/nifi/minifi/bootstrap/status/reporters/StatusLoggerTest.java b/minifi/minifi-bootstrap/src/test/java/org/apache/nifi/minifi/bootstrap/status/reporters/StatusLoggerTest.java index 918edacee7..08cb00f403 100644 --- a/minifi/minifi-bootstrap/src/test/java/org/apache/nifi/minifi/bootstrap/status/reporters/StatusLoggerTest.java +++ b/minifi/minifi-bootstrap/src/test/java/org/apache/nifi/minifi/bootstrap/status/reporters/StatusLoggerTest.java @@ -22,16 +22,18 @@ import static org.apache.nifi.minifi.commons.api.MiNiFiProperties.NIFI_MINIFI_ST import static org.apache.nifi.minifi.commons.api.MiNiFiProperties.NIFI_MINIFI_STATUS_REPORTER_LOG_PERIOD; import static org.apache.nifi.minifi.commons.api.MiNiFiProperties.NIFI_MINIFI_STATUS_REPORTER_LOG_QUERY; import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.mockito.BDDMockito.given; +import static org.mockito.Mockito.mock; import static org.mockito.Mockito.verify; import java.io.IOException; -import java.util.Properties; import java.util.concurrent.ScheduledFuture; import java.util.concurrent.ScheduledThreadPoolExecutor; import java.util.concurrent.TimeUnit; import org.apache.nifi.logging.LogLevel; import org.apache.nifi.minifi.bootstrap.QueryableStatusAggregator; import org.apache.nifi.minifi.commons.status.FlowStatusReport; +import org.apache.nifi.minifi.properties.BootstrapProperties; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.mockito.Mockito; @@ -68,28 +70,28 @@ public class StatusLoggerTest { @Test public void testFailedInitDueToFatalLogLevel() { - Properties properties = new Properties(); - properties.setProperty(NIFI_MINIFI_STATUS_REPORTER_LOG_PERIOD.getKey(), "1"); - properties.setProperty(NIFI_MINIFI_STATUS_REPORTER_LOG_LEVEL.getKey(), LogLevel.FATAL.name()); - properties.setProperty(NIFI_MINIFI_STATUS_REPORTER_LOG_QUERY.getKey(), MOCK_QUERY); + BootstrapProperties properties = mock(BootstrapProperties.class); + given(properties.getProperty(NIFI_MINIFI_STATUS_REPORTER_LOG_PERIOD.getKey())).willReturn("1"); + given(properties.getProperty(NIFI_MINIFI_STATUS_REPORTER_LOG_LEVEL.getKey())).willReturn(LogLevel.FATAL.name()); + given(properties.getProperty(NIFI_MINIFI_STATUS_REPORTER_LOG_QUERY.getKey())).willReturn(MOCK_QUERY); assertThrows(IllegalStateException.class, () -> statusLogger.initialize(properties, queryableStatusAggregator)); } @Test public void testFailedInitDueToNoPeriod() { - Properties properties = new Properties(); - properties.setProperty(NIFI_MINIFI_STATUS_REPORTER_LOG_LEVEL.getKey(), LogLevel.INFO.name()); - properties.setProperty(NIFI_MINIFI_STATUS_REPORTER_LOG_QUERY.getKey(), MOCK_QUERY); + BootstrapProperties properties = mock(BootstrapProperties.class); + given(properties.getProperty(NIFI_MINIFI_STATUS_REPORTER_LOG_LEVEL.getKey())).willReturn(LogLevel.INFO.name()); + given(properties.getProperty(NIFI_MINIFI_STATUS_REPORTER_LOG_QUERY.getKey())).willReturn(MOCK_QUERY); assertThrows(IllegalStateException.class, () -> statusLogger.initialize(properties, queryableStatusAggregator)); } @Test public void testFailedInitDueToNoQuery() { - Properties properties = new Properties(); - properties.setProperty(NIFI_MINIFI_STATUS_REPORTER_LOG_PERIOD.getKey(), "1"); - properties.setProperty(NIFI_MINIFI_STATUS_REPORTER_LOG_LEVEL.getKey(), LogLevel.INFO.name()); + BootstrapProperties properties = mock(BootstrapProperties.class); + given(properties.getProperty(NIFI_MINIFI_STATUS_REPORTER_LOG_LEVEL.getKey())).willReturn(LogLevel.INFO.name()); + given(properties.getProperty(NIFI_MINIFI_STATUS_REPORTER_LOG_QUERY.getKey())).willReturn(MOCK_QUERY); assertThrows(IllegalStateException.class, () -> statusLogger.initialize(properties, queryableStatusAggregator)); } @@ -139,13 +141,9 @@ public class StatusLoggerTest { verify(logger, Mockito.atLeastOnce()).error(MOCK_STATUS, (Throwable) null); } - // Exception testing @Test public void testTraceException() throws IOException { - Properties properties = new Properties(); - properties.setProperty(NIFI_MINIFI_STATUS_REPORTER_LOG_PERIOD.getKey(), "1"); - properties.setProperty(NIFI_MINIFI_STATUS_REPORTER_LOG_LEVEL.getKey(), LogLevel.TRACE.name()); - properties.setProperty(NIFI_MINIFI_STATUS_REPORTER_LOG_QUERY.getKey(), MOCK_QUERY); + BootstrapProperties properties = getProperties(LogLevel.TRACE); IOException ioException = new IOException("This is an expected test exception"); Mockito.when(queryableStatusAggregator.statusReport(MOCK_QUERY)).thenThrow(ioException); @@ -205,11 +203,11 @@ public class StatusLoggerTest { verify(logger, Mockito.atLeastOnce()).error(ENCOUNTERED_IO_EXCEPTION, ioException); } - private static Properties getProperties(LogLevel logLevel) { - Properties properties = new Properties(); - properties.setProperty(NIFI_MINIFI_STATUS_REPORTER_LOG_PERIOD.getKey(), "1"); - properties.setProperty(NIFI_MINIFI_STATUS_REPORTER_LOG_LEVEL.getKey(), logLevel.name()); - properties.setProperty(NIFI_MINIFI_STATUS_REPORTER_LOG_QUERY.getKey(), MOCK_QUERY); + private static BootstrapProperties getProperties(LogLevel logLevel) { + BootstrapProperties properties = mock(BootstrapProperties.class); + given(properties.getProperty(NIFI_MINIFI_STATUS_REPORTER_LOG_PERIOD.getKey())).willReturn("1"); + given(properties.getProperty(NIFI_MINIFI_STATUS_REPORTER_LOG_LEVEL.getKey())).willReturn(logLevel.name()); + given(properties.getProperty(NIFI_MINIFI_STATUS_REPORTER_LOG_QUERY.getKey())).willReturn(MOCK_QUERY); return properties; } @@ -222,7 +220,6 @@ public class StatusLoggerTest { @Override public ScheduledFuture scheduleAtFixedRate(Runnable command, long initialDelay, long period, TimeUnit unit) { command.run(); - // Return value is not used return null; } } diff --git a/minifi/minifi-c2/minifi-c2-service/pom.xml b/minifi/minifi-c2/minifi-c2-service/pom.xml index a8c96d3125..6a5fccad41 100644 --- a/minifi/minifi-c2/minifi-c2-service/pom.xml +++ b/minifi/minifi-c2/minifi-c2-service/pom.xml @@ -74,7 +74,6 @@ limitations under the License. com.google.guava guava - provided com.fasterxml.jackson.core diff --git a/minifi/minifi-commons/minifi-commons-api/src/main/java/org/apache/nifi/minifi/commons/api/MiNiFiProperties.java b/minifi/minifi-commons/minifi-commons-api/src/main/java/org/apache/nifi/minifi/commons/api/MiNiFiProperties.java index 93bf1c16a2..f685519f04 100644 --- a/minifi/minifi-commons/minifi-commons-api/src/main/java/org/apache/nifi/minifi/commons/api/MiNiFiProperties.java +++ b/minifi/minifi-commons/minifi-commons-api/src/main/java/org/apache/nifi/minifi/commons/api/MiNiFiProperties.java @@ -63,7 +63,7 @@ public enum MiNiFiProperties { NIFI_MINIFI_SECURITY_SSL_PROTOCOL("nifi.minifi.security.ssl.protocol", null, false, false, VALID), NIFI_MINIFI_FLOW_USE_PARENT_SSL("nifi.minifi.flow.use.parent.ssl", null, false, true, VALID), NIFI_MINIFI_SENSITIVE_PROPS_KEY("nifi.minifi.sensitive.props.key", null, true, false, VALID), - NIFI_MINIFI_SENSITIVE_PROPS_ALGORITHM("nifi.minifi.sensitive.props.algorithm", null, true, false, VALID), + NIFI_MINIFI_SENSITIVE_PROPS_ALGORITHM("nifi.minifi.sensitive.props.algorithm", null, false, false, VALID), C2_ENABLE("c2.enable", "false", false, true, BOOLEAN_VALIDATOR), C2_AGENT_HEARTBEAT_PERIOD("c2.agent.heartbeat.period", "1000", false, true, LONG_VALIDATOR), C2_AGENT_CLASS("c2.agent.class", "", false, true, VALID), @@ -120,6 +120,7 @@ public enum MiNiFiProperties { public static final String MINIFI_LOG_DIRECTORY = "nifi.minifi.log.directory"; public static final String MINIFI_APP_LOG_FILE = "nifi.minifi.app.log.file"; public static final String MINIFI_BOOTSTRAP_LOG_FILE = "nifi.minifi.bootstrap.log.file"; + public static final String ADDITIONAL_SENSITIVE_PROPERTIES_KEY = "nifi.minifi.sensitive.props.additional.keys"; private final String key; private final String defaultValue; diff --git a/minifi/minifi-commons/minifi-commons-utils/src/main/java/org/apache/nifi/minifi/commons/utils/PropertyUtil.java b/minifi/minifi-commons/minifi-commons-utils/src/main/java/org/apache/nifi/minifi/commons/utils/PropertyUtil.java index 10b628bf01..d90aa836cf 100644 --- a/minifi/minifi-commons/minifi-commons-utils/src/main/java/org/apache/nifi/minifi/commons/utils/PropertyUtil.java +++ b/minifi/minifi-commons/minifi-commons-utils/src/main/java/org/apache/nifi/minifi/commons/utils/PropertyUtil.java @@ -48,4 +48,5 @@ public abstract class PropertyUtil { private static Stream keyPermutations(String name) { return Stream.of(name, name.replace(DOT, UNDERSCORE), name.replace(HYPHEN, UNDERSCORE), name.replace(DOT, UNDERSCORE).replace(HYPHEN, UNDERSCORE)).distinct(); } + } diff --git a/minifi/minifi-commons/minifi-commons-utils/src/main/java/org/apache/nifi/minifi/commons/utils/SensitivePropertyUtils.java b/minifi/minifi-commons/minifi-commons-utils/src/main/java/org/apache/nifi/minifi/commons/utils/SensitivePropertyUtils.java new file mode 100644 index 0000000000..301f2637d8 --- /dev/null +++ b/minifi/minifi-commons/minifi-commons-utils/src/main/java/org/apache/nifi/minifi/commons/utils/SensitivePropertyUtils.java @@ -0,0 +1,83 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.nifi.minifi.commons.utils; + +import static java.lang.String.format; + +import java.io.BufferedInputStream; +import java.io.FileInputStream; +import java.io.InputStream; +import java.util.Optional; +import java.util.Properties; + +public class SensitivePropertyUtils { + + public static final String MINIFI_BOOTSTRAP_SENSITIVE_KEY = "minifi.bootstrap.sensitive.key"; + private static final String EMPTY = ""; + + private SensitivePropertyUtils() { + } + + public static String getFormattedKey() { + String key = getKey(System.getProperty("minifi.bootstrap.conf.file.path")); + // Format the key (check hex validity and remove spaces) + return getFormattedKey(key); + } + + public static String getFormattedKey(String unformattedKey) { + String key = formatHexKey(unformattedKey); + + if (isNotEmpty(key) && !isHexKeyValid(key)) { + throw new IllegalArgumentException("The key was not provided in valid hex format and of the correct length"); + } else { + return key; + } + } + + private static String formatHexKey(String input) { + return Optional.ofNullable(input) + .map(String::trim) + .filter(SensitivePropertyUtils::isNotEmpty) + .map(str -> str.replaceAll("[^0-9a-fA-F]", EMPTY).toLowerCase()) + .orElse(EMPTY); + } + + private static boolean isHexKeyValid(String key) { + return Optional.ofNullable(key) + .map(String::trim) + .filter(SensitivePropertyUtils::isNotEmpty) + .filter(k -> k.matches("^[0-9a-fA-F]{64}$")) + .isPresent(); + } + + private static String getKey(String bootstrapConfigFilePath) { + Properties properties = new Properties(); + + try (InputStream inputStream = new BufferedInputStream(new FileInputStream(bootstrapConfigFilePath))) { + properties.load(inputStream); + } catch (Exception e) { + throw new RuntimeException(format("Loading Bootstrap Properties [%s] failed", bootstrapConfigFilePath), e); + } + + return properties.getProperty(MINIFI_BOOTSTRAP_SENSITIVE_KEY); + } + + private static boolean isNotEmpty(String keyFilePath) { + return keyFilePath != null && !keyFilePath.isBlank(); + } +} diff --git a/minifi/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-properties-loader/pom.xml b/minifi/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-properties-loader/pom.xml new file mode 100644 index 0000000000..3b5249d22d --- /dev/null +++ b/minifi/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-properties-loader/pom.xml @@ -0,0 +1,55 @@ + + + + + 4.0.0 + + org.apache.nifi.minifi + minifi-framework + 2.0.0-SNAPSHOT + + + minifi-properties-loader + + + + org.apache.nifi + nifi-property-protection-loader + + + org.apache.nifi + nifi-property-protection-cipher + + + org.apache.nifi + nifi-properties + + + org.apache.nifi.minifi + minifi-commons-utils + provided + + + org.apache.nifi.minifi + minifi-commons-api + + + + \ No newline at end of file diff --git a/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/service/BootstrapProperties.java b/minifi/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-properties-loader/src/main/java/org/apache/nifi/minifi/properties/BootstrapProperties.java similarity index 51% rename from minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/service/BootstrapProperties.java rename to minifi/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-properties-loader/src/main/java/org/apache/nifi/minifi/properties/BootstrapProperties.java index 6484706689..29bac65691 100644 --- a/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/service/BootstrapProperties.java +++ b/minifi/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-properties-loader/src/main/java/org/apache/nifi/minifi/properties/BootstrapProperties.java @@ -14,24 +14,33 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.nifi.minifi.bootstrap.service; +package org.apache.nifi.minifi.properties; import static org.apache.nifi.minifi.commons.utils.PropertyUtil.resolvePropertyValue; +import java.util.Collections; +import java.util.Map; import java.util.Properties; +import java.util.Set; +import java.util.stream.Collectors; +import java.util.stream.Stream; +import org.apache.nifi.properties.ApplicationProperties; /** * Extends Properties functionality with System and Environment property override possibility. The property resolution also works with * dots and hyphens that are not supported in some shells. */ -public class BootstrapProperties extends Properties { +public class BootstrapProperties extends ApplicationProperties { - private BootstrapProperties() { - super(); + public BootstrapProperties() { + super(Collections.emptyMap()); } - public static BootstrapProperties getInstance() { - return new BootstrapProperties(); + public BootstrapProperties(Properties properties) { + super(properties); + } + public BootstrapProperties(Map properties) { + super(properties); } @Override @@ -41,4 +50,22 @@ public class BootstrapProperties extends Properties { .orElseGet(() -> super.getProperty(key)); } + @Override + public String getProperty(String key, String defaultValue) { + return resolvePropertyValue(key, System.getProperties()) + .or(() -> resolvePropertyValue(key, System.getenv())) + .orElseGet(() -> super.getProperty(key, defaultValue)); + } + + @Override + public Set getPropertyKeys() { + Set systemKeys = System.getProperties().keySet().stream().map(String::valueOf).collect(Collectors.toSet()); + return Stream.of(systemKeys, System.getenv().keySet(), super.getPropertyKeys()) + .flatMap(Set::stream) + .collect(Collectors.toSet()); + } + + public boolean containsKey(String key) { + return getPropertyKeys().contains(key); + } } diff --git a/minifi/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-properties-loader/src/main/java/org/apache/nifi/minifi/properties/BootstrapPropertiesLoader.java b/minifi/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-properties-loader/src/main/java/org/apache/nifi/minifi/properties/BootstrapPropertiesLoader.java new file mode 100644 index 0000000000..4c24394b8b --- /dev/null +++ b/minifi/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-properties-loader/src/main/java/org/apache/nifi/minifi/properties/BootstrapPropertiesLoader.java @@ -0,0 +1,49 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.nifi.minifi.properties; + +import static java.lang.String.format; +import static org.apache.nifi.minifi.commons.utils.SensitivePropertyUtils.MINIFI_BOOTSTRAP_SENSITIVE_KEY; +import static org.apache.nifi.minifi.commons.utils.SensitivePropertyUtils.getFormattedKey; + +import java.io.File; +import org.apache.nifi.properties.AesGcmSensitivePropertyProvider; + +public class BootstrapPropertiesLoader { + + public static BootstrapProperties load(File file) { + ProtectedBootstrapProperties protectedProperties = loadProtectedProperties(file); + if (protectedProperties.hasProtectedKeys()) { + String sensitiveKey = protectedProperties.getApplicationProperties().getProperty(MINIFI_BOOTSTRAP_SENSITIVE_KEY); + validateSensitiveKeyProperty(sensitiveKey); + String keyHex = getFormattedKey(sensitiveKey); + protectedProperties.addSensitivePropertyProvider(new AesGcmSensitivePropertyProvider(keyHex)); + } + return protectedProperties.getUnprotectedProperties(); + } + + public static ProtectedBootstrapProperties loadProtectedProperties(File file) { + return new ProtectedBootstrapProperties(PropertiesLoader.load(file, "Bootstrap")); + } + + private static void validateSensitiveKeyProperty(String sensitiveKey) { + if (sensitiveKey == null || sensitiveKey.trim().isEmpty()) { + throw new IllegalArgumentException(format("bootstrap.conf contains protected properties but %s is not found", MINIFI_BOOTSTRAP_SENSITIVE_KEY)); + } + } +} diff --git a/minifi/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-properties-loader/src/main/java/org/apache/nifi/minifi/properties/DuplicateDetectingProperties.java b/minifi/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-properties-loader/src/main/java/org/apache/nifi/minifi/properties/DuplicateDetectingProperties.java new file mode 100644 index 0000000000..92fc64430f --- /dev/null +++ b/minifi/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-properties-loader/src/main/java/org/apache/nifi/minifi/properties/DuplicateDetectingProperties.java @@ -0,0 +1,53 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.nifi.minifi.properties; + +import java.util.HashSet; +import java.util.Properties; +import java.util.Set; + +class DuplicateDetectingProperties extends Properties { + // Only need to retain Properties key. This will help prevent possible inadvertent exposure of sensitive Properties value + private final Set duplicateKeys = new HashSet<>(); + private final Set redundantKeys = new HashSet<>(); + + public DuplicateDetectingProperties() { + super(); + } + + public Set duplicateKeySet() { + return duplicateKeys; + } + + public Set redundantKeySet() { + return redundantKeys; + } + + @Override + public Object put(Object key, Object value) { + Object existingValue = super.put(key, value); + if (existingValue != null) { + if (existingValue.toString().equals(value.toString())) { + redundantKeys.add(key.toString()); + } else { + duplicateKeys.add(key.toString()); + } + } + return value; + } +} diff --git a/minifi/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-properties-loader/src/main/java/org/apache/nifi/minifi/properties/MiNiFiPropertiesLoader.java b/minifi/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-properties-loader/src/main/java/org/apache/nifi/minifi/properties/MiNiFiPropertiesLoader.java new file mode 100644 index 0000000000..d15fa3ee37 --- /dev/null +++ b/minifi/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-properties-loader/src/main/java/org/apache/nifi/minifi/properties/MiNiFiPropertiesLoader.java @@ -0,0 +1,104 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.minifi.properties; + +import java.io.File; +import org.apache.nifi.properties.AesGcmSensitivePropertyProvider; +import org.apache.nifi.util.NiFiBootstrapUtils; +import org.apache.nifi.util.NiFiProperties; + +public class MiNiFiPropertiesLoader { + + private static final String DEFAULT_APPLICATION_PROPERTIES_FILE_PATH = NiFiBootstrapUtils.getDefaultApplicationPropertiesFilePath(); + + private NiFiProperties instance; + private String keyHex; + + public MiNiFiPropertiesLoader(String keyHex) { + this.keyHex = keyHex; + } + + /** + * Returns a {@link ProtectedMiNiFiProperties} instance loaded from the + * serialized form in the file. Responsible for actually reading from disk + * and deserializing the properties. Returns a protected instance to allow + * for decryption operations. + * + * @param file the file containing serialized properties + * @return the ProtectedMiNiFiProperties instance + */ + ProtectedMiNiFiProperties loadProtectedProperties(File file) { + return new ProtectedMiNiFiProperties(PropertiesLoader.load(file, "Application")); + } + + /** + * Returns an instance of {@link NiFiProperties} loaded from the provided + * {@link File}. If any properties are protected, will attempt to use the + * {@link AesGcmSensitivePropertyProvider} to unprotect them + * transparently. + * + * @param file the File containing the serialized properties + * @return the NiFiProperties instance + */ + public NiFiProperties load(File file) { + ProtectedMiNiFiProperties protectedProperties = loadProtectedProperties(file); + if (protectedProperties.hasProtectedKeys()) { + protectedProperties.addSensitivePropertyProvider(new AesGcmSensitivePropertyProvider(keyHex)); + } + return new MultiSourceMinifiProperties(protectedProperties.getUnprotectedPropertiesAsMap()); + } + + /** + * Returns an instance of {@link NiFiProperties}. If the path is empty, this + * will load the default properties file as specified by + * {@code NiFiProperties.PROPERTY_FILE_PATH}. + * + * @param path the path of the serialized properties file + * @return the NiFiProperties instance + * @see MiNiFiPropertiesLoader#load(File) + */ + public NiFiProperties load(String path) { + if (path != null && !path.trim().isEmpty()) { + return load(new File(path)); + } else { + return loadDefault(); + } + } + + /** + * Returns the loaded {@link NiFiProperties} instance. If none is currently + * loaded, attempts to load the default instance. + *

+ *

+ * NOTE: This method is used reflectively by the process which starts MiNiFi + * so changes to it must be made in conjunction with that mechanism.

+ * + * @return the current NiFiProperties instance + */ + public NiFiProperties get() { + if (instance == null) { + instance = loadDefault(); + } + + return instance; + } + + private NiFiProperties loadDefault() { + return load(DEFAULT_APPLICATION_PROPERTIES_FILE_PATH); + } + +} diff --git a/minifi/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-properties-loader/src/main/java/org/apache/nifi/minifi/properties/MultiSourceMinifiProperties.java b/minifi/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-properties-loader/src/main/java/org/apache/nifi/minifi/properties/MultiSourceMinifiProperties.java new file mode 100644 index 0000000000..f39a97dee5 --- /dev/null +++ b/minifi/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-properties-loader/src/main/java/org/apache/nifi/minifi/properties/MultiSourceMinifiProperties.java @@ -0,0 +1,55 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.minifi.properties; + +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; +import java.util.stream.Stream; +import org.apache.nifi.minifi.commons.utils.PropertyUtil; +import org.apache.nifi.util.NiFiProperties; + +/** + * Extends NiFi properties functionality with System and Environment property override possibility. The property resolution also works with + * dots and hyphens that are not supported in some shells. + */ +public class MultiSourceMinifiProperties extends NiFiProperties { + + public MultiSourceMinifiProperties(Map props) { + super(props); + } + + @Override + public Set getPropertyKeys() { + return Stream.of(System.getProperties().stringPropertyNames(), System.getenv().keySet(), super.getPropertyKeys()) + .flatMap(Set::stream) + .collect(Collectors.toSet()); + } + + @Override + public int size() { + return getPropertyKeys().size(); + } + + @Override + public String getProperty(String key) { + return PropertyUtil.resolvePropertyValue(key, System.getProperties()) + .or(() -> PropertyUtil.resolvePropertyValue(key, System.getenv())) + .orElseGet(() -> super.getProperty(key)); + } + +} diff --git a/minifi/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-properties-loader/src/main/java/org/apache/nifi/minifi/properties/PropertiesLoader.java b/minifi/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-properties-loader/src/main/java/org/apache/nifi/minifi/properties/PropertiesLoader.java new file mode 100644 index 0000000000..9785fda997 --- /dev/null +++ b/minifi/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-properties-loader/src/main/java/org/apache/nifi/minifi/properties/PropertiesLoader.java @@ -0,0 +1,61 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.nifi.minifi.properties; + +import java.io.BufferedInputStream; +import java.io.File; +import java.io.FileInputStream; +import java.io.InputStream; +import java.util.Properties; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public interface PropertiesLoader { + + static final Logger logger = LoggerFactory.getLogger(PropertiesLoader.class); + + static Properties load(File file, String propertiesType) { + if (file == null || !file.exists() || !file.canRead()) { + throw new IllegalArgumentException(String.format("{} Properties [%s] not found", propertiesType, file)); + } + + logger.info("Loading {} Properties [{}]", propertiesType, file); + DuplicateDetectingProperties rawProperties = new DuplicateDetectingProperties(); + + try (InputStream inputStream = new BufferedInputStream(new FileInputStream(file))) { + rawProperties.load(inputStream); + } catch (Exception e) { + throw new RuntimeException(String.format("Loading {} Properties [%s] failed", propertiesType, file), e); + } + + if (!rawProperties.redundantKeySet().isEmpty()) { + logger.warn("Duplicate property keys with the same value were detected in the properties file: {}", String.join(", ", rawProperties.redundantKeySet())); + } + if (!rawProperties.duplicateKeySet().isEmpty()) { + throw new IllegalArgumentException("Duplicate property keys with different values were detected in the properties file: " + String.join(", ", rawProperties.duplicateKeySet())); + } + + Properties properties = new Properties(); + rawProperties.stringPropertyNames() + .forEach(key -> { + String property = rawProperties.getProperty(key); + properties.setProperty(key, property.trim()); + }); + return properties; + } +} diff --git a/minifi/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-properties-loader/src/main/java/org/apache/nifi/minifi/properties/ProtectedBootstrapProperties.java b/minifi/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-properties-loader/src/main/java/org/apache/nifi/minifi/properties/ProtectedBootstrapProperties.java new file mode 100644 index 0000000000..486574a822 --- /dev/null +++ b/minifi/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-properties-loader/src/main/java/org/apache/nifi/minifi/properties/ProtectedBootstrapProperties.java @@ -0,0 +1,134 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.nifi.minifi.properties; + +import static org.apache.nifi.minifi.commons.api.MiNiFiProperties.ADDITIONAL_SENSITIVE_PROPERTIES_KEY; +import static org.apache.nifi.minifi.properties.ProtectedMiNiFiProperties.DEFAULT_SENSITIVE_PROPERTIES; + +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.Properties; +import java.util.Set; +import java.util.stream.Collectors; +import java.util.stream.Stream; +import org.apache.nifi.minifi.commons.api.MiNiFiProperties; +import org.apache.nifi.properties.ApplicationPropertiesProtector; +import org.apache.nifi.properties.ProtectedProperties; +import org.apache.nifi.properties.SensitivePropertyProtectionException; +import org.apache.nifi.properties.SensitivePropertyProtector; +import org.apache.nifi.properties.SensitivePropertyProvider; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class ProtectedBootstrapProperties extends BootstrapProperties implements ProtectedProperties, + SensitivePropertyProtector { + + private static final Logger logger = LoggerFactory.getLogger(ProtectedBootstrapProperties.class); + + private BootstrapProperties bootstrapProperties; + + private final SensitivePropertyProtector propertyProtectionDelegate; + + public ProtectedBootstrapProperties(BootstrapProperties props) { + super(); + this.bootstrapProperties = props; + this.propertyProtectionDelegate = new ApplicationPropertiesProtector<>(this); + logger.debug("Loaded {} properties (including {} protection schemes) into ProtectedBootstrapProperties", getApplicationProperties().getPropertyKeys().size(), + getProtectedPropertyKeys().size()); + } + + public ProtectedBootstrapProperties(Properties rawProps) { + this(new BootstrapProperties(rawProps)); + } + + @Override + public Set getPropertyKeysIncludingProtectionSchemes() { + return propertyProtectionDelegate.getPropertyKeysIncludingProtectionSchemes(); + } + + @Override + public List getSensitivePropertyKeys() { + return propertyProtectionDelegate.getSensitivePropertyKeys(); + } + + @Override + public List getPopulatedSensitivePropertyKeys() { + return propertyProtectionDelegate.getPopulatedSensitivePropertyKeys(); + } + + @Override + public boolean hasProtectedKeys() { + return propertyProtectionDelegate.hasProtectedKeys(); + } + + @Override + public Map getProtectedPropertyKeys() { + return propertyProtectionDelegate.getProtectedPropertyKeys(); + } + + @Override + public boolean isPropertySensitive(String key) { + return propertyProtectionDelegate.isPropertySensitive(key); + } + + @Override + public boolean isPropertyProtected(String key) { + return propertyProtectionDelegate.isPropertyProtected(key); + } + + @Override + public BootstrapProperties getUnprotectedProperties() throws SensitivePropertyProtectionException { + return propertyProtectionDelegate.getUnprotectedProperties(); + } + + @Override + public void addSensitivePropertyProvider(SensitivePropertyProvider sensitivePropertyProvider) { + propertyProtectionDelegate.addSensitivePropertyProvider(sensitivePropertyProvider); + } + + @Override + public String getAdditionalSensitivePropertiesKeys() { + return getProperty(getAdditionalSensitivePropertiesKeysName()); + } + + @Override + public String getAdditionalSensitivePropertiesKeysName() { + return ADDITIONAL_SENSITIVE_PROPERTIES_KEY; + } + + @Override + public List getDefaultSensitiveProperties() { + return Stream.of(DEFAULT_SENSITIVE_PROPERTIES, Arrays.stream(MiNiFiProperties.values()).filter(MiNiFiProperties::isSensitive).map(MiNiFiProperties::getKey).collect(Collectors.toList())) + .flatMap(List::stream).distinct().collect(Collectors.toList()); + } + + @Override + public BootstrapProperties getApplicationProperties() { + if (this.bootstrapProperties == null) { + this.bootstrapProperties = new BootstrapProperties(); + } + + return this.bootstrapProperties; + } + + @Override + public BootstrapProperties createApplicationProperties(Properties rawProperties) { + return new BootstrapProperties(rawProperties); + } +} diff --git a/minifi/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-properties-loader/src/main/java/org/apache/nifi/minifi/properties/ProtectedMiNiFiProperties.java b/minifi/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-properties-loader/src/main/java/org/apache/nifi/minifi/properties/ProtectedMiNiFiProperties.java new file mode 100644 index 0000000000..238a110413 --- /dev/null +++ b/minifi/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-properties-loader/src/main/java/org/apache/nifi/minifi/properties/ProtectedMiNiFiProperties.java @@ -0,0 +1,239 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.minifi.properties; + +import static java.util.Arrays.asList; +import static org.apache.nifi.minifi.commons.api.MiNiFiProperties.ADDITIONAL_SENSITIVE_PROPERTIES_KEY; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.Properties; +import java.util.Set; +import java.util.function.Function; +import java.util.stream.Collectors; +import java.util.stream.Stream; +import org.apache.nifi.minifi.commons.api.MiNiFiProperties; +import org.apache.nifi.properties.ApplicationPropertiesProtector; +import org.apache.nifi.properties.ProtectedProperties; +import org.apache.nifi.properties.SensitivePropertyProtectionException; +import org.apache.nifi.properties.SensitivePropertyProtector; +import org.apache.nifi.properties.SensitivePropertyProvider; +import org.apache.nifi.util.NiFiProperties; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Decorator class for intermediate phase when {@link MiNiFiPropertiesLoader} loads the + * raw properties file and performs unprotection activities before returning a clean + * implementation of {@link NiFiProperties}. + * This encapsulates the sensitive property access logic from external consumers + * of {@code NiFiProperties}. + */ +public class ProtectedMiNiFiProperties extends NiFiProperties implements ProtectedProperties, + SensitivePropertyProtector { + private static final Logger logger = LoggerFactory.getLogger(ProtectedMiNiFiProperties.class); + public static final List DEFAULT_SENSITIVE_PROPERTIES = new ArrayList<>(asList( + SECURITY_KEY_PASSWD, + SECURITY_KEYSTORE_PASSWD, + SECURITY_TRUSTSTORE_PASSWD, + SENSITIVE_PROPS_KEY + )); + + private final SensitivePropertyProtector propertyProtectionDelegate; + + private NiFiProperties applicationProperties; + + public ProtectedMiNiFiProperties() { + this(new NiFiProperties()); + } + + /** + * Creates an instance containing the provided {@link NiFiProperties}. + * + * @param props the NiFiProperties to contain + */ + public ProtectedMiNiFiProperties(NiFiProperties props) { + this.applicationProperties = props; + this.propertyProtectionDelegate = new ApplicationPropertiesProtector<>(this); + logger.debug("Loaded {} properties (including {} protection schemes) into ProtectedNiFiProperties", getApplicationProperties() + .getPropertyKeys().size(), getProtectedPropertyKeys().size()); + } + + /** + * Creates an instance containing the provided raw {@link Properties}. + * + * @param rawProps the Properties to contain + */ + public ProtectedMiNiFiProperties(Properties rawProps) { + this(new NiFiProperties(rawProps)); + } + + @Override + public String getAdditionalSensitivePropertiesKeys() { + return getProperty(getAdditionalSensitivePropertiesKeysName()); + } + + @Override + public String getAdditionalSensitivePropertiesKeysName() { + return ADDITIONAL_SENSITIVE_PROPERTIES_KEY; + } + + @Override + public List getDefaultSensitiveProperties() { + return Stream.of(DEFAULT_SENSITIVE_PROPERTIES, + Arrays.stream(MiNiFiProperties.values()).filter(MiNiFiProperties::isSensitive).map(MiNiFiProperties::getKey).toList()).flatMap(List::stream).distinct().toList(); + } + + /** + * Returns the internal representation of the {@link NiFiProperties} -- protected + * or not as determined by the current state. No guarantee is made to the + * protection state of these properties. If the internal reference is null, a new + * {@link NiFiProperties} instance is created. + * + * @return the internal properties + */ + public NiFiProperties getApplicationProperties() { + if (this.applicationProperties == null) { + this.applicationProperties = new NiFiProperties(); + } + + return this.applicationProperties; + } + + @Override + public NiFiProperties createApplicationProperties(final Properties rawProperties) { + return new NiFiProperties(rawProperties); + } + + /** + * Retrieves the property value for the given property key. + * + * @param key the key of property value to lookup + * @return value of property at given key or null if not found + */ + @Override + public String getProperty(String key) { + return getApplicationProperties().getProperty(key); + } + + /** + * Retrieves all known property keys. + * + * @return all known property keys + */ + @Override + public Set getPropertyKeys() { + return propertyProtectionDelegate.getPropertyKeys(); + } + + /** + * Returns the number of properties, excluding protection scheme properties. + *

+ * Example: + *

+ * key: E(value, key) + * key.protected: aes/gcm/256 + * key2: value2 + *

+ * would return size 2 + * + * @return the count of real properties + */ + @Override + public int size() { + return propertyProtectionDelegate.size(); + } + + @Override + public Set getPropertyKeysIncludingProtectionSchemes() { + return propertyProtectionDelegate.getPropertyKeysIncludingProtectionSchemes(); + } + + @Override + public List getSensitivePropertyKeys() { + return propertyProtectionDelegate.getSensitivePropertyKeys(); + } + + @Override + public List getPopulatedSensitivePropertyKeys() { + return propertyProtectionDelegate.getPopulatedSensitivePropertyKeys(); + } + + @Override + public boolean hasProtectedKeys() { + return propertyProtectionDelegate.hasProtectedKeys(); + } + + @Override + public Map getProtectedPropertyKeys() { + return propertyProtectionDelegate.getProtectedPropertyKeys(); + } + + @Override + public boolean isPropertySensitive(final String key) { + return propertyProtectionDelegate.isPropertySensitive(key); + } + + @Override + public boolean isPropertyProtected(final String key) { + return propertyProtectionDelegate.isPropertyProtected(key); + } + + @Override + public NiFiProperties getUnprotectedProperties() throws SensitivePropertyProtectionException { + return propertyProtectionDelegate.getUnprotectedProperties(); + } + + @Override + public void addSensitivePropertyProvider(final SensitivePropertyProvider sensitivePropertyProvider) { + propertyProtectionDelegate.addSensitivePropertyProvider(sensitivePropertyProvider); + } + + public Map getUnprotectedPropertiesAsMap() { + NiFiProperties niFiProperties = propertyProtectionDelegate.getUnprotectedProperties(); + return niFiProperties.getPropertyKeys().stream().collect(Collectors.toMap(Function.identity(), niFiProperties::getProperty)); + } + + /** + * Returns the number of properties that are marked as protected in the provided {@link NiFiProperties} instance without requiring external creation of a + * {@link ProtectedMiNiFiProperties} instance. + * + * @param plainProperties the instance to count protected properties + * @return the number of protected properties + */ + public static int countProtectedProperties(final NiFiProperties plainProperties) { + return new ProtectedMiNiFiProperties(plainProperties).getProtectedPropertyKeys().size(); + } + + /** + * Returns the number of properties that are marked as sensitive in the provided {@link NiFiProperties} instance without requiring external creation of a + * {@link ProtectedMiNiFiProperties} instance. + * + * @param plainProperties the instance to count sensitive properties + * @return the number of sensitive properties + */ + public static int countSensitiveProperties(final NiFiProperties plainProperties) { + return new ProtectedMiNiFiProperties(plainProperties).getSensitivePropertyKeys().size(); + } + + @Override + public String toString() { + return String.format("%s Size [%d]", getClass().getSimpleName(), size()); + } +} diff --git a/minifi/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-properties-loader/src/test/java/org/apache/nifi/minifi/properties/BootstrapPropertiesLoaderTest.java b/minifi/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-properties-loader/src/test/java/org/apache/nifi/minifi/properties/BootstrapPropertiesLoaderTest.java new file mode 100644 index 0000000000..c3d3febcac --- /dev/null +++ b/minifi/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-properties-loader/src/test/java/org/apache/nifi/minifi/properties/BootstrapPropertiesLoaderTest.java @@ -0,0 +1,82 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.nifi.minifi.properties; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertThrows; + +import java.io.File; +import java.net.URL; +import java.util.Map; +import java.util.function.Function; +import java.util.stream.Collectors; +import org.junit.jupiter.api.Test; + +class BootstrapPropertiesLoaderTest { + + private static final String NON_EXISTING_FILE_PATH = "/conf/nonexisting.conf"; + private static final String DUPLICATED_ITEMS_FILE_PATH = "/conf/bootstrap_duplicated_items.conf"; + private static final String UNPROTECTED_ITEMS_FILE_PATH = "/conf/bootstrap_unprotected.conf"; + private static final String PROTECTED_ITEMS_FILE_PATH = "/conf/bootstrap_protected.conf"; + private static final String PROTECTED_ITEMS_WITHOUT_SENSITIVE_KEY_FILE_PATH = "/conf/bootstrap_protected_without_sensitive_key.conf"; + + private static final Map UNPROTECTED_PROPERTIES = Map.of("nifi.minifi.security.keystorePasswd", "testPassword", "nifi.minifi.sensitive.props.key", "testSensitivePropsKey"); + + @Test + void shouldThrowIllegalArgumentExceptionIfFileIsNotProvided() { + assertThrows(IllegalArgumentException.class, () -> BootstrapPropertiesLoader.load(null)); + } + + @Test + void shouldThrowIllegalArgumentExceptionIfFileDoesNotExists() { + assertThrows(IllegalArgumentException.class, () -> BootstrapPropertiesLoader.load(new File(NON_EXISTING_FILE_PATH))); + } + + @Test + void shouldThrowIllegalArgumentExceptionIfTheConfigFileContainsDuplicatedKeysWithDifferentValues() { + assertThrows(IllegalArgumentException.class, () -> BootstrapPropertiesLoader.load(getFile(DUPLICATED_ITEMS_FILE_PATH))); + } + + @Test + void shouldReturnPropertiesIfConfigFileDoesNotContainProtectedProperties() { + BootstrapProperties bootstrapProperties = BootstrapPropertiesLoader.load(getFile(UNPROTECTED_ITEMS_FILE_PATH)); + + assertEquals(UNPROTECTED_PROPERTIES, + bootstrapProperties.getPropertyKeys().stream().filter(UNPROTECTED_PROPERTIES::containsKey).collect(Collectors.toMap(Function.identity(), bootstrapProperties::getProperty))); + } + + @Test + void shouldReturnUnProtectedProperties() { + BootstrapProperties bootstrapProperties = BootstrapPropertiesLoader.load(getFile(PROTECTED_ITEMS_FILE_PATH)); + + assertEquals(UNPROTECTED_PROPERTIES, + bootstrapProperties.getPropertyKeys().stream().filter(UNPROTECTED_PROPERTIES::containsKey).collect(Collectors.toMap(Function.identity(), bootstrapProperties::getProperty))); + } + + @Test + void shouldThrowIllegalArgumentExceptionIfFileContainsProtectedPropertiesButSensitiveKeyIsMissing() { + assertThrows(IllegalArgumentException.class, () -> BootstrapPropertiesLoader.load(getFile(PROTECTED_ITEMS_WITHOUT_SENSITIVE_KEY_FILE_PATH))); + } + + private File getFile(String duplicatedItemsFilePath) { + URL resource = BootstrapPropertiesLoaderTest.class.getResource(duplicatedItemsFilePath); + assertNotNull(resource); + return new File(resource.getPath()); + } +} \ No newline at end of file diff --git a/minifi/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-properties-loader/src/test/java/org/apache/nifi/minifi/properties/MiNiFiPropertiesLoaderTest.java b/minifi/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-properties-loader/src/test/java/org/apache/nifi/minifi/properties/MiNiFiPropertiesLoaderTest.java new file mode 100644 index 0000000000..fd4c863dd0 --- /dev/null +++ b/minifi/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-properties-loader/src/test/java/org/apache/nifi/minifi/properties/MiNiFiPropertiesLoaderTest.java @@ -0,0 +1,90 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.nifi.minifi.properties; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertThrows; + +import java.io.File; +import java.net.URL; +import java.util.Map; +import java.util.function.Function; +import java.util.stream.Collectors; +import org.apache.nifi.util.NiFiProperties; +import org.junit.jupiter.api.Test; + +class MiNiFiPropertiesLoaderTest { + private static final String NON_EXISTING_FILE_PATH = "/conf/nonexisting.properties"; + private static final String DUPLICATED_ITEMS_FILE_PATH = "/conf/bootstrap_duplicated_items.conf"; + private static final String UNPROTECTED_ITEMS_FILE_PATH = "/conf/minifi_unprotected.properties"; + private static final String PROTECTED_ITEMS_FILE_PATH = "/conf/minifi_protected.properties"; + private static final Map UNPROTECTED_PROPERTIES = Map.of("nifi.security.keystorePasswd", "testPassword", "nifi.security.keyPasswd", + "testSensitivePropsKey"); + private static final String PROTECTION_KEY = "00714ae7a77b24cde1d36bd19472777e0d4ab02c38913b7f9bf41f3963147b4f"; + + @Test + void shouldThrowIllegalArgumentExceptionIfFileIsNotProvided() { + MiNiFiPropertiesLoader miNiFiPropertiesLoader = new MiNiFiPropertiesLoader(""); + assertThrows(IllegalArgumentException.class, () -> miNiFiPropertiesLoader.load((String) null)); + } + + @Test + void shouldThrowIllegalArgumentExceptionIfFileDoesNotExists() { + MiNiFiPropertiesLoader miNiFiPropertiesLoader = new MiNiFiPropertiesLoader(""); + assertThrows(IllegalArgumentException.class, () -> miNiFiPropertiesLoader.load(new File(NON_EXISTING_FILE_PATH))); + } + + @Test + void shouldThrowIllegalArgumentExceptionIfTheConfigFileContainsDuplicatedKeysWithDifferentValues() { + MiNiFiPropertiesLoader miNiFiPropertiesLoader = new MiNiFiPropertiesLoader(""); + + assertThrows(IllegalArgumentException.class, () -> miNiFiPropertiesLoader.load(getFile(DUPLICATED_ITEMS_FILE_PATH))); + } + + @Test + void shouldReturnPropertiesIfConfigFileDoesNotContainProtectedProperties() { + MiNiFiPropertiesLoader miNiFiPropertiesLoader = new MiNiFiPropertiesLoader(""); + + NiFiProperties niFiProperties = miNiFiPropertiesLoader.load(getFile(UNPROTECTED_ITEMS_FILE_PATH)); + + assertEquals(UNPROTECTED_PROPERTIES, + niFiProperties.getPropertyKeys().stream().filter(UNPROTECTED_PROPERTIES::containsKey).collect(Collectors.toMap(Function.identity(), niFiProperties::getProperty))); + } + + @Test + void shouldReturnUnProtectedProperties() { + MiNiFiPropertiesLoader miNiFiPropertiesLoader = new MiNiFiPropertiesLoader(PROTECTION_KEY); + + NiFiProperties niFiProperties = miNiFiPropertiesLoader.load(getUrl(PROTECTED_ITEMS_FILE_PATH).getPath()); + + assertEquals(UNPROTECTED_PROPERTIES, + niFiProperties.getPropertyKeys().stream().filter(UNPROTECTED_PROPERTIES::containsKey).collect(Collectors.toMap(Function.identity(), niFiProperties::getProperty))); + } + + private File getFile(String propertiesFilePath) { + URL resource = getUrl(propertiesFilePath); + return new File(resource.getPath()); + } + + private static URL getUrl(String propertiesFilePath) { + URL resource = BootstrapPropertiesLoaderTest.class.getResource(propertiesFilePath); + assertNotNull(resource); + return resource; + } +} \ No newline at end of file diff --git a/minifi/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-properties-loader/src/test/resources/conf/bootstrap_duplicated_items.conf b/minifi/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-properties-loader/src/test/resources/conf/bootstrap_duplicated_items.conf new file mode 100644 index 0000000000..dd742bafda --- /dev/null +++ b/minifi/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-properties-loader/src/test/resources/conf/bootstrap_duplicated_items.conf @@ -0,0 +1,18 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# property file that contains the same key with different values +property1=test +property1=test2 \ No newline at end of file diff --git a/minifi/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-properties-loader/src/test/resources/conf/bootstrap_protected.conf b/minifi/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-properties-loader/src/test/resources/conf/bootstrap_protected.conf new file mode 100644 index 0000000000..86966e8999 --- /dev/null +++ b/minifi/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-properties-loader/src/test/resources/conf/bootstrap_protected.conf @@ -0,0 +1,22 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# property file that contains the protected properties and a sensitive key to decrypt it +nifi.minifi.security.keystorePasswd=noBD32A0ANcz/BHv||nhtyNhlFgNNZcVhgxEOg2xUZ5UAihgyBit1drQ== +nifi.minifi.security.keystorePasswd.protected=aes/gcm/256 +nifi.minifi.sensitive.props.key=JUMeAtpD1Q7CiXHt||BppiPMoWXxkKBl5mYsxP5vkVabsXZyLu2lxjyv9LMHc6RJEE9g== +nifi.minifi.sensitive.props.key.protected=aes/gcm/256 + +minifi.bootstrap.sensitive.key=00714ae7a77b24cde1d36bd19472777e0d4ab02c38913b7f9bf41f3963147b4f diff --git a/minifi/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-properties-loader/src/test/resources/conf/bootstrap_protected_without_sensitive_key.conf b/minifi/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-properties-loader/src/test/resources/conf/bootstrap_protected_without_sensitive_key.conf new file mode 100644 index 0000000000..bcc8fe9d8a --- /dev/null +++ b/minifi/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-properties-loader/src/test/resources/conf/bootstrap_protected_without_sensitive_key.conf @@ -0,0 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# property file that contains the protected properties but without minifi.bootstrap.sensitive.key +nifi.minifi.security.keystorePasswd=noBD32A0ANcz/BHv||nhtyNhlFgNNZcVhgxEOg2xUZ5UAihgyBit1drQ== +nifi.minifi.security.keystorePasswd.protected=aes/gcm/256 +nifi.minifi.sensitive.props.key=JUMeAtpD1Q7CiXHt||BppiPMoWXxkKBl5mYsxP5vkVabsXZyLu2lxjyv9LMHc6RJEE9g== +nifi.minifi.sensitive.props.key.protected=aes/gcm/256 diff --git a/minifi/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-properties-loader/src/test/resources/conf/bootstrap_unprotected.conf b/minifi/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-properties-loader/src/test/resources/conf/bootstrap_unprotected.conf new file mode 100644 index 0000000000..e08d4790a5 --- /dev/null +++ b/minifi/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-properties-loader/src/test/resources/conf/bootstrap_unprotected.conf @@ -0,0 +1,18 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# property file that contains the unprotected properties +nifi.minifi.security.keystorePasswd=testPassword +nifi.minifi.sensitive.props.key=testSensitivePropsKey \ No newline at end of file diff --git a/minifi/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-properties-loader/src/test/resources/conf/minifi_duplicated.properties b/minifi/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-properties-loader/src/test/resources/conf/minifi_duplicated.properties new file mode 100644 index 0000000000..285d126c1d --- /dev/null +++ b/minifi/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-properties-loader/src/test/resources/conf/minifi_duplicated.properties @@ -0,0 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +nifi.security.keystorePasswd=s+jNt0t608+F0uq9||PWPtOCr5fmItmL8ZsgpheQxkkJJzXWqrNvqdCL/gcGE2cy1NTgGDhI1apuacNHjj +nifi.security.keystorePasswd=123 +nifi.security.keystorePasswd.protected=aes/gcm/256 +nifi.security.keyPasswd=JUMeAtpD1Q7CiXHt||BppiPMoWXxkKBl5mYsxP5vkVabsXZyLu2lxjyv9LMHc6RJEE9g== +nifi.security.keyPasswd.protected=aes/gcm/256 \ No newline at end of file diff --git a/minifi/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-properties-loader/src/test/resources/conf/minifi_protected.properties b/minifi/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-properties-loader/src/test/resources/conf/minifi_protected.properties new file mode 100644 index 0000000000..85319171cf --- /dev/null +++ b/minifi/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-properties-loader/src/test/resources/conf/minifi_protected.properties @@ -0,0 +1,19 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +nifi.security.keystorePasswd=noBD32A0ANcz/BHv||nhtyNhlFgNNZcVhgxEOg2xUZ5UAihgyBit1drQ== +nifi.security.keystorePasswd.protected=aes/gcm/256 +nifi.security.keyPasswd=JUMeAtpD1Q7CiXHt||BppiPMoWXxkKBl5mYsxP5vkVabsXZyLu2lxjyv9LMHc6RJEE9g== +nifi.security.keyPasswd.protected=aes/gcm/256 \ No newline at end of file diff --git a/minifi/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-properties-loader/src/test/resources/conf/minifi_unprotected.properties b/minifi/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-properties-loader/src/test/resources/conf/minifi_unprotected.properties new file mode 100644 index 0000000000..06d04d3f31 --- /dev/null +++ b/minifi/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-properties-loader/src/test/resources/conf/minifi_unprotected.properties @@ -0,0 +1,17 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +nifi.security.keystorePasswd=testPassword +nifi.security.keyPasswd=testSensitivePropsKey \ No newline at end of file diff --git a/minifi/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-runtime/pom.xml b/minifi/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-runtime/pom.xml index 8601de6541..5d9b215633 100644 --- a/minifi/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-runtime/pom.xml +++ b/minifi/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-runtime/pom.xml @@ -58,8 +58,9 @@ limitations under the License. provided - org.apache.nifi - nifi-properties-loader + org.apache.nifi.minifi + minifi-properties-loader + 2.0.0-SNAPSHOT provided @@ -76,5 +77,6 @@ limitations under the License. com.fasterxml.jackson.core jackson-databind + diff --git a/minifi/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-runtime/src/main/java/org/apache/nifi/minifi/MiNiFi.java b/minifi/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-runtime/src/main/java/org/apache/nifi/minifi/MiNiFi.java index 02ecc75e85..567381ddda 100644 --- a/minifi/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-runtime/src/main/java/org/apache/nifi/minifi/MiNiFi.java +++ b/minifi/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-runtime/src/main/java/org/apache/nifi/minifi/MiNiFi.java @@ -16,8 +16,14 @@ */ package org.apache.nifi.minifi; +import static org.apache.nifi.minifi.commons.utils.SensitivePropertyUtils.getFormattedKey; +import static org.apache.nifi.minifi.util.BootstrapClassLoaderUtils.createBootstrapClassLoader; + import java.io.File; import java.io.IOException; +import java.lang.reflect.Constructor; +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; import java.util.Set; import java.util.Timer; import java.util.TimerTask; @@ -30,7 +36,6 @@ import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicLong; import org.apache.nifi.NiFiServer; import org.apache.nifi.bundle.Bundle; -import org.apache.nifi.minifi.util.MultiSourceMinifiProperties; import org.apache.nifi.nar.ExtensionMapping; import org.apache.nifi.nar.NarClassLoaders; import org.apache.nifi.nar.NarClassLoadersHolder; @@ -46,6 +51,7 @@ import org.slf4j.bridge.SLF4JBridgeHandler; public class MiNiFi { private static final Logger logger = LoggerFactory.getLogger(MiNiFi.class); + private final MiNiFiServer minifiServer; private volatile boolean shutdown = false; @@ -155,7 +161,7 @@ public class MiNiFi { logger.info("MiNiFi server shutdown completed (nicely or otherwise)."); } catch (final Throwable t) { - logger.warn("Problem occurred ensuring MiNiFi server was properly terminated due to " + t); + logger.warn("Problem occurred ensuring MiNiFi server was properly terminated due to {}", t.getMessage()); } } @@ -218,10 +224,41 @@ public class MiNiFi { public static void main(String[] args) { logger.info("Launching MiNiFi..."); try { - NiFiProperties niFiProperties = MultiSourceMinifiProperties.getInstance(); - new MiNiFi(niFiProperties); + NiFiProperties properties = getValidatedMiNifiProperties(); + new MiNiFi(properties); } catch (final Throwable t) { logger.error("Failure to launch MiNiFi due to " + t, t); } } + + protected static NiFiProperties getValidatedMiNifiProperties() { + NiFiProperties properties = initializeProperties(createBootstrapClassLoader()); + properties.validate(); + return properties; + } + + private static NiFiProperties initializeProperties(ClassLoader boostrapLoader) { + ClassLoader contextClassLoader = Thread.currentThread().getContextClassLoader(); + String key = getFormattedKey(); + + Thread.currentThread().setContextClassLoader(boostrapLoader); + + try { + Class propsLoaderClass = Class.forName("org.apache.nifi.minifi.properties.MiNiFiPropertiesLoader", true, boostrapLoader); + Constructor constructor = propsLoaderClass.getDeclaredConstructor(String.class); + Object loaderInstance = constructor.newInstance(key); + Method getMethod = propsLoaderClass.getMethod("get"); + NiFiProperties properties = (NiFiProperties) getMethod.invoke(loaderInstance); + logger.info("Application Properties loaded [{}]", properties.size()); + return properties; + } catch (InvocationTargetException wrappedException) { + throw new IllegalArgumentException("There was an issue decrypting protected properties", wrappedException.getCause() == null ? wrappedException : wrappedException.getCause()); + } catch (final IllegalAccessException | NoSuchMethodException | ClassNotFoundException | InstantiationException reex) { + throw new IllegalArgumentException("Unable to access properties loader in the expected manner - apparent classpath or build issue", reex); + } catch (final RuntimeException e) { + throw new IllegalArgumentException("There was an issue decrypting protected properties", e); + } finally { + Thread.currentThread().setContextClassLoader(contextClassLoader); + } + } } diff --git a/minifi/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-runtime/src/main/java/org/apache/nifi/minifi/util/BootstrapClassLoaderUtils.java b/minifi/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-runtime/src/main/java/org/apache/nifi/minifi/util/BootstrapClassLoaderUtils.java new file mode 100644 index 0000000000..0c7580cc60 --- /dev/null +++ b/minifi/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-runtime/src/main/java/org/apache/nifi/minifi/util/BootstrapClassLoaderUtils.java @@ -0,0 +1,59 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.nifi.minifi.util; + +import java.io.IOException; +import java.net.MalformedURLException; +import java.net.URL; +import java.net.URLClassLoader; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Stream; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Util for creating class loader with bootstrap libs. + */ +public final class BootstrapClassLoaderUtils { + private static final Logger LOGGER = LoggerFactory.getLogger(BootstrapClassLoaderUtils.class); + private static final String LIB_BOOTSTRAP_DIR = "lib/bootstrap"; + + private BootstrapClassLoaderUtils() { + + } + + public static ClassLoader createBootstrapClassLoader() { + List urls = new ArrayList<>(); + try (Stream files = Files.list(Paths.get(LIB_BOOTSTRAP_DIR))) { + files.forEach(p -> { + try { + urls.add(p.toUri().toURL()); + } catch (MalformedURLException mef) { + LOGGER.warn("Unable to load bootstrap library [{}]", p.getFileName(), mef); + } + }); + } catch (IOException ioe) { + LOGGER.warn("Unable to access lib/bootstrap to create bootstrap classloader", ioe); + } + return new URLClassLoader(urls.toArray(new URL[0]), Thread.currentThread().getContextClassLoader()); + } +} diff --git a/minifi/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-runtime/src/main/java/org/apache/nifi/minifi/util/MultiSourceMinifiProperties.java b/minifi/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-runtime/src/main/java/org/apache/nifi/minifi/util/MultiSourceMinifiProperties.java deleted file mode 100644 index 95096b8bf7..0000000000 --- a/minifi/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-runtime/src/main/java/org/apache/nifi/minifi/util/MultiSourceMinifiProperties.java +++ /dev/null @@ -1,87 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.nifi.minifi.util; - -import static org.apache.nifi.minifi.commons.utils.PropertyUtil.resolvePropertyValue; - -import java.io.BufferedInputStream; -import java.io.File; -import java.io.FileInputStream; -import java.io.InputStream; -import java.util.Properties; -import java.util.Set; -import java.util.stream.Collectors; -import java.util.stream.Stream; -import org.apache.nifi.util.NiFiProperties; - -/** - * Extends NiFi properties functionality with System and Environment property override possibility. The property resolution also works with - * dots and hyphens that are not supported in some shells. - */ -public class MultiSourceMinifiProperties extends NiFiProperties { - - private final Properties properties = new Properties(); - - public static MultiSourceMinifiProperties getInstance() { - return new MultiSourceMinifiProperties(); - } - - private MultiSourceMinifiProperties() { - readFromPropertiesFile(); - } - - @Override - public Set getPropertyKeys() { - return Stream.of(System.getProperties().stringPropertyNames(), System.getenv().keySet(), properties.stringPropertyNames()) - .flatMap(Set::stream) - .collect(Collectors.toSet()); - } - - @Override - public int size() { - return getPropertyKeys().size(); - } - - @Override - public String getProperty(String key) { - return resolvePropertyValue(key, System.getProperties()) - .or(() -> resolvePropertyValue(key, System.getenv())) - .orElseGet(() -> properties.getProperty(key)); - } - - private void readFromPropertiesFile() { - String propertiesFilePath = System.getProperty(NiFiProperties.PROPERTIES_FILE_PATH); - - if (propertiesFilePath != null) { - File propertiesFile = new File(propertiesFilePath.trim()); - - if (!propertiesFile.exists()) { - throw new RuntimeException("Properties file doesn't exist '" + propertiesFile.getAbsolutePath() + "'"); - } - - if (!propertiesFile.canRead()) { - throw new RuntimeException("Properties file exists but cannot be read '" + propertiesFile.getAbsolutePath() + "'"); - } - - try (InputStream inStream = new BufferedInputStream(new FileInputStream(propertiesFile))) { - properties.load(inStream); - } catch (Exception ex) { - throw new RuntimeException("Cannot load properties file due to " + ex.getLocalizedMessage(), ex); - } - } - } -} diff --git a/minifi/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/pom.xml b/minifi/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/pom.xml index ff95f02a9d..8beb586c78 100644 --- a/minifi/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/pom.xml +++ b/minifi/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/pom.xml @@ -30,6 +30,7 @@ limitations under the License. minifi-runtime minifi-resources minifi-server + minifi-properties-loader diff --git a/minifi/minifi-nar-bundles/minifi-framework-bundle/pom.xml b/minifi/minifi-nar-bundles/minifi-framework-bundle/pom.xml index 2369df008f..cc6fda3dc0 100644 --- a/minifi/minifi-nar-bundles/minifi-framework-bundle/pom.xml +++ b/minifi/minifi-nar-bundles/minifi-framework-bundle/pom.xml @@ -71,8 +71,7 @@ limitations under the License. org.apache.nifi - nifi-properties-loader - 2.0.0-SNAPSHOT + minifi-properties-loader org.apache.nifi @@ -80,6 +79,16 @@ limitations under the License. 2.0.0-SNAPSHOT test + + org.apache.nifi + nifi-property-protection-loader + 2.0.0-SNAPSHOT + + + org.apache.nifi + nifi-property-protection-cipher + 2.0.0-SNAPSHOT + \ No newline at end of file diff --git a/minifi/minifi-toolkit/minifi-toolkit-assembly/README.md b/minifi/minifi-toolkit/minifi-toolkit-assembly/README.md index 17fe2f564f..29f8c859b6 100644 --- a/minifi/minifi-toolkit/minifi-toolkit-assembly/README.md +++ b/minifi/minifi-toolkit/minifi-toolkit-assembly/README.md @@ -19,18 +19,19 @@ MiNiFi is a child project effort of Apache NiFi. The MiNiFi toolkit aids in cre ## Table of Contents - [Requirements](#requirements) -- [Getting Started](#getting-started) +- [MiNiFi Toolkit Converter](#minifi-toolkit-converter) +- [Encrypting Sensitive Properties in bootstrap.conf](#encrypt-sensitive-properties-in-bootstrapconf) - [Getting Help](#getting-help) - [Documentation](#documentation) - [License](#license) - [Export Control](#export-control) ## Requirements -* JRE 1.8 +* JRE 21 -## Getting Started +The latest version of the MiNiFi Toolkit can be found at https://nifi.apache.org/minifi/download.html under the `MiNiFi Toolkit Binaries` section. -The latest version of the MiNiFi Toolkit Converter can be found at https://nifi.apache.org/minifi/download.html under the `MiNiFi Toolkit Binaries` section. +# MiNiFi Toolkit Converter After downloading the binary and extracting it, to run the MiNiFi Toolkit Converter: - Change directory to the location where you installed MiNiFi Toolkit and run it and view usage information @@ -60,6 +61,87 @@ After downloading the binary and extracting it, to run the MiNiFi Toolkit Conver ## Note It's not guaranteed in all circumstances that the migration will result in a correct flow. For example if a processor's configuration has changed between version, the conversion tool won't be aware of this, and will use the deprecated property names. You will need to fix such issues manually. +# Encrypting Sensitive Properties in bootstrap.conf + +## MiNiFi Encrypt-Config Tool +The encrypt-config command line tool (invoked in minifi-toolkit as ./bin/encrypt-config.sh or bin\encrypt-config.bat) reads from a bootstrap.conf file with plaintext sensitive configuration values and encrypts each value using a random encryption key. It replaces the plain values with the protected value in the same file, or writes to a new bootstrap.conf file if specified. + +The supported encryption algorithm utilized is AES/GCM 256-bit. + +### Usage +To show help: + +``` +./bin/encrypt-config.sh -h +``` + +The following are the available options: +* -b, --bootstrapConf Path to file containing Bootstrap Configuration [bootstrap.conf] +* -B, --outputBootstrapConf Path to output file for Bootstrap Configuration [bootstrap.conf] with root key configured +* -h, --help Show help message and exit. + +### Example +As an example of how the tool works with the following existing values in the bootstrap.conf file: +``` +nifi.sensitive.props.key=thisIsABadSensitiveKeyPassword +nifi.sensitive.props.algorithm=NIFI_PBKDF2_AES_GCM_256 +nifi.sensitive.props.additional.keys= + +nifi.security.keystore=/path/to/keystore.jks +nifi.security.keystoreType=JKS +nifi.security.keystorePasswd=thisIsABadKeystorePassword +nifi.security.keyPasswd=thisIsABadKeyPassword +nifi.security.truststore= +nifi.security.truststoreType= +nifi.security.truststorePasswd= +c2.security.truststore.location= +c2.security.truststore.password=thisIsABadTruststorePassword +c2.security.truststore.type=JKS +c2.security.keystore.location= +c2.security.keystore.password=thisIsABadKeystorePassword +c2.security.keystore.type=JKS +``` +Enter the following arguments when using the tool: +``` +encrypt-config.sh \ +-b bootstrap.conf \ +``` +As a result, the bootstrap.conf file is overwritten with protected properties and sibling encryption identifiers (aes/gcm/256, the currently supported algorithm): +``` +nifi.sensitive.props.key=4OjkrFywZb7BlGz4||Tm9pg0jV4TltvVKeiMlm9zBsqmtmYUA2QkzcLKQpspyggtQuhNAkAla5s2695A== +nifi.sensitive.props.key.protected=aes/gcm/256 +nifi.sensitive.props.algorithm=NIFI_PBKDF2_AES_GCM_256 +nifi.sensitive.props.additional.keys= + +nifi.security.keystore=/path/to/keystore.jks +nifi.security.keystoreType=JKS +nifi.security.keystorePasswd=iXDmDCadoNJ3VotZ||WvOGbrii4Gk0vr3b6mDstZg+NE0BPZUPk6LVqQlf2Sx3G5XFbUbUYAUz +nifi.security.keystorePasswd.protected=aes/gcm/256 +nifi.security.keyPasswd=199uUUgpPqB4Fuoo||KckbW7iu+HZf1r4KSMQAFn8NLJK+CnUuayqPsTsdM0Wxou1BHg== +nifi.security.keyPasswd.protected=aes/gcm/256 +nifi.security.truststore= +nifi.security.truststoreType= +nifi.security.truststorePasswd= +c2.security.truststore.location= +c2.security.truststore.password=0pHpp+l/WHsDM/sm||fXBvDAQ1BXvNQ8b4EHKa1GspsLx+UD+2EDhph0HbsdmgpVhEv4qj0q5TDo0= +c2.security.truststore.password.protected=aes/gcm/256 +c2.security.truststore.type=JKS +c2.security.keystore.location= +c2.security.keystore.password=j+80L7++RNDf9INQ||RX/QkdVFwRos6Y4XJ8YSUWoI3W5Wx50dyw7HrAA84719SvfxA9eUSDEA +c2.security.keystore.password.protected=aes/gcm/256 +c2.security.keystore.type=JKS +``` + +Additionally, the bootstrap.conf file is updated with the encryption key as follows: +``` +minifi.bootstrap.sensitive.key=c92623e798be949379d0d18f432a57f1b74732141be321cb4af9ed94aa0ae8ac +``` + +Sensitive configuration values are encrypted by the tool by default, however you can encrypt any additional properties, if desired. To encrypt additional properties, specify them as comma-separated values in the minifi.sensitive.props.additional.keys property. + +If the bootstrap.conf file already has valid protected values, those property values are not modified by the tool. + + ## Getting Help If you have questions, you can reach out to our mailing list: dev@nifi.apache.org ([archive](https://mail-archives.apache.org/mod_mbox/nifi-dev)). diff --git a/minifi/minifi-toolkit/minifi-toolkit-assembly/pom.xml b/minifi/minifi-toolkit/minifi-toolkit-assembly/pom.xml index 7f32bfcd07..90c1b1a9a7 100644 --- a/minifi/minifi-toolkit/minifi-toolkit-assembly/pom.xml +++ b/minifi/minifi-toolkit/minifi-toolkit-assembly/pom.xml @@ -62,6 +62,16 @@ limitations under the License. minifi-toolkit-configuration ${project.version} + + org.apache.nifi.minifi + minifi-toolkit-encrypt-config + ${project.version} + + + org.slf4j + slf4j-api + compile + diff --git a/minifi/minifi-toolkit/minifi-toolkit-assembly/src/main/resources/bin/encrypt-config.bat b/minifi/minifi-toolkit/minifi-toolkit-assembly/src/main/resources/bin/encrypt-config.bat new file mode 100644 index 0000000000..589d5a9690 --- /dev/null +++ b/minifi/minifi-toolkit/minifi-toolkit-assembly/src/main/resources/bin/encrypt-config.bat @@ -0,0 +1,41 @@ +@echo off +rem +rem Licensed to the Apache Software Foundation (ASF) under one or more +rem contributor license agreements. See the NOTICE file distributed with +rem this work for additional information regarding copyright ownership. +rem The ASF licenses this file to You under the Apache License, Version 2.0 +rem (the "License"); you may not use this file except in compliance with +rem the License. You may obtain a copy of the License at +rem +rem http://www.apache.org/licenses/LICENSE-2.0 +rem +rem Unless required by applicable law or agreed to in writing, software +rem distributed under the License is distributed on an "AS IS" BASIS, +rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +rem See the License for the specific language governing permissions and +rem limitations under the License. +rem + +rem Use JAVA_HOME if it's set; otherwise, just use java + +if "%JAVA_HOME%" == "" goto noJavaHome +if not exist "%JAVA_HOME%\bin\java.exe" goto noJavaHome +set JAVA_EXE=%JAVA_HOME%\bin\java.exe +goto startConfig + +:noJavaHome +echo The JAVA_HOME environment variable is not defined correctly. +echo Instead the PATH will be used to find the java executable. +echo. +set JAVA_EXE=java +goto startConfig + +:startConfig +set LIB_DIR=%~dp0..\classpath;%~dp0..\lib + +if "%JAVA_OPTS%" == "" set JAVA_OPTS=-Xms128m -Xmx256m + +SET JAVA_PARAMS=-cp %LIB_DIR%\* %JAVA_OPTS% org.apache.nifi.minifi.toolkit.config.EncryptConfigCommand + +cmd.exe /C ""%JAVA_EXE%" %JAVA_PARAMS% %* "" + diff --git a/minifi/minifi-toolkit/minifi-toolkit-assembly/src/main/resources/bin/encrypt-config.sh b/minifi/minifi-toolkit/minifi-toolkit-assembly/src/main/resources/bin/encrypt-config.sh new file mode 100644 index 0000000000..4aa5e13707 --- /dev/null +++ b/minifi/minifi-toolkit/minifi-toolkit-assembly/src/main/resources/bin/encrypt-config.sh @@ -0,0 +1,119 @@ +#!/bin/sh +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# + +# Script structure inspired from Apache Karaf and other Apache projects with similar startup approaches + +SCRIPT_DIR=$(dirname "$0") +SCRIPT_NAME=$(basename "$0") +MINIFI_TOOLKIT_HOME=$(cd "${SCRIPT_DIR}" && cd .. && pwd) +PROGNAME=$(basename "$0") + + +warn() { + echo "${PROGNAME}: $*" +} + +die() { + warn "$*" + exit 1 +} + +detectOS() { + # OS specific support (must be 'true' or 'false'). + cygwin=false; + aix=false; + os400=false; + darwin=false; + case "$(uname)" in + CYGWIN*) + cygwin=true + ;; + AIX*) + aix=true + ;; + OS400*) + os400=true + ;; + Darwin) + darwin=true + ;; + esac + # For AIX, set an environment variable + if ${aix}; then + export LDR_CNTRL=MAXDATA=0xB0000000@DSA + echo ${LDR_CNTRL} + fi +} + +locateJava() { + # Setup the Java Virtual Machine + if $cygwin ; then + [ -n "${JAVA}" ] && JAVA=$(cygpath --unix "${JAVA}") + [ -n "${JAVA_HOME}" ] && JAVA_HOME=$(cygpath --unix "${JAVA_HOME}") + fi + + if [ "x${JAVA}" = "x" ] && [ -r /etc/gentoo-release ] ; then + JAVA_HOME=$(java-config --jre-home) + fi + if [ "x${JAVA}" = "x" ]; then + if [ "x${JAVA_HOME}" != "x" ]; then + if [ ! -d "${JAVA_HOME}" ]; then + die "JAVA_HOME is not valid: ${JAVA_HOME}" + fi + JAVA="${JAVA_HOME}/bin/java" + else + warn "JAVA_HOME not set; results may vary" + JAVA=$(type java) + JAVA=$(expr "${JAVA}" : '.* \(/.*\)$') + if [ "x${JAVA}" = "x" ]; then + die "java command not found" + fi + fi + fi +} + +init() { + # Determine if there is special OS handling we must perform + detectOS + + # Locate the Java VM to execute + locateJava +} + +run() { + LIBS="${MINIFI_TOOLKIT_HOME}/lib/*" + + sudo_cmd_prefix="" + if $cygwin; then + MINIFI_TOOLKIT_HOME=$(cygpath --path --windows "${MINIFI_TOOLKIT_HOME}") + CLASSPATH="$MINIFI_TOOLKIT_HOME/classpath;$(cygpath --path --windows "${LIBS}")" + else + CLASSPATH="$MINIFI_TOOLKIT_HOME/classpath:${LIBS}" + fi + + export JAVA_HOME="$JAVA_HOME" + export MINIFI_TOOLKIT_HOME="$MINIFI_TOOLKIT_HOME" + + umask 0077 + exec "${JAVA}" -cp "${CLASSPATH}" ${JAVA_OPTS:--Xms128m -Xmx256m} org.apache.nifi.minifi.toolkit.config.EncryptConfigCommand "$@" +} + + +init +run "$@" diff --git a/minifi/minifi-toolkit/minifi-toolkit-configuration/pom.xml b/minifi/minifi-toolkit/minifi-toolkit-configuration/pom.xml index 4dcbe0730b..f95df2e635 100644 --- a/minifi/minifi-toolkit/minifi-toolkit-configuration/pom.xml +++ b/minifi/minifi-toolkit/minifi-toolkit-configuration/pom.xml @@ -35,10 +35,6 @@ limitations under the License. minifi-toolkit-schema ${project.version} - - org.apache.nifi - nifi-framework-core - org.apache.nifi nifi-property-utils @@ -48,6 +44,20 @@ limitations under the License. org.apache.nifi nifi-properties + + + commons-io + commons-io + + + com.fasterxml.jackson.module + jackson-module-jaxb-annotations + + + + com.fasterxml.jackson.module + jackson-module-jakarta-xmlbind-annotations + diff --git a/minifi/minifi-toolkit/minifi-toolkit-encrypt-config/pom.xml b/minifi/minifi-toolkit/minifi-toolkit-encrypt-config/pom.xml new file mode 100644 index 0000000000..fd936cad8d --- /dev/null +++ b/minifi/minifi-toolkit/minifi-toolkit-encrypt-config/pom.xml @@ -0,0 +1,57 @@ + + + + + org.apache.nifi.minifi + minifi-toolkit + 2.0.0-SNAPSHOT + + 4.0.0 + + minifi-toolkit-encrypt-config + Tool to encrypt sensitive configuration values + + + info.picocli + picocli + 4.7.5 + + + org.apache.nifi + nifi-property-protection-api + 2.0.0-SNAPSHOT + + + org.apache.nifi + nifi-property-protection-cipher + 2.0.0-SNAPSHOT + + + org.apache.nifi.minifi + minifi-properties-loader + 2.0.0-SNAPSHOT + + + org.slf4j + slf4j-api + compile + + + org.apache.nifi.minifi + minifi-commons-utils + + + diff --git a/minifi/minifi-toolkit/minifi-toolkit-encrypt-config/src/main/java/org/apache/nifi/minifi/toolkit/config/EncryptConfigCommand.java b/minifi/minifi-toolkit/minifi-toolkit-encrypt-config/src/main/java/org/apache/nifi/minifi/toolkit/config/EncryptConfigCommand.java new file mode 100644 index 0000000000..9e9942ac33 --- /dev/null +++ b/minifi/minifi-toolkit/minifi-toolkit-encrypt-config/src/main/java/org/apache/nifi/minifi/toolkit/config/EncryptConfigCommand.java @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.minifi.toolkit.config; + +import org.apache.nifi.minifi.toolkit.config.command.MiNiFiEncryptConfig; +import picocli.CommandLine; + +/** + * Encrypt Config Command launcher for Command Line implementation + */ +public class EncryptConfigCommand { + + /** + * Main command method launches Picocli Command Line implementation of Encrypt Config + * + * @param arguments Command line arguments + */ + public static void main(String[] arguments) { + CommandLine commandLine = new CommandLine(new MiNiFiEncryptConfig()); + if (arguments.length == 0) { + commandLine.usage(System.out); + } else { + int status = commandLine.execute(arguments); + System.exit(status); + } + } + +} diff --git a/minifi/minifi-toolkit/minifi-toolkit-encrypt-config/src/main/java/org/apache/nifi/minifi/toolkit/config/command/MiNiFiEncryptConfig.java b/minifi/minifi-toolkit/minifi-toolkit-encrypt-config/src/main/java/org/apache/nifi/minifi/toolkit/config/command/MiNiFiEncryptConfig.java new file mode 100644 index 0000000000..f5e6248240 --- /dev/null +++ b/minifi/minifi-toolkit/minifi-toolkit-encrypt-config/src/main/java/org/apache/nifi/minifi/toolkit/config/command/MiNiFiEncryptConfig.java @@ -0,0 +1,146 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.minifi.toolkit.config.command; + +import java.io.IOException; +import java.io.UncheckedIOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; +import java.security.SecureRandom; +import java.util.HashSet; +import java.util.HexFormat; +import java.util.Optional; +import java.util.Set; +import org.apache.nifi.minifi.properties.BootstrapProperties; +import org.apache.nifi.minifi.properties.BootstrapPropertiesLoader; +import org.apache.nifi.minifi.properties.ProtectedBootstrapProperties; +import org.apache.nifi.minifi.toolkit.config.transformer.ApplicationPropertiesFileTransformer; +import org.apache.nifi.minifi.toolkit.config.transformer.BootstrapConfigurationFileTransformer; +import org.apache.nifi.minifi.toolkit.config.transformer.FileTransformer; +import org.apache.nifi.properties.AesGcmSensitivePropertyProvider; +import org.apache.nifi.properties.SensitivePropertyProvider; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import picocli.CommandLine.Command; +import picocli.CommandLine.Option; + +/** + * Shared Encrypt Configuration for NiFi and NiFi Registry + */ +@Command( + name = "encrypt-config", + sortOptions = false, + mixinStandardHelpOptions = true, + usageHelpWidth = 160, + separator = " ", + version = { + "Java ${java.version} (${java.vendor} ${java.vm.name} ${java.vm.version})" + }, + descriptionHeading = "Description: ", + description = { + "encrypt-config supports protection of sensitive values in Apache MiNiFi" + } +) +public class MiNiFiEncryptConfig implements Runnable{ + + static final String BOOTSTRAP_ROOT_KEY_PROPERTY = "minifi.bootstrap.sensitive.key"; + + private static final String WORKING_FILE_NAME_FORMAT = "%s.%d.working"; + private static final int KEY_LENGTH = 32; + + @Option( + names = {"-b", "--bootstrapConf"}, + description = "Path to file containing Bootstrap Configuration [bootstrap.conf] for optional root key and property protection scheme settings" + ) + Path bootstrapConfPath; + + @Option( + names = {"-B", "--outputBootstrapConf"}, + description = "Path to output file for Bootstrap Configuration [bootstrap.conf] with root key configured" + ) + Path outputBootstrapConf; + + protected final Logger logger = LoggerFactory.getLogger(getClass()); + + @Override + public void run() { + processBootstrapConf(); + } + + /** + * Process bootstrap.conf writing new Root Key to specified Root Key Property when bootstrap.conf is specified + * + */ + protected void processBootstrapConf() { + BootstrapProperties unprotectedProperties = BootstrapPropertiesLoader.load(bootstrapConfPath.toFile()); + + logger.info("Started processing Bootstrap Configuration [{}]", bootstrapConfPath); + + String newRootKey = getRootKey(); + + Set sensitivePropertyNames = new HashSet<>((new ProtectedBootstrapProperties(unprotectedProperties)).getSensitivePropertyKeys()); + FileTransformer fileTransformer2 = new ApplicationPropertiesFileTransformer(unprotectedProperties, getInputSensitivePropertyProvider(newRootKey), sensitivePropertyNames); + runFileTransformer(fileTransformer2, bootstrapConfPath, outputBootstrapConf); + + FileTransformer fileTransformer = new BootstrapConfigurationFileTransformer(BOOTSTRAP_ROOT_KEY_PROPERTY, newRootKey); + runFileTransformer(fileTransformer, Optional.ofNullable(outputBootstrapConf).orElse(bootstrapConfPath), outputBootstrapConf); + logger.info("Completed processing Bootstrap Configuration [{}]", bootstrapConfPath); + } + + private String getRootKey() { + SecureRandom secureRandom = new SecureRandom(); + byte[] sensitivePropertiesKeyBinary = new byte[KEY_LENGTH]; + secureRandom.nextBytes(sensitivePropertiesKeyBinary); + return HexFormat.of().formatHex(sensitivePropertiesKeyBinary); + } + + /** + * Run File Transformer using working path based on output path + * + * @param fileTransformer File Transformer to be invoked + * @param inputPath Input path of file to be transformed + * @param outputPath Output path for transformed file that defaults to the input path when not specified + */ + protected void runFileTransformer(FileTransformer fileTransformer, Path inputPath, Path outputPath) { + Path configuredOutputPath = outputPath == null ? inputPath : outputPath; + Path workingPath = getWorkingPath(configuredOutputPath); + try { + fileTransformer.transform(inputPath, workingPath); + Files.move(workingPath, configuredOutputPath, StandardCopyOption.REPLACE_EXISTING); + } catch (IOException e) { + String message = String.format("Processing Configuration [%s] failed", inputPath); + throw new UncheckedIOException(message, e); + } + } + + + /** + * Get Input Sensitive Property Provider for decrypting previous values + * + * @return Input Sensitive Property Provider + */ + protected SensitivePropertyProvider getInputSensitivePropertyProvider(String keyHex) { + return new AesGcmSensitivePropertyProvider(keyHex); + } + + private Path getWorkingPath(Path resourcePath) { + Path fileName = resourcePath.getFileName(); + String workingFileName = String.format(WORKING_FILE_NAME_FORMAT, fileName, System.currentTimeMillis()); + return resourcePath.resolveSibling(workingFileName); + } +} diff --git a/minifi/minifi-toolkit/minifi-toolkit-encrypt-config/src/main/java/org/apache/nifi/minifi/toolkit/config/transformer/ApplicationPropertiesFileTransformer.java b/minifi/minifi-toolkit/minifi-toolkit-encrypt-config/src/main/java/org/apache/nifi/minifi/toolkit/config/transformer/ApplicationPropertiesFileTransformer.java new file mode 100644 index 0000000000..5cdc8668dd --- /dev/null +++ b/minifi/minifi-toolkit/minifi-toolkit-encrypt-config/src/main/java/org/apache/nifi/minifi/toolkit/config/transformer/ApplicationPropertiesFileTransformer.java @@ -0,0 +1,152 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.minifi.toolkit.config.transformer; + +import static org.apache.nifi.properties.ApplicationPropertiesProtector.PROTECTED_KEY_SUFFIX; + +import java.io.BufferedReader; +import java.io.BufferedWriter; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Objects; +import java.util.Set; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import org.apache.nifi.properties.ApplicationProperties; +import org.apache.nifi.properties.ApplicationPropertiesProtector; +import org.apache.nifi.properties.ProtectedPropertyContext; +import org.apache.nifi.properties.SensitivePropertyProvider; + +/** + * File Transformer supporting transformation of Application Properties with sensitive property values + */ +public class ApplicationPropertiesFileTransformer implements FileTransformer { + + private static final Pattern PROPERTY_VALUE_PATTERN = Pattern.compile("^([^#!][^=]+?)\\s*=\\s?(.+)"); + + private static final int NAME_GROUP = 1; + + private static final int VALUE_GROUP = 2; + + private static final char PROPERTY_VALUE_SEPARATOR = '='; + + private final ApplicationProperties applicationProperties; + + private final SensitivePropertyProvider outputSensitivePropertyProvider; + + private final Set sensitivePropertyNames; + + /** + * Application Properties File Transformer uses provided Application Properties as the source of protected values + * + * @param applicationProperties Application Properties containing decrypted source property values + * @param outputSensitivePropertyProvider Sensitive Property Provider encrypts specified sensitive property values + * @param sensitivePropertyNames Sensitive Property Names marked for encryption + */ + public ApplicationPropertiesFileTransformer( + ApplicationProperties applicationProperties, + SensitivePropertyProvider outputSensitivePropertyProvider, + Set sensitivePropertyNames + ) { + this.applicationProperties = Objects.requireNonNull(applicationProperties, "Application Properties required"); + this.outputSensitivePropertyProvider = Objects.requireNonNull(outputSensitivePropertyProvider, "Output Property Provider required"); + this.sensitivePropertyNames = Objects.requireNonNull(sensitivePropertyNames, "Sensitive Property Names required"); + } + + /** + * Transform input application properties using configured Sensitive Property Provider and write output properties + * + * @param inputPath Input file path to be transformed containing source application properties + * @param outputPath Output file path for protected application properties + * @throws IOException Thrown on transformation failures + */ + @Override + public void transform(Path inputPath, Path outputPath) throws IOException { + Objects.requireNonNull(inputPath, "Input path required"); + Objects.requireNonNull(outputPath, "Output path required"); + + try (BufferedReader reader = Files.newBufferedReader(inputPath); + BufferedWriter writer = Files.newBufferedWriter(outputPath)) { + transform(reader, writer); + } + } + + private void transform(BufferedReader reader, BufferedWriter writer) throws IOException { + String line = reader.readLine(); + while (line != null) { + Matcher matcher = PROPERTY_VALUE_PATTERN.matcher(line); + String nextLine = null; + if (matcher.matches()) { + nextLine = processPropertyLine(reader, writer, matcher, line); + } else { + writeNonSensitiveLine(writer, line); + } + + line = nextLine == null ? reader.readLine() : nextLine; + } + } + + private String processPropertyLine(BufferedReader reader, BufferedWriter writer, Matcher matcher, String line) throws IOException { + String actualPropertyName = matcher.group(NAME_GROUP); + String actualPropertyValue = matcher.group(VALUE_GROUP); + String nextLine = null; + + if (!actualPropertyName.endsWith(PROTECTED_KEY_SUFFIX)) { + nextLine = reader.readLine(); + if (sensitivePropertyNames.contains(actualPropertyName) || isNextPropertyProtected(actualPropertyName, nextLine)) { + String applicationProperty = applicationProperties.getProperty(actualPropertyName, actualPropertyValue); + writeProtectedProperty(writer, actualPropertyName, applicationProperty); + } else { + writeNonSensitiveLine(writer, line); + } + } + return nextLine; + } + + private void writeNonSensitiveLine(BufferedWriter writer, String line) throws IOException { + writer.write(line); + writer.newLine(); + } + + private boolean isNextPropertyProtected(String actualPropertyName, String nextLine) { + if (nextLine != null) { + Matcher nextLineMatcher = PROPERTY_VALUE_PATTERN.matcher(nextLine); + if (nextLineMatcher.matches()) { + String protectedActualPropertyName = ApplicationPropertiesProtector.getProtectionKey(actualPropertyName); + String nextName = nextLineMatcher.group(NAME_GROUP); + return protectedActualPropertyName.equals(nextName); + } + } + return false; + } + + private void writeProtectedProperty(BufferedWriter writer, String name, String value) throws IOException { + ProtectedPropertyContext propertyContext = ProtectedPropertyContext.defaultContext(name); + String protectedValue = outputSensitivePropertyProvider.protect(value, propertyContext); + + writer.write(name); + writer.write(PROPERTY_VALUE_SEPARATOR); + writeNonSensitiveLine(writer, protectedValue); + + String protectedName = ApplicationPropertiesProtector.getProtectionKey(name); + writer.write(protectedName); + writer.write(PROPERTY_VALUE_SEPARATOR); + String protectionIdentifierKey = outputSensitivePropertyProvider.getIdentifierKey(); + writeNonSensitiveLine(writer, protectionIdentifierKey); + } +} diff --git a/minifi/minifi-toolkit/minifi-toolkit-encrypt-config/src/main/java/org/apache/nifi/minifi/toolkit/config/transformer/BootstrapConfigurationFileTransformer.java b/minifi/minifi-toolkit/minifi-toolkit-encrypt-config/src/main/java/org/apache/nifi/minifi/toolkit/config/transformer/BootstrapConfigurationFileTransformer.java new file mode 100644 index 0000000000..8325c1ef70 --- /dev/null +++ b/minifi/minifi-toolkit/minifi-toolkit-encrypt-config/src/main/java/org/apache/nifi/minifi/toolkit/config/transformer/BootstrapConfigurationFileTransformer.java @@ -0,0 +1,108 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.minifi.toolkit.config.transformer; + +import java.io.BufferedReader; +import java.io.BufferedWriter; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Objects; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** + * File Transformer supporting Bootstrap Configuration with updated Root Key + */ +public class BootstrapConfigurationFileTransformer implements FileTransformer { + + private static final Pattern PROPERTY_VALUE_PATTERN = Pattern.compile("^([^#!][^=]+?)\\s*=.*"); + + private static final int NAME_GROUP = 1; + + private static final char PROPERTY_VALUE_SEPARATOR = '='; + + private final String rootKeyPropertyName; + + private final String rootKey; + + /** + * Bootstrap Configuration File Transformer writes provided Root Key to output files + * + * @param rootKeyPropertyName Root Key property name to be written + * @param rootKey Root Key to be written + */ + public BootstrapConfigurationFileTransformer(String rootKeyPropertyName, String rootKey) { + this.rootKeyPropertyName = Objects.requireNonNull(rootKeyPropertyName, "Root Key Property Name required"); + this.rootKey = Objects.requireNonNull(rootKey, "Root Key required"); + } + + /** + * Transform input configuration and write Root Key to output location + * + * @param inputPath Input file path to be transformed containing Bootstrap Configuration + * @param outputPath Output file path for updated configuration + * @throws IOException Thrown on transformation failures + */ + @Override + public void transform(Path inputPath, Path outputPath) throws IOException { + Objects.requireNonNull(inputPath, "Input path required"); + Objects.requireNonNull(outputPath, "Output path required"); + + try (BufferedReader reader = Files.newBufferedReader(inputPath); + BufferedWriter writer = Files.newBufferedWriter(outputPath)) { + transform(reader, writer); + } + } + + private void transform(BufferedReader reader, BufferedWriter writer) throws IOException { + boolean rootKeyPropertyNotFound = true; + + String line = reader.readLine(); + while (line != null) { + Matcher matcher = PROPERTY_VALUE_PATTERN.matcher(line); + if (matcher.matches()) { + String name = matcher.group(NAME_GROUP); + + if (rootKeyPropertyName.equals(name)) { + writeRootKey(writer); + rootKeyPropertyNotFound = false; + } else { + writer.write(line); + writer.newLine(); + } + } else { + writer.write(line); + writer.newLine(); + } + + line = reader.readLine(); + } + + if (rootKeyPropertyNotFound) { + writer.newLine(); + writeRootKey(writer); + } + } + + private void writeRootKey(BufferedWriter writer) throws IOException { + writer.write(rootKeyPropertyName); + writer.write(PROPERTY_VALUE_SEPARATOR); + writer.write(rootKey); + writer.newLine(); + } +} diff --git a/minifi/minifi-toolkit/minifi-toolkit-encrypt-config/src/main/java/org/apache/nifi/minifi/toolkit/config/transformer/FileTransformer.java b/minifi/minifi-toolkit/minifi-toolkit-encrypt-config/src/main/java/org/apache/nifi/minifi/toolkit/config/transformer/FileTransformer.java new file mode 100644 index 0000000000..ecc05a43b6 --- /dev/null +++ b/minifi/minifi-toolkit/minifi-toolkit-encrypt-config/src/main/java/org/apache/nifi/minifi/toolkit/config/transformer/FileTransformer.java @@ -0,0 +1,34 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.minifi.toolkit.config.transformer; + +import java.io.IOException; +import java.nio.file.Path; + +/** + * Abstraction for transforming Files + */ +public interface FileTransformer { + /** + * Transform input file and write contents to output file path + * + * @param inputPath Input file path to be transformed + * @param outputPath Output file path + * @throws IOException Thrown on input or output processing failures + */ + void transform(Path inputPath, Path outputPath) throws IOException; +} diff --git a/minifi/minifi-toolkit/minifi-toolkit-encrypt-config/src/test/java/org/apache/nifi/minifi/toolkit/config/transformer/ApplicationPropertiesFileTransformerTest.java b/minifi/minifi-toolkit/minifi-toolkit-encrypt-config/src/test/java/org/apache/nifi/minifi/toolkit/config/transformer/ApplicationPropertiesFileTransformerTest.java new file mode 100644 index 0000000000..11bf6b0106 --- /dev/null +++ b/minifi/minifi-toolkit/minifi-toolkit-encrypt-config/src/test/java/org/apache/nifi/minifi/toolkit/config/transformer/ApplicationPropertiesFileTransformerTest.java @@ -0,0 +1,99 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.nifi.minifi.toolkit.config.transformer; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.when; + +import java.io.IOException; +import java.io.InputStream; +import java.net.URISyntaxException; +import java.net.URL; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.Map; +import java.util.Objects; +import java.util.Properties; +import java.util.Set; +import org.apache.nifi.properties.ApplicationProperties; +import org.apache.nifi.properties.SensitivePropertyProvider; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.junit.jupiter.api.io.TempDir; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; + +@ExtendWith(MockitoExtension.class) +class ApplicationPropertiesFileTransformerTest { + + private static final String UNPROTECTED_BOOTSTRAP_CONF = "/transformer/unprotected.conf"; + private static final String PROTECTED_BOOTSTRAP_PROPERTIES = "/transformer/protected.conf"; + private static final String PROPERTIES_TRANSFORMED = "transformed.conf"; + private static final String SENSITIVE_PROPERTY_NAME_1 = "property1"; + private static final String SENSITIVE_PROPERTY_NAME_2 = "property2"; + private static final String SENSITIVE_PROPERTY_NAME_3 = "property3"; + private static final String PROVIDER_IDENTIFIER_KEY = "mocked-provider"; + private static final String UNPROTECTED = "UNPROTECTED"; + private static final String ENCRYPTED = "ENCRYPTED"; + + private static final Set SENSITIVE_PROPERTY_NAMES = Set.of(SENSITIVE_PROPERTY_NAME_1, SENSITIVE_PROPERTY_NAME_2, SENSITIVE_PROPERTY_NAME_3); + + @TempDir + private Path tempDir; + + @Mock + private SensitivePropertyProvider sensitivePropertyProvider; + + @Test + void shouldTransformProperties() throws URISyntaxException, IOException { + Path propertiesPath = getResourcePath(UNPROTECTED_BOOTSTRAP_CONF); + + Path tempPropertiesPath = tempDir.resolve(propertiesPath.getFileName()); + Files.copy(propertiesPath, tempPropertiesPath); + + Path outputPropertiesPath = tempDir.resolve(PROPERTIES_TRANSFORMED); + ApplicationProperties applicationProperties = new ApplicationProperties(Map.of(SENSITIVE_PROPERTY_NAME_3, UNPROTECTED)); + FileTransformer transformer = new ApplicationPropertiesFileTransformer(applicationProperties, sensitivePropertyProvider, SENSITIVE_PROPERTY_NAMES); + + when(sensitivePropertyProvider.getIdentifierKey()).thenReturn(PROVIDER_IDENTIFIER_KEY); + when(sensitivePropertyProvider.protect(eq(UNPROTECTED), any())).thenReturn(ENCRYPTED); + + transformer.transform(tempPropertiesPath, outputPropertiesPath); + + Properties expectedProperties = loadProperties(getResourcePath(PROTECTED_BOOTSTRAP_PROPERTIES)); + Properties resultProperties = loadProperties(outputPropertiesPath); + + assertEquals(expectedProperties, resultProperties); + } + + private Properties loadProperties(Path resourcePath) throws IOException { + Properties properties = new Properties(); + try (InputStream propertiesStream = Files.newInputStream(resourcePath)) { + properties.load(propertiesStream); + } + return properties; + } + + private Path getResourcePath(String resource) throws URISyntaxException { + final URL resourceUrl = Objects.requireNonNull(getClass().getResource(resource), String.format("Resource [%s] not found", resource)); + return Paths.get(resourceUrl.toURI()); + } +} \ No newline at end of file diff --git a/minifi/minifi-toolkit/minifi-toolkit-encrypt-config/src/test/java/org/apache/nifi/minifi/toolkit/config/transformer/BootstrapConfigurationFileTransformerTest.java b/minifi/minifi-toolkit/minifi-toolkit-encrypt-config/src/test/java/org/apache/nifi/minifi/toolkit/config/transformer/BootstrapConfigurationFileTransformerTest.java new file mode 100644 index 0000000000..cd05132aef --- /dev/null +++ b/minifi/minifi-toolkit/minifi-toolkit-encrypt-config/src/test/java/org/apache/nifi/minifi/toolkit/config/transformer/BootstrapConfigurationFileTransformerTest.java @@ -0,0 +1,74 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.nifi.minifi.toolkit.config.transformer; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import java.io.IOException; +import java.io.InputStream; +import java.net.URISyntaxException; +import java.net.URL; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.Objects; +import java.util.Properties; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.junit.jupiter.api.io.TempDir; +import org.mockito.junit.jupiter.MockitoExtension; + +@ExtendWith(MockitoExtension.class) +class BootstrapConfigurationFileTransformerTest { + private static final String BOOTSTRAP_ROOT_KEY_PROPERTY = "minifi.bootstrap.sensitive.key"; + private static final String MOCK_KEY = "mockKey"; + private static final String BOOTSTRAP_CONF_FILE_WITHOUT_KEY = "/transformer/bootstrap_without_key.conf"; + private static final String BOOTSTRAP_CONF_TRANSFORMED= "transformed.conf"; + + @TempDir + private Path tempDir; + + @Test + void shouldWriteRootPropertyKeyIfItIsNotPresent() throws URISyntaxException, IOException { + Path propertiesPath = getResourcePath(BOOTSTRAP_CONF_FILE_WITHOUT_KEY); + + Path tempPropertiesPath = tempDir.resolve(propertiesPath.getFileName()); + Files.copy(propertiesPath, tempPropertiesPath); + + Path outputPropertiesPath = tempDir.resolve(BOOTSTRAP_CONF_TRANSFORMED); + + BootstrapConfigurationFileTransformer transformer = new BootstrapConfigurationFileTransformer(BOOTSTRAP_ROOT_KEY_PROPERTY, MOCK_KEY); + transformer.transform(tempPropertiesPath, outputPropertiesPath); + + Properties properties = loadProperties(outputPropertiesPath); + assertEquals(MOCK_KEY, properties.get(BOOTSTRAP_ROOT_KEY_PROPERTY)); + } + + private Properties loadProperties(Path resourcePath) throws IOException { + Properties properties = new Properties(); + try (InputStream propertiesStream = Files.newInputStream(resourcePath)) { + properties.load(propertiesStream); + } + return properties; + } + + private Path getResourcePath(String resource) throws URISyntaxException { + final URL resourceUrl = Objects.requireNonNull(getClass().getResource(resource), String.format("Resource [%s] not found", resource)); + return Paths.get(resourceUrl.toURI()); + } +} \ No newline at end of file diff --git a/minifi/minifi-toolkit/minifi-toolkit-encrypt-config/src/test/resources/transformer/bootstrap_without_key.conf b/minifi/minifi-toolkit/minifi-toolkit-encrypt-config/src/test/resources/transformer/bootstrap_without_key.conf new file mode 100644 index 0000000000..ae1e83eeb3 --- /dev/null +++ b/minifi/minifi-toolkit/minifi-toolkit-encrypt-config/src/test/resources/transformer/bootstrap_without_key.conf @@ -0,0 +1,14 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/minifi/minifi-toolkit/minifi-toolkit-encrypt-config/src/test/resources/transformer/protected.conf b/minifi/minifi-toolkit/minifi-toolkit-encrypt-config/src/test/resources/transformer/protected.conf new file mode 100644 index 0000000000..573f76b514 --- /dev/null +++ b/minifi/minifi-toolkit/minifi-toolkit-encrypt-config/src/test/resources/transformer/protected.conf @@ -0,0 +1,22 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +property1=ENCRYPTED +property1.protected=mocked-provider +property2=ENCRYPTED +property2.protected=mocked-provider +nonsensitive.property=value +property3=ENCRYPTED +property3.protected=mocked-provider \ No newline at end of file diff --git a/minifi/minifi-toolkit/minifi-toolkit-encrypt-config/src/test/resources/transformer/unprotected.conf b/minifi/minifi-toolkit/minifi-toolkit-encrypt-config/src/test/resources/transformer/unprotected.conf new file mode 100644 index 0000000000..5efa9a484d --- /dev/null +++ b/minifi/minifi-toolkit/minifi-toolkit-encrypt-config/src/test/resources/transformer/unprotected.conf @@ -0,0 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +property1=UNPROTECTED +property2=UNPROTECTED +nonsensitive.property=value +property3=ENCRYPTED +property3.protected=mocked-provider \ No newline at end of file diff --git a/minifi/minifi-toolkit/pom.xml b/minifi/minifi-toolkit/pom.xml index f625769a64..175f7d8d51 100644 --- a/minifi/minifi-toolkit/pom.xml +++ b/minifi/minifi-toolkit/pom.xml @@ -29,5 +29,13 @@ limitations under the License. minifi-toolkit-schema minifi-toolkit-configuration minifi-toolkit-assembly + minifi-toolkit-encrypt-config + + + + org.slf4j + slf4j-simple + + diff --git a/minifi/pom.xml b/minifi/pom.xml index aa13d86aa9..6178e865e1 100644 --- a/minifi/pom.xml +++ b/minifi/pom.xml @@ -86,6 +86,11 @@ limitations under the License. minifi-resources 2.0.0-SNAPSHOT + + org.apache.nifi.minifi + minifi-properties-loader + 2.0.0-SNAPSHOT + org.apache.nifi.minifi minifi-provenance-repository-nar diff --git a/nifi-commons/nifi-property-protection-cipher/src/main/java/org/apache/nifi/properties/AesGcmSensitivePropertyProvider.java b/nifi-commons/nifi-property-protection-cipher/src/main/java/org/apache/nifi/properties/AesGcmSensitivePropertyProvider.java index 1641bc14f9..70f94486b9 100644 --- a/nifi-commons/nifi-property-protection-cipher/src/main/java/org/apache/nifi/properties/AesGcmSensitivePropertyProvider.java +++ b/nifi-commons/nifi-property-protection-cipher/src/main/java/org/apache/nifi/properties/AesGcmSensitivePropertyProvider.java @@ -39,7 +39,7 @@ import java.util.Objects; /** * Sensitive Property Provider implementation using AES-GCM with configurable key sizes */ -class AesGcmSensitivePropertyProvider implements SensitivePropertyProvider { +public class AesGcmSensitivePropertyProvider implements SensitivePropertyProvider { private static final String ALGORITHM = "AES/GCM/NoPadding"; private static final String SECRET_KEY_ALGORITHM = "AES"; private static final String DELIMITER = "||"; // "|" is not a valid Base64 character, so ensured not to be present in cipher text